Tactical Media Broadcast Hub — initial release
Self-hosted multi-platform broadcast newsroom for activist collectives: publishing pipeline with group approval and DST-proof scheduling, TweetDeck-style column wall with hostile-content-safe rendering, live control room with Auto Director, bridged chat governance with polls and quorum, distributed sandboxed transcode workers, encrypted content-addressed archive, monthly analytics. MIT licensed. See README.md and docs/.
This commit is contained in:
commit
aa5e262f3a
880 changed files with 111042 additions and 0 deletions
7
infra/peertube-workers/win/build-requirements.txt
Normal file
7
infra/peertube-workers/win/build-requirements.txt
Normal file
|
|
@ -0,0 +1,7 @@
|
|||
pyinstaller==6.16.0
|
||||
pyinstaller-hooks-contrib==2026.8
|
||||
altgraph==0.17.5
|
||||
packaging==26.3
|
||||
pefile==2023.2.7
|
||||
pywin32-ctypes==0.2.3
|
||||
setuptools==84.0.0
|
||||
24
infra/peertube-workers/win/dependencies.json
Normal file
24
infra/peertube-workers/win/dependencies.json
Normal file
|
|
@ -0,0 +1,24 @@
|
|||
{
|
||||
"node_version": "22.23.3",
|
||||
"node_msi": {
|
||||
"url": "https://nodejs.org/dist/v22.23.3/node-v22.23.3-x64.msi",
|
||||
"sha256": "1c0efc8449987e7da5d184786a0a96da83ffa11d334421201e5c09b93017cb8d"
|
||||
},
|
||||
"node_zip": {
|
||||
"url": "https://nodejs.org/dist/v22.23.3/node-v22.23.3-win-x64.zip",
|
||||
"sha256": "2b0ff57b049cda1bbcea2240eec20467018713c1efe1f7360c2681859b90ed71"
|
||||
},
|
||||
"ffmpeg": {
|
||||
"url": "https://github.com/BtbN/FFmpeg-Builds/releases/download/autobuild-2026-10-09-14-16/ffmpeg-n8.1.3-16-ge0a878dd70-win64-gpl-8.1.zip",
|
||||
"sha256": "3c1efebafb60dae9aa1322a1662595d2ee516b352af2a2f6493923998bbee612",
|
||||
"version": "n8.1.3-16-ge0a878dd70-20261009",
|
||||
"validated_original_zip_sha256": "75c912eb2c8253a9efd5e8bf75f03d4124ce1f94a3af41218f0b9da7c3986d76"
|
||||
},
|
||||
"winsw": {
|
||||
"url": "https://github.com/winsw/winsw/releases/download/v2.12.0/WinSW-x64.exe",
|
||||
"sha256": "05b82d46ad331cc16bdc00de5c6332c1ef818df8ceefcd49c726553209b3a0da",
|
||||
"version": "2.12.0"
|
||||
},
|
||||
"runner": "0.2.0",
|
||||
"net_ipc": "2.2.0"
|
||||
}
|
||||
86
infra/peertube-workers/win/e2e-peertube.mjs
Normal file
86
infra/peertube-workers/win/e2e-peertube.mjs
Normal file
|
|
@ -0,0 +1,86 @@
|
|||
// Authorized private-video E2E. No credential is accepted in argv or printed.
|
||||
import fs from 'node:fs';
|
||||
import crypto from 'node:crypto';
|
||||
import {execFileSync} from 'node:child_process';
|
||||
const emit=x=>process.stdout.write(JSON.stringify(x)+'\n');
|
||||
const sleep=ms=>new Promise(r=>setTimeout(r,ms));
|
||||
let stage='model-registration', session, db, auth, video, result={states:[]};
|
||||
try {
|
||||
if(JSON.parse(fs.readFileSync('/app/package.json','utf8')).version !== '7.3.0') throw Error('unreviewed server version');
|
||||
const dbURL=new URL('file:///app/dist/core/initializers/database.js');
|
||||
db=await import(dbURL.href);
|
||||
const {logger}=await import('file:///app/dist/core/helpers/logger.js');logger.silent=true;
|
||||
// Register the shipped models without invoking initDatabaseModels: that helper
|
||||
// recreates a SQL function and is outside this task's authorized server scope.
|
||||
const source=fs.readFileSync(dbURL,'utf8');
|
||||
const names=source.match(/sequelizeTypescript\.addModels\(\[([\s\S]*?)\]\)/)[1].split(',').map(x=>x.trim()).filter(Boolean);
|
||||
const imports=[...source.matchAll(/import \{ (\w+) \} from '(\.\.\/models\/[^']+)';/g)];
|
||||
const models=[];
|
||||
for(const name of names){const item=imports.find(m=>m[1]===name);if(!item)throw Error('model not found');models.push((await import(new URL(item[2],dbURL).href))[name]);}
|
||||
db.sequelizeTypescript.addModels(models);
|
||||
await db.sequelizeTypescript.authenticate();
|
||||
const {OAuthTokenModel}=await import('file:///app/dist/core/models/oauth/oauth-token.js');
|
||||
const {OAuthClientModel}=await import('file:///app/dist/core/models/oauth/oauth-client.js');
|
||||
const {UserModel}=await import('file:///app/dist/core/models/user/user.js');
|
||||
const {RunnerJobModel}=await import('file:///app/dist/core/models/runner/runner-job.js');
|
||||
const {RUNNER_JOB_STATES}=await import('file:///app/dist/core/initializers/constants.js');
|
||||
const {Op}=await import('sequelize');
|
||||
const user=await UserModel.findOne({where:{username:'root',role:0}}), client=await OAuthClientModel.loadFirstClient();
|
||||
if(!user||!client)throw Error('admin/client absent');
|
||||
stage='temporary-admin-session';
|
||||
const expires=new Date(Date.now()+10*60*1000);
|
||||
session=await OAuthTokenModel.create({accessToken:crypto.randomBytes(32).toString('hex'),refreshToken:crypto.randomBytes(32).toString('hex'),accessTokenExpiresAt:expires,refreshTokenExpiresAt:expires,userId:user.id,oAuthClientId:client.id,loginDevice:'authorized-pt-worker-e2e',loginIP:'127.0.0.1',loginDate:new Date()});
|
||||
auth={Authorization:'Bearer '+session.accessToken};
|
||||
const base='http://127.0.0.1:9000/api/v1';
|
||||
async function api(path,options={}) {const r=await fetch(base+path,{...options,headers:{...auth,...options.headers},signal:AbortSignal.timeout(30000)});if(!r.ok){result.http_status=r.status;throw Error('API request failed');}return r.status===204?null:r.json();}
|
||||
stage='admin-auth-check';
|
||||
const me=await api('/users/me');
|
||||
const channel=me.videoChannels?.[0];if(!channel)throw Error('root channel absent');
|
||||
const baseline=await RunnerJobModel.max('id')||0;
|
||||
stage='private-upload';
|
||||
const form=new FormData();
|
||||
for(const [k,v] of Object.entries({channelId:String(channel.id),name:'Windows NVENC worker E2E 2026-10-09',privacy:'3',nsfw:'false',commentsEnabled:'false',downloadEnabled:'true',waitTranscoding:'true'}))form.set(k,v);
|
||||
form.set('videofile',new Blob([fs.readFileSync('/tmp/pt-worker-e2e-source.mp4')],{type:'video/mp4'}),'nvenc-e2e.mp4');
|
||||
const upload=await api('/videos/upload',{method:'POST',body:form});video=upload.video;
|
||||
result.video={id:video.id,uuid:video.uuid,shortUUID:video.shortUUID,privacy:'private',watch_url:'https://tube.occupymedia.cc/w/'+video.shortUUID};
|
||||
emit({phase:'uploaded',video:result.video});
|
||||
stage='remote-job-states';
|
||||
const end=Date.now()+5*60*1000, seen=new Map();let completed;
|
||||
while(Date.now()<end){
|
||||
const jobs=await RunnerJobModel.findAll({attributes:['id','uuid','type','state','runnerId','createdAt','startedAt','updatedAt'],where:{id:{[Op.gt]:baseline}},raw:true});
|
||||
for(const j of jobs){const label=RUNNER_JOB_STATES[j.state];if(seen.get(j.uuid)!==label){seen.set(j.uuid,label);const item={...j,state:label,observed_at:new Date().toISOString()};result.states.push(item);emit({phase:'job-state',...item});}if(label==='Completed')completed=j;if(label==='Errored')throw Error('remote job errored');}
|
||||
if(completed && jobs.some(j=>j.type==='vod-hls-transcoding' && RUNNER_JOB_STATES[j.state]==='Completed'))break;
|
||||
await sleep(100);
|
||||
}
|
||||
if(!completed)throw Error('remote job deadline');
|
||||
result.job={id:completed.id,uuid:completed.uuid,type:completed.type,runner_id:completed.runnerId};
|
||||
stage='result-media-api';
|
||||
let details;
|
||||
for(let n=0;n<60;n++){details=await api('/videos/'+video.uuid);if(details.streamingPlaylists?.[0]?.files?.length)break;await sleep(500);}
|
||||
const playlist=details.streamingPlaylists?.[0];if(!playlist?.files?.length)throw Error('HLS result absent');
|
||||
let fileToken;
|
||||
async function media(url){let u=new URL(url);u.protocol='http:';u.hostname='127.0.0.1';u.port='9000';if(fileToken)u.searchParams.set('videoFileToken',fileToken);let r=await fetch(u,{headers:auth,signal:AbortSignal.timeout(15000)});if(!r.ok&&!fileToken){try{const t=await api('/videos/'+video.uuid+'/token',{method:'POST'});fileToken=t.files?.token||t.token||t.videoFileToken;}catch{}if(fileToken){u.searchParams.set('videoFileToken',fileToken);r=await fetch(u,{headers:auth,signal:AbortSignal.timeout(15000)});}}if(!r.ok){result.media_http_status=r.status;throw Error('private media fetch failed');}return r;}
|
||||
stage='playlist-playback';
|
||||
const master=await (await media(playlist.playlistUrl)).text();
|
||||
const renditionLine=master.split(/\r?\n/).find(l=>l.trim()&&!l.startsWith('#'));
|
||||
const renditionURL=new URL(renditionLine,playlist.playlistUrl).href;
|
||||
const rendition=await (await media(renditionURL)).text();
|
||||
const assetLine=rendition.split(/\r?\n/).find(l=>l.trim()&&!l.startsWith('#'));
|
||||
const assetURL=new URL(assetLine,renditionURL).href;
|
||||
const blob=Buffer.from(await (await media(assetURL)).arrayBuffer());
|
||||
const path='/tmp/pt-worker-e2e-result.mp4';fs.writeFileSync(path,blob,{mode:0o600});
|
||||
const probe=JSON.parse(execFileSync('ffprobe',['-v','error','-show_streams','-show_format','-of','json',path],{encoding:'utf8',timeout:30000}));
|
||||
stage='playback-decode';execFileSync('ffmpeg',['-v','error','-nostdin','-i',path,'-f','null','-'],{stdio:'pipe',timeout:30000});
|
||||
const stream=probe.streams.find(s=>s.codec_type==='video');
|
||||
result.playback={master_http:200,rendition_http:200,asset_http:200,decode_exit:0,bytes:blob.length,codec:stream.codec_name,width:stream.width,height:stream.height,duration:probe.format.duration};
|
||||
result.success=true;emit({phase:'e2e-completed',job:result.job,playback:result.playback});
|
||||
}catch(e){result.success=false;result.failed_stage=stage;result.error_type=e.name;emit({phase:'e2e-failed',stage,error_type:e.name,http_status:result.http_status,media_http_status:result.media_http_status});}
|
||||
finally{
|
||||
if(session){
|
||||
try{await fetch('http://127.0.0.1:9000/api/v1/users/revoke-token',{method:'POST',headers:auth,signal:AbortSignal.timeout(10000)});await sleep(300);}catch{}
|
||||
try{await session.destroy();result.temporary_admin_session_removed=true;}catch{result.temporary_admin_session_removed=false;}
|
||||
}
|
||||
fs.writeFileSync('/tmp/pt-worker-e2e-result.json',JSON.stringify(result,null,2),{mode:0o600});
|
||||
emit({phase:'final-e2e-record',...result});
|
||||
if(db)await db.sequelizeTypescript.close();
|
||||
}
|
||||
11
infra/peertube-workers/win/enroll-server.cjs
Normal file
11
infra/peertube-workers/win/enroll-server.cjs
Normal file
|
|
@ -0,0 +1,11 @@
|
|||
'use strict';
|
||||
// Enrollment listener has concurrency zero and discards upstream logs.
|
||||
const fs = require('node:fs'), path = require('node:path'), {spawn} = require('node:child_process');
|
||||
const root=process.env.PT_ROOT || 'C:\\pt-worker';
|
||||
const cfg=JSON.parse(fs.readFileSync(path.join(root,'state','worker.json'),'utf8').replace(/^\uFEFF/,''));
|
||||
const env={...process.env, PT_ROOT:root, APPDATA:path.join(root,'profile','Roaming'), LOCALAPPDATA:path.join(root,'profile','Local'), FFMPEG_PATH:path.join(root,'bin','ffmpeg-nvenc.exe'),FFPROBE_PATH:path.join(root,'bin','ffprobe.exe'), PT_STATE_DIR:path.join(root,'state'),PT_WORKER_CONFIG:path.join(root,'state','worker.json')};
|
||||
const toml=fs.readFileSync(path.join(env.APPDATA,'peertube-runner-nodejs','Config','default','config.toml'),'utf8');
|
||||
if(!/^concurrency\s*=\s*0\s*$/m.test(toml)) throw Error('Enrollment requires concurrency zero');
|
||||
const child=spawn(cfg.node_binary,[path.join(root,'runner','node_modules','@peertube','peertube-runner','dist','peertube-runner.js'),'server','--enable-job','vod-web-video-transcoding','--enable-job','vod-hls-transcoding','--enable-job','vod-audio-merge-transcoding'],{env,detached:true,windowsHide:true,stdio:'ignore'});
|
||||
fs.writeFileSync(path.join(root,'state','enroll-process.json'),JSON.stringify({pid:child.pid}));
|
||||
child.unref(); console.log(JSON.stringify({enrollment_listener_started:true,concurrency:0}));
|
||||
210
infra/peertube-workers/win/ffmpeg-nvenc.py
Normal file
210
infra/peertube-workers/win/ffmpeg-nvenc.py
Normal file
|
|
@ -0,0 +1,210 @@
|
|||
"""Fail-closed PeerTube 0.2.0 adapter; freeze with PyInstaller as a real PE .exe.
|
||||
Video encoding is always NVENC. CPU image/audio processing is permitted.
|
||||
Only reviewed argument shapes are accepted, including explicit copy-only remux.
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
HERE = Path(sys.executable if getattr(sys, 'frozen', False) else __file__).resolve().parent
|
||||
REAL = Path(os.environ.get('PT_FFMPEG_EXE', str(HERE / 'ffmpeg.exe')))
|
||||
PROBE = Path(os.environ.get('PT_FFPROBE_EXE', str(HERE / 'ffprobe.exe')))
|
||||
STATE = Path(os.environ.get('PT_STATE_DIR', r'C:\pt-worker\state'))
|
||||
DEVICE = os.environ.get('PT_NVENC_DEVICE', '0')
|
||||
TARGET = os.environ.get('PT_NVENC_TARGET', 'h264_nvenc')
|
||||
QUERIES = ('-version', '-formats', '-encoders', '-filters', '-codecs', '-hwaccels')
|
||||
FLAGS = {'-y', '-n', '-nostdin', '-hide_banner', '-an', '-vn', '-shortest', '-copyts'}
|
||||
VALUES = {'-i', '-vcodec', '-c:v', '-codec:v', '-acodec', '-c:a', '-codec:a', '-c',
|
||||
'-vf', '-filter:v', '-pix_fmt', '-preset', '-preset:v', '-b_strategy', '-bf',
|
||||
'-tune', '-crf', '-maxrate', '-maxrate:v', '-bufsize', '-bufsize:v',
|
||||
'-b:v', '-vb', '-b:a', '-ab', '-q:a', '-g', '-g:v', '-r', '-r:v',
|
||||
'-threads', '-f', '-movflags', '-map', '-map_metadata', '-max_muxing_queue_size',
|
||||
'-channel_layout', '-ac', '-ar', '-t', '-loop', '-hls_time', '-hls_list_size',
|
||||
'-hls_playlist_type', '-hls_segment_filename', '-hls_segment_type', '-hls_flags',
|
||||
'-bsf:a', '-loglevel', '-v', '-sc_threshold'}
|
||||
|
||||
|
||||
def event(kind, **fields):
|
||||
try:
|
||||
STATE.mkdir(parents=True, exist_ok=True)
|
||||
with (STATE / 'events.jsonl').open('a', encoding='utf8') as f:
|
||||
f.write(json.dumps({'time': time.time(), 'kind': kind, **fields}) + '\n')
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
def probe(filename):
|
||||
p = subprocess.run([str(PROBE), '-v', 'error', '-show_streams', '-show_format',
|
||||
'-of', 'json', filename], capture_output=True, timeout=30)
|
||||
if p.returncode:
|
||||
raise ValueError('probe failed')
|
||||
return json.loads(p.stdout)
|
||||
|
||||
|
||||
def parse(args):
|
||||
if len(args) < 2 or args[-1].startswith('-'):
|
||||
raise ValueError('missing output')
|
||||
entries, i = [], 0
|
||||
while i < len(args) - 1:
|
||||
key = args[i]
|
||||
if key in FLAGS:
|
||||
entries.append((key, None)); i += 1
|
||||
elif key in VALUES and i + 1 < len(args) - 1:
|
||||
entries.append((key, args[i + 1])); i += 2
|
||||
else:
|
||||
raise ValueError('unknown option or extra output')
|
||||
return entries
|
||||
|
||||
|
||||
def rewrite(args, input_probe=None):
|
||||
if len(args) == 1 and args[0] in QUERIES:
|
||||
return args, None
|
||||
if TARGET not in ('h264_nvenc', 'hevc_nvenc') or not DEVICE.isdigit() or (STATE / 'gpu-unavailable').exists():
|
||||
raise ValueError('GPU disabled')
|
||||
entries = parse(args)
|
||||
# No codec aliases, extra output streams, or global software encoders can escape validation.
|
||||
video = [(k, v) for k, v in entries if k in ('-vcodec', '-c:v', '-codec:v')]
|
||||
generic = [v for k, v in entries if k == '-c']
|
||||
audio = [v for k, v in entries if k in ('-acodec', '-c:a', '-codec:a')]
|
||||
if any(v not in ('copy', 'aac', 'libfdk_aac', 'libmp3lame') for v in audio):
|
||||
raise ValueError('unknown audio encoder')
|
||||
if generic:
|
||||
if generic != ['copy'] or video or audio:
|
||||
raise ValueError('unreviewed global codec')
|
||||
return args, None
|
||||
if len(video) > 1:
|
||||
raise ValueError('multiple video encoders')
|
||||
if '-vn' in args:
|
||||
if video:
|
||||
raise ValueError('contradictory video options')
|
||||
return args, None
|
||||
if video and video[0][1] == 'copy':
|
||||
return args, None
|
||||
if len(video) != 1 or video[0][1] != 'libx264':
|
||||
raise ValueError('unknown video encoder')
|
||||
inputs = [v for k, v in entries if k == '-i']
|
||||
if not inputs or len(inputs) > 2:
|
||||
raise ValueError('unreviewed inputs')
|
||||
data = input_probe if input_probe is not None else probe(inputs[0])
|
||||
stream = next(s for s in data['streams'] if s['codec_type'] == 'video')
|
||||
codec = stream['codec_name']
|
||||
still = codec in ('png', 'mjpeg', 'webp') and '-loop' in args
|
||||
if not still and codec not in ('h264', 'hevc', 'vp9'):
|
||||
raise ValueError('source has no reviewed NVDEC path')
|
||||
if not still and stream.get('pix_fmt') not in ('yuv420p', 'yuv420p10le', 'nv12', 'p010le'):
|
||||
raise ValueError('unsupported pixel format')
|
||||
if stream.get('color_transfer') in ('smpte2084', 'arib-std-b67'):
|
||||
raise ValueError('HDR needs reviewed tone mapping')
|
||||
if stream.get('tags', {}).get('rotate') or any(s.get('rotation', 0) for s in stream.get('side_data_list', [])):
|
||||
raise ValueError('rotation needs reviewed GPU path')
|
||||
width, height = stream['width'], stream['height']
|
||||
w, h = width // 2 * 2, height // 2 * 2
|
||||
filters = [v for k, v in entries if k in ('-vf', '-filter:v')]
|
||||
if len(filters) > 1:
|
||||
raise ValueError('multiple filters')
|
||||
if filters:
|
||||
value = filters[0]
|
||||
if value == 'scale=trunc(iw/2)*2:trunc(ih/2)*2' and still:
|
||||
pass
|
||||
else:
|
||||
match = re.fullmatch(r'scale=w=(-2|\d+):h=(-2|\d+)', value)
|
||||
if not match:
|
||||
raise ValueError('unreviewed filter')
|
||||
w, h = map(int, match.groups())
|
||||
if w == -2 and h > 0:
|
||||
w = max(2, round(width * h / height / 2) * 2)
|
||||
elif h == -2 and w > 0:
|
||||
h = max(2, round(height * w / width / 2) * 2)
|
||||
if w <= 0 or h <= 0 or w % 2 or h % 2 or w > 7680 or h > 4320:
|
||||
raise ValueError('invalid dimensions')
|
||||
out = ['-init_hw_device', f'cuda=nv:{DEVICE}', '-filter_hw_device', 'nv']
|
||||
first = True
|
||||
for key, value in entries:
|
||||
if key == '-i' and first:
|
||||
if not still:
|
||||
out += ['-hwaccel', 'cuda', '-hwaccel_device', DEVICE, '-hwaccel_output_format', 'cuda', '-noautorotate']
|
||||
first = False
|
||||
if key in ('-pix_fmt', '-preset', '-preset:v', '-b_strategy', '-bf', '-tune', '-crf', '-vf', '-filter:v'):
|
||||
continue
|
||||
if key in ('-vcodec', '-c:v', '-codec:v'):
|
||||
value = TARGET
|
||||
out.append(key)
|
||||
if value is not None:
|
||||
out.append(value)
|
||||
prefix = 'format=nv12,hwupload_cuda,' if still else ''
|
||||
out += ['-vf', prefix + f'scale_cuda=w={w}:h={h}:format=nv12',
|
||||
'-preset', 'p4', '-gpu', DEVICE, '-bf', '2', '-rc', 'vbr']
|
||||
rate = next((v for k, v in entries if k in ('-maxrate:v', '-maxrate')), None)
|
||||
if rate and not any(k in ('-b:v', '-vb') for k, _ in entries):
|
||||
out += ['-b:v', rate]
|
||||
out.append(args[-1])
|
||||
uuid = None
|
||||
try:
|
||||
uuid = json.loads((STATE / 'inputs' / (Path(inputs[0]).name + '.json')).read_text())['uuid']
|
||||
except (OSError, ValueError, KeyError):
|
||||
pass
|
||||
return out, {'job_uuid': uuid, 'source_codec': codec, 'target_codec': 'hevc' if TARGET == 'hevc_nvenc' else 'h264', 'encoder': TARGET,
|
||||
'source_width': width, 'source_height': height, 'target_width': w, 'target_height': h,
|
||||
'duration': float(data.get('format', {}).get('duration', 0) or 0), 'static_image_upload': still}
|
||||
|
||||
|
||||
def encoder_util():
|
||||
try:
|
||||
p = subprocess.run(['nvidia-smi.exe', '-i', DEVICE, '--query-gpu=utilization.encoder',
|
||||
'--format=csv,noheader,nounits'], capture_output=True, text=True, timeout=3)
|
||||
return float(p.stdout.strip()) if p.returncode == 0 else None
|
||||
except (OSError, ValueError, subprocess.SubprocessError):
|
||||
return None
|
||||
|
||||
|
||||
def main():
|
||||
try:
|
||||
args, info = rewrite(sys.argv[1:])
|
||||
if not REAL.is_file():
|
||||
raise ValueError('ffmpeg missing')
|
||||
if info is None:
|
||||
sys.exit(subprocess.call([str(REAL), *args]))
|
||||
start = time.monotonic()
|
||||
# Do not persist ffmpeg command lines, media paths, stderr, or signed URLs.
|
||||
p = subprocess.Popen([str(REAL), *args], stdout=sys.stdout, stderr=subprocess.PIPE,
|
||||
text=True, encoding='utf8', errors='replace')
|
||||
progress = {}
|
||||
def consume():
|
||||
for line in p.stderr:
|
||||
sys.stderr.write(line); sys.stderr.flush()
|
||||
for k, v in re.findall(r'(frame|fps|speed)=\s*([\d.]+)', line):
|
||||
progress[k] = float(v)
|
||||
reader = threading.Thread(target=consume, daemon=True); reader.start()
|
||||
samples = []
|
||||
while p.poll() is None:
|
||||
value = encoder_util()
|
||||
if value is not None:
|
||||
samples.append(value)
|
||||
time.sleep(.2)
|
||||
reader.join(timeout=2)
|
||||
seconds = time.monotonic() - start
|
||||
maximum = max(samples) if samples else None
|
||||
# Short jobs can end inside the utilization averaging interval. They still
|
||||
# explicitly use NVENC; boot/driver self-test provides mandatory GPU proof.
|
||||
no_signal = seconds >= 3 and (maximum is None or maximum == 0)
|
||||
rc = p.returncode or (78 if no_signal else 0)
|
||||
event('encode', **info, seconds=seconds, returncode=rc, ffmpeg_returncode=p.returncode,
|
||||
engine_busy_max=maximum, engine_samples=len(samples), engine_counters_readable=bool(samples),
|
||||
frames=progress.get('frame'), ffmpeg_fps=progress.get('fps'), ffmpeg_speed=progress.get('speed'),
|
||||
realtime_factor=info['duration'] / seconds if info['duration'] else None)
|
||||
if rc:
|
||||
event('gpu_failure', reason='nvenc_transcode_failed' if p.returncode else 'no_encoder_signal')
|
||||
sys.exit(rc)
|
||||
except (ValueError, OSError, subprocess.SubprocessError, KeyError, StopIteration):
|
||||
event('gpu_failure', reason='unreviewed_or_unavailable_nvenc')
|
||||
print('PT-WORKER: NVENC adapter refused command; drain required', file=sys.stderr)
|
||||
sys.exit(78)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
253
infra/peertube-workers/win/ffmpeg-qsv.py
Normal file
253
infra/peertube-workers/win/ffmpeg-qsv.py
Normal file
|
|
@ -0,0 +1,253 @@
|
|||
#!/usr/bin/env python3
|
||||
"""Pinned PeerTube runner 0.2.0 FFmpeg adapter (native Windows).
|
||||
|
||||
Port of infra/peertube-workers/ffmpeg-qsv.py from the validated Linux QSV worker.
|
||||
Same contract: rewrite known runner command shapes into explicit QSV decode ->
|
||||
hardware frames -> scale_qsv -> h264_qsv; never execute a CPU video encoder;
|
||||
fail closed (exit 78) on anything unreviewed.
|
||||
|
||||
Windows deltas from the Linux adapter:
|
||||
- real ffmpeg/ffprobe are sibling .exe files of this shim (the runner resolves
|
||||
FFMPEG_PATH with fs.exists, then spawns it without a shell, so the shim
|
||||
MUST be a real .exe: a .cmd/.bat wrapper is rejected by Node >= 18.20/20.12
|
||||
with EINVAL, and fluent-ffmpeg passes no `shell` option).
|
||||
- QSV device init is `-init_hw_device qsv=qs` (D3D11 child device by default;
|
||||
the libmfx/oneVPL dispatcher DLLs must sit next to ffmpeg.exe).
|
||||
- "hardware really did it" telemetry is the PDH GPU Engine counter
|
||||
\\GPU Engine(pid_<pid>_*)\\Utilization Percentage over engtype_VideoEncode
|
||||
instances, sampled via typeperf (i915 DRM fdinfo does not exist on Windows).
|
||||
If the counters are readable and stay exactly 0 for the whole run, the run
|
||||
is recorded as a GPU failure even on exit 0.
|
||||
- capability queries (-version/-encoders/...) are passed through untouched.
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
HERE = Path(__file__).resolve().parent
|
||||
REAL = Path(os.environ.get('PT_FFMPEG_EXE', str(HERE / 'ffmpeg.exe')))
|
||||
PROBE = Path(os.environ.get('PT_FFPROBE_EXE', str(HERE / 'ffprobe.exe')))
|
||||
# Optional child-device spec for the QSV device (e.g. 'd3d11va', 'd3d11va:1').
|
||||
# Empty means the default adapter. Verify on first hardware contact.
|
||||
DEVICE = os.environ.get('PT_QSV_DEVICE', '').strip()
|
||||
STATE = Path(os.environ.get('PT_STATE_DIR', r'C:\pt-worker\state'))
|
||||
|
||||
PASSTHROUGH = (['-version'], ['-formats'], ['-encoders'], ['-filters'],
|
||||
['-codecs'], ['-hwaccels'])
|
||||
|
||||
|
||||
def event(kind, **fields):
|
||||
STATE.mkdir(parents=True, exist_ok=True)
|
||||
line = json.dumps({'time': time.time(), 'kind': kind, **fields}) + '\n'
|
||||
try:
|
||||
with open(STATE / 'events.jsonl', 'a', encoding='utf8') as handle:
|
||||
handle.write(line)
|
||||
except OSError:
|
||||
pass # telemetry loss must never fail a job
|
||||
|
||||
|
||||
def probe(path):
|
||||
p = subprocess.run([str(PROBE), '-v', 'error', '-show_streams', '-show_format',
|
||||
'-of', 'json', str(path)], capture_output=True, text=True, timeout=30)
|
||||
if p.returncode:
|
||||
raise ValueError('input probe failed')
|
||||
return json.loads(p.stdout)
|
||||
|
||||
|
||||
def csv_cells(line):
|
||||
return [cell.strip('"') for cell in line.split('","')]
|
||||
|
||||
|
||||
def as_float(text):
|
||||
try:
|
||||
return float(text)
|
||||
except (TypeError, ValueError):
|
||||
pass
|
||||
try: # locale decimal-comma fallback
|
||||
return float(text.replace('.', '').replace(',', '.'))
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def video_encode_busy(pid):
|
||||
"""Sum of PDH \\GPU Engine(pid_<pid>_*)\\Utilization Percentage over
|
||||
engtype_VideoEncode instances, one 1-second typeperf sample.
|
||||
|
||||
Returns (busy_or_None, readable). Counter names and decimal points are
|
||||
locale-sensitive; unparseable samples count as not-readable, never as zero.
|
||||
"""
|
||||
counter = rf'\GPU Engine(pid_{pid}_*)\Utilization Percentage'
|
||||
try:
|
||||
p = subprocess.run(['typeperf', counter, '-sc', '1'], capture_output=True,
|
||||
text=True, timeout=15)
|
||||
except (OSError, subprocess.SubprocessError):
|
||||
return None, False
|
||||
if p.returncode:
|
||||
return None, False
|
||||
rows = [line for line in (p.stdout or '').splitlines() if line.startswith('"')]
|
||||
if len(rows) < 2:
|
||||
return None, False
|
||||
headers = list(csv_cells(rows[0]))
|
||||
values = list(csv_cells(rows[1]))
|
||||
total, seen = 0.0, False
|
||||
for index, header in enumerate(headers):
|
||||
if 'engtype_VideoEncode' not in header.lower() or index >= len(values):
|
||||
continue
|
||||
number = as_float(values[index])
|
||||
if number is None:
|
||||
return None, False
|
||||
seen = True
|
||||
total += number
|
||||
return (total if seen else None), seen
|
||||
|
||||
|
||||
def rewrite(args, input_probe=None):
|
||||
# fluent-ffmpeg queries capabilities before building commands. These do no encode.
|
||||
if list(args) in [list(p) for p in PASSTHROUGH]:
|
||||
return args, None
|
||||
pairs = list(zip(args, args[1:]))
|
||||
video_options = [(k, v) for k, v in pairs if k in ('-vcodec', '-c:v', '-codec:v')]
|
||||
if len(video_options) > 1:
|
||||
raise ValueError('multiple video codec options refused')
|
||||
if '-vn' in args and not video_options:
|
||||
return args, None # audio-only VOD renditions have no video encode
|
||||
if not video_options or video_options[0][1] != 'libx264':
|
||||
raise ValueError('unknown runner command or video encoder refused')
|
||||
if '-filter_complex' in args or '-hwaccel' in args or '-init_hw_device' in args:
|
||||
raise ValueError('unreviewed filter/device override refused')
|
||||
inputs = [v for k, v in pairs if k == '-i']
|
||||
if not inputs:
|
||||
raise ValueError('missing video input')
|
||||
data = input_probe if input_probe is not None else probe(inputs[0])
|
||||
stream = next((s for s in data['streams'] if s['codec_type'] == 'video'), None)
|
||||
if stream is None:
|
||||
raise ValueError('missing video stream')
|
||||
codec = stream['codec_name']
|
||||
still = codec in ('png', 'mjpeg', 'webp') and '-loop' in args
|
||||
decoder = {'h264': 'h264_qsv', 'hevc': 'hevc_qsv', 'vp9': 'vp9_qsv', 'av1': 'av1_qsv'}.get(codec)
|
||||
if not decoder and not still:
|
||||
raise ValueError('source codec has no reviewed QSV decode path')
|
||||
if stream.get('pix_fmt') not in ('yuv420p', 'yuv420p10le', 'nv12', 'p010le') and not still:
|
||||
raise ValueError('unsupported pixel format; CPU decode fallback refused')
|
||||
if stream.get('tags', {}).get('rotate') or any(s.get('rotation', 0) for s in stream.get('side_data_list', [])):
|
||||
raise ValueError('rotated source requires reviewed GPU transpose path')
|
||||
width, height = stream['width'], stream['height']
|
||||
duration = float(data.get('format', {}).get('duration', 0) or 0)
|
||||
w, h = width // 2 * 2, height // 2 * 2
|
||||
output = ['-init_hw_device', f'qsv=qs:{DEVICE}' if DEVICE else 'qsv=qs',
|
||||
'-filter_hw_device', 'qs']
|
||||
i, first_input = 0, True
|
||||
while i < len(args):
|
||||
key = args[i]
|
||||
val = args[i + 1] if i + 1 < len(args) else None
|
||||
if key == '-i' and first_input:
|
||||
if not still:
|
||||
output += ['-hwaccel', 'qsv', '-hwaccel_output_format', 'qsv', '-c:v', decoder, '-noautorotate']
|
||||
output += [key, val]
|
||||
first_input = False
|
||||
i += 2
|
||||
elif key in ('-vcodec', '-c:v', '-codec:v'):
|
||||
output += [key, 'h264_qsv']
|
||||
i += 2
|
||||
elif key in ('-pix_fmt', '-preset', '-preset:v', '-b_strategy', '-bf', '-tune', '-crf'):
|
||||
i += 2
|
||||
elif key in ('-vf', '-filter:v'):
|
||||
if val == 'scale=trunc(iw/2)*2:trunc(ih/2)*2' and still:
|
||||
w, h = width // 2 * 2, height // 2 * 2
|
||||
else:
|
||||
match = re.fullmatch(r'scale=w=(-2|\d+):h=(-2|\d+)', val or '')
|
||||
if not match:
|
||||
raise ValueError('unreviewed software filter refused')
|
||||
w, h = map(int, match.groups())
|
||||
if w == -2 and h > 0:
|
||||
w = max(2, round(width * h / height / 2) * 2)
|
||||
elif h == -2 and w > 0:
|
||||
h = max(2, round(height * w / width / 2) * 2)
|
||||
if w <= 0 or h <= 0 or w % 2 or h % 2:
|
||||
raise ValueError('invalid output dimensions')
|
||||
prefix = 'format=nv12,hwupload=extra_hw_frames=64,' if still else ''
|
||||
output += [key, prefix + f'scale_qsv=w={w}:h={h}:format=nv12']
|
||||
i += 2
|
||||
else:
|
||||
output.append(key)
|
||||
i += 1
|
||||
if '-vf' not in args and '-filter:v' not in args:
|
||||
prefix = 'format=nv12,hwupload=extra_hw_frames=64,' if still else ''
|
||||
output[-1:-1] = ['-vf', prefix + f'scale_qsv=w={width // 2 * 2}:h={height // 2 * 2}:format=nv12']
|
||||
# Keep PeerTube's rate cap and GOP; QSV has no x264 CRF mode.
|
||||
rate = next((v for k, v in pairs if k in ('-maxrate:v', '-maxrate')), None)
|
||||
extra = ['-preset', 'medium', '-bf', '2']
|
||||
if rate and not any(k in ('-b:v', '-vb') for k, _ in pairs):
|
||||
extra += ['-b:v', rate]
|
||||
output[-1:-1] = extra
|
||||
job_uuid = None
|
||||
try:
|
||||
job_uuid = json.loads((STATE / 'inputs' / (Path(inputs[0]).name + '.json')).read_text(encoding='utf8'))['uuid']
|
||||
except (OSError, ValueError, KeyError):
|
||||
pass
|
||||
return output, {'job_uuid': job_uuid, 'source_codec': codec, 'target_codec': 'h264', 'encoder': 'h264_qsv',
|
||||
'source_width': width, 'source_height': height, 'duration': duration,
|
||||
'target_width': w, 'target_height': h,
|
||||
'static_image_upload': still}
|
||||
|
||||
|
||||
def main():
|
||||
try:
|
||||
if not REAL.is_file():
|
||||
raise ValueError('real ffmpeg.exe not found next to shim')
|
||||
args, info = rewrite(sys.argv[1:])
|
||||
if info is None:
|
||||
os.execv(str(REAL), [str(REAL), *args])
|
||||
started = time.monotonic()
|
||||
p = subprocess.Popen([str(REAL), *args], stderr=subprocess.PIPE, text=True,
|
||||
encoding='utf8', errors='replace')
|
||||
progress = {}
|
||||
def stderr_progress():
|
||||
for line in p.stderr:
|
||||
sys.stderr.write(line)
|
||||
sys.stderr.flush()
|
||||
for key, value in re.findall(r'(frame|fps|speed)=\s*([\d.]+)', line):
|
||||
try:
|
||||
progress[key] = float(value)
|
||||
except ValueError:
|
||||
pass
|
||||
reader = threading.Thread(target=stderr_progress, daemon=True)
|
||||
reader.start()
|
||||
maximum, samples, readable = None, 0, False
|
||||
while p.poll() is None:
|
||||
busy, ok = video_encode_busy(p.pid)
|
||||
readable |= ok
|
||||
if ok:
|
||||
samples += 1
|
||||
if busy is not None:
|
||||
maximum = busy if maximum is None else max(maximum, busy)
|
||||
time.sleep(1.0)
|
||||
elapsed = time.monotonic() - started
|
||||
reader.join(timeout=2)
|
||||
event('encode', **info, seconds=elapsed, returncode=p.returncode,
|
||||
engine_busy_max=maximum, engine_samples=samples,
|
||||
engine_counters_readable=readable,
|
||||
ffmpeg_fps=progress.get('fps'), frames=progress.get('frame'),
|
||||
ffmpeg_speed=progress.get('speed'),
|
||||
realtime_factor=info['duration'] / elapsed if info['duration'] else None)
|
||||
if p.returncode:
|
||||
event('gpu_failure', reason='qsv_transcode_failed')
|
||||
elif readable and samples >= 3 and maximum == 0:
|
||||
# Exit 0 with a readable VideoEncode counter pinned at 0 is NOT
|
||||
# proof of hardware encoding; fail closed like the Linux fdinfo gate.
|
||||
event('gpu_failure', reason='no_video_encode_utilization')
|
||||
sys.exit(p.returncode)
|
||||
except (ValueError, OSError, subprocess.SubprocessError, KeyError) as e:
|
||||
# Messages are fixed strings; never expose source URLs or runner credentials.
|
||||
event('gpu_failure', reason=type(e).__name__)
|
||||
print('PT-WORKER: QSV adapter refused command; worker must drain', file=sys.stderr)
|
||||
sys.exit(78)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
55
infra/peertube-workers/win/ipc-nvenc.cjs
Normal file
55
infra/peertube-workers/win/ipc-nvenc.cjs
Normal file
|
|
@ -0,0 +1,55 @@
|
|||
#!/usr/bin/env node
|
||||
'use strict';
|
||||
// Same IPC protocol/library as the official runner 0.2.0; Windows named-pipe variant
|
||||
// of infra/peertube-workers/ipc.cjs. Registration secret comes from stdin, never argv.
|
||||
//
|
||||
// Pipe-name derivation (verified in the shipped 0.2.0 bundle):
|
||||
// ConfigManager id defaults to "default"; socket path is
|
||||
// join(envPaths("peertube-runner", suffix "nodejs").data, id, "peertube-runner.sock")
|
||||
// = %LOCALAPPDATA%\peertube-runner-nodejs\Data\default\peertube-runner.sock
|
||||
// The bundled @peertube/net-ipc rewrites any path on win32:
|
||||
// `\\\\.\\pipe\\${path.replace(/^\//, "").replace(/\//g, "-")}`
|
||||
// so we pass the RAW Windows path and never build the \\.\pipe\ prefix ourselves.
|
||||
const root = process.argv[3] || process.env.PT_ROOT || 'C:\\pt-worker';
|
||||
process.env.APPDATA = require('node:path').join(root, 'profile', 'Roaming');
|
||||
process.env.LOCALAPPDATA = require('node:path').join(root, 'profile', 'Local');
|
||||
const { Client } = require(require('node:path').join(root, 'runner', 'node_modules', '@peertube', 'net-ipc'));
|
||||
const fs = require('node:fs');
|
||||
const os = require('node:os');
|
||||
const path = require('node:path');
|
||||
const action = process.argv[2];
|
||||
const base = process.env.LOCALAPPDATA || path.join(os.homedir(), 'AppData', 'Local');
|
||||
const id = process.env.PT_RUNNER_ID || 'default';
|
||||
const sock = path.join(base, 'peertube-runner-nodejs', 'Data', id, 'peertube-runner.sock');
|
||||
let client = new Client({ path: sock, reconnect: false });
|
||||
const deadline = setTimeout(() => { client.destroy(); process.exit(1); }, 15000);
|
||||
(async () => {
|
||||
try {
|
||||
let connected = false;
|
||||
for (let attempt = 0; attempt < 20; attempt++) {
|
||||
try { await client.connect(); connected = true; break; }
|
||||
catch { client.destroy(); await new Promise(resolve => setTimeout(resolve, 500)); client = new Client({path: sock, reconnect: false}); }
|
||||
}
|
||||
if (!connected) throw new Error('IPC unavailable');
|
||||
let request = { type: action };
|
||||
if (action === 'register') {
|
||||
const cfg = JSON.parse(fs.readFileSync(process.env.PT_WORKER_CONFIG || path.join(root, 'state', 'worker.json'), 'utf8').replace(/^\uFEFF/, ''));
|
||||
const proof = JSON.parse(fs.readFileSync(path.join(root, 'state', 'nvenc.json'), 'utf8').replace(/^\uFEFF/, ''));
|
||||
const identity = require('node:child_process').execFileSync('nvidia-smi.exe', ['-i',String(cfg.nvenc_device || 0),'--query-gpu=uuid,driver_version','--format=csv,noheader,nounits'],{timeout:5000,encoding:'utf8'}).trim().split(/,\s*/);
|
||||
if (identity[0] !== proof.gpu_uuid || identity[1] !== proof.driver_version) throw new Error('changed GPU');
|
||||
if (!proof.nvenc_h264_ok || !proof.nvenc_hevc_ok || Date.now()/1000 - proof.tested_at > 86400) throw new Error('proof missing');
|
||||
const registrationToken = fs.readFileSync(0, 'utf8').trim();
|
||||
if (!registrationToken.startsWith('ptrrt-')) throw new Error('bad token');
|
||||
request = { type: 'register', url: cfg.peertube_url, runnerName: cfg.name, registrationToken,
|
||||
runnerDescription: 'Windows NVENC only; VOD; AC-only admission; concurrency 2' };
|
||||
} else if (!['list-jobs', 'list-registered', 'graceful-shutdown'].includes(action)) throw new Error('bad action');
|
||||
const result = await client.request(request);
|
||||
if (!result.success) throw new Error('IPC failed');
|
||||
if (action === 'list-jobs') {
|
||||
// Drop payloads (can contain URLs); status consumers only need counts.
|
||||
console.log(JSON.stringify({ concurrency: result.data.concurrency, active_jobs: result.data.processingJobs.length }));
|
||||
} else if (action === 'list-registered') console.log(JSON.stringify({ registered_count: result.data.registeredInstances?.length || result.data.instances?.length || 0 }));
|
||||
else console.log(JSON.stringify({ success: true }));
|
||||
} catch { console.error('Runner IPC failed; inspect local service health'); process.exitCode = 1; }
|
||||
finally { clearTimeout(deadline); client.destroy(); }
|
||||
})();
|
||||
43
infra/peertube-workers/win/ipc-win.cjs
Normal file
43
infra/peertube-workers/win/ipc-win.cjs
Normal file
|
|
@ -0,0 +1,43 @@
|
|||
#!/usr/bin/env node
|
||||
'use strict';
|
||||
// Same IPC protocol/library as the official runner 0.2.0; Windows named-pipe variant
|
||||
// of infra/peertube-workers/ipc.cjs. Registration secret comes from stdin, never argv.
|
||||
//
|
||||
// Pipe-name derivation (verified in the shipped 0.2.0 bundle):
|
||||
// ConfigManager id defaults to "default"; socket path is
|
||||
// join(envPaths("peertube-runner", suffix "nodejs").data, id, "peertube-runner.sock")
|
||||
// = %LOCALAPPDATA%\peertube-runner-nodejs\Data\default\peertube-runner.sock
|
||||
// The bundled @peertube/net-ipc rewrites any path on win32:
|
||||
// `\\\\.\\pipe\\${path.replace(/^\//, "").replace(/\//g, "-")}`
|
||||
// so we pass the RAW Windows path and never build the \\.\pipe\ prefix ourselves.
|
||||
const { Client } = require('./runtime/node_modules/@peertube/net-ipc');
|
||||
const fs = require('node:fs');
|
||||
const os = require('node:os');
|
||||
const path = require('node:path');
|
||||
const action = process.argv[2];
|
||||
const base = process.env.LOCALAPPDATA || path.join(os.homedir(), 'AppData', 'Local');
|
||||
const id = process.env.PT_RUNNER_ID || 'default';
|
||||
const sock = path.join(base, 'peertube-runner-nodejs', 'Data', id, 'peertube-runner.sock');
|
||||
const client = new Client({ path: sock, reconnect: false });
|
||||
const deadline = setTimeout(() => { client.destroy(); process.exit(1); }, 15000);
|
||||
(async () => {
|
||||
try {
|
||||
await client.connect();
|
||||
let request = { type: action };
|
||||
if (action === 'register') {
|
||||
const cfg = JSON.parse(fs.readFileSync(process.env.PT_WORKER_CONFIG || 'C:\\pt-worker\\state\\worker.json', 'utf8'));
|
||||
const registrationToken = fs.readFileSync(0, 'utf8').trim();
|
||||
if (!registrationToken.startsWith('ptrrt-')) throw new Error('bad token');
|
||||
request = { type: 'register', url: cfg.peertube_url, runnerName: cfg.name, registrationToken,
|
||||
runnerDescription: 'VOD only; QSV required; admission controlled' };
|
||||
} else if (!['list-jobs', 'list-registered', 'graceful-shutdown'].includes(action)) throw new Error('bad action');
|
||||
const result = await client.request(request);
|
||||
if (!result.success) throw new Error('IPC failed');
|
||||
if (action === 'list-jobs') {
|
||||
// Drop payloads (can contain URLs); status consumers only need counts.
|
||||
console.log(JSON.stringify({ concurrency: result.data.concurrency, active_jobs: result.data.processingJobs.length }));
|
||||
} else if (action === 'list-registered') console.log(JSON.stringify(result.data));
|
||||
else console.log(JSON.stringify({ success: true }));
|
||||
} catch { console.error('Runner IPC failed; inspect local service health'); process.exitCode = 1; }
|
||||
finally { clearTimeout(deadline); client.destroy(); }
|
||||
})();
|
||||
565
infra/peertube-workers/win/pt-worker-nvenc.py
Normal file
565
infra/peertube-workers/win/pt-worker-nvenc.py
Normal file
|
|
@ -0,0 +1,565 @@
|
|||
#!/usr/bin/env python3
|
||||
"""Native Windows NVENC admission controller. Linux Meter/Admission semantics,
|
||||
Windows power/thermal/GPU sensors, isolated profile, and official named-pipe IPC.
|
||||
The supervisor owns runner processes; admission only publishes fail-closed gates.
|
||||
"""
|
||||
import argparse
|
||||
import ctypes
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import time
|
||||
import tomllib
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(os.environ.get('PT_ROOT', r'C:\pt-worker'))
|
||||
os.environ.update(APPDATA=str(ROOT / 'profile' / 'Roaming'), LOCALAPPDATA=str(ROOT / 'profile' / 'Local'), PT_STATE_DIR=str(ROOT / 'state'), PT_WORKER_CONFIG=str(ROOT / 'state' / 'worker.json'))
|
||||
CONFIG = ROOT / 'state' / 'worker.json'
|
||||
STATE = ROOT / 'state'
|
||||
RUNNER_CONFIG = Path(os.environ.get('APPDATA', str(Path.home() / 'AppData' / 'Roaming'))) / 'peertube-runner-nodejs' / 'Config' / 'default' / 'config.toml'
|
||||
|
||||
DEFAULTS = {
|
||||
'require_ac': True, 'min_disk_free_bytes': 20 * 1024 ** 3, 'max_temperature_c': 85,
|
||||
'min_download_mbps': 10, 'min_upload_mbps': 10, 'max_latency_ms': 150,
|
||||
'allow_derp': False, 'recovery_factor': 1.25, 'good_samples': 3, 'bad_samples': 2,
|
||||
'bandwidth_ttl_seconds': 900, 'transfer_failure_cooldown_seconds': 60,
|
||||
'thermal_cooldown_seconds': 120, 'poll_seconds': 5,
|
||||
}
|
||||
# Last-resort resource gates that hold even while admission policy is disabled.
|
||||
# Same thresholds as the Linux installer defaults (15 GiB RAM floor for a
|
||||
# 2-concurrency QSV worker, 6 GiB min per-job scratch); no CPU fallback exists.
|
||||
HARD_FLOORS = {'ram_total_bytes': 15 * 1024 ** 3, 'min_scratch_bytes': 6 * 1024 ** 3}
|
||||
|
||||
|
||||
def run(args, timeout=10):
|
||||
return subprocess.run(args, capture_output=True, text=True, timeout=timeout)
|
||||
|
||||
|
||||
def read(path, default=None):
|
||||
try:
|
||||
return json.loads(Path(path).read_text(encoding='utf-8-sig'))
|
||||
except (OSError, ValueError):
|
||||
return default
|
||||
|
||||
|
||||
def config():
|
||||
value = read(CONFIG)
|
||||
if not value or not value.get('peertube_url') or not value.get('tailscale_peer'):
|
||||
raise ValueError('worker.json needs peertube_url and tailscale_peer')
|
||||
return {**DEFAULTS, **value}
|
||||
|
||||
|
||||
def ipc(action, node_binary='node'):
|
||||
"""Official runner IPC via ipc-win.cjs; the runner must be running in server mode."""
|
||||
env = {**os.environ, 'PT_STATE_DIR': str(STATE)}
|
||||
p = run([node_binary, str(ROOT / 'bin' / 'ipc-nvenc.cjs'), action], 15)
|
||||
if p.returncode:
|
||||
raise RuntimeError('runner IPC unavailable')
|
||||
return json.loads(p.stdout)
|
||||
|
||||
|
||||
def registered(cfg):
|
||||
try:
|
||||
data = tomllib.loads(RUNNER_CONFIG.read_text(encoding='utf-8-sig'))
|
||||
return any(r['url'].rstrip('/') == cfg['peertube_url'].rstrip('/') and
|
||||
r['runnerName'] == cfg['name'] and r.get('runnerToken')
|
||||
for r in data.get('registeredInstances', []))
|
||||
except (OSError, ValueError, KeyError):
|
||||
return False
|
||||
|
||||
|
||||
def network(cfg):
|
||||
result = {'tailscale_ip': None, 'tailscale_path': 'unknown', 'latency_ms': None,
|
||||
'network_ok': False, 'api_ok': False, 'peer_online': False}
|
||||
try:
|
||||
ts = json.loads(run([cfg.get('tailscale_binary', r'C:\Program Files\Tailscale\tailscale.exe'), 'status', '--json'], 5).stdout)
|
||||
result['tailscale_ip'] = next((a for a in ts.get('Self', {}).get('TailscaleIPs', []) if ':' not in a), None)
|
||||
peers = list(ts.get('Peer', {}).values())
|
||||
peer = next((p for p in peers if cfg['tailscale_peer'] in p.get('TailscaleIPs', []) or
|
||||
p.get('HostName') == cfg['tailscale_peer'] or
|
||||
p.get('DNSName', '').split('.')[0] == cfg['tailscale_peer']), None)
|
||||
if ts.get('BackendState') != 'Running' or not peer or not peer.get('Online'):
|
||||
return result
|
||||
result['peer_online'] = True
|
||||
p = run([cfg.get('tailscale_binary', r'C:\Program Files\Tailscale\tailscale.exe'), 'ping', '--c=1', '--until-direct=false', '--timeout=3s', cfg['tailscale_peer']], 5)
|
||||
if 'pong from' not in p.stdout:
|
||||
return result
|
||||
result['tailscale_path'] = 'DERP' if 'DERP(' in p.stdout else 'direct'
|
||||
latency = re.search(r'in ([\d.]+)ms', p.stdout)
|
||||
if latency:
|
||||
result['latency_ms'] = float(latency[1])
|
||||
started = time.monotonic()
|
||||
with urllib.request.urlopen(cfg['peertube_url'].rstrip('/') + '/api/v1/config', timeout=4) as response:
|
||||
version = json.load(response).get('serverVersion')
|
||||
result['api_ok'] = version == cfg.get('peertube_version', '7.3.0')
|
||||
result['network_ok'] = result['api_ok']
|
||||
except (OSError, ValueError, subprocess.SubprocessError, urllib.error.URLError):
|
||||
pass
|
||||
return result
|
||||
|
||||
|
||||
def transfer_probe(cfg, megabytes=8):
|
||||
"""Bounded probe against pt-worker-netprobe (same server-side probe as Linux)."""
|
||||
url = cfg.get('probe_url')
|
||||
rates = {}
|
||||
if not url:
|
||||
return rates
|
||||
payload = os.urandom(megabytes * 1024 * 1024)
|
||||
for direction, request in (('download', urllib.request.Request(url.rstrip('/') + '/download',
|
||||
data=None, method='GET')),
|
||||
('upload', None)):
|
||||
try:
|
||||
started = time.monotonic()
|
||||
if direction == 'download':
|
||||
with urllib.request.urlopen(request, timeout=30) as response:
|
||||
seen = 0
|
||||
while chunk := response.read(1 << 20):
|
||||
seen += len(chunk)
|
||||
if seen >= megabytes * 1024 * 1024:
|
||||
break
|
||||
seconds = time.monotonic() - started
|
||||
if seen < 65536:
|
||||
continue
|
||||
else:
|
||||
req = urllib.request.Request(url.rstrip('/') + '/upload', data=payload, method='PUT')
|
||||
urllib.request.urlopen(req, timeout=30).read()
|
||||
seconds = time.monotonic() - started
|
||||
rates[direction] = {'mbps': megabytes * 1024 * 1024 * 8 / 1e6 / seconds, 'time': time.time(),
|
||||
'bytes': megabytes * 1024 * 1024, 'seconds': seconds}
|
||||
except (OSError, urllib.error.URLError, ValueError):
|
||||
rates[direction + '_failed'] = True
|
||||
return rates
|
||||
|
||||
|
||||
def get_system_power_status():
|
||||
"""Win32 GetSystemPowerStatus; dict or None when unsupported/unknown."""
|
||||
class PLS(ctypes.Structure):
|
||||
_fields_ = [('ACLineStatus', ctypes.c_byte), ('BatteryFlag', ctypes.c_byte),
|
||||
('BatteryLifePercent', ctypes.c_byte), ('Reserved1', ctypes.c_byte),
|
||||
('BatteryLifeTime', ctypes.c_uint32), ('BatteryFullLifeTime', ctypes.c_uint32)]
|
||||
status = PLS()
|
||||
if not ctypes.windll.kernel32.GetSystemPowerStatus(ctypes.byref(status)):
|
||||
return None
|
||||
ac = {0: False, 1: True}.get(status.ACLineStatus) # 255 == unknown -> None
|
||||
percentage = status.BatteryLifePercent
|
||||
return {'on_ac': ac, 'battery_flag': status.BatteryFlag,
|
||||
'battery_percent': None if percentage in (0, 255) else percentage}
|
||||
|
||||
|
||||
_thermal_cache = {'time': 0, 'temperature_c': None, 'temperatures_c': []}
|
||||
|
||||
def sensors():
|
||||
result = {'temperature_c': None, 'temperatures_c': [], 'on_ac': None, 'battery': [],
|
||||
'throttle_counts': {}, 'ram_total_bytes': None, 'ram_available_bytes': None,
|
||||
'load_average': None, 'wsl2': False}
|
||||
try:
|
||||
if time.time() - _thermal_cache['time'] < 300:
|
||||
raise LookupError('cached')
|
||||
_thermal_cache['time'] = time.time()
|
||||
p = run(['powershell.exe', '-NoProfile', '-NonInteractive', '-Command',
|
||||
'(Get-CimInstance -Namespace root/wmi -ClassName MSAcpi_ThermalZoneTemperature -ErrorAction Stop | '
|
||||
'Measure-Object -Property CurrentTemperature -Maximum).Maximum'], 12)
|
||||
if p.returncode == 0:
|
||||
value = float(p.stdout.strip()) / 10.0 - 273.15
|
||||
if 0 < value < 120:
|
||||
result['temperature_c'] = round(value, 1)
|
||||
result['temperatures_c'] = [result['temperature_c']]
|
||||
_thermal_cache.update(temperature_c=result['temperature_c'], temperatures_c=result['temperatures_c'])
|
||||
except (OSError, ValueError, LookupError, subprocess.SubprocessError):
|
||||
result.update(temperature_c=_thermal_cache['temperature_c'], temperatures_c=_thermal_cache['temperatures_c'])
|
||||
power = get_system_power_status()
|
||||
if power:
|
||||
result['on_ac'] = power['on_ac']
|
||||
if power['battery_percent'] is not None:
|
||||
result['battery'] = [{'capacity': power['battery_percent'],
|
||||
'status': 'on_ac' if power['on_ac'] is True else 'discharging' if power['on_ac'] is False else 'unknown'}]
|
||||
try:
|
||||
import ctypes.wintypes # noqa: F401 (MEMORYSTATUSEX needs no extra types)
|
||||
class MEMORYSTATUSEX(ctypes.Structure):
|
||||
_fields_ = [('dwLength', ctypes.c_uint32), ('dwMemoryLoad', ctypes.c_uint32),
|
||||
('ullTotalPhys', ctypes.c_uint64), ('ullAvailPhys', ctypes.c_uint64),
|
||||
('ullTotalPageFile', ctypes.c_uint64), ('ullAvailPageFile', ctypes.c_uint64),
|
||||
('ullTotalVirtual', ctypes.c_uint64), ('ullAvailVirtual', ctypes.c_uint64),
|
||||
('ullAvailExtendedVirtual', ctypes.c_uint64)]
|
||||
memory = MEMORYSTATUSEX()
|
||||
memory.dwLength = ctypes.sizeof(MEMORYSTATUSEX)
|
||||
if ctypes.windll.kernel32.GlobalMemoryStatusEx(ctypes.byref(memory)):
|
||||
result['ram_total_bytes'] = memory.ullTotalPhys
|
||||
result['ram_available_bytes'] = memory.ullAvailPhys
|
||||
except (OSError, AttributeError, ValueError):
|
||||
pass
|
||||
return result
|
||||
|
||||
|
||||
def disk_free(path=str(ROOT)):
|
||||
try:
|
||||
free = ctypes.c_ulonglong()
|
||||
if ctypes.windll.kernel32.GetDiskFreeSpaceExW(ctypes.c_wchar_p(path), None, None, ctypes.byref(free)):
|
||||
return free.value
|
||||
except (OSError, AttributeError):
|
||||
pass
|
||||
return None
|
||||
|
||||
|
||||
def gpu_identity():
|
||||
try:
|
||||
p = run(['powershell.exe', '-NoProfile', '-NonInteractive', '-Command',
|
||||
'Get-CimInstance Win32_VideoController | Select-Object Name,DriverVersion,Status | ConvertTo-Json'], 12)
|
||||
data = json.loads(p.stdout or '[]')
|
||||
if isinstance(data, dict):
|
||||
data = [data]
|
||||
return data
|
||||
except (OSError, ValueError, subprocess.SubprocessError):
|
||||
return []
|
||||
|
||||
|
||||
class Meter:
|
||||
"""Byte-identical semantics to the Linux controller's Meter."""
|
||||
|
||||
def __init__(self):
|
||||
self.data = read(STATE / 'meter.json', {'offset': 0, 'jobs_completed': 0, 'jobs_failed': 0,
|
||||
'transfer_failures': 0, 'active': {}}) or {}
|
||||
|
||||
def invalidate(self):
|
||||
for key in ('download', 'upload'):
|
||||
self.data.pop(key, None)
|
||||
|
||||
def update(self):
|
||||
p = STATE / 'events.jsonl'
|
||||
if not p.exists():
|
||||
return
|
||||
with p.open(encoding='utf-8-sig') as f:
|
||||
if p.stat().st_size < self.data.get('offset', 0):
|
||||
self.data['offset'] = 0
|
||||
f.seek(self.data.get('offset', 0))
|
||||
while True:
|
||||
position = f.tell()
|
||||
line = f.readline()
|
||||
if not line:
|
||||
break
|
||||
if not line.endswith('\n'):
|
||||
f.seek(position)
|
||||
break
|
||||
try:
|
||||
self.consume(json.loads(line))
|
||||
except (ValueError, KeyError, TypeError):
|
||||
pass
|
||||
self.data['offset'] = f.tell()
|
||||
try:
|
||||
(STATE / 'meter.json').write_text(json.dumps(self.data), encoding='utf-8-sig')
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
def consume(self, e):
|
||||
kind, stamp = e['kind'], e['time']
|
||||
if kind == 'transfer':
|
||||
if not e['ok']:
|
||||
self.data['transfer_failures'] = self.data.get('transfer_failures', 0) + 1
|
||||
self.data['last_transfer_failure'] = stamp
|
||||
elif e['bytes'] >= 65536 and e['seconds'] > 0:
|
||||
direction = e['direction']
|
||||
rate = e['bytes'] * 8 / e['seconds'] / 1e6
|
||||
old = self.data.get(direction, {}).get('mbps', rate)
|
||||
self.data[direction] = {'mbps': .3 * rate + .7 * old, 'time': stamp,
|
||||
'bytes': e['bytes'], 'seconds': e['seconds']}
|
||||
elif kind == 'gpu_failure':
|
||||
self.data['gpu_failure_time'] = stamp
|
||||
elif kind == 'encode':
|
||||
self.data['last_encode'] = e
|
||||
if e.get('job_uuid'):
|
||||
self.data.setdefault('encodes', {})[e['job_uuid']] = e
|
||||
elif kind == 'job_accept' and e['ok']:
|
||||
self.data.setdefault('active', {})[e['uuid']] = stamp
|
||||
elif kind in ('job_success', 'job_error', 'job_abort') and e['ok']:
|
||||
begun = self.data.setdefault('active', {}).pop(e['uuid'], None)
|
||||
if kind == 'job_success':
|
||||
self.data['jobs_completed'] = self.data.get('jobs_completed', 0) + 1
|
||||
elapsed = stamp - begun if begun else None
|
||||
self.data['last_job_duration_seconds'] = elapsed
|
||||
self.data['last_job_uuid'] = e['uuid']
|
||||
encoded = self.data.setdefault('encodes', {}).pop(e['uuid'], {})
|
||||
duration = encoded.get('duration')
|
||||
self.data['last_job_realtime_factor'] = duration / elapsed if duration and elapsed else None
|
||||
self.data['last_job_encode'] = encoded or None
|
||||
elif kind == 'job_error':
|
||||
self.data['jobs_failed'] = self.data.get('jobs_failed', 0) + 1
|
||||
for path in (STATE / 'inputs').glob('*.json'):
|
||||
if read(path) and read(path).get('uuid') == e['uuid']:
|
||||
try:
|
||||
path.unlink()
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
def rates(self, now, ttl):
|
||||
return {f'recent_{k}_mbps': self.data.get(k, {}).get('mbps')
|
||||
if now - self.data.get(k, {}).get('time', 0) < ttl else None
|
||||
for k in ('download', 'upload')}
|
||||
|
||||
|
||||
class Admission:
|
||||
"""Decision gates: PAUSED, GPU_UNAVAILABLE, DRAINING (power), LOW_DISK,
|
||||
THERMAL_THROTTLED, OFFLINE, NETWORK_DEGRADED, BUSY, AVAILABLE.
|
||||
Hysteresis and thresholds identical to the Linux controller."""
|
||||
|
||||
def __init__(self, cfg):
|
||||
self.cfg, self.good, self.bad = cfg, 0, 0
|
||||
self.admitted, self.path, self.throttles = False, None, {}
|
||||
|
||||
def decide(self, sample, metric, control, now):
|
||||
cfg = self.cfg
|
||||
def refuse(state):
|
||||
self.admitted, self.good = False, 0
|
||||
return state
|
||||
if self.path and sample['tailscale_path'] != self.path:
|
||||
self.admitted = False
|
||||
self.good = 0
|
||||
metric.invalidate()
|
||||
self.path = sample['tailscale_path']
|
||||
rates = metric.rates(now, cfg.get('bandwidth_ttl_seconds', 900))
|
||||
sample.update(rates)
|
||||
if control.get('paused', True):
|
||||
return refuse('PAUSED')
|
||||
if not sample['registered']:
|
||||
return refuse('PAUSED')
|
||||
gpu_failed = metric.data.get('gpu_failure_time', 0) >= sample.get('gpu_tested_at', 0)
|
||||
if not sample['gpu_accessible'] or not sample.get('gpu_ready') or gpu_failed:
|
||||
return refuse('GPU_UNAVAILABLE')
|
||||
if cfg.get('require_ac', True) and sample['on_ac'] is not True:
|
||||
return refuse('DRAINING')
|
||||
if sample['disk_free_bytes'] is None or sample['disk_free_bytes'] < cfg.get('min_disk_free_bytes', 20 * 1024 ** 3):
|
||||
return refuse('LOW_DISK')
|
||||
rising = any(v > self.throttles.get(k, v) for k, v in sample['throttle_counts'].items())
|
||||
self.throttles = sample['throttle_counts']
|
||||
if rising or (sample['temperature_c'] is not None and sample['temperature_c'] >= cfg.get('max_temperature_c', 85)):
|
||||
self.thermal_until = now + cfg.get('thermal_cooldown_seconds', 120)
|
||||
if now < getattr(self, 'thermal_until', 0):
|
||||
return refuse('THERMAL_THROTTLED')
|
||||
if not sample['network_ok']:
|
||||
self.admitted, self.good = False, 0
|
||||
return 'OFFLINE'
|
||||
calibrating = control.get('calibrate_until', 0) > now
|
||||
factor = cfg.get('recovery_factor', 1.25) if not self.admitted else 1
|
||||
bad = sample['latency_ms'] is None or sample['latency_ms'] > cfg.get('max_latency_ms', 150)
|
||||
bad |= sample['tailscale_path'] == 'DERP' and not cfg.get('allow_derp', False)
|
||||
if not calibrating:
|
||||
bad |= any(rates[f'recent_{key}_mbps'] is None or rates[f'recent_{key}_mbps'] <
|
||||
cfg.get(f'min_{key}_mbps', 10) * factor for key in ('download', 'upload'))
|
||||
bad |= now - metric.data.get('last_transfer_failure', 0) < cfg.get('transfer_failure_cooldown_seconds', 60)
|
||||
if bad:
|
||||
self.good, self.bad = 0, self.bad + 1
|
||||
if self.bad >= cfg.get('bad_samples', 2) or not self.admitted:
|
||||
self.admitted = False
|
||||
return 'NETWORK_DEGRADED'
|
||||
else:
|
||||
self.bad, self.good = 0, self.good + 1
|
||||
if self.good >= cfg.get('good_samples', 3):
|
||||
self.admitted = True
|
||||
if not self.admitted:
|
||||
return 'NETWORK_DEGRADED'
|
||||
return 'BUSY' if sample['active_jobs'] else 'AVAILABLE'
|
||||
|
||||
|
||||
def atomic(path, value):
|
||||
path = Path(path)
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
tmp = path.with_suffix('.tmp-' + str(os.getpid()))
|
||||
tmp.write_text(json.dumps(value), encoding='utf8')
|
||||
os.replace(tmp, path)
|
||||
|
||||
|
||||
def gpu_snapshot(cfg):
|
||||
p = run(['nvidia-smi.exe', '-i', str(cfg.get('nvenc_device', 0)),
|
||||
'--query-gpu=name,uuid,driver_version,temperature.gpu,utilization.encoder',
|
||||
'--format=csv,noheader,nounits'], 5)
|
||||
if p.returncode:
|
||||
raise RuntimeError('GPU query failed')
|
||||
name, uuid, driver, temp, util = [v.strip() for v in p.stdout.strip().split(',')]
|
||||
return {'gpu_model': name, 'gpu_uuid': uuid, 'driver_version': driver,
|
||||
'temperature_c': float(temp), 'encoder_util': float(util)}
|
||||
|
||||
|
||||
def boot_hardware():
|
||||
p = run(['powershell.exe', '-NoProfile', '-NonInteractive', '-Command',
|
||||
"[pscustomobject]@{boot_id=(Get-CimInstance Win32_OperatingSystem).LastBootUpTime.ToUniversalTime().ToString('o');cpu_model=(Get-CimInstance Win32_Processor | Select-Object -First 1).Name} | ConvertTo-Json -Compress"], 30)
|
||||
return json.loads(p.stdout)
|
||||
|
||||
|
||||
def collect(cfg, meter, admission, hw):
|
||||
now = time.time()
|
||||
meter.update()
|
||||
sensor = sensors()
|
||||
proof = read(STATE / 'nvenc.json', {}) or {}
|
||||
gpu = {}
|
||||
try:
|
||||
gpu = gpu_snapshot(cfg)
|
||||
except (OSError, ValueError, RuntimeError, subprocess.SubprocessError):
|
||||
pass
|
||||
ready = bool(proof.get('nvenc_h264_ok') and proof.get('nvenc_hevc_ok') and
|
||||
proof.get('boot_id') == hw.get('boot_id') and
|
||||
proof.get('driver_version') == gpu.get('driver_version') and
|
||||
proof.get('gpu_uuid') == gpu.get('gpu_uuid') and
|
||||
proof.get('device') == cfg.get('nvenc_device', 0) and
|
||||
0 <= now - proof.get('tested_at', 0) < 7 * 86400)
|
||||
runner = read(STATE / 'runner.json', {}) or {}
|
||||
running = runner.get('running', False) and now - runner.get('updated_at', 0) < 30
|
||||
jobs, ipc_ok = {'active_jobs': 0, 'concurrency': cfg.get('concurrency', 2)}, not running
|
||||
if running:
|
||||
try:
|
||||
jobs = ipc('list-jobs', cfg['node_binary'])
|
||||
ipc_ok = True
|
||||
except (OSError, ValueError, RuntimeError, subprocess.SubprocessError):
|
||||
jobs['active_jobs'] = None
|
||||
temps = sensor['temperatures_c'] + ([gpu['temperature_c']] if gpu else [])
|
||||
sample = {**network(cfg), **sensor, **hw, 'hostname': cfg['name'], 'updated_at': now,
|
||||
'registered': registered(cfg), 'registration_state': 'registered' if registered(cfg) else 'unregistered',
|
||||
**jobs, 'gpu_accessible': bool(gpu) and not (STATE / 'gpu-unavailable').exists(),
|
||||
'gpu_ready': ready, 'gpu_tested_at': proof.get('tested_at', 0),
|
||||
'disk_free_bytes': disk_free(), 'temperature_c': max(temps, default=None),
|
||||
'temperatures_c': temps, 'throttle_counts': {}, 'qsv': {'qsv_h264_ok': False, 'qsv_hevc_ok': False},
|
||||
'qsv_h264_ok': False, 'qsv_hevc_ok': False,
|
||||
'nvenc_h264_ok': proof.get('nvenc_h264_ok', False), 'nvenc_hevc_ok': proof.get('nvenc_hevc_ok', False),
|
||||
'av1_decode_ok': proof.get('av1_decode_ok'), 'av1_encode_ok': False, 'backend': 'nvidia_nvenc',
|
||||
'gpu_model': gpu.get('gpu_model'), 'driver_version': gpu.get('driver_version'),
|
||||
'tested_at': proof.get('tested_at'), 'runner_running': bool(running),
|
||||
'draining': runner.get('draining', False), 'util': {'cpu': None, 'gpu': {'encoder': gpu.get('encoder_util')}}}
|
||||
if sample['disk_free_bytes'] is None:
|
||||
sample['disk_free_bytes'] = 0
|
||||
state = admission.decide(sample, meter, read(STATE / 'control.json', {'paused': True}), now)
|
||||
if not ipc_ok:
|
||||
state = 'OFFLINE'
|
||||
sample.update(status=state, state=state, admission_allowed=state in ('AVAILABLE', 'BUSY'),
|
||||
jobs_completed=meter.data.get('jobs_completed', 0), jobs_failed=meter.data.get('jobs_failed', 0),
|
||||
last_job_duration_seconds=meter.data.get('last_job_duration_seconds'),
|
||||
last_job_realtime_factor=meter.data.get('last_job_realtime_factor'))
|
||||
sample.update({k: meter.data.get(k) for k in ('transfer_failures', 'last_job_uuid', 'last_job_encode', 'last_encode')})
|
||||
# Linux status keys remain intact; research/fleet aliases are additive.
|
||||
sample.update(registration=sample['registered'], gpu_util=gpu.get('encoder_util'), temps=temps,
|
||||
disk_free=sample['disk_free_bytes'], last_job_duration=sample['last_job_duration_seconds'],
|
||||
last_job_rtf=sample['last_job_realtime_factor'], updated_at=time.time())
|
||||
return sample
|
||||
|
||||
|
||||
def self_test_marker_stale(hw, proof):
|
||||
return not proof.get('tested_at') or time.time() - proof['tested_at'] > 7 * 86400 or proof.get('boot_id') != hw.get('boot_id') or proof.get('driver_version') != hw.get('driver_version')
|
||||
|
||||
|
||||
def daemon():
|
||||
import msvcrt
|
||||
import signal
|
||||
lock = (STATE / 'admission.lock').open('a+b')
|
||||
lock.seek(0); lock.write(b'0'); lock.flush(); lock.seek(0)
|
||||
msvcrt.locking(lock.fileno(), msvcrt.LK_NBLCK, 1)
|
||||
cfg = config()
|
||||
meter, admission = Meter(), Admission(cfg)
|
||||
while True:
|
||||
try:
|
||||
hw = boot_hardware()
|
||||
break
|
||||
except (OSError, ValueError, subprocess.SubprocessError):
|
||||
atomic(STATE / 'status.json', {'hostname': cfg['name'], 'status': 'OFFLINE', 'state': 'OFFLINE', 'updated_at': time.time(), 'admission_allowed': False})
|
||||
time.sleep(5)
|
||||
atomic(STATE / 'hardware.json', hw)
|
||||
stopped = False
|
||||
def stop(*_):
|
||||
nonlocal stopped
|
||||
stopped = True
|
||||
signal.signal(signal.SIGINT, stop)
|
||||
signal.signal(signal.SIGTERM, stop)
|
||||
last_probe, previous_path = 0, None
|
||||
attempted_proof = None
|
||||
while not stopped:
|
||||
try:
|
||||
proof = read(STATE / 'nvenc.json', {}) or {}
|
||||
gpu = gpu_snapshot(cfg)
|
||||
identity = (hw.get('boot_id'), gpu.get('driver_version'))
|
||||
if self_test_marker_stale(hw | {'driver_version': gpu.get('driver_version')}, proof) and identity != attempted_proof and not read(STATE / 'runner.json', {}).get('running'):
|
||||
attempted_proof = identity
|
||||
atomic(STATE / 'status.json', {'updated_at': time.time(), 'status': 'GPU_UNAVAILABLE', 'state': 'GPU_UNAVAILABLE', 'admission_allowed': False})
|
||||
if get_system_power_status() and get_system_power_status()['on_ac'] is True:
|
||||
subprocess.run(['powershell.exe', '-NoProfile', '-NonInteractive', '-ExecutionPolicy', 'Bypass', '-File', str(ROOT / 'selftest-nvenc.ps1'), '-Root', str(ROOT), '-Device', str(cfg.get('nvenc_device', 0))], capture_output=True, timeout=90)
|
||||
sample = collect(cfg, meter, admission, hw)
|
||||
if previous_path and previous_path != sample['tailscale_path']:
|
||||
meter.invalidate(); last_probe = 0
|
||||
admission.admitted, admission.good, admission.path = False, 0, None
|
||||
sample['admission_allowed'] = False
|
||||
sample['state'] = sample['status'] = 'NETWORK_DEGRADED'
|
||||
previous_path = sample['tailscale_path']
|
||||
atomic(STATE / 'status.json', sample)
|
||||
control = read(STATE / 'control.json', {'paused': True})
|
||||
if not control.get('paused', True) and sample['network_ok'] and sample['on_ac'] is True and \
|
||||
sample['tailscale_path'] == 'direct' and sample['active_jobs'] == 0 and \
|
||||
time.time() - last_probe >= cfg.get('probe_interval_seconds', 300):
|
||||
# Close admission and wait for the official graceful drain before measuring.
|
||||
sample.update(admission_allowed=False, state='DRAINING', status='DRAINING')
|
||||
atomic(STATE / 'status.json', sample)
|
||||
if not sample['runner_running']:
|
||||
rates = transfer_probe(cfg)
|
||||
for direction in ('download', 'upload'):
|
||||
value = rates.get(direction)
|
||||
meter.consume({'kind': 'transfer', 'time': time.time(), 'direction': direction,
|
||||
'ok': bool(value), 'bytes': value['bytes'] if value else 0,
|
||||
'seconds': value['seconds'] if value else 1})
|
||||
atomic(STATE / 'meter.json', meter.data)
|
||||
last_probe = time.time()
|
||||
except Exception:
|
||||
# Suppress exception strings: upstream data may contain credentials/URLs.
|
||||
atomic(STATE / 'status.json', {'hostname': cfg['name'], 'status': 'OFFLINE', 'state': 'OFFLINE',
|
||||
'updated_at': time.time(), 'admission_allowed': False})
|
||||
before_sleep = time.time()
|
||||
time.sleep(cfg.get('poll_seconds', 5))
|
||||
if time.time() - before_sleep > cfg.get('poll_seconds', 5) * 3:
|
||||
meter.invalidate()
|
||||
admission.admitted, admission.good = False, 0
|
||||
last_probe = 0
|
||||
atomic(STATE / 'status.json', {'hostname': cfg['name'], 'status': 'OFFLINE', 'state': 'OFFLINE',
|
||||
'updated_at': time.time(), 'admission_allowed': False})
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('command', choices=['daemon', 'status', 'network-test', 'start', 'stop', 'drain', 'restart', 'self-test'])
|
||||
args = parser.parse_args()
|
||||
STATE.mkdir(parents=True, exist_ok=True)
|
||||
if args.command == 'daemon':
|
||||
daemon()
|
||||
elif args.command == 'status':
|
||||
value = read(STATE / 'status.json', {'hostname': config()['name'], 'status': 'OFFLINE', 'state': 'OFFLINE'})
|
||||
if time.time() - value.get('updated_at', 0) > 30:
|
||||
value.update(status='OFFLINE', state='OFFLINE', stale=True, admission_allowed=False)
|
||||
print(json.dumps(value, indent=2))
|
||||
elif args.command == 'network-test':
|
||||
meter = Meter(); meter.update()
|
||||
rates = transfer_probe(config())
|
||||
for direction in ('download', 'upload'):
|
||||
r = rates.get(direction)
|
||||
meter.consume({'kind': 'transfer', 'time': time.time(), 'direction': direction, 'ok': bool(r),
|
||||
'bytes': r['bytes'] if r else 0, 'seconds': r['seconds'] if r else 1})
|
||||
atomic(STATE / 'meter.json', meter.data)
|
||||
print(json.dumps(rates))
|
||||
return 0 if all(k in rates for k in ('download', 'upload')) else 1
|
||||
elif args.command == 'self-test':
|
||||
if read(STATE / 'runner.json', {}).get('running'):
|
||||
raise ValueError('drain before self-test')
|
||||
return subprocess.call(['powershell.exe', '-NoProfile', '-NonInteractive', '-ExecutionPolicy', 'Bypass',
|
||||
'-File', str(ROOT / 'selftest-nvenc.ps1'), '-Root', str(ROOT),
|
||||
'-Device', str(config().get('nvenc_device', 0))])
|
||||
else:
|
||||
paused = args.command in ('stop', 'drain')
|
||||
atomic(STATE / 'control.json', {'paused': paused})
|
||||
if args.command == 'restart':
|
||||
try:
|
||||
ipc('graceful-shutdown', config()['node_binary'])
|
||||
except (OSError, RuntimeError, subprocess.SubprocessError):
|
||||
pass
|
||||
print(json.dumps({'requested': args.command, 'paused': paused}))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
try:
|
||||
raise SystemExit(main())
|
||||
except (OSError, ValueError, RuntimeError, subprocess.SubprocessError):
|
||||
print('pt-worker-win: command failed; check service health', file=__import__('sys').stderr)
|
||||
raise SystemExit(1)
|
||||
459
infra/peertube-workers/win/pt-worker-win.py
Normal file
459
infra/peertube-workers/win/pt-worker-win.py
Normal file
|
|
@ -0,0 +1,459 @@
|
|||
#!/usr/bin/env python3
|
||||
"""pt-worker admission controller, native Windows port (validator for the design
|
||||
in windows-worker-result.md).
|
||||
|
||||
Ported from infra/peertube-workers/pt-worker.py: identical Meter / Admission
|
||||
state machine and policy defaults (AC-only, 20 GiB reserve, 85 C ceiling,
|
||||
10 Mbps floors with 1.25x recovery factor, DERP disallowed, 3-good/2-bad
|
||||
hysteresis, 120 s thermal cooldown, 60 s transfer-failure cooldown, 900 s
|
||||
bandwidth TTL, 5 s poll). Windows deltas:
|
||||
- power/battery via GetSystemPowerStatus (ctypes; None == unknown == drain);
|
||||
Modern Standby is mitigated with `powercfg /change standby-timeout-ac 0`
|
||||
during install, not here.
|
||||
- temperatures via MSAcpi_ThermalZoneTemperature; unsupported sensors yield
|
||||
None (unsupported measurements are never invented).
|
||||
- GPU encode telemetry from PDH GPU Engine (engtype_VideoEncode) counters;
|
||||
DRM fdinfo does not exist on Windows. Node >=18.20/20.12: `os.loadavg()`
|
||||
returns [0,0,0] on Windows and is not emitted.
|
||||
- cross-process daemon lock is a win32 mutex via msvcrt (fcntl is POSIX).
|
||||
- drain is delivered through the official runner named-pipe IPC.
|
||||
- hardware re-proof triggers on driver_version change in addition to boot_id
|
||||
(Windows Update can silently change graphics drivers).
|
||||
|
||||
Runs on CPython >= 3.11 with psutil (optional: improves disk/memory collection).
|
||||
Emitting a status JSON is safe to run standalone on Windows or elsewhere.
|
||||
"""
|
||||
import argparse
|
||||
import ctypes
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import time
|
||||
import tomllib
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(r'C:\pt-worker')
|
||||
CONFIG = ROOT / 'state' / 'worker.json'
|
||||
STATE = ROOT / 'state'
|
||||
RUNNER_CONFIG = Path(os.environ.get('APPDATA', str(Path.home() / 'AppData' / 'Roaming'))) / 'peertube-runner-nodejs' / 'Config' / 'config.toml'
|
||||
|
||||
DEFAULTS = {
|
||||
'require_ac': True, 'min_disk_free_bytes': 20 * 1024 ** 3, 'max_temperature_c': 85,
|
||||
'min_download_mbps': 10, 'min_upload_mbps': 10, 'max_latency_ms': 150,
|
||||
'allow_derp': False, 'recovery_factor': 1.25, 'good_samples': 3, 'bad_samples': 2,
|
||||
'bandwidth_ttl_seconds': 900, 'transfer_failure_cooldown_seconds': 60,
|
||||
'thermal_cooldown_seconds': 120, 'poll_seconds': 5,
|
||||
}
|
||||
# Last-resort resource gates that hold even while admission policy is disabled.
|
||||
# Same thresholds as the Linux installer defaults (15 GiB RAM floor for a
|
||||
# 2-concurrency QSV worker, 6 GiB min per-job scratch); no CPU fallback exists.
|
||||
HARD_FLOORS = {'ram_total_bytes': 15 * 1024 ** 3, 'min_scratch_bytes': 6 * 1024 ** 3}
|
||||
|
||||
|
||||
def run(args, timeout=10):
|
||||
return subprocess.run(args, capture_output=True, text=True, timeout=timeout)
|
||||
|
||||
|
||||
def read(path, default=None):
|
||||
try:
|
||||
return json.loads(Path(path).read_text(encoding='utf8'))
|
||||
except (OSError, ValueError):
|
||||
return default
|
||||
|
||||
|
||||
def config():
|
||||
value = read(CONFIG)
|
||||
if not value or not value.get('peertube_url') or not value.get('tailscale_peer'):
|
||||
raise ValueError('worker.json needs peertube_url and tailscale_peer')
|
||||
return {**DEFAULTS, **value}
|
||||
|
||||
|
||||
def ipc(action, node_binary='node'):
|
||||
"""Official runner IPC via ipc-win.cjs; the runner must be running in server mode."""
|
||||
env = {**os.environ, 'PT_STATE_DIR': str(STATE)}
|
||||
p = run([node_binary, str(ROOT / 'bin' / 'ipc-win.cjs'), action], 15)
|
||||
if p.returncode:
|
||||
raise RuntimeError('runner IPC unavailable')
|
||||
return json.loads(p.stdout)
|
||||
|
||||
|
||||
def registered(cfg):
|
||||
try:
|
||||
data = tomllib.loads(RUNNER_CONFIG.read_text(encoding='utf8'))
|
||||
return any(r['url'].rstrip('/') == cfg['peertube_url'].rstrip('/') and
|
||||
r['runnerName'] == cfg['name'] and r.get('runnerToken')
|
||||
for r in data.get('registeredInstances', []))
|
||||
except (OSError, ValueError, KeyError):
|
||||
return False
|
||||
|
||||
|
||||
def network(cfg):
|
||||
result = {'tailscale_ip': None, 'tailscale_path': 'unknown', 'latency_ms': None,
|
||||
'network_ok': False, 'api_ok': False, 'peer_online': False}
|
||||
try:
|
||||
ts = json.loads(run(['tailscale', 'status', '--json'], 5).stdout)
|
||||
result['tailscale_ip'] = next((a for a in ts.get('Self', {}).get('TailscaleIPs', []) if ':' not in a), None)
|
||||
peers = list(ts.get('Peer', {}).values())
|
||||
peer = next((p for p in peers if cfg['tailscale_peer'] in p.get('TailscaleIPs', []) or
|
||||
p.get('HostName') == cfg['tailscale_peer'] or
|
||||
p.get('DNSName', '').split('.')[0] == cfg['tailscale_peer']), None)
|
||||
if ts.get('BackendState') != 'Running' or not peer or not peer.get('Online'):
|
||||
return result
|
||||
result['peer_online'] = True
|
||||
p = run(['tailscale', 'ping', '--c=1', '--until-direct=false', '--timeout=3s', cfg['tailscale_peer']], 5)
|
||||
if 'pong from' not in p.stdout:
|
||||
return result
|
||||
result['tailscale_path'] = 'DERP' if 'DERP(' in p.stdout else 'direct'
|
||||
latency = re.search(r'in ([\d.]+)ms', p.stdout)
|
||||
if latency:
|
||||
result['latency_ms'] = float(latency[1])
|
||||
started = time.monotonic()
|
||||
with urllib.request.urlopen(cfg['peertube_url'].rstrip('/') + '/api/v1/config', timeout=4) as response:
|
||||
version = json.load(response).get('serverVersion')
|
||||
result['api_ok'] = version == cfg.get('peertube_version', '7.3.0')
|
||||
result['network_ok'] = result['api_ok']
|
||||
except (OSError, ValueError, subprocess.SubprocessError, urllib.error.URLError):
|
||||
pass
|
||||
return result
|
||||
|
||||
|
||||
def transfer_probe(cfg, megabytes=8):
|
||||
"""Bounded probe against pt-worker-netprobe (same server-side probe as Linux)."""
|
||||
url = cfg.get('probe_url')
|
||||
rates = {}
|
||||
if not url:
|
||||
return rates
|
||||
payload = os.urandom(megabytes * 1024 * 1024)
|
||||
for direction, request in (('download', urllib.request.Request(url.rstrip('/') + '/download',
|
||||
data=None, method='GET')),
|
||||
('upload', None)):
|
||||
try:
|
||||
started = time.monotonic()
|
||||
if direction == 'download':
|
||||
with urllib.request.urlopen(request, timeout=30) as response:
|
||||
seen = 0
|
||||
while len(response.read(1 << 20)):
|
||||
seen += 1 << 20
|
||||
if seen >= megabytes * 1024 * 1024:
|
||||
break
|
||||
seconds = time.monotonic() - started
|
||||
if seen < 65536:
|
||||
continue
|
||||
else:
|
||||
req = urllib.request.Request(url.rstrip('/') + '/upload', data=payload, method='POST')
|
||||
urllib.request.urlopen(req, timeout=30).read()
|
||||
seconds = time.monotonic() - started
|
||||
rates[direction] = {'mbps': megabytes * 8 / seconds, 'time': time.time(),
|
||||
'bytes': megabytes * 1024 * 1024, 'seconds': seconds}
|
||||
except (OSError, urllib.error.URLError, ValueError):
|
||||
rates[direction + '_failed'] = True
|
||||
return rates
|
||||
|
||||
|
||||
def get_system_power_status():
|
||||
"""Win32 GetSystemPowerStatus; dict or None when unsupported/unknown."""
|
||||
class PLS(ctypes.Structure):
|
||||
_fields_ = [('ACLineStatus', ctypes.c_byte), ('BatteryFlag', ctypes.c_byte),
|
||||
('BatteryLifePercent', ctypes.c_byte), ('Reserved1', ctypes.c_byte),
|
||||
('BatteryLifeTime', ctypes.c_uint32), ('BatteryFullLifeTime', ctypes.c_uint32)]
|
||||
status = PLS()
|
||||
if not ctypes.windll.kernel32.GetSystemPowerStatus(ctypes.byref(status)):
|
||||
return None
|
||||
ac = {0: False, 1: True}.get(status.ACLineStatus) # 255 == unknown -> None
|
||||
percentage = status.BatteryLifePercent
|
||||
return {'on_ac': ac, 'battery_flag': status.BatteryFlag,
|
||||
'battery_percent': None if percentage in (0, 255) else percentage}
|
||||
|
||||
|
||||
def sensors():
|
||||
result = {'temperature_c': None, 'temperatures_c': [], 'on_ac': None, 'battery': [],
|
||||
'throttle_counts': {}, 'ram_total_bytes': None, 'ram_available_bytes': None,
|
||||
'load_average': None}
|
||||
try:
|
||||
p = run(['powershell', '-NoProfile', '-NonInteractive', '-Command',
|
||||
'(Get-CimInstance -ClassName MSAcpi_ThermalZoneTemperature -ErrorAction Stop | '
|
||||
'Measure-Object -Property CurrentTemperature -Maximum).Maximum'], 12)
|
||||
if p.returncode == 0:
|
||||
value = float(p.stdout.strip()) / 10.0 - 273.15
|
||||
if 0 < value < 120:
|
||||
result['temperature_c'] = round(value, 1)
|
||||
result['temperatures_c'] = [result['temperature_c']]
|
||||
except (OSError, ValueError, subprocess.SubprocessError):
|
||||
pass
|
||||
power = get_system_power_status()
|
||||
if power:
|
||||
result['on_ac'] = power['on_ac']
|
||||
if power['battery_percent'] is not None:
|
||||
result['battery'] = [{'capacity': power['battery_percent'],
|
||||
'status': {0: 'unknown', 1: 'discharging', 2: 'on_ac'}.get(power['battery_flag'], 'other')}]
|
||||
try:
|
||||
import ctypes.wintypes # noqa: F401 (MEMORYSTATUSEX needs no extra types)
|
||||
class MEMORYSTATUSEX(ctypes.Structure):
|
||||
_fields_ = [('dwLength', ctypes.c_uint32), ('dwMemoryLoad', ctypes.c_uint32),
|
||||
('ullTotalPhys', ctypes.c_uint64), ('ullAvailPhys', ctypes.c_uint64),
|
||||
('ullTotalPageFile', ctypes.c_uint64), ('ullAvailPageFile', ctypes.c_uint64),
|
||||
('ullTotalVirtual', ctypes.c_uint64), ('ullAvailVirtual', ctypes.c_uint64),
|
||||
('ullAvailExtendedVirtual', ctypes.c_uint64)]
|
||||
memory = MEMORYSTATUSEX()
|
||||
memory.dwLength = ctypes.sizeof(MEMORYSTATUSEX)
|
||||
if ctypes.windll.kernel32.GlobalMemoryStatusEx(ctypes.byref(memory)):
|
||||
result['ram_total_bytes'] = memory.ullTotalPhys
|
||||
result['ram_available_bytes'] = memory.ullAvailPhys
|
||||
except (OSError, AttributeError, ValueError):
|
||||
pass
|
||||
return result
|
||||
|
||||
|
||||
def disk_free(path=str(ROOT)):
|
||||
try:
|
||||
free = ctypes.c_ulonglong()
|
||||
if ctypes.windll.kernel32.GetDiskFreeSpaceExW(ctypes.c_wchar_p(path), None, None, ctypes.byref(free)):
|
||||
return free.value
|
||||
except (OSError, AttributeError):
|
||||
pass
|
||||
return None
|
||||
|
||||
|
||||
def gpu_identity():
|
||||
try:
|
||||
p = run(['powershell', '-NoProfile', '-NonInteractive', '-Command',
|
||||
'Get-CimInstance Win32_VideoController | Select-Object Name,DriverVersion,Status | ConvertTo-Json'], 12)
|
||||
data = json.loads(p.stdout or '[]')
|
||||
if isinstance(data, dict):
|
||||
data = [data]
|
||||
return data
|
||||
except (OSError, ValueError, subprocess.SubprocessError):
|
||||
return []
|
||||
|
||||
|
||||
class Meter:
|
||||
"""Byte-identical semantics to the Linux controller's Meter."""
|
||||
|
||||
def __init__(self):
|
||||
self.data = read(STATE / 'meter.json', {'offset': 0, 'jobs_completed': 0, 'jobs_failed': 0,
|
||||
'transfer_failures': 0, 'active': {}}) or {}
|
||||
|
||||
def invalidate(self):
|
||||
for key in ('download', 'upload'):
|
||||
self.data.pop(key, None)
|
||||
|
||||
def update(self):
|
||||
p = STATE / 'events.jsonl'
|
||||
if not p.exists():
|
||||
return
|
||||
with p.open(encoding='utf8') as f:
|
||||
if p.stat().st_size < self.data.get('offset', 0):
|
||||
self.data['offset'] = 0
|
||||
f.seek(self.data.get('offset', 0))
|
||||
while True:
|
||||
position = f.tell()
|
||||
line = f.readline()
|
||||
if not line:
|
||||
break
|
||||
if not line.endswith('\n'):
|
||||
f.seek(position)
|
||||
break
|
||||
try:
|
||||
self.consume(json.loads(line))
|
||||
except (ValueError, KeyError, TypeError):
|
||||
pass
|
||||
self.data['offset'] = f.tell()
|
||||
try:
|
||||
(STATE / 'meter.json').write_text(json.dumps(self.data), encoding='utf8')
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
def consume(self, e):
|
||||
kind, stamp = e['kind'], e['time']
|
||||
if kind == 'transfer':
|
||||
if not e['ok']:
|
||||
self.data['transfer_failures'] = self.data.get('transfer_failures', 0) + 1
|
||||
self.data['last_transfer_failure'] = stamp
|
||||
elif e['bytes'] >= 65536 and e['seconds'] > 0:
|
||||
direction = e['direction']
|
||||
rate = e['bytes'] * 8 / e['seconds'] / 1e6
|
||||
old = self.data.get(direction, {}).get('mbps', rate)
|
||||
self.data[direction] = {'mbps': .3 * rate + .7 * old, 'time': stamp,
|
||||
'bytes': e['bytes'], 'seconds': e['seconds']}
|
||||
elif kind == 'gpu_failure':
|
||||
self.data['gpu_failure_time'] = stamp
|
||||
elif kind == 'encode':
|
||||
self.data['last_encode'] = e
|
||||
if e.get('job_uuid'):
|
||||
self.data.setdefault('encodes', {})[e['job_uuid']] = e
|
||||
elif kind == 'job_accept' and e['ok']:
|
||||
self.data.setdefault('active', {})[e['uuid']] = stamp
|
||||
elif kind in ('job_success', 'job_error', 'job_abort') and e['ok']:
|
||||
begun = self.data.setdefault('active', {}).pop(e['uuid'], None)
|
||||
if kind == 'job_success':
|
||||
self.data['jobs_completed'] = self.data.get('jobs_completed', 0) + 1
|
||||
elapsed = stamp - begun if begun else None
|
||||
self.data['last_job_duration_seconds'] = elapsed
|
||||
self.data['last_job_uuid'] = e['uuid']
|
||||
encoded = self.data.setdefault('encodes', {}).pop(e['uuid'], {})
|
||||
duration = encoded.get('duration')
|
||||
self.data['last_job_realtime_factor'] = duration / elapsed if duration and elapsed else None
|
||||
self.data['last_job_encode'] = encoded or None
|
||||
elif kind == 'job_error':
|
||||
self.data['jobs_failed'] = self.data.get('jobs_failed', 0) + 1
|
||||
for path in (STATE / 'inputs').glob('*.json'):
|
||||
if read(path) and read(path).get('uuid') == e['uuid']:
|
||||
try:
|
||||
path.unlink()
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
def rates(self, now, ttl):
|
||||
return {f'recent_{k}_mbps': self.data.get(k, {}).get('mbps')
|
||||
if now - self.data.get(k, {}).get('time', 0) < ttl else None
|
||||
for k in ('download', 'upload')}
|
||||
|
||||
|
||||
class Admission:
|
||||
"""Decision gates: PAUSED, GPU_UNAVAILABLE, DRAINING (power), LOW_DISK,
|
||||
THERMAL_THROTTLED, OFFLINE, NETWORK_DEGRADED, BUSY, AVAILABLE.
|
||||
Hysteresis and thresholds identical to the Linux controller."""
|
||||
|
||||
def __init__(self, cfg):
|
||||
self.cfg, self.good, self.bad = cfg, 0, 0
|
||||
self.admitted, self.path, self.throttles = False, None, {}
|
||||
|
||||
def decide(self, sample, metric, control, now):
|
||||
cfg = self.cfg
|
||||
def refuse(state):
|
||||
self.admitted, self.good = False, 0
|
||||
return state
|
||||
if self.path and sample['tailscale_path'] != self.path:
|
||||
self.admitted = False
|
||||
self.good = 0
|
||||
metric.invalidate()
|
||||
self.path = sample['tailscale_path']
|
||||
rates = metric.rates(now, cfg.get('bandwidth_ttl_seconds', 900))
|
||||
sample.update(rates)
|
||||
if control.get('paused', True):
|
||||
return refuse('PAUSED')
|
||||
if not sample['registered']:
|
||||
return refuse('PAUSED')
|
||||
gpu_failed = metric.data.get('gpu_failure_time', 0) >= sample.get('gpu_tested_at', 0)
|
||||
if not sample['gpu_accessible'] or not sample.get('gpu_ready') or gpu_failed:
|
||||
return refuse('GPU_UNAVAILABLE')
|
||||
if cfg.get('require_ac', True) and sample['on_ac'] is not True:
|
||||
return refuse('DRAINING')
|
||||
if sample['disk_free_bytes'] is None or sample['disk_free_bytes'] < cfg.get('min_disk_free_bytes', 20 * 1024 ** 3):
|
||||
return refuse('LOW_DISK')
|
||||
rising = any(v > self.throttles.get(k, v) for k, v in sample['throttle_counts'].items())
|
||||
self.throttles = sample['throttle_counts']
|
||||
if rising or (sample['temperature_c'] is not None and sample['temperature_c'] >= cfg.get('max_temperature_c', 85)):
|
||||
self.thermal_until = now + cfg.get('thermal_cooldown_seconds', 120)
|
||||
if now < getattr(self, 'thermal_until', 0):
|
||||
return refuse('THERMAL_THROTTLED')
|
||||
if not sample['network_ok']:
|
||||
self.admitted, self.good = False, 0
|
||||
return 'OFFLINE'
|
||||
calibrating = control.get('calibrate_until', 0) > now
|
||||
factor = cfg.get('recovery_factor', 1.25) if not self.admitted else 1
|
||||
bad = sample['latency_ms'] is None or sample['latency_ms'] > cfg.get('max_latency_ms', 150)
|
||||
bad |= sample['tailscale_path'] == 'DERP' and not cfg.get('allow_derp', False)
|
||||
if not calibrating:
|
||||
bad |= any(rates[f'recent_{key}_mbps'] is None or rates[f'recent_{key}_mbps'] <
|
||||
cfg.get(f'min_{key}_mbps', 10) * factor for key in ('download', 'upload'))
|
||||
bad |= now - metric.data.get('last_transfer_failure', 0) < cfg.get('transfer_failure_cooldown_seconds', 60)
|
||||
if bad:
|
||||
self.good, self.bad = 0, self.bad + 1
|
||||
if self.bad >= cfg.get('bad_samples', 2) or not self.admitted:
|
||||
self.admitted = False
|
||||
return 'NETWORK_DEGRADED'
|
||||
else:
|
||||
self.bad, self.good = 0, self.good + 1
|
||||
if self.good >= cfg.get('good_samples', 3):
|
||||
self.admitted = True
|
||||
if not self.admitted:
|
||||
return 'NETWORK_DEGRADED'
|
||||
return 'BUSY' if sample['active_jobs'] else 'AVAILABLE'
|
||||
|
||||
|
||||
def status():
|
||||
"""Read-only status document (same schema as the Linux worker's status JSON;
|
||||
names preserved so central fleet tooling needs no OS branches)."""
|
||||
cfg = config()
|
||||
now = time.time()
|
||||
meter = Meter()
|
||||
meter.update()
|
||||
admission = Admission(cfg)
|
||||
sensors_data = sensors()
|
||||
hw = read(STATE / 'hardware.json', {}) or {}
|
||||
gpu = hw.get('gpu_model') or gpu_identity()
|
||||
proof = read(STATE / 'qsv.json', {}) or {}
|
||||
proof_current = bool(proof.get('qsv_h264_ok')) and proof.get('tested_at', 0) > now - 7 * 86400 and \
|
||||
(proof.get('boot_id') == hw.get('boot_id') and proof.get('driver_version') == hw.get('driver_version'))
|
||||
control = read(STATE / 'control.json', {}) or {}
|
||||
active = {}
|
||||
try:
|
||||
active = ipc('list-jobs', cfg.get('node_binary', 'node'))
|
||||
except (RuntimeError, ValueError, OSError):
|
||||
active = {}
|
||||
sample = {**network(cfg), 'registered': registered(cfg),
|
||||
'gpu_accessible': bool(proof_current), 'gpu_ready': bool(proof_current),
|
||||
'gpu_tested_at': proof.get('tested_at', 0),
|
||||
'on_ac': sensors_data['on_ac'], 'disk_free_bytes': disk_free(),
|
||||
'temperature_c': sensors_data['temperature_c'],
|
||||
'throttle_counts': sensors_data['throttle_counts'],
|
||||
'active_jobs': active.get('active_jobs'), 'concurrency': active.get('concurrency')}
|
||||
state = admission.decide(sample, meter, control, now)
|
||||
rates = meter.rates(now, cfg.get('bandwidth_ttl_seconds', 900))
|
||||
last = meter.data.get('last_encode') or {}
|
||||
return {'hostname': os.environ.get('COMPUTERNAME', 'unknown'), 'state': state,
|
||||
'tailscale_ip': sample['tailscale_ip'], 'tailscale_path': sample['tailscale_path'],
|
||||
'registration': sample['registered'], 'cpu_model': hw.get('cpu_model'),
|
||||
'gpu_model': gpu, 'gpu_driver_version': hw.get('driver_version'),
|
||||
'qsv_h264_ok': proof.get('qsv_h264_ok'), 'qsv_hevc_ok': proof.get('qsv_hevc_ok'),
|
||||
'av1_decode_ok': proof.get('av1_decode_ok'), 'av1_encode_ok': proof.get('av1_encode_ok'),
|
||||
'active_jobs': sample['active_jobs'], 'concurrency': sample['concurrency'],
|
||||
'gpu_util': last.get('engine_busy_max'), 'temps': sensors_data['temperatures_c'],
|
||||
'disk_free': sample['disk_free_bytes'],
|
||||
'ram_total_bytes': sensors_data['ram_total_bytes'],
|
||||
'ram_available_bytes': sensors_data['ram_available_bytes'],
|
||||
'on_ac': sample['on_ac'], 'battery': sensors_data['battery'],
|
||||
'recent_download_mbps': rates['recent_download_mbps'],
|
||||
'recent_upload_mbps': rates['recent_upload_mbps'],
|
||||
'jobs_completed': meter.data.get('jobs_completed', 0),
|
||||
'jobs_failed': meter.data.get('jobs_failed', 0),
|
||||
'last_job_duration': meter.data.get('last_job_duration_seconds'),
|
||||
'last_job_rtf': meter.data.get('last_job_realtime_factor'),
|
||||
'tested_at': proof.get('tested_at'), 'boot_id': hw.get('boot_id'),
|
||||
'driver_version': hw.get('driver_version')}
|
||||
|
||||
|
||||
def self_test_marker_stale(hw, proof):
|
||||
"""Re-proof triggers: boot change, GPU driver change, or 7-day proof age.
|
||||
(Extension of the Linux boot-id rule: Windows Update changes graphics
|
||||
drivers without a boot.)"""
|
||||
if not proof.get('tested_at') or time.time() - proof['tested_at'] > 7 * 86400:
|
||||
return True
|
||||
return proof.get('boot_id') != hw.get('boot_id') or proof.get('driver_version') != hw.get('driver_version')
|
||||
|
||||
|
||||
def daemon():
|
||||
raise SystemExit('daemon loop is a validation item; run status/network-test here, '
|
||||
'see windows-worker-result.md section 6 before production enablement')
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('command', choices=['status', 'network-test', 'drain'])
|
||||
args = parser.parse_args()
|
||||
if args.command == 'status':
|
||||
print(json.dumps(status(), indent=1))
|
||||
elif args.command == 'network-test':
|
||||
cfg = config()
|
||||
rates = transfer_probe(cfg)
|
||||
print(json.dumps(rates, indent=1))
|
||||
elif args.command == 'drain':
|
||||
cfg = config()
|
||||
print(json.dumps(ipc('graceful-shutdown', cfg.get('node_binary', 'node'))))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
35
infra/peertube-workers/win/register-from-ssh.py
Normal file
35
infra/peertube-workers/win/register-from-ssh.py
Normal file
|
|
@ -0,0 +1,35 @@
|
|||
"""Pipe the existing registration secret directly between SSH sessions. No file/argv/log."""
|
||||
import argparse,json,re,shlex,subprocess,tempfile
|
||||
from pathlib import Path
|
||||
parser=argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('--worker',required=True,help='SSH target user@WSL-host')
|
||||
parser.add_argument('--name',required=True)
|
||||
parser.add_argument('--root',default=r'C:\pt-worker')
|
||||
parser.add_argument('--server',default='root@100.92.231.98')
|
||||
parser.add_argument('--server-key',default=str(Path.home()/'.ssh/id_rsa'))
|
||||
parser.add_argument('--worker-key',default=str(Path.home()/'.ssh/devserver'))
|
||||
args=parser.parse_args()
|
||||
assert re.fullmatch(r'[A-Za-z0-9_-]+',args.name),'Use a simple unique runner name'
|
||||
assert re.fullmatch(r'[A-Za-z]:\\[^\r\n]+',args.root),'Require local Windows drive root'
|
||||
SSH=['ssh','-o','BatchMode=yes','-o','StrictHostKeyChecking=yes']
|
||||
server=SSH+['-i',args.server_key,args.server]
|
||||
worker=SSH+['-i',args.worker_key,args.worker]
|
||||
query='SELECT "registrationToken" FROM "runnerRegistrationToken" WHERE id=1;'
|
||||
command="docker exec -i occ-tube-peertube-postgres-1 sh -c 'exec psql -X -v ON_ERROR_STOP=1 -U \"$POSTGRES_USER\" -d peertube -At'"
|
||||
# metadata check is independent of, and never prints, token material.
|
||||
p=subprocess.run(server+[command],input=f"SELECT (SELECT count(*) FROM \"runnerRegistrationToken\"),(SELECT count(*) FROM \"runner\" WHERE name='{args.name}');",text=True,capture_output=True,check=True)
|
||||
assert p.stdout.strip()=='1|0','Expected existing token and no duplicate runner'
|
||||
marker=Path(tempfile.gettempdir())/('pt-worker-'+args.name+'-enrollment.pending')
|
||||
assert not marker.exists(),'Previous result uncertain; inspect server before retrying'
|
||||
marker.write_text('Registration attempted; contains no secrets.\n')
|
||||
source=subprocess.Popen(server+[command],stdin=subprocess.PIPE,stdout=subprocess.PIPE,stderr=subprocess.PIPE)
|
||||
source.stdin.write(query.encode());source.stdin.close()
|
||||
node='/mnt/'+args.root[0].lower()+args.root[2:].replace('\\','/')+'/node/node.exe'
|
||||
remote=shlex.join([node,args.root+r'\bin\ipc-nvenc.cjs','register',args.root])
|
||||
receiver=subprocess.run(worker+[remote],stdin=source.stdout,capture_output=True)
|
||||
source.stdout.close();source.wait(timeout=20)
|
||||
if source.returncode or receiver.returncode:
|
||||
raise SystemExit('Registration failed; secret-bearing outputs suppressed, inspect safe metadata before retry')
|
||||
assert json.loads(receiver.stdout)['success'] is True
|
||||
marker.unlink()
|
||||
print(json.dumps({'runner_name':args.name,'registration_success':True,'secret_transport':'SSH stdout->SSH stdin->official named-pipe IPC','secret_saved':False}))
|
||||
65
infra/peertube-workers/win/runner-supervisor.cjs
Normal file
65
infra/peertube-workers/win/runner-supervisor.cjs
Normal file
|
|
@ -0,0 +1,65 @@
|
|||
'use strict';
|
||||
// Both WinSW and interactive validation use this supervisor. Only official IPC
|
||||
// drains the runner; loss of admission never kills an in-flight video job.
|
||||
const fs = require('node:fs');
|
||||
const path = require('node:path');
|
||||
const { spawn, execFile } = require('node:child_process');
|
||||
const root = process.env.PT_ROOT || 'C:\\pt-worker';
|
||||
const state = path.join(root, 'state');
|
||||
const cfg = JSON.parse(fs.readFileSync(path.join(state, 'worker.json'), 'utf8').replace(/^\uFEFF/, ''));
|
||||
const env = { ...process.env, PT_ROOT: root, PT_STATE_DIR: state,
|
||||
PT_WORKER_CONFIG: path.join(state, 'worker.json'),
|
||||
APPDATA: path.join(root, 'profile', 'Roaming'), LOCALAPPDATA: path.join(root, 'profile', 'Local'),
|
||||
TMP: path.join(root, 'tmp'), TEMP: path.join(root, 'tmp'),
|
||||
FFMPEG_PATH: path.join(root, 'bin', 'ffmpeg-nvenc.exe'),
|
||||
FFPROBE_PATH: path.join(root, 'bin', 'ffprobe.exe'), PT_NVENC_DEVICE: String(cfg.nvenc_device || 0) };
|
||||
let child = null, draining = false, asking = false, stopping = false, restartAfter = 0;
|
||||
function read(name) { try { return JSON.parse(fs.readFileSync(path.join(state, name), 'utf8').replace(/^\uFEFF/, '')); } catch { return {}; } }
|
||||
function emit(data) { fs.appendFileSync(path.join(state, 'events.jsonl'), JSON.stringify({ time: Date.now()/1000, ...data }) + '\n'); }
|
||||
function atomic(name, data) { const file = path.join(state, name), tmp = file+'.'+process.pid+'.tmp'; fs.writeFileSync(tmp, JSON.stringify(data)); fs.renameSync(tmp, file); }
|
||||
function drain() {
|
||||
if (!child || asking || draining) return;
|
||||
const target = child;
|
||||
asking = true;
|
||||
execFile(cfg.node_binary, [path.join(root, 'bin', 'ipc-nvenc.cjs'), 'graceful-shutdown'],
|
||||
{ env, timeout: 17000, windowsHide: true }, err => { asking = false; if (!err && child === target) draining = true; });
|
||||
}
|
||||
function tick() {
|
||||
if (fs.existsSync(path.join(state, 'supervisor-stop'))) stopping = true;
|
||||
const now = Date.now()/1000, admission = read('status.json'), control = read('control.json');
|
||||
const allow = !stopping && control.paused === false && admission.admission_allowed === true &&
|
||||
now - admission.updated_at < 30 && ['AVAILABLE', 'BUSY'].includes(admission.status) &&
|
||||
!fs.existsSync(path.join(state, 'gpu-unavailable'));
|
||||
if (child && !allow) drain();
|
||||
if (!child && allow && now >= restartAfter) {
|
||||
draining = false;
|
||||
child = spawn(cfg.node_binary, ['--require', path.join(root, 'bin', 'observe.cjs'),
|
||||
path.join(root, 'runner', 'node_modules', '@peertube', 'peertube-runner', 'dist', 'peertube-runner.js'),
|
||||
'server', '--enable-job', 'vod-web-video-transcoding', '--enable-job', 'vod-hls-transcoding',
|
||||
'--enable-job', 'vod-audio-merge-transcoding'], { env, windowsHide: true, stdio: ['ignore', 'pipe', 'pipe'] });
|
||||
// Upstream logs can contain tokens. Retain only known safe milestones.
|
||||
let buffer = '';
|
||||
for (const stream of [child.stdout, child.stderr]) stream.on('data', chunk => {
|
||||
buffer += chunk.toString();
|
||||
const lines = buffer.split(/\r?\n/); buffer = lines.pop().slice(-8192);
|
||||
for (const line of lines) {
|
||||
const match = line.match(/(Finished processing|Processing) job of type ([\w-]+): ([a-f\d-]{36})/);
|
||||
if (match) emit({ kind: 'runner_milestone', phase: match[1].startsWith('Finished') ? 'completed' : 'processing', job_type: match[2], uuid: match[3] });
|
||||
if (line.includes('Connected to ') && line.includes('/runners')) emit({kind: 'runner_connected'});
|
||||
if (line.includes('Cannot process job')) emit({kind: 'runner_error', reason: 'process_job_failed'});
|
||||
}
|
||||
});
|
||||
child.once('error', () => { emit({kind: 'runner_error', reason: 'spawn_failed'}); });
|
||||
child.once('exit', code => {
|
||||
emit({kind: 'runner_exit', code, graceful_requested: draining || asking});
|
||||
child = null; draining = false; asking = false; restartAfter = Date.now()/1000 + 15;
|
||||
});
|
||||
emit({kind: 'runner_start', concurrency: cfg.concurrency});
|
||||
}
|
||||
atomic('runner.json', {running: !!child, draining, updated_at: now, pid: child?.pid || null});
|
||||
if (stopping && !child) process.exit(0);
|
||||
}
|
||||
fs.rmSync(path.join(state, 'supervisor-stop'), {force: true});
|
||||
process.on('SIGINT', () => { stopping = true; drain(); });
|
||||
process.on('SIGTERM', () => { stopping = true; drain(); });
|
||||
setInterval(tick, 1000); tick();
|
||||
42
infra/peertube-workers/win/selftest-nvenc.ps1
Normal file
42
infra/peertube-workers/win/selftest-nvenc.ps1
Normal file
|
|
@ -0,0 +1,42 @@
|
|||
param([string]$Root='C:\pt-worker', [int]$Device=0)
|
||||
$ErrorActionPreference='Stop'
|
||||
$ProgressPreference='SilentlyContinue'
|
||||
$ffmpeg=Join-Path $Root 'bin\ffmpeg.exe'
|
||||
$state=Join-Path $Root 'state'
|
||||
New-Item -ItemType Directory -Force $state,(Join-Path $Root 'tmp') | Out-Null
|
||||
$gpu=& nvidia-smi.exe -i $Device --query-gpu=name,uuid,driver_version --format=csv,noheader,nounits
|
||||
if($LASTEXITCODE){throw 'nvidia-smi identity failed'}
|
||||
$identity=$gpu -split ',\s*'
|
||||
$boot=(Get-CimInstance Win32_OperatingSystem).LastBootUpTime.ToUniversalTime().ToString('o')
|
||||
$proof=[ordered]@{tested_at=[DateTimeOffset]::UtcNow.ToUnixTimeSeconds();boot_id=$boot;driver_version=$identity[2];gpu_model=$identity[0];gpu_uuid=$identity[1];device=$Device;nvenc_h264_ok=$false;nvenc_hevc_ok=$false;av1_encode_ok=$false;av1_decode_ok=$null;tests=@()}
|
||||
foreach($codec in @('h264','hevc')){
|
||||
$out=Join-Path $Root "tmp\selftest-$codec.mp4"
|
||||
$err=Join-Path $state "selftest-$codec.stderr"
|
||||
$stdout=Join-Path $state "selftest-$codec.stdout"
|
||||
$argv=@('-hide_banner','-nostdin','-y','-re','-f','lavfi','-i','testsrc2=size=1920x1080:rate=30','-t','5','-an','-c:v',"${codec}_nvenc",'-gpu',"$Device",'-preset','p4','-pix_fmt','yuv420p',$out)
|
||||
$start=Get-Date
|
||||
$p=Start-Process $ffmpeg -ArgumentList $argv -PassThru -NoNewWindow -RedirectStandardError $err -RedirectStandardOutput $stdout
|
||||
$null=$p.Handle
|
||||
$samples=@(); $sessions=@()
|
||||
while(!( $p.HasExited ) -and ((Get-Date)-$start).TotalSeconds -lt 30){
|
||||
$value=& nvidia-smi.exe -i $Device --query-gpu=utilization.encoder,encoder.stats.sessionCount,encoder.stats.averageFps --format=csv,noheader,nounits
|
||||
if($LASTEXITCODE -eq 0){
|
||||
$cells=$value -split ',\s*'
|
||||
$n=0.0
|
||||
if([double]::TryParse($cells[0],[ref]$n)){$samples+=$n}
|
||||
if($cells.Count -ge 3){$sessions+=@{count=$cells[1];fps=$cells[2]}}
|
||||
}
|
||||
Start-Sleep -Milliseconds 200
|
||||
$p.Refresh()
|
||||
}
|
||||
if(!$p.HasExited){$p.Kill();throw 'NVENC self-test exceeded 30 second deadline'}
|
||||
$p.WaitForExit()
|
||||
$max=if($samples.Count){($samples | Measure-Object -Maximum).Maximum}else{$null}
|
||||
$ok=$p.ExitCode -eq 0 -and $null -ne $max -and $max -gt 0
|
||||
$proof["nvenc_${codec}_ok"]=$ok
|
||||
$proof.tests+=@{codec=$codec;encoder="${codec}_nvenc";width=1920;height=1080;duration=5;exit_code=$p.ExitCode;seconds=((Get-Date)-$start).TotalSeconds;encoder_util_max=$max;encoder_util_samples=$samples;encoder_stats=$sessions;stderr_file=$err}
|
||||
if(!$ok){$proof.failure=if($p.ExitCode -ne 0){Get-Content $err -Raw}else{'No positive NVIDIA encoder utilization: hardware proof refused'};break}
|
||||
}
|
||||
$proof | ConvertTo-Json -Depth 8 | Set-Content (Join-Path $state 'nvenc.json') -Encoding UTF8
|
||||
$proof | ConvertTo-Json -Depth 8
|
||||
if(!$proof.nvenc_h264_ok -or !$proof.nvenc_hevc_ok){exit 78}
|
||||
159
infra/peertube-workers/win/selftest-qsv.ps1
Normal file
159
infra/peertube-workers/win/selftest-qsv.ps1
Normal file
|
|
@ -0,0 +1,159 @@
|
|||
# Windows QSV self-test for the PeerTube transcode worker (validator).
|
||||
#
|
||||
# Gates registration. A real encode MUST produce a positive PDH
|
||||
# \GPU Engine\Utilization Percentage (engtype_VideoEncode) signal; exit 0 alone
|
||||
# is insufficient. No CPU fallback exists anywhere in this worker.
|
||||
#
|
||||
# Usage (PowerShell 5.1+, from C:\pt-worker\bin after the ffmpeg zip is unpacked):
|
||||
# powershell -ExecutionPolicy Bypass -File selftest-qsv.ps1
|
||||
# Writes C:\pt-worker\state\qsv.json on completion.
|
||||
|
||||
$ErrorActionPreference = 'Continue'
|
||||
$State = 'C:\pt-worker\state'
|
||||
$Bin = 'C:\pt-worker\bin'
|
||||
$FFmpeg = Join-Path $Bin 'ffmpeg.exe'
|
||||
$FFprobe = Join-Path $Bin 'ffprobe.exe'
|
||||
New-Item -ItemType Directory -Force -Path $State | Out-Null
|
||||
|
||||
function HexToBytes([string]$hex) {
|
||||
$bytes = New-Object byte[] ($hex.Length / 2)
|
||||
for ($i = 0; $i -lt $bytes.Length; $i++) { $bytes[$i] = [Convert]::ToByte($hex.Substring($i * 2, 2), 16) }
|
||||
return ,$bytes
|
||||
}
|
||||
|
||||
# 5-second synthetic 1080p30 clip (testsrc2 -> x264 -> mp4). No network, no libx264 use at encode time.
|
||||
$TestClip = Join-Path $State 'self-test-source.mp4'
|
||||
& $FFmpeg -hide_banner -loglevel error -y -f lavfi -i 'testsrc2=size=1920x1080:rate=30' -t 5 -c:v libx264 -pix_fmt yuv420p $TestClip
|
||||
if ($LASTEXITCODE -ne 0) { Write-Error 'failed to build self-test source clip'; exit 2 }
|
||||
|
||||
function Get-VideoEncodeBusy([int]$ProcId, [double]$Seconds = 3.0) {
|
||||
# Sum of engtype_VideoEncode instances of \GPU Engine(pid_<id>_*)\Utilization Percentage.
|
||||
# Locale-sensitive counter names/decimal separators: unparseable => -1 (unknown, never 0).
|
||||
$out = & typeperf "\GPU Engine(pid_$ProcId`_*)\Utilization Percentage" -sc 1 2>$null
|
||||
$rows = @($out | Where-Object { $_ -like '"*' })
|
||||
if ($rows.Count -lt 2) { return -1 }
|
||||
$headers = ($rows[0] -replace '^"|"$','').split('","')
|
||||
$values = ($rows[1] -replace '^"|"$','').split('","')
|
||||
$total = 0.0; $seen = $false
|
||||
for ($i = 0; $i -lt $headers.Count; $i++) {
|
||||
if ($headers[$i] -notmatch 'engtype_VideoEncode') { continue }
|
||||
if ($i -ge $values.Count) { return -1 }
|
||||
$raw = $values[$i]
|
||||
$num = 0.0
|
||||
if (-not [double]::TryParse($raw, [Globalization.NumberStyles]::Float,
|
||||
[Globalization.CultureInfo]::InvariantCulture, [ref]$num)) {
|
||||
$raw = $raw -replace '\.', '' -replace ',', '.'
|
||||
if (-not [double]::TryParse($raw, [Globalization.NumberStyles]::Float,
|
||||
[Globalization.CultureInfo]::InvariantCulture, [ref]$num)) { return -1 }
|
||||
}
|
||||
$total += $num; $seen = $true
|
||||
}
|
||||
if (-not $seen) { return -1 }
|
||||
return $total
|
||||
}
|
||||
|
||||
function Test-RealEncode([string]$Name, [string[]]$ExtraArgs, [int]$ProcId) {
|
||||
$log = Join-Path $State "selftest-$Name.log"
|
||||
$proc = Start-Process -FilePath $FFmpeg -ArgumentList $ExtraArgs -NoNewWindow -PassThru `
|
||||
-RedirectStandardError $log -RedirectStandardOutput (Join-Path $State "selftest-$Name.out")
|
||||
$maxBusy = 0.0; $samples = 0; $readable = $false
|
||||
while (-not $proc.HasExited) {
|
||||
$busy = Get-VideoEncodeBusy -ProcId ($proc.Id)
|
||||
if ($busy -ge 0) { $readable = $true; $samples++; if ($busy -gt $maxBusy) { $maxBusy = $busy } }
|
||||
Start-Sleep -Milliseconds 800
|
||||
}
|
||||
$proc.WaitForExit() | Out-Null
|
||||
return @{ name = $Name; exit = $proc.ExitCode; max_busy = $maxBusy;
|
||||
samples = $samples; counters_readable = $readable; log = $log }
|
||||
}
|
||||
|
||||
$report = [ordered]@{
|
||||
schema = 'pt-worker-qsv-selftest/1'
|
||||
tested_at = [DateTimeOffset]::UtcNow.ToUnixTimeSeconds()
|
||||
gpu = @(); driver_version = $null
|
||||
dispatcher_dlls = @{ libmfx = (Test-Path (Join-Path $Bin 'libmfx64-gen.dll'));
|
||||
libvpl = (Test-Path (Join-Path $Bin 'libvpl64-1.dll')) }
|
||||
ffmpeg_version = $null; qsv_h264_ok = $false; qsv_hevc_ok = $false
|
||||
h264_mf_ok = $false; av1_decode_ok = $null; av1_encode_ok = $false
|
||||
tests = @{}; failures = @(); boot_id = (Get-CimInstance Win32_OperatingSystem).LastBootUpTime.ToString('o')
|
||||
}
|
||||
|
||||
$gpus = @(); try {
|
||||
$gpus = @(Get-CimInstance Win32_VideoController | Select-Object Name, DriverVersion, Status)
|
||||
} catch { }
|
||||
$report.gpu = $gpus
|
||||
$intel = $gpus | Where-Object { $_.Name -match 'Intel.*(?:Iris|UHD|Graphics)' } | Select-Object -First 1
|
||||
if (-not $intel) { $report.failures += 'no Intel graphics adapter found via WMI' }
|
||||
else { $report.driver_version = $intel.DriverVersion }
|
||||
$report.ffmpeg_version = (& $FFmpeg -version | Select-Object -First 1)
|
||||
|
||||
# --- capability listing (necessary, not sufficient) ---
|
||||
$encoders = (& $FFmpeg -hide_banner -encoders) -join "`n"
|
||||
foreach ($codec in @('h264_qsv', 'hevc_qsv', 'h264_mf', 'av1_qsv')) {
|
||||
$report.tests["encoder_$codec"] = ($encoders -match [regex]::Escape($codec))
|
||||
}
|
||||
if ($report.tests.encoder_h264_qsv -ne $true) { $report.failures += 'h264_qsv encoder missing from ffmpeg build' }
|
||||
|
||||
# --- session init proof (device open) ---
|
||||
$initLog = Join-Path $State 'selftest-qsv-init.log'
|
||||
& $FFmpeg -hide_banner -v verbose -y -init_hw_device qsv=hw -f lavfi -i 'testsrc2=size=1280x720:rate=30' `
|
||||
-frames:v 10 -c:v h264_qsv -f null - 2> $initLog
|
||||
$initExit = $LASTEXITCODE
|
||||
$initText = Get-Content $initLog -Raw
|
||||
$report.tests.qsv_device_init_exit = $initExit
|
||||
if ($initExit -ne 0) { $report.failures += "h264_qsv session init failed (exit $initExit); see selftest-qsv-init.log" }
|
||||
|
||||
# --- REAL H.264 1080p encode with mandatory VideoEncode counter proof ---
|
||||
$h264 = Test-RealEncode -Name 'qsv-h264' -ProcId 0 -ExtraArgs @(
|
||||
'-hide_banner', '-nostdin', '-v', 'info',
|
||||
'-init_hw_device', 'qsv=qs', '-filter_hw_device', 'qs',
|
||||
'-hwaccel', 'qsv', '-hwaccel_output_format', 'qsv', '-c:v', 'h264_qsv',
|
||||
'-i', $TestClip,
|
||||
'-vf', 'scale_qsv=w=1280:h=720:format=nv12',
|
||||
'-c:v', 'h264_qsv', '-preset', 'medium', '-b:v', '2M', '-maxrate', '2M', '-bufsize', '4M', '-bf', '2',
|
||||
'-an', '-f', 'null', '-'
|
||||
)
|
||||
$report.tests.h264 = $h264
|
||||
if ($h264.exit -eq 0 -and $h264.counters_readable -and $h264.samples -ge 3 -and $h264.max_busy -gt 0) {
|
||||
$report.qsv_h264_ok = $true
|
||||
} elseif ($h264.exit -eq 0) {
|
||||
$report.failures += 'h264_qsv exited 0 without a positive VideoEncode counter; not accepted as hardware proof'
|
||||
}
|
||||
|
||||
# --- HEVC encode attempt (capability discovery; not enabled in production) ---
|
||||
$hevc = Test-RealEncode -Name 'qsv-hevc' -ProcId 0 -ExtraArgs @(
|
||||
'-hide_banner', '-nostdin', '-v', 'info',
|
||||
'-init_hw_device', 'qsv=qs', '-filter_hw_device', 'qs',
|
||||
'-hwaccel', 'qsv', '-hwaccel_output_format', 'qsv', '-c:v', 'hevc_qsv',
|
||||
'-i', $TestClip,
|
||||
'-vf', 'scale_qsv=w=1280:h=720:format=nv12',
|
||||
'-c:v', 'hevc_qsv', '-preset', 'medium', '-b:v', '2M',
|
||||
'-an', '-f', 'null', '-'
|
||||
)
|
||||
$report.tests.hevc = $hevc
|
||||
$report.qsv_hevc_ok = ($hevc.exit -eq 0 -and $hevc.counters_readable -and $hevc.samples -ge 3 -and $hevc.max_busy -gt 0)
|
||||
|
||||
# --- AV1: decode-only expectation on Alder Lake; verify, never assume ---
|
||||
# Build an AV1 source only if an AV1 encoder (software) exists in this build;
|
||||
# decoding AV1 hardware-wise is the capability under test.
|
||||
$av1DecodeOk = $false
|
||||
$av1Source = Join-Path $State 'self-test-source-av1.mp4'
|
||||
& $FFmpeg -hide_banner -loglevel error -y -f lavfi -i 'testsrc2=size=320x180:rate=30' -t 1 -c:v libaom-av1 $av1Source 2>$null
|
||||
if ($LASTEXITCODE -eq 0) {
|
||||
$av1Log = Join-Path $State 'selftest-av1-decode.log'
|
||||
& $FFmpeg -hide_banner -v error -y -c:v av1_qsv -i $av1Source -frames:v 5 -f null - 2> $av1Log
|
||||
$av1DecodeOk = ($LASTEXITCODE -eq 0)
|
||||
}
|
||||
$report.av1_decode_ok = $av1DecodeOk
|
||||
$encAv1 = ($encoders -match 'av1_qsv')
|
||||
$report.av1_encode_ok = $false
|
||||
$report.tests.av1_encoder_present = $encAv1
|
||||
|
||||
# --- Record + verdict. NO CPU fallback exists; registration is blocked on failure. ---
|
||||
$report | ConvertTo-Json -Depth 5 | Set-Content -Encoding UTF8 (Join-Path $State 'qsv.json')
|
||||
if ($report.failures.Count -gt 0 -or -not $report.qsv_h264_ok) {
|
||||
Write-Host 'SELF-TEST FAILED:' ($report.failures -join '; ')
|
||||
exit 1
|
||||
}
|
||||
Write-Host 'SELF-TEST PASSED: h264_qsv with positive VideoEncode counter proof'
|
||||
exit 0
|
||||
8
infra/peertube-workers/win/service-stop.cjs
Normal file
8
infra/peertube-workers/win/service-stop.cjs
Normal file
|
|
@ -0,0 +1,8 @@
|
|||
'use strict';
|
||||
const fs=require('node:fs'),path=require('node:path');
|
||||
const root=process.env.PT_ROOT || 'C:\\pt-worker', state=path.join(root,'state');
|
||||
// The supervisor owns graceful shutdown; only stop after all processing jobs exit.
|
||||
fs.writeFileSync(path.join(state,'supervisor-stop'), 'requested');
|
||||
const timer=setInterval(()=>{
|
||||
try {const r=JSON.parse(fs.readFileSync(path.join(state,'runner.json'),'utf8'));if(!r.running){clearInterval(timer);process.exit(0);}}catch{}
|
||||
},1000);
|
||||
87
infra/peertube-workers/win/tests/test_nvenc.py
Normal file
87
infra/peertube-workers/win/tests/test_nvenc.py
Normal file
|
|
@ -0,0 +1,87 @@
|
|||
import importlib.util
|
||||
import os
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
import time
|
||||
import unittest
|
||||
|
||||
WIN = Path(__file__).resolve().parents[1]
|
||||
ROOT = WIN.parent
|
||||
|
||||
def load(name, path):
|
||||
spec = importlib.util.spec_from_file_location(name, path)
|
||||
m = importlib.util.module_from_spec(spec); spec.loader.exec_module(m)
|
||||
return m
|
||||
|
||||
shim = load('nvenc', WIN / 'ffmpeg-nvenc.py')
|
||||
worker = load('nvenc_worker', WIN / 'pt-worker-nvenc.py')
|
||||
linux = load('linux_worker', ROOT / 'pt-worker.py')
|
||||
PROBE = {'streams': [{'codec_type': 'video', 'codec_name': 'h264', 'pix_fmt': 'yuv420p', 'width': 1920, 'height': 1080}], 'format': {'duration': '10'}}
|
||||
|
||||
class AdapterTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.temp = tempfile.TemporaryDirectory(); self.old = shim.STATE
|
||||
shim.STATE = Path(self.temp.name)
|
||||
def tearDown(self):
|
||||
shim.STATE = self.old; self.temp.cleanup()
|
||||
def test_runner_default_becomes_nvdec_cuda_scale_nvenc(self):
|
||||
args, info = shim.rewrite(['-i','source.mp4','-vcodec','libx264','-vf','scale=w=-2:h=720','-pix_fmt','yuv420p','-preset','veryfast','-maxrate:v','2000000','-bufsize:v','4000000','-b_strategy','1','-bf','16','-g:v','60','-r','30','-c:a','aac','-movflags','faststart','-f','mp4','out.mp4'], PROBE)
|
||||
self.assertIn('h264_nvenc', args); self.assertIn('cuda', args)
|
||||
self.assertIn('scale_cuda=w=1280:h=720:format=nv12', args)
|
||||
self.assertIn('p4', args); self.assertNotIn('libx264', args)
|
||||
self.assertNotIn('-b_strategy', args); self.assertEqual(info['target_height'],720)
|
||||
def test_unknown_args_aliases_codecs_filters_and_extra_outputs_refused(self):
|
||||
base=['-i','a','-c:v','libx264']
|
||||
for extra in (['-unknown','1'],['-c:v:0','libx264'],['-c','libx264'],['-filter_complex','null'],['-vf','scale=640:360,eq=contrast=2'],['-hwaccel','none'],['-x264-params','no-scenecut=1'],['other.mp4'],['-c:v','libx265']):
|
||||
with self.subTest(extra=extra),self.assertRaises(ValueError):
|
||||
shim.rewrite(base+extra+['out'],PROBE)
|
||||
def test_source_rotation_hdr_unknown_codec_refused(self):
|
||||
for change in ({'codec_name':'mpeg4'},{'side_data_list':[{'rotation':90}]},{'color_transfer':'smpte2084'}):
|
||||
p={**PROBE,'streams':[{**PROBE['streams'][0],**change}]}
|
||||
with self.assertRaises(ValueError):shim.rewrite(['-i','a','-c:v','libx264','out'],p)
|
||||
def test_hevc_sdr_source_cuda_decode(self):
|
||||
p={**PROBE,'streams':[{**PROBE['streams'][0],'codec_name':'hevc','pix_fmt':'yuv420p10le'}]}
|
||||
args,_=shim.rewrite(['-i','a','-c:v','libx264','out'],p)
|
||||
self.assertIn('h264_nvenc',args);self.assertIn('cuda',args)
|
||||
def test_hevc_target_is_hardware_only_and_unknown_target_refused(self):
|
||||
old=shim.TARGET
|
||||
try:
|
||||
shim.TARGET='hevc_nvenc'
|
||||
args,info=shim.rewrite(['-i','a','-c:v','libx264','out'],PROBE)
|
||||
self.assertIn('hevc_nvenc',args);self.assertEqual(info['target_codec'],'hevc')
|
||||
shim.TARGET='libx265'
|
||||
with self.assertRaises(ValueError):shim.rewrite(['-i','a','-c:v','libx264','out'],PROBE)
|
||||
finally:shim.TARGET=old
|
||||
def test_gpu_unavailable_marker_refuses_video_and_remux(self):
|
||||
(shim.STATE/'gpu-unavailable').touch()
|
||||
for args in (['-i','a','-c:v','libx264','out'],['-i','a','-c','copy','out']):
|
||||
with self.assertRaises(ValueError):shim.rewrite(args,PROBE)
|
||||
def test_queries_audio_and_copy_are_explicit(self):
|
||||
for args in (['-version'],['-encoders'],['-i','a','-vn','-c:a','aac','out'],['-i','a','-c','copy','out']):
|
||||
out,info=shim.rewrite(args,PROBE);self.assertEqual(out,args);self.assertIsNone(info)
|
||||
with self.assertRaises(ValueError):shim.rewrite(['-i','a','-vn','-c:v','libx264','out'],PROBE)
|
||||
|
||||
class AdmissionTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.temp=tempfile.TemporaryDirectory();self.old=(linux.STATE,worker.STATE)
|
||||
linux.STATE=worker.STATE=Path(self.temp.name);self.now=time.time()
|
||||
def tearDown(self):linux.STATE,worker.STATE=self.old;self.temp.cleanup()
|
||||
def test_linux_windows_decisions_match_for_entire_sequence(self):
|
||||
overrides=[{}, {}, {}, {'active_jobs':2}, {'on_ac':False}, {'on_ac':None}, {}, {}, {}, {'temperature_c':90}, {'temperature_c':50}, {'gpu_ready':False}, {'disk_free_bytes':0}, {'network_ok':False}, {'tailscale_path':'DERP'}]
|
||||
traces=[]
|
||||
for mod in (linux,worker):
|
||||
meter=mod.Meter();policy=mod.Admission({})
|
||||
for d in ('download','upload'):meter.consume({'kind':'transfer','time':self.now,'ok':True,'direction':d,'bytes':10_000_000,'seconds':1})
|
||||
trace=[]
|
||||
for i,o in enumerate(overrides):
|
||||
sample={'tailscale_path':'direct','latency_ms':5,'network_ok':True,'registered':True,'gpu_ready':True,'gpu_accessible':True,'gpu_tested_at':self.now,'on_ac':True,'disk_free_bytes':100*1024**3,'temperature_c':50,'throttle_counts':{},'active_jobs':0,**o}
|
||||
trace.append(policy.decide(sample,meter,{'paused':False},self.now+i*130))
|
||||
traces.append(trace)
|
||||
self.assertEqual(traces[0],traces[1]);self.assertEqual(traces[0][:6],['NETWORK_DEGRADED','NETWORK_DEGRADED','AVAILABLE','BUSY','DRAINING','DRAINING'])
|
||||
def test_gpu_failure_latched_until_later_proof(self):
|
||||
policy=worker.Admission({});meter=worker.Meter()
|
||||
meter.consume({'kind':'gpu_failure','time':self.now+1})
|
||||
sample={'tailscale_path':'direct','latency_ms':1,'network_ok':True,'registered':True,'gpu_ready':True,'gpu_accessible':True,'gpu_tested_at':self.now,'on_ac':True,'disk_free_bytes':100*1024**3,'temperature_c':None,'throttle_counts':{},'active_jobs':0}
|
||||
self.assertEqual(policy.decide(sample,meter,{'paused':False},self.now+2),'GPU_UNAVAILABLE')
|
||||
|
||||
if __name__=='__main__':unittest.main()
|
||||
247
infra/peertube-workers/win/tests/test_win_worker.py
Normal file
247
infra/peertube-workers/win/tests/test_win_worker.py
Normal file
|
|
@ -0,0 +1,247 @@
|
|||
"""Tests for the Windows worker port (win/). Runs on Linux; no GPU, no Windows host.
|
||||
|
||||
Covers the three pieces the wave doc says must hold before first hardware contact:
|
||||
1. ffmpeg-qsv.py argument rewriting matches the validated Linux adapter's
|
||||
contract and fails closed on unreviewed commands (exit 78).
|
||||
2. the capability-query passthrough list matches what the shipped runner
|
||||
actually generates (verified in the 0.2.0 bundle).
|
||||
3. pt-worker-win.py Meter/Admission semantics are byte-identical to the
|
||||
Linux controller's for every decision input (states, hysteresis, floors).
|
||||
"""
|
||||
import importlib.util
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
WIN = Path(__file__).resolve().parents[2] / 'win'
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
|
||||
|
||||
def load(name, path):
|
||||
spec = importlib.util.spec_from_file_location(name, path)
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
qsv = load('qsv_win', WIN / 'ffmpeg-qsv.py')
|
||||
win = load('worker_win', WIN / 'pt-worker-win.py')
|
||||
linux = load('worker_linux', ROOT / 'pt-worker.py')
|
||||
|
||||
PROBE = {'streams': [{'codec_type': 'video', 'codec_name': 'h264', 'pix_fmt': 'yuv420p',
|
||||
'width': 1920, 'height': 1080}],
|
||||
'format': {'duration': '10.0'}}
|
||||
|
||||
|
||||
class RewriteTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.old_state, self.old_probe = qsv.STATE, qsv.probe
|
||||
self.tmp = tempfile.TemporaryDirectory()
|
||||
qsv.STATE = Path(self.tmp.name)
|
||||
def stub_probe(path): # no real ffprobe.exe on this Linux host
|
||||
name = str(path).lower()
|
||||
if name.endswith(('.png', '.mjpeg', '.webp')):
|
||||
return {'streams': [{**PROBE['streams'][0], 'codec_name': 'png',
|
||||
'pix_fmt': 'rgba'}], 'format': {'duration': '0'}}
|
||||
return PROBE
|
||||
qsv.probe = stub_probe
|
||||
|
||||
def tearDown(self):
|
||||
qsv.STATE, qsv.probe = self.old_state, self.old_probe
|
||||
self.tmp.cleanup()
|
||||
|
||||
def rewritten(self, args, data=PROBE):
|
||||
out, _ = qsv.rewrite(args, input_probe=data)
|
||||
return out
|
||||
|
||||
def test_h264_web_video_becomes_qsv_pipeline(self):
|
||||
out = self.rewritten(['-i', 'in.mp4', '-vf', 'scale=w=-2:h=720', '-c:v', 'libx264',
|
||||
'-preset', 'veryfast', '-crf', '18', '-maxrate', '1500K',
|
||||
'-bufsize', '3000K', '-g', '96', '-x264-params', 'nofast-pskip=1',
|
||||
'-movflags', '+faststart', 'out.mp4'])
|
||||
text = ' '.join(out)
|
||||
self.assertIn('-hwaccel qsv', text)
|
||||
self.assertIn('-hwaccel_output_format qsv', text)
|
||||
self.assertIn('-c:v h264_qsv', text)
|
||||
self.assertIn('scale_qsv=w=1280:h=720', text)
|
||||
self.assertIn('-preset medium', text)
|
||||
self.assertIn('-b:v 1500K', text) # runner's rate cap kept, CRF dropped
|
||||
self.assertIn('-g 96', text) # runner's GOP kept
|
||||
self.assertNotIn('libx264', text)
|
||||
self.assertNotIn('-crf', text)
|
||||
|
||||
def test_hevc_source_uses_hevc_qsv_decode(self):
|
||||
data = {**PROBE, 'streams': [{**PROBE['streams'][0], 'codec_name': 'hevc', 'pix_fmt': 'yuv420p10le'}]}
|
||||
out = self.rewritten(['-i', 'in.mkv', '-c:v', 'libx264', 'out.mp4'], data)
|
||||
self.assertIn('-c:v hevc_qsv', ' '.join(out))
|
||||
|
||||
def test_no_vf_gets_default_scale_qsv(self):
|
||||
out = self.rewritten(['-i', 'in.mp4', '-c:v', 'libx264', 'out.mp4'])
|
||||
self.assertIn('scale_qsv=w=1920:h=1080', ' '.join(out))
|
||||
|
||||
def test_audio_only_passthrough(self):
|
||||
out, info = qsv.rewrite(['-i', 'in.mp4', '-vn', '-c:a', 'aac', 'out.m4a'])
|
||||
self.assertEqual(info, None)
|
||||
|
||||
def test_capability_queries_passthrough(self):
|
||||
for args in (['-version'], ['-encoders'], ['-hwaccels']):
|
||||
out, info = qsv.rewrite(args)
|
||||
self.assertEqual((out, info), (args, None))
|
||||
|
||||
def test_fail_closed_cpu_or_unknown(self):
|
||||
# Note: -vf contents are validated after the (stubbed) probe, matching
|
||||
# the Linux adapter, which needs the probe to resolve -2 dimensions.
|
||||
for args in (['-i', 'in.mp4', '-c:v', 'mpeg4', 'out.mp4'],
|
||||
['-i', 'in.mp4', '-c:v', 'libx264', '-filter_complex', 'x', 'out.mp4'],
|
||||
['-i', 'in.mp4', '-c:v', 'libx264', '-vf', 'scale=w=-2:h=720,hue=s=0', 'out.mp4'],
|
||||
['-i', 'in.png', '-c:v', 'libx264', '-vf', 'scale=w=-2:h=720', 'out.mp4'],
|
||||
['-i', 'in.mp4', '-c:v', 'libx264', '-vf', 'scale=w=0:h=720', 'out.mp4']):
|
||||
with self.assertRaises(ValueError):
|
||||
qsv.rewrite(args)
|
||||
|
||||
def test_refusal_process_exit_78(self):
|
||||
import os
|
||||
env = {**os.environ, 'PT_STATE_DIR': str(self.tmp.name), 'PT_FFPROBE_EXE': 'ffprobe.exe'}
|
||||
p = subprocess.run([sys.executable, str(WIN / 'ffmpeg-qsv.py'),
|
||||
'-i', 'x.mp4', '-c:v', 'libx264', 'o.mp4'],
|
||||
capture_output=True, text=True, env=env)
|
||||
self.assertEqual(p.returncode, 78)
|
||||
|
||||
def test_events_have_fixed_reasons_only(self):
|
||||
import os
|
||||
env = {**os.environ, 'PT_STATE_DIR': str(self.tmp.name), 'PT_FFPROBE_EXE': 'ffprobe.exe'}
|
||||
p = subprocess.run([sys.executable, str(WIN / 'ffmpeg-qsv.py'),
|
||||
'-i', 'x.mp4', '-c:v', 'libx264', 'o.mp4'],
|
||||
capture_output=True, text=True, env=env)
|
||||
events = [json.loads(line) for line in
|
||||
(Path(self.tmp.name) / 'events.jsonl').read_text().splitlines()]
|
||||
self.assertEqual(events[0]['kind'], 'gpu_failure')
|
||||
self.assertEqual(events[0]['reason'], 'ValueError')
|
||||
self.assertNotIn('x.mp4', p.stderr)
|
||||
|
||||
|
||||
class RunnerContractTests(unittest.TestCase):
|
||||
"""Verified directly in the shipped runner 0.2.0 bundle this session."""
|
||||
|
||||
def test_queries_actually_issued_by_runner(self):
|
||||
bundle = Path('/tmp/peertube-worker-inspect/runtime/node_modules/@peertube/peertube-runner'
|
||||
'/dist/peertube-runner.js')
|
||||
if not bundle.exists():
|
||||
self.skipTest('pinned runner bundle not present')
|
||||
text = bundle.read_text(errors='replace')
|
||||
# Exactly the queries fluent-ffmpeg's capability probe issues in 0.2.0
|
||||
# (-hwaccels is NOT issued; the shim passes it through anyway, harmlessly).
|
||||
for query in ('-version', '-formats', '-encoders', '-filters', '-codecs'):
|
||||
self.assertIn(query, text)
|
||||
self.assertNotIn('-hwaccels', text)
|
||||
self.assertIn('encodersToTry = this.availableEncoders.encodersToTry', text)
|
||||
self.assertIn('"libx264"', text)
|
||||
|
||||
def test_spawn_without_shell_and_ffprobe_sibling(self):
|
||||
bundle = Path('/tmp/peertube-worker-inspect/runtime/node_modules/@peertube/peertube-runner'
|
||||
'/dist/peertube-runner.js')
|
||||
if not bundle.exists():
|
||||
self.skipTest('pinned runner bundle not present')
|
||||
text = bundle.read_text(errors='replace')
|
||||
self.assertNotIn('shell: true', text) # EINVAL on .cmd/.bat shims
|
||||
self.assertIn('utils.isWindows ? "ffprobe.exe" : "ffprobe"', text)
|
||||
self.assertIn('!utils.isWindows', text) # `nice` skipped on win32
|
||||
|
||||
|
||||
class AdmissionParityTests(unittest.TestCase):
|
||||
"""The Windows controller must decide identically to the Linux controller."""
|
||||
|
||||
def setUp(self):
|
||||
self.tmp = tempfile.TemporaryDirectory()
|
||||
self.old_win, self.old_lin = win.STATE, linux.STATE
|
||||
win.STATE = linux.STATE = Path(self.tmp.name)
|
||||
self.now = time.time()
|
||||
|
||||
def tearDown(self):
|
||||
win.STATE, linux.STATE = self.old_win, self.old_lin
|
||||
self.tmp.cleanup()
|
||||
|
||||
def meter_with_transfers(self, module, mbps):
|
||||
meter = module.Meter()
|
||||
for direction in ('download', 'upload'):
|
||||
meter.consume({'kind': 'transfer', 'time': self.now, 'ok': True, 'direction': direction,
|
||||
'bytes': int(mbps * 1e6 / 8), 'seconds': 1})
|
||||
return meter
|
||||
|
||||
def sample(self, **overrides):
|
||||
base = {'tailscale_path': 'direct', 'latency_ms': 5, 'network_ok': True,
|
||||
'registered': True, 'gpu_ready': True, 'gpu_tested_at': self.now,
|
||||
'gpu_accessible': True, 'on_ac': True, 'disk_free_bytes': 100 * 1024 ** 3,
|
||||
'temperature_c': 50, 'throttle_counts': {}, 'active_jobs': 0}
|
||||
base.update(overrides)
|
||||
return base
|
||||
|
||||
def parity(self, overrides, control=None, decisions=1):
|
||||
control = control or {'paused': False}
|
||||
outputs = []
|
||||
for module in (linux, win):
|
||||
policy = module.Admission({})
|
||||
meter = self.meter_with_transfers(module, 50)
|
||||
for _ in range(decisions):
|
||||
outputs.append(policy.decide(self.sample(**overrides), meter, control, self.now))
|
||||
self.assertEqual(outputs[0], outputs[1],
|
||||
f'Linux and Windows controllers diverge on {overrides}: {outputs}')
|
||||
return outputs[0]
|
||||
|
||||
def test_happy_path(self):
|
||||
self.parity({}, decisions=4)
|
||||
|
||||
def test_power_loss_drains_identically(self):
|
||||
self.assertEqual(self.parity({'on_ac': False}), 'DRAINING')
|
||||
|
||||
def test_unknown_power_drains_identically(self):
|
||||
self.assertEqual(self.parity({'on_ac': None}), 'DRAINING')
|
||||
|
||||
def test_missing_gpu_proof_blocks_identically(self):
|
||||
self.assertEqual(self.parity({'gpu_ready': False, 'gpu_accessible': False}), 'GPU_UNAVAILABLE')
|
||||
|
||||
def test_gpu_failure_event_after_proof_blocks_identically(self):
|
||||
outputs = []
|
||||
for module in (linux, win):
|
||||
policy = module.Admission({})
|
||||
meter = self.meter_with_transfers(module, 50)
|
||||
meter.consume({'kind': 'gpu_failure', 'time': self.now + 1})
|
||||
outputs.append(policy.decide(self.sample(gpu_tested_at=self.now), meter,
|
||||
{'paused': False}, self.now + 2))
|
||||
self.assertEqual(outputs[0], 'GPU_UNAVAILABLE')
|
||||
self.assertEqual(outputs[0], outputs[1])
|
||||
|
||||
def test_low_disk_and_thermal(self):
|
||||
self.assertEqual(self.parity({'disk_free_bytes': 10 * 1024 ** 3}), 'LOW_DISK')
|
||||
self.assertEqual(self.parity({'temperature_c': 90}), 'THERMAL_THROTTLED')
|
||||
|
||||
def test_derp_disallowed_and_latency_ceiling(self):
|
||||
self.assertEqual(self.parity({'tailscale_path': 'DERP'}), 'NETWORK_DEGRADED')
|
||||
self.assertEqual(self.parity({'latency_ms': 500}), 'NETWORK_DEGRADED')
|
||||
|
||||
def test_bandwidth_floor_hysteresis(self):
|
||||
# 5 Mbps < 10 Mbps floor: every sample is bad, both controllers degrade.
|
||||
for module in (linux, win):
|
||||
policy = module.Admission({})
|
||||
meter = self.meter_with_transfers(module, 5)
|
||||
outputs = [policy.decide(self.sample(), meter, {'paused': False}, self.now)
|
||||
for _ in range(3)]
|
||||
self.assertEqual(outputs, ['NETWORK_DEGRADED'] * 3, module.__name__)
|
||||
|
||||
def test_recovery_factor(self):
|
||||
outputs = []
|
||||
for module in (linux, win):
|
||||
policy = module.Admission({})
|
||||
meter = self.meter_with_transfers(module, 11)
|
||||
outputs.append(policy.decide(self.sample(), meter, {'paused': False}, self.now))
|
||||
outputs.append(policy.decide(self.sample(), meter, {'paused': False}, self.now))
|
||||
self.assertEqual(outputs[0], 'NETWORK_DEGRADED')
|
||||
self.assertEqual(outputs[0], outputs[1])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main(verbosity=1)
|
||||
Loading…
Add table
Add a link
Reference in a new issue