Tactical Media Broadcast Hub — initial release
Self-hosted multi-platform broadcast newsroom for activist collectives: publishing pipeline with group approval and DST-proof scheduling, TweetDeck-style column wall with hostile-content-safe rendering, live control room with Auto Director, bridged chat governance with polls and quorum, distributed sandboxed transcode workers, encrypted content-addressed archive, monthly analytics. MIT licensed. See README.md and docs/.
This commit is contained in:
commit
aa5e262f3a
880 changed files with 111042 additions and 0 deletions
565
infra/peertube-workers/win/pt-worker-nvenc.py
Normal file
565
infra/peertube-workers/win/pt-worker-nvenc.py
Normal file
|
|
@ -0,0 +1,565 @@
|
|||
#!/usr/bin/env python3
|
||||
"""Native Windows NVENC admission controller. Linux Meter/Admission semantics,
|
||||
Windows power/thermal/GPU sensors, isolated profile, and official named-pipe IPC.
|
||||
The supervisor owns runner processes; admission only publishes fail-closed gates.
|
||||
"""
|
||||
import argparse
|
||||
import ctypes
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import time
|
||||
import tomllib
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(os.environ.get('PT_ROOT', r'C:\pt-worker'))
|
||||
os.environ.update(APPDATA=str(ROOT / 'profile' / 'Roaming'), LOCALAPPDATA=str(ROOT / 'profile' / 'Local'), PT_STATE_DIR=str(ROOT / 'state'), PT_WORKER_CONFIG=str(ROOT / 'state' / 'worker.json'))
|
||||
CONFIG = ROOT / 'state' / 'worker.json'
|
||||
STATE = ROOT / 'state'
|
||||
RUNNER_CONFIG = Path(os.environ.get('APPDATA', str(Path.home() / 'AppData' / 'Roaming'))) / 'peertube-runner-nodejs' / 'Config' / 'default' / 'config.toml'
|
||||
|
||||
DEFAULTS = {
|
||||
'require_ac': True, 'min_disk_free_bytes': 20 * 1024 ** 3, 'max_temperature_c': 85,
|
||||
'min_download_mbps': 10, 'min_upload_mbps': 10, 'max_latency_ms': 150,
|
||||
'allow_derp': False, 'recovery_factor': 1.25, 'good_samples': 3, 'bad_samples': 2,
|
||||
'bandwidth_ttl_seconds': 900, 'transfer_failure_cooldown_seconds': 60,
|
||||
'thermal_cooldown_seconds': 120, 'poll_seconds': 5,
|
||||
}
|
||||
# Last-resort resource gates that hold even while admission policy is disabled.
|
||||
# Same thresholds as the Linux installer defaults (15 GiB RAM floor for a
|
||||
# 2-concurrency QSV worker, 6 GiB min per-job scratch); no CPU fallback exists.
|
||||
HARD_FLOORS = {'ram_total_bytes': 15 * 1024 ** 3, 'min_scratch_bytes': 6 * 1024 ** 3}
|
||||
|
||||
|
||||
def run(args, timeout=10):
|
||||
return subprocess.run(args, capture_output=True, text=True, timeout=timeout)
|
||||
|
||||
|
||||
def read(path, default=None):
|
||||
try:
|
||||
return json.loads(Path(path).read_text(encoding='utf-8-sig'))
|
||||
except (OSError, ValueError):
|
||||
return default
|
||||
|
||||
|
||||
def config():
|
||||
value = read(CONFIG)
|
||||
if not value or not value.get('peertube_url') or not value.get('tailscale_peer'):
|
||||
raise ValueError('worker.json needs peertube_url and tailscale_peer')
|
||||
return {**DEFAULTS, **value}
|
||||
|
||||
|
||||
def ipc(action, node_binary='node'):
|
||||
"""Official runner IPC via ipc-win.cjs; the runner must be running in server mode."""
|
||||
env = {**os.environ, 'PT_STATE_DIR': str(STATE)}
|
||||
p = run([node_binary, str(ROOT / 'bin' / 'ipc-nvenc.cjs'), action], 15)
|
||||
if p.returncode:
|
||||
raise RuntimeError('runner IPC unavailable')
|
||||
return json.loads(p.stdout)
|
||||
|
||||
|
||||
def registered(cfg):
|
||||
try:
|
||||
data = tomllib.loads(RUNNER_CONFIG.read_text(encoding='utf-8-sig'))
|
||||
return any(r['url'].rstrip('/') == cfg['peertube_url'].rstrip('/') and
|
||||
r['runnerName'] == cfg['name'] and r.get('runnerToken')
|
||||
for r in data.get('registeredInstances', []))
|
||||
except (OSError, ValueError, KeyError):
|
||||
return False
|
||||
|
||||
|
||||
def network(cfg):
|
||||
result = {'tailscale_ip': None, 'tailscale_path': 'unknown', 'latency_ms': None,
|
||||
'network_ok': False, 'api_ok': False, 'peer_online': False}
|
||||
try:
|
||||
ts = json.loads(run([cfg.get('tailscale_binary', r'C:\Program Files\Tailscale\tailscale.exe'), 'status', '--json'], 5).stdout)
|
||||
result['tailscale_ip'] = next((a for a in ts.get('Self', {}).get('TailscaleIPs', []) if ':' not in a), None)
|
||||
peers = list(ts.get('Peer', {}).values())
|
||||
peer = next((p for p in peers if cfg['tailscale_peer'] in p.get('TailscaleIPs', []) or
|
||||
p.get('HostName') == cfg['tailscale_peer'] or
|
||||
p.get('DNSName', '').split('.')[0] == cfg['tailscale_peer']), None)
|
||||
if ts.get('BackendState') != 'Running' or not peer or not peer.get('Online'):
|
||||
return result
|
||||
result['peer_online'] = True
|
||||
p = run([cfg.get('tailscale_binary', r'C:\Program Files\Tailscale\tailscale.exe'), 'ping', '--c=1', '--until-direct=false', '--timeout=3s', cfg['tailscale_peer']], 5)
|
||||
if 'pong from' not in p.stdout:
|
||||
return result
|
||||
result['tailscale_path'] = 'DERP' if 'DERP(' in p.stdout else 'direct'
|
||||
latency = re.search(r'in ([\d.]+)ms', p.stdout)
|
||||
if latency:
|
||||
result['latency_ms'] = float(latency[1])
|
||||
started = time.monotonic()
|
||||
with urllib.request.urlopen(cfg['peertube_url'].rstrip('/') + '/api/v1/config', timeout=4) as response:
|
||||
version = json.load(response).get('serverVersion')
|
||||
result['api_ok'] = version == cfg.get('peertube_version', '7.3.0')
|
||||
result['network_ok'] = result['api_ok']
|
||||
except (OSError, ValueError, subprocess.SubprocessError, urllib.error.URLError):
|
||||
pass
|
||||
return result
|
||||
|
||||
|
||||
def transfer_probe(cfg, megabytes=8):
|
||||
"""Bounded probe against pt-worker-netprobe (same server-side probe as Linux)."""
|
||||
url = cfg.get('probe_url')
|
||||
rates = {}
|
||||
if not url:
|
||||
return rates
|
||||
payload = os.urandom(megabytes * 1024 * 1024)
|
||||
for direction, request in (('download', urllib.request.Request(url.rstrip('/') + '/download',
|
||||
data=None, method='GET')),
|
||||
('upload', None)):
|
||||
try:
|
||||
started = time.monotonic()
|
||||
if direction == 'download':
|
||||
with urllib.request.urlopen(request, timeout=30) as response:
|
||||
seen = 0
|
||||
while chunk := response.read(1 << 20):
|
||||
seen += len(chunk)
|
||||
if seen >= megabytes * 1024 * 1024:
|
||||
break
|
||||
seconds = time.monotonic() - started
|
||||
if seen < 65536:
|
||||
continue
|
||||
else:
|
||||
req = urllib.request.Request(url.rstrip('/') + '/upload', data=payload, method='PUT')
|
||||
urllib.request.urlopen(req, timeout=30).read()
|
||||
seconds = time.monotonic() - started
|
||||
rates[direction] = {'mbps': megabytes * 1024 * 1024 * 8 / 1e6 / seconds, 'time': time.time(),
|
||||
'bytes': megabytes * 1024 * 1024, 'seconds': seconds}
|
||||
except (OSError, urllib.error.URLError, ValueError):
|
||||
rates[direction + '_failed'] = True
|
||||
return rates
|
||||
|
||||
|
||||
def get_system_power_status():
|
||||
"""Win32 GetSystemPowerStatus; dict or None when unsupported/unknown."""
|
||||
class PLS(ctypes.Structure):
|
||||
_fields_ = [('ACLineStatus', ctypes.c_byte), ('BatteryFlag', ctypes.c_byte),
|
||||
('BatteryLifePercent', ctypes.c_byte), ('Reserved1', ctypes.c_byte),
|
||||
('BatteryLifeTime', ctypes.c_uint32), ('BatteryFullLifeTime', ctypes.c_uint32)]
|
||||
status = PLS()
|
||||
if not ctypes.windll.kernel32.GetSystemPowerStatus(ctypes.byref(status)):
|
||||
return None
|
||||
ac = {0: False, 1: True}.get(status.ACLineStatus) # 255 == unknown -> None
|
||||
percentage = status.BatteryLifePercent
|
||||
return {'on_ac': ac, 'battery_flag': status.BatteryFlag,
|
||||
'battery_percent': None if percentage in (0, 255) else percentage}
|
||||
|
||||
|
||||
_thermal_cache = {'time': 0, 'temperature_c': None, 'temperatures_c': []}
|
||||
|
||||
def sensors():
|
||||
result = {'temperature_c': None, 'temperatures_c': [], 'on_ac': None, 'battery': [],
|
||||
'throttle_counts': {}, 'ram_total_bytes': None, 'ram_available_bytes': None,
|
||||
'load_average': None, 'wsl2': False}
|
||||
try:
|
||||
if time.time() - _thermal_cache['time'] < 300:
|
||||
raise LookupError('cached')
|
||||
_thermal_cache['time'] = time.time()
|
||||
p = run(['powershell.exe', '-NoProfile', '-NonInteractive', '-Command',
|
||||
'(Get-CimInstance -Namespace root/wmi -ClassName MSAcpi_ThermalZoneTemperature -ErrorAction Stop | '
|
||||
'Measure-Object -Property CurrentTemperature -Maximum).Maximum'], 12)
|
||||
if p.returncode == 0:
|
||||
value = float(p.stdout.strip()) / 10.0 - 273.15
|
||||
if 0 < value < 120:
|
||||
result['temperature_c'] = round(value, 1)
|
||||
result['temperatures_c'] = [result['temperature_c']]
|
||||
_thermal_cache.update(temperature_c=result['temperature_c'], temperatures_c=result['temperatures_c'])
|
||||
except (OSError, ValueError, LookupError, subprocess.SubprocessError):
|
||||
result.update(temperature_c=_thermal_cache['temperature_c'], temperatures_c=_thermal_cache['temperatures_c'])
|
||||
power = get_system_power_status()
|
||||
if power:
|
||||
result['on_ac'] = power['on_ac']
|
||||
if power['battery_percent'] is not None:
|
||||
result['battery'] = [{'capacity': power['battery_percent'],
|
||||
'status': 'on_ac' if power['on_ac'] is True else 'discharging' if power['on_ac'] is False else 'unknown'}]
|
||||
try:
|
||||
import ctypes.wintypes # noqa: F401 (MEMORYSTATUSEX needs no extra types)
|
||||
class MEMORYSTATUSEX(ctypes.Structure):
|
||||
_fields_ = [('dwLength', ctypes.c_uint32), ('dwMemoryLoad', ctypes.c_uint32),
|
||||
('ullTotalPhys', ctypes.c_uint64), ('ullAvailPhys', ctypes.c_uint64),
|
||||
('ullTotalPageFile', ctypes.c_uint64), ('ullAvailPageFile', ctypes.c_uint64),
|
||||
('ullTotalVirtual', ctypes.c_uint64), ('ullAvailVirtual', ctypes.c_uint64),
|
||||
('ullAvailExtendedVirtual', ctypes.c_uint64)]
|
||||
memory = MEMORYSTATUSEX()
|
||||
memory.dwLength = ctypes.sizeof(MEMORYSTATUSEX)
|
||||
if ctypes.windll.kernel32.GlobalMemoryStatusEx(ctypes.byref(memory)):
|
||||
result['ram_total_bytes'] = memory.ullTotalPhys
|
||||
result['ram_available_bytes'] = memory.ullAvailPhys
|
||||
except (OSError, AttributeError, ValueError):
|
||||
pass
|
||||
return result
|
||||
|
||||
|
||||
def disk_free(path=str(ROOT)):
|
||||
try:
|
||||
free = ctypes.c_ulonglong()
|
||||
if ctypes.windll.kernel32.GetDiskFreeSpaceExW(ctypes.c_wchar_p(path), None, None, ctypes.byref(free)):
|
||||
return free.value
|
||||
except (OSError, AttributeError):
|
||||
pass
|
||||
return None
|
||||
|
||||
|
||||
def gpu_identity():
|
||||
try:
|
||||
p = run(['powershell.exe', '-NoProfile', '-NonInteractive', '-Command',
|
||||
'Get-CimInstance Win32_VideoController | Select-Object Name,DriverVersion,Status | ConvertTo-Json'], 12)
|
||||
data = json.loads(p.stdout or '[]')
|
||||
if isinstance(data, dict):
|
||||
data = [data]
|
||||
return data
|
||||
except (OSError, ValueError, subprocess.SubprocessError):
|
||||
return []
|
||||
|
||||
|
||||
class Meter:
|
||||
"""Byte-identical semantics to the Linux controller's Meter."""
|
||||
|
||||
def __init__(self):
|
||||
self.data = read(STATE / 'meter.json', {'offset': 0, 'jobs_completed': 0, 'jobs_failed': 0,
|
||||
'transfer_failures': 0, 'active': {}}) or {}
|
||||
|
||||
def invalidate(self):
|
||||
for key in ('download', 'upload'):
|
||||
self.data.pop(key, None)
|
||||
|
||||
def update(self):
|
||||
p = STATE / 'events.jsonl'
|
||||
if not p.exists():
|
||||
return
|
||||
with p.open(encoding='utf-8-sig') as f:
|
||||
if p.stat().st_size < self.data.get('offset', 0):
|
||||
self.data['offset'] = 0
|
||||
f.seek(self.data.get('offset', 0))
|
||||
while True:
|
||||
position = f.tell()
|
||||
line = f.readline()
|
||||
if not line:
|
||||
break
|
||||
if not line.endswith('\n'):
|
||||
f.seek(position)
|
||||
break
|
||||
try:
|
||||
self.consume(json.loads(line))
|
||||
except (ValueError, KeyError, TypeError):
|
||||
pass
|
||||
self.data['offset'] = f.tell()
|
||||
try:
|
||||
(STATE / 'meter.json').write_text(json.dumps(self.data), encoding='utf-8-sig')
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
def consume(self, e):
|
||||
kind, stamp = e['kind'], e['time']
|
||||
if kind == 'transfer':
|
||||
if not e['ok']:
|
||||
self.data['transfer_failures'] = self.data.get('transfer_failures', 0) + 1
|
||||
self.data['last_transfer_failure'] = stamp
|
||||
elif e['bytes'] >= 65536 and e['seconds'] > 0:
|
||||
direction = e['direction']
|
||||
rate = e['bytes'] * 8 / e['seconds'] / 1e6
|
||||
old = self.data.get(direction, {}).get('mbps', rate)
|
||||
self.data[direction] = {'mbps': .3 * rate + .7 * old, 'time': stamp,
|
||||
'bytes': e['bytes'], 'seconds': e['seconds']}
|
||||
elif kind == 'gpu_failure':
|
||||
self.data['gpu_failure_time'] = stamp
|
||||
elif kind == 'encode':
|
||||
self.data['last_encode'] = e
|
||||
if e.get('job_uuid'):
|
||||
self.data.setdefault('encodes', {})[e['job_uuid']] = e
|
||||
elif kind == 'job_accept' and e['ok']:
|
||||
self.data.setdefault('active', {})[e['uuid']] = stamp
|
||||
elif kind in ('job_success', 'job_error', 'job_abort') and e['ok']:
|
||||
begun = self.data.setdefault('active', {}).pop(e['uuid'], None)
|
||||
if kind == 'job_success':
|
||||
self.data['jobs_completed'] = self.data.get('jobs_completed', 0) + 1
|
||||
elapsed = stamp - begun if begun else None
|
||||
self.data['last_job_duration_seconds'] = elapsed
|
||||
self.data['last_job_uuid'] = e['uuid']
|
||||
encoded = self.data.setdefault('encodes', {}).pop(e['uuid'], {})
|
||||
duration = encoded.get('duration')
|
||||
self.data['last_job_realtime_factor'] = duration / elapsed if duration and elapsed else None
|
||||
self.data['last_job_encode'] = encoded or None
|
||||
elif kind == 'job_error':
|
||||
self.data['jobs_failed'] = self.data.get('jobs_failed', 0) + 1
|
||||
for path in (STATE / 'inputs').glob('*.json'):
|
||||
if read(path) and read(path).get('uuid') == e['uuid']:
|
||||
try:
|
||||
path.unlink()
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
def rates(self, now, ttl):
|
||||
return {f'recent_{k}_mbps': self.data.get(k, {}).get('mbps')
|
||||
if now - self.data.get(k, {}).get('time', 0) < ttl else None
|
||||
for k in ('download', 'upload')}
|
||||
|
||||
|
||||
class Admission:
|
||||
"""Decision gates: PAUSED, GPU_UNAVAILABLE, DRAINING (power), LOW_DISK,
|
||||
THERMAL_THROTTLED, OFFLINE, NETWORK_DEGRADED, BUSY, AVAILABLE.
|
||||
Hysteresis and thresholds identical to the Linux controller."""
|
||||
|
||||
def __init__(self, cfg):
|
||||
self.cfg, self.good, self.bad = cfg, 0, 0
|
||||
self.admitted, self.path, self.throttles = False, None, {}
|
||||
|
||||
def decide(self, sample, metric, control, now):
|
||||
cfg = self.cfg
|
||||
def refuse(state):
|
||||
self.admitted, self.good = False, 0
|
||||
return state
|
||||
if self.path and sample['tailscale_path'] != self.path:
|
||||
self.admitted = False
|
||||
self.good = 0
|
||||
metric.invalidate()
|
||||
self.path = sample['tailscale_path']
|
||||
rates = metric.rates(now, cfg.get('bandwidth_ttl_seconds', 900))
|
||||
sample.update(rates)
|
||||
if control.get('paused', True):
|
||||
return refuse('PAUSED')
|
||||
if not sample['registered']:
|
||||
return refuse('PAUSED')
|
||||
gpu_failed = metric.data.get('gpu_failure_time', 0) >= sample.get('gpu_tested_at', 0)
|
||||
if not sample['gpu_accessible'] or not sample.get('gpu_ready') or gpu_failed:
|
||||
return refuse('GPU_UNAVAILABLE')
|
||||
if cfg.get('require_ac', True) and sample['on_ac'] is not True:
|
||||
return refuse('DRAINING')
|
||||
if sample['disk_free_bytes'] is None or sample['disk_free_bytes'] < cfg.get('min_disk_free_bytes', 20 * 1024 ** 3):
|
||||
return refuse('LOW_DISK')
|
||||
rising = any(v > self.throttles.get(k, v) for k, v in sample['throttle_counts'].items())
|
||||
self.throttles = sample['throttle_counts']
|
||||
if rising or (sample['temperature_c'] is not None and sample['temperature_c'] >= cfg.get('max_temperature_c', 85)):
|
||||
self.thermal_until = now + cfg.get('thermal_cooldown_seconds', 120)
|
||||
if now < getattr(self, 'thermal_until', 0):
|
||||
return refuse('THERMAL_THROTTLED')
|
||||
if not sample['network_ok']:
|
||||
self.admitted, self.good = False, 0
|
||||
return 'OFFLINE'
|
||||
calibrating = control.get('calibrate_until', 0) > now
|
||||
factor = cfg.get('recovery_factor', 1.25) if not self.admitted else 1
|
||||
bad = sample['latency_ms'] is None or sample['latency_ms'] > cfg.get('max_latency_ms', 150)
|
||||
bad |= sample['tailscale_path'] == 'DERP' and not cfg.get('allow_derp', False)
|
||||
if not calibrating:
|
||||
bad |= any(rates[f'recent_{key}_mbps'] is None or rates[f'recent_{key}_mbps'] <
|
||||
cfg.get(f'min_{key}_mbps', 10) * factor for key in ('download', 'upload'))
|
||||
bad |= now - metric.data.get('last_transfer_failure', 0) < cfg.get('transfer_failure_cooldown_seconds', 60)
|
||||
if bad:
|
||||
self.good, self.bad = 0, self.bad + 1
|
||||
if self.bad >= cfg.get('bad_samples', 2) or not self.admitted:
|
||||
self.admitted = False
|
||||
return 'NETWORK_DEGRADED'
|
||||
else:
|
||||
self.bad, self.good = 0, self.good + 1
|
||||
if self.good >= cfg.get('good_samples', 3):
|
||||
self.admitted = True
|
||||
if not self.admitted:
|
||||
return 'NETWORK_DEGRADED'
|
||||
return 'BUSY' if sample['active_jobs'] else 'AVAILABLE'
|
||||
|
||||
|
||||
def atomic(path, value):
|
||||
path = Path(path)
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
tmp = path.with_suffix('.tmp-' + str(os.getpid()))
|
||||
tmp.write_text(json.dumps(value), encoding='utf8')
|
||||
os.replace(tmp, path)
|
||||
|
||||
|
||||
def gpu_snapshot(cfg):
|
||||
p = run(['nvidia-smi.exe', '-i', str(cfg.get('nvenc_device', 0)),
|
||||
'--query-gpu=name,uuid,driver_version,temperature.gpu,utilization.encoder',
|
||||
'--format=csv,noheader,nounits'], 5)
|
||||
if p.returncode:
|
||||
raise RuntimeError('GPU query failed')
|
||||
name, uuid, driver, temp, util = [v.strip() for v in p.stdout.strip().split(',')]
|
||||
return {'gpu_model': name, 'gpu_uuid': uuid, 'driver_version': driver,
|
||||
'temperature_c': float(temp), 'encoder_util': float(util)}
|
||||
|
||||
|
||||
def boot_hardware():
|
||||
p = run(['powershell.exe', '-NoProfile', '-NonInteractive', '-Command',
|
||||
"[pscustomobject]@{boot_id=(Get-CimInstance Win32_OperatingSystem).LastBootUpTime.ToUniversalTime().ToString('o');cpu_model=(Get-CimInstance Win32_Processor | Select-Object -First 1).Name} | ConvertTo-Json -Compress"], 30)
|
||||
return json.loads(p.stdout)
|
||||
|
||||
|
||||
def collect(cfg, meter, admission, hw):
|
||||
now = time.time()
|
||||
meter.update()
|
||||
sensor = sensors()
|
||||
proof = read(STATE / 'nvenc.json', {}) or {}
|
||||
gpu = {}
|
||||
try:
|
||||
gpu = gpu_snapshot(cfg)
|
||||
except (OSError, ValueError, RuntimeError, subprocess.SubprocessError):
|
||||
pass
|
||||
ready = bool(proof.get('nvenc_h264_ok') and proof.get('nvenc_hevc_ok') and
|
||||
proof.get('boot_id') == hw.get('boot_id') and
|
||||
proof.get('driver_version') == gpu.get('driver_version') and
|
||||
proof.get('gpu_uuid') == gpu.get('gpu_uuid') and
|
||||
proof.get('device') == cfg.get('nvenc_device', 0) and
|
||||
0 <= now - proof.get('tested_at', 0) < 7 * 86400)
|
||||
runner = read(STATE / 'runner.json', {}) or {}
|
||||
running = runner.get('running', False) and now - runner.get('updated_at', 0) < 30
|
||||
jobs, ipc_ok = {'active_jobs': 0, 'concurrency': cfg.get('concurrency', 2)}, not running
|
||||
if running:
|
||||
try:
|
||||
jobs = ipc('list-jobs', cfg['node_binary'])
|
||||
ipc_ok = True
|
||||
except (OSError, ValueError, RuntimeError, subprocess.SubprocessError):
|
||||
jobs['active_jobs'] = None
|
||||
temps = sensor['temperatures_c'] + ([gpu['temperature_c']] if gpu else [])
|
||||
sample = {**network(cfg), **sensor, **hw, 'hostname': cfg['name'], 'updated_at': now,
|
||||
'registered': registered(cfg), 'registration_state': 'registered' if registered(cfg) else 'unregistered',
|
||||
**jobs, 'gpu_accessible': bool(gpu) and not (STATE / 'gpu-unavailable').exists(),
|
||||
'gpu_ready': ready, 'gpu_tested_at': proof.get('tested_at', 0),
|
||||
'disk_free_bytes': disk_free(), 'temperature_c': max(temps, default=None),
|
||||
'temperatures_c': temps, 'throttle_counts': {}, 'qsv': {'qsv_h264_ok': False, 'qsv_hevc_ok': False},
|
||||
'qsv_h264_ok': False, 'qsv_hevc_ok': False,
|
||||
'nvenc_h264_ok': proof.get('nvenc_h264_ok', False), 'nvenc_hevc_ok': proof.get('nvenc_hevc_ok', False),
|
||||
'av1_decode_ok': proof.get('av1_decode_ok'), 'av1_encode_ok': False, 'backend': 'nvidia_nvenc',
|
||||
'gpu_model': gpu.get('gpu_model'), 'driver_version': gpu.get('driver_version'),
|
||||
'tested_at': proof.get('tested_at'), 'runner_running': bool(running),
|
||||
'draining': runner.get('draining', False), 'util': {'cpu': None, 'gpu': {'encoder': gpu.get('encoder_util')}}}
|
||||
if sample['disk_free_bytes'] is None:
|
||||
sample['disk_free_bytes'] = 0
|
||||
state = admission.decide(sample, meter, read(STATE / 'control.json', {'paused': True}), now)
|
||||
if not ipc_ok:
|
||||
state = 'OFFLINE'
|
||||
sample.update(status=state, state=state, admission_allowed=state in ('AVAILABLE', 'BUSY'),
|
||||
jobs_completed=meter.data.get('jobs_completed', 0), jobs_failed=meter.data.get('jobs_failed', 0),
|
||||
last_job_duration_seconds=meter.data.get('last_job_duration_seconds'),
|
||||
last_job_realtime_factor=meter.data.get('last_job_realtime_factor'))
|
||||
sample.update({k: meter.data.get(k) for k in ('transfer_failures', 'last_job_uuid', 'last_job_encode', 'last_encode')})
|
||||
# Linux status keys remain intact; research/fleet aliases are additive.
|
||||
sample.update(registration=sample['registered'], gpu_util=gpu.get('encoder_util'), temps=temps,
|
||||
disk_free=sample['disk_free_bytes'], last_job_duration=sample['last_job_duration_seconds'],
|
||||
last_job_rtf=sample['last_job_realtime_factor'], updated_at=time.time())
|
||||
return sample
|
||||
|
||||
|
||||
def self_test_marker_stale(hw, proof):
|
||||
return not proof.get('tested_at') or time.time() - proof['tested_at'] > 7 * 86400 or proof.get('boot_id') != hw.get('boot_id') or proof.get('driver_version') != hw.get('driver_version')
|
||||
|
||||
|
||||
def daemon():
|
||||
import msvcrt
|
||||
import signal
|
||||
lock = (STATE / 'admission.lock').open('a+b')
|
||||
lock.seek(0); lock.write(b'0'); lock.flush(); lock.seek(0)
|
||||
msvcrt.locking(lock.fileno(), msvcrt.LK_NBLCK, 1)
|
||||
cfg = config()
|
||||
meter, admission = Meter(), Admission(cfg)
|
||||
while True:
|
||||
try:
|
||||
hw = boot_hardware()
|
||||
break
|
||||
except (OSError, ValueError, subprocess.SubprocessError):
|
||||
atomic(STATE / 'status.json', {'hostname': cfg['name'], 'status': 'OFFLINE', 'state': 'OFFLINE', 'updated_at': time.time(), 'admission_allowed': False})
|
||||
time.sleep(5)
|
||||
atomic(STATE / 'hardware.json', hw)
|
||||
stopped = False
|
||||
def stop(*_):
|
||||
nonlocal stopped
|
||||
stopped = True
|
||||
signal.signal(signal.SIGINT, stop)
|
||||
signal.signal(signal.SIGTERM, stop)
|
||||
last_probe, previous_path = 0, None
|
||||
attempted_proof = None
|
||||
while not stopped:
|
||||
try:
|
||||
proof = read(STATE / 'nvenc.json', {}) or {}
|
||||
gpu = gpu_snapshot(cfg)
|
||||
identity = (hw.get('boot_id'), gpu.get('driver_version'))
|
||||
if self_test_marker_stale(hw | {'driver_version': gpu.get('driver_version')}, proof) and identity != attempted_proof and not read(STATE / 'runner.json', {}).get('running'):
|
||||
attempted_proof = identity
|
||||
atomic(STATE / 'status.json', {'updated_at': time.time(), 'status': 'GPU_UNAVAILABLE', 'state': 'GPU_UNAVAILABLE', 'admission_allowed': False})
|
||||
if get_system_power_status() and get_system_power_status()['on_ac'] is True:
|
||||
subprocess.run(['powershell.exe', '-NoProfile', '-NonInteractive', '-ExecutionPolicy', 'Bypass', '-File', str(ROOT / 'selftest-nvenc.ps1'), '-Root', str(ROOT), '-Device', str(cfg.get('nvenc_device', 0))], capture_output=True, timeout=90)
|
||||
sample = collect(cfg, meter, admission, hw)
|
||||
if previous_path and previous_path != sample['tailscale_path']:
|
||||
meter.invalidate(); last_probe = 0
|
||||
admission.admitted, admission.good, admission.path = False, 0, None
|
||||
sample['admission_allowed'] = False
|
||||
sample['state'] = sample['status'] = 'NETWORK_DEGRADED'
|
||||
previous_path = sample['tailscale_path']
|
||||
atomic(STATE / 'status.json', sample)
|
||||
control = read(STATE / 'control.json', {'paused': True})
|
||||
if not control.get('paused', True) and sample['network_ok'] and sample['on_ac'] is True and \
|
||||
sample['tailscale_path'] == 'direct' and sample['active_jobs'] == 0 and \
|
||||
time.time() - last_probe >= cfg.get('probe_interval_seconds', 300):
|
||||
# Close admission and wait for the official graceful drain before measuring.
|
||||
sample.update(admission_allowed=False, state='DRAINING', status='DRAINING')
|
||||
atomic(STATE / 'status.json', sample)
|
||||
if not sample['runner_running']:
|
||||
rates = transfer_probe(cfg)
|
||||
for direction in ('download', 'upload'):
|
||||
value = rates.get(direction)
|
||||
meter.consume({'kind': 'transfer', 'time': time.time(), 'direction': direction,
|
||||
'ok': bool(value), 'bytes': value['bytes'] if value else 0,
|
||||
'seconds': value['seconds'] if value else 1})
|
||||
atomic(STATE / 'meter.json', meter.data)
|
||||
last_probe = time.time()
|
||||
except Exception:
|
||||
# Suppress exception strings: upstream data may contain credentials/URLs.
|
||||
atomic(STATE / 'status.json', {'hostname': cfg['name'], 'status': 'OFFLINE', 'state': 'OFFLINE',
|
||||
'updated_at': time.time(), 'admission_allowed': False})
|
||||
before_sleep = time.time()
|
||||
time.sleep(cfg.get('poll_seconds', 5))
|
||||
if time.time() - before_sleep > cfg.get('poll_seconds', 5) * 3:
|
||||
meter.invalidate()
|
||||
admission.admitted, admission.good = False, 0
|
||||
last_probe = 0
|
||||
atomic(STATE / 'status.json', {'hostname': cfg['name'], 'status': 'OFFLINE', 'state': 'OFFLINE',
|
||||
'updated_at': time.time(), 'admission_allowed': False})
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('command', choices=['daemon', 'status', 'network-test', 'start', 'stop', 'drain', 'restart', 'self-test'])
|
||||
args = parser.parse_args()
|
||||
STATE.mkdir(parents=True, exist_ok=True)
|
||||
if args.command == 'daemon':
|
||||
daemon()
|
||||
elif args.command == 'status':
|
||||
value = read(STATE / 'status.json', {'hostname': config()['name'], 'status': 'OFFLINE', 'state': 'OFFLINE'})
|
||||
if time.time() - value.get('updated_at', 0) > 30:
|
||||
value.update(status='OFFLINE', state='OFFLINE', stale=True, admission_allowed=False)
|
||||
print(json.dumps(value, indent=2))
|
||||
elif args.command == 'network-test':
|
||||
meter = Meter(); meter.update()
|
||||
rates = transfer_probe(config())
|
||||
for direction in ('download', 'upload'):
|
||||
r = rates.get(direction)
|
||||
meter.consume({'kind': 'transfer', 'time': time.time(), 'direction': direction, 'ok': bool(r),
|
||||
'bytes': r['bytes'] if r else 0, 'seconds': r['seconds'] if r else 1})
|
||||
atomic(STATE / 'meter.json', meter.data)
|
||||
print(json.dumps(rates))
|
||||
return 0 if all(k in rates for k in ('download', 'upload')) else 1
|
||||
elif args.command == 'self-test':
|
||||
if read(STATE / 'runner.json', {}).get('running'):
|
||||
raise ValueError('drain before self-test')
|
||||
return subprocess.call(['powershell.exe', '-NoProfile', '-NonInteractive', '-ExecutionPolicy', 'Bypass',
|
||||
'-File', str(ROOT / 'selftest-nvenc.ps1'), '-Root', str(ROOT),
|
||||
'-Device', str(config().get('nvenc_device', 0))])
|
||||
else:
|
||||
paused = args.command in ('stop', 'drain')
|
||||
atomic(STATE / 'control.json', {'paused': paused})
|
||||
if args.command == 'restart':
|
||||
try:
|
||||
ipc('graceful-shutdown', config()['node_binary'])
|
||||
except (OSError, RuntimeError, subprocess.SubprocessError):
|
||||
pass
|
||||
print(json.dumps({'requested': args.command, 'paused': paused}))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
try:
|
||||
raise SystemExit(main())
|
||||
except (OSError, ValueError, RuntimeError, subprocess.SubprocessError):
|
||||
print('pt-worker-win: command failed; check service health', file=__import__('sys').stderr)
|
||||
raise SystemExit(1)
|
||||
Loading…
Add table
Add a link
Reference in a new issue