glass beadian
aa5e262f3a
Self-hosted multi-platform broadcast newsroom for activist collectives: publishing pipeline with group approval and DST-proof scheduling, TweetDeck-style column wall with hostile-content-safe rendering, live control room with Auto Director, bridged chat governance with polls and quorum, distributed sandboxed transcode workers, encrypted content-addressed archive, monthly analytics. MIT licensed. See README.md and docs/.
565 lines
28 KiB
Python
565 lines
28 KiB
Python
#!/usr/bin/env python3
|
|
"""Native Windows NVENC admission controller. Linux Meter/Admission semantics,
|
|
Windows power/thermal/GPU sensors, isolated profile, and official named-pipe IPC.
|
|
The supervisor owns runner processes; admission only publishes fail-closed gates.
|
|
"""
|
|
import argparse
|
|
import ctypes
|
|
import json
|
|
import os
|
|
import re
|
|
import subprocess
|
|
import time
|
|
import tomllib
|
|
import urllib.request
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(os.environ.get('PT_ROOT', r'C:\pt-worker'))
|
|
os.environ.update(APPDATA=str(ROOT / 'profile' / 'Roaming'), LOCALAPPDATA=str(ROOT / 'profile' / 'Local'), PT_STATE_DIR=str(ROOT / 'state'), PT_WORKER_CONFIG=str(ROOT / 'state' / 'worker.json'))
|
|
CONFIG = ROOT / 'state' / 'worker.json'
|
|
STATE = ROOT / 'state'
|
|
RUNNER_CONFIG = Path(os.environ.get('APPDATA', str(Path.home() / 'AppData' / 'Roaming'))) / 'peertube-runner-nodejs' / 'Config' / 'default' / 'config.toml'
|
|
|
|
DEFAULTS = {
|
|
'require_ac': True, 'min_disk_free_bytes': 20 * 1024 ** 3, 'max_temperature_c': 85,
|
|
'min_download_mbps': 10, 'min_upload_mbps': 10, 'max_latency_ms': 150,
|
|
'allow_derp': False, 'recovery_factor': 1.25, 'good_samples': 3, 'bad_samples': 2,
|
|
'bandwidth_ttl_seconds': 900, 'transfer_failure_cooldown_seconds': 60,
|
|
'thermal_cooldown_seconds': 120, 'poll_seconds': 5,
|
|
}
|
|
# Last-resort resource gates that hold even while admission policy is disabled.
|
|
# Same thresholds as the Linux installer defaults (15 GiB RAM floor for a
|
|
# 2-concurrency QSV worker, 6 GiB min per-job scratch); no CPU fallback exists.
|
|
HARD_FLOORS = {'ram_total_bytes': 15 * 1024 ** 3, 'min_scratch_bytes': 6 * 1024 ** 3}
|
|
|
|
|
|
def run(args, timeout=10):
|
|
return subprocess.run(args, capture_output=True, text=True, timeout=timeout)
|
|
|
|
|
|
def read(path, default=None):
|
|
try:
|
|
return json.loads(Path(path).read_text(encoding='utf-8-sig'))
|
|
except (OSError, ValueError):
|
|
return default
|
|
|
|
|
|
def config():
|
|
value = read(CONFIG)
|
|
if not value or not value.get('peertube_url') or not value.get('tailscale_peer'):
|
|
raise ValueError('worker.json needs peertube_url and tailscale_peer')
|
|
return {**DEFAULTS, **value}
|
|
|
|
|
|
def ipc(action, node_binary='node'):
|
|
"""Official runner IPC via ipc-win.cjs; the runner must be running in server mode."""
|
|
env = {**os.environ, 'PT_STATE_DIR': str(STATE)}
|
|
p = run([node_binary, str(ROOT / 'bin' / 'ipc-nvenc.cjs'), action], 15)
|
|
if p.returncode:
|
|
raise RuntimeError('runner IPC unavailable')
|
|
return json.loads(p.stdout)
|
|
|
|
|
|
def registered(cfg):
|
|
try:
|
|
data = tomllib.loads(RUNNER_CONFIG.read_text(encoding='utf-8-sig'))
|
|
return any(r['url'].rstrip('/') == cfg['peertube_url'].rstrip('/') and
|
|
r['runnerName'] == cfg['name'] and r.get('runnerToken')
|
|
for r in data.get('registeredInstances', []))
|
|
except (OSError, ValueError, KeyError):
|
|
return False
|
|
|
|
|
|
def network(cfg):
|
|
result = {'tailscale_ip': None, 'tailscale_path': 'unknown', 'latency_ms': None,
|
|
'network_ok': False, 'api_ok': False, 'peer_online': False}
|
|
try:
|
|
ts = json.loads(run([cfg.get('tailscale_binary', r'C:\Program Files\Tailscale\tailscale.exe'), 'status', '--json'], 5).stdout)
|
|
result['tailscale_ip'] = next((a for a in ts.get('Self', {}).get('TailscaleIPs', []) if ':' not in a), None)
|
|
peers = list(ts.get('Peer', {}).values())
|
|
peer = next((p for p in peers if cfg['tailscale_peer'] in p.get('TailscaleIPs', []) or
|
|
p.get('HostName') == cfg['tailscale_peer'] or
|
|
p.get('DNSName', '').split('.')[0] == cfg['tailscale_peer']), None)
|
|
if ts.get('BackendState') != 'Running' or not peer or not peer.get('Online'):
|
|
return result
|
|
result['peer_online'] = True
|
|
p = run([cfg.get('tailscale_binary', r'C:\Program Files\Tailscale\tailscale.exe'), 'ping', '--c=1', '--until-direct=false', '--timeout=3s', cfg['tailscale_peer']], 5)
|
|
if 'pong from' not in p.stdout:
|
|
return result
|
|
result['tailscale_path'] = 'DERP' if 'DERP(' in p.stdout else 'direct'
|
|
latency = re.search(r'in ([\d.]+)ms', p.stdout)
|
|
if latency:
|
|
result['latency_ms'] = float(latency[1])
|
|
started = time.monotonic()
|
|
with urllib.request.urlopen(cfg['peertube_url'].rstrip('/') + '/api/v1/config', timeout=4) as response:
|
|
version = json.load(response).get('serverVersion')
|
|
result['api_ok'] = version == cfg.get('peertube_version', '7.3.0')
|
|
result['network_ok'] = result['api_ok']
|
|
except (OSError, ValueError, subprocess.SubprocessError, urllib.error.URLError):
|
|
pass
|
|
return result
|
|
|
|
|
|
def transfer_probe(cfg, megabytes=8):
|
|
"""Bounded probe against pt-worker-netprobe (same server-side probe as Linux)."""
|
|
url = cfg.get('probe_url')
|
|
rates = {}
|
|
if not url:
|
|
return rates
|
|
payload = os.urandom(megabytes * 1024 * 1024)
|
|
for direction, request in (('download', urllib.request.Request(url.rstrip('/') + '/download',
|
|
data=None, method='GET')),
|
|
('upload', None)):
|
|
try:
|
|
started = time.monotonic()
|
|
if direction == 'download':
|
|
with urllib.request.urlopen(request, timeout=30) as response:
|
|
seen = 0
|
|
while chunk := response.read(1 << 20):
|
|
seen += len(chunk)
|
|
if seen >= megabytes * 1024 * 1024:
|
|
break
|
|
seconds = time.monotonic() - started
|
|
if seen < 65536:
|
|
continue
|
|
else:
|
|
req = urllib.request.Request(url.rstrip('/') + '/upload', data=payload, method='PUT')
|
|
urllib.request.urlopen(req, timeout=30).read()
|
|
seconds = time.monotonic() - started
|
|
rates[direction] = {'mbps': megabytes * 1024 * 1024 * 8 / 1e6 / seconds, 'time': time.time(),
|
|
'bytes': megabytes * 1024 * 1024, 'seconds': seconds}
|
|
except (OSError, urllib.error.URLError, ValueError):
|
|
rates[direction + '_failed'] = True
|
|
return rates
|
|
|
|
|
|
def get_system_power_status():
|
|
"""Win32 GetSystemPowerStatus; dict or None when unsupported/unknown."""
|
|
class PLS(ctypes.Structure):
|
|
_fields_ = [('ACLineStatus', ctypes.c_byte), ('BatteryFlag', ctypes.c_byte),
|
|
('BatteryLifePercent', ctypes.c_byte), ('Reserved1', ctypes.c_byte),
|
|
('BatteryLifeTime', ctypes.c_uint32), ('BatteryFullLifeTime', ctypes.c_uint32)]
|
|
status = PLS()
|
|
if not ctypes.windll.kernel32.GetSystemPowerStatus(ctypes.byref(status)):
|
|
return None
|
|
ac = {0: False, 1: True}.get(status.ACLineStatus) # 255 == unknown -> None
|
|
percentage = status.BatteryLifePercent
|
|
return {'on_ac': ac, 'battery_flag': status.BatteryFlag,
|
|
'battery_percent': None if percentage in (0, 255) else percentage}
|
|
|
|
|
|
_thermal_cache = {'time': 0, 'temperature_c': None, 'temperatures_c': []}
|
|
|
|
def sensors():
|
|
result = {'temperature_c': None, 'temperatures_c': [], 'on_ac': None, 'battery': [],
|
|
'throttle_counts': {}, 'ram_total_bytes': None, 'ram_available_bytes': None,
|
|
'load_average': None, 'wsl2': False}
|
|
try:
|
|
if time.time() - _thermal_cache['time'] < 300:
|
|
raise LookupError('cached')
|
|
_thermal_cache['time'] = time.time()
|
|
p = run(['powershell.exe', '-NoProfile', '-NonInteractive', '-Command',
|
|
'(Get-CimInstance -Namespace root/wmi -ClassName MSAcpi_ThermalZoneTemperature -ErrorAction Stop | '
|
|
'Measure-Object -Property CurrentTemperature -Maximum).Maximum'], 12)
|
|
if p.returncode == 0:
|
|
value = float(p.stdout.strip()) / 10.0 - 273.15
|
|
if 0 < value < 120:
|
|
result['temperature_c'] = round(value, 1)
|
|
result['temperatures_c'] = [result['temperature_c']]
|
|
_thermal_cache.update(temperature_c=result['temperature_c'], temperatures_c=result['temperatures_c'])
|
|
except (OSError, ValueError, LookupError, subprocess.SubprocessError):
|
|
result.update(temperature_c=_thermal_cache['temperature_c'], temperatures_c=_thermal_cache['temperatures_c'])
|
|
power = get_system_power_status()
|
|
if power:
|
|
result['on_ac'] = power['on_ac']
|
|
if power['battery_percent'] is not None:
|
|
result['battery'] = [{'capacity': power['battery_percent'],
|
|
'status': 'on_ac' if power['on_ac'] is True else 'discharging' if power['on_ac'] is False else 'unknown'}]
|
|
try:
|
|
import ctypes.wintypes # noqa: F401 (MEMORYSTATUSEX needs no extra types)
|
|
class MEMORYSTATUSEX(ctypes.Structure):
|
|
_fields_ = [('dwLength', ctypes.c_uint32), ('dwMemoryLoad', ctypes.c_uint32),
|
|
('ullTotalPhys', ctypes.c_uint64), ('ullAvailPhys', ctypes.c_uint64),
|
|
('ullTotalPageFile', ctypes.c_uint64), ('ullAvailPageFile', ctypes.c_uint64),
|
|
('ullTotalVirtual', ctypes.c_uint64), ('ullAvailVirtual', ctypes.c_uint64),
|
|
('ullAvailExtendedVirtual', ctypes.c_uint64)]
|
|
memory = MEMORYSTATUSEX()
|
|
memory.dwLength = ctypes.sizeof(MEMORYSTATUSEX)
|
|
if ctypes.windll.kernel32.GlobalMemoryStatusEx(ctypes.byref(memory)):
|
|
result['ram_total_bytes'] = memory.ullTotalPhys
|
|
result['ram_available_bytes'] = memory.ullAvailPhys
|
|
except (OSError, AttributeError, ValueError):
|
|
pass
|
|
return result
|
|
|
|
|
|
def disk_free(path=str(ROOT)):
|
|
try:
|
|
free = ctypes.c_ulonglong()
|
|
if ctypes.windll.kernel32.GetDiskFreeSpaceExW(ctypes.c_wchar_p(path), None, None, ctypes.byref(free)):
|
|
return free.value
|
|
except (OSError, AttributeError):
|
|
pass
|
|
return None
|
|
|
|
|
|
def gpu_identity():
|
|
try:
|
|
p = run(['powershell.exe', '-NoProfile', '-NonInteractive', '-Command',
|
|
'Get-CimInstance Win32_VideoController | Select-Object Name,DriverVersion,Status | ConvertTo-Json'], 12)
|
|
data = json.loads(p.stdout or '[]')
|
|
if isinstance(data, dict):
|
|
data = [data]
|
|
return data
|
|
except (OSError, ValueError, subprocess.SubprocessError):
|
|
return []
|
|
|
|
|
|
class Meter:
|
|
"""Byte-identical semantics to the Linux controller's Meter."""
|
|
|
|
def __init__(self):
|
|
self.data = read(STATE / 'meter.json', {'offset': 0, 'jobs_completed': 0, 'jobs_failed': 0,
|
|
'transfer_failures': 0, 'active': {}}) or {}
|
|
|
|
def invalidate(self):
|
|
for key in ('download', 'upload'):
|
|
self.data.pop(key, None)
|
|
|
|
def update(self):
|
|
p = STATE / 'events.jsonl'
|
|
if not p.exists():
|
|
return
|
|
with p.open(encoding='utf-8-sig') as f:
|
|
if p.stat().st_size < self.data.get('offset', 0):
|
|
self.data['offset'] = 0
|
|
f.seek(self.data.get('offset', 0))
|
|
while True:
|
|
position = f.tell()
|
|
line = f.readline()
|
|
if not line:
|
|
break
|
|
if not line.endswith('\n'):
|
|
f.seek(position)
|
|
break
|
|
try:
|
|
self.consume(json.loads(line))
|
|
except (ValueError, KeyError, TypeError):
|
|
pass
|
|
self.data['offset'] = f.tell()
|
|
try:
|
|
(STATE / 'meter.json').write_text(json.dumps(self.data), encoding='utf-8-sig')
|
|
except OSError:
|
|
pass
|
|
|
|
def consume(self, e):
|
|
kind, stamp = e['kind'], e['time']
|
|
if kind == 'transfer':
|
|
if not e['ok']:
|
|
self.data['transfer_failures'] = self.data.get('transfer_failures', 0) + 1
|
|
self.data['last_transfer_failure'] = stamp
|
|
elif e['bytes'] >= 65536 and e['seconds'] > 0:
|
|
direction = e['direction']
|
|
rate = e['bytes'] * 8 / e['seconds'] / 1e6
|
|
old = self.data.get(direction, {}).get('mbps', rate)
|
|
self.data[direction] = {'mbps': .3 * rate + .7 * old, 'time': stamp,
|
|
'bytes': e['bytes'], 'seconds': e['seconds']}
|
|
elif kind == 'gpu_failure':
|
|
self.data['gpu_failure_time'] = stamp
|
|
elif kind == 'encode':
|
|
self.data['last_encode'] = e
|
|
if e.get('job_uuid'):
|
|
self.data.setdefault('encodes', {})[e['job_uuid']] = e
|
|
elif kind == 'job_accept' and e['ok']:
|
|
self.data.setdefault('active', {})[e['uuid']] = stamp
|
|
elif kind in ('job_success', 'job_error', 'job_abort') and e['ok']:
|
|
begun = self.data.setdefault('active', {}).pop(e['uuid'], None)
|
|
if kind == 'job_success':
|
|
self.data['jobs_completed'] = self.data.get('jobs_completed', 0) + 1
|
|
elapsed = stamp - begun if begun else None
|
|
self.data['last_job_duration_seconds'] = elapsed
|
|
self.data['last_job_uuid'] = e['uuid']
|
|
encoded = self.data.setdefault('encodes', {}).pop(e['uuid'], {})
|
|
duration = encoded.get('duration')
|
|
self.data['last_job_realtime_factor'] = duration / elapsed if duration and elapsed else None
|
|
self.data['last_job_encode'] = encoded or None
|
|
elif kind == 'job_error':
|
|
self.data['jobs_failed'] = self.data.get('jobs_failed', 0) + 1
|
|
for path in (STATE / 'inputs').glob('*.json'):
|
|
if read(path) and read(path).get('uuid') == e['uuid']:
|
|
try:
|
|
path.unlink()
|
|
except OSError:
|
|
pass
|
|
|
|
def rates(self, now, ttl):
|
|
return {f'recent_{k}_mbps': self.data.get(k, {}).get('mbps')
|
|
if now - self.data.get(k, {}).get('time', 0) < ttl else None
|
|
for k in ('download', 'upload')}
|
|
|
|
|
|
class Admission:
|
|
"""Decision gates: PAUSED, GPU_UNAVAILABLE, DRAINING (power), LOW_DISK,
|
|
THERMAL_THROTTLED, OFFLINE, NETWORK_DEGRADED, BUSY, AVAILABLE.
|
|
Hysteresis and thresholds identical to the Linux controller."""
|
|
|
|
def __init__(self, cfg):
|
|
self.cfg, self.good, self.bad = cfg, 0, 0
|
|
self.admitted, self.path, self.throttles = False, None, {}
|
|
|
|
def decide(self, sample, metric, control, now):
|
|
cfg = self.cfg
|
|
def refuse(state):
|
|
self.admitted, self.good = False, 0
|
|
return state
|
|
if self.path and sample['tailscale_path'] != self.path:
|
|
self.admitted = False
|
|
self.good = 0
|
|
metric.invalidate()
|
|
self.path = sample['tailscale_path']
|
|
rates = metric.rates(now, cfg.get('bandwidth_ttl_seconds', 900))
|
|
sample.update(rates)
|
|
if control.get('paused', True):
|
|
return refuse('PAUSED')
|
|
if not sample['registered']:
|
|
return refuse('PAUSED')
|
|
gpu_failed = metric.data.get('gpu_failure_time', 0) >= sample.get('gpu_tested_at', 0)
|
|
if not sample['gpu_accessible'] or not sample.get('gpu_ready') or gpu_failed:
|
|
return refuse('GPU_UNAVAILABLE')
|
|
if cfg.get('require_ac', True) and sample['on_ac'] is not True:
|
|
return refuse('DRAINING')
|
|
if sample['disk_free_bytes'] is None or sample['disk_free_bytes'] < cfg.get('min_disk_free_bytes', 20 * 1024 ** 3):
|
|
return refuse('LOW_DISK')
|
|
rising = any(v > self.throttles.get(k, v) for k, v in sample['throttle_counts'].items())
|
|
self.throttles = sample['throttle_counts']
|
|
if rising or (sample['temperature_c'] is not None and sample['temperature_c'] >= cfg.get('max_temperature_c', 85)):
|
|
self.thermal_until = now + cfg.get('thermal_cooldown_seconds', 120)
|
|
if now < getattr(self, 'thermal_until', 0):
|
|
return refuse('THERMAL_THROTTLED')
|
|
if not sample['network_ok']:
|
|
self.admitted, self.good = False, 0
|
|
return 'OFFLINE'
|
|
calibrating = control.get('calibrate_until', 0) > now
|
|
factor = cfg.get('recovery_factor', 1.25) if not self.admitted else 1
|
|
bad = sample['latency_ms'] is None or sample['latency_ms'] > cfg.get('max_latency_ms', 150)
|
|
bad |= sample['tailscale_path'] == 'DERP' and not cfg.get('allow_derp', False)
|
|
if not calibrating:
|
|
bad |= any(rates[f'recent_{key}_mbps'] is None or rates[f'recent_{key}_mbps'] <
|
|
cfg.get(f'min_{key}_mbps', 10) * factor for key in ('download', 'upload'))
|
|
bad |= now - metric.data.get('last_transfer_failure', 0) < cfg.get('transfer_failure_cooldown_seconds', 60)
|
|
if bad:
|
|
self.good, self.bad = 0, self.bad + 1
|
|
if self.bad >= cfg.get('bad_samples', 2) or not self.admitted:
|
|
self.admitted = False
|
|
return 'NETWORK_DEGRADED'
|
|
else:
|
|
self.bad, self.good = 0, self.good + 1
|
|
if self.good >= cfg.get('good_samples', 3):
|
|
self.admitted = True
|
|
if not self.admitted:
|
|
return 'NETWORK_DEGRADED'
|
|
return 'BUSY' if sample['active_jobs'] else 'AVAILABLE'
|
|
|
|
|
|
def atomic(path, value):
|
|
path = Path(path)
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
tmp = path.with_suffix('.tmp-' + str(os.getpid()))
|
|
tmp.write_text(json.dumps(value), encoding='utf8')
|
|
os.replace(tmp, path)
|
|
|
|
|
|
def gpu_snapshot(cfg):
|
|
p = run(['nvidia-smi.exe', '-i', str(cfg.get('nvenc_device', 0)),
|
|
'--query-gpu=name,uuid,driver_version,temperature.gpu,utilization.encoder',
|
|
'--format=csv,noheader,nounits'], 5)
|
|
if p.returncode:
|
|
raise RuntimeError('GPU query failed')
|
|
name, uuid, driver, temp, util = [v.strip() for v in p.stdout.strip().split(',')]
|
|
return {'gpu_model': name, 'gpu_uuid': uuid, 'driver_version': driver,
|
|
'temperature_c': float(temp), 'encoder_util': float(util)}
|
|
|
|
|
|
def boot_hardware():
|
|
p = run(['powershell.exe', '-NoProfile', '-NonInteractive', '-Command',
|
|
"[pscustomobject]@{boot_id=(Get-CimInstance Win32_OperatingSystem).LastBootUpTime.ToUniversalTime().ToString('o');cpu_model=(Get-CimInstance Win32_Processor | Select-Object -First 1).Name} | ConvertTo-Json -Compress"], 30)
|
|
return json.loads(p.stdout)
|
|
|
|
|
|
def collect(cfg, meter, admission, hw):
|
|
now = time.time()
|
|
meter.update()
|
|
sensor = sensors()
|
|
proof = read(STATE / 'nvenc.json', {}) or {}
|
|
gpu = {}
|
|
try:
|
|
gpu = gpu_snapshot(cfg)
|
|
except (OSError, ValueError, RuntimeError, subprocess.SubprocessError):
|
|
pass
|
|
ready = bool(proof.get('nvenc_h264_ok') and proof.get('nvenc_hevc_ok') and
|
|
proof.get('boot_id') == hw.get('boot_id') and
|
|
proof.get('driver_version') == gpu.get('driver_version') and
|
|
proof.get('gpu_uuid') == gpu.get('gpu_uuid') and
|
|
proof.get('device') == cfg.get('nvenc_device', 0) and
|
|
0 <= now - proof.get('tested_at', 0) < 7 * 86400)
|
|
runner = read(STATE / 'runner.json', {}) or {}
|
|
running = runner.get('running', False) and now - runner.get('updated_at', 0) < 30
|
|
jobs, ipc_ok = {'active_jobs': 0, 'concurrency': cfg.get('concurrency', 2)}, not running
|
|
if running:
|
|
try:
|
|
jobs = ipc('list-jobs', cfg['node_binary'])
|
|
ipc_ok = True
|
|
except (OSError, ValueError, RuntimeError, subprocess.SubprocessError):
|
|
jobs['active_jobs'] = None
|
|
temps = sensor['temperatures_c'] + ([gpu['temperature_c']] if gpu else [])
|
|
sample = {**network(cfg), **sensor, **hw, 'hostname': cfg['name'], 'updated_at': now,
|
|
'registered': registered(cfg), 'registration_state': 'registered' if registered(cfg) else 'unregistered',
|
|
**jobs, 'gpu_accessible': bool(gpu) and not (STATE / 'gpu-unavailable').exists(),
|
|
'gpu_ready': ready, 'gpu_tested_at': proof.get('tested_at', 0),
|
|
'disk_free_bytes': disk_free(), 'temperature_c': max(temps, default=None),
|
|
'temperatures_c': temps, 'throttle_counts': {}, 'qsv': {'qsv_h264_ok': False, 'qsv_hevc_ok': False},
|
|
'qsv_h264_ok': False, 'qsv_hevc_ok': False,
|
|
'nvenc_h264_ok': proof.get('nvenc_h264_ok', False), 'nvenc_hevc_ok': proof.get('nvenc_hevc_ok', False),
|
|
'av1_decode_ok': proof.get('av1_decode_ok'), 'av1_encode_ok': False, 'backend': 'nvidia_nvenc',
|
|
'gpu_model': gpu.get('gpu_model'), 'driver_version': gpu.get('driver_version'),
|
|
'tested_at': proof.get('tested_at'), 'runner_running': bool(running),
|
|
'draining': runner.get('draining', False), 'util': {'cpu': None, 'gpu': {'encoder': gpu.get('encoder_util')}}}
|
|
if sample['disk_free_bytes'] is None:
|
|
sample['disk_free_bytes'] = 0
|
|
state = admission.decide(sample, meter, read(STATE / 'control.json', {'paused': True}), now)
|
|
if not ipc_ok:
|
|
state = 'OFFLINE'
|
|
sample.update(status=state, state=state, admission_allowed=state in ('AVAILABLE', 'BUSY'),
|
|
jobs_completed=meter.data.get('jobs_completed', 0), jobs_failed=meter.data.get('jobs_failed', 0),
|
|
last_job_duration_seconds=meter.data.get('last_job_duration_seconds'),
|
|
last_job_realtime_factor=meter.data.get('last_job_realtime_factor'))
|
|
sample.update({k: meter.data.get(k) for k in ('transfer_failures', 'last_job_uuid', 'last_job_encode', 'last_encode')})
|
|
# Linux status keys remain intact; research/fleet aliases are additive.
|
|
sample.update(registration=sample['registered'], gpu_util=gpu.get('encoder_util'), temps=temps,
|
|
disk_free=sample['disk_free_bytes'], last_job_duration=sample['last_job_duration_seconds'],
|
|
last_job_rtf=sample['last_job_realtime_factor'], updated_at=time.time())
|
|
return sample
|
|
|
|
|
|
def self_test_marker_stale(hw, proof):
|
|
return not proof.get('tested_at') or time.time() - proof['tested_at'] > 7 * 86400 or proof.get('boot_id') != hw.get('boot_id') or proof.get('driver_version') != hw.get('driver_version')
|
|
|
|
|
|
def daemon():
|
|
import msvcrt
|
|
import signal
|
|
lock = (STATE / 'admission.lock').open('a+b')
|
|
lock.seek(0); lock.write(b'0'); lock.flush(); lock.seek(0)
|
|
msvcrt.locking(lock.fileno(), msvcrt.LK_NBLCK, 1)
|
|
cfg = config()
|
|
meter, admission = Meter(), Admission(cfg)
|
|
while True:
|
|
try:
|
|
hw = boot_hardware()
|
|
break
|
|
except (OSError, ValueError, subprocess.SubprocessError):
|
|
atomic(STATE / 'status.json', {'hostname': cfg['name'], 'status': 'OFFLINE', 'state': 'OFFLINE', 'updated_at': time.time(), 'admission_allowed': False})
|
|
time.sleep(5)
|
|
atomic(STATE / 'hardware.json', hw)
|
|
stopped = False
|
|
def stop(*_):
|
|
nonlocal stopped
|
|
stopped = True
|
|
signal.signal(signal.SIGINT, stop)
|
|
signal.signal(signal.SIGTERM, stop)
|
|
last_probe, previous_path = 0, None
|
|
attempted_proof = None
|
|
while not stopped:
|
|
try:
|
|
proof = read(STATE / 'nvenc.json', {}) or {}
|
|
gpu = gpu_snapshot(cfg)
|
|
identity = (hw.get('boot_id'), gpu.get('driver_version'))
|
|
if self_test_marker_stale(hw | {'driver_version': gpu.get('driver_version')}, proof) and identity != attempted_proof and not read(STATE / 'runner.json', {}).get('running'):
|
|
attempted_proof = identity
|
|
atomic(STATE / 'status.json', {'updated_at': time.time(), 'status': 'GPU_UNAVAILABLE', 'state': 'GPU_UNAVAILABLE', 'admission_allowed': False})
|
|
if get_system_power_status() and get_system_power_status()['on_ac'] is True:
|
|
subprocess.run(['powershell.exe', '-NoProfile', '-NonInteractive', '-ExecutionPolicy', 'Bypass', '-File', str(ROOT / 'selftest-nvenc.ps1'), '-Root', str(ROOT), '-Device', str(cfg.get('nvenc_device', 0))], capture_output=True, timeout=90)
|
|
sample = collect(cfg, meter, admission, hw)
|
|
if previous_path and previous_path != sample['tailscale_path']:
|
|
meter.invalidate(); last_probe = 0
|
|
admission.admitted, admission.good, admission.path = False, 0, None
|
|
sample['admission_allowed'] = False
|
|
sample['state'] = sample['status'] = 'NETWORK_DEGRADED'
|
|
previous_path = sample['tailscale_path']
|
|
atomic(STATE / 'status.json', sample)
|
|
control = read(STATE / 'control.json', {'paused': True})
|
|
if not control.get('paused', True) and sample['network_ok'] and sample['on_ac'] is True and \
|
|
sample['tailscale_path'] == 'direct' and sample['active_jobs'] == 0 and \
|
|
time.time() - last_probe >= cfg.get('probe_interval_seconds', 300):
|
|
# Close admission and wait for the official graceful drain before measuring.
|
|
sample.update(admission_allowed=False, state='DRAINING', status='DRAINING')
|
|
atomic(STATE / 'status.json', sample)
|
|
if not sample['runner_running']:
|
|
rates = transfer_probe(cfg)
|
|
for direction in ('download', 'upload'):
|
|
value = rates.get(direction)
|
|
meter.consume({'kind': 'transfer', 'time': time.time(), 'direction': direction,
|
|
'ok': bool(value), 'bytes': value['bytes'] if value else 0,
|
|
'seconds': value['seconds'] if value else 1})
|
|
atomic(STATE / 'meter.json', meter.data)
|
|
last_probe = time.time()
|
|
except Exception:
|
|
# Suppress exception strings: upstream data may contain credentials/URLs.
|
|
atomic(STATE / 'status.json', {'hostname': cfg['name'], 'status': 'OFFLINE', 'state': 'OFFLINE',
|
|
'updated_at': time.time(), 'admission_allowed': False})
|
|
before_sleep = time.time()
|
|
time.sleep(cfg.get('poll_seconds', 5))
|
|
if time.time() - before_sleep > cfg.get('poll_seconds', 5) * 3:
|
|
meter.invalidate()
|
|
admission.admitted, admission.good = False, 0
|
|
last_probe = 0
|
|
atomic(STATE / 'status.json', {'hostname': cfg['name'], 'status': 'OFFLINE', 'state': 'OFFLINE',
|
|
'updated_at': time.time(), 'admission_allowed': False})
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument('command', choices=['daemon', 'status', 'network-test', 'start', 'stop', 'drain', 'restart', 'self-test'])
|
|
args = parser.parse_args()
|
|
STATE.mkdir(parents=True, exist_ok=True)
|
|
if args.command == 'daemon':
|
|
daemon()
|
|
elif args.command == 'status':
|
|
value = read(STATE / 'status.json', {'hostname': config()['name'], 'status': 'OFFLINE', 'state': 'OFFLINE'})
|
|
if time.time() - value.get('updated_at', 0) > 30:
|
|
value.update(status='OFFLINE', state='OFFLINE', stale=True, admission_allowed=False)
|
|
print(json.dumps(value, indent=2))
|
|
elif args.command == 'network-test':
|
|
meter = Meter(); meter.update()
|
|
rates = transfer_probe(config())
|
|
for direction in ('download', 'upload'):
|
|
r = rates.get(direction)
|
|
meter.consume({'kind': 'transfer', 'time': time.time(), 'direction': direction, 'ok': bool(r),
|
|
'bytes': r['bytes'] if r else 0, 'seconds': r['seconds'] if r else 1})
|
|
atomic(STATE / 'meter.json', meter.data)
|
|
print(json.dumps(rates))
|
|
return 0 if all(k in rates for k in ('download', 'upload')) else 1
|
|
elif args.command == 'self-test':
|
|
if read(STATE / 'runner.json', {}).get('running'):
|
|
raise ValueError('drain before self-test')
|
|
return subprocess.call(['powershell.exe', '-NoProfile', '-NonInteractive', '-ExecutionPolicy', 'Bypass',
|
|
'-File', str(ROOT / 'selftest-nvenc.ps1'), '-Root', str(ROOT),
|
|
'-Device', str(config().get('nvenc_device', 0))])
|
|
else:
|
|
paused = args.command in ('stop', 'drain')
|
|
atomic(STATE / 'control.json', {'paused': paused})
|
|
if args.command == 'restart':
|
|
try:
|
|
ipc('graceful-shutdown', config()['node_binary'])
|
|
except (OSError, RuntimeError, subprocess.SubprocessError):
|
|
pass
|
|
print(json.dumps({'requested': args.command, 'paused': paused}))
|
|
return 0
|
|
|
|
|
|
if __name__ == '__main__':
|
|
try:
|
|
raise SystemExit(main())
|
|
except (OSError, ValueError, RuntimeError, subprocess.SubprocessError):
|
|
print('pt-worker-win: command failed; check service health', file=__import__('sys').stderr)
|
|
raise SystemExit(1)
|