Tactical Media Broadcast Hub — initial release
Self-hosted multi-platform broadcast newsroom for activist collectives: publishing pipeline with group approval and DST-proof scheduling, TweetDeck-style column wall with hostile-content-safe rendering, live control room with Auto Director, bridged chat governance with polls and quorum, distributed sandboxed transcode workers, encrypted content-addressed archive, monthly analytics. MIT licensed. See README.md and docs/.
This commit is contained in:
commit
aa5e262f3a
880 changed files with 111042 additions and 0 deletions
128
infra/peertube-workers/tests/runner_smoke.py
Normal file
128
infra/peertube-workers/tests/runner_smoke.py
Normal file
|
|
@ -0,0 +1,128 @@
|
|||
#!/usr/bin/python3
|
||||
"""Integration test with published runner against a loopback API; no production calls."""
|
||||
import argparse
|
||||
import http.server
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import tempfile
|
||||
import threading
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
UUID = '22222222-2222-4222-8222-222222222222'
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('--node', required=True)
|
||||
parser.add_argument('--runtime', required=True, help='directory containing installed runtime node_modules')
|
||||
args = parser.parse_args()
|
||||
with tempfile.TemporaryDirectory(prefix='pt-runner-smoke-') as directory:
|
||||
tmp = Path(directory)
|
||||
home, state = tmp / 'home', tmp / 'state'
|
||||
home.mkdir(); state.mkdir()
|
||||
(tmp / 'runtime').symlink_to(Path(args.runtime).resolve(), target_is_directory=True)
|
||||
shutil.copy(ROOT / 'ipc.cjs', tmp / 'ipc.cjs')
|
||||
fixture = tmp / 'source.mp4'
|
||||
subprocess.run(['/usr/bin/ffmpeg', '-v', 'error', '-f', 'lavfi', '-i', 'testsrc2=size=64x64:rate=2',
|
||||
'-t', '1', '-c:v', 'libx264', '-threads', '1', str(fixture)], check=True)
|
||||
requests, accepted, failures = [], threading.Event(), threading.Event()
|
||||
registered_count = 0
|
||||
offered = False
|
||||
class API(http.server.BaseHTTPRequestHandler):
|
||||
def log_message(self, *_): pass
|
||||
def do_POST(self):
|
||||
nonlocal registered_count, offered
|
||||
length = int(self.headers.get('Content-Length', 0))
|
||||
self.rfile.read(length) # discarded; never log credentials
|
||||
requests.append(self.path)
|
||||
if self.path == '/api/v1/runners/register':
|
||||
registered_count += 1
|
||||
return self.reply({'runnerToken': 'ptrt-local-test-only'})
|
||||
if self.path == '/api/v1/runners/jobs/request':
|
||||
jobs = [] if offered or not registered_count else [{'uuid': UUID, 'type': 'vod-web-video-transcoding',
|
||||
'payload': {'input': {'videoFileUrl': f'https://canonical.invalid/api/v1/runners/jobs/{UUID}/files/videos/source/max-quality',
|
||||
'separatedAudioFileUrl': []}, 'output': {'resolution': 64, 'fps': 2}}}]
|
||||
return self.reply({'availableJobs': jobs})
|
||||
if self.path.endswith('/accept'):
|
||||
offered = True
|
||||
accepted.set()
|
||||
return self.reply({'job': {'uuid': UUID, 'type': 'vod-web-video-transcoding', 'jobToken': 'local-job-secret',
|
||||
'payload': {'input': {'videoFileUrl': f'https://canonical.invalid/api/v1/runners/jobs/{UUID}/files/videos/source/max-quality',
|
||||
'separatedAudioFileUrl': []}, 'output': {'resolution': 64, 'fps': 2}}}})
|
||||
if '/files/' in self.path:
|
||||
blob = fixture.read_bytes()
|
||||
self.send_response(200); self.send_header('Content-Length', str(len(blob))); self.end_headers()
|
||||
self.wfile.write(blob)
|
||||
return
|
||||
if self.path.endswith('/error'):
|
||||
failures.set()
|
||||
self.send_response(204); self.end_headers()
|
||||
def reply(self, data):
|
||||
blob = json.dumps(data).encode()
|
||||
self.send_response(200); self.send_header('Content-Type', 'application/json')
|
||||
self.send_header('Content-Length', str(len(blob))); self.end_headers(); self.wfile.write(blob)
|
||||
server = http.server.ThreadingHTTPServer(('127.0.0.1', 0), API)
|
||||
threading.Thread(target=server.serve_forever, daemon=True).start()
|
||||
url = f'http://127.0.0.1:{server.server_port}'
|
||||
cfg = tmp / 'worker.json'
|
||||
cfg.write_text(json.dumps({'name': 'smoke-worker', 'peertube_url': url, 'canonical_origin': 'https://canonical.invalid'}))
|
||||
env = {**os.environ, 'HOME': str(home), 'XDG_CONFIG_HOME': str(home / '.config'),
|
||||
'XDG_CACHE_HOME': str(home / '.cache'), 'XDG_DATA_HOME': str(home / '.local/share'),
|
||||
'PT_WORKER_CONFIG': str(cfg), 'PT_STATE_DIR': str(state),
|
||||
'PT_RENDER_DEVICE': str(tmp / 'missing-render'), 'FFMPEG_PATH': str(ROOT / 'ffmpeg-qsv.py'),
|
||||
'FFPROBE_PATH': '/usr/bin/ffprobe', 'NODE_OPTIONS': '--require=' + str(ROOT / 'observe.cjs')}
|
||||
runner = Path(args.runtime) / 'node_modules/@peertube/peertube-runner/dist/peertube-runner.js'
|
||||
runner_config = home / '.config/peertube-runner-nodejs/default/config.toml'
|
||||
runner_config.parent.mkdir(parents=True)
|
||||
runner_config.write_text('[jobs]\nconcurrency=0\n[ffmpeg]\nthreads=2\nnice=20\n')
|
||||
socket = home / '.local/share/peertube-runner-nodejs/default/peertube-runner.sock'
|
||||
children = []
|
||||
def start():
|
||||
child = subprocess.Popen([args.node, str(runner), 'server', '--enable-job', 'vod-web-video-transcoding'],
|
||||
env=env, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
||||
children.append(child)
|
||||
for _ in range(100):
|
||||
if socket.exists():
|
||||
p = subprocess.run([args.node, str(tmp / 'ipc.cjs'), 'list-jobs'], env=env, capture_output=True, timeout=3)
|
||||
if p.returncode == 0: return child
|
||||
if child.poll() is not None: raise AssertionError('runner exited')
|
||||
time.sleep(.1)
|
||||
raise AssertionError('IPC socket unavailable')
|
||||
def call(command, data=None):
|
||||
return subprocess.run([args.node, str(tmp / 'ipc.cjs'), command], input=data,
|
||||
env=env, capture_output=True, text=True, timeout=20, check=True)
|
||||
try:
|
||||
child = start()
|
||||
call('register', 'ptrrt-local-test-only')
|
||||
assert registered_count == 1
|
||||
assert json.loads(call('list-jobs').stdout)['active_jobs'] == 0
|
||||
call('graceful-shutdown'); child.wait(timeout=10)
|
||||
assert not accepted.is_set(), 'enrollment accepted a job at concurrency=0'
|
||||
runner_config.write_text(runner_config.read_text().replace('concurrency = 0', 'concurrency = 1'))
|
||||
socket.unlink(missing_ok=True)
|
||||
child = start()
|
||||
assert accepted.wait(15), 'job not accepted'
|
||||
assert failures.wait(15), 'missing QSV did not report job failure'
|
||||
call('graceful-shutdown'); child.wait(timeout=10)
|
||||
assert registered_count == 1, 'restart duplicated registration'
|
||||
transcoding = home / '.cache/peertube-runner-nodejs/default/transcoding'
|
||||
assert not list(transcoding.iterdir()), 'temporary files not cleaned'
|
||||
events = (state / 'events.jsonl').read_text()
|
||||
assert 'gpu_failure' in events
|
||||
assert 'local-job-secret' not in events and 'ptrt-local-test-only' not in events
|
||||
assert not any(p.endswith('/success') for p in requests), 'failed QSV uploaded output'
|
||||
assert any('/files/' in p for p in requests), 'canonical source was not routed to loopback'
|
||||
print('PASS: official runner 0.2.0 registration via secret stdin, concurrency=0 enrollment, restart without re-registration, source routing, QSV failure reporting, graceful shutdown, tmp cleanup')
|
||||
finally:
|
||||
for child in children:
|
||||
if child.poll() is None:
|
||||
child.kill(); child.wait()
|
||||
server.shutdown(); server.server_close()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
62
infra/peertube-workers/tests/test_observer.cjs
Normal file
62
infra/peertube-workers/tests/test_observer.cjs
Normal file
|
|
@ -0,0 +1,62 @@
|
|||
'use strict';
|
||||
// Real loopback transfers exercise routing, byte counts, file/job correlation, and secret exclusion.
|
||||
const fs = require('node:fs');
|
||||
const http = require('node:http');
|
||||
const path = require('node:path');
|
||||
const os = require('node:os');
|
||||
const assert = require('node:assert/strict');
|
||||
const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'pt-observer-'));
|
||||
const uuid = '11111111-1111-4111-8111-111111111111';
|
||||
const source = Buffer.alloc(300000, 7);
|
||||
let uploaded = 0;
|
||||
const server = http.createServer((req, res) => {
|
||||
if (req.url.includes('/files/abort')) {
|
||||
res.writeHead(200, { 'Content-Length': '1000000' });
|
||||
res.write(Buffer.alloc(1000)); setTimeout(() => res.destroy(), 20);
|
||||
} else if (req.url.includes('/files/')) { res.writeHead(200); res.end(source); }
|
||||
else {
|
||||
req.on('data', chunk => { uploaded += chunk.length; });
|
||||
req.on('end', () => { res.writeHead(204); res.end(); });
|
||||
}
|
||||
});
|
||||
server.listen(0, '127.0.0.1', async () => {
|
||||
const port = server.address().port;
|
||||
const cfg = path.join(tmp, 'worker.json');
|
||||
fs.writeFileSync(cfg, JSON.stringify({ canonical_origin: 'https://canonical.invalid', peertube_url: `http://127.0.0.1:${port}` }));
|
||||
process.env.PT_WORKER_CONFIG = cfg;
|
||||
process.env.PT_STATE_DIR = tmp;
|
||||
require('../observe.cjs');
|
||||
const https = require('node:https');
|
||||
try {
|
||||
await new Promise((ok, fail) => {
|
||||
const req = https.request(`https://canonical.invalid/api/v1/runners/jobs/${uuid}/files/videos/source?secret=NEVER_RECORD_ME`, res => {
|
||||
const file = fs.createWriteStream(path.join(tmp, 'source'));
|
||||
res.pipe(file); file.on('finish', ok);
|
||||
});
|
||||
req.on('error', fail); req.end('runnerToken=NEVER_RECORD_ME');
|
||||
});
|
||||
await new Promise((ok, fail) => {
|
||||
const req = http.request({ hostname: '127.0.0.1', port, path: `/api/v1/runners/jobs/${uuid}/success`, method: 'POST' }, res => {
|
||||
res.resume(); res.on('end', ok);
|
||||
});
|
||||
req.on('error', fail);
|
||||
req.write(Buffer.alloc(100000, 1)); req.end(Buffer.alloc(200000, 2));
|
||||
});
|
||||
const events = fs.readFileSync(path.join(tmp, 'events.jsonl'), 'utf8');
|
||||
assert(!events.includes('NEVER_RECORD_ME'));
|
||||
const parsed = events.trim().split('\n').map(JSON.parse);
|
||||
assert.equal(parsed.find(e => e.kind === 'transfer' && e.direction === 'download').bytes, source.length);
|
||||
assert.equal(parsed.find(e => e.kind === 'transfer' && e.direction === 'upload').bytes, uploaded);
|
||||
assert.equal(uploaded, 300000);
|
||||
assert.equal(JSON.parse(fs.readFileSync(path.join(tmp, 'inputs/source.json'))).uuid, uuid);
|
||||
await new Promise((ok, fail) => {
|
||||
const timeout = setTimeout(() => fail(new Error('interrupted transfer did not fail promptly')), 2000);
|
||||
const req = http.request({ hostname: '127.0.0.1', port, path: `/api/v1/runners/jobs/${uuid}/files/abort` }, res => res.resume());
|
||||
req.on('error', () => { clearTimeout(timeout); ok(); }); req.end();
|
||||
});
|
||||
const failed = fs.readFileSync(path.join(tmp, 'events.jsonl'), 'utf8').trim().split('\n').map(JSON.parse);
|
||||
assert(failed.some(e => e.kind === 'transfer' && !e.ok));
|
||||
console.log('PASS: real download/upload counts, Tailscale-origin routing, UUID correlation, interrupted-transfer failure, no secrets');
|
||||
} catch (e) { console.error(e); process.exitCode = 1; }
|
||||
finally { server.close(); fs.rmSync(tmp, { recursive: true, force: true }); }
|
||||
});
|
||||
157
infra/peertube-workers/tests/test_worker.py
Normal file
157
infra/peertube-workers/tests/test_worker.py
Normal file
|
|
@ -0,0 +1,157 @@
|
|||
import importlib.util
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import tempfile
|
||||
import time
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
spec = importlib.util.spec_from_file_location('worker', ROOT / 'pt-worker.py')
|
||||
worker = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(worker)
|
||||
|
||||
|
||||
class PolicyTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.tmp = tempfile.TemporaryDirectory()
|
||||
self.old_state = worker.STATE
|
||||
worker.STATE = Path(self.tmp.name)
|
||||
self.meter = worker.Meter()
|
||||
self.now = time.time()
|
||||
for direction in ('download', 'upload'):
|
||||
self.meter.consume({'kind': 'transfer', 'time': self.now, 'ok': True,
|
||||
'direction': direction, 'bytes': 10_000_000, 'seconds': 1})
|
||||
self.policy = worker.Admission({})
|
||||
self.sample = {'tailscale_path': 'direct', 'latency_ms': 5, 'network_ok': True,
|
||||
'registered': True, 'qsv': {'qsv_h264_ok': True, 'tested_at': self.now},
|
||||
'gpu_ready': True, 'gpu_tested_at': self.now,
|
||||
'gpu_accessible': True, 'on_ac': True, 'disk_free_bytes': 100 * 1024**3,
|
||||
'temperature_c': 50, 'throttle_counts': {}, 'active_jobs': 0}
|
||||
self.control = {'paused': False}
|
||||
|
||||
def tearDown(self):
|
||||
worker.STATE = self.old_state
|
||||
self.tmp.cleanup()
|
||||
|
||||
def decide(self):
|
||||
return self.policy.decide(self.sample, self.meter, self.control, self.now)
|
||||
|
||||
def available(self):
|
||||
self.assertEqual(self.decide(), 'NETWORK_DEGRADED')
|
||||
self.assertEqual(self.decide(), 'NETWORK_DEGRADED')
|
||||
self.assertEqual(self.decide(), 'AVAILABLE')
|
||||
|
||||
def test_recovery_hysteresis_and_low_bandwidth_drain(self):
|
||||
self.available()
|
||||
self.meter.data['download']['mbps'] = 1
|
||||
self.assertEqual(self.decide(), 'AVAILABLE')
|
||||
self.assertEqual(self.decide(), 'NETWORK_DEGRADED')
|
||||
self.meter.data['download']['mbps'] = 11 # insufficient recovery margin
|
||||
self.assertEqual(self.decide(), 'NETWORK_DEGRADED')
|
||||
self.meter.data['download']['mbps'] = 20
|
||||
self.available()
|
||||
|
||||
def test_path_transition_invalidates_real_bandwidth(self):
|
||||
self.available()
|
||||
self.sample['tailscale_path'] = 'DERP'
|
||||
self.assertEqual(self.decide(), 'NETWORK_DEGRADED')
|
||||
self.assertIsNone(self.sample['recent_download_mbps'])
|
||||
self.sample['tailscale_path'] = 'direct'
|
||||
self.assertEqual(self.decide(), 'NETWORK_DEGRADED')
|
||||
|
||||
def test_battery_gpu_disk_and_network_refuse(self):
|
||||
self.available()
|
||||
self.sample['on_ac'] = False
|
||||
self.assertEqual(self.decide(), 'DRAINING')
|
||||
self.sample['on_ac'] = True
|
||||
self.sample['gpu_accessible'] = False
|
||||
self.assertEqual(self.decide(), 'GPU_UNAVAILABLE')
|
||||
self.sample['gpu_accessible'] = True
|
||||
self.sample['disk_free_bytes'] = 0
|
||||
self.assertEqual(self.decide(), 'LOW_DISK')
|
||||
self.sample['disk_free_bytes'] = 100 * 1024**3
|
||||
self.sample['network_ok'] = False
|
||||
self.assertEqual(self.decide(), 'OFFLINE')
|
||||
|
||||
def test_gpu_failure_latched_until_later_selftest(self):
|
||||
self.meter.consume({'kind': 'gpu_failure', 'time': self.now + 1})
|
||||
self.assertEqual(self.decide(), 'GPU_UNAVAILABLE')
|
||||
self.sample['qsv']['tested_at'] = self.now + 2
|
||||
self.sample['gpu_tested_at'] = self.now + 2
|
||||
self.available()
|
||||
|
||||
def test_transfer_failure_cooldown_and_stale_metrics(self):
|
||||
self.meter.consume({'kind': 'transfer', 'time': self.now, 'ok': False})
|
||||
self.assertEqual(self.decide(), 'NETWORK_DEGRADED')
|
||||
self.now += 61
|
||||
self.available()
|
||||
self.now += 900
|
||||
self.assertEqual(self.decide(), 'AVAILABLE')
|
||||
self.assertEqual(self.decide(), 'NETWORK_DEGRADED')
|
||||
|
||||
def test_thermal_throttle_and_pause(self):
|
||||
self.available()
|
||||
self.sample['throttle_counts'] = {'cpu0': 1}
|
||||
self.decide()
|
||||
self.sample['throttle_counts'] = {'cpu0': 2}
|
||||
self.assertEqual(self.decide(), 'THERMAL_THROTTLED')
|
||||
self.sample['temperature_c'] = 50
|
||||
self.now += 121
|
||||
self.available()
|
||||
self.control['paused'] = True
|
||||
self.assertEqual(self.decide(), 'PAUSED')
|
||||
|
||||
def test_job_correlation_uses_uuid_under_concurrency(self):
|
||||
for uuid in ('a', 'b'):
|
||||
self.meter.consume({'kind': 'job_accept', 'time': self.now, 'ok': True, 'uuid': uuid})
|
||||
self.meter.consume({'kind': 'encode', 'time': self.now + 2, 'job_uuid': 'b', 'duration': 200})
|
||||
self.meter.consume({'kind': 'encode', 'time': self.now + 3, 'job_uuid': 'a', 'duration': 100})
|
||||
self.meter.consume({'kind': 'job_success', 'time': self.now + 10, 'uuid': 'b', 'ok': True})
|
||||
self.assertEqual(self.meter.data['last_job_realtime_factor'], 20)
|
||||
|
||||
|
||||
class AdapterTests(unittest.TestCase):
|
||||
def probe(self, codec='h264', rotation=False):
|
||||
return {'streams': [{'codec_type': 'video', 'codec_name': codec, 'pix_fmt': 'yuv420p',
|
||||
'width': 1920, 'height': 1080,
|
||||
'side_data_list': [{'rotation': 90}] if rotation else []}],
|
||||
'format': {'duration': '10'}}
|
||||
|
||||
def test_qsv_decode_scale_encode_and_no_x264_flags(self):
|
||||
args, info = worker.qsv.rewrite(['-i', '/tmp/in', '-vcodec', 'libx264', '-vf', 'scale=w=-2:h=720',
|
||||
'-pix_fmt', 'yuv420p', '-preset', 'veryfast', '-b_strategy', '1', '-bf', '16', '/tmp/out'], self.probe())
|
||||
self.assertIn('h264_qsv', args)
|
||||
self.assertIn('scale_qsv=w=1280:h=720:format=nv12', args)
|
||||
self.assertNotIn('libx264', args)
|
||||
self.assertNotIn('veryfast', args)
|
||||
self.assertNotIn('-b_strategy', args)
|
||||
self.assertEqual(info['target_height'], 720)
|
||||
|
||||
def test_unknown_decode_rotation_and_cpu_encoder_fail(self):
|
||||
for probe in (self.probe('mpeg4'), self.probe(rotation=True)):
|
||||
with self.assertRaises(ValueError):
|
||||
worker.qsv.rewrite(['-i', '/tmp/in', '-c:v', 'libx264', '/tmp/out'], probe)
|
||||
with self.assertRaises(ValueError):
|
||||
worker.qsv.rewrite(['-i', '/tmp/in', '-c:v', 'libx265', '/tmp/out'], self.probe())
|
||||
|
||||
def test_unreviewed_software_filter_refused(self):
|
||||
with self.assertRaises(ValueError):
|
||||
worker.qsv.rewrite(['-i', '/tmp/in', '-c:v', 'libx264', '-vf', 'scale=640:360,eq=contrast=2', '/tmp/out'], self.probe())
|
||||
|
||||
def test_real_missing_qsv_refuses_without_cpu_output(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
src, out = Path(tmp) / 'src.mp4', Path(tmp) / 'out.mp4'
|
||||
subprocess.run(['/usr/bin/ffmpeg', '-v', 'error', '-f', 'lavfi', '-i', 'testsrc2=size=64x64:rate=2',
|
||||
'-t', '1', '-c:v', 'libx264', '-threads', '1', str(src)], check=True)
|
||||
env = {**os.environ, 'PT_STATE_DIR': tmp, 'PT_RENDER_DEVICE': tmp + '/missing-device'}
|
||||
p = subprocess.run(['/usr/bin/python3', str(ROOT / 'ffmpeg-qsv.py'), '-i', str(src), '-c:v',
|
||||
'libx264', str(out)], env=env, capture_output=True)
|
||||
self.assertEqual(p.returncode, 78)
|
||||
self.assertFalse(out.exists())
|
||||
self.assertIn('gpu_failure', (Path(tmp) / 'events.jsonl').read_text())
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
Loading…
Add table
Add a link
Reference in a new issue