hacklab
8f01765df3
From the hacklab/T-HUB fork (diverged at a6843e9):
- 046_file_folders: folder trees (group + personal scope, RLS forced,
workspace-immutability trigger) + created_by on chat_media_archive
- Instagram connector wired into connect + publishing
- outreach contacts: per-collective DM tracking (todo/sent/replied/declined),
RLS forced, role-scoped permissions, audit without message text,
migration renumbered 041→047
- FORCE row-level security on media_assets
The J1 api agent's folder service/controller work continues separately.
185 lines
7.2 KiB
Python
185 lines
7.2 KiB
Python
#!/usr/bin/env python3
|
|
"""Native Windows hardware transcode worker; Python standard library only."""
|
|
|
|
import argparse
|
|
import json
|
|
import logging
|
|
import math
|
|
import os
|
|
from pathlib import Path
|
|
import subprocess
|
|
import tempfile
|
|
import time
|
|
import uuid
|
|
|
|
|
|
PRESETS = {
|
|
"nvenc_h264": ["-c:v", "h264_nvenc", "-preset", "p4", "-rc", "vbr"],
|
|
"qsv_h264": ["-c:v", "h264_qsv"],
|
|
"nvenc_hevc": ["-c:v", "hevc_nvenc", "-preset", "p4", "-rc", "vbr"],
|
|
}
|
|
|
|
|
|
class WorkerLock:
|
|
"""Windows releases this byte-range lock even if the worker crashes."""
|
|
|
|
def __init__(self, inbox):
|
|
self.path = inbox / ".worker.lock"
|
|
self.file = None
|
|
|
|
def __enter__(self):
|
|
import msvcrt
|
|
|
|
self.file = self.path.open("a+b")
|
|
self.file.seek(0, os.SEEK_END)
|
|
if self.file.tell() == 0:
|
|
self.file.write(b"\0")
|
|
self.file.flush()
|
|
self.file.seek(0)
|
|
try:
|
|
msvcrt.locking(self.file.fileno(), msvcrt.LK_NBLCK, 1)
|
|
except OSError:
|
|
self.file.close()
|
|
self.file = None
|
|
raise
|
|
return self
|
|
|
|
def __exit__(self, *args):
|
|
# Closing the handle releases the lock; keep the file to avoid races.
|
|
self.file.close()
|
|
|
|
|
|
def load_job(job_path):
|
|
# Windows PowerShell 5.1's UTF8 output includes a BOM.
|
|
with job_path.open(encoding="utf-8-sig") as source:
|
|
job = json.load(source)
|
|
if not isinstance(job, dict):
|
|
raise ValueError("Job must be a JSON object")
|
|
for key in ("input", "output", "preset"):
|
|
if not isinstance(job.get(key), str) or not job[key].strip():
|
|
raise ValueError("Job requires a nonempty string: " + key)
|
|
if job["preset"] not in PRESETS:
|
|
raise ValueError("Unknown preset: " + job["preset"])
|
|
quality = job.get("crf", 23)
|
|
if not isinstance(quality, int) or isinstance(quality, bool):
|
|
raise ValueError("crf must be an integer")
|
|
|
|
def job_file(value):
|
|
path = Path(value).expanduser()
|
|
return (job_path.parent / path).resolve()
|
|
|
|
input_path = job_file(job["input"])
|
|
output_path = job_file(job["output"])
|
|
if not input_path.is_file():
|
|
raise ValueError("Input file does not exist: " + str(input_path))
|
|
if input_path == output_path:
|
|
raise ValueError("Input and output must be different files")
|
|
if output_path.exists():
|
|
raise ValueError("Output already exists: " + str(output_path))
|
|
if not output_path.suffix:
|
|
raise ValueError("Output needs a container extension, such as .mp4")
|
|
return input_path, output_path, job["preset"], quality
|
|
|
|
|
|
def archive_job(job_path, directory, error=None):
|
|
target = directory / job_path.name
|
|
while target.exists() or target.with_suffix(target.suffix + ".err").exists():
|
|
target = directory / (job_path.stem + "-" + uuid.uuid4().hex + ".json")
|
|
if error is not None:
|
|
target.with_suffix(target.suffix + ".err").write_text(error, encoding="utf-8")
|
|
job_path.rename(target)
|
|
return target
|
|
|
|
|
|
def process_job(job_path, done, failed, ffmpeg):
|
|
temporary_output = None
|
|
try:
|
|
input_path, output_path, preset, quality = load_job(job_path)
|
|
output_path.parent.mkdir(parents=True, exist_ok=True)
|
|
# Keep the container suffix so ffmpeg can infer the output format.
|
|
temporary_output = output_path.with_name(
|
|
"." + output_path.stem + "-" + uuid.uuid4().hex + ".part" + output_path.suffix
|
|
)
|
|
video_args = list(PRESETS[preset])
|
|
if preset == "qsv_h264":
|
|
video_args += ["-global_quality", str(quality)]
|
|
else:
|
|
video_args += ["-cq", str(quality), "-b:v", "0"]
|
|
command = [
|
|
ffmpeg, "-hide_banner", "-nostdin", "-n", "-i", str(input_path),
|
|
"-map", "0:v:0", "-map", "0:a:0?",
|
|
*video_args, "-pix_fmt", "nv12", "-c:a", "aac", "-b:a", "192k",
|
|
str(temporary_output),
|
|
]
|
|
logging.info("Job %s: %s", job_path.name, subprocess.list2cmdline(command))
|
|
with tempfile.TemporaryFile() as stderr:
|
|
result = subprocess.run(command, stdin=subprocess.DEVNULL,
|
|
stdout=subprocess.DEVNULL, stderr=stderr, check=False)
|
|
if result.returncode != 0:
|
|
# Bound diagnostic size even for a long-running failed encode.
|
|
stderr.seek(0, os.SEEK_END)
|
|
stderr.seek(max(0, stderr.tell() - 65536))
|
|
detail = stderr.read().decode("utf-8", errors="replace")
|
|
raise RuntimeError(
|
|
"ffmpeg exited with code {}\nCommand: {}\n\n{}".format(
|
|
result.returncode, subprocess.list2cmdline(command), detail
|
|
)
|
|
)
|
|
if not temporary_output.is_file() or temporary_output.stat().st_size == 0:
|
|
raise RuntimeError("ffmpeg returned success without a nonempty output")
|
|
# On Windows rename refuses to replace an existing destination.
|
|
temporary_output.rename(output_path)
|
|
archive_job(job_path, done)
|
|
logging.info("Completed %s -> %s", job_path.name, output_path)
|
|
return True
|
|
except Exception as error:
|
|
logging.error("Failed %s: %s", job_path.name, error)
|
|
archive_job(job_path, failed, "{}: {}\n".format(type(error).__name__, error))
|
|
return False
|
|
finally:
|
|
if temporary_output is not None:
|
|
temporary_output.unlink(missing_ok=True)
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--inbox", type=Path, default=Path(r"C:\transcode-jobs\inbox"),
|
|
help="JSON inbox; done/failed are sibling directories")
|
|
parser.add_argument("--ffmpeg", default="ffmpeg", help="ffmpeg executable (default: PATH)")
|
|
parser.add_argument("--poll-interval", type=float, default=5, help="Poll interval in seconds")
|
|
parser.add_argument("--once", action="store_true", help="Process the current inbox and exit")
|
|
args = parser.parse_args()
|
|
if os.name != "nt":
|
|
parser.error("Run this worker with Windows Python directly, not inside WSL")
|
|
if not math.isfinite(args.poll_interval) or args.poll_interval <= 0:
|
|
parser.error("--poll-interval must be a finite positive number")
|
|
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
|
|
inbox = args.inbox.resolve()
|
|
done, failed = inbox.parent / "done", inbox.parent / "failed"
|
|
for directory in (inbox, done, failed):
|
|
directory.mkdir(parents=True, exist_ok=True)
|
|
lock = WorkerLock(inbox)
|
|
try:
|
|
lock.__enter__()
|
|
except OSError as error:
|
|
logging.error("Cannot acquire worker lock (another worker may be running): %s", error)
|
|
return 1
|
|
try:
|
|
logging.info("Polling %s", inbox)
|
|
had_failure = False
|
|
while True:
|
|
for job_path in sorted(inbox.glob("*.json")):
|
|
if job_path.is_file():
|
|
had_failure = not process_job(job_path, done, failed, args.ffmpeg) or had_failure
|
|
if args.once:
|
|
return int(had_failure)
|
|
time.sleep(args.poll_interval)
|
|
except KeyboardInterrupt:
|
|
logging.info("Worker stopped")
|
|
return 0
|
|
finally:
|
|
lock.__exit__()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|