feat: adapt T-HUB features — folders schema, Instagram connector, outreach contacts, media RLS force
From the hacklab/T-HUB fork (diverged at a6843e9):
- 046_file_folders: folder trees (group + personal scope, RLS forced,
workspace-immutability trigger) + created_by on chat_media_archive
- Instagram connector wired into connect + publishing
- outreach contacts: per-collective DM tracking (todo/sent/replied/declined),
RLS forced, role-scoped permissions, audit without message text,
migration renumbered 041→047
- FORCE row-level security on media_assets
The J1 api agent's folder service/controller work continues separately.
This commit is contained in:
parent
57762f543c
commit
8f01765df3
25 changed files with 3054 additions and 4 deletions
185
apps/media-worker/scripts/windows-transcode-worker.py
Normal file
185
apps/media-worker/scripts/windows-transcode-worker.py
Normal file
|
|
@ -0,0 +1,185 @@
|
|||
#!/usr/bin/env python3
|
||||
"""Native Windows hardware transcode worker; Python standard library only."""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import logging
|
||||
import math
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import tempfile
|
||||
import time
|
||||
import uuid
|
||||
|
||||
|
||||
PRESETS = {
|
||||
"nvenc_h264": ["-c:v", "h264_nvenc", "-preset", "p4", "-rc", "vbr"],
|
||||
"qsv_h264": ["-c:v", "h264_qsv"],
|
||||
"nvenc_hevc": ["-c:v", "hevc_nvenc", "-preset", "p4", "-rc", "vbr"],
|
||||
}
|
||||
|
||||
|
||||
class WorkerLock:
|
||||
"""Windows releases this byte-range lock even if the worker crashes."""
|
||||
|
||||
def __init__(self, inbox):
|
||||
self.path = inbox / ".worker.lock"
|
||||
self.file = None
|
||||
|
||||
def __enter__(self):
|
||||
import msvcrt
|
||||
|
||||
self.file = self.path.open("a+b")
|
||||
self.file.seek(0, os.SEEK_END)
|
||||
if self.file.tell() == 0:
|
||||
self.file.write(b"\0")
|
||||
self.file.flush()
|
||||
self.file.seek(0)
|
||||
try:
|
||||
msvcrt.locking(self.file.fileno(), msvcrt.LK_NBLCK, 1)
|
||||
except OSError:
|
||||
self.file.close()
|
||||
self.file = None
|
||||
raise
|
||||
return self
|
||||
|
||||
def __exit__(self, *args):
|
||||
# Closing the handle releases the lock; keep the file to avoid races.
|
||||
self.file.close()
|
||||
|
||||
|
||||
def load_job(job_path):
|
||||
# Windows PowerShell 5.1's UTF8 output includes a BOM.
|
||||
with job_path.open(encoding="utf-8-sig") as source:
|
||||
job = json.load(source)
|
||||
if not isinstance(job, dict):
|
||||
raise ValueError("Job must be a JSON object")
|
||||
for key in ("input", "output", "preset"):
|
||||
if not isinstance(job.get(key), str) or not job[key].strip():
|
||||
raise ValueError("Job requires a nonempty string: " + key)
|
||||
if job["preset"] not in PRESETS:
|
||||
raise ValueError("Unknown preset: " + job["preset"])
|
||||
quality = job.get("crf", 23)
|
||||
if not isinstance(quality, int) or isinstance(quality, bool):
|
||||
raise ValueError("crf must be an integer")
|
||||
|
||||
def job_file(value):
|
||||
path = Path(value).expanduser()
|
||||
return (job_path.parent / path).resolve()
|
||||
|
||||
input_path = job_file(job["input"])
|
||||
output_path = job_file(job["output"])
|
||||
if not input_path.is_file():
|
||||
raise ValueError("Input file does not exist: " + str(input_path))
|
||||
if input_path == output_path:
|
||||
raise ValueError("Input and output must be different files")
|
||||
if output_path.exists():
|
||||
raise ValueError("Output already exists: " + str(output_path))
|
||||
if not output_path.suffix:
|
||||
raise ValueError("Output needs a container extension, such as .mp4")
|
||||
return input_path, output_path, job["preset"], quality
|
||||
|
||||
|
||||
def archive_job(job_path, directory, error=None):
|
||||
target = directory / job_path.name
|
||||
while target.exists() or target.with_suffix(target.suffix + ".err").exists():
|
||||
target = directory / (job_path.stem + "-" + uuid.uuid4().hex + ".json")
|
||||
if error is not None:
|
||||
target.with_suffix(target.suffix + ".err").write_text(error, encoding="utf-8")
|
||||
job_path.rename(target)
|
||||
return target
|
||||
|
||||
|
||||
def process_job(job_path, done, failed, ffmpeg):
|
||||
temporary_output = None
|
||||
try:
|
||||
input_path, output_path, preset, quality = load_job(job_path)
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
# Keep the container suffix so ffmpeg can infer the output format.
|
||||
temporary_output = output_path.with_name(
|
||||
"." + output_path.stem + "-" + uuid.uuid4().hex + ".part" + output_path.suffix
|
||||
)
|
||||
video_args = list(PRESETS[preset])
|
||||
if preset == "qsv_h264":
|
||||
video_args += ["-global_quality", str(quality)]
|
||||
else:
|
||||
video_args += ["-cq", str(quality), "-b:v", "0"]
|
||||
command = [
|
||||
ffmpeg, "-hide_banner", "-nostdin", "-n", "-i", str(input_path),
|
||||
"-map", "0:v:0", "-map", "0:a:0?",
|
||||
*video_args, "-pix_fmt", "nv12", "-c:a", "aac", "-b:a", "192k",
|
||||
str(temporary_output),
|
||||
]
|
||||
logging.info("Job %s: %s", job_path.name, subprocess.list2cmdline(command))
|
||||
with tempfile.TemporaryFile() as stderr:
|
||||
result = subprocess.run(command, stdin=subprocess.DEVNULL,
|
||||
stdout=subprocess.DEVNULL, stderr=stderr, check=False)
|
||||
if result.returncode != 0:
|
||||
# Bound diagnostic size even for a long-running failed encode.
|
||||
stderr.seek(0, os.SEEK_END)
|
||||
stderr.seek(max(0, stderr.tell() - 65536))
|
||||
detail = stderr.read().decode("utf-8", errors="replace")
|
||||
raise RuntimeError(
|
||||
"ffmpeg exited with code {}\nCommand: {}\n\n{}".format(
|
||||
result.returncode, subprocess.list2cmdline(command), detail
|
||||
)
|
||||
)
|
||||
if not temporary_output.is_file() or temporary_output.stat().st_size == 0:
|
||||
raise RuntimeError("ffmpeg returned success without a nonempty output")
|
||||
# On Windows rename refuses to replace an existing destination.
|
||||
temporary_output.rename(output_path)
|
||||
archive_job(job_path, done)
|
||||
logging.info("Completed %s -> %s", job_path.name, output_path)
|
||||
return True
|
||||
except Exception as error:
|
||||
logging.error("Failed %s: %s", job_path.name, error)
|
||||
archive_job(job_path, failed, "{}: {}\n".format(type(error).__name__, error))
|
||||
return False
|
||||
finally:
|
||||
if temporary_output is not None:
|
||||
temporary_output.unlink(missing_ok=True)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--inbox", type=Path, default=Path(r"C:\transcode-jobs\inbox"),
|
||||
help="JSON inbox; done/failed are sibling directories")
|
||||
parser.add_argument("--ffmpeg", default="ffmpeg", help="ffmpeg executable (default: PATH)")
|
||||
parser.add_argument("--poll-interval", type=float, default=5, help="Poll interval in seconds")
|
||||
parser.add_argument("--once", action="store_true", help="Process the current inbox and exit")
|
||||
args = parser.parse_args()
|
||||
if os.name != "nt":
|
||||
parser.error("Run this worker with Windows Python directly, not inside WSL")
|
||||
if not math.isfinite(args.poll_interval) or args.poll_interval <= 0:
|
||||
parser.error("--poll-interval must be a finite positive number")
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
|
||||
inbox = args.inbox.resolve()
|
||||
done, failed = inbox.parent / "done", inbox.parent / "failed"
|
||||
for directory in (inbox, done, failed):
|
||||
directory.mkdir(parents=True, exist_ok=True)
|
||||
lock = WorkerLock(inbox)
|
||||
try:
|
||||
lock.__enter__()
|
||||
except OSError as error:
|
||||
logging.error("Cannot acquire worker lock (another worker may be running): %s", error)
|
||||
return 1
|
||||
try:
|
||||
logging.info("Polling %s", inbox)
|
||||
had_failure = False
|
||||
while True:
|
||||
for job_path in sorted(inbox.glob("*.json")):
|
||||
if job_path.is_file():
|
||||
had_failure = not process_job(job_path, done, failed, args.ffmpeg) or had_failure
|
||||
if args.once:
|
||||
return int(had_failure)
|
||||
time.sleep(args.poll_interval)
|
||||
except KeyboardInterrupt:
|
||||
logging.info("Worker stopped")
|
||||
return 0
|
||||
finally:
|
||||
lock.__exit__()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Loading…
Add table
Add a link
Reference in a new issue