diff --git a/infra/occupymedia/edge/README.md b/infra/occupymedia/edge/README.md index 599a21b..69ed7d2 100644 --- a/infra/occupymedia/edge/README.md +++ b/infra/occupymedia/edge/README.md @@ -55,13 +55,20 @@ Router: forward TCP 443 (80 optional) → the CT's LAN IP. ## Activation -The workstation watcher (`edge-activate.sh`, not shipped here) probes -`ping.occupymedia.cc` through a reachable edge IP. Two consecutive 200s after -the port-forward goes live → it sets `EDGE_ENABLED=true`, flips the four -hostnames, verifies the public path for 6 minutes, and **reverts to TUNNEL -automatically** if the new pool assignment is broken too. The CT watcher then -owns ongoing failover (2 tunnel failures → ORIGIN/GREY; 3 origin failures → -TUNNEL). +The workstation watcher (`edge-activate.sh`) probes `ping.occupymedia.cc` +through a reachable edge IP. Two consecutive 200s after the port-forward goes +live → it sets `EDGE_ENABLED=true`, flips the four hostnames, verifies the +public path for 6 minutes, and **reverts to TUNNEL automatically** if the new +pool assignment is broken too. The CT watcher then owns ongoing failover (2 +tunnel failures → ORIGIN/GREY; 3 origin failures → TUNNEL). + +Run it from a machine that (a) has the repo `.env` with the Cloudflare token, +(b) can SSH to the CT, and (c) is OUTSIDE the affected region if you need to +distinguish "edge pool broken" from "origin unreachable": + +```bash +TARGET_MODE=ORIGIN ./edge-activate.sh # or GREY for direct DNS +``` Target mode: `TARGET_MODE=ORIGIN` (default) or `TARGET_MODE=GREY` in `watcher.env` / the activation environment. diff --git a/infra/occupymedia/edge/edge-activate.sh b/infra/occupymedia/edge/edge-activate.sh new file mode 100755 index 0000000..d31afe2 --- /dev/null +++ b/infra/occupymedia/edge/edge-activate.sh @@ -0,0 +1,129 @@ +#!/bin/bash +# edge-activate.sh — one-shot activation: flips the four hostnames to ORIGIN mode +# once the router port-forward verifiably works THROUGH CLOUDFLARE (the real +# production path: working edge pool -> CF -> origin:443 -> nginx -> apps). +# +# Probe: https://ping.occupymedia.cc (proxied A -> public IP). nginx returns 200 +# "edge-ok" only when the whole chain works. While the forward is down CF +# answers 521 (origin unreachable), which counts as not-live. +# +# Also keeps ping's A record synced to the current public IP (the IP is dynamic). +# Retires itself if the tunnel path heals first (flip not needed). +set -uo pipefail +source /home/moxexley/projects/SuperPoster/.env 2>/dev/null || { echo "no .env"; exit 1; } + +ZONE="4bd0ac2e13a735d2e0068ce3feea2996" +HOSTS=(app api tube chat) +TARGET_MODE=${TARGET_MODE:-ORIGIN} # ORIGIN (CF-proxied) or GREY (direct DNS) +LOG=/tmp/opencode/edge-activate.log +SSH="ssh -i $HOME/.ssh/devserver -o ConnectTimeout=10 root@proxnode2" + +log() { echo "$(date -u '+%H:%M:%S') $*" >> "$LOG"; } + +get_ip() { $SSH "curl -s -m 8 ifconfig.me" 2>/dev/null; } + +probe_edge() { curl -s -4 -o /dev/null -w '%{http_code}' -m 20 \ + --resolve "ping.occupymedia.cc:443:104.21.59.202" https://ping.occupymedia.cc 2>/dev/null; } + +revert_to_tunnel() { + T="Authorization: Bearer $CLOUDFLARE_API_TOKEN" + ZURL="https://api.cloudflare.com/client/v4/zones/$ZONE/dns_records" + for h in "${HOSTS[@]}"; do + aid=$(curl -s -m 15 -H "$T" "$ZURL?type=A&name=$h.occupymedia.cc" | python3 -c "import json,sys; r=json.load(sys.stdin)['result']; print(r[0]['id'] if r else '')") + [[ -n $aid ]] && curl -s -m 15 -X DELETE -H "$T" "$ZURL/$aid" > /dev/null + cid=$(curl -s -m 15 -H "$T" "$ZURL?type=CNAME&name=$h.occupymedia.cc" | python3 -c "import json,sys; r=json.load(sys.stdin)['result']; print(r[0]['id'] if r else '')") + if [[ -z $cid ]]; then + curl -s -m 15 -X POST -H "$T" -H "Content-Type: application/json" \ + -d "{\"type\":\"CNAME\",\"name\":\"$h.occupymedia.cc\",\"content\":\"e5695d8b-94eb-4e2b-9600-ed3cbf20e541.cfargotunnel.com\",\"proxied\":true}" \ + "$ZURL" > /dev/null + fi + done + $SSH "pct exec 150 -- bash -c 'cd /opt/anarchoposter/infra/occupymedia/edge && sed -i \"s/^EDGE_ENABLED=.*/EDGE_ENABLED=false/\" watcher.env && echo \"mode=TUNNEL failures=0 tunnel_ok=0 origin_ok=1 ip=$1\" > edge.state'" >/dev/null 2>&1 +} +probe_default() { curl -s -o /dev/null -w '%{http_code}' -m 8 https://app.occupymedia.cc 2>/dev/null; } + +ping_record_id() { curl -s -m 15 -H "Authorization: Bearer $CLOUDFLARE_API_TOKEN" \ + "https://api.cloudflare.com/client/v4/zones/$ZONE/dns_records?type=A&name=ping.occupymedia.cc" \ + | python3 -c "import json,sys; r=json.load(sys.stdin)['result']; print(r[0]['id'] if r else '')" 2>/dev/null; } + +sync_ping_ip() { # $1 = current public IP + local rid content + rid=$(ping_record_id) + [[ -n $rid ]] || { log "ping record missing — recreating"; \ + curl -s -m 15 -X POST -H "Authorization: Bearer $CLOUDFLARE_API_TOKEN" -H "Content-Type: application/json" \ + -d "{\"type\":\"A\",\"name\":\"ping.occupymedia.cc\",\"content\":\"$1\",\"proxied\":true,\"ttl\":60}" \ + "https://api.cloudflare.com/client/v4/zones/$ZONE/dns_records" > /dev/null; return; } + content=$(curl -s -m 15 -H "Authorization: Bearer $CLOUDFLARE_API_TOKEN" \ + "https://api.cloudflare.com/client/v4/zones/$ZONE/dns_records/$rid" \ + | python3 -c "import json,sys; print(json.load(sys.stdin)['result']['content'])" 2>/dev/null) + if [[ $content != "$1" ]]; then + curl -s -m 15 -X PATCH -H "Authorization: Bearer $CLOUDFLARE_API_TOKEN" -H "Content-Type: application/json" \ + -d "{\"content\":\"$1\"}" "https://api.cloudflare.com/client/v4/zones/$ZONE/dns_records/$rid" > /dev/null + log "ping A synced $content -> $1" + fi +} + +fails=0 +while true; do + sleep 60 + ip=$(get_ip) + [[ $ip =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]] || { log "ip lookup failed"; continue; } + sync_ping_ip "$ip" + + default_code=$(probe_default) + if [[ $default_code == 200 || $default_code == 307 ]]; then + log "tunnel path healed (default=$default_code) — activation not needed, retiring." + exit 0 + fi + + code=$(probe_edge) + if [[ $code == 200 ]]; then + fails=$((fails+1)) + log "edge probe via CF -> $code (live, $fails/2)" + else + fails=0 + log "edge probe via CF -> $code (forward not live yet)" + continue + fi + + if [[ $fails -ge 2 ]]; then + log "origin path confirmed live via Cloudflare ($code x2) — ACTIVATING as $TARGET_MODE" + $SSH "pct exec 150 -- sed -i 's/^EDGE_ENABLED=.*/EDGE_ENABLED=true/; s/^TARGET_MODE=.*/TARGET_MODE=$TARGET_MODE/' /opt/anarchoposter/infra/occupymedia/edge/watcher.env" >/dev/null 2>&1 + if [[ $TARGET_MODE == GREY ]]; then + $SSH "pct exec 150 -- bash -c 'cd /opt/anarchoposter/infra/occupymedia/edge && echo OPEN > firewall.mode && ./edge-firewall.sh'" >/dev/null 2>&1 + log "firewall relaxed to OPEN (grey mode: visitors connect directly)" + fi + proxied=true + [[ $TARGET_MODE == GREY ]] && proxied=false + T="Authorization: Bearer $CLOUDFLARE_API_TOKEN" + ZURL="https://api.cloudflare.com/client/v4/zones/$ZONE/dns_records" + for h in "${HOSTS[@]}"; do + cid=$(curl -s -m 15 -H "$T" "$ZURL?type=CNAME&name=$h.occupymedia.cc" | python3 -c "import json,sys; r=json.load(sys.stdin)['result']; print(r[0]['id'] if r else '')") + [[ -n $cid ]] && curl -s -m 15 -X DELETE -H "$T" "$ZURL/$cid" > /dev/null + aid=$(curl -s -m 15 -H "$T" "$ZURL?type=A&name=$h.occupymedia.cc" | python3 -c "import json,sys; r=json.load(sys.stdin)['result']; print(r[0]['id'] if r else '')") + if [[ -n $aid ]]; then + curl -s -m 15 -X PATCH -H "$T" -H "Content-Type: application/json" -d "{\"content\":\"$ip\",\"proxied\":$proxied}" "$ZURL/$aid" > /dev/null + else + curl -s -m 15 -X POST -H "$T" -H "Content-Type: application/json" -d "{\"type\":\"A\",\"name\":\"$h.occupymedia.cc\",\"content\":\"$ip\",\"proxied\":$proxied}" "$ZURL" > /dev/null + fi + log "$h -> A $ip proxied=$proxied" + done + $SSH "pct exec 150 -- bash -c 'cd /opt/anarchoposter/infra/occupymedia/edge && echo \"mode=$TARGET_MODE failures=0 tunnel_ok=0 origin_ok=1 ip=$ip\" > edge.state'" >/dev/null 2>&1 + log "DNS flipped to $TARGET_MODE mode. Verifying public path for 6 min..." + verified=0 + for i in 1 2 3 4 5 6 7 8 9 10 11 12; do + sleep 30 + dc=$(probe_default) + if [[ $dc == 200 || $dc == 307 ]]; then verified=1; break; fi + log "post-flip check $i/12: default=$dc" + done + if [[ $verified == 1 ]]; then + log "PUBLIC PATH VERIFIED — $TARGET_MODE mode live, site reachable. CT watcher owns failover." + else + log "default path STILL broken post-flip — $TARGET_MODE pool assignment is dead too. REVERTING to TUNNEL." + revert_to_tunnel "$ip" + log "Reverted to TUNNEL + watcher gated off. If guaranteed reachability is required: rerun with TARGET_MODE=GREY (direct DNS, origin IP exposed, firewall OPEN)." + fi + exit 0 + fi +done