#!/bin/bash # edge-watcher.sh — keeps occupymedia.cc reachable regardless of which path is sick. # # Two ingress paths behind Cloudflare's proxy: # TUNNEL proxied CNAMEs -> .cfargotunnel.com (edge pool can go bad per-region) # ORIGIN proxied A records -> current public IP -> nginx edge on CT 150 (443 CF-only) # # Every run: # 1. sync ORIGIN A records to the current public IP (the IP is dynamic) # 2. health-check both paths (tunnel = what visitors see via default DNS) # 3. fail over with hysteresis, log every decision # # Gated by EDGE_ENABLED=true in watcher.env — no DNS writes until the port # forward is verified. State: edge.state. Log: edge-watcher.log (rotated by size). set -uo pipefail cd "$(dirname "$0")" # shellcheck disable=SC1091 source ./watcher.env ZONE="4bd0ac2e13a735d2e0068ce3feea2996" TUNNEL_NAME="e5695d8b-94eb-4e2b-9600-ed3cbf20e541.cfargotunnel.com" HOSTS=(app api tube chat) STATE_FILE="./edge.state" LOG="./edge-watcher.log" MAXLOG=1048576 log() { echo "$(date -u '+%Y-%m-%dT%H:%M:%SZ') $*" >> "$LOG"; } # keep the log bounded if [[ -f $LOG && $(stat -c%s "$LOG") -gt $MAXLOG ]]; then tail -n 2000 "$LOG" > "$LOG.tmp" && mv "$LOG.tmp" "$LOG" fi api() { curl -s -m 15 -H "Authorization: Bearer $CF_API_TOKEN" "$@"; } record_id() { # host type -> id or empty api "https://api.cloudflare.com/client/v4/zones/$ZONE/dns_records?type=$2&name=$1.occupymedia.cc" \ | python3 -c "import json,sys; r=json.load(sys.stdin)['result']; print(r[0]['id'] if r else '')" 2>/dev/null } public_ip() { curl -s -m 10 ifconfig.me || curl -s -m 10 api.ipify.org; } set_mode() { # TUNNEL|ORIGIN|GREY local mode=$1 ip=$2 local proxied=true [[ $mode == GREY ]] && proxied=false for h in "${HOSTS[@]}"; do if [[ $mode == ORIGIN || $mode == GREY ]]; then local cid; cid=$(record_id "$h" CNAME) [[ -n $cid ]] && api -X DELETE "https://api.cloudflare.com/client/v4/zones/$ZONE/dns_records/$cid" > /dev/null local aid; aid=$(record_id "$h" A) if [[ -n $aid ]]; then api -X PATCH -H "Content-Type: application/json" \ -d "{\"content\":\"$ip\",\"proxied\":$proxied}" \ "https://api.cloudflare.com/client/v4/zones/$ZONE/dns_records/$aid" > /dev/null else api -X POST -H "Content-Type: application/json" \ -d "{\"type\":\"A\",\"name\":\"$h.occupymedia.cc\",\"content\":\"$ip\",\"proxied\":$proxied}" \ "https://api.cloudflare.com/client/v4/zones/$ZONE/dns_records" > /dev/null fi else local aid; aid=$(record_id "$h" A) [[ -n $aid ]] && api -X DELETE "https://api.cloudflare.com/client/v4/zones/$ZONE/dns_records/$aid" > /dev/null local cid; cid=$(record_id "$h" CNAME) if [[ -z $cid ]]; then api -X POST -H "Content-Type: application/json" \ -d "{\"type\":\"CNAME\",\"name\":\"$h.occupymedia.cc\",\"content\":\"$TUNNEL_NAME\",\"proxied\":true}" \ "https://api.cloudflare.com/client/v4/zones/$ZONE/dns_records" > /dev/null fi fi done echo "mode=$mode failures=0 tunnel_ok=0 origin_ok=0 ip=$ip" > "$STATE_FILE" } # ---- state ---- if [[ -f $STATE_FILE ]]; then source "$STATE_FILE"; else mode=TUNNEL; fi : "${mode:=TUNNEL}"; : "${failures:=0}"; : "${tunnel_ok:=0}"; : "${origin_ok:=0}"; : "${ip:=}" : "${TARGET_MODE:=ORIGIN}" now_ip=$(public_ip) [[ $now_ip =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]] || { log "ERROR public ip lookup failed ('$now_ip')"; exit 0; } # ---- health checks ---- tunnel_code=$(curl -s -o /dev/null -w '%{http_code}' -m 12 "https://api.occupymedia.cc/healthz" 2>/dev/null) [[ $tunnel_code == 200 ]] && tunnel_healthy=1 || tunnel_healthy=0 origin_code=$(curl -s -o /dev/null -w '%{http_code}' -m 12 --resolve api.occupymedia.cc:443:127.0.0.1 \ --cacert ./certs/live/occupymedia.cc/chain.pem 2>/dev/null "https://api.occupymedia.cc/healthz") [[ $origin_code == 200 ]] && origin_healthy=1 || origin_healthy=0 # ---- IP drift (only meaningful in ORIGIN mode; harmless otherwise) ---- if [[ $mode == ORIGIN && -n ${EDGE_ENABLED:-} && $ip != "$now_ip" ]]; then log "public ip changed $ip -> $now_ip; updating A records" set_mode ORIGIN "$now_ip"; mode=ORIGIN fi # ---- decisions (gated) ---- if [[ ${EDGE_ENABLED:-} != true ]]; then echo "mode=$mode failures=$failures tunnel_ok=$tunnel_ok origin_ok=$origin_ok ip=$now_ip" > "$STATE_FILE" log "EDGE_ENABLED!=true — observation only (tunnel=$tunnel_code origin=$origin_code)" exit 0 fi if [[ $mode == TUNNEL ]]; then if [[ $tunnel_healthy == 1 ]]; then failures=0; else failures=$((failures+1)); fi if [[ $failures -ge 2 && $origin_healthy == 1 ]]; then log "tunnel failed $failuresx, origin healthy — FAILING OVER to $TARGET_MODE ($now_ip)" set_mode "$TARGET_MODE" "$now_ip" else echo "mode=TUNNEL failures=$failures tunnel_ok=$tunnel_ok origin_ok=$origin_ok ip=$now_ip" > "$STATE_FILE" fi else if [[ $origin_healthy == 1 ]]; then failures=0; else failures=$((failures+1)); fi if [[ $failures -ge 3 ]]; then log "origin path failing ($failuresx) — FAILING BACK to TUNNEL" set_mode TUNNEL "$now_ip" else echo "mode=$mode failures=$failures tunnel_ok=$tunnel_ok origin_ok=$origin_ok ip=$now_ip" > "$STATE_FILE" fi fi