T-HUB/infra/occupymedia/edge/edge-watcher.sh

#!/bin/bash
# edge-watcher.sh — keeps occupymedia.cc reachable regardless of which path is sick.
#
# Two ingress paths behind Cloudflare's proxy:
#   TUNNEL  proxied CNAMEs -> <tunnel>.cfargotunnel.com   (edge pool can go bad per-region)
#   ORIGIN  proxied A records -> current public IP -> nginx edge on CT 150 (443 CF-only)
#
# Every run:
#   1. sync ORIGIN A records to the current public IP (the IP is dynamic)
#   2. health-check both paths (tunnel = what visitors see via default DNS)
#   3. fail over with hysteresis, log every decision
#
# Gated by EDGE_ENABLED=true in watcher.env — no DNS writes until the port
# forward is verified. State: edge.state. Log: edge-watcher.log (rotated by size).
set -uo pipefail

cd "$(dirname "$0")"
# shellcheck disable=SC1091
source ./watcher.env

ZONE="4bd0ac2e13a735d2e0068ce3feea2996"
TUNNEL_NAME="e5695d8b-94eb-4e2b-9600-ed3cbf20e541.cfargotunnel.com"
HOSTS=(app api tube chat)
STATE_FILE="./edge.state"
LOG="./edge-watcher.log"
MAXLOG=1048576

log() { echo "$(date -u '+%Y-%m-%dT%H:%M:%SZ') $*" >> "$LOG"; }
# keep the log bounded
if [[ -f $LOG && $(stat -c%s "$LOG") -gt $MAXLOG ]]; then
  tail -n 2000 "$LOG" > "$LOG.tmp" && mv "$LOG.tmp" "$LOG"
fi

api() { curl -s -m 15 -H "Authorization: Bearer $CF_API_TOKEN" "$@"; }

record_id() { # host type -> id or empty
  api "https://api.cloudflare.com/client/v4/zones/$ZONE/dns_records?type=$2&name=$1.occupymedia.cc" \
    | python3 -c "import json,sys; r=json.load(sys.stdin)['result']; print(r[0]['id'] if r else '')" 2>/dev/null
}

public_ip() { curl -s -m 10 ifconfig.me || curl -s -m 10 api.ipify.org; }

set_mode() { # TUNNEL|ORIGIN|GREY
  local mode=$1 ip=$2
  local proxied=true
  [[ $mode == GREY ]] && proxied=false
  for h in "${HOSTS[@]}"; do
    if [[ $mode == ORIGIN || $mode == GREY ]]; then
      local cid; cid=$(record_id "$h" CNAME)
      [[ -n $cid ]] && api -X DELETE "https://api.cloudflare.com/client/v4/zones/$ZONE/dns_records/$cid" > /dev/null
      local aid; aid=$(record_id "$h" A)
      if [[ -n $aid ]]; then
        api -X PATCH -H "Content-Type: application/json" \
          -d "{\"content\":\"$ip\",\"proxied\":$proxied}" \
          "https://api.cloudflare.com/client/v4/zones/$ZONE/dns_records/$aid" > /dev/null
      else
        api -X POST -H "Content-Type: application/json" \
          -d "{\"type\":\"A\",\"name\":\"$h.occupymedia.cc\",\"content\":\"$ip\",\"proxied\":$proxied}" \
          "https://api.cloudflare.com/client/v4/zones/$ZONE/dns_records" > /dev/null
      fi
    else
      local aid; aid=$(record_id "$h" A)
      [[ -n $aid ]] && api -X DELETE "https://api.cloudflare.com/client/v4/zones/$ZONE/dns_records/$aid" > /dev/null
      local cid; cid=$(record_id "$h" CNAME)
      if [[ -z $cid ]]; then
        api -X POST -H "Content-Type: application/json" \
          -d "{\"type\":\"CNAME\",\"name\":\"$h.occupymedia.cc\",\"content\":\"$TUNNEL_NAME\",\"proxied\":true}" \
          "https://api.cloudflare.com/client/v4/zones/$ZONE/dns_records" > /dev/null
      fi
    fi
  done
  echo "mode=$mode failures=0 tunnel_ok=0 origin_ok=0 ip=$ip" > "$STATE_FILE"
}

# ---- state ----
if [[ -f $STATE_FILE ]]; then source "$STATE_FILE"; else mode=TUNNEL; fi
: "${mode:=TUNNEL}"; : "${failures:=0}"; : "${tunnel_ok:=0}"; : "${origin_ok:=0}"; : "${ip:=}"
: "${TARGET_MODE:=ORIGIN}"

now_ip=$(public_ip)
[[ $now_ip =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]] || { log "ERROR public ip lookup failed ('$now_ip')"; exit 0; }

# ---- health checks ----
tunnel_code=$(curl -s -o /dev/null -w '%{http_code}' -m 12 "https://api.occupymedia.cc/healthz" 2>/dev/null)
[[ $tunnel_code == 200 ]] && tunnel_healthy=1 || tunnel_healthy=0

origin_code=$(curl -s -o /dev/null -w '%{http_code}' -m 12 --resolve api.occupymedia.cc:443:127.0.0.1 \
  --cacert ./certs/live/occupymedia.cc/chain.pem 2>/dev/null "https://api.occupymedia.cc/healthz")
[[ $origin_code == 200 ]] && origin_healthy=1 || origin_healthy=0

# ---- IP drift (only meaningful in ORIGIN mode; harmless otherwise) ----
if [[ $mode == ORIGIN && -n ${EDGE_ENABLED:-} && $ip != "$now_ip" ]]; then
  log "public ip changed $ip -> $now_ip; updating A records"
  set_mode ORIGIN "$now_ip"; mode=ORIGIN
fi

# ---- decisions (gated) ----
if [[ ${EDGE_ENABLED:-} != true ]]; then
  echo "mode=$mode failures=$failures tunnel_ok=$tunnel_ok origin_ok=$origin_ok ip=$now_ip" > "$STATE_FILE"
  log "EDGE_ENABLED!=true — observation only (tunnel=$tunnel_code origin=$origin_code)"
  exit 0
fi

if [[ $mode == TUNNEL ]]; then
  if [[ $tunnel_healthy == 1 ]]; then failures=0; else failures=$((failures+1)); fi
  if [[ $failures -ge 2 && $origin_healthy == 1 ]]; then
    log "tunnel failed $failuresx, origin healthy — FAILING OVER to $TARGET_MODE ($now_ip)"
    set_mode "$TARGET_MODE" "$now_ip"
  else
    echo "mode=TUNNEL failures=$failures tunnel_ok=$tunnel_ok origin_ok=$origin_ok ip=$now_ip" > "$STATE_FILE"
  fi
else
  if [[ $origin_healthy == 1 ]]; then failures=0; else failures=$((failures+1)); fi
  if [[ $failures -ge 3 ]]; then
    log "origin path failing ($failuresx) — FAILING BACK to TUNNEL"
    set_mode TUNNEL "$now_ip"
  else
    echo "mode=$mode failures=$failures tunnel_ok=$tunnel_ok origin_ok=$origin_ok ip=$now_ip" > "$STATE_FILE"
  fi
fi