#!/bin/bash # ============================================================================ # rq-historical — eng begrenzter root-Wrapper fuer Red Queen (rq-host) # RED-QUEEN-VPS-HOST-ACCESS + PHASE-13.5-READONLY + DEV/TEST/DEPLOY-CHANNEL # | Rain Ocampo (Hermes) | 2026-08-29 # # LEAST-PRIVILEGE-Design: # - rq-host hat KEINE docker-Gruppe, KEINEN Docker-Socket, KEIN sudo ALL, # KEINE Root-Shell. Einziger Root-Kanal: NOPASSWD:/usr/local/bin/rq-historical # - Dieser Wrapper ist der EINZIGE Kanal zu historical-service. Er laeuft als # root (via sudoers), erzwingt aber fuer Dev/Test/Deploy strikte Fixierung: # * feste Staging-Roots, feste Allowlist (KEY->Staging-Datei->Containerpfad) # * keine freien Pfade, keine ../, kein Symlink-Escape, kein Mount-Arg # * automatisches Backup + Healthcheck + Hash-Reverify vor/nach Deploy # * deploy != execution: nie automatischer Backfill/Runner/Watchdog # - Deploy-Methode: atomic docker cp der validierten Stage-Datei in den # laufenden Container (Container ist beschreibbar, ReadonlyRootfs=false). # Rebuild via compose-build NICHT freigegeben (wuerde rq-host Docker-Build # + Container-Recreation geben -> Punkt 2/10-Verletzung). # - Test: py_compile der Stage-Kopie in Container-/tmp (keine Mutation des # laufenden Produktivcodes). # # SUBKOMMANDOS: # ps | inspect | logs [--tail N] | run [--start --end --limit-days] (bestehend) # code | hashes | code-list (bestehend, read-only) # stage-dir (zeigt Staging-Pfade) # stage-check (validiert Staging-Kandidat) # test (py_compile in /tmp, isoliert) # search-writes (fixe Write-Schreibpfad-Suche) # healthcheck (/health + /health/ready 200) # backup (aktuellen Container-Code sichern) # deploy (Backup->stage-check->test->cp->reverify->health) # rollback (zurueck zum letzten Backup) # ============================================================================ set -euo pipefail CONTAINER=historical-service # --------------------------------------------------------- Allowlist (fest) # KEY -> Container-Zielpfad (identisch mit 'code'-Read-Allowlist) declare -A TARGET_PATHS=( [runner]="/app/app/run_backfill_year.py" [backfill]="/app/app/backfill.py" [pipeline]="/app/app/pipeline_v2.py" [repository]="/app/app/persistence/repository.py" [provenance]="/app/app/trust/provenance.py" [trust]="/app/app/trust/trust.py" [sanity]="/app/app/trust/sanity.py" [trust_gate]="/app/app/trust/trust_gate.py" ) # KEY -> Staging-Dateiname (fest, NUR Dateiname im Staging-Root) declare -A STAGE_FILES=( [runner]="run_backfill_year.py" [backfill]="backfill.py" [pipeline]="pipeline_v2.py" [repository]="repository.py" [provenance]="provenance.py" [trust]="trust.py" [sanity]="sanity.py" [trust_gate]="trust_gate.py" ) STAGING_ROOT=/opt/rq-historical-staging BACKUP_ROOT=/opt/rq-historical-backups # Relevanter Pfad fuer den 'run'-Subbefehl (Backfill-Entrypoint) SCRIPT=/app/app/run_backfill_year.py # ------------------------------------------------------------ Validierung die() { echo "ERR: $*" >&2; exit 2; } req_ok() { docker inspect -f '{{.State.Running}}' "$CONTAINER" >/dev/null 2>&1 \ || die "Container '$CONTAINER' nicht vorhanden oder nicht am Laufen" } key_path() { echo "${TARGET_PATHS[$1]:-}"; } # $1=key -> Containerpfad stage_file() { echo "$STAGING_ROOT/${STAGE_FILES[$1]:-}"; } # $1=key -> abs Staging-Pfad key_ok() { # $1=key; exit 2 wenn unbekannt [ -n "${TARGET_PATHS[$1]:-}" ] && [ -n "${STAGE_FILES[$1]:-}" ] || die "unbekannter KEY '$1' (code-list)" } expect_zero() { [ $# -eq 0 ] || die "unerlaubtes Argument '$*'"; } # stage-check: FAIL-CLOSED-Validierung eines Staging-Kandidaten validate_stage() { # $1=key local key="$1" file real owner dmode file="$(stage_file "$key")" [ -e "$file" ] || die "Staging-Kandidat fehlt: $file (lege zuerst die Datei ab)" real="$(realpath -m "$file")" [[ "$real" == "$STAGING_ROOT/"* ]] || die "Traversal blockiert: realpath '$real' ausserhalb Staging" [ "$real" = "$file" ] || die "Alias erkannt (realpath != Pfad): $file" [ -f "$file" ] || die "keine reguläre Datei: $file" [ -L "$file" ] && die "Symlink nicht erlaubt: $file" owner="$(stat -c '%U' "$file")" [ "$owner" = "rq-host" ] || die "Staging-Kandidat Owner = $owner (erwartet rq-host)" dmode="$(stat -c '%a' "$STAGING_ROOT")" [[ "$dmode" != *002 ]] && [[ "$dmode" != *022 ]] || die "Staging-Root global schreibbar '$dmode' (FAIL CLOSED)" echo "OK: $key -> $file (regular, no symlink, owner rq-host, in staging)" } # ---------------------------------------------------- dispatch SUBCMD="${1:-}" shift || true case "$SUBCMD" in # ---------------- bestehend (read-only / run) ---------------- ps) expect_zero "$@"; exec docker ps ;; inspect) req_ok; expect_zero "$@"; exec docker inspect "$CONTAINER" ;; logs) req_ok tail_n=200 while [ $# -gt 0 ]; do case "$1" in --tail) [ $# -ge 2 ] || die "--tail braucht Wert"; [[ "$2" =~ ^[0-9]+$ ]] || die "ungueltiges --tail"; tail_n="$2"; shift 2 ;; *) die "unerlaubtes Argument 'logs $1'" ;; esac done exec docker logs --tail "$tail_n" "$CONTAINER" ;; run) req_ok declare -a RUN=() while [ $# -gt 0 ]; do case "$1" in --start) [[ "${2:-}" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}$ ]] || die "ungueltiges --start"; RUN+=("--start" "$2"); shift 2 ;; --end) [[ "${2:-}" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}$ ]] || die "ungueltiges --end"; RUN+=("--end" "$2"); shift 2 ;; --limit-days) [[ "${2:-}" =~ ^[0-9]+$ ]] || die "ungueltiges --limit-days"; RUN+=("--limit-days" "$2"); shift 2 ;; *) die "unerlaubtes Argument 'run $1'" ;; esac done echo "[rq-historical] Start: docker exec $CONTAINER python $SCRIPT ${RUN[*]}" >&2 exec docker exec "$CONTAINER" python "$SCRIPT" "${RUN[@]}" ;; # ------------- read-only Code ------------- code) req_ok [ $# -eq 1 ] || die "code braucht genau EINEN Key (code-list)" p="$(key_path "$1")"; [ -n "$p" ] || die "unbekannter Key 'code $1'" exec docker exec "$CONTAINER" cat "$p" ;; hashes) req_ok; expect_zero "$@" PATHS=(); for k in "${!TARGET_PATHS[@]}"; do PATHS+=("${TARGET_PATHS[$k]}"); done exec docker exec "$CONTAINER" sha256sum "${PATHS[@]}" ;; code-list) req_ok; expect_zero "$@" echo "Verfuegbare READ-ONLY-Code-Keys (historical-service):" for k in "${!TARGET_PATHS[@]}"; do printf ' %-11s | staging: %-22s -> %s\n' "$k" "${STAGE_FILES[$k]}" "${TARGET_PATHS[$k]}"; done ;; # ------------- dev/test/deploy-kanal ------------- stage-dir) expect_zero "$@" echo "Staging-Root: $STAGING_ROOT (Owner rq-host, keine globalen Schreibrechte)" for k in runner backfill pipeline repository provenance trust sanity trust_gate; do printf ' %-11s %s\n' "$k" "$(stage_file "$k")" done ;; stage-check) [ $# -eq 1 ] || die "stage-check braucht genau EINEN Key"; key_ok "$1" validate_stage "$1" ;; test) req_ok; [ $# -eq 1 ] || die "test braucht genau EINEN Key"; key_ok "$1" validate_stage "$1" file="$(stage_file "$1")" tmp="/tmp/rqtest_${1}_$(date +%s).py" docker cp "$file" "$CONTAINER:$tmp" if docker exec "$CONTAINER" python -m py_compile "$tmp" >/dev/null 2>&1; then echo "PASS: py_compile OK ($file) [isoliert in Container-/tmp, Produktivcode unberuehrt]" else echo "FAIL: py_compile fehlgeschlagen ($file)" docker exec "$CONTAINER" rm -f "$tmp" exit 2 fi docker exec "$CONTAINER" rm -f "$tmp" ;; search-writes) req_ok; expect_zero "$@" docker exec "$CONTAINER" sh -c 'grep -rnE "insert_raw_bars|insert_derived_bars|BackfillOrchestrator|trust_gate|TrustGate" /app/app --include="*.py" 2>/dev/null | head -100 || true' ;; healthcheck) req_ok; expect_zero "$@" rh="$(docker exec "$CONTAINER" python -c 'import urllib.request; print(urllib.request.urlopen("http://localhost:8000/health", timeout=5).status)' 2>/dev/null || echo N/A)" rr="$(docker exec "$CONTAINER" python -c 'import urllib.request; print(urllib.request.urlopen("http://localhost:8000/health/ready", timeout=5).status)' 2>/dev/null || echo N/A)" running="$(docker inspect -f '{{.State.Running}}' "$CONTAINER")" echo "Container.Running=$running /health=$rh /health/ready=$rr" [ "$running" = "true" ] && [ "$rh" = "200" ] && [ "$rr" = "200" ] || { echo "HEALTHCHECK FAILED"; exit 2; } echo "HEALTHCHECK OK" ;; backup) req_ok; [ $# -eq 1 ] || die "backup braucht genau EINEN Key"; key_ok "$1" p="$(key_path "$1")" ts="$(date +%Y%m%d_%H%M%S)" bdir="$BACKUP_ROOT/$1/$ts" mkdir -p "$bdir" docker exec "$CONTAINER" cat "$p" > "$bdir/$(basename "$p")" sha256sum "$bdir/$(basename "$p")" > "$bdir/SHA256" echo "BACKUP: $1 -> $bdir" cat "$bdir/SHA256" ;; deploy) req_ok; [ $# -eq 1 ] || die "deploy braucht genau EINEN Key"; key_ok "$1" key="$1"; p="$(key_path "$key")"; file="$(stage_file "$key")" # 1) validate (FAIL CLOSED) validate_stage "$key" # 2) backup erzwingen (muss gelingen) ts="$(date +%Y%m%d_%H%M%S)"; bdir="$BACKUP_ROOT/$key/$ts" mkdir -p "$bdir" docker exec "$CONTAINER" cat "$p" > "$bdir/$(basename "$p")" || die "Backup fehlgeschlagen -> deploy abgebrochen" sha256sum "$bdir/$(basename "$p")" > "$bdir/SHA256" echo "BACKUP: $bdir" # 3) test (py_compile isoliert) tmp="/tmp/rqdeploy_${key}_$(date +%s).py" docker cp "$file" "$CONTAINER:$tmp" if ! docker exec "$CONTAINER" python -m py_compile "$tmp" >/dev/null 2>&1; then docker exec "$CONTAINER" rm -f "$tmp" die "py_compile FAIL -> deploy abgebrochen (nichts deployt, kein Rollback noetig)" fi docker exec "$CONTAINER" rm -f "$tmp" # 4) atomic deploy docker cp "$file" "$CONTAINER:$p" # 5) hash-reverify sha_stage="$(sha256sum "$file" | awk '{print $1}')" sha_deployed="$(docker exec "$CONTAINER" sha256sum "$p" | awk '{print $1}')" echo "SHA stage : $sha_stage" echo "SHA deployed: $sha_deployed" if [ "$sha_stage" != "$sha_deployed" ]; then echo "HASH-MISMATCH -> rollback" docker cp "$bdir/$(basename "$p")" "$CONTAINER:$p" die "Deploy Hash-Mismatch, zurueckgerollt" fi # 6) healthcheck (nur health/ready, KEIN run) rh="$(docker exec "$CONTAINER" python -c 'import urllib.request; print(urllib.request.urlopen("http://localhost:8000/health", timeout=5).status)' 2>/dev/null || echo N/A)" rr="$(docker exec "$CONTAINER" python -c 'import urllib.request; print(urllib.request.urlopen("http://localhost:8000/health/ready", timeout=5).status)' 2>/dev/null || echo N/A)" echo "Health: /health=$rh /health/ready=$rr" if [ "$rh" != "200" ] || [ "$rr" != "200" ]; then echo "DEPLOY-HEALTHCHECK FAILED -> rollback" docker cp "$bdir/$(basename "$p")" "$CONTAINER:$p" die "Rollback ausgefuehrt" fi echo "DEPLOY OK: $key -> $p (Backup: $bdir). KEIN backfill/run gestartet." ;; rollback) req_ok; [ $# -eq 1 ] || die "rollback braucht genau EINEN Key"; key_ok "$1" p="$(key_path "$1")" bdir="$(ls -1t "$BACKUP_ROOT/$1" 2>/dev/null | head -1)" [ -n "$bdir" ] || die "kein Backup fuer '$1' vorhanden" bdir="$BACKUP_ROOT/$1/$bdir" src="$bdir/$(basename "$p")" [ -f "$src" ] || die "Backup-Datei fehlt: $src" sha_bak="$(sha256sum "$src" | awk '{print $1}')" sha_cur="$(docker exec "$CONTAINER" sha256sum "$p" | awk '{print $1}')" echo "aktuell: $sha_cur / backup: $sha_bak" [ "$sha_bak" != "$sha_cur" ] || die "kein Unterschied (bereits auf Backup-Stand)" docker cp "$src" "$CONTAINER:$p" sha_cur="$(docker exec "$CONTAINER" sha256sum "$p" | awk '{print $1}')" [ "$sha_cur" = "$sha_bak" ] || die "ROLLBACK-HASH-MISMATCH" echo "ROLLBACK OK: $key -> $p auf $bdir" ;; *) die "Aufruf: ps|inspect|logs|run|code |hashes|code-list|stage-dir|stage-check |test |search-writes|healthcheck|backup |deploy |rollback " ;; esac