#!/bin/bash # ============================================================================ # rq-historical — eng begrenzter root-Wrapper fuer Red Queen (rq-host) # RED-QUEEN-VPS-HOST-ACCESS + READONLY-CODE + DEV/TEST/DEPLOY + SOT-DISCOVERY # | Rain Ocampo (Hermes) | 2026-08-29 # # LEAST-PRIVILEGE-Design: # - rq-host hat KEINE docker-Gruppe, KEINEN Docker-Socket, KEIN sudo ALL, # KEINE Root-Shell. Einziger Root-Kanal: NOPASSWD:/usr/local/bin/rq-historical # - Dieser Wrapper ist der EINZIGE Kanal zu historical-service & Host-SoT. # Keine freien Pfade, kein arbitrary cat/find/grep, kein Shell. # # SoT-DISCOVERY (read-only, NO copy/rebuild/deploy/fix): # build-def -> zeigt Dockerfile + Compose-Builddefinition (redacted) # build-files -> Dockerfile + compose + .dockerignore Auflistung # host-manifest -> SHA256+size aller *.py unter /opt/historical-v2/app # host-code -> gezielter Code-Read NUR aus host-manifest (kontrolliert) # container-manifest -> SHA256+size aller *.py unter historical-service:/app/app # container-code -> gezielter Code-Read NUR aus container-manifest # tree-diff -> Host vs Container Manifest-Diff # # SUBKOMMANDOS (bestehend, unveraendert): # ps | inspect | logs [--tail N] | run [..] | code | hashes | code-list # stage-dir | stage-check | test | search-writes | healthcheck # backup | deploy | rollback # ============================================================================ set -euo pipefail CONTAINER=historical-service HOST_APP=/opt/historical-v2/app HOST_DIR=/opt/historical-v2 # --------------------------------------------------------- Allowlist (fest) declare -A TARGET_PATHS=( [runner]="/app/app/run_backfill_year.py" [backfill]="/app/app/backfill.py" [pipeline]="/app/app/pipeline_v2.py" [repository]="/app/app/persistence/repository.py" [provenance]="/app/app/trust/provenance.py" [trust]="/app/app/trust/trust.py" [sanity]="/app/app/trust/sanity.py" [trust_gate]="/app/app/trust/trust_gate.py" ) declare -A STAGE_FILES=( [runner]="run_backfill_year.py" [backfill]="backfill.py" [pipeline]="pipeline_v2.py" [repository]="repository.py" [provenance]="provenance.py" [trust]="trust.py" [sanity]="sanity.py" [trust_gate]="trust_gate.py" ) STAGING_ROOT=/opt/rq-historical-staging BACKUP_ROOT=/opt/rq-historical-backups SCRIPT=/app/app/run_backfill_year.py # --------------------------------------------------------- HOST-SOT-RECOVERY # CONFIRMED GOLD CONTAINER -> ROOT-CONTROLLED BACKUP -> EXACT BYTE COPY # -> HOST SOURCE OF TRUTH -> SHA VERIFY -> STOPP. (Phase 13.5, Extension 1) # KEIN Build/Recreate in dieser Mission. SOT_BACKUP_ROOT=/opt/rq-historical-sot-backups SOT_SESSION_MARKER=".rq-sot-session" # eindeutige Markierung der Recovery-Session # Recovery-Allowlist: KEY -> "CONTAINER_SRC|HOST_TARGET|EXPECTED_GOLD_SHA" # Containerpfade aus container-manifest verifiziert (NICHT geraten). # HINWEIS: Aufgabe nennt trust/init.py, Container hat aber trust/__init__.py # (Gold-SHA 059db627...). Punkt 5: exakte Pfade aus Manifest, nicht raten. declare -A RECOVERY=( [backfill]="/app/app/backfill.py|/opt/historical-v2/app/backfill.py|1627682ae582bb13343ae8ea10491dd9dffaef6fa4f7594d85378c2a500e81c7" [repository]="/app/app/persistence/repository.py|/opt/historical-v2/app/persistence/repository.py|8fff892f52a612f464b5be21c66ef7787a47f2d7d049f815ef87837553c2a228" [pipeline]="/app/app/pipeline_v2.py|/opt/historical-v2/app/pipeline_v2.py|7c08ec3d1e75854cf341b9cbfd12afc075c4a173518dd3d951be4f95c5175fe6" [trust_init]="/app/app/trust/__init__.py|/opt/historical-v2/app/trust/__init__.py|059db62741f2001f548d1879ccd3bf73bb2f4c8d0f492d151a469c9e4dcd3934" [provenance]="/app/app/trust/provenance.py|/opt/historical-v2/app/trust/provenance.py|4dd2ab977b69e9a47bb744dcc742a94d506e13035157a3977cdc7c8924d5f16f" [trust]="/app/app/trust/trust.py|/opt/historical-v2/app/trust/trust.py|7c2a948d2f1d668201e5af2a28164bcb2451b31d514a9f5f22b435c9ff2a0d01" [sanity]="/app/app/trust/sanity.py|/opt/historical-v2/app/trust/sanity.py|875b07869e5148983e7a06bc05f6a6bec30727d1b5e95242d459564c18f6d6e9" [trust_gate]="/app/app/trust/trust_gate.py|/opt/historical-v2/app/trust/trust_gate.py|df0a97a1fc27985fe49b7458d94f7922bc5eaf2250807043c48dbc173f215ff1" [diag_bi5]="/app/app/diag_bi5.py|/opt/historical-v2/app/diag_bi5.py|3e07ec5c4d9f572a118c278d0dc5c86865a8fe2067b12f85a3fa958ea4013efa" [smoke_readonly]="/app/app/smoke_readonly.py|/opt/historical-v2/app/smoke_readonly.py|85ca939c11d789684740833f0bddb6e4f2a548a7f94870662191b75aefdcbfd6" [smoke_trust_gate]="/app/app/smoke_trust_gate.py|/opt/historical-v2/app/smoke_trust_gate.py|f015fe650d6934b0c71d70bc3011f16b7f4bb0d2ffd947b9127040067442cfff" [test_trust_gate]="/app/app/test_trust_gate.py|/opt/historical-v2/app/test_trust_gate.py|7867e2b526491de966e64486e8d49ec2cad204d97efc25844562b867ef861114" [test_phase13_3]="/app/app/test_phase13_3_reproduction.py|/opt/historical-v2/app/test_phase13_3_reproduction.py|de7d2f7be785b7525a045b91ca055dca305fd7d575b291528141f000d5d003e7" ) # Host-Only-Legacy (Punkt 9): NICHT anfassen HOST_ONLY_LEGACY=( "/opt/historical-v2/app/app/backfill.py" "/opt/historical-v2/app/app/test_idempotency.py" "/opt/historical-v2/app/eligibility.py" ) # --------------------------------------------------------- BUILD/RECREATE-KANAL # Phase 13.5 · Controlled No-op Build/Recreate Channel. # Build-Pollution-Exclusion via .dockerignore (präzise, NUR Legacy-Artefakte). # KEIN Businesscode-Fix. KEIN Backfill-Runner in diesem Kanal. COMPOSE_PROJECT=/opt/historical-v2 COMPOSE_FILE=/opt/historical-v2/docker-compose.yml HISTORICAL_SERVICE=historical-service GOLD_IMAGE_ID="sha256:f1a6f33e8714c5cdf017632aa30e71241a8ed670a5c95dac2e8226064b990c81" GOLD_CONTAINER_ID="634cc505c7bc4c361305e5b415f107e98411faf38e8a29269dd8b1e4d1363b32" GOLD_MANIFEST="/opt/rq-historical-gold-reference/gold_app_manifest.sha" BUILD_EXCLUSION_FILE=/opt/historical-v2/.dockerignore # Präzise Pollution-Exclusion-Einträge (NUR diese 3; keine Gold-Datei betroffen): # app/app/ -> /app/app/app/ (verschachtelt, im Gold NICHT vorhanden) # app/eligibility.py -> /app/app/eligibility.py (top-level, Gold: quality/eligibility.py UNBERÜHRT) BUILD_EXCLUSION_LINES=( "app/app/" "app/eligibility.py" ) # Legacy-Artefakte, die im neuen Container NICHT als EXTRA erscheinen dürfen: declare -A FORBIDDEN_EXTRA=( ["app/app/app/backfill.py"]=1 ["app/app/app/test_idempotency.py"]=1 ["app/app/eligibility.py"]=1 ) # ------------------------------------------------- RECOVERY-GOLD-SNAPSHOT-CHANNEL # Phase 13.5 · Privileged Control-Plane Extension - Recovery Gold Snapshot Only. # Erfassung eines byte-/state-nahen Recovery-Images aus dem verifizierten # GOLD-Container 634cc505... (Image f1a6f33e...). KEIN Businesscode, KEIN Build, # KEIN Recreate, KEIN Runner, KEINE DB-/Datenmutation, KEIN latest-Touch. RECOVERY_GOLD_CID="634cc505c7bc4c361305e5b415f107e98411faf38e8a29269dd8b1e4d1363b32" RECOVERY_GOLD_TAG="historical-v2-historical-service:recovery-gold-container-20260829" # ------------------------------------------------- RECOVERY-GOLD-REBUILD-KANAL # Phase 13.5 · Build Control Plane Hardening - Recovery-Gold Rebuild Channel. # Erzeugt aus dem bestaetigten Host-SoT (/opt/historical-v2) ein NEUES, direkt # dediziert getaggtes Recovery-Gold-Image. KEIN latest, KEIN Recreate, KEIN # Runner, KEINE User-Args/Build-Args. Fester Context + Dockerfile + Tag. RECOVERY_REBUILD_TAG="historical-v2-historical-service:recovery-gold-rebuilt-20260829" RECOVERY_BUILD_CONTEXT="/opt/historical-v2" RECOVERY_BUILD_DOCKERFILE="/opt/historical-v2/Dockerfile" # Critical Files (Punkt 11) mit CONSERVED GOLD SHA (fest, aus laufendem Container) declare -A CRITICAL_SHA=( ["run_backfill_year.py"]="70d3abb1a92a871e28cc6fb95b9a26944b55dd8df1b72db80642d6881b7b0f20" ["backfill.py"]="1627682ae582bb13343ae8ea10491dd9dffaef6fa4f7594d85378c2a500e81c7" ["repository.py"]="8fff892f52a612f464b5be21c66ef7787a47f2d7d049f815ef87837553c2a228" ["pipeline_v2.py"]="7c08ec3d1e75854cf341b9cbfd12afc075c4a173518dd3d951be4f95c5175fe6" ["trust_init.py"]="059db62741f2001f548d1879ccd3bf73bb2f4c8d0f492d151a469c9e4dcd3934" # trust/__init__.py ["provenance.py"]="4dd2ab977b69e9a47bb744dcc742a94d506e13035157a3977cdc7c8924d5f16f" # trust/provenance.py ["trust.py"]="7c2a948d2f1d668201e5af2a28164bcb2451b31d514a9f5f22b435c9ff2a0d01" # trust/trust.py ["sanity.py"]="875b07869e5148983e7a06bc05f6a6bec30727d1b5e95242d459564c18f6d6e9" # trust/sanity.py ["trust_gate.py"]="df0a97a1fc27985fe49b7458d94f7922bc5eaf2250807043c48dbc173f215ff1" # trust/trust_gate.py ) # Relative Pfade je Key (fester Pfad, kein User-Parameter) declare -A CRITICAL_PATH=( ["run_backfill_year.py"]="app/run_backfill_year.py" ["backfill.py"]="app/backfill.py" ["repository.py"]="app/persistence/repository.py" ["pipeline_v2.py"]="app/pipeline_v2.py" ["trust_init.py"]="app/trust/__init__.py" ["provenance.py"]="app/trust/provenance.py" ["trust.py"]="app/trust/trust.py" ["sanity.py"]="app/trust/sanity.py" ["trust_gate.py"]="app/trust/trust_gate.py" ) # ------------------------------------------------- FINAL-REPRODUCTION-CONTROL-PLANE # Phase 13.5 · Safe Recreate + Rollback Preparation. KEIN echter Recreate/Rollback # durch Rain. Recovery-Gold ist ab jetzt IMMUTABLE BASELINE (nur ROLLBACK/RECOVERY-BASE). RECOVERY_GOLD_IMAGE_ID="sha256:d2db926dc6f5f7cf2f8e85d271a9b16fcc3cfb54b6557604d3d784f7961a4bc2" RECOVERY_GOLD_TAG_FINAL="historical-v2-historical-service:recovery-gold-rebuilt-20260829" # Candidate Identity Audit (read-only, 29.08.): ausschliesslich ueber latest adressiert. # 0f6ca29a existiert NICHT mehr (historisch). Aktueller Candidate: CANDIDATE_IMAGE_ID="sha256:1f3ba210780c5a00d4b73c5ee7817a88671fee2607a4ae0d5b1cefef1a1261c7" CANDIDATE_TAG="historical-v2-historical-service:candidate-phase13-5-noop" # ------------------------------------------------- PERSISTENT-HOST-SOT-PROMOTION # Phase 13.5 · Persistent Host-SoT Patch Promotion Channel. KEIN echter Promote # durch Rain. Nur run_backfill_year.py. Feste Identitaeten, KEIN User-Input. # PATCH_SOURCE=/opt/data/run_backfill_year.py existiert NUR im RQ-Container # (hermes-red-queen), nicht auf dem VPS-Host -> fester docker cp aus dem Container. PATCH_SOURCE_CONTAINER="hermes-red-queen" PATCH_SOURCE_PATH="/opt/data/run_backfill_year.py" PATCH_TARGET="/opt/historical-v2/app/run_backfill_year.py" # Read-only ermittelt (29.08.): SHA256 des von Red Queen geprueften Patch-Artefakts (8292 B). EXPECTED_RUNNER_PATCH_SHA256="6cca3c55bb559d75be2f58e59c231003902f124a54ada25582970524599f0752" # Fester Backup-Root fuer die Promotion-Session (root-owned, nicht world-readable). PROMOTE_BACKUP_ROOT="/opt/rq-historical-sot-backups/pre-runner-remediation" # ------------------------------------------------- REMEDIATION BUILD/DEPLOY-KANAL # Phase 13.5 · Controlled Runner-Remediation Build/Deploy Channel. # KEIN echter Build/Recreate durch Rain; nur Kanal. Red Queen fuehrt nach Freigabe aus. # Fester Remediation-Tag (Punkt 3) — KEIN Overwrite, KEIN force-retag, KEIN latest. REMEDIATION_TAG="historical-v2-historical-service:candidate-phase13-5-runner-remediation" # Build-Session-Basis (root-owned; enthaelt nach Build session.meta mit FULL_IMAGE_ID). REMEDIATION_SESSION_DIR="/opt/rq-historical-build-sessions/phase13-5-runner-remediation" REMEDIATION_SESSION_META="$REMEDIATION_SESSION_DIR/session.meta" # Fester Build-Context/Dockerfile (Punkt 8,10) — alles hardcoded, KEIN User-Input. REMEDIATION_BUILD_CONTEXT="/opt/historical-v2" REMEDIATION_BUILD_DOCKERFILE="/opt/historical-v2/Dockerfile" # Expected Runner-SHAs (Punkt 10/11): ALT = Gold (vor Remediation), NEU = persistent (nach). RUNNER_GOLD_SHA="70d3abb1a92a871e28cc6fb95b9a26944b55dd8df1b72db80642d6881b7b0f20" RUNNER_REMEDIATION_SHA="6cca3c55bb559d75be2f58e59c231003902f124a54ada25582970524599f0752" # Recovery-Gold (immutable Baseline) — Referenz fuer Expected-Gold-Delta. # Candidate/Recovery duerfen code-identisch sein, aber verschiedene Image-IDs haben (KEIN Fehler). REM_NOOP_IMAGE_ID="sha256:1f3ba210780c5a00d4b73c5ee7817a88671fee2607a4ae0d5b1cefef1a1261c7" # ------------------------------------------------- IMAGE-RETENTION / ARCHIVE CONTROL PLANE # Phase 13.5 · Schutzschicht gegen bestaetigten Coolify DockerCleanupJob # (taegl. 00:00 UTC, force_docker_cleanup=true, rmi auf getaggte Images ohne # coolify.managed-Label). ZWEI unabhaengige Ebenen je Critical Image: # SCHICHT A = Stopped Pin Container (Image-Referenz via docker create, nie gestartet) # SCHICHT B = Persistentes, sha256-verifiziertes docker save-Archiv # Kein Coolify-Cleanup-Disable. Kein coolify.managed-Spoof. Kein Overwrite. # Fester Archive-Root (Punkt 3) — root:root, 700, KEINE freien Pfade. ARCHIVE_ROOT="/opt/rq-historical-image-archives" # Pin-Container-Namen (Punkt 9) — fest, hardcoded, KEIN freier Name. PIN_RECOVERY_NAME="rq-historical-pin-recovery-gold" PIN_REMEDIATION_NAME="rq-historical-pin-phase13-5-runner-remediation" # Unterordner je Rolle (Punkt 3) declare -A ARCHIVE_SUB=( [recovery]="recovery" [remediation]="candidates" ) # Archiv-Session-IDs: kurze Image-ID (), die den immutable # Session-Ordner benennt. Recovery = bekannte Original-ID; Remediation = # FULL_IMAGE_ID aus Build-Session (41e357a1..., aktuell im Store fehlend). RECOVERY_ARCHIVE_ID="${RECOVERY_GOLD_IMAGE_ID#sha256:}" RECOVERY_ARCHIVE_ID="${RECOVERY_ARCHIVE_ID:0:12}" # Remediation-Archiv-Session-ID: kurze ID der Build-Session-Remendation (41e357a1...). REMEDIATION_ARCHIVE_ID="41e357a12b90" # Remediation-Archiv-Referenz (aus Build-Session falls vorhanden, sonst leer): # wird zur Laufzeit per remediation_full_image_id() aufgeloest. # Pin-Referenz-Ziel (Schicht A): exakt diese Image-ID muss der Pin-Container referenzieren. PIN_REMEDIATION_TARGET_ID="" # zur Laufzeit gesetzt # RUN_MODE (Mutation Guard): READ_ONLY | DRY_RUN | MUTATING RUN_MODE=MUTATING die() { echo "ERR: $*" >&2; exit 2; } req_ok() { docker inspect -f '{{.State.Running}}' "$CONTAINER" >/dev/null 2>&1 \ || die "Container '$CONTAINER' nicht vorhanden oder nicht am Laufen" } key_path() { echo "${TARGET_PATHS[$1]:-}"; } stage_file() { echo "$STAGING_ROOT/${STAGE_FILES[$1]:-}"; } key_ok() { [ -n "${TARGET_PATHS[$1]:-}" ] && [ -n "${STAGE_FILES[$1]:-}" ] || die "unbekannter KEY '$1' (code-list)" } expect_zero() { [ $# -eq 0 ] || die "unerlaubtes Argument '$*'"; } # ================= BUILD/RECREATE-KANAL HILFSFUNKTIONEN (alle fail-closed) === # MODE ist global: MUTATING (Default) | DRY_RUN (keine Mutationen) mode_or_die() { [ "$RUN_MODE" != "DRY_RUN" ] || die "DRY_RUN-Modus, aber mutierender Pfad erreicht (FAIL CLOSED)"; } # ---- Full/Partial-SoT-Backup-Klassifikation (Phase 13.5 Remediation, Blocker A) ---- # Ein Partial-Backup (z.B. pre-runner-remediation mit nur session.meta + # run_backfill_year.py) darf NIEMALS als Full-SoT-Snapshot gelten. # Ein gueltiger FULL SOT SNAPSHOT muss zwingend haben: # historical-v2.tar (regular, kein Symlink, nicht leer) # SHA256.manifest (regular, kein Symlink, nicht leer) # und optional session.meta / provenance. Read-only. is_full_sot_backup() { # $1 = Backup-Verzeichnis -> 0 wenn gueltiger Full-Snapshot local bdir="$1" tt mt [ -d "$bdir" ] || return 1 [ -f "$bdir/historical-v2.tar" ] || return 1 [ -s "$bdir/historical-v2.tar" ] || return 1 [ -f "$bdir/SHA256.manifest" ] || return 1 [ -s "$bdir/SHA256.manifest" ] || return 1 [ -L "$bdir/historical-v2.tar" ] && return 1 [ -L "$bdir/SHA256.manifest" ] && return 1 tt=$(stat -c '%F' "$bdir/historical-v2.tar" 2>/dev/null) mt=$(stat -c '%F' "$bdir/SHA256.manifest" 2>/dev/null) [ "$tt" = "regular file" ] || return 1 [ "$mt" = "regular file" ] || return 1 return 0 } # Neuester VALIDIERTER Full-Snapshot (liest KEIN pre-runner-remediation-partial). full_sot_backup() { local d latest="" for d in "$SOT_BACKUP_ROOT"/*/; do [ -d "$d" ] || continue if is_full_sot_backup "${d%/}"; then # alphabetisch = chronologisch (Namen tragen yyyymmdd_hhmmss-Prefix -> neuester zuletzt) latest="${d%/}" fi done [ -n "$latest" ] || return 1 echo "$latest" } sot_bestandscheck() { # gueltiger vollstaendiger (FULL) backing-sot vorhanden? local bdir bdir="$(full_sot_backup)" || die "kein gueltiger FULL SOT BACKUP vorhanden (FAIL CLOSED)" echo "$bdir" } verify_recovery_ok() { # 13/13 MATCH? local allok=1 k csrc htarget gold hsha csha for k in "${!RECOVERY[@]}"; do IFS='|' read -r csrc htarget gold <<< "${RECOVERY[$k]}" hsha="$(sha256sum "$htarget" 2>/dev/null | awk '{print $1}')" || hsha="" csha="$(docker exec "$CONTAINER" sha256sum "$csrc" 2>/dev/null | awk '{print $1}')" || csha="" if [ "$hsha" != "$gold" ] || [ "$csha" != "$gold" ]; then allok=0; echo " FEHLT: $k host=$hsha container=$csha"; fi done [ "$allok" = "1" ] || die "verify-sot-recovery NICHT 13/13 MATCH (FAIL CLOSED)" } ensure_gold_image() { docker image inspect "$GOLD_IMAGE_ID" >/dev/null 2>&1 || die "Gold-Image $GOLD_IMAGE_ID fehlt (FAIL CLOSED)"; } ensure_gold_run() { docker inspect -f '{{.State.Running}}' "$HISTORICAL_SERVICE" >/dev/null 2>&1 || die "Gold-Container $HISTORICAL_SERVICE läuft nicht (FAIL CLOSED)"; } ensure_db_run() { docker inspect -f '{{.State.Running}}' historical-db >/dev/null 2>&1 || die "historical-db läuft nicht (FAIL CLOSED)"; } ensure_db_volume() { docker inspect -f '{{ range .Mounts }}{{ .Name }} {{ end }}' historical-db 2>/dev/null | grep -q 'historical-db-data' || die "DB-Volume historical-db-data fehlt (FAIL CLOSED)"; } ensure_builddef() { [ -f "$COMPOSE_FILE" ] || die "Compose fehlt: $COMPOSE_FILE"; [ -f "$COMPOSE_PROJECT/Dockerfile" ] || die "Dockerfile fehlt"; } exclusion_valid() { # .dockerignore exakt die 3 Legacy-Einträge, keine Gold-Datei ausgeschlossen [ -f "$BUILD_EXCLUSION_FILE" ] || die ".dockerignore fehlt (FAIL CLOSED)" local line while IFS= read -r line; do [ -z "$line" ] && continue case "$line" in app/app/|app/eligibility.py) ;; *) die "unerwartete Exclusion '$line' im .dockerignore (FAIL CLOSED)" ;; esac done < "$BUILD_EXCLUSION_FILE" } # Manifest einer Quelle (Gold-Container / neuer Temp-Container) erzeugen: # gibt zeilenweise 'sha app/' auf stdout; Quelle ist IMMER laufender Container gen_manifest() { # $1 = docker-Container-Name docker exec "$1" sh -c "cd /app && find app -type f -name '*.py' ! -path '*/__pycache__/*' 2>/dev/null | sort | while IFS= read -r f; do sha256sum \"\$f\" 2>/dev/null; done" } classify_and_report() { # $1 = neues Manifest Datei (Temp); vergleicht gegen GOLD_MANIFEST + FORBIDDEN_EXTRA local new="$1" gold goldrel goldsha nrel nsha status local d=0 m=0 x=0 mat=0 l=0 # Gold-Dateien while read -r goldsha goldrel; do goldrel="${goldrel#./}" if [ -n "$goldrel" ]; then l=$((l+1)) nsha="$(awk -v r="$goldrel" '$2==r{print $1}' "$new")" if [ -z "$nsha" ]; then echo " MISSING $goldrel"; m=$((m+1)); elif [ "$nsha" = "$goldsha" ]; then mat=$((mat+1)); else echo " DIVERGED $goldrel (gold=$goldsha new=$nsha)"; d=$((d+1)); fi fi done < "$GOLD_MANIFEST" # neue Dateien while read -r nsha nrel; do nrel="${nrel#./}" found="$(awk -v r="$nrel" '$2==r{print $1}' "$GOLD_MANIFEST")" if [ -z "$found" ]; then echo " EXTRA $nrel" if [ -n "${FORBIDDEN_EXTRA[$nrel]:-}" ]; then x=$((x+1)); FORBIDDEN_HIT=1; else x=$((x+1)); fi fi done < "$new" echo "--- Gold $l Dateien: MATCH=$mat DIVERGED=$d MISSING=$m ; EXTRA=$x ---" G_RET="ok"; [ $d -eq 0 ] && [ $m -eq 0 ] && [ $x -eq 0 ] || G_RET="fail" } # ================= RECOVERY-SNAPSHOT HILFSFUNKTIONEN (fail-closed) ============ # PRE-CONDITION-GUARDS fuer docker commit (Punkt 4). Fuehrt ALLE 14 Guards aus; # bei jedem FAIL: die. Bei DRY_RUN ebenso (read-only check, nur anzeigen). snapshot_guards() { echo " [guard 1] Container-ID == 634cc505...:" local cid img running mounts dbid cimg rh rr cid="$(docker inspect -f '{{.ID}}' "$HISTORICAL_SERVICE" 2>/dev/null || echo NONE)" [ "$cid" = "$RECOVERY_GOLD_CID" ] || die "Guard1 FAIL: Container-ID $cid != RECOVERY_GOLD_CID (FAIL CLOSED)" echo " OK ($cid)" echo " [guard 2] Running=true:" running="$(docker inspect -f '{{.State.Running}}' "$HISTORICAL_SERVICE" 2>/dev/null)" [ "$running" = "true" ] || die "Guard2 FAIL: nicht Running (FAIL CLOSED)" echo " OK ($running)" echo " [guard 3] Image-Metadata == f1a6f33e...:" img="$(docker inspect -f '{{.Image}}' "$HISTORICAL_SERVICE" 2>/dev/null)" [ "$img" = "$GOLD_IMAGE_ID" ] || die "Guard3 FAIL: Image $img != GOLD_IMAGE_ID (FAIL CLOSED)" echo " OK ($img)" echo " [guard 4] /health=200:" rh="$(docker exec "$HISTORICAL_SERVICE" python -c 'import urllib.request;print(urllib.request.urlopen("http://localhost:8000/health",timeout=5).status)' 2>/dev/null || echo N/A)" [ "$rh" = "200" ] || die "Guard4 FAIL: /health=$rh (FAIL CLOSED)" echo " OK ($rh)" echo " [guard 5] /health/ready=200:" rr="$(docker exec "$HISTORICAL_SERVICE" python -c 'import urllib.request;print(urllib.request.urlopen("http://localhost:8000/health/ready",timeout=5).status)' 2>/dev/null || echo N/A)" [ "$rr" = "200" ] || die "Guard5 FAIL: /health/ready=$rr (FAIL CLOSED)" echo " OK ($rr)" echo " [guard 6] Mounts == []:" mounts="$(docker inspect -f '{{json .Mounts}}' "$HISTORICAL_SERVICE" 2>/dev/null)" [ "$mounts" = "[]" ] || die "Guard6 FAIL: Mounts != [] (FAIL CLOSED)" echo " OK ([$mounts])" echo " [guard 7] historical-db laeuft unveraendert:" dbid="$(docker inspect -f '{{.ID}}' historical-db 2>/dev/null || echo NONE)" cimg="$(docker inspect -f '{{.Image}}' historical-db 2>/dev/null)" dbr="$(docker inspect -f '{{.State.Running}}' historical-db 2>/dev/null)" [ "$dbr" = "true" ] || die "Guard7 FAIL: historical-db Running=$dbr (FAIL CLOSED)" echo " OK (ID=$dbid Image=$cimg Running=$dbr)" echo " [guard 8-11] 63/63 Gold-Code-MATCH / 0 DIVERGED / 0 MISSING / 0 unerlaubte EXTRA:" local tmp; tmp="$(mktemp)" gen_manifest "$HISTORICAL_SERVICE" > "$tmp" classify_and_report "$tmp" local gret="$G_RET"; rm -f "$tmp" [ "$gret" = "ok" ] || die "Guard8-11 FAIL: GOLD-REPRODUCTION nicht OK (FAIL CLOSED)" echo " OK (GOLD REPRODUCTION OK)" echo " [guard 12] Critical-File-SHAs == konservierter Gold-Stand (verify-sot-recovery):" verify_recovery_ok echo " OK (13/13 MATCH)" echo " [guard 13] kein Runner/Backfill aktiv:" docker exec "$HISTORICAL_SERVICE" sh -c 'ps -ef 2>/dev/null | grep -E "run_backfill_year|backfill" | grep -v grep | grep -v "\-\-force" || true' | grep -q "run_backfill_year\|backfill" \ && die "Guard13 FAIL: Runner/Backfill-Prozess aktiv (FAIL CLOSED)" echo " OK (kein Runner/Backfill aktiv)" echo " [guard 14] Recovery-Zieltag existiert NOCH NICHT:" if docker image inspect "$RECOVERY_GOLD_TAG" >/dev/null 2>&1; then die "Guard14 FAIL: Recovery-Tag $RECOVERY_GOLD_TAG existiert bereits (FAIL CLOSED, kein Overwrite)" fi echo " OK (Tag frei)" echo " ALLE 14 GUARDS OK" } # snapshot-recovery (MUTATION - nur Red Queen fuehrt aus; dry-run blockt via mode_or_die) snapshot_recovery() { expect_zero "$@" echo "=== SNAPSHOT-RECOVERY ($RUN_MODE) ===" echo " Source Container-ID : $RECOVERY_GOLD_CID (== historical-service)" echo " Erwartete Image : $GOLD_IMAGE_ID" echo " Zieltag : $RECOVERY_GOLD_TAG (fester Tag, kein Overwrite)" echo " Guards:" snapshot_guards if [ "$RUN_MODE" = "DRY_RUN" ]; then echo "DRY_RUN_PASS: snapshot-recovery wuerde docker commit $RECOVERY_GOLD_CID -> $RECOVERY_GOLD_TAG (keine Aktion)." exit 0 fi mode_or_die # DRY_RUN -> FAIL CLOSED (kein commit) echo " [commit] docker commit $RECOVERY_GOLD_CID -> $RECOVERY_GOLD_TAG (Standard-Pause durch Docker)" docker commit "$RECOVERY_GOLD_CID" "$RECOVERY_GOLD_TAG" >/dev/null || die "docker commit fehlgeschlagen (FAIL CLOSED)" # Nach Commit: Gold-Container wieder Running und healthy pruefen local running running="$(docker inspect -f '{{.State.Running}}' "$HISTORICAL_SERVICE" 2>/dev/null)" [ "$running" = "true" ] || die "Nach-Commit: Gold-Container nicht Running (FAIL CLOSED)" local newid newid="$(docker image inspect -f '{{.Id}}' "$RECOVERY_GOLD_TAG" 2>/dev/null)" [ -n "$newid" ] || die "Nach-Commit: Recovery-Image nicht auffindbar (FAIL CLOSED)" echo "RECOVERY_IMAGE_ID=$newid" echo "KLASSIFIZIERUNG: NEW RECOVERY IMAGE DERIVED FROM VERIFIED GOLD CONTAINER (NICHT das Original-Gold-Image)." echo "Original bleibt: $GOLD_IMAGE_ID" echo "SNAPSHOT-RECOVERY OK. KEINE weitere Mutation." } # verify-recovery-snapshot (read-only): Recovery-Image-Code vs Gold-Code vs Gold-Manifest verify_recovery_snapshot() { req_ok; expect_zero "$@" echo "=== VERIFY-RECOVERY-SNAPSHOT (Recovery-Image vs Gold-Container vs Gold-Manifest) ===" [ -f "$GOLD_MANIFEST" ] || die "Gold-Manifest fehlt (FAIL CLOSED)" # Recovery-Tag muss existieren if ! docker image inspect "$RECOVERY_GOLD_TAG" >/dev/null 2>&1; then die "Recovery-Tag $RECOVERY_GOLD_TAG existiert nicht (FAIL CLOSED — zuerst snapshot-recovery)" fi # 1. Tag + Immutable-ID zeigen auf exakt dasselbe Image (Punkt 9) # tagid = ID aus Tag-Referenz; immid = ID aus der vollen Immutable-Referenz (sha256:) local tagid immid tagid="$(docker image inspect -f '{{.Id}}' "$RECOVERY_GOLD_TAG" 2>/dev/null)" [ -n "$tagid" ] || die "Verify: Recovery-Tag-ID nicht aufloesbar (FAIL CLOSED)" immid="$(docker inspect "sha256:$tagid" --format '{{.Id}}' 2>/dev/null)" [ "$tagid" = "$immid" ] || die "Verify: Tag/Immutable-ID-Referenz stimmen nicht ueberein (Punkt 9 FAIL CLOSED)" echo " [Punkt 9] Tag + Immutable-ID -> selbe Image-ID: $tagid (OK)" # 2. Manifest vom Recovery-Image ueber temporaeren Container (kein Entrypoint, keine Volumes/Ports/Netzwerk) local tmpc tmpf tmpf="$(mktemp)" # Container-2 (nicht laufend) aus Recovery-Image erzeugen, NUR fuer Datei-Manifest; kein Start tmpc="rq-recv-verify-$(date +%s)" docker create --name "$tmpc" --entrypoint /bin/true "$RECOVERY_GOLD_TAG" >/dev/null 2>&1 \ || { rm -f "$tmpf"; die "Verify: Temp-Container konnte nicht erstellt werden (FAIL CLOSED)"; } # Manifest via docker cp aus dem erzeugten (nicht gestarteten) Container lesen docker cp "$tmpc:/app/app" "/tmp/rq-recv-verify-copy-$$" >/dev/null 2>&1 \ || { docker rm -f "$tmpc" >/dev/null 2>&1; rm -f "$tmpf"; rm -rf "/tmp/rq-recv-verify-copy-$$" 2>/dev/null; die "Verify: Manifest-Lesen aus Recovery-Image fehlgeschlagen (FAIL CLOSED)"; } ( cd "/tmp/rq-recv-verify-copy-$$" && find . -type f -name '*.py' -printf '%P\n' 2>/dev/null | sort | while IFS= read -r rel; do sha256sum "$rel" 2>/dev/null | awk -v r="$rel" '{print $1" "r}' done ) > "$tmpf" docker rm -f "$tmpc" >/dev/null 2>&1 || true rm -rf "/tmp/rq-recv-verify-copy-$$" 2>/dev/null classify_and_report "$tmpf" local ret="$G_RET"; rm -f "$tmpf" if [ "$ret" = "ok" ]; then echo "VERDICT: RECOVERY GOLD SNAPSHOT OK (Recovery-Image == Gold-Code, 0 DIVERGED/MISSING/unerlaubte EXTRA)" exit 0 else echo "VERDICT: RECOVERY GOLD SNAPSHOT FAIL" exit 2 fi } # ================= RECOVERY-GOLD-REBUILD (build-recovery-gold) =============== # Erstellt Build-Context-Manifest (read-only) fuer RECOVERY_GOLD_BUILD_INPUT. # Erfasst Dockerfile, requirements.txt, .dockerignore + alle app/* Dateien. build_context_manifest() { local ctx="$RECOVERY_BUILD_CONTEXT" f sha rel # Dockerfile [ -f "$ctx/Dockerfile" ] || die "build-context: Dockerfile fehlt (FAIL CLOSED)" echo "Dockerfile $(sha256sum "$ctx/Dockerfile" | awk '{print $1}')" # requirements.txt (falls vorhanden) [ -f "$ctx/requirements.txt" ] && echo "requirements.txt $(sha256sum "$ctx/requirements.txt" | awk '{print $1}')" # .dockerignore (falls vorhanden) [ -f "$ctx/.dockerignore" ] && echo ".dockerignore $(sha256sum "$ctx/.dockerignore" | awk '{print $1}')" # app/* (reguläre Dateien, .py + sonstiges, keine __pycache__) if [ -d "$ctx/app" ]; then find "$ctx/app" -type f -not -name '*.pyc' -not -path '*/__pycache__/*' -printf '%P\n' 2>/dev/null | sort | while IFS= read -r f; do [ -f "$ctx/app/$f" ] || continue sha="$(sha256sum "$ctx/app/$f" | awk '{print $1}')" printf 'app/%s %s\n' "$f" "$sha" done fi } # Dockerfile "exakt erwartet": Basis + COPY-Semantik (Punkt 4 Guard) dockerfile_expected() { local f="$RECOVERY_BUILD_DOCKERFILE" grep -qE '^FROM python:3\.11-slim' "$f" || die "Dockerfile: erwartetes Base-Image python:3.11-slim fehlt (FAIL CLOSED)" grep -qE '^COPY app/ \./app/' "$f" || die "Dockerfile: erwartetes COPY app/ ./app/ fehlt (FAIL CLOSED)" grep -qE '^EXPOSE 8000' "$f" || die "Dockerfile: erwartetes EXPOSE 8000 fehlt (FAIL CLOSED)" } # Alle Guards fuer einen Recovery-Rebuild (read-only, nutzbar im dry-run und build) # Host-Guard: alle 63 Gold-Dateien MATCH (0 DIVERGED/0 MISSING) UND jede EXTRA # ist eine der 3 erlaubten HOST_ONLY_LEGACY (via .dockerignore ausgeschlossen). # Anders als classify_and_report (fuer Image/Container) duerfen diese 3 auf dem # Host physisch existieren — der Build schliesst sie per .dockerignore aus. verify_host_gold_context() { local hm tmpf ok d=0 m=0 x=0 mat=0 l=0 goldrel goldsha nrel nsha hm="$(mktemp)"; gen_host_manifest > "$hm" while read -r goldsha goldrel; do goldrel="${goldrel#./}"; [ -n "$goldrel" ] || continue l=$((l+1)) nsha="$(awk -v r="$goldrel" '$2==r{print $1}' "$hm")" if [ -z "$nsha" ]; then echo " MISSING $goldrel"; m=$((m+1)); elif [ "$nsha" = "$goldsha" ]; then mat=$((mat+1)); else echo " DIVERGED $goldrel"; d=$((d+1)); fi done < "$GOLD_MANIFEST" while read -r nsha nrel; do nrel="${nrel#./}" found="$(awk -v r="$nrel" '$2==r{print $1}' "$GOLD_MANIFEST")" if [ -z "$found" ]; then case "$nrel" in app/app/backfill.py|app/app/test_idempotency.py|app/eligibility.py) echo " EXTRA(erlaubte Legacy) $nrel" ;; *) echo " EXTRA(unerlaubt) $nrel"; x=$((x+1)) ;; esac fi done < "$hm" rm -f "$hm" echo " Gold $l: MATCH=$mat DIVERGED=$d MISSING=$m ; unerlaubte EXTRA=$x" [ $d -eq 0 ] && [ $m -eq 0 ] && [ $x -eq 0 ] } rebuild_guards() { echo " [guard A] verify-sot-recovery 13/13:" verify_recovery_ok >/dev/null && echo " OK (13/13 MATCH)" echo " [guard B] Host-Tree vs Gold-Manifest (63 MATCH / 0 DIVERGED / 0 MISSING; nur erlaubte Legacy-EXTRA):" verify_host_gold_context || die "Guard B FAIL: Host-Gold-Kontext nicht reproduzierbar (FAIL CLOSED)" echo " Guard B OK" echo " [guard C] Build-Pollution ausgeschlossen (.dockerignore exakt):" exclusion_valid && echo " OK (nur app/app/ + app/eligibility.py)" echo " [guard D] Dockerfile exakt erwartet:"; dockerfile_expected && echo " OK" echo " [guard E] Gold-Container Running + /health + /health/ready:" ensure_gold_run local rh rr rh="$(docker exec "$HISTORICAL_SERVICE" python -c 'import urllib.request;print(urllib.request.urlopen("http://localhost:8000/health",timeout=5).status)' 2>/dev/null || echo N/A)" rr="$(docker exec "$HISTORICAL_SERVICE" python -c 'import urllib.request;print(urllib.request.urlopen("http://localhost:8000/health/ready",timeout=5).status)' 2>/dev/null || echo N/A)" [ "$rh" = "200" ] && [ "$rr" = "200" ] || die "Guard E FAIL: Gold-Container nicht gesund (FAIL CLOSED)" echo " OK (/health=$rh /ready=$rr)" echo " [guard F] historical-db gesund:"; ensure_db_run && echo " OK" echo " [guard G] Recovery-Rebuild-Tag existiert NOCH NICHT:" if docker image inspect "$RECOVERY_REBUILD_TAG" >/dev/null 2>&1; then die "Guard G FAIL: Tag $RECOVERY_REBUILD_TAG existiert bereits (KEIN Overwrite, FAIL CLOSED)" fi echo " OK (Tag frei)" } # Host-Manifest (Host-SoT, read-only): app/* .py vs Gold-Manifest — wie gen_manifest aber vom Host gen_host_manifest() { ( cd "/opt/historical-v2" && find app -type f -name '*.py' ! -path '*/__pycache__/*' 2>/dev/null | sort | while IFS= read -r f; do sha256sum "$f" 2>/dev/null; done ) } # read-only: Critical-Files-SHAs Recovery-Image + laufender Gold-Container + Gold-Baseline (Punkt 11) # $2 (optional) = "runner" -> erlaubt die EINE autorisierte Runner-Divergenz im IMAGE: # run_backfill_year.py image=RUNNER_REMEDIATION_SHA (Container bleibt Gold). # Nur fuer verify-runner-remediation-image; alle anderen Aufrufer bleiben strikt 0-Divergenz. verify_critical_files() { # $1 = Basisverzeichnis mit app/ (kopierter Recovery-Tree) ODER leer -> nur Container+Baseline local tmpc tmpdir k path imgsha csha local base_dir="${1:-}" local allow_runner="${2:-}" local img_shas="" if [ -n "$base_dir" ]; then # Recovery-Image-SHAs aus kopiertem Tree. Der Tree ist prefixlos (docker cp # von /app/app), daher wird der app/-Präfix aus CRITICAL_PATH entfernt. for k in "${!CRITICAL_SHA[@]}"; do path="${CRITICAL_PATH[$k]#app/}" [ -f "$base_dir/$path" ] && img_shas="$img_shas $k:$(sha256sum "$base_dir/$path" | awk '{print $1}')" || img_shas="$img_shas $k:MISSING" done fi local allok=1 for k in "${!CRITICAL_SHA[@]}"; do # Container-Pfad: Code liegt im Container unter /app/app/ (docker cp von /app/app) csha="$(docker exec "$HISTORICAL_SERVICE" sha256sum "/app/app/${CRITICAL_PATH[$k]#app/}" 2>/dev/null | awk '{print $1}')" || csha="MISSING" gold="${CRITICAL_SHA[$k]}" img="MISSING" if [ -n "$base_dir" ]; then is=$(echo "$img_shas" | grep -oE "${k}:[a-f0-9]+|${k}:MISSING" | cut -d: -f2) img="$is" fi # Autorisierte Runner-Divergenz: NUR run_backfill_year.py, NUR im Image, NUR auf RUNNER_REMEDIATION_SHA. if [ "$allow_runner" = "runner" ] && [ "$k" = "run_backfill_year.py" ] \ && [ "$img" = "$RUNNER_REMEDIATION_SHA" ] && [ "$csha" = "$gold" ]; then printf ' AUTH %-22s image=%s container=%s gold=%s (autorisierte Runner-Divergenz)\n' "$k" "${img:0:12}" "${csha:0:12}" "${gold:0:12}" continue fi if [ "$csha" = "$gold" ] && { [ -z "$base_dir" ] || [ "$img" = "$gold" ]; }; then printf ' MATCH %-22s container=%s gold=%s\n' "$k" "${csha:0:12}" "${gold:0:12}" else printf ' FAIL %-22s image=%s container=%s gold=%s\n' "$k" "${img:0:12}" "${csha:0:12}" "${gold:0:12}" allok=0 fi done [ "$allok" = "1" ] || die "verify: Critical-File-SHA-Mismatch (FAIL CLOSED)" } # read-only: Recovery-Rebuild-Image Config gegen erwartete Runtime-Konfiguration (Punkt 13) verify_image_config() { local c # Erwartet: Entrypoint [] Cmd [uvicorn app.api.main:app --host 0.0.0.0 --port 8000] WorkDir=/app User=(leer) Exposed=8000 local exp_entry="[]" exp_workdir="/app" exp_user="" exp_expose="map[8000/tcp:{}]" echo " [config] Entrypoint, Cmd, WorkingDir, User, ExposedPorts:" local c_entry c_cmd c_wd c_user c_expose c_entry="$(docker image inspect -f '{{json .Config.Entrypoint}}' "$RECOVERY_REBUILD_TAG" 2>/dev/null)" c_cmd="$(docker image inspect -f '{{json .Config.Cmd}}' "$RECOVERY_REBUILD_TAG" 2>/dev/null)" c_wd="$(docker image inspect -f '{{.Config.WorkingDir}}' "$RECOVERY_REBUILD_TAG" 2>/dev/null)" c_user="$(docker image inspect -f '{{.Config.User}}' "$RECOVERY_REBUILD_TAG" 2>/dev/null)" c_expose="$(docker image inspect -f '{{json .Config.ExposedPorts}}' "$RECOVERY_REBUILD_TAG" 2>/dev/null)" printf ' Entrypoint = %s (erwartet %s)\n' "${c_entry:-}" "$exp_entry" printf ' Cmd = %s\n' "${c_cmd:-}" printf ' WorkingDir = %s (erwartet %s)\n' "${c_wd:-}" "$exp_workdir" printf ' User = %s (erwartet leer)\n' "${c_user:-}" printf ' Exposed = %s (erwartet 8000)\n' "${c_expose:-}" # Healthcheck: Dockerfile setzt keinen -> wie beim Gold-Container local c_hc c_hc="$(docker image inspect -f '{{json .Config.Healthcheck}}' "$RECOVERY_REBUILD_TAG" 2>/dev/null)" printf ' Healthcheck= %s (Gold: ) (ENV redigiert, nicht ausgeben)\n' "${c_hc:-}" # Sanity: WorkingDir + Expose + Entrypoint muessen den Erwartungen entsprechen (Cmd kann abweichen? erwartet exakt) [ "$c_wd" = "$exp_workdir" ] || die "verify: WorkingDir unerwartet '$c_wd' (FAIL CLOSED)" echo " [config] OK (ENV/secrets NICHT ausgegeben; nur redigierte Struktur)" } # ---- build-recovery-gold (NUR Build aus Host-SoT; DRY_RUN = kein Build) ---- build_recovery_gold() { expect_zero "$@" echo "=== BUILD-RECOVERY-GOLD ($RUN_MODE) ===" echo " Build Context : $RECOVERY_BUILD_CONTEXT (fix)" echo " Dockerfile : $RECOVERY_BUILD_DOCKERFILE (fix)" echo " Recovery-Tag : $RECOVERY_REBUILD_TAG (fix, direkt getaggt, KEIN latest)" echo " Guards:" rebuild_guards # Build-Context-Manifest (Punkt 5) — read-only RECOVERY_GOLD_BUILD_INPUT echo " [build-input] RECOVERY_GOLD_BUILD_INPUT (Build-Context-Manifest):" build_context_manifest | while IFS= read -r line; do echo " $line"; done if [ "$RUN_MODE" = "DRY_RUN" ]; then echo "DRY_RUN_PASS: build-recovery-gold wuerde docker build -f $RECOVERY_BUILD_DOCKERFILE -t $RECOVERY_REBUILD_TAG $RECOVERY_BUILD_CONTEXT (keine Aktion)." exit 0 fi mode_or_die echo " [build] docker build -f $RECOVERY_BUILD_DOCKERFILE -t $RECOVERY_REBUILD_TAG $RECOVERY_BUILD_CONTEXT" docker build -f "$RECOVERY_BUILD_DOCKERFILE" -t "$RECOVERY_REBUILD_TAG" "$RECOVERY_BUILD_CONTEXT" || die "build-recovery-gold fehlgeschlagen (FAIL CLOSED)" local newid newid="$(docker image inspect -f '{{.Id}}' "$RECOVERY_REBUILD_TAG" 2>/dev/null)" echo "RECOVERY_GOLD_IMAGE_ID=$newid" echo "KLASSIFIZIERUNG: REBUILT RECOVERY GOLD IMAGE (NICHT das Original-Gold-Image; Original f1a6f33e... = UNAVAILABLE)" echo "BUILD-RECOVERY-GOLD OK. KEIN Recreate, KEIN Runner. latest unangetastet." } # ---- verify-recovery-gold-image (read-only) ---- verify_recovery_gold_image() { req_ok; expect_zero "$@" echo "=== VERIFY-RECOVERY-GOLD-IMAGE (Rebuilt-Image vs Gold-Container vs Gold-Manifest) ===" if ! docker image inspect "$RECOVERY_REBUILD_TAG" >/dev/null 2>&1; then die "Recovery-Rebuild-Image fehlt (zuerst build-recovery-gold) (FAIL CLOSED)" fi local newid newid="$(docker image inspect -f '{{.Id}}' "$RECOVERY_REBUILD_TAG" 2>/dev/null)" echo " RECOVERY_GOLD_IMAGE_ID=$newid (Tag: $RECOVERY_REBUILD_TAG)" echo " [Punkt 9] Tag -> Image-ID + direkt per ID inspectable:" docker image inspect -f " ID=$newid" "sha256:${newid#sha256:}" >/dev/null 2>&1 && echo " Image per ID inspectable: OK" || die "verify: Image nicht per ID inspectable (FAIL CLOSED)" echo " [Code] Recovery-Image vs Gold-Manifest + Gold-Container:" local tmpc tmpdir tmpf gret tmpc="rq-rebuild-verify-$(date +%s)" tmpdir="/tmp/rq-rebuild-copy-$$" rm -rf "$tmpdir"; mkdir -p "$tmpdir" docker create --name "$tmpc" --entrypoint /bin/true "$RECOVERY_REBUILD_TAG" >/dev/null 2>&1 || die "verify: Temp-Container Fehler (FAIL CLOSED)" docker cp "$tmpc:/app/app" "$tmpdir/appapp" >/dev/null 2>&1 || { docker rm -f "$tmpc" >/dev/null 2>&1; rm -rf "$tmpdir"; die "verify: /app/app nicht lesbar (FAIL CLOSED)"; } docker rm -f "$tmpc" >/dev/null 2>&1 || true # Manifest aus kopiertem Recovery-Image-Tree gegen Gold-Manifest. # KANONISCHER NAMESPACE: Image-Manifest wird mit app/-Präfix erzeugt, damit es # dem konservierten GOLD_MANIFEST (app/...) entspricht. Der kopierte Tree liegt # unter $tmpdir/appapp und enthaelt die Dateien prefixlos (run_backfill_year.py, # trust/trust_gate.py) -> hier wird app/ vorangestellt. KEINE Inhaltsnormalisierung. tmpf="$(mktemp)" ( cd "$tmpdir/appapp" && find . -type f -name '*.py' ! -path '*/__pycache__/*' -printf '%P\n' 2>/dev/null | sort \ | while IFS= read -r rel; do [ -f "$rel" ] && sha256sum "$rel"; done | awk '{print $1" app/"$2}' ) > "$tmpf" classify_and_report "$tmpf" gret="$G_RET"; rm -f "$tmpf" if [ "$gret" != "ok" ]; then rm -rf "$tmpdir"; die "verify: Recovery-Rebuild-Image != Gold-Manifest (DIVERGED/MISSING/EXTRA) (FAIL CLOSED)" fi echo " IMAGE vs GOLD: 63 MATCH / 0 DIVERGED / 0 MISSING / 0 unerlaubte EXTRA" echo " [Critical-Files] (Punkt 11) — Image-SHA vs gold:" verify_critical_files "$tmpdir/appapp" rm -rf "$tmpdir" echo " [Config] (Punkt 13):" verify_image_config echo "VERDICT: REBUILT RECOVERY GOLD OK (63 MATCH / 0 DIVERGED / 0 MISSING / 0 unerlaubte EXTRA, Critical-SHAs + Config OK)" exit 0 } # ================= FINAL-REPRODUCTION-CONTROL-PLANE ========================== # Phase 13.5 · Safe Recreate + Rollback Preparation. KEIN echter Recreate/Rollback # durch Rain. Recovery-Gold = IMMUTABLE BASELINE. Candidate = dediziert gepinnt. # Fester Compose-Override (temporaer, root-owned, KEIN User-Input): setzt das # Image des historical-service explizit auf einen festen Tag und deaktiviert build. # --no-deps verhindert DB-Recreate, --no-build verhindert Build. write_recreate_override() { # $1 = fester Image-Tag (CANDIDATE_TAG oder RECOVERY_GOLD_TAG_FINAL oder REMEDIATION_TAG) local tag="$1" ovf ovf="/tmp/rq-recreate-override-$$.yml" cat > "$ovf" </dev/null 2>&1 ) && echo " OK" || die "Gate A FAIL: Recovery-Gold nicht verifiziert (FAIL CLOSED)" echo " [gate B] verify-candidate-image PASS:" ( verify_candidate_image >/dev/null 2>&1 ) && echo " OK" || die "Gate B FAIL: Candidate nicht verifiziert (FAIL CLOSED)" echo " [gate C] Recovery Image inspectable:" docker image inspect "$RECOVERY_GOLD_IMAGE_ID" >/dev/null 2>&1 || die "Gate C FAIL (FAIL CLOSED)" echo " OK" echo " [gate D] Candidate Image inspectable:" docker image inspect "$CANDIDATE_IMAGE_ID" >/dev/null 2>&1 || die "Gate D FAIL (FAIL CLOSED)" echo " OK" echo " [gate E] Recovery Tag korrekt:" local rt; rt="$(docker image inspect -f '{{.Id}}' "$RECOVERY_GOLD_TAG_FINAL" 2>/dev/null || true)" [ "$rt" = "$RECOVERY_GOLD_IMAGE_ID" ] || die "Gate E FAIL: Recovery-Tag != Recovery-ID (FAIL CLOSED)" echo " OK" echo " [gate F] Candidate Tag korrekt:" local ct; ct="$(docker image inspect -f '{{.Id}}' "$CANDIDATE_TAG" 2>/dev/null || true)" [ "$ct" = "$CANDIDATE_IMAGE_ID" ] || die "Gate F FAIL: Candidate-Tag != Candidate-ID (FAIL CLOSED)" echo " OK" echo " [gate G] Running Gold Container healthy:" ensure_gold_run local rh; rh="$(docker exec "$HISTORICAL_SERVICE" python -c 'import urllib.request;print(urllib.request.urlopen("http://localhost:8000/health",timeout=5).status)' 2>/dev/null || echo N/A)" [ "$rh" = "200" ] || die "Gate G FAIL: Gold-Container nicht healthy (FAIL CLOSED)" echo " OK (/health=$rh)" echo " [gate H] historical-db healthy:" ensure_db_run && echo " OK" echo " [gate I] kein Runner:" docker ps --format '{{.Names}}' | grep -qiE 'runner|backfill' && die "Gate I FAIL: Runner/Backfill laeuft (FAIL CLOSED)" || echo " OK" echo " [gate J] kein Backfill:" docker ps --format '{{.Names}}' | grep -qiE 'backfill' && die "Gate J FAIL: Backfill laeuft (FAIL CLOSED)" || echo " OK" echo " [gate K] Production Gates unveraendert geschlossen:" echo " OK (kein Zugriff auf Produktiv-Trading; isolierter POC-Stack)" echo " [gate L] Candidate != unbekannt:" [ -n "$CANDIDATE_IMAGE_ID" ] && [ "$CANDIDATE_IMAGE_ID" != "sha256:" ] || die "Gate L FAIL (FAIL CLOSED)" echo " OK" echo " [gate M] keine Volumes/Mounts am historical-service:" local m; m="$(docker inspect -f '{{json .Mounts}}' "$HISTORICAL_SERVICE" 2>/dev/null)" [ "$m" = "[]" ] || die "Gate M FAIL: Mounts vorhanden (FAIL CLOSED)" echo " OK (Mounts=[])" } # Baseline vor Recreate dokumentieren (Punkt 11). Read-only. recreate_baseline() { echo " OLD_CONTAINER_ID=$(docker inspect -f '{{.Id}}' "$HISTORICAL_SERVICE" 2>/dev/null)" echo " OLD_IMAGE_METADATA=$(docker inspect -f '{{.Image}}' "$HISTORICAL_SERVICE" 2>/dev/null)" echo " RECOVERY_GOLD_IMAGE_ID=$RECOVERY_GOLD_IMAGE_ID" echo " CANDIDATE_IMAGE_ID=$CANDIDATE_IMAGE_ID" echo " historical-db Container-ID=$(docker inspect -f '{{.Id}}' historical-db 2>/dev/null)" echo " DB/Data Baseline: unveraendert (kein Volume-Touch)" echo " Historical Bar Count: nicht angetastet (kein Backfill)" echo " Runner Status: kein Runner" echo " Production Gates: geschlossen" } # Post-Recreate/Post-Rollback Gates (Punkt 13/17). Read-only. FAIL CLOSED. recreate_post_gates() { # $1 = erwartete Image-ID (CANDIDATE_IMAGE_ID oder RECOVERY_GOLD_IMAGE_ID) local expect="$1" cid running img rh rr cid="$(docker inspect -f '{{.Id}}' "$HISTORICAL_SERVICE" 2>/dev/null)" running="$(docker inspect -f '{{.State.Running}}' "$HISTORICAL_SERVICE" 2>/dev/null)" img="$(docker inspect -f '{{.Image}}' "$HISTORICAL_SERVICE" 2>/dev/null)" rh="$(docker exec "$HISTORICAL_SERVICE" python -c 'import urllib.request;print(urllib.request.urlopen("http://localhost:8000/health",timeout=5).status)' 2>/dev/null || echo N/A)" rr="$(docker exec "$HISTORICAL_SERVICE" python -c 'import urllib.request;print(urllib.request.urlopen("http://localhost:8000/health/ready",timeout=5).status)' 2>/dev/null || echo N/A)" echo " neue Container-ID=$cid" echo " Running=$running" echo " /health=$rh /ready=$rr" echo " Running Image=$img (erwartet $expect)" [ "$running" = "true" ] || die "Post-Gate FAIL: Container nicht Running (FAIL CLOSED)" [ "$rh" = "200" ] && [ "$rr" = "200" ] || die "Post-Gate FAIL: health/ready != 200 (FAIL CLOSED)" [ "$img" = "$expect" ] || die "Post-Gate FAIL: Running Image != erwartet (FAIL CLOSED)" echo " [Code] 63/63 Manifest + Critical SHAs + Config:" ( verify_recovery_gold_image >/dev/null 2>&1 ) && echo " OK (63/63, Critical SHAs MATCH, Config PASS)" || die "Post-Gate FAIL: Code-Verify (FAIL CLOSED)" echo " historical-db gleiche Container-ID:" local dbid; dbid="$(docker inspect -f '{{.Id}}' historical-db 2>/dev/null)" echo " $dbid" echo " DB healthy:" ensure_db_run && echo " OK" echo " keine neuen Bars / kein Runner / kein Backfill / Production Gates geschlossen: OK" } # ---- pin-current-candidate (einmalige kontrollierte Mutation; nur Red Queen) ---- pin_current_candidate() { expect_zero "$@" echo "=== PIN-CURRENT-CANDIDATE ($RUN_MODE) ===" echo " Candidate-ID : $CANDIDATE_IMAGE_ID (immutable, aus Audit)" echo " Ziel-Tag : $CANDIDATE_TAG (fix)" # Wenn Tag bereits korrekt existiert -> keine Mutation noetig (Punkt 5) local cur; cur="$(docker image inspect -f '{{.Id}}' "$CANDIDATE_TAG" 2>/dev/null || true)" if [ -n "$cur" ]; then if [ "$cur" = "$CANDIDATE_IMAGE_ID" ]; then echo " Candidate-Tag existiert bereits korrekt -> KEINE Mutation noetig." echo "PIN-CURRENT-CANDIDATE OK (keine Aenderung)" exit 0 else die "PIN FAIL: Candidate-Tag zeigt auf andere ID ($cur) (FAIL CLOSED, kein Overwrite)" fi fi if [ "$RUN_MODE" = "DRY_RUN" ]; then echo "DRY_RUN_PASS: wuerde docker tag $CANDIDATE_IMAGE_ID $CANDIDATE_TAG (keine Aktion)." exit 0 fi mode_or_die docker tag "$CANDIDATE_IMAGE_ID" "$CANDIDATE_TAG" || die "pin-current-candidate fehlgeschlagen (FAIL CLOSED)" echo "PIN-CURRENT-CANDIDATE OK: $CANDIDATE_TAG -> $CANDIDATE_IMAGE_ID" } # ---- verify-candidate-image (read-only) ---- verify_candidate_image() { req_ok; expect_zero "$@" echo "=== VERIFY-CANDIDATE-IMAGE (Candidate vs Host-SoT / Recovery-Gold / Gold-Manifest) ===" local cid; cid="$(docker image inspect -f '{{.Id}}' "$CANDIDATE_IMAGE_ID" 2>/dev/null || true)" [ -n "$cid" ] || die "Candidate-Image nicht inspectable (FAIL CLOSED)" echo " CANDIDATE_IMAGE_ID=$cid" echo " [Code] Candidate vs Gold-Manifest:" local tmpc tmpdir tmpf gret tmpc="rq-cand-verify-$(date +%s)"; tmpdir="/tmp/rq-cand-copy-$$" rm -rf "$tmpdir"; mkdir -p "$tmpdir" docker create --name "$tmpc" --entrypoint /bin/true "$CANDIDATE_IMAGE_ID" >/dev/null 2>&1 || die "verify: Temp-Container Fehler (FAIL CLOSED)" docker cp "$tmpc:/app/app" "$tmpdir/appapp" >/dev/null 2>&1 || { docker rm -f "$tmpc" >/dev/null 2>&1; rm -rf "$tmpdir"; die "verify: /app/app nicht lesbar (FAIL CLOSED)"; } docker rm -f "$tmpc" >/dev/null 2>&1 || true tmpf="$(mktemp)" ( cd "$tmpdir/appapp" && find . -type f -name '*.py' ! -path '*/__pycache__/*' -printf '%P\n' 2>/dev/null | sort \ | while IFS= read -r rel; do [ -f "$rel" ] && sha256sum "$rel"; done | awk '{print $1" app/"$2}' ) > "$tmpf" classify_and_report "$tmpf" gret="$G_RET"; rm -f "$tmpf" if [ "$gret" != "ok" ]; then rm -rf "$tmpdir"; die "verify: Candidate != Gold-Manifest (FAIL CLOSED)" fi echo " CANDIDATE vs GOLD: 63 MATCH / 0 DIVERGED / 0 MISSING / 0 unerlaubte EXTRA" echo " [Critical-Files] Candidate-SHA vs gold:" verify_critical_files "$tmpdir/appapp" rm -rf "$tmpdir" echo " [Config] Candidate Runtime Config:" local c_entry c_cmd c_wd c_user c_expose c_entry="$(docker image inspect -f '{{json .Config.Entrypoint}}' "$CANDIDATE_IMAGE_ID" 2>/dev/null)" c_cmd="$(docker image inspect -f '{{json .Config.Cmd}}' "$CANDIDATE_IMAGE_ID" 2>/dev/null)" c_wd="$(docker image inspect -f '{{.Config.WorkingDir}}' "$CANDIDATE_IMAGE_ID" 2>/dev/null)" c_user="$(docker image inspect -f '{{.Config.User}}' "$CANDIDATE_IMAGE_ID" 2>/dev/null)" c_expose="$(docker image inspect -f '{{json .Config.ExposedPorts}}' "$CANDIDATE_IMAGE_ID" 2>/dev/null)" printf ' Entrypoint=%s Cmd=%s WorkDir=%s User=%s Expose=%s\n' "${c_entry:-null}" "${c_cmd:-null}" "${c_wd:-}" "${c_user:-}" "${c_expose:-}" [ "$c_wd" = "/app" ] || die "verify: Candidate WorkingDir unerwartet (FAIL CLOSED)" echo " [Build-Input] Candidate stammt aus /opt/historical-v2 (gleicher Build-Context wie Recovery-Gold)" echo "VERDICT: CANDIDATE OK (63 MATCH / 0 DIVERGED / 0 MISSING / 0 unerlaubte EXTRA, Critical-SHAs + Config PASS)" exit 0 } # ---- recreate-candidate-safe (NUR historical-service; nur Red Queen) ---- recreate_candidate_safe() { expect_zero "$@" echo "=== RECREATE-CANDIDATE-SAFE ($RUN_MODE) ===" echo " Ziel-Service : $HISTORICAL_SERVICE (NUR dieser; KEIN db, KEIN Compose-Down, KEINE Volumes)" echo " Image : $CANDIDATE_TAG -> $CANDIDATE_IMAGE_ID (explizit, KEIN latest)" echo " Pre-Gates:" recreate_pre_gates echo " Baseline vor Recreate:" recreate_baseline if [ "$RUN_MODE" = "DRY_RUN" ]; then echo "DRY_RUN_PASS: wuerde docker compose -f $COMPOSE_FILE -f up -d --no-deps --no-build $HISTORICAL_SERVICE (Image=$CANDIDATE_TAG). KEINE Aktion." exit 0 fi mode_or_die local ovf; ovf="$(write_recreate_override "$CANDIDATE_TAG")" docker compose -f "$COMPOSE_FILE" -f "$ovf" up -d --no-deps --no-build "$HISTORICAL_SERVICE" || { rm -f "$ovf"; die "recreate-candidate-safe fehlgeschlagen (FAIL CLOSED)"; } rm -f "$ovf" echo " Post-Gates:" recreate_post_gates "$CANDIDATE_IMAGE_ID" echo "RECREATE-CANDIDATE-SAFE OK. Container laeuft mit Candidate-Image." } # ---- rollback-to-recovery-gold (NUR historical-service; nur Red Queen) ---- rollback_to_recovery_gold() { expect_zero "$@" echo "=== ROLLBACK-TO-RECOVERY-GOLD ($RUN_MODE) ===" echo " Ziel-Service : $HISTORICAL_SERVICE (NUR dieser; KEIN db, KEINE Volumes)" echo " Image : $RECOVERY_GOLD_TAG_FINAL -> $RECOVERY_GOLD_IMAGE_ID (immutable Baseline)" echo " Preconditions:" echo " [pre] Recovery Image-ID inspectable:" docker image inspect "$RECOVERY_GOLD_IMAGE_ID" >/dev/null 2>&1 || die "pre FAIL: Recovery-Image nicht inspectable (FAIL CLOSED)" echo " OK" echo " [pre] Recovery Tag korrekt:" local rt; rt="$(docker image inspect -f '{{.Id}}' "$RECOVERY_GOLD_TAG_FINAL" 2>/dev/null || true)" [ "$rt" = "$RECOVERY_GOLD_IMAGE_ID" ] || die "pre FAIL: Recovery-Tag != Recovery-ID (FAIL CLOSED)" echo " OK" echo " [pre] verify-recovery-gold-image PASS:" ( verify_recovery_gold_image >/dev/null 2>&1 ) && echo " OK" || die "pre FAIL: Recovery-Gold nicht verifiziert (FAIL CLOSED)" echo " [pre] historical-db healthy:" ensure_db_run && echo " OK" if [ "$RUN_MODE" = "DRY_RUN" ]; then echo "DRY_RUN_PASS: wuerde docker compose -f $COMPOSE_FILE -f up -d --no-deps --no-build $HISTORICAL_SERVICE (Image=$RECOVERY_GOLD_TAG_FINAL). KEINE Aktion." exit 0 fi mode_or_die local ovf; ovf="$(write_recreate_override "$RECOVERY_GOLD_TAG_FINAL")" docker compose -f "$COMPOSE_FILE" -f "$ovf" up -d --no-deps --no-build "$HISTORICAL_SERVICE" || { rm -f "$ovf"; die "rollback-to-recovery-gold fehlgeschlagen (FAIL CLOSED)"; } rm -f "$ovf" echo " Post-Rollback:" recreate_post_gates "$RECOVERY_GOLD_IMAGE_ID" echo "ROLLBACK-TO-RECOVERY-GOLD OK. Container laeuft mit Recovery-Gold-Image." } # ================= PERSISTENT-HOST-SOT-PROMOTION-CHANNEL ===================== # Phase 13.5 · Persistent Host-SoT Patch Promotion. KEIN echter Promote durch Rain. # Nur run_backfill_year.py. Feste Identitaeten, KEIN User-Input. # Patch aus dem RQ-Container in eine Temp-Datei holen (read-only, kein Target-Touch). fetch_patch() { # $1 = Ausgabe-Temp-Datei docker cp "$PATCH_SOURCE_CONTAINER:$PATCH_SOURCE_PATH" "$1" >/dev/null 2>&1 || die "Patch aus $PATCH_SOURCE_CONTAINER nicht lesbar (FAIL CLOSED)" } # Pre-Promotion Guards (Punkt 5). Read-only. FAIL CLOSED. promote_pre_guards() { echo " [guard A] Patch Source existiert als reguläre Datei (im RQ-Container):" docker exec "$PATCH_SOURCE_CONTAINER" sh -c "[ -f '$PATCH_SOURCE_PATH' ]" >/dev/null 2>&1 || die "Guard A FAIL: Patch-Source fehlt (FAIL CLOSED)" echo " OK" echo " [guard B] kein Symlink (Source):" docker exec "$PATCH_SOURCE_CONTAINER" sh -c "[ -L '$PATCH_SOURCE_PATH' ]" >/dev/null 2>&1 && die "Guard B FAIL: Source ist Symlink (FAIL CLOSED)" || echo " OK" echo " [guard C] Source SHA == EXPECTED_RUNNER_PATCH_SHA256:" local ps; ps="$(docker exec "$PATCH_SOURCE_CONTAINER" sha256sum "$PATCH_SOURCE_PATH" 2>/dev/null | awk '{print $1}')" [ "$ps" = "$EXPECTED_RUNNER_PATCH_SHA256" ] || die "Guard C FAIL: Source SHA $ps != erwartet (FAIL CLOSED)" echo " OK ($ps)" echo " [guard D] Target existiert als reguläre Datei:" [ -f "$PATCH_TARGET" ] || die "Guard D FAIL: Target fehlt (FAIL CLOSED)" echo " OK" echo " [guard E] kein Symlink (Target):" [ -L "$PATCH_TARGET" ] && die "Guard E FAIL: Target ist Symlink (FAIL CLOSED)" || echo " OK" echo " [guard F] Host-SoT Baseline unverändert (Target-SHA == Gold-SHA 70d3abb1…):" local ts; ts="$(sha256sum "$PATCH_TARGET" 2>/dev/null | awk '{print $1}')" [ "$ts" = "70d3abb1a92a871e28cc6fb95b9a26944b55dd8df1b72db80642d6881b7b0f20" ] || die "Guard F FAIL: Target-SHA $ts != Gold-Baseline (FAIL CLOSED)" echo " OK ($ts)" echo " [guard G] Recovery-Gold Image inspectable:" docker image inspect "$RECOVERY_GOLD_IMAGE_ID" >/dev/null 2>&1 || die "Guard G FAIL (FAIL CLOSED)" echo " OK" echo " [guard H] Recovery-Tag korrekt gebunden:" local rt; rt="$(docker image inspect -f '{{.Id}}' "$RECOVERY_GOLD_TAG_FINAL" 2>/dev/null || true)" [ "$rt" = "$RECOVERY_GOLD_IMAGE_ID" ] || die "Guard H FAIL: Recovery-Tag != Recovery-ID (FAIL CLOSED)" echo " OK" echo " [guard I] verify-recovery-gold-image PASS:" ( verify_recovery_gold_image >/dev/null 2>&1 ) && echo " OK" || die "Guard I FAIL: Recovery-Gold nicht verifiziert (FAIL CLOSED)" echo " [guard J] historical-db healthy:" ensure_db_run && echo " OK" echo " [guard K] kein Runner / kein Backfill aktiv:" docker ps --format '{{.Names}}' | grep -qiE 'runner|backfill' && die "Guard K FAIL: Runner/Backfill aktiv (FAIL CLOSED)" || echo " OK" echo " [guard L] Production Gates geschlossen:" echo " OK (isolierter POC-Stack, kein Produktiv-Trading-Zugriff)" } # Promotion-Session-Backup (Punkt 6). Root-owned, nicht world-readable. promote_backup() { # $1 = Session-Timestamp local ts="$1" bdir bfile bdir="$PROMOTE_BACKUP_ROOT/$ts" mkdir -p "$bdir" || die "Backup-Verzeichnis nicht erstellbar (FAIL CLOSED)" chmod 700 "$bdir" bfile="$bdir/run_backfill_year.py" cp -a "$PATCH_TARGET" "$bfile" || die "Backup fehlgeschlagen (FAIL CLOSED)" chown root:root "$bfile"; chmod 600 "$bfile" echo "$bfile" } # Promotion-Session-Metadaten (Punkt 6/8). Root-owned. promote_session_meta() { # $1 = Session-Timestamp, $2 = Backup-Datei, $3 = TARGET_SHA_BEFORE local ts="$1" bfile="$2" before="$3" mfile mfile="$PROMOTE_BACKUP_ROOT/$ts/session.meta" { echo "SESSION_TS=$ts" echo "PATCH_SOURCE=$PATCH_SOURCE_CONTAINER:$PATCH_SOURCE_PATH" echo "PATCH_TARGET=$PATCH_TARGET" echo "EXPECTED_RUNNER_PATCH_SHA256=$EXPECTED_RUNNER_PATCH_SHA256" echo "TARGET_SHA_BEFORE=$before" echo "BACKUP_SHA=$(sha256sum "$bfile" | awk '{print $1}')" echo "TARGET_OWNER=$(stat -c '%U' "$PATCH_TARGET")" echo "TARGET_GROUP=$(stat -c '%G' "$PATCH_TARGET")" echo "TARGET_MODE=$(stat -c '%a' "$PATCH_TARGET")" } > "$mfile" chown root:root "$mfile"; chmod 600 "$mfile" } # ---- promote-runner-patch (MUTATION; nur Red Queen fuehrt aus) ---- promote_runner_patch() { expect_zero "$@" echo "=== PROMOTE-RUNNER-PATCH ($RUN_MODE) ===" echo " SOURCE : $PATCH_SOURCE_CONTAINER:$PATCH_SOURCE_PATH" echo " TARGET : $PATCH_TARGET" echo " EXPECTED PATCH SHA : $EXPECTED_RUNNER_PATCH_SHA256" local ts; ts="$(date +%Y%m%d_%H%M%S)" echo " Pre-Guards:" promote_pre_guards local tbefore; tbefore="$(sha256sum "$PATCH_TARGET" | awk '{print $1}')" echo " CURRENT TARGET SHA : $tbefore" echo " geplante Backup-Position : $PROMOTE_BACKUP_ROOT/$ts/run_backfill_year.py" echo " geplante eine Datei-Mutation : $PATCH_TARGET" if [ "$RUN_MODE" = "DRY_RUN" ]; then echo "DRY_RUN_PASS: wuerde Patch promoten (Backup + atomarer rename). KEINE Aktion." exit 0 fi mode_or_die # Backup + Session-Metadaten local bfile; bfile="$(promote_backup "$ts")" promote_session_meta "$ts" "$bfile" "$tbefore" # Patch in Temp-Datei im Zielverzeichnis holen + SHA pruefen local tmpf; tmpf="$PATCH_TARGET.tmp.$$" fetch_patch "$tmpf" || { rm -f "$tmpf"; die "Promotion: Patch nicht lesbar (FAIL CLOSED)"; } local ps; ps="$(sha256sum "$tmpf" | awk '{print $1}')" [ "$ps" = "$EXPECTED_RUNNER_PATCH_SHA256" ] || { rm -f "$tmpf"; die "Promotion: Temp-SHA $ps != erwartet (FAIL CLOSED)"; } # Besitzer/Mode exakt wie bestehendes Target herstellen chown "$(stat -c '%U' "$PATCH_TARGET"):$(stat -c '%G' "$PATCH_TARGET")" "$tmpf" chmod "$(stat -c '%a' "$PATCH_TARGET")" "$tmpf" # atomarer rename mv -f "$tmpf" "$PATCH_TARGET" || { rm -f "$tmpf"; die "Promotion: atomarer rename fehlgeschlagen (FAIL CLOSED)"; } echo " Promotion ausgeführt (Backup=$bfile)." echo " Post-Promotion Verify:" local tafter; tafter="$(sha256sum "$PATCH_TARGET" | awk '{print $1}')" echo " TARGET_SHA_AFTER : $tafter" [ "$tafter" = "$EXPECTED_RUNNER_PATCH_SHA256" ] || die "Post-Promotion FAIL: Target-SHA != erwartet (FAIL CLOSED)" echo " py_compile gegen persistenten Host-SoT Runner:" python3 -m py_compile "$PATCH_TARGET" 2>&1 && echo " OK" || die "Post-Promotion FAIL: py_compile (FAIL CLOSED)" echo "PROMOTE-RUNNER-PATCH OK. Host-SoT Runner ist jetzt der gepruefte Patch." } # ---- rollback-runner-patch (MUTATION; nur Red Queen fuehrt aus) ---- rollback_runner_patch() { expect_zero "$@" echo "=== ROLLBACK-RUNNER-PATCH ($RUN_MODE) ===" echo " TARGET : $PATCH_TARGET" echo " Rollback-Sicherheit (Punkt 11):" # gueltige Promotion-Session vorhanden? local latest; latest="$(ls -1 "$PROMOTE_BACKUP_ROOT" 2>/dev/null | sort | tail -1 || true)" [ -n "$latest" ] || die "Rollback FAIL: keine gueltige Promotion-Session (FAIL CLOSED)" local mfile bfile mfile="$PROMOTE_BACKUP_ROOT/$latest/session.meta" bfile="$PROMOTE_BACKUP_ROOT/$latest/run_backfill_year.py" [ -f "$mfile" ] || die "Rollback FAIL: Session-Metadaten fehlen (FAIL CLOSED)" [ -f "$bfile" ] || die "Rollback FAIL: Backup fehlt (FAIL CLOSED)" [ -L "$bfile" ] && die "Rollback FAIL: Backup ist Symlink (FAIL CLOSED)" local bsha; bsha="$(sha256sum "$bfile" | awk '{print $1}')" local tbefore; tbefore="$(sha256sum "$PATCH_TARGET" | awk '{print $1}')" echo " Session : $latest" echo " Backup SHA : $bsha" echo " aktueller Target-Zustand : $tbefore" echo " geplante eine Datei-Mutation : $PATCH_TARGET (zurueck auf Backup)" if [ "$RUN_MODE" = "DRY_RUN" ]; then echo "DRY_RUN_PASS: wuerde Backup $bfile auf $PATCH_TARGET zurueckspielen. KEINE Aktion." exit 0 fi mode_or_die # aktuellen Target-Zustand dokumentieren (vor Rollback) local cur; cur="$(sha256sum "$PATCH_TARGET" | awk '{print $1}')" echo " aktueller Target-SHA vor Rollback : $cur" # Backup in Temp + SHA pruefen + atomarer rename local tmpf; tmpf="$PATCH_TARGET.tmp.$$" cp -a "$bfile" "$tmpf" || die "Rollback: Backup nicht lesbar (FAIL CLOSED)" local ps; ps="$(sha256sum "$tmpf" | awk '{print $1}')" [ "$ps" = "$bsha" ] || { rm -f "$tmpf"; die "Rollback: Temp-SHA != Backup-SHA (FAIL CLOSED)"; } chown "$(stat -c '%U' "$PATCH_TARGET"):$(stat -c '%G' "$PATCH_TARGET")" "$tmpf" chmod "$(stat -c '%a' "$PATCH_TARGET")" "$tmpf" mv -f "$tmpf" "$PATCH_TARGET" || { rm -f "$tmpf"; die "Rollback: atomarer rename fehlgeschlagen (FAIL CLOSED)"; } local tafter; tafter="$(sha256sum "$PATCH_TARGET" | awk '{print $1}')" echo " TARGET_SHA_AFTER : $tafter" # TARGET_SHA_BEFORE aus Session-Metadaten local expected; expected="$(grep '^TARGET_SHA_BEFORE=' "$mfile" | cut -d= -f2)" [ "$tafter" = "$expected" ] || die "Rollback FAIL: Target-SHA != TARGET_SHA_BEFORE (FAIL CLOSED)" echo "ROLLBACK-RUNNER-PATCH OK. Target zurueck auf TARGET_SHA_BEFORE=$expected." } # ---- REMEDIATION BUILD/DEPLOY CONTROL-PLANE (Phase 13.5) ---- # KEIN echter Build/Recreate durch Rain. Red Queen fuehrt nach Freigabe aus. # Helfer: Read-only-Session-Bestand ("Build-Session vorhanden?") — muss zusaetzlich # FULL_IMAGE_ID tragen, da Verify/Recreate nur der gespeicherten Image-ID vertrauen. remediation_session() { # 0 wenn giltige Build-Session mit FULL_IMAGE_ID vorhanden [ -f "$REMEDIATION_SESSION_META" ] || return 1 grep -q '^FULL_IMAGE_ID=sha256:[a-f0-9]\{64\}$' "$REMEDIATION_SESSION_META" || return 1 [ "$(stat -c '%F' "$REMEDIATION_SESSION_META")" = "regular file" ] || return 1 [ -L "$REMEDIATION_SESSION_META" ] && return 1 return 0 } remediation_full_image_id() { # FULL_IMAGE_ID aus Session (echo) — nur wenn rem_remediation_session OK grep '^FULL_IMAGE_ID=' "$REMEDIATION_SESSION_META" | cut -d= -f2 } # Pre-Build-Guards (Punkt 12: dry-run-Gates A–O). Read-only; alle FAIL CLOSED. remediation_guards() { echo " [guard A] Host-SoT vorhanden:" [ -f "/opt/historical-v2/app/run_backfill_year.py" ] || die "Guard A FAIL: Host-Runner fehlt (FAIL CLOSED)" echo " OK" echo " [guard B] Runner-SHA persistent == erwartet:" local hs; hs="$(sha256sum "/opt/historical-v2/app/run_backfill_year.py" | awk '{print $1}')" [ "$hs" = "$RUNNER_REMEDIATION_SHA" ] || die "Guard B FAIL: Host-Runner-SHA $hs != $RUNNER_REMEDIATION_SHA (FAIL CLOSED)" echo " OK ($hs)" echo " [guard C] Build-Input korrekt (Context/Dockerfile/.dockerignore):" [ -d "$REMEDIATION_BUILD_CONTEXT" ] || die "Guard C FAIL: Build-Context fehlt (FAIL CLOSED)" [ -f "$REMEDIATION_BUILD_DOCKERFILE" ] || die "Guard C FAIL: Dockerfile fehlt (FAIL CLOSED)" exclusion_valid || die "Guard C FAIL: .dockerignore nicht exakt (FAIL CLOSED)" echo " OK" echo " [guard D] nur erwartete Gold-Divergenz (app/run_backfill_year.py):" verify_remediation_build_input || die "Guard D FAIL: unerwartete Build-Input-Divergenz (FAIL CLOSED)" echo " OK (nur Runner-Divergenz autorisiert)" echo " [guard E] valider Full-SoT-Backup vorhanden (Partial ausgeschlossen):" full_sot_backup >/dev/null || die "Guard E FAIL: kein gueltiger FULL SOT BACKUP (FAIL CLOSED)" echo " OK ($(full_sot_backup))" echo " [guard F] Recovery-Gold inspectable:" docker image inspect "$RECOVERY_GOLD_IMAGE_ID" >/dev/null 2>&1 || die "Guard F FAIL: Recovery-Image nicht inspectable (FAIL CLOSED)" echo " OK" echo " [guard G] Recovery-Tag -> Recovery-ID:" local rt; rt="$(docker image inspect -f '{{.Id}}' "$RECOVERY_GOLD_TAG_FINAL" 2>/dev/null || true)" [ "$rt" = "$RECOVERY_GOLD_IMAGE_ID" ] || die "Guard G FAIL: Recovery-Tag != Recovery-ID (FAIL CLOSED)" echo " OK" echo " [guard H] verify-recovery-gold-image PASS:" ( verify_recovery_gold_image >/dev/null 2>&1 ) || die "Guard H FAIL: Recovery-Gold nicht verifiziert (FAIL CLOSED)" echo " OK" echo " [guard I] No-op-Candidate unveraendert:" local ni; ni="$(docker image inspect -f '{{.Id}}' "$CANDIDATE_TAG" 2>/dev/null || true)" [ "$ni" = "$CANDIDATE_IMAGE_ID" ] && [ "$CANDIDATE_IMAGE_ID" = "$REM_NOOP_IMAGE_ID" ] || die "Guard I FAIL: No-op-Candidate veraendert (FAIL CLOSED)" echo " OK ($ni)" echo " [guard J] Remediation-Tag existiert NOCH NICHT (KEIN Overwrite):" if docker image inspect "$REMEDIATION_TAG" >/dev/null 2>&1; then die "Guard J FAIL: Remediation-Tag existiert bereits — KEIN Ueberschreiben, kein force-retag (FAIL CLOSED)" fi echo " OK (Tag frei)" echo " [guard K] historical-db healthy:" ensure_db_run || die "Guard K FAIL: historical-db nicht Running (FAIL CLOSED)" echo " OK" echo " [guard L] kein Runner/Backfill laeuft:" docker ps --format '{{.Names}}' | grep -qiE 'runner|backfill' && die "Guard L FAIL: Runner/Backfill laeuft (FAIL CLOSED)" || echo " OK" echo " [guard M] Production Safety geschlossen (isolierter POC-Stack):" echo " OK (eigene db, eigenes Netz, GET-only API, kein Produktivnetz)" echo " [guard N] Dockerfile/Context exakt erwartet:" dockerfile_expected && echo " OK" echo " [guard O] kein freier Parameter/User-Input:" echo " OK (keine Argumente akzeptiert)" } # ---- GEMEINSAME AUTHORIZED-GOLD-DELTA-POLICY (Phase 13.5) ---- # Einzige autorisierte Divergenz gegen GOLD_MANIFEST: # app/run_backfill_year.py ALT=RUNNER_GOLD_SHA -> NEU=RUNNER_REMEDIATION_SHA # (exakte Pfad- UND SHA-Bindung; KEINE generische "1 Divergenz ok"-Lockerung). # Setzt globale Zaehler AD_MATCH/AD_AUTH/AD_UNAUTH/AD_MISSING/AD_EXTRA und # AD_RET (ok|fail). fail wenn UNAUTH>0 || MISSING>0 || EXTRA>0. # Wird von verify_remediation_build_input UND verify_runner_remediation_image # verwendet -> EINE Policy, keine unterschiedlichen Policies. authorized_gold_delta() { local new="$1" gold goldrel goldsha nsha nrel found AD_MATCH=0; AD_AUTH=0; AD_UNAUTH=0; AD_MISSING=0; AD_EXTRA=0 while read -r goldsha goldrel; do goldrel="${goldrel#./}"; [ -n "$goldrel" ] || continue nsha="$(awk -v r="$goldrel" '$2==r{print $1}' "$new")" if [ -z "$nsha" ]; then AD_MISSING=$((AD_MISSING+1)) elif [ "$nsha" = "$goldsha" ]; then AD_MATCH=$((AD_MATCH+1)) else if [ "$goldrel" = "app/run_backfill_year.py" ] \ && [ "$goldsha" = "$RUNNER_GOLD_SHA" ] \ && [ "$nsha" = "$RUNNER_REMEDIATION_SHA" ]; then AD_AUTH=$((AD_AUTH+1)) else AD_UNAUTH=$((AD_UNAUTH+1)) fi fi done < "$GOLD_MANIFEST" while read -r nsha nrel; do nrel="${nrel#./}" found="$(awk -v r="$nrel" '$2==r{print $1}' "$GOLD_MANIFEST")" if [ -z "$found" ]; then case "$nrel" in app/app/backfill.py|app/app/test_idempotency.py|app/eligibility.py) ;; *) AD_EXTRA=$((AD_EXTRA+1)) ;; esac fi done < "$new" AD_RET="ok" [ $AD_UNAUTH -eq 0 ] && [ $AD_MISSING -eq 0 ] && [ $AD_EXTRA -eq 0 ] || AD_RET="fail" } # Read-only Build-Input-Check: nur app/run_backfill_year.py darf vom Gold-Manifest # divergieren (und zwar exakt auf RUNNER_REMEDIATION_SHA). alles andere -> FAIL. verify_remediation_build_input() { local hm hm="$(mktemp)"; gen_host_manifest > "$hm" authorized_gold_delta "$hm" rm -f "$hm" echo " Gold $((AD_MATCH+AD_AUTH+AD_UNAUTH+AD_MISSING)): MATCH=$AD_MATCH AUTHORIZED_DIVERGED=$AD_AUTH UNAUTHORIZED_DIVERGED=$AD_UNAUTH MISSING=$AD_MISSING ; unerlaubte EXTRA=$AD_EXTRA" [ "$AD_AUTH" = "1" ] || die "BUILD-INPUT FAIL: autorisierte Runner-Divergenz fehlt (FAIL CLOSED)" [ "$AD_RET" = "ok" ] || die "BUILD-INPUT FAIL: unerwartete Divergenz (FAIL CLOSED)" return 0 } # ---- build-runner-remediation (DRY_RUN = kein Build; KEIN Overwrite) ---- build_runner_remediation() { expect_zero "$@" echo "=== BUILD-RUNNER-REMEDIATION ($RUN_MODE) ===" echo " Build Context : $REMEDIATION_BUILD_CONTEXT (fix)" echo " Dockerfile : $REMEDIATION_BUILD_DOCKERFILE (fix)" echo " Ziel-Tag : $REMEDIATION_TAG (fix, KEIN latest, KEIN Overwrite)" echo " Guards:" remediation_guards echo " [build-input] REMEDIATION_BUILD_INPUT (Build-Context-Manifest):" build_context_manifest | while IFS= read -r line; do echo " $line"; done if [ "$RUN_MODE" = "DRY_RUN" ]; then echo "DRY_RUN_PASS: build-runner-remediation wuerde docker build -f $REMEDIATION_BUILD_DOCKERFILE -t $REMEDIATION_TAG $REMEDIATION_BUILD_CONTEXT (keine Aktion)." exit 0 fi mode_or_die echo " [build] docker build -f $REMEDIATION_BUILD_DOCKERFILE -t $REMEDIATION_TAG $REMEDIATION_BUILD_CONTEXT" docker build -f "$REMEDIATION_BUILD_DOCKERFILE" -t "$REMEDIATION_TAG" "$REMEDIATION_BUILD_CONTEXT" || die "build-runner-remediation fehlgeschlagen (FAIL CLOSED)" local newid now newid="$(docker image inspect -f '{{.Id}}' "$REMEDIATION_TAG" 2>/dev/null)" [ -n "$newid" ] || die "build: Image-ID nicht ermittelbar (FAIL CLOSED)" echo " FULL_IMAGE_ID=$newid" now="$(date -u +%Y-%m-%dT%H:%M:%SZ)" # Build-Session root-owned anlegen (einmaliges Schreiben der Image-ID) mkdir -p "$REMEDIATION_SESSION_DIR" || die "build: Session-Dir nicht anlegbar (FAIL CLOSED)" cat > "$REMEDIATION_SESSION_META" </dev/null 2>&1 || die "verify: Image-ID nicht inspectable (FAIL CLOSED)" echo " [Binding] Remediation-Tag -> FULL Image-ID:" local rt; rt="$(docker image inspect -f '{{.Id}}' "$REMEDIATION_TAG" 2>/dev/null || true)" [ "$rt" = "$imgid" ] || die "verify: Remediation-Tag ($rt) != gespeicherte Image-ID ($imgid) (FAIL CLOSED)" echo " OK" echo " [Host-SoT App-Manifest vs Image] (Candidate == Host-SoT):" local tmpc tmpdir tmpf tmpc="rq-rem-verify-$(date +%s)"; tmpdir="/tmp/rq-rem-copy-$$" rm -rf "$tmpdir"; mkdir -p "$tmpdir" docker create --name "$tmpc" --entrypoint /bin/true "$imgid" >/dev/null 2>&1 || die "verify: Temp-Container Fehler (FAIL CLOSED)" docker cp "$tmpc:/app/app" "$tmpdir/appapp" >/dev/null 2>&1 || { docker rm -f "$tmpc" >/dev/null 2>&1; rm -rf "$tmpdir"; die "verify: /app/app nicht lesbar (FAIL CLOSED)"; } docker rm -f "$tmpc" >/dev/null 2>&1 || true tmpf="$(mktemp)" ( cd "$tmpdir/appapp" && find . -type f -name '*.py' ! -path '*/__pycache__/*' -printf '%P\n' 2>/dev/null | sort \ | while IFS= read -r rel; do [ -f "$rel" ] && sha256sum "$rel"; done | awk '{print $1" app/"$2}' ) > "$tmpf" # [Gold-Delta] Image vs GOLD_MANIFEST: exakt 1 autorisierte Runner-Divergenz erlaubt. authorized_gold_delta "$tmpf" echo " Image vs GOLD: MATCH=$AD_MATCH AUTHORIZED_DIVERGED=$AD_AUTH UNAUTHORIZED_DIVERGED=$AD_UNAUTH MISSING=$AD_MISSING unerlaubte EXTRA=$AD_EXTRA" [ "$AD_AUTH" = "1" ] || { rm -rf "$tmpdir"; die "verify: Gold-Delta != exakt 1 autorisierte Runner-Divergenz (FAIL CLOSED)"; } [ "$AD_RET" = "ok" ] || { rm -rf "$tmpdir"; die "verify: Image != Gold-Manifest (unzulaessige Divergenz) (FAIL CLOSED)"; } # [Candidate vs Host-SoT] Image-App-Manifest == Host-SoT-App-Manifest (direkter # Manifest-zu-Manifest-Vergleich, OHNE Gold-Bezug; vollstaendig MATCH). # Host-only-Legacy-Dateien (im Image via .dockerignore ausgeschlossen) sind erlaubt. local hmf cmiss cextra hmf="$(mktemp)"; gen_host_manifest > "$hmf" cmiss=0; cextra=0 while read -r hsha hrel; do hrel="${hrel#./}"; [ -n "$hrel" ] || continue case "$hrel" in app/app/backfill.py|app/app/test_idempotency.py|app/eligibility.py) continue ;; esac isha="$(awk -v r="$hrel" '$2==r{print $1}' "$tmpf")" if [ -z "$isha" ]; then cmiss=$((cmiss+1)) elif [ "$isha" != "$hsha" ]; then cextra=$((cextra+1)); fi done < "$hmf" rm -f "$hmf" echo " Candidate vs Host-SoT: MATCH (direkt) MISSING=$cmiss DIVERGED=$cextra" [ "$cmiss" = "0" ] && [ "$cextra" = "0" ] || { rm -rf "$tmpdir"; die "verify: Candidate != Host-SoT (FAIL CLOSED)"; } rm -f "$tmpf" echo " [Critical-Files] $REMEDIATION_TAG vs gold:" verify_critical_files "$tmpdir/appapp" runner rm -rf "$tmpdir" echo " [Runner] Image vs Host-SoT persistent:" local ir; ir="$(docker run --rm --entrypoint /bin/sh "$imgid" -c "sha256sum /app/app/run_backfill_year.py 2>/dev/null" | awk '{print $1}')" || ir="" local hr; hr="$(sha256sum /opt/historical-v2/app/run_backfill_year.py | awk '{print $1}')" [ "$ir" = "$hr" ] && [ "$ir" = "$RUNNER_REMEDIATION_SHA" ] || die "verify: Runner-Image-SHA ($ir) != Host-SoT ($hr) (FAIL CLOSED)" echo " Image=$ir Host=$hr (erwartet $RUNNER_REMEDIATION_SHA)" echo " [Expected Gold Delta] GENAU EINE fachliche Divergenz = app/run_backfill_year.py:" echo " ALT(Gold)=$RUNNER_GOLD_SHA NEU(Remediation)=$RUNNER_REMEDIATION_SHA" echo " [Runtime Config] $REMEDIATION_TAG:" local c_entry c_cmd c_wd c_user c_expose c_entry="$(docker image inspect -f '{{json .Config.Entrypoint}}' "$imgid" 2>/dev/null)" c_cmd="$(docker image inspect -f '{{json .Config.Cmd}}' "$imgid" 2>/dev/null)" c_wd="$(docker image inspect -f '{{.Config.WorkingDir}}' "$imgid" 2>/dev/null)" c_user="$(docker image inspect -f '{{.Config.User}}' "$imgid" 2>/dev/null)" c_expose="$(docker image inspect -f '{{json .Config.ExposedPorts}}' "$imgid" 2>/dev/null)" printf ' Entrypoint=%s Cmd=%s WorkDir=%s User=%s Expose=%s\n' "${c_entry:-null}" "${c_cmd:-null}" "${c_wd:-}" "${c_user:-}" "${c_expose:-}" [ "$c_wd" = "/app" ] || die "verify: WorkingDir unerwartet (FAIL CLOSED)" echo " [Build-Input] Session-Host-Runner == persistent Host-Runner:" local srunner; srunner="$(grep '^HOST_SOT_RUNNER_SHA=' "$REMEDIATION_SESSION_META" | cut -d= -f2)" [ "$srunner" = "$RUNNER_REMEDIATION_SHA" ] || die "verify: Session-Runner != erwartet (FAIL CLOSED)" echo " OK ($srunner)" echo "VERDICT: REMEDIATION IMAGE OK (Candidate==Host-SoT; Gold-Delta exakt 1 autorisierte Runner-Divergenz; Critical-SHAs + Config PASS)" exit 0 } # ---- recreate-runner-remediation-safe (NUR historical-service; nur Red Queen) ---- recreate_runner_remediation_safe() { expect_zero "$@" echo "=== RECREATE-RUNNER-REMEDIATION-SAFE ($RUN_MODE) ===" echo " Ziel-Service : $HISTORICAL_SERVICE (NUR dieser; KEIN db, KEIN Compose-Down, KEINE Volumes)" remediation_session || die "recreate: keine gueltige Build-Session (zuerst build-runner-remediation) (FAIL CLOSED)" local imgid; imgid="$(remediation_full_image_id)" echo " Image : $REMEDIATION_TAG -> $imgid (explizit, KEIN latest)" echo " Pre-Gates:" recreate_pre_gates echo " [gate N] Remediation-Tag -> FULL Image-ID:" local rt; rt="$(docker image inspect -f '{{.Id}}' "$REMEDIATION_TAG" 2>/dev/null || true)" [ "$rt" = "$imgid" ] || die "Gate N FAIL: Remediation-Tag != FULL Image-ID (FAIL CLOSED)" echo " OK" echo " [gate O] verify-runner-remediation-image PASS:" ( verify_runner_remediation_image >/dev/null 2>&1 ) || die "Gate O FAIL: Remediation-Image nicht verifiziert (FAIL CLOSED)" echo " OK" echo " Baseline vor Recreate:" recreate_baseline if [ "$RUN_MODE" = "DRY_RUN" ]; then echo "DRY_RUN_PASS: wuerde docker compose -f $COMPOSE_FILE -f up -d --no-deps --no-build $HISTORICAL_SERVICE (Image=$REMEDIATION_TAG, gespeicherte ID=$imgid). KEINE Aktion." exit 0 fi mode_or_die local ovf; ovf="$(write_recreate_override "$REMEDIATION_TAG")" docker compose -f "$COMPOSE_FILE" -f "$ovf" up -d --no-deps --no-build "$HISTORICAL_SERVICE" || { rm -f "$ovf"; die "recreate-runner-remediation-safe fehlgeschlagen (FAIL CLOSED)"; } rm -f "$ovf" echo " Post-Gates:" recreate_post_gates "$imgid" echo "RECREATE-RUNNER-REMEDIATION-SAFE OK. Container laeuft mit Remediation-Image (gespeicherte ID)." } # ---- build-historical (NUR Build, kein recreate; DRY_RUN = kein Bau) ---- build_historical() { expect_zero "$@" echo "=== BUILD-HISTORICAL ($RUN_MODE) ===" echo " Projekt : $COMPOSE_PROJECT" echo " Compose : $COMPOSE_FILE" echo " Service : $HISTORICAL_SERVICE" echo " Kontext : $COMPOSE_PROJECT (Dockerfile: $COMPOSE_PROJECT/Dockerfile)" # 1. Dry-run zeigt Guards geplant, aber fuehrt sie (bis auf reine Checks) nicht aus sot_bestandscheck >/dev/null echo " [guard] backup-sot vorhanden: OK" # verify-recovery check ist read-only, kann im dry-run laufen echo " [guard] verify-sot-recovery:" verify_recovery_ok echo " [guard] Gold-Image $GOLD_IMAGE_ID: $(ensure_gold_image && echo OK)" echo " [guard] Gold-Container Running: $(ensure_gold_run && echo OK)" echo " [guard] historical-db Running: $(ensure_db_run && echo OK)" echo " [guard] DB-Volume: $(ensure_db_volume && echo OK)" echo " [guard] Builddefinition (Dockerfile+Compose): $(ensure_builddef && echo OK)" echo " [guard] Build-Pollution-Exclusion valid:" exclusion_valid && echo " .dockerignore OK (nur app/app/ + app/eligibility.py)" if [ "$RUN_MODE" = "DRY_RUN" ]; then echo "DRY_RUN_PASS: build-historical wuerde aus $COMPOSE_PROJECT bauen (keine Aktion)." exit 0 fi mode_or_die echo " [build] docker compose -f $COMPOSE_FILE build $HISTORICAL_SERVICE" docker compose -f "$COMPOSE_FILE" build "$HISTORICAL_SERVICE" || die "build fehlgeschlagen" echo " [result]" docker images --filter reference="${HISTORICAL_SERVICE}" --format 'Image {{.ID}} Created {{.CreatedSince}}' echo "BUILD-HISTORICAL OK. Gold-Image NICHT geloescht ($GOLD_IMAGE_ID). Nur Build, kein Recreate." } # ---- recreate-historical (NUR historical-service, kein Runner; DRY_RUN = kein Recreate) ---- recreate_historical() { expect_zero "$@" echo "=== RECREATE-HISTORICAL ($RUN_MODE) ===" echo " Service : $HISTORICAL_SERVICE (NUR dieser; NICHT historical-db)" sot_bestandscheck >/dev/null echo " [guard] backup-sot vorhanden: OK" echo " [guard] Gold-Image $GOLD_IMAGE_ID: $(ensure_gold_image && echo OK)" echo " [guard] Gold-Manifest: $([ -f "$GOLD_MANIFEST" ] && echo OK || echo FEHLT)" echo " [guard] historical-db Running: $(ensure_db_run && echo OK)" echo " [guard] DB-Volume: $(ensure_db_volume && echo OK)" if [ "$RUN_MODE" = "DRY_RUN" ]; then echo "DRY_RUN_PASS: recreate-historical wuerde historical-service neu erstellen (keine Aktion)." exit 0 fi mode_or_die echo " [recreate] docker compose -f $COMPOSE_FILE up -d --no-deps $HISTORICAL_SERVICE" docker compose -f "$COMPOSE_FILE" up -d --no-deps "$HISTORICAL_SERVICE" || die "recreate fehlgeschlagen" sleep 4 local running running="$(docker inspect -f '{{.State.Running}}' "$HISTORICAL_SERVICE" 2>/dev/null)" [ "$running" = "true" ] || die "neuer Container laeuft nicht (FAIL CLOSED)" echo " [Container] ID: $(docker inspect -f '{{.ID}}' "$HISTORICAL_SERVICE")" echo " [Image] $(docker inspect -f '{{.Image}}' "$HISTORICAL_SERVICE")" echo "RECREATE-HISTORICAL OK. KEIN Runner, KEIN compose down, KEIN Volume-Remove." } # ---- verify-gold-reproduction (read-only) ---- verify_gold_reproduction() { req_ok; expect_zero "$@" echo "=== VERIFY-GOLD-REPRODUCTION (Gold-Manifest vs Running-Container) ===" [ -f "$GOLD_MANIFEST" ] || die "Gold-Manifest fehlt (FAIL CLOSED)" local tmp; tmp="$(mktemp)" gen_manifest "$HISTORICAL_SERVICE" > "$tmp" classify_and_report "$tmp" local ret="$G_RET" rm -f "$tmp" if [ "$ret" = "ok" ]; then echo "VERDICT: GOLD REPRODUCTION OK (0 DIVERGED/MISSING/unerlaubte EXTRA)" exit 0 else echo "VERDICT: GOLD REPRODUCTION FAIL" exit 2 fi } validate_stage() { local key="$1" file real owner dmode file="$(stage_file "$key")" [ -e "$file" ] || die "Staging-Kandidat fehlt: $file (lege zuerst die Datei ab)" real="$(realpath -m "$file")" [[ "$real" == "$STAGING_ROOT/"* ]] || die "Traversal blockiert: realpath '$real' ausserhalb Staging" [ "$real" = "$file" ] || die "Alias erkannt (realpath != Pfad): $file" [ -f "$file" ] || die "keine reguläre Datei: $file" [ -L "$file" ] && die "Symlink nicht erlaubt: $file" owner="$(stat -c '%U' "$file")" [ "$owner" = "rq-host" ] || die "Staging-Kandidat Owner = $owner (erwartet rq-host)" dmode="$(stat -c '%a' "$STAGING_ROOT")" [[ "$dmode" != *002 ]] && [[ "$dmode" != *022 ]] || die "Staging-Root global schreibbar '$dmode' (FAIL CLOSED)" echo "OK: $key -> $file (regular, no symlink, owner rq-host, in staging)" } # ---- redact: maskiert Passwort-/Secret-Werte (fuer build-def Anzeige) ---- redact_line() { # Zeilen mit "PASSWORD=" / "SECRET=" / "TOKEN=" : Wert maskieren echo "$1" | sed -E 's/(PASSWORD|PASSWORDS|SECRET|SECRETS|TOKEN|TOKENS|API_KEY|CLIENT_SECRET|PRIVATE_KEY)(["'"'"']?[[:space:]]*=[[:space:]]*["'"'"']?)[^[:space:]"'"'"']+/\\1\\2***REDACTED***/Ig' } # ============================================================================= # IMAGE-RETENTION / ARCHIVE CONTROL PLANE (Phase 13.5 Schutzschicht) # Zwei unabhängige Ebenen je Critical Image: # A) Stopped Pin Container (docker create, nie gestartet) -> Image-Referenz # B) sha256-verifiziertes docker save-Archiv -> externes Archiv # READY-Gate: IMAGE VERIFY + PIN VERIFY + ARCHIVE VERIFY alle PASS. # KEIN Overwrite. KEIN Coolify-Cleanup-Disable/Managed-Spoof. Kein echter # Rebuild/Restore/Recreate. Kein freies Input (feste Rollen: recovery / remediation). # ============================================================================= role_ok() { # $1 = role -> 0 wenn gueltig case "$1" in recovery|remediation) return 0 ;; *) return 1 ;; esac } # Feste Rollen->Image/Identity-Bindung (Punkt 2/4/8/9/10) — hardcoded. role_image_id() { # $1 = role -> echo erwartete FULL_IMAGE_ID case "$1" in recovery) echo "$RECOVERY_GOLD_IMAGE_ID" ;; # sha256:d2db926d... remediation) echo "$REMEDIATION_TAG" ;; # Tag -> aufgeloest via inspect (Image fehlt aktuell) esac } role_tag() { case "$1" in recovery) echo "$RECOVERY_GOLD_TAG_FINAL" ;; remediation) echo "$REMEDIATION_TAG" ;; esac } role_pin() { case "$1" in recovery) echo "$PIN_RECOVERY_NAME" ;; remediation) echo "$PIN_REMEDIATION_NAME" ;; esac } role_arch_sub() { echo "${ARCHIVE_SUB[$1]}" ; } role_arch_dir() { echo "$ARCHIVE_ROOT/$(role_arch_sub "$1")" ; } # --- Schicht A: Stopped Pin Container --- # Container-Referenz schuetzt Image-Objekt vor Coolify rmi (empirisch belegt: # No-op ueberlebte, weil Container 2fe3d30c... es referenzierte). pin_container_identity_ok() { # $1 = role, $2 = actual container id -> 0 wenn ok local role="$1" actual="$2" expected case "$role" in recovery) expected="$RECOVERY_GOLD_IMAGE_ID" ;; remediation) expected="$PIN_REMEDIATION_TARGET_ID" ;; esac [ "$actual" = "$expected" ] } # --- Schicht B: Archive Verify --- verify_archive() { # $1 = role (read-only) local role="$1" adir sid adir="$(role_arch_dir "$role")" case "$role" in recovery) sid="$RECOVERY_ARCHIVE_ID" ;; remediation) sid="$REMEDIATION_ARCHIVE_ID" ;; esac # Archive existiert [ -d "$adir" ] || die "verify-archive($role): kein Archiv-Root (FAIL CLOSED)" local base="$adir/$sid" [ -d "$base" ] || die "verify-archive($role): kein Session-Ordner $base (FAIL CLOSED)" # Pflichtdateien local f img meta manifest runtime img="$base/image.tar"; meta="$base/session.meta"; manifest="$base/app.manifest"; runtime="$base/runtime-config.txt" [ -f "$img" ] && [ ! -L "$img" ] || die "verify-archive($role): image.tar fehlt/kein File (FAIL CLOSED)" [ -s "$img" ] || die "verify-archive($role): image.tar leer (FAIL CLOSED)" for f in "$meta" "$manifest" "$runtime"; do [ -f "$f" ] && [ ! -L "$f" ] || die "verify-archive($role): fehlt $(basename "$f") (FAIL CLOSED)" done [ "$(stat -c '%U' "$img")" = "root" ] || die "verify-archive($role): image.tar Owner != root (FAIL CLOSED)" case "$(stat -c '%a' "$img")" in 600|640|700|400) ;; *) die "verify-archive($role): image.tar Mode unsicher ($(stat -c '%a' "$img")) (FAIL CLOSED)" ;; esac # Sha256 stimmt (Punkt 7) local want got want="$(awk -F'=' '$1=="ARCHIVE_SHA256"{print $2}' "$meta" 2>/dev/null)" [ -n "$want" ] && [[ "$want" =~ ^[a-f0-9]{64}$ ]] || die "verify-archive($role): ARCHIVE_SHA256 fehlt/ungueltig in session.meta (FAIL CLOSED)" got="$(sha256sum "$img" | awk '{print $1}')" [ "$got" = "$want" ] || die "verify-archive($role): image.tar SHA mismatch (FAIL CLOSED)" # session.meta Vollstaendigkeit (Punkt 5) local key for key in ROLE TAG FULL_IMAGE_ID ARCHIVE_SHA256 ARCHIVE_SIZE CREATED_AT HOST_SOT_RUNNER_SHA BUILD_INPUT_SHA WRAPPER_SHA APP_MANIFEST_SHA RUNTIME_CONFIG_SHA; do grep -q "^$key=.*" "$meta" || die "verify-archive($role): session.meta fehlt $key (FAIL CLOSED)" done local mrole mrole="$(awk -F'=' '$1=="ROLE"{print $2}' "$meta" 2>/dev/null)" [ "$mrole" = "$role" ] || die "verify-archive($role): ROLE in session.meta = '$mrole' (FAIL CLOSED)" echo "VERIFY-ARCHIVE($role): PASS ($base)" } # --- Ready-Gate (Punkt 14) --- verify_ready() { # $1 = role local role="$1" label case "$role" in recovery) label="RECOVERY READY" ;; remediation) label="CANDIDATE READY" ;; esac echo "=== $label (READY-Gate: IMAGE VERIFY + PIN VERIFY + ARCHIVE VERIFY) ===" # IMAGE VERIFY (zwingend inspectable + korrekt) echo " [IMAGE] $(role_tag "$role"):" verify_image_present "$role" || die "READY-Gate FAIL: Image nicht inspectable (FAIL CLOSED)" echo " OK" echo " [PIN] $(role_pin "$role"):" verify_pin "$role" || die "READY-Gate FAIL: Pin nicht PASS (FAIL CLOSED)" echo " OK" echo " [ARCHIVE] $(role_arch_dir "$role"):" verify_archive "$role" >/dev/null || die "READY-Gate FAIL: Archive nicht PASS (FAIL CLOSED)" echo " OK" echo "$label: PASS" } # --- Image-Present-Verify (Schicht-Pruefung: Image inspectable + Identity) --- verify_image_present() { # $1 = role (read-only) -> 0 wenn Image vorhanden+bindung ok local role="$1" id tag actualid tag="$(role_tag "$role")" case "$role" in recovery) id="$RECOVERY_GOLD_IMAGE_ID" # Tag muss exakt auf die immutable Recovery-ID zeigen (kein Overwrite-Erwartung) actualid="$(docker image inspect -f '{{.Id}}' "$tag" 2>/dev/null || true)" [ "$actualid" = "$id" ] || return 1 ;; remediation) # Candidate: FULL_IMAGE_ID aus Build-Session; Tag -> ID Bindung if ! remediation_session; then die "verify-image(remediation): keine gueltige Build-Session (FAIL CLOSED)"; fi id="$(remediation_full_image_id)" actualid="$(docker image inspect -f '{{.Id}}' "$id" 2>/dev/null || true)" [ -n "$actualid" ] || return 1 ;; esac return 0 } # --- Schicht A: Pin-Verify (Punkt 12) --- verify_pin() { # $1 = role (read-only) -> 0 wenn Pin PASS local role="$1" name imgid nw mounts ports rp priv capadd expected state name="$(role_pin "$role")" docker container inspect "$name" >/dev/null 2>&1 || { echo " PIN($role): Container $name fehlt"; return 1; } expected="$(role_pin_target_id "$role")" imgid="$(docker container inspect -f '{{.Image}}' "$name" 2>/dev/null)" # .Image ist die Image-ID des referenzierten Images (exakt FULL_IMAGE_ID) [ -n "$expected" ] || die "verify-pin($role): Ziel-Image-ID nicht bekannt (FAIL CLOSED)" # Image-ID kann lang (sha256:...) sein — Bindung: .Image == erwartete FULL_IMAGE_ID [ "$imgid" = "$expected" ] || { echo " PIN($role): Image-ID $imgid != erwartet $expected"; return 1; } state="$(docker container inspect -f '{{.State.Status}}' "$name" 2>/dev/null)" case "$state" in created|exited|stopped) ;; *) echo " PIN($role): State=$state (nicht created/stopped — VERBOTEN running)"; return 1 ;; esac nw="$(docker container inspect -f '{{.HostConfig.NetworkMode}}' "$name" 2>/dev/null)" [ "$nw" = "none" ] || { echo " PIN($role): NetworkMode=$nw != none"; return 1; } mounts="$(docker container inspect -f '{{json .Mounts}}' "$name" 2>/dev/null)" [ "$mounts" = "[]" ] || { echo " PIN($role): Mounts != []"; return 1; } ports="$(docker container inspect -f '{{json .HostConfig.PortBindings}}' "$name" 2>/dev/null)" [ "$ports" = "{}" ] || { echo " PIN($role): PortBindings != {}"; return 1; } rp="$(docker container inspect -f '{{.HostConfig.RestartPolicy.Name}}' "$name" 2>/dev/null)" [ "$rp" = "no" ] || { echo " PIN($role): RestartPolicy=$rp != no"; return 1; } priv="$(docker container inspect -f '{{.HostConfig.Privileged}}' "$name" 2>/dev/null)" [ "$priv" = "false" ] || { echo " PIN($role): Privileged=$priv"; return 1; } capadd="$(docker container inspect -f '{{json .HostConfig.CapAdd}}' "$name" 2>/dev/null)" [ "$capadd" = "null" ] || [ "$capadd" = "[]" ] || { echo " PIN($role): CapAdd=$capadd"; return 1; } return 0 } # --- Ziel-Image-ID fuer Pin (Schicht A bindet exakt FULL_IMAGE_ID) --- role_pin_target_id() { # $1 = role case "$1" in recovery) echo "$RECOVERY_GOLD_IMAGE_ID" ;; remediation) if remediation_session; then echo "$(remediation_full_image_id)"; else echo ""; fi ;; esac } # --- Archive-Metadaten-Generator (Punkt 5; kein Secret) --- gen_archive_meta() { # $1 = role, $2 = session dir, $3 = image id, $4 = archive size, $5 = archive sha local role="$1" dir="$2" image_id="$3" asize="$4" asha="$5" now now="$(date -u +%Y-%m-%dT%H:%M:%SZ)" printf 'ROLE=%s\nTAG=%s\nFULL_IMAGE_ID=%s\nARCHIVE_SHA256=%s\nARCHIVE_SIZE=%s\nCREATED_AT=%s\nHOST_SOT_RUNNER_SHA=%s\nBUILD_INPUT_SHA=%s\nWRAPPER_SHA=%s\nAPP_MANIFEST_SHA=%s\nRUNTIME_CONFIG_SHA=%s\n' \ "$role" "$(role_tag "$role")" "$image_id" "$asha" "$asize" "$now" \ "$HOST_SOT_RUNNER_SHA_CUR" "$BUILD_INPUT_SHA_CUR" "$WRAPPER_SHA_CUR" "$APP_MANIFEST_SHA_CUR" "$RUNTIME_CONFIG_SHA_CUR" \ > "$dir/session.meta" chown root:root "$dir/session.meta"; chmod 600 "$dir/session.meta" } # --- runtime-config Evidence (Punkt 4: runtime-config.txt) — keine Secrets --- write_runtime_config() { # $1 = role, $2 = session dir local role="$1" dir="$2" tag="$(role_tag "$role")" id id="$(docker image inspect -f '{{.Id}}' "$tag" 2>/dev/null)" { echo "RUNTIME CONFIG EVIDENCE (Phase 13.5 Archive) — keine Secrets" echo "ROLE=$role" echo "FULL_IMAGE_ID=$id" echo "Entrypoint=$(docker image inspect -f '{{json .Config.Entrypoint}}' "$tag" 2>/dev/null)" echo "Cmd=$(docker image inspect -f '{{json .Config.Cmd}}' "$tag" 2>/dev/null)" echo "WorkingDir=$(docker image inspect -f '{{.Config.WorkingDir}}' "$tag" 2>/dev/null)" echo "User=$(docker image inspect -f '{{.Config.User}}' "$tag" 2>/dev/null)" echo "ExposedPorts=$(docker image inspect -f '{{json .Config.ExposedPorts}}' "$tag" 2>/dev/null)" echo "ENV_KEYS=$(docker image inspect -f '{{range .Config.Env}}{{println .}}{{end}}' "$tag" 2>/dev/null | cut -d= -f1 | tr '\n' ',' | sed 's/,$//')" } > "$dir/runtime-config.txt" chown root:root "$dir/runtime-config.txt"; chmod 600 "$dir/runtime-config.txt" } # --- app.manifest Evidence (Punkt 4) — Sha256 aller *.py im Image-Layer --- write_app_manifest() { # $1 = role, $2 = session dir, $3 = image id/tag local role="$1" dir="$2" ref="$3" tmpc tmpd tmpc="rq-arch-manifest-$$" tmpd="/tmp/rq-arch-manifest-$$" rm -rf "$tmpd"; mkdir -p "$tmpd" docker create --name "$tmpc" --entrypoint /bin/true "$ref" >/dev/null 2>&1 \ || { rm -rf "$tmpd"; die "archive($role): Temp-Container Fehler (FAIL CLOSED)"; } if ! docker cp "$tmpc:/app/app" "$tmpd/appapp" >/dev/null 2>&1; then docker rm -f "$tmpc" >/dev/null 2>&1 || true; rm -rf "$tmpd" die "archive($role): /app/app nicht lesbar (FAIL CLOSED)" fi docker rm -f "$tmpc" >/dev/null 2>&1 || true ( cd "$tmpd/appapp" && find . -type f -name '*.py' ! -path '*/__pycache__/*' -printf '%P\n' 2>/dev/null | sort \ | while IFS= read -r rel; do [ -f "$rel" ] && sha256sum "$rel"; done | awk '{print $1" app/"$2}' ) > "$dir/app.manifest" rm -rf "$tmpd" chown root:root "$dir/app.manifest"; chmod 600 "$dir/app.manifest" } # ============================================================================= # COMMAND-FUNKTIONEN (Phase 13.5 Archive/Pin) — fail-closed, kein Overwrite, # kein echter Rebuild/Restore/Recreate in dieser Mission. # ============================================================================= # archive-recovery-image / archive-remediation-image archive_image() { # $1 = role local role="$1" expect_zero "${@:2}" echo "=== ARCHIVE-$role ($RUN_MODE) ===" local tag id subdir sdir tag="$(role_tag "$role")" case "$role" in recovery) id="$RECOVERY_GOLD_IMAGE_ID" ;; remediation) remediation_session || die "archive(remediation): keine gueltige Build-Session (FAIL CLOSED)" id="$(remediation_full_image_id)" PIN_REMEDIATION_TARGET_ID="$id" ;; esac echo " FULL_IMAGE_ID=$id" echo " Tag=$tag" # Punkt 6: Image-inspect + Tag->ID Gate VOR save echo " [inspect] Tag -> ID Gate:" local tagid tagid="$(docker image inspect -f '{{.Id}}' "$tag" 2>/dev/null || true)" if [ -z "$tagid" ]; then die "archive($role): Source Image nicht inspectable (SOURCE IMAGE NOT INSPECTABLE — FAIL CLOSED)" fi [ "$tagid" = "$id" ] || die "archive($role): Tag->ID Gate FAIL ($tagid != $id) (FAIL CLOSED)" echo " OK ($tagid)" # Punkt 7: Immutability — Archiv-Session darf NICHT existieren local sdir sdir="$(role_arch_dir "$role")" # = $ARCHIVE_ROOT/ local sid case "$role" in recovery) sid="$RECOVERY_ARCHIVE_ID" ;; remediation) sid="${id#sha256:}"; sid="${sid:0:12}" ;; esac local base="$sdir/$sid" # Fail-closed wenn bereits existiert (kein Overwrite) if [ -e "$base" ]; then die "archive($role): Session-Ordner existiert bereits $base (KEIN Overwrite, FAIL CLOSED)"; fi if [ "$RUN_MODE" = "DRY_RUN" ]; then echo "DRY_RUN_PASS: archive($role) wuerde anlegen: $base/ (image.tar + sha256 + session.meta + app.manifest + runtime-config.txt). Keine Aktion." exit 0 fi mode_or_die # MUTATION: Archive anlegen + docker save mkdir -p "$base" || die "archive($role): Session-Ordner nicht anlegbar (FAIL CLOSED)" # Session.meta ZUSAMMENBAUEN VOR save (fuer Size/SHA brauchen aber erst nach save) # Reihenfolge: save -> size/sha -> meta -> manifest -> runtime echo " [save] docker image save $id > image.tar" if ! docker image save "$id" > "$base/image.tar" 2>/dev/null; then rm -rf "$base"; die "archive($role): docker save fehlgeschlagen (FAIL CLOSED)" fi chown root:root "$base/image.tar"; chmod 600 "$base/image.tar" local asha asize asha="$(sha256sum "$base/image.tar" | awk '{print $1}')" asize="$(stat -c '%s' "$base/image.tar")" # SHAs der Metadaten (zur Laufzeit, aus Build-Session / Host-SoT) HOST_SOT_RUNNER_SHA_CUR="$(sha256sum /opt/historical-v2/app/run_backfill_year.py 2>/dev/null | awk '{print $1}')" BUILD_INPUT_SHA_CUR="$(build_context_manifest 2>/dev/null | sha256sum | awk '{print $1}')" WRAPPER_SHA_CUR="$(sha256sum /usr/local/bin/rq-historical | awk '{print $1}')" APP_MANIFEST_SHA_CUR="$(sha256sum "$base/app.manifest" 2>/dev/null | awk '{print $1}')" RUNTIME_CONFIG_SHA_CUR="$(sha256sum "$base/runtime-config.txt" 2>/dev/null | awk '{print $1}')" gen_archive_meta "$role" "$base" "$id" "$asize" "$asha" write_runtime_config "$role" "$base" write_app_manifest "$role" "$base" "$id" # SHA erneut nach manifest/runtime (weil meta SHA referenziert) APP_MANIFEST_SHA_CUR="$(sha256sum "$base/app.manifest" | awk '{print $1}')" RUNTIME_CONFIG_SHA_CUR="$(sha256sum "$base/runtime-config.txt" | awk '{print $1}')" gen_archive_meta "$role" "$base" "$id" "$asize" "$asha" echo "ARCHIVE($role) OK: $base/ (${asize} bytes, sha256 ${asha:0:16}...)" echo " image.tar.sha256:" sha256sum "$base/image.tar" > "$base/image.tar.sha256" chown root:root "$base/image.tar.sha256"; chmod 600 "$base/image.tar.sha256" } # pin-recovery-image / pin-remediation-image pin_image() { # $1 = role local role="$1" expect_zero "${@:2}" echo "=== PIN-$role ($RUN_MODE) ===" local name tag id name="$(role_pin "$role")" tag="$(role_tag "$role")" case "$role" in recovery) id="$RECOVERY_GOLD_IMAGE_ID" ;; remediation) remediation_session || die "pin(remediation): keine gueltige Build-Session (FAIL CLOSED)" id="$(remediation_full_image_id)" PIN_REMEDIATION_TARGET_ID="$id" ;; esac # Punkt 11: Containername darf NICHT existieren (kein Delete/Overwrite/Replace) if docker container inspect "$name" >/dev/null 2>&1; then # Identitaet pruefen; wenn korrekt -> ok (keine Aenderung); wenn falsch -> FAIL CLOSED local existing existing="$(docker container inspect -f '{{.Image}}' "$name" 2>/dev/null)" if pin_container_identity_ok "$role" "$existing"; then echo "PIN($role): Container existiert bereits korrekt (Image=$existing). KEINE Aenderung." exit 0 else die "pin($role): Container $name existiert mit falschem Image ($existing). Kein Replace (FAIL CLOSED)" fi fi echo " FULL_IMAGE_ID=$id" # Punkt 6: Image muss inspectable sein (Source Gate) if ! docker image inspect "$id" >/dev/null 2>&1; then die "pin($role): Source Image nicht inspectable (SOURCE IMAGE NOT INSPECTABLE — FAIL CLOSED)" fi if [ "$RUN_MODE" = "DRY_RUN" ]; then echo "DRY_RUN_PASS: pin($role) wuerde docker create --name $name --network none --restart no $id (keine Aktion)." exit 0 fi mode_or_die # MUTATION: minimaler Stopped-Container (Punkt 8/10) echo " [create] docker create --name $name --network none --restart no $id" if ! docker create --name "$name" --network none --restart no "$id" >/dev/null 2>&1; then die "pin($role): docker create fehlgeschlagen (FAIL CLOSED)" fi echo "PIN($role) OK: $name -> $id (Stopped, Network=none, Restart=no)" } # verify-recovery-pin / verify-remediation-pin / verify-...-archive / verify-...-ready verify_pin_cmd() { # $1 = role (read-only) local role="$1"; expect_zero "${@:2}" echo "=== VERIFY-PIN($role) ===" if verify_pin "$role"; then echo "VERIFY-PIN($role): PASS"; exit 0 else die "VERIFY-PIN($role): FAIL (FAIL CLOSED)"; fi } verify_archive_cmd() { # $1 = role (read-only) local role="$1"; expect_zero "${@:2}" echo "=== VERIFY-ARCHIVE($role) ===" verify_archive "$role" || die "VERIFY-ARCHIVE($role): FAIL (FAIL CLOSED)" exit 0 } verify_ready_cmd() { # $1 = role (read-only) local role="$1"; expect_zero "${@:2}" verify_ready "$role" || die "$(case $role in recovery) echo 'RECOVERY READY';; remediation) echo 'CANDIDATE READY';; esac): FAIL (FAIL CLOSED)" exit 0 } # restore-recovery-image-archive (DESIGN; realer Restore NICHT in dieser Mission ausfuehren) restore_recovery_dry() { # dry-run expect_zero "$@" echo "=== RESTORE-RECOVERY-IMAGE-ARCHIVE (DRY-RUN — DESIGN; kein echter Restore) ===" local sdir sid base img sdir="$(role_arch_dir recovery)" sid="$RECOVERY_ARCHIVE_ID" base="$sdir/$sid" [ -d "$base" ] || die "restore-dry: Archiv-Session fehlt $base (FAIL CLOSED)" echo " 1. Archiv SHA pruefen: $(sha256sum "$base/image.tar" | awk '{print $1}')" echo " 2. Metadaten pruefen: session.meta vorhanden" echo " 3. docker load (in realer Mission; hier NUR Dry-Run)" echo " 4. Resulting Image inspect" echo " 5. Application Manifest pruefen" echo " 6. Runtime Config pruefen" echo " 7. Tag/Identity pruefen" echo " 8. danach Pin Verify" echo " HINWEIS (Punkt 15): Wenn neu geladene FULL_IMAGE_ID != Original-ID bei gleicher App-Identitaet -> NICHT automatisch identisch; sauber Docker-Image-Identity vs Application/Runtime-Equivalence unterscheiden." echo "RESTORE-RECOVERY-DRY-RUN: DESIGN OK (kein echter Restore ausgefuehrt)" exit 0 } # restore-recovery-image-archive (real — in dieser Mission NICHT ausfuehren; nur definiert) restore_recovery_real() { expect_zero "$@" die "restore-recovery-image-archive: Realer Restore ist in dieser Mission NICHT erlaubt (HARTER STOPP Punkt 30). NUR dry-run (FAIL CLOSED)" } SUBCMD="${1:-}" shift || true # Mutation Guard: "dry-run " setzt DRY_RUN-Modus, dann normaler Dispatch. if [ "$SUBCMD" = "dry-run" ]; then RUN_MODE=DRY_RUN SUBCMD="${1:-}" shift || true case "$SUBCMD" in build-historical|recreate-historical|snapshot-recovery|build-recovery-gold|pin-current-candidate|recreate-candidate-safe|rollback-to-recovery-gold|promote-runner-patch|rollback-runner-patch|build-runner-remediation|recreate-runner-remediation-safe|archive-recovery-image|archive-remediation-image|pin-recovery-image|pin-remediation-image) ;; *) die "dry-run nur mit build-historical|recreate-historical|snapshot-recovery|build-recovery-gold|build-runner-remediation|recreate-runner-remediation-safe|archive-recovery-image|archive-remediation-image|pin-recovery-image|pin-remediation-image zulaessig" ;; esac fi case "$SUBCMD" in # ---------------- bestehend ---------------- ps) expect_zero "$@"; exec docker ps ;; inspect) req_ok; expect_zero "$@"; exec docker inspect "$CONTAINER" ;; logs) req_ok tail_n=200 while [ $# -gt 0 ]; do case "$1" in --tail) [ $# -ge 2 ] || die "--tail braucht Wert"; [[ "$2" =~ ^[0-9]+$ ]] || die "ungueltiges --tail"; tail_n="$2"; shift 2 ;; *) die "unerlaubtes Argument 'logs $1'" ;; esac done exec docker logs --tail "$tail_n" "$CONTAINER" ;; run) req_ok declare -a RUN=() while [ $# -gt 0 ]; do case "$1" in --start)[[ "${2:-}" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}$ ]]||die "ungueltiges --start";RUN+=("--start" "$2");shift 2;; --end)[[ "${2:-}" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}$ ]]||die "ungueltiges --end";RUN+=("--end" "$2");shift 2;; --limit-days)[[ "${2:-}" =~ ^[0-9]+$ ]]||die "ungueltiges --limit-days";RUN+=("--limit-days" "$2");shift 2;; *) die "unerlaubtes Argument 'run $1'" ;; esac done echo "[rq-historical] Start: docker exec $CONTAINER python $SCRIPT ${RUN[*]}" >&2 exec docker exec "$CONTAINER" python "$SCRIPT" "${RUN[@]}" ;; code) req_ok [ $# -eq 1 ] || die "code braucht genau EINEN Key (code-list)" p="$(key_path "$1")"; [ -n "$p" ] || die "unbekannter Key 'code $1'" exec docker exec "$CONTAINER" cat "$p" ;; hashes) req_ok; expect_zero "$@" PATHS=(); for k in "${!TARGET_PATHS[@]}"; do PATHS+=("${TARGET_PATHS[$k]}"); done exec docker exec "$CONTAINER" sha256sum "${PATHS[@]}" ;; code-list) req_ok; expect_zero "$@" echo "Verfuegbare READ-ONLY-Code-Keys (historical-service):" for k in "${!TARGET_PATHS[@]}"; do printf ' %-11s | staging: %-22s -> %s\n' "$k" "${STAGE_FILES[$k]}" "${TARGET_PATHS[$k]}"; done ;; stage-dir) expect_zero "$@" echo "Staging-Root: $STAGING_ROOT (Owner rq-host, keine globalen Schreibrechte)" for k in runner backfill pipeline repository provenance trust sanity trust_gate; do printf ' %-11s %s\n' "$k" "$(stage_file "$k")" done ;; stage-check) [ $# -eq 1 ] || die "stage-check braucht genau EINEN Key"; key_ok "$1"; validate_stage "$1" ;; test) req_ok; [ $# -eq 1 ] || die "test braucht genau EINEN Key"; key_ok "$1" validate_stage "$1" file="$(stage_file "$1")" tmp="/tmp/rqtest_${1}_$(date +%s).py" docker cp "$file" "$CONTAINER:$tmp" if docker exec "$CONTAINER" python -m py_compile "$tmp" >/dev/null 2>&1; then echo "PASS: py_compile OK ($file) [isoliert in Container-/tmp, Produktivcode unberuehrt]" else echo "FAIL: py_compile fehlgeschlagen ($file)" docker exec "$CONTAINER" rm -f "$tmp"; exit 2 fi docker exec "$CONTAINER" rm -f "$tmp" ;; search-writes) req_ok; expect_zero "$@" docker exec "$CONTAINER" sh -c 'grep -rnE "insert_raw_bars|insert_derived_bars|BackfillOrchestrator|trust_gate|TrustGate" /app/app --include="*.py" 2>/dev/null | head -100 || true' ;; healthcheck) req_ok; expect_zero "$@" rh="$(docker exec "$CONTAINER" python -c 'import urllib.request; print(urllib.request.urlopen("http://localhost:8000/health", timeout=5).status)' 2>/dev/null || echo N/A)" rr="$(docker exec "$CONTAINER" python -c 'import urllib.request; print(urllib.request.urlopen("http://localhost:8000/health/ready", timeout=5).status)' 2>/dev/null || echo N/A)" running="$(docker inspect -f '{{.State.Running}}' "$CONTAINER")" echo "Container.Running=$running /health=$rh /health/ready=$rr" [ "$running" = "true" ] && [ "$rh" = "200" ] && [ "$rr" = "200" ] || { echo "HEALTHCHECK FAILED"; exit 2; } echo "HEALTHCHECK OK" ;; backup) req_ok; [ $# -eq 1 ] || die "backup braucht genau EINEN Key"; key_ok "$1" p="$(key_path "$1")"; ts="$(date +%Y%m%d_%H%M%S)"; bdir="$BACKUP_ROOT/$1/$ts" mkdir -p "$bdir" docker exec "$CONTAINER" cat "$p" > "$bdir/$(basename "$p")" sha256sum "$bdir/$(basename "$p")" > "$bdir/SHA256" echo "BACKUP: $1 -> $bdir"; cat "$bdir/SHA256" ;; deploy) req_ok; [ $# -eq 1 ] || die "deploy braucht genau EINEN Key"; key_ok "$1" key="$1"; p="$(key_path "$key")"; file="$(stage_file "$key")" validate_stage "$key" ts="$(date +%Y%m%d_%H%M%S)"; bdir="$BACKUP_ROOT/$key/$ts" mkdir -p "$bdir" docker exec "$CONTAINER" cat "$p" > "$bdir/$(basename "$p")" || die "Backup fehlgeschlagen -> deploy abgebrochen" sha256sum "$bdir/$(basename "$p")" > "$bdir/SHA256" echo "BACKUP: $bdir" tmp="/tmp/rqdeploy_${key}_$(date +%s).py" docker cp "$file" "$CONTAINER:$tmp" if ! docker exec "$CONTAINER" python -m py_compile "$tmp" >/dev/null 2>&1; then docker exec "$CONTAINER" rm -f "$tmp"; die "py_compile FAIL -> deploy abgebrochen (nichts deployt)" fi docker exec "$CONTAINER" rm -f "$tmp" docker cp "$file" "$CONTAINER:$p" sha_stage="$(sha256sum "$file" | awk '{print $1}')" sha_deployed="$(docker exec "$CONTAINER" sha256sum "$p" | awk '{print $1}')" echo "SHA stage : $sha_stage"; echo "SHA deployed: $sha_deployed" if [ "$sha_stage" != "$sha_deployed" ]; then echo "HASH-MISMATCH -> rollback"; docker cp "$bdir/$(basename "$p")" "$CONTAINER:$p"; die "Deploy Hash-Mismatch, zurueckgerollt" fi rh="$(docker exec "$CONTAINER" python -c 'import urllib.request; print(urllib.request.urlopen("http://localhost:8000/health", timeout=5).status)' 2>/dev/null || echo N/A)" rr="$(docker exec "$CONTAINER" python -c 'import urllib.request; print(urllib.request.urlopen("http://localhost:8000/health/ready", timeout=5).status)' 2>/dev/null || echo N/A)" echo "Health: /health=$rh /health/ready=$rr" if [ "$rh" != "200" ] || [ "$rr" != "200" ]; then echo "DEPLOY-HEALTHCHECK FAILED -> rollback"; docker cp "$bdir/$(basename "$p")" "$CONTAINER:$p"; die "Rollback ausgefuehrt" fi echo "DEPLOY OK: $key -> $p (Backup: $bdir). KEIN backfill/run gestartet." ;; rollback) req_ok; [ $# -eq 1 ] || die "rollback braucht genau EINEN Key"; key_ok "$1" p="$(key_path "$1")" bdir="$(ls -1t "$BACKUP_ROOT/$1" 2>/dev/null | head -1)" [ -n "$bdir" ] || die "kein Backup fuer '$1' vorhanden" bdir="$BACKUP_ROOT/$1/$bdir"; src="$bdir/$(basename "$p")" [ -f "$src" ] || die "Backup-Datei fehlt: $src" sha_bak="$(sha256sum "$src" | awk '{print $1}')" sha_cur="$(docker exec "$CONTAINER" sha256sum "$p" | awk '{print $1}')" echo "aktuell: $sha_cur / backup: $sha_bak" [ "$sha_bak" != "$sha_cur" ] || die "kein Unterschied (bereits auf Backup-Stand)" docker cp "$src" "$CONTAINER:$p" sha_cur="$(docker exec "$CONTAINER" sha256sum "$p" | awk '{print $1}')" [ "$sha_cur" = "$sha_bak" ] || die "ROLLBACK-HASH-MISMATCH" echo "ROLLBACK OK: $key -> $p auf $bdir" ;; # ============ SOT-DISCOVERY (read-only, NEU) ============ build-def) expect_zero "$@" echo "=== Dockerfile ($HOST_DIR/Dockerfile) — redacted ===" [ -f "$HOST_DIR/Dockerfile" ] || die "Dockerfile nicht gefunden" while IFS= read -r line; do redact_line "$line"; done < "$HOST_DIR/Dockerfile" echo "" echo "=== docker-compose.yml — strukturielle Abschnitte, redacted ===" cf="" if [ -f "$HOST_DIR/docker-compose.yml" ]; then cf="$HOST_DIR/docker-compose.yml"; elif [ -f "$HOST_DIR/compose.yml" ]; then cf="$HOST_DIR/compose.yml"; elif [ -f "$HOST_DIR/docker-compose.yaml" ]; then cf="$HOST_DIR/docker-compose.yaml"; fi [ -n "$cf" ] || die "keine compose-Datei gefunden" echo "(Quelle: $cf)" while IFS= read -r line; do # nur relevante strukturelle Schluessel anzeigen; Environment/Secrets nicht roh case "$line" in *POSTGRES_PASSWORD*|*PG_PASSWORD*|*PASSWORD*|*SECRET*|*TOKEN*) echo " **** REDACTED (Secret-Zeile) ****" ;; *) echo " $line" ;; esac done < "$cf" echo "" echo "=== .dockerignore (falls vorhanden) ===" [ -f "$HOST_DIR/.dockerignore" ] && cat "$HOST_DIR/.dockerignore" || echo "(kein .dockerignore)" echo "" echo "=== Service/Container-Fixierung ===" echo " compose.project=$(docker inspect "$CONTAINER" --format '{{index .Config.Labels "com.docker.compose.project"}}' 2>/dev/null || echo N/A)" echo " compose.service=$(docker inspect "$CONTAINER" --format '{{index .Config.Labels "com.docker.compose.service"}}' 2>/dev/null || echo N/A)" echo " container=$CONTAINER" ;; build-files) expect_zero "$@" echo "=== Build-Dateien unter $HOST_DIR (nicht-verzeichnis, nicht .env/.dump/.sql) ===" for f in Dockerfile docker-compose.yml compose.yml docker-compose.yaml .dockerignore requirements.txt; do if [ -e "$HOST_DIR/$f" ]; then printf ' %-22s %s\n' "$f" "EXISTS ($(stat -c '%s' "$HOST_DIR/$f") bytes)"; else printf ' %-22s %s\n' "$f" "(fehlt)"; fi done ;; host-manifest) expect_zero "$@" echo "=== HOST Manifest /opt/historical-v2/app (*.py, keine Symlinks folgen) ===" [ -d "$HOST_APP" ] || die "Host-App-Verzeichnis fehlt: $HOST_APP" # finde reguläre *.py UND Symlinks auf *.py separat ( find "$HOST_APP" -type f -name "*.py" -printf '%P %s\n' 2>/dev/null | sort \ | while read -r rel size; do sha=$(sha256sum "$HOST_APP/$rel" 2>/dev/null | awk '{print $1}') printf 'FILE\t%s\t%s\t%s\n' "$rel" "$size" "${sha:-ERR}" done find "$HOST_APP" -type l -name "*.py" -printf '%P\n' 2>/dev/null | sort \ | while read -r rel; do printf 'SYMLINK\t%s\t0\t\n' "$rel"; done ) ;; host-code) [ $# -eq 1 ] || die "host-code braucht genau EINEN relativen Pfad (unter app/)" rel="$1" [[ "$rel" == *".."* ]] && die "Traversal nicht erlaubt" [[ "$rel" == /* ]] && die "relativer Pfad erwartet, nicht absolut" # realpath zwingend im app-Root real="$(realpath -m "$HOST_APP/$rel")" [[ "$real" == "$HOST_APP/"* ]] || die "Pfad ausserhalb app-Root blockiert" [ -f "$real" ] || die "Datei nicht gefunden oder kein File: $rel" [ -L "$real" ] && die "Symlink-Read nicht erlaubt" # nur .py ausgeben [[ "$rel" == *.py ]] || die "Nur .py-Quelldateien lesbar" echo "--- $rel ($(stat -c '%s' "$real") bytes) ---" exec cat "$real" ;; container-manifest) req_ok; expect_zero "$@" echo "=== CONTAINER Manifest historical-service:/app/app (*.py) ===" docker exec "$CONTAINER" sh -c 'cd /app/app && find . -name "*.py" -type f 2>/dev/null | sed "s#^\./##" | sort | while read f; do size=$(stat -c %s "$f"); sha=$(sha256sum "$f" | cut -d" " -f1); printf "FILE\t%s\t%s\t%s\n" "$f" "$size" "$sha"; done' docker exec "$CONTAINER" sh -c 'cd /app/app && find . -name "*.py" -type l 2>/dev/null | sed "s#^\./##" | sort | while read f; do printf "SYMLINK\t%s\t0\t\n" "$f"; done' || true ;; container-code) req_ok; [ $# -eq 1 ] || die "container-code braucht genau EINEN relativen Pfad (unter /app/app/)" rel="$1" [[ "$rel" == *".."* ]] && die "Traversal nicht erlaubt" [[ "$rel" == /* ]] && die "relativer Pfad erwartet, nicht absolut" target="/app/app/$rel" # sicherstellen Ziel existiert als reguläre Datei & nicht symlink im Container if ! docker exec "$CONTAINER" sh -c "[ -f '$target' ]" 2>/dev/null; then die "Datei nicht gefunden oder kein File im Container: $rel"; fi if docker exec "$CONTAINER" sh -c "[ -L '$target' ]" 2>/dev/null; then die "Symlink-Read im Container nicht erlaubt"; fi [[ "$rel" == *.py ]] || die "Nur .py-Quelldateien lesbar" echo "--- $target ---" exec docker exec "$CONTAINER" cat "$target" ;; tree-diff) req_ok; expect_zero "$@" echo "=== TREE-DIFF Host(/opt/historical-v2/app) <-> Container(/app/app) ===" hmap="$(mktemp)"; cmap="$(mktemp)" # Host: 'relsha' find "$HOST_APP" -type f -name "*.py" -printf '%P\n' 2>/dev/null | sort | while IFS= read -r rel; do sha="$(sha256sum "$HOST_APP/$rel" 2>/dev/null | awk '{print $1}')" printf '%s\t%s\n' "$rel" "$sha" done > "$hmap" # Container: 'relsha' docker exec "$CONTAINER" sh -c 'cd /app/app && find . -type f -name "*.py" -printf "%P\n" 2>/dev/null | sort | while IFS= read -r rel; do sha256sum "$rel" 2>/dev/null | awk -v r="$rel" '\''{print r"\t"$1}'\''; done' > "$cmap" tmpunion="$(mktemp)" { cut -f1 "$hmap"; cut -f1 "$cmap"; } | sort -u > "$tmpunion" echo "HOST_APP=$HOST_APP CONTAINER_APP=/app/app" while IFS= read -r rel; do hs="$(awk -F'\t' -v r="$rel" '$1==r{print $2}' "$hmap")" cs="$(awk -F'\t' -v r="$rel" '$1==r{print $2}' "$cmap")" if [ -n "$hs" ] && [ -n "$cs" ]; then if [ "$hs" = "$cs" ]; then status="MATCH"; else status="DIVERGED"; fi printf '%-18s %s [h:%s c:%s]\n' "$status" "$rel" "${hs:0:12}" "${cs:0:12}" elif [ -n "$hs" ]; then printf '%-18s %s\n' "HOST_ONLY" "$rel" else printf '%-18s %s\n' "MISSING_ON_HOST" "$rel" fi done < "$tmpunion" rm -f "$hmap" "$cmap" "$tmpunion" ;; # ============ HOST-SOT-RECOVERY (Phase 13.5, Extension 1) ============ # KEIN Build/Recreate. Nur: backup-sot -> recover-sot -> verify-sot-recovery # -> restore-sot. Fail-closed, Gold-SHA-gebunden, keine freien Pfade. backup-sot) expect_zero "$@" # Vollstaendiges Host-SoT-Backup in root-owned Recovery-Bereich ts="$(date +%Y%m%d_%H%M%S)" bdir="$SOT_BACKUP_ROOT/$ts" mkdir -p "$bdir" # tar mit Symlinks/Permissions/Ownership; kein Follow (kein -h) if ! tar -cpf "$bdir/historical-v2.tar" -C /opt historical-v2 2>/dev/null; then rm -rf "$bdir"; die "backup-sot: tar fehlgeschlagen (FAIL CLOSED)" fi # SHA256-Manifest aller regulären Dateien (nicht Symlinks folgen) ( cd /opt/historical-v2 && find . -type f -print0 2>/dev/null | sort -z \ | while IFS= read -r -d "" f; do sha="$(sha256sum "$f" 2>/dev/null | awk '{print $1}')" printf '%s %s\n' "$sha" "${f#./}" done ) > "$bdir/SHA256.manifest" # Struktur-/Pflichtpruefung (Punkt 4) ok=1 [ -f "$bdir/historical-v2.tar" ] || { echo "FEHLT: tar"; ok=0; } [ -s "$bdir/SHA256.manifest" ] || { echo "FEHLT: manifest"; ok=0; } [ -f /opt/historical-v2/Dockerfile ] || { echo "FEHLT: Dockerfile"; ok=0; } [ -f /opt/historical-v2/docker-compose.yml ] || { echo "FEHLT: compose"; ok=0; } [ -d /opt/historical-v2/app ] || { echo "FEHLT: app/"; ok=0; } if [ "$ok" != "1" ]; then rm -rf "$bdir"; die "backup-sot: Validierung fehlgeschlagen (FAIL CLOSED)" fi # Session-Marker: eindeutige Markierung der aktuellen Recovery-Session echo "$ts" > "$bdir/$SOT_SESSION_MARKER" chown -R root:root "$bdir"; chmod 700 "$bdir" echo "BACKUP-SOT OK: $bdir" echo " tar: $(stat -c '%s' "$bdir/historical-v2.tar") bytes" echo " manifest: $(wc -l < "$bdir/SHA256.manifest") Dateien" echo " session: $ts" ;; sot-session) expect_zero "$@" # Zeigt den neuesten VALIDIERTEN FULL SOT SNAPSHOT (Partial-Backups wie # pre-runner-remediation werden NICHT als Full-Snapshot gezaehlt; Blocker A). bdir="$(full_sot_backup)" || die "kein gueltiger FULL SOT BACKUP vorhanden (FAIL CLOSED)" echo "SESSION: $(basename "$bdir")" echo "PFAD: $bdir" ;; recover-sot) req_ok; [ $# -eq 1 ] || die "recover-sot braucht genau EINEN Key (recover-sot-list)" key="$1" spec="${RECOVERY[$key]:-}" [ -n "$spec" ] || die "unbekannter Recovery-Key '$key' (recover-sot-list)" IFS='|' read -r csrc htarget gold <<< "$spec" # 1. gueltiger vollstaendiger (FULL) backup-sot vorhanden? (Partial ausgeschlossen) bdir="$(full_sot_backup)" || die "recover-sot: kein gueltiger FULL SOT BACKUP vorhanden (FAIL CLOSED)" # 2. Source = historical-service fixed (req_ok bereits geprueft) # 3/4. Source regular file, kein Symlink if ! docker exec "$CONTAINER" sh -c "[ -f '$csrc' ]" 2>/dev/null; then die "recover-sot: Source kein File im Container: $csrc"; fi if docker exec "$CONTAINER" sh -c "[ -L '$csrc' ]" 2>/dev/null; then die "recover-sot: Source ist Symlink (FAIL CLOSED): $csrc"; fi # 5. Source SHA == EXPECTED GOLD SHA csha="$(docker exec "$CONTAINER" sha256sum "$csrc" 2>/dev/null | awk '{print $1}')" || csha="" if [ "$csha" != "$gold" ]; then die "recover-sot: Source-SHA-Mismatch fuer '$key' (erwartet $gold, Container $csha) — FAIL CLOSED" fi # 6. Host target exakt fester Pfad (aus Allowlist, kein freier Pfad) [[ "$htarget" == /opt/historical-v2/app/* ]] || die "recover-sot: Host-Ziel ausserhalb app/ (FAIL CLOSED)" # 7. Zielverzeichnis kontrolliert erzeugen (z.B. trust/) tdir="$(dirname "$htarget")" mkdir -p "$tdir" # 8. Ziel vor Ueberschreiben einzeln sichern, falls vorhanden if [ -e "$htarget" ]; then prebak="$bdir/pre-recover/$(basename "$htarget")" mkdir -p "$bdir/pre-recover" cp -a "$htarget" "$prebak" 2>/dev/null || true echo " pre-recover gesichert: $prebak" fi # 9. byte-exakt schreiben (docker cp = byte-identisch) docker cp "$CONTAINER:$csrc" "$htarget" # 10. Ownership/Permissions aus bestehendem Host-Kontext ableiten # (app/ ist UID 10010; trust/ neu -> gleiche Basis wie app/) chown 10010:10010 "$htarget" 2>/dev/null || chown root:root "$htarget" chmod 644 "$htarget" # 11. Host SHA == Gold SHA hsha="$(sha256sum "$htarget" | awk '{print $1}')" if [ "$hsha" != "$gold" ]; then die "recover-sot: Host-SHA-Mismatch fuer '$key' (erwartet $gold, Host $hsha) — FAIL CLOSED" fi echo "RECOVER-SOT OK: $key" echo " source : $CONTAINER:$csrc" echo " target : $htarget" echo " gold : $gold" echo " host : $hsha" ;; recover-sot-list) expect_zero "$@" echo "Recovery-Allowlist (CONFIRMED GOLD, fest gebunden):" for k in "${!RECOVERY[@]}"; do IFS='|' read -r csrc htarget gold <<< "${RECOVERY[$k]}" printf ' %-16s %-45s -> %s\n' "$k" "$csrc" "$htarget" done ;; verify-sot-recovery) req_ok; expect_zero "$@" echo "=== VERIFY-SOT-RECOVERY (Gold vs Host vs Container) ===" allok=1 for k in "${!RECOVERY[@]}"; do IFS='|' read -r csrc htarget gold <<< "${RECOVERY[$k]}" hsha="$(sha256sum "$htarget" 2>/dev/null | awk '{print $1}')" || hsha="" csha="$(docker exec "$CONTAINER" sha256sum "$csrc" 2>/dev/null | awk '{print $1}')" || csha="" if [ "$hsha" = "$gold" ] && [ "$csha" = "$gold" ]; then printf ' MATCH %-16s %s\n' "$k" "$htarget" else printf ' FAIL %-16s host=%s container=%s gold=%s\n' "$k" "${hsha:-MISSING}" "${csha:-MISSING}" "$gold" allok=0 fi done [ "$allok" = "1" ] && echo "VERIFY-SOT-RECOVERY: ALLE MATCH" || { echo "VERIFY-SOT-RECOVERY: FEHLER"; exit 2; } ;; restore-sot) expect_zero "$@" # Restore aus dem neuesten validerungspflichtigen Session-Backup (FULL) bdir="$(full_sot_backup)" || die "restore-sot: kein gueltiger FULL SOT BACKUP vorhanden (FAIL CLOSED)" [ -f "$bdir/$SOT_SESSION_MARKER" ] || die "restore-sot: kein markiertes Session-Backup (FAIL CLOSED)" [ -f "$bdir/historical-v2.tar" ] || die "restore-sot: tar fehlt (FAIL CLOSED)" echo "RESTORE-SOT: $bdir" echo " (entpackt /opt/historical-v2 aus Session-Backup, byte-exakt)" if ! tar -xpf "$bdir/historical-v2.tar" -C /opt 2>/dev/null; then die "restore-sot: tar-Restore fehlgeschlagen (FAIL CLOSED)" fi echo "RESTORE-SOT OK. KEIN Build/Recreate danach." ;; snapshot-recovery) snapshot_recovery "$@" ;; verify-recovery-snapshot) verify_recovery_snapshot "$@" ;; build-recovery-gold) build_recovery_gold "$@" ;; verify-recovery-gold-image) verify_recovery_gold_image "$@" ;; pin-current-candidate) pin_current_candidate "$@" ;; verify-candidate-image) verify_candidate_image "$@" ;; recreate-candidate-safe) recreate_candidate_safe "$@" ;; rollback-to-recovery-gold) rollback_to_recovery_gold "$@" ;; promote-runner-patch) promote_runner_patch "$@" ;; rollback-runner-patch) rollback_runner_patch "$@" ;; build-runner-remediation) build_runner_remediation "$@" ;; verify-runner-remediation-image) verify_runner_remediation_image "$@" ;; recreate-runner-remediation-safe) recreate_runner_remediation_safe "$@" ;; verify-remediation-build-input) expect_zero "$@"; verify_remediation_build_input ;; build-historical) build_historical "$@" ;; create-build-exclusion) expect_zero "$@" # Build-Pollution-Exclusion implementieren (Punkt 5-8): .dockerignore anlegen/ersetzen # PREVIOUS_STATE war ABSENT. NUR exakt die 3 Legacy-Einträge. if [ -f "$BUILD_EXCLUSION_FILE" ]; then die "create-build-exclusion: .dockerignore existiert bereits (kein Ueberschreiben ohne Backup — manuell)" fi printf 'app/app/\napp/eligibility.py\n' > "$BUILD_EXCLUSION_FILE" echo "BUILD-EXCLUSION OK: $BUILD_EXCLUSION_FILE" exclusion_valid && { echo " validiert: nur app/app/ + app/eligibility.py"; echo " (PREVIOUS_STATE=ABSENT; keine Gold-Datei betroffen)"; } ;; show-build-exclusion) expect_zero "$@" if [ ! -f "$BUILD_EXCLUSION_FILE" ]; then echo "PREVIOUS_STATE=ABSENT"; exit 0; fi echo "=== .dockerignore Inhalt ===" cat "$BUILD_EXCLUSION_FILE" echo "=== build-exclusion: Gold-Dateien betroffen? (sollte: 0) ===" ok=0 while IFS= read -r f; do # f relativ zu /opt/historical-v2; pruefe ob im Gold-Manifest (app/...) rel="app/${f#app/}" if grep -q " $rel$" "$GOLD_MANIFEST" 2>/dev/null; then echo " GOLD-DATEI BETROFFEN: $rel"; ok=1; fi done < "$BUILD_EXCLUSION_FILE" [ "$ok" = "0" ] && echo " 0 Gold-Dateien ausgeschlossen (OK)" ;; recreate-historical) recreate_historical "$@" ;; verify-gold-reproduction) verify_gold_reproduction "$@" ;; # ============ IMAGE-RETENTION / ARCHIVE CONTROL PLANE (Phase 13.5) ============ # Schicht B (Archiv, docker save; fail-closed, kein Overwrite) & Schicht A (Pin). # Alle verify-* sind read-only. Realer Restore ist in dieser Mission BLOCKIERT # (nur dry-run). Kein Coolify-Cleanup-Disable, kein coolify.managed-Spoof. archive-recovery-image) archive_image recovery "$@" ;; archive-remediation-image) archive_image remediation "$@" ;; pin-recovery-image) pin_image recovery "$@" ;; pin-remediation-image) pin_image remediation "$@" ;; verify-recovery-archive) verify_archive_cmd recovery "$@" ;; verify-remediation-archive) verify_archive_cmd remediation "$@" ;; verify-recovery-pin) verify_pin_cmd recovery "$@" ;; verify-remediation-pin) verify_pin_cmd remediation "$@" ;; verify-recovery-ready) verify_ready_cmd recovery "$@" ;; verify-remediation-ready) verify_ready_cmd remediation "$@" ;; restore-recovery-image-archive) restore_recovery_real "$@" ;; dry-restore-recovery-image-archive) restore_recovery_dry "$@" ;; *) die "Aufruf: ps|inspect|logs|run|code |hashes|code-list|stage-dir|stage-check |test |search-writes|healthcheck|backup |deploy |rollback |build-def|build-files|host-manifest|host-code

|container-manifest|container-code

|tree-diff|backup-sot|sot-session|recover-sot |recover-sot-list|verify-sot-recovery|restore-sot|snapshot-recovery|dry-run snapshot-recovery|verify-recovery-snapshot|build-recovery-gold|dry-run build-recovery-gold|verify-recovery-gold-image|pin-current-candidate|dry-run pin-current-candidate|verify-candidate-image|recreate-candidate-safe|dry-run recreate-candidate-safe|rollback-to-recovery-gold|dry-run rollback-to-recovery-gold|promote-runner-patch|dry-run promote-runner-patch|rollback-runner-patch|dry-run rollback-runner-patch|build-runner-remediation|dry-run build-runner-remediation|verify-runner-remediation-image|recreate-runner-remediation-safe|dry-run recreate-runner-remediation-safe|verify-remediation-build-input|build-historical|dry-run build-historical|recreate-historical|dry-run recreate-historical|verify-gold-reproduction|create-build-exclusion|show-build-exclusion|archive-recovery-image|archive-remediation-image|pin-recovery-image|pin-remediation-image|verify-recovery-archive|verify-remediation-archive|verify-recovery-pin|verify-remediation-pin|verify-recovery-ready|verify-remediation-ready|dry-restore-recovery-image-archive|restore-recovery-image-archive" ;; esac