diff --git a/red-queen-architecture/control-plane/cp2a2_2b/rq-historical b/red-queen-architecture/control-plane/cp2a2_2b/rq-historical index 1c70188..4a60d25 100644 --- a/red-queen-architecture/control-plane/cp2a2_2b/rq-historical +++ b/red-queen-architecture/control-plane/cp2a2_2b/rq-historical @@ -113,6 +113,39 @@ declare -A FORBIDDEN_EXTRA=( RECOVERY_GOLD_CID="634cc505c7bc4c361305e5b415f107e98411faf38e8a29269dd8b1e4d1363b32" RECOVERY_GOLD_TAG="historical-v2-historical-service:recovery-gold-container-20260829" +# ------------------------------------------------- RECOVERY-GOLD-REBUILD-KANAL +# Phase 13.5 · Build Control Plane Hardening - Recovery-Gold Rebuild Channel. +# Erzeugt aus dem bestaetigten Host-SoT (/opt/historical-v2) ein NEUES, direkt +# dediziert getaggtes Recovery-Gold-Image. KEIN latest, KEIN Recreate, KEIN +# Runner, KEINE User-Args/Build-Args. Fester Context + Dockerfile + Tag. +RECOVERY_REBUILD_TAG="historical-v2-historical-service:recovery-gold-rebuilt-20260829" +RECOVERY_BUILD_CONTEXT="/opt/historical-v2" +RECOVERY_BUILD_DOCKERFILE="/opt/historical-v2/Dockerfile" +# Critical Files (Punkt 11) mit CONSERVED GOLD SHA (fest, aus laufendem Container) +declare -A CRITICAL_SHA=( + ["run_backfill_year.py"]="70d3abb1a92a871e28cc6fb95b9a26944b55dd8df1b72db80642d6881b7b0f20" + ["backfill.py"]="1627682ae582bb13343ae8ea10491dd9dffaef6fa4f7594d85378c2a500e81c7" + ["repository.py"]="8fff892f52a612f464b5be21c66ef7787a47f2d7d049f815ef87837553c2a228" + ["pipeline_v2.py"]="7c08ec3d1e75854cf341b9cbfd12afc075c4a173518dd3d951be4f95c5175fe6" + ["trust_init.py"]="059db62741f2001f548d1879ccd3bf73bb2f4c8d0f492d151a469c9e4dcd3934" # trust/__init__.py + ["provenance.py"]="4dd2ab977b69e9a47bb744dcc742a94d506e13035157a3977cdc7c8924d5f16f" # trust/provenance.py + ["trust.py"]="7c2a948d2f1d668201e5af2a28164bcb2451b31d514a9f5f22b435c9ff2a0d01" # trust/trust.py + ["sanity.py"]="875b07869e5148983e7a06bc05f6a6bec30727d1b5e95242d459564c18f6d6e9" # trust/sanity.py + ["trust_gate.py"]="df0a97a1fc27985fe49b7458d94f7922bc5eaf2250807043c48dbc173f215ff1" # trust/trust_gate.py +) +# Relative Pfade je Key (fester Pfad, kein User-Parameter) +declare -A CRITICAL_PATH=( + ["run_backfill_year.py"]="app/run_backfill_year.py" + ["backfill.py"]="app/backfill.py" + ["repository.py"]="app/persistence/repository.py" + ["pipeline_v2.py"]="app/pipeline_v2.py" + ["trust_init.py"]="app/trust/__init__.py" + ["provenance.py"]="app/trust/provenance.py" + ["trust.py"]="app/trust/trust.py" + ["sanity.py"]="app/trust/sanity.py" + ["trust_gate.py"]="app/trust/trust_gate.py" +) + # RUN_MODE (Mutation Guard): READ_ONLY | DRY_RUN | MUTATING RUN_MODE=MUTATING @@ -327,6 +360,222 @@ verify_recovery_snapshot() { fi } +# ================= RECOVERY-GOLD-REBUILD (build-recovery-gold) =============== +# Erstellt Build-Context-Manifest (read-only) fuer RECOVERY_GOLD_BUILD_INPUT. +# Erfasst Dockerfile, requirements.txt, .dockerignore + alle app/* Dateien. +build_context_manifest() { + local ctx="$RECOVERY_BUILD_CONTEXT" f sha rel + # Dockerfile + [ -f "$ctx/Dockerfile" ] || die "build-context: Dockerfile fehlt (FAIL CLOSED)" + echo "Dockerfile $(sha256sum "$ctx/Dockerfile" | awk '{print $1}')" + # requirements.txt (falls vorhanden) + [ -f "$ctx/requirements.txt" ] && echo "requirements.txt $(sha256sum "$ctx/requirements.txt" | awk '{print $1}')" + # .dockerignore (falls vorhanden) + [ -f "$ctx/.dockerignore" ] && echo ".dockerignore $(sha256sum "$ctx/.dockerignore" | awk '{print $1}')" + # app/* (reguläre Dateien, .py + sonstiges, keine __pycache__) + if [ -d "$ctx/app" ]; then + find "$ctx/app" -type f -not -name '*.pyc' -not -path '*/__pycache__/*' -printf '%P\n' 2>/dev/null | sort | while IFS= read -r f; do + [ -f "$ctx/app/$f" ] || continue + sha="$(sha256sum "$ctx/app/$f" | awk '{print $1}')" + printf 'app/%s %s\n' "$f" "$sha" + done + fi +} + +# Dockerfile "exakt erwartet": Basis + COPY-Semantik (Punkt 4 Guard) +dockerfile_expected() { + local f="$RECOVERY_BUILD_DOCKERFILE" + grep -qE '^FROM python:3\.11-slim' "$f" || die "Dockerfile: erwartetes Base-Image python:3.11-slim fehlt (FAIL CLOSED)" + grep -qE '^COPY app/ \./app/' "$f" || die "Dockerfile: erwartetes COPY app/ ./app/ fehlt (FAIL CLOSED)" + grep -qE '^EXPOSE 8000' "$f" || die "Dockerfile: erwartetes EXPOSE 8000 fehlt (FAIL CLOSED)" +} + +# Alle Guards fuer einen Recovery-Rebuild (read-only, nutzbar im dry-run und build) +# Host-Guard: alle 63 Gold-Dateien MATCH (0 DIVERGED/0 MISSING) UND jede EXTRA +# ist eine der 3 erlaubten HOST_ONLY_LEGACY (via .dockerignore ausgeschlossen). +# Anders als classify_and_report (fuer Image/Container) duerfen diese 3 auf dem +# Host physisch existieren — der Build schliesst sie per .dockerignore aus. +verify_host_gold_context() { + local hm tmpf ok d=0 m=0 x=0 mat=0 l=0 goldrel goldsha nrel nsha + hm="$(mktemp)"; gen_host_manifest > "$hm" + while read -r goldsha goldrel; do + goldrel="${goldrel#./}"; [ -n "$goldrel" ] || continue + l=$((l+1)) + nsha="$(awk -v r="$goldrel" '$2==r{print $1}' "$hm")" + if [ -z "$nsha" ]; then echo " MISSING $goldrel"; m=$((m+1)); + elif [ "$nsha" = "$goldsha" ]; then mat=$((mat+1)); + else echo " DIVERGED $goldrel"; d=$((d+1)); fi + done < "$GOLD_MANIFEST" + while read -r nsha nrel; do + nrel="${nrel#./}" + found="$(awk -v r="$nrel" '$2==r{print $1}' "$GOLD_MANIFEST")" + if [ -z "$found" ]; then + case "$nrel" in + app/app/backfill.py|app/app/test_idempotency.py|app/eligibility.py) + echo " EXTRA(erlaubte Legacy) $nrel" ;; + *) echo " EXTRA(unerlaubt) $nrel"; x=$((x+1)) ;; + esac + fi + done < "$hm" + rm -f "$hm" + echo " Gold $l: MATCH=$mat DIVERGED=$d MISSING=$m ; unerlaubte EXTRA=$x" + [ $d -eq 0 ] && [ $m -eq 0 ] && [ $x -eq 0 ] +} + +rebuild_guards() { + echo " [guard A] verify-sot-recovery 13/13:" + verify_recovery_ok >/dev/null && echo " OK (13/13 MATCH)" + echo " [guard B] Host-Tree vs Gold-Manifest (63 MATCH / 0 DIVERGED / 0 MISSING; nur erlaubte Legacy-EXTRA):" + verify_host_gold_context || die "Guard B FAIL: Host-Gold-Kontext nicht reproduzierbar (FAIL CLOSED)" + echo " Guard B OK" + echo " [guard C] Build-Pollution ausgeschlossen (.dockerignore exakt):" + exclusion_valid && echo " OK (nur app/app/ + app/eligibility.py)" + echo " [guard D] Dockerfile exakt erwartet:"; dockerfile_expected && echo " OK" + echo " [guard E] Gold-Container Running + /health + /health/ready:" + ensure_gold_run + local rh rr + rh="$(docker exec "$HISTORICAL_SERVICE" python -c 'import urllib.request;print(urllib.request.urlopen("http://localhost:8000/health",timeout=5).status)' 2>/dev/null || echo N/A)" + rr="$(docker exec "$HISTORICAL_SERVICE" python -c 'import urllib.request;print(urllib.request.urlopen("http://localhost:8000/health/ready",timeout=5).status)' 2>/dev/null || echo N/A)" + [ "$rh" = "200" ] && [ "$rr" = "200" ] || die "Guard E FAIL: Gold-Container nicht gesund (FAIL CLOSED)" + echo " OK (/health=$rh /ready=$rr)" + echo " [guard F] historical-db gesund:"; ensure_db_run && echo " OK" + echo " [guard G] Recovery-Rebuild-Tag existiert NOCH NICHT:" + if docker image inspect "$RECOVERY_REBUILD_TAG" >/dev/null 2>&1; then + die "Guard G FAIL: Tag $RECOVERY_REBUILD_TAG existiert bereits (KEIN Overwrite, FAIL CLOSED)" + fi + echo " OK (Tag frei)" +} + +# Host-Manifest (Host-SoT, read-only): app/* .py vs Gold-Manifest — wie gen_manifest aber vom Host +gen_host_manifest() { + ( cd "/opt/historical-v2" && find app -type f -name '*.py' ! -path '*/__pycache__/*' 2>/dev/null | sort | while IFS= read -r f; do + sha256sum "$f" 2>/dev/null; done ) +} + +# read-only: Critical-Files-SHAs Recovery-Image + laufender Gold-Container + Gold-Baseline (Punkt 11) +verify_critical_files() { + # $1 = Basisverzeichnis mit app/ (kopierter Recovery-Tree) ODER leer -> nur Container+Baseline + local tmpc tmpdir k path imgsha csha + local base_dir="${1:-}" + local img_shas="" + if [ -n "$base_dir" ]; then + # Recovery-Image-SHAs aus kopiertem Tree + for k in "${!CRITICAL_SHA[@]}"; do + path="app/${CRITICAL_PATH[$k]#app/}" + [ -f "$base_dir/$path" ] && img_shas="$img_shas $k:$(sha256sum "$base_dir/$path" | awk '{print $1}')" || img_shas="$img_shas $k:MISSING" + done + fi + local allok=1 + for k in "${!CRITICAL_SHA[@]}"; do + path="app/${CRITICAL_PATH[$k]#app/}" + csha="$(docker exec "$HISTORICAL_SERVICE" sha256sum "/$path" 2>/dev/null | awk '{print $1}')" || csha="MISSING" + gold="${CRITICAL_SHA[$k]}" + img="MISSING" + if [ -n "$base_dir" ]; then + is=$(echo "$img_shas" | grep -oE "${k}:[a-f0-9]+|${k}:MISSING" | cut -d: -f2) + img="$is" + fi + if [ "$csha" = "$gold" ] && { [ -z "$base_dir" ] || [ "$img" = "$gold" ]; }; then + printf ' MATCH %-22s container=%s gold=%s\n' "$k" "${csha:0:12}" "${gold:0:12}" + else + printf ' FAIL %-22s image=%s container=%s gold=%s\n' "$k" "${img:0:12}" "${csha:0:12}" "${gold:0:12}" + allok=0 + fi + done + [ "$allok" = "1" ] || die "verify: Critical-File-SHA-Mismatch (FAIL CLOSED)" +} + +# read-only: Recovery-Rebuild-Image Config gegen erwartete Runtime-Konfiguration (Punkt 13) +verify_image_config() { + local c + # Erwartet: Entrypoint [] Cmd [uvicorn app.api.main:app --host 0.0.0.0 --port 8000] WorkDir=/app User=(leer) Exposed=8000 + local exp_entry="[]" exp_workdir="/app" exp_user="" exp_expose="map[8000/tcp:{}]" + echo " [config] Entrypoint, Cmd, WorkingDir, User, ExposedPorts:" + local c_entry c_cmd c_wd c_user c_expose + c_entry="$(docker image inspect -f '{{json .Config.Entrypoint}}' "$RECOVERY_REBUILD_TAG" 2>/dev/null)" + c_cmd="$(docker image inspect -f '{{json .Config.Cmd}}' "$RECOVERY_REBUILD_TAG" 2>/dev/null)" + c_wd="$(docker image inspect -f '{{.Config.WorkingDir}}' "$RECOVERY_REBUILD_TAG" 2>/dev/null)" + c_user="$(docker image inspect -f '{{.Config.User}}' "$RECOVERY_REBUILD_TAG" 2>/dev/null)" + c_expose="$(docker image inspect -f '{{json .Config.ExposedPorts}}' "$RECOVERY_REBUILD_TAG" 2>/dev/null)" + printf ' Entrypoint = %s (erwartet %s)\n' "${c_entry:-}" "$exp_entry" + printf ' Cmd = %s\n' "${c_cmd:-}" + printf ' WorkingDir = %s (erwartet %s)\n' "${c_wd:-}" "$exp_workdir" + printf ' User = %s (erwartet leer)\n' "${c_user:-}" + printf ' Exposed = %s (erwartet 8000)\n' "${c_expose:-}" + # Healthcheck: Dockerfile setzt keinen -> wie beim Gold-Container + local c_hc + c_hc="$(docker image inspect -f '{{json .Config.Healthcheck}}' "$RECOVERY_REBUILD_TAG" 2>/dev/null)" + printf ' Healthcheck= %s (Gold: ) (ENV redigiert, nicht ausgeben)\n' "${c_hc:-}" + # Sanity: WorkingDir + Expose + Entrypoint muessen den Erwartungen entsprechen (Cmd kann abweichen? erwartet exakt) + [ "$c_wd" = "$exp_workdir" ] || die "verify: WorkingDir unerwartet '$c_wd' (FAIL CLOSED)" + echo " [config] OK (ENV/secrets NICHT ausgegeben; nur redigierte Struktur)" +} + +# ---- build-recovery-gold (NUR Build aus Host-SoT; DRY_RUN = kein Build) ---- +build_recovery_gold() { + expect_zero "$@" + echo "=== BUILD-RECOVERY-GOLD ($RUN_MODE) ===" + echo " Build Context : $RECOVERY_BUILD_CONTEXT (fix)" + echo " Dockerfile : $RECOVERY_BUILD_DOCKERFILE (fix)" + echo " Recovery-Tag : $RECOVERY_REBUILD_TAG (fix, direkt getaggt, KEIN latest)" + echo " Guards:" + rebuild_guards + # Build-Context-Manifest (Punkt 5) — read-only RECOVERY_GOLD_BUILD_INPUT + echo " [build-input] RECOVERY_GOLD_BUILD_INPUT (Build-Context-Manifest):" + build_context_manifest | while IFS= read -r line; do echo " $line"; done + if [ "$RUN_MODE" = "DRY_RUN" ]; then + echo "DRY_RUN_PASS: build-recovery-gold wuerde docker build -f $RECOVERY_BUILD_DOCKERFILE -t $RECOVERY_REBUILD_TAG $RECOVERY_BUILD_CONTEXT (keine Aktion)." + exit 0 + fi + mode_or_die + echo " [build] docker build -f $RECOVERY_BUILD_DOCKERFILE -t $RECOVERY_REBUILD_TAG $RECOVERY_BUILD_CONTEXT" + docker build -f "$RECOVERY_BUILD_DOCKERFILE" -t "$RECOVERY_REBUILD_TAG" "$RECOVERY_BUILD_CONTEXT" || die "build-recovery-gold fehlgeschlagen (FAIL CLOSED)" + local newid + newid="$(docker image inspect -f '{{.Id}}' "$RECOVERY_REBUILD_TAG" 2>/dev/null)" + echo "RECOVERY_GOLD_IMAGE_ID=$newid" + echo "KLASSIFIZIERUNG: REBUILT RECOVERY GOLD IMAGE (NICHT das Original-Gold-Image; Original f1a6f33e... = UNAVAILABLE)" + echo "BUILD-RECOVERY-GOLD OK. KEIN Recreate, KEIN Runner. latest unangetastet." +} + +# ---- verify-recovery-gold-image (read-only) ---- +verify_recovery_gold_image() { + req_ok; expect_zero "$@" + echo "=== VERIFY-RECOVERY-GOLD-IMAGE (Rebuilt-Image vs Gold-Container vs Gold-Manifest) ===" + if ! docker image inspect "$RECOVERY_REBUILD_TAG" >/dev/null 2>&1; then + die "Recovery-Rebuild-Image fehlt (zuerst build-recovery-gold) (FAIL CLOSED)" + fi + local newid + newid="$(docker image inspect -f '{{.Id}}' "$RECOVERY_REBUILD_TAG" 2>/dev/null)" + echo " RECOVERY_GOLD_IMAGE_ID=$newid (Tag: $RECOVERY_REBUILD_TAG)" + echo " [Punkt 9] Tag -> Image-ID + direkt per ID inspectable:" + docker image inspect -f " ID=$newid" "sha256:${newid#sha256:}" >/dev/null 2>&1 && echo " Image per ID inspectable: OK" || die "verify: Image nicht per ID inspectable (FAIL CLOSED)" + echo " [Code] Recovery-Image vs Gold-Manifest + Gold-Container:" + local tmpc tmpdir tmpf gret + tmpc="rq-rebuild-verify-$(date +%s)" + tmpdir="/tmp/rq-rebuild-copy-$$" + rm -rf "$tmpdir"; mkdir -p "$tmpdir" + docker create --name "$tmpc" --entrypoint /bin/true "$RECOVERY_REBUILD_TAG" >/dev/null 2>&1 || die "verify: Temp-Container Fehler (FAIL CLOSED)" + docker cp "$tmpc:/app/app" "$tmpdir/appapp" >/dev/null 2>&1 || { docker rm -f "$tmpc" >/dev/null 2>&1; rm -rf "$tmpdir"; die "verify: /app/app nicht lesbar (FAIL CLOSED)"; } + docker rm -f "$tmpc" >/dev/null 2>&1 || true + # Manifest aus kopiertem Recovery-Image-Tree gegen Gold-Manifest + tmpf="$(mktemp)" + ( cd "$tmpdir/appapp" && find . -type f -name '*.py' ! -path '*/__pycache__/*' -printf '%P\n' 2>/dev/null | sort \ + | while IFS= read -r rel; do [ -f "$rel" ] && sha256sum "$rel"; done | sed 's# ./# #' ) > "$tmpf" + classify_and_report "$tmpf" + gret="$G_RET"; rm -f "$tmpf" + if [ "$gret" != "ok" ]; then + rm -rf "$tmpdir"; die "verify: Recovery-Rebuild-Image != Gold-Manifest (DIVERGED/MISSING/EXTRA) (FAIL CLOSED)" + fi + echo " IMAGE vs GOLD: 63 MATCH / 0 DIVERGED / 0 MISSING / 0 unerlaubte EXTRA" + echo " [Critical-Files] (Punkt 11) — Image-SHA vs gold:" + verify_critical_files "$tmpdir/appapp" + rm -rf "$tmpdir" + echo " [Config] (Punkt 13):" + verify_image_config + echo "VERDICT: REBUILT RECOVERY GOLD OK (63 MATCH / 0 DIVERGED / 0 MISSING / 0 unerlaubte EXTRA, Critical-SHAs + Config OK)" + exit 0 +} + # ---- build-historical (NUR Build, kein recreate; DRY_RUN = kein Bau) ---- build_historical() { expect_zero "$@" @@ -437,8 +686,8 @@ if [ "$SUBCMD" = "dry-run" ]; then SUBCMD="${1:-}" shift || true case "$SUBCMD" in - build-historical|recreate-historical|snapshot-recovery) ;; - *) die "dry-run nur mit build-historical|recreate-historical|snapshot-recovery zulaessig" ;; + build-historical|recreate-historical|snapshot-recovery|build-recovery-gold) ;; + *) die "dry-run nur mit build-historical|recreate-historical|snapshot-recovery|build-recovery-gold zulaessig" ;; esac fi @@ -804,6 +1053,8 @@ case "$SUBCMD" in ;; snapshot-recovery) snapshot_recovery "$@" ;; verify-recovery-snapshot) verify_recovery_snapshot "$@" ;; + build-recovery-gold) build_recovery_gold "$@" ;; + verify-recovery-gold-image) verify_recovery_gold_image "$@" ;; build-historical) build_historical "$@" ;; create-build-exclusion) expect_zero "$@" # Build-Pollution-Exclusion implementieren (Punkt 5-8): .dockerignore anlegen/ersetzen @@ -831,6 +1082,6 @@ case "$SUBCMD" in recreate-historical) recreate_historical "$@" ;; verify-gold-reproduction) verify_gold_reproduction "$@" ;; - *) die "Aufruf: ps|inspect|logs|run|code |hashes|code-list|stage-dir|stage-check |test |search-writes|healthcheck|backup |deploy |rollback |build-def|build-files|host-manifest|host-code

|container-manifest|container-code

|tree-diff|backup-sot|sot-session|recover-sot |recover-sot-list|verify-sot-recovery|restore-sot|snapshot-recovery|dry-run snapshot-recovery|verify-recovery-snapshot|build-historical|dry-run build-historical|recreate-historical|dry-run recreate-historical|verify-gold-reproduction|create-build-exclusion|show-build-exclusion" + *) die "Aufruf: ps|inspect|logs|run|code |hashes|code-list|stage-dir|stage-check |test |search-writes|healthcheck|backup |deploy |rollback |build-def|build-files|host-manifest|host-code

|container-manifest|container-code

|tree-diff|backup-sot|sot-session|recover-sot |recover-sot-list|verify-sot-recovery|restore-sot|snapshot-recovery|dry-run snapshot-recovery|verify-recovery-snapshot|build-recovery-gold|dry-run build-recovery-gold|verify-recovery-gold-image|build-historical|dry-run build-historical|recreate-historical|dry-run recreate-historical|verify-gold-reproduction|create-build-exclusion|show-build-exclusion" ;; esac