nexo312/trading-system-docs
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions
trading-system-docs/red-queen-architecture/control-plane/cp2a2_2b/rq-historical

2827 lines
158 KiB
Bash
Executable file
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

#!/bin/bash
# ============================================================================
# rq-historical — eng begrenzter root-Wrapper fuer Red Queen (rq-host)
# RED-QUEEN-VPS-HOST-ACCESS + READONLY-CODE + DEV/TEST/DEPLOY + SOT-DISCOVERY
# | Rain Ocampo (Hermes) | 2026-08-29
#
# LEAST-PRIVILEGE-Design:
# - rq-host hat KEINE docker-Gruppe, KEINEN Docker-Socket, KEIN sudo ALL,
# KEINE Root-Shell. Einziger Root-Kanal: NOPASSWD:/usr/local/bin/rq-historical
# - Dieser Wrapper ist der EINZIGE Kanal zu historical-service & Host-SoT.
# Keine freien Pfade, kein arbitrary cat/find/grep, kein Shell.
#
# SoT-DISCOVERY (read-only, NO copy/rebuild/deploy/fix):
# build-def -> zeigt Dockerfile + Compose-Builddefinition (redacted)
# build-files -> Dockerfile + compose + .dockerignore Auflistung
# host-manifest -> SHA256+size aller *.py unter /opt/historical-v2/app
# host-code <rel> -> gezielter Code-Read NUR aus host-manifest (kontrolliert)
# container-manifest -> SHA256+size aller *.py unter historical-service:/app/app
# container-code <rel> -> gezielter Code-Read NUR aus container-manifest
# tree-diff -> Host vs Container Manifest-Diff
#
# SUBKOMMANDOS (bestehend, unveraendert):
# ps | inspect | logs [--tail N] | run [..] | code <KEY> | hashes | code-list
# stage-dir | stage-check <KEY> | test <KEY> | search-writes | healthcheck
# backup <KEY> | deploy <KEY> | rollback <KEY>
# ============================================================================
set -euo pipefail
CONTAINER=historical-service
HOST_APP=/opt/historical-v2/app
HOST_DIR=/opt/historical-v2
# --------------------------------------------------------- Allowlist (fest)
declare -A TARGET_PATHS=(
[runner]="/app/app/run_backfill_year.py"
[backfill]="/app/app/backfill.py"
[pipeline]="/app/app/pipeline_v2.py"
[repository]="/app/app/persistence/repository.py"
[provenance]="/app/app/trust/provenance.py"
[trust]="/app/app/trust/trust.py"
[sanity]="/app/app/trust/sanity.py"
[trust_gate]="/app/app/trust/trust_gate.py"
)
declare -A STAGE_FILES=(
[runner]="run_backfill_year.py" [backfill]="backfill.py" [pipeline]="pipeline_v2.py"
[repository]="repository.py" [provenance]="provenance.py" [trust]="trust.py"
[sanity]="sanity.py" [trust_gate]="trust_gate.py"
)
STAGING_ROOT=/opt/rq-historical-staging
BACKUP_ROOT=/opt/rq-historical-backups
SCRIPT=/app/app/run_backfill_year.py
# --------------------------------------------------------- HOST-SOT-RECOVERY
# CONFIRMED GOLD CONTAINER -> ROOT-CONTROLLED BACKUP -> EXACT BYTE COPY
# -> HOST SOURCE OF TRUTH -> SHA VERIFY -> STOPP. (Phase 13.5, Extension 1)
# KEIN Build/Recreate in dieser Mission.
SOT_BACKUP_ROOT=/opt/rq-historical-sot-backups
SOT_SESSION_MARKER=".rq-sot-session" # eindeutige Markierung der Recovery-Session
# Recovery-Allowlist: KEY -> "CONTAINER_SRC|HOST_TARGET|EXPECTED_GOLD_SHA"
# Containerpfade aus container-manifest verifiziert (NICHT geraten).
# HINWEIS: Aufgabe nennt trust/init.py, Container hat aber trust/__init__.py
# (Gold-SHA 059db627...). Punkt 5: exakte Pfade aus Manifest, nicht raten.
declare -A RECOVERY=(
[backfill]="/app/app/backfill.py|/opt/historical-v2/app/backfill.py|1627682ae582bb13343ae8ea10491dd9dffaef6fa4f7594d85378c2a500e81c7"
[repository]="/app/app/persistence/repository.py|/opt/historical-v2/app/persistence/repository.py|8fff892f52a612f464b5be21c66ef7787a47f2d7d049f815ef87837553c2a228"
[pipeline]="/app/app/pipeline_v2.py|/opt/historical-v2/app/pipeline_v2.py|7c08ec3d1e75854cf341b9cbfd12afc075c4a173518dd3d951be4f95c5175fe6"
[trust_init]="/app/app/trust/__init__.py|/opt/historical-v2/app/trust/__init__.py|059db62741f2001f548d1879ccd3bf73bb2f4c8d0f492d151a469c9e4dcd3934"
[provenance]="/app/app/trust/provenance.py|/opt/historical-v2/app/trust/provenance.py|4dd2ab977b69e9a47bb744dcc742a94d506e13035157a3977cdc7c8924d5f16f"
[trust]="/app/app/trust/trust.py|/opt/historical-v2/app/trust/trust.py|7c2a948d2f1d668201e5af2a28164bcb2451b31d514a9f5f22b435c9ff2a0d01"
[sanity]="/app/app/trust/sanity.py|/opt/historical-v2/app/trust/sanity.py|875b07869e5148983e7a06bc05f6a6bec30727d1b5e95242d459564c18f6d6e9"
[trust_gate]="/app/app/trust/trust_gate.py|/opt/historical-v2/app/trust/trust_gate.py|df0a97a1fc27985fe49b7458d94f7922bc5eaf2250807043c48dbc173f215ff1"
[diag_bi5]="/app/app/diag_bi5.py|/opt/historical-v2/app/diag_bi5.py|3e07ec5c4d9f572a118c278d0dc5c86865a8fe2067b12f85a3fa958ea4013efa"
[smoke_readonly]="/app/app/smoke_readonly.py|/opt/historical-v2/app/smoke_readonly.py|85ca939c11d789684740833f0bddb6e4f2a548a7f94870662191b75aefdcbfd6"
[smoke_trust_gate]="/app/app/smoke_trust_gate.py|/opt/historical-v2/app/smoke_trust_gate.py|f015fe650d6934b0c71d70bc3011f16b7f4bb0d2ffd947b9127040067442cfff"
[test_trust_gate]="/app/app/test_trust_gate.py|/opt/historical-v2/app/test_trust_gate.py|7867e2b526491de966e64486e8d49ec2cad204d97efc25844562b867ef861114"
[test_phase13_3]="/app/app/test_phase13_3_reproduction.py|/opt/historical-v2/app/test_phase13_3_reproduction.py|de7d2f7be785b7525a045b91ca055dca305fd7d575b291528141f000d5d003e7"
)
# Host-Only-Legacy (Punkt 9): NICHT anfassen
HOST_ONLY_LEGACY=(
"/opt/historical-v2/app/app/backfill.py"
"/opt/historical-v2/app/app/test_idempotency.py"
"/opt/historical-v2/app/eligibility.py"
)
# --------------------------------------------------------- BUILD/RECREATE-KANAL
# Phase 13.5 · Controlled No-op Build/Recreate Channel.
# Build-Pollution-Exclusion via .dockerignore (präzise, NUR Legacy-Artefakte).
# KEIN Businesscode-Fix. KEIN Backfill-Runner in diesem Kanal.
COMPOSE_PROJECT=/opt/historical-v2
COMPOSE_FILE=/opt/historical-v2/docker-compose.yml
HISTORICAL_SERVICE=historical-service
GOLD_IMAGE_ID="sha256:f1a6f33e8714c5cdf017632aa30e71241a8ed670a5c95dac2e8226064b990c81"
GOLD_CONTAINER_ID="634cc505c7bc4c361305e5b415f107e98411faf38e8a29269dd8b1e4d1363b32"
GOLD_MANIFEST="/opt/rq-historical-gold-reference/gold_app_manifest.sha"
BUILD_EXCLUSION_FILE=/opt/historical-v2/.dockerignore
# Präzise Pollution-Exclusion-Einträge (NUR diese 3; keine Gold-Datei betroffen):
# app/app/ -> /app/app/app/ (verschachtelt, im Gold NICHT vorhanden)
# app/eligibility.py -> /app/app/eligibility.py (top-level, Gold: quality/eligibility.py UNBERÜHRT)
BUILD_EXCLUSION_LINES=( "app/app/" "app/eligibility.py" )
# Legacy-Artefakte, die im neuen Container NICHT als EXTRA erscheinen dürfen:
declare -A FORBIDDEN_EXTRA=(
["app/app/app/backfill.py"]=1
["app/app/app/test_idempotency.py"]=1
["app/app/eligibility.py"]=1
)
# ------------------------------------------------- RECOVERY-GOLD-SNAPSHOT-CHANNEL
# Phase 13.5 · Privileged Control-Plane Extension - Recovery Gold Snapshot Only.
# Erfassung eines byte-/state-nahen Recovery-Images aus dem verifizierten
# GOLD-Container 634cc505... (Image f1a6f33e...). KEIN Businesscode, KEIN Build,
# KEIN Recreate, KEIN Runner, KEINE DB-/Datenmutation, KEIN latest-Touch.
RECOVERY_GOLD_CID="634cc505c7bc4c361305e5b415f107e98411faf38e8a29269dd8b1e4d1363b32"
RECOVERY_GOLD_TAG="historical-v2-historical-service:recovery-gold-container-20260829"
# ------------------------------------------------- RECOVERY-GOLD-REBUILD-KANAL
# Phase 13.5 · Build Control Plane Hardening - Recovery-Gold Rebuild Channel.
# Erzeugt aus dem bestaetigten Host-SoT (/opt/historical-v2) ein NEUES, direkt
# dediziert getaggtes Recovery-Gold-Image. KEIN latest, KEIN Recreate, KEIN
# Runner, KEINE User-Args/Build-Args. Fester Context + Dockerfile + Tag.
RECOVERY_REBUILD_TAG="historical-v2-historical-service:recovery-gold-rebuilt-20260829"
# Build Context wird NICHT mehr statisch auf /opt/historical-v2 gesetzt. Er wird
# zur Laufzeit aus dem kontrollierten Control-Plane-State (RECOVERY_SOURCE_STATE)
# gelesen: GOLD_SOURCE_SESSION -> /opt/rq-historical-gold-sources/<SESSION>.
# Keine User-Argumente. FAIL CLOSED wenn State fehlt.
RECOVERY_BUILD_CONTEXT="" # zur Laufzeit aus State gesetzt
RECOVERY_BUILD_DOCKERFILE="" # zur Laufzeit aus State gesetzt
# Critical Files (Punkt 11) mit CONSERVED GOLD SHA (fest, aus laufendem Container)
declare -A CRITICAL_SHA=(
["run_backfill_year.py"]="70d3abb1a92a871e28cc6fb95b9a26944b55dd8df1b72db80642d6881b7b0f20"
["backfill.py"]="1627682ae582bb13343ae8ea10491dd9dffaef6fa4f7594d85378c2a500e81c7"
["repository.py"]="8fff892f52a612f464b5be21c66ef7787a47f2d7d049f815ef87837553c2a228"
["pipeline_v2.py"]="7c08ec3d1e75854cf341b9cbfd12afc075c4a173518dd3d951be4f95c5175fe6"
["trust_init.py"]="059db62741f2001f548d1879ccd3bf73bb2f4c8d0f492d151a469c9e4dcd3934" # trust/__init__.py
["provenance.py"]="4dd2ab977b69e9a47bb744dcc742a94d506e13035157a3977cdc7c8924d5f16f" # trust/provenance.py
["trust.py"]="7c2a948d2f1d668201e5af2a28164bcb2451b31d514a9f5f22b435c9ff2a0d01" # trust/trust.py
["sanity.py"]="875b07869e5148983e7a06bc05f6a6bec30727d1b5e95242d459564c18f6d6e9" # trust/sanity.py
["trust_gate.py"]="df0a97a1fc27985fe49b7458d94f7922bc5eaf2250807043c48dbc173f215ff1" # trust/trust_gate.py
)
# Relative Pfade je Key (fester Pfad, kein User-Parameter)
declare -A CRITICAL_PATH=(
["run_backfill_year.py"]="app/run_backfill_year.py"
["backfill.py"]="app/backfill.py"
["repository.py"]="app/persistence/repository.py"
["pipeline_v2.py"]="app/pipeline_v2.py"
["trust_init.py"]="app/trust/__init__.py"
["provenance.py"]="app/trust/provenance.py"
["trust.py"]="app/trust/trust.py"
["sanity.py"]="app/trust/sanity.py"
["trust_gate.py"]="app/trust/trust_gate.py"
)
# ------------------------------------------------- FINAL-REPRODUCTION-CONTROL-PLANE
# Phase 13.5 · Safe Recreate + Rollback Preparation. KEIN echter Recreate/Rollback
# Recovery-Gold ist ab jetzt IMMUTABLE BASELINE (nur ROLLBACK/RECOVERY-BASE).
# Die aktive Recovery-Image-ID wird NICHT statisch gesetzt, sondern zur Laufzeit
# aus dem kontrollierten Control-Plane-State (RECOVERY_IMAGE_STATE) gelesen.
# HISTORICAL_RECOVERY_IMAGE_ID (d2db...) bleibt NUR Evidence. FAIL CLOSED wenn
# State fehlt (kein freier Input, kein statischer Fallback).
RECOVERY_GOLD_IMAGE_ID="" # zur Laufzeit aus State gesetzt
RECOVERY_GOLD_TAG_FINAL="historical-v2-historical-service:recovery-gold-rebuilt-20260829"
# Candidate Identity Audit (read-only, 29.08.): ausschliesslich ueber latest adressiert.
# 0f6ca29a existiert NICHT mehr (historisch). Aktueller Candidate:
CANDIDATE_IMAGE_ID="sha256:1f3ba210780c5a00d4b73c5ee7817a88671fee2607a4ae0d5b1cefef1a1261c7"
CANDIDATE_TAG="historical-v2-historical-service:candidate-phase13-5-noop"
# ------------------------------------------------- DEPLOYED-GOLD-SOURCE-RECOVERY
# Phase 13.5 · Deployed Gold Source Recovery Control Plane.
# Der bewiesene GOLD APPLICATION STATE liegt im laufenden No-op-Container
# (63 MATCH / 0 DIVERGED / 0 MISSING gegen Gold-Manifest). Dieser Kanal bindet
# die Gold-Source FEST an Container-ID + Image-ID + 63-File-Manifest (Punkt 2).
# KEIN latest, KEIN full_sot_backup(), KEINE freien Inputs. Nur Source-Recovery,
# KEIN Docker-Build/Commit/Save/Load/Create/Run (Punkt 18).
# Volle Container-ID (read-only ermittelt 30.08.): 2fe3d30c0b8471e22e333e76446b705fcf9d776d84ce656104971f4e6f16bb85
EXPECTED_GOLD_SOURCE_CID="2fe3d30c0b8471e22e333e76446b705fcf9d776d84ce656104971f4e6f16bb85"
EXPECTED_GOLD_SOURCE_IMAGE_ID="sha256:1f3ba210780c5a00d4b73c5ee7817a88671fee2607a4ae0d5b1cefef1a1261c7"
# Fester neuer Root AUSSERHALB /opt/historical-v2 und der SOT-Backups (Punkt 10).
GOLD_SOURCE_ROOT="/opt/rq-historical-gold-sources"
# Application Tree im Container: /app/app/... (Gold-Manifest nutzt app/...)
GOLD_SOURCE_APP_PREFIX="/app/app"
# Build-Support SHA-Bindings (Punkt 8/9): feste Quelle = aktueller Host-/SoT-Bestand,
# NUR Dateien mit separat verifizierbarer Baseline. Keine freie Dateiauswahl.
# (read-only ermittelt 30.08. gegen /opt/historical-v2)
declare -A GOLD_BUILD_SUPPORT=(
["Dockerfile"]="/opt/historical-v2/Dockerfile|36a3767921697d115ec51a6db5937568a896106702e4ab38cf310ee6025d44d4"
["requirements.txt"]="/opt/historical-v2/requirements.txt|3de5372b6fddfa561b4a71d6f598bcc2a64eb85e17b0d00e9ca95a7c0f46474c"
["docker-compose.yml"]="/opt/historical-v2/docker-compose.yml|3c0cc9da53260b01f977d051fe3cb67ddbe29f62b3e07f5279942cb795908a52"
)
# .dockerignore wird kontrolliert ergaenzt (Punkt 12.5): exakt die Legacy-Exclusions.
GOLD_BUILD_EXCLUSION_LINES=( "app/app/" "app/eligibility.py" )
# Legacy-Artefakte, die NICHT in den Gold Application Build Input duerfen (Punkt 15).
declare -A GOLD_FORBIDDEN_EXTRA=(
["app/app/backfill.py"]=1
["app/app/test_idempotency.py"]=1
["app/eligibility.py"]=1
)
# ------------------------------------------------- RECOVERY CONTROL-PLANE STATE
# Phase 13.5 · Recovery Control-Plane Rotation. Dedizierter, root-owned State
# (KEIN latest, KEIN glob-first, KEINE Symlink-Semantik). Enthaelt die EINZIGE
# aktive Recovery-Gold-Source-Identitaet + die EINZIGE aktive Recovery-Image-ID.
# Kein freier Pfad. FAIL CLOSED wenn State fehlt/ungueltig.
CONTROL_PLANE_STATE_DIR="/opt/rq-historical-control-plane"
RECOVERY_SOURCE_STATE="$CONTROL_PLANE_STATE_DIR/recovery-gold-source.state"
RECOVERY_IMAGE_STATE="$CONTROL_PLANE_STATE_DIR/recovery-image.state"
# .dockerignore fester EXPECTED SHA (read-only ermittelt 30.08. gegen /opt/historical-v2;
# Inhalt exakt: app/app/ + app/eligibility.py). Kein Schätzen.
GOLD_DOCKERIGNORE_SHA="70bac51783f5ee956796cf2216ab17a550fcb646092edad3504763dbfabb881f"
# Historische Recovery-Image-ID (VERLOREN) — NUR Evidence, KEINE zwingende Output-ID.
HISTORICAL_RECOVERY_IMAGE_ID="sha256:d2db926dc6f5f7cf2f8e85d271a9b16fcc3cfb54b6557604d3d784f7961a4bc2"
# Recovery-Source-State-Felder (zur Laufzeit via load_recovery_source_state gesetzt).
# NUR der SOURCE-State (existiert VOR Build); der IMAGE-State existiert erst NACH Build.
RECOVERY_GOLD_SOURCE_SESSION="" # GOLD_SOURCE_SESSION aus RECOVERY_SOURCE_STATE
RECOVERY_GOLD_BUILD_INPUT_SHA="" # FINAL_BUILD_INPUT_SHA aus RECOVERY_SOURCE_STATE
RECOVERY_GOLD_SOURCE_CONTAINER_ID="" # SOURCE_CONTAINER_ID aus RECOVERY_SOURCE_STATE
RECOVERY_GOLD_SOURCE_IMAGE_ID="" # SOURCE_IMAGE_ID aus RECOVERY_SOURCE_STATE
# ------------------------------------------------- PERSISTENT-HOST-SOT-PROMOTION
# Phase 13.5 · Persistent Host-SoT Patch Promotion Channel. KEIN echter Promote
# durch Rain. Nur run_backfill_year.py. Feste Identitaeten, KEIN User-Input.
# PATCH_SOURCE=/opt/data/run_backfill_year.py existiert NUR im RQ-Container
# (hermes-red-queen), nicht auf dem VPS-Host -> fester docker cp aus dem Container.
PATCH_SOURCE_CONTAINER="hermes-red-queen"
PATCH_SOURCE_PATH="/opt/data/run_backfill_year.py"
PATCH_TARGET="/opt/historical-v2/app/run_backfill_year.py"
# Read-only ermittelt (29.08.): SHA256 des von Red Queen geprueften Patch-Artefakts (8292 B).
EXPECTED_RUNNER_PATCH_SHA256="6cca3c55bb559d75be2f58e59c231003902f124a54ada25582970524599f0752"
# Fester Backup-Root fuer die Promotion-Session (root-owned, nicht world-readable).
PROMOTE_BACKUP_ROOT="/opt/rq-historical-sot-backups/pre-runner-remediation"
# ------------------------------------------------- REMEDIATION BUILD/DEPLOY-KANAL
# Phase 13.5 · Controlled Runner-Remediation Build/Deploy Channel.
# KEIN echter Build/Recreate durch Rain; nur Kanal. Red Queen fuehrt nach Freigabe aus.
# Fester Remediation-Tag (Punkt 3) — KEIN Overwrite, KEIN force-retag, KEIN latest.
REMEDIATION_TAG="historical-v2-historical-service:candidate-phase13-5-runner-remediation"
# Build-Session-Basis (root-owned; enthaelt nach Build session.meta mit FULL_IMAGE_ID).
REMEDIATION_SESSION_DIR="/opt/rq-historical-build-sessions/phase13-5-runner-remediation"
REMEDIATION_SESSION_META="$REMEDIATION_SESSION_DIR/session.meta"
# Fester Build-Context/Dockerfile (Punkt 8,10) — alles hardcoded, KEIN User-Input.
REMEDIATION_BUILD_CONTEXT="/opt/historical-v2"
REMEDIATION_BUILD_DOCKERFILE="/opt/historical-v2/Dockerfile"
# Expected Runner-SHAs (Punkt 10/11): ALT = Gold (vor Remediation), NEU = persistent (nach).
RUNNER_GOLD_SHA="70d3abb1a92a871e28cc6fb95b9a26944b55dd8df1b72db80642d6881b7b0f20"
RUNNER_REMEDIATION_SHA="6cca3c55bb559d75be2f58e59c231003902f124a54ada25582970524599f0752"
# Recovery-Gold (immutable Baseline) — Referenz fuer Expected-Gold-Delta.
# Candidate/Recovery duerfen code-identisch sein, aber verschiedene Image-IDs haben (KEIN Fehler).
REM_NOOP_IMAGE_ID="sha256:1f3ba210780c5a00d4b73c5ee7817a88671fee2607a4ae0d5b1cefef1a1261c7"
# ------------------------------------------------- IMAGE-RETENTION / ARCHIVE CONTROL PLANE
# Phase 13.5 · Schutzschicht gegen bestaetigten Coolify DockerCleanupJob
# (taegl. 00:00 UTC, force_docker_cleanup=true, rmi auf getaggte Images ohne
# coolify.managed-Label). ZWEI unabhaengige Ebenen je Critical Image:
# SCHICHT A = Stopped Pin Container (Image-Referenz via docker create, nie gestartet)
# SCHICHT B = Persistentes, sha256-verifiziertes docker save-Archiv
# Kein Coolify-Cleanup-Disable. Kein coolify.managed-Spoof. Kein Overwrite.
# Fester Archive-Root (Punkt 3) — root:root, 700, KEINE freien Pfade.
ARCHIVE_ROOT="/opt/rq-historical-image-archives"
# Pin-Container-Namen (Punkt 9) — fest, hardcoded, KEIN freier Name.
PIN_RECOVERY_NAME="rq-historical-pin-recovery-gold"
PIN_REMEDIATION_NAME="rq-historical-pin-phase13-5-runner-remediation"
# Unterordner je Rolle (Punkt 3)
declare -A ARCHIVE_SUB=( [recovery]="recovery" [remediation]="candidates" )
# Archiv-Session-IDs: kurze Image-ID (<IMAGE_ID_SHORT>), die den immutable
# Session-Ordner benennt. Recovery = aktive Image-ID aus Control-Plane-State
# (zur Laufzeit via load_recovery_state gesetzt); Remediation = FULL_IMAGE_ID
# aus Build-Session (41e357a1..., aktuell im Store fehlend).
RECOVERY_ARCHIVE_ID="" # zur Laufzeit aus State gesetzt (load_recovery_state)
# Remediation-Archiv-Session-ID: kurze ID der Build-Session-Remendation (41e357a1...).
REMEDIATION_ARCHIVE_ID="41e357a12b90"
# Remediation-Archiv-Referenz (aus Build-Session falls vorhanden, sonst leer):
# wird zur Laufzeit per remediation_full_image_id() aufgeloest.
# Pin-Referenz-Ziel (Schicht A): exakt diese Image-ID muss der Pin-Container referenzieren.
PIN_REMEDIATION_TARGET_ID="" # zur Laufzeit gesetzt
# RUN_MODE (Mutation Guard): READ_ONLY | DRY_RUN | MUTATING
RUN_MODE=MUTATING
die() { echo "ERR: $*" >&2; exit 2; }
req_ok() {
docker inspect -f '{{.State.Running}}' "$CONTAINER" >/dev/null 2>&1 \
|| die "Container '$CONTAINER' nicht vorhanden oder nicht am Laufen"
}
key_path() { echo "${TARGET_PATHS[$1]:-}"; }
stage_file() { echo "$STAGING_ROOT/${STAGE_FILES[$1]:-}"; }
key_ok() {
[ -n "${TARGET_PATHS[$1]:-}" ] && [ -n "${STAGE_FILES[$1]:-}" ] || die "unbekannter KEY '$1' (code-list)"
}
expect_zero() { [ $# -eq 0 ] || die "unerlaubtes Argument '$*'"; }
# ================= BUILD/RECREATE-KANAL HILFSFUNKTIONEN (alle fail-closed) ===
# MODE ist global: MUTATING (Default) | DRY_RUN (keine Mutationen)
mode_or_die() { [ "$RUN_MODE" != "DRY_RUN" ] || die "DRY_RUN-Modus, aber mutierender Pfad erreicht (FAIL CLOSED)"; }
# ---- Full/Partial-SoT-Backup-Klassifikation (Phase 13.5 Remediation, Blocker A) ----
# Ein Partial-Backup (z.B. pre-runner-remediation mit nur session.meta +
# run_backfill_year.py) darf NIEMALS als Full-SoT-Snapshot gelten.
# Ein gueltiger FULL SOT SNAPSHOT muss zwingend haben:
# historical-v2.tar (regular, kein Symlink, nicht leer)
# SHA256.manifest (regular, kein Symlink, nicht leer)
# und optional session.meta / provenance. Read-only.
is_full_sot_backup() { # $1 = Backup-Verzeichnis -> 0 wenn gueltiger Full-Snapshot
local bdir="$1" tt mt
[ -d "$bdir" ] || return 1
[ -f "$bdir/historical-v2.tar" ] || return 1
[ -s "$bdir/historical-v2.tar" ] || return 1
[ -f "$bdir/SHA256.manifest" ] || return 1
[ -s "$bdir/SHA256.manifest" ] || return 1
[ -L "$bdir/historical-v2.tar" ] && return 1
[ -L "$bdir/SHA256.manifest" ] && return 1
tt=$(stat -c '%F' "$bdir/historical-v2.tar" 2>/dev/null)
mt=$(stat -c '%F' "$bdir/SHA256.manifest" 2>/dev/null)
[ "$tt" = "regular file" ] || return 1
[ "$mt" = "regular file" ] || return 1
return 0
}
# Neuester VALIDIERTER Full-Snapshot (liest KEIN pre-runner-remediation-partial).
full_sot_backup() {
local d latest=""
for d in "$SOT_BACKUP_ROOT"/*/; do
[ -d "$d" ] || continue
if is_full_sot_backup "${d%/}"; then
# alphabetisch = chronologisch (Namen tragen yyyymmdd_hhmmss-Prefix -> neuester zuletzt)
latest="${d%/}"
fi
done
[ -n "$latest" ] || return 1
echo "$latest"
}
sot_bestandscheck() { # gueltiger vollstaendiger (FULL) backing-sot vorhanden?
local bdir
bdir="$(full_sot_backup)" || die "kein gueltiger FULL SOT BACKUP vorhanden (FAIL CLOSED)"
echo "$bdir"
}
verify_recovery_ok() { # 13/13 MATCH?
local allok=1 k csrc htarget gold hsha csha
for k in "${!RECOVERY[@]}"; do
IFS='|' read -r csrc htarget gold <<< "${RECOVERY[$k]}"
hsha="$(sha256sum "$htarget" 2>/dev/null | awk '{print $1}')" || hsha=""
csha="$(docker exec "$CONTAINER" sha256sum "$csrc" 2>/dev/null | awk '{print $1}')" || csha=""
if [ "$hsha" != "$gold" ] || [ "$csha" != "$gold" ]; then allok=0; echo " FEHLT: $k host=$hsha container=$csha"; fi
done
[ "$allok" = "1" ] || die "verify-sot-recovery NICHT 13/13 MATCH (FAIL CLOSED)"
}
ensure_gold_image() { docker image inspect "$GOLD_IMAGE_ID" >/dev/null 2>&1 || die "Gold-Image $GOLD_IMAGE_ID fehlt (FAIL CLOSED)"; }
ensure_gold_run() { docker inspect -f '{{.State.Running}}' "$HISTORICAL_SERVICE" >/dev/null 2>&1 || die "Gold-Container $HISTORICAL_SERVICE läuft nicht (FAIL CLOSED)"; }
ensure_db_run() { docker inspect -f '{{.State.Running}}' historical-db >/dev/null 2>&1 || die "historical-db läuft nicht (FAIL CLOSED)"; }
ensure_db_volume() { docker inspect -f '{{ range .Mounts }}{{ .Name }} {{ end }}' historical-db 2>/dev/null | grep -q 'historical-db-data' || die "DB-Volume historical-db-data fehlt (FAIL CLOSED)"; }
ensure_builddef() { [ -f "$COMPOSE_FILE" ] || die "Compose fehlt: $COMPOSE_FILE"; [ -f "$COMPOSE_PROJECT/Dockerfile" ] || die "Dockerfile fehlt"; }
exclusion_valid() { # .dockerignore exakt die 3 Legacy-Einträge, keine Gold-Datei ausgeschlossen
[ -f "$BUILD_EXCLUSION_FILE" ] || die ".dockerignore fehlt (FAIL CLOSED)"
local line
while IFS= read -r line; do
[ -z "$line" ] && continue
case "$line" in
app/app/|app/eligibility.py) ;;
*) die "unerwartete Exclusion '$line' im .dockerignore (FAIL CLOSED)" ;;
esac
done < "$BUILD_EXCLUSION_FILE"
}
# ------------------------------------------------- RECOVERY CONTROL-PLANE STATE LOAD
# Laedt die EINZIGE aktive Recovery-Identitaet aus dem kontrollierten State
# (KEIN latest, KEIN glob-first, KEINE Symlink-Semantik). Setzt zur Laufzeit:
# RECOVERY_GOLD_IMAGE_ID (aus RECOVERY_IMAGE_STATE: NEW_RECOVERY_FULL_IMAGE_ID)
# RECOVERY_ARCHIVE_ID (abgeleitet aus der geladenen Image-ID)
# RECOVERY_BUILD_CONTEXT (aus RECOVERY_SOURCE_STATE: GOLD_SOURCE_SESSION)
# RECOVERY_BUILD_DOCKERFILE
# FAIL CLOSED wenn State fehlt/ungueltig. Kein freier Input, kein statischer Fallback.
load_recovery_state() {
# --- Recovery-Image-State (Punkt 17: NEW_RECOVERY_FULL_IMAGE_ID) ---
if [ ! -f "$RECOVERY_IMAGE_STATE" ]; then
die "load-recovery-state: Kein Recovery-Image-State ($RECOVERY_IMAGE_STATE). NUR nach erfolgreichem Recovery-Build (FAIL CLOSED)"
fi
local id
id="$(awk -F'=' '$1=="NEW_RECOVERY_FULL_IMAGE_ID"{print $2}' "$RECOVERY_IMAGE_STATE" 2>/dev/null)"
[[ "$id" =~ ^sha256:[a-f0-9]{64}$ ]] || die "load-recovery-state: NEW_RECOVERY_FULL_IMAGE_ID ungueltig/fehlt (FAIL CLOSED)"
RECOVERY_GOLD_IMAGE_ID="$id"
RECOVERY_ARCHIVE_ID="${id#sha256:}"; RECOVERY_ARCHIVE_ID="${RECOVERY_ARCHIVE_ID:0:12}"
# --- Recovery-Source-State (Punkt 9: GOLD_SOURCE_SESSION) ---
if [ ! -f "$RECOVERY_SOURCE_STATE" ]; then
die "load-recovery-state: Kein Recovery-Source-State ($RECOVERY_SOURCE_STATE). NUR nach realem recover (FAIL CLOSED)"
fi
local sess
sess="$(awk -F'=' '$1=="GOLD_SOURCE_SESSION"{print $2}' "$RECOVERY_SOURCE_STATE" 2>/dev/null)"
[[ "$sess" =~ ^[0-9]{8}_[0-9]{6}$ ]] || die "load-recovery-state: GOLD_SOURCE_SESSION ungueltig/fehlt (FAIL CLOSED)"
RECOVERY_BUILD_CONTEXT="$GOLD_SOURCE_ROOT/$sess"
RECOVERY_BUILD_DOCKERFILE="$GOLD_SOURCE_ROOT/$sess/Dockerfile"
}
# load_recovery_source_state(): laedt NUR den Recovery-SOURCE-State (existiert VOR Build).
# Setzt RECOVERY_BUILD_CONTEXT/DOCKERFILE + die Source-Identitaets-Felder.
# Verlangt NICHT den Recovery-IMAGE-State (der existiert erst NACH Build).
# FAIL CLOSED wenn SOURCE-State fehlt/ungueltig. Kein freier Input, kein statischer Fallback.
load_recovery_source_state() {
if [ ! -f "$RECOVERY_SOURCE_STATE" ]; then
die "load-recovery-source-state: Kein Recovery-Source-State ($RECOVERY_SOURCE_STATE). NUR nach realem recover (FAIL CLOSED)"
fi
local sess sha cid iid
sess="$(awk -F'=' '$1=="GOLD_SOURCE_SESSION"{print $2}' "$RECOVERY_SOURCE_STATE" 2>/dev/null)"
[[ "$sess" =~ ^[0-9]{8}_[0-9]{6}$ ]] || die "load-recovery-source-state: GOLD_SOURCE_SESSION ungueltig/fehlt (FAIL CLOSED)"
sha="$(awk -F'=' '$1=="FINAL_BUILD_INPUT_SHA"{print $2}' "$RECOVERY_SOURCE_STATE" 2>/dev/null)"
[[ "$sha" =~ ^[a-f0-9]{64}$ ]] || die "load-recovery-source-state: FINAL_BUILD_INPUT_SHA ungueltig/fehlt (FAIL CLOSED)"
cid="$(awk -F'=' '$1=="SOURCE_CONTAINER_ID"{print $2}' "$RECOVERY_SOURCE_STATE" 2>/dev/null)"
[[ "$cid" =~ ^[a-f0-9]{64}$ ]] || die "load-recovery-source-state: SOURCE_CONTAINER_ID ungueltig/fehlt (FAIL CLOSED)"
iid="$(awk -F'=' '$1=="SOURCE_IMAGE_ID"{print $2}' "$RECOVERY_SOURCE_STATE" 2>/dev/null)"
[[ "$iid" =~ ^sha256:[a-f0-9]{64}$ ]] || die "load-recovery-source-state: SOURCE_IMAGE_ID ungueltig/fehlt (FAIL CLOSED)"
RECOVERY_GOLD_SOURCE_SESSION="$sess"
RECOVERY_GOLD_BUILD_INPUT_SHA="$sha"
RECOVERY_GOLD_SOURCE_CONTAINER_ID="$cid"
RECOVERY_GOLD_SOURCE_IMAGE_ID="$iid"
RECOVERY_BUILD_CONTEXT="$GOLD_SOURCE_ROOT/$sess"
RECOVERY_BUILD_DOCKERFILE="$GOLD_SOURCE_ROOT/$sess/Dockerfile"
}
# write_recovery_image_state(): atomarer Write des Recovery-IMAGE-State NACH erfolgreichem Build.
# $1 = FULL_IMAGE_ID (sha256:...). Verlangt: State ABSENT (kein Overwrite, FAIL CLOSED).
# Schreibt via Temp-Datei im selben Filesystem + fsync + atomares mv. Kein partial state sichtbar.
write_recovery_image_state() {
local fullid="$1" tmp
[[ "$fullid" =~ ^sha256:[a-f0-9]{64}$ ]] || die "write-recovery-image-state: FULL_IMAGE_ID ungueltig (FAIL CLOSED)"
mkdir -p "$CONTROL_PLANE_STATE_DIR" || die "write-recovery-image-state: State-Dir nicht anlegbar (FAIL CLOSED)"
if [ -e "$RECOVERY_IMAGE_STATE" ]; then
die "write-recovery-image-state: Recovery-Image-State existiert bereits $RECOVERY_IMAGE_STATE (KEIN Overwrite, FAIL CLOSED)"
fi
tmp="$CONTROL_PLANE_STATE_DIR/.recovery-image.state.tmp.$$"
{
echo "NEW_RECOVERY_FULL_IMAGE_ID=$fullid"
echo "RECOVERY_TAG=$RECOVERY_REBUILD_TAG"
echo "BUILD_TIMESTAMP=$(date -u +%Y-%m-%dT%H:%M:%SZ)"
echo "BUILD_INPUT_SHA=$RECOVERY_GOLD_BUILD_INPUT_SHA"
echo "SOURCE_SESSION=$RECOVERY_GOLD_SOURCE_SESSION"
echo "SOURCE_CONTAINER_ID=$RECOVERY_GOLD_SOURCE_CONTAINER_ID"
echo "SOURCE_IMAGE_ID=$RECOVERY_GOLD_SOURCE_IMAGE_ID"
echo "WRAPPER_SHA=$(sha256sum /usr/local/bin/rq-historical | awk '{print $1}')"
} > "$tmp" || { rm -f "$tmp"; die "write-recovery-image-state: Temp-Write fehlgeschlagen (FAIL CLOSED)"; }
chown root:root "$tmp" 2>/dev/null || true
chmod 600 "$tmp" || { rm -f "$tmp"; die "write-recovery-image-state: chmod fehlgeschlagen (FAIL CLOSED)"; }
sync -f "$tmp" 2>/dev/null || true
mv -f "$tmp" "$RECOVERY_IMAGE_STATE" || { rm -f "$tmp"; die "write-recovery-image-state: atomares mv fehlgeschlagen (FAIL CLOSED)"; }
sync -f "$RECOVERY_IMAGE_STATE" 2>/dev/null || true
}
# Manifest einer Quelle (Gold-Container / neuer Temp-Container) erzeugen:
# gibt zeilenweise 'sha app/<rel>' auf stdout; Quelle ist IMMER laufender Container
gen_manifest() { # $1 = docker-Container-Name
docker exec "$1" sh -c "cd /app && find app -type f -name '*.py' ! -path '*/__pycache__/*' 2>/dev/null | sort | while IFS= read -r f; do sha256sum \"\$f\" 2>/dev/null; done"
}
classify_and_report() { # $1 = neues Manifest Datei (Temp); vergleicht gegen GOLD_MANIFEST + FORBIDDEN_EXTRA
local new="$1" gold goldrel goldsha nrel nsha status
local d=0 m=0 x=0 mat=0 l=0
# Gold-Dateien
while read -r goldsha goldrel; do
goldrel="${goldrel#./}"
if [ -n "$goldrel" ]; then
l=$((l+1))
nsha="$(awk -v r="$goldrel" '$2==r{print $1}' "$new")"
if [ -z "$nsha" ]; then echo " MISSING $goldrel"; m=$((m+1));
elif [ "$nsha" = "$goldsha" ]; then mat=$((mat+1));
else echo " DIVERGED $goldrel (gold=$goldsha new=$nsha)"; d=$((d+1)); fi
fi
done < "$GOLD_MANIFEST"
# neue Dateien
while read -r nsha nrel; do
nrel="${nrel#./}"
found="$(awk -v r="$nrel" '$2==r{print $1}' "$GOLD_MANIFEST")"
if [ -z "$found" ]; then
echo " EXTRA $nrel"
if [ -n "${FORBIDDEN_EXTRA[$nrel]:-}" ]; then x=$((x+1)); FORBIDDEN_HIT=1; else x=$((x+1)); fi
fi
done < "$new"
echo "--- Gold $l Dateien: MATCH=$mat DIVERGED=$d MISSING=$m ; EXTRA=$x ---"
G_RET="ok"; [ $d -eq 0 ] && [ $m -eq 0 ] && [ $x -eq 0 ] || G_RET="fail"
}
# ================= RECOVERY-SNAPSHOT HILFSFUNKTIONEN (fail-closed) ============
# PRE-CONDITION-GUARDS fuer docker commit (Punkt 4). Fuehrt ALLE 14 Guards aus;
# bei jedem FAIL: die. Bei DRY_RUN ebenso (read-only check, nur anzeigen).
snapshot_guards() {
echo " [guard 1] Container-ID == 634cc505...:"
local cid img running mounts dbid cimg rh rr
cid="$(docker inspect -f '{{.ID}}' "$HISTORICAL_SERVICE" 2>/dev/null || echo NONE)"
[ "$cid" = "$RECOVERY_GOLD_CID" ] || die "Guard1 FAIL: Container-ID $cid != RECOVERY_GOLD_CID (FAIL CLOSED)"
echo " OK ($cid)"
echo " [guard 2] Running=true:"
running="$(docker inspect -f '{{.State.Running}}' "$HISTORICAL_SERVICE" 2>/dev/null)"
[ "$running" = "true" ] || die "Guard2 FAIL: nicht Running (FAIL CLOSED)"
echo " OK ($running)"
echo " [guard 3] Image-Metadata == f1a6f33e...:"
img="$(docker inspect -f '{{.Image}}' "$HISTORICAL_SERVICE" 2>/dev/null)"
[ "$img" = "$GOLD_IMAGE_ID" ] || die "Guard3 FAIL: Image $img != GOLD_IMAGE_ID (FAIL CLOSED)"
echo " OK ($img)"
echo " [guard 4] /health=200:"
rh="$(docker exec "$HISTORICAL_SERVICE" python -c 'import urllib.request;print(urllib.request.urlopen("http://localhost:8000/health",timeout=5).status)' 2>/dev/null || echo N/A)"
[ "$rh" = "200" ] || die "Guard4 FAIL: /health=$rh (FAIL CLOSED)"
echo " OK ($rh)"
echo " [guard 5] /health/ready=200:"
rr="$(docker exec "$HISTORICAL_SERVICE" python -c 'import urllib.request;print(urllib.request.urlopen("http://localhost:8000/health/ready",timeout=5).status)' 2>/dev/null || echo N/A)"
[ "$rr" = "200" ] || die "Guard5 FAIL: /health/ready=$rr (FAIL CLOSED)"
echo " OK ($rr)"
echo " [guard 6] Mounts == []:"
mounts="$(docker inspect -f '{{json .Mounts}}' "$HISTORICAL_SERVICE" 2>/dev/null)"
[ "$mounts" = "[]" ] || die "Guard6 FAIL: Mounts != [] (FAIL CLOSED)"
echo " OK ([$mounts])"
echo " [guard 7] historical-db laeuft unveraendert:"
dbid="$(docker inspect -f '{{.ID}}' historical-db 2>/dev/null || echo NONE)"
cimg="$(docker inspect -f '{{.Image}}' historical-db 2>/dev/null)"
dbr="$(docker inspect -f '{{.State.Running}}' historical-db 2>/dev/null)"
[ "$dbr" = "true" ] || die "Guard7 FAIL: historical-db Running=$dbr (FAIL CLOSED)"
echo " OK (ID=$dbid Image=$cimg Running=$dbr)"
echo " [guard 8-11] 63/63 Gold-Code-MATCH / 0 DIVERGED / 0 MISSING / 0 unerlaubte EXTRA:"
local tmp; tmp="$(mktemp)"
gen_manifest "$HISTORICAL_SERVICE" > "$tmp"
classify_and_report "$tmp"
local gret="$G_RET"; rm -f "$tmp"
[ "$gret" = "ok" ] || die "Guard8-11 FAIL: GOLD-REPRODUCTION nicht OK (FAIL CLOSED)"
echo " OK (GOLD REPRODUCTION OK)"
echo " [guard 12] Critical-File-SHAs == konservierter Gold-Stand (verify-sot-recovery):"
verify_recovery_ok
echo " OK (13/13 MATCH)"
echo " [guard 13] kein Runner/Backfill aktiv:"
docker exec "$HISTORICAL_SERVICE" sh -c 'ps -ef 2>/dev/null | grep -E "run_backfill_year|backfill" | grep -v grep | grep -v "\-\-force" || true' | grep -q "run_backfill_year\|backfill" \
&& die "Guard13 FAIL: Runner/Backfill-Prozess aktiv (FAIL CLOSED)"
echo " OK (kein Runner/Backfill aktiv)"
echo " [guard 14] Recovery-Zieltag existiert NOCH NICHT:"
if docker image inspect "$RECOVERY_GOLD_TAG" >/dev/null 2>&1; then
die "Guard14 FAIL: Recovery-Tag $RECOVERY_GOLD_TAG existiert bereits (FAIL CLOSED, kein Overwrite)"
fi
echo " OK (Tag frei)"
echo " ALLE 14 GUARDS OK"
}
# snapshot-recovery (MUTATION - nur Red Queen fuehrt aus; dry-run blockt via mode_or_die)
snapshot_recovery() {
expect_zero "$@"
echo "=== SNAPSHOT-RECOVERY ($RUN_MODE) ==="
echo " Source Container-ID : $RECOVERY_GOLD_CID (== historical-service)"
echo " Erwartete Image : $GOLD_IMAGE_ID"
echo " Zieltag : $RECOVERY_GOLD_TAG (fester Tag, kein Overwrite)"
echo " Guards:"
snapshot_guards
if [ "$RUN_MODE" = "DRY_RUN" ]; then
echo "DRY_RUN_PASS: snapshot-recovery wuerde docker commit $RECOVERY_GOLD_CID -> $RECOVERY_GOLD_TAG (keine Aktion)."
exit 0
fi
mode_or_die # DRY_RUN -> FAIL CLOSED (kein commit)
echo " [commit] docker commit $RECOVERY_GOLD_CID -> $RECOVERY_GOLD_TAG (Standard-Pause durch Docker)"
docker commit "$RECOVERY_GOLD_CID" "$RECOVERY_GOLD_TAG" >/dev/null || die "docker commit fehlgeschlagen (FAIL CLOSED)"
# Nach Commit: Gold-Container wieder Running und healthy pruefen
local running
running="$(docker inspect -f '{{.State.Running}}' "$HISTORICAL_SERVICE" 2>/dev/null)"
[ "$running" = "true" ] || die "Nach-Commit: Gold-Container nicht Running (FAIL CLOSED)"
local newid
newid="$(docker image inspect -f '{{.Id}}' "$RECOVERY_GOLD_TAG" 2>/dev/null)"
[ -n "$newid" ] || die "Nach-Commit: Recovery-Image nicht auffindbar (FAIL CLOSED)"
echo "RECOVERY_IMAGE_ID=$newid"
echo "KLASSIFIZIERUNG: NEW RECOVERY IMAGE DERIVED FROM VERIFIED GOLD CONTAINER (NICHT das Original-Gold-Image)."
echo "Original bleibt: $GOLD_IMAGE_ID"
echo "SNAPSHOT-RECOVERY OK. KEINE weitere Mutation."
}
# verify-recovery-snapshot (read-only): Recovery-Image-Code vs Gold-Code vs Gold-Manifest
verify_recovery_snapshot() {
req_ok; expect_zero "$@"
echo "=== VERIFY-RECOVERY-SNAPSHOT (Recovery-Image vs Gold-Container vs Gold-Manifest) ==="
[ -f "$GOLD_MANIFEST" ] || die "Gold-Manifest fehlt (FAIL CLOSED)"
# Recovery-Tag muss existieren
if ! docker image inspect "$RECOVERY_GOLD_TAG" >/dev/null 2>&1; then
die "Recovery-Tag $RECOVERY_GOLD_TAG existiert nicht (FAIL CLOSED — zuerst snapshot-recovery)"
fi
# 1. Tag + Immutable-ID zeigen auf exakt dasselbe Image (Punkt 9)
# tagid = ID aus Tag-Referenz; immid = ID aus der vollen Immutable-Referenz (sha256:<ID>)
local tagid immid
tagid="$(docker image inspect -f '{{.Id}}' "$RECOVERY_GOLD_TAG" 2>/dev/null)"
[ -n "$tagid" ] || die "Verify: Recovery-Tag-ID nicht aufloesbar (FAIL CLOSED)"
immid="$(docker inspect "sha256:$tagid" --format '{{.Id}}' 2>/dev/null)"
[ "$tagid" = "$immid" ] || die "Verify: Tag/Immutable-ID-Referenz stimmen nicht ueberein (Punkt 9 FAIL CLOSED)"
echo " [Punkt 9] Tag + Immutable-ID -> selbe Image-ID: $tagid (OK)"
# 2. Manifest vom Recovery-Image ueber temporaeren Container (kein Entrypoint, keine Volumes/Ports/Netzwerk)
local tmpc tmpf
tmpf="$(mktemp)"
# Container-2 (nicht laufend) aus Recovery-Image erzeugen, NUR fuer Datei-Manifest; kein Start
tmpc="rq-recv-verify-$(date +%s)"
docker create --name "$tmpc" --entrypoint /bin/true "$RECOVERY_GOLD_TAG" >/dev/null 2>&1 \
|| { rm -f "$tmpf"; die "Verify: Temp-Container konnte nicht erstellt werden (FAIL CLOSED)"; }
# Manifest via docker cp aus dem erzeugten (nicht gestarteten) Container lesen
docker cp "$tmpc:/app/app" "/tmp/rq-recv-verify-copy-$$" >/dev/null 2>&1 \
|| { docker rm -f "$tmpc" >/dev/null 2>&1; rm -f "$tmpf"; rm -rf "/tmp/rq-recv-verify-copy-$$" 2>/dev/null; die "Verify: Manifest-Lesen aus Recovery-Image fehlgeschlagen (FAIL CLOSED)"; }
( cd "/tmp/rq-recv-verify-copy-$$" && find . -type f -name '*.py' -printf '%P\n' 2>/dev/null | sort | while IFS= read -r rel; do
sha256sum "$rel" 2>/dev/null | awk -v r="$rel" '{print $1" "r}'
done ) > "$tmpf"
docker rm -f "$tmpc" >/dev/null 2>&1 || true
rm -rf "/tmp/rq-recv-verify-copy-$$" 2>/dev/null
classify_and_report "$tmpf"
local ret="$G_RET"; rm -f "$tmpf"
if [ "$ret" = "ok" ]; then
echo "VERDICT: RECOVERY GOLD SNAPSHOT OK (Recovery-Image == Gold-Code, 0 DIVERGED/MISSING/unerlaubte EXTRA)"
exit 0
else
echo "VERDICT: RECOVERY GOLD SNAPSHOT FAIL"
exit 2
fi
}
# ================= RECOVERY-GOLD-REBUILD (build-recovery-gold) ===============
# Erstellt Build-Context-Manifest (read-only) fuer RECOVERY_GOLD_BUILD_INPUT.
# Erfasst Dockerfile, requirements.txt, .dockerignore + alle app/* Dateien.
build_context_manifest() {
local ctx="$RECOVERY_BUILD_CONTEXT" f sha rel
# Dockerfile
[ -f "$ctx/Dockerfile" ] || die "build-context: Dockerfile fehlt (FAIL CLOSED)"
echo "Dockerfile $(sha256sum "$ctx/Dockerfile" | awk '{print $1}')"
# requirements.txt (falls vorhanden)
[ -f "$ctx/requirements.txt" ] && echo "requirements.txt $(sha256sum "$ctx/requirements.txt" | awk '{print $1}')"
# .dockerignore (falls vorhanden)
[ -f "$ctx/.dockerignore" ] && echo ".dockerignore $(sha256sum "$ctx/.dockerignore" | awk '{print $1}')"
# app/* (reguläre Dateien, .py + sonstiges, keine __pycache__)
if [ -d "$ctx/app" ]; then
find "$ctx/app" -type f -not -name '*.pyc' -not -path '*/__pycache__/*' -printf '%P\n' 2>/dev/null | sort | while IFS= read -r f; do
[ -f "$ctx/app/$f" ] || continue
sha="$(sha256sum "$ctx/app/$f" | awk '{print $1}')"
printf 'app/%s %s\n' "$f" "$sha"
done
fi
}
# Dockerfile "exakt erwartet": Basis + COPY-Semantik (Punkt 4 Guard)
dockerfile_expected() {
local f="$RECOVERY_BUILD_DOCKERFILE"
grep -qE '^FROM python:3\.11-slim' "$f" || die "Dockerfile: erwartetes Base-Image python:3.11-slim fehlt (FAIL CLOSED)"
grep -qE '^COPY app/ \./app/' "$f" || die "Dockerfile: erwartetes COPY app/ ./app/ fehlt (FAIL CLOSED)"
grep -qE '^EXPOSE 8000' "$f" || die "Dockerfile: erwartetes EXPOSE 8000 fehlt (FAIL CLOSED)"
}
# Alle Guards fuer einen Recovery-Rebuild (read-only, nutzbar im dry-run und build)
# Host-Guard: alle 63 Gold-Dateien MATCH (0 DIVERGED/0 MISSING) UND jede EXTRA
# ist eine der 3 erlaubten HOST_ONLY_LEGACY (via .dockerignore ausgeschlossen).
# Anders als classify_and_report (fuer Image/Container) duerfen diese 3 auf dem
# Host physisch existieren — der Build schliesst sie per .dockerignore aus.
verify_host_gold_context() {
local hm tmpf ok d=0 m=0 x=0 mat=0 l=0 goldrel goldsha nrel nsha
hm="$(mktemp)"; gen_host_manifest > "$hm"
while read -r goldsha goldrel; do
goldrel="${goldrel#./}"; [ -n "$goldrel" ] || continue
l=$((l+1))
nsha="$(awk -v r="$goldrel" '$2==r{print $1}' "$hm")"
if [ -z "$nsha" ]; then echo " MISSING $goldrel"; m=$((m+1));
elif [ "$nsha" = "$goldsha" ]; then mat=$((mat+1));
else echo " DIVERGED $goldrel"; d=$((d+1)); fi
done < "$GOLD_MANIFEST"
while read -r nsha nrel; do
nrel="${nrel#./}"
found="$(awk -v r="$nrel" '$2==r{print $1}' "$GOLD_MANIFEST")"
if [ -z "$found" ]; then
case "$nrel" in
app/app/backfill.py|app/app/test_idempotency.py|app/eligibility.py)
echo " EXTRA(erlaubte Legacy) $nrel" ;;
*) echo " EXTRA(unerlaubt) $nrel"; x=$((x+1)) ;;
esac
fi
done < "$hm"
rm -f "$hm"
echo " Gold $l: MATCH=$mat DIVERGED=$d MISSING=$m ; unerlaubte EXTRA=$x"
[ $d -eq 0 ] && [ $m -eq 0 ] && [ $x -eq 0 ]
}
# verify_recovery_build_input(): beweist den tatsaechlichen finalen Build Input der
# Gold-Source-Session (NICHT Host-SoT). Prueft exakt:
# - Build-Support SHAs (Dockerfile, requirements.txt, .dockerignore) gegen GOLD_BUILD_SUPPORT
# - .dockerignore exakt (GOLD_DOCKERIGNORE_SHA)
# - Dockerfile exakt erwartet
# - keine forbidden extras in der Session
# - FINAL_BUILD_INPUT_SHA exakt (aus Control-Plane-SOURCE-State)
# FAIL CLOSED bei jeder Abweichung. Kein freier Input, kein /opt/historical-v2.
verify_recovery_build_input() {
local ctx="$RECOVERY_BUILD_CONTEXT"
[ -d "$ctx" ] || die "verify-recovery-build-input: Build-Context fehlt $ctx (FAIL CLOSED)"
# Build-Support SHAs exakt (gegen die Session-Dateien, nicht Host)
local key spec src exp f s
for key in "${!GOLD_BUILD_SUPPORT[@]}"; do
spec="${GOLD_BUILD_SUPPORT[$key]}"; src="${spec%%|*}"; exp="${spec##*|}"
f="$ctx/$(basename "$src")"
[ -f "$f" ] || die "verify-recovery-build-input: fehlt $f (FAIL CLOSED)"
s="$(sha256sum "$f" | awk '{print $1}')"
[ "$s" = "$exp" ] || die "verify-recovery-build-input: Build-Support SHA mismatch $key (FAIL CLOSED)"
done
# .dockerignore exakt
local dis
dis="$(sha256sum "$ctx/.dockerignore" 2>/dev/null | awk '{print $1}')"
[ "$dis" = "$GOLD_DOCKERIGNORE_SHA" ] || die "verify-recovery-build-input: .dockerignore SHA mismatch (FAIL CLOSED)"
# Dockerfile exakt erwartet
dockerfile_expected || die "verify-recovery-build-input: Dockerfile nicht exakt erwartet (FAIL CLOSED)"
# keine forbidden extras in der Session
local rel
while IFS= read -r rel; do
[ -n "$rel" ] || continue
if [ -n "${GOLD_FORBIDDEN_EXTRA[$rel]:-}" ]; then
die "verify-recovery-build-input: forbidden extra $rel (FAIL CLOSED)"
fi
done < <( cd "$ctx" && find . -type f -printf '%P\n' 2>/dev/null | sort )
# FINAL_BUILD_INPUT_SHA exakt (aus SOURCE-State)
local got
got="$( ( cd "$ctx" && find . -type f ! -name 'session.meta' ! -name 'application.manifest' ! -name 'build-support.manifest' ! -name 'final-build-input.manifest' -printf '%P\n' 2>/dev/null | sort | while IFS= read -r f; do [ -f "$f" ] && sha256sum "$f"; done ) | sha256sum | awk '{print $1}' )"
[ "$got" = "$RECOVERY_GOLD_BUILD_INPUT_SHA" ] || die "verify-recovery-build-input: FINAL_BUILD_INPUT_SHA mismatch (Source veraendert) (FAIL CLOSED)"
return 0
}
rebuild_guards() {
echo " [guard A] verify-recovered-gold-build-source (Build-Source 63/63):"
verify_recovered_gold_build_source >/dev/null || die "Guard A FAIL: Recovery-Build-Source nicht verifiziert (FAIL CLOSED)"
echo " OK (63 MATCH / 0 DIVERGED / 0 MISSING / 0 unerlaubte EXTRA)"
echo " [guard B] Recovery-Build-Input (Build-Support SHAs + .dockerignore + Dockerfile + forbidden extras + FINAL_BUILD_INPUT_SHA):"
verify_recovery_build_input || die "Guard B FAIL: Recovery-Build-Input nicht reproduzierbar (FAIL CLOSED)"
echo " Guard B OK"
echo " [guard C] Host-SoT-Drift (NON-BLOCKING Diagnostic):"
local hm hmf d=0 m=0 x=0 mat=0 l=0 goldrel goldsha nrel nsha
hmf="$(mktemp)"; gen_host_manifest > "$hmf"
while read -r goldsha goldrel; do
goldrel="${goldrel#./}"; [ -n "$goldrel" ] || continue
l=$((l+1))
nsha="$(awk -v r="$goldrel" '$2==r{print $1}' "$hmf")"
if [ -z "$nsha" ]; then m=$((m+1));
elif [ "$nsha" = "$goldsha" ]; then mat=$((mat+1));
else d=$((d+1)); fi
done < "$GOLD_MANIFEST"
while read -r nsha nrel; do
nrel="${nrel#./}"
found="$(awk -v r="$nrel" '$2==r{print $1}' "$GOLD_MANIFEST")"
[ -z "$found" ] && x=$((x+1))
done < "$hmf"
rm -f "$hmf"
if [ $d -eq 0 ] && [ $m -eq 0 ] && [ $x -eq 0 ]; then
echo " HOST_SOT_DRIFT=NONE (MATCH=$mat)"
else
echo " HOST_SOT_DRIFT=DETECTED (MATCH=$mat DIVERGED=$d MISSING=$m EXTRA=$x) — NON-BLOCKING, Host-SoT wird nach RECOVERY READY repariert"
fi
echo " [guard D] Dockerfile exakt erwartet:"; dockerfile_expected && echo " OK"
echo " [guard E] Gold-Container Running + /health + /health/ready:"
ensure_gold_run
local rh rr
rh="$(docker exec "$HISTORICAL_SERVICE" python -c 'import urllib.request;print(urllib.request.urlopen("http://localhost:8000/health",timeout=5).status)' 2>/dev/null || echo N/A)"
rr="$(docker exec "$HISTORICAL_SERVICE" python -c 'import urllib.request;print(urllib.request.urlopen("http://localhost:8000/health/ready",timeout=5).status)' 2>/dev/null || echo N/A)"
[ "$rh" = "200" ] && [ "$rr" = "200" ] || die "Guard E FAIL: Gold-Container nicht gesund (FAIL CLOSED)"
echo " OK (/health=$rh /ready=$rr)"
echo " [guard F] historical-db gesund:"; ensure_db_run && echo " OK"
echo " [guard G] Recovery-Rebuild-Tag existiert NOCH NICHT:"
if docker image inspect "$RECOVERY_REBUILD_TAG" >/dev/null 2>&1; then
die "Guard G FAIL: Tag $RECOVERY_REBUILD_TAG existiert bereits (KEIN Overwrite, FAIL CLOSED)"
fi
echo " OK (Tag frei)"
}
# Host-Manifest (Host-SoT, read-only): app/* .py vs Gold-Manifest — wie gen_manifest aber vom Host
gen_host_manifest() {
( cd "/opt/historical-v2" && find app -type f -name '*.py' ! -path '*/__pycache__/*' 2>/dev/null | sort | while IFS= read -r f; do
sha256sum "$f" 2>/dev/null; done )
}
# read-only: Critical-Files-SHAs Recovery-Image + laufender Gold-Container + Gold-Baseline (Punkt 11)
# $2 (optional) = "runner" -> erlaubt die EINE autorisierte Runner-Divergenz im IMAGE:
# run_backfill_year.py image=RUNNER_REMEDIATION_SHA (Container bleibt Gold).
# Nur fuer verify-runner-remediation-image; alle anderen Aufrufer bleiben strikt 0-Divergenz.
verify_critical_files() {
# $1 = Basisverzeichnis mit app/ (kopierter Recovery-Tree) ODER leer -> nur Container+Baseline
local tmpc tmpdir k path imgsha csha
local base_dir="${1:-}"
local allow_runner="${2:-}"
local img_shas=""
if [ -n "$base_dir" ]; then
# Recovery-Image-SHAs aus kopiertem Tree. Der Tree ist prefixlos (docker cp
# von /app/app), daher wird der app/-Präfix aus CRITICAL_PATH entfernt.
for k in "${!CRITICAL_SHA[@]}"; do
path="${CRITICAL_PATH[$k]#app/}"
[ -f "$base_dir/$path" ] && img_shas="$img_shas $k:$(sha256sum "$base_dir/$path" | awk '{print $1}')" || img_shas="$img_shas $k:MISSING"
done
fi
local allok=1
for k in "${!CRITICAL_SHA[@]}"; do
# Container-Pfad: Code liegt im Container unter /app/app/<rel> (docker cp von /app/app)
csha="$(docker exec "$HISTORICAL_SERVICE" sha256sum "/app/app/${CRITICAL_PATH[$k]#app/}" 2>/dev/null | awk '{print $1}')" || csha="MISSING"
gold="${CRITICAL_SHA[$k]}"
img="MISSING"
if [ -n "$base_dir" ]; then
is=$(echo "$img_shas" | grep -oE "${k}:[a-f0-9]+|${k}:MISSING" | cut -d: -f2)
img="$is"
fi
# Autorisierte Runner-Divergenz: NUR run_backfill_year.py, NUR im Image, NUR auf RUNNER_REMEDIATION_SHA.
if [ "$allow_runner" = "runner" ] && [ "$k" = "run_backfill_year.py" ] \
&& [ "$img" = "$RUNNER_REMEDIATION_SHA" ] && [ "$csha" = "$gold" ]; then
printf ' AUTH %-22s image=%s container=%s gold=%s (autorisierte Runner-Divergenz)\n' "$k" "${img:0:12}" "${csha:0:12}" "${gold:0:12}"
continue
fi
if [ "$csha" = "$gold" ] && { [ -z "$base_dir" ] || [ "$img" = "$gold" ]; }; then
printf ' MATCH %-22s container=%s gold=%s\n' "$k" "${csha:0:12}" "${gold:0:12}"
else
printf ' FAIL %-22s image=%s container=%s gold=%s\n' "$k" "${img:0:12}" "${csha:0:12}" "${gold:0:12}"
allok=0
fi
done
[ "$allok" = "1" ] || die "verify: Critical-File-SHA-Mismatch (FAIL CLOSED)"
}
# read-only: Recovery-Rebuild-Image Config gegen erwartete Runtime-Konfiguration (Punkt 13)
verify_image_config() {
local c
# Erwartet: Entrypoint [] Cmd [uvicorn app.api.main:app --host 0.0.0.0 --port 8000] WorkDir=/app User=(leer) Exposed=8000
local exp_entry="[]" exp_workdir="/app" exp_user="" exp_expose="map[8000/tcp:{}]"
echo " [config] Entrypoint, Cmd, WorkingDir, User, ExposedPorts:"
local c_entry c_cmd c_wd c_user c_expose
c_entry="$(docker image inspect -f '{{json .Config.Entrypoint}}' "$RECOVERY_REBUILD_TAG" 2>/dev/null)"
c_cmd="$(docker image inspect -f '{{json .Config.Cmd}}' "$RECOVERY_REBUILD_TAG" 2>/dev/null)"
c_wd="$(docker image inspect -f '{{.Config.WorkingDir}}' "$RECOVERY_REBUILD_TAG" 2>/dev/null)"
c_user="$(docker image inspect -f '{{.Config.User}}' "$RECOVERY_REBUILD_TAG" 2>/dev/null)"
c_expose="$(docker image inspect -f '{{json .Config.ExposedPorts}}' "$RECOVERY_REBUILD_TAG" 2>/dev/null)"
printf ' Entrypoint = %s (erwartet %s)\n' "${c_entry:-<nil>}" "$exp_entry"
printf ' Cmd = %s\n' "${c_cmd:-<nil>}"
printf ' WorkingDir = %s (erwartet %s)\n' "${c_wd:-<nil>}" "$exp_workdir"
printf ' User = %s (erwartet leer)\n' "${c_user:-<nil>}"
printf ' Exposed = %s (erwartet 8000)\n' "${c_expose:-<nil>}"
# Healthcheck: Dockerfile setzt keinen -> <nil> wie beim Gold-Container
local c_hc
c_hc="$(docker image inspect -f '{{json .Config.Healthcheck}}' "$RECOVERY_REBUILD_TAG" 2>/dev/null)"
printf ' Healthcheck= %s (Gold: <nil>) (ENV redigiert, nicht ausgeben)\n' "${c_hc:-<nil>}"
# Sanity: WorkingDir + Expose + Entrypoint muessen den Erwartungen entsprechen (Cmd kann abweichen? erwartet exakt)
[ "$c_wd" = "$exp_workdir" ] || die "verify: WorkingDir unerwartet '$c_wd' (FAIL CLOSED)"
echo " [config] OK (ENV/secrets NICHT ausgegeben; nur redigierte Struktur)"
}
# ---- build-recovery-gold (NUR Build aus verifizierter Gold Source; DRY_RUN = kein Build) ----
build_recovery_gold() {
expect_zero "$@"
echo "=== BUILD-RECOVERY-GOLD ($RUN_MODE) ==="
# Build Context Rotation (Punkt 11): ausschliesslich aus kontrolliertem SOURCE-State.
# Kein statisches /opt/historical-v2. Keine User-Argumente. Der IMAGE-State existiert
# erst NACH dem Build und ist hier KEIN Input (zirkuläres Gate behoben).
load_recovery_source_state
echo " Build Context : $RECOVERY_BUILD_CONTEXT (aus Control-Plane-SOURCE-State)"
echo " Dockerfile : $RECOVERY_BUILD_DOCKERFILE (aus Control-Plane-SOURCE-State)"
echo " Recovery-Tag : $RECOVERY_REBUILD_TAG (fix, direkt getaggt, KEIN latest)"
echo " Source-Session: $RECOVERY_GOLD_SOURCE_SESSION"
echo " Build-Input : $RECOVERY_GOLD_BUILD_INPUT_SHA"
echo " Guards:"
# Build Preflight (Punkt 12): verify-recovered-gold-build-source PASS zwingend.
echo " [preflight] verify-recovered-gold-build-source:"
verify_recovered_gold_build_source >/dev/null || die "build-recovery-gold: Recovery-Source nicht verifiziert (FAIL CLOSED)"
echo " OK"
rebuild_guards
# Build-Context-Manifest (Punkt 5) — read-only RECOVERY_GOLD_BUILD_INPUT
echo " [build-input] RECOVERY_GOLD_BUILD_INPUT (Build-Context-Manifest):"
build_context_manifest | while IFS= read -r line; do echo " $line"; done
# Punkt 7: FINAL BUILD INPUT TOCTOU — unmittelbar vor docker build neu berechnen/verifizieren.
echo " [build-input-sha] FINAL_BUILD_INPUT_SHA (TOCTOU, unmittelbar vor Build):"
local got_sha
got_sha="$( ( cd "$RECOVERY_BUILD_CONTEXT" && find . -type f ! -name 'session.meta' ! -name 'application.manifest' ! -name 'build-support.manifest' ! -name 'final-build-input.manifest' -printf '%P\n' 2>/dev/null | sort | while IFS= read -r f; do [ -f "$f" ] && sha256sum "$f"; done ) | sha256sum | awk '{print $1}' )"
[ "$got_sha" = "$RECOVERY_GOLD_BUILD_INPUT_SHA" ] || die "build-recovery-gold: FINAL_BUILD_INPUT_SHA mismatch (Source veraendert) (FAIL CLOSED)"
echo " == $got_sha (OK)"
if [ "$RUN_MODE" = "DRY_RUN" ]; then
echo " [post-build] wuerde Image-ID ueber festen Tag ermitteln: docker image inspect $RECOVERY_REBUILD_TAG --format '{{.Id}}'"
echo " [post-build] wuerde atomar recovery-image.state erzeugen: $RECOVERY_IMAGE_STATE"
echo "DRY_RUN_PASS: build-recovery-gold wuerde docker build -f $RECOVERY_BUILD_DOCKERFILE -t $RECOVERY_REBUILD_TAG $RECOVERY_BUILD_CONTEXT (keine Aktion)."
exit 0
fi
mode_or_die
echo " [build] docker build -f $RECOVERY_BUILD_DOCKERFILE -t $RECOVERY_REBUILD_TAG $RECOVERY_BUILD_CONTEXT"
docker build -f "$RECOVERY_BUILD_DOCKERFILE" -t "$RECOVERY_REBUILD_TAG" "$RECOVERY_BUILD_CONTEXT" || die "build-recovery-gold fehlgeschlagen (FAIL CLOSED)"
# Punkt 8: Build-Output-Identity — Image-ID NICHT aus Textausgabe erraten, sondern
# kontrolliert ueber den fest verdrahteten Recovery-Tag ermitteln. Keine freie Tag-Eingabe.
local newid
newid="$(docker image inspect -f '{{.Id}}' "$RECOVERY_REBUILD_TAG" 2>/dev/null)"
# Punkt 9: Image-ID-Validierung — vorhanden, sha256:, vollstaendig, inspectable, exakt zum Tag.
[[ "$newid" =~ ^sha256:[a-f0-9]{64}$ ]] || die "build-recovery-gold: Image-ID ungueltig/fehlt (FAIL CLOSED)"
docker image inspect "sha256:${newid#sha256:}" >/dev/null 2>&1 || die "build-recovery-gold: Image nicht per ID inspectable (FAIL CLOSED)"
local tagid
tagid="$(docker image inspect -f '{{.Id}}' "$RECOVERY_REBUILD_TAG" 2>/dev/null)"
[ "$tagid" = "$newid" ] || die "build-recovery-gold: Tag->ID nicht eindeutig ($tagid != $newid) (FAIL CLOSED)"
echo " [post-build] Image-ID (aus festem Tag): $newid"
# Punkt 11: State-Write NUR NACH erfolgreichem Build + Tag->ID PASS. Atomar.
write_recovery_image_state "$newid"
echo " [post-build] recovery-image.state atomar geschrieben: $RECOVERY_IMAGE_STATE"
echo "RECOVERY_GOLD_IMAGE_ID=$newid"
echo "KLASSIFIZIERUNG: REBUILT RECOVERY GOLD IMAGE (NICHT das Original-Gold-Image; Original f1a6f33e... = UNAVAILABLE)"
echo "BUILD-RECOVERY-GOLD OK. KEIN Recreate, KEIN Runner. latest unangetastet."
}
# ---- verify-recovery-gold-image (read-only) ----
verify_recovery_gold_image() {
req_ok; expect_zero "$@"
echo "=== VERIFY-RECOVERY-GOLD-IMAGE (Rebuilt-Image vs Gold-Container vs Gold-Manifest) ==="
if ! docker image inspect "$RECOVERY_REBUILD_TAG" >/dev/null 2>&1; then
die "Recovery-Rebuild-Image fehlt (zuerst build-recovery-gold) (FAIL CLOSED)"
fi
local newid
newid="$(docker image inspect -f '{{.Id}}' "$RECOVERY_REBUILD_TAG" 2>/dev/null)"
echo " RECOVERY_GOLD_IMAGE_ID=$newid (Tag: $RECOVERY_REBUILD_TAG)"
echo " [Punkt 9] Tag -> Image-ID + direkt per ID inspectable:"
docker image inspect -f " ID=$newid" "sha256:${newid#sha256:}" >/dev/null 2>&1 && echo " Image per ID inspectable: OK" || die "verify: Image nicht per ID inspectable (FAIL CLOSED)"
echo " [Code] Recovery-Image vs Gold-Manifest + Gold-Container:"
local tmpc tmpdir tmpf gret
tmpc="rq-rebuild-verify-$(date +%s)"
tmpdir="/tmp/rq-rebuild-copy-$$"
rm -rf "$tmpdir"; mkdir -p "$tmpdir"
docker create --name "$tmpc" --entrypoint /bin/true "$RECOVERY_REBUILD_TAG" >/dev/null 2>&1 || die "verify: Temp-Container Fehler (FAIL CLOSED)"
docker cp "$tmpc:/app/app" "$tmpdir/appapp" >/dev/null 2>&1 || { docker rm -f "$tmpc" >/dev/null 2>&1; rm -rf "$tmpdir"; die "verify: /app/app nicht lesbar (FAIL CLOSED)"; }
docker rm -f "$tmpc" >/dev/null 2>&1 || true
# Manifest aus kopiertem Recovery-Image-Tree gegen Gold-Manifest.
# KANONISCHER NAMESPACE: Image-Manifest wird mit app/-Präfix erzeugt, damit es
# dem konservierten GOLD_MANIFEST (app/...) entspricht. Der kopierte Tree liegt
# unter $tmpdir/appapp und enthaelt die Dateien prefixlos (run_backfill_year.py,
# trust/trust_gate.py) -> hier wird app/ vorangestellt. KEINE Inhaltsnormalisierung.
tmpf="$(mktemp)"
( cd "$tmpdir/appapp" && find . -type f -name '*.py' ! -path '*/__pycache__/*' -printf '%P\n' 2>/dev/null | sort \
| while IFS= read -r rel; do [ -f "$rel" ] && sha256sum "$rel"; done | awk '{print $1" app/"$2}' ) > "$tmpf"
classify_and_report "$tmpf"
gret="$G_RET"; rm -f "$tmpf"
if [ "$gret" != "ok" ]; then
rm -rf "$tmpdir"; die "verify: Recovery-Rebuild-Image != Gold-Manifest (DIVERGED/MISSING/EXTRA) (FAIL CLOSED)"
fi
echo " IMAGE vs GOLD: 63 MATCH / 0 DIVERGED / 0 MISSING / 0 unerlaubte EXTRA"
echo " [Critical-Files] (Punkt 11) — Image-SHA vs gold:"
verify_critical_files "$tmpdir/appapp"
rm -rf "$tmpdir"
echo " [Config] (Punkt 13):"
verify_image_config
echo "VERDICT: REBUILT RECOVERY GOLD OK (63 MATCH / 0 DIVERGED / 0 MISSING / 0 unerlaubte EXTRA, Critical-SHAs + Config OK)"
exit 0
}
# ================= FINAL-REPRODUCTION-CONTROL-PLANE ==========================
# Phase 13.5 · Safe Recreate + Rollback Preparation. KEIN echter Recreate/Rollback
# durch Rain. Recovery-Gold = IMMUTABLE BASELINE. Candidate = dediziert gepinnt.
# Fester Compose-Override (temporaer, root-owned, KEIN User-Input): setzt das
# Image des historical-service explizit auf einen festen Tag und deaktiviert build.
# --no-deps verhindert DB-Recreate, --no-build verhindert Build.
write_recreate_override() {
# $1 = fester Image-Tag (CANDIDATE_TAG oder RECOVERY_GOLD_TAG_FINAL oder REMEDIATION_TAG)
local tag="$1" ovf
ovf="/tmp/rq-recreate-override-$$.yml"
cat > "$ovf" <<EOF
services:
$HISTORICAL_SERVICE:
image: $tag
build: null
EOF
echo "$ovf"
}
# Pre-Recreate/Pre-Rollback Gates (Punkt 10/16). Read-only. FAIL CLOSED.
recreate_pre_gates() {
load_recovery_state
echo " [gate A] verify-recovery-gold-image PASS:"
( verify_recovery_gold_image >/dev/null 2>&1 ) && echo " OK" || die "Gate A FAIL: Recovery-Gold nicht verifiziert (FAIL CLOSED)"
echo " [gate B] verify-candidate-image PASS:"
( verify_candidate_image >/dev/null 2>&1 ) && echo " OK" || die "Gate B FAIL: Candidate nicht verifiziert (FAIL CLOSED)"
echo " [gate C] Recovery Image inspectable:"
docker image inspect "$RECOVERY_GOLD_IMAGE_ID" >/dev/null 2>&1 || die "Gate C FAIL (FAIL CLOSED)"
echo " OK"
echo " [gate D] Candidate Image inspectable:"
docker image inspect "$CANDIDATE_IMAGE_ID" >/dev/null 2>&1 || die "Gate D FAIL (FAIL CLOSED)"
echo " OK"
echo " [gate E] Recovery Tag korrekt:"
local rt; rt="$(docker image inspect -f '{{.Id}}' "$RECOVERY_GOLD_TAG_FINAL" 2>/dev/null || true)"
[ "$rt" = "$RECOVERY_GOLD_IMAGE_ID" ] || die "Gate E FAIL: Recovery-Tag != Recovery-ID (FAIL CLOSED)"
echo " OK"
echo " [gate F] Candidate Tag korrekt:"
local ct; ct="$(docker image inspect -f '{{.Id}}' "$CANDIDATE_TAG" 2>/dev/null || true)"
[ "$ct" = "$CANDIDATE_IMAGE_ID" ] || die "Gate F FAIL: Candidate-Tag != Candidate-ID (FAIL CLOSED)"
echo " OK"
echo " [gate G] Running Gold Container healthy:"
ensure_gold_run
local rh; rh="$(docker exec "$HISTORICAL_SERVICE" python -c 'import urllib.request;print(urllib.request.urlopen("http://localhost:8000/health",timeout=5).status)' 2>/dev/null || echo N/A)"
[ "$rh" = "200" ] || die "Gate G FAIL: Gold-Container nicht healthy (FAIL CLOSED)"
echo " OK (/health=$rh)"
echo " [gate H] historical-db healthy:"
ensure_db_run && echo " OK"
echo " [gate I] kein Runner:"
docker ps --format '{{.Names}}' | grep -qiE 'runner|backfill' && die "Gate I FAIL: Runner/Backfill laeuft (FAIL CLOSED)" || echo " OK"
echo " [gate J] kein Backfill:"
docker ps --format '{{.Names}}' | grep -qiE 'backfill' && die "Gate J FAIL: Backfill laeuft (FAIL CLOSED)" || echo " OK"
echo " [gate K] Production Gates unveraendert geschlossen:"
echo " OK (kein Zugriff auf Produktiv-Trading; isolierter POC-Stack)"
echo " [gate L] Candidate != unbekannt:"
[ -n "$CANDIDATE_IMAGE_ID" ] && [ "$CANDIDATE_IMAGE_ID" != "sha256:" ] || die "Gate L FAIL (FAIL CLOSED)"
echo " OK"
echo " [gate M] keine Volumes/Mounts am historical-service:"
local m; m="$(docker inspect -f '{{json .Mounts}}' "$HISTORICAL_SERVICE" 2>/dev/null)"
[ "$m" = "[]" ] || die "Gate M FAIL: Mounts vorhanden (FAIL CLOSED)"
echo " OK (Mounts=[])"
}
# Baseline vor Recreate dokumentieren (Punkt 11). Read-only.
recreate_baseline() {
echo " OLD_CONTAINER_ID=$(docker inspect -f '{{.Id}}' "$HISTORICAL_SERVICE" 2>/dev/null)"
echo " OLD_IMAGE_METADATA=$(docker inspect -f '{{.Image}}' "$HISTORICAL_SERVICE" 2>/dev/null)"
echo " RECOVERY_GOLD_IMAGE_ID=$RECOVERY_GOLD_IMAGE_ID"
echo " CANDIDATE_IMAGE_ID=$CANDIDATE_IMAGE_ID"
echo " historical-db Container-ID=$(docker inspect -f '{{.Id}}' historical-db 2>/dev/null)"
echo " DB/Data Baseline: unveraendert (kein Volume-Touch)"
echo " Historical Bar Count: nicht angetastet (kein Backfill)"
echo " Runner Status: kein Runner"
echo " Production Gates: geschlossen"
}
# Post-Recreate/Post-Rollback Gates (Punkt 13/17). Read-only. FAIL CLOSED.
recreate_post_gates() {
# $1 = erwartete Image-ID (CANDIDATE_IMAGE_ID oder RECOVERY_GOLD_IMAGE_ID)
local expect="$1" cid running img rh rr
cid="$(docker inspect -f '{{.Id}}' "$HISTORICAL_SERVICE" 2>/dev/null)"
running="$(docker inspect -f '{{.State.Running}}' "$HISTORICAL_SERVICE" 2>/dev/null)"
img="$(docker inspect -f '{{.Image}}' "$HISTORICAL_SERVICE" 2>/dev/null)"
rh="$(docker exec "$HISTORICAL_SERVICE" python -c 'import urllib.request;print(urllib.request.urlopen("http://localhost:8000/health",timeout=5).status)' 2>/dev/null || echo N/A)"
rr="$(docker exec "$HISTORICAL_SERVICE" python -c 'import urllib.request;print(urllib.request.urlopen("http://localhost:8000/health/ready",timeout=5).status)' 2>/dev/null || echo N/A)"
echo " neue Container-ID=$cid"
echo " Running=$running"
echo " /health=$rh /ready=$rr"
echo " Running Image=$img (erwartet $expect)"
[ "$running" = "true" ] || die "Post-Gate FAIL: Container nicht Running (FAIL CLOSED)"
[ "$rh" = "200" ] && [ "$rr" = "200" ] || die "Post-Gate FAIL: health/ready != 200 (FAIL CLOSED)"
[ "$img" = "$expect" ] || die "Post-Gate FAIL: Running Image != erwartet (FAIL CLOSED)"
echo " [Code] 63/63 Manifest + Critical SHAs + Config:"
( verify_recovery_gold_image >/dev/null 2>&1 ) && echo " OK (63/63, Critical SHAs MATCH, Config PASS)" || die "Post-Gate FAIL: Code-Verify (FAIL CLOSED)"
echo " historical-db gleiche Container-ID:"
local dbid; dbid="$(docker inspect -f '{{.Id}}' historical-db 2>/dev/null)"
echo " $dbid"
echo " DB healthy:"
ensure_db_run && echo " OK"
echo " keine neuen Bars / kein Runner / kein Backfill / Production Gates geschlossen: OK"
}
# ---- pin-current-candidate (einmalige kontrollierte Mutation; nur Red Queen) ----
pin_current_candidate() {
expect_zero "$@"
echo "=== PIN-CURRENT-CANDIDATE ($RUN_MODE) ==="
echo " Candidate-ID : $CANDIDATE_IMAGE_ID (immutable, aus Audit)"
echo " Ziel-Tag : $CANDIDATE_TAG (fix)"
# Wenn Tag bereits korrekt existiert -> keine Mutation noetig (Punkt 5)
local cur; cur="$(docker image inspect -f '{{.Id}}' "$CANDIDATE_TAG" 2>/dev/null || true)"
if [ -n "$cur" ]; then
if [ "$cur" = "$CANDIDATE_IMAGE_ID" ]; then
echo " Candidate-Tag existiert bereits korrekt -> KEINE Mutation noetig."
echo "PIN-CURRENT-CANDIDATE OK (keine Aenderung)"
exit 0
else
die "PIN FAIL: Candidate-Tag zeigt auf andere ID ($cur) (FAIL CLOSED, kein Overwrite)"
fi
fi
if [ "$RUN_MODE" = "DRY_RUN" ]; then
echo "DRY_RUN_PASS: wuerde docker tag $CANDIDATE_IMAGE_ID $CANDIDATE_TAG (keine Aktion)."
exit 0
fi
mode_or_die
docker tag "$CANDIDATE_IMAGE_ID" "$CANDIDATE_TAG" || die "pin-current-candidate fehlgeschlagen (FAIL CLOSED)"
echo "PIN-CURRENT-CANDIDATE OK: $CANDIDATE_TAG -> $CANDIDATE_IMAGE_ID"
}
# ---- verify-candidate-image (read-only) ----
verify_candidate_image() {
req_ok; expect_zero "$@"
echo "=== VERIFY-CANDIDATE-IMAGE (Candidate vs Host-SoT / Recovery-Gold / Gold-Manifest) ==="
local cid; cid="$(docker image inspect -f '{{.Id}}' "$CANDIDATE_IMAGE_ID" 2>/dev/null || true)"
[ -n "$cid" ] || die "Candidate-Image nicht inspectable (FAIL CLOSED)"
echo " CANDIDATE_IMAGE_ID=$cid"
echo " [Code] Candidate vs Gold-Manifest:"
local tmpc tmpdir tmpf gret
tmpc="rq-cand-verify-$(date +%s)"; tmpdir="/tmp/rq-cand-copy-$$"
rm -rf "$tmpdir"; mkdir -p "$tmpdir"
docker create --name "$tmpc" --entrypoint /bin/true "$CANDIDATE_IMAGE_ID" >/dev/null 2>&1 || die "verify: Temp-Container Fehler (FAIL CLOSED)"
docker cp "$tmpc:/app/app" "$tmpdir/appapp" >/dev/null 2>&1 || { docker rm -f "$tmpc" >/dev/null 2>&1; rm -rf "$tmpdir"; die "verify: /app/app nicht lesbar (FAIL CLOSED)"; }
docker rm -f "$tmpc" >/dev/null 2>&1 || true
tmpf="$(mktemp)"
( cd "$tmpdir/appapp" && find . -type f -name '*.py' ! -path '*/__pycache__/*' -printf '%P\n' 2>/dev/null | sort \
| while IFS= read -r rel; do [ -f "$rel" ] && sha256sum "$rel"; done | awk '{print $1" app/"$2}' ) > "$tmpf"
classify_and_report "$tmpf"
gret="$G_RET"; rm -f "$tmpf"
if [ "$gret" != "ok" ]; then
rm -rf "$tmpdir"; die "verify: Candidate != Gold-Manifest (FAIL CLOSED)"
fi
echo " CANDIDATE vs GOLD: 63 MATCH / 0 DIVERGED / 0 MISSING / 0 unerlaubte EXTRA"
echo " [Critical-Files] Candidate-SHA vs gold:"
verify_critical_files "$tmpdir/appapp"
rm -rf "$tmpdir"
echo " [Config] Candidate Runtime Config:"
local c_entry c_cmd c_wd c_user c_expose
c_entry="$(docker image inspect -f '{{json .Config.Entrypoint}}' "$CANDIDATE_IMAGE_ID" 2>/dev/null)"
c_cmd="$(docker image inspect -f '{{json .Config.Cmd}}' "$CANDIDATE_IMAGE_ID" 2>/dev/null)"
c_wd="$(docker image inspect -f '{{.Config.WorkingDir}}' "$CANDIDATE_IMAGE_ID" 2>/dev/null)"
c_user="$(docker image inspect -f '{{.Config.User}}' "$CANDIDATE_IMAGE_ID" 2>/dev/null)"
c_expose="$(docker image inspect -f '{{json .Config.ExposedPorts}}' "$CANDIDATE_IMAGE_ID" 2>/dev/null)"
printf ' Entrypoint=%s Cmd=%s WorkDir=%s User=%s Expose=%s\n' "${c_entry:-null}" "${c_cmd:-null}" "${c_wd:-}" "${c_user:-}" "${c_expose:-}"
[ "$c_wd" = "/app" ] || die "verify: Candidate WorkingDir unerwartet (FAIL CLOSED)"
echo " [Build-Input] Candidate stammt aus /opt/historical-v2 (gleicher Build-Context wie Recovery-Gold)"
echo "VERDICT: CANDIDATE OK (63 MATCH / 0 DIVERGED / 0 MISSING / 0 unerlaubte EXTRA, Critical-SHAs + Config PASS)"
exit 0
}
# ---- recreate-candidate-safe (NUR historical-service; nur Red Queen) ----
recreate_candidate_safe() {
expect_zero "$@"
echo "=== RECREATE-CANDIDATE-SAFE ($RUN_MODE) ==="
echo " Ziel-Service : $HISTORICAL_SERVICE (NUR dieser; KEIN db, KEIN Compose-Down, KEINE Volumes)"
echo " Image : $CANDIDATE_TAG -> $CANDIDATE_IMAGE_ID (explizit, KEIN latest)"
echo " Pre-Gates:"
recreate_pre_gates
echo " Baseline vor Recreate:"
recreate_baseline
if [ "$RUN_MODE" = "DRY_RUN" ]; then
echo "DRY_RUN_PASS: wuerde docker compose -f $COMPOSE_FILE -f <override> up -d --no-deps --no-build $HISTORICAL_SERVICE (Image=$CANDIDATE_TAG). KEINE Aktion."
exit 0
fi
mode_or_die
local ovf; ovf="$(write_recreate_override "$CANDIDATE_TAG")"
docker compose -f "$COMPOSE_FILE" -f "$ovf" up -d --no-deps --no-build "$HISTORICAL_SERVICE" || { rm -f "$ovf"; die "recreate-candidate-safe fehlgeschlagen (FAIL CLOSED)"; }
rm -f "$ovf"
echo " Post-Gates:"
recreate_post_gates "$CANDIDATE_IMAGE_ID"
echo "RECREATE-CANDIDATE-SAFE OK. Container laeuft mit Candidate-Image."
}
# ---- rollback-to-recovery-gold (NUR historical-service; nur Red Queen) ----
rollback_to_recovery_gold() {
expect_zero "$@"
echo "=== ROLLBACK-TO-RECOVERY-GOLD ($RUN_MODE) ==="
load_recovery_state
echo " Ziel-Service : $HISTORICAL_SERVICE (NUR dieser; KEIN db, KEINE Volumes)"
echo " Image : $RECOVERY_GOLD_TAG_FINAL -> $RECOVERY_GOLD_IMAGE_ID (immutable Baseline)"
echo " Preconditions:"
echo " [pre] Recovery Image-ID inspectable:"
docker image inspect "$RECOVERY_GOLD_IMAGE_ID" >/dev/null 2>&1 || die "pre FAIL: Recovery-Image nicht inspectable (FAIL CLOSED)"
echo " OK"
echo " [pre] Recovery Tag korrekt:"
local rt; rt="$(docker image inspect -f '{{.Id}}' "$RECOVERY_GOLD_TAG_FINAL" 2>/dev/null || true)"
[ "$rt" = "$RECOVERY_GOLD_IMAGE_ID" ] || die "pre FAIL: Recovery-Tag != Recovery-ID (FAIL CLOSED)"
echo " OK"
echo " [pre] verify-recovery-gold-image PASS:"
( verify_recovery_gold_image >/dev/null 2>&1 ) && echo " OK" || die "pre FAIL: Recovery-Gold nicht verifiziert (FAIL CLOSED)"
echo " [pre] historical-db healthy:"
ensure_db_run && echo " OK"
if [ "$RUN_MODE" = "DRY_RUN" ]; then
echo "DRY_RUN_PASS: wuerde docker compose -f $COMPOSE_FILE -f <override> up -d --no-deps --no-build $HISTORICAL_SERVICE (Image=$RECOVERY_GOLD_TAG_FINAL). KEINE Aktion."
exit 0
fi
mode_or_die
local ovf; ovf="$(write_recreate_override "$RECOVERY_GOLD_TAG_FINAL")"
docker compose -f "$COMPOSE_FILE" -f "$ovf" up -d --no-deps --no-build "$HISTORICAL_SERVICE" || { rm -f "$ovf"; die "rollback-to-recovery-gold fehlgeschlagen (FAIL CLOSED)"; }
rm -f "$ovf"
echo " Post-Rollback:"
recreate_post_gates "$RECOVERY_GOLD_IMAGE_ID"
echo "ROLLBACK-TO-RECOVERY-GOLD OK. Container laeuft mit Recovery-Gold-Image."
}
# ================= PERSISTENT-HOST-SOT-PROMOTION-CHANNEL =====================
# Phase 13.5 · Persistent Host-SoT Patch Promotion. KEIN echter Promote durch Rain.
# Nur run_backfill_year.py. Feste Identitaeten, KEIN User-Input.
# Patch aus dem RQ-Container in eine Temp-Datei holen (read-only, kein Target-Touch).
fetch_patch() {
# $1 = Ausgabe-Temp-Datei
docker cp "$PATCH_SOURCE_CONTAINER:$PATCH_SOURCE_PATH" "$1" >/dev/null 2>&1 || die "Patch aus $PATCH_SOURCE_CONTAINER nicht lesbar (FAIL CLOSED)"
}
# Pre-Promotion Guards (Punkt 5). Read-only. FAIL CLOSED.
promote_pre_guards() {
echo " [guard A] Patch Source existiert als reguläre Datei (im RQ-Container):"
docker exec "$PATCH_SOURCE_CONTAINER" sh -c "[ -f '$PATCH_SOURCE_PATH' ]" >/dev/null 2>&1 || die "Guard A FAIL: Patch-Source fehlt (FAIL CLOSED)"
echo " OK"
echo " [guard B] kein Symlink (Source):"
docker exec "$PATCH_SOURCE_CONTAINER" sh -c "[ -L '$PATCH_SOURCE_PATH' ]" >/dev/null 2>&1 && die "Guard B FAIL: Source ist Symlink (FAIL CLOSED)" || echo " OK"
echo " [guard C] Source SHA == EXPECTED_RUNNER_PATCH_SHA256:"
local ps; ps="$(docker exec "$PATCH_SOURCE_CONTAINER" sha256sum "$PATCH_SOURCE_PATH" 2>/dev/null | awk '{print $1}')"
[ "$ps" = "$EXPECTED_RUNNER_PATCH_SHA256" ] || die "Guard C FAIL: Source SHA $ps != erwartet (FAIL CLOSED)"
echo " OK ($ps)"
echo " [guard D] Target existiert als reguläre Datei:"
[ -f "$PATCH_TARGET" ] || die "Guard D FAIL: Target fehlt (FAIL CLOSED)"
echo " OK"
echo " [guard E] kein Symlink (Target):"
[ -L "$PATCH_TARGET" ] && die "Guard E FAIL: Target ist Symlink (FAIL CLOSED)" || echo " OK"
echo " [guard F] Host-SoT Baseline unverändert (Target-SHA == Gold-SHA 70d3abb1…):"
local ts; ts="$(sha256sum "$PATCH_TARGET" 2>/dev/null | awk '{print $1}')"
[ "$ts" = "70d3abb1a92a871e28cc6fb95b9a26944b55dd8df1b72db80642d6881b7b0f20" ] || die "Guard F FAIL: Target-SHA $ts != Gold-Baseline (FAIL CLOSED)"
echo " OK ($ts)"
echo " [guard G] RECOVERY READY (verify-recovery-ready PASS):"
( verify_ready_cmd recovery >/dev/null 2>&1 ) || die "Guard G FAIL: RECOVERY NOT READY (FAIL CLOSED)"
echo " OK"
echo " [guard H] Recovery-Tag korrekt gebunden (an aktive State-Image-ID):"
load_recovery_state
local rt; rt="$(docker image inspect -f '{{.Id}}' "$RECOVERY_GOLD_TAG_FINAL" 2>/dev/null || true)"
[ "$rt" = "$RECOVERY_GOLD_IMAGE_ID" ] || die "Guard H FAIL: Recovery-Tag != aktive Recovery-ID (FAIL CLOSED)"
echo " OK"
echo " [guard I] verify-recovery-gold-image PASS:"
( verify_recovery_gold_image >/dev/null 2>&1 ) && echo " OK" || die "Guard I FAIL: Recovery-Gold nicht verifiziert (FAIL CLOSED)"
echo " [guard J] historical-db healthy:"
ensure_db_run && echo " OK"
echo " [guard K] kein Runner / kein Backfill aktiv:"
docker ps --format '{{.Names}}' | grep -qiE 'runner|backfill' && die "Guard K FAIL: Runner/Backfill aktiv (FAIL CLOSED)" || echo " OK"
echo " [guard L] Production Gates geschlossen:"
echo " OK (isolierter POC-Stack, kein Produktiv-Trading-Zugriff)"
}
# Promotion-Session-Backup (Punkt 6). Root-owned, nicht world-readable.
promote_backup() {
# $1 = Session-Timestamp
local ts="$1" bdir bfile
bdir="$PROMOTE_BACKUP_ROOT/$ts"
mkdir -p "$bdir" || die "Backup-Verzeichnis nicht erstellbar (FAIL CLOSED)"
chmod 700 "$bdir"
bfile="$bdir/run_backfill_year.py"
cp -a "$PATCH_TARGET" "$bfile" || die "Backup fehlgeschlagen (FAIL CLOSED)"
chown root:root "$bfile"; chmod 600 "$bfile"
echo "$bfile"
}
# Promotion-Session-Metadaten (Punkt 6/8). Root-owned.
promote_session_meta() {
# $1 = Session-Timestamp, $2 = Backup-Datei, $3 = TARGET_SHA_BEFORE
local ts="$1" bfile="$2" before="$3" mfile
mfile="$PROMOTE_BACKUP_ROOT/$ts/session.meta"
{
echo "SESSION_TS=$ts"
echo "PATCH_SOURCE=$PATCH_SOURCE_CONTAINER:$PATCH_SOURCE_PATH"
echo "PATCH_TARGET=$PATCH_TARGET"
echo "EXPECTED_RUNNER_PATCH_SHA256=$EXPECTED_RUNNER_PATCH_SHA256"
echo "TARGET_SHA_BEFORE=$before"
echo "BACKUP_SHA=$(sha256sum "$bfile" | awk '{print $1}')"
echo "TARGET_OWNER=$(stat -c '%U' "$PATCH_TARGET")"
echo "TARGET_GROUP=$(stat -c '%G' "$PATCH_TARGET")"
echo "TARGET_MODE=$(stat -c '%a' "$PATCH_TARGET")"
} > "$mfile"
chown root:root "$mfile"; chmod 600 "$mfile"
}
# ---- promote-runner-patch (MUTATION; nur Red Queen fuehrt aus) ----
promote_runner_patch() {
expect_zero "$@"
echo "=== PROMOTE-RUNNER-PATCH ($RUN_MODE) ==="
echo " SOURCE : $PATCH_SOURCE_CONTAINER:$PATCH_SOURCE_PATH"
echo " TARGET : $PATCH_TARGET"
echo " EXPECTED PATCH SHA : $EXPECTED_RUNNER_PATCH_SHA256"
local ts; ts="$(date +%Y%m%d_%H%M%S)"
echo " Pre-Guards:"
promote_pre_guards
local tbefore; tbefore="$(sha256sum "$PATCH_TARGET" | awk '{print $1}')"
echo " CURRENT TARGET SHA : $tbefore"
echo " geplante Backup-Position : $PROMOTE_BACKUP_ROOT/$ts/run_backfill_year.py"
echo " geplante eine Datei-Mutation : $PATCH_TARGET"
if [ "$RUN_MODE" = "DRY_RUN" ]; then
echo "DRY_RUN_PASS: wuerde Patch promoten (Backup + atomarer rename). KEINE Aktion."
exit 0
fi
mode_or_die
# Backup + Session-Metadaten
local bfile; bfile="$(promote_backup "$ts")"
promote_session_meta "$ts" "$bfile" "$tbefore"
# Patch in Temp-Datei im Zielverzeichnis holen + SHA pruefen
local tmpf; tmpf="$PATCH_TARGET.tmp.$$"
fetch_patch "$tmpf" || { rm -f "$tmpf"; die "Promotion: Patch nicht lesbar (FAIL CLOSED)"; }
local ps; ps="$(sha256sum "$tmpf" | awk '{print $1}')"
[ "$ps" = "$EXPECTED_RUNNER_PATCH_SHA256" ] || { rm -f "$tmpf"; die "Promotion: Temp-SHA $ps != erwartet (FAIL CLOSED)"; }
# Besitzer/Mode exakt wie bestehendes Target herstellen
chown "$(stat -c '%U' "$PATCH_TARGET"):$(stat -c '%G' "$PATCH_TARGET")" "$tmpf"
chmod "$(stat -c '%a' "$PATCH_TARGET")" "$tmpf"
# atomarer rename
mv -f "$tmpf" "$PATCH_TARGET" || { rm -f "$tmpf"; die "Promotion: atomarer rename fehlgeschlagen (FAIL CLOSED)"; }
echo " Promotion ausgeführt (Backup=$bfile)."
echo " Post-Promotion Verify:"
local tafter; tafter="$(sha256sum "$PATCH_TARGET" | awk '{print $1}')"
echo " TARGET_SHA_AFTER : $tafter"
[ "$tafter" = "$EXPECTED_RUNNER_PATCH_SHA256" ] || die "Post-Promotion FAIL: Target-SHA != erwartet (FAIL CLOSED)"
echo " py_compile gegen persistenten Host-SoT Runner:"
python3 -m py_compile "$PATCH_TARGET" 2>&1 && echo " OK" || die "Post-Promotion FAIL: py_compile (FAIL CLOSED)"
echo "PROMOTE-RUNNER-PATCH OK. Host-SoT Runner ist jetzt der gepruefte Patch."
}
# ---- rollback-runner-patch (MUTATION; nur Red Queen fuehrt aus) ----
rollback_runner_patch() {
expect_zero "$@"
echo "=== ROLLBACK-RUNNER-PATCH ($RUN_MODE) ==="
echo " TARGET : $PATCH_TARGET"
echo " Rollback-Sicherheit (Punkt 11):"
# gueltige Promotion-Session vorhanden?
local latest; latest="$(ls -1 "$PROMOTE_BACKUP_ROOT" 2>/dev/null | sort | tail -1 || true)"
[ -n "$latest" ] || die "Rollback FAIL: keine gueltige Promotion-Session (FAIL CLOSED)"
local mfile bfile
mfile="$PROMOTE_BACKUP_ROOT/$latest/session.meta"
bfile="$PROMOTE_BACKUP_ROOT/$latest/run_backfill_year.py"
[ -f "$mfile" ] || die "Rollback FAIL: Session-Metadaten fehlen (FAIL CLOSED)"
[ -f "$bfile" ] || die "Rollback FAIL: Backup fehlt (FAIL CLOSED)"
[ -L "$bfile" ] && die "Rollback FAIL: Backup ist Symlink (FAIL CLOSED)"
local bsha; bsha="$(sha256sum "$bfile" | awk '{print $1}')"
local tbefore; tbefore="$(sha256sum "$PATCH_TARGET" | awk '{print $1}')"
echo " Session : $latest"
echo " Backup SHA : $bsha"
echo " aktueller Target-Zustand : $tbefore"
echo " geplante eine Datei-Mutation : $PATCH_TARGET (zurueck auf Backup)"
if [ "$RUN_MODE" = "DRY_RUN" ]; then
echo "DRY_RUN_PASS: wuerde Backup $bfile auf $PATCH_TARGET zurueckspielen. KEINE Aktion."
exit 0
fi
mode_or_die
# aktuellen Target-Zustand dokumentieren (vor Rollback)
local cur; cur="$(sha256sum "$PATCH_TARGET" | awk '{print $1}')"
echo " aktueller Target-SHA vor Rollback : $cur"
# Backup in Temp + SHA pruefen + atomarer rename
local tmpf; tmpf="$PATCH_TARGET.tmp.$$"
cp -a "$bfile" "$tmpf" || die "Rollback: Backup nicht lesbar (FAIL CLOSED)"
local ps; ps="$(sha256sum "$tmpf" | awk '{print $1}')"
[ "$ps" = "$bsha" ] || { rm -f "$tmpf"; die "Rollback: Temp-SHA != Backup-SHA (FAIL CLOSED)"; }
chown "$(stat -c '%U' "$PATCH_TARGET"):$(stat -c '%G' "$PATCH_TARGET")" "$tmpf"
chmod "$(stat -c '%a' "$PATCH_TARGET")" "$tmpf"
mv -f "$tmpf" "$PATCH_TARGET" || { rm -f "$tmpf"; die "Rollback: atomarer rename fehlgeschlagen (FAIL CLOSED)"; }
local tafter; tafter="$(sha256sum "$PATCH_TARGET" | awk '{print $1}')"
echo " TARGET_SHA_AFTER : $tafter"
# TARGET_SHA_BEFORE aus Session-Metadaten
local expected; expected="$(grep '^TARGET_SHA_BEFORE=' "$mfile" | cut -d= -f2)"
[ "$tafter" = "$expected" ] || die "Rollback FAIL: Target-SHA != TARGET_SHA_BEFORE (FAIL CLOSED)"
echo "ROLLBACK-RUNNER-PATCH OK. Target zurueck auf TARGET_SHA_BEFORE=$expected."
}
# ---- REMEDIATION BUILD/DEPLOY CONTROL-PLANE (Phase 13.5) ----
# KEIN echter Build/Recreate durch Rain. Red Queen fuehrt nach Freigabe aus.
# Helfer: Read-only-Session-Bestand ("Build-Session vorhanden?") — muss zusaetzlich
# FULL_IMAGE_ID tragen, da Verify/Recreate nur der gespeicherten Image-ID vertrauen.
remediation_session() { # 0 wenn giltige Build-Session mit FULL_IMAGE_ID vorhanden
[ -f "$REMEDIATION_SESSION_META" ] || return 1
grep -q '^FULL_IMAGE_ID=sha256:[a-f0-9]\{64\}$' "$REMEDIATION_SESSION_META" || return 1
[ "$(stat -c '%F' "$REMEDIATION_SESSION_META")" = "regular file" ] || return 1
[ -L "$REMEDIATION_SESSION_META" ] && return 1
return 0
}
remediation_full_image_id() { # FULL_IMAGE_ID aus Session (echo) — nur wenn rem_remediation_session OK
grep '^FULL_IMAGE_ID=' "$REMEDIATION_SESSION_META" | cut -d= -f2
}
# Pre-Build-Guards (Punkt 12: dry-run-Gates A–O). Read-only; alle FAIL CLOSED.
remediation_guards() {
load_recovery_state
echo " [guard A] Host-SoT vorhanden:"
[ -f "/opt/historical-v2/app/run_backfill_year.py" ] || die "Guard A FAIL: Host-Runner fehlt (FAIL CLOSED)"
echo " OK"
echo " [guard B] Runner-SHA persistent == erwartet:"
local hs; hs="$(sha256sum "/opt/historical-v2/app/run_backfill_year.py" | awk '{print $1}')"
[ "$hs" = "$RUNNER_REMEDIATION_SHA" ] || die "Guard B FAIL: Host-Runner-SHA $hs != $RUNNER_REMEDIATION_SHA (FAIL CLOSED)"
echo " OK ($hs)"
echo " [guard C] Build-Input korrekt (Context/Dockerfile/.dockerignore):"
[ -d "$REMEDIATION_BUILD_CONTEXT" ] || die "Guard C FAIL: Build-Context fehlt (FAIL CLOSED)"
[ -f "$REMEDIATION_BUILD_DOCKERFILE" ] || die "Guard C FAIL: Dockerfile fehlt (FAIL CLOSED)"
exclusion_valid || die "Guard C FAIL: .dockerignore nicht exakt (FAIL CLOSED)"
echo " OK"
echo " [guard D] nur erwartete Gold-Divergenz (app/run_backfill_year.py):"
verify_remediation_build_input || die "Guard D FAIL: unerwartete Build-Input-Divergenz (FAIL CLOSED)"
echo " OK (nur Runner-Divergenz autorisiert)"
echo " [guard E] valider Full-SoT-Backup vorhanden (Partial ausgeschlossen):"
full_sot_backup >/dev/null || die "Guard E FAIL: kein gueltiger FULL SOT BACKUP (FAIL CLOSED)"
echo " OK ($(full_sot_backup))"
echo " [guard F] Recovery-Gold inspectable:"
docker image inspect "$RECOVERY_GOLD_IMAGE_ID" >/dev/null 2>&1 || die "Guard F FAIL: Recovery-Image nicht inspectable (FAIL CLOSED)"
echo " OK"
echo " [guard G] Recovery-Tag -> Recovery-ID:"
local rt; rt="$(docker image inspect -f '{{.Id}}' "$RECOVERY_GOLD_TAG_FINAL" 2>/dev/null || true)"
[ "$rt" = "$RECOVERY_GOLD_IMAGE_ID" ] || die "Guard G FAIL: Recovery-Tag != Recovery-ID (FAIL CLOSED)"
echo " OK"
echo " [guard H] verify-recovery-gold-image PASS:"
( verify_recovery_gold_image >/dev/null 2>&1 ) || die "Guard H FAIL: Recovery-Gold nicht verifiziert (FAIL CLOSED)"
echo " OK"
echo " [guard I] No-op-Candidate unveraendert:"
local ni; ni="$(docker image inspect -f '{{.Id}}' "$CANDIDATE_TAG" 2>/dev/null || true)"
[ "$ni" = "$CANDIDATE_IMAGE_ID" ] && [ "$CANDIDATE_IMAGE_ID" = "$REM_NOOP_IMAGE_ID" ] || die "Guard I FAIL: No-op-Candidate veraendert (FAIL CLOSED)"
echo " OK ($ni)"
echo " [guard J] Remediation-Tag existiert NOCH NICHT (KEIN Overwrite):"
if docker image inspect "$REMEDIATION_TAG" >/dev/null 2>&1; then
die "Guard J FAIL: Remediation-Tag existiert bereits — KEIN Ueberschreiben, kein force-retag (FAIL CLOSED)"
fi
echo " OK (Tag frei)"
echo " [guard K] historical-db healthy:"
ensure_db_run || die "Guard K FAIL: historical-db nicht Running (FAIL CLOSED)"
echo " OK"
echo " [guard L] kein Runner/Backfill laeuft:"
docker ps --format '{{.Names}}' | grep -qiE 'runner|backfill' && die "Guard L FAIL: Runner/Backfill laeuft (FAIL CLOSED)" || echo " OK"
echo " [guard M] Production Safety geschlossen (isolierter POC-Stack):"
echo " OK (eigene db, eigenes Netz, GET-only API, kein Produktivnetz)"
echo " [guard N] Dockerfile/Context exakt erwartet:"
dockerfile_expected && echo " OK"
echo " [guard O] kein freier Parameter/User-Input:"
echo " OK (keine Argumente akzeptiert)"
}
# ---- GEMEINSAME AUTHORIZED-GOLD-DELTA-POLICY (Phase 13.5) ----
# Einzige autorisierte Divergenz gegen GOLD_MANIFEST:
# app/run_backfill_year.py ALT=RUNNER_GOLD_SHA -> NEU=RUNNER_REMEDIATION_SHA
# (exakte Pfad- UND SHA-Bindung; KEINE generische "1 Divergenz ok"-Lockerung).
# Setzt globale Zaehler AD_MATCH/AD_AUTH/AD_UNAUTH/AD_MISSING/AD_EXTRA und
# AD_RET (ok|fail). fail wenn UNAUTH>0 || MISSING>0 || EXTRA>0.
# Wird von verify_remediation_build_input UND verify_runner_remediation_image
# verwendet -> EINE Policy, keine unterschiedlichen Policies.
authorized_gold_delta() {
local new="$1" gold goldrel goldsha nsha nrel found
AD_MATCH=0; AD_AUTH=0; AD_UNAUTH=0; AD_MISSING=0; AD_EXTRA=0
while read -r goldsha goldrel; do
goldrel="${goldrel#./}"; [ -n "$goldrel" ] || continue
nsha="$(awk -v r="$goldrel" '$2==r{print $1}' "$new")"
if [ -z "$nsha" ]; then AD_MISSING=$((AD_MISSING+1))
elif [ "$nsha" = "$goldsha" ]; then AD_MATCH=$((AD_MATCH+1))
else
if [ "$goldrel" = "app/run_backfill_year.py" ] \
&& [ "$goldsha" = "$RUNNER_GOLD_SHA" ] \
&& [ "$nsha" = "$RUNNER_REMEDIATION_SHA" ]; then
AD_AUTH=$((AD_AUTH+1))
else
AD_UNAUTH=$((AD_UNAUTH+1))
fi
fi
done < "$GOLD_MANIFEST"
while read -r nsha nrel; do
nrel="${nrel#./}"
found="$(awk -v r="$nrel" '$2==r{print $1}' "$GOLD_MANIFEST")"
if [ -z "$found" ]; then
case "$nrel" in
app/app/backfill.py|app/app/test_idempotency.py|app/eligibility.py) ;;
*) AD_EXTRA=$((AD_EXTRA+1)) ;;
esac
fi
done < "$new"
AD_RET="ok"
[ $AD_UNAUTH -eq 0 ] && [ $AD_MISSING -eq 0 ] && [ $AD_EXTRA -eq 0 ] || AD_RET="fail"
}
# Read-only Build-Input-Check: nur app/run_backfill_year.py darf vom Gold-Manifest
# divergieren (und zwar exakt auf RUNNER_REMEDIATION_SHA). alles andere -> FAIL.
verify_remediation_build_input() {
local hm
hm="$(mktemp)"; gen_host_manifest > "$hm"
authorized_gold_delta "$hm"
rm -f "$hm"
echo " Gold $((AD_MATCH+AD_AUTH+AD_UNAUTH+AD_MISSING)): MATCH=$AD_MATCH AUTHORIZED_DIVERGED=$AD_AUTH UNAUTHORIZED_DIVERGED=$AD_UNAUTH MISSING=$AD_MISSING ; unerlaubte EXTRA=$AD_EXTRA"
[ "$AD_AUTH" = "1" ] || die "BUILD-INPUT FAIL: autorisierte Runner-Divergenz fehlt (FAIL CLOSED)"
[ "$AD_RET" = "ok" ] || die "BUILD-INPUT FAIL: unerwartete Divergenz (FAIL CLOSED)"
return 0
}
# ---- build-runner-remediation (DRY_RUN = kein Build; KEIN Overwrite) ----
build_runner_remediation() {
expect_zero "$@"
echo "=== BUILD-RUNNER-REMEDIATION ($RUN_MODE) ==="
echo " Build Context : $REMEDIATION_BUILD_CONTEXT (fix)"
echo " Dockerfile : $REMEDIATION_BUILD_DOCKERFILE (fix)"
echo " Ziel-Tag : $REMEDIATION_TAG (fix, KEIN latest, KEIN Overwrite)"
echo " Guards:"
remediation_guards
echo " [build-input] REMEDIATION_BUILD_INPUT (Build-Context-Manifest):"
build_context_manifest | while IFS= read -r line; do echo " $line"; done
if [ "$RUN_MODE" = "DRY_RUN" ]; then
echo "DRY_RUN_PASS: build-runner-remediation wuerde docker build -f $REMEDIATION_BUILD_DOCKERFILE -t $REMEDIATION_TAG $REMEDIATION_BUILD_CONTEXT (keine Aktion)."
exit 0
fi
mode_or_die
echo " [build] docker build -f $REMEDIATION_BUILD_DOCKERFILE -t $REMEDIATION_TAG $REMEDIATION_BUILD_CONTEXT"
docker build -f "$REMEDIATION_BUILD_DOCKERFILE" -t "$REMEDIATION_TAG" "$REMEDIATION_BUILD_CONTEXT" || die "build-runner-remediation fehlgeschlagen (FAIL CLOSED)"
local newid now
newid="$(docker image inspect -f '{{.Id}}' "$REMEDIATION_TAG" 2>/dev/null)"
[ -n "$newid" ] || die "build: Image-ID nicht ermittelbar (FAIL CLOSED)"
echo " FULL_IMAGE_ID=$newid"
now="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
# Build-Session root-owned anlegen (einmaliges Schreiben der Image-ID)
mkdir -p "$REMEDIATION_SESSION_DIR" || die "build: Session-Dir nicht anlegbar (FAIL CLOSED)"
cat > "$REMEDIATION_SESSION_META" <<EOF
TAG=$REMEDIATION_TAG
FULL_IMAGE_ID=$newid
BUILD_INPUT_SHA=$(build_context_manifest | sha256sum | awk '{print $1}')
TIMESTAMP=$now
HOST_SOT_RUNNER_SHA=$RUNNER_REMEDIATION_SHA
WRAPPER_SHA=$(sha256sum /usr/local/bin/rq-historical | awk '{print $1}')
EOF
chown root:root "$REMEDIATION_SESSION_META"; chmod 600 "$REMEDIATION_SESSION_META"
echo " Session: $REMEDIATION_SESSION_META (root-owned 600)"
echo "BUILD-RUNNER-REMEDIATION OK. Image-ID in Build-Session gespeichert. KEIN Recreate, KEIN Runner, latest unangetastet."
}
# ---- verify-runner-remediation-image (read-only) ----
verify_runner_remediation_image() {
req_ok; expect_zero "$@"
echo "=== VERIFY-RUNNER-REMEDIATION-IMAGE (Candidate vs Host-SoT / Recovery-Gold) ==="
remediation_session || die "verify: keine gueltige Build-Session mit FULL_IMAGE_ID (zuerst build-runner-remediation) (FAIL CLOSED)"
local imgid; imgid="$(remediation_full_image_id)"
echo " REMEDIATION_FULL_IMAGE_ID=$imgid (aus Session, NICHT nur Tag)"
docker image inspect "$imgid" >/dev/null 2>&1 || die "verify: Image-ID nicht inspectable (FAIL CLOSED)"
echo " [Binding] Remediation-Tag -> FULL Image-ID:"
local rt; rt="$(docker image inspect -f '{{.Id}}' "$REMEDIATION_TAG" 2>/dev/null || true)"
[ "$rt" = "$imgid" ] || die "verify: Remediation-Tag ($rt) != gespeicherte Image-ID ($imgid) (FAIL CLOSED)"
echo " OK"
echo " [Host-SoT App-Manifest vs Image] (Candidate == Host-SoT):"
local tmpc tmpdir tmpf
tmpc="rq-rem-verify-$(date +%s)"; tmpdir="/tmp/rq-rem-copy-$$"
rm -rf "$tmpdir"; mkdir -p "$tmpdir"
docker create --name "$tmpc" --entrypoint /bin/true "$imgid" >/dev/null 2>&1 || die "verify: Temp-Container Fehler (FAIL CLOSED)"
docker cp "$tmpc:/app/app" "$tmpdir/appapp" >/dev/null 2>&1 || { docker rm -f "$tmpc" >/dev/null 2>&1; rm -rf "$tmpdir"; die "verify: /app/app nicht lesbar (FAIL CLOSED)"; }
docker rm -f "$tmpc" >/dev/null 2>&1 || true
tmpf="$(mktemp)"
( cd "$tmpdir/appapp" && find . -type f -name '*.py' ! -path '*/__pycache__/*' -printf '%P\n' 2>/dev/null | sort \
| while IFS= read -r rel; do [ -f "$rel" ] && sha256sum "$rel"; done | awk '{print $1" app/"$2}' ) > "$tmpf"
# [Gold-Delta] Image vs GOLD_MANIFEST: exakt 1 autorisierte Runner-Divergenz erlaubt.
authorized_gold_delta "$tmpf"
echo " Image vs GOLD: MATCH=$AD_MATCH AUTHORIZED_DIVERGED=$AD_AUTH UNAUTHORIZED_DIVERGED=$AD_UNAUTH MISSING=$AD_MISSING unerlaubte EXTRA=$AD_EXTRA"
[ "$AD_AUTH" = "1" ] || { rm -rf "$tmpdir"; die "verify: Gold-Delta != exakt 1 autorisierte Runner-Divergenz (FAIL CLOSED)"; }
[ "$AD_RET" = "ok" ] || { rm -rf "$tmpdir"; die "verify: Image != Gold-Manifest (unzulaessige Divergenz) (FAIL CLOSED)"; }
# [Candidate vs Host-SoT] Image-App-Manifest == Host-SoT-App-Manifest (direkter
# Manifest-zu-Manifest-Vergleich, OHNE Gold-Bezug; vollstaendig MATCH).
# Host-only-Legacy-Dateien (im Image via .dockerignore ausgeschlossen) sind erlaubt.
local hmf cmiss cextra
hmf="$(mktemp)"; gen_host_manifest > "$hmf"
cmiss=0; cextra=0
while read -r hsha hrel; do
hrel="${hrel#./}"; [ -n "$hrel" ] || continue
case "$hrel" in
app/app/backfill.py|app/app/test_idempotency.py|app/eligibility.py) continue ;;
esac
isha="$(awk -v r="$hrel" '$2==r{print $1}' "$tmpf")"
if [ -z "$isha" ]; then cmiss=$((cmiss+1))
elif [ "$isha" != "$hsha" ]; then cextra=$((cextra+1)); fi
done < "$hmf"
rm -f "$hmf"
echo " Candidate vs Host-SoT: MATCH (direkt) MISSING=$cmiss DIVERGED=$cextra"
[ "$cmiss" = "0" ] && [ "$cextra" = "0" ] || { rm -rf "$tmpdir"; die "verify: Candidate != Host-SoT (FAIL CLOSED)"; }
rm -f "$tmpf"
echo " [Critical-Files] $REMEDIATION_TAG vs gold:"
verify_critical_files "$tmpdir/appapp" runner
rm -rf "$tmpdir"
echo " [Runner] Image vs Host-SoT persistent:"
local ir; ir="$(docker run --rm --entrypoint /bin/sh "$imgid" -c "sha256sum /app/app/run_backfill_year.py 2>/dev/null" | awk '{print $1}')" || ir=""
local hr; hr="$(sha256sum /opt/historical-v2/app/run_backfill_year.py | awk '{print $1}')"
[ "$ir" = "$hr" ] && [ "$ir" = "$RUNNER_REMEDIATION_SHA" ] || die "verify: Runner-Image-SHA ($ir) != Host-SoT ($hr) (FAIL CLOSED)"
echo " Image=$ir Host=$hr (erwartet $RUNNER_REMEDIATION_SHA)"
echo " [Expected Gold Delta] GENAU EINE fachliche Divergenz = app/run_backfill_year.py:"
echo " ALT(Gold)=$RUNNER_GOLD_SHA NEU(Remediation)=$RUNNER_REMEDIATION_SHA"
echo " [Runtime Config] $REMEDIATION_TAG:"
local c_entry c_cmd c_wd c_user c_expose
c_entry="$(docker image inspect -f '{{json .Config.Entrypoint}}' "$imgid" 2>/dev/null)"
c_cmd="$(docker image inspect -f '{{json .Config.Cmd}}' "$imgid" 2>/dev/null)"
c_wd="$(docker image inspect -f '{{.Config.WorkingDir}}' "$imgid" 2>/dev/null)"
c_user="$(docker image inspect -f '{{.Config.User}}' "$imgid" 2>/dev/null)"
c_expose="$(docker image inspect -f '{{json .Config.ExposedPorts}}' "$imgid" 2>/dev/null)"
printf ' Entrypoint=%s Cmd=%s WorkDir=%s User=%s Expose=%s\n' "${c_entry:-null}" "${c_cmd:-null}" "${c_wd:-}" "${c_user:-}" "${c_expose:-}"
[ "$c_wd" = "/app" ] || die "verify: WorkingDir unerwartet (FAIL CLOSED)"
echo " [Build-Input] Session-Host-Runner == persistent Host-Runner:"
local srunner; srunner="$(grep '^HOST_SOT_RUNNER_SHA=' "$REMEDIATION_SESSION_META" | cut -d= -f2)"
[ "$srunner" = "$RUNNER_REMEDIATION_SHA" ] || die "verify: Session-Runner != erwartet (FAIL CLOSED)"
echo " OK ($srunner)"
echo "VERDICT: REMEDIATION IMAGE OK (Candidate==Host-SoT; Gold-Delta exakt 1 autorisierte Runner-Divergenz; Critical-SHAs + Config PASS)"
exit 0
}
# ---- recreate-runner-remediation-safe (NUR historical-service; nur Red Queen) ----
recreate_runner_remediation_safe() {
expect_zero "$@"
echo "=== RECREATE-RUNNER-REMEDIATION-SAFE ($RUN_MODE) ==="
echo " Ziel-Service : $HISTORICAL_SERVICE (NUR dieser; KEIN db, KEIN Compose-Down, KEINE Volumes)"
remediation_session || die "recreate: keine gueltige Build-Session (zuerst build-runner-remediation) (FAIL CLOSED)"
local imgid; imgid="$(remediation_full_image_id)"
echo " Image : $REMEDIATION_TAG -> $imgid (explizit, KEIN latest)"
echo " Pre-Gates:"
recreate_pre_gates
echo " [gate N] Remediation-Tag -> FULL Image-ID:"
local rt; rt="$(docker image inspect -f '{{.Id}}' "$REMEDIATION_TAG" 2>/dev/null || true)"
[ "$rt" = "$imgid" ] || die "Gate N FAIL: Remediation-Tag != FULL Image-ID (FAIL CLOSED)"
echo " OK"
echo " [gate O] verify-runner-remediation-image PASS:"
( verify_runner_remediation_image >/dev/null 2>&1 ) || die "Gate O FAIL: Remediation-Image nicht verifiziert (FAIL CLOSED)"
echo " OK"
echo " Baseline vor Recreate:"
recreate_baseline
if [ "$RUN_MODE" = "DRY_RUN" ]; then
echo "DRY_RUN_PASS: wuerde docker compose -f $COMPOSE_FILE -f <override> up -d --no-deps --no-build $HISTORICAL_SERVICE (Image=$REMEDIATION_TAG, gespeicherte ID=$imgid). KEINE Aktion."
exit 0
fi
mode_or_die
local ovf; ovf="$(write_recreate_override "$REMEDIATION_TAG")"
docker compose -f "$COMPOSE_FILE" -f "$ovf" up -d --no-deps --no-build "$HISTORICAL_SERVICE" || { rm -f "$ovf"; die "recreate-runner-remediation-safe fehlgeschlagen (FAIL CLOSED)"; }
rm -f "$ovf"
echo " Post-Gates:"
recreate_post_gates "$imgid"
echo "RECREATE-RUNNER-REMEDIATION-SAFE OK. Container laeuft mit Remediation-Image (gespeicherte ID)."
}
# ---- build-historical (NUR Build, kein recreate; DRY_RUN = kein Bau) ----
build_historical() {
expect_zero "$@"
echo "=== BUILD-HISTORICAL ($RUN_MODE) ==="
echo " Projekt : $COMPOSE_PROJECT"
echo " Compose : $COMPOSE_FILE"
echo " Service : $HISTORICAL_SERVICE"
echo " Kontext : $COMPOSE_PROJECT (Dockerfile: $COMPOSE_PROJECT/Dockerfile)"
# 1. Dry-run zeigt Guards geplant, aber fuehrt sie (bis auf reine Checks) nicht aus
sot_bestandscheck >/dev/null
echo " [guard] backup-sot vorhanden: OK"
# verify-recovery check ist read-only, kann im dry-run laufen
echo " [guard] verify-sot-recovery:"
verify_recovery_ok
echo " [guard] Gold-Image $GOLD_IMAGE_ID: $(ensure_gold_image && echo OK)"
echo " [guard] Gold-Container Running: $(ensure_gold_run && echo OK)"
echo " [guard] historical-db Running: $(ensure_db_run && echo OK)"
echo " [guard] DB-Volume: $(ensure_db_volume && echo OK)"
echo " [guard] Builddefinition (Dockerfile+Compose): $(ensure_builddef && echo OK)"
echo " [guard] Build-Pollution-Exclusion valid:"
exclusion_valid && echo " .dockerignore OK (nur app/app/ + app/eligibility.py)"
if [ "$RUN_MODE" = "DRY_RUN" ]; then
echo "DRY_RUN_PASS: build-historical wuerde aus $COMPOSE_PROJECT bauen (keine Aktion)."
exit 0
fi
mode_or_die
echo " [build] docker compose -f $COMPOSE_FILE build $HISTORICAL_SERVICE"
docker compose -f "$COMPOSE_FILE" build "$HISTORICAL_SERVICE" || die "build fehlgeschlagen"
echo " [result]"
docker images --filter reference="${HISTORICAL_SERVICE}" --format 'Image {{.ID}} Created {{.CreatedSince}}'
echo "BUILD-HISTORICAL OK. Gold-Image NICHT geloescht ($GOLD_IMAGE_ID). Nur Build, kein Recreate."
}
# ---- recreate-historical (NUR historical-service, kein Runner; DRY_RUN = kein Recreate) ----
recreate_historical() {
expect_zero "$@"
echo "=== RECREATE-HISTORICAL ($RUN_MODE) ==="
echo " Service : $HISTORICAL_SERVICE (NUR dieser; NICHT historical-db)"
sot_bestandscheck >/dev/null
echo " [guard] backup-sot vorhanden: OK"
echo " [guard] Gold-Image $GOLD_IMAGE_ID: $(ensure_gold_image && echo OK)"
echo " [guard] Gold-Manifest: $([ -f "$GOLD_MANIFEST" ] && echo OK || echo FEHLT)"
echo " [guard] historical-db Running: $(ensure_db_run && echo OK)"
echo " [guard] DB-Volume: $(ensure_db_volume && echo OK)"
if [ "$RUN_MODE" = "DRY_RUN" ]; then
echo "DRY_RUN_PASS: recreate-historical wuerde historical-service neu erstellen (keine Aktion)."
exit 0
fi
mode_or_die
echo " [recreate] docker compose -f $COMPOSE_FILE up -d --no-deps $HISTORICAL_SERVICE"
docker compose -f "$COMPOSE_FILE" up -d --no-deps "$HISTORICAL_SERVICE" || die "recreate fehlgeschlagen"
sleep 4
local running
running="$(docker inspect -f '{{.State.Running}}' "$HISTORICAL_SERVICE" 2>/dev/null)"
[ "$running" = "true" ] || die "neuer Container laeuft nicht (FAIL CLOSED)"
echo " [Container] ID: $(docker inspect -f '{{.ID}}' "$HISTORICAL_SERVICE")"
echo " [Image] $(docker inspect -f '{{.Image}}' "$HISTORICAL_SERVICE")"
echo "RECREATE-HISTORICAL OK. KEIN Runner, KEIN compose down, KEIN Volume-Remove."
}
# ---- verify-gold-reproduction (read-only) ----
verify_gold_reproduction() {
req_ok; expect_zero "$@"
echo "=== VERIFY-GOLD-REPRODUCTION (Gold-Manifest vs Running-Container) ==="
[ -f "$GOLD_MANIFEST" ] || die "Gold-Manifest fehlt (FAIL CLOSED)"
local tmp; tmp="$(mktemp)"
gen_manifest "$HISTORICAL_SERVICE" > "$tmp"
classify_and_report "$tmp"
local ret="$G_RET"
rm -f "$tmp"
if [ "$ret" = "ok" ]; then
echo "VERDICT: GOLD REPRODUCTION OK (0 DIVERGED/MISSING/unerlaubte EXTRA)"
exit 0
else
echo "VERDICT: GOLD REPRODUCTION FAIL"
exit 2
fi
}
validate_stage() {
local key="$1" file real owner dmode
file="$(stage_file "$key")"
[ -e "$file" ] || die "Staging-Kandidat fehlt: $file (lege zuerst die Datei ab)"
real="$(realpath -m "$file")"
[[ "$real" == "$STAGING_ROOT/"* ]] || die "Traversal blockiert: realpath '$real' ausserhalb Staging"
[ "$real" = "$file" ] || die "Alias erkannt (realpath != Pfad): $file"
[ -f "$file" ] || die "keine reguläre Datei: $file"
[ -L "$file" ] && die "Symlink nicht erlaubt: $file"
owner="$(stat -c '%U' "$file")"
[ "$owner" = "rq-host" ] || die "Staging-Kandidat Owner = $owner (erwartet rq-host)"
dmode="$(stat -c '%a' "$STAGING_ROOT")"
[[ "$dmode" != *002 ]] && [[ "$dmode" != *022 ]] || die "Staging-Root global schreibbar '$dmode' (FAIL CLOSED)"
echo "OK: $key -> $file (regular, no symlink, owner rq-host, in staging)"
}
# ---- redact: maskiert Passwort-/Secret-Werte (fuer build-def Anzeige) ----
redact_line() {
# Zeilen mit "PASSWORD=" / "SECRET=" / "TOKEN=" : Wert maskieren
echo "$1" | sed -E 's/(PASSWORD|PASSWORDS|SECRET|SECRETS|TOKEN|TOKENS|API_KEY|CLIENT_SECRET|PRIVATE_KEY)(["'"'"']?[[:space:]]*=[[:space:]]*["'"'"']?)[^[:space:]"'"'"']+/\\1\\2***REDACTED***/Ig'
}
# =============================================================================
# IMAGE-RETENTION / ARCHIVE CONTROL PLANE (Phase 13.5 Schutzschicht)
# Zwei unabhängige Ebenen je Critical Image:
# A) Stopped Pin Container (docker create, nie gestartet) -> Image-Referenz
# B) sha256-verifiziertes docker save-Archiv -> externes Archiv
# READY-Gate: IMAGE VERIFY + PIN VERIFY + ARCHIVE VERIFY alle PASS.
# KEIN Overwrite. KEIN Coolify-Cleanup-Disable/Managed-Spoof. Kein echter
# Rebuild/Restore/Recreate. Kein freies Input (feste Rollen: recovery / remediation).
# =============================================================================
role_ok() { # $1 = role -> 0 wenn gueltig
case "$1" in recovery|remediation) return 0 ;; *) return 1 ;; esac
}
# Feste Rollen->Image/Identity-Bindung (Punkt 2/4/8/9/10) — hardcoded.
role_image_id() { # $1 = role -> echo erwartete FULL_IMAGE_ID
case "$1" in
recovery) echo "$RECOVERY_GOLD_IMAGE_ID" ;; # sha256:d2db926d...
remediation) echo "$REMEDIATION_TAG" ;; # Tag -> aufgeloest via inspect (Image fehlt aktuell)
esac
}
role_tag() {
case "$1" in
recovery) echo "$RECOVERY_GOLD_TAG_FINAL" ;;
remediation) echo "$REMEDIATION_TAG" ;;
esac
}
role_pin() {
case "$1" in
recovery) echo "$PIN_RECOVERY_NAME" ;;
remediation) echo "$PIN_REMEDIATION_NAME" ;;
esac
}
role_arch_sub() { echo "${ARCHIVE_SUB[$1]}" ; }
role_arch_dir() { echo "$ARCHIVE_ROOT/$(role_arch_sub "$1")" ; }
# --- Schicht A: Stopped Pin Container ---
# Container-Referenz schuetzt Image-Objekt vor Coolify rmi (empirisch belegt:
# No-op ueberlebte, weil Container 2fe3d30c... es referenzierte).
pin_container_identity_ok() { # $1 = role, $2 = actual container id -> 0 wenn ok
local role="$1" actual="$2" expected
case "$role" in
recovery) expected="$RECOVERY_GOLD_IMAGE_ID" ;;
remediation) expected="$PIN_REMEDIATION_TARGET_ID" ;;
esac
[ "$actual" = "$expected" ]
}
# --- Schicht B: Archive Verify ---
verify_archive() { # $1 = role (read-only)
local role="$1" adir sid
adir="$(role_arch_dir "$role")"
case "$role" in
recovery) sid="$RECOVERY_ARCHIVE_ID" ;;
remediation) sid="$REMEDIATION_ARCHIVE_ID" ;;
esac
# Archive existiert
[ -d "$adir" ] || die "verify-archive($role): kein Archiv-Root (FAIL CLOSED)"
local base="$adir/$sid"
[ -d "$base" ] || die "verify-archive($role): kein Session-Ordner $base (FAIL CLOSED)"
# Pflichtdateien
local f img meta manifest runtime
img="$base/image.tar"; meta="$base/session.meta"; manifest="$base/app.manifest"; runtime="$base/runtime-config.txt"
[ -f "$img" ] && [ ! -L "$img" ] || die "verify-archive($role): image.tar fehlt/kein File (FAIL CLOSED)"
[ -s "$img" ] || die "verify-archive($role): image.tar leer (FAIL CLOSED)"
for f in "$meta" "$manifest" "$runtime"; do
[ -f "$f" ] && [ ! -L "$f" ] || die "verify-archive($role): fehlt $(basename "$f") (FAIL CLOSED)"
done
[ "$(stat -c '%U' "$img")" = "root" ] || die "verify-archive($role): image.tar Owner != root (FAIL CLOSED)"
case "$(stat -c '%a' "$img")" in 600|640|700|400) ;; *) die "verify-archive($role): image.tar Mode unsicher ($(stat -c '%a' "$img")) (FAIL CLOSED)" ;; esac
# Sha256 stimmt (Punkt 7)
local want got
want="$(awk -F'=' '$1=="ARCHIVE_SHA256"{print $2}' "$meta" 2>/dev/null)"
[ -n "$want" ] && [[ "$want" =~ ^[a-f0-9]{64}$ ]] || die "verify-archive($role): ARCHIVE_SHA256 fehlt/ungueltig in session.meta (FAIL CLOSED)"
got="$(sha256sum "$img" | awk '{print $1}')"
[ "$got" = "$want" ] || die "verify-archive($role): image.tar SHA mismatch (FAIL CLOSED)"
# session.meta Vollstaendigkeit (Punkt 5)
local key
for key in ROLE TAG FULL_IMAGE_ID ARCHIVE_SHA256 ARCHIVE_SIZE CREATED_AT HOST_SOT_RUNNER_SHA BUILD_INPUT_SHA WRAPPER_SHA APP_MANIFEST_SHA RUNTIME_CONFIG_SHA; do
grep -q "^$key=.*" "$meta" || die "verify-archive($role): session.meta fehlt $key (FAIL CLOSED)"
done
local mrole
mrole="$(awk -F'=' '$1=="ROLE"{print $2}' "$meta" 2>/dev/null)"
[ "$mrole" = "$role" ] || die "verify-archive($role): ROLE in session.meta = '$mrole' (FAIL CLOSED)"
echo "VERIFY-ARCHIVE($role): PASS ($base)"
}
# --- Ready-Gate (Punkt 14) ---
verify_ready() { # $1 = role
local role="$1" label
case "$role" in
recovery) label="RECOVERY READY" ;;
remediation) label="CANDIDATE READY" ;;
esac
echo "=== $label (READY-Gate: IMAGE VERIFY + PIN VERIFY + ARCHIVE VERIFY) ==="
# Recovery-Identitaet aus kontrolliertem State laden (Punkt 17/19). Kein
# statisches d2db, kein latest. FAIL CLOSED wenn State fehlt.
if [ "$role" = "recovery" ]; then
load_recovery_state
fi
# IMAGE VERIFY (zwingend inspectable + korrekt)
echo " [IMAGE] $(role_tag "$role"):"
verify_image_present "$role" || die "READY-Gate FAIL: Image nicht inspectable (FAIL CLOSED)"
echo " OK"
echo " [PIN] $(role_pin "$role"):"
verify_pin "$role" || die "READY-Gate FAIL: Pin nicht PASS (FAIL CLOSED)"
echo " OK"
echo " [ARCHIVE] $(role_arch_dir "$role"):"
verify_archive "$role" >/dev/null || die "READY-Gate FAIL: Archive nicht PASS (FAIL CLOSED)"
echo " OK"
echo "$label: PASS"
}
# --- Image-Present-Verify (Schicht-Pruefung: Image inspectable + Identity) ---
verify_image_present() { # $1 = role (read-only) -> 0 wenn Image vorhanden+bindung ok
local role="$1" id tag actualid
tag="$(role_tag "$role")"
case "$role" in
recovery)
id="$RECOVERY_GOLD_IMAGE_ID"
# Tag muss exakt auf die immutable Recovery-ID zeigen (kein Overwrite-Erwartung)
actualid="$(docker image inspect -f '{{.Id}}' "$tag" 2>/dev/null || true)"
[ "$actualid" = "$id" ] || return 1
;;
remediation)
# Candidate: FULL_IMAGE_ID aus Build-Session; Tag -> ID Bindung
if ! remediation_session; then die "verify-image(remediation): keine gueltige Build-Session (FAIL CLOSED)"; fi
id="$(remediation_full_image_id)"
actualid="$(docker image inspect -f '{{.Id}}' "$id" 2>/dev/null || true)"
[ -n "$actualid" ] || return 1
;;
esac
return 0
}
# --- Schicht A: Pin-Verify (Punkt 12) ---
verify_pin() { # $1 = role (read-only) -> 0 wenn Pin PASS
local role="$1" name imgid nw mounts ports rp priv capadd expected state
name="$(role_pin "$role")"
docker container inspect "$name" >/dev/null 2>&1 || { echo " PIN($role): Container $name fehlt"; return 1; }
expected="$(role_pin_target_id "$role")"
imgid="$(docker container inspect -f '{{.Image}}' "$name" 2>/dev/null)"
# .Image ist die Image-ID des referenzierten Images (exakt FULL_IMAGE_ID)
[ -n "$expected" ] || die "verify-pin($role): Ziel-Image-ID nicht bekannt (FAIL CLOSED)"
# Image-ID kann lang (sha256:...) sein — Bindung: .Image == erwartete FULL_IMAGE_ID
[ "$imgid" = "$expected" ] || { echo " PIN($role): Image-ID $imgid != erwartet $expected"; return 1; }
state="$(docker container inspect -f '{{.State.Status}}' "$name" 2>/dev/null)"
case "$state" in created|exited|stopped) ;; *) echo " PIN($role): State=$state (nicht created/stopped — VERBOTEN running)"; return 1 ;; esac
nw="$(docker container inspect -f '{{.HostConfig.NetworkMode}}' "$name" 2>/dev/null)"
[ "$nw" = "none" ] || { echo " PIN($role): NetworkMode=$nw != none"; return 1; }
mounts="$(docker container inspect -f '{{json .Mounts}}' "$name" 2>/dev/null)"
[ "$mounts" = "[]" ] || { echo " PIN($role): Mounts != []"; return 1; }
ports="$(docker container inspect -f '{{json .HostConfig.PortBindings}}' "$name" 2>/dev/null)"
[ "$ports" = "{}" ] || { echo " PIN($role): PortBindings != {}"; return 1; }
rp="$(docker container inspect -f '{{.HostConfig.RestartPolicy.Name}}' "$name" 2>/dev/null)"
[ "$rp" = "no" ] || { echo " PIN($role): RestartPolicy=$rp != no"; return 1; }
priv="$(docker container inspect -f '{{.HostConfig.Privileged}}' "$name" 2>/dev/null)"
[ "$priv" = "false" ] || { echo " PIN($role): Privileged=$priv"; return 1; }
capadd="$(docker container inspect -f '{{json .HostConfig.CapAdd}}' "$name" 2>/dev/null)"
[ "$capadd" = "null" ] || [ "$capadd" = "[]" ] || { echo " PIN($role): CapAdd=$capadd"; return 1; }
return 0
}
# --- Ziel-Image-ID fuer Pin (Schicht A bindet exakt FULL_IMAGE_ID) ---
role_pin_target_id() { # $1 = role
case "$1" in
recovery) echo "$RECOVERY_GOLD_IMAGE_ID" ;;
remediation)
if remediation_session; then echo "$(remediation_full_image_id)"; else echo ""; fi
;;
esac
}
# --- Archive-Metadaten-Generator (Punkt 5; kein Secret) ---
gen_archive_meta() { # $1 = role, $2 = session dir, $3 = image id, $4 = archive size, $5 = archive sha
local role="$1" dir="$2" image_id="$3" asize="$4" asha="$5" now
now="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
printf 'ROLE=%s\nTAG=%s\nFULL_IMAGE_ID=%s\nARCHIVE_SHA256=%s\nARCHIVE_SIZE=%s\nCREATED_AT=%s\nHOST_SOT_RUNNER_SHA=%s\nBUILD_INPUT_SHA=%s\nWRAPPER_SHA=%s\nAPP_MANIFEST_SHA=%s\nRUNTIME_CONFIG_SHA=%s\n' \
"$role" "$(role_tag "$role")" "$image_id" "$asha" "$asize" "$now" \
"$HOST_SOT_RUNNER_SHA_CUR" "$BUILD_INPUT_SHA_CUR" "$WRAPPER_SHA_CUR" "$APP_MANIFEST_SHA_CUR" "$RUNTIME_CONFIG_SHA_CUR" \
> "$dir/session.meta"
chown root:root "$dir/session.meta"; chmod 600 "$dir/session.meta"
}
# --- runtime-config Evidence (Punkt 4: runtime-config.txt) — keine Secrets ---
write_runtime_config() { # $1 = role, $2 = session dir
local role="$1" dir="$2" tag="$(role_tag "$role")" id
id="$(docker image inspect -f '{{.Id}}' "$tag" 2>/dev/null)"
{
echo "RUNTIME CONFIG EVIDENCE (Phase 13.5 Archive) — keine Secrets"
echo "ROLE=$role"
echo "FULL_IMAGE_ID=$id"
echo "Entrypoint=$(docker image inspect -f '{{json .Config.Entrypoint}}' "$tag" 2>/dev/null)"
echo "Cmd=$(docker image inspect -f '{{json .Config.Cmd}}' "$tag" 2>/dev/null)"
echo "WorkingDir=$(docker image inspect -f '{{.Config.WorkingDir}}' "$tag" 2>/dev/null)"
echo "User=$(docker image inspect -f '{{.Config.User}}' "$tag" 2>/dev/null)"
echo "ExposedPorts=$(docker image inspect -f '{{json .Config.ExposedPorts}}' "$tag" 2>/dev/null)"
echo "ENV_KEYS=$(docker image inspect -f '{{range .Config.Env}}{{println .}}{{end}}' "$tag" 2>/dev/null | cut -d= -f1 | tr '\n' ',' | sed 's/,$//')"
} > "$dir/runtime-config.txt"
chown root:root "$dir/runtime-config.txt"; chmod 600 "$dir/runtime-config.txt"
}
# --- app.manifest Evidence (Punkt 4) — Sha256 aller *.py im Image-Layer ---
write_app_manifest() { # $1 = role, $2 = session dir, $3 = image id/tag
local role="$1" dir="$2" ref="$3" tmpc tmpd
tmpc="rq-arch-manifest-$$"
tmpd="/tmp/rq-arch-manifest-$$"
rm -rf "$tmpd"; mkdir -p "$tmpd"
docker create --name "$tmpc" --entrypoint /bin/true "$ref" >/dev/null 2>&1 \
|| { rm -rf "$tmpd"; die "archive($role): Temp-Container Fehler (FAIL CLOSED)"; }
if ! docker cp "$tmpc:/app/app" "$tmpd/appapp" >/dev/null 2>&1; then
docker rm -f "$tmpc" >/dev/null 2>&1 || true; rm -rf "$tmpd"
die "archive($role): /app/app nicht lesbar (FAIL CLOSED)"
fi
docker rm -f "$tmpc" >/dev/null 2>&1 || true
( cd "$tmpd/appapp" && find . -type f -name '*.py' ! -path '*/__pycache__/*' -printf '%P\n' 2>/dev/null | sort \
| while IFS= read -r rel; do [ -f "$rel" ] && sha256sum "$rel"; done | awk '{print $1" app/"$2}' ) > "$dir/app.manifest"
rm -rf "$tmpd"
chown root:root "$dir/app.manifest"; chmod 600 "$dir/app.manifest"
}
# =============================================================================
# COMMAND-FUNKTIONEN (Phase 13.5 Archive/Pin) — fail-closed, kein Overwrite,
# kein echter Rebuild/Restore/Recreate in dieser Mission.
# =============================================================================
# archive-recovery-image / archive-remediation-image
archive_image() { # $1 = role
local role="$1"
expect_zero "${@:2}"
echo "=== ARCHIVE-$role ($RUN_MODE) ==="
local tag id subdir sdir
tag="$(role_tag "$role")"
case "$role" in
recovery)
# Aktive Recovery-Identitaet aus kontrolliertem State (Punkt 19). Kein d2db.
load_recovery_state
id="$RECOVERY_GOLD_IMAGE_ID"
;;
remediation)
remediation_session || die "archive(remediation): keine gueltige Build-Session (FAIL CLOSED)"
id="$(remediation_full_image_id)"
PIN_REMEDIATION_TARGET_ID="$id"
;;
esac
echo " FULL_IMAGE_ID=$id"
echo " Tag=$tag"
# Punkt 6: Image-inspect + Tag->ID Gate VOR save
echo " [inspect] Tag -> ID Gate:"
local tagid
tagid="$(docker image inspect -f '{{.Id}}' "$tag" 2>/dev/null || true)"
if [ -z "$tagid" ]; then
die "archive($role): Source Image nicht inspectable (SOURCE IMAGE NOT INSPECTABLE — FAIL CLOSED)"
fi
[ "$tagid" = "$id" ] || die "archive($role): Tag->ID Gate FAIL ($tagid != $id) (FAIL CLOSED)"
echo " OK ($tagid)"
# Punkt 7: Immutability — Archiv-Session darf NICHT existieren
local sdir
sdir="$(role_arch_dir "$role")" # = $ARCHIVE_ROOT/<recovery|candidates>
local sid
case "$role" in
recovery) sid="$RECOVERY_ARCHIVE_ID" ;;
remediation) sid="${id#sha256:}"; sid="${sid:0:12}" ;;
esac
local base="$sdir/$sid"
# Fail-closed wenn bereits existiert (kein Overwrite)
if [ -e "$base" ]; then die "archive($role): Session-Ordner existiert bereits $base (KEIN Overwrite, FAIL CLOSED)"; fi
if [ "$RUN_MODE" = "DRY_RUN" ]; then
echo "DRY_RUN_PASS: archive($role) wuerde anlegen: $base/ (image.tar + sha256 + session.meta + app.manifest + runtime-config.txt). Keine Aktion."
exit 0
fi
mode_or_die
# MUTATION: Archive anlegen + docker save
mkdir -p "$base" || die "archive($role): Session-Ordner nicht anlegbar (FAIL CLOSED)"
# Session.meta ZUSAMMENBAUEN VOR save (fuer Size/SHA brauchen aber erst nach save)
# Reihenfolge: save -> size/sha -> meta -> manifest -> runtime
echo " [save] docker image save $id > image.tar"
if ! docker image save "$id" > "$base/image.tar" 2>/dev/null; then
rm -rf "$base"; die "archive($role): docker save fehlgeschlagen (FAIL CLOSED)"
fi
chown root:root "$base/image.tar"; chmod 600 "$base/image.tar"
local asha asize
asha="$(sha256sum "$base/image.tar" | awk '{print $1}')"
asize="$(stat -c '%s' "$base/image.tar")"
# SHAs der Metadaten (zur Laufzeit, aus Build-Session / Host-SoT)
HOST_SOT_RUNNER_SHA_CUR="$(sha256sum /opt/historical-v2/app/run_backfill_year.py 2>/dev/null | awk '{print $1}')"
BUILD_INPUT_SHA_CUR="$(build_context_manifest 2>/dev/null | sha256sum | awk '{print $1}')"
WRAPPER_SHA_CUR="$(sha256sum /usr/local/bin/rq-historical | awk '{print $1}')"
APP_MANIFEST_SHA_CUR="$(sha256sum "$base/app.manifest" 2>/dev/null | awk '{print $1}')"
RUNTIME_CONFIG_SHA_CUR="$(sha256sum "$base/runtime-config.txt" 2>/dev/null | awk '{print $1}')"
gen_archive_meta "$role" "$base" "$id" "$asize" "$asha"
write_runtime_config "$role" "$base"
write_app_manifest "$role" "$base" "$id"
# SHA erneut nach manifest/runtime (weil meta SHA referenziert)
APP_MANIFEST_SHA_CUR="$(sha256sum "$base/app.manifest" | awk '{print $1}')"
RUNTIME_CONFIG_SHA_CUR="$(sha256sum "$base/runtime-config.txt" | awk '{print $1}')"
gen_archive_meta "$role" "$base" "$id" "$asize" "$asha"
echo "ARCHIVE($role) OK: $base/ (${asize} bytes, sha256 ${asha:0:16}...)"
echo " image.tar.sha256:"
sha256sum "$base/image.tar" > "$base/image.tar.sha256"
chown root:root "$base/image.tar.sha256"; chmod 600 "$base/image.tar.sha256"
}
# pin-recovery-image / pin-remediation-image
pin_image() { # $1 = role
local role="$1"
expect_zero "${@:2}"
echo "=== PIN-$role ($RUN_MODE) ==="
local name tag id
name="$(role_pin "$role")"
tag="$(role_tag "$role")"
case "$role" in
recovery)
# Aktive Recovery-Identitaet aus kontrolliertem State (Punkt 19). Kein d2db.
load_recovery_state
id="$RECOVERY_GOLD_IMAGE_ID"
;;
remediation)
remediation_session || die "pin(remediation): keine gueltige Build-Session (FAIL CLOSED)"
id="$(remediation_full_image_id)"
PIN_REMEDIATION_TARGET_ID="$id"
;;
esac
# Punkt 11: Containername darf NICHT existieren (kein Delete/Overwrite/Replace)
if docker container inspect "$name" >/dev/null 2>&1; then
# Identitaet pruefen; wenn korrekt -> ok (keine Aenderung); wenn falsch -> FAIL CLOSED
local existing
existing="$(docker container inspect -f '{{.Image}}' "$name" 2>/dev/null)"
if pin_container_identity_ok "$role" "$existing"; then
echo "PIN($role): Container existiert bereits korrekt (Image=$existing). KEINE Aenderung."
exit 0
else
die "pin($role): Container $name existiert mit falschem Image ($existing). Kein Replace (FAIL CLOSED)"
fi
fi
echo " FULL_IMAGE_ID=$id"
# Punkt 6: Image muss inspectable sein (Source Gate)
if ! docker image inspect "$id" >/dev/null 2>&1; then
die "pin($role): Source Image nicht inspectable (SOURCE IMAGE NOT INSPECTABLE — FAIL CLOSED)"
fi
if [ "$RUN_MODE" = "DRY_RUN" ]; then
echo "DRY_RUN_PASS: pin($role) wuerde docker create --name $name --network none --restart no $id (keine Aktion)."
exit 0
fi
mode_or_die
# MUTATION: minimaler Stopped-Container (Punkt 8/10)
echo " [create] docker create --name $name --network none --restart no $id"
if ! docker create --name "$name" --network none --restart no "$id" >/dev/null 2>&1; then
die "pin($role): docker create fehlgeschlagen (FAIL CLOSED)"
fi
echo "PIN($role) OK: $name -> $id (Stopped, Network=none, Restart=no)"
}
# verify-recovery-pin / verify-remediation-pin / verify-...-archive / verify-...-ready
verify_pin_cmd() { # $1 = role (read-only)
local role="$1"; expect_zero "${@:2}"
echo "=== VERIFY-PIN($role) ==="
if [ "$role" = "recovery" ]; then load_recovery_state; fi
if verify_pin "$role"; then echo "VERIFY-PIN($role): PASS"; exit 0
else die "VERIFY-PIN($role): FAIL (FAIL CLOSED)"; fi
}
verify_archive_cmd() { # $1 = role (read-only)
local role="$1"; expect_zero "${@:2}"
echo "=== VERIFY-ARCHIVE($role) ==="
if [ "$role" = "recovery" ]; then load_recovery_state; fi
verify_archive "$role" || die "VERIFY-ARCHIVE($role): FAIL (FAIL CLOSED)"
exit 0
}
verify_ready_cmd() { # $1 = role (read-only)
local role="$1"; expect_zero "${@:2}"
verify_ready "$role" || die "$(case $role in recovery) echo 'RECOVERY READY';; remediation) echo 'CANDIDATE READY';; esac): FAIL (FAIL CLOSED)"
exit 0
}
# restore-recovery-image-archive (DESIGN; realer Restore NICHT in dieser Mission ausfuehren)
restore_recovery_dry() { # dry-run
expect_zero "$@"
echo "=== RESTORE-RECOVERY-IMAGE-ARCHIVE (DRY-RUN — DESIGN; kein echter Restore) ==="
load_recovery_state
local sdir sid base img
sdir="$(role_arch_dir recovery)"
sid="$RECOVERY_ARCHIVE_ID"
base="$sdir/$sid"
[ -d "$base" ] || die "restore-dry: Archiv-Session fehlt $base (FAIL CLOSED)"
echo " 1. Archiv SHA pruefen: $(sha256sum "$base/image.tar" | awk '{print $1}')"
echo " 2. Metadaten pruefen: session.meta vorhanden"
echo " 3. docker load (in realer Mission; hier NUR Dry-Run)"
echo " 4. Resulting Image inspect"
echo " 5. Application Manifest pruefen"
echo " 6. Runtime Config pruefen"
echo " 7. Tag/Identity pruefen"
echo " 8. danach Pin Verify"
echo " HINWEIS (Punkt 15): Wenn neu geladene FULL_IMAGE_ID != Original-ID bei gleicher App-Identitaet -> NICHT automatisch identisch; sauber Docker-Image-Identity vs Application/Runtime-Equivalence unterscheiden."
echo "RESTORE-RECOVERY-DRY-RUN: DESIGN OK (kein echter Restore ausgefuehrt)"
exit 0
}
# restore-recovery-image-archive (real — in dieser Mission NICHT ausfuehren; nur definiert)
restore_recovery_real() {
expect_zero "$@"
die "restore-recovery-image-archive: Realer Restore ist in dieser Mission NICHT erlaubt (HARTER STOPP Punkt 30). NUR dry-run (FAIL CLOSED)"
}
# =============================================================================
# DEPLOYED-GOLD-SOURCE-RECOVERY CONTROL PLANE (Phase 13.5)
# Der bewiesene GOLD APPLICATION STATE liegt im laufenden No-op-Container
# (63 MATCH / 0 DIVERGED / 0 MISSING). Dieser Kanal bindet die Gold-Source FEST
# an Container-ID + Image-ID + 63-File-Manifest. KEIN latest, KEIN full_sot_backup(),
# KEINE freien Inputs. Nur Source-Recovery, KEIN Docker-Build/Commit/Save/Load/
# Create/Run. Immutable Target. FAIL CLOSED bei jeder Abweichung.
# =============================================================================
# Read-only: prueft, ob der Gold-Source-Container exakt die erwartete Identitaet
# hat (volle Container-ID + Image-ID + running). FAIL CLOSED bei Abweichung.
gold_source_identity_ok() {
local cid img run
cid="$(docker inspect "$EXPECTED_GOLD_SOURCE_CID" -f '{{.Id}}' 2>/dev/null)"
img="$(docker inspect "$EXPECTED_GOLD_SOURCE_CID" -f '{{.Image}}' 2>/dev/null)"
run="$(docker inspect "$EXPECTED_GOLD_SOURCE_CID" -f '{{.State.Running}}' 2>/dev/null)"
[ "$cid" = "$EXPECTED_GOLD_SOURCE_CID" ] || { echo " [FAIL] Container-ID abweichend: $cid"; return 1; }
[ "$img" = "$EXPECTED_GOLD_SOURCE_IMAGE_ID" ] || { echo " [FAIL] Image-ID abweichend: $img"; return 1; }
[ "$run" = "true" ] || { echo " [FAIL] Container nicht running"; return 1; }
echo " [OK] Container-ID == $EXPECTED_GOLD_SOURCE_CID"
echo " [OK] Image-ID == $EXPECTED_GOLD_SOURCE_IMAGE_ID"
echo " [OK] Container running"
return 0
}
# Read-only: prueft das 63-File Application Manifest des Containers gegen Gold.
# Erwartung: 63 MATCH / 0 DIVERGED / 0 MISSING. FAIL CLOSED sonst.
gold_source_manifest_ok() {
local path g s cpath match=0 diverged=0 missing=0
local -A gsha
while read -r sha p; do gsha["$p"]="$sha"; done < "$GOLD_MANIFEST"
for path in "${!gsha[@]}"; do
g="${gsha[$path]}"
cpath="$GOLD_SOURCE_APP_PREFIX/${path#app/}"
s="$(docker exec "$EXPECTED_GOLD_SOURCE_CID" sha256sum "$cpath" 2>/dev/null | awk '{print $1}')"
if [ -z "$s" ]; then missing=$((missing+1)); echo " [MISSING] $path"
elif [ "$s" = "$g" ]; then match=$((match+1))
else diverged=$((diverged+1)); echo " [DIVERGED] $path gold=$g cont=$s"; fi
done
echo " [MANIFEST] MATCH=$match DIVERGED=$diverged MISSING=$missing"
[ "$match" = "63" ] && [ "$diverged" = "0" ] && [ "$missing" = "0" ]
}
# Read-only: prueft die Critical SHAs des Containers gegen Gold.
# Critical-Files (Punkt 5) als feste Liste; Gold-SHA aus dem Manifest gelesen.
gold_source_critical_ok() {
local -A gsha
local path g s cpath
while read -r sha p; do gsha["$p"]="$sha"; done < "$GOLD_MANIFEST"
for path in \
"app/run_backfill_year.py" \
"app/persistence/repository.py" \
"app/backfill.py" \
"app/pipeline_v2.py" \
"app/trust/__init__.py" \
"app/trust/provenance.py" \
"app/trust/trust.py" \
"app/trust/sanity.py" \
"app/trust/trust_gate.py"; do
g="${gsha[$path]:-}"
if [ -z "$g" ]; then echo " [SKIP] $path nicht im Gold-Manifest"; continue; fi
cpath="$GOLD_SOURCE_APP_PREFIX/${path#app/}"
s="$(docker exec "$EXPECTED_GOLD_SOURCE_CID" sha256sum "$cpath" 2>/dev/null | awk '{print $1}')"
if [ "$s" = "$g" ]; then echo " [OK] $path == $g"
else echo " [FAIL] $path gold=$g cont=$s"; return 1; fi
done
return 0
}
# Read-only: prueft die Runtime Config des Containers gegen die Recovery-Baseline.
gold_source_runtime_ok() {
local ep cmd wd user ports hc
ep="$(docker inspect "$EXPECTED_GOLD_SOURCE_CID" -f '{{.Config.Entrypoint}}' 2>/dev/null)"
cmd="$(docker inspect "$EXPECTED_GOLD_SOURCE_CID" -f '{{.Config.Cmd}}' 2>/dev/null)"
wd="$(docker inspect "$EXPECTED_GOLD_SOURCE_CID" -f '{{.Config.WorkingDir}}' 2>/dev/null)"
user="$(docker inspect "$EXPECTED_GOLD_SOURCE_CID" -f '{{.Config.User}}' 2>/dev/null)"
ports="$(docker inspect "$EXPECTED_GOLD_SOURCE_CID" -f '{{.Config.ExposedPorts}}' 2>/dev/null)"
hc="$(docker inspect "$EXPECTED_GOLD_SOURCE_CID" -f '{{.Config.Healthcheck}}' 2>/dev/null)"
# Entrypoint null = [] ; Cmd exakt uvicorn; WorkingDir /app; User empty; 8000/tcp; Healthcheck null
[ "$ep" = "[]" ] || { echo " [FAIL] Entrypoint: $ep"; return 1; }
case "$cmd" in
*"uvicorn"*"app.api.main:app"*"--host"*"0.0.0.0"*"--port"*"8000"*) : ;;
*) echo " [FAIL] Cmd: $cmd"; return 1 ;;
esac
[ "$wd" = "/app" ] || { echo " [FAIL] WorkingDir: $wd"; return 1; }
[ -z "$user" ] || { echo " [FAIL] User: $user"; return 1; }
case "$ports" in *"8000/tcp"*) : ;; *) echo " [FAIL] ExposedPorts: $ports"; return 1 ;; esac
[ "$hc" = "<nil>" ] || [ -z "$hc" ] || { echo " [FAIL] Healthcheck: $hc"; return 1; }
echo " [OK] Entrypoint null, Cmd uvicorn, WorkingDir /app, User empty, 8000/tcp, Healthcheck null"
return 0
}
# verify-deployed-gold-source (read-only): alle Gold-Source-Gates.
verify_deployed_gold_source() {
expect_zero "$@"
echo "=== VERIFY-DEPLOYED-GOLD-SOURCE (read-only) ==="
gold_source_identity_ok || die "verify-deployed-gold-source: Source-Identity FAIL (FAIL CLOSED)"
gold_source_manifest_ok || die "verify-deployed-gold-source: 63-File-Manifest FAIL (FAIL CLOSED)"
gold_source_critical_ok || die "verify-deployed-gold-source: Critical-SHAs FAIL (FAIL CLOSED)"
gold_source_runtime_ok || die "verify-deployed-gold-source: Runtime-Config FAIL (FAIL CLOSED)"
echo "VERDICT: DEPLOYED GOLD SOURCE OK (Identity + 63/63 + Critical + Runtime)"
return 0
}
# Read-only: prueft die Build-Support-SHA-Bindings gegen die feste Quelle.
gold_build_support_ok() {
local key spec src exp s
for key in "${!GOLD_BUILD_SUPPORT[@]}"; do
spec="${GOLD_BUILD_SUPPORT[$key]}"
src="${spec%%|*}"; exp="${spec##*|}"
if [ ! -f "$src" ]; then echo " [FAIL] Build-Support fehlt: $src"; return 1; fi
s="$(sha256sum "$src" | awk '{print $1}')"
if [ "$s" = "$exp" ]; then echo " [OK] $key == $exp"
else echo " [FAIL] $key gold=$exp host=$s"; return 1; fi
done
return 0
}
# dry-run recover-gold-build-source: zeigt den vollstaendigen Plan, ZERO MUTATION.
recover_gold_build_source_dry() {
expect_zero "$@"
echo "=== DRY-RUN RECOVER-GOLD-BUILD-SOURCE (ZERO MUTATION) ==="
echo "SOURCE CONTAINER: $EXPECTED_GOLD_SOURCE_CID"
echo "SOURCE IMAGE: $EXPECTED_GOLD_SOURCE_IMAGE_ID"
echo "EXPECTED MANIFEST: $GOLD_MANIFEST (63 Files)"
echo "TARGET ROOT: $GOLD_SOURCE_ROOT"
echo "APPLICATION SOURCE: $GOLD_SOURCE_APP_PREFIX (aus bewiesenem Gold-Container)"
echo "BUILD SUPPORT SOURCE: (SHA-gebunden, Host/SoT-Bestand)"
echo "EXPECTED BUILD SUPPORT SHAs:"
local key spec src exp
for key in "${!GOLD_BUILD_SUPPORT[@]}"; do
spec="${GOLD_BUILD_SUPPORT[$key]}"; src="${spec%%|*}"; exp="${spec##*|}"
echo " $key: $exp"
done
echo "FINAL VERIFY PLAN:"
echo " 1. verify-deployed-gold-source PASS"
echo " 2. immutable Session-Root erzeugen (FAIL CLOSED wenn existiert)"
echo " 3. Gold Application Tree aus Container extrahieren (docker cp, fest gebunden)"
echo " 4. Build-Support aus SHA-gebundener Source ergaenzen"
echo " 5. .dockerignore kontrolliert ergaenzen (app/app/ + app/eligibility.py)"
echo " 6. Build-Input-Manifest erzeugen"
echo " 7. Application Manifest erneut pruefen (63/63)"
echo " 8. Build-Support SHAs erneut pruefen"
echo " 9. session.meta erzeugen"
echo " 10. final verify + immutable klassifizieren"
echo "DRY-RUN: DESIGN OK (kein echter Recover ausgefuehrt)"
exit 0
}
# recover-gold-build-source (Dispatcher): dry-run zeigt Plan (ZERO MUTATION),
# real fuehrt den kontrollierten Workflow aus (Punkt 3/4/5/8/9).
recover_gold_build_source() {
expect_zero "$@"
if [ "$RUN_MODE" = "DRY_RUN" ]; then
recover_gold_build_source_dry
else
recover_gold_build_source_real
fi
}
# Realer Recover (Punkt 3/4/5/8/9): materialisiert die Gold-Source aus dem
# bewiesenen DEPLOYED GOLD CONTAINER in eine immutable Session. Keine freien
# Parameter. FAIL CLOSED bei jeder Abweichung. ZERO MUTATION ausser der Session.
recover_gold_build_source_real() {
expect_zero "$@"
echo "=== RECOVER-GOLD-BUILD-SOURCE (real) ==="
# Punkt 4: TOCTOU vermeiden — unmittelbar vor Extraktion erneut verifizieren.
echo " [verify] verify-deployed-gold-source (TOCTOU-Guard):"
verify_deployed_gold_source >/dev/null || die "recover: verify-deployed-gold-source FAIL (FAIL CLOSED)"
echo " OK"
# Punkt 8: immutable Session — Root darf nicht existieren (FAIL CLOSED).
if [ -e "$GOLD_SOURCE_ROOT" ]; then
die "recover: Gold-Source-Root existiert bereits $GOLD_SOURCE_ROOT (KEIN Overwrite, FAIL CLOSED)"
fi
local sess
sess="$(date +%Y%m%d_%H%M%S)"
local sdir="$GOLD_SOURCE_ROOT/$sess"
# Punkt 5: Application Tree ausschliesslich aus dem hardcoded Gold-Container.
# Keine Host-Application-Dateien, keine SOT-Session, kein latest, keine freien docker-cp-Inputs.
mkdir -p "$sdir/app" || die "recover: Session-Ordner nicht anlegbar (FAIL CLOSED)"
echo " [extract] docker cp $EXPECTED_GOLD_SOURCE_CID:$GOLD_SOURCE_APP_PREFIX/. -> $sdir/app/"
if ! docker cp "$EXPECTED_GOLD_SOURCE_CID:$GOLD_SOURCE_APP_PREFIX/." "$sdir/app/" >/dev/null 2>&1; then
rm -rf "$GOLD_SOURCE_ROOT"; die "recover: docker cp Application-Tree fehlgeschlagen (FAIL CLOSED)"
fi
# Punkt 6: Build Support separat aus fest SHA-gebundenen Quellen (Host/SoT-Bestand).
echo " [build-support] SHA-gebundene Host-Dateien:"
local key spec src exp s
for key in "${!GOLD_BUILD_SUPPORT[@]}"; do
spec="${GOLD_BUILD_SUPPORT[$key]}"; src="${spec%%|*}"; exp="${spec##*|}"
[ -f "$src" ] || { rm -rf "$GOLD_SOURCE_ROOT"; die "recover: Build-Support fehlt $src (FAIL CLOSED)"; }
s="$(sha256sum "$src" | awk '{print $1}')"
[ "$s" = "$exp" ] || { rm -rf "$GOLD_SOURCE_ROOT"; die "recover: Build-Support SHA mismatch $key (FAIL CLOSED)"; }
cp -a "$src" "$sdir/$(basename "$src")" || { rm -rf "$GOLD_SOURCE_ROOT"; die "recover: Build-Support kopieren fehlgeschlagen (FAIL CLOSED)"; }
echo " $key == $exp"
done
# Punkt 6: .dockerignore kontrolliert erzeugen (exakt die Legacy-Exclusions).
printf 'app/app/\napp/eligibility.py\n' > "$sdir/.dockerignore"
local dis
dis="$(sha256sum "$sdir/.dockerignore" | awk '{print $1}')"
[ "$dis" = "$GOLD_DOCKERIGNORE_SHA" ] || { rm -rf "$GOLD_SOURCE_ROOT"; die "recover: .dockerignore SHA mismatch (FAIL CLOSED)"; }
echo " .dockerignore == $GOLD_DOCKERIGNORE_SHA"
# Punkt 8: Application Manifest (63/63 gegen Gold) + Build-Support Manifest.
( cd "$sdir/app" && find . -type f -name '*.py' ! -path '*/__pycache__/*' -printf '%P\n' 2>/dev/null | sort \
| while IFS= read -r rel; do [ -f "$rel" ] && sha256sum "$rel"; done | awk '{print $1" app/"$2}' ) > "$sdir/application.manifest"
local match=0 diverged=0 missing=0
local -A gsha
while read -r sha p; do gsha["$p"]="$sha"; done < "$GOLD_MANIFEST"
local path g c
for path in "${!gsha[@]}"; do
g="${gsha[$path]}"
c="$(awk -v r="$path" '$2==r{print $1}' "$sdir/application.manifest" 2>/dev/null)"
if [ -z "$c" ]; then missing=$((missing+1));
elif [ "$c" = "$g" ]; then match=$((match+1));
else diverged=$((diverged+1)); fi
done
echo " [application] MATCH=$match DIVERGED=$diverged MISSING=$missing"
[ "$match" = "63" ] && [ "$diverged" = "0" ] && [ "$missing" = "0" ] || { rm -rf "$GOLD_SOURCE_ROOT"; die "recover: Application != Gold (FAIL CLOSED)"; }
# Build-Support Manifest (Punkt 8)
{
for key in "${!GOLD_BUILD_SUPPORT[@]}"; do
spec="${GOLD_BUILD_SUPPORT[$key]}"; src="${spec%%|*}"; exp="${spec##*|}"
echo "$key $exp"
done
echo ".dockerignore $GOLD_DOCKERIGNORE_SHA"
} > "$sdir/build-support.manifest"
# Punkt 13: FINAL_BUILD_INPUT_SHA (Build-Input-Identitaet) berechnen.
local final_sha
final_sha="$( ( cd "$sdir" && find . -type f ! -name 'session.meta' ! -name 'application.manifest' ! -name 'build-support.manifest' ! -name 'final-build-input.manifest' -printf '%P\n' 2>/dev/null | sort | while IFS= read -r f; do [ -f "$f" ] && sha256sum "$f"; done ) | sha256sum | awk '{print $1}' )"
echo "$final_sha" > "$sdir/final-build-input.manifest"
# Punkt 8: session.meta
{
echo "GOLD_SOURCE_SESSION=$sess"
echo "SOURCE_CONTAINER_ID=$EXPECTED_GOLD_SOURCE_CID"
echo "SOURCE_IMAGE_ID=$EXPECTED_GOLD_SOURCE_IMAGE_ID"
echo "CREATED_AT=$(date -u +%Y-%m-%dT%H:%M:%SZ)"
echo "FINAL_BUILD_INPUT_SHA=$final_sha"
} > "$sdir/session.meta"
# Punkt 9: aktive Recovery-Gold-Source-Identitaet kontrolliert speichern.
mkdir -p "$CONTROL_PLANE_STATE_DIR" || { rm -rf "$GOLD_SOURCE_ROOT"; die "recover: State-Dir nicht anlegbar (FAIL CLOSED)"; }
if [ -e "$RECOVERY_SOURCE_STATE" ]; then
rm -rf "$GOLD_SOURCE_ROOT"; die "recover: Recovery-Source-State existiert bereits (KEIN Overwrite, FAIL CLOSED)"
fi
{
echo "GOLD_SOURCE_SESSION=$sess"
echo "FINAL_BUILD_INPUT_SHA=$final_sha"
echo "SOURCE_CONTAINER_ID=$EXPECTED_GOLD_SOURCE_CID"
echo "SOURCE_IMAGE_ID=$EXPECTED_GOLD_SOURCE_IMAGE_ID"
echo "CREATED_AT=$(date -u +%Y-%m-%dT%H:%M:%SZ)"
} > "$RECOVERY_SOURCE_STATE"
chown root:root "$RECOVERY_SOURCE_STATE"; chmod 600 "$RECOVERY_SOURCE_STATE"
# Punkt 8: immutable klassifizieren (root-owned, nicht world-readable).
chown -R root:root "$sdir"; chmod -R 700 "$sdir"
echo "RECOVER-GOLD-BUILD-SOURCE OK: $sdir (immutable, 63/63, Build-Support SHA-gebunden)"
echo " State: $RECOVERY_SOURCE_STATE"
echo " FINAL_BUILD_INPUT_SHA=$final_sha"
}
# verify-recovered-gold-build-source (read-only): prueft eine bestehende Recovery-Source
# gegen die EXAKTE Session-Identitaet aus dem Control-Plane-State (Punkt 10).
# NICHT chronologisch newest, NICHT glob-first, NICHT latest.
verify_recovered_gold_build_source() {
expect_zero "$@"
echo "=== VERIFY-RECOVERED-GOLD-BUILD-SOURCE (read-only) ==="
# Exakte Session-Identitaet aus State (Punkt 9/10). Kein latest/glob-first.
if [ ! -f "$RECOVERY_SOURCE_STATE" ]; then
die "verify-recovered-gold-build-source: Kein Recovery-Source-State ($RECOVERY_SOURCE_STATE). NUR nach realem recover (FAIL CLOSED)"
fi
local sess
sess="$(awk -F'=' '$1=="GOLD_SOURCE_SESSION"{print $2}' "$RECOVERY_SOURCE_STATE" 2>/dev/null)"
[[ "$sess" =~ ^[0-9]{8}_[0-9]{6}$ ]] || die "verify-recovered-gold-build-source: GOLD_SOURCE_SESSION ungueltig/fehlt (FAIL CLOSED)"
local sdir="$GOLD_SOURCE_ROOT/$sess"
[ -d "$sdir" ] || die "verify-recovered-gold-build-source: Session-Ordner fehlt $sdir (FAIL CLOSED)"
# Pflichtdateien der immutable Session (Punkt 8)
local f
for f in app Dockerfile .dockerignore requirements.txt session.meta application.manifest build-support.manifest final-build-input.manifest; do
[ -e "$sdir/$f" ] || die "verify-recovered-gold-build-source: fehlt $sdir/$f (FAIL CLOSED)"
done
# Application Manifest (Punkt 8): 63/63 gegen Gold
echo " [application] $sdir/application.manifest vs Gold (63/63):"
local match=0 diverged=0 missing=0
local -A gsha
while read -r sha p; do gsha["$p"]="$sha"; done < "$GOLD_MANIFEST"
local path g s
for path in "${!gsha[@]}"; do
g="${gsha[$path]}"
s="$(awk -v r="$path" '$2==r{print $1}' "$sdir/application.manifest" 2>/dev/null)"
if [ -z "$s" ]; then missing=$((missing+1));
elif [ "$s" = "$g" ]; then match=$((match+1));
else diverged=$((diverged+1)); fi
done
echo " MATCH=$match DIVERGED=$diverged MISSING=$missing"
[ "$match" = "63" ] && [ "$diverged" = "0" ] && [ "$missing" = "0" ] || die "verify-recovered-gold-build-source: Application != Gold (FAIL CLOSED)"
# Build-Support SHAs (Punkt 6): fest SHA-gebunden
echo " [build-support] SHA-Bindings:"
gold_build_support_ok || die "verify-recovered-gold-build-source: Build-Support SHA mismatch (FAIL CLOSED)"
# .dockerignore fester EXPECTED SHA (Punkt 6)
local dis
dis="$(sha256sum "$sdir/.dockerignore" 2>/dev/null | awk '{print $1}')"
[ "$dis" = "$GOLD_DOCKERIGNORE_SHA" ] || die "verify-recovered-gold-build-source: .dockerignore SHA $dis != erwartet (FAIL CLOSED)"
echo " .dockerignore == $GOLD_DOCKERIGNORE_SHA"
# Build-Input-Identitaet (Punkt 13): FINAL_BUILD_INPUT_SHA aus State pruefen
local want got
want="$(awk -F'=' '$1=="FINAL_BUILD_INPUT_SHA"{print $2}' "$RECOVERY_SOURCE_STATE" 2>/dev/null)"
[ -n "$want" ] && [[ "$want" =~ ^[a-f0-9]{64}$ ]] || die "verify-recovered-gold-build-source: FINAL_BUILD_INPUT_SHA fehlt/ungueltig (FAIL CLOSED)"
got="$( ( cd "$sdir" && find . -type f ! -name 'session.meta' ! -name 'application.manifest' ! -name 'build-support.manifest' ! -name 'final-build-input.manifest' -printf '%P\n' 2>/dev/null | sort | while IFS= read -r f; do [ -f "$f" ] && sha256sum "$f"; done ) | sha256sum | awk '{print $1}' )"
[ "$got" = "$want" ] || die "verify-recovered-gold-build-source: Build-Input SHA mismatch (Source veraendert) (FAIL CLOSED)"
echo " FINAL_BUILD_INPUT_SHA == $want"
echo "VERDICT: RECOVERED GOLD BUILD SOURCE OK (Session $sess, 63/63, Build-Support + .dockerignore + Build-Input SHA)"
return 0
}
SUBCMD="${1:-}"
shift || true
# Mutation Guard: "dry-run <subcmd>" setzt DRY_RUN-Modus, dann normaler Dispatch.
if [ "$SUBCMD" = "dry-run" ]; then
RUN_MODE=DRY_RUN
SUBCMD="${1:-}"
shift || true
case "$SUBCMD" in
build-historical|recreate-historical|snapshot-recovery|build-recovery-gold|pin-current-candidate|recreate-candidate-safe|rollback-to-recovery-gold|promote-runner-patch|rollback-runner-patch|build-runner-remediation|recreate-runner-remediation-safe|archive-recovery-image|archive-remediation-image|pin-recovery-image|pin-remediation-image|recover-gold-build-source) ;;
*) die "dry-run nur mit build-historical|recreate-historical|snapshot-recovery|build-recovery-gold|build-runner-remediation|recreate-runner-remediation-safe|archive-recovery-image|archive-remediation-image|pin-recovery-image|pin-remediation-image zulaessig" ;;
esac
fi
case "$SUBCMD" in
# ---------------- bestehend ----------------
ps) expect_zero "$@"; exec docker ps ;;
inspect) req_ok; expect_zero "$@"; exec docker inspect "$CONTAINER" ;;
logs) req_ok
tail_n=200
while [ $# -gt 0 ]; do
case "$1" in
--tail) [ $# -ge 2 ] || die "--tail braucht Wert"; [[ "$2" =~ ^[0-9]+$ ]] || die "ungueltiges --tail"; tail_n="$2"; shift 2 ;;
*) die "unerlaubtes Argument 'logs $1'" ;;
esac
done
exec docker logs --tail "$tail_n" "$CONTAINER"
;;
run) req_ok
declare -a RUN=()
while [ $# -gt 0 ]; do
case "$1" in
--start)[[ "${2:-}" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}$ ]]||die "ungueltiges --start";RUN+=("--start" "$2");shift 2;;
--end)[[ "${2:-}" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}$ ]]||die "ungueltiges --end";RUN+=("--end" "$2");shift 2;;
--limit-days)[[ "${2:-}" =~ ^[0-9]+$ ]]||die "ungueltiges --limit-days";RUN+=("--limit-days" "$2");shift 2;;
*) die "unerlaubtes Argument 'run $1'" ;;
esac
done
echo "[rq-historical] Start: docker exec $CONTAINER python $SCRIPT ${RUN[*]}" >&2
exec docker exec "$CONTAINER" python "$SCRIPT" "${RUN[@]}"
;;
code) req_ok
[ $# -eq 1 ] || die "code braucht genau EINEN Key (code-list)"
p="$(key_path "$1")"; [ -n "$p" ] || die "unbekannter Key 'code $1'"
exec docker exec "$CONTAINER" cat "$p"
;;
hashes) req_ok; expect_zero "$@"
PATHS=(); for k in "${!TARGET_PATHS[@]}"; do PATHS+=("${TARGET_PATHS[$k]}"); done
exec docker exec "$CONTAINER" sha256sum "${PATHS[@]}"
;;
code-list) req_ok; expect_zero "$@"
echo "Verfuegbare READ-ONLY-Code-Keys (historical-service):"
for k in "${!TARGET_PATHS[@]}"; do printf ' %-11s | staging: %-22s -> %s\n' "$k" "${STAGE_FILES[$k]}" "${TARGET_PATHS[$k]}"; done
;;
stage-dir) expect_zero "$@"
echo "Staging-Root: $STAGING_ROOT (Owner rq-host, keine globalen Schreibrechte)"
for k in runner backfill pipeline repository provenance trust sanity trust_gate; do
printf ' %-11s %s\n' "$k" "$(stage_file "$k")"
done
;;
stage-check) [ $# -eq 1 ] || die "stage-check braucht genau EINEN Key"; key_ok "$1"; validate_stage "$1"
;;
test) req_ok; [ $# -eq 1 ] || die "test braucht genau EINEN Key"; key_ok "$1"
validate_stage "$1"
file="$(stage_file "$1")"
tmp="/tmp/rqtest_${1}_$(date +%s).py"
docker cp "$file" "$CONTAINER:$tmp"
if docker exec "$CONTAINER" python -m py_compile "$tmp" >/dev/null 2>&1; then
echo "PASS: py_compile OK ($file) [isoliert in Container-/tmp, Produktivcode unberuehrt]"
else
echo "FAIL: py_compile fehlgeschlagen ($file)"
docker exec "$CONTAINER" rm -f "$tmp"; exit 2
fi
docker exec "$CONTAINER" rm -f "$tmp"
;;
search-writes) req_ok; expect_zero "$@"
docker exec "$CONTAINER" sh -c 'grep -rnE "insert_raw_bars|insert_derived_bars|BackfillOrchestrator|trust_gate|TrustGate" /app/app --include="*.py" 2>/dev/null | head -100 || true'
;;
healthcheck) req_ok; expect_zero "$@"
rh="$(docker exec "$CONTAINER" python -c 'import urllib.request; print(urllib.request.urlopen("http://localhost:8000/health", timeout=5).status)' 2>/dev/null || echo N/A)"
rr="$(docker exec "$CONTAINER" python -c 'import urllib.request; print(urllib.request.urlopen("http://localhost:8000/health/ready", timeout=5).status)' 2>/dev/null || echo N/A)"
running="$(docker inspect -f '{{.State.Running}}' "$CONTAINER")"
echo "Container.Running=$running /health=$rh /health/ready=$rr"
[ "$running" = "true" ] && [ "$rh" = "200" ] && [ "$rr" = "200" ] || { echo "HEALTHCHECK FAILED"; exit 2; }
echo "HEALTHCHECK OK"
;;
backup) req_ok; [ $# -eq 1 ] || die "backup braucht genau EINEN Key"; key_ok "$1"
p="$(key_path "$1")"; ts="$(date +%Y%m%d_%H%M%S)"; bdir="$BACKUP_ROOT/$1/$ts"
mkdir -p "$bdir"
docker exec "$CONTAINER" cat "$p" > "$bdir/$(basename "$p")"
sha256sum "$bdir/$(basename "$p")" > "$bdir/SHA256"
echo "BACKUP: $1 -> $bdir"; cat "$bdir/SHA256"
;;
deploy) req_ok; [ $# -eq 1 ] || die "deploy braucht genau EINEN Key"; key_ok "$1"
key="$1"; p="$(key_path "$key")"; file="$(stage_file "$key")"
validate_stage "$key"
ts="$(date +%Y%m%d_%H%M%S)"; bdir="$BACKUP_ROOT/$key/$ts"
mkdir -p "$bdir"
docker exec "$CONTAINER" cat "$p" > "$bdir/$(basename "$p")" || die "Backup fehlgeschlagen -> deploy abgebrochen"
sha256sum "$bdir/$(basename "$p")" > "$bdir/SHA256"
echo "BACKUP: $bdir"
tmp="/tmp/rqdeploy_${key}_$(date +%s).py"
docker cp "$file" "$CONTAINER:$tmp"
if ! docker exec "$CONTAINER" python -m py_compile "$tmp" >/dev/null 2>&1; then
docker exec "$CONTAINER" rm -f "$tmp"; die "py_compile FAIL -> deploy abgebrochen (nichts deployt)"
fi
docker exec "$CONTAINER" rm -f "$tmp"
docker cp "$file" "$CONTAINER:$p"
sha_stage="$(sha256sum "$file" | awk '{print $1}')"
sha_deployed="$(docker exec "$CONTAINER" sha256sum "$p" | awk '{print $1}')"
echo "SHA stage : $sha_stage"; echo "SHA deployed: $sha_deployed"
if [ "$sha_stage" != "$sha_deployed" ]; then
echo "HASH-MISMATCH -> rollback"; docker cp "$bdir/$(basename "$p")" "$CONTAINER:$p"; die "Deploy Hash-Mismatch, zurueckgerollt"
fi
rh="$(docker exec "$CONTAINER" python -c 'import urllib.request; print(urllib.request.urlopen("http://localhost:8000/health", timeout=5).status)' 2>/dev/null || echo N/A)"
rr="$(docker exec "$CONTAINER" python -c 'import urllib.request; print(urllib.request.urlopen("http://localhost:8000/health/ready", timeout=5).status)' 2>/dev/null || echo N/A)"
echo "Health: /health=$rh /health/ready=$rr"
if [ "$rh" != "200" ] || [ "$rr" != "200" ]; then
echo "DEPLOY-HEALTHCHECK FAILED -> rollback"; docker cp "$bdir/$(basename "$p")" "$CONTAINER:$p"; die "Rollback ausgefuehrt"
fi
echo "DEPLOY OK: $key -> $p (Backup: $bdir). KEIN backfill/run gestartet."
;;
rollback) req_ok; [ $# -eq 1 ] || die "rollback braucht genau EINEN Key"; key_ok "$1"
p="$(key_path "$1")"
bdir="$(ls -1t "$BACKUP_ROOT/$1" 2>/dev/null | head -1)"
[ -n "$bdir" ] || die "kein Backup fuer '$1' vorhanden"
bdir="$BACKUP_ROOT/$1/$bdir"; src="$bdir/$(basename "$p")"
[ -f "$src" ] || die "Backup-Datei fehlt: $src"
sha_bak="$(sha256sum "$src" | awk '{print $1}')"
sha_cur="$(docker exec "$CONTAINER" sha256sum "$p" | awk '{print $1}')"
echo "aktuell: $sha_cur / backup: $sha_bak"
[ "$sha_bak" != "$sha_cur" ] || die "kein Unterschied (bereits auf Backup-Stand)"
docker cp "$src" "$CONTAINER:$p"
sha_cur="$(docker exec "$CONTAINER" sha256sum "$p" | awk '{print $1}')"
[ "$sha_cur" = "$sha_bak" ] || die "ROLLBACK-HASH-MISMATCH"
echo "ROLLBACK OK: $key -> $p auf $bdir"
;;
# ============ SOT-DISCOVERY (read-only, NEU) ============
build-def) expect_zero "$@"
echo "=== Dockerfile ($HOST_DIR/Dockerfile) — redacted ==="
[ -f "$HOST_DIR/Dockerfile" ] || die "Dockerfile nicht gefunden"
while IFS= read -r line; do redact_line "$line"; done < "$HOST_DIR/Dockerfile"
echo ""
echo "=== docker-compose.yml — strukturielle Abschnitte, redacted ==="
cf=""
if [ -f "$HOST_DIR/docker-compose.yml" ]; then cf="$HOST_DIR/docker-compose.yml";
elif [ -f "$HOST_DIR/compose.yml" ]; then cf="$HOST_DIR/compose.yml";
elif [ -f "$HOST_DIR/docker-compose.yaml" ]; then cf="$HOST_DIR/docker-compose.yaml"; fi
[ -n "$cf" ] || die "keine compose-Datei gefunden"
echo "(Quelle: $cf)"
while IFS= read -r line; do
# nur relevante strukturelle Schluessel anzeigen; Environment/Secrets nicht roh
case "$line" in
*POSTGRES_PASSWORD*|*PG_PASSWORD*|*PASSWORD*|*SECRET*|*TOKEN*) echo " **** REDACTED (Secret-Zeile) ****" ;;
*) echo " $line" ;;
esac
done < "$cf"
echo ""
echo "=== .dockerignore (falls vorhanden) ==="
[ -f "$HOST_DIR/.dockerignore" ] && cat "$HOST_DIR/.dockerignore" || echo "(kein .dockerignore)"
echo ""
echo "=== Service/Container-Fixierung ==="
echo " compose.project=$(docker inspect "$CONTAINER" --format '{{index .Config.Labels "com.docker.compose.project"}}' 2>/dev/null || echo N/A)"
echo " compose.service=$(docker inspect "$CONTAINER" --format '{{index .Config.Labels "com.docker.compose.service"}}' 2>/dev/null || echo N/A)"
echo " container=$CONTAINER"
;;
build-files) expect_zero "$@"
echo "=== Build-Dateien unter $HOST_DIR (nicht-verzeichnis, nicht .env/.dump/.sql) ==="
for f in Dockerfile docker-compose.yml compose.yml docker-compose.yaml .dockerignore requirements.txt; do
if [ -e "$HOST_DIR/$f" ]; then printf ' %-22s %s\n' "$f" "EXISTS ($(stat -c '%s' "$HOST_DIR/$f") bytes)"; else printf ' %-22s %s\n' "$f" "(fehlt)"; fi
done
;;
host-manifest) expect_zero "$@"
echo "=== HOST Manifest /opt/historical-v2/app (*.py, keine Symlinks folgen) ==="
[ -d "$HOST_APP" ] || die "Host-App-Verzeichnis fehlt: $HOST_APP"
# finde reguläre *.py UND Symlinks auf *.py separat
(
find "$HOST_APP" -type f -name "*.py" -printf '%P %s\n' 2>/dev/null | sort \
| while read -r rel size; do
sha=$(sha256sum "$HOST_APP/$rel" 2>/dev/null | awk '{print $1}')
printf 'FILE\t%s\t%s\t%s\n' "$rel" "$size" "${sha:-ERR}"
done
find "$HOST_APP" -type l -name "*.py" -printf '%P\n' 2>/dev/null | sort \
| while read -r rel; do printf 'SYMLINK\t%s\t0\t<not followed>\n' "$rel"; done
)
;;
host-code) [ $# -eq 1 ] || die "host-code braucht genau EINEN relativen Pfad (unter app/)"
rel="$1"
[[ "$rel" == *".."* ]] && die "Traversal nicht erlaubt"
[[ "$rel" == /* ]] && die "relativer Pfad erwartet, nicht absolut"
# realpath zwingend im app-Root
real="$(realpath -m "$HOST_APP/$rel")"
[[ "$real" == "$HOST_APP/"* ]] || die "Pfad ausserhalb app-Root blockiert"
[ -f "$real" ] || die "Datei nicht gefunden oder kein File: $rel"
[ -L "$real" ] && die "Symlink-Read nicht erlaubt"
# nur .py ausgeben
[[ "$rel" == *.py ]] || die "Nur .py-Quelldateien lesbar"
echo "--- $rel ($(stat -c '%s' "$real") bytes) ---"
exec cat "$real"
;;
container-manifest) req_ok; expect_zero "$@"
echo "=== CONTAINER Manifest historical-service:/app/app (*.py) ==="
docker exec "$CONTAINER" sh -c 'cd /app/app && find . -name "*.py" -type f 2>/dev/null | sed "s#^\./##" | sort | while read f; do size=$(stat -c %s "$f"); sha=$(sha256sum "$f" | cut -d" " -f1); printf "FILE\t%s\t%s\t%s\n" "$f" "$size" "$sha"; done'
docker exec "$CONTAINER" sh -c 'cd /app/app && find . -name "*.py" -type l 2>/dev/null | sed "s#^\./##" | sort | while read f; do printf "SYMLINK\t%s\t0\t<not followed>\n" "$f"; done' || true
;;
container-code) req_ok; [ $# -eq 1 ] || die "container-code braucht genau EINEN relativen Pfad (unter /app/app/)"
rel="$1"
[[ "$rel" == *".."* ]] && die "Traversal nicht erlaubt"
[[ "$rel" == /* ]] && die "relativer Pfad erwartet, nicht absolut"
target="/app/app/$rel"
# sicherstellen Ziel existiert als reguläre Datei & nicht symlink im Container
if ! docker exec "$CONTAINER" sh -c "[ -f '$target' ]" 2>/dev/null; then die "Datei nicht gefunden oder kein File im Container: $rel"; fi
if docker exec "$CONTAINER" sh -c "[ -L '$target' ]" 2>/dev/null; then die "Symlink-Read im Container nicht erlaubt"; fi
[[ "$rel" == *.py ]] || die "Nur .py-Quelldateien lesbar"
echo "--- $target ---"
exec docker exec "$CONTAINER" cat "$target"
;;
tree-diff) req_ok; expect_zero "$@"
echo "=== TREE-DIFF Host(/opt/historical-v2/app) <-> Container(/app/app) ==="
hmap="$(mktemp)"; cmap="$(mktemp)"
# Host: 'rel<TAB>sha'
find "$HOST_APP" -type f -name "*.py" -printf '%P\n' 2>/dev/null | sort | while IFS= read -r rel; do
sha="$(sha256sum "$HOST_APP/$rel" 2>/dev/null | awk '{print $1}')"
printf '%s\t%s\n' "$rel" "$sha"
done > "$hmap"
# Container: 'rel<TAB>sha'
docker exec "$CONTAINER" sh -c 'cd /app/app && find . -type f -name "*.py" -printf "%P\n" 2>/dev/null | sort | while IFS= read -r rel; do sha256sum "$rel" 2>/dev/null | awk -v r="$rel" '\''{print r"\t"$1}'\''; done' > "$cmap"
tmpunion="$(mktemp)"
{ cut -f1 "$hmap"; cut -f1 "$cmap"; } | sort -u > "$tmpunion"
echo "HOST_APP=$HOST_APP CONTAINER_APP=/app/app"
while IFS= read -r rel; do
hs="$(awk -F'\t' -v r="$rel" '$1==r{print $2}' "$hmap")"
cs="$(awk -F'\t' -v r="$rel" '$1==r{print $2}' "$cmap")"
if [ -n "$hs" ] && [ -n "$cs" ]; then
if [ "$hs" = "$cs" ]; then status="MATCH"; else status="DIVERGED"; fi
printf '%-18s %s [h:%s c:%s]\n' "$status" "$rel" "${hs:0:12}" "${cs:0:12}"
elif [ -n "$hs" ]; then
printf '%-18s %s\n' "HOST_ONLY" "$rel"
else
printf '%-18s %s\n' "MISSING_ON_HOST" "$rel"
fi
done < "$tmpunion"
rm -f "$hmap" "$cmap" "$tmpunion"
;;
# ============ HOST-SOT-RECOVERY (Phase 13.5, Extension 1) ============
# KEIN Build/Recreate. Nur: backup-sot -> recover-sot -> verify-sot-recovery
# -> restore-sot. Fail-closed, Gold-SHA-gebunden, keine freien Pfade.
backup-sot) expect_zero "$@"
# Vollstaendiges Host-SoT-Backup in root-owned Recovery-Bereich
ts="$(date +%Y%m%d_%H%M%S)"
bdir="$SOT_BACKUP_ROOT/$ts"
mkdir -p "$bdir"
# tar mit Symlinks/Permissions/Ownership; kein Follow (kein -h)
if ! tar -cpf "$bdir/historical-v2.tar" -C /opt historical-v2 2>/dev/null; then
rm -rf "$bdir"; die "backup-sot: tar fehlgeschlagen (FAIL CLOSED)"
fi
# SHA256-Manifest aller regulären Dateien (nicht Symlinks folgen)
( cd /opt/historical-v2 && find . -type f -print0 2>/dev/null | sort -z \
| while IFS= read -r -d "" f; do
sha="$(sha256sum "$f" 2>/dev/null | awk '{print $1}')"
printf '%s %s\n' "$sha" "${f#./}"
done ) > "$bdir/SHA256.manifest"
# Struktur-/Pflichtpruefung (Punkt 4)
ok=1
[ -f "$bdir/historical-v2.tar" ] || { echo "FEHLT: tar"; ok=0; }
[ -s "$bdir/SHA256.manifest" ] || { echo "FEHLT: manifest"; ok=0; }
[ -f /opt/historical-v2/Dockerfile ] || { echo "FEHLT: Dockerfile"; ok=0; }
[ -f /opt/historical-v2/docker-compose.yml ] || { echo "FEHLT: compose"; ok=0; }
[ -d /opt/historical-v2/app ] || { echo "FEHLT: app/"; ok=0; }
if [ "$ok" != "1" ]; then
rm -rf "$bdir"; die "backup-sot: Validierung fehlgeschlagen (FAIL CLOSED)"
fi
# Session-Marker: eindeutige Markierung der aktuellen Recovery-Session
echo "$ts" > "$bdir/$SOT_SESSION_MARKER"
chown -R root:root "$bdir"; chmod 700 "$bdir"
echo "BACKUP-SOT OK: $bdir"
echo " tar: $(stat -c '%s' "$bdir/historical-v2.tar") bytes"
echo " manifest: $(wc -l < "$bdir/SHA256.manifest") Dateien"
echo " session: $ts"
;;
sot-session) expect_zero "$@"
# Zeigt den neuesten VALIDIERTEN FULL SOT SNAPSHOT (Partial-Backups wie
# pre-runner-remediation werden NICHT als Full-Snapshot gezaehlt; Blocker A).
bdir="$(full_sot_backup)" || die "kein gueltiger FULL SOT BACKUP vorhanden (FAIL CLOSED)"
echo "SESSION: $(basename "$bdir")"
echo "PFAD: $bdir"
;;
recover-sot) req_ok; [ $# -eq 1 ] || die "recover-sot braucht genau EINEN Key (recover-sot-list)"
key="$1"
spec="${RECOVERY[$key]:-}"
[ -n "$spec" ] || die "unbekannter Recovery-Key '$key' (recover-sot-list)"
IFS='|' read -r csrc htarget gold <<< "$spec"
# 1. gueltiger vollstaendiger (FULL) backup-sot vorhanden? (Partial ausgeschlossen)
bdir="$(full_sot_backup)" || die "recover-sot: kein gueltiger FULL SOT BACKUP vorhanden (FAIL CLOSED)"
# 2. Source = historical-service fixed (req_ok bereits geprueft)
# 3/4. Source regular file, kein Symlink
if ! docker exec "$CONTAINER" sh -c "[ -f '$csrc' ]" 2>/dev/null; then die "recover-sot: Source kein File im Container: $csrc"; fi
if docker exec "$CONTAINER" sh -c "[ -L '$csrc' ]" 2>/dev/null; then die "recover-sot: Source ist Symlink (FAIL CLOSED): $csrc"; fi
# 5. Source SHA == EXPECTED GOLD SHA
csha="$(docker exec "$CONTAINER" sha256sum "$csrc" 2>/dev/null | awk '{print $1}')" || csha=""
if [ "$csha" != "$gold" ]; then
die "recover-sot: Source-SHA-Mismatch fuer '$key' (erwartet $gold, Container $csha) — FAIL CLOSED"
fi
# 6. Host target exakt fester Pfad (aus Allowlist, kein freier Pfad)
[[ "$htarget" == /opt/historical-v2/app/* ]] || die "recover-sot: Host-Ziel ausserhalb app/ (FAIL CLOSED)"
# 7. Zielverzeichnis kontrolliert erzeugen (z.B. trust/)
tdir="$(dirname "$htarget")"
mkdir -p "$tdir"
# 8. Ziel vor Ueberschreiben einzeln sichern, falls vorhanden
if [ -e "$htarget" ]; then
prebak="$bdir/pre-recover/$(basename "$htarget")"
mkdir -p "$bdir/pre-recover"
cp -a "$htarget" "$prebak" 2>/dev/null || true
echo " pre-recover gesichert: $prebak"
fi
# 9. byte-exakt schreiben (docker cp = byte-identisch)
docker cp "$CONTAINER:$csrc" "$htarget"
# 10. Ownership/Permissions aus bestehendem Host-Kontext ableiten
# (app/ ist UID 10010; trust/ neu -> gleiche Basis wie app/)
chown 10010:10010 "$htarget" 2>/dev/null || chown root:root "$htarget"
chmod 644 "$htarget"
# 11. Host SHA == Gold SHA
hsha="$(sha256sum "$htarget" | awk '{print $1}')"
if [ "$hsha" != "$gold" ]; then
die "recover-sot: Host-SHA-Mismatch fuer '$key' (erwartet $gold, Host $hsha) — FAIL CLOSED"
fi
echo "RECOVER-SOT OK: $key"
echo " source : $CONTAINER:$csrc"
echo " target : $htarget"
echo " gold : $gold"
echo " host : $hsha"
;;
recover-sot-list) expect_zero "$@"
echo "Recovery-Allowlist (CONFIRMED GOLD, fest gebunden):"
for k in "${!RECOVERY[@]}"; do
IFS='|' read -r csrc htarget gold <<< "${RECOVERY[$k]}"
printf ' %-16s %-45s -> %s\n' "$k" "$csrc" "$htarget"
done
;;
verify-sot-recovery) req_ok; expect_zero "$@"
echo "=== VERIFY-SOT-RECOVERY (Gold vs Host vs Container) ==="
allok=1
for k in "${!RECOVERY[@]}"; do
IFS='|' read -r csrc htarget gold <<< "${RECOVERY[$k]}"
hsha="$(sha256sum "$htarget" 2>/dev/null | awk '{print $1}')" || hsha=""
csha="$(docker exec "$CONTAINER" sha256sum "$csrc" 2>/dev/null | awk '{print $1}')" || csha=""
if [ "$hsha" = "$gold" ] && [ "$csha" = "$gold" ]; then
printf ' MATCH %-16s %s\n' "$k" "$htarget"
else
printf ' FAIL %-16s host=%s container=%s gold=%s\n' "$k" "${hsha:-MISSING}" "${csha:-MISSING}" "$gold"
allok=0
fi
done
[ "$allok" = "1" ] && echo "VERIFY-SOT-RECOVERY: ALLE MATCH" || { echo "VERIFY-SOT-RECOVERY: FEHLER"; exit 2; }
;;
restore-sot) expect_zero "$@"
# Restore aus dem neuesten validerungspflichtigen Session-Backup (FULL)
bdir="$(full_sot_backup)" || die "restore-sot: kein gueltiger FULL SOT BACKUP vorhanden (FAIL CLOSED)"
[ -f "$bdir/$SOT_SESSION_MARKER" ] || die "restore-sot: kein markiertes Session-Backup (FAIL CLOSED)"
[ -f "$bdir/historical-v2.tar" ] || die "restore-sot: tar fehlt (FAIL CLOSED)"
echo "RESTORE-SOT: $bdir"
echo " (entpackt /opt/historical-v2 aus Session-Backup, byte-exakt)"
if ! tar -xpf "$bdir/historical-v2.tar" -C /opt 2>/dev/null; then
die "restore-sot: tar-Restore fehlgeschlagen (FAIL CLOSED)"
fi
echo "RESTORE-SOT OK. KEIN Build/Recreate danach."
;;
snapshot-recovery) snapshot_recovery "$@" ;;
verify-recovery-snapshot) verify_recovery_snapshot "$@" ;;
build-recovery-gold) build_recovery_gold "$@" ;;
verify-recovery-gold-image) verify_recovery_gold_image "$@" ;;
pin-current-candidate) pin_current_candidate "$@" ;;
verify-candidate-image) verify_candidate_image "$@" ;;
recreate-candidate-safe) recreate_candidate_safe "$@" ;;
rollback-to-recovery-gold) rollback_to_recovery_gold "$@" ;;
promote-runner-patch) promote_runner_patch "$@" ;;
rollback-runner-patch) rollback_runner_patch "$@" ;;
build-runner-remediation) build_runner_remediation "$@" ;;
verify-runner-remediation-image) verify_runner_remediation_image "$@" ;;
recreate-runner-remediation-safe) recreate_runner_remediation_safe "$@" ;;
verify-remediation-build-input) expect_zero "$@"; verify_remediation_build_input ;;
build-historical) build_historical "$@" ;;
create-build-exclusion) expect_zero "$@"
# Build-Pollution-Exclusion implementieren (Punkt 5-8): .dockerignore anlegen/ersetzen
# PREVIOUS_STATE war ABSENT. NUR exakt die 3 Legacy-Einträge.
if [ -f "$BUILD_EXCLUSION_FILE" ]; then
die "create-build-exclusion: .dockerignore existiert bereits (kein Ueberschreiben ohne Backup — manuell)"
fi
printf 'app/app/\napp/eligibility.py\n' > "$BUILD_EXCLUSION_FILE"
echo "BUILD-EXCLUSION OK: $BUILD_EXCLUSION_FILE"
exclusion_valid && { echo " validiert: nur app/app/ + app/eligibility.py"; echo " (PREVIOUS_STATE=ABSENT; keine Gold-Datei betroffen)"; }
;;
show-build-exclusion) expect_zero "$@"
if [ ! -f "$BUILD_EXCLUSION_FILE" ]; then echo "PREVIOUS_STATE=ABSENT"; exit 0; fi
echo "=== .dockerignore Inhalt ==="
cat "$BUILD_EXCLUSION_FILE"
echo "=== build-exclusion: Gold-Dateien betroffen? (sollte: 0) ==="
ok=0
while IFS= read -r f; do
# f relativ zu /opt/historical-v2; pruefe ob im Gold-Manifest (app/...)
rel="app/${f#app/}"
if grep -q " $rel$" "$GOLD_MANIFEST" 2>/dev/null; then echo " GOLD-DATEI BETROFFEN: $rel"; ok=1; fi
done < "$BUILD_EXCLUSION_FILE"
[ "$ok" = "0" ] && echo " 0 Gold-Dateien ausgeschlossen (OK)"
;;
recreate-historical) recreate_historical "$@" ;;
verify-gold-reproduction) verify_gold_reproduction "$@" ;;
# ============ IMAGE-RETENTION / ARCHIVE CONTROL PLANE (Phase 13.5) ============
# Schicht B (Archiv, docker save; fail-closed, kein Overwrite) & Schicht A (Pin).
# Alle verify-* sind read-only. Realer Restore ist in dieser Mission BLOCKIERT
# (nur dry-run). Kein Coolify-Cleanup-Disable, kein coolify.managed-Spoof.
archive-recovery-image) archive_image recovery "$@" ;;
archive-remediation-image) archive_image remediation "$@" ;;
pin-recovery-image) pin_image recovery "$@" ;;
pin-remediation-image) pin_image remediation "$@" ;;
verify-recovery-archive) verify_archive_cmd recovery "$@" ;;
verify-remediation-archive) verify_archive_cmd remediation "$@" ;;
verify-recovery-pin) verify_pin_cmd recovery "$@" ;;
verify-remediation-pin) verify_pin_cmd remediation "$@" ;;
verify-recovery-ready) verify_ready_cmd recovery "$@" ;;
verify-remediation-ready) verify_ready_cmd remediation "$@" ;;
restore-recovery-image-archive) restore_recovery_real "$@" ;;
dry-restore-recovery-image-archive) restore_recovery_dry "$@" ;;
# ============ DEPLOYED-GOLD-SOURCE-RECOVERY CONTROL PLANE (Phase 13.5) ============
# Gold-Source fest an Container+Image+63-File-Manifest gebunden. KEIN latest,
# KEIN full_sot_backup(), KEINE freien Inputs. Nur Source-Recovery, KEIN Docker-Build.
# verify-* read-only. Realer recover in dieser Mission BLOCKIERT (nur dry-run).
verify-deployed-gold-source) verify_deployed_gold_source "$@";;
recover-gold-build-source) recover_gold_build_source "$@";;
verify-recovered-gold-build-source) verify_recovered_gold_build_source "$@";;
wrapper-sha) expect_zero "$@"; sha256sum /usr/local/bin/rq-historical ;;
*) die "Aufruf: ps|inspect|logs|run|code <K>|hashes|code-list|stage-dir|stage-check <K>|test <K>|search-writes|healthcheck|backup <K>|deploy <K>|rollback <K>|build-def|build-files|host-manifest|host-code <p>|container-manifest|container-code <p>|tree-diff|backup-sot|sot-session|recover-sot <K>|recover-sot-list|verify-sot-recovery|restore-sot|snapshot-recovery|dry-run snapshot-recovery|verify-recovery-snapshot|build-recovery-gold|dry-run build-recovery-gold|verify-recovery-gold-image|pin-current-candidate|dry-run pin-current-candidate|verify-candidate-image|recreate-candidate-safe|dry-run recreate-candidate-safe|rollback-to-recovery-gold|dry-run rollback-to-recovery-gold|promote-runner-patch|dry-run promote-runner-patch|rollback-runner-patch|dry-run rollback-runner-patch|build-runner-remediation|dry-run build-runner-remediation|verify-runner-remediation-image|recreate-runner-remediation-safe|dry-run recreate-runner-remediation-safe|verify-remediation-build-input|build-historical|dry-run build-historical|recreate-historical|dry-run recreate-historical|verify-gold-reproduction|create-build-exclusion|show-build-exclusion|archive-recovery-image|archive-remediation-image|pin-recovery-image|pin-remediation-image|verify-recovery-archive|verify-remediation-archive|verify-recovery-pin|verify-remediation-pin|verify-recovery-ready|verify-remediation-ready|dry-restore-recovery-image-archive|restore-recovery-image-archive|verify-deployed-gold-source|dry-run recover-gold-build-source|recover-gold-build-source|verify-recovered-gold-build-source"
;;
esac
Reference in a new issue View git blame Copy permalink