3763 lines
214 KiB
Bash
Executable file
3763 lines
214 KiB
Bash
Executable file
#!/bin/bash
|
||
# ============================================================================
|
||
# rq-historical — eng begrenzter root-Wrapper fuer Red Queen (rq-host)
|
||
# RED-QUEEN-VPS-HOST-ACCESS + READONLY-CODE + DEV/TEST/DEPLOY + SOT-DISCOVERY
|
||
# | Rain Ocampo (Hermes) | 2026-08-29
|
||
#
|
||
# LEAST-PRIVILEGE-Design:
|
||
# - rq-host hat KEINE docker-Gruppe, KEINEN Docker-Socket, KEIN sudo ALL,
|
||
# KEINE Root-Shell. Einziger Root-Kanal: NOPASSWD:/usr/local/bin/rq-historical
|
||
# - Dieser Wrapper ist der EINZIGE Kanal zu historical-service & Host-SoT.
|
||
# Keine freien Pfade, kein arbitrary cat/find/grep, kein Shell.
|
||
#
|
||
# SoT-DISCOVERY (read-only, NO copy/rebuild/deploy/fix):
|
||
# build-def -> zeigt Dockerfile + Compose-Builddefinition (redacted)
|
||
# build-files -> Dockerfile + compose + .dockerignore Auflistung
|
||
# host-manifest -> SHA256+size aller *.py unter /opt/historical-v2/app
|
||
# host-code <rel> -> gezielter Code-Read NUR aus host-manifest (kontrolliert)
|
||
# container-manifest -> SHA256+size aller *.py unter historical-service:/app/app
|
||
# container-code <rel> -> gezielter Code-Read NUR aus container-manifest
|
||
# tree-diff -> Host vs Container Manifest-Diff
|
||
#
|
||
# SUBKOMMANDOS (bestehend, unveraendert):
|
||
# ps | inspect | logs [--tail N] | run [..] | code <KEY> | hashes | code-list
|
||
# stage-dir | stage-check <KEY> | test <KEY> | search-writes | healthcheck
|
||
# backup <KEY> | deploy <KEY> | rollback <KEY>
|
||
# ============================================================================
|
||
set -euo pipefail
|
||
|
||
CONTAINER=historical-service
|
||
HOST_APP=/opt/historical-v2/app
|
||
HOST_DIR=/opt/historical-v2
|
||
|
||
# --------------------------------------------------------- Allowlist (fest)
|
||
declare -A TARGET_PATHS=(
|
||
[runner]="/app/app/run_backfill_year.py"
|
||
[backfill]="/app/app/backfill.py"
|
||
[pipeline]="/app/app/pipeline_v2.py"
|
||
[repository]="/app/app/persistence/repository.py"
|
||
[provenance]="/app/app/trust/provenance.py"
|
||
[trust]="/app/app/trust/trust.py"
|
||
[sanity]="/app/app/trust/sanity.py"
|
||
[trust_gate]="/app/app/trust/trust_gate.py"
|
||
)
|
||
declare -A STAGE_FILES=(
|
||
[runner]="run_backfill_year.py" [backfill]="backfill.py" [pipeline]="pipeline_v2.py"
|
||
[repository]="repository.py" [provenance]="provenance.py" [trust]="trust.py"
|
||
[sanity]="sanity.py" [trust_gate]="trust_gate.py"
|
||
)
|
||
STAGING_ROOT=/opt/rq-historical-staging
|
||
BACKUP_ROOT=/opt/rq-historical-backups
|
||
SCRIPT=/app/app/run_backfill_year.py
|
||
|
||
# --------------------------------------------------------- HOST-SOT-RECOVERY
|
||
# CONFIRMED GOLD CONTAINER -> ROOT-CONTROLLED BACKUP -> EXACT BYTE COPY
|
||
# -> HOST SOURCE OF TRUTH -> SHA VERIFY -> STOPP. (Phase 13.5, Extension 1)
|
||
# KEIN Build/Recreate in dieser Mission.
|
||
SOT_BACKUP_ROOT=/opt/rq-historical-sot-backups
|
||
SOT_SESSION_MARKER=".rq-sot-session" # eindeutige Markierung der Recovery-Session
|
||
|
||
# Recovery-Allowlist: KEY -> "CONTAINER_SRC|HOST_TARGET|EXPECTED_GOLD_SHA"
|
||
# Containerpfade aus container-manifest verifiziert (NICHT geraten).
|
||
# HINWEIS: Aufgabe nennt trust/init.py, Container hat aber trust/__init__.py
|
||
# (Gold-SHA 059db627...). Punkt 5: exakte Pfade aus Manifest, nicht raten.
|
||
declare -A RECOVERY=(
|
||
[backfill]="/app/app/backfill.py|/opt/historical-v2/app/backfill.py|1627682ae582bb13343ae8ea10491dd9dffaef6fa4f7594d85378c2a500e81c7"
|
||
[repository]="/app/app/persistence/repository.py|/opt/historical-v2/app/persistence/repository.py|8fff892f52a612f464b5be21c66ef7787a47f2d7d049f815ef87837553c2a228"
|
||
[pipeline]="/app/app/pipeline_v2.py|/opt/historical-v2/app/pipeline_v2.py|7c08ec3d1e75854cf341b9cbfd12afc075c4a173518dd3d951be4f95c5175fe6"
|
||
[trust_init]="/app/app/trust/__init__.py|/opt/historical-v2/app/trust/__init__.py|059db62741f2001f548d1879ccd3bf73bb2f4c8d0f492d151a469c9e4dcd3934"
|
||
[provenance]="/app/app/trust/provenance.py|/opt/historical-v2/app/trust/provenance.py|4dd2ab977b69e9a47bb744dcc742a94d506e13035157a3977cdc7c8924d5f16f"
|
||
[trust]="/app/app/trust/trust.py|/opt/historical-v2/app/trust/trust.py|7c2a948d2f1d668201e5af2a28164bcb2451b31d514a9f5f22b435c9ff2a0d01"
|
||
[sanity]="/app/app/trust/sanity.py|/opt/historical-v2/app/trust/sanity.py|875b07869e5148983e7a06bc05f6a6bec30727d1b5e95242d459564c18f6d6e9"
|
||
[trust_gate]="/app/app/trust/trust_gate.py|/opt/historical-v2/app/trust/trust_gate.py|df0a97a1fc27985fe49b7458d94f7922bc5eaf2250807043c48dbc173f215ff1"
|
||
[diag_bi5]="/app/app/diag_bi5.py|/opt/historical-v2/app/diag_bi5.py|3e07ec5c4d9f572a118c278d0dc5c86865a8fe2067b12f85a3fa958ea4013efa"
|
||
[smoke_readonly]="/app/app/smoke_readonly.py|/opt/historical-v2/app/smoke_readonly.py|85ca939c11d789684740833f0bddb6e4f2a548a7f94870662191b75aefdcbfd6"
|
||
[smoke_trust_gate]="/app/app/smoke_trust_gate.py|/opt/historical-v2/app/smoke_trust_gate.py|f015fe650d6934b0c71d70bc3011f16b7f4bb0d2ffd947b9127040067442cfff"
|
||
[test_trust_gate]="/app/app/test_trust_gate.py|/opt/historical-v2/app/test_trust_gate.py|7867e2b526491de966e64486e8d49ec2cad204d97efc25844562b867ef861114"
|
||
[test_phase13_3]="/app/app/test_phase13_3_reproduction.py|/opt/historical-v2/app/test_phase13_3_reproduction.py|de7d2f7be785b7525a045b91ca055dca305fd7d575b291528141f000d5d003e7"
|
||
)
|
||
# Host-Only-Legacy (Punkt 9): NICHT anfassen
|
||
HOST_ONLY_LEGACY=(
|
||
"/opt/historical-v2/app/app/backfill.py"
|
||
"/opt/historical-v2/app/app/test_idempotency.py"
|
||
"/opt/historical-v2/app/eligibility.py"
|
||
)
|
||
|
||
# --------------------------------------------------------- BUILD/RECREATE-KANAL
|
||
# Phase 13.5 · Controlled No-op Build/Recreate Channel.
|
||
# Build-Pollution-Exclusion via .dockerignore (präzise, NUR Legacy-Artefakte).
|
||
# KEIN Businesscode-Fix. KEIN Backfill-Runner in diesem Kanal.
|
||
COMPOSE_PROJECT=/opt/historical-v2
|
||
COMPOSE_FILE=/opt/historical-v2/docker-compose.yml
|
||
HISTORICAL_SERVICE=historical-service
|
||
GOLD_IMAGE_ID="sha256:f1a6f33e8714c5cdf017632aa30e71241a8ed670a5c95dac2e8226064b990c81"
|
||
GOLD_CONTAINER_ID="634cc505c7bc4c361305e5b415f107e98411faf38e8a29269dd8b1e4d1363b32"
|
||
GOLD_MANIFEST="/opt/rq-historical-gold-reference/gold_app_manifest.sha"
|
||
BUILD_EXCLUSION_FILE=/opt/historical-v2/.dockerignore
|
||
# Präzise Pollution-Exclusion-Einträge (NUR diese 3; keine Gold-Datei betroffen):
|
||
# app/app/ -> /app/app/app/ (verschachtelt, im Gold NICHT vorhanden)
|
||
# app/eligibility.py -> /app/app/eligibility.py (top-level, Gold: quality/eligibility.py UNBERÜHRT)
|
||
BUILD_EXCLUSION_LINES=( "app/app/" "app/eligibility.py" )
|
||
# Legacy-Artefakte, die im neuen Container NICHT als EXTRA erscheinen dürfen:
|
||
declare -A FORBIDDEN_EXTRA=(
|
||
["app/app/app/backfill.py"]=1
|
||
["app/app/app/test_idempotency.py"]=1
|
||
["app/app/eligibility.py"]=1
|
||
)
|
||
|
||
# ------------------------------------------------- RECOVERY-GOLD-SNAPSHOT-CHANNEL
|
||
# Phase 13.5 · Privileged Control-Plane Extension - Recovery Gold Snapshot Only.
|
||
# Erfassung eines byte-/state-nahen Recovery-Images aus dem verifizierten
|
||
# GOLD-Container 634cc505... (Image f1a6f33e...). KEIN Businesscode, KEIN Build,
|
||
# KEIN Recreate, KEIN Runner, KEINE DB-/Datenmutation, KEIN latest-Touch.
|
||
RECOVERY_GOLD_CID="634cc505c7bc4c361305e5b415f107e98411faf38e8a29269dd8b1e4d1363b32"
|
||
RECOVERY_GOLD_TAG="historical-v2-historical-service:recovery-gold-container-20260829"
|
||
|
||
# ------------------------------------------------- RECOVERY-GOLD-REBUILD-KANAL
|
||
# Phase 13.5 · Build Control Plane Hardening - Recovery-Gold Rebuild Channel.
|
||
# Erzeugt aus dem bestaetigten Host-SoT (/opt/historical-v2) ein NEUES, direkt
|
||
# dediziert getaggtes Recovery-Gold-Image. KEIN latest, KEIN Recreate, KEIN
|
||
# Runner, KEINE User-Args/Build-Args. Fester Context + Dockerfile + Tag.
|
||
RECOVERY_REBUILD_TAG="historical-v2-historical-service:recovery-gold-rebuilt-20260829"
|
||
# Build Context wird NICHT mehr statisch auf /opt/historical-v2 gesetzt. Er wird
|
||
# zur Laufzeit aus dem kontrollierten Control-Plane-State (RECOVERY_SOURCE_STATE)
|
||
# gelesen: GOLD_SOURCE_SESSION -> /opt/rq-historical-gold-sources/<SESSION>.
|
||
# Keine User-Argumente. FAIL CLOSED wenn State fehlt.
|
||
RECOVERY_BUILD_CONTEXT="" # zur Laufzeit aus State gesetzt
|
||
RECOVERY_BUILD_DOCKERFILE="" # zur Laufzeit aus State gesetzt
|
||
# Critical Files (Punkt 11) mit CONSERVED GOLD SHA (fest, aus laufendem Container)
|
||
declare -A CRITICAL_SHA=(
|
||
["run_backfill_year.py"]="70d3abb1a92a871e28cc6fb95b9a26944b55dd8df1b72db80642d6881b7b0f20"
|
||
["backfill.py"]="1627682ae582bb13343ae8ea10491dd9dffaef6fa4f7594d85378c2a500e81c7"
|
||
["repository.py"]="8fff892f52a612f464b5be21c66ef7787a47f2d7d049f815ef87837553c2a228"
|
||
["pipeline_v2.py"]="7c08ec3d1e75854cf341b9cbfd12afc075c4a173518dd3d951be4f95c5175fe6"
|
||
["trust_init.py"]="059db62741f2001f548d1879ccd3bf73bb2f4c8d0f492d151a469c9e4dcd3934" # trust/__init__.py
|
||
["provenance.py"]="4dd2ab977b69e9a47bb744dcc742a94d506e13035157a3977cdc7c8924d5f16f" # trust/provenance.py
|
||
["trust.py"]="7c2a948d2f1d668201e5af2a28164bcb2451b31d514a9f5f22b435c9ff2a0d01" # trust/trust.py
|
||
["sanity.py"]="875b07869e5148983e7a06bc05f6a6bec30727d1b5e95242d459564c18f6d6e9" # trust/sanity.py
|
||
["trust_gate.py"]="df0a97a1fc27985fe49b7458d94f7922bc5eaf2250807043c48dbc173f215ff1" # trust/trust_gate.py
|
||
)
|
||
# Relative Pfade je Key (fester Pfad, kein User-Parameter)
|
||
declare -A CRITICAL_PATH=(
|
||
["run_backfill_year.py"]="app/run_backfill_year.py"
|
||
["backfill.py"]="app/backfill.py"
|
||
["repository.py"]="app/persistence/repository.py"
|
||
["pipeline_v2.py"]="app/pipeline_v2.py"
|
||
["trust_init.py"]="app/trust/__init__.py"
|
||
["provenance.py"]="app/trust/provenance.py"
|
||
["trust.py"]="app/trust/trust.py"
|
||
["sanity.py"]="app/trust/sanity.py"
|
||
["trust_gate.py"]="app/trust/trust_gate.py"
|
||
)
|
||
|
||
# ------------------------------------------------- FINAL-REPRODUCTION-CONTROL-PLANE
|
||
# Phase 13.5 · Safe Recreate + Rollback Preparation. KEIN echter Recreate/Rollback
|
||
# Recovery-Gold ist ab jetzt IMMUTABLE BASELINE (nur ROLLBACK/RECOVERY-BASE).
|
||
# Die aktive Recovery-Image-ID wird NICHT statisch gesetzt, sondern zur Laufzeit
|
||
# aus dem kontrollierten Control-Plane-State (RECOVERY_IMAGE_STATE) gelesen.
|
||
# HISTORICAL_RECOVERY_IMAGE_ID (d2db...) bleibt NUR Evidence. FAIL CLOSED wenn
|
||
# State fehlt (kein freier Input, kein statischer Fallback).
|
||
RECOVERY_GOLD_IMAGE_ID="" # zur Laufzeit aus State gesetzt
|
||
RECOVERY_GOLD_TAG_FINAL="historical-v2-historical-service:recovery-gold-rebuilt-20260829"
|
||
# Candidate Identity Audit (read-only, 29.08.): ausschliesslich ueber latest adressiert.
|
||
# 0f6ca29a existiert NICHT mehr (historisch). Aktueller Candidate:
|
||
CANDIDATE_IMAGE_ID="sha256:1f3ba210780c5a00d4b73c5ee7817a88671fee2607a4ae0d5b1cefef1a1261c7"
|
||
CANDIDATE_TAG="historical-v2-historical-service:candidate-phase13-5-noop"
|
||
|
||
# ------------------------------------------------- DEPLOYED-GOLD-SOURCE-RECOVERY
|
||
# Phase 13.5 · Deployed Gold Source Recovery Control Plane.
|
||
# Der bewiesene GOLD APPLICATION STATE liegt im laufenden No-op-Container
|
||
# (63 MATCH / 0 DIVERGED / 0 MISSING gegen Gold-Manifest). Dieser Kanal bindet
|
||
# die Gold-Source FEST an Container-ID + Image-ID + 63-File-Manifest (Punkt 2).
|
||
# KEIN latest, KEIN full_sot_backup(), KEINE freien Inputs. Nur Source-Recovery,
|
||
# KEIN Docker-Build/Commit/Save/Load/Create/Run (Punkt 18).
|
||
# Volle Container-ID (read-only ermittelt 30.08.): 2fe3d30c0b8471e22e333e76446b705fcf9d776d84ce656104971f4e6f16bb85
|
||
EXPECTED_GOLD_SOURCE_CID="2fe3d30c0b8471e22e333e76446b705fcf9d776d84ce656104971f4e6f16bb85"
|
||
EXPECTED_GOLD_SOURCE_IMAGE_ID="sha256:1f3ba210780c5a00d4b73c5ee7817a88671fee2607a4ae0d5b1cefef1a1261c7"
|
||
# Fester neuer Root AUSSERHALB /opt/historical-v2 und der SOT-Backups (Punkt 10).
|
||
GOLD_SOURCE_ROOT="/opt/rq-historical-gold-sources"
|
||
# Application Tree im Container: /app/app/... (Gold-Manifest nutzt app/...)
|
||
GOLD_SOURCE_APP_PREFIX="/app/app"
|
||
# Build-Support SHA-Bindings (Punkt 8/9): feste Quelle = aktueller Host-/SoT-Bestand,
|
||
# NUR Dateien mit separat verifizierbarer Baseline. Keine freie Dateiauswahl.
|
||
# (read-only ermittelt 30.08. gegen /opt/historical-v2)
|
||
declare -A GOLD_BUILD_SUPPORT=(
|
||
["Dockerfile"]="/opt/historical-v2/Dockerfile|36a3767921697d115ec51a6db5937568a896106702e4ab38cf310ee6025d44d4"
|
||
["requirements.txt"]="/opt/historical-v2/requirements.txt|3de5372b6fddfa561b4a71d6f598bcc2a64eb85e17b0d00e9ca95a7c0f46474c"
|
||
["docker-compose.yml"]="/opt/historical-v2/docker-compose.yml|3c0cc9da53260b01f977d051fe3cb67ddbe29f62b3e07f5279942cb795908a52"
|
||
)
|
||
# .dockerignore wird kontrolliert ergaenzt (Punkt 12.5): exakt die Legacy-Exclusions.
|
||
GOLD_BUILD_EXCLUSION_LINES=( "app/app/" "app/eligibility.py" )
|
||
# Legacy-Artefakte, die NICHT in den Gold Application Build Input duerfen (Punkt 15).
|
||
declare -A GOLD_FORBIDDEN_EXTRA=(
|
||
["app/app/backfill.py"]=1
|
||
["app/app/test_idempotency.py"]=1
|
||
["app/eligibility.py"]=1
|
||
)
|
||
|
||
# ------------------------------------------------- RECOVERY CONTROL-PLANE STATE
|
||
# Phase 13.5 · Recovery Control-Plane Rotation. Dedizierter, root-owned State
|
||
# (KEIN latest, KEIN glob-first, KEINE Symlink-Semantik). Enthaelt die EINZIGE
|
||
# aktive Recovery-Gold-Source-Identitaet + die EINZIGE aktive Recovery-Image-ID.
|
||
# Kein freier Pfad. FAIL CLOSED wenn State fehlt/ungueltig.
|
||
CONTROL_PLANE_STATE_DIR="/opt/rq-historical-control-plane"
|
||
RECOVERY_SOURCE_STATE="$CONTROL_PLANE_STATE_DIR/recovery-gold-source.state"
|
||
RECOVERY_IMAGE_STATE="$CONTROL_PLANE_STATE_DIR/recovery-image.state"
|
||
# .dockerignore fester EXPECTED SHA (read-only ermittelt 30.08. gegen /opt/historical-v2;
|
||
# Inhalt exakt: app/app/ + app/eligibility.py). Kein Schätzen.
|
||
GOLD_DOCKERIGNORE_SHA="70bac51783f5ee956796cf2216ab17a550fcb646092edad3504763dbfabb881f"
|
||
# Historische Recovery-Image-ID (VERLOREN) — NUR Evidence, KEINE zwingende Output-ID.
|
||
HISTORICAL_RECOVERY_IMAGE_ID="sha256:d2db926dc6f5f7cf2f8e85d271a9b16fcc3cfb54b6557604d3d784f7961a4bc2"
|
||
# Recovery-Source-State-Felder (zur Laufzeit via load_recovery_source_state gesetzt).
|
||
# NUR der SOURCE-State (existiert VOR Build); der IMAGE-State existiert erst NACH Build.
|
||
RECOVERY_GOLD_SOURCE_SESSION="" # GOLD_SOURCE_SESSION aus RECOVERY_SOURCE_STATE
|
||
RECOVERY_GOLD_BUILD_INPUT_SHA="" # FINAL_BUILD_INPUT_SHA aus RECOVERY_SOURCE_STATE
|
||
RECOVERY_GOLD_SOURCE_CONTAINER_ID="" # SOURCE_CONTAINER_ID aus RECOVERY_SOURCE_STATE
|
||
RECOVERY_GOLD_SOURCE_IMAGE_ID="" # SOURCE_IMAGE_ID aus RECOVERY_SOURCE_STATE
|
||
|
||
|
||
# ------------------------------------------------- PERSISTENT-HOST-SOT-PROMOTION
|
||
# Phase 13.5 · Persistent Host-SoT Patch Promotion Channel. KEIN echter Promote
|
||
# durch Rain. Nur run_backfill_year.py. Feste Identitaeten, KEIN User-Input.
|
||
# PATCH_SOURCE=/opt/data/run_backfill_year.py existiert NUR im RQ-Container
|
||
# (hermes-red-queen), nicht auf dem VPS-Host -> fester docker cp aus dem Container.
|
||
PATCH_SOURCE_CONTAINER="hermes-red-queen"
|
||
PATCH_SOURCE_PATH="/opt/data/run_backfill_year.py"
|
||
PATCH_TARGET="/opt/historical-v2/app/run_backfill_year.py"
|
||
# Read-only ermittelt (29.08.): SHA256 des von Red Queen geprueften Patch-Artefakts (8292 B).
|
||
EXPECTED_RUNNER_PATCH_SHA256="6cca3c55bb559d75be2f58e59c231003902f124a54ada25582970524599f0752"
|
||
# Fester Backup-Root fuer die Promotion-Session (root-owned, nicht world-readable).
|
||
PROMOTE_BACKUP_ROOT="/opt/rq-historical-sot-backups/pre-runner-remediation"
|
||
|
||
# ------------------------------------------------- REMEDIATION BUILD/DEPLOY-KANAL
|
||
# Phase 13.5 · Controlled Runner-Remediation Build/Deploy Channel.
|
||
# KEIN echter Build/Recreate durch Rain; nur Kanal. Red Queen fuehrt nach Freigabe aus.
|
||
# Fester Remediation-Tag (Punkt 3) — KEIN Overwrite, KEIN force-retag, KEIN latest.
|
||
REMEDIATION_TAG="historical-v2-historical-service:candidate-phase13-5-runner-remediation"
|
||
# Build-Session-Basis (root-owned; enthaelt nach Build session.meta mit FULL_IMAGE_ID).
|
||
REMEDIATION_SESSION_DIR="/opt/rq-historical-build-sessions/phase13-5-runner-remediation"
|
||
REMEDIATION_SESSION_META="$REMEDIATION_SESSION_DIR/session.meta"
|
||
# Fester Build-Context/Dockerfile (Punkt 8,10) — NICHT mehr statisch auf /opt/historical-v2.
|
||
# Wird zur Laufzeit aus der kontrollierten Gold-Source-Session (RECOVERY_SOURCE_STATE)
|
||
# + autorisiertem Runner-Overlay als kontrollierter Staging-Context aufgeloest.
|
||
# Kein User-Input. FAIL CLOSED wenn State fehlt. Host-SoT ist NICHT Build-Input.
|
||
REMEDIATION_BUILD_CONTEXT="" # zur Laufzeit aus Gold-Source-Session + Overlay (Staging)
|
||
REMEDIATION_BUILD_DOCKERFILE="" # zur Laufzeit aus Staging
|
||
REMEDIATION_STAGING_ROOT="/opt/rq-historical-remediation-staging"
|
||
REMEDIATION_STAGING_DIR="" # zur Laufzeit = $REMEDIATION_STAGING_ROOT/$SESSION
|
||
REMEDIATION_BUILD_INPUT_SHA="" # zur Laufzeit deterministisch (Gold-Source + Overlay)
|
||
# Autorisiertes Runner-Overlay (Punkt 6): EINZIGE Quelle fuer run_backfill_year.py.
|
||
# Regulaere Datei, kein Symlink, SHA fest gebunden (RUNNER_REMEDIATION_SHA).
|
||
RUNNER_PATCH_SOURCE="/opt/data/run_backfill_year.py"
|
||
# Expected Runner-SHAs (Punkt 10/11): ALT = Gold (vor Remediation), NEU = persistent (nach).
|
||
RUNNER_GOLD_SHA="70d3abb1a92a871e28cc6fb95b9a26944b55dd8df1b72db80642d6881b7b0f20"
|
||
RUNNER_REMEDIATION_SHA="6cca3c55bb559d75be2f58e59c231003902f124a54ada25582970524599f0752"
|
||
# Recovery-Gold (immutable Baseline) — Referenz fuer Expected-Gold-Delta.
|
||
# Candidate/Recovery duerfen code-identisch sein, aber verschiedene Image-IDs haben (KEIN Fehler).
|
||
REM_NOOP_IMAGE_ID="sha256:1f3ba210780c5a00d4b73c5ee7817a88671fee2607a4ae0d5b1cefef1a1261c7"
|
||
|
||
# ------------------------------------------------- IMAGE-RETENTION / ARCHIVE CONTROL PLANE
|
||
# Phase 13.5 · Schutzschicht gegen bestaetigten Coolify DockerCleanupJob
|
||
# (taegl. 00:00 UTC, force_docker_cleanup=true, rmi auf getaggte Images ohne
|
||
# coolify.managed-Label). ZWEI unabhaengige Ebenen je Critical Image:
|
||
# SCHICHT A = Stopped Pin Container (Image-Referenz via docker create, nie gestartet)
|
||
# SCHICHT B = Persistentes, sha256-verifiziertes docker save-Archiv
|
||
# Kein Coolify-Cleanup-Disable. Kein coolify.managed-Spoof. Kein Overwrite.
|
||
# Fester Archive-Root (Punkt 3) — root:root, 700, KEINE freien Pfade.
|
||
ARCHIVE_ROOT="/opt/rq-historical-image-archives"
|
||
# Pin-Container-Namen (Punkt 9) — fest, hardcoded, KEIN freier Name.
|
||
PIN_RECOVERY_NAME="rq-historical-pin-recovery-gold"
|
||
PIN_REMEDIATION_NAME="rq-historical-pin-phase13-5-runner-remediation"
|
||
# Unterordner je Rolle (Punkt 3)
|
||
declare -A ARCHIVE_SUB=( [recovery]="recovery" [remediation]="candidates" )
|
||
# Archiv-Session-IDs: kurze Image-ID (<IMAGE_ID_SHORT>), die den immutable
|
||
# Session-Ordner benennt. Recovery = aktive Image-ID aus Control-Plane-State
|
||
# (zur Laufzeit via load_recovery_state gesetzt); Remediation = FULL_IMAGE_ID
|
||
# aus Build-Session (41e357a1..., aktuell im Store fehlend).
|
||
RECOVERY_ARCHIVE_ID="" # zur Laufzeit aus State gesetzt (load_recovery_state)
|
||
# Remediation-Archiv-Session-ID: kurze ID der Build-Session-Remendation (41e357a1...).
|
||
# (RAIN Hotfix) Statische REMEDIATION_ARCHIVE_ID ENTFERNT — historische 41e357a12b90
|
||
# darf keine aktive Archive-Identity sein. Aktive ID ausschliesslich aus der aktuellen,
|
||
# inspectable Build-Session ableiten (remediation_session / remediation_full_image_id).
|
||
# Remediation-Archiv-Referenz (aus Build-Session falls vorhanden, sonst leer):
|
||
# wird zur Laufzeit per remediation_full_image_id() aufgeloest.
|
||
# Pin-Referenz-Ziel (Schicht A): exakt diese Image-ID muss der Pin-Container referenzieren.
|
||
PIN_REMEDIATION_TARGET_ID="" # zur Laufzeit gesetzt
|
||
|
||
# RUN_MODE (Mutation Guard): READ_ONLY | DRY_RUN | MUTATING
|
||
RUN_MODE=MUTATING
|
||
|
||
die() { echo "ERR: $*" >&2; exit 2; }
|
||
req_ok() {
|
||
docker inspect -f '{{.State.Running}}' "$CONTAINER" >/dev/null 2>&1 \
|
||
|| die "Container '$CONTAINER' nicht vorhanden oder nicht am Laufen"
|
||
}
|
||
key_path() { echo "${TARGET_PATHS[$1]:-}"; }
|
||
stage_file() { echo "$STAGING_ROOT/${STAGE_FILES[$1]:-}"; }
|
||
key_ok() {
|
||
[ -n "${TARGET_PATHS[$1]:-}" ] && [ -n "${STAGE_FILES[$1]:-}" ] || die "unbekannter KEY '$1' (code-list)"
|
||
}
|
||
expect_zero() { [ $# -eq 0 ] || die "unerlaubtes Argument '$*'"; }
|
||
|
||
# ================= BUILD/RECREATE-KANAL HILFSFUNKTIONEN (alle fail-closed) ===
|
||
# MODE ist global: MUTATING (Default) | DRY_RUN (keine Mutationen)
|
||
mode_or_die() { [ "$RUN_MODE" != "DRY_RUN" ] || die "DRY_RUN-Modus, aber mutierender Pfad erreicht (FAIL CLOSED)"; }
|
||
|
||
# ---- Full/Partial-SoT-Backup-Klassifikation (Phase 13.5 Remediation, Blocker A) ----
|
||
# Ein Partial-Backup (z.B. pre-runner-remediation mit nur session.meta +
|
||
# run_backfill_year.py) darf NIEMALS als Full-SoT-Snapshot gelten.
|
||
# Ein gueltiger FULL SOT SNAPSHOT muss zwingend haben:
|
||
# historical-v2.tar (regular, kein Symlink, nicht leer)
|
||
# SHA256.manifest (regular, kein Symlink, nicht leer)
|
||
# und optional session.meta / provenance. Read-only.
|
||
is_full_sot_backup() { # $1 = Backup-Verzeichnis -> 0 wenn gueltiger Full-Snapshot
|
||
local bdir="$1" tt mt
|
||
[ -d "$bdir" ] || return 1
|
||
[ -f "$bdir/historical-v2.tar" ] || return 1
|
||
[ -s "$bdir/historical-v2.tar" ] || return 1
|
||
[ -f "$bdir/SHA256.manifest" ] || return 1
|
||
[ -s "$bdir/SHA256.manifest" ] || return 1
|
||
[ -L "$bdir/historical-v2.tar" ] && return 1
|
||
[ -L "$bdir/SHA256.manifest" ] && return 1
|
||
tt=$(stat -c '%F' "$bdir/historical-v2.tar" 2>/dev/null)
|
||
mt=$(stat -c '%F' "$bdir/SHA256.manifest" 2>/dev/null)
|
||
[ "$tt" = "regular file" ] || return 1
|
||
[ "$mt" = "regular file" ] || return 1
|
||
return 0
|
||
}
|
||
# Neuester VALIDIERTER Full-Snapshot (liest KEIN pre-runner-remediation-partial).
|
||
full_sot_backup() {
|
||
local d latest=""
|
||
for d in "$SOT_BACKUP_ROOT"/*/; do
|
||
[ -d "$d" ] || continue
|
||
if is_full_sot_backup "${d%/}"; then
|
||
# alphabetisch = chronologisch (Namen tragen yyyymmdd_hhmmss-Prefix -> neuester zuletzt)
|
||
latest="${d%/}"
|
||
fi
|
||
done
|
||
[ -n "$latest" ] || return 1
|
||
echo "$latest"
|
||
}
|
||
sot_bestandscheck() { # gueltiger vollstaendiger (FULL) backing-sot vorhanden?
|
||
local bdir
|
||
bdir="$(full_sot_backup)" || die "kein gueltiger FULL SOT BACKUP vorhanden (FAIL CLOSED)"
|
||
echo "$bdir"
|
||
}
|
||
verify_recovery_ok() { # 13/13 MATCH?
|
||
local allok=1 k csrc htarget gold hsha csha
|
||
for k in "${!RECOVERY[@]}"; do
|
||
IFS='|' read -r csrc htarget gold <<< "${RECOVERY[$k]}"
|
||
hsha="$(sha256sum "$htarget" 2>/dev/null | awk '{print $1}')" || hsha=""
|
||
csha="$(docker exec "$CONTAINER" sha256sum "$csrc" 2>/dev/null | awk '{print $1}')" || csha=""
|
||
if [ "$hsha" != "$gold" ] || [ "$csha" != "$gold" ]; then allok=0; echo " FEHLT: $k host=$hsha container=$csha"; fi
|
||
done
|
||
[ "$allok" = "1" ] || die "verify-sot-recovery NICHT 13/13 MATCH (FAIL CLOSED)"
|
||
}
|
||
ensure_gold_image() { docker image inspect "$GOLD_IMAGE_ID" >/dev/null 2>&1 || die "Gold-Image $GOLD_IMAGE_ID fehlt (FAIL CLOSED)"; }
|
||
ensure_gold_run() { docker inspect -f '{{.State.Running}}' "$HISTORICAL_SERVICE" >/dev/null 2>&1 || die "Gold-Container $HISTORICAL_SERVICE läuft nicht (FAIL CLOSED)"; }
|
||
ensure_db_run() { docker inspect -f '{{.State.Running}}' historical-db >/dev/null 2>&1 || die "historical-db läuft nicht (FAIL CLOSED)"; }
|
||
ensure_db_volume() { docker inspect -f '{{ range .Mounts }}{{ .Name }} {{ end }}' historical-db 2>/dev/null | grep -q 'historical-db-data' || die "DB-Volume historical-db-data fehlt (FAIL CLOSED)"; }
|
||
ensure_builddef() { [ -f "$COMPOSE_FILE" ] || die "Compose fehlt: $COMPOSE_FILE"; [ -f "$COMPOSE_PROJECT/Dockerfile" ] || die "Dockerfile fehlt"; }
|
||
exclusion_valid() { # .dockerignore exakt die 3 Legacy-Einträge, keine Gold-Datei ausgeschlossen
|
||
[ -f "$BUILD_EXCLUSION_FILE" ] || die ".dockerignore fehlt (FAIL CLOSED)"
|
||
local line
|
||
while IFS= read -r line; do
|
||
[ -z "$line" ] && continue
|
||
case "$line" in
|
||
app/app/|app/eligibility.py) ;;
|
||
*) die "unerwartete Exclusion '$line' im .dockerignore (FAIL CLOSED)" ;;
|
||
esac
|
||
done < "$BUILD_EXCLUSION_FILE"
|
||
}
|
||
|
||
# ------------------------------------------------- RECOVERY CONTROL-PLANE STATE LOAD
|
||
# Laedt die EINZIGE aktive Recovery-Identitaet aus dem kontrollierten State
|
||
# (KEIN latest, KEIN glob-first, KEINE Symlink-Semantik). Setzt zur Laufzeit:
|
||
# RECOVERY_GOLD_IMAGE_ID (aus RECOVERY_IMAGE_STATE: NEW_RECOVERY_FULL_IMAGE_ID)
|
||
# RECOVERY_ARCHIVE_ID (abgeleitet aus der geladenen Image-ID)
|
||
# RECOVERY_BUILD_CONTEXT (aus RECOVERY_SOURCE_STATE: GOLD_SOURCE_SESSION)
|
||
# RECOVERY_BUILD_DOCKERFILE
|
||
# FAIL CLOSED wenn State fehlt/ungueltig. Kein freier Input, kein statischer Fallback.
|
||
load_recovery_state() {
|
||
# --- Recovery-Image-State (Punkt 17: NEW_RECOVERY_FULL_IMAGE_ID) ---
|
||
if [ ! -f "$RECOVERY_IMAGE_STATE" ]; then
|
||
die "load-recovery-state: Kein Recovery-Image-State ($RECOVERY_IMAGE_STATE). NUR nach erfolgreichem Recovery-Build (FAIL CLOSED)"
|
||
fi
|
||
local id
|
||
id="$(awk -F'=' '$1=="NEW_RECOVERY_FULL_IMAGE_ID"{print $2}' "$RECOVERY_IMAGE_STATE" 2>/dev/null)"
|
||
[[ "$id" =~ ^sha256:[a-f0-9]{64}$ ]] || die "load-recovery-state: NEW_RECOVERY_FULL_IMAGE_ID ungueltig/fehlt (FAIL CLOSED)"
|
||
RECOVERY_GOLD_IMAGE_ID="$id"
|
||
RECOVERY_ARCHIVE_ID="${id#sha256:}"; RECOVERY_ARCHIVE_ID="${RECOVERY_ARCHIVE_ID:0:12}"
|
||
# --- Recovery-Source-State (Punkt 9: GOLD_SOURCE_SESSION) ---
|
||
if [ ! -f "$RECOVERY_SOURCE_STATE" ]; then
|
||
die "load-recovery-state: Kein Recovery-Source-State ($RECOVERY_SOURCE_STATE). NUR nach realem recover (FAIL CLOSED)"
|
||
fi
|
||
local sess
|
||
sess="$(awk -F'=' '$1=="GOLD_SOURCE_SESSION"{print $2}' "$RECOVERY_SOURCE_STATE" 2>/dev/null)"
|
||
[[ "$sess" =~ ^[0-9]{8}_[0-9]{6}$ ]] || die "load-recovery-state: GOLD_SOURCE_SESSION ungueltig/fehlt (FAIL CLOSED)"
|
||
RECOVERY_BUILD_CONTEXT="$GOLD_SOURCE_ROOT/$sess"
|
||
RECOVERY_BUILD_DOCKERFILE="$GOLD_SOURCE_ROOT/$sess/Dockerfile"
|
||
}
|
||
|
||
# load_recovery_source_state(): laedt NUR den Recovery-SOURCE-State (existiert VOR Build).
|
||
# Setzt RECOVERY_BUILD_CONTEXT/DOCKERFILE + die Source-Identitaets-Felder.
|
||
# Verlangt NICHT den Recovery-IMAGE-State (der existiert erst NACH Build).
|
||
# FAIL CLOSED wenn SOURCE-State fehlt/ungueltig. Kein freier Input, kein statischer Fallback.
|
||
load_recovery_source_state() {
|
||
if [ ! -f "$RECOVERY_SOURCE_STATE" ]; then
|
||
die "load-recovery-source-state: Kein Recovery-Source-State ($RECOVERY_SOURCE_STATE). NUR nach realem recover (FAIL CLOSED)"
|
||
fi
|
||
local sess sha cid iid
|
||
sess="$(awk -F'=' '$1=="GOLD_SOURCE_SESSION"{print $2}' "$RECOVERY_SOURCE_STATE" 2>/dev/null)"
|
||
[[ "$sess" =~ ^[0-9]{8}_[0-9]{6}$ ]] || die "load-recovery-source-state: GOLD_SOURCE_SESSION ungueltig/fehlt (FAIL CLOSED)"
|
||
sha="$(awk -F'=' '$1=="FINAL_BUILD_INPUT_SHA"{print $2}' "$RECOVERY_SOURCE_STATE" 2>/dev/null)"
|
||
[[ "$sha" =~ ^[a-f0-9]{64}$ ]] || die "load-recovery-source-state: FINAL_BUILD_INPUT_SHA ungueltig/fehlt (FAIL CLOSED)"
|
||
cid="$(awk -F'=' '$1=="SOURCE_CONTAINER_ID"{print $2}' "$RECOVERY_SOURCE_STATE" 2>/dev/null)"
|
||
[[ "$cid" =~ ^[a-f0-9]{64}$ ]] || die "load-recovery-source-state: SOURCE_CONTAINER_ID ungueltig/fehlt (FAIL CLOSED)"
|
||
iid="$(awk -F'=' '$1=="SOURCE_IMAGE_ID"{print $2}' "$RECOVERY_SOURCE_STATE" 2>/dev/null)"
|
||
[[ "$iid" =~ ^sha256:[a-f0-9]{64}$ ]] || die "load-recovery-source-state: SOURCE_IMAGE_ID ungueltig/fehlt (FAIL CLOSED)"
|
||
RECOVERY_GOLD_SOURCE_SESSION="$sess"
|
||
RECOVERY_GOLD_BUILD_INPUT_SHA="$sha"
|
||
RECOVERY_GOLD_SOURCE_CONTAINER_ID="$cid"
|
||
RECOVERY_GOLD_SOURCE_IMAGE_ID="$iid"
|
||
RECOVERY_BUILD_CONTEXT="$GOLD_SOURCE_ROOT/$sess"
|
||
RECOVERY_BUILD_DOCKERFILE="$GOLD_SOURCE_ROOT/$sess/Dockerfile"
|
||
}
|
||
|
||
# ------------------------------------------------- REMEDIATION BUILD-CONTEXT ROTATION
|
||
# Phase 13.5 · Remediation Build Context Control-Plane Rotation.
|
||
# Der Remediation-Build kommt NICHT mehr aus /opt/historical-v2 (Host-SoT), sondern
|
||
# aus der kontrollierten, immutable Gold-Source-Session + EXAKT EINEM autorisierten
|
||
# Runner-Overlay (run_backfill_year.py = RUNNER_REMEDIATION_SHA).
|
||
# Host-SoT bleibt NUR diagnostisch (HOST_SOT_DRIFT=NON-BLOCKING_FOR_REMEDIATION_BUILD).
|
||
# Kein globaler Pfadtausch: Recovery/No-op/Host-SoT-Verify/Promotion bleiben unveraendert.
|
||
|
||
# load_remediation_source_state(): bindet die EINZIGE aktive Gold-Source-Session aus
|
||
# RECOVERY_SOURCE_STATE (KEIN latest/auto-select/freie Session). Setzt:
|
||
# REMEDIATION_SESSION_ID, REMEDIATION_STAGING_DIR, REMEDIATION_BUILD_CONTEXT/DOCKERFILE
|
||
# FAIL CLOSED wenn State fehlt/ungueltig oder Session-Ordner fehlt.
|
||
load_remediation_source_state() {
|
||
if [ ! -f "$RECOVERY_SOURCE_STATE" ]; then
|
||
die "load-remediation-source-state: Kein Recovery-Source-State ($RECOVERY_SOURCE_STATE). NUR nach realem recover (FAIL CLOSED)"
|
||
fi
|
||
local sess
|
||
sess="$(awk -F'=' '$1=="GOLD_SOURCE_SESSION"{print $2}' "$RECOVERY_SOURCE_STATE" 2>/dev/null)"
|
||
[[ "$sess" =~ ^[0-9]{8}_[0-9]{6}$ ]] || die "load-remediation-source-state: GOLD_SOURCE_SESSION ungueltig/fehlt (FAIL CLOSED)"
|
||
local sdir="$GOLD_SOURCE_ROOT/$sess"
|
||
[ -d "$sdir" ] || die "load-remediation-source-state: Gold-Source-Session fehlt $sdir (FAIL CLOSED)"
|
||
REMEDIATION_SESSION_ID="$sess"
|
||
REMEDIATION_STAGING_DIR="$REMEDIATION_STAGING_ROOT/$sess"
|
||
REMEDIATION_BUILD_CONTEXT="$REMEDIATION_STAGING_DIR"
|
||
REMEDIATION_BUILD_DOCKERFILE="$REMEDIATION_STAGING_DIR/Dockerfile"
|
||
}
|
||
|
||
# remediation_expected_manifest(): erzeugt das ERWARTETE Remediation-Build-Input-Manifest
|
||
# (zeilenweise '<sha> app/<rel>', konsistent mit GOLD_MANIFEST) aus der Gold-Source-Session
|
||
# + autorisiertem Runner-Overlay. Basis = Gold-Source-Session (immutable, 63 Gold-Dateien),
|
||
# dann run_backfill_year.py durch RUNNER_REMEDIATION_SHA ersetzt. KEIN Host-SoT. Read-only.
|
||
remediation_expected_manifest() {
|
||
local sdir="$GOLD_SOURCE_ROOT/$REMEDIATION_SESSION_ID"
|
||
[ -d "$sdir" ] || die "remediation-expected-manifest: Session fehlt $sdir (FAIL CLOSED)"
|
||
( cd "$sdir" && find app -type f -name '*.py' ! -path '*/__pycache__/*' -printf '%P\n' 2>/dev/null | sort \
|
||
| while IFS= read -r rel; do
|
||
[ -f "app/$rel" ] || continue
|
||
if [ "$rel" = "run_backfill_year.py" ]; then
|
||
printf '%s app/%s\n' "$RUNNER_REMEDIATION_SHA" "$rel"
|
||
else
|
||
printf '%s app/%s\n' "$(sha256sum "app/$rel" | awk '{print $1}')" "$rel"
|
||
fi
|
||
done )
|
||
}
|
||
|
||
# remediation_expected_build_input_sha(): deterministischer REMEDIATION_BUILD_INPUT_SHA
|
||
# ueber den finalen tatsaechlich gebauten Input: 62 unveraenderte Gold-Dateien + Runner
|
||
# 6cca3c55... + Dockerfile + requirements.txt + docker-compose.yml + .dockerignore.
|
||
# Basis = $1 (default: Gold-Source-Session, immutable), Runner-Overlay ersetzt. KEIN Host-SoT.
|
||
remediation_expected_build_input_sha() {
|
||
local sdir="${1:-$GOLD_SOURCE_ROOT/$REMEDIATION_SESSION_ID}"
|
||
[ -d "$sdir" ] || die "remediation-build-input-sha: Basis fehlt $sdir (FAIL CLOSED)"
|
||
( cd "$sdir" && find . -type f ! -name 'session.meta' ! -name 'application.manifest' ! -name 'build-support.manifest' ! -name 'final-build-input.manifest' -printf '%P\n' 2>/dev/null | sort \
|
||
| while IFS= read -r f; do
|
||
[ -f "$f" ] || continue
|
||
if [ "$f" = "app/run_backfill_year.py" ]; then
|
||
printf '%s %s\n' "$RUNNER_REMEDIATION_SHA" "$f"
|
||
else
|
||
printf '%s %s\n' "$(sha256sum "$f" | awk '{print $1}')" "$f"
|
||
fi
|
||
done ) | sha256sum | awk '{print $1}'
|
||
}
|
||
|
||
# remediation_expected_manifest_ok(): klassifiziert das ERWARTETE Remediation-Build-Input
|
||
# (Gold-Source-Session + autorisierter Runner-Overlay, via remediation_expected_manifest)
|
||
# gegen GOLD_MANIFEST. Erwartung: 62 Gold-MATCH + 1 AUTHORIZED_DIVERGED (Runner 6cca3c55)
|
||
# + 0 UNAUTH + 0 MISSING + 0 EXTRA. Setzt globale Zaehler RM_MATCH/RM_AUTH/RM_UNAUTH/
|
||
# RM_MISSING/RM_EXTRA. Read-only. FAIL CLOSED bei jeder Abweichung.
|
||
remediation_expected_manifest_ok() {
|
||
local sdir="$GOLD_SOURCE_ROOT/$REMEDIATION_SESSION_ID"
|
||
[ -d "$sdir" ] || die "remediation-manifest-ok: Session fehlt $sdir (FAIL CLOSED)"
|
||
local -A gsha
|
||
while read -r sha p; do gsha["$p"]="$sha"; done < "$GOLD_MANIFEST"
|
||
RM_MATCH=0; RM_AUTH=0; RM_UNAUTH=0; RM_MISSING=0; RM_EXTRA=0
|
||
local path g s
|
||
# Gold-Dateien: 62 MATCH + 1 AUTHORIZED (Runner 6cca3c55)
|
||
for path in "${!gsha[@]}"; do
|
||
g="${gsha[$path]}"
|
||
if [ "$path" = "app/run_backfill_year.py" ]; then
|
||
# autorisierte Divergenz: Gold 70d3abb1... -> Remediation 6cca3c55...
|
||
s="$(remediation_expected_manifest | awk -v r="$path" '$2==r{print $1}')"
|
||
if [ "$s" = "$RUNNER_REMEDIATION_SHA" ]; then RM_AUTH=$((RM_AUTH+1))
|
||
elif [ "$s" = "$g" ]; then RM_MATCH=$((RM_MATCH+1))
|
||
else RM_UNAUTH=$((RM_UNAUTH+1)); fi
|
||
else
|
||
s="$(remediation_expected_manifest | awk -v r="$path" '$2==r{print $1}')"
|
||
if [ -z "$s" ]; then RM_MISSING=$((RM_MISSING+1))
|
||
elif [ "$s" = "$g" ]; then RM_MATCH=$((RM_MATCH+1))
|
||
else RM_UNAUTH=$((RM_UNAUTH+1)); fi
|
||
fi
|
||
done
|
||
# Extra-Dateien (forbidden extras + unerlaubte) im ERWARTETEN Build-Input
|
||
local rel
|
||
while IFS= read -r rel; do
|
||
[ -n "$rel" ] || continue
|
||
case "$rel" in
|
||
app/app/backfill.py|app/app/test_idempotency.py|app/eligibility.py) RM_EXTRA=$((RM_EXTRA+1)) ;;
|
||
app/*) [ -n "${gsha[$rel]:-}" ] || RM_EXTRA=$((RM_EXTRA+1)) ;;
|
||
esac
|
||
done < <( remediation_expected_manifest | awk '{print $2}' | sort )
|
||
[ "$RM_AUTH" = "1" ] && [ "$RM_UNAUTH" = "0" ] && [ "$RM_MISSING" = "0" ] && [ "$RM_EXTRA" = "0" ]
|
||
}
|
||
|
||
# remediation_staging_matches_expected(): manifestiert das TATSÄCHLICHE Staging-Dir
|
||
# (nach Overlay) und vergleicht es exakt mit dem erwarteten Build-Input
|
||
# (remediation_expected_manifest). Zusaetzlich: forbidden extras im Staging ABSENT.
|
||
# Read-only. FAIL CLOSED bei jeder Abweichung.
|
||
remediation_staging_matches_expected() {
|
||
local sdir="$REMEDIATION_STAGING_DIR"
|
||
[ -d "$sdir" ] || die "remediation-staging-check: Staging fehlt $sdir (FAIL CLOSED)"
|
||
local exp act
|
||
exp="$(remediation_expected_manifest)"
|
||
act="$( cd "$sdir" && find app -type f -name '*.py' ! -path '*/__pycache__/*' -printf '%P\n' 2>/dev/null | sort \
|
||
| while IFS= read -r rel; do
|
||
[ -f "app/$rel" ] || continue
|
||
printf '%s app/%s\n' "$(sha256sum "app/$rel" | awk '{print $1}')" "$rel"
|
||
done )"
|
||
[ "$act" = "$exp" ] || die "remediation-staging-check: Staging != erwartetes Build-Input (FAIL CLOSED)"
|
||
# forbidden extras im Staging zwingend ABSENT
|
||
for fe in app/app/backfill.py app/app/test_idempotency.py app/eligibility.py; do
|
||
[ -e "$sdir/$fe" ] && die "remediation-staging-check: forbidden extra $fe im Staging (FAIL CLOSED)"
|
||
done
|
||
return 0
|
||
}
|
||
|
||
# write_recovery_image_state(): atomarer Write des Recovery-IMAGE-State NACH erfolgreichem Build.
|
||
# $1 = FULL_IMAGE_ID (sha256:...). Verlangt: State ABSENT (kein Overwrite, FAIL CLOSED).
|
||
# Schreibt via Temp-Datei im selben Filesystem + fsync + atomares mv. Kein partial state sichtbar.
|
||
write_recovery_image_state() {
|
||
local fullid="$1" tmp
|
||
[[ "$fullid" =~ ^sha256:[a-f0-9]{64}$ ]] || die "write-recovery-image-state: FULL_IMAGE_ID ungueltig (FAIL CLOSED)"
|
||
mkdir -p "$CONTROL_PLANE_STATE_DIR" || die "write-recovery-image-state: State-Dir nicht anlegbar (FAIL CLOSED)"
|
||
if [ -e "$RECOVERY_IMAGE_STATE" ]; then
|
||
die "write-recovery-image-state: Recovery-Image-State existiert bereits $RECOVERY_IMAGE_STATE (KEIN Overwrite, FAIL CLOSED)"
|
||
fi
|
||
tmp="$CONTROL_PLANE_STATE_DIR/.recovery-image.state.tmp.$$"
|
||
{
|
||
echo "NEW_RECOVERY_FULL_IMAGE_ID=$fullid"
|
||
echo "RECOVERY_TAG=$RECOVERY_REBUILD_TAG"
|
||
echo "BUILD_TIMESTAMP=$(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
||
echo "BUILD_INPUT_SHA=$RECOVERY_GOLD_BUILD_INPUT_SHA"
|
||
echo "SOURCE_SESSION=$RECOVERY_GOLD_SOURCE_SESSION"
|
||
echo "SOURCE_CONTAINER_ID=$RECOVERY_GOLD_SOURCE_CONTAINER_ID"
|
||
echo "SOURCE_IMAGE_ID=$RECOVERY_GOLD_SOURCE_IMAGE_ID"
|
||
echo "WRAPPER_SHA=$(sha256sum /usr/local/bin/rq-historical | awk '{print $1}')"
|
||
} > "$tmp" || { rm -f "$tmp"; die "write-recovery-image-state: Temp-Write fehlgeschlagen (FAIL CLOSED)"; }
|
||
chown root:root "$tmp" 2>/dev/null || true
|
||
chmod 600 "$tmp" || { rm -f "$tmp"; die "write-recovery-image-state: chmod fehlgeschlagen (FAIL CLOSED)"; }
|
||
sync -f "$tmp" 2>/dev/null || true
|
||
mv -f "$tmp" "$RECOVERY_IMAGE_STATE" || { rm -f "$tmp"; die "write-recovery-image-state: atomares mv fehlgeschlagen (FAIL CLOSED)"; }
|
||
sync -f "$RECOVERY_IMAGE_STATE" 2>/dev/null || true
|
||
}
|
||
|
||
# Manifest einer Quelle (Gold-Container / neuer Temp-Container) erzeugen:
|
||
# gibt zeilenweise 'sha app/<rel>' auf stdout; Quelle ist IMMER laufender Container
|
||
gen_manifest() { # $1 = docker-Container-Name
|
||
docker exec "$1" sh -c "cd /app && find app -type f -name '*.py' ! -path '*/__pycache__/*' 2>/dev/null | sort | while IFS= read -r f; do sha256sum \"\$f\" 2>/dev/null; done"
|
||
}
|
||
classify_and_report() { # $1 = neues Manifest Datei (Temp); vergleicht gegen GOLD_MANIFEST + FORBIDDEN_EXTRA
|
||
local new="$1" gold goldrel goldsha nrel nsha status
|
||
local d=0 m=0 x=0 mat=0 l=0
|
||
# Gold-Dateien
|
||
while read -r goldsha goldrel; do
|
||
goldrel="${goldrel#./}"
|
||
if [ -n "$goldrel" ]; then
|
||
l=$((l+1))
|
||
nsha="$(awk -v r="$goldrel" '$2==r{print $1}' "$new")"
|
||
if [ -z "$nsha" ]; then echo " MISSING $goldrel"; m=$((m+1));
|
||
elif [ "$nsha" = "$goldsha" ]; then mat=$((mat+1));
|
||
else echo " DIVERGED $goldrel (gold=$goldsha new=$nsha)"; d=$((d+1)); fi
|
||
fi
|
||
done < "$GOLD_MANIFEST"
|
||
# neue Dateien
|
||
while read -r nsha nrel; do
|
||
nrel="${nrel#./}"
|
||
found="$(awk -v r="$nrel" '$2==r{print $1}' "$GOLD_MANIFEST")"
|
||
if [ -z "$found" ]; then
|
||
echo " EXTRA $nrel"
|
||
if [ -n "${FORBIDDEN_EXTRA[$nrel]:-}" ]; then x=$((x+1)); FORBIDDEN_HIT=1; else x=$((x+1)); fi
|
||
fi
|
||
done < "$new"
|
||
echo "--- Gold $l Dateien: MATCH=$mat DIVERGED=$d MISSING=$m ; EXTRA=$x ---"
|
||
G_RET="ok"; [ $d -eq 0 ] && [ $m -eq 0 ] && [ $x -eq 0 ] || G_RET="fail"
|
||
}
|
||
|
||
# ================= RECOVERY-SNAPSHOT HILFSFUNKTIONEN (fail-closed) ============
|
||
# PRE-CONDITION-GUARDS fuer docker commit (Punkt 4). Fuehrt ALLE 14 Guards aus;
|
||
# bei jedem FAIL: die. Bei DRY_RUN ebenso (read-only check, nur anzeigen).
|
||
snapshot_guards() {
|
||
echo " [guard 1] Container-ID == 634cc505...:"
|
||
local cid img running mounts dbid cimg rh rr
|
||
cid="$(docker inspect -f '{{.ID}}' "$HISTORICAL_SERVICE" 2>/dev/null || echo NONE)"
|
||
[ "$cid" = "$RECOVERY_GOLD_CID" ] || die "Guard1 FAIL: Container-ID $cid != RECOVERY_GOLD_CID (FAIL CLOSED)"
|
||
echo " OK ($cid)"
|
||
echo " [guard 2] Running=true:"
|
||
running="$(docker inspect -f '{{.State.Running}}' "$HISTORICAL_SERVICE" 2>/dev/null)"
|
||
[ "$running" = "true" ] || die "Guard2 FAIL: nicht Running (FAIL CLOSED)"
|
||
echo " OK ($running)"
|
||
echo " [guard 3] Image-Metadata == f1a6f33e...:"
|
||
img="$(docker inspect -f '{{.Image}}' "$HISTORICAL_SERVICE" 2>/dev/null)"
|
||
[ "$img" = "$GOLD_IMAGE_ID" ] || die "Guard3 FAIL: Image $img != GOLD_IMAGE_ID (FAIL CLOSED)"
|
||
echo " OK ($img)"
|
||
echo " [guard 4] /health=200:"
|
||
rh="$(docker exec "$HISTORICAL_SERVICE" python -c 'import urllib.request;print(urllib.request.urlopen("http://localhost:8000/health",timeout=5).status)' 2>/dev/null || echo N/A)"
|
||
[ "$rh" = "200" ] || die "Guard4 FAIL: /health=$rh (FAIL CLOSED)"
|
||
echo " OK ($rh)"
|
||
echo " [guard 5] /health/ready=200:"
|
||
rr="$(docker exec "$HISTORICAL_SERVICE" python -c 'import urllib.request;print(urllib.request.urlopen("http://localhost:8000/health/ready",timeout=5).status)' 2>/dev/null || echo N/A)"
|
||
[ "$rr" = "200" ] || die "Guard5 FAIL: /health/ready=$rr (FAIL CLOSED)"
|
||
echo " OK ($rr)"
|
||
echo " [guard 6] Mounts == []:"
|
||
mounts="$(docker inspect -f '{{json .Mounts}}' "$HISTORICAL_SERVICE" 2>/dev/null)"
|
||
[ "$mounts" = "[]" ] || die "Guard6 FAIL: Mounts != [] (FAIL CLOSED)"
|
||
echo " OK ([$mounts])"
|
||
echo " [guard 7] historical-db laeuft unveraendert:"
|
||
dbid="$(docker inspect -f '{{.ID}}' historical-db 2>/dev/null || echo NONE)"
|
||
cimg="$(docker inspect -f '{{.Image}}' historical-db 2>/dev/null)"
|
||
dbr="$(docker inspect -f '{{.State.Running}}' historical-db 2>/dev/null)"
|
||
[ "$dbr" = "true" ] || die "Guard7 FAIL: historical-db Running=$dbr (FAIL CLOSED)"
|
||
echo " OK (ID=$dbid Image=$cimg Running=$dbr)"
|
||
echo " [guard 8-11] 63/63 Gold-Code-MATCH / 0 DIVERGED / 0 MISSING / 0 unerlaubte EXTRA:"
|
||
local tmp; tmp="$(mktemp)"
|
||
gen_manifest "$HISTORICAL_SERVICE" > "$tmp"
|
||
classify_and_report "$tmp"
|
||
local gret="$G_RET"; rm -f "$tmp"
|
||
[ "$gret" = "ok" ] || die "Guard8-11 FAIL: GOLD-REPRODUCTION nicht OK (FAIL CLOSED)"
|
||
echo " OK (GOLD REPRODUCTION OK)"
|
||
echo " [guard 12] Critical-File-SHAs == konservierter Gold-Stand (verify-sot-recovery):"
|
||
verify_recovery_ok
|
||
echo " OK (13/13 MATCH)"
|
||
echo " [guard 13] kein Runner/Backfill aktiv:"
|
||
docker exec "$HISTORICAL_SERVICE" sh -c 'ps -ef 2>/dev/null | grep -E "run_backfill_year|backfill" | grep -v grep | grep -v "\-\-force" || true' | grep -q "run_backfill_year\|backfill" \
|
||
&& die "Guard13 FAIL: Runner/Backfill-Prozess aktiv (FAIL CLOSED)"
|
||
echo " OK (kein Runner/Backfill aktiv)"
|
||
echo " [guard 14] Recovery-Zieltag existiert NOCH NICHT:"
|
||
if docker image inspect "$RECOVERY_GOLD_TAG" >/dev/null 2>&1; then
|
||
die "Guard14 FAIL: Recovery-Tag $RECOVERY_GOLD_TAG existiert bereits (FAIL CLOSED, kein Overwrite)"
|
||
fi
|
||
echo " OK (Tag frei)"
|
||
echo " ALLE 14 GUARDS OK"
|
||
}
|
||
|
||
# snapshot-recovery (MUTATION - nur Red Queen fuehrt aus; dry-run blockt via mode_or_die)
|
||
snapshot_recovery() {
|
||
expect_zero "$@"
|
||
echo "=== SNAPSHOT-RECOVERY ($RUN_MODE) ==="
|
||
echo " Source Container-ID : $RECOVERY_GOLD_CID (== historical-service)"
|
||
echo " Erwartete Image : $GOLD_IMAGE_ID"
|
||
echo " Zieltag : $RECOVERY_GOLD_TAG (fester Tag, kein Overwrite)"
|
||
echo " Guards:"
|
||
snapshot_guards
|
||
if [ "$RUN_MODE" = "DRY_RUN" ]; then
|
||
echo "DRY_RUN_PASS: snapshot-recovery wuerde docker commit $RECOVERY_GOLD_CID -> $RECOVERY_GOLD_TAG (keine Aktion)."
|
||
exit 0
|
||
fi
|
||
mode_or_die # DRY_RUN -> FAIL CLOSED (kein commit)
|
||
echo " [commit] docker commit $RECOVERY_GOLD_CID -> $RECOVERY_GOLD_TAG (Standard-Pause durch Docker)"
|
||
docker commit "$RECOVERY_GOLD_CID" "$RECOVERY_GOLD_TAG" >/dev/null || die "docker commit fehlgeschlagen (FAIL CLOSED)"
|
||
# Nach Commit: Gold-Container wieder Running und healthy pruefen
|
||
local running
|
||
running="$(docker inspect -f '{{.State.Running}}' "$HISTORICAL_SERVICE" 2>/dev/null)"
|
||
[ "$running" = "true" ] || die "Nach-Commit: Gold-Container nicht Running (FAIL CLOSED)"
|
||
local newid
|
||
newid="$(docker image inspect -f '{{.Id}}' "$RECOVERY_GOLD_TAG" 2>/dev/null)"
|
||
[ -n "$newid" ] || die "Nach-Commit: Recovery-Image nicht auffindbar (FAIL CLOSED)"
|
||
echo "RECOVERY_IMAGE_ID=$newid"
|
||
echo "KLASSIFIZIERUNG: NEW RECOVERY IMAGE DERIVED FROM VERIFIED GOLD CONTAINER (NICHT das Original-Gold-Image)."
|
||
echo "Original bleibt: $GOLD_IMAGE_ID"
|
||
echo "SNAPSHOT-RECOVERY OK. KEINE weitere Mutation."
|
||
}
|
||
|
||
# verify-recovery-snapshot (read-only): Recovery-Image-Code vs Gold-Code vs Gold-Manifest
|
||
verify_recovery_snapshot() {
|
||
req_ok; expect_zero "$@"
|
||
echo "=== VERIFY-RECOVERY-SNAPSHOT (Recovery-Image vs Gold-Container vs Gold-Manifest) ==="
|
||
[ -f "$GOLD_MANIFEST" ] || die "Gold-Manifest fehlt (FAIL CLOSED)"
|
||
# Recovery-Tag muss existieren
|
||
if ! docker image inspect "$RECOVERY_GOLD_TAG" >/dev/null 2>&1; then
|
||
die "Recovery-Tag $RECOVERY_GOLD_TAG existiert nicht (FAIL CLOSED — zuerst snapshot-recovery)"
|
||
fi
|
||
# 1. Tag + Immutable-ID zeigen auf exakt dasselbe Image (Punkt 9)
|
||
# tagid = ID aus Tag-Referenz; immid = ID aus der vollen Immutable-Referenz (sha256:<ID>)
|
||
local tagid immid
|
||
tagid="$(docker image inspect -f '{{.Id}}' "$RECOVERY_GOLD_TAG" 2>/dev/null)"
|
||
[ -n "$tagid" ] || die "Verify: Recovery-Tag-ID nicht aufloesbar (FAIL CLOSED)"
|
||
immid="$(docker inspect "sha256:$tagid" --format '{{.Id}}' 2>/dev/null)"
|
||
[ "$tagid" = "$immid" ] || die "Verify: Tag/Immutable-ID-Referenz stimmen nicht ueberein (Punkt 9 FAIL CLOSED)"
|
||
echo " [Punkt 9] Tag + Immutable-ID -> selbe Image-ID: $tagid (OK)"
|
||
# 2. Manifest vom Recovery-Image ueber temporaeren Container (kein Entrypoint, keine Volumes/Ports/Netzwerk)
|
||
local tmpc tmpf
|
||
tmpf="$(mktemp)"
|
||
# Container-2 (nicht laufend) aus Recovery-Image erzeugen, NUR fuer Datei-Manifest; kein Start
|
||
tmpc="rq-recv-verify-$(date +%s)"
|
||
docker create --name "$tmpc" --entrypoint /bin/true "$RECOVERY_GOLD_TAG" >/dev/null 2>&1 \
|
||
|| { rm -f "$tmpf"; die "Verify: Temp-Container konnte nicht erstellt werden (FAIL CLOSED)"; }
|
||
# Manifest via docker cp aus dem erzeugten (nicht gestarteten) Container lesen
|
||
docker cp "$tmpc:/app/app" "/tmp/rq-recv-verify-copy-$$" >/dev/null 2>&1 \
|
||
|| { docker rm -f "$tmpc" >/dev/null 2>&1; rm -f "$tmpf"; rm -rf "/tmp/rq-recv-verify-copy-$$" 2>/dev/null; die "Verify: Manifest-Lesen aus Recovery-Image fehlgeschlagen (FAIL CLOSED)"; }
|
||
( cd "/tmp/rq-recv-verify-copy-$$" && find . -type f -name '*.py' -printf '%P\n' 2>/dev/null | sort | while IFS= read -r rel; do
|
||
sha256sum "$rel" 2>/dev/null | awk -v r="$rel" '{print $1" "r}'
|
||
done ) > "$tmpf"
|
||
docker rm -f "$tmpc" >/dev/null 2>&1 || true
|
||
rm -rf "/tmp/rq-recv-verify-copy-$$" 2>/dev/null
|
||
classify_and_report "$tmpf"
|
||
local ret="$G_RET"; rm -f "$tmpf"
|
||
if [ "$ret" = "ok" ]; then
|
||
echo "VERDICT: RECOVERY GOLD SNAPSHOT OK (Recovery-Image == Gold-Code, 0 DIVERGED/MISSING/unerlaubte EXTRA)"
|
||
exit 0
|
||
else
|
||
echo "VERDICT: RECOVERY GOLD SNAPSHOT FAIL"
|
||
exit 2
|
||
fi
|
||
}
|
||
|
||
# ================= RECOVERY-GOLD-REBUILD (build-recovery-gold) ===============
|
||
# Erstellt Build-Context-Manifest (read-only) fuer RECOVERY_GOLD_BUILD_INPUT.
|
||
# Erfasst Dockerfile, requirements.txt, .dockerignore + alle app/* Dateien.
|
||
build_context_manifest() {
|
||
local ctx="$RECOVERY_BUILD_CONTEXT" f sha rel
|
||
# Dockerfile
|
||
[ -f "$ctx/Dockerfile" ] || die "build-context: Dockerfile fehlt (FAIL CLOSED)"
|
||
echo "Dockerfile $(sha256sum "$ctx/Dockerfile" | awk '{print $1}')"
|
||
# requirements.txt (falls vorhanden)
|
||
[ -f "$ctx/requirements.txt" ] && echo "requirements.txt $(sha256sum "$ctx/requirements.txt" | awk '{print $1}')"
|
||
# .dockerignore (falls vorhanden)
|
||
[ -f "$ctx/.dockerignore" ] && echo ".dockerignore $(sha256sum "$ctx/.dockerignore" | awk '{print $1}')"
|
||
# app/* (reguläre Dateien, .py + sonstiges, keine __pycache__)
|
||
if [ -d "$ctx/app" ]; then
|
||
find "$ctx/app" -type f -not -name '*.pyc' -not -path '*/__pycache__/*' -printf '%P\n' 2>/dev/null | sort | while IFS= read -r f; do
|
||
[ -f "$ctx/app/$f" ] || continue
|
||
sha="$(sha256sum "$ctx/app/$f" | awk '{print $1}')"
|
||
printf 'app/%s %s\n' "$f" "$sha"
|
||
done
|
||
fi
|
||
}
|
||
|
||
# Dockerfile "exakt erwartet": Basis + COPY-Semantik (Punkt 4 Guard)
|
||
dockerfile_expected() {
|
||
local f="$RECOVERY_BUILD_DOCKERFILE"
|
||
grep -qE '^FROM python:3\.11-slim' "$f" || die "Dockerfile: erwartetes Base-Image python:3.11-slim fehlt (FAIL CLOSED)"
|
||
grep -qE '^COPY app/ \./app/' "$f" || die "Dockerfile: erwartetes COPY app/ ./app/ fehlt (FAIL CLOSED)"
|
||
grep -qE '^EXPOSE 8000' "$f" || die "Dockerfile: erwartetes EXPOSE 8000 fehlt (FAIL CLOSED)"
|
||
}
|
||
|
||
# Alle Guards fuer einen Recovery-Rebuild (read-only, nutzbar im dry-run und build)
|
||
# Host-Guard: alle 63 Gold-Dateien MATCH (0 DIVERGED/0 MISSING) UND jede EXTRA
|
||
# ist eine der 3 erlaubten HOST_ONLY_LEGACY (via .dockerignore ausgeschlossen).
|
||
# Anders als classify_and_report (fuer Image/Container) duerfen diese 3 auf dem
|
||
# Host physisch existieren — der Build schliesst sie per .dockerignore aus.
|
||
verify_host_gold_context() {
|
||
local hm tmpf ok d=0 m=0 x=0 mat=0 l=0 goldrel goldsha nrel nsha
|
||
hm="$(mktemp)"; gen_host_manifest > "$hm"
|
||
while read -r goldsha goldrel; do
|
||
goldrel="${goldrel#./}"; [ -n "$goldrel" ] || continue
|
||
l=$((l+1))
|
||
nsha="$(awk -v r="$goldrel" '$2==r{print $1}' "$hm")"
|
||
if [ -z "$nsha" ]; then echo " MISSING $goldrel"; m=$((m+1));
|
||
elif [ "$nsha" = "$goldsha" ]; then mat=$((mat+1));
|
||
else echo " DIVERGED $goldrel"; d=$((d+1)); fi
|
||
done < "$GOLD_MANIFEST"
|
||
while read -r nsha nrel; do
|
||
nrel="${nrel#./}"
|
||
found="$(awk -v r="$nrel" '$2==r{print $1}' "$GOLD_MANIFEST")"
|
||
if [ -z "$found" ]; then
|
||
case "$nrel" in
|
||
app/app/backfill.py|app/app/test_idempotency.py|app/eligibility.py)
|
||
echo " EXTRA(erlaubte Legacy) $nrel" ;;
|
||
*) echo " EXTRA(unerlaubt) $nrel"; x=$((x+1)) ;;
|
||
esac
|
||
fi
|
||
done < "$hm"
|
||
rm -f "$hm"
|
||
echo " Gold $l: MATCH=$mat DIVERGED=$d MISSING=$m ; unerlaubte EXTRA=$x"
|
||
[ $d -eq 0 ] && [ $m -eq 0 ] && [ $x -eq 0 ]
|
||
}
|
||
|
||
# verify_recovery_build_input(): beweist den tatsaechlichen finalen Build Input der
|
||
# Gold-Source-Session (NICHT Host-SoT). Prueft exakt:
|
||
# - Build-Support SHAs (Dockerfile, requirements.txt, .dockerignore) gegen GOLD_BUILD_SUPPORT
|
||
# - .dockerignore exakt (GOLD_DOCKERIGNORE_SHA)
|
||
# - Dockerfile exakt erwartet
|
||
# - keine forbidden extras in der Session
|
||
# - FINAL_BUILD_INPUT_SHA exakt (aus Control-Plane-SOURCE-State)
|
||
# FAIL CLOSED bei jeder Abweichung. Kein freier Input, kein /opt/historical-v2.
|
||
verify_recovery_build_input() {
|
||
local ctx="$RECOVERY_BUILD_CONTEXT"
|
||
[ -d "$ctx" ] || die "verify-recovery-build-input: Build-Context fehlt $ctx (FAIL CLOSED)"
|
||
# Build-Support SHAs exakt (gegen die Session-Dateien, nicht Host)
|
||
local key spec src exp f s
|
||
for key in "${!GOLD_BUILD_SUPPORT[@]}"; do
|
||
spec="${GOLD_BUILD_SUPPORT[$key]}"; src="${spec%%|*}"; exp="${spec##*|}"
|
||
f="$ctx/$(basename "$src")"
|
||
[ -f "$f" ] || die "verify-recovery-build-input: fehlt $f (FAIL CLOSED)"
|
||
s="$(sha256sum "$f" | awk '{print $1}')"
|
||
[ "$s" = "$exp" ] || die "verify-recovery-build-input: Build-Support SHA mismatch $key (FAIL CLOSED)"
|
||
done
|
||
# .dockerignore exakt
|
||
local dis
|
||
dis="$(sha256sum "$ctx/.dockerignore" 2>/dev/null | awk '{print $1}')"
|
||
[ "$dis" = "$GOLD_DOCKERIGNORE_SHA" ] || die "verify-recovery-build-input: .dockerignore SHA mismatch (FAIL CLOSED)"
|
||
# Dockerfile exakt erwartet
|
||
dockerfile_expected || die "verify-recovery-build-input: Dockerfile nicht exakt erwartet (FAIL CLOSED)"
|
||
# keine forbidden extras in der Session
|
||
local rel
|
||
while IFS= read -r rel; do
|
||
[ -n "$rel" ] || continue
|
||
if [ -n "${GOLD_FORBIDDEN_EXTRA[$rel]:-}" ]; then
|
||
die "verify-recovery-build-input: forbidden extra $rel (FAIL CLOSED)"
|
||
fi
|
||
done < <( cd "$ctx" && find . -type f -printf '%P\n' 2>/dev/null | sort )
|
||
# FINAL_BUILD_INPUT_SHA exakt (aus SOURCE-State)
|
||
local got
|
||
got="$( ( cd "$ctx" && find . -type f ! -name 'session.meta' ! -name 'application.manifest' ! -name 'build-support.manifest' ! -name 'final-build-input.manifest' -printf '%P\n' 2>/dev/null | sort | while IFS= read -r f; do [ -f "$f" ] && sha256sum "$f"; done ) | sha256sum | awk '{print $1}' )"
|
||
[ "$got" = "$RECOVERY_GOLD_BUILD_INPUT_SHA" ] || die "verify-recovery-build-input: FINAL_BUILD_INPUT_SHA mismatch (Source veraendert) (FAIL CLOSED)"
|
||
return 0
|
||
}
|
||
|
||
rebuild_guards() {
|
||
echo " [guard A] verify-recovered-gold-build-source (Build-Source 63/63):"
|
||
verify_recovered_gold_build_source >/dev/null || die "Guard A FAIL: Recovery-Build-Source nicht verifiziert (FAIL CLOSED)"
|
||
echo " OK (63 MATCH / 0 DIVERGED / 0 MISSING / 0 unerlaubte EXTRA)"
|
||
echo " [guard B] Recovery-Build-Input (Build-Support SHAs + .dockerignore + Dockerfile + forbidden extras + FINAL_BUILD_INPUT_SHA):"
|
||
verify_recovery_build_input || die "Guard B FAIL: Recovery-Build-Input nicht reproduzierbar (FAIL CLOSED)"
|
||
echo " Guard B OK"
|
||
echo " [guard C] Host-SoT-Drift (NON-BLOCKING Diagnostic):"
|
||
local hm hmf d=0 m=0 x=0 mat=0 l=0 goldrel goldsha nrel nsha
|
||
hmf="$(mktemp)"; gen_host_manifest > "$hmf"
|
||
while read -r goldsha goldrel; do
|
||
goldrel="${goldrel#./}"; [ -n "$goldrel" ] || continue
|
||
l=$((l+1))
|
||
nsha="$(awk -v r="$goldrel" '$2==r{print $1}' "$hmf")"
|
||
if [ -z "$nsha" ]; then m=$((m+1));
|
||
elif [ "$nsha" = "$goldsha" ]; then mat=$((mat+1));
|
||
else d=$((d+1)); fi
|
||
done < "$GOLD_MANIFEST"
|
||
while read -r nsha nrel; do
|
||
nrel="${nrel#./}"
|
||
found="$(awk -v r="$nrel" '$2==r{print $1}' "$GOLD_MANIFEST")"
|
||
[ -z "$found" ] && x=$((x+1))
|
||
done < "$hmf"
|
||
rm -f "$hmf"
|
||
if [ $d -eq 0 ] && [ $m -eq 0 ] && [ $x -eq 0 ]; then
|
||
echo " HOST_SOT_DRIFT=NONE (MATCH=$mat)"
|
||
else
|
||
echo " HOST_SOT_DRIFT=DETECTED (MATCH=$mat DIVERGED=$d MISSING=$m EXTRA=$x) — NON-BLOCKING, Host-SoT wird nach RECOVERY READY repariert"
|
||
fi
|
||
echo " [guard D] Dockerfile exakt erwartet:"; dockerfile_expected && echo " OK"
|
||
echo " [guard E] Gold-Container Running + /health + /health/ready:"
|
||
ensure_gold_run
|
||
local rh rr
|
||
rh="$(docker exec "$HISTORICAL_SERVICE" python -c 'import urllib.request;print(urllib.request.urlopen("http://localhost:8000/health",timeout=5).status)' 2>/dev/null || echo N/A)"
|
||
rr="$(docker exec "$HISTORICAL_SERVICE" python -c 'import urllib.request;print(urllib.request.urlopen("http://localhost:8000/health/ready",timeout=5).status)' 2>/dev/null || echo N/A)"
|
||
[ "$rh" = "200" ] && [ "$rr" = "200" ] || die "Guard E FAIL: Gold-Container nicht gesund (FAIL CLOSED)"
|
||
echo " OK (/health=$rh /ready=$rr)"
|
||
echo " [guard F] historical-db gesund:"; ensure_db_run && echo " OK"
|
||
echo " [guard G] Recovery-Rebuild-Tag existiert NOCH NICHT:"
|
||
if docker image inspect "$RECOVERY_REBUILD_TAG" >/dev/null 2>&1; then
|
||
die "Guard G FAIL: Tag $RECOVERY_REBUILD_TAG existiert bereits (KEIN Overwrite, FAIL CLOSED)"
|
||
fi
|
||
echo " OK (Tag frei)"
|
||
}
|
||
|
||
# Host-Manifest (Host-SoT, read-only): app/* .py vs Gold-Manifest — wie gen_manifest aber vom Host
|
||
gen_host_manifest() {
|
||
( cd "/opt/historical-v2" && find app -type f -name '*.py' ! -path '*/__pycache__/*' 2>/dev/null | sort | while IFS= read -r f; do
|
||
sha256sum "$f" 2>/dev/null; done )
|
||
}
|
||
|
||
# read-only: Critical-Files-SHAs Recovery-Image + laufender Gold-Container + Gold-Baseline (Punkt 11)
|
||
# $2 (optional) = "runner" -> erlaubt die EINE autorisierte Runner-Divergenz im IMAGE:
|
||
# run_backfill_year.py image=RUNNER_REMEDIATION_SHA (Container bleibt Gold).
|
||
# Nur fuer verify-runner-remediation-image; alle anderen Aufrufer bleiben strikt 0-Divergenz.
|
||
verify_critical_files() {
|
||
# $1 = Basisverzeichnis mit app/ (kopierter Recovery-Tree) ODER leer -> nur Container+Baseline
|
||
local tmpc tmpdir k path imgsha csha
|
||
local base_dir="${1:-}"
|
||
local allow_runner="${2:-}"
|
||
local img_shas=""
|
||
if [ -n "$base_dir" ]; then
|
||
# Recovery-Image-SHAs aus kopiertem Tree. Der Tree ist prefixlos (docker cp
|
||
# von /app/app), daher wird der app/-Präfix aus CRITICAL_PATH entfernt.
|
||
for k in "${!CRITICAL_SHA[@]}"; do
|
||
path="${CRITICAL_PATH[$k]#app/}"
|
||
[ -f "$base_dir/$path" ] && img_shas="$img_shas $k:$(sha256sum "$base_dir/$path" | awk '{print $1}')" || img_shas="$img_shas $k:MISSING"
|
||
done
|
||
fi
|
||
local allok=1
|
||
for k in "${!CRITICAL_SHA[@]}"; do
|
||
# Container-Pfad: Code liegt im Container unter /app/app/<rel> (docker cp von /app/app)
|
||
csha="$(docker exec "$HISTORICAL_SERVICE" sha256sum "/app/app/${CRITICAL_PATH[$k]#app/}" 2>/dev/null | awk '{print $1}')" || csha="MISSING"
|
||
gold="${CRITICAL_SHA[$k]}"
|
||
img="MISSING"
|
||
if [ -n "$base_dir" ]; then
|
||
is=$(echo "$img_shas" | grep -oE "${k}:[a-f0-9]+|${k}:MISSING" | cut -d: -f2)
|
||
img="$is"
|
||
fi
|
||
# Autorisierte Runner-Divergenz: NUR run_backfill_year.py, NUR im Image, NUR auf RUNNER_REMEDIATION_SHA.
|
||
if [ "$allow_runner" = "runner" ] && [ "$k" = "run_backfill_year.py" ] \
|
||
&& [ "$img" = "$RUNNER_REMEDIATION_SHA" ] && [ "$csha" = "$gold" ]; then
|
||
printf ' AUTH %-22s image=%s container=%s gold=%s (autorisierte Runner-Divergenz)\n' "$k" "${img:0:12}" "${csha:0:12}" "${gold:0:12}"
|
||
continue
|
||
fi
|
||
if [ "$csha" = "$gold" ] && { [ -z "$base_dir" ] || [ "$img" = "$gold" ]; }; then
|
||
printf ' MATCH %-22s container=%s gold=%s\n' "$k" "${csha:0:12}" "${gold:0:12}"
|
||
else
|
||
printf ' FAIL %-22s image=%s container=%s gold=%s\n' "$k" "${img:0:12}" "${csha:0:12}" "${gold:0:12}"
|
||
allok=0
|
||
fi
|
||
done
|
||
[ "$allok" = "1" ] || die "verify: Critical-File-SHA-Mismatch (FAIL CLOSED)"
|
||
}
|
||
|
||
# read-only: Recovery-Rebuild-Image Config gegen erwartete Runtime-Konfiguration (Punkt 13)
|
||
verify_image_config() {
|
||
local c
|
||
# Erwartet: Entrypoint [] Cmd [uvicorn app.api.main:app --host 0.0.0.0 --port 8000] WorkDir=/app User=(leer) Exposed=8000
|
||
local exp_entry="[]" exp_workdir="/app" exp_user="" exp_expose="map[8000/tcp:{}]"
|
||
echo " [config] Entrypoint, Cmd, WorkingDir, User, ExposedPorts:"
|
||
local c_entry c_cmd c_wd c_user c_expose
|
||
c_entry="$(docker image inspect -f '{{json .Config.Entrypoint}}' "$RECOVERY_REBUILD_TAG" 2>/dev/null)"
|
||
c_cmd="$(docker image inspect -f '{{json .Config.Cmd}}' "$RECOVERY_REBUILD_TAG" 2>/dev/null)"
|
||
c_wd="$(docker image inspect -f '{{.Config.WorkingDir}}' "$RECOVERY_REBUILD_TAG" 2>/dev/null)"
|
||
c_user="$(docker image inspect -f '{{.Config.User}}' "$RECOVERY_REBUILD_TAG" 2>/dev/null)"
|
||
c_expose="$(docker image inspect -f '{{json .Config.ExposedPorts}}' "$RECOVERY_REBUILD_TAG" 2>/dev/null)"
|
||
printf ' Entrypoint = %s (erwartet %s)\n' "${c_entry:-<nil>}" "$exp_entry"
|
||
printf ' Cmd = %s\n' "${c_cmd:-<nil>}"
|
||
printf ' WorkingDir = %s (erwartet %s)\n' "${c_wd:-<nil>}" "$exp_workdir"
|
||
printf ' User = %s (erwartet leer)\n' "${c_user:-<nil>}"
|
||
printf ' Exposed = %s (erwartet 8000)\n' "${c_expose:-<nil>}"
|
||
# Healthcheck: Dockerfile setzt keinen -> <nil> wie beim Gold-Container
|
||
local c_hc
|
||
c_hc="$(docker image inspect -f '{{json .Config.Healthcheck}}' "$RECOVERY_REBUILD_TAG" 2>/dev/null)"
|
||
printf ' Healthcheck= %s (Gold: <nil>) (ENV redigiert, nicht ausgeben)\n' "${c_hc:-<nil>}"
|
||
# Sanity: WorkingDir + Expose + Entrypoint muessen den Erwartungen entsprechen (Cmd kann abweichen? erwartet exakt)
|
||
[ "$c_wd" = "$exp_workdir" ] || die "verify: WorkingDir unerwartet '$c_wd' (FAIL CLOSED)"
|
||
echo " [config] OK (ENV/secrets NICHT ausgegeben; nur redigierte Struktur)"
|
||
}
|
||
|
||
# ---- build-recovery-gold (NUR Build aus verifizierter Gold Source; DRY_RUN = kein Build) ----
|
||
build_recovery_gold() {
|
||
expect_zero "$@"
|
||
echo "=== BUILD-RECOVERY-GOLD ($RUN_MODE) ==="
|
||
# Build Context Rotation (Punkt 11): ausschliesslich aus kontrolliertem SOURCE-State.
|
||
# Kein statisches /opt/historical-v2. Keine User-Argumente. Der IMAGE-State existiert
|
||
# erst NACH dem Build und ist hier KEIN Input (zirkuläres Gate behoben).
|
||
load_recovery_source_state
|
||
echo " Build Context : $RECOVERY_BUILD_CONTEXT (aus Control-Plane-SOURCE-State)"
|
||
echo " Dockerfile : $RECOVERY_BUILD_DOCKERFILE (aus Control-Plane-SOURCE-State)"
|
||
echo " Recovery-Tag : $RECOVERY_REBUILD_TAG (fix, direkt getaggt, KEIN latest)"
|
||
echo " Source-Session: $RECOVERY_GOLD_SOURCE_SESSION"
|
||
echo " Build-Input : $RECOVERY_GOLD_BUILD_INPUT_SHA"
|
||
echo " Guards:"
|
||
# Build Preflight (Punkt 12): verify-recovered-gold-build-source PASS zwingend.
|
||
echo " [preflight] verify-recovered-gold-build-source:"
|
||
verify_recovered_gold_build_source >/dev/null || die "build-recovery-gold: Recovery-Source nicht verifiziert (FAIL CLOSED)"
|
||
echo " OK"
|
||
rebuild_guards
|
||
# Build-Context-Manifest (Punkt 5) — read-only RECOVERY_GOLD_BUILD_INPUT
|
||
echo " [build-input] RECOVERY_GOLD_BUILD_INPUT (Build-Context-Manifest):"
|
||
build_context_manifest | while IFS= read -r line; do echo " $line"; done
|
||
# Punkt 7: FINAL BUILD INPUT TOCTOU — unmittelbar vor docker build neu berechnen/verifizieren.
|
||
echo " [build-input-sha] FINAL_BUILD_INPUT_SHA (TOCTOU, unmittelbar vor Build):"
|
||
local got_sha
|
||
got_sha="$( ( cd "$RECOVERY_BUILD_CONTEXT" && find . -type f ! -name 'session.meta' ! -name 'application.manifest' ! -name 'build-support.manifest' ! -name 'final-build-input.manifest' -printf '%P\n' 2>/dev/null | sort | while IFS= read -r f; do [ -f "$f" ] && sha256sum "$f"; done ) | sha256sum | awk '{print $1}' )"
|
||
[ "$got_sha" = "$RECOVERY_GOLD_BUILD_INPUT_SHA" ] || die "build-recovery-gold: FINAL_BUILD_INPUT_SHA mismatch (Source veraendert) (FAIL CLOSED)"
|
||
echo " == $got_sha (OK)"
|
||
if [ "$RUN_MODE" = "DRY_RUN" ]; then
|
||
echo " [post-build] wuerde Image-ID ueber festen Tag ermitteln: docker image inspect $RECOVERY_REBUILD_TAG --format '{{.Id}}'"
|
||
echo " [post-build] wuerde atomar recovery-image.state erzeugen: $RECOVERY_IMAGE_STATE"
|
||
echo "DRY_RUN_PASS: build-recovery-gold wuerde docker build -f $RECOVERY_BUILD_DOCKERFILE -t $RECOVERY_REBUILD_TAG $RECOVERY_BUILD_CONTEXT (keine Aktion)."
|
||
exit 0
|
||
fi
|
||
mode_or_die
|
||
echo " [build] docker build -f $RECOVERY_BUILD_DOCKERFILE -t $RECOVERY_REBUILD_TAG $RECOVERY_BUILD_CONTEXT"
|
||
docker build -f "$RECOVERY_BUILD_DOCKERFILE" -t "$RECOVERY_REBUILD_TAG" "$RECOVERY_BUILD_CONTEXT" || die "build-recovery-gold fehlgeschlagen (FAIL CLOSED)"
|
||
# Punkt 8: Build-Output-Identity — Image-ID NICHT aus Textausgabe erraten, sondern
|
||
# kontrolliert ueber den fest verdrahteten Recovery-Tag ermitteln. Keine freie Tag-Eingabe.
|
||
local newid
|
||
newid="$(docker image inspect -f '{{.Id}}' "$RECOVERY_REBUILD_TAG" 2>/dev/null)"
|
||
# Punkt 9: Image-ID-Validierung — vorhanden, sha256:, vollstaendig, inspectable, exakt zum Tag.
|
||
[[ "$newid" =~ ^sha256:[a-f0-9]{64}$ ]] || die "build-recovery-gold: Image-ID ungueltig/fehlt (FAIL CLOSED)"
|
||
docker image inspect "sha256:${newid#sha256:}" >/dev/null 2>&1 || die "build-recovery-gold: Image nicht per ID inspectable (FAIL CLOSED)"
|
||
local tagid
|
||
tagid="$(docker image inspect -f '{{.Id}}' "$RECOVERY_REBUILD_TAG" 2>/dev/null)"
|
||
[ "$tagid" = "$newid" ] || die "build-recovery-gold: Tag->ID nicht eindeutig ($tagid != $newid) (FAIL CLOSED)"
|
||
echo " [post-build] Image-ID (aus festem Tag): $newid"
|
||
# Punkt 11: State-Write NUR NACH erfolgreichem Build + Tag->ID PASS. Atomar.
|
||
write_recovery_image_state "$newid"
|
||
echo " [post-build] recovery-image.state atomar geschrieben: $RECOVERY_IMAGE_STATE"
|
||
echo "RECOVERY_GOLD_IMAGE_ID=$newid"
|
||
echo "KLASSIFIZIERUNG: REBUILT RECOVERY GOLD IMAGE (NICHT das Original-Gold-Image; Original f1a6f33e... = UNAVAILABLE)"
|
||
echo "BUILD-RECOVERY-GOLD OK. KEIN Recreate, KEIN Runner. latest unangetastet."
|
||
}
|
||
|
||
# ---- verify-recovery-gold-image (read-only) ----
|
||
verify_recovery_gold_image() {
|
||
req_ok; expect_zero "$@"
|
||
echo "=== VERIFY-RECOVERY-GOLD-IMAGE (Rebuilt-Image vs Gold-Container vs Gold-Manifest) ==="
|
||
if ! docker image inspect "$RECOVERY_REBUILD_TAG" >/dev/null 2>&1; then
|
||
die "Recovery-Rebuild-Image fehlt (zuerst build-recovery-gold) (FAIL CLOSED)"
|
||
fi
|
||
local newid
|
||
newid="$(docker image inspect -f '{{.Id}}' "$RECOVERY_REBUILD_TAG" 2>/dev/null)"
|
||
echo " RECOVERY_GOLD_IMAGE_ID=$newid (Tag: $RECOVERY_REBUILD_TAG)"
|
||
echo " [Punkt 9] Tag -> Image-ID + direkt per ID inspectable:"
|
||
docker image inspect -f " ID=$newid" "sha256:${newid#sha256:}" >/dev/null 2>&1 && echo " Image per ID inspectable: OK" || die "verify: Image nicht per ID inspectable (FAIL CLOSED)"
|
||
echo " [Code] Recovery-Image vs Gold-Manifest + Gold-Container:"
|
||
local tmpc tmpdir tmpf gret
|
||
tmpc="rq-rebuild-verify-$(date +%s)"
|
||
tmpdir="/tmp/rq-rebuild-copy-$$"
|
||
rm -rf "$tmpdir"; mkdir -p "$tmpdir"
|
||
docker create --name "$tmpc" --entrypoint /bin/true "$RECOVERY_REBUILD_TAG" >/dev/null 2>&1 || die "verify: Temp-Container Fehler (FAIL CLOSED)"
|
||
docker cp "$tmpc:/app/app" "$tmpdir/appapp" >/dev/null 2>&1 || { docker rm -f "$tmpc" >/dev/null 2>&1; rm -rf "$tmpdir"; die "verify: /app/app nicht lesbar (FAIL CLOSED)"; }
|
||
docker rm -f "$tmpc" >/dev/null 2>&1 || true
|
||
# Manifest aus kopiertem Recovery-Image-Tree gegen Gold-Manifest.
|
||
# KANONISCHER NAMESPACE: Image-Manifest wird mit app/-Präfix erzeugt, damit es
|
||
# dem konservierten GOLD_MANIFEST (app/...) entspricht. Der kopierte Tree liegt
|
||
# unter $tmpdir/appapp und enthaelt die Dateien prefixlos (run_backfill_year.py,
|
||
# trust/trust_gate.py) -> hier wird app/ vorangestellt. KEINE Inhaltsnormalisierung.
|
||
tmpf="$(mktemp)"
|
||
( cd "$tmpdir/appapp" && find . -type f -name '*.py' ! -path '*/__pycache__/*' -printf '%P\n' 2>/dev/null | sort \
|
||
| while IFS= read -r rel; do [ -f "$rel" ] && sha256sum "$rel"; done | awk '{print $1" app/"$2}' ) > "$tmpf"
|
||
classify_and_report "$tmpf"
|
||
gret="$G_RET"; rm -f "$tmpf"
|
||
if [ "$gret" != "ok" ]; then
|
||
rm -rf "$tmpdir"; die "verify: Recovery-Rebuild-Image != Gold-Manifest (DIVERGED/MISSING/EXTRA) (FAIL CLOSED)"
|
||
fi
|
||
echo " IMAGE vs GOLD: 63 MATCH / 0 DIVERGED / 0 MISSING / 0 unerlaubte EXTRA"
|
||
echo " [Critical-Files] (Punkt 11) — Image-SHA vs gold:"
|
||
verify_critical_files "$tmpdir/appapp"
|
||
rm -rf "$tmpdir"
|
||
echo " [Config] (Punkt 13):"
|
||
verify_image_config
|
||
echo "VERDICT: REBUILT RECOVERY GOLD OK (63 MATCH / 0 DIVERGED / 0 MISSING / 0 unerlaubte EXTRA, Critical-SHAs + Config OK)"
|
||
exit 0
|
||
}
|
||
|
||
# ================= FINAL-REPRODUCTION-CONTROL-PLANE ==========================
|
||
# Phase 13.5 · Safe Recreate + Rollback Preparation. KEIN echter Recreate/Rollback
|
||
# durch Rain. Recovery-Gold = IMMUTABLE BASELINE. Candidate = dediziert gepinnt.
|
||
|
||
# Fester Compose-Override (temporaer, root-owned, KEIN User-Input): setzt das
|
||
# Image des historical-service explizit auf einen festen Tag und deaktiviert build.
|
||
# --no-deps verhindert DB-Recreate, --no-build verhindert Build.
|
||
write_recreate_override() {
|
||
# $1 = fester Image-Tag (CANDIDATE_TAG oder RECOVERY_GOLD_TAG_FINAL oder REMEDIATION_TAG)
|
||
local tag="$1" ovf
|
||
ovf="/tmp/rq-recreate-override-$$.yml"
|
||
cat > "$ovf" <<EOF
|
||
services:
|
||
$HISTORICAL_SERVICE:
|
||
image: $tag
|
||
build: null
|
||
EOF
|
||
echo "$ovf"
|
||
}
|
||
|
||
# Pre-Recreate/Pre-Rollback Gates (Punkt 10/16). Read-only. FAIL CLOSED.
|
||
recreate_pre_gates() {
|
||
load_recovery_state
|
||
echo " [gate A] verify-recovery-gold-image PASS:"
|
||
( verify_recovery_gold_image >/dev/null 2>&1 ) && echo " OK" || die "Gate A FAIL: Recovery-Gold nicht verifiziert (FAIL CLOSED)"
|
||
echo " [gate B] verify-candidate-image PASS:"
|
||
( verify_candidate_image >/dev/null 2>&1 ) && echo " OK" || die "Gate B FAIL: Candidate nicht verifiziert (FAIL CLOSED)"
|
||
echo " [gate C] Recovery Image inspectable:"
|
||
docker image inspect "$RECOVERY_GOLD_IMAGE_ID" >/dev/null 2>&1 || die "Gate C FAIL (FAIL CLOSED)"
|
||
echo " OK"
|
||
echo " [gate D] Candidate Image inspectable:"
|
||
docker image inspect "$CANDIDATE_IMAGE_ID" >/dev/null 2>&1 || die "Gate D FAIL (FAIL CLOSED)"
|
||
echo " OK"
|
||
echo " [gate E] Recovery Tag korrekt:"
|
||
local rt; rt="$(docker image inspect -f '{{.Id}}' "$RECOVERY_GOLD_TAG_FINAL" 2>/dev/null || true)"
|
||
[ "$rt" = "$RECOVERY_GOLD_IMAGE_ID" ] || die "Gate E FAIL: Recovery-Tag != Recovery-ID (FAIL CLOSED)"
|
||
echo " OK"
|
||
echo " [gate F] Candidate Tag korrekt:"
|
||
local ct; ct="$(docker image inspect -f '{{.Id}}' "$CANDIDATE_TAG" 2>/dev/null || true)"
|
||
[ "$ct" = "$CANDIDATE_IMAGE_ID" ] || die "Gate F FAIL: Candidate-Tag != Candidate-ID (FAIL CLOSED)"
|
||
echo " OK"
|
||
echo " [gate G] Running Gold Container healthy:"
|
||
ensure_gold_run
|
||
local rh; rh="$(docker exec "$HISTORICAL_SERVICE" python -c 'import urllib.request;print(urllib.request.urlopen("http://localhost:8000/health",timeout=5).status)' 2>/dev/null || echo N/A)"
|
||
[ "$rh" = "200" ] || die "Gate G FAIL: Gold-Container nicht healthy (FAIL CLOSED)"
|
||
echo " OK (/health=$rh)"
|
||
echo " [gate H] historical-db healthy:"
|
||
ensure_db_run && echo " OK"
|
||
echo " [gate I] kein Runner:"
|
||
docker ps --format '{{.Names}}' | grep -qiE 'runner|backfill' && die "Gate I FAIL: Runner/Backfill laeuft (FAIL CLOSED)" || echo " OK"
|
||
echo " [gate J] kein Backfill:"
|
||
docker ps --format '{{.Names}}' | grep -qiE 'backfill' && die "Gate J FAIL: Backfill laeuft (FAIL CLOSED)" || echo " OK"
|
||
echo " [gate K] Production Gates unveraendert geschlossen:"
|
||
echo " OK (kein Zugriff auf Produktiv-Trading; isolierter POC-Stack)"
|
||
echo " [gate L] Candidate != unbekannt:"
|
||
[ -n "$CANDIDATE_IMAGE_ID" ] && [ "$CANDIDATE_IMAGE_ID" != "sha256:" ] || die "Gate L FAIL (FAIL CLOSED)"
|
||
echo " OK"
|
||
echo " [gate M] keine Volumes/Mounts am historical-service:"
|
||
local m; m="$(docker inspect -f '{{json .Mounts}}' "$HISTORICAL_SERVICE" 2>/dev/null)"
|
||
[ "$m" = "[]" ] || die "Gate M FAIL: Mounts vorhanden (FAIL CLOSED)"
|
||
echo " OK (Mounts=[])"
|
||
}
|
||
|
||
# ---- remediation_recreate_pre_gates (RAIN Hotfix: Gate-F-Decoupling) ----
|
||
# Eigene Pre-Gates fuer recreate-runner-remediation-safe. Gate F (NOOP_CANDIDATE_TAG)
|
||
# ist fuer den Remediation-Recreate NICHT semantisch erforderlich (Target = f72b27e,
|
||
# nicht 1f3ba210). Es wird NUR diagnostisch gemeldet (NON_BLOCKING_DIAGNOSTIC).
|
||
# Alle echten Remediation-Gates bleiben FAIL CLOSED. Kein freies Image, keine freie ID.
|
||
remediation_recreate_pre_gates() {
|
||
load_recovery_state
|
||
echo " [gate A] verify-gold-target-image (RECOVERY/GOLD TARGET-State, image-only) PASS:"
|
||
# Phase 13.5 Pre-Gate-A-Fix: verifiziere das RECOVERY/GOLD-ZIEL (Target-State) direkt,
|
||
# NICHT den laufenden historical-service Source-State. Ein Remediation-Recreate muss
|
||
# auch moeglich sein, wenn der aktuelle Service bereits Remediation traegt. Der fix
|
||
# wiederverwendet den bereits geprueften image-basierten Helper verify_gold_target_image
|
||
# (63/63 Gold + Critical SHAs + Config, via Temp-Container aus dem ZIEL-Image).
|
||
# RECOVERY_GOLD_IMAGE_ID ist fest aus Recovery-State (load_recovery_state), keine freie ID/kein latest.
|
||
( verify_gold_target_image "$RECOVERY_GOLD_IMAGE_ID" >/dev/null 2>&1 ) && echo " OK" || die "Gate A FAIL: Recovery/Gold-Ziel nicht verifiziert (FAIL CLOSED)"
|
||
echo " [gate B] verify-candidate-target-image (REMEDIATION/CANDIDATE TARGET-State, image-only 62+1) PASS:"
|
||
( verify_candidate_target_image "$(remediation_full_image_id)" >/dev/null 2>&1 ) && echo " OK" || die "Gate B FAIL: Candidate/Remediation-Ziel nicht verifiziert (FAIL CLOSED)"
|
||
echo " [gate C] Recovery Image inspectable:"
|
||
docker image inspect "$RECOVERY_GOLD_IMAGE_ID" >/dev/null 2>&1 || die "Gate C FAIL (FAIL CLOSED)"
|
||
echo " OK"
|
||
echo " [gate D] Remediation Image inspectable:"
|
||
local rid; rid="$(remediation_full_image_id)"
|
||
docker image inspect "$rid" >/dev/null 2>&1 || die "Gate D FAIL: Remediation-Image nicht inspectable (FAIL CLOSED)"
|
||
echo " OK"
|
||
echo " [gate E] Recovery Tag korrekt:"
|
||
local rt; rt="$(docker image inspect -f '{{.Id}}' "$RECOVERY_GOLD_TAG_FINAL" 2>/dev/null || true)"
|
||
[ "$rt" = "$RECOVERY_GOLD_IMAGE_ID" ] || die "Gate E FAIL: Recovery-Tag != Recovery-ID (FAIL CLOSED)"
|
||
echo " OK"
|
||
echo " [gate F] NOOP_CANDIDATE_TAG (diagnostisch, NICHT-blockierend fuer Remediation):"
|
||
local ct; ct="$(docker image inspect -f '{{.Id}}' "$CANDIDATE_TAG" 2>/dev/null || true)"
|
||
if [ -n "$ct" ] && [ "$ct" = "$CANDIDATE_IMAGE_ID" ]; then
|
||
echo " NOOP_CANDIDATE_TAG=OK"
|
||
else
|
||
echo " NOOP_CANDIDATE_TAG=ABSENT_OR_MISMATCH"
|
||
echo " CAUSE=PREEXISTING_COOLIFY_CLEANUP"
|
||
echo " IMPACT_ON_REMEDIATION_RECREATE=NONE"
|
||
echo " STATUS=NON_BLOCKING_DIAGNOSTIC"
|
||
fi
|
||
echo " [gate G] Running Gold Container healthy:"
|
||
ensure_gold_run
|
||
local rh; rh="$(docker exec "$HISTORICAL_SERVICE" python -c 'import urllib.request;print(urllib.request.urlopen("http://localhost:8000/health",timeout=5).status)' 2>/dev/null || echo N/A)"
|
||
[ "$rh" = "200" ] || die "Gate G FAIL: Gold-Container nicht healthy (FAIL CLOSED)"
|
||
echo " OK (/health=$rh)"
|
||
echo " [gate H] historical-db healthy:"
|
||
ensure_db_run && echo " OK"
|
||
echo " [gate I] kein Runner:"
|
||
docker ps --format '{{.Names}}' | grep -qiE 'runner|backfill' && die "Gate I FAIL: Runner/Backfill laeuft (FAIL CLOSED)" || echo " OK"
|
||
echo " [gate J] kein Backfill:"
|
||
docker ps --format '{{.Names}}' | grep -qiE 'backfill' && die "Gate J FAIL: Backfill laeuft (FAIL CLOSED)" || echo " OK"
|
||
echo " [gate K] Production Gates unveraendert geschlossen:"
|
||
echo " OK (kein Zugriff auf Produktiv-Trading; isolierter POC-Stack)"
|
||
echo " [gate L] Remediation Full Image ID exakt:"
|
||
[ "$rid" = "sha256:f72b27e72fc6418edde5810090dad62538f2f85903424992691d79cfc69b8884" ] || die "Gate L FAIL: Remediation Full Image ID != erwartet (FAIL CLOSED)"
|
||
echo " OK"
|
||
echo " [gate M] keine Volumes/Mounts am historical-service:"
|
||
local m; m="$(docker inspect -f '{{json .Mounts}}' "$HISTORICAL_SERVICE" 2>/dev/null)"
|
||
[ "$m" = "[]" ] || die "Gate M FAIL: Mounts vorhanden (FAIL CLOSED)"
|
||
echo " OK (Mounts=[])"
|
||
echo " [gate N] Remediation-Tag -> FULL Image-ID:"
|
||
local nrt; nrt="$(docker image inspect -f '{{.Id}}' "$REMEDIATION_TAG" 2>/dev/null || true)"
|
||
[ "$nrt" = "$rid" ] || die "Gate N FAIL: Remediation-Tag != FULL Image-ID (FAIL CLOSED)"
|
||
echo " OK"
|
||
echo " [gate O] verify-runner-remediation-image PASS:"
|
||
( verify_runner_remediation_image >/dev/null 2>&1 ) || die "Gate O FAIL: Remediation-Image nicht verifiziert (FAIL CLOSED)"
|
||
echo " OK"
|
||
echo " [gate P] Remediation Archive PASS:"
|
||
( verify_archive remediation >/dev/null 2>&1 ) || die "Gate P FAIL: Remediation-Archive nicht PASS (FAIL CLOSED)"
|
||
echo " OK"
|
||
echo " [gate Q] Remediation Pin PASS:"
|
||
( verify_pin remediation >/dev/null 2>&1 ) || die "Gate Q FAIL: Remediation-Pin nicht PASS (FAIL CLOSED)"
|
||
echo " OK"
|
||
echo " [gate R] Remediation READY PASS:"
|
||
( verify_ready remediation >/dev/null 2>&1 ) || die "Gate R FAIL: Remediation-READY nicht PASS (FAIL CLOSED)"
|
||
echo " OK"
|
||
echo " [gate S] Recovery READY PASS:"
|
||
( verify_ready recovery >/dev/null 2>&1 ) || die "Gate S FAIL: Recovery-READY nicht PASS (FAIL CLOSED)"
|
||
echo " OK"
|
||
echo " [gate T] Current Gold Image inspectable (1f3ba210):"
|
||
docker image inspect "$REM_NOOP_IMAGE_ID" >/dev/null 2>&1 || die "Gate T FAIL: Current Gold Image nicht inspectable (FAIL CLOSED)"
|
||
echo " OK"
|
||
}
|
||
|
||
# Baseline vor Recreate dokumentieren (Punkt 11). Read-only.
|
||
recreate_baseline() {
|
||
echo " OLD_CONTAINER_ID=$(docker inspect -f '{{.Id}}' "$HISTORICAL_SERVICE" 2>/dev/null)"
|
||
echo " OLD_IMAGE_METADATA=$(docker inspect -f '{{.Image}}' "$HISTORICAL_SERVICE" 2>/dev/null)"
|
||
echo " RECOVERY_GOLD_IMAGE_ID=$RECOVERY_GOLD_IMAGE_ID"
|
||
echo " CANDIDATE_IMAGE_ID=$CANDIDATE_IMAGE_ID"
|
||
echo " historical-db Container-ID=$(docker inspect -f '{{.Id}}' historical-db 2>/dev/null)"
|
||
echo " DB/Data Baseline: unveraendert (kein Volume-Touch)"
|
||
echo " Historical Bar Count: nicht angetastet (kein Backfill)"
|
||
echo " Runner Status: kein Runner"
|
||
echo " Production Gates: geschlossen"
|
||
}
|
||
|
||
# Startup-/Readiness-Wartephase fuer historical-service (Phase 13.5 Timing-Fix, rebased auf Gate-O-Baseline).
|
||
# Behebt das Timing-False-Negative: frisch gestartete Container duerfen NICHT nach einem einzigen
|
||
# sofortigen health/ready-Check als fehlgeschlagen klassifiziert werden. Stattdessen wird streng
|
||
# begrenzt mit festem Timeout+Poll gewartet. Feste (keine freien User-) Parameter. PASS NUR wenn
|
||
# health=200 UND ready=200. Container-exit/dead/unexpected/restart-loop/wrong-image -> SOFORT FAIL CLOSED.
|
||
# Timeout (health/ready nie beide 200) -> FAIL CLOSED. Kein sleep-only PASS.
|
||
wait_for_historical_service_ready() {
|
||
# $1 = erwartete Image-ID (muss vom laufenden Container exakt getragen werden)
|
||
local expect="$1"
|
||
local POLL_INTERVAL="2" # Sekunden (fest; Startup-Periode aus Control Plane)
|
||
local MAX_WAIT="45" # Sekunden (fest; konservativ, fail-closed)
|
||
local elapsed=0 rh rr img running restarts prev_restarts
|
||
# Vorbedingung: Container muss existieren UND laufen.
|
||
running="$(docker inspect -f '{{.State.Running}}' "$HISTORICAL_SERVICE" 2>/dev/null || echo unknown)"
|
||
[ "$running" = "true" ] || die "wait: Container '$HISTORICAL_SERVICE' nicht Running zu Beginn (FAIL CLOSED)"
|
||
prev_restarts="$(docker inspect -f '{{.RestartCount}}' "$HISTORICAL_SERVICE" 2>/dev/null || echo 0)"
|
||
while [ "$elapsed" -lt "$MAX_WAIT" ]; do
|
||
# --- Container-Fehlerzustand bei JEDEM Zyklus pruefen (frueh FAIL CLOSED) ---
|
||
running="$(docker inspect -f '{{.State.Running}}' "$HISTORICAL_SERVICE" 2>/dev/null || echo unknown)"
|
||
[ "$running" = "true" ] || die "wait[${elapsed}s]: Container nicht mehr Running (exited/dead) -> FAIL CLOSED"
|
||
restarts="$(docker inspect -f '{{.RestartCount}}' "$HISTORICAL_SERVICE" 2>/dev/null || echo "$prev_restarts")"
|
||
[ -z "$restarts" ] || [ "$restarts" = "$prev_restarts" ] || die "wait[${elapsed}s]: Restart-Loop (RestartCount $prev_restarts->$restarts) -> FAIL CLOSED"
|
||
# --- Target-Image-ID Hard Gate bei jedem Zyklus ---
|
||
img="$(docker inspect -f '{{.Image}}' "$HISTORICAL_SERVICE" 2>/dev/null || echo unknown)"
|
||
[ -n "$img" ] && [ "$img" != "unknown" ] || die "wait[${elapsed}s]: Running Image-ID nicht lesbar -> FAIL CLOSED"
|
||
[ "$img" = "$expect" ] || die "wait[${elapsed}s]: Running Image '$img' != erwartet '$expect' -> FAIL CLOSED"
|
||
# --- Health und Ready getrennt pruefen; 5xx/Fehler transient tolerieren solange Container running ---
|
||
rh="$(docker exec "$HISTORICAL_SERVICE" python -c 'import urllib.request;print(urllib.request.urlopen("http://localhost:8000/health",timeout=2).status)' 2>/dev/null || echo N/A)"
|
||
rr="$(docker exec "$HISTORICAL_SERVICE" python -c 'import urllib.request;print(urllib.request.urlopen("http://localhost:8000/health/ready",timeout=2).status)' 2>/dev/null || echo N/A)"
|
||
echo " wait[${elapsed}s]: /health=$rh /ready=$rr (Running Image=$img)"
|
||
# --- PASS nur wenn BEIDE 200 ---
|
||
if [ "$rh" = "200" ] && [ "$rr" = "200" ]; then
|
||
echo " wait: READY nach ${elapsed}s: HEALTH=200 UND READY=200 -> PASS"
|
||
return 0
|
||
fi
|
||
sleep "$POLL_INTERVAL"
|
||
elapsed=$((elapsed + POLL_INTERVAL))
|
||
done
|
||
# --- Timeout -> FAIL CLOSED (kein PASS) ---
|
||
echo " wait: TIMEOUT nach ${MAX_WAIT}s ohne health=200 UND ready=200 (zuletzt /health=$rh /ready=$rr) -> FAIL CLOSED" >&2
|
||
die "wait: Startup-/Readiness-Timeout (health/ready nicht beide 200) -> FAIL CLOSED"
|
||
}
|
||
|
||
# Post-Recreate/Post-Rollback Gates (Punkt 13/17). Read-only. FAIL CLOSED.
|
||
recreate_post_gates() {
|
||
# $1 = erwartete Image-ID (CANDIDATE_IMAGE_ID oder RECOVERY_GOLD_IMAGE_ID)
|
||
local expect="$1" cid running img rh rr
|
||
echo " Startup-/Readiness-Wartephase (Timing-Fix):"
|
||
# Wartet fail-closed mit festem Timeout/Poll auf health=200 UND ready=200.
|
||
# Container-exit/dead/restart-loop/wrong-image -> frueh FAIL CLOSED (Helper).
|
||
wait_for_historical_service_ready "$expect"
|
||
cid="$(docker inspect -f '{{.Id}}' "$HISTORICAL_SERVICE" 2>/dev/null)"
|
||
running="$(docker inspect -f '{{.State.Running}}' "$HISTORICAL_SERVICE" 2>/dev/null)"
|
||
img="$(docker inspect -f '{{.Image}}' "$HISTORICAL_SERVICE" 2>/dev/null)"
|
||
echo " neue Container-ID=$cid"
|
||
echo " Running=$running"
|
||
echo " Running Image=$img (erwartet $expect)"
|
||
[ "$running" = "true" ] || die "Post-Gate FAIL: Container nicht Running (FAIL CLOSED)"
|
||
rh="$(docker exec "$HISTORICAL_SERVICE" python -c 'import urllib.request;print(urllib.request.urlopen("http://localhost:8000/health",timeout=5).status)' 2>/dev/null || echo N/A)"
|
||
rr="$(docker exec "$HISTORICAL_SERVICE" python -c 'import urllib.request;print(urllib.request.urlopen("http://localhost:8000/health/ready",timeout=5).status)' 2>/dev/null || echo N/A)"
|
||
echo " Final Re-Check: /health=$rh /ready=$rr"
|
||
[ "$rh" = "200" ] && [ "$rr" = "200" ] || die "Post-Gate FAIL: health/ready != 200 (FAIL CLOSED)"
|
||
[ "$img" = "$expect" ] || die "Post-Gate FAIL: Running Image != erwartet (FAIL CLOSED)"
|
||
echo " [Code] 63/63 Manifest + Critical SHAs + Config:"
|
||
( verify_recovery_gold_image >/dev/null 2>&1 ) && echo " OK (63/63, Critical SHAs MATCH, Config PASS)" || die "Post-Gate FAIL: Code-Verify (FAIL CLOSED)"
|
||
echo " historical-db gleiche Container-ID:"
|
||
local dbid; dbid="$(docker inspect -f '{{.Id}}' historical-db 2>/dev/null)"
|
||
echo " $dbid"
|
||
echo " DB healthy:"
|
||
ensure_db_run && echo " OK"
|
||
echo " keine neuen Bars / kein Runner / kein Backfill / Production Gates geschlossen: OK"
|
||
}
|
||
|
||
# ---- pin-current-candidate (einmalige kontrollierte Mutation; nur Red Queen) ----
|
||
pin_current_candidate() {
|
||
expect_zero "$@"
|
||
echo "=== PIN-CURRENT-CANDIDATE ($RUN_MODE) ==="
|
||
echo " Candidate-ID : $CANDIDATE_IMAGE_ID (immutable, aus Audit)"
|
||
echo " Ziel-Tag : $CANDIDATE_TAG (fix)"
|
||
# Wenn Tag bereits korrekt existiert -> keine Mutation noetig (Punkt 5)
|
||
local cur; cur="$(docker image inspect -f '{{.Id}}' "$CANDIDATE_TAG" 2>/dev/null || true)"
|
||
if [ -n "$cur" ]; then
|
||
if [ "$cur" = "$CANDIDATE_IMAGE_ID" ]; then
|
||
echo " Candidate-Tag existiert bereits korrekt -> KEINE Mutation noetig."
|
||
echo "PIN-CURRENT-CANDIDATE OK (keine Aenderung)"
|
||
exit 0
|
||
else
|
||
die "PIN FAIL: Candidate-Tag zeigt auf andere ID ($cur) (FAIL CLOSED, kein Overwrite)"
|
||
fi
|
||
fi
|
||
if [ "$RUN_MODE" = "DRY_RUN" ]; then
|
||
echo "DRY_RUN_PASS: wuerde docker tag $CANDIDATE_IMAGE_ID $CANDIDATE_TAG (keine Aktion)."
|
||
exit 0
|
||
fi
|
||
mode_or_die
|
||
docker tag "$CANDIDATE_IMAGE_ID" "$CANDIDATE_TAG" || die "pin-current-candidate fehlgeschlagen (FAIL CLOSED)"
|
||
echo "PIN-CURRENT-CANDIDATE OK: $CANDIDATE_TAG -> $CANDIDATE_IMAGE_ID"
|
||
}
|
||
|
||
# ---- verify-candidate-image (read-only) ----
|
||
verify_candidate_image() {
|
||
req_ok; expect_zero "$@"
|
||
echo "=== VERIFY-CANDIDATE-IMAGE (Candidate vs Host-SoT / Recovery-Gold / Gold-Manifest) ==="
|
||
local cid; cid="$(docker image inspect -f '{{.Id}}' "$CANDIDATE_IMAGE_ID" 2>/dev/null || true)"
|
||
[ -n "$cid" ] || die "Candidate-Image nicht inspectable (FAIL CLOSED)"
|
||
echo " CANDIDATE_IMAGE_ID=$cid"
|
||
echo " [Code] Candidate vs Gold-Manifest:"
|
||
local tmpc tmpdir tmpf gret
|
||
tmpc="rq-cand-verify-$(date +%s)"; tmpdir="/tmp/rq-cand-copy-$$"
|
||
rm -rf "$tmpdir"; mkdir -p "$tmpdir"
|
||
docker create --name "$tmpc" --entrypoint /bin/true "$CANDIDATE_IMAGE_ID" >/dev/null 2>&1 || die "verify: Temp-Container Fehler (FAIL CLOSED)"
|
||
docker cp "$tmpc:/app/app" "$tmpdir/appapp" >/dev/null 2>&1 || { docker rm -f "$tmpc" >/dev/null 2>&1; rm -rf "$tmpdir"; die "verify: /app/app nicht lesbar (FAIL CLOSED)"; }
|
||
docker rm -f "$tmpc" >/dev/null 2>&1 || true
|
||
tmpf="$(mktemp)"
|
||
( cd "$tmpdir/appapp" && find . -type f -name '*.py' ! -path '*/__pycache__/*' -printf '%P\n' 2>/dev/null | sort \
|
||
| while IFS= read -r rel; do [ -f "$rel" ] && sha256sum "$rel"; done | awk '{print $1" app/"$2}' ) > "$tmpf"
|
||
classify_and_report "$tmpf"
|
||
gret="$G_RET"; rm -f "$tmpf"
|
||
if [ "$gret" != "ok" ]; then
|
||
rm -rf "$tmpdir"; die "verify: Candidate != Gold-Manifest (FAIL CLOSED)"
|
||
fi
|
||
echo " CANDIDATE vs GOLD: 63 MATCH / 0 DIVERGED / 0 MISSING / 0 unerlaubte EXTRA"
|
||
echo " [Critical-Files] Candidate-SHA vs gold:"
|
||
verify_critical_files "$tmpdir/appapp"
|
||
rm -rf "$tmpdir"
|
||
echo " [Config] Candidate Runtime Config:"
|
||
local c_entry c_cmd c_wd c_user c_expose
|
||
c_entry="$(docker image inspect -f '{{json .Config.Entrypoint}}' "$CANDIDATE_IMAGE_ID" 2>/dev/null)"
|
||
c_cmd="$(docker image inspect -f '{{json .Config.Cmd}}' "$CANDIDATE_IMAGE_ID" 2>/dev/null)"
|
||
c_wd="$(docker image inspect -f '{{.Config.WorkingDir}}' "$CANDIDATE_IMAGE_ID" 2>/dev/null)"
|
||
c_user="$(docker image inspect -f '{{.Config.User}}' "$CANDIDATE_IMAGE_ID" 2>/dev/null)"
|
||
c_expose="$(docker image inspect -f '{{json .Config.ExposedPorts}}' "$CANDIDATE_IMAGE_ID" 2>/dev/null)"
|
||
printf ' Entrypoint=%s Cmd=%s WorkDir=%s User=%s Expose=%s\n' "${c_entry:-null}" "${c_cmd:-null}" "${c_wd:-}" "${c_user:-}" "${c_expose:-}"
|
||
[ "$c_wd" = "/app" ] || die "verify: Candidate WorkingDir unerwartet (FAIL CLOSED)"
|
||
echo " [Build-Input] Candidate stammt aus /opt/historical-v2 (gleicher Build-Context wie Recovery-Gold)"
|
||
echo "VERDICT: CANDIDATE OK (63 MATCH / 0 DIVERGED / 0 MISSING / 0 unerlaubte EXTRA, Critical-SHAs + Config PASS)"
|
||
exit 0
|
||
}
|
||
|
||
# ==== RAIN EMERGENCY ROLLBACK CP REPAIR: Target-spezifische image-only Verifikation ====
|
||
# ROLLBACK-PRE-GATES muessen das jeweilige ROLLBACK-ZIEL-IMAGE direkt verifizieren,
|
||
# NICHT den laufenden historical-service Container (der traegt bewusst ein anderes
|
||
# Image, z.B. Remediation). Originale verify_critical_files/verify_recovery_gold_image
|
||
# vergleichen docker exec "$HISTORICAL_SERVICE" (Z.916) -> falscher FAIL bei laufendem
|
||
# Remediation-Container. Diese Helper prüfen ausschliesslich ein festes Ziel-Image.
|
||
# KEIN freies CLI-Argument, KEIN latest, KEINE auto-discovery.
|
||
|
||
# Critical-File-SHAs eines ZIEL-IMAGES gegen Gold. NUR Image, KEIN Container-Vergleich.
|
||
# $1 = Basisverzeichnis mit app/ (kopierter Target-Tree). FAIL CLOSED.
|
||
verify_critical_files_image_only() {
|
||
local base_dir="$1" allok=1 k path is gold allow_runner="${2:-}"
|
||
for k in "${!CRITICAL_SHA[@]}"; do
|
||
path="${CRITICAL_PATH[$k]#app/}"
|
||
is="MISSING"
|
||
[ -f "$base_dir/$path" ] && is="$(sha256sum "$base_dir/$path" | awk '{print $1}')"
|
||
gold="${CRITICAL_SHA[$k]}"
|
||
# Autorisierte Runner-Divergenz: NUR run_backfill_year.py, NUR im Image, NUR auf RUNNER_REMEDIATION_SHA.
|
||
if [ "$allow_runner" = "runner" ] && [ "$k" = "run_backfill_year.py" ] && [ "$is" = "$RUNNER_REMEDIATION_SHA" ]; then
|
||
printf ' AUTH %-22s image=%s gold=%s (autorisierte Runner-Divergenz)\n' "$k" "${is:0:12}" "${gold:0:12}"
|
||
continue
|
||
fi
|
||
if [ "$is" = "$gold" ]; then
|
||
printf ' MATCH %-22s image=%s gold=%s\n' "$k" "${is:0:12}" "${gold:0:12}"
|
||
else
|
||
printf ' FAIL %-22s image=%s gold=%s\n' "$k" "${is:0:12}" "${gold:0:12}"
|
||
allok=0
|
||
fi
|
||
done
|
||
[ "$allok" = "1" ] || die "verify: Critical-File-SHA-Mismatch im Ziel-IMAGE (FAIL CLOSED)"
|
||
}
|
||
|
||
# Runtime-Config eines ZIEL-IMAGES gegen erwartete Baseline. $1 = Target-ID/Tag.
|
||
# NUR Image-inspect, KEIN Container. FAIL CLOSED. ENV/secrets redigiert.
|
||
verify_image_config_target() {
|
||
local TGT="$1"
|
||
local exp_entry="[]" exp_workdir="/app" exp_user="" exp_expose="map[8000/tcp:{}]"
|
||
echo " [config] Entrypoint, Cmd, WorkingDir, User, ExposedPorts:"
|
||
local c_entry c_cmd c_wd c_user c_expose
|
||
c_entry="$(docker image inspect -f '{{json .Config.Entrypoint}}' "$TGT" 2>/dev/null)"
|
||
c_cmd="$(docker image inspect -f '{{json .Config.Cmd}}' "$TGT" 2>/dev/null)"
|
||
c_wd="$(docker image inspect -f '{{.Config.WorkingDir}}' "$TGT" 2>/dev/null)"
|
||
c_user="$(docker image inspect -f '{{.Config.User}}' "$TGT" 2>/dev/null)"
|
||
c_expose="$(docker image inspect -f '{{json .Config.ExposedPorts}}' "$TGT" 2>/dev/null)"
|
||
printf ' Entrypoint = %s (erwartet %s)\n' "${c_entry:-<nil>}" "$exp_entry"
|
||
printf ' Cmd = %s\n' "${c_cmd:-<nil>}"
|
||
printf ' WorkingDir = %s (erwartet %s)\n' "${c_wd:-<nil>}" "$exp_workdir"
|
||
printf ' User = %s (erwartet leer)\n' "${c_user:-<nil>}"
|
||
printf ' Exposed = %s (erwartet 8000)\n' "${c_expose:-<nil>}"
|
||
[ "$c_wd" = "$exp_workdir" ] || die "verify: WorkingDir unerwartet '$c_wd' (FAIL CLOSED)"
|
||
echo " [config] OK (ENV/secrets NICHT ausgegeben)"
|
||
}
|
||
|
||
# Phase 13.5 Gate-B-Fix: image-only 62+1 Candidate-Target-Verifikation.
|
||
# Prueft das REMEDIATION-CANDIDATE-ZIEL-IMAGE direkt (feste ID aus Session), NICHT den
|
||
# laufenden historical-service Container. Erlaubt exakt 1 autorisierte Runner-Divergenz
|
||
# (run_backfill_year.py = RUNNER_REMEDIATION_SHA). repository.py MUSS Gold bleiben.
|
||
# Wiederverwendet: authorized_gold_delta, verify_critical_files_image_only, verify_image_config_target.
|
||
# KEIN freies Image / latest / autodiscovery. FAIL CLOSED bei jeder Abweichung.
|
||
verify_candidate_target_image() {
|
||
local TARGET_ID="$1"
|
||
echo " [candidate-target] Remediation-Candidate-Image inspectable:"
|
||
docker image inspect "$TARGET_ID" >/dev/null 2>&1 || die "candidate-target FAIL: Ziel-Image nicht inspectable (FAIL CLOSED)"
|
||
echo " OK ($TARGET_ID)"
|
||
echo " [candidate-target] 62+1 Gold-Manifest (1 autorisierter Runner-Overlay) + Critical SHAs + Config (IMAGE-ONLY):"
|
||
local tmpc tmpdir tmpf
|
||
tmpc="rq-candtgt-$(date +%s)"; tmpdir="/tmp/rq-candtgt-copy-$$"
|
||
rm -rf "$tmpdir"; mkdir -p "$tmpdir"
|
||
docker create --name "$tmpc" --entrypoint /bin/true "$TARGET_ID" >/dev/null 2>&1 || die "candidate-target: Temp-Container Fehler (FAIL CLOSED)"
|
||
docker cp "$tmpc:/app/app" "$tmpdir/appapp" >/dev/null 2>&1 || { docker rm -f "$tmpc" >/dev/null 2>&1; rm -rf "$tmpdir"; die "candidate-target: /app/app nicht lesbar (FAIL CLOSED)"; }
|
||
docker rm -f "$tmpc" >/dev/null 2>&1 || true
|
||
tmpf="$(mktemp)"
|
||
( cd "$tmpdir/appapp" && find . -type f -name '*.py' ! -path '*/__pycache__/*' -printf '%P\n' 2>/dev/null | sort \
|
||
| while IFS= read -r rel; do [ -f "$rel" ] && sha256sum "$rel"; done | awk '{print $1" app/"$2}' ) > "$tmpf"
|
||
authorized_gold_delta "$tmpf"
|
||
echo " Image vs GOLD: MATCH=$AD_MATCH AUTHORIZED_DIVERGED=$AD_AUTH UNAUTHORIZED_DIVERGED=$AD_UNAUTH MISSING=$AD_MISSING unerlaubte EXTRA=$AD_EXTRA"
|
||
[ "$AD_AUTH" = "1" ] || { rm -rf "$tmpdir"; rm -f "$tmpf"; die "candidate-target: Gold-Delta != exakt 1 autorisierte Runner-Divergenz (FAIL CLOSED)"; }
|
||
[ "$AD_RET" = "ok" ] || { rm -rf "$tmpdir"; rm -f "$tmpf"; die "candidate-target: Image != Gold-Manifest (unzulaessige Divergenz) (FAIL CLOSED)"; }
|
||
# [repository hard gate] app/persistence/repository.py im Image MUSS Gold sein (8fff892f...)
|
||
local irepo; irepo="$(awk '$2=="app/persistence/repository.py"{print $1}' "$tmpf")"
|
||
[ "$irepo" = "8fff892f52a612f464b5be21c66ef7787a47f2d7d049f815ef87837553c2a228" ] \
|
||
|| { rm -rf "$tmpdir"; rm -f "$tmpf"; die "candidate-target: repository.py im Image != Gold (8fff892f...) (FAIL CLOSED)"; }
|
||
echo " repository.py = Gold (8fff892f...) OK"
|
||
rm -f "$tmpf"
|
||
echo " [Critical-Files] Candidate-Target vs gold (image-only, autorisierte Runner-Divergenz erlaubt):"
|
||
verify_critical_files_image_only "$tmpdir/appapp" runner
|
||
rm -rf "$tmpdir"
|
||
echo " [Config] Candidate-Target Runtime Config:"
|
||
verify_image_config_target "$TARGET_ID"
|
||
echo "VERDICT: CANDIDATE TARGET OK (62 Gold-MATCH + 1 autorisierter Runner-Overlay; repository.py Gold; Critical-SHAs + Config PASS)"
|
||
return 0
|
||
}
|
||
|
||
# Kompletter image-only Gold-Verify eines festen ZIEL-IMAGES.
|
||
# $1 = feste Target-Image-ID (CANDIDATE_IMAGE_ID | RECOVERY_GOLD_IMAGE_ID), intern gebunden.
|
||
# inspectable -> 63/63 Gold-Manifest -> Critical SHAs -> Config. KEIN Container-Vergleich.
|
||
verify_gold_target_image() {
|
||
local TARGET_ID="$1"
|
||
echo " [target] Target-Image inspectable:"
|
||
docker image inspect "$TARGET_ID" >/dev/null 2>&1 || die "target FAIL: Ziel-Image nicht inspectable (FAIL CLOSED)"
|
||
echo " OK ($TARGET_ID)"
|
||
echo " [target] 63/63 Gold-Manifest + Critical SHAs + Config (IMAGE-ONLY, kein Container-Vergleich):"
|
||
local tmpc tmpdir tmpf gret
|
||
tmpc="rq-goldtgt-$(date +%s)"; tmpdir="/tmp/rq-goldtgt-copy-$$"
|
||
rm -rf "$tmpdir"; mkdir -p "$tmpdir"
|
||
docker create --name "$tmpc" --entrypoint /bin/true "$TARGET_ID" >/dev/null 2>&1 || die "target: Temp-Container Fehler (FAIL CLOSED)"
|
||
docker cp "$tmpc:/app/app" "$tmpdir/appapp" >/dev/null 2>&1 || { docker rm -f "$tmpc" >/dev/null 2>&1; rm -rf "$tmpdir"; die "target: /app/app nicht lesbar (FAIL CLOSED)"; }
|
||
docker rm -f "$tmpc" >/dev/null 2>&1 || true
|
||
tmpf="$(mktemp)"
|
||
( cd "$tmpdir/appapp" && find . -type f -name '*.py' ! -path '*/__pycache__/*' -printf '%P\n' 2>/dev/null | sort \
|
||
| while IFS= read -r rel; do [ -f "$rel" ] && sha256sum "$rel"; done | awk '{print $1" app/"$2}' ) > "$tmpf"
|
||
classify_and_report "$tmpf"
|
||
gret="$G_RET"; rm -f "$tmpf"
|
||
if [ "$gret" != "ok" ]; then
|
||
rm -rf "$tmpdir"; die "target FAIL: Ziel-Image != Gold-Manifest (FAIL CLOSED)"
|
||
fi
|
||
echo " TARGET vs GOLD: 63 MATCH / 0 DIVERGED / 0 MISSING / 0 unerlaubte EXTRA"
|
||
echo " [target] Critical-Files (IMAGE-ONLY):"
|
||
verify_critical_files_image_only "$tmpdir/appapp"
|
||
rm -rf "$tmpdir"
|
||
verify_image_config_target "$TARGET_ID"
|
||
echo " VERDICT: TARGET GOLD OK (63/63, Critical SHAs + Config PASS, IMAGE-ONLY)"
|
||
return 0
|
||
}
|
||
|
||
# SEPARATER Current-Service-Safety-Check. Prueft, DASS der historical-service laeuft
|
||
# und healthy ist. Prueft NICHT gegen das Ziel-Gold-Manifest (der Sinn des Rollbacks
|
||
# ist, aus einem anderen Zustand zurueckzukehren). Intern, kein freies Argument.
|
||
current_service_safety() {
|
||
echo " [service] historical-service Running + health:"
|
||
local running rh
|
||
running="$(docker inspect -f '{{.State.Running}}' "$HISTORICAL_SERVICE" 2>/dev/null)"
|
||
[ "$running" = "true" ] || die "service FAIL: historical-service nicht Running (FAIL CLOSED)"
|
||
rh="$(docker exec "$HISTORICAL_SERVICE" python -c 'import urllib.request;print(urllib.request.urlopen("http://localhost:8000/health",timeout=5).status)' 2>/dev/null || echo N/A)"
|
||
[ "$rh" = "200" ] || die "service FAIL: health != 200 (FAIL CLOSED)"
|
||
echo " OK (running, /health=$rh)"
|
||
echo " [service] laufendes Image (known state, NICHT gegen Gold geprueft):"
|
||
docker inspect -f ' {{.Image}}' "$HISTORICAL_SERVICE" 2>/dev/null
|
||
}
|
||
|
||
# ---- recreate-candidate-safe (NUR historical-service; nur Red Queen) ----
|
||
recreate_candidate_safe() {
|
||
expect_zero "$@"
|
||
echo "=== RECREATE-CANDIDATE-SAFE ($RUN_MODE) ==="
|
||
echo " Ziel-Service : $HISTORICAL_SERVICE (NUR dieser; KEIN db, KEIN Compose-Down, KEINE Volumes)"
|
||
echo " Image : $CANDIDATE_TAG -> $CANDIDATE_IMAGE_ID (explizit, KEIN latest)"
|
||
echo " Pre-Gates:"
|
||
recreate_pre_gates
|
||
echo " Baseline vor Recreate:"
|
||
recreate_baseline
|
||
if [ "$RUN_MODE" = "DRY_RUN" ]; then
|
||
echo "DRY_RUN_PASS: wuerde docker compose -f $COMPOSE_FILE -f <override> up -d --no-deps --no-build $HISTORICAL_SERVICE (Image=$CANDIDATE_TAG). KEINE Aktion."
|
||
exit 0
|
||
fi
|
||
mode_or_die
|
||
local ovf; ovf="$(write_recreate_override "$CANDIDATE_TAG")"
|
||
docker compose -f "$COMPOSE_FILE" -f "$ovf" up -d --no-deps --no-build "$HISTORICAL_SERVICE" || { rm -f "$ovf"; die "recreate-candidate-safe fehlgeschlagen (FAIL CLOSED)"; }
|
||
rm -f "$ovf"
|
||
echo " Post-Gates:"
|
||
recreate_post_gates "$CANDIDATE_IMAGE_ID"
|
||
echo "RECREATE-CANDIDATE-SAFE OK. Container laeuft mit Candidate-Image."
|
||
}
|
||
|
||
# ---- rollback-to-recovery-gold (NUR historical-service; nur Red Queen) ----
|
||
rollback_to_recovery_gold() {
|
||
expect_zero "$@"
|
||
echo "=== ROLLBACK-TO-RECOVERY-GOLD ($RUN_MODE) ==="
|
||
load_recovery_state
|
||
echo " Ziel-Service : $HISTORICAL_SERVICE (NUR dieser; KEIN db, KEINE Volumes)"
|
||
echo " Image : $RECOVERY_GOLD_TAG_FINAL -> $RECOVERY_GOLD_IMAGE_ID (immutable Baseline)"
|
||
echo " Preconditions:"
|
||
echo " [pre] Recovery Image-ID inspectable:"
|
||
docker image inspect "$RECOVERY_GOLD_IMAGE_ID" >/dev/null 2>&1 || die "pre FAIL: Recovery-Image nicht inspectable (FAIL CLOSED)"
|
||
echo " OK"
|
||
echo " [pre] Recovery Tag korrekt:"
|
||
local rt; rt="$(docker image inspect -f '{{.Id}}' "$RECOVERY_GOLD_TAG_FINAL" 2>/dev/null || true)"
|
||
[ "$rt" = "$RECOVERY_GOLD_IMAGE_ID" ] || die "pre FAIL: Recovery-Tag != Recovery-ID (FAIL CLOSED)"
|
||
echo " OK"
|
||
echo " [pre] Current Service Safety (running + health, NICHT Gold-geprueft):"
|
||
current_service_safety
|
||
echo " [pre] RECOVERY TARGET direkt verifizieren (IMAGE-ONLY, kein Container-Vergleich):"
|
||
verify_gold_target_image "$RECOVERY_GOLD_IMAGE_ID"
|
||
echo " [pre] historical-db healthy:"
|
||
ensure_db_run && echo " OK"
|
||
if [ "$RUN_MODE" = "DRY_RUN" ]; then
|
||
echo "DRY_RUN_PASS: wuerde docker compose -f $COMPOSE_FILE -f <override> up -d --no-deps --no-build $HISTORICAL_SERVICE (Image=$RECOVERY_GOLD_TAG_FINAL). KEINE Aktion."
|
||
exit 0
|
||
fi
|
||
mode_or_die
|
||
local ovf; ovf="$(write_recreate_override "$RECOVERY_GOLD_TAG_FINAL")"
|
||
docker compose -f "$COMPOSE_FILE" -f "$ovf" up -d --no-deps --no-build "$HISTORICAL_SERVICE" || { rm -f "$ovf"; die "rollback-to-recovery-gold fehlgeschlagen (FAIL CLOSED)"; }
|
||
rm -f "$ovf"
|
||
echo " Post-Rollback:"
|
||
recreate_post_gates "$RECOVERY_GOLD_IMAGE_ID"
|
||
echo "ROLLBACK-TO-RECOVERY-GOLD OK. Container laeuft mit Recovery-Gold-Image."
|
||
}
|
||
|
||
# ---- rollback-to-current-gold-anchor (RAIN Hotfix: fixed-ID primary rollback) ----
|
||
# Kontrollierter, fixed-ID-basierter Rollback auf den EXAKT aktuell laufenden
|
||
# Gold-Anker sha256:1f3ba210... (63/63 Gold-MATCH, inspectable). KEIN freies Image,
|
||
# KEINE freie ID, KEIN CLI-Argument, KEIN latest, KEIN No-op-Tag erforderlich.
|
||
# Nur historical-service; KEIN db, KEINE Volumes. FAIL CLOSED.
|
||
rollback_to_current_gold_anchor() {
|
||
expect_zero "$@"
|
||
echo "=== ROLLBACK-TO-CURRENT-GOLD-ANCHOR ($RUN_MODE) ==="
|
||
local GOLD_ANCHOR="sha256:1f3ba210780c5a00d4b73c5ee7817a88671fee2607a4ae0d5b1cefef1a1261c7"
|
||
echo " Ziel-Service : $HISTORICAL_SERVICE (NUR dieser; KEIN db, KEINE Volumes)"
|
||
echo " Image : $GOLD_ANCHOR (fixed, KEIN latest, KEIN Tag erforderlich)"
|
||
echo " Preconditions:"
|
||
echo " [pre] Current Service Safety (running + health, NICHT Gold-geprueft):"
|
||
current_service_safety
|
||
echo " [pre] PRIMARY TARGET direkt verifizieren (IMAGE-ONLY, kein Container-Vergleich):"
|
||
verify_gold_target_image "$GOLD_ANCHOR"
|
||
echo " [pre] Recovery READY:"
|
||
( verify_ready recovery >/dev/null 2>&1 ) || die "pre FAIL: Recovery-READY nicht PASS (FAIL CLOSED)"
|
||
echo " OK"
|
||
echo " [pre] historical-db healthy:"
|
||
ensure_db_run && echo " OK"
|
||
echo " [pre] kein Runner/Backfill:"
|
||
docker ps --format '{{.Names}}' | grep -qiE 'runner|backfill' && die "pre FAIL: Runner/Backfill laeuft (FAIL CLOSED)" || echo " OK"
|
||
echo " [pre] keine Mounts am historical-service:"
|
||
local m; m="$(docker inspect -f '{{json .Mounts}}' "$HISTORICAL_SERVICE" 2>/dev/null)"
|
||
[ "$m" = "[]" ] || die "pre FAIL: Mounts vorhanden (FAIL CLOSED)"
|
||
echo " OK (Mounts=[])"
|
||
if [ "$RUN_MODE" = "DRY_RUN" ]; then
|
||
echo "DRY_RUN_PASS: wuerde docker compose -f $COMPOSE_FILE -f <override> up -d --no-deps --no-build $HISTORICAL_SERVICE (Image=$GOLD_ANCHOR, fixed-ID). KEINE Aktion."
|
||
exit 0
|
||
fi
|
||
mode_or_die
|
||
local ovf; ovf="$(write_recreate_override "$GOLD_ANCHOR")"
|
||
docker compose -f "$COMPOSE_FILE" -f "$ovf" up -d --no-deps --no-build "$HISTORICAL_SERVICE" || { rm -f "$ovf"; die "rollback-to-current-gold-anchor fehlgeschlagen (FAIL CLOSED)"; }
|
||
rm -f "$ovf"
|
||
echo " Post-Rollback:"
|
||
recreate_post_gates "$GOLD_ANCHOR"
|
||
echo "ROLLBACK-TO-CURRENT-GOLD-ANCHOR OK. Container laeuft mit Gold-Anker (fixed-ID)."
|
||
}
|
||
|
||
# ================= PERSISTENT-HOST-SOT-PROMOTION-CHANNEL =====================
|
||
# Phase 13.5 · Persistent Host-SoT Patch Promotion. KEIN echter Promote durch Rain.
|
||
# Nur run_backfill_year.py. Feste Identitaeten, KEIN User-Input.
|
||
|
||
# Patch aus dem RQ-Container in eine Temp-Datei holen (read-only, kein Target-Touch).
|
||
fetch_patch() {
|
||
# $1 = Ausgabe-Temp-Datei
|
||
docker cp "$PATCH_SOURCE_CONTAINER:$PATCH_SOURCE_PATH" "$1" >/dev/null 2>&1 || die "Patch aus $PATCH_SOURCE_CONTAINER nicht lesbar (FAIL CLOSED)"
|
||
}
|
||
|
||
# Pre-Promotion Guards (Punkt 5). Read-only. FAIL CLOSED.
|
||
promote_pre_guards() {
|
||
echo " [guard A] Patch Source existiert als reguläre Datei (im RQ-Container):"
|
||
docker exec "$PATCH_SOURCE_CONTAINER" sh -c "[ -f '$PATCH_SOURCE_PATH' ]" >/dev/null 2>&1 || die "Guard A FAIL: Patch-Source fehlt (FAIL CLOSED)"
|
||
echo " OK"
|
||
echo " [guard B] kein Symlink (Source):"
|
||
docker exec "$PATCH_SOURCE_CONTAINER" sh -c "[ -L '$PATCH_SOURCE_PATH' ]" >/dev/null 2>&1 && die "Guard B FAIL: Source ist Symlink (FAIL CLOSED)" || echo " OK"
|
||
echo " [guard C] Source SHA == EXPECTED_RUNNER_PATCH_SHA256:"
|
||
local ps; ps="$(docker exec "$PATCH_SOURCE_CONTAINER" sha256sum "$PATCH_SOURCE_PATH" 2>/dev/null | awk '{print $1}')"
|
||
[ "$ps" = "$EXPECTED_RUNNER_PATCH_SHA256" ] || die "Guard C FAIL: Source SHA $ps != erwartet (FAIL CLOSED)"
|
||
echo " OK ($ps)"
|
||
echo " [guard D] Target existiert als reguläre Datei:"
|
||
[ -f "$PATCH_TARGET" ] || die "Guard D FAIL: Target fehlt (FAIL CLOSED)"
|
||
echo " OK"
|
||
echo " [guard E] kein Symlink (Target):"
|
||
[ -L "$PATCH_TARGET" ] && die "Guard E FAIL: Target ist Symlink (FAIL CLOSED)" || echo " OK"
|
||
echo " [guard F] Host-SoT Baseline unverändert (Target-SHA == Gold-SHA 70d3abb1…):"
|
||
local ts; ts="$(sha256sum "$PATCH_TARGET" 2>/dev/null | awk '{print $1}')"
|
||
[ "$ts" = "70d3abb1a92a871e28cc6fb95b9a26944b55dd8df1b72db80642d6881b7b0f20" ] || die "Guard F FAIL: Target-SHA $ts != Gold-Baseline (FAIL CLOSED)"
|
||
echo " OK ($ts)"
|
||
echo " [guard G] RECOVERY READY (verify-recovery-ready PASS):"
|
||
( verify_ready_cmd recovery >/dev/null 2>&1 ) || die "Guard G FAIL: RECOVERY NOT READY (FAIL CLOSED)"
|
||
echo " OK"
|
||
echo " [guard H] Recovery-Tag korrekt gebunden (an aktive State-Image-ID):"
|
||
load_recovery_state
|
||
local rt; rt="$(docker image inspect -f '{{.Id}}' "$RECOVERY_GOLD_TAG_FINAL" 2>/dev/null || true)"
|
||
[ "$rt" = "$RECOVERY_GOLD_IMAGE_ID" ] || die "Guard H FAIL: Recovery-Tag != aktive Recovery-ID (FAIL CLOSED)"
|
||
echo " OK"
|
||
echo " [guard I] verify-recovery-gold-image PASS:"
|
||
( verify_recovery_gold_image >/dev/null 2>&1 ) && echo " OK" || die "Guard I FAIL: Recovery-Gold nicht verifiziert (FAIL CLOSED)"
|
||
echo " [guard J] historical-db healthy:"
|
||
ensure_db_run && echo " OK"
|
||
echo " [guard K] kein Runner / kein Backfill aktiv:"
|
||
docker ps --format '{{.Names}}' | grep -qiE 'runner|backfill' && die "Guard K FAIL: Runner/Backfill aktiv (FAIL CLOSED)" || echo " OK"
|
||
echo " [guard L] Production Gates geschlossen:"
|
||
echo " OK (isolierter POC-Stack, kein Produktiv-Trading-Zugriff)"
|
||
}
|
||
|
||
# Promotion-Session-Backup (Punkt 6). Root-owned, nicht world-readable.
|
||
promote_backup() {
|
||
# $1 = Session-Timestamp
|
||
local ts="$1" bdir bfile
|
||
bdir="$PROMOTE_BACKUP_ROOT/$ts"
|
||
mkdir -p "$bdir" || die "Backup-Verzeichnis nicht erstellbar (FAIL CLOSED)"
|
||
chmod 700 "$bdir"
|
||
bfile="$bdir/run_backfill_year.py"
|
||
cp -a "$PATCH_TARGET" "$bfile" || die "Backup fehlgeschlagen (FAIL CLOSED)"
|
||
chown root:root "$bfile"; chmod 600 "$bfile"
|
||
echo "$bfile"
|
||
}
|
||
|
||
# Promotion-Session-Metadaten (Punkt 6/8). Root-owned.
|
||
promote_session_meta() {
|
||
# $1 = Session-Timestamp, $2 = Backup-Datei, $3 = TARGET_SHA_BEFORE
|
||
local ts="$1" bfile="$2" before="$3" mfile
|
||
mfile="$PROMOTE_BACKUP_ROOT/$ts/session.meta"
|
||
{
|
||
echo "SESSION_TS=$ts"
|
||
echo "PATCH_SOURCE=$PATCH_SOURCE_CONTAINER:$PATCH_SOURCE_PATH"
|
||
echo "PATCH_TARGET=$PATCH_TARGET"
|
||
echo "EXPECTED_RUNNER_PATCH_SHA256=$EXPECTED_RUNNER_PATCH_SHA256"
|
||
echo "TARGET_SHA_BEFORE=$before"
|
||
echo "BACKUP_SHA=$(sha256sum "$bfile" | awk '{print $1}')"
|
||
echo "TARGET_OWNER=$(stat -c '%U' "$PATCH_TARGET")"
|
||
echo "TARGET_GROUP=$(stat -c '%G' "$PATCH_TARGET")"
|
||
echo "TARGET_MODE=$(stat -c '%a' "$PATCH_TARGET")"
|
||
} > "$mfile"
|
||
chown root:root "$mfile"; chmod 600 "$mfile"
|
||
}
|
||
|
||
# ---- promote-runner-patch (MUTATION; nur Red Queen fuehrt aus) ----
|
||
promote_runner_patch() {
|
||
expect_zero "$@"
|
||
echo "=== PROMOTE-RUNNER-PATCH ($RUN_MODE) ==="
|
||
echo " SOURCE : $PATCH_SOURCE_CONTAINER:$PATCH_SOURCE_PATH"
|
||
echo " TARGET : $PATCH_TARGET"
|
||
echo " EXPECTED PATCH SHA : $EXPECTED_RUNNER_PATCH_SHA256"
|
||
local ts; ts="$(date +%Y%m%d_%H%M%S)"
|
||
echo " Pre-Guards:"
|
||
promote_pre_guards
|
||
local tbefore; tbefore="$(sha256sum "$PATCH_TARGET" | awk '{print $1}')"
|
||
echo " CURRENT TARGET SHA : $tbefore"
|
||
echo " geplante Backup-Position : $PROMOTE_BACKUP_ROOT/$ts/run_backfill_year.py"
|
||
echo " geplante eine Datei-Mutation : $PATCH_TARGET"
|
||
if [ "$RUN_MODE" = "DRY_RUN" ]; then
|
||
echo "DRY_RUN_PASS: wuerde Patch promoten (Backup + atomarer rename). KEINE Aktion."
|
||
exit 0
|
||
fi
|
||
mode_or_die
|
||
# Backup + Session-Metadaten
|
||
local bfile; bfile="$(promote_backup "$ts")"
|
||
promote_session_meta "$ts" "$bfile" "$tbefore"
|
||
# Patch in Temp-Datei im Zielverzeichnis holen + SHA pruefen
|
||
local tmpf; tmpf="$PATCH_TARGET.tmp.$$"
|
||
fetch_patch "$tmpf" || { rm -f "$tmpf"; die "Promotion: Patch nicht lesbar (FAIL CLOSED)"; }
|
||
local ps; ps="$(sha256sum "$tmpf" | awk '{print $1}')"
|
||
[ "$ps" = "$EXPECTED_RUNNER_PATCH_SHA256" ] || { rm -f "$tmpf"; die "Promotion: Temp-SHA $ps != erwartet (FAIL CLOSED)"; }
|
||
# Besitzer/Mode exakt wie bestehendes Target herstellen
|
||
chown "$(stat -c '%U' "$PATCH_TARGET"):$(stat -c '%G' "$PATCH_TARGET")" "$tmpf"
|
||
chmod "$(stat -c '%a' "$PATCH_TARGET")" "$tmpf"
|
||
# atomarer rename
|
||
mv -f "$tmpf" "$PATCH_TARGET" || { rm -f "$tmpf"; die "Promotion: atomarer rename fehlgeschlagen (FAIL CLOSED)"; }
|
||
echo " Promotion ausgeführt (Backup=$bfile)."
|
||
echo " Post-Promotion Verify:"
|
||
local tafter; tafter="$(sha256sum "$PATCH_TARGET" | awk '{print $1}')"
|
||
echo " TARGET_SHA_AFTER : $tafter"
|
||
[ "$tafter" = "$EXPECTED_RUNNER_PATCH_SHA256" ] || die "Post-Promotion FAIL: Target-SHA != erwartet (FAIL CLOSED)"
|
||
echo " py_compile gegen persistenten Host-SoT Runner:"
|
||
python3 -m py_compile "$PATCH_TARGET" 2>&1 && echo " OK" || die "Post-Promotion FAIL: py_compile (FAIL CLOSED)"
|
||
echo "PROMOTE-RUNNER-PATCH OK. Host-SoT Runner ist jetzt der gepruefte Patch."
|
||
}
|
||
|
||
# ---- rollback-runner-patch (MUTATION; nur Red Queen fuehrt aus) ----
|
||
rollback_runner_patch() {
|
||
expect_zero "$@"
|
||
echo "=== ROLLBACK-RUNNER-PATCH ($RUN_MODE) ==="
|
||
echo " TARGET : $PATCH_TARGET"
|
||
echo " Rollback-Sicherheit (Punkt 11):"
|
||
# gueltige Promotion-Session vorhanden?
|
||
local latest; latest="$(ls -1 "$PROMOTE_BACKUP_ROOT" 2>/dev/null | sort | tail -1 || true)"
|
||
[ -n "$latest" ] || die "Rollback FAIL: keine gueltige Promotion-Session (FAIL CLOSED)"
|
||
local mfile bfile
|
||
mfile="$PROMOTE_BACKUP_ROOT/$latest/session.meta"
|
||
bfile="$PROMOTE_BACKUP_ROOT/$latest/run_backfill_year.py"
|
||
[ -f "$mfile" ] || die "Rollback FAIL: Session-Metadaten fehlen (FAIL CLOSED)"
|
||
[ -f "$bfile" ] || die "Rollback FAIL: Backup fehlt (FAIL CLOSED)"
|
||
[ -L "$bfile" ] && die "Rollback FAIL: Backup ist Symlink (FAIL CLOSED)"
|
||
local bsha; bsha="$(sha256sum "$bfile" | awk '{print $1}')"
|
||
local tbefore; tbefore="$(sha256sum "$PATCH_TARGET" | awk '{print $1}')"
|
||
echo " Session : $latest"
|
||
echo " Backup SHA : $bsha"
|
||
echo " aktueller Target-Zustand : $tbefore"
|
||
echo " geplante eine Datei-Mutation : $PATCH_TARGET (zurueck auf Backup)"
|
||
if [ "$RUN_MODE" = "DRY_RUN" ]; then
|
||
echo "DRY_RUN_PASS: wuerde Backup $bfile auf $PATCH_TARGET zurueckspielen. KEINE Aktion."
|
||
exit 0
|
||
fi
|
||
mode_or_die
|
||
# aktuellen Target-Zustand dokumentieren (vor Rollback)
|
||
local cur; cur="$(sha256sum "$PATCH_TARGET" | awk '{print $1}')"
|
||
echo " aktueller Target-SHA vor Rollback : $cur"
|
||
# Backup in Temp + SHA pruefen + atomarer rename
|
||
local tmpf; tmpf="$PATCH_TARGET.tmp.$$"
|
||
cp -a "$bfile" "$tmpf" || die "Rollback: Backup nicht lesbar (FAIL CLOSED)"
|
||
local ps; ps="$(sha256sum "$tmpf" | awk '{print $1}')"
|
||
[ "$ps" = "$bsha" ] || { rm -f "$tmpf"; die "Rollback: Temp-SHA != Backup-SHA (FAIL CLOSED)"; }
|
||
chown "$(stat -c '%U' "$PATCH_TARGET"):$(stat -c '%G' "$PATCH_TARGET")" "$tmpf"
|
||
chmod "$(stat -c '%a' "$PATCH_TARGET")" "$tmpf"
|
||
mv -f "$tmpf" "$PATCH_TARGET" || { rm -f "$tmpf"; die "Rollback: atomarer rename fehlgeschlagen (FAIL CLOSED)"; }
|
||
local tafter; tafter="$(sha256sum "$PATCH_TARGET" | awk '{print $1}')"
|
||
echo " TARGET_SHA_AFTER : $tafter"
|
||
# TARGET_SHA_BEFORE aus Session-Metadaten
|
||
local expected; expected="$(grep '^TARGET_SHA_BEFORE=' "$mfile" | cut -d= -f2)"
|
||
[ "$tafter" = "$expected" ] || die "Rollback FAIL: Target-SHA != TARGET_SHA_BEFORE (FAIL CLOSED)"
|
||
echo "ROLLBACK-RUNNER-PATCH OK. Target zurueck auf TARGET_SHA_BEFORE=$expected."
|
||
}
|
||
|
||
# ---- REMEDIATION BUILD/DEPLOY CONTROL-PLANE (Phase 13.5) ----
|
||
# KEIN echter Build/Recreate durch Rain. Red Queen fuehrt nach Freigabe aus.
|
||
# Helfer: Read-only-Session-Bestand ("Build-Session vorhanden?") — muss zusaetzlich
|
||
# FULL_IMAGE_ID tragen, da Verify/Recreate nur der gespeicherten Image-ID vertrauen.
|
||
remediation_session() { # 0 NUR wenn AKTUELL erfolgreicher Build-State mit inspectable Image existiert
|
||
# (RAIN Hotfix §9/§10) Eine historische/stale FULL_IMAGE_ID darf KEINE aktive Identity
|
||
# sein. Nur wenn das referenzierte Image im Docker-Store tatsaechlich inspectable ist,
|
||
# ist die Session "aktiv und aktuell". Sonst -> NO ACTIVE REMEDIATION IMAGE STATE.
|
||
[ -f "$REMEDIATION_SESSION_META" ] || return 1
|
||
grep -q '^FULL_IMAGE_ID=sha256:[a-f0-9]\{64\}$' "$REMEDIATION_SESSION_META" || return 1
|
||
[ "$(stat -c '%F' "$REMEDIATION_SESSION_META")" = "regular file" ] || return 1
|
||
[ -L "$REMEDIATION_SESSION_META" ] && return 1
|
||
local rid
|
||
rid="$(grep '^FULL_IMAGE_ID=' "$REMEDIATION_SESSION_META" | cut -d= -f2)"
|
||
# NO ACTIVE fallback: historische ID ist nur Evidence. Image muss inspectable sein.
|
||
docker image inspect "$rid" >/dev/null 2>&1 || return 1
|
||
return 0
|
||
}
|
||
remediation_full_image_id() { # FULL_IMAGE_ID aus Session (echo) — nur wenn rem_remediation_session OK
|
||
grep '^FULL_IMAGE_ID=' "$REMEDIATION_SESSION_META" | cut -d= -f2
|
||
}
|
||
# Pre-Build-Guards (Punkt 12: dry-run-Gates A–O). Read-only; alle FAIL CLOSED.
|
||
remediation_guards() {
|
||
load_remediation_source_state
|
||
load_recovery_state
|
||
echo " [guard A] Gold-Source-Session (immutable) gebunden:"
|
||
[ -d "$GOLD_SOURCE_ROOT/$REMEDIATION_SESSION_ID" ] || die "Guard A FAIL: Gold-Source-Session fehlt (FAIL CLOSED)"
|
||
echo " OK (Session $REMEDIATION_SESSION_ID)"
|
||
echo " [guard B] Runner-Overlay-SHA == erwartet (6cca3c55...):"
|
||
local hs; hs="$(sha256sum "$RUNNER_PATCH_SOURCE" | awk '{print $1}')"
|
||
[ "$hs" = "$RUNNER_REMEDIATION_SHA" ] || die "Guard B FAIL: Runner-Overlay-SHA $hs != $RUNNER_REMEDIATION_SHA (FAIL CLOSED)"
|
||
echo " OK ($hs)"
|
||
echo " [guard C] Build-Context-Basis (Gold-Source-Session) + Dockerfile/.dockerignore:"
|
||
[ -d "$GOLD_SOURCE_ROOT/$REMEDIATION_SESSION_ID" ] || die "Guard C FAIL: Gold-Source-Session fehlt (FAIL CLOSED)"
|
||
[ -f "$GOLD_SOURCE_ROOT/$REMEDIATION_SESSION_ID/Dockerfile" ] || die "Guard C FAIL: Dockerfile fehlt (FAIL CLOSED)"
|
||
[ -f "$GOLD_SOURCE_ROOT/$REMEDIATION_SESSION_ID/.dockerignore" ] || die "Guard C FAIL: .dockerignore fehlt (FAIL CLOSED)"
|
||
echo " OK"
|
||
echo " [guard D] erwartetes Build-Input (62 Gold + 1 autorisierter Runner-Overlay):"
|
||
remediation_expected_manifest_ok || die "Guard D FAIL: erwartetes Remediation-Build-Input nicht exakt (FAIL CLOSED)"
|
||
echo " OK (MATCH=$RM_MATCH AUTHORIZED_DIVERGED=$RM_AUTH UNAUTHORIZED_DIVERGED=$RM_UNAUTH MISSING=$RM_MISSING EXTRA=$RM_EXTRA)"
|
||
echo " [guard E] valider Full-SoT-Backup vorhanden (Partial ausgeschlossen):"
|
||
full_sot_backup >/dev/null || die "Guard E FAIL: kein gueltiger FULL SOT BACKUP (FAIL CLOSED)"
|
||
echo " OK ($(full_sot_backup))"
|
||
echo " [guard F] Recovery-Gold inspectable:"
|
||
docker image inspect "$RECOVERY_GOLD_IMAGE_ID" >/dev/null 2>&1 || die "Guard F FAIL: Recovery-Image nicht inspectable (FAIL CLOSED)"
|
||
echo " OK"
|
||
echo " [guard G] Recovery-Tag -> Recovery-ID:"
|
||
local rt; rt="$(docker image inspect -f '{{.Id}}' "$RECOVERY_GOLD_TAG_FINAL" 2>/dev/null || true)"
|
||
[ "$rt" = "$RECOVERY_GOLD_IMAGE_ID" ] || die "Guard G FAIL: Recovery-Tag != Recovery-ID (FAIL CLOSED)"
|
||
echo " OK"
|
||
echo " [guard H] verify-recovery-gold-image PASS:"
|
||
( verify_recovery_gold_image >/dev/null 2>&1 ) || die "Guard H FAIL: Recovery-Gold nicht verifiziert (FAIL CLOSED)"
|
||
echo " OK"
|
||
echo " [guard I] No-op-Candidate (DIAGNOSTIC, NON-BLOCKING fuer Remediation-Build):"
|
||
local ni; ni="$(docker image inspect -f '{{.Id}}' "$CANDIDATE_TAG" 2>/dev/null || true)"
|
||
if [ -n "$ni" ] && [ "$ni" = "$CANDIDATE_IMAGE_ID" ] && [ "$CANDIDATE_IMAGE_ID" = "$REM_NOOP_IMAGE_ID" ]; then
|
||
echo " NOOP_CANDIDATE_TAG=PRESENT"
|
||
echo " STATUS=OK"
|
||
else
|
||
echo " NOOP_CANDIDATE_TAG=ABSENT"
|
||
echo " CAUSE=PREEXISTING_COOLIFY_CLEANUP"
|
||
echo " IMPACT_ON_REMEDIATION_BUILD=NONE"
|
||
echo " STATUS=NON_BLOCKING_DIAGNOSTIC"
|
||
echo " (No-op-Candidate ist weder Build Source, Build Input, Overlay Source,"
|
||
echo " Recovery Anchor noch Remediation Target Identity -> kein fail-closed Build-Gate)"
|
||
fi
|
||
echo " [guard J] Remediation-Tag existiert NOCH NICHT (KEIN Overwrite):"
|
||
if docker image inspect "$REMEDIATION_TAG" >/dev/null 2>&1; then
|
||
die "Guard J FAIL: Remediation-Tag existiert bereits — KEIN Ueberschreiben, kein force-retag (FAIL CLOSED)"
|
||
fi
|
||
echo " OK (Tag frei)"
|
||
echo " [guard K] historical-db healthy:"
|
||
ensure_db_run || die "Guard K FAIL: historical-db nicht Running (FAIL CLOSED)"
|
||
echo " OK"
|
||
echo " [guard L] kein Runner/Backfill laeuft:"
|
||
docker ps --format '{{.Names}}' | grep -qiE 'runner|backfill' && die "Guard L FAIL: Runner/Backfill laeuft (FAIL CLOSED)" || echo " OK"
|
||
echo " [guard M] Production Safety geschlossen (isolierter POC-Stack):"
|
||
echo " OK (eigene db, eigenes Netz, GET-only API, kein Produktivnetz)"
|
||
echo " [guard N] Dockerfile/Context exakt erwartet (aus Gold-Source-Session):"
|
||
local gdf="$GOLD_SOURCE_ROOT/$REMEDIATION_SESSION_ID/Dockerfile"
|
||
[ -f "$gdf" ] || die "Guard N FAIL: Gold-Source-Dockerfile fehlt (FAIL CLOSED)"
|
||
grep -qE '^FROM python:3\.11-slim' "$gdf" || die "Guard N FAIL: Dockerfile Base-Image python:3.11-slim fehlt (FAIL CLOSED)"
|
||
grep -qE '^COPY app/ \./app/' "$gdf" || die "Guard N FAIL: Dockerfile COPY app/ ./app/ fehlt (FAIL CLOSED)"
|
||
grep -qE '^EXPOSE 8000' "$gdf" || die "Guard N FAIL: Dockerfile EXPOSE 8000 fehlt (FAIL CLOSED)"
|
||
echo " OK"
|
||
echo " [guard O] kein freier Parameter/User-Input:"
|
||
echo " OK (keine Argumente akzeptiert)"
|
||
echo " [diagnostic] Host-SoT (NICHT Build-Input):"
|
||
local hrepo; hrepo="$(sha256sum /opt/historical-v2/app/persistence/repository.py 2>/dev/null | awk '{print $1}')"
|
||
echo " HOST_SOT_DRIFT=DETECTED (repository.py $hrepo vs Gold 8fff892f...)"
|
||
echo " HOST_SOT_DRIFT=NON-BLOCKING_FOR_REMEDIATION_BUILD"
|
||
}
|
||
# ---- GEMEINSAME AUTHORIZED-GOLD-DELTA-POLICY (Phase 13.5) ----
|
||
# Einzige autorisierte Divergenz gegen GOLD_MANIFEST:
|
||
# app/run_backfill_year.py ALT=RUNNER_GOLD_SHA -> NEU=RUNNER_REMEDIATION_SHA
|
||
# (exakte Pfad- UND SHA-Bindung; KEINE generische "1 Divergenz ok"-Lockerung).
|
||
# Setzt globale Zaehler AD_MATCH/AD_AUTH/AD_UNAUTH/AD_MISSING/AD_EXTRA und
|
||
# AD_RET (ok|fail). fail wenn UNAUTH>0 || MISSING>0 || EXTRA>0.
|
||
# Wird von verify_remediation_build_input UND verify_runner_remediation_image
|
||
# verwendet -> EINE Policy, keine unterschiedlichen Policies.
|
||
authorized_gold_delta() {
|
||
local new="$1" gold goldrel goldsha nsha nrel found
|
||
AD_MATCH=0; AD_AUTH=0; AD_UNAUTH=0; AD_MISSING=0; AD_EXTRA=0
|
||
while read -r goldsha goldrel; do
|
||
goldrel="${goldrel#./}"; [ -n "$goldrel" ] || continue
|
||
nsha="$(awk -v r="$goldrel" '$2==r{print $1}' "$new")"
|
||
if [ -z "$nsha" ]; then AD_MISSING=$((AD_MISSING+1))
|
||
elif [ "$nsha" = "$goldsha" ]; then AD_MATCH=$((AD_MATCH+1))
|
||
else
|
||
if [ "$goldrel" = "app/run_backfill_year.py" ] \
|
||
&& [ "$goldsha" = "$RUNNER_GOLD_SHA" ] \
|
||
&& [ "$nsha" = "$RUNNER_REMEDIATION_SHA" ]; then
|
||
AD_AUTH=$((AD_AUTH+1))
|
||
else
|
||
AD_UNAUTH=$((AD_UNAUTH+1))
|
||
fi
|
||
fi
|
||
done < "$GOLD_MANIFEST"
|
||
while read -r nsha nrel; do
|
||
nrel="${nrel#./}"
|
||
found="$(awk -v r="$nrel" '$2==r{print $1}' "$GOLD_MANIFEST")"
|
||
if [ -z "$found" ]; then
|
||
case "$nrel" in
|
||
app/app/backfill.py|app/app/test_idempotency.py|app/eligibility.py) ;;
|
||
*) AD_EXTRA=$((AD_EXTRA+1)) ;;
|
||
esac
|
||
fi
|
||
done < "$new"
|
||
AD_RET="ok"
|
||
[ $AD_UNAUTH -eq 0 ] && [ $AD_MISSING -eq 0 ] && [ $AD_EXTRA -eq 0 ] || AD_RET="fail"
|
||
}
|
||
# Read-only Build-Input-Check: nur app/run_backfill_year.py darf vom Gold-Manifest
|
||
# divergieren (und zwar exakt auf RUNNER_REMEDIATION_SHA). alles andere -> FAIL.
|
||
verify_remediation_build_input() {
|
||
load_remediation_source_state
|
||
echo " BUILD_SOURCE=VERIFIED_GOLD_SOURCE"
|
||
echo " SOURCE_SESSION=$REMEDIATION_SESSION_ID"
|
||
echo " AUTHORIZED_OVERLAY=run_backfill_year.py"
|
||
echo " AUTHORIZED_RUNNER_SHA=$RUNNER_REMEDIATION_SHA"
|
||
local hm
|
||
hm="$(mktemp)"; remediation_expected_manifest > "$hm"
|
||
authorized_gold_delta "$hm"
|
||
rm -f "$hm"
|
||
echo " Gold $((AD_MATCH+AD_AUTH+AD_UNAUTH+AD_MISSING)): MATCH=$AD_MATCH AUTHORIZED_DIVERGED=$AD_AUTH UNAUTHORIZED_DIVERGED=$AD_UNAUTH MISSING=$AD_MISSING ; unerlaubte EXTRA=$AD_EXTRA"
|
||
[ "$AD_AUTH" = "1" ] || die "BUILD-INPUT FAIL: autorisierte Runner-Divergenz fehlt (FAIL CLOSED)"
|
||
[ "$AD_RET" = "ok" ] || die "BUILD-INPUT FAIL: unerwartete Divergenz (FAIL CLOSED)"
|
||
echo " [diagnostic] Host-SoT (NICHT Build-Input):"
|
||
local hrepo; hrepo="$(sha256sum /opt/historical-v2/app/persistence/repository.py 2>/dev/null | awk '{print $1}')"
|
||
echo " HOST_SOT_DRIFT=DETECTED (repository.py $hrepo vs Gold 8fff892f...)"
|
||
echo " HOST_SOT_DRIFT=NON-BLOCKING_FOR_REMEDIATION_BUILD"
|
||
return 0
|
||
}
|
||
# ---- build-runner-remediation (DRY_RUN = kein Build, kein Staging; KEIN Overwrite) ----
|
||
# Build-Context = kontrolliertes Staging aus der immutable Gold-Source-Session
|
||
# + EXAKT EIN autorisierter Runner-Overlay (RUNNER_PATCH_SOURCE). KEIN /opt/historical-v2.
|
||
# DRY_RUN: Guards + erwartetes Manifest + deterministischer REMEDIATION_BUILD_INPUT_SHA,
|
||
# ZERO MUTATION (kein Staging-Dir erzeugt). Real: Staging erzeugen, Guards, Build.
|
||
build_runner_remediation() {
|
||
expect_zero "$@"
|
||
echo "=== BUILD-RUNNER-REMEDIATION ($RUN_MODE) ==="
|
||
load_remediation_source_state
|
||
echo " BUILD_SOURCE=VERIFIED_GOLD_SOURCE"
|
||
echo " SOURCE_SESSION=$REMEDIATION_SESSION_ID"
|
||
echo " AUTHORIZED_OVERLAY=run_backfill_year.py"
|
||
echo " AUTHORIZED_RUNNER_SHA=$RUNNER_REMEDIATION_SHA"
|
||
echo " Build Context : $REMEDIATION_BUILD_CONTEXT (Staging aus Gold-Source-Session)"
|
||
echo " Dockerfile : $REMEDIATION_BUILD_DOCKERFILE (aus Gold-Source-Session)"
|
||
echo " Ziel-Tag : $REMEDIATION_TAG (fix, KEIN latest, KEIN Overwrite)"
|
||
echo " Guards:"
|
||
remediation_guards
|
||
echo " [expected-manifest] REMEDIATION_BUILD_INPUT (Gold-Source + Overlay):"
|
||
remediation_expected_manifest | while IFS= read -r line; do echo " $line"; done
|
||
echo " [expected] MATCH=$RM_MATCH AUTHORIZED_DIVERGED=$RM_AUTH UNAUTHORIZED_DIVERGED=$RM_UNAUTH MISSING=$RM_MISSING EXTRA=$RM_EXTRA"
|
||
local ebi; ebi="$(remediation_expected_build_input_sha)"
|
||
echo " REMEDIATION_BUILD_INPUT_SHA=$ebi (deterministisch, Gold-Source + Overlay)"
|
||
if [ "$RUN_MODE" = "DRY_RUN" ]; then
|
||
echo "DRY_RUN_PASS: build-runner-remediation wuerde Staging aus Gold-Source-Session erzeugen, run_backfill_year.py durch $RUNNER_REMEDIATION_SHA ersetzen und docker build -f $REMEDIATION_BUILD_DOCKERFILE -t $REMEDIATION_TAG $REMEDIATION_BUILD_CONTEXT ausfuehren (keine Aktion)."
|
||
exit 0
|
||
fi
|
||
mode_or_die
|
||
# --- Staging erzeugen (Punkt 7/8): Gold-Source-Session -> kontrollierte Kopie -> Overlay ---
|
||
if [ -e "$REMEDIATION_STAGING_DIR" ]; then
|
||
die "build: Staging-Ziel existiert bereits unerwartet $REMEDIATION_STAGING_DIR (FAIL CLOSED)"
|
||
fi
|
||
mkdir -p "$REMEDIATION_STAGING_ROOT" || die "build: Staging-Root nicht anlegbar (FAIL CLOSED)"
|
||
cp -a "$GOLD_SOURCE_ROOT/$REMEDIATION_SESSION_ID" "$REMEDIATION_STAGING_DIR" || die "build: Gold-Source-Kopie fehlgeschlagen (FAIL CLOSED)"
|
||
# Overlay: run_backfill_year.py durch autorisierten Patch ersetzen (regulaere Datei, kein Symlink)
|
||
[ -L "$RUNNER_PATCH_SOURCE" ] && die "build: Runner-Patch ist Symlink (FAIL CLOSED)"
|
||
[ -f "$RUNNER_PATCH_SOURCE" ] || die "build: Runner-Patch fehlt (FAIL CLOSED)"
|
||
cp -f "$RUNNER_PATCH_SOURCE" "$REMEDIATION_STAGING_DIR/app/run_backfill_year.py" || die "build: Overlay-Kopie fehlgeschlagen (FAIL CLOSED)"
|
||
# Staging verifizieren: erwartetes Manifest + Build-Input-SHA
|
||
REMEDIATION_SESSION_ID="$REMEDIATION_SESSION_ID"
|
||
REMEDIATION_BUILD_CONTEXT="$REMEDIATION_STAGING_DIR"
|
||
REMEDIATION_BUILD_DOCKERFILE="$REMEDIATION_STAGING_DIR/Dockerfile"
|
||
remediation_staging_matches_expected || die "build: Staging != erwartetes Build-Input (FAIL CLOSED)"
|
||
local sbi; sbi="$(remediation_expected_build_input_sha)"
|
||
[ "$sbi" = "$ebi" ] || die "build: Staging-Build-Input-SHA $sbi != erwartet $ebi (FAIL CLOSED)"
|
||
echo " [staging] Staging verifiziert: MATCH=$RM_MATCH AUTHORIZED_DIVERGED=$RM_AUTH UNAUTHORIZED_DIVERGED=$RM_UNAUTH MISSING=$RM_MISSING EXTRA=$RM_EXTRA"
|
||
echo " [build] docker build -f $REMEDIATION_BUILD_DOCKERFILE -t $REMEDIATION_TAG $REMEDIATION_BUILD_CONTEXT"
|
||
docker build -f "$REMEDIATION_BUILD_DOCKERFILE" -t "$REMEDIATION_TAG" "$REMEDIATION_BUILD_CONTEXT" || die "build-runner-remediation fehlgeschlagen (FAIL CLOSED)"
|
||
local newid now
|
||
newid="$(docker image inspect -f '{{.Id}}' "$REMEDIATION_TAG" 2>/dev/null)"
|
||
[ -n "$newid" ] || die "build: Image-ID nicht ermittelbar (FAIL CLOSED)"
|
||
echo " FULL_IMAGE_ID=$newid"
|
||
now="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
||
# Build-Session root-owned anlegen (einmaliges Schreiben der Image-ID)
|
||
mkdir -p "$REMEDIATION_SESSION_DIR" || die "build: Session-Dir nicht anlegbar (FAIL CLOSED)"
|
||
cat > "$REMEDIATION_SESSION_META" <<EOF
|
||
TAG=$REMEDIATION_TAG
|
||
FULL_IMAGE_ID=$newid
|
||
REMEDIATION_BUILD_INPUT_SHA=$sbi
|
||
SOURCE_SESSION=$REMEDIATION_SESSION_ID
|
||
AUTHORIZED_RUNNER_SHA=$RUNNER_REMEDIATION_SHA
|
||
TIMESTAMP=$now
|
||
WRAPPER_SHA=$(sha256sum /usr/local/bin/rq-historical | awk '{print $1}')
|
||
EOF
|
||
chown root:root "$REMEDIATION_SESSION_META"; chmod 600 "$REMEDIATION_SESSION_META"
|
||
echo " Session: $REMEDIATION_SESSION_META (root-owned 600)"
|
||
echo "BUILD-RUNNER-REMEDIATION OK. Image-ID in Build-Session gespeichert. KEIN Recreate, KEIN Runner, latest unangetastet."
|
||
}
|
||
# ---- verify-runner-remediation-image (read-only) ----
|
||
verify_runner_remediation_image() {
|
||
req_ok; expect_zero "$@"
|
||
echo "=== VERIFY-RUNNER-REMEDIATION-IMAGE (Candidate vs Host-SoT / Recovery-Gold) ==="
|
||
remediation_session || die "verify: keine gueltige Build-Session mit FULL_IMAGE_ID (zuerst build-runner-remediation) (FAIL CLOSED)"
|
||
# (RAIN Hotfix) Source-State-Variablen (REMEDIATION_SESSION_ID u.a.) deterministisch aus
|
||
# autorisiertem Recovery-Source-State initialisieren, bevor sie unten verwendet werden.
|
||
# Ohne diesen Aufruf ist REMEDIATION_SESSION_ID unter set -u unbound -> Exit 1.
|
||
load_remediation_source_state
|
||
local imgid; imgid="$(remediation_full_image_id)"
|
||
echo " REMEDIATION_FULL_IMAGE_ID=$imgid (aus Session, NICHT nur Tag)"
|
||
docker image inspect "$imgid" >/dev/null 2>&1 || die "verify: Image-ID nicht inspectable (FAIL CLOSED)"
|
||
echo " [Binding] Remediation-Tag -> FULL Image-ID:"
|
||
local rt; rt="$(docker image inspect -f '{{.Id}}' "$REMEDIATION_TAG" 2>/dev/null || true)"
|
||
[ "$rt" = "$imgid" ] || die "verify: Remediation-Tag ($rt) != gespeicherte Image-ID ($imgid) (FAIL CLOSED)"
|
||
echo " OK"
|
||
echo " [Gold-Delta] Image vs GOLD_MANIFEST (62 MATCH + 1 autorisierter Runner-Overlay):"
|
||
local tmpc tmpdir tmpf
|
||
tmpc="rq-rem-verify-$(date +%s)"; tmpdir="/tmp/rq-rem-copy-$$"
|
||
rm -rf "$tmpdir"; mkdir -p "$tmpdir"
|
||
docker create --name "$tmpc" --entrypoint /bin/true "$imgid" >/dev/null 2>&1 || die "verify: Temp-Container Fehler (FAIL CLOSED)"
|
||
docker cp "$tmpc:/app/app" "$tmpdir/appapp" >/dev/null 2>&1 || { docker rm -f "$tmpc" >/dev/null 2>&1; rm -rf "$tmpdir"; die "verify: /app/app nicht lesbar (FAIL CLOSED)"; }
|
||
docker rm -f "$tmpc" >/dev/null 2>&1 || true
|
||
tmpf="$(mktemp)"
|
||
( cd "$tmpdir/appapp" && find . -type f -name '*.py' ! -path '*/__pycache__/*' -printf '%P\n' 2>/dev/null | sort \
|
||
| while IFS= read -r rel; do [ -f "$rel" ] && sha256sum "$rel"; done | awk '{print $1" app/"$2}' ) > "$tmpf"
|
||
# [Gold-Delta] Image vs GOLD_MANIFEST: exakt 1 autorisierte Runner-Divergenz erlaubt.
|
||
authorized_gold_delta "$tmpf"
|
||
echo " Image vs GOLD: MATCH=$AD_MATCH AUTHORIZED_DIVERGED=$AD_AUTH UNAUTHORIZED_DIVERGED=$AD_UNAUTH MISSING=$AD_MISSING unerlaubte EXTRA=$AD_EXTRA"
|
||
[ "$AD_AUTH" = "1" ] || { rm -rf "$tmpdir"; die "verify: Gold-Delta != exakt 1 autorisierte Runner-Divergenz (FAIL CLOSED)"; }
|
||
[ "$AD_RET" = "ok" ] || { rm -rf "$tmpdir"; die "verify: Image != Gold-Manifest (unzulaessige Divergenz) (FAIL CLOSED)"; }
|
||
# [repository hard gate] app/persistence/repository.py im Image MUSS Gold sein (8fff892f...)
|
||
local irepo; irepo="$(awk '$2=="app/persistence/repository.py"{print $1}' "$tmpf")"
|
||
[ "$irepo" = "8fff892f52a612f464b5be21c66ef7787a47f2d7d049f815ef87837553c2a228" ] \
|
||
|| { rm -rf "$tmpdir"; die "verify: repository.py im Image != Gold (8fff892f...) (FAIL CLOSED)"; }
|
||
echo " repository.py = Gold (8fff892f...) OK"
|
||
# [diagnostic] Host-SoT (NICHT Build-Input): repository.py 3fc0f962... vs Gold 8fff892f...
|
||
local hrepo; hrepo="$(sha256sum /opt/historical-v2/app/persistence/repository.py 2>/dev/null | awk '{print $1}')"
|
||
echo " HOST_SOT_DRIFT=DETECTED (repository.py $hrepo vs Gold 8fff892f...)"
|
||
echo " HOST_SOT_DRIFT=NON-BLOCKING_FOR_REMEDIATION_BUILD"
|
||
rm -f "$tmpf"
|
||
echo " [Critical-Files] $REMEDIATION_TAG vs gold (image-only, autorisierte Runner-Divergenz erlaubt):"
|
||
verify_critical_files_image_only "$tmpdir/appapp" runner
|
||
rm -rf "$tmpdir"
|
||
echo " [Runner] Image == autorisierter Overlay (6cca3c55...):"
|
||
local ir; ir="$(docker run --rm --entrypoint /bin/sh "$imgid" -c "sha256sum /app/app/run_backfill_year.py 2>/dev/null" | awk '{print $1}')" || ir=""
|
||
[ "$ir" = "$RUNNER_REMEDIATION_SHA" ] || die "verify: Runner-Image-SHA ($ir) != $RUNNER_REMEDIATION_SHA (FAIL CLOSED)"
|
||
echo " Image=$ir (erwartet $RUNNER_REMEDIATION_SHA)"
|
||
echo " [Expected Gold Delta] GENAU EINE fachliche Divergenz = app/run_backfill_year.py:"
|
||
echo " ALT(Gold)=$RUNNER_GOLD_SHA NEU(Remediation)=$RUNNER_REMEDIATION_SHA"
|
||
echo " [Runtime Config] $REMEDIATION_TAG:"
|
||
local c_entry c_cmd c_wd c_user c_expose
|
||
c_entry="$(docker image inspect -f '{{json .Config.Entrypoint}}' "$imgid" 2>/dev/null)"
|
||
c_cmd="$(docker image inspect -f '{{json .Config.Cmd}}' "$imgid" 2>/dev/null)"
|
||
c_wd="$(docker image inspect -f '{{.Config.WorkingDir}}' "$imgid" 2>/dev/null)"
|
||
c_user="$(docker image inspect -f '{{.Config.User}}' "$imgid" 2>/dev/null)"
|
||
c_expose="$(docker image inspect -f '{{json .Config.ExposedPorts}}' "$imgid" 2>/dev/null)"
|
||
printf ' Entrypoint=%s Cmd=%s WorkDir=%s User=%s Expose=%s\n' "${c_entry:-null}" "${c_cmd:-null}" "${c_wd:-}" "${c_user:-}" "${c_expose:-}"
|
||
[ "$c_wd" = "/app" ] || die "verify: WorkingDir unerwartet (FAIL CLOSED)"
|
||
echo " [Build-Input] Session-Binding (Gold-Source + Overlay):"
|
||
local ssrc srunner sbi
|
||
ssrc="$(grep '^SOURCE_SESSION=' "$REMEDIATION_SESSION_META" | cut -d= -f2)"
|
||
srunner="$(grep '^AUTHORIZED_RUNNER_SHA=' "$REMEDIATION_SESSION_META" | cut -d= -f2)"
|
||
sbi="$(grep '^REMEDIATION_BUILD_INPUT_SHA=' "$REMEDIATION_SESSION_META" | cut -d= -f2)"
|
||
[ "$ssrc" = "$REMEDIATION_SESSION_ID" ] || die "verify: Session-SOURCE_SESSION != erwartet (FAIL CLOSED)"
|
||
[ "$srunner" = "$RUNNER_REMEDIATION_SHA" ] || die "verify: Session-AUTHORIZED_RUNNER_SHA != erwartet (FAIL CLOSED)"
|
||
[[ "$sbi" =~ ^[a-f0-9]{64}$ ]] || die "verify: Session-REMEDIATION_BUILD_INPUT_SHA ungueltig (FAIL CLOSED)"
|
||
echo " SOURCE_SESSION=$ssrc AUTHORIZED_RUNNER_SHA=$srunner REMEDIATION_BUILD_INPUT_SHA=$sbi"
|
||
echo "VERDICT: REMEDIATION IMAGE OK (62 Gold-MATCH + 1 autorisierter Runner-Overlay; repository.py Gold; Critical-SHAs + Config PASS)"
|
||
exit 0
|
||
}
|
||
# ---- recreate-runner-remediation-safe (NUR historical-service; nur Red Queen) ----
|
||
recreate_runner_remediation_safe() {
|
||
expect_zero "$@"
|
||
echo "=== RECREATE-RUNNER-REMEDIATION-SAFE ($RUN_MODE) ==="
|
||
echo " Ziel-Service : $HISTORICAL_SERVICE (NUR dieser; KEIN db, KEIN Compose-Down, KEINE Volumes)"
|
||
remediation_session || die "recreate: keine gueltige Build-Session (zuerst build-runner-remediation) (FAIL CLOSED)"
|
||
local imgid; imgid="$(remediation_full_image_id)"
|
||
echo " Image : $REMEDIATION_TAG -> $imgid (explizit, KEIN latest)"
|
||
echo " Pre-Gates:"
|
||
remediation_recreate_pre_gates
|
||
echo " Baseline vor Recreate:"
|
||
recreate_baseline
|
||
if [ "$RUN_MODE" = "DRY_RUN" ]; then
|
||
echo "DRY_RUN_PASS: wuerde docker compose -f $COMPOSE_FILE -f <override> up -d --no-deps --no-build $HISTORICAL_SERVICE (Image=$REMEDIATION_TAG, gespeicherte ID=$imgid). KEINE Aktion."
|
||
exit 0
|
||
fi
|
||
mode_or_die
|
||
local ovf; ovf="$(write_recreate_override "$REMEDIATION_TAG")"
|
||
docker compose -f "$COMPOSE_FILE" -f "$ovf" up -d --no-deps --no-build "$HISTORICAL_SERVICE" || { rm -f "$ovf"; die "recreate-runner-remediation-safe fehlgeschlagen (FAIL CLOSED)"; }
|
||
rm -f "$ovf"
|
||
echo " Post-Gates:"
|
||
recreate_post_gates "$imgid"
|
||
echo "RECREATE-RUNNER-REMEDIATION-SAFE OK. Container laeuft mit Remediation-Image (gespeicherte ID)."
|
||
}
|
||
|
||
# ---- build-historical (NUR Build, kein recreate; DRY_RUN = kein Bau) ----
|
||
build_historical() {
|
||
expect_zero "$@"
|
||
echo "=== BUILD-HISTORICAL ($RUN_MODE) ==="
|
||
echo " Projekt : $COMPOSE_PROJECT"
|
||
echo " Compose : $COMPOSE_FILE"
|
||
echo " Service : $HISTORICAL_SERVICE"
|
||
echo " Kontext : $COMPOSE_PROJECT (Dockerfile: $COMPOSE_PROJECT/Dockerfile)"
|
||
# 1. Dry-run zeigt Guards geplant, aber fuehrt sie (bis auf reine Checks) nicht aus
|
||
sot_bestandscheck >/dev/null
|
||
echo " [guard] backup-sot vorhanden: OK"
|
||
# verify-recovery check ist read-only, kann im dry-run laufen
|
||
echo " [guard] verify-sot-recovery:"
|
||
verify_recovery_ok
|
||
echo " [guard] Gold-Image $GOLD_IMAGE_ID: $(ensure_gold_image && echo OK)"
|
||
echo " [guard] Gold-Container Running: $(ensure_gold_run && echo OK)"
|
||
echo " [guard] historical-db Running: $(ensure_db_run && echo OK)"
|
||
echo " [guard] DB-Volume: $(ensure_db_volume && echo OK)"
|
||
echo " [guard] Builddefinition (Dockerfile+Compose): $(ensure_builddef && echo OK)"
|
||
echo " [guard] Build-Pollution-Exclusion valid:"
|
||
exclusion_valid && echo " .dockerignore OK (nur app/app/ + app/eligibility.py)"
|
||
if [ "$RUN_MODE" = "DRY_RUN" ]; then
|
||
echo "DRY_RUN_PASS: build-historical wuerde aus $COMPOSE_PROJECT bauen (keine Aktion)."
|
||
exit 0
|
||
fi
|
||
mode_or_die
|
||
echo " [build] docker compose -f $COMPOSE_FILE build $HISTORICAL_SERVICE"
|
||
docker compose -f "$COMPOSE_FILE" build "$HISTORICAL_SERVICE" || die "build fehlgeschlagen"
|
||
echo " [result]"
|
||
docker images --filter reference="${HISTORICAL_SERVICE}" --format 'Image {{.ID}} Created {{.CreatedSince}}'
|
||
echo "BUILD-HISTORICAL OK. Gold-Image NICHT geloescht ($GOLD_IMAGE_ID). Nur Build, kein Recreate."
|
||
}
|
||
|
||
# ---- recreate-historical (NUR historical-service, kein Runner; DRY_RUN = kein Recreate) ----
|
||
recreate_historical() {
|
||
expect_zero "$@"
|
||
echo "=== RECREATE-HISTORICAL ($RUN_MODE) ==="
|
||
echo " Service : $HISTORICAL_SERVICE (NUR dieser; NICHT historical-db)"
|
||
sot_bestandscheck >/dev/null
|
||
echo " [guard] backup-sot vorhanden: OK"
|
||
echo " [guard] Gold-Image $GOLD_IMAGE_ID: $(ensure_gold_image && echo OK)"
|
||
echo " [guard] Gold-Manifest: $([ -f "$GOLD_MANIFEST" ] && echo OK || echo FEHLT)"
|
||
echo " [guard] historical-db Running: $(ensure_db_run && echo OK)"
|
||
echo " [guard] DB-Volume: $(ensure_db_volume && echo OK)"
|
||
if [ "$RUN_MODE" = "DRY_RUN" ]; then
|
||
echo "DRY_RUN_PASS: recreate-historical wuerde historical-service neu erstellen (keine Aktion)."
|
||
exit 0
|
||
fi
|
||
mode_or_die
|
||
echo " [recreate] docker compose -f $COMPOSE_FILE up -d --no-deps $HISTORICAL_SERVICE"
|
||
docker compose -f "$COMPOSE_FILE" up -d --no-deps "$HISTORICAL_SERVICE" || die "recreate fehlgeschlagen"
|
||
sleep 4
|
||
local running
|
||
running="$(docker inspect -f '{{.State.Running}}' "$HISTORICAL_SERVICE" 2>/dev/null)"
|
||
[ "$running" = "true" ] || die "neuer Container laeuft nicht (FAIL CLOSED)"
|
||
echo " [Container] ID: $(docker inspect -f '{{.ID}}' "$HISTORICAL_SERVICE")"
|
||
echo " [Image] $(docker inspect -f '{{.Image}}' "$HISTORICAL_SERVICE")"
|
||
echo "RECREATE-HISTORICAL OK. KEIN Runner, KEIN compose down, KEIN Volume-Remove."
|
||
}
|
||
|
||
# ---- verify-gold-reproduction (read-only) ----
|
||
verify_gold_reproduction() {
|
||
req_ok; expect_zero "$@"
|
||
echo "=== VERIFY-GOLD-REPRODUCTION (Gold-Manifest vs Running-Container) ==="
|
||
[ -f "$GOLD_MANIFEST" ] || die "Gold-Manifest fehlt (FAIL CLOSED)"
|
||
local tmp; tmp="$(mktemp)"
|
||
gen_manifest "$HISTORICAL_SERVICE" > "$tmp"
|
||
classify_and_report "$tmp"
|
||
local ret="$G_RET"
|
||
rm -f "$tmp"
|
||
if [ "$ret" = "ok" ]; then
|
||
echo "VERDICT: GOLD REPRODUCTION OK (0 DIVERGED/MISSING/unerlaubte EXTRA)"
|
||
exit 0
|
||
else
|
||
echo "VERDICT: GOLD REPRODUCTION FAIL"
|
||
exit 2
|
||
fi
|
||
}
|
||
|
||
validate_stage() {
|
||
local key="$1" file real owner dmode
|
||
file="$(stage_file "$key")"
|
||
[ -e "$file" ] || die "Staging-Kandidat fehlt: $file (lege zuerst die Datei ab)"
|
||
real="$(realpath -m "$file")"
|
||
[[ "$real" == "$STAGING_ROOT/"* ]] || die "Traversal blockiert: realpath '$real' ausserhalb Staging"
|
||
[ "$real" = "$file" ] || die "Alias erkannt (realpath != Pfad): $file"
|
||
[ -f "$file" ] || die "keine reguläre Datei: $file"
|
||
[ -L "$file" ] && die "Symlink nicht erlaubt: $file"
|
||
owner="$(stat -c '%U' "$file")"
|
||
[ "$owner" = "rq-host" ] || die "Staging-Kandidat Owner = $owner (erwartet rq-host)"
|
||
dmode="$(stat -c '%a' "$STAGING_ROOT")"
|
||
[[ "$dmode" != *002 ]] && [[ "$dmode" != *022 ]] || die "Staging-Root global schreibbar '$dmode' (FAIL CLOSED)"
|
||
echo "OK: $key -> $file (regular, no symlink, owner rq-host, in staging)"
|
||
}
|
||
|
||
# ---- redact: maskiert Passwort-/Secret-Werte (fuer build-def Anzeige) ----
|
||
redact_line() {
|
||
# Zeilen mit "PASSWORD=" / "SECRET=" / "TOKEN=" : Wert maskieren
|
||
echo "$1" | sed -E 's/(PASSWORD|PASSWORDS|SECRET|SECRETS|TOKEN|TOKENS|API_KEY|CLIENT_SECRET|PRIVATE_KEY)(["'"'"']?[[:space:]]*=[[:space:]]*["'"'"']?)[^[:space:]"'"'"']+/\\1\\2***REDACTED***/Ig'
|
||
}
|
||
|
||
# =============================================================================
|
||
# IMAGE-RETENTION / ARCHIVE CONTROL PLANE (Phase 13.5 Schutzschicht)
|
||
# Zwei unabhängige Ebenen je Critical Image:
|
||
# A) Stopped Pin Container (docker create, nie gestartet) -> Image-Referenz
|
||
# B) sha256-verifiziertes docker save-Archiv -> externes Archiv
|
||
# READY-Gate: IMAGE VERIFY + PIN VERIFY + ARCHIVE VERIFY alle PASS.
|
||
# KEIN Overwrite. KEIN Coolify-Cleanup-Disable/Managed-Spoof. Kein echter
|
||
# Rebuild/Restore/Recreate. Kein freies Input (feste Rollen: recovery / remediation).
|
||
# =============================================================================
|
||
role_ok() { # $1 = role -> 0 wenn gueltig
|
||
case "$1" in recovery|remediation) return 0 ;; *) return 1 ;; esac
|
||
}
|
||
# Feste Rollen->Image/Identity-Bindung (Punkt 2/4/8/9/10) — hardcoded.
|
||
role_image_id() { # $1 = role -> echo erwartete FULL_IMAGE_ID
|
||
case "$1" in
|
||
recovery) echo "$RECOVERY_GOLD_IMAGE_ID" ;; # sha256:d2db926d...
|
||
remediation) echo "$REMEDIATION_TAG" ;; # Tag -> aufgeloest via inspect (Image fehlt aktuell)
|
||
esac
|
||
}
|
||
role_tag() {
|
||
case "$1" in
|
||
recovery) echo "$RECOVERY_GOLD_TAG_FINAL" ;;
|
||
remediation) echo "$REMEDIATION_TAG" ;;
|
||
esac
|
||
}
|
||
role_pin() {
|
||
case "$1" in
|
||
recovery) echo "$PIN_RECOVERY_NAME" ;;
|
||
remediation) echo "$PIN_REMEDIATION_NAME" ;;
|
||
esac
|
||
}
|
||
role_arch_sub() { echo "${ARCHIVE_SUB[$1]}" ; }
|
||
role_arch_dir() { echo "$ARCHIVE_ROOT/$(role_arch_sub "$1")" ; }
|
||
|
||
# --- Schicht A: Stopped Pin Container ---
|
||
# Container-Referenz schuetzt Image-Objekt vor Coolify rmi (empirisch belegt:
|
||
# No-op ueberlebte, weil Container 2fe3d30c... es referenzierte).
|
||
pin_container_identity_ok() { # $1 = role, $2 = actual container id -> 0 wenn ok
|
||
local role="$1" actual="$2" expected
|
||
case "$role" in
|
||
recovery) expected="$RECOVERY_GOLD_IMAGE_ID" ;;
|
||
remediation) expected="$PIN_REMEDIATION_TARGET_ID" ;;
|
||
esac
|
||
[ "$actual" = "$expected" ]
|
||
}
|
||
|
||
# --- Schicht B: Archive Verify ---
|
||
verify_archive() { # $1 = role (read-only)
|
||
local role="$1" adir sid
|
||
adir="$(role_arch_dir "$role")"
|
||
case "$role" in
|
||
recovery) sid="$RECOVERY_ARCHIVE_ID" ;;
|
||
remediation)
|
||
# Aktive Remediation-Identitaet aus der Build-Session (Punkt 21). Keine
|
||
# historische 41e357a1... als aktive Identity. FAIL CLOSED ohne Build-Session.
|
||
remediation_session || die "verify-archive(remediation): keine gueltige Build-Session (FAIL CLOSED)"
|
||
local rid; rid="$(remediation_full_image_id)"
|
||
sid="${rid#sha256:}"; sid="${sid:0:12}"
|
||
;;
|
||
esac
|
||
# Archive existiert
|
||
[ -d "$adir" ] || die "verify-archive($role): kein Archiv-Root (FAIL CLOSED)"
|
||
local base="$adir/$sid"
|
||
[ -d "$base" ] || die "verify-archive($role): kein Session-Ordner $base (FAIL CLOSED)"
|
||
# Pflichtdateien
|
||
local f img meta manifest runtime
|
||
img="$base/image.tar"; meta="$base/session.meta"; manifest="$base/app.manifest"; runtime="$base/runtime-config.txt"
|
||
[ -f "$img" ] && [ ! -L "$img" ] || die "verify-archive($role): image.tar fehlt/kein File (FAIL CLOSED)"
|
||
[ -s "$img" ] || die "verify-archive($role): image.tar leer (FAIL CLOSED)"
|
||
for f in "$meta" "$manifest" "$runtime"; do
|
||
[ -f "$f" ] && [ ! -L "$f" ] || die "verify-archive($role): fehlt $(basename "$f") (FAIL CLOSED)"
|
||
done
|
||
[ "$(stat -c '%U' "$img")" = "root" ] || die "verify-archive($role): image.tar Owner != root (FAIL CLOSED)"
|
||
case "$(stat -c '%a' "$img")" in 600|640|700|400) ;; *) die "verify-archive($role): image.tar Mode unsicher ($(stat -c '%a' "$img")) (FAIL CLOSED)" ;; esac
|
||
# Sha256 stimmt (Punkt 7)
|
||
local want got
|
||
want="$(awk -F'=' '$1=="ARCHIVE_SHA256"{print $2}' "$meta" 2>/dev/null)"
|
||
[ -n "$want" ] && [[ "$want" =~ ^[a-f0-9]{64}$ ]] || die "verify-archive($role): ARCHIVE_SHA256 fehlt/ungueltig in session.meta (FAIL CLOSED)"
|
||
got="$(sha256sum "$img" | awk '{print $1}')"
|
||
[ "$got" = "$want" ] || die "verify-archive($role): image.tar SHA mismatch (FAIL CLOSED)"
|
||
# session.meta Vollstaendigkeit (Punkt 5)
|
||
local key
|
||
for key in ROLE TAG FULL_IMAGE_ID ARCHIVE_SHA256 ARCHIVE_SIZE CREATED_AT HOST_SOT_RUNNER_SHA BUILD_INPUT_SHA WRAPPER_SHA APP_MANIFEST_SHA RUNTIME_CONFIG_SHA; do
|
||
grep -q "^$key=.*" "$meta" || die "verify-archive($role): session.meta fehlt $key (FAIL CLOSED)"
|
||
done
|
||
local mrole
|
||
mrole="$(awk -F'=' '$1=="ROLE"{print $2}' "$meta" 2>/dev/null)"
|
||
[ "$mrole" = "$role" ] || die "verify-archive($role): ROLE in session.meta = '$mrole' (FAIL CLOSED)"
|
||
echo "VERIFY-ARCHIVE($role): PASS ($base)"
|
||
}
|
||
|
||
# --- Ready-Gate (Punkt 14) ---
|
||
verify_ready() { # $1 = role
|
||
local role="$1" label
|
||
case "$role" in
|
||
recovery) label="RECOVERY READY" ;;
|
||
remediation) label="CANDIDATE READY" ;;
|
||
esac
|
||
echo "=== $label (READY-Gate: IMAGE VERIFY + PIN VERIFY + ARCHIVE VERIFY) ==="
|
||
# Recovery-Identitaet aus kontrolliertem State laden (Punkt 17/19). Kein
|
||
# statisches d2db, kein latest. FAIL CLOSED wenn State fehlt.
|
||
if [ "$role" = "recovery" ]; then
|
||
load_recovery_state
|
||
fi
|
||
# IMAGE VERIFY (zwingend inspectable + korrekt)
|
||
echo " [IMAGE] $(role_tag "$role"):"
|
||
verify_image_present "$role" || die "READY-Gate FAIL: Image nicht inspectable (FAIL CLOSED)"
|
||
echo " OK"
|
||
echo " [PIN] $(role_pin "$role"):"
|
||
verify_pin "$role" || die "READY-Gate FAIL: Pin nicht PASS (FAIL CLOSED)"
|
||
echo " OK"
|
||
echo " [ARCHIVE] $(role_arch_dir "$role"):"
|
||
verify_archive "$role" >/dev/null || die "READY-Gate FAIL: Archive nicht PASS (FAIL CLOSED)"
|
||
echo " OK"
|
||
echo "$label: PASS"
|
||
}
|
||
|
||
# --- Image-Present-Verify (Schicht-Pruefung: Image inspectable + Identity) ---
|
||
verify_image_present() { # $1 = role (read-only) -> 0 wenn Image vorhanden+bindung ok
|
||
local role="$1" id tag actualid
|
||
tag="$(role_tag "$role")"
|
||
case "$role" in
|
||
recovery)
|
||
id="$RECOVERY_GOLD_IMAGE_ID"
|
||
# Tag muss exakt auf die immutable Recovery-ID zeigen (kein Overwrite-Erwartung)
|
||
actualid="$(docker image inspect -f '{{.Id}}' "$tag" 2>/dev/null || true)"
|
||
[ "$actualid" = "$id" ] || return 1
|
||
;;
|
||
remediation)
|
||
# Candidate: FULL_IMAGE_ID aus Build-Session; Tag -> ID Bindung
|
||
if ! remediation_session; then die "verify-image(remediation): keine gueltige Build-Session (FAIL CLOSED)"; fi
|
||
id="$(remediation_full_image_id)"
|
||
actualid="$(docker image inspect -f '{{.Id}}' "$id" 2>/dev/null || true)"
|
||
[ -n "$actualid" ] || return 1
|
||
;;
|
||
esac
|
||
return 0
|
||
}
|
||
|
||
# --- Schicht A: Pin-Verify (Punkt 12) ---
|
||
verify_pin() { # $1 = role (read-only) -> 0 wenn Pin PASS
|
||
local role="$1" name imgid nw mounts ports rp priv capadd expected state
|
||
name="$(role_pin "$role")"
|
||
docker container inspect "$name" >/dev/null 2>&1 || { echo " PIN($role): Container $name fehlt"; return 1; }
|
||
expected="$(role_pin_target_id "$role")"
|
||
imgid="$(docker container inspect -f '{{.Image}}' "$name" 2>/dev/null)"
|
||
# .Image ist die Image-ID des referenzierten Images (exakt FULL_IMAGE_ID)
|
||
[ -n "$expected" ] || die "verify-pin($role): Ziel-Image-ID nicht bekannt (FAIL CLOSED)"
|
||
# Image-ID kann lang (sha256:...) sein — Bindung: .Image == erwartete FULL_IMAGE_ID
|
||
[ "$imgid" = "$expected" ] || { echo " PIN($role): Image-ID $imgid != erwartet $expected"; return 1; }
|
||
state="$(docker container inspect -f '{{.State.Status}}' "$name" 2>/dev/null)"
|
||
case "$state" in created|exited|stopped) ;; *) echo " PIN($role): State=$state (nicht created/stopped — VERBOTEN running)"; return 1 ;; esac
|
||
nw="$(docker container inspect -f '{{.HostConfig.NetworkMode}}' "$name" 2>/dev/null)"
|
||
[ "$nw" = "none" ] || { echo " PIN($role): NetworkMode=$nw != none"; return 1; }
|
||
mounts="$(docker container inspect -f '{{json .Mounts}}' "$name" 2>/dev/null)"
|
||
[ "$mounts" = "[]" ] || { echo " PIN($role): Mounts != []"; return 1; }
|
||
ports="$(docker container inspect -f '{{json .HostConfig.PortBindings}}' "$name" 2>/dev/null)"
|
||
[ "$ports" = "{}" ] || { echo " PIN($role): PortBindings != {}"; return 1; }
|
||
rp="$(docker container inspect -f '{{.HostConfig.RestartPolicy.Name}}' "$name" 2>/dev/null)"
|
||
[ "$rp" = "no" ] || { echo " PIN($role): RestartPolicy=$rp != no"; return 1; }
|
||
priv="$(docker container inspect -f '{{.HostConfig.Privileged}}' "$name" 2>/dev/null)"
|
||
[ "$priv" = "false" ] || { echo " PIN($role): Privileged=$priv"; return 1; }
|
||
capadd="$(docker container inspect -f '{{json .HostConfig.CapAdd}}' "$name" 2>/dev/null)"
|
||
[ "$capadd" = "null" ] || [ "$capadd" = "[]" ] || { echo " PIN($role): CapAdd=$capadd"; return 1; }
|
||
return 0
|
||
}
|
||
|
||
# --- Ziel-Image-ID fuer Pin (Schicht A bindet exakt FULL_IMAGE_ID) ---
|
||
role_pin_target_id() { # $1 = role
|
||
case "$1" in
|
||
recovery) echo "$RECOVERY_GOLD_IMAGE_ID" ;;
|
||
remediation)
|
||
if remediation_session; then echo "$(remediation_full_image_id)"; else echo ""; fi
|
||
;;
|
||
esac
|
||
}
|
||
|
||
# --- Archive-Metadaten-Generator (Punkt 5; kein Secret) ---
|
||
gen_archive_meta() { # $1 = role, $2 = session dir, $3 = image id, $4 = archive size, $5 = archive sha
|
||
local role="$1" dir="$2" image_id="$3" asize="$4" asha="$5" now
|
||
now="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
||
printf 'ROLE=%s\nTAG=%s\nFULL_IMAGE_ID=%s\nARCHIVE_SHA256=%s\nARCHIVE_SIZE=%s\nCREATED_AT=%s\nHOST_SOT_RUNNER_SHA=%s\nBUILD_INPUT_SHA=%s\nWRAPPER_SHA=%s\nAPP_MANIFEST_SHA=%s\nRUNTIME_CONFIG_SHA=%s\n' \
|
||
"$role" "$(role_tag "$role")" "$image_id" "$asha" "$asize" "$now" \
|
||
"$HOST_SOT_RUNNER_SHA_CUR" "$BUILD_INPUT_SHA_CUR" "$WRAPPER_SHA_CUR" "$APP_MANIFEST_SHA_CUR" "$RUNTIME_CONFIG_SHA_CUR" \
|
||
> "$dir/session.meta"
|
||
chown root:root "$dir/session.meta"; chmod 600 "$dir/session.meta"
|
||
}
|
||
|
||
# --- runtime-config Evidence (Punkt 4: runtime-config.txt) — keine Secrets ---
|
||
write_runtime_config() { # $1 = role, $2 = session dir
|
||
local role="$1" dir="$2" tag="$(role_tag "$role")" id
|
||
id="$(docker image inspect -f '{{.Id}}' "$tag" 2>/dev/null)"
|
||
{
|
||
echo "RUNTIME CONFIG EVIDENCE (Phase 13.5 Archive) — keine Secrets"
|
||
echo "ROLE=$role"
|
||
echo "FULL_IMAGE_ID=$id"
|
||
echo "Entrypoint=$(docker image inspect -f '{{json .Config.Entrypoint}}' "$tag" 2>/dev/null)"
|
||
echo "Cmd=$(docker image inspect -f '{{json .Config.Cmd}}' "$tag" 2>/dev/null)"
|
||
echo "WorkingDir=$(docker image inspect -f '{{.Config.WorkingDir}}' "$tag" 2>/dev/null)"
|
||
echo "User=$(docker image inspect -f '{{.Config.User}}' "$tag" 2>/dev/null)"
|
||
echo "ExposedPorts=$(docker image inspect -f '{{json .Config.ExposedPorts}}' "$tag" 2>/dev/null)"
|
||
echo "ENV_KEYS=$(docker image inspect -f '{{range .Config.Env}}{{println .}}{{end}}' "$tag" 2>/dev/null | cut -d= -f1 | tr '\n' ',' | sed 's/,$//')"
|
||
} > "$dir/runtime-config.txt"
|
||
chown root:root "$dir/runtime-config.txt"; chmod 600 "$dir/runtime-config.txt"
|
||
}
|
||
|
||
# --- app.manifest Evidence (Punkt 4) — Sha256 aller *.py im Image-Layer ---
|
||
write_app_manifest() { # $1 = role, $2 = session dir, $3 = image id/tag
|
||
local role="$1" dir="$2" ref="$3" tmpc tmpd
|
||
tmpc="rq-arch-manifest-$$"
|
||
tmpd="/tmp/rq-arch-manifest-$$"
|
||
rm -rf "$tmpd"; mkdir -p "$tmpd"
|
||
docker create --name "$tmpc" --entrypoint /bin/true "$ref" >/dev/null 2>&1 \
|
||
|| { rm -rf "$tmpd"; die "archive($role): Temp-Container Fehler (FAIL CLOSED)"; }
|
||
if ! docker cp "$tmpc:/app/app" "$tmpd/appapp" >/dev/null 2>&1; then
|
||
docker rm -f "$tmpc" >/dev/null 2>&1 || true; rm -rf "$tmpd"
|
||
die "archive($role): /app/app nicht lesbar (FAIL CLOSED)"
|
||
fi
|
||
docker rm -f "$tmpc" >/dev/null 2>&1 || true
|
||
( cd "$tmpd/appapp" && find . -type f -name '*.py' ! -path '*/__pycache__/*' -printf '%P\n' 2>/dev/null | sort \
|
||
| while IFS= read -r rel; do [ -f "$rel" ] && sha256sum "$rel"; done | awk '{print $1" app/"$2}' ) > "$dir/app.manifest"
|
||
rm -rf "$tmpd"
|
||
chown root:root "$dir/app.manifest"; chmod 600 "$dir/app.manifest"
|
||
}
|
||
|
||
# =============================================================================
|
||
# COMMAND-FUNKTIONEN (Phase 13.5 Archive/Pin) — fail-closed, kein Overwrite,
|
||
# kein echter Rebuild/Restore/Recreate in dieser Mission.
|
||
# =============================================================================
|
||
|
||
# archive-recovery-image / archive-remediation-image
|
||
archive_image() { # $1 = role
|
||
local role="$1"
|
||
expect_zero "${@:2}"
|
||
echo "=== ARCHIVE-$role ($RUN_MODE) ==="
|
||
local tag id subdir sdir
|
||
tag="$(role_tag "$role")"
|
||
case "$role" in
|
||
recovery)
|
||
# Aktive Recovery-Identitaet aus kontrolliertem State (Punkt 19). Kein d2db.
|
||
load_recovery_state
|
||
id="$RECOVERY_GOLD_IMAGE_ID"
|
||
;;
|
||
remediation)
|
||
remediation_session || die "archive(remediation): keine gueltige Build-Session (FAIL CLOSED)"
|
||
# (RAIN Hotfix) Source-State-Variablen deterministisch aus autorisiertem State initialisieren
|
||
load_remediation_source_state
|
||
id="$(remediation_full_image_id)"
|
||
PIN_REMEDIATION_TARGET_ID="$id"
|
||
;;
|
||
esac
|
||
echo " FULL_IMAGE_ID=$id"
|
||
echo " Tag=$tag"
|
||
# Punkt 6: Image-inspect + Tag->ID Gate VOR save
|
||
echo " [inspect] Tag -> ID Gate:"
|
||
local tagid
|
||
tagid="$(docker image inspect -f '{{.Id}}' "$tag" 2>/dev/null || true)"
|
||
if [ -z "$tagid" ]; then
|
||
die "archive($role): Source Image nicht inspectable (SOURCE IMAGE NOT INSPECTABLE — FAIL CLOSED)"
|
||
fi
|
||
[ "$tagid" = "$id" ] || die "archive($role): Tag->ID Gate FAIL ($tagid != $id) (FAIL CLOSED)"
|
||
echo " OK ($tagid)"
|
||
# Punkt 7: Immutability — Archiv-Session darf NICHT existieren
|
||
local sdir
|
||
sdir="$(role_arch_dir "$role")" # = $ARCHIVE_ROOT/<recovery|candidates>
|
||
local sid
|
||
case "$role" in
|
||
recovery) sid="$RECOVERY_ARCHIVE_ID" ;;
|
||
remediation) sid="${id#sha256:}"; sid="${sid:0:12}" ;;
|
||
esac
|
||
local base="$sdir/$sid"
|
||
# Fail-closed wenn bereits existiert (kein Overwrite)
|
||
if [ -e "$base" ]; then die "archive($role): Session-Ordner existiert bereits $base (KEIN Overwrite, FAIL CLOSED)"; fi
|
||
if [ "$RUN_MODE" = "DRY_RUN" ]; then
|
||
echo "DRY_RUN_PASS: archive($role) wuerde anlegen: $base/ (image.tar + sha256 + session.meta + app.manifest + runtime-config.txt). Keine Aktion."
|
||
exit 0
|
||
fi
|
||
mode_or_die
|
||
# MUTATION: Archive anlegen + docker save
|
||
mkdir -p "$base" || die "archive($role): Session-Ordner nicht anlegbar (FAIL CLOSED)"
|
||
# Session.meta ZUSAMMENBAUEN VOR save (fuer Size/SHA brauchen aber erst nach save)
|
||
# Reihenfolge: save -> size/sha -> meta -> manifest -> runtime
|
||
echo " [save] docker image save $id > image.tar"
|
||
if ! docker image save "$id" > "$base/image.tar" 2>/dev/null; then
|
||
rm -rf "$base"; die "archive($role): docker save fehlgeschlagen (FAIL CLOSED)"
|
||
fi
|
||
chown root:root "$base/image.tar"; chmod 600 "$base/image.tar"
|
||
local asha asize
|
||
asha="$(sha256sum "$base/image.tar" | awk '{print $1}')"
|
||
asize="$(stat -c '%s' "$base/image.tar")"
|
||
# SHAs der Metadaten (zur Laufzeit, aus Build-Session / Host-SoT)
|
||
# HINWEIS: APP_MANIFEST_SHA_CUR / RUNTIME_CONFIG_SHA_CUR werden hier NICHT
|
||
# berechnet — app.manifest und runtime-config.txt existieren zu diesem Zeitpunkt
|
||
# noch nicht (werden erst unten erzeugt). sha256sum auf nicht-existente Datei
|
||
# wuerde unter set -euo pipefail mit EXIT=1 abbrechen (bewiesener Root Cause).
|
||
# Die korrekte Berechnung erfolgt NACH write_runtime_config/write_app_manifest.
|
||
HOST_SOT_RUNNER_SHA_CUR="$(sha256sum /opt/historical-v2/app/run_backfill_year.py 2>/dev/null | awk '{print $1}')"
|
||
# (RAIN Hotfix) Branch-spezifische Build-Input-Semantik: Recovery nutzt build_context_manifest
|
||
# (RECOVERY_BUILD_CONTEXT), Remediation nutzt remediation_expected_build_input_sha
|
||
# (REMEDIATION_BUILD_CONTEXT, Gold-Source + autorisierter Runner-Overlay). KEIN globaler Tausch.
|
||
case "$role" in
|
||
recovery) BUILD_INPUT_SHA_CUR="$(build_context_manifest 2>/dev/null | sha256sum | awk '{print $1}')" ;;
|
||
remediation) BUILD_INPUT_SHA_CUR="$(remediation_expected_build_input_sha)" ;;
|
||
*) die "archive: unbekannte Rolle $role (FAIL CLOSED)" ;;
|
||
esac
|
||
WRAPPER_SHA_CUR="$(sha256sum /usr/local/bin/rq-historical | awk '{print $1}')"
|
||
# Reihenfolge (Punkt 5): erst Manifest + Runtime erzeugen, DANN deren SHAs
|
||
# berechnen, DANN gen_archive_meta mit allen Pflicht-SHAs aufrufen.
|
||
# Der fruehere gen_archive_meta-Aufruf VOR write_runtime_config/write_app_manifest
|
||
# las APP_MANIFEST_SHA_CUR/RUNTIME_CONFIG_SHA_CUR, die zu diesem Zeitpunkt noch
|
||
# nicht gesetzt waren -> unter set -u "unbound variable" (bewiesener Root Cause).
|
||
write_runtime_config "$role" "$base"
|
||
write_app_manifest "$role" "$base" "$id"
|
||
# SHA der im aktuellen Vorgang erzeugten Metadaten (Punkt 3/4/7)
|
||
APP_MANIFEST_SHA_CUR="$(sha256sum "$base/app.manifest" | awk '{print $1}')"
|
||
RUNTIME_CONFIG_SHA_CUR="$(sha256sum "$base/runtime-config.txt" | awk '{print $1}')"
|
||
# Validierung: exakt 64 hex, Datei existiert, reguläre Datei, kein Symlink
|
||
[[ "$APP_MANIFEST_SHA_CUR" =~ ^[a-f0-9]{64}$ ]] || die "archive($role): APP_MANIFEST_SHA_CUR ungueltig (FAIL CLOSED)"
|
||
[[ "$RUNTIME_CONFIG_SHA_CUR" =~ ^[a-f0-9]{64}$ ]] || die "archive($role): RUNTIME_CONFIG_SHA_CUR ungueltig (FAIL CLOSED)"
|
||
[ -f "$base/app.manifest" ] && [ ! -L "$base/app.manifest" ] || die "archive($role): app.manifest fehlt/ist Symlink (FAIL CLOSED)"
|
||
[ -f "$base/runtime-config.txt" ] && [ ! -L "$base/runtime-config.txt" ] || die "archive($role): runtime-config.txt fehlt/ist Symlink (FAIL CLOSED)"
|
||
gen_archive_meta "$role" "$base" "$id" "$asize" "$asha"
|
||
echo "ARCHIVE($role) OK: $base/ (${asize} bytes, sha256 ${asha:0:16}...)"
|
||
echo " image.tar.sha256:"
|
||
sha256sum "$base/image.tar" > "$base/image.tar.sha256"
|
||
chown root:root "$base/image.tar.sha256"; chmod 600 "$base/image.tar.sha256"
|
||
}
|
||
|
||
# pin-recovery-image / pin-remediation-image
|
||
pin_image() { # $1 = role
|
||
local role="$1"
|
||
expect_zero "${@:2}"
|
||
echo "=== PIN-$role ($RUN_MODE) ==="
|
||
local name tag id
|
||
name="$(role_pin "$role")"
|
||
tag="$(role_tag "$role")"
|
||
case "$role" in
|
||
recovery)
|
||
# Aktive Recovery-Identitaet aus kontrolliertem State (Punkt 19). Kein d2db.
|
||
load_recovery_state
|
||
id="$RECOVERY_GOLD_IMAGE_ID"
|
||
;;
|
||
remediation)
|
||
remediation_session || die "pin(remediation): keine gueltige Build-Session (FAIL CLOSED)"
|
||
id="$(remediation_full_image_id)"
|
||
PIN_REMEDIATION_TARGET_ID="$id"
|
||
;;
|
||
esac
|
||
# Punkt 11: Containername darf NICHT existieren (kein Delete/Overwrite/Replace)
|
||
if docker container inspect "$name" >/dev/null 2>&1; then
|
||
# Identitaet pruefen; wenn korrekt -> ok (keine Aenderung); wenn falsch -> FAIL CLOSED
|
||
local existing
|
||
existing="$(docker container inspect -f '{{.Image}}' "$name" 2>/dev/null)"
|
||
if pin_container_identity_ok "$role" "$existing"; then
|
||
echo "PIN($role): Container existiert bereits korrekt (Image=$existing). KEINE Aenderung."
|
||
exit 0
|
||
else
|
||
die "pin($role): Container $name existiert mit falschem Image ($existing). Kein Replace (FAIL CLOSED)"
|
||
fi
|
||
fi
|
||
echo " FULL_IMAGE_ID=$id"
|
||
# Punkt 6: Image muss inspectable sein (Source Gate)
|
||
if ! docker image inspect "$id" >/dev/null 2>&1; then
|
||
die "pin($role): Source Image nicht inspectable (SOURCE IMAGE NOT INSPECTABLE — FAIL CLOSED)"
|
||
fi
|
||
if [ "$RUN_MODE" = "DRY_RUN" ]; then
|
||
echo "DRY_RUN_PASS: pin($role) wuerde docker create --name $name --network none --restart no $id (keine Aktion)."
|
||
exit 0
|
||
fi
|
||
mode_or_die
|
||
# MUTATION: minimaler Stopped-Container (Punkt 8/10)
|
||
echo " [create] docker create --name $name --network none --restart no $id"
|
||
if ! docker create --name "$name" --network none --restart no "$id" >/dev/null 2>&1; then
|
||
die "pin($role): docker create fehlgeschlagen (FAIL CLOSED)"
|
||
fi
|
||
echo "PIN($role) OK: $name -> $id (Stopped, Network=none, Restart=no)"
|
||
}
|
||
|
||
# verify-recovery-pin / verify-remediation-pin / verify-...-archive / verify-...-ready
|
||
verify_pin_cmd() { # $1 = role (read-only)
|
||
local role="$1"; expect_zero "${@:2}"
|
||
echo "=== VERIFY-PIN($role) ==="
|
||
if [ "$role" = "recovery" ]; then load_recovery_state; fi
|
||
if verify_pin "$role"; then echo "VERIFY-PIN($role): PASS"; exit 0
|
||
else die "VERIFY-PIN($role): FAIL (FAIL CLOSED)"; fi
|
||
}
|
||
verify_archive_cmd() { # $1 = role (read-only)
|
||
local role="$1"; expect_zero "${@:2}"
|
||
echo "=== VERIFY-ARCHIVE($role) ==="
|
||
if [ "$role" = "recovery" ]; then load_recovery_state; fi
|
||
verify_archive "$role" || die "VERIFY-ARCHIVE($role): FAIL (FAIL CLOSED)"
|
||
exit 0
|
||
}
|
||
|
||
# =============================================================================
|
||
# cleanup-partial-recovery-archive (Phase 13.5 · Recovery Archive Hotfix)
|
||
# Kontrollierter Cleanup eines NACHWEISLICH unvollstaendigen Recovery-Archive-
|
||
# Ordners. KEINE freie Pfadangabe. Nur der exakte Partial-Ordner fuer
|
||
# CURRENT_RECOVERY_FULL_IMAGE_ID. Verified Archive wird strikt DENY.
|
||
# Safety-Model (Punkt 9): Pfad exakt unter festem Root, Ordnername exakt aus
|
||
# CURRENT_RECOVERY_FULL_IMAGE_ID abgeleitet, kein freier Pfad, kein Symlink,
|
||
# kein Mountpoint, kein verified Archive, session.meta fehlt ODER INVALID/PARTIAL,
|
||
# CURRENT_RECOVERY_FULL_IMAGE_ID stimmt weiterhin, keine andere Recovery-ID.
|
||
# =============================================================================
|
||
cleanup_partial_recovery_archive() {
|
||
expect_zero "$@"
|
||
echo "=== CLEANUP-PARTIAL-RECOVERY-ARCHIVE ($RUN_MODE) ==="
|
||
# Aktive Recovery-Identitaet aus kontrolliertem State (Punkt 17). Kein d2db.
|
||
load_recovery_state
|
||
local sdir sid base
|
||
sdir="$(role_arch_dir recovery)" # = $ARCHIVE_ROOT/recovery
|
||
sid="$RECOVERY_ARCHIVE_ID" # exakt aus CURRENT_RECOVERY_FULL_IMAGE_ID abgeleitet
|
||
base="$sdir/$sid"
|
||
echo " TARGET=$base"
|
||
echo " CURRENT_RECOVERY_FULL_IMAGE_ID=$RECOVERY_GOLD_IMAGE_ID"
|
||
# --- Safety-Checks (alle FAIL CLOSED) ---
|
||
# 1. Root exakt unter festem Recovery Archive Root
|
||
[ "$sdir" = "$ARCHIVE_ROOT/recovery" ] || die "cleanup: Root nicht exakt $ARCHIVE_ROOT/recovery (FAIL CLOSED)"
|
||
# 2. Ordnername exakt 12 hex (aus FULL_IMAGE_ID abgeleitet, kein freier Pfad)
|
||
[[ "$sid" =~ ^[a-f0-9]{12}$ ]] || die "cleanup: Ordnername ungueltig '$sid' (FAIL CLOSED)"
|
||
# 3. Ziel existiert
|
||
[ -e "$base" ] || die "cleanup: Ziel existiert nicht $base (FAIL CLOSED)"
|
||
# 4. kein Symlink
|
||
[ ! -L "$base" ] || die "cleanup: Ziel ist Symlink (FAIL CLOSED)"
|
||
# 5. kein Mountpoint
|
||
if mountpoint -q "$base" 2>/dev/null; then die "cleanup: Ziel ist Mountpoint (FAIL CLOSED)"; fi
|
||
# 6. CURRENT_RECOVERY_FULL_IMAGE_ID stimmt weiterhin (Tag->ID Gate)
|
||
local tagid
|
||
tagid="$(docker image inspect -f '{{.Id}}' "$RECOVERY_GOLD_TAG_FINAL" 2>/dev/null || true)"
|
||
[ "$tagid" = "$RECOVERY_GOLD_IMAGE_ID" ] || die "cleanup: Tag->ID Gate FAIL ($tagid != $RECOVERY_GOLD_IMAGE_ID) (FAIL CLOSED)"
|
||
# 7. kein verified Archive (Punkt 10: verified Archive strikt DENY)
|
||
# verify_archive macht bei FAIL intern exit 2 -> in Subshell kapseln,
|
||
# damit ein INVALID/PARTIAL-Ergebnis das Skript nicht beendet.
|
||
local verify_rc=0
|
||
( verify_archive recovery ) >/dev/null 2>&1 || verify_rc=$?
|
||
if [ "$verify_rc" -eq 0 ]; then
|
||
die "cleanup: Archive ist VERIFIED (verify-recovery-archive PASS) — Cleanup strikt DENY (FAIL CLOSED)"
|
||
fi
|
||
# 8. Klassifikation: PARTIAL wenn session.meta fehlt; sonst INVALID (verify FAIL)
|
||
local classification
|
||
if [ ! -f "$base/session.meta" ]; then
|
||
classification="PARTIAL"
|
||
else
|
||
classification="INVALID"
|
||
fi
|
||
echo " CLASSIFICATION=$classification"
|
||
# 9. Nur PARTIAL/INVALID darf bereinigt werden (verified bereits oben DENY)
|
||
[ "$classification" = "PARTIAL" ] || [ "$classification" = "INVALID" ] || die "cleanup: Klassifikation ungueltig (FAIL CLOSED)"
|
||
# 10. keine andere Recovery-ID betroffen — nur exakt dieser sid
|
||
local other
|
||
for other in "$sdir"/*/; do
|
||
[ -d "$other" ] || continue
|
||
local oname="${other%/}"; oname="${oname##*/}"
|
||
[ "$oname" = "$sid" ] && continue
|
||
die "cleanup: andere Recovery-ID betroffen: $oname (FAIL CLOSED)"
|
||
done
|
||
if [ "$RUN_MODE" = "DRY_RUN" ]; then
|
||
echo " WOULD_REMOVE=$base"
|
||
echo "DRY_RUN_PASS: cleanup-partial-recovery-archive wuerde $base entfernen (keine Aktion)."
|
||
exit 0
|
||
fi
|
||
mode_or_die
|
||
# MUTATION: kontrollierter Cleanup des Partial-Ordners
|
||
rm -rf -- "$base" || die "cleanup: Entfernen fehlgeschlagen (FAIL CLOSED)"
|
||
[ ! -e "$base" ] || die "cleanup: Ziel nach Entfernen noch vorhanden (FAIL CLOSED)"
|
||
echo "CLEANUP-PARTIAL-RECOVERY-ARCHIVE OK: $base entfernt"
|
||
echo " Retry bereit: archive-recovery-image wird denselben CURRENT_RECOVERY_FULL_IMAGE_ID verwenden."
|
||
}
|
||
verify_ready_cmd() { # $1 = role (read-only)
|
||
local role="$1"; expect_zero "${@:2}"
|
||
verify_ready "$role" || die "$(case $role in recovery) echo 'RECOVERY READY';; remediation) echo 'CANDIDATE READY';; esac): FAIL (FAIL CLOSED)"
|
||
exit 0
|
||
}
|
||
|
||
# cleanup-partial-remediation-staging (Phase 13.5 · Orphan Remediation Staging Cleanup)
|
||
# Kontrollierter Cleanup eines NACHWEISLICH verwaisten Remediation-Staging-Ordners.
|
||
# KEINE freie Pfadangabe. Zielidentitaet vollstaendig aus kontrolliertem State
|
||
# (Gold-Source-Session) abgeleitet. Fail-closed bei jeder Abweichung.
|
||
# Safety-Model (Punkt 3-8): Path Hard Gates, Session Binding an feste autorisierte
|
||
# Gold-Source-Session, Orphan-Proof (kein aktiver Image State/Image/Archive/Pin/Prozess/Mount),
|
||
# Content-Verify gegen erwartetes Build-Input, Build-Support-Verify, Build-Input-SHA.
|
||
# =============================================================================
|
||
cleanup_partial_remediation_staging() {
|
||
expect_zero "$@"
|
||
echo "=== CLEANUP-PARTIAL-REMEDIATION-STAGING ($RUN_MODE) ==="
|
||
# Session Binding: feste autorisierte Gold-Source-Session aus kontrolliertem State
|
||
load_remediation_source_state
|
||
echo " SOURCE_SESSION=$REMEDIATION_SESSION_ID"
|
||
# Punkt 4: Session muss exakt die feste autorisierte Gold-Source-Session sein
|
||
[ "$REMEDIATION_SESSION_ID" = "20260830_104127" ] || die "cleanup: Session $REMEDIATION_SESSION_ID != autorisierte 20260830_104127 (FAIL CLOSED)"
|
||
local sdir base
|
||
sdir="$REMEDIATION_STAGING_ROOT"
|
||
base="$REMEDIATION_STAGING_DIR"
|
||
echo " STAGING_PATH=$base"
|
||
# --- PATH HARD GATES (Punkt 3) ---
|
||
# 1. Pfad ist absolut
|
||
case "$base" in /*) ;; *) die "cleanup: Pfad nicht absolut $base (FAIL CLOSED)";; esac
|
||
# 2. Pfad exakt aus State abgeleitet
|
||
[ "$base" = "$REMEDIATION_STAGING_ROOT/$REMEDIATION_SESSION_ID" ] || die "cleanup: Pfad nicht exakt aus State (FAIL CLOSED)"
|
||
# 3. Parent exakt Staging-Root
|
||
[ "$sdir" = "/opt/rq-historical-remediation-staging" ] || die "cleanup: Parent nicht exakt Staging-Root (FAIL CLOSED)"
|
||
# 4. Ziel ist nicht /
|
||
[ "$base" != "/" ] || die "cleanup: Ziel ist / (FAIL CLOSED)"
|
||
# 5. Ziel ist nicht Parent selbst
|
||
[ "$base" != "$sdir" ] || die "cleanup: Ziel ist Parent selbst (FAIL CLOSED)"
|
||
# 6. Ziel enthaelt keine ..
|
||
case "$base" in *..*) die "cleanup: Ziel enthaelt .. (FAIL CLOSED)";; esac
|
||
# 7. Ziel ist kein Symlink
|
||
[ ! -L "$base" ] || die "cleanup: Ziel ist Symlink (FAIL CLOSED)"
|
||
# 8. Parent ist kein Symlink
|
||
[ ! -L "$sdir" ] || die "cleanup: Parent ist Symlink (FAIL CLOSED)"
|
||
# 9. Ziel ist Directory
|
||
[ -d "$base" ] || die "cleanup: Ziel ist kein Directory (FAIL CLOSED)"
|
||
# 10. kein Mountpoint (Ziel + Parent)
|
||
if mountpoint -q "$base" 2>/dev/null; then die "cleanup: Ziel ist Mountpoint (FAIL CLOSED)"; fi
|
||
if mountpoint -q "$sdir" 2>/dev/null; then die "cleanup: Parent ist Mountpoint (FAIL CLOSED)"; fi
|
||
# 11. kein Bind Mount / Docker Volume Mount
|
||
if grep -q " $base " /proc/mounts 2>/dev/null; then die "cleanup: Ziel ist Bind/Volume Mount (FAIL CLOSED)"; fi
|
||
if grep -q " $sdir " /proc/mounts 2>/dev/null; then die "cleanup: Parent ist Bind/Volume Mount (FAIL CLOSED)"; fi
|
||
echo " PATH_GATES=PASS"
|
||
# --- ORPHAN CLASSIFICATION (Punkt 5) ---
|
||
# A) Kein aktiver Remediation Image State (stale State mit fehlendem Image = kein aktiver)
|
||
if remediation_session; then
|
||
local sid; sid="$(remediation_full_image_id)"
|
||
if docker image inspect "$sid" >/dev/null 2>&1; then
|
||
die "cleanup: aktiver Remediation Image State mit existierendem Image $sid (FAIL CLOSED)"
|
||
else
|
||
echo " A) Remediation Image State stale (Image $sid fehlt) -> kein aktiver State"
|
||
fi
|
||
else
|
||
echo " A) Kein Remediation Image State"
|
||
fi
|
||
# B) Kein aktuelles Remediation Image aus dieser Build-Session
|
||
if docker images --format '{{.Repository}}:{{.Tag}}' | grep -q "candidate-phase13-5-runner-remediation"; then
|
||
die "cleanup: Remediation Image existiert (FAIL CLOSED)"
|
||
fi
|
||
echo " B) Kein Remediation Image"
|
||
# C) VALID-Archive-Semantik (RAIN Hotfix §4): NICHT nackte Directory-Existence.
|
||
# VALID -> strikt DENY (gueltiges Archive blockiert Staging-Cleanup).
|
||
# PARTIAL -> erst sanktionierten cleanup-partial-remediation-archive ausfuehren,
|
||
# dann Retry (Lifecycle-getrennt, bevorzugte Variante).
|
||
# NO -> kein Blocker, Staging-Cleanup faehrt fort.
|
||
# INVALID/UNKNOWN -> FAIL CLOSED.
|
||
local cls
|
||
cls="$(remediation_archive_classification)"
|
||
case "$cls" in
|
||
VALID) die "cleanup: VALID Remediation Archive existiert — Cleanup strikt DENY (FAIL CLOSED)" ;;
|
||
PARTIAL)
|
||
echo " C) PARTIAL_ARCHIVE_PRESENT=YES"
|
||
echo " PARTIAL_ARCHIVE_CLASSIFICATION=SAFE_ORPHAN"
|
||
die "cleanup: PARTIAL Remediation Archive vorhanden — zuerst cleanup-partial-remediation-archive, dann Retry (Lifecycle-getrennt)" ;;
|
||
INVALID) die "cleanup: Archive-Zustand INVALID — FAIL CLOSED, kein automatischer Umgang" ;;
|
||
UNKNOWN) die "cleanup: Archive-Zustand UNKNOWN — FAIL CLOSED, kein automatischer Umgang" ;;
|
||
NO) echo " C) Remediation Archive = NO (kein VALID-Archive-Blocker)" ;;
|
||
*) die "cleanup: ungueltige Archive-Klassifikation (FAIL CLOSED)" ;;
|
||
esac
|
||
# D) Kein Remediation Pin
|
||
if docker ps -a --format '{{.Names}}' | grep -q "rq-historical-pin-phase13-5-runner-remediation"; then
|
||
die "cleanup: Remediation Pin existiert (FAIL CLOSED)"
|
||
fi
|
||
echo " D) Kein Remediation Pin"
|
||
# E) Kein laufender Build-/Runner-Prozess
|
||
if ps aux | grep -E "docker build|run_backfill" | grep -v grep >/dev/null 2>&1; then
|
||
die "cleanup: Build-/Runner-Prozess aktiv (FAIL CLOSED)"
|
||
fi
|
||
echo " E) Kein Build-/Runner-Prozess"
|
||
# F) Kein Docker Build mit diesem Context
|
||
if docker ps --format '{{.Names}}' | grep -qE "build|remediation"; then
|
||
die "cleanup: Build-Container aktiv (FAIL CLOSED)"
|
||
fi
|
||
echo " F) Kein Docker Build"
|
||
# G/H) historical-service/db nutzen Staging nicht als Mount
|
||
for c in historical-service historical-db; do
|
||
if docker inspect "$c" --format '{{range .Mounts}}{{.Source}}{{println}}{{end}}' 2>/dev/null | grep -q "$sdir"; then
|
||
die "cleanup: $c mountet Staging (FAIL CLOSED)"
|
||
fi
|
||
done
|
||
echo " G/H) Kein Service/DB Mount"
|
||
echo " ORPHAN_STAGING=CONFIRMED"
|
||
# --- CONTENT-VALIDIERUNG (Punkt 6) ---
|
||
# Staging gegen erwartetes Build-Input (Gold-Source + Overlay)
|
||
remediation_staging_matches_expected || die "cleanup: Staging != erwartetes Build-Input (FAIL CLOSED)"
|
||
# 62+1 Klassifikation
|
||
remediation_expected_manifest_ok || die "cleanup: erwartetes Build-Input nicht exakt (FAIL CLOSED)"
|
||
echo " CONTENT_VERIFY=PASS (MATCH=$RM_MATCH AUTH=$RM_AUTH UNAUTH=$RM_UNAUTH MISSING=$RM_MISSING EXTRA=$RM_EXTRA)"
|
||
# repository.py == Gold
|
||
local repo_sha
|
||
repo_sha="$(sha256sum "$base/app/persistence/repository.py" | awk '{print $1}')"
|
||
[ "$repo_sha" = "8fff892f52a612f464b5be21c66ef7787a47f2d7d049f815ef87837553c2a228" ] || die "cleanup: repository.py != Gold (FAIL CLOSED)"
|
||
echo " repository.py=8fff892f (Gold)"
|
||
# --- BUILD SUPPORT VALIDIEREN (Punkt 7) ---
|
||
local gs="$GOLD_SOURCE_ROOT/$REMEDIATION_SESSION_ID"
|
||
for bf in Dockerfile requirements.txt docker-compose.yml .dockerignore; do
|
||
local gs_sha st_sha
|
||
gs_sha="$(sha256sum "$gs/$bf" 2>/dev/null | awk '{print $1}')"
|
||
st_sha="$(sha256sum "$base/$bf" 2>/dev/null | awk '{print $1}')"
|
||
[ -n "$gs_sha" ] && [ "$gs_sha" = "$st_sha" ] || die "cleanup: Build-Support $bf != Gold (FAIL CLOSED)"
|
||
done
|
||
echo " BUILD_SUPPORT_VERIFY=PASS"
|
||
# --- EXPECTED BUILD INPUT SHA (Punkt 8) ---
|
||
local ebi; ebi="$(remediation_expected_build_input_sha "$base")"
|
||
[ "$ebi" = "75555a411b64e0ac0ef0f8502180ee0a331bab93748515d24b5c8d8f255f2d84" ] || die "cleanup: Build-Input-SHA $ebi != erwartet (FAIL CLOSED)"
|
||
echo " REMEDIATION_BUILD_INPUT_SHA=$ebi"
|
||
# --- ACTION ---
|
||
if [ "$RUN_MODE" = "DRY_RUN" ]; then
|
||
echo " ACTION=WOULD_REMOVE_STAGING"
|
||
echo "DRY_RUN_PASS: cleanup-partial-remediation-staging wuerde $base entfernen (keine Aktion)."
|
||
exit 0
|
||
fi
|
||
mode_or_die
|
||
# MUTATION: kontrollierter Cleanup des verwaisten Staging-Ordners
|
||
rm -rf -- "$base" || die "cleanup: Entfernen fehlgeschlagen (FAIL CLOSED)"
|
||
[ ! -e "$base" ] || die "cleanup: Ziel nach Entfernen noch vorhanden (FAIL CLOSED)"
|
||
echo "CLEANUP-PARTIAL-REMEDIATION-STAGING OK: $base entfernt"
|
||
echo " Retry bereit: build-runner-remediation wird neues Staging materialisieren."
|
||
}
|
||
|
||
|
||
# =============================================================================
|
||
# cleanup-partial-remediation-archive (Phase 13.5 · RAIN Hotfix)
|
||
# Kontrollierter Cleanup eines NACHWEISLICH unvollstaendigen / verwaisten
|
||
# Remediation-Archive-Ordners. KEINE freie Pfadangabe, KEINE freie Session.
|
||
# Zielidentitaet vollstaendig aus kontrolliertem State / Build-Session abgeleitet.
|
||
# Sicher entfernbar NUR klar klassifizierbare PARTIAL-Orphans. Valid/Invalid/
|
||
# Unknown/aktive Zustände -> FAIL CLOSED.
|
||
# =============================================================================
|
||
# --- Archive-Zustands-Klassifikation (Mission §2) ---
|
||
# A) NO_ARCHIVE : kein Archive-Ziel vorhanden.
|
||
# B) PARTIAL_ARCHIVE : Verzeichnis vorhanden, aber mind. ein Pflichtartefakt fehlt
|
||
# oder ist unvollstaendig (z.B. image.tar fehlt/leer).
|
||
# C) VALID_ARCHIVE : alle Pflichtartefakte da UND konsistent gebunden UND verify PASS.
|
||
# D) INVALID_ARCHIVE : vollstaendig erscheinend, aber SHA/Metadata/Binding falsch.
|
||
# E) UNKNOWN_ARCHIVE_STATE : nicht sicher klassifizierbar.
|
||
# Fail-closed: VALID blockiert Cleanup; PARTIAL separat bereinigbar; INVALID/UNKNOWN FAIL.
|
||
remediation_archive_classification() { # echo: NO|PARTIAL|VALID|INVALID|UNKNOWN
|
||
local sdir base img meta manifest runtime
|
||
sdir="$(role_arch_dir remediation)" # = $ARCHIVE_ROOT/candidates
|
||
local sid
|
||
if remediation_session; then
|
||
local ridsess; ridsess="$(remediation_full_image_id)"
|
||
sid="${ridsess#sha256:}"; sid="${sid:0:12}"
|
||
else
|
||
sid=""
|
||
fi
|
||
echo " REMEDIATION_ARCHIVE_BASE=$sdir" >&2
|
||
[ -d "$sdir" ] || { echo "NO"; return; }
|
||
if [ -n "$sid" ]; then
|
||
base="$sdir/$sid"
|
||
else
|
||
local found="" cnt=0
|
||
for d in "$sdir"/*/; do
|
||
[ -d "$d" ] || continue
|
||
cnt=$((cnt+1)); found="${d%/}"; found="${found##*/}"
|
||
done
|
||
if [ "$cnt" = "0" ]; then echo "NO"; return; fi
|
||
base="$sdir/$found"
|
||
fi
|
||
[ -d "$base" ] || { echo "NO"; return; }
|
||
img="$base/image.tar"; meta="$base/session.meta"; manifest="$base/app.manifest"; runtime="$base/runtime-config.txt"
|
||
# --- VALID-Pruefung (Mission §3) ---
|
||
local ok=1
|
||
[ -f "$img" ] && [ ! -L "$img" ] && [ -s "$img" ] || ok=0
|
||
[ -f "$meta" ] && [ ! -L "$meta" ] || ok=0
|
||
[ -f "$manifest" ] && [ ! -L "$manifest" ] || ok=0
|
||
[ -f "$runtime" ] && [ ! -L "$runtime" ] || ok=0
|
||
if [ "$ok" = "1" ]; then
|
||
local want got
|
||
want="$(awk -F'=' '$1=="ARCHIVE_SHA256"{print $2}' "$meta" 2>/dev/null)"
|
||
got="$(sha256sum "$img" 2>/dev/null | awk '{print $1}')"
|
||
if [ -n "$want" ] && [[ "$want" =~ ^[a-f0-9]{64}$ ]] && [ "$got" = "$want" ]; then
|
||
local role
|
||
role=remediation
|
||
if ( verify_archive remediation ) >/dev/null 2>&1; then echo "VALID"; return; fi
|
||
echo "INVALID"; return
|
||
fi
|
||
echo "INVALID"; return
|
||
fi
|
||
# --- PARTIAL vs UNKNOWN (RAIN Hotfix: fail-closed-Semantik) ---
|
||
# PARTIAL nur wenn sicher erkennbar unvollstaendig UND alle Safety-Gates PASS:
|
||
# - kontrollierter candidates-Pfad (base = $sdir/$sid, gebunden an aktive Session)
|
||
# - mindestens ein Archive-Artefakt vorhanden
|
||
# - vollstaendige VALID-Struktur fehlt
|
||
# - kein widerspruechliches/unerwartetes Fremdartefakt
|
||
# - keine Symlinks, keine Mountpoints, keine falsche Image-ID
|
||
# - kein anderes Session-/Image-Metadata-Binding
|
||
# - kein valides vollstaendiges Archive
|
||
# UNKNOWN wenn Zustand nicht sicher klassifizierbar (fail closed).
|
||
local has_any=0
|
||
[ -e "$base/image.tar" ] && has_any=1
|
||
[ -e "$base/session.meta" ] && has_any=1
|
||
[ -e "$base/app.manifest" ] && has_any=1
|
||
[ -e "$base/runtime-config.txt" ] && has_any=1
|
||
[ -e "$base/image.tar.sha256" ] && has_any=1
|
||
[ "$has_any" = "0" ] && { echo "NO"; return; } # kein Artefakt -> NO
|
||
# Safety: keine Symlinks im Ziel (Datei ODER Verzeichnis)
|
||
if [ -L "$base" ] || [ -L "$base/image.tar" ] || [ -L "$base/session.meta" ] || \
|
||
[ -L "$base/app.manifest" ] || [ -L "$base/runtime-config.txt" ] || [ -L "$base/image.tar.sha256" ]; then
|
||
echo "UNKNOWN"; return
|
||
fi
|
||
# Safety: kein Mountpoint / Bind-Mount
|
||
if mountpoint -q "$base" 2>/dev/null || grep -qE " $base( |$)" /proc/mounts 2>/dev/null; then
|
||
echo "UNKNOWN"; return
|
||
fi
|
||
# Safety: keine unerwarteten Extrafiles (nur die 5 bekannten Artefakte erlaubt)
|
||
local extra=""
|
||
for f in "$base"/*; do
|
||
[ -e "$f" ] || continue
|
||
local bn; bn="${f##*/}"
|
||
case "$bn" in
|
||
image.tar|session.meta|app.manifest|runtime-config.txt|image.tar.sha256) : ;;
|
||
*) extra="$extra $bn" ;;
|
||
esac
|
||
done
|
||
[ -n "$extra" ] && { echo "UNKNOWN"; return; } # Fremdartefakt -> UNKNOWN
|
||
# PARTIAL: image.tar valide vorhanden, aber Metadaten-Struktur unvollstaendig
|
||
# (session.meta/app.manifest/runtime-config.txt/image.tar.sha256 fehlt mind. eines)
|
||
if [ -f "$base/image.tar" ] && [ -s "$base/image.tar" ] && \
|
||
{ [ ! -f "$base/session.meta" ] || [ ! -f "$base/app.manifest" ] || \
|
||
[ ! -f "$base/runtime-config.txt" ] || [ ! -f "$base/image.tar.sha256" ]; }; then
|
||
echo "PARTIAL"; return
|
||
fi
|
||
# PARTIAL: image.tar fehlt/leer, aber andere Artefakte vorhanden (orphan)
|
||
if [ ! -f "$base/image.tar" ] || [ ! -s "$base/image.tar" ]; then
|
||
echo "PARTIAL"; return
|
||
fi
|
||
# Sonst: nicht sicher klassifizierbar -> UNKNOWN (fail closed)
|
||
echo "UNKNOWN"; return
|
||
}
|
||
|
||
# cleanup-partial-remediation-archive — dry-run + real (Mission §5/§6/§7)
|
||
cleanup_partial_remediation_archive() {
|
||
expect_zero "$@"
|
||
echo "=== CLEANUP-PARTIAL-REMEDIATION-ARCHIVE ($RUN_MODE) ==="
|
||
local sdir sid base
|
||
sdir="$(role_arch_dir remediation)" # = $ARCHIVE_ROOT/candidates
|
||
if remediation_session; then
|
||
local ridsess; ridsess="$(remediation_full_image_id)"
|
||
sid="${ridsess#sha256:}"; sid="${sid:0:12}"
|
||
else
|
||
sid=""
|
||
fi
|
||
if [ -n "$sid" ]; then
|
||
base="$sdir/$sid"
|
||
else
|
||
local found="" cnt=0
|
||
for d in "$sdir"/*/; do
|
||
[ -d "$d" ] || continue
|
||
cnt=$((cnt+1)); found="${d%/}"; found="${found##*/}"
|
||
done
|
||
if [ "$cnt" = "0" ]; then die "cleanup-partial-remediation-archive: kein Archive-Ordner (FAIL CLOSED)"; fi
|
||
if [ "$cnt" != "1" ]; then die "cleanup-partial-remediation-archive: mehrere Candidate-Ordner — Ziel nicht eindeutig (FAIL CLOSED)"; fi
|
||
sid="$found"; base="$sdir/$sid"
|
||
fi
|
||
echo " TARGET=$base"
|
||
# --- SAFETY GATES (Mission §6) ---
|
||
[ "$sdir" = "$ARCHIVE_ROOT/candidates" ] || die "cleanup-partial: Root != $ARCHIVE_ROOT/candidates (FAIL CLOSED)"
|
||
[[ "$sid" =~ ^[a-f0-9]{12}$ ]] || die "cleanup-partial: Ordnername ungueltig '$sid' (FAIL CLOSED)"
|
||
[ -e "$base" ] || { echo " PARTIAL_ARCHIVE=NO (kein Ziel)"; echo "DRY_RUN_PASS: nichts zu entfernen."; exit 0; }
|
||
[ ! -L "$base" ] || die "cleanup-partial: Ziel ist Symlink (FAIL CLOSED)"
|
||
if mountpoint -q "$base" 2>/dev/null; then die "cleanup-partial: Ziel ist Mountpoint (FAIL CLOSED)"; fi
|
||
if mountpoint -q "$sdir" 2>/dev/null; then die "cleanup-partial: Parent ist Mountpoint (FAIL CLOSED)"; fi
|
||
if grep -qE " $base( |$)" /proc/mounts 2>/dev/null; then die "cleanup-partial: Ziel ist Bind/Volume Mount (FAIL CLOSED)"; fi
|
||
if ps aux 2>/dev/null | grep -E "docker save|docker build|docker tag" | grep -v grep | grep -q "$sdir"; then
|
||
die "cleanup-partial: Build-/Save-Prozess aktiv (FAIL CLOSED)"
|
||
fi
|
||
if docker ps --format '{{.Names}}' 2>/dev/null | grep -qE "build|remediation"; then
|
||
die "cleanup-partial: Build-Container aktiv (FAIL CLOSED)"
|
||
fi
|
||
# --- KLASSIFIKATION (Mission §2/§7) ---
|
||
local cls
|
||
cls="$(remediation_archive_classification)"
|
||
echo " CLASSIFICATION=$cls"
|
||
case "$cls" in
|
||
VALID) die "cleanup-partial: Archive ist VALID/verify PASS — Cleanup strikt DENY (FAIL CLOSED)" ;;
|
||
INVALID) die "cleanup-partial: Archive INVALID (voll, aber SHA/Metadata/Binding falsch) — FAIL CLOSED, kein Cleanup" ;;
|
||
UNKNOWN) die "cleanup-partial: Archive-Zustand UNKNOWN — FAIL CLOSED, kein Cleanup" ;;
|
||
PARTIAL) : ;;
|
||
NO) echo " PARTIAL_ARCHIVE=NO"; echo "DRY_RUN_PASS: kein partielles Archive."; exit 0 ;;
|
||
*) die "cleanup-partial: ungueltige Klassifikation (FAIL CLOSED)" ;;
|
||
esac
|
||
echo " PARTIAL_ARCHIVE=CONFIRMED"
|
||
echo " IMAGE_TAR=$( [ -f "$base/image.tar" ] && [ -s "$base/image.tar" ] && echo PRESENT || echo ABSENT )"
|
||
echo " ACTIVE_REMEDIATION_IMAGE=$( if remediation_session && docker image inspect "$(remediation_full_image_id)" >/dev/null 2>&1; then echo PRESENT; else echo NONE; fi )"
|
||
echo " ACTIVE_PIN=$( docker ps -a --format '{{.Names}}' 2>/dev/null | grep -Eq 'rq-historical-pin-phase13-5-runner-remediation' && echo PRESENT || echo NONE )"
|
||
echo " ACTIVE_CONTAINER=$( docker ps --format '{{.Names}}' 2>/dev/null | grep -q 'remediation' && echo PRESENT || echo NONE )"
|
||
echo " VALID_ARCHIVE=NO"
|
||
if [ "$RUN_MODE" = "DRY_RUN" ]; then
|
||
echo " ACTION=WOULD_REMOVE_PARTIAL_REMEDIATION_ARCHIVE"
|
||
echo "DRY_RUN_PASS: cleanup-partial-remediation-archive wuerde $base entfernen (keine Aktion)."
|
||
echo " ZERO MUTATION."
|
||
exit 0
|
||
fi
|
||
mode_or_die
|
||
rm -rf -- "$base" || die "cleanup-partial: Entfernen fehlgeschlagen (FAIL CLOSED)"
|
||
[ ! -e "$base" ] || die "cleanup-partial: Ziel nach Entfernen noch vorhanden (FAIL CLOSED)"
|
||
echo "CLEANUP-PARTIAL-REMEDIATION-ARCHIVE OK: $base entfernt"
|
||
echo " Retry bereit: cleanup-partial-remediation-staging ist nun logisch freigegeben."
|
||
}
|
||
|
||
# restore-recovery-image-archive (DESIGN; realer Restore NICHT in dieser Mission ausfuehren)
|
||
restore_recovery_dry() { # dry-run
|
||
expect_zero "$@"
|
||
echo "=== RESTORE-RECOVERY-IMAGE-ARCHIVE (DRY-RUN — DESIGN; kein echter Restore) ==="
|
||
load_recovery_state
|
||
local sdir sid base img
|
||
sdir="$(role_arch_dir recovery)"
|
||
sid="$RECOVERY_ARCHIVE_ID"
|
||
base="$sdir/$sid"
|
||
[ -d "$base" ] || die "restore-dry: Archiv-Session fehlt $base (FAIL CLOSED)"
|
||
echo " 1. Archiv SHA pruefen: $(sha256sum "$base/image.tar" | awk '{print $1}')"
|
||
echo " 2. Metadaten pruefen: session.meta vorhanden"
|
||
echo " 3. docker load (in realer Mission; hier NUR Dry-Run)"
|
||
echo " 4. Resulting Image inspect"
|
||
echo " 5. Application Manifest pruefen"
|
||
echo " 6. Runtime Config pruefen"
|
||
echo " 7. Tag/Identity pruefen"
|
||
echo " 8. danach Pin Verify"
|
||
echo " HINWEIS (Punkt 15): Wenn neu geladene FULL_IMAGE_ID != Original-ID bei gleicher App-Identitaet -> NICHT automatisch identisch; sauber Docker-Image-Identity vs Application/Runtime-Equivalence unterscheiden."
|
||
echo "RESTORE-RECOVERY-DRY-RUN: DESIGN OK (kein echter Restore ausgefuehrt)"
|
||
exit 0
|
||
}
|
||
|
||
# restore-recovery-image-archive (real — in dieser Mission NICHT ausfuehren; nur definiert)
|
||
restore_recovery_real() {
|
||
expect_zero "$@"
|
||
die "restore-recovery-image-archive: Realer Restore ist in dieser Mission NICHT erlaubt (HARTER STOPP Punkt 30). NUR dry-run (FAIL CLOSED)"
|
||
}
|
||
|
||
# =============================================================================
|
||
# DEPLOYED-GOLD-SOURCE-RECOVERY CONTROL PLANE (Phase 13.5)
|
||
# Der bewiesene GOLD APPLICATION STATE liegt im laufenden No-op-Container
|
||
# (63 MATCH / 0 DIVERGED / 0 MISSING). Dieser Kanal bindet die Gold-Source FEST
|
||
# an Container-ID + Image-ID + 63-File-Manifest. KEIN latest, KEIN full_sot_backup(),
|
||
# KEINE freien Inputs. Nur Source-Recovery, KEIN Docker-Build/Commit/Save/Load/
|
||
# Create/Run. Immutable Target. FAIL CLOSED bei jeder Abweichung.
|
||
# =============================================================================
|
||
|
||
# Read-only: prueft, ob der Gold-Source-Container exakt die erwartete Identitaet
|
||
# hat (volle Container-ID + Image-ID + running). FAIL CLOSED bei Abweichung.
|
||
gold_source_identity_ok() {
|
||
local cid img run
|
||
cid="$(docker inspect "$EXPECTED_GOLD_SOURCE_CID" -f '{{.Id}}' 2>/dev/null)"
|
||
img="$(docker inspect "$EXPECTED_GOLD_SOURCE_CID" -f '{{.Image}}' 2>/dev/null)"
|
||
run="$(docker inspect "$EXPECTED_GOLD_SOURCE_CID" -f '{{.State.Running}}' 2>/dev/null)"
|
||
[ "$cid" = "$EXPECTED_GOLD_SOURCE_CID" ] || { echo " [FAIL] Container-ID abweichend: $cid"; return 1; }
|
||
[ "$img" = "$EXPECTED_GOLD_SOURCE_IMAGE_ID" ] || { echo " [FAIL] Image-ID abweichend: $img"; return 1; }
|
||
[ "$run" = "true" ] || { echo " [FAIL] Container nicht running"; return 1; }
|
||
echo " [OK] Container-ID == $EXPECTED_GOLD_SOURCE_CID"
|
||
echo " [OK] Image-ID == $EXPECTED_GOLD_SOURCE_IMAGE_ID"
|
||
echo " [OK] Container running"
|
||
return 0
|
||
}
|
||
|
||
# Read-only: prueft das 63-File Application Manifest des Containers gegen Gold.
|
||
# Erwartung: 63 MATCH / 0 DIVERGED / 0 MISSING. FAIL CLOSED sonst.
|
||
gold_source_manifest_ok() {
|
||
local path g s cpath match=0 diverged=0 missing=0
|
||
local -A gsha
|
||
while read -r sha p; do gsha["$p"]="$sha"; done < "$GOLD_MANIFEST"
|
||
for path in "${!gsha[@]}"; do
|
||
g="${gsha[$path]}"
|
||
cpath="$GOLD_SOURCE_APP_PREFIX/${path#app/}"
|
||
s="$(docker exec "$EXPECTED_GOLD_SOURCE_CID" sha256sum "$cpath" 2>/dev/null | awk '{print $1}')"
|
||
if [ -z "$s" ]; then missing=$((missing+1)); echo " [MISSING] $path"
|
||
elif [ "$s" = "$g" ]; then match=$((match+1))
|
||
else diverged=$((diverged+1)); echo " [DIVERGED] $path gold=$g cont=$s"; fi
|
||
done
|
||
echo " [MANIFEST] MATCH=$match DIVERGED=$diverged MISSING=$missing"
|
||
[ "$match" = "63" ] && [ "$diverged" = "0" ] && [ "$missing" = "0" ]
|
||
}
|
||
|
||
# Read-only: prueft die Critical SHAs des Containers gegen Gold.
|
||
# Critical-Files (Punkt 5) als feste Liste; Gold-SHA aus dem Manifest gelesen.
|
||
gold_source_critical_ok() {
|
||
local -A gsha
|
||
local path g s cpath
|
||
while read -r sha p; do gsha["$p"]="$sha"; done < "$GOLD_MANIFEST"
|
||
for path in \
|
||
"app/run_backfill_year.py" \
|
||
"app/persistence/repository.py" \
|
||
"app/backfill.py" \
|
||
"app/pipeline_v2.py" \
|
||
"app/trust/__init__.py" \
|
||
"app/trust/provenance.py" \
|
||
"app/trust/trust.py" \
|
||
"app/trust/sanity.py" \
|
||
"app/trust/trust_gate.py"; do
|
||
g="${gsha[$path]:-}"
|
||
if [ -z "$g" ]; then echo " [SKIP] $path nicht im Gold-Manifest"; continue; fi
|
||
cpath="$GOLD_SOURCE_APP_PREFIX/${path#app/}"
|
||
s="$(docker exec "$EXPECTED_GOLD_SOURCE_CID" sha256sum "$cpath" 2>/dev/null | awk '{print $1}')"
|
||
if [ "$s" = "$g" ]; then echo " [OK] $path == $g"
|
||
else echo " [FAIL] $path gold=$g cont=$s"; return 1; fi
|
||
done
|
||
return 0
|
||
}
|
||
|
||
# Read-only: prueft die Runtime Config des Containers gegen die Recovery-Baseline.
|
||
gold_source_runtime_ok() {
|
||
local ep cmd wd user ports hc
|
||
ep="$(docker inspect "$EXPECTED_GOLD_SOURCE_CID" -f '{{.Config.Entrypoint}}' 2>/dev/null)"
|
||
cmd="$(docker inspect "$EXPECTED_GOLD_SOURCE_CID" -f '{{.Config.Cmd}}' 2>/dev/null)"
|
||
wd="$(docker inspect "$EXPECTED_GOLD_SOURCE_CID" -f '{{.Config.WorkingDir}}' 2>/dev/null)"
|
||
user="$(docker inspect "$EXPECTED_GOLD_SOURCE_CID" -f '{{.Config.User}}' 2>/dev/null)"
|
||
ports="$(docker inspect "$EXPECTED_GOLD_SOURCE_CID" -f '{{.Config.ExposedPorts}}' 2>/dev/null)"
|
||
hc="$(docker inspect "$EXPECTED_GOLD_SOURCE_CID" -f '{{.Config.Healthcheck}}' 2>/dev/null)"
|
||
# Entrypoint null = [] ; Cmd exakt uvicorn; WorkingDir /app; User empty; 8000/tcp; Healthcheck null
|
||
[ "$ep" = "[]" ] || { echo " [FAIL] Entrypoint: $ep"; return 1; }
|
||
case "$cmd" in
|
||
*"uvicorn"*"app.api.main:app"*"--host"*"0.0.0.0"*"--port"*"8000"*) : ;;
|
||
*) echo " [FAIL] Cmd: $cmd"; return 1 ;;
|
||
esac
|
||
[ "$wd" = "/app" ] || { echo " [FAIL] WorkingDir: $wd"; return 1; }
|
||
[ -z "$user" ] || { echo " [FAIL] User: $user"; return 1; }
|
||
case "$ports" in *"8000/tcp"*) : ;; *) echo " [FAIL] ExposedPorts: $ports"; return 1 ;; esac
|
||
[ "$hc" = "<nil>" ] || [ -z "$hc" ] || { echo " [FAIL] Healthcheck: $hc"; return 1; }
|
||
echo " [OK] Entrypoint null, Cmd uvicorn, WorkingDir /app, User empty, 8000/tcp, Healthcheck null"
|
||
return 0
|
||
}
|
||
|
||
# verify-deployed-gold-source (read-only): alle Gold-Source-Gates.
|
||
verify_deployed_gold_source() {
|
||
expect_zero "$@"
|
||
echo "=== VERIFY-DEPLOYED-GOLD-SOURCE (read-only) ==="
|
||
gold_source_identity_ok || die "verify-deployed-gold-source: Source-Identity FAIL (FAIL CLOSED)"
|
||
gold_source_manifest_ok || die "verify-deployed-gold-source: 63-File-Manifest FAIL (FAIL CLOSED)"
|
||
gold_source_critical_ok || die "verify-deployed-gold-source: Critical-SHAs FAIL (FAIL CLOSED)"
|
||
gold_source_runtime_ok || die "verify-deployed-gold-source: Runtime-Config FAIL (FAIL CLOSED)"
|
||
echo "VERDICT: DEPLOYED GOLD SOURCE OK (Identity + 63/63 + Critical + Runtime)"
|
||
return 0
|
||
}
|
||
|
||
# Read-only: prueft die Build-Support-SHA-Bindings gegen die feste Quelle.
|
||
gold_build_support_ok() {
|
||
local key spec src exp s
|
||
for key in "${!GOLD_BUILD_SUPPORT[@]}"; do
|
||
spec="${GOLD_BUILD_SUPPORT[$key]}"
|
||
src="${spec%%|*}"; exp="${spec##*|}"
|
||
if [ ! -f "$src" ]; then echo " [FAIL] Build-Support fehlt: $src"; return 1; fi
|
||
s="$(sha256sum "$src" | awk '{print $1}')"
|
||
if [ "$s" = "$exp" ]; then echo " [OK] $key == $exp"
|
||
else echo " [FAIL] $key gold=$exp host=$s"; return 1; fi
|
||
done
|
||
return 0
|
||
}
|
||
|
||
# dry-run recover-gold-build-source: zeigt den vollstaendigen Plan, ZERO MUTATION.
|
||
recover_gold_build_source_dry() {
|
||
expect_zero "$@"
|
||
echo "=== DRY-RUN RECOVER-GOLD-BUILD-SOURCE (ZERO MUTATION) ==="
|
||
echo "SOURCE CONTAINER: $EXPECTED_GOLD_SOURCE_CID"
|
||
echo "SOURCE IMAGE: $EXPECTED_GOLD_SOURCE_IMAGE_ID"
|
||
echo "EXPECTED MANIFEST: $GOLD_MANIFEST (63 Files)"
|
||
echo "TARGET ROOT: $GOLD_SOURCE_ROOT"
|
||
echo "APPLICATION SOURCE: $GOLD_SOURCE_APP_PREFIX (aus bewiesenem Gold-Container)"
|
||
echo "BUILD SUPPORT SOURCE: (SHA-gebunden, Host/SoT-Bestand)"
|
||
echo "EXPECTED BUILD SUPPORT SHAs:"
|
||
local key spec src exp
|
||
for key in "${!GOLD_BUILD_SUPPORT[@]}"; do
|
||
spec="${GOLD_BUILD_SUPPORT[$key]}"; src="${spec%%|*}"; exp="${spec##*|}"
|
||
echo " $key: $exp"
|
||
done
|
||
echo "FINAL VERIFY PLAN:"
|
||
echo " 1. verify-deployed-gold-source PASS"
|
||
echo " 2. immutable Session-Root erzeugen (FAIL CLOSED wenn existiert)"
|
||
echo " 3. Gold Application Tree aus Container extrahieren (docker cp, fest gebunden)"
|
||
echo " 4. Build-Support aus SHA-gebundener Source ergaenzen"
|
||
echo " 5. .dockerignore kontrolliert ergaenzen (app/app/ + app/eligibility.py)"
|
||
echo " 6. Build-Input-Manifest erzeugen"
|
||
echo " 7. Application Manifest erneut pruefen (63/63)"
|
||
echo " 8. Build-Support SHAs erneut pruefen"
|
||
echo " 9. session.meta erzeugen"
|
||
echo " 10. final verify + immutable klassifizieren"
|
||
echo "DRY-RUN: DESIGN OK (kein echter Recover ausgefuehrt)"
|
||
exit 0
|
||
}
|
||
|
||
# recover-gold-build-source (Dispatcher): dry-run zeigt Plan (ZERO MUTATION),
|
||
# real fuehrt den kontrollierten Workflow aus (Punkt 3/4/5/8/9).
|
||
recover_gold_build_source() {
|
||
expect_zero "$@"
|
||
if [ "$RUN_MODE" = "DRY_RUN" ]; then
|
||
recover_gold_build_source_dry
|
||
else
|
||
recover_gold_build_source_real
|
||
fi
|
||
}
|
||
|
||
# Realer Recover (Punkt 3/4/5/8/9): materialisiert die Gold-Source aus dem
|
||
# bewiesenen DEPLOYED GOLD CONTAINER in eine immutable Session. Keine freien
|
||
# Parameter. FAIL CLOSED bei jeder Abweichung. ZERO MUTATION ausser der Session.
|
||
recover_gold_build_source_real() {
|
||
expect_zero "$@"
|
||
echo "=== RECOVER-GOLD-BUILD-SOURCE (real) ==="
|
||
# Punkt 4: TOCTOU vermeiden — unmittelbar vor Extraktion erneut verifizieren.
|
||
echo " [verify] verify-deployed-gold-source (TOCTOU-Guard):"
|
||
verify_deployed_gold_source >/dev/null || die "recover: verify-deployed-gold-source FAIL (FAIL CLOSED)"
|
||
echo " OK"
|
||
# Punkt 8: immutable Session — Root darf nicht existieren (FAIL CLOSED).
|
||
if [ -e "$GOLD_SOURCE_ROOT" ]; then
|
||
die "recover: Gold-Source-Root existiert bereits $GOLD_SOURCE_ROOT (KEIN Overwrite, FAIL CLOSED)"
|
||
fi
|
||
local sess
|
||
sess="$(date +%Y%m%d_%H%M%S)"
|
||
local sdir="$GOLD_SOURCE_ROOT/$sess"
|
||
# Punkt 5: Application Tree ausschliesslich aus dem hardcoded Gold-Container.
|
||
# Keine Host-Application-Dateien, keine SOT-Session, kein latest, keine freien docker-cp-Inputs.
|
||
mkdir -p "$sdir/app" || die "recover: Session-Ordner nicht anlegbar (FAIL CLOSED)"
|
||
echo " [extract] docker cp $EXPECTED_GOLD_SOURCE_CID:$GOLD_SOURCE_APP_PREFIX/. -> $sdir/app/"
|
||
if ! docker cp "$EXPECTED_GOLD_SOURCE_CID:$GOLD_SOURCE_APP_PREFIX/." "$sdir/app/" >/dev/null 2>&1; then
|
||
rm -rf "$GOLD_SOURCE_ROOT"; die "recover: docker cp Application-Tree fehlgeschlagen (FAIL CLOSED)"
|
||
fi
|
||
# Punkt 6: Build Support separat aus fest SHA-gebundenen Quellen (Host/SoT-Bestand).
|
||
echo " [build-support] SHA-gebundene Host-Dateien:"
|
||
local key spec src exp s
|
||
for key in "${!GOLD_BUILD_SUPPORT[@]}"; do
|
||
spec="${GOLD_BUILD_SUPPORT[$key]}"; src="${spec%%|*}"; exp="${spec##*|}"
|
||
[ -f "$src" ] || { rm -rf "$GOLD_SOURCE_ROOT"; die "recover: Build-Support fehlt $src (FAIL CLOSED)"; }
|
||
s="$(sha256sum "$src" | awk '{print $1}')"
|
||
[ "$s" = "$exp" ] || { rm -rf "$GOLD_SOURCE_ROOT"; die "recover: Build-Support SHA mismatch $key (FAIL CLOSED)"; }
|
||
cp -a "$src" "$sdir/$(basename "$src")" || { rm -rf "$GOLD_SOURCE_ROOT"; die "recover: Build-Support kopieren fehlgeschlagen (FAIL CLOSED)"; }
|
||
echo " $key == $exp"
|
||
done
|
||
# Punkt 6: .dockerignore kontrolliert erzeugen (exakt die Legacy-Exclusions).
|
||
printf 'app/app/\napp/eligibility.py\n' > "$sdir/.dockerignore"
|
||
local dis
|
||
dis="$(sha256sum "$sdir/.dockerignore" | awk '{print $1}')"
|
||
[ "$dis" = "$GOLD_DOCKERIGNORE_SHA" ] || { rm -rf "$GOLD_SOURCE_ROOT"; die "recover: .dockerignore SHA mismatch (FAIL CLOSED)"; }
|
||
echo " .dockerignore == $GOLD_DOCKERIGNORE_SHA"
|
||
# Punkt 8: Application Manifest (63/63 gegen Gold) + Build-Support Manifest.
|
||
( cd "$sdir/app" && find . -type f -name '*.py' ! -path '*/__pycache__/*' -printf '%P\n' 2>/dev/null | sort \
|
||
| while IFS= read -r rel; do [ -f "$rel" ] && sha256sum "$rel"; done | awk '{print $1" app/"$2}' ) > "$sdir/application.manifest"
|
||
local match=0 diverged=0 missing=0
|
||
local -A gsha
|
||
while read -r sha p; do gsha["$p"]="$sha"; done < "$GOLD_MANIFEST"
|
||
local path g c
|
||
for path in "${!gsha[@]}"; do
|
||
g="${gsha[$path]}"
|
||
c="$(awk -v r="$path" '$2==r{print $1}' "$sdir/application.manifest" 2>/dev/null)"
|
||
if [ -z "$c" ]; then missing=$((missing+1));
|
||
elif [ "$c" = "$g" ]; then match=$((match+1));
|
||
else diverged=$((diverged+1)); fi
|
||
done
|
||
echo " [application] MATCH=$match DIVERGED=$diverged MISSING=$missing"
|
||
[ "$match" = "63" ] && [ "$diverged" = "0" ] && [ "$missing" = "0" ] || { rm -rf "$GOLD_SOURCE_ROOT"; die "recover: Application != Gold (FAIL CLOSED)"; }
|
||
# Build-Support Manifest (Punkt 8)
|
||
{
|
||
for key in "${!GOLD_BUILD_SUPPORT[@]}"; do
|
||
spec="${GOLD_BUILD_SUPPORT[$key]}"; src="${spec%%|*}"; exp="${spec##*|}"
|
||
echo "$key $exp"
|
||
done
|
||
echo ".dockerignore $GOLD_DOCKERIGNORE_SHA"
|
||
} > "$sdir/build-support.manifest"
|
||
# Punkt 13: FINAL_BUILD_INPUT_SHA (Build-Input-Identitaet) berechnen.
|
||
local final_sha
|
||
final_sha="$( ( cd "$sdir" && find . -type f ! -name 'session.meta' ! -name 'application.manifest' ! -name 'build-support.manifest' ! -name 'final-build-input.manifest' -printf '%P\n' 2>/dev/null | sort | while IFS= read -r f; do [ -f "$f" ] && sha256sum "$f"; done ) | sha256sum | awk '{print $1}' )"
|
||
echo "$final_sha" > "$sdir/final-build-input.manifest"
|
||
# Punkt 8: session.meta
|
||
{
|
||
echo "GOLD_SOURCE_SESSION=$sess"
|
||
echo "SOURCE_CONTAINER_ID=$EXPECTED_GOLD_SOURCE_CID"
|
||
echo "SOURCE_IMAGE_ID=$EXPECTED_GOLD_SOURCE_IMAGE_ID"
|
||
echo "CREATED_AT=$(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
||
echo "FINAL_BUILD_INPUT_SHA=$final_sha"
|
||
} > "$sdir/session.meta"
|
||
# Punkt 9: aktive Recovery-Gold-Source-Identitaet kontrolliert speichern.
|
||
mkdir -p "$CONTROL_PLANE_STATE_DIR" || { rm -rf "$GOLD_SOURCE_ROOT"; die "recover: State-Dir nicht anlegbar (FAIL CLOSED)"; }
|
||
if [ -e "$RECOVERY_SOURCE_STATE" ]; then
|
||
rm -rf "$GOLD_SOURCE_ROOT"; die "recover: Recovery-Source-State existiert bereits (KEIN Overwrite, FAIL CLOSED)"
|
||
fi
|
||
{
|
||
echo "GOLD_SOURCE_SESSION=$sess"
|
||
echo "FINAL_BUILD_INPUT_SHA=$final_sha"
|
||
echo "SOURCE_CONTAINER_ID=$EXPECTED_GOLD_SOURCE_CID"
|
||
echo "SOURCE_IMAGE_ID=$EXPECTED_GOLD_SOURCE_IMAGE_ID"
|
||
echo "CREATED_AT=$(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
||
} > "$RECOVERY_SOURCE_STATE"
|
||
chown root:root "$RECOVERY_SOURCE_STATE"; chmod 600 "$RECOVERY_SOURCE_STATE"
|
||
# Punkt 8: immutable klassifizieren (root-owned, nicht world-readable).
|
||
chown -R root:root "$sdir"; chmod -R 700 "$sdir"
|
||
echo "RECOVER-GOLD-BUILD-SOURCE OK: $sdir (immutable, 63/63, Build-Support SHA-gebunden)"
|
||
echo " State: $RECOVERY_SOURCE_STATE"
|
||
echo " FINAL_BUILD_INPUT_SHA=$final_sha"
|
||
}
|
||
|
||
# verify-recovered-gold-build-source (read-only): prueft eine bestehende Recovery-Source
|
||
# gegen die EXAKTE Session-Identitaet aus dem Control-Plane-State (Punkt 10).
|
||
# NICHT chronologisch newest, NICHT glob-first, NICHT latest.
|
||
verify_recovered_gold_build_source() {
|
||
expect_zero "$@"
|
||
echo "=== VERIFY-RECOVERED-GOLD-BUILD-SOURCE (read-only) ==="
|
||
# Exakte Session-Identitaet aus State (Punkt 9/10). Kein latest/glob-first.
|
||
if [ ! -f "$RECOVERY_SOURCE_STATE" ]; then
|
||
die "verify-recovered-gold-build-source: Kein Recovery-Source-State ($RECOVERY_SOURCE_STATE). NUR nach realem recover (FAIL CLOSED)"
|
||
fi
|
||
local sess
|
||
sess="$(awk -F'=' '$1=="GOLD_SOURCE_SESSION"{print $2}' "$RECOVERY_SOURCE_STATE" 2>/dev/null)"
|
||
[[ "$sess" =~ ^[0-9]{8}_[0-9]{6}$ ]] || die "verify-recovered-gold-build-source: GOLD_SOURCE_SESSION ungueltig/fehlt (FAIL CLOSED)"
|
||
local sdir="$GOLD_SOURCE_ROOT/$sess"
|
||
[ -d "$sdir" ] || die "verify-recovered-gold-build-source: Session-Ordner fehlt $sdir (FAIL CLOSED)"
|
||
# Pflichtdateien der immutable Session (Punkt 8)
|
||
local f
|
||
for f in app Dockerfile .dockerignore requirements.txt session.meta application.manifest build-support.manifest final-build-input.manifest; do
|
||
[ -e "$sdir/$f" ] || die "verify-recovered-gold-build-source: fehlt $sdir/$f (FAIL CLOSED)"
|
||
done
|
||
# Application Manifest (Punkt 8): 63/63 gegen Gold
|
||
echo " [application] $sdir/application.manifest vs Gold (63/63):"
|
||
local match=0 diverged=0 missing=0
|
||
local -A gsha
|
||
while read -r sha p; do gsha["$p"]="$sha"; done < "$GOLD_MANIFEST"
|
||
local path g s
|
||
for path in "${!gsha[@]}"; do
|
||
g="${gsha[$path]}"
|
||
s="$(awk -v r="$path" '$2==r{print $1}' "$sdir/application.manifest" 2>/dev/null)"
|
||
if [ -z "$s" ]; then missing=$((missing+1));
|
||
elif [ "$s" = "$g" ]; then match=$((match+1));
|
||
else diverged=$((diverged+1)); fi
|
||
done
|
||
echo " MATCH=$match DIVERGED=$diverged MISSING=$missing"
|
||
[ "$match" = "63" ] && [ "$diverged" = "0" ] && [ "$missing" = "0" ] || die "verify-recovered-gold-build-source: Application != Gold (FAIL CLOSED)"
|
||
# Build-Support SHAs (Punkt 6): fest SHA-gebunden
|
||
echo " [build-support] SHA-Bindings:"
|
||
gold_build_support_ok || die "verify-recovered-gold-build-source: Build-Support SHA mismatch (FAIL CLOSED)"
|
||
# .dockerignore fester EXPECTED SHA (Punkt 6)
|
||
local dis
|
||
dis="$(sha256sum "$sdir/.dockerignore" 2>/dev/null | awk '{print $1}')"
|
||
[ "$dis" = "$GOLD_DOCKERIGNORE_SHA" ] || die "verify-recovered-gold-build-source: .dockerignore SHA $dis != erwartet (FAIL CLOSED)"
|
||
echo " .dockerignore == $GOLD_DOCKERIGNORE_SHA"
|
||
# Build-Input-Identitaet (Punkt 13): FINAL_BUILD_INPUT_SHA aus State pruefen
|
||
local want got
|
||
want="$(awk -F'=' '$1=="FINAL_BUILD_INPUT_SHA"{print $2}' "$RECOVERY_SOURCE_STATE" 2>/dev/null)"
|
||
[ -n "$want" ] && [[ "$want" =~ ^[a-f0-9]{64}$ ]] || die "verify-recovered-gold-build-source: FINAL_BUILD_INPUT_SHA fehlt/ungueltig (FAIL CLOSED)"
|
||
got="$( ( cd "$sdir" && find . -type f ! -name 'session.meta' ! -name 'application.manifest' ! -name 'build-support.manifest' ! -name 'final-build-input.manifest' -printf '%P\n' 2>/dev/null | sort | while IFS= read -r f; do [ -f "$f" ] && sha256sum "$f"; done ) | sha256sum | awk '{print $1}' )"
|
||
[ "$got" = "$want" ] || die "verify-recovered-gold-build-source: Build-Input SHA mismatch (Source veraendert) (FAIL CLOSED)"
|
||
echo " FINAL_BUILD_INPUT_SHA == $want"
|
||
echo "VERDICT: RECOVERED GOLD BUILD SOURCE OK (Session $sess, 63/63, Build-Support + .dockerignore + Build-Input SHA)"
|
||
return 0
|
||
}
|
||
|
||
SUBCMD="${1:-}"
|
||
shift || true
|
||
|
||
# Mutation Guard: "dry-run <subcmd>" setzt DRY_RUN-Modus, dann normaler Dispatch.
|
||
if [ "$SUBCMD" = "dry-run" ]; then
|
||
RUN_MODE=DRY_RUN
|
||
SUBCMD="${1:-}"
|
||
shift || true
|
||
case "$SUBCMD" in
|
||
build-historical|recreate-historical|snapshot-recovery|build-recovery-gold|pin-current-candidate|recreate-candidate-safe|rollback-to-recovery-gold|rollback-to-current-gold-anchor|promote-runner-patch|rollback-runner-patch|build-runner-remediation|recreate-runner-remediation-safe|archive-recovery-image|archive-remediation-image|pin-recovery-image|pin-remediation-image|cleanup-partial-recovery-archive|cleanup-partial-remediation-staging|cleanup-partial-remediation-archive|recover-gold-build-source) ;;
|
||
*) die "dry-run nur mit build-historical|recreate-historical|snapshot-recovery|build-recovery-gold|build-runner-remediation|recreate-runner-remediation-safe|archive-recovery-image|archive-remediation-image|pin-recovery-image|pin-remediation-image|cleanup-partial-recovery-archive zulaessig" ;;
|
||
esac
|
||
fi
|
||
|
||
case "$SUBCMD" in
|
||
# ---------------- bestehend ----------------
|
||
ps) expect_zero "$@"; exec docker ps ;;
|
||
inspect) req_ok; expect_zero "$@"; exec docker inspect "$CONTAINER" ;;
|
||
logs) req_ok
|
||
tail_n=200
|
||
while [ $# -gt 0 ]; do
|
||
case "$1" in
|
||
--tail) [ $# -ge 2 ] || die "--tail braucht Wert"; [[ "$2" =~ ^[0-9]+$ ]] || die "ungueltiges --tail"; tail_n="$2"; shift 2 ;;
|
||
*) die "unerlaubtes Argument 'logs $1'" ;;
|
||
esac
|
||
done
|
||
exec docker logs --tail "$tail_n" "$CONTAINER"
|
||
;;
|
||
run) req_ok
|
||
declare -a RUN=()
|
||
while [ $# -gt 0 ]; do
|
||
case "$1" in
|
||
--start)[[ "${2:-}" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}$ ]]||die "ungueltiges --start";RUN+=("--start" "$2");shift 2;;
|
||
--end)[[ "${2:-}" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}$ ]]||die "ungueltiges --end";RUN+=("--end" "$2");shift 2;;
|
||
--limit-days)[[ "${2:-}" =~ ^[0-9]+$ ]]||die "ungueltiges --limit-days";RUN+=("--limit-days" "$2");shift 2;;
|
||
*) die "unerlaubtes Argument 'run $1'" ;;
|
||
esac
|
||
done
|
||
echo "[rq-historical] Start: docker exec $CONTAINER python $SCRIPT ${RUN[*]}" >&2
|
||
exec docker exec "$CONTAINER" python "$SCRIPT" "${RUN[@]}"
|
||
;;
|
||
code) req_ok
|
||
[ $# -eq 1 ] || die "code braucht genau EINEN Key (code-list)"
|
||
p="$(key_path "$1")"; [ -n "$p" ] || die "unbekannter Key 'code $1'"
|
||
exec docker exec "$CONTAINER" cat "$p"
|
||
;;
|
||
hashes) req_ok; expect_zero "$@"
|
||
PATHS=(); for k in "${!TARGET_PATHS[@]}"; do PATHS+=("${TARGET_PATHS[$k]}"); done
|
||
exec docker exec "$CONTAINER" sha256sum "${PATHS[@]}"
|
||
;;
|
||
code-list) req_ok; expect_zero "$@"
|
||
echo "Verfuegbare READ-ONLY-Code-Keys (historical-service):"
|
||
for k in "${!TARGET_PATHS[@]}"; do printf ' %-11s | staging: %-22s -> %s\n' "$k" "${STAGE_FILES[$k]}" "${TARGET_PATHS[$k]}"; done
|
||
;;
|
||
stage-dir) expect_zero "$@"
|
||
echo "Staging-Root: $STAGING_ROOT (Owner rq-host, keine globalen Schreibrechte)"
|
||
for k in runner backfill pipeline repository provenance trust sanity trust_gate; do
|
||
printf ' %-11s %s\n' "$k" "$(stage_file "$k")"
|
||
done
|
||
;;
|
||
stage-check) [ $# -eq 1 ] || die "stage-check braucht genau EINEN Key"; key_ok "$1"; validate_stage "$1"
|
||
;;
|
||
test) req_ok; [ $# -eq 1 ] || die "test braucht genau EINEN Key"; key_ok "$1"
|
||
validate_stage "$1"
|
||
file="$(stage_file "$1")"
|
||
tmp="/tmp/rqtest_${1}_$(date +%s).py"
|
||
docker cp "$file" "$CONTAINER:$tmp"
|
||
if docker exec "$CONTAINER" python -m py_compile "$tmp" >/dev/null 2>&1; then
|
||
echo "PASS: py_compile OK ($file) [isoliert in Container-/tmp, Produktivcode unberuehrt]"
|
||
else
|
||
echo "FAIL: py_compile fehlgeschlagen ($file)"
|
||
docker exec "$CONTAINER" rm -f "$tmp"; exit 2
|
||
fi
|
||
docker exec "$CONTAINER" rm -f "$tmp"
|
||
;;
|
||
search-writes) req_ok; expect_zero "$@"
|
||
docker exec "$CONTAINER" sh -c 'grep -rnE "insert_raw_bars|insert_derived_bars|BackfillOrchestrator|trust_gate|TrustGate" /app/app --include="*.py" 2>/dev/null | head -100 || true'
|
||
;;
|
||
healthcheck) req_ok; expect_zero "$@"
|
||
rh="$(docker exec "$CONTAINER" python -c 'import urllib.request; print(urllib.request.urlopen("http://localhost:8000/health", timeout=5).status)' 2>/dev/null || echo N/A)"
|
||
rr="$(docker exec "$CONTAINER" python -c 'import urllib.request; print(urllib.request.urlopen("http://localhost:8000/health/ready", timeout=5).status)' 2>/dev/null || echo N/A)"
|
||
running="$(docker inspect -f '{{.State.Running}}' "$CONTAINER")"
|
||
echo "Container.Running=$running /health=$rh /health/ready=$rr"
|
||
[ "$running" = "true" ] && [ "$rh" = "200" ] && [ "$rr" = "200" ] || { echo "HEALTHCHECK FAILED"; exit 2; }
|
||
echo "HEALTHCHECK OK"
|
||
;;
|
||
backup) req_ok; [ $# -eq 1 ] || die "backup braucht genau EINEN Key"; key_ok "$1"
|
||
p="$(key_path "$1")"; ts="$(date +%Y%m%d_%H%M%S)"; bdir="$BACKUP_ROOT/$1/$ts"
|
||
mkdir -p "$bdir"
|
||
docker exec "$CONTAINER" cat "$p" > "$bdir/$(basename "$p")"
|
||
sha256sum "$bdir/$(basename "$p")" > "$bdir/SHA256"
|
||
echo "BACKUP: $1 -> $bdir"; cat "$bdir/SHA256"
|
||
;;
|
||
deploy) req_ok; [ $# -eq 1 ] || die "deploy braucht genau EINEN Key"; key_ok "$1"
|
||
key="$1"; p="$(key_path "$key")"; file="$(stage_file "$key")"
|
||
validate_stage "$key"
|
||
ts="$(date +%Y%m%d_%H%M%S)"; bdir="$BACKUP_ROOT/$key/$ts"
|
||
mkdir -p "$bdir"
|
||
docker exec "$CONTAINER" cat "$p" > "$bdir/$(basename "$p")" || die "Backup fehlgeschlagen -> deploy abgebrochen"
|
||
sha256sum "$bdir/$(basename "$p")" > "$bdir/SHA256"
|
||
echo "BACKUP: $bdir"
|
||
tmp="/tmp/rqdeploy_${key}_$(date +%s).py"
|
||
docker cp "$file" "$CONTAINER:$tmp"
|
||
if ! docker exec "$CONTAINER" python -m py_compile "$tmp" >/dev/null 2>&1; then
|
||
docker exec "$CONTAINER" rm -f "$tmp"; die "py_compile FAIL -> deploy abgebrochen (nichts deployt)"
|
||
fi
|
||
docker exec "$CONTAINER" rm -f "$tmp"
|
||
docker cp "$file" "$CONTAINER:$p"
|
||
sha_stage="$(sha256sum "$file" | awk '{print $1}')"
|
||
sha_deployed="$(docker exec "$CONTAINER" sha256sum "$p" | awk '{print $1}')"
|
||
echo "SHA stage : $sha_stage"; echo "SHA deployed: $sha_deployed"
|
||
if [ "$sha_stage" != "$sha_deployed" ]; then
|
||
echo "HASH-MISMATCH -> rollback"; docker cp "$bdir/$(basename "$p")" "$CONTAINER:$p"; die "Deploy Hash-Mismatch, zurueckgerollt"
|
||
fi
|
||
rh="$(docker exec "$CONTAINER" python -c 'import urllib.request; print(urllib.request.urlopen("http://localhost:8000/health", timeout=5).status)' 2>/dev/null || echo N/A)"
|
||
rr="$(docker exec "$CONTAINER" python -c 'import urllib.request; print(urllib.request.urlopen("http://localhost:8000/health/ready", timeout=5).status)' 2>/dev/null || echo N/A)"
|
||
echo "Health: /health=$rh /health/ready=$rr"
|
||
if [ "$rh" != "200" ] || [ "$rr" != "200" ]; then
|
||
echo "DEPLOY-HEALTHCHECK FAILED -> rollback"; docker cp "$bdir/$(basename "$p")" "$CONTAINER:$p"; die "Rollback ausgefuehrt"
|
||
fi
|
||
echo "DEPLOY OK: $key -> $p (Backup: $bdir). KEIN backfill/run gestartet."
|
||
;;
|
||
rollback) req_ok; [ $# -eq 1 ] || die "rollback braucht genau EINEN Key"; key_ok "$1"
|
||
p="$(key_path "$1")"
|
||
bdir="$(ls -1t "$BACKUP_ROOT/$1" 2>/dev/null | head -1)"
|
||
[ -n "$bdir" ] || die "kein Backup fuer '$1' vorhanden"
|
||
bdir="$BACKUP_ROOT/$1/$bdir"; src="$bdir/$(basename "$p")"
|
||
[ -f "$src" ] || die "Backup-Datei fehlt: $src"
|
||
sha_bak="$(sha256sum "$src" | awk '{print $1}')"
|
||
sha_cur="$(docker exec "$CONTAINER" sha256sum "$p" | awk '{print $1}')"
|
||
echo "aktuell: $sha_cur / backup: $sha_bak"
|
||
[ "$sha_bak" != "$sha_cur" ] || die "kein Unterschied (bereits auf Backup-Stand)"
|
||
docker cp "$src" "$CONTAINER:$p"
|
||
sha_cur="$(docker exec "$CONTAINER" sha256sum "$p" | awk '{print $1}')"
|
||
[ "$sha_cur" = "$sha_bak" ] || die "ROLLBACK-HASH-MISMATCH"
|
||
echo "ROLLBACK OK: $key -> $p auf $bdir"
|
||
;;
|
||
|
||
# ============ SOT-DISCOVERY (read-only, NEU) ============
|
||
build-def) expect_zero "$@"
|
||
echo "=== Dockerfile ($HOST_DIR/Dockerfile) — redacted ==="
|
||
[ -f "$HOST_DIR/Dockerfile" ] || die "Dockerfile nicht gefunden"
|
||
while IFS= read -r line; do redact_line "$line"; done < "$HOST_DIR/Dockerfile"
|
||
echo ""
|
||
echo "=== docker-compose.yml — strukturielle Abschnitte, redacted ==="
|
||
cf=""
|
||
if [ -f "$HOST_DIR/docker-compose.yml" ]; then cf="$HOST_DIR/docker-compose.yml";
|
||
elif [ -f "$HOST_DIR/compose.yml" ]; then cf="$HOST_DIR/compose.yml";
|
||
elif [ -f "$HOST_DIR/docker-compose.yaml" ]; then cf="$HOST_DIR/docker-compose.yaml"; fi
|
||
[ -n "$cf" ] || die "keine compose-Datei gefunden"
|
||
echo "(Quelle: $cf)"
|
||
while IFS= read -r line; do
|
||
# nur relevante strukturelle Schluessel anzeigen; Environment/Secrets nicht roh
|
||
case "$line" in
|
||
*POSTGRES_PASSWORD*|*PG_PASSWORD*|*PASSWORD*|*SECRET*|*TOKEN*) echo " **** REDACTED (Secret-Zeile) ****" ;;
|
||
*) echo " $line" ;;
|
||
esac
|
||
done < "$cf"
|
||
echo ""
|
||
echo "=== .dockerignore (falls vorhanden) ==="
|
||
[ -f "$HOST_DIR/.dockerignore" ] && cat "$HOST_DIR/.dockerignore" || echo "(kein .dockerignore)"
|
||
echo ""
|
||
echo "=== Service/Container-Fixierung ==="
|
||
echo " compose.project=$(docker inspect "$CONTAINER" --format '{{index .Config.Labels "com.docker.compose.project"}}' 2>/dev/null || echo N/A)"
|
||
echo " compose.service=$(docker inspect "$CONTAINER" --format '{{index .Config.Labels "com.docker.compose.service"}}' 2>/dev/null || echo N/A)"
|
||
echo " container=$CONTAINER"
|
||
;;
|
||
build-files) expect_zero "$@"
|
||
echo "=== Build-Dateien unter $HOST_DIR (nicht-verzeichnis, nicht .env/.dump/.sql) ==="
|
||
for f in Dockerfile docker-compose.yml compose.yml docker-compose.yaml .dockerignore requirements.txt; do
|
||
if [ -e "$HOST_DIR/$f" ]; then printf ' %-22s %s\n' "$f" "EXISTS ($(stat -c '%s' "$HOST_DIR/$f") bytes)"; else printf ' %-22s %s\n' "$f" "(fehlt)"; fi
|
||
done
|
||
;;
|
||
host-manifest) expect_zero "$@"
|
||
echo "=== HOST Manifest /opt/historical-v2/app (*.py, keine Symlinks folgen) ==="
|
||
[ -d "$HOST_APP" ] || die "Host-App-Verzeichnis fehlt: $HOST_APP"
|
||
# finde reguläre *.py UND Symlinks auf *.py separat
|
||
(
|
||
find "$HOST_APP" -type f -name "*.py" -printf '%P %s\n' 2>/dev/null | sort \
|
||
| while read -r rel size; do
|
||
sha=$(sha256sum "$HOST_APP/$rel" 2>/dev/null | awk '{print $1}')
|
||
printf 'FILE\t%s\t%s\t%s\n' "$rel" "$size" "${sha:-ERR}"
|
||
done
|
||
find "$HOST_APP" -type l -name "*.py" -printf '%P\n' 2>/dev/null | sort \
|
||
| while read -r rel; do printf 'SYMLINK\t%s\t0\t<not followed>\n' "$rel"; done
|
||
)
|
||
;;
|
||
host-code) [ $# -eq 1 ] || die "host-code braucht genau EINEN relativen Pfad (unter app/)"
|
||
rel="$1"
|
||
[[ "$rel" == *".."* ]] && die "Traversal nicht erlaubt"
|
||
[[ "$rel" == /* ]] && die "relativer Pfad erwartet, nicht absolut"
|
||
# realpath zwingend im app-Root
|
||
real="$(realpath -m "$HOST_APP/$rel")"
|
||
[[ "$real" == "$HOST_APP/"* ]] || die "Pfad ausserhalb app-Root blockiert"
|
||
[ -f "$real" ] || die "Datei nicht gefunden oder kein File: $rel"
|
||
[ -L "$real" ] && die "Symlink-Read nicht erlaubt"
|
||
# nur .py ausgeben
|
||
[[ "$rel" == *.py ]] || die "Nur .py-Quelldateien lesbar"
|
||
echo "--- $rel ($(stat -c '%s' "$real") bytes) ---"
|
||
exec cat "$real"
|
||
;;
|
||
container-manifest) req_ok; expect_zero "$@"
|
||
echo "=== CONTAINER Manifest historical-service:/app/app (*.py) ==="
|
||
docker exec "$CONTAINER" sh -c 'cd /app/app && find . -name "*.py" -type f 2>/dev/null | sed "s#^\./##" | sort | while read f; do size=$(stat -c %s "$f"); sha=$(sha256sum "$f" | cut -d" " -f1); printf "FILE\t%s\t%s\t%s\n" "$f" "$size" "$sha"; done'
|
||
docker exec "$CONTAINER" sh -c 'cd /app/app && find . -name "*.py" -type l 2>/dev/null | sed "s#^\./##" | sort | while read f; do printf "SYMLINK\t%s\t0\t<not followed>\n" "$f"; done' || true
|
||
;;
|
||
container-code) req_ok; [ $# -eq 1 ] || die "container-code braucht genau EINEN relativen Pfad (unter /app/app/)"
|
||
rel="$1"
|
||
[[ "$rel" == *".."* ]] && die "Traversal nicht erlaubt"
|
||
[[ "$rel" == /* ]] && die "relativer Pfad erwartet, nicht absolut"
|
||
target="/app/app/$rel"
|
||
# sicherstellen Ziel existiert als reguläre Datei & nicht symlink im Container
|
||
if ! docker exec "$CONTAINER" sh -c "[ -f '$target' ]" 2>/dev/null; then die "Datei nicht gefunden oder kein File im Container: $rel"; fi
|
||
if docker exec "$CONTAINER" sh -c "[ -L '$target' ]" 2>/dev/null; then die "Symlink-Read im Container nicht erlaubt"; fi
|
||
[[ "$rel" == *.py ]] || die "Nur .py-Quelldateien lesbar"
|
||
echo "--- $target ---"
|
||
exec docker exec "$CONTAINER" cat "$target"
|
||
;;
|
||
tree-diff) req_ok; expect_zero "$@"
|
||
echo "=== TREE-DIFF Host(/opt/historical-v2/app) <-> Container(/app/app) ==="
|
||
hmap="$(mktemp)"; cmap="$(mktemp)"
|
||
# Host: 'rel<TAB>sha'
|
||
find "$HOST_APP" -type f -name "*.py" -printf '%P\n' 2>/dev/null | sort | while IFS= read -r rel; do
|
||
sha="$(sha256sum "$HOST_APP/$rel" 2>/dev/null | awk '{print $1}')"
|
||
printf '%s\t%s\n' "$rel" "$sha"
|
||
done > "$hmap"
|
||
# Container: 'rel<TAB>sha'
|
||
docker exec "$CONTAINER" sh -c 'cd /app/app && find . -type f -name "*.py" -printf "%P\n" 2>/dev/null | sort | while IFS= read -r rel; do sha256sum "$rel" 2>/dev/null | awk -v r="$rel" '\''{print r"\t"$1}'\''; done' > "$cmap"
|
||
tmpunion="$(mktemp)"
|
||
{ cut -f1 "$hmap"; cut -f1 "$cmap"; } | sort -u > "$tmpunion"
|
||
echo "HOST_APP=$HOST_APP CONTAINER_APP=/app/app"
|
||
while IFS= read -r rel; do
|
||
hs="$(awk -F'\t' -v r="$rel" '$1==r{print $2}' "$hmap")"
|
||
cs="$(awk -F'\t' -v r="$rel" '$1==r{print $2}' "$cmap")"
|
||
if [ -n "$hs" ] && [ -n "$cs" ]; then
|
||
if [ "$hs" = "$cs" ]; then status="MATCH"; else status="DIVERGED"; fi
|
||
printf '%-18s %s [h:%s c:%s]\n' "$status" "$rel" "${hs:0:12}" "${cs:0:12}"
|
||
elif [ -n "$hs" ]; then
|
||
printf '%-18s %s\n' "HOST_ONLY" "$rel"
|
||
else
|
||
printf '%-18s %s\n' "MISSING_ON_HOST" "$rel"
|
||
fi
|
||
done < "$tmpunion"
|
||
rm -f "$hmap" "$cmap" "$tmpunion"
|
||
;;
|
||
|
||
# ============ HOST-SOT-RECOVERY (Phase 13.5, Extension 1) ============
|
||
# KEIN Build/Recreate. Nur: backup-sot -> recover-sot -> verify-sot-recovery
|
||
# -> restore-sot. Fail-closed, Gold-SHA-gebunden, keine freien Pfade.
|
||
backup-sot) expect_zero "$@"
|
||
# Vollstaendiges Host-SoT-Backup in root-owned Recovery-Bereich
|
||
ts="$(date +%Y%m%d_%H%M%S)"
|
||
bdir="$SOT_BACKUP_ROOT/$ts"
|
||
mkdir -p "$bdir"
|
||
# tar mit Symlinks/Permissions/Ownership; kein Follow (kein -h)
|
||
if ! tar -cpf "$bdir/historical-v2.tar" -C /opt historical-v2 2>/dev/null; then
|
||
rm -rf "$bdir"; die "backup-sot: tar fehlgeschlagen (FAIL CLOSED)"
|
||
fi
|
||
# SHA256-Manifest aller regulären Dateien (nicht Symlinks folgen)
|
||
( cd /opt/historical-v2 && find . -type f -print0 2>/dev/null | sort -z \
|
||
| while IFS= read -r -d "" f; do
|
||
sha="$(sha256sum "$f" 2>/dev/null | awk '{print $1}')"
|
||
printf '%s %s\n' "$sha" "${f#./}"
|
||
done ) > "$bdir/SHA256.manifest"
|
||
# Struktur-/Pflichtpruefung (Punkt 4)
|
||
ok=1
|
||
[ -f "$bdir/historical-v2.tar" ] || { echo "FEHLT: tar"; ok=0; }
|
||
[ -s "$bdir/SHA256.manifest" ] || { echo "FEHLT: manifest"; ok=0; }
|
||
[ -f /opt/historical-v2/Dockerfile ] || { echo "FEHLT: Dockerfile"; ok=0; }
|
||
[ -f /opt/historical-v2/docker-compose.yml ] || { echo "FEHLT: compose"; ok=0; }
|
||
[ -d /opt/historical-v2/app ] || { echo "FEHLT: app/"; ok=0; }
|
||
if [ "$ok" != "1" ]; then
|
||
rm -rf "$bdir"; die "backup-sot: Validierung fehlgeschlagen (FAIL CLOSED)"
|
||
fi
|
||
# Session-Marker: eindeutige Markierung der aktuellen Recovery-Session
|
||
echo "$ts" > "$bdir/$SOT_SESSION_MARKER"
|
||
chown -R root:root "$bdir"; chmod 700 "$bdir"
|
||
echo "BACKUP-SOT OK: $bdir"
|
||
echo " tar: $(stat -c '%s' "$bdir/historical-v2.tar") bytes"
|
||
echo " manifest: $(wc -l < "$bdir/SHA256.manifest") Dateien"
|
||
echo " session: $ts"
|
||
;;
|
||
sot-session) expect_zero "$@"
|
||
# Zeigt den neuesten VALIDIERTEN FULL SOT SNAPSHOT (Partial-Backups wie
|
||
# pre-runner-remediation werden NICHT als Full-Snapshot gezaehlt; Blocker A).
|
||
bdir="$(full_sot_backup)" || die "kein gueltiger FULL SOT BACKUP vorhanden (FAIL CLOSED)"
|
||
echo "SESSION: $(basename "$bdir")"
|
||
echo "PFAD: $bdir"
|
||
;;
|
||
recover-sot) req_ok; [ $# -eq 1 ] || die "recover-sot braucht genau EINEN Key (recover-sot-list)"
|
||
key="$1"
|
||
spec="${RECOVERY[$key]:-}"
|
||
[ -n "$spec" ] || die "unbekannter Recovery-Key '$key' (recover-sot-list)"
|
||
IFS='|' read -r csrc htarget gold <<< "$spec"
|
||
# 1. gueltiger vollstaendiger (FULL) backup-sot vorhanden? (Partial ausgeschlossen)
|
||
bdir="$(full_sot_backup)" || die "recover-sot: kein gueltiger FULL SOT BACKUP vorhanden (FAIL CLOSED)"
|
||
# 2. Source = historical-service fixed (req_ok bereits geprueft)
|
||
# 3/4. Source regular file, kein Symlink
|
||
if ! docker exec "$CONTAINER" sh -c "[ -f '$csrc' ]" 2>/dev/null; then die "recover-sot: Source kein File im Container: $csrc"; fi
|
||
if docker exec "$CONTAINER" sh -c "[ -L '$csrc' ]" 2>/dev/null; then die "recover-sot: Source ist Symlink (FAIL CLOSED): $csrc"; fi
|
||
# 5. Source SHA == EXPECTED GOLD SHA
|
||
csha="$(docker exec "$CONTAINER" sha256sum "$csrc" 2>/dev/null | awk '{print $1}')" || csha=""
|
||
if [ "$csha" != "$gold" ]; then
|
||
die "recover-sot: Source-SHA-Mismatch fuer '$key' (erwartet $gold, Container $csha) — FAIL CLOSED"
|
||
fi
|
||
# 6. Host target exakt fester Pfad (aus Allowlist, kein freier Pfad)
|
||
[[ "$htarget" == /opt/historical-v2/app/* ]] || die "recover-sot: Host-Ziel ausserhalb app/ (FAIL CLOSED)"
|
||
# 7. Zielverzeichnis kontrolliert erzeugen (z.B. trust/)
|
||
tdir="$(dirname "$htarget")"
|
||
mkdir -p "$tdir"
|
||
# 8. Ziel vor Ueberschreiben einzeln sichern, falls vorhanden
|
||
if [ -e "$htarget" ]; then
|
||
prebak="$bdir/pre-recover/$(basename "$htarget")"
|
||
mkdir -p "$bdir/pre-recover"
|
||
cp -a "$htarget" "$prebak" 2>/dev/null || true
|
||
echo " pre-recover gesichert: $prebak"
|
||
fi
|
||
# 9. byte-exakt schreiben (docker cp = byte-identisch)
|
||
docker cp "$CONTAINER:$csrc" "$htarget"
|
||
# 10. Ownership/Permissions aus bestehendem Host-Kontext ableiten
|
||
# (app/ ist UID 10010; trust/ neu -> gleiche Basis wie app/)
|
||
chown 10010:10010 "$htarget" 2>/dev/null || chown root:root "$htarget"
|
||
chmod 644 "$htarget"
|
||
# 11. Host SHA == Gold SHA
|
||
hsha="$(sha256sum "$htarget" | awk '{print $1}')"
|
||
if [ "$hsha" != "$gold" ]; then
|
||
die "recover-sot: Host-SHA-Mismatch fuer '$key' (erwartet $gold, Host $hsha) — FAIL CLOSED"
|
||
fi
|
||
echo "RECOVER-SOT OK: $key"
|
||
echo " source : $CONTAINER:$csrc"
|
||
echo " target : $htarget"
|
||
echo " gold : $gold"
|
||
echo " host : $hsha"
|
||
;;
|
||
recover-sot-list) expect_zero "$@"
|
||
echo "Recovery-Allowlist (CONFIRMED GOLD, fest gebunden):"
|
||
for k in "${!RECOVERY[@]}"; do
|
||
IFS='|' read -r csrc htarget gold <<< "${RECOVERY[$k]}"
|
||
printf ' %-16s %-45s -> %s\n' "$k" "$csrc" "$htarget"
|
||
done
|
||
;;
|
||
verify-sot-recovery) req_ok; expect_zero "$@"
|
||
echo "=== VERIFY-SOT-RECOVERY (Gold vs Host vs Container) ==="
|
||
allok=1
|
||
for k in "${!RECOVERY[@]}"; do
|
||
IFS='|' read -r csrc htarget gold <<< "${RECOVERY[$k]}"
|
||
hsha="$(sha256sum "$htarget" 2>/dev/null | awk '{print $1}')" || hsha=""
|
||
csha="$(docker exec "$CONTAINER" sha256sum "$csrc" 2>/dev/null | awk '{print $1}')" || csha=""
|
||
if [ "$hsha" = "$gold" ] && [ "$csha" = "$gold" ]; then
|
||
printf ' MATCH %-16s %s\n' "$k" "$htarget"
|
||
else
|
||
printf ' FAIL %-16s host=%s container=%s gold=%s\n' "$k" "${hsha:-MISSING}" "${csha:-MISSING}" "$gold"
|
||
allok=0
|
||
fi
|
||
done
|
||
[ "$allok" = "1" ] && echo "VERIFY-SOT-RECOVERY: ALLE MATCH" || { echo "VERIFY-SOT-RECOVERY: FEHLER"; exit 2; }
|
||
;;
|
||
restore-sot) expect_zero "$@"
|
||
# Restore aus dem neuesten validerungspflichtigen Session-Backup (FULL)
|
||
bdir="$(full_sot_backup)" || die "restore-sot: kein gueltiger FULL SOT BACKUP vorhanden (FAIL CLOSED)"
|
||
[ -f "$bdir/$SOT_SESSION_MARKER" ] || die "restore-sot: kein markiertes Session-Backup (FAIL CLOSED)"
|
||
[ -f "$bdir/historical-v2.tar" ] || die "restore-sot: tar fehlt (FAIL CLOSED)"
|
||
echo "RESTORE-SOT: $bdir"
|
||
echo " (entpackt /opt/historical-v2 aus Session-Backup, byte-exakt)"
|
||
if ! tar -xpf "$bdir/historical-v2.tar" -C /opt 2>/dev/null; then
|
||
die "restore-sot: tar-Restore fehlgeschlagen (FAIL CLOSED)"
|
||
fi
|
||
echo "RESTORE-SOT OK. KEIN Build/Recreate danach."
|
||
;;
|
||
snapshot-recovery) snapshot_recovery "$@" ;;
|
||
verify-recovery-snapshot) verify_recovery_snapshot "$@" ;;
|
||
build-recovery-gold) build_recovery_gold "$@" ;;
|
||
verify-recovery-gold-image) verify_recovery_gold_image "$@" ;;
|
||
pin-current-candidate) pin_current_candidate "$@" ;;
|
||
verify-candidate-image) verify_candidate_image "$@" ;;
|
||
recreate-candidate-safe) recreate_candidate_safe "$@" ;;
|
||
rollback-to-recovery-gold) rollback_to_recovery_gold "$@" ;;
|
||
rollback-to-current-gold-anchor) rollback_to_current_gold_anchor "$@" ;;
|
||
promote-runner-patch) promote_runner_patch "$@" ;;
|
||
rollback-runner-patch) rollback_runner_patch "$@" ;;
|
||
build-runner-remediation) build_runner_remediation "$@" ;;
|
||
verify-runner-remediation-image) verify_runner_remediation_image "$@" ;;
|
||
recreate-runner-remediation-safe) recreate_runner_remediation_safe "$@" ;;
|
||
verify-remediation-build-input) expect_zero "$@"; verify_remediation_build_input ;;
|
||
build-historical) build_historical "$@" ;;
|
||
create-build-exclusion) expect_zero "$@"
|
||
# Build-Pollution-Exclusion implementieren (Punkt 5-8): .dockerignore anlegen/ersetzen
|
||
# PREVIOUS_STATE war ABSENT. NUR exakt die 3 Legacy-Einträge.
|
||
if [ -f "$BUILD_EXCLUSION_FILE" ]; then
|
||
die "create-build-exclusion: .dockerignore existiert bereits (kein Ueberschreiben ohne Backup — manuell)"
|
||
fi
|
||
printf 'app/app/\napp/eligibility.py\n' > "$BUILD_EXCLUSION_FILE"
|
||
echo "BUILD-EXCLUSION OK: $BUILD_EXCLUSION_FILE"
|
||
exclusion_valid && { echo " validiert: nur app/app/ + app/eligibility.py"; echo " (PREVIOUS_STATE=ABSENT; keine Gold-Datei betroffen)"; }
|
||
;;
|
||
show-build-exclusion) expect_zero "$@"
|
||
if [ ! -f "$BUILD_EXCLUSION_FILE" ]; then echo "PREVIOUS_STATE=ABSENT"; exit 0; fi
|
||
echo "=== .dockerignore Inhalt ==="
|
||
cat "$BUILD_EXCLUSION_FILE"
|
||
echo "=== build-exclusion: Gold-Dateien betroffen? (sollte: 0) ==="
|
||
ok=0
|
||
while IFS= read -r f; do
|
||
# f relativ zu /opt/historical-v2; pruefe ob im Gold-Manifest (app/...)
|
||
rel="app/${f#app/}"
|
||
if grep -q " $rel$" "$GOLD_MANIFEST" 2>/dev/null; then echo " GOLD-DATEI BETROFFEN: $rel"; ok=1; fi
|
||
done < "$BUILD_EXCLUSION_FILE"
|
||
[ "$ok" = "0" ] && echo " 0 Gold-Dateien ausgeschlossen (OK)"
|
||
;;
|
||
recreate-historical) recreate_historical "$@" ;;
|
||
verify-gold-reproduction) verify_gold_reproduction "$@" ;;
|
||
|
||
# ============ IMAGE-RETENTION / ARCHIVE CONTROL PLANE (Phase 13.5) ============
|
||
# Schicht B (Archiv, docker save; fail-closed, kein Overwrite) & Schicht A (Pin).
|
||
# Alle verify-* sind read-only. Realer Restore ist in dieser Mission BLOCKIERT
|
||
# (nur dry-run). Kein Coolify-Cleanup-Disable, kein coolify.managed-Spoof.
|
||
archive-recovery-image) archive_image recovery "$@" ;;
|
||
archive-remediation-image) archive_image remediation "$@" ;;
|
||
pin-recovery-image) pin_image recovery "$@" ;;
|
||
pin-remediation-image) pin_image remediation "$@" ;;
|
||
verify-recovery-archive) verify_archive_cmd recovery "$@" ;;
|
||
verify-remediation-archive) verify_archive_cmd remediation "$@" ;;
|
||
cleanup-partial-recovery-archive) cleanup_partial_recovery_archive "$@" ;;
|
||
cleanup-partial-remediation-staging) cleanup_partial_remediation_staging "$@" ;;
|
||
cleanup-partial-remediation-archive) cleanup_partial_remediation_archive "$@" ;;
|
||
verify-recovery-pin) verify_pin_cmd recovery "$@" ;;
|
||
verify-remediation-pin) verify_pin_cmd remediation "$@" ;;
|
||
verify-recovery-ready) verify_ready_cmd recovery "$@" ;;
|
||
verify-remediation-ready) verify_ready_cmd remediation "$@" ;;
|
||
restore-recovery-image-archive) restore_recovery_real "$@" ;;
|
||
dry-restore-recovery-image-archive) restore_recovery_dry "$@" ;;
|
||
|
||
# ============ DEPLOYED-GOLD-SOURCE-RECOVERY CONTROL PLANE (Phase 13.5) ============
|
||
# Gold-Source fest an Container+Image+63-File-Manifest gebunden. KEIN latest,
|
||
# KEIN full_sot_backup(), KEINE freien Inputs. Nur Source-Recovery, KEIN Docker-Build.
|
||
# verify-* read-only. Realer recover in dieser Mission BLOCKIERT (nur dry-run).
|
||
verify-deployed-gold-source) verify_deployed_gold_source "$@";;
|
||
recover-gold-build-source) recover_gold_build_source "$@";;
|
||
verify-recovered-gold-build-source) verify_recovered_gold_build_source "$@";;
|
||
wrapper-sha) expect_zero "$@"; sha256sum /usr/local/bin/rq-historical ;;
|
||
|
||
*) die "Aufruf: ps|inspect|logs|run|code <K>|hashes|code-list|stage-dir|stage-check <K>|test <K>|search-writes|healthcheck|backup <K>|deploy <K>|rollback <K>|build-def|build-files|host-manifest|host-code <p>|container-manifest|container-code <p>|tree-diff|backup-sot|sot-session|recover-sot <K>|recover-sot-list|verify-sot-recovery|restore-sot|snapshot-recovery|dry-run snapshot-recovery|verify-recovery-snapshot|build-recovery-gold|dry-run build-recovery-gold|verify-recovery-gold-image|pin-current-candidate|dry-run pin-current-candidate|verify-candidate-image|recreate-candidate-safe|dry-run recreate-candidate-safe|rollback-to-recovery-gold|dry-run rollback-to-recovery-gold|promote-runner-patch|dry-run promote-runner-patch|rollback-runner-patch|dry-run rollback-runner-patch|build-runner-remediation|dry-run build-runner-remediation|verify-runner-remediation-image|recreate-runner-remediation-safe|dry-run recreate-runner-remediation-safe|verify-remediation-build-input|build-historical|dry-run build-historical|recreate-historical|dry-run recreate-historical|verify-gold-reproduction|create-build-exclusion|show-build-exclusion|archive-recovery-image|archive-remediation-image|pin-recovery-image|pin-remediation-image|verify-recovery-archive|verify-remediation-archive|cleanup-partial-recovery-archive|cleanup-partial-remediation-staging|verify-recovery-pin|verify-remediation-pin|verify-recovery-ready|verify-remediation-ready|dry-restore-recovery-image-archive|restore-recovery-image-archive|verify-deployed-gold-source|dry-run recover-gold-build-source|recover-gold-build-source|verify-recovered-gold-build-source"
|
||
;;
|
||
esac
|