Phase 13.5: Recovery-Gold-Snapshot-Kanal (snapshot-recovery/verify/dry-run, 14 Guards, fail-closed) | Rain Ocampo 2026-08-29
This commit is contained in:
root
parent
c9b3dafd9f
commit
efe13ec92d
1 changed files with 143 additions and 4 deletions
|
|
@ -91,7 +91,7 @@ COMPOSE_PROJECT=/opt/historical-v2
|
||||||
COMPOSE_FILE=/opt/historical-v2/docker-compose.yml
|
COMPOSE_FILE=/opt/historical-v2/docker-compose.yml
|
||||||
HISTORICAL_SERVICE=historical-service
|
HISTORICAL_SERVICE=historical-service
|
||||||
GOLD_IMAGE_ID="sha256:f1a6f33e8714c5cdf017632aa30e71241a8ed670a5c95dac2e8226064b990c81"
|
GOLD_IMAGE_ID="sha256:f1a6f33e8714c5cdf017632aa30e71241a8ed670a5c95dac2e8226064b990c81"
|
||||||
GOLD_CONTAINER_ID="634cc505c7bc" # 634cc505c7bc (laufender Gold-Container)
|
GOLD_CONTAINER_ID="634cc505c7bc4c361305e5b415f107e98411faf38e8a29269dd8b1e4d1363b32"
|
||||||
GOLD_MANIFEST="/opt/rq-historical-gold-reference/gold_app_manifest.sha"
|
GOLD_MANIFEST="/opt/rq-historical-gold-reference/gold_app_manifest.sha"
|
||||||
BUILD_EXCLUSION_FILE=/opt/historical-v2/.dockerignore
|
BUILD_EXCLUSION_FILE=/opt/historical-v2/.dockerignore
|
||||||
# Präzise Pollution-Exclusion-Einträge (NUR diese 3; keine Gold-Datei betroffen):
|
# Präzise Pollution-Exclusion-Einträge (NUR diese 3; keine Gold-Datei betroffen):
|
||||||
|
|
@ -105,6 +105,14 @@ declare -A FORBIDDEN_EXTRA=(
|
||||||
["app/app/eligibility.py"]=1
|
["app/app/eligibility.py"]=1
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# ------------------------------------------------- RECOVERY-GOLD-SNAPSHOT-CHANNEL
|
||||||
|
# Phase 13.5 · Privileged Control-Plane Extension - Recovery Gold Snapshot Only.
|
||||||
|
# Erfassung eines byte-/state-nahen Recovery-Images aus dem verifizierten
|
||||||
|
# GOLD-Container 634cc505... (Image f1a6f33e...). KEIN Businesscode, KEIN Build,
|
||||||
|
# KEIN Recreate, KEIN Runner, KEINE DB-/Datenmutation, KEIN latest-Touch.
|
||||||
|
RECOVERY_GOLD_CID="634cc505c7bc4c361305e5b415f107e98411faf38e8a29269dd8b1e4d1363b32"
|
||||||
|
RECOVERY_GOLD_TAG="historical-v2-historical-service:recovery-gold-container-20260829"
|
||||||
|
|
||||||
# RUN_MODE (Mutation Guard): READ_ONLY | DRY_RUN | MUTATING
|
# RUN_MODE (Mutation Guard): READ_ONLY | DRY_RUN | MUTATING
|
||||||
RUN_MODE=MUTATING
|
RUN_MODE=MUTATING
|
||||||
|
|
||||||
|
|
@ -190,6 +198,135 @@ classify_and_report() { # $1 = neues Manifest Datei (Temp); vergleicht gegen GO
|
||||||
G_RET="ok"; [ $d -eq 0 ] && [ $m -eq 0 ] && [ $x -eq 0 ] || G_RET="fail"
|
G_RET="ok"; [ $d -eq 0 ] && [ $m -eq 0 ] && [ $x -eq 0 ] || G_RET="fail"
|
||||||
}
|
}
|
||||||
|
|
||||||
|
# ================= RECOVERY-SNAPSHOT HILFSFUNKTIONEN (fail-closed) ============
|
||||||
|
# PRE-CONDITION-GUARDS fuer docker commit (Punkt 4). Fuehrt ALLE 14 Guards aus;
|
||||||
|
# bei jedem FAIL: die. Bei DRY_RUN ebenso (read-only check, nur anzeigen).
|
||||||
|
snapshot_guards() {
|
||||||
|
echo " [guard 1] Container-ID == 634cc505...:"
|
||||||
|
local cid img running mounts dbid cimg rh rr
|
||||||
|
cid="$(docker inspect -f '{{.ID}}' "$HISTORICAL_SERVICE" 2>/dev/null || echo NONE)"
|
||||||
|
[ "$cid" = "$RECOVERY_GOLD_CID" ] || die "Guard1 FAIL: Container-ID $cid != RECOVERY_GOLD_CID (FAIL CLOSED)"
|
||||||
|
echo " OK ($cid)"
|
||||||
|
echo " [guard 2] Running=true:"
|
||||||
|
running="$(docker inspect -f '{{.State.Running}}' "$HISTORICAL_SERVICE" 2>/dev/null)"
|
||||||
|
[ "$running" = "true" ] || die "Guard2 FAIL: nicht Running (FAIL CLOSED)"
|
||||||
|
echo " OK ($running)"
|
||||||
|
echo " [guard 3] Image-Metadata == f1a6f33e...:"
|
||||||
|
img="$(docker inspect -f '{{.Image}}' "$HISTORICAL_SERVICE" 2>/dev/null)"
|
||||||
|
[ "$img" = "$GOLD_IMAGE_ID" ] || die "Guard3 FAIL: Image $img != GOLD_IMAGE_ID (FAIL CLOSED)"
|
||||||
|
echo " OK ($img)"
|
||||||
|
echo " [guard 4] /health=200:"
|
||||||
|
rh="$(docker exec "$HISTORICAL_SERVICE" python -c 'import urllib.request;print(urllib.request.urlopen("http://localhost:8000/health",timeout=5).status)' 2>/dev/null || echo N/A)"
|
||||||
|
[ "$rh" = "200" ] || die "Guard4 FAIL: /health=$rh (FAIL CLOSED)"
|
||||||
|
echo " OK ($rh)"
|
||||||
|
echo " [guard 5] /health/ready=200:"
|
||||||
|
rr="$(docker exec "$HISTORICAL_SERVICE" python -c 'import urllib.request;print(urllib.request.urlopen("http://localhost:8000/health/ready",timeout=5).status)' 2>/dev/null || echo N/A)"
|
||||||
|
[ "$rr" = "200" ] || die "Guard5 FAIL: /health/ready=$rr (FAIL CLOSED)"
|
||||||
|
echo " OK ($rr)"
|
||||||
|
echo " [guard 6] Mounts == []:"
|
||||||
|
mounts="$(docker inspect -f '{{json .Mounts}}' "$HISTORICAL_SERVICE" 2>/dev/null)"
|
||||||
|
[ "$mounts" = "[]" ] || die "Guard6 FAIL: Mounts != [] (FAIL CLOSED)"
|
||||||
|
echo " OK ([$mounts])"
|
||||||
|
echo " [guard 7] historical-db laeuft unveraendert:"
|
||||||
|
dbid="$(docker inspect -f '{{.ID}}' historical-db 2>/dev/null || echo NONE)"
|
||||||
|
cimg="$(docker inspect -f '{{.Image}}' historical-db 2>/dev/null)"
|
||||||
|
dbr="$(docker inspect -f '{{.State.Running}}' historical-db 2>/dev/null)"
|
||||||
|
[ "$dbr" = "true" ] || die "Guard7 FAIL: historical-db Running=$dbr (FAIL CLOSED)"
|
||||||
|
echo " OK (ID=$dbid Image=$cimg Running=$dbr)"
|
||||||
|
echo " [guard 8-11] 63/63 Gold-Code-MATCH / 0 DIVERGED / 0 MISSING / 0 unerlaubte EXTRA:"
|
||||||
|
local tmp; tmp="$(mktemp)"
|
||||||
|
gen_manifest "$HISTORICAL_SERVICE" > "$tmp"
|
||||||
|
classify_and_report "$tmp"
|
||||||
|
local gret="$G_RET"; rm -f "$tmp"
|
||||||
|
[ "$gret" = "ok" ] || die "Guard8-11 FAIL: GOLD-REPRODUCTION nicht OK (FAIL CLOSED)"
|
||||||
|
echo " OK (GOLD REPRODUCTION OK)"
|
||||||
|
echo " [guard 12] Critical-File-SHAs == konservierter Gold-Stand (verify-sot-recovery):"
|
||||||
|
verify_recovery_ok
|
||||||
|
echo " OK (13/13 MATCH)"
|
||||||
|
echo " [guard 13] kein Runner/Backfill aktiv:"
|
||||||
|
docker exec "$HISTORICAL_SERVICE" sh -c 'ps -ef 2>/dev/null | grep -E "run_backfill_year|backfill" | grep -v grep | grep -v "\-\-force" || true' | grep -q "run_backfill_year\|backfill" \
|
||||||
|
&& die "Guard13 FAIL: Runner/Backfill-Prozess aktiv (FAIL CLOSED)"
|
||||||
|
echo " OK (kein Runner/Backfill aktiv)"
|
||||||
|
echo " [guard 14] Recovery-Zieltag existiert NOCH NICHT:"
|
||||||
|
if docker image inspect "$RECOVERY_GOLD_TAG" >/dev/null 2>&1; then
|
||||||
|
die "Guard14 FAIL: Recovery-Tag $RECOVERY_GOLD_TAG existiert bereits (FAIL CLOSED, kein Overwrite)"
|
||||||
|
fi
|
||||||
|
echo " OK (Tag frei)"
|
||||||
|
echo " ALLE 14 GUARDS OK"
|
||||||
|
}
|
||||||
|
|
||||||
|
# snapshot-recovery (MUTATION - nur Red Queen fuehrt aus; dry-run blockt via mode_or_die)
|
||||||
|
snapshot_recovery() {
|
||||||
|
expect_zero "$@"
|
||||||
|
echo "=== SNAPSHOT-RECOVERY ($RUN_MODE) ==="
|
||||||
|
echo " Source Container-ID : $RECOVERY_GOLD_CID (== historical-service)"
|
||||||
|
echo " Erwartete Image : $GOLD_IMAGE_ID"
|
||||||
|
echo " Zieltag : $RECOVERY_GOLD_TAG (fester Tag, kein Overwrite)"
|
||||||
|
echo " Guards:"
|
||||||
|
snapshot_guards
|
||||||
|
if [ "$RUN_MODE" = "DRY_RUN" ]; then
|
||||||
|
echo "DRY_RUN_PASS: snapshot-recovery wuerde docker commit $RECOVERY_GOLD_CID -> $RECOVERY_GOLD_TAG (keine Aktion)."
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
mode_or_die # DRY_RUN -> FAIL CLOSED (kein commit)
|
||||||
|
echo " [commit] docker commit $RECOVERY_GOLD_CID -> $RECOVERY_GOLD_TAG (Standard-Pause durch Docker)"
|
||||||
|
docker commit "$RECOVERY_GOLD_CID" "$RECOVERY_GOLD_TAG" >/dev/null || die "docker commit fehlgeschlagen (FAIL CLOSED)"
|
||||||
|
# Nach Commit: Gold-Container wieder Running und healthy pruefen
|
||||||
|
local running
|
||||||
|
running="$(docker inspect -f '{{.State.Running}}' "$HISTORICAL_SERVICE" 2>/dev/null)"
|
||||||
|
[ "$running" = "true" ] || die "Nach-Commit: Gold-Container nicht Running (FAIL CLOSED)"
|
||||||
|
local newid
|
||||||
|
newid="$(docker image inspect -f '{{.Id}}' "$RECOVERY_GOLD_TAG" 2>/dev/null)"
|
||||||
|
[ -n "$newid" ] || die "Nach-Commit: Recovery-Image nicht auffindbar (FAIL CLOSED)"
|
||||||
|
echo "RECOVERY_IMAGE_ID=$newid"
|
||||||
|
echo "KLASSIFIZIERUNG: NEW RECOVERY IMAGE DERIVED FROM VERIFIED GOLD CONTAINER (NICHT das Original-Gold-Image)."
|
||||||
|
echo "Original bleibt: $GOLD_IMAGE_ID"
|
||||||
|
echo "SNAPSHOT-RECOVERY OK. KEINE weitere Mutation."
|
||||||
|
}
|
||||||
|
|
||||||
|
# verify-recovery-snapshot (read-only): Recovery-Image-Code vs Gold-Code vs Gold-Manifest
|
||||||
|
verify_recovery_snapshot() {
|
||||||
|
req_ok; expect_zero "$@"
|
||||||
|
echo "=== VERIFY-RECOVERY-SNAPSHOT (Recovery-Image vs Gold-Container vs Gold-Manifest) ==="
|
||||||
|
[ -f "$GOLD_MANIFEST" ] || die "Gold-Manifest fehlt (FAIL CLOSED)"
|
||||||
|
# Recovery-Tag muss existieren
|
||||||
|
if ! docker image inspect "$RECOVERY_GOLD_TAG" >/dev/null 2>&1; then
|
||||||
|
die "Recovery-Tag $RECOVERY_GOLD_TAG existiert nicht (FAIL CLOSED — zuerst snapshot-recovery)"
|
||||||
|
fi
|
||||||
|
# 1. Tag + Immutable-ID zeigen auf exakt dasselbe Image (Punkt 9)
|
||||||
|
# tagid = ID aus Tag-Referenz; immid = ID aus der vollen Immutable-Referenz (sha256:<ID>)
|
||||||
|
local tagid immid
|
||||||
|
tagid="$(docker image inspect -f '{{.Id}}' "$RECOVERY_GOLD_TAG" 2>/dev/null)"
|
||||||
|
[ -n "$tagid" ] || die "Verify: Recovery-Tag-ID nicht aufloesbar (FAIL CLOSED)"
|
||||||
|
immid="$(docker inspect "sha256:$tagid" --format '{{.Id}}' 2>/dev/null)"
|
||||||
|
[ "$tagid" = "$immid" ] || die "Verify: Tag/Immutable-ID-Referenz stimmen nicht ueberein (Punkt 9 FAIL CLOSED)"
|
||||||
|
echo " [Punkt 9] Tag + Immutable-ID -> selbe Image-ID: $tagid (OK)"
|
||||||
|
# 2. Manifest vom Recovery-Image ueber temporaeren Container (kein Entrypoint, keine Volumes/Ports/Netzwerk)
|
||||||
|
local tmpc tmpf
|
||||||
|
tmpf="$(mktemp)"
|
||||||
|
# Container-2 (nicht laufend) aus Recovery-Image erzeugen, NUR fuer Datei-Manifest; kein Start
|
||||||
|
tmpc="rq-recv-verify-$(date +%s)"
|
||||||
|
docker create --name "$tmpc" --entrypoint /bin/true "$RECOVERY_GOLD_TAG" >/dev/null 2>&1 \
|
||||||
|
|| { rm -f "$tmpf"; die "Verify: Temp-Container konnte nicht erstellt werden (FAIL CLOSED)"; }
|
||||||
|
# Manifest via docker cp aus dem erzeugten (nicht gestarteten) Container lesen
|
||||||
|
docker cp "$tmpc:/app/app" "/tmp/rq-recv-verify-copy-$$" >/dev/null 2>&1 \
|
||||||
|
|| { docker rm -f "$tmpc" >/dev/null 2>&1; rm -f "$tmpf"; rm -rf "/tmp/rq-recv-verify-copy-$$" 2>/dev/null; die "Verify: Manifest-Lesen aus Recovery-Image fehlgeschlagen (FAIL CLOSED)"; }
|
||||||
|
( cd "/tmp/rq-recv-verify-copy-$$" && find . -type f -name '*.py' -printf '%P\n' 2>/dev/null | sort | while IFS= read -r rel; do
|
||||||
|
sha256sum "$rel" 2>/dev/null | awk -v r="$rel" '{print $1" "r}'
|
||||||
|
done ) > "$tmpf"
|
||||||
|
docker rm -f "$tmpc" >/dev/null 2>&1 || true
|
||||||
|
rm -rf "/tmp/rq-recv-verify-copy-$$" 2>/dev/null
|
||||||
|
classify_and_report "$tmpf"
|
||||||
|
local ret="$G_RET"; rm -f "$tmpf"
|
||||||
|
if [ "$ret" = "ok" ]; then
|
||||||
|
echo "VERDICT: RECOVERY GOLD SNAPSHOT OK (Recovery-Image == Gold-Code, 0 DIVERGED/MISSING/unerlaubte EXTRA)"
|
||||||
|
exit 0
|
||||||
|
else
|
||||||
|
echo "VERDICT: RECOVERY GOLD SNAPSHOT FAIL"
|
||||||
|
exit 2
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
# ---- build-historical (NUR Build, kein recreate; DRY_RUN = kein Bau) ----
|
# ---- build-historical (NUR Build, kein recreate; DRY_RUN = kein Bau) ----
|
||||||
build_historical() {
|
build_historical() {
|
||||||
expect_zero "$@"
|
expect_zero "$@"
|
||||||
|
|
@ -300,8 +437,8 @@ if [ "$SUBCMD" = "dry-run" ]; then
|
||||||
SUBCMD="${1:-}"
|
SUBCMD="${1:-}"
|
||||||
shift || true
|
shift || true
|
||||||
case "$SUBCMD" in
|
case "$SUBCMD" in
|
||||||
build-historical|recreate-historical) ;;
|
build-historical|recreate-historical|snapshot-recovery) ;;
|
||||||
*) die "dry-run nur mit build-historical|recreate-historical zulaessig" ;;
|
*) die "dry-run nur mit build-historical|recreate-historical|snapshot-recovery zulaessig" ;;
|
||||||
esac
|
esac
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
|
@ -665,6 +802,8 @@ case "$SUBCMD" in
|
||||||
fi
|
fi
|
||||||
echo "RESTORE-SOT OK. KEIN Build/Recreate danach."
|
echo "RESTORE-SOT OK. KEIN Build/Recreate danach."
|
||||||
;;
|
;;
|
||||||
|
snapshot-recovery) snapshot_recovery "$@" ;;
|
||||||
|
verify-recovery-snapshot) verify_recovery_snapshot "$@" ;;
|
||||||
build-historical) build_historical "$@" ;;
|
build-historical) build_historical "$@" ;;
|
||||||
create-build-exclusion) expect_zero "$@"
|
create-build-exclusion) expect_zero "$@"
|
||||||
# Build-Pollution-Exclusion implementieren (Punkt 5-8): .dockerignore anlegen/ersetzen
|
# Build-Pollution-Exclusion implementieren (Punkt 5-8): .dockerignore anlegen/ersetzen
|
||||||
|
|
@ -692,6 +831,6 @@ case "$SUBCMD" in
|
||||||
recreate-historical) recreate_historical "$@" ;;
|
recreate-historical) recreate_historical "$@" ;;
|
||||||
verify-gold-reproduction) verify_gold_reproduction "$@" ;;
|
verify-gold-reproduction) verify_gold_reproduction "$@" ;;
|
||||||
|
|
||||||
*) die "Aufruf: ps|inspect|logs|run|code <K>|hashes|code-list|stage-dir|stage-check <K>|test <K>|search-writes|healthcheck|backup <K>|deploy <K>|rollback <K>|build-def|build-files|host-manifest|host-code <p>|container-manifest|container-code <p>|tree-diff|backup-sot|sot-session|recover-sot <K>|recover-sot-list|verify-sot-recovery|restore-sot|build-historical|dry-run build-historical|recreate-historical|dry-run recreate-historical|verify-gold-reproduction|create-build-exclusion|show-build-exclusion"
|
*) die "Aufruf: ps|inspect|logs|run|code <K>|hashes|code-list|stage-dir|stage-check <K>|test <K>|search-writes|healthcheck|backup <K>|deploy <K>|rollback <K>|build-def|build-files|host-manifest|host-code <p>|container-manifest|container-code <p>|tree-diff|backup-sot|sot-session|recover-sot <K>|recover-sot-list|verify-sot-recovery|restore-sot|snapshot-recovery|dry-run snapshot-recovery|verify-recovery-snapshot|build-historical|dry-run build-historical|recreate-historical|dry-run recreate-historical|verify-gold-reproduction|create-build-exclusion|show-build-exclusion"
|
||||||
;;
|
;;
|
||||||
esac
|
esac
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue