rq-historical: HOST-SOT-RECOVERY Kanal (backup-sot, recover-sot, verify-sot-recovery, restore-sot) — Gold-SHA-gebunden, fail-closed, kein Build
This commit is contained in:
Rain Ocampo
parent
9129e76a6d
commit
64240ea3b1
1 changed files with 163 additions and 1 deletions
|
|
@ -50,6 +50,39 @@ STAGING_ROOT=/opt/rq-historical-staging
|
||||||
BACKUP_ROOT=/opt/rq-historical-backups
|
BACKUP_ROOT=/opt/rq-historical-backups
|
||||||
SCRIPT=/app/app/run_backfill_year.py
|
SCRIPT=/app/app/run_backfill_year.py
|
||||||
|
|
||||||
|
# --------------------------------------------------------- HOST-SOT-RECOVERY
|
||||||
|
# CONFIRMED GOLD CONTAINER -> ROOT-CONTROLLED BACKUP -> EXACT BYTE COPY
|
||||||
|
# -> HOST SOURCE OF TRUTH -> SHA VERIFY -> STOPP. (Phase 13.5, Extension 1)
|
||||||
|
# KEIN Build/Recreate in dieser Mission.
|
||||||
|
SOT_BACKUP_ROOT=/opt/rq-historical-sot-backups
|
||||||
|
SOT_SESSION_MARKER=".rq-sot-session" # eindeutige Markierung der Recovery-Session
|
||||||
|
|
||||||
|
# Recovery-Allowlist: KEY -> "CONTAINER_SRC|HOST_TARGET|EXPECTED_GOLD_SHA"
|
||||||
|
# Containerpfade aus container-manifest verifiziert (NICHT geraten).
|
||||||
|
# HINWEIS: Aufgabe nennt trust/init.py, Container hat aber trust/__init__.py
|
||||||
|
# (Gold-SHA 059db627...). Punkt 5: exakte Pfade aus Manifest, nicht raten.
|
||||||
|
declare -A RECOVERY=(
|
||||||
|
[backfill]="/app/app/backfill.py|/opt/historical-v2/app/backfill.py|1627682ae582bb13343ae8ea10491dd9dffaef6fa4f7594d85378c2a500e81c7"
|
||||||
|
[repository]="/app/app/persistence/repository.py|/opt/historical-v2/app/persistence/repository.py|8fff892f52a612f464b5be21c66ef7787a47f2d7d049f815ef87837553c2a228"
|
||||||
|
[pipeline]="/app/app/pipeline_v2.py|/opt/historical-v2/app/pipeline_v2.py|7c08ec3d1e75854cf341b9cbfd12afc075c4a173518dd3d951be4f95c5175fe6"
|
||||||
|
[trust_init]="/app/app/trust/__init__.py|/opt/historical-v2/app/trust/__init__.py|059db62741f2001f548d1879ccd3bf73bb2f4c8d0f492d151a469c9e4dcd3934"
|
||||||
|
[provenance]="/app/app/trust/provenance.py|/opt/historical-v2/app/trust/provenance.py|4dd2ab977b69e9a47bb744dcc742a94d506e13035157a3977cdc7c8924d5f16f"
|
||||||
|
[trust]="/app/app/trust/trust.py|/opt/historical-v2/app/trust/trust.py|7c2a948d2f1d668201e5af2a28164bcb2451b31d514a9f5f22b435c9ff2a0d01"
|
||||||
|
[sanity]="/app/app/trust/sanity.py|/opt/historical-v2/app/trust/sanity.py|875b07869e5148983e7a06bc05f6a6bec30727d1b5e95242d459564c18f6d6e9"
|
||||||
|
[trust_gate]="/app/app/trust/trust_gate.py|/opt/historical-v2/app/trust/trust_gate.py|df0a97a1fc27985fe49b7458d94f7922bc5eaf2250807043c48dbc173f215ff1"
|
||||||
|
[diag_bi5]="/app/app/diag_bi5.py|/opt/historical-v2/app/diag_bi5.py|3e07ec5c4d9f572a118c278d0dc5c86865a8fe2067b12f85a3fa958ea4013efa"
|
||||||
|
[smoke_readonly]="/app/app/smoke_readonly.py|/opt/historical-v2/app/smoke_readonly.py|85ca939c11d789684740833f0bddb6e4f2a548a7f94870662191b75aefdcbfd6"
|
||||||
|
[smoke_trust_gate]="/app/app/smoke_trust_gate.py|/opt/historical-v2/app/smoke_trust_gate.py|f015fe650d6934b0c71d70bc3011f16b7f4bb0d2ffd947b9127040067442cfff"
|
||||||
|
[test_trust_gate]="/app/app/test_trust_gate.py|/opt/historical-v2/app/test_trust_gate.py|7867e2b526491de966e64486e8d49ec2cad204d97efc25844562b867ef861114"
|
||||||
|
[test_phase13_3]="/app/app/test_phase13_3_reproduction.py|/opt/historical-v2/app/test_phase13_3_reproduction.py|de7d2f7be785b7525a045b91ca055dca305fd7d575b291528141f000d5d003e7"
|
||||||
|
)
|
||||||
|
# Host-Only-Legacy (Punkt 9): NICHT anfassen
|
||||||
|
HOST_ONLY_LEGACY=(
|
||||||
|
"/opt/historical-v2/app/app/backfill.py"
|
||||||
|
"/opt/historical-v2/app/app/test_idempotency.py"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
die() { echo "ERR: $*" >&2; exit 2; }
|
die() { echo "ERR: $*" >&2; exit 2; }
|
||||||
req_ok() {
|
req_ok() {
|
||||||
docker inspect -f '{{.State.Running}}' "$CONTAINER" >/dev/null 2>&1 \
|
docker inspect -f '{{.State.Running}}' "$CONTAINER" >/dev/null 2>&1 \
|
||||||
|
|
@ -319,6 +352,135 @@ case "$SUBCMD" in
|
||||||
rm -f "$hmap" "$cmap" "$tmpunion"
|
rm -f "$hmap" "$cmap" "$tmpunion"
|
||||||
;;
|
;;
|
||||||
|
|
||||||
*) die "Aufruf: ps|inspect|logs|run|code <K>|hashes|code-list|stage-dir|stage-check <K>|test <K>|search-writes|healthcheck|backup <K>|deploy <K>|rollback <K>|build-def|build-files|host-manifest|host-code <p>|container-manifest|container-code <p>|tree-diff"
|
# ============ HOST-SOT-RECOVERY (Phase 13.5, Extension 1) ============
|
||||||
|
# KEIN Build/Recreate. Nur: backup-sot -> recover-sot -> verify-sot-recovery
|
||||||
|
# -> restore-sot. Fail-closed, Gold-SHA-gebunden, keine freien Pfade.
|
||||||
|
backup-sot) expect_zero "$@"
|
||||||
|
# Vollstaendiges Host-SoT-Backup in root-owned Recovery-Bereich
|
||||||
|
ts="$(date +%Y%m%d_%H%M%S)"
|
||||||
|
bdir="$SOT_BACKUP_ROOT/$ts"
|
||||||
|
mkdir -p "$bdir"
|
||||||
|
# tar mit Symlinks/Permissions/Ownership; kein Follow (kein -h)
|
||||||
|
if ! tar -cpf "$bdir/historical-v2.tar" -C /opt historical-v2 2>/dev/null; then
|
||||||
|
rm -rf "$bdir"; die "backup-sot: tar fehlgeschlagen (FAIL CLOSED)"
|
||||||
|
fi
|
||||||
|
# SHA256-Manifest aller regulären Dateien (nicht Symlinks folgen)
|
||||||
|
( cd /opt/historical-v2 && find . -type f -print0 2>/dev/null | sort -z \
|
||||||
|
| while IFS= read -r -d "" f; do
|
||||||
|
sha="$(sha256sum "$f" 2>/dev/null | awk '{print $1}')"
|
||||||
|
printf '%s %s\n' "$sha" "${f#./}"
|
||||||
|
done ) > "$bdir/SHA256.manifest"
|
||||||
|
# Struktur-/Pflichtpruefung (Punkt 4)
|
||||||
|
ok=1
|
||||||
|
[ -f "$bdir/historical-v2.tar" ] || { echo "FEHLT: tar"; ok=0; }
|
||||||
|
[ -s "$bdir/SHA256.manifest" ] || { echo "FEHLT: manifest"; ok=0; }
|
||||||
|
[ -f /opt/historical-v2/Dockerfile ] || { echo "FEHLT: Dockerfile"; ok=0; }
|
||||||
|
[ -f /opt/historical-v2/docker-compose.yml ] || { echo "FEHLT: compose"; ok=0; }
|
||||||
|
[ -d /opt/historical-v2/app ] || { echo "FEHLT: app/"; ok=0; }
|
||||||
|
if [ "$ok" != "1" ]; then
|
||||||
|
rm -rf "$bdir"; die "backup-sot: Validierung fehlgeschlagen (FAIL CLOSED)"
|
||||||
|
fi
|
||||||
|
# Session-Marker: eindeutige Markierung der aktuellen Recovery-Session
|
||||||
|
echo "$ts" > "$bdir/$SOT_SESSION_MARKER"
|
||||||
|
chown -R root:root "$bdir"; chmod 700 "$bdir"
|
||||||
|
echo "BACKUP-SOT OK: $bdir"
|
||||||
|
echo " tar: $(stat -c '%s' "$bdir/historical-v2.tar") bytes"
|
||||||
|
echo " manifest: $(wc -l < "$bdir/SHA256.manifest") Dateien"
|
||||||
|
echo " session: $ts"
|
||||||
|
;;
|
||||||
|
sot-session) expect_zero "$@"
|
||||||
|
# Zeigt die aktuelle (neueste) Recovery-Session-Backup-ID
|
||||||
|
latest="$(ls -1t "$SOT_BACKUP_ROOT" 2>/dev/null | head -1)"
|
||||||
|
[ -n "$latest" ] || die "kein backup-sot vorhanden"
|
||||||
|
echo "SESSION: $latest"
|
||||||
|
echo "PFAD: $SOT_BACKUP_ROOT/$latest"
|
||||||
|
;;
|
||||||
|
recover-sot) req_ok; [ $# -eq 1 ] || die "recover-sot braucht genau EINEN Key (recover-sot-list)"
|
||||||
|
key="$1"
|
||||||
|
spec="${RECOVERY[$key]:-}"
|
||||||
|
[ -n "$spec" ] || die "unbekannter Recovery-Key '$key' (recover-sot-list)"
|
||||||
|
IFS='|' read -r csrc htarget gold <<< "$spec"
|
||||||
|
# 1. gültiges vollständiges backup-sot vorhanden?
|
||||||
|
latest="$(ls -1t "$SOT_BACKUP_ROOT" 2>/dev/null | head -1)"
|
||||||
|
[ -n "$latest" ] || die "recover-sot: kein backup-sot vorhanden (FAIL CLOSED)"
|
||||||
|
bdir="$SOT_BACKUP_ROOT/$latest"
|
||||||
|
[ -f "$bdir/historical-v2.tar" ] && [ -s "$bdir/SHA256.manifest" ] || die "recover-sot: Backup unvollständig (FAIL CLOSED)"
|
||||||
|
# 2. Source = historical-service fixed (req_ok bereits geprueft)
|
||||||
|
# 3/4. Source regular file, kein Symlink
|
||||||
|
if ! docker exec "$CONTAINER" sh -c "[ -f '$csrc' ]" 2>/dev/null; then die "recover-sot: Source kein File im Container: $csrc"; fi
|
||||||
|
if docker exec "$CONTAINER" sh -c "[ -L '$csrc' ]" 2>/dev/null; then die "recover-sot: Source ist Symlink (FAIL CLOSED): $csrc"; fi
|
||||||
|
# 5. Source SHA == EXPECTED GOLD SHA
|
||||||
|
csha="$(docker exec "$CONTAINER" sha256sum "$csrc" 2>/dev/null | awk '{print $1}')" || csha=""
|
||||||
|
if [ "$csha" != "$gold" ]; then
|
||||||
|
die "recover-sot: Source-SHA-Mismatch fuer '$key' (erwartet $gold, Container $csha) — FAIL CLOSED"
|
||||||
|
fi
|
||||||
|
# 6. Host target exakt fester Pfad (aus Allowlist, kein freier Pfad)
|
||||||
|
[[ "$htarget" == /opt/historical-v2/app/* ]] || die "recover-sot: Host-Ziel ausserhalb app/ (FAIL CLOSED)"
|
||||||
|
# 7. Zielverzeichnis kontrolliert erzeugen (z.B. trust/)
|
||||||
|
tdir="$(dirname "$htarget")"
|
||||||
|
mkdir -p "$tdir"
|
||||||
|
# 8. Ziel vor Ueberschreiben einzeln sichern, falls vorhanden
|
||||||
|
if [ -e "$htarget" ]; then
|
||||||
|
prebak="$bdir/pre-recover/$(basename "$htarget")"
|
||||||
|
mkdir -p "$bdir/pre-recover"
|
||||||
|
cp -a "$htarget" "$prebak" 2>/dev/null || true
|
||||||
|
echo " pre-recover gesichert: $prebak"
|
||||||
|
fi
|
||||||
|
# 9. byte-exakt schreiben (docker cp = byte-identisch)
|
||||||
|
docker cp "$CONTAINER:$csrc" "$htarget"
|
||||||
|
# 10. Ownership/Permissions aus bestehendem Host-Kontext ableiten
|
||||||
|
# (app/ ist UID 10010; trust/ neu -> gleiche Basis wie app/)
|
||||||
|
chown 10010:10010 "$htarget" 2>/dev/null || chown root:root "$htarget"
|
||||||
|
chmod 644 "$htarget"
|
||||||
|
# 11. Host SHA == Gold SHA
|
||||||
|
hsha="$(sha256sum "$htarget" | awk '{print $1}')"
|
||||||
|
if [ "$hsha" != "$gold" ]; then
|
||||||
|
die "recover-sot: Host-SHA-Mismatch fuer '$key' (erwartet $gold, Host $hsha) — FAIL CLOSED"
|
||||||
|
fi
|
||||||
|
echo "RECOVER-SOT OK: $key"
|
||||||
|
echo " source : $CONTAINER:$csrc"
|
||||||
|
echo " target : $htarget"
|
||||||
|
echo " gold : $gold"
|
||||||
|
echo " host : $hsha"
|
||||||
|
;;
|
||||||
|
recover-sot-list) expect_zero "$@"
|
||||||
|
echo "Recovery-Allowlist (CONFIRMED GOLD, fest gebunden):"
|
||||||
|
for k in "${!RECOVERY[@]}"; do
|
||||||
|
IFS='|' read -r csrc htarget gold <<< "${RECOVERY[$k]}"
|
||||||
|
printf ' %-16s %-45s -> %s\n' "$k" "$csrc" "$htarget"
|
||||||
|
done
|
||||||
|
;;
|
||||||
|
verify-sot-recovery) req_ok; expect_zero "$@"
|
||||||
|
echo "=== VERIFY-SOT-RECOVERY (Gold vs Host vs Container) ==="
|
||||||
|
allok=1
|
||||||
|
for k in "${!RECOVERY[@]}"; do
|
||||||
|
IFS='|' read -r csrc htarget gold <<< "${RECOVERY[$k]}"
|
||||||
|
hsha="$(sha256sum "$htarget" 2>/dev/null | awk '{print $1}')" || hsha=""
|
||||||
|
csha="$(docker exec "$CONTAINER" sha256sum "$csrc" 2>/dev/null | awk '{print $1}')" || csha=""
|
||||||
|
if [ "$hsha" = "$gold" ] && [ "$csha" = "$gold" ]; then
|
||||||
|
printf ' MATCH %-16s %s\n' "$k" "$htarget"
|
||||||
|
else
|
||||||
|
printf ' FAIL %-16s host=%s container=%s gold=%s\n' "$k" "${hsha:-MISSING}" "${csha:-MISSING}" "$gold"
|
||||||
|
allok=0
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
[ "$allok" = "1" ] && echo "VERIFY-SOT-RECOVERY: ALLE MATCH" || { echo "VERIFY-SOT-RECOVERY: FEHLER"; exit 2; }
|
||||||
|
;;
|
||||||
|
restore-sot) expect_zero "$@"
|
||||||
|
# Restore des eindeutig markierten aktuellen Recovery-Session-Backups
|
||||||
|
latest="$(ls -1t "$SOT_BACKUP_ROOT" 2>/dev/null | head -1)"
|
||||||
|
[ -n "$latest" ] || die "restore-sot: kein backup-sot vorhanden"
|
||||||
|
bdir="$SOT_BACKUP_ROOT/$latest"
|
||||||
|
[ -f "$bdir/$SOT_SESSION_MARKER" ] || die "restore-sot: kein markiertes Session-Backup (FAIL CLOSED)"
|
||||||
|
[ -f "$bdir/historical-v2.tar" ] || die "restore-sot: tar fehlt (FAIL CLOSED)"
|
||||||
|
echo "RESTORE-SOT: $bdir"
|
||||||
|
echo " (entpackt /opt/historical-v2 aus Session-Backup, byte-exakt)"
|
||||||
|
if ! tar -xpf "$bdir/historical-v2.tar" -C /opt 2>/dev/null; then
|
||||||
|
die "restore-sot: tar-Restore fehlgeschlagen (FAIL CLOSED)"
|
||||||
|
fi
|
||||||
|
echo "RESTORE-SOT OK. KEIN Build/Recreate danach."
|
||||||
|
;;
|
||||||
|
|
||||||
|
*) die "Aufruf: ps|inspect|logs|run|code <K>|hashes|code-list|stage-dir|stage-check <K>|test <K>|search-writes|healthcheck|backup <K>|deploy <K>|rollback <K>|build-def|build-files|host-manifest|host-code <p>|container-manifest|container-code <p>|tree-diff|backup-sot|sot-session|recover-sot <K>|recover-sot-list|verify-sot-recovery|restore-sot"
|
||||||
;;
|
;;
|
||||||
esac
|
esac
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue