nexo312/trading-system-docs
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions
trading-system-docs/red-queen-architecture/control-plane/cp2a2_2b/rq-historical

486 lines
26 KiB
Bash
Raw Blame History

#!/bin/bash
# ============================================================================
# rq-historical — eng begrenzter root-Wrapper fuer Red Queen (rq-host)
# RED-QUEEN-VPS-HOST-ACCESS + READONLY-CODE + DEV/TEST/DEPLOY + SOT-DISCOVERY
# | Rain Ocampo (Hermes) | 2026-08-29
#
# LEAST-PRIVILEGE-Design:
# - rq-host hat KEINE docker-Gruppe, KEINEN Docker-Socket, KEIN sudo ALL,
# KEINE Root-Shell. Einziger Root-Kanal: NOPASSWD:/usr/local/bin/rq-historical
# - Dieser Wrapper ist der EINZIGE Kanal zu historical-service & Host-SoT.
# Keine freien Pfade, kein arbitrary cat/find/grep, kein Shell.
#
# SoT-DISCOVERY (read-only, NO copy/rebuild/deploy/fix):
# build-def -> zeigt Dockerfile + Compose-Builddefinition (redacted)
# build-files -> Dockerfile + compose + .dockerignore Auflistung
# host-manifest -> SHA256+size aller *.py unter /opt/historical-v2/app
# host-code <rel> -> gezielter Code-Read NUR aus host-manifest (kontrolliert)
# container-manifest -> SHA256+size aller *.py unter historical-service:/app/app
# container-code <rel> -> gezielter Code-Read NUR aus container-manifest
# tree-diff -> Host vs Container Manifest-Diff
#
# SUBKOMMANDOS (bestehend, unveraendert):
# ps | inspect | logs [--tail N] | run [..] | code <KEY> | hashes | code-list
# stage-dir | stage-check <KEY> | test <KEY> | search-writes | healthcheck
# backup <KEY> | deploy <KEY> | rollback <KEY>
# ============================================================================
set -euo pipefail
CONTAINER=historical-service
HOST_APP=/opt/historical-v2/app
HOST_DIR=/opt/historical-v2
# --------------------------------------------------------- Allowlist (fest)
declare -A TARGET_PATHS=(
[runner]="/app/app/run_backfill_year.py"
[backfill]="/app/app/backfill.py"
[pipeline]="/app/app/pipeline_v2.py"
[repository]="/app/app/persistence/repository.py"
[provenance]="/app/app/trust/provenance.py"
[trust]="/app/app/trust/trust.py"
[sanity]="/app/app/trust/sanity.py"
[trust_gate]="/app/app/trust/trust_gate.py"
)
declare -A STAGE_FILES=(
[runner]="run_backfill_year.py" [backfill]="backfill.py" [pipeline]="pipeline_v2.py"
[repository]="repository.py" [provenance]="provenance.py" [trust]="trust.py"
[sanity]="sanity.py" [trust_gate]="trust_gate.py"
)
STAGING_ROOT=/opt/rq-historical-staging
BACKUP_ROOT=/opt/rq-historical-backups
SCRIPT=/app/app/run_backfill_year.py
# --------------------------------------------------------- HOST-SOT-RECOVERY
# CONFIRMED GOLD CONTAINER -> ROOT-CONTROLLED BACKUP -> EXACT BYTE COPY
# -> HOST SOURCE OF TRUTH -> SHA VERIFY -> STOPP. (Phase 13.5, Extension 1)
# KEIN Build/Recreate in dieser Mission.
SOT_BACKUP_ROOT=/opt/rq-historical-sot-backups
SOT_SESSION_MARKER=".rq-sot-session" # eindeutige Markierung der Recovery-Session
# Recovery-Allowlist: KEY -> "CONTAINER_SRC|HOST_TARGET|EXPECTED_GOLD_SHA"
# Containerpfade aus container-manifest verifiziert (NICHT geraten).
# HINWEIS: Aufgabe nennt trust/init.py, Container hat aber trust/__init__.py
# (Gold-SHA 059db627...). Punkt 5: exakte Pfade aus Manifest, nicht raten.
declare -A RECOVERY=(
[backfill]="/app/app/backfill.py|/opt/historical-v2/app/backfill.py|1627682ae582bb13343ae8ea10491dd9dffaef6fa4f7594d85378c2a500e81c7"
[repository]="/app/app/persistence/repository.py|/opt/historical-v2/app/persistence/repository.py|8fff892f52a612f464b5be21c66ef7787a47f2d7d049f815ef87837553c2a228"
[pipeline]="/app/app/pipeline_v2.py|/opt/historical-v2/app/pipeline_v2.py|7c08ec3d1e75854cf341b9cbfd12afc075c4a173518dd3d951be4f95c5175fe6"
[trust_init]="/app/app/trust/__init__.py|/opt/historical-v2/app/trust/__init__.py|059db62741f2001f548d1879ccd3bf73bb2f4c8d0f492d151a469c9e4dcd3934"
[provenance]="/app/app/trust/provenance.py|/opt/historical-v2/app/trust/provenance.py|4dd2ab977b69e9a47bb744dcc742a94d506e13035157a3977cdc7c8924d5f16f"
[trust]="/app/app/trust/trust.py|/opt/historical-v2/app/trust/trust.py|7c2a948d2f1d668201e5af2a28164bcb2451b31d514a9f5f22b435c9ff2a0d01"
[sanity]="/app/app/trust/sanity.py|/opt/historical-v2/app/trust/sanity.py|875b07869e5148983e7a06bc05f6a6bec30727d1b5e95242d459564c18f6d6e9"
[trust_gate]="/app/app/trust/trust_gate.py|/opt/historical-v2/app/trust/trust_gate.py|df0a97a1fc27985fe49b7458d94f7922bc5eaf2250807043c48dbc173f215ff1"
[diag_bi5]="/app/app/diag_bi5.py|/opt/historical-v2/app/diag_bi5.py|3e07ec5c4d9f572a118c278d0dc5c86865a8fe2067b12f85a3fa958ea4013efa"
[smoke_readonly]="/app/app/smoke_readonly.py|/opt/historical-v2/app/smoke_readonly.py|85ca939c11d789684740833f0bddb6e4f2a548a7f94870662191b75aefdcbfd6"
[smoke_trust_gate]="/app/app/smoke_trust_gate.py|/opt/historical-v2/app/smoke_trust_gate.py|f015fe650d6934b0c71d70bc3011f16b7f4bb0d2ffd947b9127040067442cfff"
[test_trust_gate]="/app/app/test_trust_gate.py|/opt/historical-v2/app/test_trust_gate.py|7867e2b526491de966e64486e8d49ec2cad204d97efc25844562b867ef861114"
[test_phase13_3]="/app/app/test_phase13_3_reproduction.py|/opt/historical-v2/app/test_phase13_3_reproduction.py|de7d2f7be785b7525a045b91ca055dca305fd7d575b291528141f000d5d003e7"
)
# Host-Only-Legacy (Punkt 9): NICHT anfassen
HOST_ONLY_LEGACY=(
"/opt/historical-v2/app/app/backfill.py"
"/opt/historical-v2/app/app/test_idempotency.py"
)
die() { echo "ERR: $*" >&2; exit 2; }
req_ok() {
docker inspect -f '{{.State.Running}}' "$CONTAINER" >/dev/null 2>&1 \
|| die "Container '$CONTAINER' nicht vorhanden oder nicht am Laufen"
}
key_path() { echo "${TARGET_PATHS[$1]:-}"; }
stage_file() { echo "$STAGING_ROOT/${STAGE_FILES[$1]:-}"; }
key_ok() {
[ -n "${TARGET_PATHS[$1]:-}" ] && [ -n "${STAGE_FILES[$1]:-}" ] || die "unbekannter KEY '$1' (code-list)"
}
expect_zero() { [ $# -eq 0 ] || die "unerlaubtes Argument '$*'"; }
validate_stage() {
local key="$1" file real owner dmode
file="$(stage_file "$key")"
[ -e "$file" ] || die "Staging-Kandidat fehlt: $file (lege zuerst die Datei ab)"
real="$(realpath -m "$file")"
[[ "$real" == "$STAGING_ROOT/"* ]] || die "Traversal blockiert: realpath '$real' ausserhalb Staging"
[ "$real" = "$file" ] || die "Alias erkannt (realpath != Pfad): $file"
[ -f "$file" ] || die "keine reguläre Datei: $file"
[ -L "$file" ] && die "Symlink nicht erlaubt: $file"
owner="$(stat -c '%U' "$file")"
[ "$owner" = "rq-host" ] || die "Staging-Kandidat Owner = $owner (erwartet rq-host)"
dmode="$(stat -c '%a' "$STAGING_ROOT")"
[[ "$dmode" != *002 ]] && [[ "$dmode" != *022 ]] || die "Staging-Root global schreibbar '$dmode' (FAIL CLOSED)"
echo "OK: $key -> $file (regular, no symlink, owner rq-host, in staging)"
}
# ---- redact: maskiert Passwort-/Secret-Werte (fuer build-def Anzeige) ----
redact_line() {
# Zeilen mit "PASSWORD=" / "SECRET=" / "TOKEN=" : Wert maskieren
echo "$1" | sed -E 's/(PASSWORD|PASSWORDS|SECRET|SECRETS|TOKEN|TOKENS|API_KEY|CLIENT_SECRET|PRIVATE_KEY)(["'"'"']?[[:space:]]*=[[:space:]]*["'"'"']?)[^[:space:]"'"'"']+/\1\2***REDACTED***/Ig'
}
SUBCMD="${1:-}"
shift || true
case "$SUBCMD" in
# ---------------- bestehend ----------------
ps) expect_zero "$@"; exec docker ps ;;
inspect) req_ok; expect_zero "$@"; exec docker inspect "$CONTAINER" ;;
logs) req_ok
tail_n=200
while [ $# -gt 0 ]; do
case "$1" in
--tail) [ $# -ge 2 ] || die "--tail braucht Wert"; [[ "$2" =~ ^[0-9]+$ ]] || die "ungueltiges --tail"; tail_n="$2"; shift 2 ;;
*) die "unerlaubtes Argument 'logs $1'" ;;
esac
done
exec docker logs --tail "$tail_n" "$CONTAINER"
;;
run) req_ok
declare -a RUN=()
while [ $# -gt 0 ]; do
case "$1" in
--start)[[ "${2:-}" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}$ ]]||die "ungueltiges --start";RUN+=("--start" "$2");shift 2;;
--end)[[ "${2:-}" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}$ ]]||die "ungueltiges --end";RUN+=("--end" "$2");shift 2;;
--limit-days)[[ "${2:-}" =~ ^[0-9]+$ ]]||die "ungueltiges --limit-days";RUN+=("--limit-days" "$2");shift 2;;
*) die "unerlaubtes Argument 'run $1'" ;;
esac
done
echo "[rq-historical] Start: docker exec $CONTAINER python $SCRIPT ${RUN[*]}" >&2
exec docker exec "$CONTAINER" python "$SCRIPT" "${RUN[@]}"
;;
code) req_ok
[ $# -eq 1 ] || die "code braucht genau EINEN Key (code-list)"
p="$(key_path "$1")"; [ -n "$p" ] || die "unbekannter Key 'code $1'"
exec docker exec "$CONTAINER" cat "$p"
;;
hashes) req_ok; expect_zero "$@"
PATHS=(); for k in "${!TARGET_PATHS[@]}"; do PATHS+=("${TARGET_PATHS[$k]}"); done
exec docker exec "$CONTAINER" sha256sum "${PATHS[@]}"
;;
code-list) req_ok; expect_zero "$@"
echo "Verfuegbare READ-ONLY-Code-Keys (historical-service):"
for k in "${!TARGET_PATHS[@]}"; do printf ' %-11s | staging: %-22s -> %s\n' "$k" "${STAGE_FILES[$k]}" "${TARGET_PATHS[$k]}"; done
;;
stage-dir) expect_zero "$@"
echo "Staging-Root: $STAGING_ROOT (Owner rq-host, keine globalen Schreibrechte)"
for k in runner backfill pipeline repository provenance trust sanity trust_gate; do
printf ' %-11s %s\n' "$k" "$(stage_file "$k")"
done
;;
stage-check) [ $# -eq 1 ] || die "stage-check braucht genau EINEN Key"; key_ok "$1"; validate_stage "$1"
;;
test) req_ok; [ $# -eq 1 ] || die "test braucht genau EINEN Key"; key_ok "$1"
validate_stage "$1"
file="$(stage_file "$1")"
tmp="/tmp/rqtest_${1}_$(date +%s).py"
docker cp "$file" "$CONTAINER:$tmp"
if docker exec "$CONTAINER" python -m py_compile "$tmp" >/dev/null 2>&1; then
echo "PASS: py_compile OK ($file) [isoliert in Container-/tmp, Produktivcode unberuehrt]"
else
echo "FAIL: py_compile fehlgeschlagen ($file)"
docker exec "$CONTAINER" rm -f "$tmp"; exit 2
fi
docker exec "$CONTAINER" rm -f "$tmp"
;;
search-writes) req_ok; expect_zero "$@"
docker exec "$CONTAINER" sh -c 'grep -rnE "insert_raw_bars|insert_derived_bars|BackfillOrchestrator|trust_gate|TrustGate" /app/app --include="*.py" 2>/dev/null | head -100 || true'
;;
healthcheck) req_ok; expect_zero "$@"
rh="$(docker exec "$CONTAINER" python -c 'import urllib.request; print(urllib.request.urlopen("http://localhost:8000/health", timeout=5).status)' 2>/dev/null || echo N/A)"
rr="$(docker exec "$CONTAINER" python -c 'import urllib.request; print(urllib.request.urlopen("http://localhost:8000/health/ready", timeout=5).status)' 2>/dev/null || echo N/A)"
running="$(docker inspect -f '{{.State.Running}}' "$CONTAINER")"
echo "Container.Running=$running /health=$rh /health/ready=$rr"
[ "$running" = "true" ] && [ "$rh" = "200" ] && [ "$rr" = "200" ] || { echo "HEALTHCHECK FAILED"; exit 2; }
echo "HEALTHCHECK OK"
;;
backup) req_ok; [ $# -eq 1 ] || die "backup braucht genau EINEN Key"; key_ok "$1"
p="$(key_path "$1")"; ts="$(date +%Y%m%d_%H%M%S)"; bdir="$BACKUP_ROOT/$1/$ts"
mkdir -p "$bdir"
docker exec "$CONTAINER" cat "$p" > "$bdir/$(basename "$p")"
sha256sum "$bdir/$(basename "$p")" > "$bdir/SHA256"
echo "BACKUP: $1 -> $bdir"; cat "$bdir/SHA256"
;;
deploy) req_ok; [ $# -eq 1 ] || die "deploy braucht genau EINEN Key"; key_ok "$1"
key="$1"; p="$(key_path "$key")"; file="$(stage_file "$key")"
validate_stage "$key"
ts="$(date +%Y%m%d_%H%M%S)"; bdir="$BACKUP_ROOT/$key/$ts"
mkdir -p "$bdir"
docker exec "$CONTAINER" cat "$p" > "$bdir/$(basename "$p")" || die "Backup fehlgeschlagen -> deploy abgebrochen"
sha256sum "$bdir/$(basename "$p")" > "$bdir/SHA256"
echo "BACKUP: $bdir"
tmp="/tmp/rqdeploy_${key}_$(date +%s).py"
docker cp "$file" "$CONTAINER:$tmp"
if ! docker exec "$CONTAINER" python -m py_compile "$tmp" >/dev/null 2>&1; then
docker exec "$CONTAINER" rm -f "$tmp"; die "py_compile FAIL -> deploy abgebrochen (nichts deployt)"
fi
docker exec "$CONTAINER" rm -f "$tmp"
docker cp "$file" "$CONTAINER:$p"
sha_stage="$(sha256sum "$file" | awk '{print $1}')"
sha_deployed="$(docker exec "$CONTAINER" sha256sum "$p" | awk '{print $1}')"
echo "SHA stage : $sha_stage"; echo "SHA deployed: $sha_deployed"
if [ "$sha_stage" != "$sha_deployed" ]; then
echo "HASH-MISMATCH -> rollback"; docker cp "$bdir/$(basename "$p")" "$CONTAINER:$p"; die "Deploy Hash-Mismatch, zurueckgerollt"
fi
rh="$(docker exec "$CONTAINER" python -c 'import urllib.request; print(urllib.request.urlopen("http://localhost:8000/health", timeout=5).status)' 2>/dev/null || echo N/A)"
rr="$(docker exec "$CONTAINER" python -c 'import urllib.request; print(urllib.request.urlopen("http://localhost:8000/health/ready", timeout=5).status)' 2>/dev/null || echo N/A)"
echo "Health: /health=$rh /health/ready=$rr"
if [ "$rh" != "200" ] || [ "$rr" != "200" ]; then
echo "DEPLOY-HEALTHCHECK FAILED -> rollback"; docker cp "$bdir/$(basename "$p")" "$CONTAINER:$p"; die "Rollback ausgefuehrt"
fi
echo "DEPLOY OK: $key -> $p (Backup: $bdir). KEIN backfill/run gestartet."
;;
rollback) req_ok; [ $# -eq 1 ] || die "rollback braucht genau EINEN Key"; key_ok "$1"
p="$(key_path "$1")"
bdir="$(ls -1t "$BACKUP_ROOT/$1" 2>/dev/null | head -1)"
[ -n "$bdir" ] || die "kein Backup fuer '$1' vorhanden"
bdir="$BACKUP_ROOT/$1/$bdir"; src="$bdir/$(basename "$p")"
[ -f "$src" ] || die "Backup-Datei fehlt: $src"
sha_bak="$(sha256sum "$src" | awk '{print $1}')"
sha_cur="$(docker exec "$CONTAINER" sha256sum "$p" | awk '{print $1}')"
echo "aktuell: $sha_cur / backup: $sha_bak"
[ "$sha_bak" != "$sha_cur" ] || die "kein Unterschied (bereits auf Backup-Stand)"
docker cp "$src" "$CONTAINER:$p"
sha_cur="$(docker exec "$CONTAINER" sha256sum "$p" | awk '{print $1}')"
[ "$sha_cur" = "$sha_bak" ] || die "ROLLBACK-HASH-MISMATCH"
echo "ROLLBACK OK: $key -> $p auf $bdir"
;;
# ============ SOT-DISCOVERY (read-only, NEU) ============
build-def) expect_zero "$@"
echo "=== Dockerfile ($HOST_DIR/Dockerfile) — redacted ==="
[ -f "$HOST_DIR/Dockerfile" ] || die "Dockerfile nicht gefunden"
while IFS= read -r line; do redact_line "$line"; done < "$HOST_DIR/Dockerfile"
echo ""
echo "=== docker-compose.yml — strukturielle Abschnitte, redacted ==="
cf=""
if [ -f "$HOST_DIR/docker-compose.yml" ]; then cf="$HOST_DIR/docker-compose.yml";
elif [ -f "$HOST_DIR/compose.yml" ]; then cf="$HOST_DIR/compose.yml";
elif [ -f "$HOST_DIR/docker-compose.yaml" ]; then cf="$HOST_DIR/docker-compose.yaml"; fi
[ -n "$cf" ] || die "keine compose-Datei gefunden"
echo "(Quelle: $cf)"
while IFS= read -r line; do
# nur relevante strukturelle Schluessel anzeigen; Environment/Secrets nicht roh
case "$line" in
*POSTGRES_PASSWORD*|*PG_PASSWORD*|*PASSWORD*|*SECRET*|*TOKEN*) echo " **** REDACTED (Secret-Zeile) ****" ;;
*) echo " $line" ;;
esac
done < "$cf"
echo ""
echo "=== .dockerignore (falls vorhanden) ==="
[ -f "$HOST_DIR/.dockerignore" ] && cat "$HOST_DIR/.dockerignore" || echo "(kein .dockerignore)"
echo ""
echo "=== Service/Container-Fixierung ==="
echo " compose.project=$(docker inspect "$CONTAINER" --format '{{index .Config.Labels "com.docker.compose.project"}}' 2>/dev/null || echo N/A)"
echo " compose.service=$(docker inspect "$CONTAINER" --format '{{index .Config.Labels "com.docker.compose.service"}}' 2>/dev/null || echo N/A)"
echo " container=$CONTAINER"
;;
build-files) expect_zero "$@"
echo "=== Build-Dateien unter $HOST_DIR (nicht-verzeichnis, nicht .env/.dump/.sql) ==="
for f in Dockerfile docker-compose.yml compose.yml docker-compose.yaml .dockerignore requirements.txt; do
if [ -e "$HOST_DIR/$f" ]; then printf ' %-22s %s\n' "$f" "EXISTS ($(stat -c '%s' "$HOST_DIR/$f") bytes)"; else printf ' %-22s %s\n' "$f" "(fehlt)"; fi
done
;;
host-manifest) expect_zero "$@"
echo "=== HOST Manifest /opt/historical-v2/app (*.py, keine Symlinks folgen) ==="
[ -d "$HOST_APP" ] || die "Host-App-Verzeichnis fehlt: $HOST_APP"
# finde reguläre *.py UND Symlinks auf *.py separat
(
find "$HOST_APP" -type f -name "*.py" -printf '%P %s\n' 2>/dev/null | sort \
| while read -r rel size; do
sha=$(sha256sum "$HOST_APP/$rel" 2>/dev/null | awk '{print $1}')
printf 'FILE\t%s\t%s\t%s\n' "$rel" "$size" "${sha:-ERR}"
done
find "$HOST_APP" -type l -name "*.py" -printf '%P\n' 2>/dev/null | sort \
| while read -r rel; do printf 'SYMLINK\t%s\t0\t<not followed>\n' "$rel"; done
)
;;
host-code) [ $# -eq 1 ] || die "host-code braucht genau EINEN relativen Pfad (unter app/)"
rel="$1"
[[ "$rel" == *".."* ]] && die "Traversal nicht erlaubt"
[[ "$rel" == /* ]] && die "relativer Pfad erwartet, nicht absolut"
# realpath zwingend im app-Root
real="$(realpath -m "$HOST_APP/$rel")"
[[ "$real" == "$HOST_APP/"* ]] || die "Pfad ausserhalb app-Root blockiert"
[ -f "$real" ] || die "Datei nicht gefunden oder kein File: $rel"
[ -L "$real" ] && die "Symlink-Read nicht erlaubt"
# nur .py ausgeben
[[ "$rel" == *.py ]] || die "Nur .py-Quelldateien lesbar"
echo "--- $rel ($(stat -c '%s' "$real") bytes) ---"
exec cat "$real"
;;
container-manifest) req_ok; expect_zero "$@"
echo "=== CONTAINER Manifest historical-service:/app/app (*.py) ==="
docker exec "$CONTAINER" sh -c 'cd /app/app && find . -name "*.py" -type f 2>/dev/null | sed "s#^\./##" | sort | while read f; do size=$(stat -c %s "$f"); sha=$(sha256sum "$f" | cut -d" " -f1); printf "FILE\t%s\t%s\t%s\n" "$f" "$size" "$sha"; done'
docker exec "$CONTAINER" sh -c 'cd /app/app && find . -name "*.py" -type l 2>/dev/null | sed "s#^\./##" | sort | while read f; do printf "SYMLINK\t%s\t0\t<not followed>\n" "$f"; done' || true
;;
container-code) req_ok; [ $# -eq 1 ] || die "container-code braucht genau EINEN relativen Pfad (unter /app/app/)"
rel="$1"
[[ "$rel" == *".."* ]] && die "Traversal nicht erlaubt"
[[ "$rel" == /* ]] && die "relativer Pfad erwartet, nicht absolut"
target="/app/app/$rel"
# sicherstellen Ziel existiert als reguläre Datei & nicht symlink im Container
if ! docker exec "$CONTAINER" sh -c "[ -f '$target' ]" 2>/dev/null; then die "Datei nicht gefunden oder kein File im Container: $rel"; fi
if docker exec "$CONTAINER" sh -c "[ -L '$target' ]" 2>/dev/null; then die "Symlink-Read im Container nicht erlaubt"; fi
[[ "$rel" == *.py ]] || die "Nur .py-Quelldateien lesbar"
echo "--- $target ---"
exec docker exec "$CONTAINER" cat "$target"
;;
tree-diff) req_ok; expect_zero "$@"
echo "=== TREE-DIFF Host(/opt/historical-v2/app) <-> Container(/app/app) ==="
hmap="$(mktemp)"; cmap="$(mktemp)"
# Host: 'rel<TAB>sha'
find "$HOST_APP" -type f -name "*.py" -printf '%P\n' 2>/dev/null | sort | while IFS= read -r rel; do
sha="$(sha256sum "$HOST_APP/$rel" 2>/dev/null | awk '{print $1}')"
printf '%s\t%s\n' "$rel" "$sha"
done > "$hmap"
# Container: 'rel<TAB>sha'
docker exec "$CONTAINER" sh -c 'cd /app/app && find . -type f -name "*.py" -printf "%P\n" 2>/dev/null | sort | while IFS= read -r rel; do sha256sum "$rel" 2>/dev/null | awk -v r="$rel" '\''{print r"\t"$1}'\''; done' > "$cmap"
tmpunion="$(mktemp)"
{ cut -f1 "$hmap"; cut -f1 "$cmap"; } | sort -u > "$tmpunion"
echo "HOST_APP=$HOST_APP CONTAINER_APP=/app/app"
while IFS= read -r rel; do
hs="$(awk -F'\t' -v r="$rel" '$1==r{print $2}' "$hmap")"
cs="$(awk -F'\t' -v r="$rel" '$1==r{print $2}' "$cmap")"
if [ -n "$hs" ] && [ -n "$cs" ]; then
if [ "$hs" = "$cs" ]; then status="MATCH"; else status="DIVERGED"; fi
printf '%-18s %s [h:%s c:%s]\n' "$status" "$rel" "${hs:0:12}" "${cs:0:12}"
elif [ -n "$hs" ]; then
printf '%-18s %s\n' "HOST_ONLY" "$rel"
else
printf '%-18s %s\n' "MISSING_ON_HOST" "$rel"
fi
done < "$tmpunion"
rm -f "$hmap" "$cmap" "$tmpunion"
;;
# ============ HOST-SOT-RECOVERY (Phase 13.5, Extension 1) ============
# KEIN Build/Recreate. Nur: backup-sot -> recover-sot -> verify-sot-recovery
# -> restore-sot. Fail-closed, Gold-SHA-gebunden, keine freien Pfade.
backup-sot) expect_zero "$@"
# Vollstaendiges Host-SoT-Backup in root-owned Recovery-Bereich
ts="$(date +%Y%m%d_%H%M%S)"
bdir="$SOT_BACKUP_ROOT/$ts"
mkdir -p "$bdir"
# tar mit Symlinks/Permissions/Ownership; kein Follow (kein -h)
if ! tar -cpf "$bdir/historical-v2.tar" -C /opt historical-v2 2>/dev/null; then
rm -rf "$bdir"; die "backup-sot: tar fehlgeschlagen (FAIL CLOSED)"
fi
# SHA256-Manifest aller regulären Dateien (nicht Symlinks folgen)
( cd /opt/historical-v2 && find . -type f -print0 2>/dev/null | sort -z \
| while IFS= read -r -d "" f; do
sha="$(sha256sum "$f" 2>/dev/null | awk '{print $1}')"
printf '%s %s\n' "$sha" "${f#./}"
done ) > "$bdir/SHA256.manifest"
# Struktur-/Pflichtpruefung (Punkt 4)
ok=1
[ -f "$bdir/historical-v2.tar" ] || { echo "FEHLT: tar"; ok=0; }
[ -s "$bdir/SHA256.manifest" ] || { echo "FEHLT: manifest"; ok=0; }
[ -f /opt/historical-v2/Dockerfile ] || { echo "FEHLT: Dockerfile"; ok=0; }
[ -f /opt/historical-v2/docker-compose.yml ] || { echo "FEHLT: compose"; ok=0; }
[ -d /opt/historical-v2/app ] || { echo "FEHLT: app/"; ok=0; }
if [ "$ok" != "1" ]; then
rm -rf "$bdir"; die "backup-sot: Validierung fehlgeschlagen (FAIL CLOSED)"
fi
# Session-Marker: eindeutige Markierung der aktuellen Recovery-Session
echo "$ts" > "$bdir/$SOT_SESSION_MARKER"
chown -R root:root "$bdir"; chmod 700 "$bdir"
echo "BACKUP-SOT OK: $bdir"
echo " tar: $(stat -c '%s' "$bdir/historical-v2.tar") bytes"
echo " manifest: $(wc -l < "$bdir/SHA256.manifest") Dateien"
echo " session: $ts"
;;
sot-session) expect_zero "$@"
# Zeigt die aktuelle (neueste) Recovery-Session-Backup-ID
latest="$(ls -1t "$SOT_BACKUP_ROOT" 2>/dev/null | head -1)"
[ -n "$latest" ] || die "kein backup-sot vorhanden"
echo "SESSION: $latest"
echo "PFAD: $SOT_BACKUP_ROOT/$latest"
;;
recover-sot) req_ok; [ $# -eq 1 ] || die "recover-sot braucht genau EINEN Key (recover-sot-list)"
key="$1"
spec="${RECOVERY[$key]:-}"
[ -n "$spec" ] || die "unbekannter Recovery-Key '$key' (recover-sot-list)"
IFS='|' read -r csrc htarget gold <<< "$spec"
# 1. gültiges vollständiges backup-sot vorhanden?
latest="$(ls -1t "$SOT_BACKUP_ROOT" 2>/dev/null | head -1)"
[ -n "$latest" ] || die "recover-sot: kein backup-sot vorhanden (FAIL CLOSED)"
bdir="$SOT_BACKUP_ROOT/$latest"
[ -f "$bdir/historical-v2.tar" ] && [ -s "$bdir/SHA256.manifest" ] || die "recover-sot: Backup unvollständig (FAIL CLOSED)"
# 2. Source = historical-service fixed (req_ok bereits geprueft)
# 3/4. Source regular file, kein Symlink
if ! docker exec "$CONTAINER" sh -c "[ -f '$csrc' ]" 2>/dev/null; then die "recover-sot: Source kein File im Container: $csrc"; fi
if docker exec "$CONTAINER" sh -c "[ -L '$csrc' ]" 2>/dev/null; then die "recover-sot: Source ist Symlink (FAIL CLOSED): $csrc"; fi
# 5. Source SHA == EXPECTED GOLD SHA
csha="$(docker exec "$CONTAINER" sha256sum "$csrc" 2>/dev/null | awk '{print $1}')" || csha=""
if [ "$csha" != "$gold" ]; then
die "recover-sot: Source-SHA-Mismatch fuer '$key' (erwartet $gold, Container $csha) — FAIL CLOSED"
fi
# 6. Host target exakt fester Pfad (aus Allowlist, kein freier Pfad)
[[ "$htarget" == /opt/historical-v2/app/* ]] || die "recover-sot: Host-Ziel ausserhalb app/ (FAIL CLOSED)"
# 7. Zielverzeichnis kontrolliert erzeugen (z.B. trust/)
tdir="$(dirname "$htarget")"
mkdir -p "$tdir"
# 8. Ziel vor Ueberschreiben einzeln sichern, falls vorhanden
if [ -e "$htarget" ]; then
prebak="$bdir/pre-recover/$(basename "$htarget")"
mkdir -p "$bdir/pre-recover"
cp -a "$htarget" "$prebak" 2>/dev/null || true
echo " pre-recover gesichert: $prebak"
fi
# 9. byte-exakt schreiben (docker cp = byte-identisch)
docker cp "$CONTAINER:$csrc" "$htarget"
# 10. Ownership/Permissions aus bestehendem Host-Kontext ableiten
# (app/ ist UID 10010; trust/ neu -> gleiche Basis wie app/)
chown 10010:10010 "$htarget" 2>/dev/null || chown root:root "$htarget"
chmod 644 "$htarget"
# 11. Host SHA == Gold SHA
hsha="$(sha256sum "$htarget" | awk '{print $1}')"
if [ "$hsha" != "$gold" ]; then
die "recover-sot: Host-SHA-Mismatch fuer '$key' (erwartet $gold, Host $hsha) — FAIL CLOSED"
fi
echo "RECOVER-SOT OK: $key"
echo " source : $CONTAINER:$csrc"
echo " target : $htarget"
echo " gold : $gold"
echo " host : $hsha"
;;
recover-sot-list) expect_zero "$@"
echo "Recovery-Allowlist (CONFIRMED GOLD, fest gebunden):"
for k in "${!RECOVERY[@]}"; do
IFS='|' read -r csrc htarget gold <<< "${RECOVERY[$k]}"
printf ' %-16s %-45s -> %s\n' "$k" "$csrc" "$htarget"
done
;;
verify-sot-recovery) req_ok; expect_zero "$@"
echo "=== VERIFY-SOT-RECOVERY (Gold vs Host vs Container) ==="
allok=1
for k in "${!RECOVERY[@]}"; do
IFS='|' read -r csrc htarget gold <<< "${RECOVERY[$k]}"
hsha="$(sha256sum "$htarget" 2>/dev/null | awk '{print $1}')" || hsha=""
csha="$(docker exec "$CONTAINER" sha256sum "$csrc" 2>/dev/null | awk '{print $1}')" || csha=""
if [ "$hsha" = "$gold" ] && [ "$csha" = "$gold" ]; then
printf ' MATCH %-16s %s\n' "$k" "$htarget"
else
printf ' FAIL %-16s host=%s container=%s gold=%s\n' "$k" "${hsha:-MISSING}" "${csha:-MISSING}" "$gold"
allok=0
fi
done
[ "$allok" = "1" ] && echo "VERIFY-SOT-RECOVERY: ALLE MATCH" || { echo "VERIFY-SOT-RECOVERY: FEHLER"; exit 2; }
;;
restore-sot) expect_zero "$@"
# Restore des eindeutig markierten aktuellen Recovery-Session-Backups
latest="$(ls -1t "$SOT_BACKUP_ROOT" 2>/dev/null | head -1)"
[ -n "$latest" ] || die "restore-sot: kein backup-sot vorhanden"
bdir="$SOT_BACKUP_ROOT/$latest"
[ -f "$bdir/$SOT_SESSION_MARKER" ] || die "restore-sot: kein markiertes Session-Backup (FAIL CLOSED)"
[ -f "$bdir/historical-v2.tar" ] || die "restore-sot: tar fehlt (FAIL CLOSED)"
echo "RESTORE-SOT: $bdir"
echo " (entpackt /opt/historical-v2 aus Session-Backup, byte-exakt)"
if ! tar -xpf "$bdir/historical-v2.tar" -C /opt 2>/dev/null; then
die "restore-sot: tar-Restore fehlgeschlagen (FAIL CLOSED)"
fi
echo "RESTORE-SOT OK. KEIN Build/Recreate danach."
;;
*) die "Aufruf: ps|inspect|logs|run|code <K>|hashes|code-list|stage-dir|stage-check <K>|test <K>|search-writes|healthcheck|backup <K>|deploy <K>|rollback <K>|build-def|build-files|host-manifest|host-code <p>|container-manifest|container-code <p>|tree-diff|backup-sot|sot-session|recover-sot <K>|recover-sot-list|verify-sot-recovery|restore-sot"
;;
esac
Reference in a new issue View git blame Copy permalink