trading-system-docs/red-queen-architecture/control-plane/cp2a2_2b/rq-historical

324 lines
17 KiB
Bash

#!/bin/bash
# ============================================================================
# rq-historical — eng begrenzter root-Wrapper fuer Red Queen (rq-host)
# RED-QUEEN-VPS-HOST-ACCESS + READONLY-CODE + DEV/TEST/DEPLOY + SOT-DISCOVERY
# | Rain Ocampo (Hermes) | 2026-08-29
#
# LEAST-PRIVILEGE-Design:
# - rq-host hat KEINE docker-Gruppe, KEINEN Docker-Socket, KEIN sudo ALL,
# KEINE Root-Shell. Einziger Root-Kanal: NOPASSWD:/usr/local/bin/rq-historical
# - Dieser Wrapper ist der EINZIGE Kanal zu historical-service & Host-SoT.
# Keine freien Pfade, kein arbitrary cat/find/grep, kein Shell.
#
# SoT-DISCOVERY (read-only, NO copy/rebuild/deploy/fix):
# build-def -> zeigt Dockerfile + Compose-Builddefinition (redacted)
# build-files -> Dockerfile + compose + .dockerignore Auflistung
# host-manifest -> SHA256+size aller *.py unter /opt/historical-v2/app
# host-code <rel> -> gezielter Code-Read NUR aus host-manifest (kontrolliert)
# container-manifest -> SHA256+size aller *.py unter historical-service:/app/app
# container-code <rel> -> gezielter Code-Read NUR aus container-manifest
# tree-diff -> Host vs Container Manifest-Diff
#
# SUBKOMMANDOS (bestehend, unveraendert):
# ps | inspect | logs [--tail N] | run [..] | code <KEY> | hashes | code-list
# stage-dir | stage-check <KEY> | test <KEY> | search-writes | healthcheck
# backup <KEY> | deploy <KEY> | rollback <KEY>
# ============================================================================
set -euo pipefail
CONTAINER=historical-service
HOST_APP=/opt/historical-v2/app
HOST_DIR=/opt/historical-v2
# --------------------------------------------------------- Allowlist (fest)
declare -A TARGET_PATHS=(
[runner]="/app/app/run_backfill_year.py"
[backfill]="/app/app/backfill.py"
[pipeline]="/app/app/pipeline_v2.py"
[repository]="/app/app/persistence/repository.py"
[provenance]="/app/app/trust/provenance.py"
[trust]="/app/app/trust/trust.py"
[sanity]="/app/app/trust/sanity.py"
[trust_gate]="/app/app/trust/trust_gate.py"
)
declare -A STAGE_FILES=(
[runner]="run_backfill_year.py" [backfill]="backfill.py" [pipeline]="pipeline_v2.py"
[repository]="repository.py" [provenance]="provenance.py" [trust]="trust.py"
[sanity]="sanity.py" [trust_gate]="trust_gate.py"
)
STAGING_ROOT=/opt/rq-historical-staging
BACKUP_ROOT=/opt/rq-historical-backups
SCRIPT=/app/app/run_backfill_year.py
die() { echo "ERR: $*" >&2; exit 2; }
req_ok() {
docker inspect -f '{{.State.Running}}' "$CONTAINER" >/dev/null 2>&1 \
|| die "Container '$CONTAINER' nicht vorhanden oder nicht am Laufen"
}
key_path() { echo "${TARGET_PATHS[$1]:-}"; }
stage_file() { echo "$STAGING_ROOT/${STAGE_FILES[$1]:-}"; }
key_ok() {
[ -n "${TARGET_PATHS[$1]:-}" ] && [ -n "${STAGE_FILES[$1]:-}" ] || die "unbekannter KEY '$1' (code-list)"
}
expect_zero() { [ $# -eq 0 ] || die "unerlaubtes Argument '$*'"; }
validate_stage() {
local key="$1" file real owner dmode
file="$(stage_file "$key")"
[ -e "$file" ] || die "Staging-Kandidat fehlt: $file (lege zuerst die Datei ab)"
real="$(realpath -m "$file")"
[[ "$real" == "$STAGING_ROOT/"* ]] || die "Traversal blockiert: realpath '$real' ausserhalb Staging"
[ "$real" = "$file" ] || die "Alias erkannt (realpath != Pfad): $file"
[ -f "$file" ] || die "keine reguläre Datei: $file"
[ -L "$file" ] && die "Symlink nicht erlaubt: $file"
owner="$(stat -c '%U' "$file")"
[ "$owner" = "rq-host" ] || die "Staging-Kandidat Owner = $owner (erwartet rq-host)"
dmode="$(stat -c '%a' "$STAGING_ROOT")"
[[ "$dmode" != *002 ]] && [[ "$dmode" != *022 ]] || die "Staging-Root global schreibbar '$dmode' (FAIL CLOSED)"
echo "OK: $key -> $file (regular, no symlink, owner rq-host, in staging)"
}
# ---- redact: maskiert Passwort-/Secret-Werte (fuer build-def Anzeige) ----
redact_line() {
# Zeilen mit "PASSWORD=" / "SECRET=" / "TOKEN=" : Wert maskieren
echo "$1" | sed -E 's/(PASSWORD|PASSWORDS|SECRET|SECRETS|TOKEN|TOKENS|API_KEY|CLIENT_SECRET|PRIVATE_KEY)(["'"'"']?[[:space:]]*=[[:space:]]*["'"'"']?)[^[:space:]"'"'"']+/\1\2***REDACTED***/Ig'
}
SUBCMD="${1:-}"
shift || true
case "$SUBCMD" in
# ---------------- bestehend ----------------
ps) expect_zero "$@"; exec docker ps ;;
inspect) req_ok; expect_zero "$@"; exec docker inspect "$CONTAINER" ;;
logs) req_ok
tail_n=200
while [ $# -gt 0 ]; do
case "$1" in
--tail) [ $# -ge 2 ] || die "--tail braucht Wert"; [[ "$2" =~ ^[0-9]+$ ]] || die "ungueltiges --tail"; tail_n="$2"; shift 2 ;;
*) die "unerlaubtes Argument 'logs $1'" ;;
esac
done
exec docker logs --tail "$tail_n" "$CONTAINER"
;;
run) req_ok
declare -a RUN=()
while [ $# -gt 0 ]; do
case "$1" in
--start)[[ "${2:-}" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}$ ]]||die "ungueltiges --start";RUN+=("--start" "$2");shift 2;;
--end)[[ "${2:-}" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}$ ]]||die "ungueltiges --end";RUN+=("--end" "$2");shift 2;;
--limit-days)[[ "${2:-}" =~ ^[0-9]+$ ]]||die "ungueltiges --limit-days";RUN+=("--limit-days" "$2");shift 2;;
*) die "unerlaubtes Argument 'run $1'" ;;
esac
done
echo "[rq-historical] Start: docker exec $CONTAINER python $SCRIPT ${RUN[*]}" >&2
exec docker exec "$CONTAINER" python "$SCRIPT" "${RUN[@]}"
;;
code) req_ok
[ $# -eq 1 ] || die "code braucht genau EINEN Key (code-list)"
p="$(key_path "$1")"; [ -n "$p" ] || die "unbekannter Key 'code $1'"
exec docker exec "$CONTAINER" cat "$p"
;;
hashes) req_ok; expect_zero "$@"
PATHS=(); for k in "${!TARGET_PATHS[@]}"; do PATHS+=("${TARGET_PATHS[$k]}"); done
exec docker exec "$CONTAINER" sha256sum "${PATHS[@]}"
;;
code-list) req_ok; expect_zero "$@"
echo "Verfuegbare READ-ONLY-Code-Keys (historical-service):"
for k in "${!TARGET_PATHS[@]}"; do printf ' %-11s | staging: %-22s -> %s\n' "$k" "${STAGE_FILES[$k]}" "${TARGET_PATHS[$k]}"; done
;;
stage-dir) expect_zero "$@"
echo "Staging-Root: $STAGING_ROOT (Owner rq-host, keine globalen Schreibrechte)"
for k in runner backfill pipeline repository provenance trust sanity trust_gate; do
printf ' %-11s %s\n' "$k" "$(stage_file "$k")"
done
;;
stage-check) [ $# -eq 1 ] || die "stage-check braucht genau EINEN Key"; key_ok "$1"; validate_stage "$1"
;;
test) req_ok; [ $# -eq 1 ] || die "test braucht genau EINEN Key"; key_ok "$1"
validate_stage "$1"
file="$(stage_file "$1")"
tmp="/tmp/rqtest_${1}_$(date +%s).py"
docker cp "$file" "$CONTAINER:$tmp"
if docker exec "$CONTAINER" python -m py_compile "$tmp" >/dev/null 2>&1; then
echo "PASS: py_compile OK ($file) [isoliert in Container-/tmp, Produktivcode unberuehrt]"
else
echo "FAIL: py_compile fehlgeschlagen ($file)"
docker exec "$CONTAINER" rm -f "$tmp"; exit 2
fi
docker exec "$CONTAINER" rm -f "$tmp"
;;
search-writes) req_ok; expect_zero "$@"
docker exec "$CONTAINER" sh -c 'grep -rnE "insert_raw_bars|insert_derived_bars|BackfillOrchestrator|trust_gate|TrustGate" /app/app --include="*.py" 2>/dev/null | head -100 || true'
;;
healthcheck) req_ok; expect_zero "$@"
rh="$(docker exec "$CONTAINER" python -c 'import urllib.request; print(urllib.request.urlopen("http://localhost:8000/health", timeout=5).status)' 2>/dev/null || echo N/A)"
rr="$(docker exec "$CONTAINER" python -c 'import urllib.request; print(urllib.request.urlopen("http://localhost:8000/health/ready", timeout=5).status)' 2>/dev/null || echo N/A)"
running="$(docker inspect -f '{{.State.Running}}' "$CONTAINER")"
echo "Container.Running=$running /health=$rh /health/ready=$rr"
[ "$running" = "true" ] && [ "$rh" = "200" ] && [ "$rr" = "200" ] || { echo "HEALTHCHECK FAILED"; exit 2; }
echo "HEALTHCHECK OK"
;;
backup) req_ok; [ $# -eq 1 ] || die "backup braucht genau EINEN Key"; key_ok "$1"
p="$(key_path "$1")"; ts="$(date +%Y%m%d_%H%M%S)"; bdir="$BACKUP_ROOT/$1/$ts"
mkdir -p "$bdir"
docker exec "$CONTAINER" cat "$p" > "$bdir/$(basename "$p")"
sha256sum "$bdir/$(basename "$p")" > "$bdir/SHA256"
echo "BACKUP: $1 -> $bdir"; cat "$bdir/SHA256"
;;
deploy) req_ok; [ $# -eq 1 ] || die "deploy braucht genau EINEN Key"; key_ok "$1"
key="$1"; p="$(key_path "$key")"; file="$(stage_file "$key")"
validate_stage "$key"
ts="$(date +%Y%m%d_%H%M%S)"; bdir="$BACKUP_ROOT/$key/$ts"
mkdir -p "$bdir"
docker exec "$CONTAINER" cat "$p" > "$bdir/$(basename "$p")" || die "Backup fehlgeschlagen -> deploy abgebrochen"
sha256sum "$bdir/$(basename "$p")" > "$bdir/SHA256"
echo "BACKUP: $bdir"
tmp="/tmp/rqdeploy_${key}_$(date +%s).py"
docker cp "$file" "$CONTAINER:$tmp"
if ! docker exec "$CONTAINER" python -m py_compile "$tmp" >/dev/null 2>&1; then
docker exec "$CONTAINER" rm -f "$tmp"; die "py_compile FAIL -> deploy abgebrochen (nichts deployt)"
fi
docker exec "$CONTAINER" rm -f "$tmp"
docker cp "$file" "$CONTAINER:$p"
sha_stage="$(sha256sum "$file" | awk '{print $1}')"
sha_deployed="$(docker exec "$CONTAINER" sha256sum "$p" | awk '{print $1}')"
echo "SHA stage : $sha_stage"; echo "SHA deployed: $sha_deployed"
if [ "$sha_stage" != "$sha_deployed" ]; then
echo "HASH-MISMATCH -> rollback"; docker cp "$bdir/$(basename "$p")" "$CONTAINER:$p"; die "Deploy Hash-Mismatch, zurueckgerollt"
fi
rh="$(docker exec "$CONTAINER" python -c 'import urllib.request; print(urllib.request.urlopen("http://localhost:8000/health", timeout=5).status)' 2>/dev/null || echo N/A)"
rr="$(docker exec "$CONTAINER" python -c 'import urllib.request; print(urllib.request.urlopen("http://localhost:8000/health/ready", timeout=5).status)' 2>/dev/null || echo N/A)"
echo "Health: /health=$rh /health/ready=$rr"
if [ "$rh" != "200" ] || [ "$rr" != "200" ]; then
echo "DEPLOY-HEALTHCHECK FAILED -> rollback"; docker cp "$bdir/$(basename "$p")" "$CONTAINER:$p"; die "Rollback ausgefuehrt"
fi
echo "DEPLOY OK: $key -> $p (Backup: $bdir). KEIN backfill/run gestartet."
;;
rollback) req_ok; [ $# -eq 1 ] || die "rollback braucht genau EINEN Key"; key_ok "$1"
p="$(key_path "$1")"
bdir="$(ls -1t "$BACKUP_ROOT/$1" 2>/dev/null | head -1)"
[ -n "$bdir" ] || die "kein Backup fuer '$1' vorhanden"
bdir="$BACKUP_ROOT/$1/$bdir"; src="$bdir/$(basename "$p")"
[ -f "$src" ] || die "Backup-Datei fehlt: $src"
sha_bak="$(sha256sum "$src" | awk '{print $1}')"
sha_cur="$(docker exec "$CONTAINER" sha256sum "$p" | awk '{print $1}')"
echo "aktuell: $sha_cur / backup: $sha_bak"
[ "$sha_bak" != "$sha_cur" ] || die "kein Unterschied (bereits auf Backup-Stand)"
docker cp "$src" "$CONTAINER:$p"
sha_cur="$(docker exec "$CONTAINER" sha256sum "$p" | awk '{print $1}')"
[ "$sha_cur" = "$sha_bak" ] || die "ROLLBACK-HASH-MISMATCH"
echo "ROLLBACK OK: $key -> $p auf $bdir"
;;
# ============ SOT-DISCOVERY (read-only, NEU) ============
build-def) expect_zero "$@"
echo "=== Dockerfile ($HOST_DIR/Dockerfile) — redacted ==="
[ -f "$HOST_DIR/Dockerfile" ] || die "Dockerfile nicht gefunden"
while IFS= read -r line; do redact_line "$line"; done < "$HOST_DIR/Dockerfile"
echo ""
echo "=== docker-compose.yml — strukturielle Abschnitte, redacted ==="
cf=""
if [ -f "$HOST_DIR/docker-compose.yml" ]; then cf="$HOST_DIR/docker-compose.yml";
elif [ -f "$HOST_DIR/compose.yml" ]; then cf="$HOST_DIR/compose.yml";
elif [ -f "$HOST_DIR/docker-compose.yaml" ]; then cf="$HOST_DIR/docker-compose.yaml"; fi
[ -n "$cf" ] || die "keine compose-Datei gefunden"
echo "(Quelle: $cf)"
while IFS= read -r line; do
# nur relevante strukturelle Schluessel anzeigen; Environment/Secrets nicht roh
case "$line" in
*POSTGRES_PASSWORD*|*PG_PASSWORD*|*PASSWORD*|*SECRET*|*TOKEN*) echo " **** REDACTED (Secret-Zeile) ****" ;;
*) echo " $line" ;;
esac
done < "$cf"
echo ""
echo "=== .dockerignore (falls vorhanden) ==="
[ -f "$HOST_DIR/.dockerignore" ] && cat "$HOST_DIR/.dockerignore" || echo "(kein .dockerignore)"
echo ""
echo "=== Service/Container-Fixierung ==="
echo " compose.project=$(docker inspect "$CONTAINER" --format '{{index .Config.Labels "com.docker.compose.project"}}' 2>/dev/null || echo N/A)"
echo " compose.service=$(docker inspect "$CONTAINER" --format '{{index .Config.Labels "com.docker.compose.service"}}' 2>/dev/null || echo N/A)"
echo " container=$CONTAINER"
;;
build-files) expect_zero "$@"
echo "=== Build-Dateien unter $HOST_DIR (nicht-verzeichnis, nicht .env/.dump/.sql) ==="
for f in Dockerfile docker-compose.yml compose.yml docker-compose.yaml .dockerignore requirements.txt; do
if [ -e "$HOST_DIR/$f" ]; then printf ' %-22s %s\n' "$f" "EXISTS ($(stat -c '%s' "$HOST_DIR/$f") bytes)"; else printf ' %-22s %s\n' "$f" "(fehlt)"; fi
done
;;
host-manifest) expect_zero "$@"
echo "=== HOST Manifest /opt/historical-v2/app (*.py, keine Symlinks folgen) ==="
[ -d "$HOST_APP" ] || die "Host-App-Verzeichnis fehlt: $HOST_APP"
# finde reguläre *.py UND Symlinks auf *.py separat
(
find "$HOST_APP" -type f -name "*.py" -printf '%P %s\n' 2>/dev/null | sort \
| while read -r rel size; do
sha=$(sha256sum "$HOST_APP/$rel" 2>/dev/null | awk '{print $1}')
printf 'FILE\t%s\t%s\t%s\n' "$rel" "$size" "${sha:-ERR}"
done
find "$HOST_APP" -type l -name "*.py" -printf '%P\n' 2>/dev/null | sort \
| while read -r rel; do printf 'SYMLINK\t%s\t0\t<not followed>\n' "$rel"; done
)
;;
host-code) [ $# -eq 1 ] || die "host-code braucht genau EINEN relativen Pfad (unter app/)"
rel="$1"
[[ "$rel" == *".."* ]] && die "Traversal nicht erlaubt"
[[ "$rel" == /* ]] && die "relativer Pfad erwartet, nicht absolut"
# realpath zwingend im app-Root
real="$(realpath -m "$HOST_APP/$rel")"
[[ "$real" == "$HOST_APP/"* ]] || die "Pfad ausserhalb app-Root blockiert"
[ -f "$real" ] || die "Datei nicht gefunden oder kein File: $rel"
[ -L "$real" ] && die "Symlink-Read nicht erlaubt"
# nur .py ausgeben
[[ "$rel" == *.py ]] || die "Nur .py-Quelldateien lesbar"
echo "--- $rel ($(stat -c '%s' "$real") bytes) ---"
exec cat "$real"
;;
container-manifest) req_ok; expect_zero "$@"
echo "=== CONTAINER Manifest historical-service:/app/app (*.py) ==="
docker exec "$CONTAINER" sh -c 'cd /app/app && find . -name "*.py" -type f 2>/dev/null | sed "s#^\./##" | sort | while read f; do size=$(stat -c %s "$f"); sha=$(sha256sum "$f" | cut -d" " -f1); printf "FILE\t%s\t%s\t%s\n" "$f" "$size" "$sha"; done'
docker exec "$CONTAINER" sh -c 'cd /app/app && find . -name "*.py" -type l 2>/dev/null | sed "s#^\./##" | sort | while read f; do printf "SYMLINK\t%s\t0\t<not followed>\n" "$f"; done' || true
;;
container-code) req_ok; [ $# -eq 1 ] || die "container-code braucht genau EINEN relativen Pfad (unter /app/app/)"
rel="$1"
[[ "$rel" == *".."* ]] && die "Traversal nicht erlaubt"
[[ "$rel" == /* ]] && die "relativer Pfad erwartet, nicht absolut"
target="/app/app/$rel"
# sicherstellen Ziel existiert als reguläre Datei & nicht symlink im Container
if ! docker exec "$CONTAINER" sh -c "[ -f '$target' ]" 2>/dev/null; then die "Datei nicht gefunden oder kein File im Container: $rel"; fi
if docker exec "$CONTAINER" sh -c "[ -L '$target' ]" 2>/dev/null; then die "Symlink-Read im Container nicht erlaubt"; fi
[[ "$rel" == *.py ]] || die "Nur .py-Quelldateien lesbar"
echo "--- $target ---"
exec docker exec "$CONTAINER" cat "$target"
;;
tree-diff) req_ok; expect_zero "$@"
echo "=== TREE-DIFF Host(/opt/historical-v2/app) <-> Container(/app/app) ==="
hmap="$(mktemp)"; cmap="$(mktemp)"
# Host: 'rel<TAB>sha'
find "$HOST_APP" -type f -name "*.py" -printf '%P\n' 2>/dev/null | sort | while IFS= read -r rel; do
sha="$(sha256sum "$HOST_APP/$rel" 2>/dev/null | awk '{print $1}')"
printf '%s\t%s\n' "$rel" "$sha"
done > "$hmap"
# Container: 'rel<TAB>sha'
docker exec "$CONTAINER" sh -c 'cd /app/app && find . -type f -name "*.py" -printf "%P\n" 2>/dev/null | sort | while IFS= read -r rel; do sha256sum "$rel" 2>/dev/null | awk -v r="$rel" '\''{print r"\t"$1}'\''; done' > "$cmap"
tmpunion="$(mktemp)"
{ cut -f1 "$hmap"; cut -f1 "$cmap"; } | sort -u > "$tmpunion"
echo "HOST_APP=$HOST_APP CONTAINER_APP=/app/app"
while IFS= read -r rel; do
hs="$(awk -F'\t' -v r="$rel" '$1==r{print $2}' "$hmap")"
cs="$(awk -F'\t' -v r="$rel" '$1==r{print $2}' "$cmap")"
if [ -n "$hs" ] && [ -n "$cs" ]; then
if [ "$hs" = "$cs" ]; then status="MATCH"; else status="DIVERGED"; fi
printf '%-18s %s [h:%s c:%s]\n' "$status" "$rel" "${hs:0:12}" "${cs:0:12}"
elif [ -n "$hs" ]; then
printf '%-18s %s\n' "HOST_ONLY" "$rel"
else
printf '%-18s %s\n' "MISSING_ON_HOST" "$rel"
fi
done < "$tmpunion"
rm -f "$hmap" "$cmap" "$tmpunion"
;;
*) die "Aufruf: ps|inspect|logs|run|code <K>|hashes|code-list|stage-dir|stage-check <K>|test <K>|search-writes|healthcheck|backup <K>|deploy <K>|rollback <K>|build-def|build-files|host-manifest|host-code <p>|container-manifest|container-code <p>|tree-diff"
;;
esac