rq-historical: add dev/test/deploy channel (stage-check,test,deploy,backup,rollback,search-writes,healthcheck)
This commit is contained in:
root
parent
bbfd85e472
commit
6324c8aaac
1 changed files with 188 additions and 71 deletions
|
|
@ -1,42 +1,44 @@
|
|||
#!/bin/bash
|
||||
# ============================================================================
|
||||
# rq-historical — eng begrenzter root-Wrapper fuer Red Queen (rq-host)
|
||||
# RED-QUEEN-VPS-HOST-ACCESS + PHASE-13.5-READONLY | Rain Ocampo (Hermes) | 2026-08-29
|
||||
# RED-QUEEN-VPS-HOST-ACCESS + PHASE-13.5-READONLY + DEV/TEST/DEPLOY-CHANNEL
|
||||
# | Rain Ocampo (Hermes) | 2026-08-29
|
||||
#
|
||||
# EINZIGER ZWECK: kontrollierter READ-ONLY-Zugriff + freigegebener Backfill-
|
||||
# Start ausschliesslich auf den Historical-Service. Least-Privilege.
|
||||
# LEAST-PRIVILEGE-Design:
|
||||
# - rq-host hat KEINE docker-Gruppe, KEINEN Docker-Socket, KEIN sudo ALL,
|
||||
# KEINE Root-Shell. Einziger Root-Kanal: NOPASSWD:/usr/local/bin/rq-historical
|
||||
# - Dieser Wrapper ist der EINZIGE Kanal zu historical-service. Er laeuft als
|
||||
# root (via sudoers), erzwingt aber fuer Dev/Test/Deploy strikte Fixierung:
|
||||
# * feste Staging-Roots, feste Allowlist (KEY->Staging-Datei->Containerpfad)
|
||||
# * keine freien Pfade, keine ../, kein Symlink-Escape, kein Mount-Arg
|
||||
# * automatisches Backup + Healthcheck + Hash-Reverify vor/nach Deploy
|
||||
# * deploy != execution: nie automatischer Backfill/Runner/Watchdog
|
||||
# - Deploy-Methode: atomic docker cp der validierten Stage-Datei in den
|
||||
# laufenden Container (Container ist beschreibbar, ReadonlyRootfs=false).
|
||||
# Rebuild via compose-build NICHT freigegeben (wuerde rq-host Docker-Build
|
||||
# + Container-Recreation geben -> Punkt 2/10-Verletzung).
|
||||
# - Test: py_compile der Stage-Kopie in Container-/tmp (keine Mutation des
|
||||
# laufenden Produktivcodes).
|
||||
#
|
||||
# SICHERHEIT:
|
||||
# - Container-/Skript-Namen sind FEST im Script (kein Benutzer-Argument
|
||||
# fuer Container/Skript/Pfad).
|
||||
# - Alle Subkommandos akzeptieren NUR strikt validierte, vollstaendig
|
||||
# konsumierte Argumente; unbekannte/ueberzaehlige Argumente -> ERR.
|
||||
# - code/hashes/code-list nutzen eine FESTE Allowlist: User uebergibt nur
|
||||
# einen Key, der intern auf einen festen Containerpfad gemappt wird.
|
||||
# KEIN frei durchgereichter Pfad, KEIN Shell-String aus User-Input.
|
||||
# - Argumente werden als argv-Array an `docker exec` uebergeben
|
||||
# (KEINE Shell-Interpolation von Benutzereingaben).
|
||||
# - KEIN sudo ALL / NOPASSWD ALL / keine Root-Shell / kein docker run.
|
||||
#
|
||||
# AUFRUF (ueber /etc/sudoers.d/rq-historical, NUR als root):
|
||||
# sudo rq-historical ps
|
||||
# sudo rq-historical inspect
|
||||
# sudo rq-historical logs [--tail N]
|
||||
# sudo rq-historical run [--start YYYY-MM-DD] [--end YYYY-MM-DD] [--limit-days N]
|
||||
# sudo rq-historical code <runner|backfill|pipeline|repository|provenance|trust|sanity|trust_gate>
|
||||
# sudo rq-historical hashes
|
||||
# sudo rq-historical code-list
|
||||
# SUBKOMMANDOS:
|
||||
# ps | inspect | logs [--tail N] | run [--start --end --limit-days] (bestehend)
|
||||
# code <KEY> | hashes | code-list (bestehend, read-only)
|
||||
# stage-dir (zeigt Staging-Pfade)
|
||||
# stage-check <KEY> (validiert Staging-Kandidat)
|
||||
# test <KEY> (py_compile in /tmp, isoliert)
|
||||
# search-writes (fixe Write-Schreibpfad-Suche)
|
||||
# healthcheck (/health + /health/ready 200)
|
||||
# backup <KEY> (aktuellen Container-Code sichern)
|
||||
# deploy <KEY> (Backup->stage-check->test->cp->reverify->health)
|
||||
# rollback <KEY> (zurueck zum letzten Backup)
|
||||
# ============================================================================
|
||||
set -euo pipefail
|
||||
|
||||
CONTAINER=historical-service
|
||||
|
||||
# ------------------------------------------------------------------ Allowlist
|
||||
# FESTE Mapping-Tabelle (Key -> Containerpfad). Read-only, hart im Script.
|
||||
# Keys NICHT benutzerdefiniert erweiterbar.
|
||||
# NOTE: Bei assign-assoc-array die Assignments OHNE Leerzeichen um '=' schreiben,
|
||||
# sonst Syntaxfehler. (Korrektur unten: einzeilig mit declare -A init.)
|
||||
declare -A CODE_FILES=(
|
||||
# --------------------------------------------------------- Allowlist (fest)
|
||||
# KEY -> Container-Zielpfad (identisch mit 'code'-Read-Allowlist)
|
||||
declare -A TARGET_PATHS=(
|
||||
[runner]="/app/app/run_backfill_year.py"
|
||||
[backfill]="/app/app/backfill.py"
|
||||
[pipeline]="/app/app/pipeline_v2.py"
|
||||
|
|
@ -47,46 +49,73 @@ declare -A CODE_FILES=(
|
|||
[trust_gate]="/app/app/trust/trust_gate.py"
|
||||
)
|
||||
|
||||
# KEY -> Staging-Dateiname (fest, NUR Dateiname im Staging-Root)
|
||||
declare -A STAGE_FILES=(
|
||||
[runner]="run_backfill_year.py"
|
||||
[backfill]="backfill.py"
|
||||
[pipeline]="pipeline_v2.py"
|
||||
[repository]="repository.py"
|
||||
[provenance]="provenance.py"
|
||||
[trust]="trust.py"
|
||||
[sanity]="sanity.py"
|
||||
[trust_gate]="trust_gate.py"
|
||||
)
|
||||
|
||||
STAGING_ROOT=/opt/rq-historical-staging
|
||||
BACKUP_ROOT=/opt/rq-historical-backups
|
||||
|
||||
# Relevanter Pfad fuer den 'run'-Subbefehl (Backfill-Entrypoint)
|
||||
SCRIPT=/app/app/run_backfill_year.py
|
||||
|
||||
# ---------------------------------------------------------------- Validierung
|
||||
# ------------------------------------------------------------ Validierung
|
||||
die() { echo "ERR: $*" >&2; exit 2; }
|
||||
req_ok() {
|
||||
docker inspect -f '{{.State.Running}}' "$CONTAINER" >/dev/null 2>&1 \
|
||||
|| die "Container '$CONTAINER' nicht vorhanden oder nicht am Laufen"
|
||||
}
|
||||
key_path() { echo "${TARGET_PATHS[$1]:-}"; } # $1=key -> Containerpfad
|
||||
stage_file() { echo "$STAGING_ROOT/${STAGE_FILES[$1]:-}"; } # $1=key -> abs Staging-Pfad
|
||||
key_ok() { # $1=key; exit 2 wenn unbekannt
|
||||
[ -n "${TARGET_PATHS[$1]:-}" ] && [ -n "${STAGE_FILES[$1]:-}" ] || die "unbekannter KEY '$1' (code-list)"
|
||||
}
|
||||
expect_zero() { [ $# -eq 0 ] || die "unerlaubtes Argument '$*'"; }
|
||||
|
||||
# -------------------------------------------------------------------- dispatch
|
||||
# stage-check: FAIL-CLOSED-Validierung eines Staging-Kandidaten
|
||||
validate_stage() { # $1=key
|
||||
local key="$1" file real owner dmode
|
||||
file="$(stage_file "$key")"
|
||||
[ -e "$file" ] || die "Staging-Kandidat fehlt: $file (lege zuerst die Datei ab)"
|
||||
real="$(realpath -m "$file")"
|
||||
[[ "$real" == "$STAGING_ROOT/"* ]] || die "Traversal blockiert: realpath '$real' ausserhalb Staging"
|
||||
[ "$real" = "$file" ] || die "Alias erkannt (realpath != Pfad): $file"
|
||||
[ -f "$file" ] || die "keine reguläre Datei: $file"
|
||||
[ -L "$file" ] && die "Symlink nicht erlaubt: $file"
|
||||
owner="$(stat -c '%U' "$file")"
|
||||
[ "$owner" = "rq-host" ] || die "Staging-Kandidat Owner = $owner (erwartet rq-host)"
|
||||
dmode="$(stat -c '%a' "$STAGING_ROOT")"
|
||||
[[ "$dmode" != *002 ]] && [[ "$dmode" != *022 ]] || die "Staging-Root global schreibbar '$dmode' (FAIL CLOSED)"
|
||||
echo "OK: $key -> $file (regular, no symlink, owner rq-host, in staging)"
|
||||
}
|
||||
|
||||
# ---------------------------------------------------- dispatch
|
||||
SUBCMD="${1:-}"
|
||||
shift || true
|
||||
|
||||
case "$SUBCMD" in
|
||||
ps)
|
||||
[ $# -eq 0 ] || die "unerlaubtes Argument 'ps $*'"
|
||||
exec docker ps
|
||||
;;
|
||||
inspect)
|
||||
req_ok
|
||||
[ $# -eq 0 ] || die "unerlaubtes Argument 'inspect $*'"
|
||||
exec docker inspect "$CONTAINER"
|
||||
;;
|
||||
logs)
|
||||
req_ok
|
||||
# ---------------- bestehend (read-only / run) ----------------
|
||||
ps) expect_zero "$@"; exec docker ps ;;
|
||||
inspect) req_ok; expect_zero "$@"; exec docker inspect "$CONTAINER" ;;
|
||||
logs) req_ok
|
||||
tail_n=200
|
||||
while [ $# -gt 0 ]; do
|
||||
case "$1" in
|
||||
--tail)
|
||||
[ $# -ge 2 ] || die "--tail braucht Wert"
|
||||
[[ "$2" =~ ^[0-9]+$ ]] || die "ungueltiges --tail"
|
||||
tail_n="$2"; shift 2 ;;
|
||||
--tail) [ $# -ge 2 ] || die "--tail braucht Wert"; [[ "$2" =~ ^[0-9]+$ ]] || die "ungueltiges --tail"; tail_n="$2"; shift 2 ;;
|
||||
*) die "unerlaubtes Argument 'logs $1'" ;;
|
||||
esac
|
||||
done
|
||||
exec docker logs --tail "$tail_n" "$CONTAINER"
|
||||
;;
|
||||
run)
|
||||
req_ok
|
||||
run) req_ok
|
||||
declare -a RUN=()
|
||||
while [ $# -gt 0 ]; do
|
||||
case "$1" in
|
||||
|
|
@ -99,34 +128,122 @@ case "$SUBCMD" in
|
|||
echo "[rq-historical] Start: docker exec $CONTAINER python $SCRIPT ${RUN[*]}" >&2
|
||||
exec docker exec "$CONTAINER" python "$SCRIPT" "${RUN[@]}"
|
||||
;;
|
||||
code)
|
||||
# Fester Allowlist-Key -> fester Containerpfad. KEIN frei gereichter Pfad.
|
||||
req_ok
|
||||
[ $# -eq 1 ] || die "code braucht genau EINEN Key (siehe code-list)"
|
||||
key="$1"
|
||||
path="${CODE_FILES[$key]:-}"
|
||||
[ -n "$path" ] || die "unbekannter Key 'code $key' (siehe code-list)"
|
||||
exec docker exec "$CONTAINER" cat "$path"
|
||||
# ------------- read-only Code -------------
|
||||
code) req_ok
|
||||
[ $# -eq 1 ] || die "code braucht genau EINEN Key (code-list)"
|
||||
p="$(key_path "$1")"; [ -n "$p" ] || die "unbekannter Key 'code $1'"
|
||||
exec docker exec "$CONTAINER" cat "$p"
|
||||
;;
|
||||
hashes)
|
||||
# SHA256 der festen Allowlist-Dateien aus dem laufenden Container.
|
||||
req_ok
|
||||
[ $# -eq 0 ] || die "unerlaubtes Argument 'hashes $*'"
|
||||
# Feste, vollstaendig konsumierte Dateiliste -> argv-Array (keine Interpolation).
|
||||
declare -a PATHS=()
|
||||
for k in "${!CODE_FILES[@]}"; do PATHS+=("${CODE_FILES[$k]}"); done
|
||||
hashes) req_ok; expect_zero "$@"
|
||||
PATHS=(); for k in "${!TARGET_PATHS[@]}"; do PATHS+=("${TARGET_PATHS[$k]}"); done
|
||||
exec docker exec "$CONTAINER" sha256sum "${PATHS[@]}"
|
||||
;;
|
||||
code-list)
|
||||
# Nur die verfuegbaren fixen Keys + Containerpfade anzeigen. Kein FS-Zugriff.
|
||||
req_ok
|
||||
[ $# -eq 0 ] || die "unerlaubtes Argument 'code-list $*'"
|
||||
code-list) req_ok; expect_zero "$@"
|
||||
echo "Verfuegbare READ-ONLY-Code-Keys (historical-service):"
|
||||
for k in "${!CODE_FILES[@]}"; do
|
||||
printf ' %-11s -> %s\n' "$k" "${CODE_FILES[$k]}"
|
||||
for k in "${!TARGET_PATHS[@]}"; do printf ' %-11s | staging: %-22s -> %s\n' "$k" "${STAGE_FILES[$k]}" "${TARGET_PATHS[$k]}"; done
|
||||
;;
|
||||
# ------------- dev/test/deploy-kanal -------------
|
||||
stage-dir) expect_zero "$@"
|
||||
echo "Staging-Root: $STAGING_ROOT (Owner rq-host, keine globalen Schreibrechte)"
|
||||
for k in runner backfill pipeline repository provenance trust sanity trust_gate; do
|
||||
printf ' %-11s %s\n' "$k" "$(stage_file "$k")"
|
||||
done
|
||||
;;
|
||||
*)
|
||||
die "Aufruf: rq-historical {ps|inspect|logs [--tail N]|run [--start --end --limit-days]|code <KEY>|hashes|code-list}"
|
||||
stage-check) [ $# -eq 1 ] || die "stage-check braucht genau EINEN Key"; key_ok "$1"
|
||||
validate_stage "$1"
|
||||
;;
|
||||
test) req_ok; [ $# -eq 1 ] || die "test braucht genau EINEN Key"; key_ok "$1"
|
||||
validate_stage "$1"
|
||||
file="$(stage_file "$1")"
|
||||
tmp="/tmp/rqtest_${1}_$(date +%s).py"
|
||||
docker cp "$file" "$CONTAINER:$tmp"
|
||||
if docker exec "$CONTAINER" python -m py_compile "$tmp" >/dev/null 2>&1; then
|
||||
echo "PASS: py_compile OK ($file) [isoliert in Container-/tmp, Produktivcode unberuehrt]"
|
||||
else
|
||||
echo "FAIL: py_compile fehlgeschlagen ($file)"
|
||||
docker exec "$CONTAINER" rm -f "$tmp"
|
||||
exit 2
|
||||
fi
|
||||
docker exec "$CONTAINER" rm -f "$tmp"
|
||||
;;
|
||||
search-writes) req_ok; expect_zero "$@"
|
||||
docker exec "$CONTAINER" sh -c 'grep -rnE "insert_raw_bars|insert_derived_bars|BackfillOrchestrator|trust_gate|TrustGate" /app/app --include="*.py" 2>/dev/null | head -100 || true'
|
||||
;;
|
||||
healthcheck) req_ok; expect_zero "$@"
|
||||
rh="$(docker exec "$CONTAINER" python -c 'import urllib.request; print(urllib.request.urlopen("http://localhost:8000/health", timeout=5).status)' 2>/dev/null || echo N/A)"
|
||||
rr="$(docker exec "$CONTAINER" python -c 'import urllib.request; print(urllib.request.urlopen("http://localhost:8000/health/ready", timeout=5).status)' 2>/dev/null || echo N/A)"
|
||||
running="$(docker inspect -f '{{.State.Running}}' "$CONTAINER")"
|
||||
echo "Container.Running=$running /health=$rh /health/ready=$rr"
|
||||
[ "$running" = "true" ] && [ "$rh" = "200" ] && [ "$rr" = "200" ] || { echo "HEALTHCHECK FAILED"; exit 2; }
|
||||
echo "HEALTHCHECK OK"
|
||||
;;
|
||||
backup) req_ok; [ $# -eq 1 ] || die "backup braucht genau EINEN Key"; key_ok "$1"
|
||||
p="$(key_path "$1")"
|
||||
ts="$(date +%Y%m%d_%H%M%S)"
|
||||
bdir="$BACKUP_ROOT/$1/$ts"
|
||||
mkdir -p "$bdir"
|
||||
docker exec "$CONTAINER" cat "$p" > "$bdir/$(basename "$p")"
|
||||
sha256sum "$bdir/$(basename "$p")" > "$bdir/SHA256"
|
||||
echo "BACKUP: $1 -> $bdir"
|
||||
cat "$bdir/SHA256"
|
||||
;;
|
||||
deploy) req_ok; [ $# -eq 1 ] || die "deploy braucht genau EINEN Key"; key_ok "$1"
|
||||
key="$1"; p="$(key_path "$key")"; file="$(stage_file "$key")"
|
||||
# 1) validate (FAIL CLOSED)
|
||||
validate_stage "$key"
|
||||
# 2) backup erzwingen (muss gelingen)
|
||||
ts="$(date +%Y%m%d_%H%M%S)"; bdir="$BACKUP_ROOT/$key/$ts"
|
||||
mkdir -p "$bdir"
|
||||
docker exec "$CONTAINER" cat "$p" > "$bdir/$(basename "$p")" || die "Backup fehlgeschlagen -> deploy abgebrochen"
|
||||
sha256sum "$bdir/$(basename "$p")" > "$bdir/SHA256"
|
||||
echo "BACKUP: $bdir"
|
||||
# 3) test (py_compile isoliert)
|
||||
tmp="/tmp/rqdeploy_${key}_$(date +%s).py"
|
||||
docker cp "$file" "$CONTAINER:$tmp"
|
||||
if ! docker exec "$CONTAINER" python -m py_compile "$tmp" >/dev/null 2>&1; then
|
||||
docker exec "$CONTAINER" rm -f "$tmp"
|
||||
die "py_compile FAIL -> deploy abgebrochen (nichts deployt, kein Rollback noetig)"
|
||||
fi
|
||||
docker exec "$CONTAINER" rm -f "$tmp"
|
||||
# 4) atomic deploy
|
||||
docker cp "$file" "$CONTAINER:$p"
|
||||
# 5) hash-reverify
|
||||
sha_stage="$(sha256sum "$file" | awk '{print $1}')"
|
||||
sha_deployed="$(docker exec "$CONTAINER" sha256sum "$p" | awk '{print $1}')"
|
||||
echo "SHA stage : $sha_stage"
|
||||
echo "SHA deployed: $sha_deployed"
|
||||
if [ "$sha_stage" != "$sha_deployed" ]; then
|
||||
echo "HASH-MISMATCH -> rollback"
|
||||
docker cp "$bdir/$(basename "$p")" "$CONTAINER:$p"
|
||||
die "Deploy Hash-Mismatch, zurueckgerollt"
|
||||
fi
|
||||
# 6) healthcheck (nur health/ready, KEIN run)
|
||||
rh="$(docker exec "$CONTAINER" python -c 'import urllib.request; print(urllib.request.urlopen("http://localhost:8000/health", timeout=5).status)' 2>/dev/null || echo N/A)"
|
||||
rr="$(docker exec "$CONTAINER" python -c 'import urllib.request; print(urllib.request.urlopen("http://localhost:8000/health/ready", timeout=5).status)' 2>/dev/null || echo N/A)"
|
||||
echo "Health: /health=$rh /health/ready=$rr"
|
||||
if [ "$rh" != "200" ] || [ "$rr" != "200" ]; then
|
||||
echo "DEPLOY-HEALTHCHECK FAILED -> rollback"
|
||||
docker cp "$bdir/$(basename "$p")" "$CONTAINER:$p"
|
||||
die "Rollback ausgefuehrt"
|
||||
fi
|
||||
echo "DEPLOY OK: $key -> $p (Backup: $bdir). KEIN backfill/run gestartet."
|
||||
;;
|
||||
rollback) req_ok; [ $# -eq 1 ] || die "rollback braucht genau EINEN Key"; key_ok "$1"
|
||||
p="$(key_path "$1")"
|
||||
bdir="$(ls -1t "$BACKUP_ROOT/$1" 2>/dev/null | head -1)"
|
||||
[ -n "$bdir" ] || die "kein Backup fuer '$1' vorhanden"
|
||||
bdir="$BACKUP_ROOT/$1/$bdir"
|
||||
src="$bdir/$(basename "$p")"
|
||||
[ -f "$src" ] || die "Backup-Datei fehlt: $src"
|
||||
sha_bak="$(sha256sum "$src" | awk '{print $1}')"
|
||||
sha_cur="$(docker exec "$CONTAINER" sha256sum "$p" | awk '{print $1}')"
|
||||
echo "aktuell: $sha_cur / backup: $sha_bak"
|
||||
[ "$sha_bak" != "$sha_cur" ] || die "kein Unterschied (bereits auf Backup-Stand)"
|
||||
docker cp "$src" "$CONTAINER:$p"
|
||||
sha_cur="$(docker exec "$CONTAINER" sha256sum "$p" | awk '{print $1}')"
|
||||
[ "$sha_cur" = "$sha_bak" ] || die "ROLLBACK-HASH-MISMATCH"
|
||||
echo "ROLLBACK OK: $key -> $p auf $bdir"
|
||||
;;
|
||||
*) die "Aufruf: ps|inspect|logs|run|code <KEY>|hashes|code-list|stage-dir|stage-check <KEY>|test <KEY>|search-writes|healthcheck|backup <KEY>|deploy <KEY>|rollback <KEY>"
|
||||
;;
|
||||
esac
|
||||
|
|
|
|||
Loading…
Reference in a new issue