From 35bbdd89e72056659b6968367d428705e09b187b Mon Sep 17 00:00:00 2001 From: root Date: Mon, 31 Aug 2026 17:29:06 +0000 Subject: [PATCH] phase13.5: add bounded recreate startup readiness gate --- .../control-plane/cp2a2_2b/rq-historical | 53 +++++++++++++++++-- 1 file changed, 50 insertions(+), 3 deletions(-) diff --git a/red-queen-architecture/control-plane/cp2a2_2b/rq-historical b/red-queen-architecture/control-plane/cp2a2_2b/rq-historical index e46a44e..a3f2069 100755 --- a/red-queen-architecture/control-plane/cp2a2_2b/rq-historical +++ b/red-queen-architecture/control-plane/cp2a2_2b/rq-historical @@ -1219,20 +1219,67 @@ recreate_baseline() { echo " Production Gates: geschlossen" } +# Startup-/Readiness-Wartephase fuer historical-service (Phase 13.5 Timing-Fix, rebased auf Gate-O-Baseline). +# Behebt das Timing-False-Negative: frisch gestartete Container duerfen NICHT nach einem einzigen +# sofortigen health/ready-Check als fehlgeschlagen klassifiziert werden. Stattdessen wird streng +# begrenzt mit festem Timeout+Poll gewartet. Feste (keine freien User-) Parameter. PASS NUR wenn +# health=200 UND ready=200. Container-exit/dead/unexpected/restart-loop/wrong-image -> SOFORT FAIL CLOSED. +# Timeout (health/ready nie beide 200) -> FAIL CLOSED. Kein sleep-only PASS. +wait_for_historical_service_ready() { + # $1 = erwartete Image-ID (muss vom laufenden Container exakt getragen werden) + local expect="$1" + local POLL_INTERVAL="2" # Sekunden (fest; Startup-Periode aus Control Plane) + local MAX_WAIT="45" # Sekunden (fest; konservativ, fail-closed) + local elapsed=0 rh rr img running restarts prev_restarts + # Vorbedingung: Container muss existieren UND laufen. + running="$(docker inspect -f '{{.State.Running}}' "$HISTORICAL_SERVICE" 2>/dev/null || echo unknown)" + [ "$running" = "true" ] || die "wait: Container '$HISTORICAL_SERVICE' nicht Running zu Beginn (FAIL CLOSED)" + prev_restarts="$(docker inspect -f '{{.RestartCount}}' "$HISTORICAL_SERVICE" 2>/dev/null || echo 0)" + while [ "$elapsed" -lt "$MAX_WAIT" ]; do + # --- Container-Fehlerzustand bei JEDEM Zyklus pruefen (frueh FAIL CLOSED) --- + running="$(docker inspect -f '{{.State.Running}}' "$HISTORICAL_SERVICE" 2>/dev/null || echo unknown)" + [ "$running" = "true" ] || die "wait[${elapsed}s]: Container nicht mehr Running (exited/dead) -> FAIL CLOSED" + restarts="$(docker inspect -f '{{.RestartCount}}' "$HISTORICAL_SERVICE" 2>/dev/null || echo "$prev_restarts")" + [ -z "$restarts" ] || [ "$restarts" = "$prev_restarts" ] || die "wait[${elapsed}s]: Restart-Loop (RestartCount $prev_restarts->$restarts) -> FAIL CLOSED" + # --- Target-Image-ID Hard Gate bei jedem Zyklus --- + img="$(docker inspect -f '{{.Image}}' "$HISTORICAL_SERVICE" 2>/dev/null || echo unknown)" + [ -n "$img" ] && [ "$img" != "unknown" ] || die "wait[${elapsed}s]: Running Image-ID nicht lesbar -> FAIL CLOSED" + [ "$img" = "$expect" ] || die "wait[${elapsed}s]: Running Image '$img' != erwartet '$expect' -> FAIL CLOSED" + # --- Health und Ready getrennt pruefen; 5xx/Fehler transient tolerieren solange Container running --- + rh="$(docker exec "$HISTORICAL_SERVICE" python -c 'import urllib.request;print(urllib.request.urlopen("http://localhost:8000/health",timeout=2).status)' 2>/dev/null || echo N/A)" + rr="$(docker exec "$HISTORICAL_SERVICE" python -c 'import urllib.request;print(urllib.request.urlopen("http://localhost:8000/health/ready",timeout=2).status)' 2>/dev/null || echo N/A)" + echo " wait[${elapsed}s]: /health=$rh /ready=$rr (Running Image=$img)" + # --- PASS nur wenn BEIDE 200 --- + if [ "$rh" = "200" ] && [ "$rr" = "200" ]; then + echo " wait: READY nach ${elapsed}s: HEALTH=200 UND READY=200 -> PASS" + return 0 + fi + sleep "$POLL_INTERVAL" + elapsed=$((elapsed + POLL_INTERVAL)) + done + # --- Timeout -> FAIL CLOSED (kein PASS) --- + echo " wait: TIMEOUT nach ${MAX_WAIT}s ohne health=200 UND ready=200 (zuletzt /health=$rh /ready=$rr) -> FAIL CLOSED" >&2 + die "wait: Startup-/Readiness-Timeout (health/ready nicht beide 200) -> FAIL CLOSED" +} + # Post-Recreate/Post-Rollback Gates (Punkt 13/17). Read-only. FAIL CLOSED. recreate_post_gates() { # $1 = erwartete Image-ID (CANDIDATE_IMAGE_ID oder RECOVERY_GOLD_IMAGE_ID) local expect="$1" cid running img rh rr + echo " Startup-/Readiness-Wartephase (Timing-Fix):" + # Wartet fail-closed mit festem Timeout/Poll auf health=200 UND ready=200. + # Container-exit/dead/restart-loop/wrong-image -> frueh FAIL CLOSED (Helper). + wait_for_historical_service_ready "$expect" cid="$(docker inspect -f '{{.Id}}' "$HISTORICAL_SERVICE" 2>/dev/null)" running="$(docker inspect -f '{{.State.Running}}' "$HISTORICAL_SERVICE" 2>/dev/null)" img="$(docker inspect -f '{{.Image}}' "$HISTORICAL_SERVICE" 2>/dev/null)" - rh="$(docker exec "$HISTORICAL_SERVICE" python -c 'import urllib.request;print(urllib.request.urlopen("http://localhost:8000/health",timeout=5).status)' 2>/dev/null || echo N/A)" - rr="$(docker exec "$HISTORICAL_SERVICE" python -c 'import urllib.request;print(urllib.request.urlopen("http://localhost:8000/health/ready",timeout=5).status)' 2>/dev/null || echo N/A)" echo " neue Container-ID=$cid" echo " Running=$running" - echo " /health=$rh /ready=$rr" echo " Running Image=$img (erwartet $expect)" [ "$running" = "true" ] || die "Post-Gate FAIL: Container nicht Running (FAIL CLOSED)" + rh="$(docker exec "$HISTORICAL_SERVICE" python -c 'import urllib.request;print(urllib.request.urlopen("http://localhost:8000/health",timeout=5).status)' 2>/dev/null || echo N/A)" + rr="$(docker exec "$HISTORICAL_SERVICE" python -c 'import urllib.request;print(urllib.request.urlopen("http://localhost:8000/health/ready",timeout=5).status)' 2>/dev/null || echo N/A)" + echo " Final Re-Check: /health=$rh /ready=$rr" [ "$rh" = "200" ] && [ "$rr" = "200" ] || die "Post-Gate FAIL: health/ready != 200 (FAIL CLOSED)" [ "$img" = "$expect" ] || die "Post-Gate FAIL: Running Image != erwartet (FAIL CLOSED)" echo " [Code] 63/63 Manifest + Critical SHAs + Config:"