nexo312/trading-system-docs
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

phase13.5: add bounded recreate startup readiness gate

Browse source
This commit is contained in:
root 2026-08-31 17:29:06 +00:00
parent b61acd3f08
commit 35bbdd89e7
1 changed files with 50 additions and 3 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

53
red-queen-architecture/control-plane/cp2a2_2b/rq-historical
View file

@ -1219,20 +1219,67 @@ recreate_baseline() {
echo " Production Gates: geschlossen"
}
# Startup-/Readiness-Wartephase fuer historical-service (Phase 13.5 Timing-Fix, rebased auf Gate-O-Baseline).
# Behebt das Timing-False-Negative: frisch gestartete Container duerfen NICHT nach einem einzigen
# sofortigen health/ready-Check als fehlgeschlagen klassifiziert werden. Stattdessen wird streng
# begrenzt mit festem Timeout+Poll gewartet. Feste (keine freien User-) Parameter. PASS NUR wenn
# health=200 UND ready=200. Container-exit/dead/unexpected/restart-loop/wrong-image -> SOFORT FAIL CLOSED.
# Timeout (health/ready nie beide 200) -> FAIL CLOSED. Kein sleep-only PASS.
wait_for_historical_service_ready() {
# $1 = erwartete Image-ID (muss vom laufenden Container exakt getragen werden)
local expect="$1"
local POLL_INTERVAL="2" # Sekunden (fest; Startup-Periode aus Control Plane)
local MAX_WAIT="45" # Sekunden (fest; konservativ, fail-closed)
local elapsed=0 rh rr img running restarts prev_restarts
# Vorbedingung: Container muss existieren UND laufen.
running="$(docker inspect -f '{{.State.Running}}' "$HISTORICAL_SERVICE" 2>/dev/null || echo unknown)"
[ "$running" = "true" ] || die "wait: Container '$HISTORICAL_SERVICE' nicht Running zu Beginn (FAIL CLOSED)"
prev_restarts="$(docker inspect -f '{{.RestartCount}}' "$HISTORICAL_SERVICE" 2>/dev/null || echo 0)"
while [ "$elapsed" -lt "$MAX_WAIT" ]; do
# --- Container-Fehlerzustand bei JEDEM Zyklus pruefen (frueh FAIL CLOSED) ---
running="$(docker inspect -f '{{.State.Running}}' "$HISTORICAL_SERVICE" 2>/dev/null || echo unknown)"
[ "$running" = "true" ] || die "wait[${elapsed}s]: Container nicht mehr Running (exited/dead) -> FAIL CLOSED"
restarts="$(docker inspect -f '{{.RestartCount}}' "$HISTORICAL_SERVICE" 2>/dev/null || echo "$prev_restarts")"
[ -z "$restarts" ] || [ "$restarts" = "$prev_restarts" ] || die "wait[${elapsed}s]: Restart-Loop (RestartCount $prev_restarts->$restarts) -> FAIL CLOSED"
# --- Target-Image-ID Hard Gate bei jedem Zyklus ---
img="$(docker inspect -f '{{.Image}}' "$HISTORICAL_SERVICE" 2>/dev/null || echo unknown)"
[ -n "$img" ] && [ "$img" != "unknown" ] || die "wait[${elapsed}s]: Running Image-ID nicht lesbar -> FAIL CLOSED"
[ "$img" = "$expect" ] || die "wait[${elapsed}s]: Running Image '$img' != erwartet '$expect' -> FAIL CLOSED"
# --- Health und Ready getrennt pruefen; 5xx/Fehler transient tolerieren solange Container running ---
rh="$(docker exec "$HISTORICAL_SERVICE" python -c 'import urllib.request;print(urllib.request.urlopen("http://localhost:8000/health",timeout=2).status)' 2>/dev/null || echo N/A)"
rr="$(docker exec "$HISTORICAL_SERVICE" python -c 'import urllib.request;print(urllib.request.urlopen("http://localhost:8000/health/ready",timeout=2).status)' 2>/dev/null || echo N/A)"
echo " wait[${elapsed}s]: /health=$rh /ready=$rr (Running Image=$img)"
# --- PASS nur wenn BEIDE 200 ---
if [ "$rh" = "200" ] && [ "$rr" = "200" ]; then
echo " wait: READY nach ${elapsed}s: HEALTH=200 UND READY=200 -> PASS"
return 0
fi
sleep "$POLL_INTERVAL"
elapsed=$((elapsed + POLL_INTERVAL))
done
# --- Timeout -> FAIL CLOSED (kein PASS) ---
echo " wait: TIMEOUT nach ${MAX_WAIT}s ohne health=200 UND ready=200 (zuletzt /health=$rh /ready=$rr) -> FAIL CLOSED" >&2
die "wait: Startup-/Readiness-Timeout (health/ready nicht beide 200) -> FAIL CLOSED"
}
# Post-Recreate/Post-Rollback Gates (Punkt 13/17). Read-only. FAIL CLOSED.
recreate_post_gates() {
# $1 = erwartete Image-ID (CANDIDATE_IMAGE_ID oder RECOVERY_GOLD_IMAGE_ID)
local expect="$1" cid running img rh rr
echo " Startup-/Readiness-Wartephase (Timing-Fix):"
# Wartet fail-closed mit festem Timeout/Poll auf health=200 UND ready=200.
# Container-exit/dead/restart-loop/wrong-image -> frueh FAIL CLOSED (Helper).
wait_for_historical_service_ready "$expect"
cid="$(docker inspect -f '{{.Id}}' "$HISTORICAL_SERVICE" 2>/dev/null)"
running="$(docker inspect -f '{{.State.Running}}' "$HISTORICAL_SERVICE" 2>/dev/null)"
img="$(docker inspect -f '{{.Image}}' "$HISTORICAL_SERVICE" 2>/dev/null)"
rh="$(docker exec "$HISTORICAL_SERVICE" python -c 'import urllib.request;print(urllib.request.urlopen("http://localhost:8000/health",timeout=5).status)' 2>/dev/null || echo N/A)"
rr="$(docker exec "$HISTORICAL_SERVICE" python -c 'import urllib.request;print(urllib.request.urlopen("http://localhost:8000/health/ready",timeout=5).status)' 2>/dev/null || echo N/A)"
echo " neue Container-ID=$cid"
echo " Running=$running"
echo " /health=$rh /ready=$rr"
echo " Running Image=$img (erwartet $expect)"
[ "$running" = "true" ] || die "Post-Gate FAIL: Container nicht Running (FAIL CLOSED)"
rh="$(docker exec "$HISTORICAL_SERVICE" python -c 'import urllib.request;print(urllib.request.urlopen("http://localhost:8000/health",timeout=5).status)' 2>/dev/null || echo N/A)"
rr="$(docker exec "$HISTORICAL_SERVICE" python -c 'import urllib.request;print(urllib.request.urlopen("http://localhost:8000/health/ready",timeout=5).status)' 2>/dev/null || echo N/A)"
echo " Final Re-Check: /health=$rh /ready=$rr"
[ "$rh" = "200" ] && [ "$rr" = "200" ] || die "Post-Gate FAIL: health/ready != 200 (FAIL CLOSED)"
[ "$img" = "$expect" ] || die "Post-Gate FAIL: Running Image != erwartet (FAIL CLOSED)"
echo " [Code] 63/63 Manifest + Critical SHAs + Config:"