phase13.5: add bounded recreate startup readiness gate

red-queen-architecture/control-plane/cp2a2_2b/rq-historical

@@ -1219,20 +1219,67 @@ recreate_baseline() {
   echo "  Production Gates: geschlossen"
 }
 
+# Startup-/Readiness-Wartephase fuer historical-service (Phase 13.5 Timing-Fix, rebased auf Gate-O-Baseline).
+# Behebt das Timing-False-Negative: frisch gestartete Container duerfen NICHT nach einem einzigen
+# sofortigen health/ready-Check als fehlgeschlagen klassifiziert werden. Stattdessen wird streng
+# begrenzt mit festem Timeout+Poll gewartet. Feste (keine freien User-) Parameter. PASS NUR wenn
+# health=200 UND ready=200. Container-exit/dead/unexpected/restart-loop/wrong-image -> SOFORT FAIL CLOSED.
+# Timeout (health/ready nie beide 200) -> FAIL CLOSED. Kein sleep-only PASS.
+wait_for_historical_service_ready() {
+  # $1 = erwartete Image-ID (muss vom laufenden Container exakt getragen werden)
+  local expect="$1"
+  local POLL_INTERVAL="2"   # Sekunden (fest; Startup-Periode aus Control Plane)
+  local MAX_WAIT="45"        # Sekunden (fest; konservativ, fail-closed)
+  local elapsed=0 rh rr img running restarts prev_restarts
+  # Vorbedingung: Container muss existieren UND laufen.
+  running="$(docker inspect -f '{{.State.Running}}' "$HISTORICAL_SERVICE" 2>/dev/null || echo unknown)"
+  [ "$running" = "true" ] || die "wait: Container '$HISTORICAL_SERVICE' nicht Running zu Beginn (FAIL CLOSED)"
+  prev_restarts="$(docker inspect -f '{{.RestartCount}}' "$HISTORICAL_SERVICE" 2>/dev/null || echo 0)"
+  while [ "$elapsed" -lt "$MAX_WAIT" ]; do
+    # --- Container-Fehlerzustand bei JEDEM Zyklus pruefen (frueh FAIL CLOSED) ---
+    running="$(docker inspect -f '{{.State.Running}}' "$HISTORICAL_SERVICE" 2>/dev/null || echo unknown)"
+    [ "$running" = "true" ] || die "wait[${elapsed}s]: Container nicht mehr Running (exited/dead) -> FAIL CLOSED"
+    restarts="$(docker inspect -f '{{.RestartCount}}' "$HISTORICAL_SERVICE" 2>/dev/null || echo "$prev_restarts")"
+    [ -z "$restarts" ] || [ "$restarts" = "$prev_restarts" ] || die "wait[${elapsed}s]: Restart-Loop (RestartCount $prev_restarts->$restarts) -> FAIL CLOSED"
+    # --- Target-Image-ID Hard Gate bei jedem Zyklus ---
+    img="$(docker inspect -f '{{.Image}}' "$HISTORICAL_SERVICE" 2>/dev/null || echo unknown)"
+    [ -n "$img" ] && [ "$img" != "unknown" ] || die "wait[${elapsed}s]: Running Image-ID nicht lesbar -> FAIL CLOSED"
+    [ "$img" = "$expect" ] || die "wait[${elapsed}s]: Running Image '$img' != erwartet '$expect' -> FAIL CLOSED"
+    # --- Health und Ready getrennt pruefen; 5xx/Fehler transient tolerieren solange Container running ---
+    rh="$(docker exec "$HISTORICAL_SERVICE" python -c 'import urllib.request;print(urllib.request.urlopen("http://localhost:8000/health",timeout=2).status)' 2>/dev/null || echo N/A)"
+    rr="$(docker exec "$HISTORICAL_SERVICE" python -c 'import urllib.request;print(urllib.request.urlopen("http://localhost:8000/health/ready",timeout=2).status)' 2>/dev/null || echo N/A)"
+    echo "  wait[${elapsed}s]: /health=$rh /ready=$rr (Running Image=$img)"
+    # --- PASS nur wenn BEIDE 200 ---
+    if [ "$rh" = "200" ] && [ "$rr" = "200" ]; then
+      echo "  wait: READY nach ${elapsed}s: HEALTH=200 UND READY=200 -> PASS"
+      return 0
+    fi
+    sleep "$POLL_INTERVAL"
+    elapsed=$((elapsed + POLL_INTERVAL))
+  done
+  # --- Timeout -> FAIL CLOSED (kein PASS) ---
+  echo "  wait: TIMEOUT nach ${MAX_WAIT}s ohne health=200 UND ready=200 (zuletzt /health=$rh /ready=$rr) -> FAIL CLOSED" >&2
+  die "wait: Startup-/Readiness-Timeout (health/ready nicht beide 200) -> FAIL CLOSED"
+}
+
 # Post-Recreate/Post-Rollback Gates (Punkt 13/17). Read-only. FAIL CLOSED.
 recreate_post_gates() {
   # $1 = erwartete Image-ID (CANDIDATE_IMAGE_ID oder RECOVERY_GOLD_IMAGE_ID)
   local expect="$1" cid running img rh rr
+  echo "  Startup-/Readiness-Wartephase (Timing-Fix):"
+  # Wartet fail-closed mit festem Timeout/Poll auf health=200 UND ready=200.
+  # Container-exit/dead/restart-loop/wrong-image -> frueh FAIL CLOSED (Helper).
+  wait_for_historical_service_ready "$expect"
   cid="$(docker inspect -f '{{.Id}}' "$HISTORICAL_SERVICE" 2>/dev/null)"
   running="$(docker inspect -f '{{.State.Running}}' "$HISTORICAL_SERVICE" 2>/dev/null)"
   img="$(docker inspect -f '{{.Image}}' "$HISTORICAL_SERVICE" 2>/dev/null)"
-  rh="$(docker exec "$HISTORICAL_SERVICE" python -c 'import urllib.request;print(urllib.request.urlopen("http://localhost:8000/health",timeout=5).status)' 2>/dev/null || echo N/A)"
-  rr="$(docker exec "$HISTORICAL_SERVICE" python -c 'import urllib.request;print(urllib.request.urlopen("http://localhost:8000/health/ready",timeout=5).status)' 2>/dev/null || echo N/A)"
   echo "  neue Container-ID=$cid"
   echo "  Running=$running"
-  echo "  /health=$rh /ready=$rr"
   echo "  Running Image=$img (erwartet $expect)"
   [ "$running" = "true" ] || die "Post-Gate FAIL: Container nicht Running (FAIL CLOSED)"
+  rh="$(docker exec "$HISTORICAL_SERVICE" python -c 'import urllib.request;print(urllib.request.urlopen("http://localhost:8000/health",timeout=5).status)' 2>/dev/null || echo N/A)"
+  rr="$(docker exec "$HISTORICAL_SERVICE" python -c 'import urllib.request;print(urllib.request.urlopen("http://localhost:8000/health/ready",timeout=5).status)' 2>/dev/null || echo N/A)"
+  echo "  Final Re-Check: /health=$rh /ready=$rr"
   [ "$rh" = "200" ] && [ "$rr" = "200" ] || die "Post-Gate FAIL: health/ready != 200 (FAIL CLOSED)"
   [ "$img" = "$expect" ] || die "Post-Gate FAIL: Running Image != erwartet (FAIL CLOSED)"
   echo "  [Code] 63/63 Manifest + Critical SHAs + Config:"