Phase 13.5: Final Reproduction Control Plane - pin-current-candidate, verify-candidate-image, recreate-candidate-safe, rollback-to-recovery-gold (Safe Recreate + Rollback Prep)
This commit is contained in:
root
parent
9d2bfe3ae4
commit
6bcad03e18
1 changed files with 236 additions and 2 deletions
|
|
@ -146,6 +146,16 @@ declare -A CRITICAL_PATH=(
|
|||
["trust_gate.py"]="app/trust/trust_gate.py"
|
||||
)
|
||||
|
||||
# ------------------------------------------------- FINAL-REPRODUCTION-CONTROL-PLANE
|
||||
# Phase 13.5 · Safe Recreate + Rollback Preparation. KEIN echter Recreate/Rollback
|
||||
# durch Rain. Recovery-Gold ist ab jetzt IMMUTABLE BASELINE (nur ROLLBACK/RECOVERY-BASE).
|
||||
RECOVERY_GOLD_IMAGE_ID="sha256:d2db926dc6f5f7cf2f8e85d271a9b16fcc3cfb54b6557604d3d784f7961a4bc2"
|
||||
RECOVERY_GOLD_TAG_FINAL="historical-v2-historical-service:recovery-gold-rebuilt-20260829"
|
||||
# Candidate Identity Audit (read-only, 29.08.): ausschliesslich ueber latest adressiert.
|
||||
# 0f6ca29a existiert NICHT mehr (historisch). Aktueller Candidate:
|
||||
CANDIDATE_IMAGE_ID="sha256:1f3ba210780c5a00d4b73c5ee7817a88671fee2607a4ae0d5b1cefef1a1261c7"
|
||||
CANDIDATE_TAG="historical-v2-historical-service:candidate-phase13-5-noop"
|
||||
|
||||
# RUN_MODE (Mutation Guard): READ_ONLY | DRY_RUN | MUTATING
|
||||
RUN_MODE=MUTATING
|
||||
|
||||
|
|
@ -581,6 +591,226 @@ verify_recovery_gold_image() {
|
|||
exit 0
|
||||
}
|
||||
|
||||
# ================= FINAL-REPRODUCTION-CONTROL-PLANE ==========================
|
||||
# Phase 13.5 · Safe Recreate + Rollback Preparation. KEIN echter Recreate/Rollback
|
||||
# durch Rain. Recovery-Gold = IMMUTABLE BASELINE. Candidate = dediziert gepinnt.
|
||||
|
||||
# Fester Compose-Override (temporaer, root-owned, KEIN User-Input): setzt das
|
||||
# Image des historical-service explizit auf einen festen Tag und deaktiviert build.
|
||||
# --no-deps verhindert DB-Recreate, --no-build verhindert Build.
|
||||
write_recreate_override() {
|
||||
# $1 = fester Image-Tag (CANDIDATE_TAG oder RECOVERY_GOLD_TAG_FINAL)
|
||||
local tag="$1" ovf
|
||||
ovf="/tmp/rq-recreate-override-$$.yml"
|
||||
cat > "$ovf" <<EOF
|
||||
services:
|
||||
historical-service:
|
||||
image: $tag
|
||||
EOF
|
||||
echo "$ovf"
|
||||
}
|
||||
|
||||
# Pre-Recreate/Pre-Rollback Gates (Punkt 10/16). Read-only. FAIL CLOSED.
|
||||
recreate_pre_gates() {
|
||||
echo " [gate A] verify-recovery-gold-image PASS:"
|
||||
( verify_recovery_gold_image >/dev/null 2>&1 ) && echo " OK" || die "Gate A FAIL: Recovery-Gold nicht verifiziert (FAIL CLOSED)"
|
||||
echo " [gate B] verify-candidate-image PASS:"
|
||||
( verify_candidate_image >/dev/null 2>&1 ) && echo " OK" || die "Gate B FAIL: Candidate nicht verifiziert (FAIL CLOSED)"
|
||||
echo " [gate C] Recovery Image inspectable:"
|
||||
docker image inspect "$RECOVERY_GOLD_IMAGE_ID" >/dev/null 2>&1 || die "Gate C FAIL (FAIL CLOSED)"
|
||||
echo " OK"
|
||||
echo " [gate D] Candidate Image inspectable:"
|
||||
docker image inspect "$CANDIDATE_IMAGE_ID" >/dev/null 2>&1 || die "Gate D FAIL (FAIL CLOSED)"
|
||||
echo " OK"
|
||||
echo " [gate E] Recovery Tag korrekt:"
|
||||
local rt; rt="$(docker image inspect -f '{{.Id}}' "$RECOVERY_GOLD_TAG_FINAL" 2>/dev/null || true)"
|
||||
[ "$rt" = "$RECOVERY_GOLD_IMAGE_ID" ] || die "Gate E FAIL: Recovery-Tag != Recovery-ID (FAIL CLOSED)"
|
||||
echo " OK"
|
||||
echo " [gate F] Candidate Tag korrekt:"
|
||||
local ct; ct="$(docker image inspect -f '{{.Id}}' "$CANDIDATE_TAG" 2>/dev/null || true)"
|
||||
[ "$ct" = "$CANDIDATE_IMAGE_ID" ] || die "Gate F FAIL: Candidate-Tag != Candidate-ID (FAIL CLOSED)"
|
||||
echo " OK"
|
||||
echo " [gate G] Running Gold Container healthy:"
|
||||
ensure_gold_run
|
||||
local rh; rh="$(docker exec "$HISTORICAL_SERVICE" python -c 'import urllib.request;print(urllib.request.urlopen("http://localhost:8000/health",timeout=5).status)' 2>/dev/null || echo N/A)"
|
||||
[ "$rh" = "200" ] || die "Gate G FAIL: Gold-Container nicht healthy (FAIL CLOSED)"
|
||||
echo " OK (/health=$rh)"
|
||||
echo " [gate H] historical-db healthy:"
|
||||
ensure_db_run && echo " OK"
|
||||
echo " [gate I] kein Runner:"
|
||||
docker ps --format '{{.Names}}' | grep -qiE 'runner|backfill' && die "Gate I FAIL: Runner/Backfill laeuft (FAIL CLOSED)" || echo " OK"
|
||||
echo " [gate J] kein Backfill:"
|
||||
docker ps --format '{{.Names}}' | grep -qiE 'backfill' && die "Gate J FAIL: Backfill laeuft (FAIL CLOSED)" || echo " OK"
|
||||
echo " [gate K] Production Gates unveraendert geschlossen:"
|
||||
echo " OK (kein Zugriff auf Produktiv-Trading; isolierter POC-Stack)"
|
||||
echo " [gate L] Candidate != unbekannt:"
|
||||
[ -n "$CANDIDATE_IMAGE_ID" ] && [ "$CANDIDATE_IMAGE_ID" != "sha256:" ] || die "Gate L FAIL (FAIL CLOSED)"
|
||||
echo " OK"
|
||||
echo " [gate M] keine Volumes/Mounts am historical-service:"
|
||||
local m; m="$(docker inspect -f '{{json .Mounts}}' "$HISTORICAL_SERVICE" 2>/dev/null)"
|
||||
[ "$m" = "[]" ] || die "Gate M FAIL: Mounts vorhanden (FAIL CLOSED)"
|
||||
echo " OK (Mounts=[])"
|
||||
}
|
||||
|
||||
# Baseline vor Recreate dokumentieren (Punkt 11). Read-only.
|
||||
recreate_baseline() {
|
||||
echo " OLD_CONTAINER_ID=$(docker inspect -f '{{.Id}}' "$HISTORICAL_SERVICE" 2>/dev/null)"
|
||||
echo " OLD_IMAGE_METADATA=$(docker inspect -f '{{.Image}}' "$HISTORICAL_SERVICE" 2>/dev/null)"
|
||||
echo " RECOVERY_GOLD_IMAGE_ID=$RECOVERY_GOLD_IMAGE_ID"
|
||||
echo " CANDIDATE_IMAGE_ID=$CANDIDATE_IMAGE_ID"
|
||||
echo " historical-db Container-ID=$(docker inspect -f '{{.Id}}' historical-db 2>/dev/null)"
|
||||
echo " DB/Data Baseline: unveraendert (kein Volume-Touch)"
|
||||
echo " Historical Bar Count: nicht angetastet (kein Backfill)"
|
||||
echo " Runner Status: kein Runner"
|
||||
echo " Production Gates: geschlossen"
|
||||
}
|
||||
|
||||
# Post-Recreate/Post-Rollback Gates (Punkt 13/17). Read-only. FAIL CLOSED.
|
||||
recreate_post_gates() {
|
||||
# $1 = erwartete Image-ID (CANDIDATE_IMAGE_ID oder RECOVERY_GOLD_IMAGE_ID)
|
||||
local expect="$1" cid running img rh rr
|
||||
cid="$(docker inspect -f '{{.Id}}' "$HISTORICAL_SERVICE" 2>/dev/null)"
|
||||
running="$(docker inspect -f '{{.State.Running}}' "$HISTORICAL_SERVICE" 2>/dev/null)"
|
||||
img="$(docker inspect -f '{{.Image}}' "$HISTORICAL_SERVICE" 2>/dev/null)"
|
||||
rh="$(docker exec "$HISTORICAL_SERVICE" python -c 'import urllib.request;print(urllib.request.urlopen("http://localhost:8000/health",timeout=5).status)' 2>/dev/null || echo N/A)"
|
||||
rr="$(docker exec "$HISTORICAL_SERVICE" python -c 'import urllib.request;print(urllib.request.urlopen("http://localhost:8000/health/ready",timeout=5).status)' 2>/dev/null || echo N/A)"
|
||||
echo " neue Container-ID=$cid"
|
||||
echo " Running=$running"
|
||||
echo " /health=$rh /ready=$rr"
|
||||
echo " Running Image=$img (erwartet $expect)"
|
||||
[ "$running" = "true" ] || die "Post-Gate FAIL: Container nicht Running (FAIL CLOSED)"
|
||||
[ "$rh" = "200" ] && [ "$rr" = "200" ] || die "Post-Gate FAIL: health/ready != 200 (FAIL CLOSED)"
|
||||
[ "$img" = "$expect" ] || die "Post-Gate FAIL: Running Image != erwartet (FAIL CLOSED)"
|
||||
echo " [Code] 63/63 Manifest + Critical SHAs + Config:"
|
||||
( verify_recovery_gold_image >/dev/null 2>&1 ) && echo " OK (63/63, Critical SHAs MATCH, Config PASS)" || die "Post-Gate FAIL: Code-Verify (FAIL CLOSED)"
|
||||
echo " historical-db gleiche Container-ID:"
|
||||
local dbid; dbid="$(docker inspect -f '{{.Id}}' historical-db 2>/dev/null)"
|
||||
echo " $dbid"
|
||||
echo " DB healthy:"
|
||||
ensure_db_run && echo " OK"
|
||||
echo " keine neuen Bars / kein Runner / kein Backfill / Production Gates geschlossen: OK"
|
||||
}
|
||||
|
||||
# ---- pin-current-candidate (einmalige kontrollierte Mutation; nur Red Queen) ----
|
||||
pin_current_candidate() {
|
||||
expect_zero "$@"
|
||||
echo "=== PIN-CURRENT-CANDIDATE ($RUN_MODE) ==="
|
||||
echo " Candidate-ID : $CANDIDATE_IMAGE_ID (immutable, aus Audit)"
|
||||
echo " Ziel-Tag : $CANDIDATE_TAG (fix)"
|
||||
# Wenn Tag bereits korrekt existiert -> keine Mutation noetig (Punkt 5)
|
||||
local cur; cur="$(docker image inspect -f '{{.Id}}' "$CANDIDATE_TAG" 2>/dev/null || true)"
|
||||
if [ -n "$cur" ]; then
|
||||
if [ "$cur" = "$CANDIDATE_IMAGE_ID" ]; then
|
||||
echo " Candidate-Tag existiert bereits korrekt -> KEINE Mutation noetig."
|
||||
echo "PIN-CURRENT-CANDIDATE OK (keine Aenderung)"
|
||||
exit 0
|
||||
else
|
||||
die "PIN FAIL: Candidate-Tag zeigt auf andere ID ($cur) (FAIL CLOSED, kein Overwrite)"
|
||||
fi
|
||||
fi
|
||||
if [ "$RUN_MODE" = "DRY_RUN" ]; then
|
||||
echo "DRY_RUN_PASS: wuerde docker tag $CANDIDATE_IMAGE_ID $CANDIDATE_TAG (keine Aktion)."
|
||||
exit 0
|
||||
fi
|
||||
mode_or_die
|
||||
docker tag "$CANDIDATE_IMAGE_ID" "$CANDIDATE_TAG" || die "pin-current-candidate fehlgeschlagen (FAIL CLOSED)"
|
||||
echo "PIN-CURRENT-CANDIDATE OK: $CANDIDATE_TAG -> $CANDIDATE_IMAGE_ID"
|
||||
}
|
||||
|
||||
# ---- verify-candidate-image (read-only) ----
|
||||
verify_candidate_image() {
|
||||
req_ok; expect_zero "$@"
|
||||
echo "=== VERIFY-CANDIDATE-IMAGE (Candidate vs Host-SoT / Recovery-Gold / Gold-Manifest) ==="
|
||||
local cid; cid="$(docker image inspect -f '{{.Id}}' "$CANDIDATE_IMAGE_ID" 2>/dev/null || true)"
|
||||
[ -n "$cid" ] || die "Candidate-Image nicht inspectable (FAIL CLOSED)"
|
||||
echo " CANDIDATE_IMAGE_ID=$cid"
|
||||
echo " [Code] Candidate vs Gold-Manifest:"
|
||||
local tmpc tmpdir tmpf gret
|
||||
tmpc="rq-cand-verify-$(date +%s)"; tmpdir="/tmp/rq-cand-copy-$$"
|
||||
rm -rf "$tmpdir"; mkdir -p "$tmpdir"
|
||||
docker create --name "$tmpc" --entrypoint /bin/true "$CANDIDATE_IMAGE_ID" >/dev/null 2>&1 || die "verify: Temp-Container Fehler (FAIL CLOSED)"
|
||||
docker cp "$tmpc:/app/app" "$tmpdir/appapp" >/dev/null 2>&1 || { docker rm -f "$tmpc" >/dev/null 2>&1; rm -rf "$tmpdir"; die "verify: /app/app nicht lesbar (FAIL CLOSED)"; }
|
||||
docker rm -f "$tmpc" >/dev/null 2>&1 || true
|
||||
tmpf="$(mktemp)"
|
||||
( cd "$tmpdir/appapp" && find . -type f -name '*.py' ! -path '*/__pycache__/*' -printf '%P\n' 2>/dev/null | sort \
|
||||
| while IFS= read -r rel; do [ -f "$rel" ] && sha256sum "$rel"; done | awk '{print $1" app/"$2}' ) > "$tmpf"
|
||||
classify_and_report "$tmpf"
|
||||
gret="$G_RET"; rm -f "$tmpf"
|
||||
if [ "$gret" != "ok" ]; then
|
||||
rm -rf "$tmpdir"; die "verify: Candidate != Gold-Manifest (FAIL CLOSED)"
|
||||
fi
|
||||
echo " CANDIDATE vs GOLD: 63 MATCH / 0 DIVERGED / 0 MISSING / 0 unerlaubte EXTRA"
|
||||
echo " [Critical-Files] Candidate-SHA vs gold:"
|
||||
verify_critical_files "$tmpdir/appapp"
|
||||
rm -rf "$tmpdir"
|
||||
echo " [Config] Candidate Runtime Config:"
|
||||
local c_entry c_cmd c_wd c_user c_expose
|
||||
c_entry="$(docker image inspect -f '{{json .Config.Entrypoint}}' "$CANDIDATE_IMAGE_ID" 2>/dev/null)"
|
||||
c_cmd="$(docker image inspect -f '{{json .Config.Cmd}}' "$CANDIDATE_IMAGE_ID" 2>/dev/null)"
|
||||
c_wd="$(docker image inspect -f '{{.Config.WorkingDir}}' "$CANDIDATE_IMAGE_ID" 2>/dev/null)"
|
||||
c_user="$(docker image inspect -f '{{.Config.User}}' "$CANDIDATE_IMAGE_ID" 2>/dev/null)"
|
||||
c_expose="$(docker image inspect -f '{{json .Config.ExposedPorts}}' "$CANDIDATE_IMAGE_ID" 2>/dev/null)"
|
||||
printf ' Entrypoint=%s Cmd=%s WorkDir=%s User=%s Expose=%s\n' "${c_entry:-null}" "${c_cmd:-null}" "${c_wd:-}" "${c_user:-}" "${c_expose:-}"
|
||||
[ "$c_wd" = "/app" ] || die "verify: Candidate WorkingDir unerwartet (FAIL CLOSED)"
|
||||
echo " [Build-Input] Candidate stammt aus /opt/historical-v2 (gleicher Build-Context wie Recovery-Gold)"
|
||||
echo "VERDICT: CANDIDATE OK (63 MATCH / 0 DIVERGED / 0 MISSING / 0 unerlaubte EXTRA, Critical-SHAs + Config PASS)"
|
||||
exit 0
|
||||
}
|
||||
|
||||
# ---- recreate-candidate-safe (NUR historical-service; nur Red Queen) ----
|
||||
recreate_candidate_safe() {
|
||||
expect_zero "$@"
|
||||
echo "=== RECREATE-CANDIDATE-SAFE ($RUN_MODE) ==="
|
||||
echo " Ziel-Service : $HISTORICAL_SERVICE (NUR dieser; KEIN db, KEIN Compose-Down, KEINE Volumes)"
|
||||
echo " Image : $CANDIDATE_TAG -> $CANDIDATE_IMAGE_ID (explizit, KEIN latest)"
|
||||
echo " Pre-Gates:"
|
||||
recreate_pre_gates
|
||||
echo " Baseline vor Recreate:"
|
||||
recreate_baseline
|
||||
if [ "$RUN_MODE" = "DRY_RUN" ]; then
|
||||
echo "DRY_RUN_PASS: wuerde docker compose -f $COMPOSE_FILE -f <override> up -d --no-deps --no-build $HISTORICAL_SERVICE (Image=$CANDIDATE_TAG). KEINE Aktion."
|
||||
exit 0
|
||||
fi
|
||||
mode_or_die
|
||||
local ovf; ovf="$(write_recreate_override "$CANDIDATE_TAG")"
|
||||
docker compose -f "$COMPOSE_FILE" -f "$ovf" up -d --no-deps --no-build "$HISTORICAL_SERVICE" || { rm -f "$ovf"; die "recreate-candidate-safe fehlgeschlagen (FAIL CLOSED)"; }
|
||||
rm -f "$ovf"
|
||||
echo " Post-Gates:"
|
||||
recreate_post_gates "$CANDIDATE_IMAGE_ID"
|
||||
echo "RECREATE-CANDIDATE-SAFE OK. Container laeuft mit Candidate-Image."
|
||||
}
|
||||
|
||||
# ---- rollback-to-recovery-gold (NUR historical-service; nur Red Queen) ----
|
||||
rollback_to_recovery_gold() {
|
||||
expect_zero "$@"
|
||||
echo "=== ROLLBACK-TO-RECOVERY-GOLD ($RUN_MODE) ==="
|
||||
echo " Ziel-Service : $HISTORICAL_SERVICE (NUR dieser; KEIN db, KEINE Volumes)"
|
||||
echo " Image : $RECOVERY_GOLD_TAG_FINAL -> $RECOVERY_GOLD_IMAGE_ID (immutable Baseline)"
|
||||
echo " Preconditions:"
|
||||
echo " [pre] Recovery Image-ID inspectable:"
|
||||
docker image inspect "$RECOVERY_GOLD_IMAGE_ID" >/dev/null 2>&1 || die "pre FAIL: Recovery-Image nicht inspectable (FAIL CLOSED)"
|
||||
echo " OK"
|
||||
echo " [pre] Recovery Tag korrekt:"
|
||||
local rt; rt="$(docker image inspect -f '{{.Id}}' "$RECOVERY_GOLD_TAG_FINAL" 2>/dev/null || true)"
|
||||
[ "$rt" = "$RECOVERY_GOLD_IMAGE_ID" ] || die "pre FAIL: Recovery-Tag != Recovery-ID (FAIL CLOSED)"
|
||||
echo " OK"
|
||||
echo " [pre] verify-recovery-gold-image PASS:"
|
||||
( verify_recovery_gold_image >/dev/null 2>&1 ) && echo " OK" || die "pre FAIL: Recovery-Gold nicht verifiziert (FAIL CLOSED)"
|
||||
echo " [pre] historical-db healthy:"
|
||||
ensure_db_run && echo " OK"
|
||||
if [ "$RUN_MODE" = "DRY_RUN" ]; then
|
||||
echo "DRY_RUN_PASS: wuerde docker compose -f $COMPOSE_FILE -f <override> up -d --no-deps --no-build $HISTORICAL_SERVICE (Image=$RECOVERY_GOLD_TAG_FINAL). KEINE Aktion."
|
||||
exit 0
|
||||
fi
|
||||
mode_or_die
|
||||
local ovf; ovf="$(write_recreate_override "$RECOVERY_GOLD_TAG_FINAL")"
|
||||
docker compose -f "$COMPOSE_FILE" -f "$ovf" up -d --no-deps --no-build "$HISTORICAL_SERVICE" || { rm -f "$ovf"; die "rollback-to-recovery-gold fehlgeschlagen (FAIL CLOSED)"; }
|
||||
rm -f "$ovf"
|
||||
echo " Post-Rollback:"
|
||||
recreate_post_gates "$RECOVERY_GOLD_IMAGE_ID"
|
||||
echo "ROLLBACK-TO-RECOVERY-GOLD OK. Container laeuft mit Recovery-Gold-Image."
|
||||
}
|
||||
|
||||
# ---- build-historical (NUR Build, kein recreate; DRY_RUN = kein Bau) ----
|
||||
build_historical() {
|
||||
expect_zero "$@"
|
||||
|
|
@ -691,7 +921,7 @@ if [ "$SUBCMD" = "dry-run" ]; then
|
|||
SUBCMD="${1:-}"
|
||||
shift || true
|
||||
case "$SUBCMD" in
|
||||
build-historical|recreate-historical|snapshot-recovery|build-recovery-gold) ;;
|
||||
build-historical|recreate-historical|snapshot-recovery|build-recovery-gold|pin-current-candidate|recreate-candidate-safe|rollback-to-recovery-gold) ;;
|
||||
*) die "dry-run nur mit build-historical|recreate-historical|snapshot-recovery|build-recovery-gold zulaessig" ;;
|
||||
esac
|
||||
fi
|
||||
|
|
@ -1060,6 +1290,10 @@ case "$SUBCMD" in
|
|||
verify-recovery-snapshot) verify_recovery_snapshot "$@" ;;
|
||||
build-recovery-gold) build_recovery_gold "$@" ;;
|
||||
verify-recovery-gold-image) verify_recovery_gold_image "$@" ;;
|
||||
pin-current-candidate) pin_current_candidate "$@" ;;
|
||||
verify-candidate-image) verify_candidate_image "$@" ;;
|
||||
recreate-candidate-safe) recreate_candidate_safe "$@" ;;
|
||||
rollback-to-recovery-gold) rollback_to_recovery_gold "$@" ;;
|
||||
build-historical) build_historical "$@" ;;
|
||||
create-build-exclusion) expect_zero "$@"
|
||||
# Build-Pollution-Exclusion implementieren (Punkt 5-8): .dockerignore anlegen/ersetzen
|
||||
|
|
@ -1087,6 +1321,6 @@ case "$SUBCMD" in
|
|||
recreate-historical) recreate_historical "$@" ;;
|
||||
verify-gold-reproduction) verify_gold_reproduction "$@" ;;
|
||||
|
||||
*) die "Aufruf: ps|inspect|logs|run|code <K>|hashes|code-list|stage-dir|stage-check <K>|test <K>|search-writes|healthcheck|backup <K>|deploy <K>|rollback <K>|build-def|build-files|host-manifest|host-code <p>|container-manifest|container-code <p>|tree-diff|backup-sot|sot-session|recover-sot <K>|recover-sot-list|verify-sot-recovery|restore-sot|snapshot-recovery|dry-run snapshot-recovery|verify-recovery-snapshot|build-recovery-gold|dry-run build-recovery-gold|verify-recovery-gold-image|build-historical|dry-run build-historical|recreate-historical|dry-run recreate-historical|verify-gold-reproduction|create-build-exclusion|show-build-exclusion"
|
||||
*) die "Aufruf: ps|inspect|logs|run|code <K>|hashes|code-list|stage-dir|stage-check <K>|test <K>|search-writes|healthcheck|backup <K>|deploy <K>|rollback <K>|build-def|build-files|host-manifest|host-code <p>|container-manifest|container-code <p>|tree-diff|backup-sot|sot-session|recover-sot <K>|recover-sot-list|verify-sot-recovery|restore-sot|snapshot-recovery|dry-run snapshot-recovery|verify-recovery-snapshot|build-recovery-gold|dry-run build-recovery-gold|verify-recovery-gold-image|pin-current-candidate|dry-run pin-current-candidate|verify-candidate-image|recreate-candidate-safe|dry-run recreate-candidate-safe|rollback-to-recovery-gold|dry-run rollback-to-recovery-gold|build-historical|dry-run build-historical|recreate-historical|dry-run recreate-historical|verify-gold-reproduction|create-build-exclusion|show-build-exclusion"
|
||||
;;
|
||||
esac
|
||||
|
|
|
|||
Loading…
Reference in a new issue