Red Queen
968094498a
- a5/rq_heartbeat.py: HeartbeatStore + Heartbeat.run_tick (thin scheduler/resume layer) - a5/rq_heartbeat_cli.py: status/tick/enable/disable/resume/approve/deny/priority - a5/test_a5.py: 31 deterministic tests (tick-lock incl. ownership, eligibility, kill-switch, approval, circuit, git-conflict, bounded single A4) - a5/DESIGN.md, a5/README.md, a5/scripts/a5_heartbeat_tick.sh - a2/rq_mission.py: add read-only mission_list() for A5 enumeration (additive) - Fresh checker: PASS after tick-lock ownership repair (Regressionschutz lock_owned)
426 lines
17 KiB
Python
426 lines
17 KiB
Python
#!/usr/bin/env python3
|
|
"""Red Queen — A5: deterministische Tests (CONTROLLED HEARTBEAT & RESUME v1).
|
|
|
|
Lauf:
|
|
python3 test_a5.py
|
|
Exit-Code 0 = alle Tests gruen; 1 = mindestens ein Fehler.
|
|
Nutzt ausschliesslich temporaere DBs. Kein Netz, kein echter A4-Dispatch (Stub)
|
|
ausser einem echten bounded Orchestrator-Test (ohne dispatcher).
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import os
|
|
import sys
|
|
import tempfile
|
|
from pathlib import Path
|
|
|
|
_HERE = Path(__file__).resolve().parent
|
|
_REPO = _HERE.parent
|
|
for _p in (str(_REPO), str(_REPO / "a2"), str(_REPO / "a3"), str(_REPO / "a4"), str(_HERE)):
|
|
if _p not in sys.path:
|
|
sys.path.insert(0, _p)
|
|
|
|
from a2.rq_mission import MissionStore
|
|
from a3.rq_safety import SafetyStore
|
|
from a4.rq_orchestrator import ( # noqa: E402
|
|
ST_NEEDS_DISPATCH,
|
|
ST_APPROVAL_REQUIRED,
|
|
ST_NO_RUNNABLE_WORK,
|
|
Orchestrator,
|
|
)
|
|
from a5.rq_heartbeat import ( # noqa: E402
|
|
HB_APPROVAL_PENDING,
|
|
HB_APPROVAL_REQUIRED,
|
|
HB_BOUNDED_RUN_COMPLETE,
|
|
HB_CANCELLED,
|
|
HB_CIRCUIT_OPEN,
|
|
HB_GIT_CONFLICT,
|
|
HB_KILL_SWITCH_OFF,
|
|
HB_LOCKED,
|
|
HB_MISSION_COMPLETED,
|
|
HB_NO_ACTIVE_MISSION,
|
|
HB_NO_RUNNABLE_WORK,
|
|
HB_PAUSED,
|
|
HB_SAFETY_BLOCKED,
|
|
HB_TERMINAL,
|
|
Heartbeat,
|
|
)
|
|
|
|
PASS = 0
|
|
FAIL = 0
|
|
FAILURES = []
|
|
|
|
|
|
def check(name: str, cond: bool, extra: str = ""):
|
|
global PASS, FAIL
|
|
if cond:
|
|
PASS += 1
|
|
print(f" [PASS] {name}")
|
|
else:
|
|
FAIL += 1
|
|
FAILURES.append(name)
|
|
print(f" [FAIL] {name} {extra}")
|
|
|
|
|
|
def _mk_env():
|
|
d = tempfile.mkdtemp(prefix="a5_test_")
|
|
return {
|
|
"dir": d,
|
|
"registry": os.path.join(d, "heartbeat.db"),
|
|
"mission": os.path.join(d, "missions.db"),
|
|
"safety": os.path.join(d, "safety.db"),
|
|
}
|
|
|
|
|
|
class StubOrchestrator:
|
|
"""Deterministischer A4-Stub: liefert genau den injizierten Status."""
|
|
|
|
def __init__(self, status=ST_NEEDS_DISPATCH, **extra):
|
|
self._status = status
|
|
self._extra = extra
|
|
self.run_bounded_calls = 0
|
|
|
|
def run_bounded(self, mission_id, *, max_steps=None, actor="red-queen"):
|
|
self.run_bounded_calls += 1
|
|
return {"status": self._status, "mission_id": mission_id, **self._extra}
|
|
|
|
def mission_completion_gate(self, mission_id, *, final_review_pass=False):
|
|
return {"mission_id": mission_id, "valid": False, "reasons": ["no wps done"]}
|
|
|
|
def attempt_mission_complete(self, mission_id, *, final_review_pass=False, actor="red-queen"):
|
|
return {"status": "MISSION_NOT_COMPLETED", "mission_id": mission_id, "valid": False}
|
|
|
|
|
|
def _mission_running(ms: MissionStore, mid: str, wp_ids=None):
|
|
ms.mission_create(title=f"M {mid}", goal="g", scope="s", mission_id=mid)
|
|
for c, t in (("CREATED", "PLANNING"), ("PLANNING", "READY"), ("READY", "RUNNING")):
|
|
ms.mission_transition(mid, t, evidence=f"to {t}")
|
|
if wp_ids:
|
|
for w in wp_ids:
|
|
ms.wp_create(mid, f"WP {w}", wp_id=f"{mid}-{w}")
|
|
ms.wp_transition(f"{mid}-{w}", "READY")
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# 1. Kill Switch
|
|
# --------------------------------------------------------------------------- #
|
|
def test_kill_switch_off_blocks_mutation():
|
|
env = _mk_env()
|
|
ms = MissionStore(env["mission"])
|
|
_mission_running(ms, "M-KS")
|
|
stub = StubOrchestrator(ST_NEEDS_DISPATCH)
|
|
hb = Heartbeat(env["registry"], env["mission"], env["safety"], orchestrator=stub)
|
|
check("kill switch default OFF", hb.registry.kill_switch_on() is False)
|
|
res = hb.run_tick(mission_id="M-KS")
|
|
check("kill switch OFF -> HB_KILL_SWITCH_OFF", res["code"] == HB_KILL_SWITCH_OFF, res["code"])
|
|
check("kill switch OFF -> keine A4-Mutation", stub.run_bounded_calls == 0, str(stub.run_bounded_calls))
|
|
|
|
|
|
def test_kill_switch_on_allows_bounded():
|
|
env = _mk_env()
|
|
ms = MissionStore(env["mission"])
|
|
_mission_running(ms, "M-KSON")
|
|
stub = StubOrchestrator(ST_NEEDS_DISPATCH)
|
|
hb = Heartbeat(env["registry"], env["mission"], env["safety"], orchestrator=stub)
|
|
hb.registry.set_kill_switch(True)
|
|
res = hb.run_tick(mission_id="M-KSON")
|
|
check("kill switch ON -> bounded run", res["code"] == HB_BOUNDED_RUN_COMPLETE, res["code"])
|
|
check("exactly one A4 call", stub.run_bounded_calls == 1, str(stub.run_bounded_calls))
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# 2. Tick Lock
|
|
# --------------------------------------------------------------------------- #
|
|
def test_tick_lock_prevents_overlap():
|
|
env = _mk_env()
|
|
ms = MissionStore(env["mission"])
|
|
_mission_running(ms, "M-TL")
|
|
hb = Heartbeat(env["registry"], env["mission"], env["safety"],
|
|
orchestrator=StubOrchestrator(ST_NEEDS_DISPATCH))
|
|
hb.registry.set_kill_switch(True)
|
|
assert hb.registry.acquire_lock("RUN-A") == "ok"
|
|
res = hb.run_tick(mission_id="M-TL")
|
|
check("overlapping tick -> HB_LOCKED", res["code"] == HB_LOCKED, res["code"])
|
|
|
|
|
|
def test_tick_lock_preserved_on_overlap():
|
|
"""Regressionsschutz: Ein HB_LOCKED-Tick darf das FREMD-aktive Lock NICHT
|
|
freigeben. Defekt, den der Fresh Checker fand: blindes release_lock im
|
|
finally loeschte das Lock von Tick A, sodass Tick C sofort wieder mutieren
|
|
konnte. Jetzt: nur der Owner-Tick gibt sein eigenes Lock frei."""
|
|
env = _mk_env()
|
|
ms = MissionStore(env["mission"])
|
|
_mission_running(ms, "M-TLP")
|
|
hb = Heartbeat(env["registry"], env["mission"], env["safety"],
|
|
orchestrator=StubOrchestrator(ST_NEEDS_DISPATCH))
|
|
hb.registry.set_kill_switch(True)
|
|
# Tick A haelt das Lock aktiv
|
|
assert hb.registry.acquire_lock("RUN-A") == "ok"
|
|
# Tick B (ueberlappend) -> HB_LOCKED; darf A's Lock NICHT loeschen
|
|
res = hb.run_tick(mission_id="M-TLP")
|
|
check("overlapping tick -> HB_LOCKED", res["code"] == HB_LOCKED, res["code"])
|
|
# Das Lock von A MUSS weiterhin bestehen bleiben (Owner A)
|
|
lock = hb.registry.acquire_lock("RUN-C")
|
|
check("fremdes Lock nach HB_LOCKED NICHT freigegeben", lock == "locked", lock)
|
|
|
|
|
|
def test_tick_lock_release_after_run():
|
|
env = _mk_env()
|
|
ms = MissionStore(env["mission"])
|
|
_mission_running(ms, "M-TL2")
|
|
hb = Heartbeat(env["registry"], env["mission"], env["safety"],
|
|
orchestrator=StubOrchestrator(ST_NEEDS_DISPATCH))
|
|
hb.registry.set_kill_switch(True)
|
|
res = hb.run_tick(mission_id="M-TL2")
|
|
check("run completes", res["code"] == HB_BOUNDED_RUN_COMPLETE, res["code"])
|
|
lock = hb.registry.acquire_lock("RUN-NEXT")
|
|
check("lock released after run", lock == "ok", lock)
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# 3. Mission Eligibility
|
|
# --------------------------------------------------------------------------- #
|
|
def test_eligibility_cancelled():
|
|
env = _mk_env()
|
|
ms = MissionStore(env["mission"])
|
|
_mission_running(ms, "M-TERM")
|
|
ms.mission_transition("M-TERM", "CANCELLED")
|
|
hb = Heartbeat(env["registry"], env["mission"], env["safety"],
|
|
orchestrator=StubOrchestrator(ST_NEEDS_DISPATCH))
|
|
hb.registry.set_kill_switch(True)
|
|
res = hb.run_tick(mission_id="M-TERM")
|
|
check("cancelled -> HB_CANCELLED", res["code"] == HB_CANCELLED, res["code"])
|
|
|
|
|
|
def test_eligibility_paused_not_resumable():
|
|
env = _mk_env()
|
|
ms = MissionStore(env["mission"])
|
|
_mission_running(ms, "M-PAUSE")
|
|
ms.mission_transition("M-PAUSE", "PAUSED")
|
|
hb = Heartbeat(env["registry"], env["mission"], env["safety"],
|
|
orchestrator=StubOrchestrator(ST_NEEDS_DISPATCH))
|
|
hb.registry.set_kill_switch(True)
|
|
res = hb.run_tick(mission_id="M-PAUSE")
|
|
check("paused (no resume) -> HB_PAUSED", res["code"] == HB_PAUSED, res["code"])
|
|
|
|
|
|
def test_eligibility_paused_resumable():
|
|
env = _mk_env()
|
|
ms = MissionStore(env["mission"])
|
|
_mission_running(ms, "M-PR")
|
|
ms.mission_transition("M-PR", "PAUSED")
|
|
hb = Heartbeat(env["registry"], env["mission"], env["safety"],
|
|
orchestrator=StubOrchestrator(ST_NEEDS_DISPATCH))
|
|
hb.registry.set_kill_switch(True)
|
|
hb.registry.set_resume_pending("M-PR", True)
|
|
res = hb.run_tick(mission_id="M-PR")
|
|
check("paused (resumable) -> bounded", res["code"] == HB_BOUNDED_RUN_COMPLETE, res["code"])
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# 4. Priority (deterministisch)
|
|
# --------------------------------------------------------------------------- #
|
|
def test_priority_selection():
|
|
env = _mk_env()
|
|
ms = MissionStore(env["mission"])
|
|
_mission_running(ms, "M-B")
|
|
_mission_running(ms, "M-A")
|
|
hb = Heartbeat(env["registry"], env["mission"], env["safety"],
|
|
orchestrator=StubOrchestrator(ST_NEEDS_DISPATCH))
|
|
hb.registry.set_kill_switch(True)
|
|
hb.registry.set_priority("M-A", 10)
|
|
res = hb.run_tick()
|
|
check("priority selects M-A", res["mission_id"] == "M-A", str(res["mission_id"]))
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# 5. Bounded: EXACTLY ONE A4 run
|
|
# --------------------------------------------------------------------------- #
|
|
def test_bounded_single_a4_call():
|
|
env = _mk_env()
|
|
ms = MissionStore(env["mission"])
|
|
_mission_running(ms, "M-BD")
|
|
stub = StubOrchestrator(ST_NEEDS_DISPATCH)
|
|
hb = Heartbeat(env["registry"], env["mission"], env["safety"], orchestrator=stub)
|
|
hb.registry.set_kill_switch(True)
|
|
res = hb.run_tick(mission_id="M-BD")
|
|
check("exactly one A4 run", stub.run_bounded_calls == 1, str(stub.run_bounded_calls))
|
|
check("bounded complete code", res["code"] == HB_BOUNDED_RUN_COMPLETE, res["code"])
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# 6. NO RUNNABLE WORK
|
|
# --------------------------------------------------------------------------- #
|
|
def test_no_runnable_work():
|
|
env = _mk_env()
|
|
ms = MissionStore(env["mission"])
|
|
_mission_running(ms, "M-NR")
|
|
hb = Heartbeat(env["registry"], env["mission"], env["safety"],
|
|
orchestrator=StubOrchestrator(ST_NO_RUNNABLE_WORK))
|
|
hb.registry.set_kill_switch(True)
|
|
res = hb.run_tick(mission_id="M-NR")
|
|
check("no runnable -> HB_NO_RUNNABLE_WORK", res["code"] == HB_NO_RUNNABLE_WORK, res["code"])
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# 7. Approval
|
|
# --------------------------------------------------------------------------- #
|
|
def test_approval_required_blocks_retry():
|
|
env = _mk_env()
|
|
ms = MissionStore(env["mission"])
|
|
_mission_running(ms, "M-APP")
|
|
stub = StubOrchestrator(ST_APPROVAL_REQUIRED, wp_id="M-APP-WP1", risk="CRITICAL",
|
|
approval={"WP": "M-APP-WP1", "REQUESTED_ACTION": "a"})
|
|
hb = Heartbeat(env["registry"], env["mission"], env["safety"], orchestrator=stub)
|
|
hb.registry.set_kill_switch(True)
|
|
res = hb.run_tick(mission_id="M-APP")
|
|
check("approval required", res["code"] == HB_APPROVAL_REQUIRED, res["code"])
|
|
check("gate persisted", hb.registry.approval_pending("M-APP") is not None)
|
|
# naechster Tick darf NICHT erneut dispatchen
|
|
stub2 = StubOrchestrator(ST_NEEDS_DISPATCH)
|
|
hb2 = Heartbeat(env["registry"], env["mission"], env["safety"], orchestrator=stub2)
|
|
hb2.registry.set_kill_switch(True)
|
|
res2 = hb2.run_tick(mission_id="M-APP")
|
|
check("approval pending blocks re-dispatch", res2["code"] == HB_APPROVAL_PENDING, res2["code"])
|
|
check("no re-dispatch A4 call", stub2.run_bounded_calls == 0, str(stub2.run_bounded_calls))
|
|
|
|
|
|
def test_approval_grant_resumes():
|
|
env = _mk_env()
|
|
ms = MissionStore(env["mission"])
|
|
_mission_running(ms, "M-AGR")
|
|
stub = StubOrchestrator(ST_APPROVAL_REQUIRED, wp_id="M-AGR-WP1", risk="CRITICAL",
|
|
approval={"WP": "M-AGR-WP1", "REQUESTED_ACTION": "a"})
|
|
hb = Heartbeat(env["registry"], env["mission"], env["safety"], orchestrator=stub)
|
|
hb.registry.set_kill_switch(True)
|
|
hb.run_tick(mission_id="M-AGR")
|
|
gate = hb.registry.approval_pending("M-AGR")
|
|
check("gate exists", gate is not None)
|
|
hb.registry.approval_grant(gate["gate_id"], "christian", "approved")
|
|
check("gate cleared after grant", hb.registry.approval_pending("M-AGR") is None)
|
|
stub2 = StubOrchestrator(ST_NEEDS_DISPATCH)
|
|
hb2 = Heartbeat(env["registry"], env["mission"], env["safety"], orchestrator=stub2)
|
|
hb2.registry.set_kill_switch(True)
|
|
res = hb2.run_tick(mission_id="M-AGR")
|
|
check("resume after approval", res["code"] == HB_BOUNDED_RUN_COMPLETE, res["code"])
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# 8. Circuit
|
|
# --------------------------------------------------------------------------- #
|
|
def test_circuit_open_blocks():
|
|
env = _mk_env()
|
|
ms = MissionStore(env["mission"])
|
|
_mission_running(ms, "M-CIR")
|
|
ss = SafetyStore(env["safety"])
|
|
ss.open_circuit("MISSION", "M-CIR", trigger="test circuit", reason="test")
|
|
stub = StubOrchestrator(ST_NEEDS_DISPATCH)
|
|
hb = Heartbeat(env["registry"], env["mission"], env["safety"], orchestrator=stub)
|
|
hb.registry.set_kill_switch(True)
|
|
res = hb.run_tick(mission_id="M-CIR")
|
|
check("circuit open -> HB_CIRCUIT_OPEN", res["code"] == HB_CIRCUIT_OPEN, res["code"])
|
|
check("circuit open -> kein A4", stub.run_bounded_calls == 0, str(stub.run_bounded_calls))
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# 9. Git Conflict
|
|
# --------------------------------------------------------------------------- #
|
|
def test_git_conflict_blocks_resume():
|
|
env = _mk_env()
|
|
ms = MissionStore(env["mission"])
|
|
_mission_running(ms, "M-GIT")
|
|
hb = Heartbeat(env["registry"], env["mission"], env["safety"],
|
|
orchestrator=StubOrchestrator(ST_NEEDS_DISPATCH),
|
|
git_preflight=lambda mid: {"ok": False, "reason": "uncommitted"})
|
|
hb.registry.set_kill_switch(True)
|
|
res = hb.run_tick(mission_id="M-GIT")
|
|
check("git conflict -> HB_GIT_CONFLICT", res["code"] == HB_GIT_CONFLICT, res["code"])
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# 10. Unknown mission / state conflict
|
|
# --------------------------------------------------------------------------- #
|
|
def test_unknown_mission():
|
|
env = _mk_env()
|
|
hb = Heartbeat(env["registry"], env["mission"], env["safety"],
|
|
orchestrator=StubOrchestrator(ST_NEEDS_DISPATCH))
|
|
hb.registry.set_kill_switch(True)
|
|
res = hb.run_tick(mission_id="M-UNKNOWN")
|
|
check("unknown mission -> HB_NO_ACTIVE_MISSION", res["code"] == HB_NO_ACTIVE_MISSION, res["code"])
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# 11. Real bounded orchestrator (kein dispatcher)
|
|
# --------------------------------------------------------------------------- #
|
|
def test_real_orchestrator_bounded_dispatch_contract():
|
|
env = _mk_env()
|
|
ms = MissionStore(env["mission"])
|
|
_mission_running(ms, "M-REAL")
|
|
a4 = Orchestrator(mission_db=env["mission"], safety_db=env["safety"])
|
|
hb = Heartbeat(env["registry"], env["mission"], env["safety"], orchestrator=a4)
|
|
hb.registry.set_kill_switch(True)
|
|
res = hb.run_tick(mission_id="M-REAL", execute=True)
|
|
check("real orchestrator bounded -> complete|no_runnable",
|
|
res["code"] in (HB_BOUNDED_RUN_COMPLETE, HB_NO_RUNNABLE_WORK), res["code"])
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# 12. Health (read-only, Kill-Switch OFF erlaubt)
|
|
# --------------------------------------------------------------------------- #
|
|
def test_health_readonly():
|
|
env = _mk_env()
|
|
ms = MissionStore(env["mission"])
|
|
_mission_running(ms, "M-HEALTH")
|
|
hb = Heartbeat(env["registry"], env["mission"], env["safety"],
|
|
orchestrator=StubOrchestrator(ST_NEEDS_DISPATCH))
|
|
h = hb.health()
|
|
check("health kill switch OFF", h["kill_switch"] == "OFF")
|
|
check("health lists mission", any(m["id"] == "M-HEALTH" for m in h["missions"]))
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# Test Runner
|
|
# --------------------------------------------------------------------------- #
|
|
ALL_TESTS = [
|
|
test_kill_switch_off_blocks_mutation,
|
|
test_kill_switch_on_allows_bounded,
|
|
test_tick_lock_prevents_overlap,
|
|
test_tick_lock_preserved_on_overlap,
|
|
test_tick_lock_release_after_run,
|
|
test_eligibility_cancelled,
|
|
test_eligibility_paused_not_resumable,
|
|
test_eligibility_paused_resumable,
|
|
test_priority_selection,
|
|
test_bounded_single_a4_call,
|
|
test_no_runnable_work,
|
|
test_approval_required_blocks_retry,
|
|
test_approval_grant_resumes,
|
|
test_circuit_open_blocks,
|
|
test_git_conflict_blocks_resume,
|
|
test_unknown_mission,
|
|
test_real_orchestrator_bounded_dispatch_contract,
|
|
test_health_readonly,
|
|
]
|
|
|
|
|
|
def main():
|
|
for fn in ALL_TESTS:
|
|
try:
|
|
fn()
|
|
except Exception as e: # noqa
|
|
global FAIL
|
|
FAIL += 1
|
|
FAILURES.append(fn.__name__)
|
|
print(f" [ERROR] {fn.__name__}: {type(e).__name__}: {e}")
|
|
print("\n" + "=" * 60)
|
|
print(f"PASS={PASS} FAIL={FAIL}")
|
|
if FAIL:
|
|
print("FAILURES:", FAILURES)
|
|
return 1
|
|
print("ALL TESTS PASSED")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|