Red Queen
968094498a
- a5/rq_heartbeat.py: HeartbeatStore + Heartbeat.run_tick (thin scheduler/resume layer) - a5/rq_heartbeat_cli.py: status/tick/enable/disable/resume/approve/deny/priority - a5/test_a5.py: 31 deterministic tests (tick-lock incl. ownership, eligibility, kill-switch, approval, circuit, git-conflict, bounded single A4) - a5/DESIGN.md, a5/README.md, a5/scripts/a5_heartbeat_tick.sh - a2/rq_mission.py: add read-only mission_list() for A5 enumeration (additive) - Fresh checker: PASS after tick-lock ownership repair (Regressionschutz lock_owned)
707 lines
29 KiB
Python
707 lines
29 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
Red Queen — A5: CONTROLLED HEARTBEAT & RESUME v1 (dünne Scheduling-/Resume-Schicht).
|
|
|
|
VERBINDLICHER RAHMEN
|
|
====================
|
|
A5 ist KEINE neue Mission-Engine und KEIN Orchestrator. A5 ist eine DUENNE,
|
|
deterministische Scheduler-/Resume-Schicht, die Red Queen (Hermes) aufweckt,
|
|
einen GENAU BEGRENZTEN A4-Arbeitszyklus anstösst, den Zustand persistiert und
|
|
dann stoppt. Es gibt KEINEN while-true, keinen unbounded Agent-Loop, kein
|
|
Selbst-Reschedule, kein Auto-Rain, kein Self-Improvement.
|
|
|
|
Authority:
|
|
* A2 `MissionStore` = Mission-/WP-State Authority (Transitions, Dependencies).
|
|
* A3 `SafetyStore` = Safety Authority (Circuit, Retry, Attempt-Ledger).
|
|
* A4 `Orchestrator` = Orchestrierungs-Logik (bounded run, Approval, Completion).
|
|
A5 erfindet NICHTS davon — es delegiert ausschliesslich und bewertet den
|
|
Rueckgabecode. A5 implementiert NUR: Kill-Switch, Tick-Lock, Run-ID, Mission-
|
|
Eligibility & -Priorität, exakt-ein-bounded-A4-Aufruf pro Tick, Notification-
|
|
Dedup, Approval-Gate-Warten, Recovery-Reconciliation (fail-closed) und das
|
|
Heartbeat-Registry.
|
|
|
|
Kill-Switch (§25/§26): persistierter globaler Gate AUTONOMOUS_EXECUTION_ENABLED.
|
|
Default OFF. Wenn OFF -> KEINE A4-Mutation; read-only Health erlaubt. Das Gate
|
|
wird nie von einem LLM-Satz überschrieben (Gated Config, >= SI-3).
|
|
|
|
V1: MAX 1 Mission pro Heartbeat. Pro Tick genau EIN A4 bounded run (max_steps=1).
|
|
Keine parallelen Missionen.
|
|
|
|
RESTART-FEST: Alle Zustände (Registry, Tick-Log, Dedup, Approval-Gate,
|
|
Kill-Switch) liegen in einer eigenen, isolierten SQLite-DB (heartbeat.db).
|
|
Nach Container-/Hermes-Restart werden die DBs neu geladen und deterministisch
|
|
fortgesetzt.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import datetime
|
|
import hashlib
|
|
import json
|
|
import sqlite3
|
|
import sys
|
|
import uuid
|
|
from pathlib import Path
|
|
from typing import Any, Callable, Dict, List, Optional
|
|
|
|
_REPO_ROOT = Path(__file__).resolve().parent.parent
|
|
for _p in (str(_REPO_ROOT), str(_REPO_ROOT / "a2"), str(_REPO_ROOT / "a3"), str(_REPO_ROOT / "a4")):
|
|
if _p not in sys.path:
|
|
sys.path.insert(0, _p)
|
|
|
|
from a2.rq_mission import MissionStore, RqError # noqa: E402
|
|
from a3.rq_safety import ( # noqa: E402
|
|
CIRCUIT_OPEN,
|
|
SafetyError,
|
|
SafetyStore,
|
|
redact_secret,
|
|
)
|
|
from a4.rq_orchestrator import ( # noqa: E402
|
|
CIRCUIT_CLOSED,
|
|
ST_APPROVAL_REQUIRED,
|
|
ST_BUDGET_REACHED,
|
|
ST_CHECKER_PASS,
|
|
ST_DEBUG_REQUIRED,
|
|
ST_MISSION_COMPLETED,
|
|
ST_MISSION_NOT_COMPLETED,
|
|
ST_MUTATION_BLOCKED,
|
|
ST_NEEDS_CHECKER_DISPATCH,
|
|
ST_NEEDS_DISPATCH,
|
|
ST_NO_RUNNABLE_WORK,
|
|
ST_SECOND_OPINION_REQUIRED,
|
|
ST_STATE_CONFLICT,
|
|
ST_TERMINAL,
|
|
ST_WAITING_PENDING,
|
|
ST_WP_DONE,
|
|
Orchestrator,
|
|
OrchestratorError,
|
|
)
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# A5-Fehler
|
|
# --------------------------------------------------------------------------- #
|
|
class HeartbeatError(Exception):
|
|
def __init__(self, code: str, message: str, detail: Optional[Dict[str, Any]] = None):
|
|
super().__init__(message)
|
|
self.code = code
|
|
self.message = message
|
|
self.detail = detail or {}
|
|
|
|
|
|
def _err(code: str, msg: str, **detail) -> HeartbeatError:
|
|
return HeartbeatError(code, msg, detail)
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# A5-Konstanten (deterministisch)
|
|
# --------------------------------------------------------------------------- #
|
|
HB_NO_ACTIVE_MISSION = "HB_NO_ACTIVE_MISSION"
|
|
HB_MISSION_NOT_ELIGIBLE = "HB_MISSION_NOT_ELIGIBLE"
|
|
HB_SAFETY_BLOCKED = "HB_SAFETY_BLOCKED"
|
|
HB_STATE_CONFLICT = "HB_STATE_CONFLICT"
|
|
HB_NO_RUNNABLE_WORK = "HB_NO_RUNNABLE_WORK"
|
|
HB_BOUNDED_RUN_COMPLETE = "HB_BOUNDED_RUN_COMPLETE"
|
|
HB_MISSION_COMPLETED = "HB_MISSION_COMPLETED"
|
|
HB_APPROVAL_REQUIRED = "HB_APPROVAL_REQUIRED"
|
|
HB_ESCALATED = "HB_ESCALATED"
|
|
HB_CIRCUIT_OPEN = "HB_CIRCUIT_OPEN"
|
|
HB_TIMEOUT = "HB_TIMEOUT"
|
|
HB_INTERNAL_ERROR = "HB_INTERNAL_ERROR"
|
|
HB_LOCKED = "HB_LOCKED"
|
|
HB_KILL_SWITCH_OFF = "HB_KILL_SWITCH_OFF"
|
|
HB_PAUSED = "HB_PAUSED"
|
|
HB_CANCELLED = "HB_CANCELLED"
|
|
HB_TERMINAL = "HB_TERMINAL"
|
|
HB_GIT_CONFLICT = "HB_GIT_CONFLICT"
|
|
HB_RECOVERY_REQUIRED = "HB_RECOVERY_REQUIRED"
|
|
HB_APPROVAL_PENDING = "HB_APPROVAL_PENDING"
|
|
HB_MISSION_NOT_COMPLETED = "HB_MISSION_NOT_COMPLETED"
|
|
|
|
ELIGIBLE_STATES = frozenset({"READY", "RUNNING", "REVIEW"})
|
|
RESUME_CAPABLE = frozenset({"PAUSED"})
|
|
|
|
TICK_LOCK_STALE_SECONDS = 600 # 10 min ohne Heartbeat -> stale (recovery)
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# DB-Hilfsfunktionen
|
|
# --------------------------------------------------------------------------- #
|
|
def _connect(path: str) -> sqlite3.Connection:
|
|
conn = sqlite3.connect(path)
|
|
conn.row_factory = sqlite3.Row
|
|
conn.execute("PRAGMA journal_mode=WAL")
|
|
conn.execute("PRAGMA foreign_keys=ON")
|
|
return conn
|
|
|
|
|
|
def _utcnow() -> str:
|
|
return datetime.datetime.now(datetime.timezone.utc).isoformat(timespec="seconds")
|
|
|
|
|
|
def _now_epoch() -> float:
|
|
return datetime.datetime.now(datetime.timezone.utc).timestamp()
|
|
|
|
|
|
def _new_run_id() -> str:
|
|
return f"HB-{datetime.datetime.now(datetime.timezone.utc).strftime('%Y%m%dT%H%M%S')}-{uuid.uuid4().hex[:8]}"
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# HeartbeatStore: persistente A5-Registry
|
|
# --------------------------------------------------------------------------- #
|
|
class HeartbeatStore:
|
|
"""Isolierte, durable A5-Registry (heartbeat.db)."""
|
|
|
|
def __init__(self, db_path: str):
|
|
self.db_path = db_path
|
|
self._ensure_schema()
|
|
|
|
def _ensure_schema(self) -> None:
|
|
conn = _connect(self.db_path)
|
|
try:
|
|
conn.executescript(
|
|
"""
|
|
CREATE TABLE IF NOT EXISTS config (
|
|
key TEXT PRIMARY KEY,
|
|
value TEXT NOT NULL
|
|
);
|
|
CREATE TABLE IF NOT EXISTS tick_log (
|
|
run_id TEXT PRIMARY KEY,
|
|
started_at TEXT NOT NULL,
|
|
finished_at TEXT,
|
|
result_code TEXT,
|
|
mission_id TEXT,
|
|
exit_reason TEXT,
|
|
detail TEXT
|
|
);
|
|
CREATE TABLE IF NOT EXISTS notification_dedup (
|
|
dedup_key TEXT PRIMARY KEY,
|
|
sent_at TEXT NOT NULL,
|
|
mission_id TEXT,
|
|
reason_code TEXT,
|
|
message_id TEXT
|
|
);
|
|
CREATE TABLE IF NOT EXISTS lock (
|
|
id INTEGER PRIMARY KEY CHECK (id = 1),
|
|
owner TEXT,
|
|
acquired_at TEXT,
|
|
heartbeat_at TEXT
|
|
);
|
|
CREATE TABLE IF NOT EXISTS approval_gates (
|
|
gate_id TEXT PRIMARY KEY,
|
|
mission_id TEXT NOT NULL,
|
|
wp_id TEXT NOT NULL,
|
|
action TEXT NOT NULL,
|
|
requested_action TEXT,
|
|
risk TEXT,
|
|
state TEXT NOT NULL,
|
|
requested_at TEXT NOT NULL,
|
|
approved_at TEXT,
|
|
approved_by TEXT,
|
|
evidence TEXT
|
|
);
|
|
"""
|
|
)
|
|
conn.commit()
|
|
finally:
|
|
conn.close()
|
|
|
|
# -- config ---------------------------------------------------------------
|
|
def get_config(self, key: str, default: Optional[str] = None) -> Optional[str]:
|
|
conn = _connect(self.db_path)
|
|
try:
|
|
row = conn.execute("SELECT value FROM config WHERE key = ?", (key,)).fetchone()
|
|
return row["value"] if row else default
|
|
finally:
|
|
conn.close()
|
|
|
|
def set_config(self, key: str, value: str) -> None:
|
|
conn = _connect(self.db_path)
|
|
try:
|
|
conn.execute(
|
|
"INSERT INTO config (key,value) VALUES (?,?) "
|
|
"ON CONFLICT(key) DO UPDATE SET value=excluded.value",
|
|
(key, value),
|
|
)
|
|
conn.commit()
|
|
finally:
|
|
conn.close()
|
|
|
|
# -- Kill Switch ----------------------------------------------------------
|
|
def kill_switch_on(self) -> bool:
|
|
v = self.get_config("AUTONOMOUS_EXECUTION_ENABLED", "OFF")
|
|
return str(v).strip().upper() == "ON"
|
|
|
|
def set_kill_switch(self, on: bool) -> None:
|
|
self.set_config("AUTONOMOUS_EXECUTION_ENABLED", "ON" if on else "OFF")
|
|
|
|
# -- Resume-Flag (PAUSED -> resume-faehig) --------------------------------
|
|
def set_resume_pending(self, mission_id: str, pending: bool) -> None:
|
|
self.set_config(f"resume_pending:{mission_id}", "1" if pending else "0")
|
|
|
|
def resume_pending(self, mission_id: str) -> bool:
|
|
return self.get_config(f"resume_pending:{mission_id}", "0") == "1"
|
|
|
|
def set_priority(self, mission_id: str, priority: int) -> None:
|
|
self.set_config(f"priority:{mission_id}", str(int(priority)))
|
|
|
|
# -- Tick Lock ------------------------------------------------------------
|
|
def acquire_lock(self, run_id: str) -> str:
|
|
"""EIN-Tick-Lock. Rueckgabe 'ok'|'locked'|'recover'."""
|
|
now = _now_epoch()
|
|
conn = _connect(self.db_path)
|
|
try:
|
|
row = conn.execute("SELECT * FROM lock WHERE id=1").fetchone()
|
|
if row is None:
|
|
conn.execute(
|
|
"INSERT INTO lock(id,owner,acquired_at,heartbeat_at) VALUES (1,?,?,?)",
|
|
(run_id, _utcnow(), now),
|
|
)
|
|
conn.commit()
|
|
return "ok"
|
|
heartbeat_at = float(row["heartbeat_at"]) if row["heartbeat_at"] else 0
|
|
if now - heartbeat_at <= TICK_LOCK_STALE_SECONDS:
|
|
return "locked"
|
|
return "recover"
|
|
finally:
|
|
conn.close()
|
|
|
|
def heartbeat_mark(self, run_id: str) -> None:
|
|
conn = _connect(self.db_path)
|
|
try:
|
|
conn.execute(
|
|
"INSERT INTO lock(id,owner,acquired_at,heartbeat_at) VALUES (1,?,?,?) "
|
|
"ON CONFLICT(id) DO UPDATE SET owner=excluded.owner, heartbeat_at=excluded.heartbeat_at",
|
|
(run_id, _utcnow(), _now_epoch()),
|
|
)
|
|
conn.commit()
|
|
finally:
|
|
conn.close()
|
|
|
|
def release_lock(self) -> None:
|
|
conn = _connect(self.db_path)
|
|
try:
|
|
conn.execute("DELETE FROM lock WHERE id=1")
|
|
conn.commit()
|
|
finally:
|
|
conn.close()
|
|
|
|
# -- tick_log -------------------------------------------------------------
|
|
def log_tick(self, run_id, started_at, finished_at, result_code, mission_id,
|
|
exit_reason, detail=None) -> None:
|
|
conn = _connect(self.db_path)
|
|
try:
|
|
conn.execute(
|
|
"INSERT OR REPLACE INTO tick_log "
|
|
"(run_id,started_at,finished_at,result_code,mission_id,exit_reason,detail) "
|
|
"VALUES (?,?,?,?,?,?,?)",
|
|
(run_id, started_at, finished_at, result_code, mission_id, exit_reason,
|
|
json.dumps(detail or {}, default=str)),
|
|
)
|
|
conn.commit()
|
|
finally:
|
|
conn.close()
|
|
|
|
def last_tick(self) -> Optional[Dict[str, Any]]:
|
|
conn = _connect(self.db_path)
|
|
try:
|
|
row = conn.execute("SELECT * FROM tick_log ORDER BY started_at DESC LIMIT 1").fetchone()
|
|
if row is None:
|
|
return None
|
|
d = dict(row)
|
|
try:
|
|
d["detail"] = json.loads(d["detail"]) if d["detail"] else {}
|
|
except (ValueError, TypeError):
|
|
d["detail"] = {}
|
|
return d
|
|
finally:
|
|
conn.close()
|
|
|
|
def run_id_known(self, run_id: str) -> bool:
|
|
conn = _connect(self.db_path)
|
|
try:
|
|
return conn.execute("SELECT 1 FROM tick_log WHERE run_id=?", (run_id,)).fetchone() is not None
|
|
finally:
|
|
conn.close()
|
|
|
|
# -- Notification-Dedup ---------------------------------------------------
|
|
def notification_key(self, mission_id: str, reason_code: str, evidence: str) -> str:
|
|
raw = f"{mission_id}|{reason_code}|{evidence}"
|
|
return hashlib.sha256(raw.encode("utf-8")).hexdigest()
|
|
|
|
def notification_seen(self, dedup_key: str) -> bool:
|
|
conn = _connect(self.db_path)
|
|
try:
|
|
return conn.execute("SELECT 1 FROM notification_dedup WHERE dedup_key=?", (dedup_key,)).fetchone() is not None
|
|
finally:
|
|
conn.close()
|
|
|
|
def notification_record(self, dedup_key: str, mission_id: str, reason_code: str, message_id: str) -> None:
|
|
conn = _connect(self.db_path)
|
|
try:
|
|
conn.execute(
|
|
"INSERT INTO notification_dedup (dedup_key,sent_at,mission_id,reason_code,message_id) "
|
|
"VALUES (?,?,?,?,?)",
|
|
(dedup_key, _utcnow(), mission_id, reason_code, message_id),
|
|
)
|
|
conn.commit()
|
|
finally:
|
|
conn.close()
|
|
|
|
# -- Approval Gate --------------------------------------------------------
|
|
def approval_pending(self, mission_id: str) -> Optional[Dict[str, Any]]:
|
|
conn = _connect(self.db_path)
|
|
try:
|
|
row = conn.execute(
|
|
"SELECT * FROM approval_gates WHERE mission_id=? AND state='PENDING' "
|
|
"ORDER BY requested_at LIMIT 1",
|
|
(mission_id,),
|
|
).fetchone()
|
|
return dict(row) if row else None
|
|
finally:
|
|
conn.close()
|
|
|
|
def approval_issue(self, mission_id: str, wp_id: str, action: str, risk: str) -> str:
|
|
"""Persistiert einen wartenden Approval-Gate. Idempotent (kein Duplikat
|
|
fuer denselben WP im PENDING-Zustand)."""
|
|
conn = _connect(self.db_path)
|
|
try:
|
|
existing = conn.execute(
|
|
"SELECT * FROM approval_gates WHERE mission_id=? AND wp_id=? AND state='PENDING'",
|
|
(mission_id, wp_id),
|
|
).fetchone()
|
|
if existing:
|
|
return existing["gate_id"]
|
|
gate_id = str(uuid.uuid4())
|
|
conn.execute(
|
|
"INSERT INTO approval_gates "
|
|
"(gate_id,mission_id,wp_id,action,requested_action,risk,state,requested_at) "
|
|
"VALUES (?,?,?,?,?,?,?,?)",
|
|
(gate_id, mission_id, wp_id, "APPROVE", action, risk, "PENDING", _utcnow()),
|
|
)
|
|
conn.commit()
|
|
return gate_id
|
|
finally:
|
|
conn.close()
|
|
|
|
def approval_grant(self, gate_id: str, approved_by: str, evidence: str) -> Dict[str, Any]:
|
|
conn = _connect(self.db_path)
|
|
try:
|
|
row = conn.execute("SELECT * FROM approval_gates WHERE gate_id=?", (gate_id,)).fetchone()
|
|
if row is None:
|
|
raise _err("UNKNOWN_GATE", f"unknown approval gate {gate_id!r}", gate_id=gate_id)
|
|
if row["state"] == "APPROVED":
|
|
return {"gate_id": gate_id, "state": "APPROVED", "idempotent": True}
|
|
conn.execute(
|
|
"UPDATE approval_gates SET state='APPROVED', approved_at=?, approved_by=?, evidence=? "
|
|
"WHERE gate_id=?",
|
|
(_utcnow(), approved_by, evidence, gate_id),
|
|
)
|
|
conn.commit()
|
|
return {"gate_id": gate_id, "state": "APPROVED", "idempotent": False}
|
|
finally:
|
|
conn.close()
|
|
|
|
def approval_deny(self, gate_id: str, denied_by: str, evidence: str) -> Dict[str, Any]:
|
|
conn = _connect(self.db_path)
|
|
try:
|
|
row = conn.execute("SELECT * FROM approval_gates WHERE gate_id=?", (gate_id,)).fetchone()
|
|
if row is None:
|
|
raise _err("UNKNOWN_GATE", f"unknown approval gate {gate_id!r}", gate_id=gate_id)
|
|
conn.execute(
|
|
"UPDATE approval_gates SET state='DENIED', approved_at=?, approved_by=?, evidence=? "
|
|
"WHERE gate_id=?",
|
|
(_utcnow(), denied_by, evidence, gate_id),
|
|
)
|
|
conn.commit()
|
|
return {"gate_id": gate_id, "state": "DENIED"}
|
|
finally:
|
|
conn.close()
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# Heartbeat
|
|
# --------------------------------------------------------------------------- #
|
|
class Heartbeat:
|
|
"""A5: Kontrollierter Heartbeat & Resume. `run_tick()` = genau ein bounded cycle."""
|
|
|
|
def __init__(
|
|
self,
|
|
registry_db: str,
|
|
mission_db: str,
|
|
safety_db: str,
|
|
orchestrator: Optional[Orchestrator] = None,
|
|
*,
|
|
max_bounded_steps: int = 1,
|
|
git_preflight: Optional[Callable[[str], Dict[str, Any]]] = None,
|
|
):
|
|
self.registry = HeartbeatStore(registry_db)
|
|
self.missions = MissionStore(mission_db)
|
|
self.safety = SafetyStore(safety_db)
|
|
self.orchestrator = orchestrator
|
|
self.max_bounded_steps = max_bounded_steps
|
|
self.git_preflight = git_preflight
|
|
|
|
# -- Helper: Mission-Enumeration ------------------------------------------
|
|
def _mission(self) -> List[Dict[str, Any]]:
|
|
try:
|
|
return self.missions.mission_list()
|
|
except RqError:
|
|
return []
|
|
|
|
# -- Eligibility & Priority (deterministisch) ------------------------------
|
|
def _eligible_states(self, state: str, resume_pending: bool) -> bool:
|
|
if state in ELIGIBLE_STATES:
|
|
return True
|
|
if state == "PAUSED" and resume_pending:
|
|
return True
|
|
return False
|
|
|
|
def _mission_circuit(self, mission_id: str) -> Optional[str]:
|
|
try:
|
|
g = self.safety.circuit_state("GLOBAL", "global")
|
|
if g["state"] == CIRCUIT_OPEN:
|
|
return "global"
|
|
except SafetyError:
|
|
pass
|
|
try:
|
|
m = self.safety.circuit_state("MISSION", mission_id)
|
|
if m["state"] == CIRCUIT_OPEN:
|
|
return f"mission:{mission_id}"
|
|
except SafetyError:
|
|
pass
|
|
return None
|
|
|
|
def _not_eligible_code(self, state: str) -> str:
|
|
if state in ("COMPLETED", "FAILED", "ESCALATED"):
|
|
return HB_TERMINAL
|
|
if state == "CANCELLED":
|
|
return HB_CANCELLED
|
|
if state == "PAUSED":
|
|
return HB_PAUSED
|
|
return HB_MISSION_NOT_ELIGIBLE
|
|
|
|
def select_eligible(self, mission_id: Optional[str] = None) -> Dict[str, Any]:
|
|
"""Deterministische Auswahl EINER heartbeat-eligible Mission.
|
|
|
|
Rueckgabe {'eligible', 'mission', 'reason', 'code'}."""
|
|
if mission_id is not None:
|
|
try:
|
|
m = self.missions.mission_read(mission_id)
|
|
except RqError as e:
|
|
return {"eligible": False, "mission": None, "reason": e.message, "code": HB_NO_ACTIVE_MISSION}
|
|
st = m["state"]
|
|
resume_pending = self.registry.resume_pending(mission_id)
|
|
if not self._eligible_states(m["state"], resume_pending):
|
|
return {"eligible": False, "mission": {"id": m["id"], "state": st},
|
|
"reason": f"mission {st}; not eligible", "code": self._not_eligible_code(st)}
|
|
circ = self._mission_circuit(mission_id)
|
|
if circ is not None:
|
|
return {"eligible": False, "mission": {"id": m["id"], "state": st},
|
|
"reason": f"circuit open: {circ}", "code": HB_CIRCUIT_OPEN}
|
|
return {"eligible": True, "mission": {"id": m["id"], "state": st, "created_at": m["created_at"]},
|
|
"reason": "", "code": "OK"}
|
|
|
|
cands = []
|
|
for m in self._mission():
|
|
resume_pending = self.registry.resume_pending(m["id"])
|
|
if not self._eligible_states(m["state"], resume_pending):
|
|
continue
|
|
if self._mission_circuit(m["id"]) is not None:
|
|
continue
|
|
cands.append({"id": m["id"], "state": m["state"], "created_at": m["created_at"],
|
|
"priority": int(self.registry.get_config(f"priority:{m['id']}", "0") or 0)})
|
|
if not cands:
|
|
return {"eligible": False, "mission": None, "reason": "no active eligible mission",
|
|
"code": HB_NO_ACTIVE_MISSION}
|
|
cands.sort(key=lambda x: (-x["priority"], x["created_at"], x["id"]))
|
|
return {"eligible": True, "mission": cands[0], "reason": "", "code": "OK"}
|
|
|
|
# -- run_tick --------------------------------------------------------------
|
|
def run_tick(self, mission_id: Optional[str] = None, *, execute: bool = True,
|
|
actor: str = "red-queen") -> Dict[str, Any]:
|
|
"""EIN Heartbeat-Tick. Endet garantiert."""
|
|
started = _utcnow()
|
|
run_id = _new_run_id()
|
|
result_code = HB_INTERNAL_ERROR
|
|
exit_reason = ""
|
|
mission_processed: Optional[str] = None
|
|
detail: Dict[str, Any] = {}
|
|
lock_owned = False # nur TRUE, wenn DIESER Tick das Tick-Lock erworben hat
|
|
|
|
try:
|
|
# Idempotenz: gleiche Run-ID nie doppelt
|
|
if self.registry.run_id_known(run_id):
|
|
return self._finalize(run_id, started, HB_NO_RUNNABLE_WORK, None,
|
|
"run id already processed", {"idempotent": True})
|
|
|
|
# Kill-Switch-Gate
|
|
if not self.registry.kill_switch_on():
|
|
return self._finalize(run_id, started, HB_KILL_SWITCH_OFF, None,
|
|
"autonomous execution disabled (kill switch OFF)",
|
|
{"mutating": False})
|
|
|
|
# Tick-Lock
|
|
lock = self.registry.acquire_lock(run_id)
|
|
if lock == "locked":
|
|
return self._finalize(run_id, started, HB_LOCKED, None,
|
|
"tick lock held: overlapping heartbeat prevented",
|
|
{"lock": "locked"})
|
|
if lock == "recover":
|
|
return self._finalize(run_id, started, HB_RECOVERY_REQUIRED, None,
|
|
"stale tick lock: recovery required, no mutation",
|
|
{"lock": "recover"})
|
|
lock_owned = True
|
|
self.registry.heartbeat_mark(run_id)
|
|
|
|
try:
|
|
# Eligibility
|
|
sel = self.select_eligible(mission_id)
|
|
if not sel["eligible"]:
|
|
mission_processed = sel.get("mission", {}).get("id") if sel.get("mission") else None
|
|
return self._finalize(run_id, started, sel["code"], mission_processed,
|
|
sel["reason"], {"mission": sel.get("mission")})
|
|
|
|
mid = sel["mission"]["id"]
|
|
mission_processed = mid
|
|
|
|
# A2-Load
|
|
try:
|
|
m = self.missions.mission_read(mid)
|
|
except RqError as e:
|
|
return self._finalize(run_id, started, HB_STATE_CONFLICT, mid,
|
|
f"mission read failed: {e.message}", {})
|
|
if m["state"] in ("COMPLETED", "FAILED", "CANCELLED", "ESCALATED"):
|
|
return self._finalize(run_id, started, HB_TERMINAL, mid,
|
|
f"mission {m['state']}; terminal", {})
|
|
|
|
# A3 Safety
|
|
try:
|
|
self.safety.check_safety_state(mid)
|
|
except SafetyError as e:
|
|
return self._finalize(run_id, started, HB_SAFETY_BLOCKED, mid,
|
|
f"safety state: {e.message}", {"reason_code": e.code})
|
|
|
|
# Git-Preflight
|
|
if self.git_preflight is not None:
|
|
git = self.git_preflight(mid)
|
|
if not git.get("ok", False):
|
|
return self._finalize(run_id, started, HB_GIT_CONFLICT, mid,
|
|
f"git state conflict: {git.get('reason')}", {"git": git})
|
|
|
|
# Approval-Pending (persistierter Gate)
|
|
pending = self.registry.approval_pending(mid)
|
|
if pending is not None:
|
|
return self._finalize(run_id, started, HB_APPROVAL_PENDING, mid,
|
|
"approval gate pending; cannot dispatch",
|
|
{"gate_id": pending["gate_id"], "wp_id": pending["wp_id"]})
|
|
|
|
# Execute-Gate (read-only)
|
|
if not execute:
|
|
return self._finalize(run_id, started, HB_NO_RUNNABLE_WORK, mid,
|
|
"execute=False (read-only); no A4 mutation", {})
|
|
|
|
# EXACTLY ONE bounded A4 run
|
|
a4 = self._orchestrator()
|
|
a4_result = a4.run_bounded(mid, max_steps=self.max_bounded_steps, actor=actor)
|
|
st = a4_result.get("status")
|
|
detail["a4"] = a4_result
|
|
|
|
# Map A4 -> A5
|
|
if st in (ST_NEEDS_DISPATCH, ST_NEEDS_CHECKER_DISPATCH, ST_WAITING_PENDING,
|
|
ST_CHECKER_PASS, ST_WP_DONE, ST_BUDGET_REACHED):
|
|
result_code = HB_BOUNDED_RUN_COMPLETE
|
|
exit_reason = f"bounded a4 cycle complete: {st}"
|
|
elif st == ST_NO_RUNNABLE_WORK:
|
|
# Completion-Gate pruefen
|
|
cc = a4.mission_completion_gate(mid, final_review_pass=False)
|
|
if cc["valid"]:
|
|
cmp = a4.attempt_mission_complete(mid, final_review_pass=False)
|
|
if cmp["status"] == ST_MISSION_COMPLETED:
|
|
result_code = HB_MISSION_COMPLETED
|
|
exit_reason = "mission completed (all wps done)"
|
|
detail["completed"] = True
|
|
else:
|
|
result_code = HB_MISSION_NOT_COMPLETED
|
|
exit_reason = f"completion gate passed but complete failed: {cmp.get('reasons')}"
|
|
else:
|
|
result_code = HB_NO_RUNNABLE_WORK
|
|
exit_reason = "no runnable work"
|
|
elif st == ST_APPROVAL_REQUIRED:
|
|
result_code = HB_APPROVAL_REQUIRED
|
|
wp_id = a4_result.get("wp_id") or a4_result.get("approval", {}).get("WP")
|
|
act = a4_result.get("approval", {}).get("REQUESTED_ACTION") or ""
|
|
risk = a4_result.get("risk") or "CRITICAL"
|
|
if wp_id:
|
|
self.registry.approval_issue(mid, wp_id, act, risk)
|
|
exit_reason = "approval required; gate persisted"
|
|
elif st == ST_MUTATION_BLOCKED:
|
|
result_code = HB_CIRCUIT_OPEN
|
|
exit_reason = f"mutation blocked: {a4_result.get('reason')}"
|
|
elif st == ST_STATE_CONFLICT:
|
|
result_code = HB_STATE_CONFLICT
|
|
exit_reason = a4_result.get("reason", "orchestrator state conflict")
|
|
elif st in (ST_DEBUG_REQUIRED, ST_SECOND_OPINION_REQUIRED):
|
|
result_code = HB_ESCALATED
|
|
exit_reason = f"escalated: {st}"
|
|
elif st == ST_MISSION_COMPLETED:
|
|
result_code = HB_MISSION_COMPLETED
|
|
exit_reason = "mission completed"
|
|
elif st == ST_TERMINAL:
|
|
result_code = HB_TERMINAL
|
|
exit_reason = "mission terminal"
|
|
else:
|
|
result_code = HB_INTERNAL_ERROR
|
|
exit_reason = f"unhandled orchestrator status {st!r}"
|
|
|
|
return self._finalize(run_id, started, result_code, mid, exit_reason, detail)
|
|
|
|
except HeartbeatError as e:
|
|
return self._finalize(run_id, started, HB_INTERNAL_ERROR, mission_processed,
|
|
e.message, {"code": e.code})
|
|
finally:
|
|
if lock_owned:
|
|
self.registry.release_lock()
|
|
|
|
except HeartbeatError as e:
|
|
return self._finalize(run_id, started, HB_INTERNAL_ERROR, mission_processed,
|
|
e.message, {"code": e.code})
|
|
except Exception as e: # noqa
|
|
return self._finalize(run_id, started, HB_INTERNAL_ERROR, mission_processed,
|
|
f"internal error: {redact_secret(str(e))}", {})
|
|
finally:
|
|
if lock_owned:
|
|
try:
|
|
self.registry.release_lock()
|
|
except Exception:
|
|
pass
|
|
|
|
def _orchestrator(self) -> Orchestrator:
|
|
if self.orchestrator is not None:
|
|
return self.orchestrator
|
|
return Orchestrator(mission_db=self.missions.db_path, safety_db=self.safety.db_path)
|
|
|
|
def _finalize(self, run_id, started, result_code, mission_id, exit_reason, detail) -> Dict[str, Any]:
|
|
finished = _utcnow()
|
|
self.registry.log_tick(run_id, started, finished, result_code, mission_id, exit_reason, detail)
|
|
return {
|
|
"run_id": run_id,
|
|
"code": result_code,
|
|
"result": result_code,
|
|
"mission_id": mission_id,
|
|
"exit_reason": exit_reason,
|
|
"detail": detail,
|
|
"started_at": started,
|
|
"finished_at": finished,
|
|
}
|
|
|
|
# -- Health (read-only) ----------------------------------------------------
|
|
def health(self) -> Dict[str, Any]:
|
|
return {
|
|
"kill_switch": "ON" if self.registry.kill_switch_on() else "OFF",
|
|
"last_tick": self.registry.last_tick(),
|
|
"missions": [{"id": m["id"], "state": m["state"]} for m in self._mission()],
|
|
}
|
|
|
|
|
|
# Referenz: Wir brauchen einen Code fuer "Completion-Gate ok aber complete failed".
|
|
HB_MISSION_NOT_FOUND_CODE = "HB_MISSION_NOT_COMPLETED"
|