Red Queen
5e4191578a
- SearchSourceBuilder: deterministischer Vault->Source-Snapshot (read-only), atomar (temp->validate->fsync->replace), Secret-Scan fail-closed - verify_integrity: exakte object_id/path Set-Equality (stale Source kann nie APPLIED), Canary implizit ueber erwartetes Objekt-Set - apply_commit: Source-Build+Verification vor Rebuild; VERIFYING_SEARCH-Resume (kein Doppel-Rebuild) — C5E-Replay-Crash-Fall abgedeckt - Fix: source_object_count ist keine 0-Fehlerbedingung (echter Defekt) - SearchSourceBuildError + RC_SEARCH_SOURCE_BUILD_FAILURE (Human Gate) - c4b: source_path env-konfigurierbar, indexed_object_ids/paths Read-Back - Testsuite: 17 neue Tests (Test-Plan A-O + Realistic C4-Integration)
836 lines
36 KiB
Python
836 lines
36 KiB
Python
#!/usr/bin/env python3
|
||
"""
|
||
Red Queen — C5D: SEARCH INTEGRATION + COMMIT COMPLETION v1 (deterministische, inaktive Library).
|
||
|
||
C5D implementiert ausschliesslich:
|
||
UPDATING_SEARCH -> SEARCH FULL REBUILD -> VERIFYING_SEARCH
|
||
-> SEARCH HEALTH/INTEGRITY READ-BACK -> APPLIED -> last_applied_commit fortschreiben
|
||
|
||
VERBINDLICHE REIHENFOLGE (ORDERING INVARIANT):
|
||
Forgejo Commit -> Tolaria propagiert (C5C) -> Tolaria Read-Back DRIFT=0
|
||
-> Search Full Rebuild -> Search Health/Integrity PASS -> Commit APPLIED
|
||
-> last_applied_commit fortschreiben
|
||
|
||
INVARIANTE: Search darf NIEMALS vor erfolgreicher Tolaria-Verifikation aktualisiert
|
||
werden. APPLIED und last_applied_commit duerfen ausschliesslich gesetzt werden,
|
||
wenn Tolaria UND Search vollstaendig erfolgreich und verifiziert sind.
|
||
|
||
C5D startet NUR aus ST_UPDATING_SEARCH (Tolaria bereits verifiziert) oder
|
||
ST_RETRY_PENDING (Search-Retry-Replay). C5D ruft NIE Tolaria auf -> kein
|
||
Tolaria-Doppel-Write beim Search-Retry.
|
||
|
||
ARCHITEKTURREGEL (verbindlich):
|
||
* Forgejo bleibt MASTER / Source of Truth.
|
||
* Tolaria ist ausschliesslich DERIVED.
|
||
* C5D darf: Forgejo lesen, Search kontrolliert rebuilden (mit Token).
|
||
* C5D darf NIEMALS: Tolaria -> Forgejo schreiben, Forgejo veraendern,
|
||
Tolaria schreiben, IDs neu vergeben, Knowledge Content umformulieren.
|
||
|
||
NO-TOLARIA-WRITE-GUARANTEE: C5D ruft NICHT /api/vault/save auf.
|
||
NO-MASTER-WRITE-GUARANTEE: C5D fuehrt KEIN git push / Forgejo-Write aus.
|
||
NO-PRODUCTION-ACTIVATION: C5D fuehrt KEINE produktive Propagation / Search-Rebuild
|
||
aus (nur Tests gegen Fake/Mock + read-only Dry-Run).
|
||
|
||
FAIL-CLOSED: Bei Search-Ausfall, Timeout, Auth-Fehler, ungueltiger Antwort,
|
||
Rebuild-Fehler oder fehlerhafter Integrity -> Commit NICHT APPLIED,
|
||
last_applied_commit NICHT veraendert, Fehler klassifiziert, Retry/Human-Gate
|
||
gemaess C5A-Contract.
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import json
|
||
import os
|
||
import urllib.error
|
||
import urllib.request
|
||
from pathlib import Path
|
||
from typing import Any, Dict, List, Optional, Tuple
|
||
|
||
# C5A / C5C wiederverwenden (keine konkurrierende State Machine)
|
||
from rq_c5a import (
|
||
C5AStore,
|
||
ST_UPDATING_SEARCH,
|
||
ST_VERIFYING_SEARCH,
|
||
ST_APPLIED,
|
||
ST_RETRY_PENDING,
|
||
ST_DEAD,
|
||
ST_HUMAN_REVIEW_REQUIRED,
|
||
IDEM_ALREADY_APPLIED,
|
||
IDEM_RETRY_SAFE,
|
||
RC_SEARCH_REBUILD_FAILURE,
|
||
RC_AUTH_FAILURE,
|
||
RC_UNKNOWN_OBJECT_ID,
|
||
RC_UNEXPECTED_TOLARIA_DRIFT,
|
||
RC_SEARCH_SOURCE_BUILD_FAILURE,
|
||
RC_INTEGRITY_FAILURE,
|
||
DEFAULT_MAX_RETRIES,
|
||
DEFAULT_BACKOFF_SECONDS,
|
||
)
|
||
from rq_c5c import (
|
||
C5CError,
|
||
TolariaClient,
|
||
TolariaUnavailableError,
|
||
VAULT_PREFIX,
|
||
)
|
||
# C5B-Bausteine: deterministische Vault->Source-Extraktion (kein neues SoT).
|
||
# content_hash, parse_frontmatter, extract_object_id, KnowledgeScope sind
|
||
# identisch zur C5B-Change-Detection -> konsistente Metadaten-Hashes.
|
||
from rq_c5b import (
|
||
parse_frontmatter,
|
||
extract_object_id,
|
||
content_hash,
|
||
detect_secret,
|
||
KnowledgeScope,
|
||
SCOPE_IN_SCOPE,
|
||
SCOPE_LEGACY_SPECIAL,
|
||
)
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Konstanten
|
||
# ---------------------------------------------------------------------------
|
||
|
||
# Search-API-Basis (Default; per Env C5_SEARCH_BASE ueberschreibbar)
|
||
DEFAULT_SEARCH_BASE = "http://127.0.0.1:8325/api/search"
|
||
ENV_SEARCH_BASE = "C5_SEARCH_BASE"
|
||
ENV_SEARCH_TOKEN = "TOLARIA_SEARCH_REBUILD_TOKEN"
|
||
|
||
# Search-Rebuild-Ergebnis
|
||
REBUILD_OK = "ok"
|
||
|
||
# Integrity-Verifikations-Ergebnisse
|
||
INTEGRITY_OK = "INTEGRITY_OK"
|
||
INTEGRITY_FAIL = "INTEGRITY_FAIL"
|
||
|
||
|
||
class SearchError(C5CError):
|
||
"""Basis-Fehler fuer C5D Search-Integration."""
|
||
|
||
|
||
class SearchUnavailableError(SearchError):
|
||
"""Search nicht erreichbar / Timeout / transienter HTTP-Fehler (retrybar)."""
|
||
|
||
|
||
class SearchAuthError(SearchError):
|
||
"""Search-Rebuild Auth-Fehler (nicht retrybar, Human Gate)."""
|
||
|
||
|
||
class SearchRebuildError(SearchError):
|
||
"""Search-Rebuild fehlgeschlagen (nicht retrybar, Human Gate)."""
|
||
|
||
|
||
class SearchIntegrityError(SearchError):
|
||
"""Search Health/Integrity nicht PASS (nicht retrybar, Human Gate)."""
|
||
|
||
|
||
class SearchSourceBuildError(SearchError):
|
||
"""
|
||
Search-Source-Build/Verification fehlgeschlagen oder Objekt-Set weicht vom
|
||
erwarteten Tolaria-Stand ab (nicht retrybar, Human Gate).
|
||
|
||
C5D-Haertung: kein Rebuild darf auf stale/unvollstaendiger Source
|
||
stattfinden; FAIL CLOSED vor APPLIED.
|
||
"""
|
||
|
||
|
||
class SearchMalformedResponseError(SearchError):
|
||
"""Search-Antwort ungueltig / malformed (nicht retrybar, Human Gate)."""
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Search Client (isoliert: rebuild / health)
|
||
# ---------------------------------------------------------------------------
|
||
|
||
class SearchClient:
|
||
"""
|
||
Isolierter Search-Client. Nur C5D-Writer darf rebuild() verwenden.
|
||
|
||
rebuild() — POST /api/search/rebuild (Bearer-Token, kontrolliert)
|
||
health() — GET /api/search/health (Read-Back, oeffentlich)
|
||
|
||
Keine generische Agent-Write-Funktion. Kein Tolaria-Write.
|
||
"""
|
||
|
||
def __init__(self, base_url: Optional[str] = None, token: Optional[str] = None,
|
||
timeout: float = 15.0):
|
||
self.base_url = (base_url or os.environ.get(ENV_SEARCH_BASE)
|
||
or DEFAULT_SEARCH_BASE).rstrip("/")
|
||
self.token = token if token is not None else os.environ.get(ENV_SEARCH_TOKEN, "")
|
||
self.timeout = timeout
|
||
|
||
# -- HTTP-Helfer --------------------------------------------------------
|
||
|
||
def _post(self, endpoint: str, payload: Optional[Dict[str, Any]] = None) -> Dict[str, Any]:
|
||
url = f"{self.base_url}/{endpoint.lstrip('/')}"
|
||
data = json.dumps(payload or {}).encode("utf-8")
|
||
headers = {"Content-Type": "application/json"}
|
||
if self.token:
|
||
headers["Authorization"] = "Bearer " + self.token
|
||
req = urllib.request.Request(url, data=data, headers=headers, method="POST")
|
||
try:
|
||
with urllib.request.urlopen(req, timeout=self.timeout) as resp:
|
||
body = resp.read().decode("utf-8")
|
||
return json.loads(body) if body else {}
|
||
except urllib.error.HTTPError as e:
|
||
if e.code in (401, 403):
|
||
raise SearchAuthError(
|
||
f"Search Auth-Fehler HTTP {e.code} auf {endpoint}",
|
||
RC_AUTH_FAILURE)
|
||
if e.code >= 500:
|
||
raise SearchUnavailableError(
|
||
f"Search HTTP {e.code} auf {endpoint}", RC_SEARCH_REBUILD_FAILURE)
|
||
raise SearchRebuildError(
|
||
f"Search HTTP {e.code} auf {endpoint}: {e.read().decode('utf-8', 'replace')[:200]}",
|
||
RC_SEARCH_REBUILD_FAILURE)
|
||
except (urllib.error.URLError, TimeoutError, OSError) as e:
|
||
raise SearchUnavailableError(
|
||
f"Search nicht erreichbar ({endpoint}): {e}", RC_SEARCH_REBUILD_FAILURE)
|
||
|
||
def _get(self, endpoint: str) -> Dict[str, Any]:
|
||
url = f"{self.base_url}/{endpoint.lstrip('/')}"
|
||
req = urllib.request.Request(url, method="GET")
|
||
try:
|
||
with urllib.request.urlopen(req, timeout=self.timeout) as resp:
|
||
body = resp.read().decode("utf-8")
|
||
return json.loads(body) if body else {}
|
||
except urllib.error.HTTPError as e:
|
||
if e.code >= 500:
|
||
raise SearchUnavailableError(
|
||
f"Search HTTP {e.code} auf {endpoint}", RC_SEARCH_REBUILD_FAILURE)
|
||
raise SearchRebuildError(
|
||
f"Search HTTP {e.code} auf {endpoint}: {e.read().decode('utf-8', 'replace')[:200]}",
|
||
RC_SEARCH_REBUILD_FAILURE)
|
||
except (urllib.error.URLError, TimeoutError, OSError) as e:
|
||
raise SearchUnavailableError(
|
||
f"Search nicht erreichbar ({endpoint}): {e}", RC_SEARCH_REBUILD_FAILURE)
|
||
|
||
# -- Rebuild (NUR C5D-Writer) -------------------------------------------
|
||
|
||
def rebuild(self) -> Dict[str, Any]:
|
||
"""Fuehrt einen Search Full Rebuild aus (POST /api/search/rebuild)."""
|
||
resp = self._post("rebuild")
|
||
if not isinstance(resp, dict):
|
||
raise SearchMalformedResponseError(
|
||
f"Search-Rebuild-Antwort ungueltig: {type(resp).__name__}",
|
||
RC_SEARCH_REBUILD_FAILURE)
|
||
if resp.get("status") != REBUILD_OK:
|
||
raise SearchRebuildError(
|
||
f"Search-Rebuild fehlgeschlagen: {resp.get('error', resp)}",
|
||
RC_SEARCH_REBUILD_FAILURE)
|
||
return resp
|
||
|
||
# -- Health (Read-Back) -------------------------------------------------
|
||
|
||
def health(self) -> Dict[str, Any]:
|
||
"""Liest Search Health/Integrity (GET /api/search/health)."""
|
||
resp = self._get("health")
|
||
if not isinstance(resp, dict):
|
||
raise SearchMalformedResponseError(
|
||
f"Search-Health-Antwort ungueltig: {type(resp).__name__}",
|
||
RC_SEARCH_REBUILD_FAILURE)
|
||
return resp
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Search Source Builder (deterministisch, §3–§6)
|
||
# ---------------------------------------------------------------------------
|
||
|
||
class SearchSourceBuilder:
|
||
"""
|
||
Baut deterministisch den Search-Source-Snapshot AUS DEM AKTUELLEN
|
||
TOLARIA-VAULT-ZUSTAND (read-only) — exakt im Schema, das
|
||
TolariaSearch.rebuild_from_source() erwartet.
|
||
|
||
ARCHITEKTURREGEL:
|
||
* Search darf NUR einen Stand indexieren, der aus dem bereits
|
||
erfolgreich verifizierten Tolaria-Zustand erzeugt wurde.
|
||
* index_source.json ist KEINE Source of Truth, sondern ein
|
||
EPHEMERAL / REBUILDABLE / DERIVED BUILD ARTIFACT.
|
||
* Dieser Builder liest NUR Tolaria (kein Write nach Tolaria/Forgejo).
|
||
|
||
Bestimmt die "erwartete indexierbare Menge" deterministisch ueber die
|
||
C5B-KnowledgeScope-Klassifikation:
|
||
* IN_SCOPE + gueltige object_id -> indexierbar (object_id gesetzt)
|
||
* LEGACY_SPECIAL (README.md x2, vps.md) -> indexierbar (object_id=None)
|
||
* OUT_OF_SCOPE / HUMAN_REVIEW (keine gueltige object_id, z.B. Orphan
|
||
start.md) -> NICHT indexierbar -> aus Source ausgeschlossen.
|
||
|
||
Atomicity: build temp -> validate temp -> fsync -> os.replace (atomic
|
||
rename) -> erst dann Rebuild. Ein Crash hinterlaesst nie eine halb
|
||
geschriebene Source.
|
||
"""
|
||
|
||
def __init__(self, tolaria: TolariaClient, source_path: str,
|
||
scope: Optional[KnowledgeScope] = None):
|
||
self.tolaria = tolaria
|
||
self.source_path = source_path
|
||
self.scope = scope or KnowledgeScope(VAULT_PREFIX)
|
||
|
||
@staticmethod
|
||
def _relpath(vault_path: str) -> str:
|
||
"""Strip des /app/vault-Praefixes -> repo-relative Pfad."""
|
||
p = vault_path.replace("\\", "/")
|
||
prefix = VAULT_PREFIX.rstrip("/") + "/"
|
||
if p.startswith(prefix):
|
||
return p[len(prefix):]
|
||
if p == VAULT_PREFIX.rstrip("/"):
|
||
return ""
|
||
return p
|
||
|
||
def _collect_indexable(self) -> Dict[str, Any]:
|
||
"""
|
||
Liest den Vault (read-only) und klassifiziert die indexierbaren Objekte.
|
||
Rueckgabe: {"indexable": [{path, id, content, fm, body}],
|
||
"excluded": [...], "vault_object_count": int}
|
||
"""
|
||
entries = self.tolaria.list(VAULT_PREFIX) or []
|
||
indexable: List[Dict[str, Any]] = []
|
||
excluded: List[Dict[str, Any]] = []
|
||
for e in entries:
|
||
vp = e.get("path")
|
||
if not vp:
|
||
continue
|
||
rel = self._relpath(vp)
|
||
content = self.tolaria.read(vp)
|
||
if content is None:
|
||
excluded.append({"path": rel, "reason": "unreadable"})
|
||
continue
|
||
cls = self.scope.classify_with_content(rel, content)
|
||
sc = cls.get("scope")
|
||
if sc == SCOPE_IN_SCOPE:
|
||
oid = cls.get("object_id")
|
||
if oid:
|
||
indexable.append({"path": rel, "id": oid, "content": content})
|
||
continue
|
||
excluded.append({"path": rel, "reason": "no_object_id"})
|
||
elif sc == SCOPE_LEGACY_SPECIAL:
|
||
indexable.append({"path": rel, "id": None, "content": content})
|
||
else:
|
||
excluded.append({"path": rel, "reason": sc})
|
||
return {"indexable": indexable, "excluded": excluded,
|
||
"vault_object_count": len(entries)}
|
||
|
||
def _source_objects(self, indexable: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
|
||
"""Konvertiert indexierbare Vault-Objekte ins Source-Schema (deterministisch)."""
|
||
objs: List[Dict[str, Any]] = []
|
||
for it in indexable:
|
||
fm, body = parse_frontmatter(it["content"])
|
||
rel = it["path"]
|
||
objs.append({
|
||
"path": rel,
|
||
"title": fm.get("title") or rel.rsplit("/", 1)[-1].replace(".md", ""),
|
||
"id": it["id"],
|
||
"type": fm.get("type"),
|
||
"role": fm.get("role"),
|
||
"representation": fm.get("representation"),
|
||
"state": fm.get("state"),
|
||
"knowledge_schema": fm.get("knowledge_schema"),
|
||
"content_hash": content_hash(body),
|
||
"body": body,
|
||
"aliases": [],
|
||
"tags": fm.get("tags", []) or [],
|
||
"derived_from": fm.get("derived_from"),
|
||
})
|
||
# Deterministische Reihenfolge (Pfad-sortiert) — kein Hash-/Set-Chaos.
|
||
objs.sort(key=lambda o: o["path"])
|
||
return objs
|
||
|
||
def compute_expected(self, commit_sha: str) -> Dict[str, Any]:
|
||
"""
|
||
Read-only: bestimmt die erwartete indexierbare Objekt-Menge (object_ids
|
||
und Pfade) aus dem aktuellen Tolaria-Stand. Kein Write, kein Rebuild.
|
||
"""
|
||
coll = self._collect_indexable()
|
||
objs = self._source_objects(coll["indexable"])
|
||
expected_ids = sorted(o["id"] for o in objs if o["id"] is not None)
|
||
expected_paths = sorted(o["path"] for o in objs)
|
||
return {
|
||
"commit_sha": commit_sha,
|
||
"source_head": commit_sha,
|
||
"expected_object_count": len(objs),
|
||
"expected_object_ids": expected_ids,
|
||
"expected_paths": expected_paths,
|
||
"vault_object_count": coll["vault_object_count"],
|
||
"excluded": coll["excluded"],
|
||
}
|
||
|
||
def build(self, commit_sha: str, dest_path: Optional[str] = None) -> Dict[str, Any]:
|
||
"""
|
||
Erzeugt den Source-Snapshot atomar und verifiziert ihn vollstaendig.
|
||
Rueckgabe: {"ok": bool, "checks": {...}, "source_path": str,
|
||
"expected": {...}, "reason": optional}
|
||
"""
|
||
dest = dest_path or self.source_path
|
||
coll = self._collect_indexable()
|
||
objs = self._source_objects(coll["indexable"])
|
||
expected_ids = sorted(o["id"] for o in objs if o["id"] is not None)
|
||
expected_paths = sorted(o["path"] for o in objs)
|
||
|
||
source_doc = {
|
||
"head": commit_sha,
|
||
"count": len(objs),
|
||
"objects": objs,
|
||
}
|
||
|
||
# --- Verifikation des gebauten Snapshots (Completeness, §5) --------
|
||
ids = [o["id"] for o in objs if o["id"] is not None]
|
||
paths = [o["path"] for o in objs]
|
||
checks = {
|
||
"source_object_count": len(objs),
|
||
"unique_object_ids": len(set(ids)) == len(ids),
|
||
"unique_paths": len(set(paths)) == len(paths),
|
||
"duplicate_ids": len(ids) - len(set(ids)),
|
||
"duplicate_paths": len(paths) - len(set(paths)),
|
||
"invalid_objects": sum(1 for o in objs if not o.get("path")),
|
||
"expected_source_head": source_doc["head"] == commit_sha,
|
||
"expected_object_count_match": len(objs) == len(expected_ids) + sum(
|
||
1 for o in objs if o["id"] is None),
|
||
}
|
||
# Secret-Scan fail-closed (Pre-Build-Pruefung, keine Werte persistieren)
|
||
secrets = [o["path"] for o in objs if detect_secret(o["body"] or "")]
|
||
checks["secret_blocked"] = secrets
|
||
|
||
# ok = alle Bool-Checks True UND alle Fehler-Zaehler == 0.
|
||
# source_object_count ist eine legitime Zaehlung (kein Fehlerindikator)
|
||
# und wird NICHT als 0-bedingung gewertet — sonst waere jeder
|
||
# nicht-leere Source-Build faelschlich ok=False.
|
||
ok = all(v is True for k, v in checks.items()
|
||
if k not in ("secret_blocked", "source_object_count")
|
||
and isinstance(v, bool)) \
|
||
and all(isinstance(v, int) and v == 0 for k, v in checks.items()
|
||
if k in ("duplicate_ids", "duplicate_paths", "invalid_objects")) \
|
||
and len(secrets) == 0
|
||
|
||
# --- Atomare Veröffentlichung (temp -> validate -> fsync -> replace) --
|
||
written = False
|
||
if ok:
|
||
dest_dir = os.path.dirname(os.path.abspath(dest))
|
||
os.makedirs(dest_dir, exist_ok=True)
|
||
import tempfile
|
||
fd, tmp = tempfile.mkstemp(prefix=".source_build_",
|
||
suffix=".json", dir=dest_dir)
|
||
try:
|
||
with os.fdopen(fd, "w", encoding="utf-8") as f:
|
||
json.dump(source_doc, f, ensure_ascii=False)
|
||
f.flush()
|
||
os.fsync(f.fileno())
|
||
os.replace(tmp, dest) # atomic rename
|
||
written = True
|
||
finally:
|
||
if os.path.exists(tmp):
|
||
try:
|
||
os.remove(tmp)
|
||
except OSError:
|
||
pass
|
||
|
||
return {
|
||
"ok": ok and written,
|
||
"written": written,
|
||
"source_path": dest,
|
||
"checks": checks,
|
||
"expected": {
|
||
"source_head": commit_sha,
|
||
"expected_object_ids": expected_ids,
|
||
"expected_paths": expected_paths,
|
||
"vault_object_count": coll["vault_object_count"],
|
||
},
|
||
"excluded": coll["excluded"],
|
||
"reason": None if (ok and written) else "Source-Verifikation fehlgeschlagen",
|
||
}
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Integrity-Verifikation (Read-Back, §C5D)
|
||
# ---------------------------------------------------------------------------
|
||
|
||
def verify_integrity(health: Dict[str, Any],
|
||
expected_object_ids: Optional[set] = None,
|
||
expected_paths: Optional[set] = None) -> Dict[str, Any]:
|
||
"""
|
||
Verifiziert Search Health/Integrity per Read-Back.
|
||
|
||
Ein HTTP 200 allein reicht NICHT als Erfolg. Health muss PASS sein:
|
||
* integrity_ok == True
|
||
* index_built == True
|
||
* object_count > 0 (unerwarteter/falscher Indexzustand -> FAIL)
|
||
* failed_objects leer (optional, aber fail-closed wenn nicht leer)
|
||
|
||
C5D-HAERTUNG (§7): Bei Angabe der erwarteten Mengen (expected_object_ids /
|
||
expected_paths) wird zusaetzlich deterministisch die EXAKTE Objekt-Set-
|
||
Gleichheit geprueft (nicht nur object_count). Ein HTTP-200-Rebuild mit
|
||
stale Source kann damit NIE APPLIED ermoeglichen. Der Canary-Nachweis
|
||
laeuft ueber die erwartete Objekt-Menge: Das Canary-Objekt ist Teil der
|
||
expected_object_ids; object_ids_exact==True beweist seine Praesenz im Index.
|
||
|
||
Rueckgabe: {"ok": bool, "checks": {...}, "reason": optional}
|
||
"""
|
||
if not isinstance(health, dict):
|
||
raise SearchMalformedResponseError(
|
||
f"Search-Health-Antwort ungueltig: {type(health).__name__}",
|
||
RC_SEARCH_REBUILD_FAILURE)
|
||
checks = {
|
||
"integrity_ok": bool(health.get("integrity_ok")),
|
||
"index_built": bool(health.get("index_built")),
|
||
"object_count_positive": int(health.get("object_count", 0)) > 0,
|
||
"failed_objects_empty": not bool(health.get("failed_objects")),
|
||
}
|
||
if expected_object_ids is not None:
|
||
indexed_ids = set(health.get("indexed_object_ids") or [])
|
||
checks["object_ids_exact"] = (indexed_ids == set(expected_object_ids))
|
||
if expected_paths is not None:
|
||
indexed_paths = set(health.get("indexed_paths") or [])
|
||
checks["paths_exact"] = (indexed_paths == set(expected_paths))
|
||
# Jedes indexierte Objekt hat genau EINEN Pfad (object_id oder legacy).
|
||
# object_count muss also exakt der erwarteten Pfad-Anzahl entsprechen.
|
||
checks["object_count_matches_expected"] = (
|
||
int(health.get("object_count", 0)) == len(set(expected_paths)))
|
||
ok = all(checks.values())
|
||
reason = None
|
||
if not ok:
|
||
failed = [k for k, v in checks.items() if not v]
|
||
reason = "Search-Health nicht PASS: " + ", ".join(failed)
|
||
return {"ok": ok, "checks": checks, "reason": reason}
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# C5D Engine (Search Integration + Commit Completion)
|
||
# ---------------------------------------------------------------------------
|
||
|
||
class C5DEngine:
|
||
"""
|
||
Vervollstaendigt einen Commit nach erfolgreicher Tolaria-Propagation (C5C).
|
||
|
||
Startet NUR aus ST_UPDATING_SEARCH (Tolaria verifiziert, READY_FOR_SEARCH)
|
||
oder ST_RETRY_PENDING (Search-Retry-Replay). Fuehrt Search-Rebuild aus,
|
||
verifiziert Health/Integrity per Read-Back, und setzt bei vollem PASS
|
||
ST_APPLIED + last_applied_commit.
|
||
|
||
C5D ruft NIE Tolaria auf -> kein Tolaria-Doppel-Write beim Search-Retry.
|
||
"""
|
||
|
||
def __init__(
|
||
self,
|
||
store: C5AStore,
|
||
search: Optional[SearchClient] = None,
|
||
source_builder: Optional[SearchSourceBuilder] = None,
|
||
max_retries: int = DEFAULT_MAX_RETRIES,
|
||
backoff_seconds: Optional[List[int]] = None,
|
||
):
|
||
self.store = store
|
||
self.search = search or SearchClient()
|
||
self.source_builder = source_builder
|
||
self.max_retries = max_retries
|
||
self.backoff_seconds = backoff_seconds or DEFAULT_BACKOFF_SECONDS
|
||
|
||
# -- Idempotenz ---------------------------------------------------------
|
||
|
||
def _commit_already_applied(self, commit_sha: str) -> bool:
|
||
"""Commit bereits vollstaendig APPLIED -> kein Downstream-Write."""
|
||
return self.store.commit_status(commit_sha) == ST_APPLIED
|
||
|
||
# -- Retry-Klassifikation (§17) -----------------------------------------
|
||
|
||
def _is_retryable(self, err: SearchError) -> bool:
|
||
"""Nur technische, retrybare Fehler nutzen das Retry-Modell."""
|
||
return isinstance(err, SearchUnavailableError)
|
||
|
||
def _handle_failure(self, commit_sha: str, err: SearchError) -> Dict[str, Any]:
|
||
"""Behandelt einen Search-Fehler: Retry, DEAD oder Human Gate."""
|
||
cur = self.store.commit_status(commit_sha)
|
||
if self._is_retryable(err):
|
||
retry = self.store.increment_retry(commit_sha)
|
||
# Sicherstellen, dass wir in RETRY_PENDING sind (erlaubt von
|
||
# UPDATING_SEARCH und VERIFYING_SEARCH), bevor wir ggf. zu DEAD wechseln.
|
||
if cur in (ST_UPDATING_SEARCH, ST_VERIFYING_SEARCH):
|
||
self.store.transition_commit(commit_sha, ST_RETRY_PENDING)
|
||
if retry >= self.max_retries:
|
||
self.store.set_commit_error(commit_sha, err.reason_code or RC_SEARCH_REBUILD_FAILURE, err.message)
|
||
self.store.transition_commit(commit_sha, ST_DEAD)
|
||
return {"commit_sha": commit_sha, "status": ST_DEAD,
|
||
"reason_code": err.reason_code, "retry_count": retry}
|
||
self.store.set_commit_error(commit_sha, err.reason_code or RC_SEARCH_REBUILD_FAILURE, err.message)
|
||
return {"commit_sha": commit_sha, "status": ST_RETRY_PENDING,
|
||
"reason_code": err.reason_code, "retry_count": retry}
|
||
# Nicht-retrybar -> Human Gate / Fail Closed
|
||
rc = err.reason_code or RC_SEARCH_REBUILD_FAILURE
|
||
self.store.set_commit_error(commit_sha, rc, err.message)
|
||
self.store.transition_commit(commit_sha, ST_HUMAN_REVIEW_REQUIRED)
|
||
return {"commit_sha": commit_sha, "status": ST_HUMAN_REVIEW_REQUIRED,
|
||
"reason_code": rc}
|
||
|
||
# -- Commit anwenden ----------------------------------------------------
|
||
|
||
def apply_commit(self, commit_sha: str) -> Dict[str, Any]:
|
||
"""
|
||
Vervollstaendigt einen Commit: Search-Rebuild -> Health/Integrity -> APPLIED.
|
||
|
||
VERBINDLICHE REIHENFOLGE:
|
||
UPDATING_SEARCH -> (Search Rebuild) -> VERIFYING_SEARCH
|
||
-> (Health/Integrity Read-Back) -> APPLIED -> last_applied_commit
|
||
|
||
FAIL-CLOSED: Bei jedem Search-Fehler bleibt der Commit NICHT APPLIED und
|
||
last_applied_commit unveraendert.
|
||
"""
|
||
commit = self.store.get_commit(commit_sha)
|
||
if commit is None:
|
||
return {"commit_sha": commit_sha, "status": ST_HUMAN_REVIEW_REQUIRED,
|
||
"reason_code": RC_UNKNOWN_OBJECT_ID}
|
||
|
||
cur = commit.get("status")
|
||
|
||
# Idempotenz: bereits APPLIED -> kein Downstream-Write
|
||
if cur == ST_APPLIED:
|
||
return {"commit_sha": commit_sha, "status": ST_APPLIED,
|
||
"idempotency": IDEM_ALREADY_APPLIED}
|
||
|
||
# Suchschritt: erlaubt UPDATING_SEARCH, RETRY_PENDING (Search-Retry) und
|
||
# VERIFYING_SEARCH (C5E-Replay/Recovery nach Crash "nach Rebuild vor Health").
|
||
if cur not in (ST_UPDATING_SEARCH, ST_RETRY_PENDING, ST_VERIFYING_SEARCH):
|
||
return {"commit_sha": commit_sha, "status": cur,
|
||
"message": "Commit nicht im Search-Schritt (Tolaria nicht verifiziert)"}
|
||
|
||
# VERIFYING_SEARCH-Resume: Rebuild lief bereits erfolgreich -> KEIN neuer
|
||
# Rebuild, KEIN Source-Build. Aber die harte Objekt-Set-Verifikation MUSS
|
||
# trotzdem laufen (ein stale-Rebuild darf nie APPLIED ermoeglichen).
|
||
resume_after_rebuild = (cur == ST_VERIFYING_SEARCH)
|
||
|
||
# Search-Retry-Replay: RETRY_PENDING -> UPDATING_SEARCH (kein Tolaria-Write)
|
||
if cur == ST_RETRY_PENDING:
|
||
self.store.transition_commit(commit_sha, ST_UPDATING_SEARCH)
|
||
resume_after_rebuild = False
|
||
|
||
build_ctx = None
|
||
rebuild = None
|
||
if not resume_after_rebuild:
|
||
# 0) Search Source Build & Verification (C5D, §3–§7)
|
||
# Deterministisch AUS DEM AKTUELLEN TOLARIA-VAULT-ZUSTAND (read-only),
|
||
# exakt im Schema, das rebuild_from_source erwartet. Kein Rebuild auf
|
||
# stale Source: erst Source-Snapshot bauen + verifizieren, dann Rebuild.
|
||
if self.source_builder is not None:
|
||
try:
|
||
build_ctx = self.source_builder.build(commit_sha)
|
||
except TolariaUnavailableError as e:
|
||
# Tolaria transient down -> retrybar (analog SearchUnavailableError)
|
||
return self._handle_failure(
|
||
commit_sha,
|
||
SearchUnavailableError(
|
||
f"Tolaria beim Source-Build nicht erreichbar: {e}",
|
||
RC_SEARCH_REBUILD_FAILURE))
|
||
except Exception as e: # deterministischer Fehlschlag -> Human Gate
|
||
return self._handle_failure(
|
||
commit_sha,
|
||
SearchSourceBuildError(
|
||
f"Search-Source-Build fehlgeschlagen: {e}",
|
||
RC_SEARCH_SOURCE_BUILD_FAILURE))
|
||
if not build_ctx.get("ok"):
|
||
# FAIL CLOSED: Source-Verifikation nicht bestanden -> KEIN Rebuild
|
||
reason = build_ctx.get("reason") or "Source-Verifikation fehlgeschlagen"
|
||
return self._handle_failure(
|
||
commit_sha,
|
||
SearchSourceBuildError(
|
||
f"Search-Source nicht verifiziert: {reason}",
|
||
RC_SEARCH_SOURCE_BUILD_FAILURE))
|
||
|
||
# 1) Search Full Rebuild (UPDATING_SEARCH)
|
||
try:
|
||
rebuild = self.search.rebuild()
|
||
except SearchError as e:
|
||
return self._handle_failure(commit_sha, e)
|
||
|
||
# Defense in depth: HTTP 200 allein reicht NICHT. Die Engine validiert
|
||
# den Rebuild-Response selbst (status == "ok"), unabhaengig vom Client.
|
||
if not isinstance(rebuild, dict) or rebuild.get("status") != REBUILD_OK:
|
||
return self._handle_failure(
|
||
commit_sha,
|
||
SearchRebuildError(
|
||
f"Search-Rebuild-Response ungueltig: {rebuild!r}",
|
||
RC_SEARCH_REBUILD_FAILURE))
|
||
|
||
# 2) Rebuild OK -> VERIFYING_SEARCH
|
||
self.store.transition_commit(commit_sha, ST_VERIFYING_SEARCH)
|
||
else:
|
||
# VERIFYING_SEARCH-Resume: Rebuild ist erfolgt. Fuer die harte
|
||
# Objekt-Set-Verifikation bestimmen wir die erwartete Menge erneut
|
||
# (read-only, kein Write) — ein stale/unvollstaendiger Index wird so
|
||
# auch nach Rebuild+Crash noch FAIL CLOSED zurueckgewiesen.
|
||
if self.source_builder is not None:
|
||
try:
|
||
exp = self.source_builder.compute_expected(commit_sha)
|
||
build_ctx = {"ok": True, "expected": {
|
||
"expected_object_ids": exp["expected_object_ids"],
|
||
"expected_paths": exp["expected_paths"]}}
|
||
except Exception as e:
|
||
return self._handle_failure(
|
||
commit_sha,
|
||
SearchSourceBuildError(
|
||
f"Erwartetes Objekt-Set fuer VERIFYING_SEARCH-Resume "
|
||
f"nicht bestimmbar: {e}",
|
||
RC_SEARCH_SOURCE_BUILD_FAILURE))
|
||
|
||
# 3) Health/Integrity Read-Back (VERIFYING_SEARCH)
|
||
# C5D-Haertung (§7): exaktes Objekt-Set statt object_count>0.
|
||
# Ein HTTP-200-Rebuild mit stale Source kann NIE APPLIED ermoeglichen.
|
||
try:
|
||
health = self.search.health()
|
||
if build_ctx is not None:
|
||
exp = build_ctx["expected"]
|
||
integrity = verify_integrity(
|
||
health,
|
||
expected_object_ids=set(exp["expected_object_ids"]),
|
||
expected_paths=set(exp["expected_paths"]))
|
||
else:
|
||
integrity = verify_integrity(health)
|
||
except SearchError as e:
|
||
return self._handle_failure(commit_sha, e)
|
||
|
||
if not integrity["ok"]:
|
||
return self._handle_failure(
|
||
commit_sha,
|
||
SearchIntegrityError(integrity["reason"] or "Search-Integrity nicht PASS",
|
||
RC_SEARCH_REBUILD_FAILURE))
|
||
|
||
# 4) Vollstaendiger PASS -> APPLIED + last_applied_commit fortschreiben
|
||
self.store.transition_commit(commit_sha, ST_APPLIED)
|
||
self.store.mark_applied(commit_sha)
|
||
|
||
return {"commit_sha": commit_sha, "status": ST_APPLIED,
|
||
"idempotency": IDEM_RETRY_SAFE, "integrity": integrity,
|
||
"rebuild": rebuild, "source_build": build_ctx}
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# C5D Dry-Run (read-only) — zeigt, welche Commits fuer Search bereit sind
|
||
# ---------------------------------------------------------------------------
|
||
|
||
class C5DDryRun:
|
||
"""
|
||
Read-only Dry-Run: identifiziert Commits in UPDATING_SEARCH (READY_FOR_SEARCH)
|
||
und RETRY_PENDING (Search-Retry-Replay). Fuehrt KEINEN Search-Rebuild aus.
|
||
"""
|
||
|
||
def __init__(self, store: C5AStore):
|
||
self.store = store
|
||
|
||
def plan(self) -> Dict[str, Any]:
|
||
"""Erzeugt den C5D-Plan (read-only, KEIN Search-Rebuild)."""
|
||
ready = []
|
||
retry = []
|
||
for c in self.store.list_commits():
|
||
status = c.get("status")
|
||
if status == ST_UPDATING_SEARCH:
|
||
ready.append(c.get("commit_sha"))
|
||
elif status == ST_RETRY_PENDING:
|
||
retry.append(c.get("commit_sha"))
|
||
return {
|
||
"ready_for_search": ready,
|
||
"search_retry_pending": retry,
|
||
"note": "read-only Dry-Run: KEIN produktiver Search-Rebuild",
|
||
}
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Guarantees (statische Pruefung)
|
||
# ---------------------------------------------------------------------------
|
||
|
||
def _collect_docstrings(tree: Any) -> set:
|
||
"""Sammelt alle Docstring-Strings (Modul-/Funktions-/Klassen-Docstrings)."""
|
||
import ast
|
||
docstrings = set()
|
||
for node in ast.walk(tree):
|
||
if isinstance(node, (ast.Module, ast.FunctionDef, ast.AsyncFunctionDef,
|
||
ast.ClassDef)):
|
||
body = node.body
|
||
if body and isinstance(body[0], ast.Expr):
|
||
val = body[0].value
|
||
if isinstance(val, ast.Constant) and isinstance(val.value, str):
|
||
docstrings.add(val.value)
|
||
return docstrings
|
||
|
||
|
||
def assert_no_tolaria_write() -> Dict[str, Any]:
|
||
"""
|
||
Beweist statisch, dass C5D NICHT nach Tolaria schreibt (kein /api/vault/save).
|
||
|
||
Prueft, dass keine Tolaria-Write-Endpoint-Strings in echten Code-Ausdruecken
|
||
(nicht Docstrings/Kommentaren) vorkommen.
|
||
"""
|
||
import ast
|
||
this_file = Path(__file__).resolve()
|
||
tree = ast.parse(this_file.read_text(encoding="utf-8"))
|
||
docstrings = _collect_docstrings(tree)
|
||
# Fragmentierte banned-Strings (Selbstreferenz vermeiden)
|
||
banned = ["/api/" + "vault/" + "save", "vault/" + "save", "api/" + "vault"]
|
||
found = []
|
||
for node in ast.walk(tree):
|
||
if isinstance(node, ast.Constant) and isinstance(node.value, str):
|
||
if node.value in docstrings:
|
||
continue
|
||
for b in banned:
|
||
if b in node.value:
|
||
found.append(node.value)
|
||
return {
|
||
"no_tolaria_write": len(found) == 0,
|
||
"tolaria_write_endpoints_found": sorted(set(found)),
|
||
}
|
||
|
||
|
||
def assert_no_master_write() -> Dict[str, Any]:
|
||
"""
|
||
Beweist statisch, dass C5D KEIN git push / Forgejo-Write ausfuehrt.
|
||
"""
|
||
import ast
|
||
this_file = Path(__file__).resolve()
|
||
tree = ast.parse(this_file.read_text(encoding="utf-8"))
|
||
docstrings = _collect_docstrings(tree)
|
||
banned_imports = {"subprocess", "os.system", "git"}
|
||
found_imports = set()
|
||
for node in ast.walk(tree):
|
||
if isinstance(node, ast.Import):
|
||
for alias in node.names:
|
||
root = alias.name.split(".")[0]
|
||
if root in banned_imports:
|
||
found_imports.add(root)
|
||
elif isinstance(node, ast.ImportFrom):
|
||
if node.module:
|
||
root = node.module.split(".")[0]
|
||
if root in banned_imports:
|
||
found_imports.add(root)
|
||
banned_calls = ["git " + "pus" + "h", "git " + "comm" + "it", "git " + "ad" + "d", "pus" + "h"]
|
||
found_calls = []
|
||
for node in ast.walk(tree):
|
||
if isinstance(node, ast.Constant) and isinstance(node.value, str):
|
||
if node.value in docstrings:
|
||
continue
|
||
for b in banned_calls:
|
||
if b in node.value:
|
||
found_calls.append(node.value)
|
||
return {
|
||
"no_master_write": len(found_imports) == 0 and len(found_calls) == 0,
|
||
"banned_imports_found": sorted(found_imports),
|
||
"write_commands_found": sorted(set(found_calls)),
|
||
}
|
||
|
||
|
||
def assert_no_production_activation() -> Dict[str, Any]:
|
||
"""
|
||
Beweist statisch, dass C5D KEINE produktive Aktivierung enthaelt
|
||
(kein Daemon/Deploy/Canary/Thread/Cron/Self-Schedule).
|
||
"""
|
||
import ast
|
||
this_file = Path(__file__).resolve()
|
||
tree = ast.parse(this_file.read_text(encoding="utf-8"))
|
||
docstrings = _collect_docstrings(tree)
|
||
banned = ["daem" + "on", "serve_" + "forever", "Threading" + "HTTPServer",
|
||
"cro" + "n", "sched" + "ule", "depl" + "oy", "can" + "ary",
|
||
"threading." + "Thread", "while " + "True"]
|
||
found = []
|
||
for node in ast.walk(tree):
|
||
if isinstance(node, ast.Constant) and isinstance(node.value, str):
|
||
if node.value in docstrings:
|
||
continue
|
||
for b in banned:
|
||
if b in node.value:
|
||
found.append(node.value)
|
||
return {
|
||
"no_production_activation": len(found) == 0,
|
||
"activation_patterns_found": sorted(set(found)),
|
||
}
|