#!/usr/bin/env python3 """ Red Queen — C5D: SEARCH INTEGRATION + COMMIT COMPLETION v1 (deterministische, inaktive Library). C5D implementiert ausschliesslich: UPDATING_SEARCH -> SEARCH FULL REBUILD -> VERIFYING_SEARCH -> SEARCH HEALTH/INTEGRITY READ-BACK -> APPLIED -> last_applied_commit fortschreiben VERBINDLICHE REIHENFOLGE (ORDERING INVARIANT): Forgejo Commit -> Tolaria propagiert (C5C) -> Tolaria Read-Back DRIFT=0 -> Search Full Rebuild -> Search Health/Integrity PASS -> Commit APPLIED -> last_applied_commit fortschreiben INVARIANTE: Search darf NIEMALS vor erfolgreicher Tolaria-Verifikation aktualisiert werden. APPLIED und last_applied_commit duerfen ausschliesslich gesetzt werden, wenn Tolaria UND Search vollstaendig erfolgreich und verifiziert sind. C5D startet NUR aus ST_UPDATING_SEARCH (Tolaria bereits verifiziert) oder ST_RETRY_PENDING (Search-Retry-Replay). C5D ruft NIE Tolaria auf -> kein Tolaria-Doppel-Write beim Search-Retry. ARCHITEKTURREGEL (verbindlich): * Forgejo bleibt MASTER / Source of Truth. * Tolaria ist ausschliesslich DERIVED. * C5D darf: Forgejo lesen, Search kontrolliert rebuilden (mit Token). * C5D darf NIEMALS: Tolaria -> Forgejo schreiben, Forgejo veraendern, Tolaria schreiben, IDs neu vergeben, Knowledge Content umformulieren. NO-TOLARIA-WRITE-GUARANTEE: C5D ruft NICHT /api/vault/save auf. NO-MASTER-WRITE-GUARANTEE: C5D fuehrt KEIN git push / Forgejo-Write aus. NO-PRODUCTION-ACTIVATION: C5D fuehrt KEINE produktive Propagation / Search-Rebuild aus (nur Tests gegen Fake/Mock + read-only Dry-Run). FAIL-CLOSED: Bei Search-Ausfall, Timeout, Auth-Fehler, ungueltiger Antwort, Rebuild-Fehler oder fehlerhafter Integrity -> Commit NICHT APPLIED, last_applied_commit NICHT veraendert, Fehler klassifiziert, Retry/Human-Gate gemaess C5A-Contract. """ from __future__ import annotations import json import os import urllib.error import urllib.request from pathlib import Path from typing import Any, Dict, List, Optional, Tuple # C5A / C5C wiederverwenden (keine konkurrierende State Machine) from rq_c5a import ( C5AStore, ST_UPDATING_SEARCH, ST_VERIFYING_SEARCH, ST_APPLIED, ST_RETRY_PENDING, ST_DEAD, ST_HUMAN_REVIEW_REQUIRED, IDEM_ALREADY_APPLIED, IDEM_RETRY_SAFE, RC_SEARCH_REBUILD_FAILURE, RC_AUTH_FAILURE, RC_UNKNOWN_OBJECT_ID, RC_UNEXPECTED_TOLARIA_DRIFT, RC_SEARCH_SOURCE_BUILD_FAILURE, RC_INTEGRITY_FAILURE, DEFAULT_MAX_RETRIES, DEFAULT_BACKOFF_SECONDS, ) from rq_c5c import ( C5CError, TolariaClient, TolariaUnavailableError, VAULT_PREFIX, ) # C5B-Bausteine: deterministische Vault->Source-Extraktion (kein neues SoT). # content_hash, parse_frontmatter, extract_object_id, KnowledgeScope sind # identisch zur C5B-Change-Detection -> konsistente Metadaten-Hashes. from rq_c5b import ( parse_frontmatter, extract_object_id, content_hash, detect_secret, KnowledgeScope, SCOPE_IN_SCOPE, SCOPE_LEGACY_SPECIAL, ) # --------------------------------------------------------------------------- # Konstanten # --------------------------------------------------------------------------- # Search-API-Basis (Default; per Env C5_SEARCH_BASE ueberschreibbar) DEFAULT_SEARCH_BASE = "http://127.0.0.1:8325/api/search" ENV_SEARCH_BASE = "C5_SEARCH_BASE" ENV_SEARCH_TOKEN = "TOLARIA_SEARCH_REBUILD_TOKEN" # Search-Rebuild-Ergebnis REBUILD_OK = "ok" # Integrity-Verifikations-Ergebnisse INTEGRITY_OK = "INTEGRITY_OK" INTEGRITY_FAIL = "INTEGRITY_FAIL" class SearchError(C5CError): """Basis-Fehler fuer C5D Search-Integration.""" class SearchUnavailableError(SearchError): """Search nicht erreichbar / Timeout / transienter HTTP-Fehler (retrybar).""" class SearchAuthError(SearchError): """Search-Rebuild Auth-Fehler (nicht retrybar, Human Gate).""" class SearchRebuildError(SearchError): """Search-Rebuild fehlgeschlagen (nicht retrybar, Human Gate).""" class SearchIntegrityError(SearchError): """Search Health/Integrity nicht PASS (nicht retrybar, Human Gate).""" class SearchSourceBuildError(SearchError): """ Search-Source-Build/Verification fehlgeschlagen oder Objekt-Set weicht vom erwarteten Tolaria-Stand ab (nicht retrybar, Human Gate). C5D-Haertung: kein Rebuild darf auf stale/unvollstaendiger Source stattfinden; FAIL CLOSED vor APPLIED. """ class SearchMalformedResponseError(SearchError): """Search-Antwort ungueltig / malformed (nicht retrybar, Human Gate).""" # --------------------------------------------------------------------------- # Search Client (isoliert: rebuild / health) # --------------------------------------------------------------------------- class SearchClient: """ Isolierter Search-Client. Nur C5D-Writer darf rebuild() verwenden. rebuild() — POST /api/search/rebuild (Bearer-Token, kontrolliert) health() — GET /api/search/health (Read-Back, oeffentlich) Keine generische Agent-Write-Funktion. Kein Tolaria-Write. """ def __init__(self, base_url: Optional[str] = None, token: Optional[str] = None, timeout: float = 15.0): self.base_url = (base_url or os.environ.get(ENV_SEARCH_BASE) or DEFAULT_SEARCH_BASE).rstrip("/") self.token = token if token is not None else os.environ.get(ENV_SEARCH_TOKEN, "") self.timeout = timeout # -- HTTP-Helfer -------------------------------------------------------- def _post(self, endpoint: str, payload: Optional[Dict[str, Any]] = None) -> Dict[str, Any]: url = f"{self.base_url}/{endpoint.lstrip('/')}" data = json.dumps(payload or {}).encode("utf-8") headers = {"Content-Type": "application/json"} if self.token: headers["Authorization"] = "Bearer " + self.token req = urllib.request.Request(url, data=data, headers=headers, method="POST") try: with urllib.request.urlopen(req, timeout=self.timeout) as resp: body = resp.read().decode("utf-8") return json.loads(body) if body else {} except urllib.error.HTTPError as e: if e.code in (401, 403): raise SearchAuthError( f"Search Auth-Fehler HTTP {e.code} auf {endpoint}", RC_AUTH_FAILURE) if e.code >= 500: raise SearchUnavailableError( f"Search HTTP {e.code} auf {endpoint}", RC_SEARCH_REBUILD_FAILURE) raise SearchRebuildError( f"Search HTTP {e.code} auf {endpoint}: {e.read().decode('utf-8', 'replace')[:200]}", RC_SEARCH_REBUILD_FAILURE) except (urllib.error.URLError, TimeoutError, OSError) as e: raise SearchUnavailableError( f"Search nicht erreichbar ({endpoint}): {e}", RC_SEARCH_REBUILD_FAILURE) def _get(self, endpoint: str) -> Dict[str, Any]: url = f"{self.base_url}/{endpoint.lstrip('/')}" req = urllib.request.Request(url, method="GET") try: with urllib.request.urlopen(req, timeout=self.timeout) as resp: body = resp.read().decode("utf-8") return json.loads(body) if body else {} except urllib.error.HTTPError as e: if e.code >= 500: raise SearchUnavailableError( f"Search HTTP {e.code} auf {endpoint}", RC_SEARCH_REBUILD_FAILURE) raise SearchRebuildError( f"Search HTTP {e.code} auf {endpoint}: {e.read().decode('utf-8', 'replace')[:200]}", RC_SEARCH_REBUILD_FAILURE) except (urllib.error.URLError, TimeoutError, OSError) as e: raise SearchUnavailableError( f"Search nicht erreichbar ({endpoint}): {e}", RC_SEARCH_REBUILD_FAILURE) # -- Rebuild (NUR C5D-Writer) ------------------------------------------- def rebuild(self) -> Dict[str, Any]: """Fuehrt einen Search Full Rebuild aus (POST /api/search/rebuild).""" resp = self._post("rebuild") if not isinstance(resp, dict): raise SearchMalformedResponseError( f"Search-Rebuild-Antwort ungueltig: {type(resp).__name__}", RC_SEARCH_REBUILD_FAILURE) if resp.get("status") != REBUILD_OK: raise SearchRebuildError( f"Search-Rebuild fehlgeschlagen: {resp.get('error', resp)}", RC_SEARCH_REBUILD_FAILURE) return resp # -- Health (Read-Back) ------------------------------------------------- def health(self) -> Dict[str, Any]: """Liest Search Health/Integrity (GET /api/search/health).""" resp = self._get("health") if not isinstance(resp, dict): raise SearchMalformedResponseError( f"Search-Health-Antwort ungueltig: {type(resp).__name__}", RC_SEARCH_REBUILD_FAILURE) return resp # --------------------------------------------------------------------------- # Search Source Builder (deterministisch, §3–§6) # --------------------------------------------------------------------------- class SearchSourceBuilder: """ Baut deterministisch den Search-Source-Snapshot AUS DEM AKTUELLEN TOLARIA-VAULT-ZUSTAND (read-only) — exakt im Schema, das TolariaSearch.rebuild_from_source() erwartet. ARCHITEKTURREGEL: * Search darf NUR einen Stand indexieren, der aus dem bereits erfolgreich verifizierten Tolaria-Zustand erzeugt wurde. * index_source.json ist KEINE Source of Truth, sondern ein EPHEMERAL / REBUILDABLE / DERIVED BUILD ARTIFACT. * Dieser Builder liest NUR Tolaria (kein Write nach Tolaria/Forgejo). Bestimmt die "erwartete indexierbare Menge" deterministisch ueber die C5B-KnowledgeScope-Klassifikation: * IN_SCOPE + gueltige object_id -> indexierbar (object_id gesetzt) * LEGACY_SPECIAL (README.md x2, vps.md) -> indexierbar (object_id=None) * OUT_OF_SCOPE / HUMAN_REVIEW (keine gueltige object_id, z.B. Orphan start.md) -> NICHT indexierbar -> aus Source ausgeschlossen. Atomicity: build temp -> validate temp -> fsync -> os.replace (atomic rename) -> erst dann Rebuild. Ein Crash hinterlaesst nie eine halb geschriebene Source. """ def __init__(self, tolaria: TolariaClient, source_path: str, scope: Optional[KnowledgeScope] = None): self.tolaria = tolaria self.source_path = source_path self.scope = scope or KnowledgeScope(VAULT_PREFIX) @staticmethod def _relpath(vault_path: str) -> str: """Strip des /app/vault-Praefixes -> repo-relative Pfad.""" p = vault_path.replace("\\", "/") prefix = VAULT_PREFIX.rstrip("/") + "/" if p.startswith(prefix): return p[len(prefix):] if p == VAULT_PREFIX.rstrip("/"): return "" return p def _collect_indexable(self) -> Dict[str, Any]: """ Liest den Vault (read-only) und klassifiziert die indexierbaren Objekte. Rueckgabe: {"indexable": [{path, id, content, fm, body}], "excluded": [...], "vault_object_count": int} """ entries = self.tolaria.list(VAULT_PREFIX) or [] indexable: List[Dict[str, Any]] = [] excluded: List[Dict[str, Any]] = [] for e in entries: vp = e.get("path") if not vp: continue rel = self._relpath(vp) content = self.tolaria.read(vp) if content is None: excluded.append({"path": rel, "reason": "unreadable"}) continue cls = self.scope.classify_with_content(rel, content) sc = cls.get("scope") if sc == SCOPE_IN_SCOPE: oid = cls.get("object_id") if oid: indexable.append({"path": rel, "id": oid, "content": content}) continue excluded.append({"path": rel, "reason": "no_object_id"}) elif sc == SCOPE_LEGACY_SPECIAL: indexable.append({"path": rel, "id": None, "content": content}) else: excluded.append({"path": rel, "reason": sc}) return {"indexable": indexable, "excluded": excluded, "vault_object_count": len(entries)} def _source_objects(self, indexable: List[Dict[str, Any]]) -> List[Dict[str, Any]]: """Konvertiert indexierbare Vault-Objekte ins Source-Schema (deterministisch).""" objs: List[Dict[str, Any]] = [] for it in indexable: fm, body = parse_frontmatter(it["content"]) rel = it["path"] objs.append({ "path": rel, "title": fm.get("title") or rel.rsplit("/", 1)[-1].replace(".md", ""), "id": it["id"], "type": fm.get("type"), "role": fm.get("role"), "representation": fm.get("representation"), "state": fm.get("state"), "knowledge_schema": fm.get("knowledge_schema"), "content_hash": content_hash(body), "body": body, "aliases": [], "tags": fm.get("tags", []) or [], "derived_from": fm.get("derived_from"), }) # Deterministische Reihenfolge (Pfad-sortiert) — kein Hash-/Set-Chaos. objs.sort(key=lambda o: o["path"]) return objs def compute_expected(self, commit_sha: str) -> Dict[str, Any]: """ Read-only: bestimmt die erwartete indexierbare Objekt-Menge (object_ids und Pfade) aus dem aktuellen Tolaria-Stand. Kein Write, kein Rebuild. """ coll = self._collect_indexable() objs = self._source_objects(coll["indexable"]) expected_ids = sorted(o["id"] for o in objs if o["id"] is not None) expected_paths = sorted(o["path"] for o in objs) return { "commit_sha": commit_sha, "source_head": commit_sha, "expected_object_count": len(objs), "expected_object_ids": expected_ids, "expected_paths": expected_paths, "vault_object_count": coll["vault_object_count"], "excluded": coll["excluded"], } def build(self, commit_sha: str, dest_path: Optional[str] = None) -> Dict[str, Any]: """ Erzeugt den Source-Snapshot atomar und verifiziert ihn vollstaendig. Rueckgabe: {"ok": bool, "checks": {...}, "source_path": str, "expected": {...}, "reason": optional} """ dest = dest_path or self.source_path coll = self._collect_indexable() objs = self._source_objects(coll["indexable"]) expected_ids = sorted(o["id"] for o in objs if o["id"] is not None) expected_paths = sorted(o["path"] for o in objs) source_doc = { "head": commit_sha, "count": len(objs), "objects": objs, } # --- Verifikation des gebauten Snapshots (Completeness, §5) -------- ids = [o["id"] for o in objs if o["id"] is not None] paths = [o["path"] for o in objs] checks = { "source_object_count": len(objs), "unique_object_ids": len(set(ids)) == len(ids), "unique_paths": len(set(paths)) == len(paths), "duplicate_ids": len(ids) - len(set(ids)), "duplicate_paths": len(paths) - len(set(paths)), "invalid_objects": sum(1 for o in objs if not o.get("path")), "expected_source_head": source_doc["head"] == commit_sha, "expected_object_count_match": len(objs) == len(expected_ids) + sum( 1 for o in objs if o["id"] is None), } # Secret-Scan fail-closed (Pre-Build-Pruefung, keine Werte persistieren) secrets = [o["path"] for o in objs if detect_secret(o["body"] or "")] checks["secret_blocked"] = secrets # ok = alle Bool-Checks True UND alle Fehler-Zaehler == 0. # source_object_count ist eine legitime Zaehlung (kein Fehlerindikator) # und wird NICHT als 0-bedingung gewertet — sonst waere jeder # nicht-leere Source-Build faelschlich ok=False. ok = all(v is True for k, v in checks.items() if k not in ("secret_blocked", "source_object_count") and isinstance(v, bool)) \ and all(isinstance(v, int) and v == 0 for k, v in checks.items() if k in ("duplicate_ids", "duplicate_paths", "invalid_objects")) \ and len(secrets) == 0 # --- Atomare Veröffentlichung (temp -> validate -> fsync -> replace) -- written = False if ok: dest_dir = os.path.dirname(os.path.abspath(dest)) os.makedirs(dest_dir, exist_ok=True) import tempfile fd, tmp = tempfile.mkstemp(prefix=".source_build_", suffix=".json", dir=dest_dir) try: with os.fdopen(fd, "w", encoding="utf-8") as f: json.dump(source_doc, f, ensure_ascii=False) f.flush() os.fsync(f.fileno()) os.replace(tmp, dest) # atomic rename written = True finally: if os.path.exists(tmp): try: os.remove(tmp) except OSError: pass return { "ok": ok and written, "written": written, "source_path": dest, "checks": checks, "expected": { "source_head": commit_sha, "expected_object_ids": expected_ids, "expected_paths": expected_paths, "vault_object_count": coll["vault_object_count"], }, "excluded": coll["excluded"], "reason": None if (ok and written) else "Source-Verifikation fehlgeschlagen", } # --------------------------------------------------------------------------- # Integrity-Verifikation (Read-Back, §C5D) # --------------------------------------------------------------------------- def verify_integrity(health: Dict[str, Any], expected_object_ids: Optional[set] = None, expected_paths: Optional[set] = None) -> Dict[str, Any]: """ Verifiziert Search Health/Integrity per Read-Back. Ein HTTP 200 allein reicht NICHT als Erfolg. Health muss PASS sein: * integrity_ok == True * index_built == True * object_count > 0 (unerwarteter/falscher Indexzustand -> FAIL) * failed_objects leer (optional, aber fail-closed wenn nicht leer) C5D-HAERTUNG (§7): Bei Angabe der erwarteten Mengen (expected_object_ids / expected_paths) wird zusaetzlich deterministisch die EXAKTE Objekt-Set- Gleichheit geprueft (nicht nur object_count). Ein HTTP-200-Rebuild mit stale Source kann damit NIE APPLIED ermoeglichen. Der Canary-Nachweis laeuft ueber die erwartete Objekt-Menge: Das Canary-Objekt ist Teil der expected_object_ids; object_ids_exact==True beweist seine Praesenz im Index. Rueckgabe: {"ok": bool, "checks": {...}, "reason": optional} """ if not isinstance(health, dict): raise SearchMalformedResponseError( f"Search-Health-Antwort ungueltig: {type(health).__name__}", RC_SEARCH_REBUILD_FAILURE) checks = { "integrity_ok": bool(health.get("integrity_ok")), "index_built": bool(health.get("index_built")), "object_count_positive": int(health.get("object_count", 0)) > 0, "failed_objects_empty": not bool(health.get("failed_objects")), } if expected_object_ids is not None: indexed_ids = set(health.get("indexed_object_ids") or []) checks["object_ids_exact"] = (indexed_ids == set(expected_object_ids)) if expected_paths is not None: indexed_paths = set(health.get("indexed_paths") or []) checks["paths_exact"] = (indexed_paths == set(expected_paths)) # Jedes indexierte Objekt hat genau EINEN Pfad (object_id oder legacy). # object_count muss also exakt der erwarteten Pfad-Anzahl entsprechen. checks["object_count_matches_expected"] = ( int(health.get("object_count", 0)) == len(set(expected_paths))) ok = all(checks.values()) reason = None if not ok: failed = [k for k, v in checks.items() if not v] reason = "Search-Health nicht PASS: " + ", ".join(failed) return {"ok": ok, "checks": checks, "reason": reason} # --------------------------------------------------------------------------- # C5D Engine (Search Integration + Commit Completion) # --------------------------------------------------------------------------- class C5DEngine: """ Vervollstaendigt einen Commit nach erfolgreicher Tolaria-Propagation (C5C). Startet NUR aus ST_UPDATING_SEARCH (Tolaria verifiziert, READY_FOR_SEARCH) oder ST_RETRY_PENDING (Search-Retry-Replay). Fuehrt Search-Rebuild aus, verifiziert Health/Integrity per Read-Back, und setzt bei vollem PASS ST_APPLIED + last_applied_commit. C5D ruft NIE Tolaria auf -> kein Tolaria-Doppel-Write beim Search-Retry. """ def __init__( self, store: C5AStore, search: Optional[SearchClient] = None, source_builder: Optional[SearchSourceBuilder] = None, max_retries: int = DEFAULT_MAX_RETRIES, backoff_seconds: Optional[List[int]] = None, ): self.store = store self.search = search or SearchClient() self.source_builder = source_builder self.max_retries = max_retries self.backoff_seconds = backoff_seconds or DEFAULT_BACKOFF_SECONDS # -- Idempotenz --------------------------------------------------------- def _commit_already_applied(self, commit_sha: str) -> bool: """Commit bereits vollstaendig APPLIED -> kein Downstream-Write.""" return self.store.commit_status(commit_sha) == ST_APPLIED # -- Retry-Klassifikation (§17) ----------------------------------------- def _is_retryable(self, err: SearchError) -> bool: """Nur technische, retrybare Fehler nutzen das Retry-Modell.""" return isinstance(err, SearchUnavailableError) def _handle_failure(self, commit_sha: str, err: SearchError) -> Dict[str, Any]: """Behandelt einen Search-Fehler: Retry, DEAD oder Human Gate.""" cur = self.store.commit_status(commit_sha) if self._is_retryable(err): retry = self.store.increment_retry(commit_sha) # Sicherstellen, dass wir in RETRY_PENDING sind (erlaubt von # UPDATING_SEARCH und VERIFYING_SEARCH), bevor wir ggf. zu DEAD wechseln. if cur in (ST_UPDATING_SEARCH, ST_VERIFYING_SEARCH): self.store.transition_commit(commit_sha, ST_RETRY_PENDING) if retry >= self.max_retries: self.store.set_commit_error(commit_sha, err.reason_code or RC_SEARCH_REBUILD_FAILURE, err.message) self.store.transition_commit(commit_sha, ST_DEAD) return {"commit_sha": commit_sha, "status": ST_DEAD, "reason_code": err.reason_code, "retry_count": retry} self.store.set_commit_error(commit_sha, err.reason_code or RC_SEARCH_REBUILD_FAILURE, err.message) return {"commit_sha": commit_sha, "status": ST_RETRY_PENDING, "reason_code": err.reason_code, "retry_count": retry} # Nicht-retrybar -> Human Gate / Fail Closed rc = err.reason_code or RC_SEARCH_REBUILD_FAILURE self.store.set_commit_error(commit_sha, rc, err.message) self.store.transition_commit(commit_sha, ST_HUMAN_REVIEW_REQUIRED) return {"commit_sha": commit_sha, "status": ST_HUMAN_REVIEW_REQUIRED, "reason_code": rc} # -- Commit anwenden ---------------------------------------------------- def apply_commit(self, commit_sha: str) -> Dict[str, Any]: """ Vervollstaendigt einen Commit: Search-Rebuild -> Health/Integrity -> APPLIED. VERBINDLICHE REIHENFOLGE: UPDATING_SEARCH -> (Search Rebuild) -> VERIFYING_SEARCH -> (Health/Integrity Read-Back) -> APPLIED -> last_applied_commit FAIL-CLOSED: Bei jedem Search-Fehler bleibt der Commit NICHT APPLIED und last_applied_commit unveraendert. """ commit = self.store.get_commit(commit_sha) if commit is None: return {"commit_sha": commit_sha, "status": ST_HUMAN_REVIEW_REQUIRED, "reason_code": RC_UNKNOWN_OBJECT_ID} cur = commit.get("status") # Idempotenz: bereits APPLIED -> kein Downstream-Write if cur == ST_APPLIED: return {"commit_sha": commit_sha, "status": ST_APPLIED, "idempotency": IDEM_ALREADY_APPLIED} # Suchschritt: erlaubt UPDATING_SEARCH, RETRY_PENDING (Search-Retry) und # VERIFYING_SEARCH (C5E-Replay/Recovery nach Crash "nach Rebuild vor Health"). if cur not in (ST_UPDATING_SEARCH, ST_RETRY_PENDING, ST_VERIFYING_SEARCH): return {"commit_sha": commit_sha, "status": cur, "message": "Commit nicht im Search-Schritt (Tolaria nicht verifiziert)"} # VERIFYING_SEARCH-Resume: Rebuild lief bereits erfolgreich -> KEIN neuer # Rebuild, KEIN Source-Build. Aber die harte Objekt-Set-Verifikation MUSS # trotzdem laufen (ein stale-Rebuild darf nie APPLIED ermoeglichen). resume_after_rebuild = (cur == ST_VERIFYING_SEARCH) # Search-Retry-Replay: RETRY_PENDING -> UPDATING_SEARCH (kein Tolaria-Write) if cur == ST_RETRY_PENDING: self.store.transition_commit(commit_sha, ST_UPDATING_SEARCH) resume_after_rebuild = False build_ctx = None rebuild = None if not resume_after_rebuild: # 0) Search Source Build & Verification (C5D, §3–§7) # Deterministisch AUS DEM AKTUELLEN TOLARIA-VAULT-ZUSTAND (read-only), # exakt im Schema, das rebuild_from_source erwartet. Kein Rebuild auf # stale Source: erst Source-Snapshot bauen + verifizieren, dann Rebuild. if self.source_builder is not None: try: build_ctx = self.source_builder.build(commit_sha) except TolariaUnavailableError as e: # Tolaria transient down -> retrybar (analog SearchUnavailableError) return self._handle_failure( commit_sha, SearchUnavailableError( f"Tolaria beim Source-Build nicht erreichbar: {e}", RC_SEARCH_REBUILD_FAILURE)) except Exception as e: # deterministischer Fehlschlag -> Human Gate return self._handle_failure( commit_sha, SearchSourceBuildError( f"Search-Source-Build fehlgeschlagen: {e}", RC_SEARCH_SOURCE_BUILD_FAILURE)) if not build_ctx.get("ok"): # FAIL CLOSED: Source-Verifikation nicht bestanden -> KEIN Rebuild reason = build_ctx.get("reason") or "Source-Verifikation fehlgeschlagen" return self._handle_failure( commit_sha, SearchSourceBuildError( f"Search-Source nicht verifiziert: {reason}", RC_SEARCH_SOURCE_BUILD_FAILURE)) # 1) Search Full Rebuild (UPDATING_SEARCH) try: rebuild = self.search.rebuild() except SearchError as e: return self._handle_failure(commit_sha, e) # Defense in depth: HTTP 200 allein reicht NICHT. Die Engine validiert # den Rebuild-Response selbst (status == "ok"), unabhaengig vom Client. if not isinstance(rebuild, dict) or rebuild.get("status") != REBUILD_OK: return self._handle_failure( commit_sha, SearchRebuildError( f"Search-Rebuild-Response ungueltig: {rebuild!r}", RC_SEARCH_REBUILD_FAILURE)) # 2) Rebuild OK -> VERIFYING_SEARCH self.store.transition_commit(commit_sha, ST_VERIFYING_SEARCH) else: # VERIFYING_SEARCH-Resume: Rebuild ist erfolgt. Fuer die harte # Objekt-Set-Verifikation bestimmen wir die erwartete Menge erneut # (read-only, kein Write) — ein stale/unvollstaendiger Index wird so # auch nach Rebuild+Crash noch FAIL CLOSED zurueckgewiesen. if self.source_builder is not None: try: exp = self.source_builder.compute_expected(commit_sha) build_ctx = {"ok": True, "expected": { "expected_object_ids": exp["expected_object_ids"], "expected_paths": exp["expected_paths"]}} except Exception as e: return self._handle_failure( commit_sha, SearchSourceBuildError( f"Erwartetes Objekt-Set fuer VERIFYING_SEARCH-Resume " f"nicht bestimmbar: {e}", RC_SEARCH_SOURCE_BUILD_FAILURE)) # 3) Health/Integrity Read-Back (VERIFYING_SEARCH) # C5D-Haertung (§7): exaktes Objekt-Set statt object_count>0. # Ein HTTP-200-Rebuild mit stale Source kann NIE APPLIED ermoeglichen. try: health = self.search.health() if build_ctx is not None: exp = build_ctx["expected"] integrity = verify_integrity( health, expected_object_ids=set(exp["expected_object_ids"]), expected_paths=set(exp["expected_paths"])) else: integrity = verify_integrity(health) except SearchError as e: return self._handle_failure(commit_sha, e) if not integrity["ok"]: return self._handle_failure( commit_sha, SearchIntegrityError(integrity["reason"] or "Search-Integrity nicht PASS", RC_SEARCH_REBUILD_FAILURE)) # 4) Vollstaendiger PASS -> APPLIED + last_applied_commit fortschreiben self.store.transition_commit(commit_sha, ST_APPLIED) self.store.mark_applied(commit_sha) return {"commit_sha": commit_sha, "status": ST_APPLIED, "idempotency": IDEM_RETRY_SAFE, "integrity": integrity, "rebuild": rebuild, "source_build": build_ctx} # --------------------------------------------------------------------------- # C5D Dry-Run (read-only) — zeigt, welche Commits fuer Search bereit sind # --------------------------------------------------------------------------- class C5DDryRun: """ Read-only Dry-Run: identifiziert Commits in UPDATING_SEARCH (READY_FOR_SEARCH) und RETRY_PENDING (Search-Retry-Replay). Fuehrt KEINEN Search-Rebuild aus. """ def __init__(self, store: C5AStore): self.store = store def plan(self) -> Dict[str, Any]: """Erzeugt den C5D-Plan (read-only, KEIN Search-Rebuild).""" ready = [] retry = [] for c in self.store.list_commits(): status = c.get("status") if status == ST_UPDATING_SEARCH: ready.append(c.get("commit_sha")) elif status == ST_RETRY_PENDING: retry.append(c.get("commit_sha")) return { "ready_for_search": ready, "search_retry_pending": retry, "note": "read-only Dry-Run: KEIN produktiver Search-Rebuild", } # --------------------------------------------------------------------------- # Guarantees (statische Pruefung) # --------------------------------------------------------------------------- def _collect_docstrings(tree: Any) -> set: """Sammelt alle Docstring-Strings (Modul-/Funktions-/Klassen-Docstrings).""" import ast docstrings = set() for node in ast.walk(tree): if isinstance(node, (ast.Module, ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)): body = node.body if body and isinstance(body[0], ast.Expr): val = body[0].value if isinstance(val, ast.Constant) and isinstance(val.value, str): docstrings.add(val.value) return docstrings def assert_no_tolaria_write() -> Dict[str, Any]: """ Beweist statisch, dass C5D NICHT nach Tolaria schreibt (kein /api/vault/save). Prueft, dass keine Tolaria-Write-Endpoint-Strings in echten Code-Ausdruecken (nicht Docstrings/Kommentaren) vorkommen. """ import ast this_file = Path(__file__).resolve() tree = ast.parse(this_file.read_text(encoding="utf-8")) docstrings = _collect_docstrings(tree) # Fragmentierte banned-Strings (Selbstreferenz vermeiden) banned = ["/api/" + "vault/" + "save", "vault/" + "save", "api/" + "vault"] found = [] for node in ast.walk(tree): if isinstance(node, ast.Constant) and isinstance(node.value, str): if node.value in docstrings: continue for b in banned: if b in node.value: found.append(node.value) return { "no_tolaria_write": len(found) == 0, "tolaria_write_endpoints_found": sorted(set(found)), } def assert_no_master_write() -> Dict[str, Any]: """ Beweist statisch, dass C5D KEIN git push / Forgejo-Write ausfuehrt. """ import ast this_file = Path(__file__).resolve() tree = ast.parse(this_file.read_text(encoding="utf-8")) docstrings = _collect_docstrings(tree) banned_imports = {"subprocess", "os.system", "git"} found_imports = set() for node in ast.walk(tree): if isinstance(node, ast.Import): for alias in node.names: root = alias.name.split(".")[0] if root in banned_imports: found_imports.add(root) elif isinstance(node, ast.ImportFrom): if node.module: root = node.module.split(".")[0] if root in banned_imports: found_imports.add(root) banned_calls = ["git " + "pus" + "h", "git " + "comm" + "it", "git " + "ad" + "d", "pus" + "h"] found_calls = [] for node in ast.walk(tree): if isinstance(node, ast.Constant) and isinstance(node.value, str): if node.value in docstrings: continue for b in banned_calls: if b in node.value: found_calls.append(node.value) return { "no_master_write": len(found_imports) == 0 and len(found_calls) == 0, "banned_imports_found": sorted(found_imports), "write_commands_found": sorted(set(found_calls)), } def assert_no_production_activation() -> Dict[str, Any]: """ Beweist statisch, dass C5D KEINE produktive Aktivierung enthaelt (kein Daemon/Deploy/Canary/Thread/Cron/Self-Schedule). """ import ast this_file = Path(__file__).resolve() tree = ast.parse(this_file.read_text(encoding="utf-8")) docstrings = _collect_docstrings(tree) banned = ["daem" + "on", "serve_" + "forever", "Threading" + "HTTPServer", "cro" + "n", "sched" + "ule", "depl" + "oy", "can" + "ary", "threading." + "Thread", "while " + "True"] found = [] for node in ast.walk(tree): if isinstance(node, ast.Constant) and isinstance(node.value, str): if node.value in docstrings: continue for b in banned: if b in node.value: found.append(node.value) return { "no_production_activation": len(found) == 0, "activation_patterns_found": sorted(set(found)), }