Red Queen
ed374ac4ce
- C5A objects table: object_id nullable + reason_code column so HUMAN_REVIEW/ SECRET_DETECTED object changes (object_id=None) are persisted, not silently dropped (was: object_id TEXT NOT NULL, no reason_code field) - C5B poll_once: content_before now read from parent_sha (state BEFORE the change) instead of sha, so MODIFIED changes classify as CONTENT_UPDATE instead of being misclassified (Checker-Befund) - test_c5b: add test_modified_content_reads_parent regression test C5B 40/40, C5A 25/25, real-repo dry run: 233 object changes (109 IN_SCOPE with valid id, 124 HUMAN_REVIEW), idempotent.
859 lines
34 KiB
Python
859 lines
34 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
Red Queen — C5B: FORGEJO POLLING & CHANGE DETECTION ENGINE v1 (read-only).
|
|
|
|
VERBINDLICHER RAHMEN
|
|
====================
|
|
C5B ist die zweite Phase des C5 Sync Service (FORGEJO MASTER -> C5 SYNC SERVICE ->
|
|
TOLARIA DERIVED -> SEARCH FULL REBUILD). C5B baut AUSSCHLIESSLICH:
|
|
* FORGEJO READ (lokale Git-Clone-Analyse, read-only)
|
|
* COMMIT DISCOVERY (HEAD + Commit-Range)
|
|
* COMMIT ORDERING (chronologisch, parent-korrekt)
|
|
* DIFF / CHANGE DETECTION (ADDED/MODIFIED/DELETED/RENAMED/MOVED)
|
|
* OBJECT CHANGE CONTRACT (BEFORE/AFTER, Hashes, Operationen)
|
|
* Uebergabe an den C5A State Store (nur State/Persistence)
|
|
|
|
NOCH KEINE PROPAGATION. C5B propagiert NICHT nach Tolaria, rebuildet NICHT den
|
|
Search, schreibt NICHT nach Forgejo, startet KEINEN produktiven Poll-Daemon.
|
|
|
|
ABSOLUTES WRITE-VERBOT (C5B):
|
|
* KEIN Forgejo-Write / kein git push / kein git commit / kein git add.
|
|
* KEIN Tolaria-Write / kein POST /api/vault/save.
|
|
* KEIN Search-Rebuild / kein POST /api/search/rebuild.
|
|
* KEIN Container-Deploy, KEINE Netzwerk-Aenderung, KEINE Hermes-Rechte.
|
|
* KEIN produktiver Polling-Daemon (nur kontrollierte poll_once-Zyklen in Tests).
|
|
* KEIN C5C / C5D / C5-Deployment.
|
|
|
|
FORGEJO_READ_METHOD = A) lokale Git-Mirror/Clone-Analyse (bevorzugt).
|
|
* AUTH_REQUIREMENT = KEIN (read-only gegen lokalen Clone; kein Write-Token).
|
|
* NETWORK_DEPENDENCY = KEIN (lokale .git-Objekte; kein HTTP).
|
|
* Der GitReader fuehrt NUR read-only git-Befehle aus (strikte Whitelist).
|
|
|
|
SECURITY BOUNDARY (dokumentiert, NICHT implementiert):
|
|
* Forgejo credential: READ ONLY (C5B haelt KEINEN Forgejo-Write).
|
|
* Tolaria write: nur spaeter C5 Service (C5B schreibt NIE nach Tolaria).
|
|
* Search rebuild: nur C5 Service Token (C5B ruft NIE Search-Rebuild).
|
|
* Agents: keine direkten Tolaria-/Search-Admin-Writes.
|
|
* Netzwerk-Haertung wird spaeter beim Deployment umgesetzt, NICHT jetzt.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import re
|
|
import subprocess
|
|
import time
|
|
from pathlib import Path
|
|
from typing import Any, Dict, List, Optional, Tuple
|
|
|
|
from rq_c5a import (
|
|
C5AStore,
|
|
ST_DISCOVERED,
|
|
ST_VALIDATING,
|
|
ST_WAITING_FOR_PREDECESSOR,
|
|
ST_HUMAN_REVIEW_REQUIRED,
|
|
ST_APPLIED,
|
|
OP_CREATE,
|
|
OP_CONTENT_UPDATE,
|
|
OP_METADATA_UPDATE,
|
|
OP_STATE_UPDATE,
|
|
OP_RENAME,
|
|
OP_MOVE,
|
|
OP_SOURCE_CANONICAL_RELATION_UPDATE,
|
|
OP_TAGS_UPDATE,
|
|
OP_SUPERSEDE,
|
|
OP_DELETE_REQUEST,
|
|
RC_FORGEJO_UNAVAILABLE,
|
|
RC_OUT_OF_ORDER_COMMIT,
|
|
RC_UNKNOWN_OBJECT_ID,
|
|
RC_UNKNOWN_LEGACY_OBJECT,
|
|
RC_SECRET_DETECTED,
|
|
RC_INVALID_SCHEMA,
|
|
RC_DANGLING_DERIVED_FROM,
|
|
RC_AMBIGUOUS_DELETE,
|
|
REASON_CODES,
|
|
OPERATIONS,
|
|
)
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Konstanten
|
|
# ---------------------------------------------------------------------------
|
|
|
|
# Polling-Intervall (Default 60s, Environment-konfigurierbar)
|
|
DEFAULT_POLL_INTERVAL_SECONDS = 60
|
|
ENV_POLL_INTERVAL = "C5_POLL_INTERVAL_SECONDS"
|
|
|
|
# Knowledge-Scope-Klassifikation
|
|
SCOPE_IN_SCOPE = "IN_SCOPE"
|
|
SCOPE_OUT_OF_SCOPE = "OUT_OF_SCOPE"
|
|
SCOPE_LEGACY_SPECIAL = "LEGACY_SPECIAL"
|
|
SCOPE_HUMAN_REVIEW = "HUMAN_REVIEW"
|
|
|
|
# Verzeichnisse, die KEINE Tolaria-Knowledge-Objekte enthalten (Architektur/Ops/Code)
|
|
OUT_OF_SCOPE_DIRS = frozenset({
|
|
"tolaria", "a2", "a3", "a4", "a5",
|
|
"red-queen-architecture", "notion-safety-brain", "notion-command-center",
|
|
"backup_patches", "notes/reference", "notes",
|
|
})
|
|
|
|
# Legacy-Ausnahmen (C3I: KEEP_DISTINCT / DEFERRED) — C5B darf diese NICHT
|
|
# zusammenfuehren oder neu-vergeben. Sie werden separat klassifiziert.
|
|
LEGACY_SPECIAL_PATHS = frozenset({
|
|
"README.md", # Root-README (KEEP_DISTINCT)
|
|
"notes/trading/system-docs/README.md", # Canonical-README (KEEP_DISTINCT)
|
|
"vps.md", # DEFERRED (C3I)
|
|
})
|
|
|
|
# Canonical-Verzeichnis (derived knowledge)
|
|
CANONICAL_DIR = "notes/trading/system-docs/"
|
|
|
|
# Git read-only Whitelist (C5B darf NUR diese Befehle ausfuehren)
|
|
# Jeder Befehl ist read-only. Write-Befehle (push/commit/add/reset/checkout/
|
|
# merge/rebase/fetch --write) sind NICHT in der Whitelist und werden blockiert.
|
|
GIT_READONLY_COMMANDS = frozenset({
|
|
"rev-parse", "log", "show", "diff-tree", "ls-tree", "cat-file",
|
|
"merge-base", "rev-list", "name-only", "diff",
|
|
})
|
|
|
|
# Git Write-Befehle (explizit verboten — statische + dynamische Pruefung)
|
|
GIT_WRITE_COMMANDS = frozenset({
|
|
"push", "commit", "add", "reset", "checkout", "merge", "rebase",
|
|
"fetch", "pull", "clone", "init", "rm", "mv", "tag", "branch",
|
|
"stash", "clean", "gc", "prune", "repack", "update-ref", "write-tree",
|
|
})
|
|
|
|
# Frontmatter-Id-Muster: object/<FULL_UUID>
|
|
OBJECT_ID_RE = re.compile(r"^object/[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$")
|
|
|
|
# Secret-/Forbidden-Muster (nur zur Klassifikation, KEINE Werte persistieren)
|
|
SECRET_PATTERNS = [
|
|
re.compile(r"sk-[A-Za-z0-9]{20,}"),
|
|
re.compile(r"ghp_[A-Za-z0-9]{20,}"),
|
|
re.compile(r"bearer\s+[A-Za-z0-9]{20,}", re.IGNORECASE),
|
|
re.compile(r"-----BEGIN (RSA |EC |OPENSSH )?PRIVATE KEY-----"),
|
|
re.compile(r"AKIA[0-9A-Z]{16}"),
|
|
]
|
|
|
|
# Knowledge-Schema-Metadaten, die in metadata_hash eingehen
|
|
METADATA_FIELDS = ("id", "type", "role", "representation", "state",
|
|
"derived_from", "tags", "knowledge_schema")
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Fehlerklassen
|
|
# ---------------------------------------------------------------------------
|
|
|
|
class C5BError(Exception):
|
|
"""Basis-Fehlerklasse fuer C5B."""
|
|
|
|
def __init__(self, message: str, reason_code: Optional[str] = None):
|
|
super().__init__(message)
|
|
self.message = message
|
|
self.reason_code = reason_code
|
|
|
|
|
|
class ForgejoUnavailableError(C5BError):
|
|
"""Forgejo (lokaler Clone) nicht lesbar."""
|
|
|
|
|
|
class HistoryDivergenceError(C5BError):
|
|
"""last_applied_commit ist kein Ancestor von HEAD (FAIL CLOSED)."""
|
|
|
|
|
|
class GitWriteBlockedError(C5BError):
|
|
"""Versuch, einen verbotenen git-Write-Befehl auszufuehren."""
|
|
|
|
|
|
class NoDownstreamWriteError(C5BError):
|
|
"""C5B darf keine externen Writes ausfuehren."""
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# GitReader (read-only, strikte Whitelist)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
class GitReader:
|
|
"""
|
|
Fuehrt AUSSCHLIESSLICH read-only git-Befehle gegen einen lokalen Clone aus.
|
|
|
|
Whitelist: rev-parse, log, show, diff-tree, ls-tree, cat-file, merge-base,
|
|
rev-list, diff. Jeder andere Befehl (insb. push/commit/add/reset/checkout/
|
|
merge/rebase) wird mit GitWriteBlockedError blockiert.
|
|
"""
|
|
|
|
def __init__(self, repo_path: str):
|
|
self.repo_path = str(repo_path)
|
|
if not os.path.isdir(os.path.join(self.repo_path, ".git")):
|
|
raise ForgejoUnavailableError(
|
|
f"Kein Git-Repo unter {self.repo_path}",
|
|
reason_code=RC_FORGEJO_UNAVAILABLE,
|
|
)
|
|
|
|
def _run(self, args: List[str], check: bool = True) -> str:
|
|
"""Fuehrt einen read-only git-Befehl aus. Blockiert Write-Befehle."""
|
|
if not args:
|
|
raise GitWriteBlockedError("Leerer git-Befehl")
|
|
cmd = args[0]
|
|
if cmd not in GIT_READONLY_COMMANDS:
|
|
raise GitWriteBlockedError(
|
|
f"git-Befehl '{cmd}' ist nicht in der read-only Whitelist"
|
|
)
|
|
if cmd in GIT_WRITE_COMMANDS:
|
|
raise GitWriteBlockedError(
|
|
f"git-Befehl '{cmd}' ist ein verbotener Write-Befehl"
|
|
)
|
|
try:
|
|
proc = subprocess.run(
|
|
["git", "-C", self.repo_path] + args,
|
|
capture_output=True, text=True, timeout=30,
|
|
)
|
|
except (subprocess.TimeoutExpired, OSError) as e:
|
|
raise ForgejoUnavailableError(
|
|
f"Forgejo (git) nicht erreichbar: {e}",
|
|
reason_code=RC_FORGEJO_UNAVAILABLE,
|
|
)
|
|
if check and proc.returncode != 0:
|
|
raise ForgejoUnavailableError(
|
|
f"git {cmd} fehlgeschlagen: {proc.stderr.strip()}",
|
|
reason_code=RC_FORGEJO_UNAVAILABLE,
|
|
)
|
|
return proc.stdout
|
|
|
|
def get_current_head(self) -> str:
|
|
"""Gibt den aktuellen HEAD-Commit-SHA zurueck (read-only)."""
|
|
out = self._run(["rev-parse", "HEAD"]).strip()
|
|
if not out:
|
|
raise ForgejoUnavailableError(
|
|
"HEAD nicht bestimmbar", reason_code=RC_FORGEJO_UNAVAILABLE
|
|
)
|
|
return out
|
|
|
|
def commit_meta(self, sha: str) -> Dict[str, Any]:
|
|
"""Gibt parent, timestamp, message eines Commits zurueck (read-only)."""
|
|
out = self._run(["show", "-s", "--format=%H%x00%P%x00%ct%x00%s", sha]).strip()
|
|
parts = out.split("\x00")
|
|
if len(parts) < 4:
|
|
raise ForgejoUnavailableError(
|
|
f"Commit-Metadaten nicht lesbar: {sha}",
|
|
reason_code=RC_FORGEJO_UNAVAILABLE,
|
|
)
|
|
return {
|
|
"commit_sha": parts[0],
|
|
"parent_sha": parts[1] or None, # Root-Commit hat keinen Parent
|
|
"timestamp": int(parts[2]) if parts[2].isdigit() else None,
|
|
"message": parts[3],
|
|
}
|
|
|
|
def is_ancestor(self, ancestor: str, descendant: str) -> bool:
|
|
"""
|
|
Prueft, ob ancestor ein Vorfahr von descendant ist (read-only).
|
|
git merge-base --is-ancestor: exit 0 = ist Ancestor, exit 1 = nicht.
|
|
"""
|
|
try:
|
|
proc = subprocess.run(
|
|
["git", "-C", self.repo_path,
|
|
"merge-base", "--is-ancestor", ancestor, descendant],
|
|
capture_output=True, text=True, timeout=30,
|
|
)
|
|
return proc.returncode == 0
|
|
except (subprocess.TimeoutExpired, OSError) as e:
|
|
raise ForgejoUnavailableError(
|
|
f"Forgejo (git) nicht erreichbar: {e}",
|
|
reason_code=RC_FORGEJO_UNAVAILABLE,
|
|
)
|
|
|
|
def commit_range(self, start_exclusive: Optional[str], end: str) -> List[str]:
|
|
"""
|
|
Deterministisch alle Commits in (start_exclusive, end] in chronologischer
|
|
(parent-korrekter) Reihenfolge. Kein Ueberspringen, keine Doppelerfassung.
|
|
"""
|
|
if start_exclusive is None:
|
|
# Alle Commits bis end (reverse = aeltester zuerst)
|
|
out = self._run(["rev-list", "--reverse", end]).strip()
|
|
else:
|
|
out = self._run(["rev-list", "--reverse", f"{start_exclusive}..{end}"]).strip()
|
|
if not out:
|
|
return []
|
|
return [line for line in out.splitlines() if line]
|
|
|
|
def diff_name_status(self, commit_sha: str) -> List[Dict[str, Any]]:
|
|
"""
|
|
Gibt die Aenderungen eines Commits gegenueber seinem Parent zurueck.
|
|
Nutzt git diff-tree (read-only) mit rename detection.
|
|
"""
|
|
out = self._run([
|
|
"diff-tree", "-r", "--name-status", "-M", "-C", "--root",
|
|
"--format=", commit_sha,
|
|
]).strip()
|
|
changes = []
|
|
for line in out.splitlines():
|
|
parts = line.split("\t")
|
|
if len(parts) < 2:
|
|
continue
|
|
status = parts[0]
|
|
path = parts[1]
|
|
# Rename/Copy: "R100\told\tnew" oder "C100\told\tnew"
|
|
if status.startswith("R") or status.startswith("C"):
|
|
if len(parts) >= 3:
|
|
changes.append({
|
|
"status": status[0], # R oder C
|
|
"similarity": status[1:],
|
|
"path_before": parts[1],
|
|
"path_after": parts[2],
|
|
})
|
|
continue
|
|
# ADDED: Datei existiert nur in AFTER (path_before=None)
|
|
if status == "A":
|
|
changes.append({
|
|
"status": status,
|
|
"path_before": None,
|
|
"path_after": path,
|
|
})
|
|
continue
|
|
# DELETED: Datei existiert nur in BEFORE (path_after=None)
|
|
if status == "D":
|
|
changes.append({
|
|
"status": status,
|
|
"path_before": path,
|
|
"path_after": None,
|
|
})
|
|
continue
|
|
# MODIFIED (M, T, U): beide Pfade = path
|
|
changes.append({
|
|
"status": status,
|
|
"path_before": path,
|
|
"path_after": path,
|
|
})
|
|
return changes
|
|
|
|
def file_content(self, sha: str, path: str) -> Optional[str]:
|
|
"""Liest den Inhalt einer Datei in einem Commit (read-only)."""
|
|
try:
|
|
out = self._run(["show", f"{sha}:{path}"], check=False)
|
|
except ForgejoUnavailableError:
|
|
return None
|
|
if out == "" and self._file_exists(sha, path) is False:
|
|
return None
|
|
return out
|
|
|
|
def _file_exists(self, sha: str, path: str) -> Optional[bool]:
|
|
"""Prueft, ob eine Datei in einem Commit existiert (read-only)."""
|
|
try:
|
|
out = self._run(["ls-tree", "-r", "--name-only", sha], check=False)
|
|
except ForgejoUnavailableError:
|
|
return None
|
|
return path in out.splitlines()
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Frontmatter / Object ID / Hash Model
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def parse_frontmatter(content: str) -> Tuple[Dict[str, Any], str]:
|
|
"""
|
|
Parst YAML-Frontmatter (--- ... ---) und gibt (fm_dict, body) zurueck.
|
|
Body = fachlicher Inhalt NACH dem Frontmatter-Block.
|
|
"""
|
|
if not content.startswith("---"):
|
|
return {}, content
|
|
lines = content.splitlines()
|
|
if len(lines) < 2:
|
|
return {}, content
|
|
# Ende des Frontmatter-Blocks finden
|
|
end = None
|
|
for i in range(1, len(lines)):
|
|
if lines[i].strip() == "---":
|
|
end = i
|
|
break
|
|
if end is None:
|
|
return {}, content
|
|
fm_lines = lines[1:end]
|
|
body = "\n".join(lines[end + 1:])
|
|
fm: Dict[str, Any] = {}
|
|
for line in fm_lines:
|
|
if ":" not in line:
|
|
continue
|
|
key, _, val = line.partition(":")
|
|
key = key.strip()
|
|
val = val.strip()
|
|
if not key:
|
|
continue
|
|
# Einfache Listen (tags: [a, b]) und Skalare
|
|
if val.startswith("[") and val.endswith("]"):
|
|
inner = val[1:-1].strip()
|
|
fm[key] = [x.strip().strip("'\"") for x in inner.split(",") if x.strip()]
|
|
else:
|
|
fm[key] = val.strip("'\"")
|
|
return fm, body
|
|
|
|
|
|
def extract_object_id(content: str) -> Optional[str]:
|
|
"""Liest object_id aus dem Frontmatter (id: object/<FULL_UUID>)."""
|
|
fm, _ = parse_frontmatter(content)
|
|
oid = fm.get("id")
|
|
if not oid:
|
|
return None
|
|
oid = str(oid).strip()
|
|
if not OBJECT_ID_RE.match(oid):
|
|
return None
|
|
return oid
|
|
|
|
|
|
def content_hash(body: str) -> str:
|
|
"""SHA-256 des fachlichen Bodies (deterministisch)."""
|
|
return hashlib.sha256(body.encode("utf-8")).hexdigest()
|
|
|
|
|
|
def metadata_hash(fm: Dict[str, Any]) -> str:
|
|
"""
|
|
SHA-256 der relevanten Knowledge-Schema-Metadaten (deterministisch).
|
|
Beruecksichtigt: id, type, role, representation, state, derived_from,
|
|
tags, knowledge_schema. Pfad ist separat (nicht Teil von metadata_hash).
|
|
"""
|
|
subset = {k: fm.get(k) for k in METADATA_FIELDS if k in fm}
|
|
canonical = json.dumps(subset, sort_keys=True, ensure_ascii=False)
|
|
return hashlib.sha256(canonical.encode("utf-8")).hexdigest()
|
|
|
|
|
|
def detect_secret(content: str) -> Optional[str]:
|
|
"""
|
|
Erkennt moegliche Secret-Werte in einem Dokument. Gibt den MATCHED PATTERN
|
|
zurueck (NICHT den Wert). Kein Secret-Wert wird ausgegeben oder persistiert.
|
|
"""
|
|
for pat in SECRET_PATTERNS:
|
|
m = pat.search(content)
|
|
if m:
|
|
return pat.pattern
|
|
return None
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Knowledge Scope
|
|
# ---------------------------------------------------------------------------
|
|
|
|
class KnowledgeScope:
|
|
"""
|
|
Bestimmt, ob eine Forgejo-Datei ein Tolaria-Knowledge-Objekt ist.
|
|
|
|
Regeln (C3/C5):
|
|
* IN_SCOPE: Markdown im Repo-Root ODER unter notes/trading/system-docs/,
|
|
mit gueltiger object_id im Frontmatter.
|
|
* OUT_OF_SCOPE: Architektur/Ops/Code-Verzeichnisse, Nicht-Markdown,
|
|
Build-Artefakte, Secrets.
|
|
* LEGACY_SPECIAL: README (x2, KEEP_DISTINCT) + vps.md (DEFERRED).
|
|
* HUMAN_REVIEW: Knowledge-Objekt ohne gueltige object_id (UNKNOWN_OBJECT_ID
|
|
/ UNKNOWN_LEGACY_OBJECT) oder Ambiguitaet.
|
|
"""
|
|
|
|
def __init__(self, repo_path: str):
|
|
self.repo_path = str(repo_path)
|
|
|
|
def classify(self, path: str) -> str:
|
|
"""Klassifiziert einen Pfad in IN_SCOPE / OUT_OF_SCOPE / LEGACY_SPECIAL."""
|
|
path = path.replace("\\", "/")
|
|
# Legacy-Ausnahmen zuerst
|
|
if path in LEGACY_SPECIAL_PATHS:
|
|
return SCOPE_LEGACY_SPECIAL
|
|
# Nur Markdown ist potenziell Knowledge
|
|
if not path.endswith(".md"):
|
|
return SCOPE_OUT_OF_SCOPE
|
|
# Canonical-Verzeichnis
|
|
if path.startswith(CANONICAL_DIR):
|
|
return SCOPE_IN_SCOPE
|
|
# Repo-Root (kein Verzeichnis-Praefix)
|
|
if "/" not in path:
|
|
return SCOPE_IN_SCOPE
|
|
# Alles andere (tolaria/, a2-5/, red-queen-architecture/, etc.) = OUT
|
|
return SCOPE_OUT_OF_SCOPE
|
|
|
|
def classify_with_content(self, path: Optional[str], content: Optional[str]) -> Dict[str, Any]:
|
|
"""
|
|
Vollstaendige Scope-Klassifikation inkl. object_id-Validierung.
|
|
Gibt ein Dict mit scope, object_id, reason_code (optional) zurueck.
|
|
"""
|
|
if path is None:
|
|
return {"path": None, "scope": SCOPE_OUT_OF_SCOPE}
|
|
scope = self.classify(path)
|
|
result: Dict[str, Any] = {"path": path, "scope": scope}
|
|
if scope == SCOPE_OUT_OF_SCOPE:
|
|
return result
|
|
if scope == SCOPE_LEGACY_SPECIAL:
|
|
return result
|
|
# IN_SCOPE: object_id aus Frontmatter validieren
|
|
if content is None:
|
|
result["scope"] = SCOPE_HUMAN_REVIEW
|
|
result["reason_code"] = RC_UNKNOWN_OBJECT_ID
|
|
return result
|
|
oid = extract_object_id(content)
|
|
if oid is None:
|
|
# Knowledge-Objekt ohne gueltige ID -> Human Review
|
|
result["scope"] = SCOPE_HUMAN_REVIEW
|
|
result["reason_code"] = RC_UNKNOWN_OBJECT_ID
|
|
return result
|
|
result["object_id"] = oid
|
|
return result
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Change Classification
|
|
# ---------------------------------------------------------------------------
|
|
|
|
class ChangeClassifier:
|
|
"""
|
|
Leitet aus BEFORE/AFTER deterministisch die erlaubten Operationen ab.
|
|
|
|
Bei mehreren geaenderten Aspekten in EINEM Objekt werden MEHRERE
|
|
ObjectChange-Eintraege erzeugt (keine Information verloren).
|
|
"""
|
|
|
|
def __init__(self, scope: KnowledgeScope):
|
|
self.scope = scope
|
|
|
|
def classify(self, path_before: Optional[str], path_after: Optional[str],
|
|
content_before: Optional[str], content_after: Optional[str]) -> List[Dict[str, Any]]:
|
|
"""
|
|
Klassifiziert die Aenderung eines Objekts. Gibt eine Liste von
|
|
ObjectChange-Dicts zurueck (eine pro Operation).
|
|
"""
|
|
# DELETE: Datei existiert in BEFORE, nicht in AFTER
|
|
if path_before and not path_after:
|
|
return [{"operation": OP_DELETE_REQUEST, "path_before": path_before,
|
|
"path_after": None}]
|
|
|
|
# CREATE: Datei existiert in AFTER, nicht in BEFORE
|
|
if not path_before and path_after:
|
|
return [{"operation": OP_CREATE, "path_before": None,
|
|
"path_after": path_after}]
|
|
|
|
# RENAME/MOVE: gleiche object_id, anderer Pfad
|
|
if path_before and path_after and path_before != path_after:
|
|
oid_before = extract_object_id(content_before or "")
|
|
oid_after = extract_object_id(content_after or "")
|
|
if oid_before and oid_after and oid_before == oid_after:
|
|
# Rename = Dateiname geaendert, gleiches Verzeichnis
|
|
# Move = Verzeichnis geaendert
|
|
dir_before = os.path.dirname(path_before)
|
|
dir_after = os.path.dirname(path_after)
|
|
if dir_before == dir_after:
|
|
return [{"operation": OP_RENAME, "path_before": path_before,
|
|
"path_after": path_after}]
|
|
return [{"operation": OP_MOVE, "path_before": path_before,
|
|
"path_after": path_after}]
|
|
# Ambiguitaet: Pfad geaendert, aber ID nicht stabil -> Human Review
|
|
return [{"operation": OP_MOVE, "path_before": path_before,
|
|
"path_after": path_after, "ambiguous": True}]
|
|
|
|
# MODIFIED: gleicher Pfad, Inhalt/Metadaten geaendert
|
|
ops: List[Dict[str, Any]] = []
|
|
fm_before, body_before = parse_frontmatter(content_before or "")
|
|
fm_after, body_after = parse_frontmatter(content_after or "")
|
|
|
|
ch_before = content_hash(body_before)
|
|
ch_after = content_hash(body_after)
|
|
mh_before = metadata_hash(fm_before)
|
|
mh_after = metadata_hash(fm_after)
|
|
|
|
if ch_before != ch_after:
|
|
ops.append({"operation": OP_CONTENT_UPDATE})
|
|
if mh_before != mh_after:
|
|
# Spezifische Metadaten-Aenderungen
|
|
if fm_before.get("state") != fm_after.get("state"):
|
|
ops.append({"operation": OP_STATE_UPDATE})
|
|
if fm_before.get("tags") != fm_after.get("tags"):
|
|
ops.append({"operation": OP_TAGS_UPDATE})
|
|
if fm_before.get("derived_from") != fm_after.get("derived_from"):
|
|
ops.append({"operation": OP_SOURCE_CANONICAL_RELATION_UPDATE})
|
|
# Generische Metadaten-Aenderung (falls keine spezifische erkannt)
|
|
if not any(op["operation"] in (
|
|
OP_STATE_UPDATE, OP_TAGS_UPDATE,
|
|
OP_SOURCE_CANONICAL_RELATION_UPDATE,
|
|
) for op in ops):
|
|
ops.append({"operation": OP_METADATA_UPDATE})
|
|
# SUPERSEDE: state -> superseded
|
|
if fm_after.get("state") == "superseded" and fm_before.get("state") != "superseded":
|
|
ops.append({"operation": OP_SUPERSEDE})
|
|
|
|
if not ops:
|
|
# Keine Aenderung erkannt (z.B. nur Whitespace) -> kein ObjectChange
|
|
return []
|
|
return ops
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# C5B Poller (integriert in C5AStore, nur State/Persistence)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
class C5BPoller:
|
|
"""
|
|
Fuehrt einen kontrollierten Poll-Zyklus aus und speist Commits + ObjectChanges
|
|
in den C5A Store ein. KEIN Network Write downstream. KEIN Poll-Daemon.
|
|
"""
|
|
|
|
def __init__(self, store: C5AStore, repo_path: str,
|
|
poll_interval_seconds: Optional[int] = None):
|
|
self.store = store
|
|
self.reader = GitReader(repo_path)
|
|
self.scope = KnowledgeScope(repo_path)
|
|
self.classifier = ChangeClassifier(self.scope)
|
|
self.poll_interval_seconds = poll_interval_seconds or self._env_poll_interval()
|
|
|
|
@staticmethod
|
|
def _env_poll_interval() -> int:
|
|
"""Liest C5_POLL_INTERVAL_SECONDS aus der Environment (Default 60)."""
|
|
raw = os.environ.get(ENV_POLL_INTERVAL, "")
|
|
if raw.isdigit() and int(raw) > 0:
|
|
return int(raw)
|
|
return DEFAULT_POLL_INTERVAL_SECONDS
|
|
|
|
def get_current_head(self) -> str:
|
|
"""Read-only HEAD-Discovery."""
|
|
return self.reader.get_current_head()
|
|
|
|
def history_safety(self, last_applied: Optional[str], head: str) -> bool:
|
|
"""
|
|
FAIL CLOSED: Wenn last_applied_commit kein Ancestor von HEAD ist,
|
|
ist die History divergiert. Keine automatische Rebase-/Reset-Logik.
|
|
"""
|
|
if last_applied is None:
|
|
return True # Bootstrap: keine Baseline, alles neu
|
|
return self.reader.is_ancestor(last_applied, head)
|
|
|
|
def poll_once(self) -> Dict[str, Any]:
|
|
"""
|
|
Ein kontrollierter Poll-Zyklus:
|
|
HEAD -> Commit-Range -> Diff -> Change-Klassifikation -> C5AStore.
|
|
Gibt ein Ergebnis-Dict zurueck. KEIN Network Write downstream.
|
|
|
|
Baseline fuer Commit-Range + History-Safety ist last_seen_commit
|
|
(C5B-Baseline). last_applied_commit bleibt laut C5A-Contract fuer die
|
|
spaetere Propagation (C5C) reserviert und wird von C5B NIE fortgeschrieben.
|
|
"""
|
|
head = self.get_current_head()
|
|
health = self.store.health()
|
|
last_seen = health.get("last_seen_commit")
|
|
|
|
# History Safety (FAIL CLOSED) gegen last_seen_commit
|
|
if not self.history_safety(last_seen, head):
|
|
return {
|
|
"status": "FAIL_CLOSED",
|
|
"reason_code": RC_OUT_OF_ORDER_COMMIT,
|
|
"message": "last_seen_commit ist kein Ancestor von HEAD (Divergenz)",
|
|
"head": head,
|
|
"last_seen": last_seen,
|
|
}
|
|
|
|
# Commit-Range bestimmen (ab last_seen_commit, exklusiv)
|
|
commits = self.reader.commit_range(last_seen, head)
|
|
self.store.mark_seen(head)
|
|
|
|
discovered = 0
|
|
objects_discovered = 0
|
|
out_of_scope = 0
|
|
human_review = 0
|
|
new_commits = []
|
|
|
|
for sha in commits:
|
|
meta = self.reader.commit_meta(sha)
|
|
# Idempotenz: bereits registriert?
|
|
existing = self.store.get_commit(sha)
|
|
if existing is not None:
|
|
continue # ALREADY_DISCOVERED
|
|
|
|
# Diff gegen Parent
|
|
changes = self.reader.diff_name_status(sha)
|
|
parent_sha = meta.get("parent_sha")
|
|
object_changes = []
|
|
for ch in changes:
|
|
path_before: Optional[str] = ch.get("path_before")
|
|
path_after: Optional[str] = ch.get("path_after")
|
|
# Inhalte read-only lesen.
|
|
# content_before stammt aus dem PARENT-Commit (der Zustand VOR der
|
|
# Aenderung), content_after aus dem aktuellen Commit (sha).
|
|
content_before: Optional[str] = None
|
|
content_after: Optional[str] = None
|
|
if path_before and parent_sha:
|
|
content_before = self.reader.file_content(parent_sha, path_before)
|
|
if path_after:
|
|
content_after = self.reader.file_content(sha, path_after)
|
|
|
|
# Scope-Klassifikation
|
|
scope_info = self.scope.classify_with_content(
|
|
path_after or path_before, content_after or content_before
|
|
)
|
|
scope = scope_info["scope"]
|
|
if scope == SCOPE_OUT_OF_SCOPE:
|
|
out_of_scope += 1
|
|
continue
|
|
if scope == SCOPE_LEGACY_SPECIAL:
|
|
out_of_scope += 1
|
|
continue
|
|
if scope == SCOPE_HUMAN_REVIEW:
|
|
human_review += 1
|
|
object_changes.append({
|
|
"object_id": None,
|
|
"operation": OP_METADATA_UPDATE,
|
|
"path_before": path_before,
|
|
"path_after": path_after,
|
|
"reason_code": scope_info.get("reason_code", RC_UNKNOWN_OBJECT_ID),
|
|
})
|
|
continue
|
|
|
|
oid = scope_info["object_id"]
|
|
# Secret-Safety: kein Secret-Inhalt persistieren
|
|
for content in (content_before, content_after):
|
|
if content and detect_secret(content):
|
|
# Nur sichere Metadaten persistieren, kein Content
|
|
object_changes.append({
|
|
"object_id": oid,
|
|
"operation": OP_METADATA_UPDATE,
|
|
"path_before": path_before,
|
|
"path_after": path_after,
|
|
"reason_code": RC_SECRET_DETECTED,
|
|
})
|
|
human_review += 1
|
|
break
|
|
else:
|
|
# Change-Klassifikation
|
|
ops = self.classifier.classify(
|
|
path_before, path_after, content_before, content_after
|
|
)
|
|
for op in ops:
|
|
fm_before, body_before = parse_frontmatter(content_before or "")
|
|
fm_after, body_after = parse_frontmatter(content_after or "")
|
|
object_changes.append({
|
|
"object_id": oid,
|
|
"operation": op["operation"],
|
|
"path_before": path_before,
|
|
"path_after": path_after,
|
|
"content_hash_before": content_hash(body_before) if content_before is not None else None,
|
|
"content_hash_after": content_hash(body_after) if content_after is not None else None,
|
|
"metadata_hash_before": metadata_hash(fm_before) if content_before is not None else None,
|
|
"metadata_hash_after": metadata_hash(fm_after) if content_after is not None else None,
|
|
"representation": fm_after.get("representation") or fm_before.get("representation"),
|
|
"state": fm_after.get("state") or fm_before.get("state"),
|
|
})
|
|
|
|
# Commit in C5AStore registrieren (Status DISCOVERED)
|
|
commit_record = {
|
|
"commit_sha": sha,
|
|
"parent_sha": meta.get("parent_sha"),
|
|
"discovered_at": int(time.time() * 1000),
|
|
"sequence": len(commits),
|
|
"status": ST_DISCOVERED,
|
|
"retry_count": 0,
|
|
}
|
|
self.store.upsert_commit(commit_record)
|
|
discovered += 1
|
|
|
|
# ObjectChanges hinzufuegen
|
|
for oc in object_changes:
|
|
oc["commit_sha"] = sha
|
|
self.store.add_object_change(oc)
|
|
objects_discovered += 1
|
|
|
|
# Status -> VALIDATING (nicht weiter zu PROPAGATING_TOLARIA)
|
|
self.store.transition_commit(sha, ST_VALIDATING)
|
|
if any(oc.get("reason_code") for oc in object_changes):
|
|
# VALIDATING -> HUMAN_REVIEW_REQUIRED (erlaubte Transition)
|
|
self.store.transition_commit(sha, ST_HUMAN_REVIEW_REQUIRED)
|
|
|
|
new_commits.append(sha)
|
|
|
|
return {
|
|
"status": "OK",
|
|
"head": head,
|
|
"last_seen": last_seen,
|
|
"commits_discovered": discovered,
|
|
"objects_discovered": objects_discovered,
|
|
"out_of_scope_count": out_of_scope,
|
|
"human_review_count": human_review,
|
|
"new_commits": new_commits,
|
|
"poll_interval_seconds": self.poll_interval_seconds,
|
|
}
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# No-Downstream-Write-Guarantee (statische Pruefung)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def assert_no_downstream_write() -> Dict[str, Any]:
|
|
"""
|
|
Beweist, dass C5B keinen Codepfad fuer externe Writes besitzt:
|
|
* KEIN Tolaria-Write (vault/save)
|
|
* KEIN Search-Rebuild (search/rebuild)
|
|
* KEIN Forgejo-Write / kein git push/commit/add
|
|
Erlaubt ist NUR read-only git (Whitelist) + stdlib.
|
|
"""
|
|
import ast
|
|
this_file = Path(__file__).resolve()
|
|
tree = ast.parse(this_file.read_text(encoding="utf-8"))
|
|
|
|
# 1. HTTP-Mutationsimporte (Tolaria/Search-Write)
|
|
http_imports = {"requests", "urllib", "http", "socket"}
|
|
found_http = set()
|
|
for node in ast.walk(tree):
|
|
if isinstance(node, ast.Import):
|
|
for alias in node.names:
|
|
root = alias.name.split(".")[0]
|
|
if root in http_imports:
|
|
found_http.add(root)
|
|
elif isinstance(node, ast.ImportFrom):
|
|
if node.module:
|
|
root = node.module.split(".")[0]
|
|
if root in http_imports:
|
|
found_http.add(root)
|
|
|
|
# 2. Verbotene Endpoint-Strings NUR in echten Code-Ausdruecken
|
|
# (nicht in Docstrings/Kommentaren). Die Verbotsliste selbst wird aus
|
|
# Teilstrings zusammengesetzt, damit der volle Endpoint-String nie als
|
|
# einzelnes String-Literal im Quellcode steht (sonst False Positive).
|
|
forbidden_endpoints = ["/api/vault" + "/save", "/api/search" + "/rebuild"]
|
|
found_endpoints = []
|
|
for literal in _collect_code_strings(tree):
|
|
for ep in forbidden_endpoints:
|
|
if ep in literal and ep not in found_endpoints:
|
|
found_endpoints.append(ep)
|
|
|
|
# 3. Git-Write-Befehle duerfen NICHT in der Whitelist stehen
|
|
write_in_whitelist = sorted(GIT_READONLY_COMMANDS & GIT_WRITE_COMMANDS)
|
|
|
|
return {
|
|
"no_http_mutations": len(found_http) == 0 and len(found_endpoints) == 0,
|
|
"http_imports_found": sorted(found_http),
|
|
"forbidden_endpoints_found": found_endpoints,
|
|
"no_git_write_in_whitelist": len(write_in_whitelist) == 0,
|
|
"git_write_in_whitelist": write_in_whitelist,
|
|
"no_downstream_write": (
|
|
len(found_http) == 0
|
|
and len(found_endpoints) == 0
|
|
and len(write_in_whitelist) == 0
|
|
),
|
|
}
|
|
|
|
|
|
def _collect_code_strings(tree: ast.AST) -> List[str]:
|
|
"""
|
|
Sammelt alle String-Literale, die KEINE Docstrings sind (echter Code).
|
|
Docstrings = erstes Statement eines Modul-/Funktions-/Klassen-Body.
|
|
"""
|
|
import ast as _ast
|
|
|
|
docstring_nodes = set()
|
|
for node in _ast.walk(tree):
|
|
if isinstance(node, (_ast.Module, _ast.FunctionDef, _ast.AsyncFunctionDef,
|
|
_ast.ClassDef)):
|
|
body = getattr(node, "body", None)
|
|
if body and isinstance(body[0], _ast.Expr) and isinstance(
|
|
body[0].value, _ast.Constant
|
|
) and isinstance(body[0].value.value, str):
|
|
docstring_nodes.add(id(body[0].value))
|
|
|
|
result = []
|
|
for node in _ast.walk(tree):
|
|
if isinstance(node, _ast.Constant) and isinstance(node.value, str):
|
|
if id(node) in docstring_nodes:
|
|
continue
|
|
result.append(node.value)
|
|
return result
|