186 lines
7.2 KiB
Python
186 lines
7.2 KiB
Python
"""
|
|
TOLARIA SEARCH SERVICE — C4B Keyword & Metadata Search Engine
|
|
================================================================
|
|
Agentenunabhängiger, abgeleiteter Retrieval-Service (Weg A, von Christian freigegeben).
|
|
|
|
Architekturrolle:
|
|
Forgejo MASTER -> Tolaria (derived Second Brain) -> TOLARIA SEARCH SERVICE
|
|
(derived retrieval) -> Search API -> Konsumenten (Red Queen / Hermes / Rain / Alice)
|
|
|
|
C4B implementiert NUR: exact, keyword, metadata, Ranking, Filtering, Collapse
|
|
(soweit möglich), Pagination, Health, rebuildbares Keyword/Metadata-Indexing,
|
|
Secret-Safety. NOCH NICHT: pgvector, Embeddings, Vector, Semantic, Hybrid.
|
|
|
|
Nicht implementierte Modi (semantic/vector/hybrid) -> HONEST MODE: ehrlicher
|
|
Fehler laut C4A-Contract, NIEMALS Fake-Ergebnisse.
|
|
|
|
Der Service ist DERIVED. Indexquelle = Forgejo-Master (SoT). Der Suchindex ist
|
|
jederzeit vollständig rebuildbar und NIE Source of Truth. Dieser Service schreibt
|
|
NICHT nach Tolaria/Forgejo (read-only ggü. Knowledge-Bestand).
|
|
|
|
Es wird NUR die Python-Standardbibliothek verwendet: keine externe DB, kein
|
|
pgvector, keine Trading-/Forgejo-DB. Persistenz = derived, rebuildbares Index-JSON.
|
|
(spätere produktive Search-Persistence = eigener isolierter pgvector-Service, C4D)
|
|
|
|
Secret-Safety: Pre-Index Secret-Scan (Muster-basiert, fail-closed). Kein erkannter
|
|
Secret-Wert wird indexiert, in Snippets ausgegeben oder geloggt.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import json
|
|
import re
|
|
import time
|
|
from dataclasses import dataclass, field
|
|
from typing import Any, Optional
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Konstanten (Index-Identity & Contract)
|
|
# ---------------------------------------------------------------------------
|
|
INDEX_VERSION = "c4b-keyword-metadata-v1"
|
|
RRF_K = 60
|
|
DEFAULT_LIMIT = 20
|
|
MAX_LIMIT = 100
|
|
REPRESENTATION_RANK = {"canonical": 3, "source": 2, "standalone": 1}
|
|
STATE_RANK = {"current": 2, "historical": 1, "superseded": 0, "archived": 0,
|
|
"draft": 1}
|
|
# Role-Aware-Downrank für overview/index bei allgemeinen Fachqueries (README-Policy)
|
|
INDEX_ROLES = {"index", "overview", "hub"}
|
|
INDEX_ROLES_PENALTY = 0.8
|
|
|
|
|
|
@dataclass
|
|
class Doc:
|
|
"""Ein zu indexierendes Knowledge-Objekt (C3)."""
|
|
path: str
|
|
title: str
|
|
id: Optional[str] # C3 object_id (object/<uuid>) oder None (README/vps)
|
|
type: Optional[str]
|
|
role: Optional[str]
|
|
representation: Optional[str]
|
|
state: Optional[str]
|
|
content_hash: str
|
|
body: str
|
|
aliases: list[str] = field(default_factory=list)
|
|
tags: list[str] = field(default_factory=list)
|
|
derived_from: Optional[str] = None
|
|
is_legacy: bool = False # True, wenn kein echtes object_id
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Secret-Scan & Markdown-Hilfen (Pre-Index)
|
|
# ---------------------------------------------------------------------------
|
|
# Erkennt secret-artige Zeilen: key=value mit langem Wert oder bekannte Secret-Header.
|
|
_SECRET_RE = re.compile(
|
|
r"(?i)(api[_-]?key|secret|token|password|passwd|bearer|client[_-]?secret|"
|
|
r"private[_-]?key)\s*[=:]\s*['\"]?([A-Za-z0-9_\-]{12,})['\"]?"
|
|
r"|(\bsk-[A-Za-z0-9]{16,}\b)"
|
|
r"|(\bAKIA[0-9A-Z]{16}\b)"
|
|
r"|(\bghp_[A-Za-z0-9]{20,}\b)",
|
|
re.IGNORECASE,
|
|
)
|
|
|
|
|
|
def scan_for_secrets(text: str) -> list[str]:
|
|
"""Liefert gefundene Secret-ähnliche Treffer (gekürzt). fail-closed."""
|
|
out: list[str] = []
|
|
for m in _SECRET_RE.finditer(text):
|
|
val = m.group(0)
|
|
out.append(val[:32])
|
|
return out
|
|
|
|
|
|
def _sha256(s: str) -> str:
|
|
return hashlib.sha256(s.encode("utf-8", "replace")).hexdigest()
|
|
|
|
|
|
def tokenize(text: str) -> list[str]:
|
|
"""Tokenisierung: Wörter, Modulnamen (modul-09), Akronyme (OHLCV, IG),
|
|
env/Config-Tokens (POSTGRES_DB). Bindestriche/Unterstriche bleiben Teil des Tokens."""
|
|
return re.findall(r"[a-zäöüß0-9]+(?:[_-][a-zäöüß0-9]+)*", text.lower())
|
|
|
|
|
|
def tokenize_phrase(text: str) -> list[str]:
|
|
return tokenize(text)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Index: invertierte Postings, rebuildbar, persistiert als JSON (derived)
|
|
# ---------------------------------------------------------------------------
|
|
class SearchIndex:
|
|
"""Deterministischer, rebuildbarer Keyword/Metadata-Index (in-memory + persisted)."""
|
|
|
|
def __init__(self):
|
|
self.docs: list[Doc] = []
|
|
self.postings: dict[str, list[int]] = {}
|
|
self.path_to_idx: dict[str, int] = {}
|
|
self.title_tokens: dict[int, set[str]] = {}
|
|
self.secret_blocked: list[str] = [] # Pfade, die fail-closed ausgeschlossen wurden
|
|
self._ready = False
|
|
|
|
def _add_token(self, tok: str, idx: int):
|
|
self.postings.setdefault(tok, []).append(idx)
|
|
|
|
def build(self, docs: list[Doc], secret_filter: bool = True, on_progress=None):
|
|
"""Vollständiger Rebuild (atomic-ish: buildt in neuer Struktur, erst am Ende swap)."""
|
|
new_docs: list[Doc] = []
|
|
new_postings: dict[str, list[int]] = {}
|
|
new_doc_by: dict[str, int] = {}
|
|
new_title: dict[int, list[str]] = {}
|
|
blocked: list[str] = []
|
|
|
|
for d in docs:
|
|
if secret_filter and scan_for_secrets(d.title + "\n" + d.body):
|
|
blocked.append(d.path)
|
|
continue
|
|
idx = len(new_docs)
|
|
new_docs.append(d)
|
|
new_doc_by[d.path] = idx
|
|
# Titel-Tokens für exact/phrase/prefix
|
|
new_title[idx] = tokenize(d.title)
|
|
# Body+title postings
|
|
seen = set()
|
|
for tok in tokenize(d.title) + tokenize(d.body):
|
|
if tok in seen:
|
|
continue
|
|
seen.add(tok)
|
|
new_postings.setdefault(tok, []).append(idx)
|
|
# Atomisches Swap
|
|
self.docs = new_docs
|
|
self.postings = new_postings
|
|
self.doc_by_key = new_doc_by
|
|
self.title_tokens = new_title
|
|
self.secret_blocked = blocked
|
|
self._ready = True
|
|
return {"indexed": len(self.docs), "blocked": len(blocked)}
|
|
|
|
def search_postings(self, tokens: list[str]) -> dict[int, int]:
|
|
"""Term-Dokument-Frequenz: doc_idx -> Anzahl Treffer für die gegebenen Tokens."""
|
|
freq: dict[int, int] = {}
|
|
for tok in tokens:
|
|
for idx in self.postings.get(tok, []):
|
|
freq[idx] = freq.get(idx, 0) + 1
|
|
return freq
|
|
|
|
def query_tokens(self, query: str, phrase=False) -> list[str]:
|
|
return tokenize(query)
|
|
|
|
def title_exact(self, q_low: str) -> list[int]:
|
|
"""Alle Docs, deren Titel den Query-String als Teilstring enthält (case-insens)."""
|
|
return [i for i, d in enumerate(self.docs) if q_low in d.title.lower()]
|
|
|
|
def title_word(self, tokens: list[str]) -> list[int]:
|
|
"""Docs, deren Titel ALLE Query-Tokens als Wörter enthält."""
|
|
res = set()
|
|
for i, t in enumerate(self.title_tokens):
|
|
if all(tok in t for tok in tokens):
|
|
res.add(i)
|
|
return sorted(res)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Ranking / RRF (deterministisch) — RRF-Konstante wird von Search-API genutzt
|
|
# ---------------------------------------------------------------------------
|
|
def rrf_score(rank: int, k: int = RRF_K) -> float:
|
|
return 1.0 / (k + rank + 1)
|