"""
TOLARIA SEARCH SERVICE — C4B Keyword & Metadata Search Engine
================================================================
Agentenunabhängiger, abgeleiteter Retrieval-Service (Weg A, von Christian freigegeben).

Architekturrolle:
  Forgejo MASTER -> Tolaria (derived Second Brain) -> TOLARIA SEARCH SERVICE
  (derived retrieval) -> Search API -> Konsumenten (Red Queen / Hermes / Rain / Alice)

C4B implementiert NUR: exact, keyword, metadata, Ranking, Filtering, Collapse
(soweit möglich), Pagination, Health, rebuildbares Keyword/Metadata-Indexing,
Secret-Safety. NOCH NICHT: pgvector, Embeddings, Vector, Semantic, Hybrid.

Nicht implementierte Modi (semantic/vector/hybrid) -> HONEST MODE: ehrlicher
Fehler laut C4A-Contract, NIEMALS Fake-Ergebnisse.

Der Service ist DERIVED. Indexquelle = Forgejo-Master (SoT). Der Suchindex ist
jederzeit vollständig rebuildbar und NIE Source of Truth. Dieser Service schreibt
NICHT nach Tolaria/Forgejo (read-only ggü. Knowledge-Bestand).

Es wird NUR die Python-Standardbibliothek verwendet: keine externe DB, kein
pgvector, keine Trading-/Forgejo-DB. Persistenz = derived, rebuildbares Index-JSON.
(spätere produktive Search-Persistence = eigener isolierter pgvector-Service, C4D)

Secret-Safety: Pre-Index Secret-Scan (Muster-basiert, fail-closed). Kein erkannter
Secret-Wert wird indexiert, in Snippets ausgegeben oder geloggt.
"""

from __future__ import annotations

import hashlib
import json
import re
import time
from dataclasses import dataclass, field
from typing import Any, Optional

# ---------------------------------------------------------------------------
# Konstanten (Index-Identity & Contract)
# ---------------------------------------------------------------------------
INDEX_VERSION = "c4b-keyword-metadata-v1"
RRF_K = 60
DEFAULT_LIMIT = 20
MAX_LIMIT = 100
REPRESENTATION_RANK = {"canonical": 3, "source": 2, "standalone": 1}
STATE_RANK = {"current": 2, "historical": 1, "superseded": 0, "archived": 0,
              "draft": 1}
# Role-Aware-Downrank für overview/index bei allgemeinen Fachqueries (README-Policy)
INDEX_ROLES = {"index", "overview", "hub"}
INDEX_ROLES_PENALTY = 0.8


@dataclass
class Doc:
    """Ein zu indexierendes Knowledge-Objekt (C3)."""
    path: str
    title: str
    id: Optional[str]              # C3 object_id (object/<uuid>) oder None (README/vps)
    type: Optional[str]
    role: Optional[str]
    representation: Optional[str]
    state: Optional[str]
    content_hash: str
    body: str
    aliases: list[str] = field(default_factory=list)
    tags: list[str] = field(default_factory=list)
    derived_from: Optional[str] = None
    is_legacy: bool = False        # True, wenn kein echtes object_id


# ---------------------------------------------------------------------------
# Secret-Scan & Markdown-Hilfen (Pre-Index)
# ---------------------------------------------------------------------------
# Erkennt secret-artige Zeilen: key=value mit langem Wert oder bekannte Secret-Header.
_SECRET_RE = re.compile(
    r"(?i)(api[_-]?key|secret|token|password|passwd|bearer|client[_-]?secret|"
    r"private[_-]?key)\s*[=:]\s*['\"]?([A-Za-z0-9_\-]{12,})['\"]?"
    r"|(\bsk-[A-Za-z0-9]{16,}\b)"
    r"|(\bAKIA[0-9A-Z]{16}\b)"
    r"|(\bghp_[A-Za-z0-9]{20,}\b)",
    re.IGNORECASE,
)


def scan_for_secrets(text: str) -> list[str]:
    """Liefert gefundene Secret-ähnliche Treffer (gekürzt). fail-closed."""
    out: list[str] = []
    for m in _SECRET_RE.finditer(text):
        val = m.group(0)
        out.append(val[:32])
    return out


def _sha256(s: str) -> str:
    return hashlib.sha256(s.encode("utf-8", "replace")).hexdigest()


def tokenize(text: str) -> list[str]:
    """Tokenisierung: Wörter, Modulnamen (modul-09), Akronyme (OHLCV, IG),
    env/Config-Tokens (POSTGRES_DB). Bindestriche/Unterstriche bleiben Teil des Tokens."""
    return re.findall(r"[a-zäöüß0-9]+(?:[_-][a-zäöüß0-9]+)*", text.lower())


def tokenize_phrase(text: str) -> list[str]:
    return tokenize(text)


# ---------------------------------------------------------------------------
# Index: invertierte Postings, rebuildbar, persistiert als JSON (derived)
# ---------------------------------------------------------------------------
class SearchIndex:
    """Deterministischer, rebuildbarer Keyword/Metadata-Index (in-memory + persisted)."""

    def __init__(self):
        self.docs: list[Doc] = []
        self.postings: dict[str, list[int]] = {}
        self.path_to_idx: dict[str, int] = {}
        self.title_tokens: dict[int, set[str]] = {}
        self.secret_blocked: list[str] = []      # Pfade, die fail-closed ausgeschlossen wurden
        self._ready = False

    def _add_token(self, tok: str, idx: int):
        self.postings.setdefault(tok, []).append(idx)

    def build(self, docs: list[Doc], secret_filter: bool = True, on_progress=None):
        """Vollständiger Rebuild (atomic-ish: buildt in neuer Struktur, erst am Ende swap)."""
        new_docs: list[Doc] = []
        new_postings: dict[str, list[int]] = {}
        new_doc_by: dict[str, int] = {}
        new_title: dict[int, list[str]] = {}
        blocked: list[str] = []

        for d in docs:
            if secret_filter and scan_for_secrets(d.title + "\n" + d.body):
                blocked.append(d.path)
                continue
            idx = len(new_docs)
            new_docs.append(d)
            new_doc_by[d.path] = idx
            # Titel-Tokens für exact/phrase/prefix
            new_title[idx] = tokenize(d.title)
            # Body+title postings
            seen = set()
            for tok in tokenize(d.title) + tokenize(d.body):
                if tok in seen:
                    continue
                seen.add(tok)
                new_postings.setdefault(tok, []).append(idx)
        # Atomisches Swap
        self.docs = new_docs
        self.postings = new_postings
        self.doc_by_key = new_doc_by
        self.title_tokens = new_title
        self.secret_blocked = blocked
        self._ready = True
        return {"indexed": len(self.docs), "blocked": len(blocked)}

    def search_postings(self, tokens: list[str]) -> dict[int, int]:
        """Term-Dokument-Frequenz: doc_idx -> Anzahl Treffer für die gegebenen Tokens."""
        freq: dict[int, int] = {}
        for tok in tokens:
            for idx in self.postings.get(tok, []):
                freq[idx] = freq.get(idx, 0) + 1
        return freq

    def query_tokens(self, query: str, phrase=False) -> list[str]:
        return tokenize(query)

    def title_exact(self, q_low: str) -> list[int]:
        """Alle Docs, deren Titel den Query-String als Teilstring enthält (case-insens)."""
        return [i for i, d in enumerate(self.docs) if q_low in d.title.lower()]

    def title_word(self, tokens: list[str]) -> list[int]:
        """Docs, deren Titel ALLE Query-Tokens als Wörter enthält."""
        res = set()
        for i, t in enumerate(self.title_tokens):
            if all(tok in t for tok in tokens):
                res.add(i)
        return sorted(res)


# ---------------------------------------------------------------------------
# Ranking / RRF (deterministisch) — RRF-Konstante wird von Search-API genutzt
# ---------------------------------------------------------------------------
def rrf_score(rank: int, k: int = RRF_K) -> float:
    return 1.0 / (k + rank + 1)