trading-system-docs/tolaria/c5-sync-service/test_c5a.py
Red Queen 3db1c71b68 C5: Human-Gated DELETE Execution Contract (Todo 7-13)
- DeleteExecutor (rq_c5_delete.py): Pre-Gates, Read-Back, idempotenter replay
- TolariaClient.delete() (rq_c5c.py): kontrollierter DELETE, keine Probes
- Approval-Store + Reason Codes RC_DELETE_APPROVAL_MISSING/MISMATCH (rq_c5a.py)
- CLI: c5-delete-approve/execute/replay/status (rq_c5_cli.py)
- C5E: recover()/replay() DELETE-Integration
- C5D: verify_integrity prueft secret_blocked_objects (FAIL CLOSED)
- Security: Path-Traversal-Block in _normalize_vault_path
- Drift nach DELETE -> FAIL CLOSED zurueck zu HUMAN_REVIEW_REQUIRED
- Tests: test_c5_delete (19), test_c5_delete_integration (22),
  test_c5_delete_fresh_checker (17) — alle gruen
- ADR: C5_DELETE_EXECUTION_ARCHITECTURE_DECISION.md (ACCEPTED)
2026-08-26 19:22:44 +00:00

488 lines
18 KiB
Python

#!/usr/bin/env python3
"""
Red Queen — C5A: CONTRACT & STATE MACHINE Testsuite.
Deterministische Unit Tests. Jeder Test nutzt eine frische temp-DB (tempfile.mkdtemp),
niemals die Produkt-DB. Beweist die No-Write-Guarantee und alle geforderten
State-Machine-, Idempotenz-, Ordering-, Retry-, Human-Gate-, Bootstrap- und
Persistenz-Verhalten.
"""
from __future__ import annotations
import json
import os
import sys
import tempfile
import traceback
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parent))
from rq_c5a import (
C5AStore,
SyncStateMachine,
assert_no_write_guarantee,
C5AError,
InvalidTransitionError,
InvalidReasonCodeError,
InvalidOperationError,
ST_DISCOVERED, ST_VALIDATING, ST_READY,
ST_PROPAGATING_TOLARIA, ST_VERIFYING_TOLARIA,
ST_UPDATING_SEARCH, ST_VERIFYING_SEARCH, ST_APPLIED,
ST_RETRY_PENDING, ST_FAILED, ST_DEAD, ST_HUMAN_REVIEW_REQUIRED,
ST_WAITING_FOR_PREDECESSOR,
BS_UNINITIALIZED, BS_RECONCILING, BS_BASELINE_READY, BS_ACTIVE,
OP_CREATE, OP_CONTENT_UPDATE, OP_DELETE_REQUEST,
IDEM_ALREADY_APPLIED, IDEM_ALREADY_AT_TARGET, IDEM_RETRY_SAFE, IDEM_CONFLICT,
RC_UNEXPECTED_TOLARIA_DRIFT, RC_ID_COLLISION, RC_UNKNOWN_OBJECT_ID,
RC_INVALID_SCHEMA, RC_DANGLING_DERIVED_FROM, RC_AMBIGUOUS_DELETE,
RC_OUT_OF_ORDER_COMMIT, RC_SECRET_DETECTED,
RC_UNKNOWN_LEGACY_OBJECT, RC_AUTH_FAILURE, RC_SEARCH_REBUILD_FAILURE,
RC_TOLARIA_UNAVAILABLE, RC_FORGEJO_UNAVAILABLE,
RC_SEARCH_UNAVAILABLE, RC_NETWORK_TIMEOUT, RC_MALFORMED_RESPONSE, RC_INTEGRITY_FAILURE,
RC_SEARCH_SOURCE_BUILD_FAILURE,
RC_DELETE_APPROVAL_MISSING, RC_DELETE_APPROVAL_MISMATCH,
REASON_CODES, OPERATIONS, SYNC_STATES, BOOTSTRAP_STATES,
)
def _new_store():
tmp = tempfile.mkdtemp(prefix="c5a_test_")
return C5AStore(os.path.join(tmp, "c5a.db"))
def _commit(sha, parent=None, seq=1, objects=None):
return {
"commit_sha": sha,
"parent_sha": parent,
"discovered_at": 1000,
"sequence": seq,
"changed_objects": objects or [],
}
def _obj(object_id, op=OP_CREATE, **kw):
d = {"object_id": object_id, "operation": op}
d.update(kw)
return d
# ---------------------------------------------------------------------------
# Testfälle
# ---------------------------------------------------------------------------
def test_normal_commit_lifecycle():
"""Normaler Commit-Lebenszyklus: DISCOVERED -> ... -> APPLIED."""
store = _new_store()
sm = SyncStateMachine(store)
c = _commit("c1", objects=[_obj("obj-1")])
store.upsert_commit(c)
store.add_object_change({"commit_sha": "c1", "object_id": "obj-1", "operation": OP_CREATE})
result = sm.process_commit(c)
assert result["status"] == ST_APPLIED, result
assert store.commit_status("c1") == ST_APPLIED
assert store.health()["last_applied_commit"] == "c1"
store.close()
def test_multi_object_commit():
"""Multi-Object-Commit: mehrere Object-Changes in einem Commit."""
store = _new_store()
sm = SyncStateMachine(store)
c = _commit("c1", objects=[
_obj("obj-1"), _obj("obj-2"), _obj("obj-3"),
])
for o in c["changed_objects"]:
store.add_object_change({"commit_sha": "c1", **o})
result = sm.process_commit(c)
assert result["status"] == ST_APPLIED
assert len(store.list_object_changes("c1")) == 3
store.close()
def test_duplicate_commit():
"""Doppelter Commit: zweite Verarbeitung -> ALREADY_APPLIED, kein Re-Apply."""
store = _new_store()
sm = SyncStateMachine(store)
c = _commit("c1", objects=[_obj("obj-1")])
store.add_object_change({"commit_sha": "c1", "object_id": "obj-1", "operation": OP_CREATE})
r1 = sm.process_commit(c)
assert r1["status"] == ST_APPLIED
r2 = sm.process_commit(c)
assert r2["status"] == ST_APPLIED
assert r2["idempotency"] == IDEM_ALREADY_APPLIED
store.close()
def test_already_applied_idempotency():
"""Idempotenz: bereits angewendeter Commit -> ALREADY_APPLIED."""
store = _new_store()
sm = SyncStateMachine(store)
c = _commit("c1", objects=[_obj("obj-1")])
store.add_object_change({"commit_sha": "c1", "object_id": "obj-1", "operation": OP_CREATE})
sm.process_commit(c)
assert sm.idempotency_check("c1", "obj-1", OP_CREATE) == IDEM_ALREADY_APPLIED
store.close()
def test_out_of_order_commit():
"""Out-of-Order-Commit: fehlender Vorgänger -> WAITING_FOR_PREDECESSOR."""
store = _new_store()
sm = SyncStateMachine(store)
c = _commit("c2", parent="c1", seq=2, objects=[_obj("obj-1")])
store.add_object_change({"commit_sha": "c2", "object_id": "obj-1", "operation": OP_CREATE})
result = sm.process_commit(c)
assert result["status"] == ST_WAITING_FOR_PREDECESSOR, result
assert result["reason_code"] == RC_OUT_OF_ORDER_COMMIT
store.close()
def test_missing_predecessor_then_applied():
"""Fehlender Vorgänger -> nach Anwendung des Vorgängers verarbeitbar."""
store = _new_store()
sm = SyncStateMachine(store)
c1 = _commit("c1", seq=1, objects=[_obj("obj-1")])
store.add_object_change({"commit_sha": "c1", "object_id": "obj-1", "operation": OP_CREATE})
sm.process_commit(c1)
c2 = _commit("c2", parent="c1", seq=2, objects=[_obj("obj-2")])
store.add_object_change({"commit_sha": "c2", "object_id": "obj-2", "operation": OP_CREATE})
r2 = sm.process_commit(c2)
assert r2["status"] == ST_APPLIED, r2
store.close()
def test_retry_progression():
"""Retry-Progression: Fehler -> RETRY_PENDING mit Backoff."""
store = _new_store()
sm = SyncStateMachine(store)
c = _commit("c1", objects=[_obj("obj-1")])
store.upsert_commit(c)
store.add_object_change({"commit_sha": "c1", "object_id": "obj-1", "operation": OP_CREATE})
r = sm.retry_commit("c1", RC_TOLARIA_UNAVAILABLE, "tolaria down")
assert r["status"] == ST_RETRY_PENDING
assert r["retry_count"] == 1
assert r["backoff_seconds"] == 1
store.close()
def test_max_retry_to_dead():
"""Max-Retry -> DEAD, kein Endlos-Retry."""
store = _new_store()
sm = SyncStateMachine(store, max_retries=3)
c = _commit("c1", objects=[_obj("obj-1")])
store.upsert_commit(c)
store.add_object_change({"commit_sha": "c1", "object_id": "obj-1", "operation": OP_CREATE})
for i in range(3):
r = sm.retry_commit("c1", RC_TOLARIA_UNAVAILABLE, "down")
assert r["status"] == ST_DEAD, r
assert store.commit_status("c1") == ST_DEAD
store.close()
def test_human_gate_transition():
"""Human-Gate-Übergang: DELETE_REQUEST -> HUMAN_REVIEW_REQUIRED."""
store = _new_store()
sm = SyncStateMachine(store)
c = _commit("c1", objects=[_obj("obj-1", op=OP_DELETE_REQUEST)])
store.add_object_change({"commit_sha": "c1", "object_id": "obj-1", "operation": OP_DELETE_REQUEST})
result = sm.process_commit(c)
assert result["status"] == ST_HUMAN_REVIEW_REQUIRED, result
assert result["reason_code"] == RC_AMBIGUOUS_DELETE
store.close()
def test_restart_reload_persistence():
"""Restart-Persistenz: neuer Store auf gleicher DB behält State."""
tmp = tempfile.mkdtemp(prefix="c5a_test_")
db = os.path.join(tmp, "c5a.db")
s1 = C5AStore(db)
sm1 = SyncStateMachine(s1)
c = _commit("c1", objects=[_obj("obj-1")])
s1.add_object_change({"commit_sha": "c1", "object_id": "obj-1", "operation": OP_CREATE})
sm1.process_commit(c)
s1.close()
s2 = C5AStore(db)
assert s2.commit_status("c1") == ST_APPLIED
assert s2.health()["last_applied_commit"] == "c1"
s2.close()
def test_state_corruption_handling():
"""State-Corruption: ungültiger Übergang -> InvalidTransitionError, kein Mutation."""
store = _new_store()
sm = SyncStateMachine(store)
c = _commit("c1", objects=[_obj("obj-1")])
store.upsert_commit(c)
store.add_object_change({"commit_sha": "c1", "object_id": "obj-1", "operation": OP_CREATE})
# Direkt von DISCOVERED nach APPLIED ist nicht erlaubt
try:
store.transition_commit("c1", ST_APPLIED)
assert False, "sollte InvalidTransitionError werfen"
except InvalidTransitionError:
pass
assert store.commit_status("c1") == ST_DISCOVERED
store.close()
def test_idempotency_conflict():
"""Idempotenz-Konflikt: existierender Change mit abweichendem Hash -> CONFLICT."""
store = _new_store()
sm = SyncStateMachine(store)
c = _commit("c1", objects=[_obj("obj-1")])
store.upsert_commit(c)
store.add_object_change({"commit_sha": "c1", "object_id": "obj-1",
"operation": OP_CREATE, "content_hash_after": "hashA"})
# Nicht angewendet, Hash weicht ab -> CONFLICT
assert sm.idempotency_check("c1", "obj-1", OP_CREATE, content_hash="hashB") == IDEM_CONFLICT
# Gleicher Hash -> ALREADY_AT_TARGET
assert sm.idempotency_check("c1", "obj-1", OP_CREATE, content_hash="hashA") == IDEM_ALREADY_AT_TARGET
store.close()
def test_delete_request_human_gate():
"""DELETE_REQUEST -> Human Gate, kein automatisches Hard Delete."""
store = _new_store()
sm = SyncStateMachine(store)
c = _commit("c1", objects=[_obj("obj-1", op=OP_DELETE_REQUEST)])
store.add_object_change({"commit_sha": "c1", "object_id": "obj-1", "operation": OP_DELETE_REQUEST})
result = sm.process_commit(c)
assert result["status"] == ST_HUMAN_REVIEW_REQUIRED
assert result["reason_code"] == RC_AMBIGUOUS_DELETE
store.close()
def test_dangling_relation_human_gate():
"""Dangling derived_from -> Human Gate."""
store = _new_store()
sm = SyncStateMachine(store)
c = _commit("c1", objects=[_obj("obj-1", derived_from="nonexistent-obj")])
store.add_object_change({"commit_sha": "c1", "object_id": "obj-1",
"operation": OP_CREATE, "derived_from": "nonexistent-obj"})
result = sm.process_commit(c)
assert result["status"] == ST_HUMAN_REVIEW_REQUIRED
assert result["reason_code"] == RC_DANGLING_DERIVED_FROM
store.close()
def test_unexpected_drift_human_gate():
"""Unexpected Drift -> Human Gate."""
store = _new_store()
sm = SyncStateMachine(store)
c = _commit("c1", objects=[_obj("obj-1", drift=True)])
store.add_object_change({"commit_sha": "c1", "object_id": "obj-1",
"operation": OP_CREATE, "drift": True})
result = sm.process_commit(c)
assert result["status"] == ST_HUMAN_REVIEW_REQUIRED
assert result["reason_code"] == RC_UNEXPECTED_TOLARIA_DRIFT
store.close()
def test_no_write_guarantee():
"""No-Write-Guarantee: keine Netzwerk-/HTTP-/Socket-/Subprocess-Imports."""
result = assert_no_write_guarantee()
assert result["no_write_guarantee"] is True, result
assert result["banned_imports_found"] == []
store.close() if False else None
def test_bootstrap_lifecycle():
"""Bootstrap: UNINITIALIZED -> RECONCILING -> BASELINE_READY -> ACTIVE."""
store = _new_store()
assert store.bootstrap_state() == BS_UNINITIALIZED
store.bootstrap_transition(BS_RECONCILING)
assert store.bootstrap_state() == BS_RECONCILING
store.bootstrap_transition(BS_BASELINE_READY)
assert store.bootstrap_state() == BS_BASELINE_READY
store.set_baseline("base-1")
assert store.baseline_commit() == "base-1"
store.bootstrap_transition(BS_ACTIVE)
assert store.bootstrap_state() == BS_ACTIVE
store.close()
def test_baseline_only_from_baseline_ready():
"""Baseline darf nur aus BASELINE_READY gesetzt werden."""
store = _new_store()
try:
store.set_baseline("base-1")
assert False, "sollte C5AError werfen"
except C5AError:
pass
store.close()
def test_reason_codes_closed_set():
"""Reason Codes: feste, geschlossene Menge — kein freier String-Wildwuchs."""
assert RC_UNEXPECTED_TOLARIA_DRIFT in REASON_CODES
assert RC_ID_COLLISION in REASON_CODES
assert RC_UNKNOWN_OBJECT_ID in REASON_CODES
assert RC_INVALID_SCHEMA in REASON_CODES
assert RC_DANGLING_DERIVED_FROM in REASON_CODES
assert RC_AMBIGUOUS_DELETE in REASON_CODES
assert RC_UNKNOWN_LEGACY_OBJECT in REASON_CODES
assert RC_AUTH_FAILURE in REASON_CODES
assert RC_SEARCH_REBUILD_FAILURE in REASON_CODES
assert RC_TOLARIA_UNAVAILABLE in REASON_CODES
assert RC_FORGEJO_UNAVAILABLE in REASON_CODES
assert RC_OUT_OF_ORDER_COMMIT in REASON_CODES
assert RC_SECRET_DETECTED in REASON_CODES
# C5E-Erweiterung (§2): Failure-/Recovery-Contract
assert RC_SEARCH_UNAVAILABLE in REASON_CODES
assert RC_NETWORK_TIMEOUT in REASON_CODES
assert RC_MALFORMED_RESPONSE in REASON_CODES
assert RC_INTEGRITY_FAILURE in REASON_CODES
# C5D-Erweiterung (search-source pipeline, commit 5e41915)
assert RC_SEARCH_SOURCE_BUILD_FAILURE in REASON_CODES
# C5 DELETE-Execution-Erweiterung (Human-Gated DELETE Contract)
assert RC_DELETE_APPROVAL_MISSING in REASON_CODES
assert RC_DELETE_APPROVAL_MISMATCH in REASON_CODES
assert len(REASON_CODES) == 20
store = _new_store()
try:
store.set_commit_error("c1", "FREIER_STRING", "x")
assert False, "sollte InvalidReasonCodeError werfen"
except InvalidReasonCodeError:
pass
store.close()
def test_operations_closed_set():
"""Operation Model: geschlossene Menge."""
assert OP_CREATE in OPERATIONS
assert OP_CONTENT_UPDATE in OPERATIONS
assert OP_DELETE_REQUEST in OPERATIONS
assert len(OPERATIONS) == 10
store = _new_store()
try:
store.add_object_change({"commit_sha": "c1", "object_id": "o",
"operation": "BOGUS"})
assert False, "sollte InvalidOperationError werfen"
except InvalidOperationError:
pass
store.close()
def test_health_contract_fields():
"""Health Contract: alle geforderten Felder vorhanden."""
store = _new_store()
h = store.health()
for field in ["status", "bootstrap_state", "last_seen_commit", "last_applied_commit",
"pending_commits", "failed_commits", "dead_commits",
"human_review_required", "forgejo_status", "tolaria_status",
"search_status", "drift_count", "last_success_at", "last_error_code"]:
assert field in h, f"fehlendes Health-Feld: {field}"
store.close()
def test_last_applied_only_after_full_pass():
"""last_applied_commit wird NUR nach vollständigem Tolaria+Search PASS fortgeschrieben."""
store = _new_store()
sm = SyncStateMachine(store)
# Commit mit DELETE_REQUEST -> Human Gate, darf NICHT applied werden
c = _commit("c1", objects=[_obj("obj-1", op=OP_DELETE_REQUEST)])
store.add_object_change({"commit_sha": "c1", "object_id": "obj-1", "operation": OP_DELETE_REQUEST})
sm.process_commit(c)
assert store.health()["last_applied_commit"] is None
store.close()
def test_retry_available_bound():
"""retry_available: false nach Erreichen von max_retries."""
store = _new_store()
sm = SyncStateMachine(store, max_retries=2)
c = _commit("c1", objects=[_obj("obj-1")])
store.upsert_commit(c)
store.add_object_change({"commit_sha": "c1", "object_id": "obj-1", "operation": OP_CREATE})
assert sm.retry_available("c1") is True
sm.retry_commit("c1", RC_TOLARIA_UNAVAILABLE, "down")
assert sm.retry_available("c1") is True
sm.retry_commit("c1", RC_TOLARIA_UNAVAILABLE, "down")
assert sm.retry_available("c1") is False
store.close()
def test_backoff_sequence():
"""Backoff: 1/2/4/8/16 Sekunden."""
store = _new_store()
sm = SyncStateMachine(store)
assert sm.backoff_for(1) == 1
assert sm.backoff_for(2) == 2
assert sm.backoff_for(3) == 4
assert sm.backoff_for(4) == 8
assert sm.backoff_for(5) == 16
assert sm.backoff_for(6) == 16 # gekappt
store.close()
def test_invalid_reason_code_rejected():
"""Unbekannter Reason Code wird abgelehnt."""
store = _new_store()
sm = SyncStateMachine(store)
c = _commit("c1", objects=[_obj("obj-1")])
store.upsert_commit(c)
store.add_object_change({"commit_sha": "c1", "object_id": "obj-1", "operation": OP_CREATE})
try:
sm.retry_commit("c1", "NOT_A_REASON_CODE", "x")
assert False, "sollte InvalidReasonCodeError werfen"
except InvalidReasonCodeError:
pass
store.close()
# ---------------------------------------------------------------------------
# Runner
# ---------------------------------------------------------------------------
ALL_TESTS = [
test_normal_commit_lifecycle,
test_multi_object_commit,
test_duplicate_commit,
test_already_applied_idempotency,
test_out_of_order_commit,
test_missing_predecessor_then_applied,
test_retry_progression,
test_max_retry_to_dead,
test_human_gate_transition,
test_restart_reload_persistence,
test_state_corruption_handling,
test_idempotency_conflict,
test_delete_request_human_gate,
test_dangling_relation_human_gate,
test_unexpected_drift_human_gate,
test_no_write_guarantee,
test_bootstrap_lifecycle,
test_baseline_only_from_baseline_ready,
test_reason_codes_closed_set,
test_operations_closed_set,
test_health_contract_fields,
test_last_applied_only_after_full_pass,
test_retry_available_bound,
test_backoff_sequence,
test_invalid_reason_code_rejected,
]
def main() -> int:
passed = 0
failed = 0
failures = []
for t in ALL_TESTS:
try:
t()
passed += 1
print(f"PASS {t.__name__}")
except Exception as e:
failed += 1
failures.append((t.__name__, e))
print(f"FAIL {t.__name__}: {e}")
traceback.print_exc()
print(f"\n=== C5A: {passed} PASS / {failed} FAIL ===")
if failures:
for name, e in failures:
print(f" FAILED: {name} -> {e}")
return 1 if failed else 0
if __name__ == "__main__":
sys.exit(main())