475 lines
17 KiB
Python
475 lines
17 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
Red Queen — C5A: CONTRACT & STATE MACHINE Testsuite.
|
|
|
|
Deterministische Unit Tests. Jeder Test nutzt eine frische temp-DB (tempfile.mkdtemp),
|
|
niemals die Produkt-DB. Beweist die No-Write-Guarantee und alle geforderten
|
|
State-Machine-, Idempotenz-, Ordering-, Retry-, Human-Gate-, Bootstrap- und
|
|
Persistenz-Verhalten.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import os
|
|
import sys
|
|
import tempfile
|
|
import traceback
|
|
from pathlib import Path
|
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
|
|
|
from rq_c5a import (
|
|
C5AStore,
|
|
SyncStateMachine,
|
|
assert_no_write_guarantee,
|
|
C5AError,
|
|
InvalidTransitionError,
|
|
InvalidReasonCodeError,
|
|
InvalidOperationError,
|
|
ST_DISCOVERED, ST_VALIDATING, ST_READY,
|
|
ST_PROPAGATING_TOLARIA, ST_VERIFYING_TOLARIA,
|
|
ST_UPDATING_SEARCH, ST_VERIFYING_SEARCH, ST_APPLIED,
|
|
ST_RETRY_PENDING, ST_FAILED, ST_DEAD, ST_HUMAN_REVIEW_REQUIRED,
|
|
ST_WAITING_FOR_PREDECESSOR,
|
|
BS_UNINITIALIZED, BS_RECONCILING, BS_BASELINE_READY, BS_ACTIVE,
|
|
OP_CREATE, OP_CONTENT_UPDATE, OP_DELETE_REQUEST,
|
|
IDEM_ALREADY_APPLIED, IDEM_ALREADY_AT_TARGET, IDEM_RETRY_SAFE, IDEM_CONFLICT,
|
|
RC_UNEXPECTED_TOLARIA_DRIFT, RC_ID_COLLISION, RC_UNKNOWN_OBJECT_ID,
|
|
RC_INVALID_SCHEMA, RC_DANGLING_DERIVED_FROM, RC_AMBIGUOUS_DELETE,
|
|
RC_OUT_OF_ORDER_COMMIT, RC_SECRET_DETECTED,
|
|
RC_UNKNOWN_LEGACY_OBJECT, RC_AUTH_FAILURE, RC_SEARCH_REBUILD_FAILURE,
|
|
RC_TOLARIA_UNAVAILABLE, RC_FORGEJO_UNAVAILABLE,
|
|
REASON_CODES, OPERATIONS, SYNC_STATES, BOOTSTRAP_STATES,
|
|
)
|
|
|
|
|
|
def _new_store():
|
|
tmp = tempfile.mkdtemp(prefix="c5a_test_")
|
|
return C5AStore(os.path.join(tmp, "c5a.db"))
|
|
|
|
|
|
def _commit(sha, parent=None, seq=1, objects=None):
|
|
return {
|
|
"commit_sha": sha,
|
|
"parent_sha": parent,
|
|
"discovered_at": 1000,
|
|
"sequence": seq,
|
|
"changed_objects": objects or [],
|
|
}
|
|
|
|
|
|
def _obj(object_id, op=OP_CREATE, **kw):
|
|
d = {"object_id": object_id, "operation": op}
|
|
d.update(kw)
|
|
return d
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Testfälle
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def test_normal_commit_lifecycle():
|
|
"""Normaler Commit-Lebenszyklus: DISCOVERED -> ... -> APPLIED."""
|
|
store = _new_store()
|
|
sm = SyncStateMachine(store)
|
|
c = _commit("c1", objects=[_obj("obj-1")])
|
|
store.upsert_commit(c)
|
|
store.add_object_change({"commit_sha": "c1", "object_id": "obj-1", "operation": OP_CREATE})
|
|
result = sm.process_commit(c)
|
|
assert result["status"] == ST_APPLIED, result
|
|
assert store.commit_status("c1") == ST_APPLIED
|
|
assert store.health()["last_applied_commit"] == "c1"
|
|
store.close()
|
|
|
|
|
|
def test_multi_object_commit():
|
|
"""Multi-Object-Commit: mehrere Object-Changes in einem Commit."""
|
|
store = _new_store()
|
|
sm = SyncStateMachine(store)
|
|
c = _commit("c1", objects=[
|
|
_obj("obj-1"), _obj("obj-2"), _obj("obj-3"),
|
|
])
|
|
for o in c["changed_objects"]:
|
|
store.add_object_change({"commit_sha": "c1", **o})
|
|
result = sm.process_commit(c)
|
|
assert result["status"] == ST_APPLIED
|
|
assert len(store.list_object_changes("c1")) == 3
|
|
store.close()
|
|
|
|
|
|
def test_duplicate_commit():
|
|
"""Doppelter Commit: zweite Verarbeitung -> ALREADY_APPLIED, kein Re-Apply."""
|
|
store = _new_store()
|
|
sm = SyncStateMachine(store)
|
|
c = _commit("c1", objects=[_obj("obj-1")])
|
|
store.add_object_change({"commit_sha": "c1", "object_id": "obj-1", "operation": OP_CREATE})
|
|
r1 = sm.process_commit(c)
|
|
assert r1["status"] == ST_APPLIED
|
|
r2 = sm.process_commit(c)
|
|
assert r2["status"] == ST_APPLIED
|
|
assert r2["idempotency"] == IDEM_ALREADY_APPLIED
|
|
store.close()
|
|
|
|
|
|
def test_already_applied_idempotency():
|
|
"""Idempotenz: bereits angewendeter Commit -> ALREADY_APPLIED."""
|
|
store = _new_store()
|
|
sm = SyncStateMachine(store)
|
|
c = _commit("c1", objects=[_obj("obj-1")])
|
|
store.add_object_change({"commit_sha": "c1", "object_id": "obj-1", "operation": OP_CREATE})
|
|
sm.process_commit(c)
|
|
assert sm.idempotency_check("c1", "obj-1", OP_CREATE) == IDEM_ALREADY_APPLIED
|
|
store.close()
|
|
|
|
|
|
def test_out_of_order_commit():
|
|
"""Out-of-Order-Commit: fehlender Vorgänger -> WAITING_FOR_PREDECESSOR."""
|
|
store = _new_store()
|
|
sm = SyncStateMachine(store)
|
|
c = _commit("c2", parent="c1", seq=2, objects=[_obj("obj-1")])
|
|
store.add_object_change({"commit_sha": "c2", "object_id": "obj-1", "operation": OP_CREATE})
|
|
result = sm.process_commit(c)
|
|
assert result["status"] == ST_WAITING_FOR_PREDECESSOR, result
|
|
assert result["reason_code"] == RC_OUT_OF_ORDER_COMMIT
|
|
store.close()
|
|
|
|
|
|
def test_missing_predecessor_then_applied():
|
|
"""Fehlender Vorgänger -> nach Anwendung des Vorgängers verarbeitbar."""
|
|
store = _new_store()
|
|
sm = SyncStateMachine(store)
|
|
c1 = _commit("c1", seq=1, objects=[_obj("obj-1")])
|
|
store.add_object_change({"commit_sha": "c1", "object_id": "obj-1", "operation": OP_CREATE})
|
|
sm.process_commit(c1)
|
|
c2 = _commit("c2", parent="c1", seq=2, objects=[_obj("obj-2")])
|
|
store.add_object_change({"commit_sha": "c2", "object_id": "obj-2", "operation": OP_CREATE})
|
|
r2 = sm.process_commit(c2)
|
|
assert r2["status"] == ST_APPLIED, r2
|
|
store.close()
|
|
|
|
|
|
def test_retry_progression():
|
|
"""Retry-Progression: Fehler -> RETRY_PENDING mit Backoff."""
|
|
store = _new_store()
|
|
sm = SyncStateMachine(store)
|
|
c = _commit("c1", objects=[_obj("obj-1")])
|
|
store.upsert_commit(c)
|
|
store.add_object_change({"commit_sha": "c1", "object_id": "obj-1", "operation": OP_CREATE})
|
|
r = sm.retry_commit("c1", RC_TOLARIA_UNAVAILABLE, "tolaria down")
|
|
assert r["status"] == ST_RETRY_PENDING
|
|
assert r["retry_count"] == 1
|
|
assert r["backoff_seconds"] == 1
|
|
store.close()
|
|
|
|
|
|
def test_max_retry_to_dead():
|
|
"""Max-Retry -> DEAD, kein Endlos-Retry."""
|
|
store = _new_store()
|
|
sm = SyncStateMachine(store, max_retries=3)
|
|
c = _commit("c1", objects=[_obj("obj-1")])
|
|
store.upsert_commit(c)
|
|
store.add_object_change({"commit_sha": "c1", "object_id": "obj-1", "operation": OP_CREATE})
|
|
for i in range(3):
|
|
r = sm.retry_commit("c1", RC_TOLARIA_UNAVAILABLE, "down")
|
|
assert r["status"] == ST_DEAD, r
|
|
assert store.commit_status("c1") == ST_DEAD
|
|
store.close()
|
|
|
|
|
|
def test_human_gate_transition():
|
|
"""Human-Gate-Übergang: DELETE_REQUEST -> HUMAN_REVIEW_REQUIRED."""
|
|
store = _new_store()
|
|
sm = SyncStateMachine(store)
|
|
c = _commit("c1", objects=[_obj("obj-1", op=OP_DELETE_REQUEST)])
|
|
store.add_object_change({"commit_sha": "c1", "object_id": "obj-1", "operation": OP_DELETE_REQUEST})
|
|
result = sm.process_commit(c)
|
|
assert result["status"] == ST_HUMAN_REVIEW_REQUIRED, result
|
|
assert result["reason_code"] == RC_AMBIGUOUS_DELETE
|
|
store.close()
|
|
|
|
|
|
def test_restart_reload_persistence():
|
|
"""Restart-Persistenz: neuer Store auf gleicher DB behält State."""
|
|
tmp = tempfile.mkdtemp(prefix="c5a_test_")
|
|
db = os.path.join(tmp, "c5a.db")
|
|
s1 = C5AStore(db)
|
|
sm1 = SyncStateMachine(s1)
|
|
c = _commit("c1", objects=[_obj("obj-1")])
|
|
s1.add_object_change({"commit_sha": "c1", "object_id": "obj-1", "operation": OP_CREATE})
|
|
sm1.process_commit(c)
|
|
s1.close()
|
|
s2 = C5AStore(db)
|
|
assert s2.commit_status("c1") == ST_APPLIED
|
|
assert s2.health()["last_applied_commit"] == "c1"
|
|
s2.close()
|
|
|
|
|
|
def test_state_corruption_handling():
|
|
"""State-Corruption: ungültiger Übergang -> InvalidTransitionError, kein Mutation."""
|
|
store = _new_store()
|
|
sm = SyncStateMachine(store)
|
|
c = _commit("c1", objects=[_obj("obj-1")])
|
|
store.upsert_commit(c)
|
|
store.add_object_change({"commit_sha": "c1", "object_id": "obj-1", "operation": OP_CREATE})
|
|
# Direkt von DISCOVERED nach APPLIED ist nicht erlaubt
|
|
try:
|
|
store.transition_commit("c1", ST_APPLIED)
|
|
assert False, "sollte InvalidTransitionError werfen"
|
|
except InvalidTransitionError:
|
|
pass
|
|
assert store.commit_status("c1") == ST_DISCOVERED
|
|
store.close()
|
|
|
|
|
|
def test_idempotency_conflict():
|
|
"""Idempotenz-Konflikt: existierender Change mit abweichendem Hash -> CONFLICT."""
|
|
store = _new_store()
|
|
sm = SyncStateMachine(store)
|
|
c = _commit("c1", objects=[_obj("obj-1")])
|
|
store.upsert_commit(c)
|
|
store.add_object_change({"commit_sha": "c1", "object_id": "obj-1",
|
|
"operation": OP_CREATE, "content_hash_after": "hashA"})
|
|
# Nicht angewendet, Hash weicht ab -> CONFLICT
|
|
assert sm.idempotency_check("c1", "obj-1", OP_CREATE, content_hash="hashB") == IDEM_CONFLICT
|
|
# Gleicher Hash -> ALREADY_AT_TARGET
|
|
assert sm.idempotency_check("c1", "obj-1", OP_CREATE, content_hash="hashA") == IDEM_ALREADY_AT_TARGET
|
|
store.close()
|
|
|
|
|
|
def test_delete_request_human_gate():
|
|
"""DELETE_REQUEST -> Human Gate, kein automatisches Hard Delete."""
|
|
store = _new_store()
|
|
sm = SyncStateMachine(store)
|
|
c = _commit("c1", objects=[_obj("obj-1", op=OP_DELETE_REQUEST)])
|
|
store.add_object_change({"commit_sha": "c1", "object_id": "obj-1", "operation": OP_DELETE_REQUEST})
|
|
result = sm.process_commit(c)
|
|
assert result["status"] == ST_HUMAN_REVIEW_REQUIRED
|
|
assert result["reason_code"] == RC_AMBIGUOUS_DELETE
|
|
store.close()
|
|
|
|
|
|
def test_dangling_relation_human_gate():
|
|
"""Dangling derived_from -> Human Gate."""
|
|
store = _new_store()
|
|
sm = SyncStateMachine(store)
|
|
c = _commit("c1", objects=[_obj("obj-1", derived_from="nonexistent-obj")])
|
|
store.add_object_change({"commit_sha": "c1", "object_id": "obj-1",
|
|
"operation": OP_CREATE, "derived_from": "nonexistent-obj"})
|
|
result = sm.process_commit(c)
|
|
assert result["status"] == ST_HUMAN_REVIEW_REQUIRED
|
|
assert result["reason_code"] == RC_DANGLING_DERIVED_FROM
|
|
store.close()
|
|
|
|
|
|
def test_unexpected_drift_human_gate():
|
|
"""Unexpected Drift -> Human Gate."""
|
|
store = _new_store()
|
|
sm = SyncStateMachine(store)
|
|
c = _commit("c1", objects=[_obj("obj-1", drift=True)])
|
|
store.add_object_change({"commit_sha": "c1", "object_id": "obj-1",
|
|
"operation": OP_CREATE, "drift": True})
|
|
result = sm.process_commit(c)
|
|
assert result["status"] == ST_HUMAN_REVIEW_REQUIRED
|
|
assert result["reason_code"] == RC_UNEXPECTED_TOLARIA_DRIFT
|
|
store.close()
|
|
|
|
|
|
def test_no_write_guarantee():
|
|
"""No-Write-Guarantee: keine Netzwerk-/HTTP-/Socket-/Subprocess-Imports."""
|
|
result = assert_no_write_guarantee()
|
|
assert result["no_write_guarantee"] is True, result
|
|
assert result["banned_imports_found"] == []
|
|
store.close() if False else None
|
|
|
|
|
|
def test_bootstrap_lifecycle():
|
|
"""Bootstrap: UNINITIALIZED -> RECONCILING -> BASELINE_READY -> ACTIVE."""
|
|
store = _new_store()
|
|
assert store.bootstrap_state() == BS_UNINITIALIZED
|
|
store.bootstrap_transition(BS_RECONCILING)
|
|
assert store.bootstrap_state() == BS_RECONCILING
|
|
store.bootstrap_transition(BS_BASELINE_READY)
|
|
assert store.bootstrap_state() == BS_BASELINE_READY
|
|
store.set_baseline("base-1")
|
|
assert store.baseline_commit() == "base-1"
|
|
store.bootstrap_transition(BS_ACTIVE)
|
|
assert store.bootstrap_state() == BS_ACTIVE
|
|
store.close()
|
|
|
|
|
|
def test_baseline_only_from_baseline_ready():
|
|
"""Baseline darf nur aus BASELINE_READY gesetzt werden."""
|
|
store = _new_store()
|
|
try:
|
|
store.set_baseline("base-1")
|
|
assert False, "sollte C5AError werfen"
|
|
except C5AError:
|
|
pass
|
|
store.close()
|
|
|
|
|
|
def test_reason_codes_closed_set():
|
|
"""Reason Codes: feste, geschlossene Menge — kein freier String-Wildwuchs."""
|
|
assert RC_UNEXPECTED_TOLARIA_DRIFT in REASON_CODES
|
|
assert RC_ID_COLLISION in REASON_CODES
|
|
assert RC_UNKNOWN_OBJECT_ID in REASON_CODES
|
|
assert RC_INVALID_SCHEMA in REASON_CODES
|
|
assert RC_DANGLING_DERIVED_FROM in REASON_CODES
|
|
assert RC_AMBIGUOUS_DELETE in REASON_CODES
|
|
assert RC_UNKNOWN_LEGACY_OBJECT in REASON_CODES
|
|
assert RC_AUTH_FAILURE in REASON_CODES
|
|
assert RC_SEARCH_REBUILD_FAILURE in REASON_CODES
|
|
assert RC_TOLARIA_UNAVAILABLE in REASON_CODES
|
|
assert RC_FORGEJO_UNAVAILABLE in REASON_CODES
|
|
assert RC_OUT_OF_ORDER_COMMIT in REASON_CODES
|
|
assert RC_SECRET_DETECTED in REASON_CODES
|
|
assert len(REASON_CODES) == 13
|
|
store = _new_store()
|
|
try:
|
|
store.set_commit_error("c1", "FREIER_STRING", "x")
|
|
assert False, "sollte InvalidReasonCodeError werfen"
|
|
except InvalidReasonCodeError:
|
|
pass
|
|
store.close()
|
|
|
|
|
|
def test_operations_closed_set():
|
|
"""Operation Model: geschlossene Menge."""
|
|
assert OP_CREATE in OPERATIONS
|
|
assert OP_CONTENT_UPDATE in OPERATIONS
|
|
assert OP_DELETE_REQUEST in OPERATIONS
|
|
assert len(OPERATIONS) == 10
|
|
store = _new_store()
|
|
try:
|
|
store.add_object_change({"commit_sha": "c1", "object_id": "o",
|
|
"operation": "BOGUS"})
|
|
assert False, "sollte InvalidOperationError werfen"
|
|
except InvalidOperationError:
|
|
pass
|
|
store.close()
|
|
|
|
|
|
def test_health_contract_fields():
|
|
"""Health Contract: alle geforderten Felder vorhanden."""
|
|
store = _new_store()
|
|
h = store.health()
|
|
for field in ["status", "bootstrap_state", "last_seen_commit", "last_applied_commit",
|
|
"pending_commits", "failed_commits", "dead_commits",
|
|
"human_review_required", "forgejo_status", "tolaria_status",
|
|
"search_status", "drift_count", "last_success_at", "last_error_code"]:
|
|
assert field in h, f"fehlendes Health-Feld: {field}"
|
|
store.close()
|
|
|
|
|
|
def test_last_applied_only_after_full_pass():
|
|
"""last_applied_commit wird NUR nach vollständigem Tolaria+Search PASS fortgeschrieben."""
|
|
store = _new_store()
|
|
sm = SyncStateMachine(store)
|
|
# Commit mit DELETE_REQUEST -> Human Gate, darf NICHT applied werden
|
|
c = _commit("c1", objects=[_obj("obj-1", op=OP_DELETE_REQUEST)])
|
|
store.add_object_change({"commit_sha": "c1", "object_id": "obj-1", "operation": OP_DELETE_REQUEST})
|
|
sm.process_commit(c)
|
|
assert store.health()["last_applied_commit"] is None
|
|
store.close()
|
|
|
|
|
|
def test_retry_available_bound():
|
|
"""retry_available: false nach Erreichen von max_retries."""
|
|
store = _new_store()
|
|
sm = SyncStateMachine(store, max_retries=2)
|
|
c = _commit("c1", objects=[_obj("obj-1")])
|
|
store.upsert_commit(c)
|
|
store.add_object_change({"commit_sha": "c1", "object_id": "obj-1", "operation": OP_CREATE})
|
|
assert sm.retry_available("c1") is True
|
|
sm.retry_commit("c1", RC_TOLARIA_UNAVAILABLE, "down")
|
|
assert sm.retry_available("c1") is True
|
|
sm.retry_commit("c1", RC_TOLARIA_UNAVAILABLE, "down")
|
|
assert sm.retry_available("c1") is False
|
|
store.close()
|
|
|
|
|
|
def test_backoff_sequence():
|
|
"""Backoff: 1/2/4/8/16 Sekunden."""
|
|
store = _new_store()
|
|
sm = SyncStateMachine(store)
|
|
assert sm.backoff_for(1) == 1
|
|
assert sm.backoff_for(2) == 2
|
|
assert sm.backoff_for(3) == 4
|
|
assert sm.backoff_for(4) == 8
|
|
assert sm.backoff_for(5) == 16
|
|
assert sm.backoff_for(6) == 16 # gekappt
|
|
store.close()
|
|
|
|
|
|
def test_invalid_reason_code_rejected():
|
|
"""Unbekannter Reason Code wird abgelehnt."""
|
|
store = _new_store()
|
|
sm = SyncStateMachine(store)
|
|
c = _commit("c1", objects=[_obj("obj-1")])
|
|
store.upsert_commit(c)
|
|
store.add_object_change({"commit_sha": "c1", "object_id": "obj-1", "operation": OP_CREATE})
|
|
try:
|
|
sm.retry_commit("c1", "NOT_A_REASON_CODE", "x")
|
|
assert False, "sollte InvalidReasonCodeError werfen"
|
|
except InvalidReasonCodeError:
|
|
pass
|
|
store.close()
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Runner
|
|
# ---------------------------------------------------------------------------
|
|
|
|
ALL_TESTS = [
|
|
test_normal_commit_lifecycle,
|
|
test_multi_object_commit,
|
|
test_duplicate_commit,
|
|
test_already_applied_idempotency,
|
|
test_out_of_order_commit,
|
|
test_missing_predecessor_then_applied,
|
|
test_retry_progression,
|
|
test_max_retry_to_dead,
|
|
test_human_gate_transition,
|
|
test_restart_reload_persistence,
|
|
test_state_corruption_handling,
|
|
test_idempotency_conflict,
|
|
test_delete_request_human_gate,
|
|
test_dangling_relation_human_gate,
|
|
test_unexpected_drift_human_gate,
|
|
test_no_write_guarantee,
|
|
test_bootstrap_lifecycle,
|
|
test_baseline_only_from_baseline_ready,
|
|
test_reason_codes_closed_set,
|
|
test_operations_closed_set,
|
|
test_health_contract_fields,
|
|
test_last_applied_only_after_full_pass,
|
|
test_retry_available_bound,
|
|
test_backoff_sequence,
|
|
test_invalid_reason_code_rejected,
|
|
]
|
|
|
|
|
|
def main() -> int:
|
|
passed = 0
|
|
failed = 0
|
|
failures = []
|
|
for t in ALL_TESTS:
|
|
try:
|
|
t()
|
|
passed += 1
|
|
print(f"PASS {t.__name__}")
|
|
except Exception as e:
|
|
failed += 1
|
|
failures.append((t.__name__, e))
|
|
print(f"FAIL {t.__name__}: {e}")
|
|
traceback.print_exc()
|
|
print(f"\n=== C5A: {passed} PASS / {failed} FAIL ===")
|
|
if failures:
|
|
for name, e in failures:
|
|
print(f" FAILED: {name} -> {e}")
|
|
return 1 if failed else 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|