From 8d7647b11380f204f0677121f7f15ef9f5a821eb Mon Sep 17 00:00:00 2001 From: Red Queen Date: Wed, 26 Aug 2026 05:57:55 +0000 Subject: [PATCH] fix(tolaria): repair C5C retry state transitions and add retry tests --- tolaria/c5-sync-service/rq_c5a.py | 2 ++ tolaria/c5-sync-service/test_c5c.py | 36 +++++++++++++++++++++++++++++ 2 files changed, 38 insertions(+) diff --git a/tolaria/c5-sync-service/rq_c5a.py b/tolaria/c5-sync-service/rq_c5a.py index ddca63b..f710d25 100644 --- a/tolaria/c5-sync-service/rq_c5a.py +++ b/tolaria/c5-sync-service/rq_c5a.py @@ -175,6 +175,7 @@ _ALLOWED_TRANSITIONS = { (ST_READY, ST_PROPAGATING_TOLARIA), (ST_PROPAGATING_TOLARIA, ST_VERIFYING_TOLARIA), (ST_PROPAGATING_TOLARIA, ST_RETRY_PENDING), # Tolaria-Write-Fehler + (ST_PROPAGATING_TOLARIA, ST_DEAD), # C5C: Retry-Limit erreicht waehrend Propagation (ST_PROPAGATING_TOLARIA, ST_HUMAN_REVIEW_REQUIRED), # C5C: Drift/Secret/dangling waehrend Propagation (ST_VERIFYING_TOLARIA, ST_UPDATING_SEARCH), (ST_VERIFYING_TOLARIA, ST_RETRY_PENDING), # DRIFT != 0 / Tolaria-Fehler @@ -186,6 +187,7 @@ _ALLOWED_TRANSITIONS = { (ST_VERIFYING_SEARCH, ST_RETRY_PENDING), # Search-Health-Fehler (ST_VERIFYING_SEARCH, ST_HUMAN_REVIEW_REQUIRED), (ST_RETRY_PENDING, ST_READY), # Retry -> erneut propagieren + (ST_RETRY_PENDING, ST_PROPAGATING_TOLARIA), # C5C: Retry-Replay -> erneut propagieren (ST_RETRY_PENDING, ST_RETRY_PENDING), # idempotenter erneuter Retry-Versuch (ST_RETRY_PENDING, ST_FAILED), # Retry-Limit erreicht (ST_RETRY_PENDING, ST_DEAD), # Max-Retry -> DEAD diff --git a/tolaria/c5-sync-service/test_c5c.py b/tolaria/c5-sync-service/test_c5c.py index febbf59..cf8a8d5 100644 --- a/tolaria/c5-sync-service/test_c5c.py +++ b/tolaria/c5-sync-service/test_c5c.py @@ -670,6 +670,42 @@ class TestRetry(unittest.TestCase): self.assertEqual(res["status"], ST_RETRY_PENDING) self.assertEqual(res["results"][0]["retry_count"], 1) # _handle_failure hat increment_retry aufgerufen + def test_retry_replay_after_recovery(self): + # Commit in RETRY_PENDING -> Tolaria wieder up -> Replay moeglich + content = _fm(UUID_A, state="current") + "body" + self.repo.write("modul-09.md", content) + sha = self.repo.commit("create") + _seed_commit(self.store, sha, None, [{ + "object_id": UUID_A, "operation": OP_CREATE, + "path_before": None, "path_after": "modul-09.md", + "content_hash_after": content_hash(parse_frontmatter(content)[1]), + }]) + # Erster Lauf: Tolaria down -> RETRY_PENDING + self.fake.unavailable = True + res1 = self.prop.propagate_commit(sha) + self.assertEqual(res1["status"], ST_RETRY_PENDING) + # Zweiter Lauf: Tolaria up -> Replay muss PROPAGATING_TOLARIA erreichen + self.fake.unavailable = False + res2 = self.prop.propagate_commit(sha) + self.assertEqual(res2["status"], ST_UPDATING_SEARCH) # vollstaendig propagiert + self.assertEqual(self.fake.write_count, 1) # genau ein Write + + def test_max_retries_reaches_dead(self): + # max_retries=1 -> nach 1 Fehlschlag DEAD + content = _fm(UUID_A, state="current") + "body" + self.repo.write("modul-09.md", content) + sha = self.repo.commit("create") + _seed_commit(self.store, sha, None, [{ + "object_id": UUID_A, "operation": OP_CREATE, + "path_before": None, "path_after": "modul-09.md", + "content_hash_after": content_hash(parse_frontmatter(content)[1]), + }]) + self.fake.unavailable = True + prop = C5CPropagator(self.store, self.reader, self.client, max_retries=1) + res = prop.propagate_commit(sha) + self.assertEqual(res["status"], ST_DEAD) + self.assertEqual(self.store.commit_status(sha), ST_DEAD) + def test_drift_not_retryable(self): # Drift ist NICHT retrybar -> direkt Human Gate before = _fm(UUID_A, state="current") + "body v1"