fix: Retry sentinel startup on failure

In particular, retry N times (N = 3 for now), increasing the retry delay on each failure (linearly, for the moment). Co-authored-by: Michael L. Szulczewski <[email protected]> Signed-off-by: Sam Stuewe <[email protected]>
mit-dci · Dec 6, 2022 · 3411d13 · 3411d13
1 parent 5b3e8b1
commit 3411d13
Showing 1 changed file with 8 additions and 2 deletions.
diff --git a/src/uhs/twophase/sentinel_2pc/controller.cpp b/src/uhs/twophase/sentinel_2pc/controller.cpp
@@ -49,8 +49,14 @@ namespace cbdc::sentinel_2pc {
             m_logger->info("Sentinel public key:", cbdc::to_string(pubkey));
         }
 
-        if(!m_coordinator_client.init()) {
-            m_logger->warn("Failed to start coordinator client");
+        auto retry_delay = std::chrono::seconds(1);
+        auto retry_threshold = 3;
+        while(!m_coordinator_client.init() && retry_threshold-- > 0) {
+            m_logger->warn("Failed to start coordinator client.");
+            if(retry_threshold > 0) {
+                m_logger->warn("Retrying...");
+            }
+            std::this_thread::sleep_for(retry_delay++);
         }
 
         for(const auto& ep : m_opts.m_sentinel_endpoints) {