diff --git a/src/main/kotlin/deployment/DeploymentRunner.kt b/src/main/kotlin/deployment/DeploymentRunner.kt
index ad66afc..2cb78c9 100644
--- a/src/main/kotlin/deployment/DeploymentRunner.kt
+++ b/src/main/kotlin/deployment/DeploymentRunner.kt
@@ -33,7 +33,8 @@ internal class DeploymentRunner(
     private val cohortStorage: CohortStorage?,
     private val metrics: LocalEvaluationMetrics = LocalEvaluationMetricsWrapper()
 ) {
-    private val lock = Once()
+    private val startLock = Once()
+    private val stopLock = Once()
     private val poller = Executors.newScheduledThreadPool(1, daemonFactory)
     private val cohortLoader = if (cohortApi != null && cohortStorage != null) {
         CohortLoader(cohortApi, cohortStorage, metrics)
@@ -71,7 +72,7 @@ internal class DeploymentRunner(
         else
             amplitudeFlagConfigUpdater
 
-    fun start() = lock.once {
+    fun start() = startLock.once {
         flagConfigUpdater.start()
         if (cohortLoader != null) {
             poller.scheduleWithFixedDelay(
@@ -97,7 +98,8 @@ internal class DeploymentRunner(
     }
 
     fun stop() {
-        lock.once {
+        if (!startLock.done) return
+        stopLock.once {
             poller.shutdown()
             flagConfigUpdater.shutdown()
         }
diff --git a/src/main/kotlin/util/Once.kt b/src/main/kotlin/util/Once.kt
index 9e25dc8..a9f98b1 100644
--- a/src/main/kotlin/util/Once.kt
+++ b/src/main/kotlin/util/Once.kt
@@ -1,7 +1,7 @@
 package com.amplitude.experiment.util
 
 class Once {
-    private var done = false
+    var done = false
     fun once(block: () -> Unit) {
         synchronized(this) {
             if (done) return@once