From c46e4aef66f1680548d4d7e50d476d130894563e Mon Sep 17 00:00:00 2001 From: Gustav Karlsson Date: Wed, 20 May 2020 21:38:19 +0200 Subject: [PATCH 1/2] Commit explicitly if auto-commit=false. Add compatibility test for such data-source. --- db-scheduler/pom.xml | 7 ++- .../scheduler/SchedulerBuilder.java | 7 ++- .../kagkarlsson/scheduler/SchedulerName.java | 7 +++ .../jdbc/AutodetectJdbcCustomization.java | 1 + .../scheduler/testhelper/TestHelper.java | 3 +- .../kagkarlsson/scheduler/ClusterTest.java | 4 +- .../EmbeddedPostgresqlExtension.java | 15 +++--- .../compatibility/CompatibilityTest.java | 1 + .../compatibility/MysqlCompatibilityTest.java | 1 + ...AutoCommitPostgresqlCompatibilityTest.java | 48 +++++++++++++++++++ .../functional/DeadExecutionTest.java | 4 +- .../functional/ExecutorPoolTest.java | 6 ++- .../functional/ImmediateExecutionTest.java | 4 +- pom.xml | 2 +- 14 files changed, 94 insertions(+), 16 deletions(-) create mode 100644 db-scheduler/src/test/java/com/github/kagkarlsson/scheduler/compatibility/NoAutoCommitPostgresqlCompatibilityTest.java diff --git a/db-scheduler/pom.xml b/db-scheduler/pom.xml index cfaef1ff..84b33c11 100644 --- a/db-scheduler/pom.xml +++ b/db-scheduler/pom.xml @@ -19,7 +19,7 @@ 2.3.3 1.6 3.1.12 - 0.1 + 0.2 0.11.4 9.4.1207 1.7.7 @@ -110,6 +110,11 @@ test + + org.testcontainers + postgresql + test + org.testcontainers mysql diff --git a/db-scheduler/src/main/java/com/github/kagkarlsson/scheduler/SchedulerBuilder.java b/db-scheduler/src/main/java/com/github/kagkarlsson/scheduler/SchedulerBuilder.java index 4d4e4a39..c8eb251e 100644 --- a/db-scheduler/src/main/java/com/github/kagkarlsson/scheduler/SchedulerBuilder.java +++ b/db-scheduler/src/main/java/com/github/kagkarlsson/scheduler/SchedulerBuilder.java @@ -41,7 +41,7 @@ public class SchedulerBuilder { protected Clock clock = new SystemClock(); // if this is set, waiter-clocks must be updated protected final DataSource dataSource; - protected SchedulerName schedulerName = new SchedulerName.Hostname(); + protected SchedulerName schedulerName; protected int executorThreads = 10; protected final List> knownTasks = new ArrayList<>(); protected final List startTasks = new ArrayList<>(); @@ -150,6 +150,11 @@ public Scheduler build() { if (pollingLimit < executorThreads) { LOG.warn("Polling-limit is less than number of threads. Should be equal or higher."); } + + if (schedulerName == null) { + schedulerName = new SchedulerName.Hostname(); + } + final TaskResolver taskResolver = new TaskResolver(statsRegistry, clock, knownTasks); final JdbcCustomization jdbcCustomization = ofNullable(this.jdbcCustomization).orElse(new AutodetectJdbcCustomization(dataSource)); final JdbcTaskRepository taskRepository = new JdbcTaskRepository(dataSource, jdbcCustomization, tableName, taskResolver, schedulerName, serializer); diff --git a/db-scheduler/src/main/java/com/github/kagkarlsson/scheduler/SchedulerName.java b/db-scheduler/src/main/java/com/github/kagkarlsson/scheduler/SchedulerName.java index 7c26d4ef..fad3da01 100644 --- a/db-scheduler/src/main/java/com/github/kagkarlsson/scheduler/SchedulerName.java +++ b/db-scheduler/src/main/java/com/github/kagkarlsson/scheduler/SchedulerName.java @@ -46,7 +46,14 @@ class Hostname implements SchedulerName { public Hostname() { try { + long start = System.currentTimeMillis(); + LOG.debug("Resolving hostname.."); cachedHostname = InetAddress.getLocalHost().getHostName(); + LOG.debug("Resolved hostname.."); + long duration = System.currentTimeMillis() - start; + if (duration > 1000) { + LOG.warn("Hostname-lookup took {}ms", duration); + } } catch (UnknownHostException e) { LOG.warn("Failed to resolve hostname. Using dummy-name for scheduler."); cachedHostname = "failed.hostname.lookup"; diff --git a/db-scheduler/src/main/java/com/github/kagkarlsson/scheduler/jdbc/AutodetectJdbcCustomization.java b/db-scheduler/src/main/java/com/github/kagkarlsson/scheduler/jdbc/AutodetectJdbcCustomization.java index 6d8c7a33..3c6f8cb8 100644 --- a/db-scheduler/src/main/java/com/github/kagkarlsson/scheduler/jdbc/AutodetectJdbcCustomization.java +++ b/db-scheduler/src/main/java/com/github/kagkarlsson/scheduler/jdbc/AutodetectJdbcCustomization.java @@ -34,6 +34,7 @@ public class AutodetectJdbcCustomization implements JdbcCustomization { public AutodetectJdbcCustomization(DataSource dataSource) { JdbcCustomization detectedCustomization = new DefaultJdbcCustomization(); + LOG.debug("Detecting database..."); try (Connection c = dataSource.getConnection()) { String databaseProductName = c.getMetaData().getDatabaseProductName(); LOG.info("Detected database {}.", databaseProductName); diff --git a/db-scheduler/src/main/java/com/github/kagkarlsson/scheduler/testhelper/TestHelper.java b/db-scheduler/src/main/java/com/github/kagkarlsson/scheduler/testhelper/TestHelper.java index 0df2d416..ff5d4593 100644 --- a/db-scheduler/src/main/java/com/github/kagkarlsson/scheduler/testhelper/TestHelper.java +++ b/db-scheduler/src/main/java/com/github/kagkarlsson/scheduler/testhelper/TestHelper.java @@ -15,6 +15,7 @@ */ package com.github.kagkarlsson.scheduler.testhelper; +import com.github.kagkarlsson.scheduler.SchedulerName; import com.github.kagkarlsson.scheduler.jdbc.DefaultJdbcCustomization; import com.github.kagkarlsson.scheduler.SchedulerBuilder; import com.github.kagkarlsson.scheduler.JdbcTaskRepository; @@ -65,7 +66,7 @@ public ManualSchedulerBuilder statsRegistry(StatsRegistry statsRegistry) { public ManualScheduler build() { final TaskResolver taskResolver = new TaskResolver(statsRegistry, clock, knownTasks); - final JdbcTaskRepository taskRepository = new JdbcTaskRepository(dataSource, new DefaultJdbcCustomization(), tableName, taskResolver, schedulerName, serializer); + final JdbcTaskRepository taskRepository = new JdbcTaskRepository(dataSource, new DefaultJdbcCustomization(), tableName, taskResolver, new SchedulerName.Fixed("manual"), serializer); return new ManualScheduler(clock, taskRepository, taskResolver, executorThreads, new DirectExecutorService(), schedulerName, waiter, heartbeatInterval, enableImmediateExecution, statsRegistry, pollingLimit, deleteUnresolvedAfter, startTasks); } diff --git a/db-scheduler/src/test/java/com/github/kagkarlsson/scheduler/ClusterTest.java b/db-scheduler/src/test/java/com/github/kagkarlsson/scheduler/ClusterTest.java index 0e1915c5..45abed83 100644 --- a/db-scheduler/src/test/java/com/github/kagkarlsson/scheduler/ClusterTest.java +++ b/db-scheduler/src/test/java/com/github/kagkarlsson/scheduler/ClusterTest.java @@ -38,7 +38,7 @@ public class ClusterTest { @Test public void test_concurrency() throws InterruptedException { - Assertions.assertTimeoutPreemptively(Duration.ofSeconds(10), () -> { + Assertions.assertTimeoutPreemptively(Duration.ofSeconds(20), () -> { final List ids = IntStream.range(1, 1001).mapToObj(String::valueOf).collect(toList()); @@ -75,7 +75,7 @@ public void test_concurrency() throws InterruptedException { @Test public void test_concurrency_recurring() throws InterruptedException { - Assertions.assertTimeoutPreemptively(Duration.ofSeconds(10), () -> { + Assertions.assertTimeoutPreemptively(Duration.ofSeconds(20), () -> { final RecurringTask task1 = Tasks.recurring("task1", Schedules.fixedDelay(Duration.ofMillis(0))) .execute((taskInstance, executionContext) -> { diff --git a/db-scheduler/src/test/java/com/github/kagkarlsson/scheduler/EmbeddedPostgresqlExtension.java b/db-scheduler/src/test/java/com/github/kagkarlsson/scheduler/EmbeddedPostgresqlExtension.java index fde7ee55..6b4c9c3a 100644 --- a/db-scheduler/src/test/java/com/github/kagkarlsson/scheduler/EmbeddedPostgresqlExtension.java +++ b/db-scheduler/src/test/java/com/github/kagkarlsson/scheduler/EmbeddedPostgresqlExtension.java @@ -29,15 +29,18 @@ public EmbeddedPostgresqlExtension(Consumer initializeSchema, Consum this.initializeSchema = initializeSchema; this.cleanupAfter = cleanupAfter; try { - if (embeddedPostgresql == null) { - embeddedPostgresql = initPostgres(); + synchronized (this) { - HikariConfig config = new HikariConfig(); - config.setDataSource(embeddedPostgresql.getDatabase("test", "test")); + if (embeddedPostgresql == null) { + embeddedPostgresql = initPostgres(); - dataSource = new HikariDataSource(config); + HikariConfig config = new HikariConfig(); + config.setDataSource(embeddedPostgresql.getDatabase("test", "test")); - initializeSchema.accept(dataSource); + dataSource = new HikariDataSource(config); + + initializeSchema.accept(dataSource); + } } } catch (IOException e) { throw new RuntimeException(e); diff --git a/db-scheduler/src/test/java/com/github/kagkarlsson/scheduler/compatibility/CompatibilityTest.java b/db-scheduler/src/test/java/com/github/kagkarlsson/scheduler/compatibility/CompatibilityTest.java index c2480432..5a515757 100644 --- a/db-scheduler/src/test/java/com/github/kagkarlsson/scheduler/compatibility/CompatibilityTest.java +++ b/db-scheduler/src/test/java/com/github/kagkarlsson/scheduler/compatibility/CompatibilityTest.java @@ -71,6 +71,7 @@ public void setUp() { scheduler = Scheduler.create(getDataSource(), Lists.newArrayList(oneTime, recurring)) .pollingInterval(Duration.ofMillis(10)) .heartbeatInterval(Duration.ofMillis(100)) + .schedulerName(new SchedulerName.Fixed("test")) .statsRegistry(statsRegistry) .build(); stopScheduler.register(scheduler); diff --git a/db-scheduler/src/test/java/com/github/kagkarlsson/scheduler/compatibility/MysqlCompatibilityTest.java b/db-scheduler/src/test/java/com/github/kagkarlsson/scheduler/compatibility/MysqlCompatibilityTest.java index 4ead644c..5d9f24c7 100644 --- a/db-scheduler/src/test/java/com/github/kagkarlsson/scheduler/compatibility/MysqlCompatibilityTest.java +++ b/db-scheduler/src/test/java/com/github/kagkarlsson/scheduler/compatibility/MysqlCompatibilityTest.java @@ -5,6 +5,7 @@ import com.zaxxer.hikari.HikariDataSource; import com.zaxxer.hikari.util.DriverDataSource; import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Tag; import org.testcontainers.containers.MySQLContainer; import org.testcontainers.junit.jupiter.Container; diff --git a/db-scheduler/src/test/java/com/github/kagkarlsson/scheduler/compatibility/NoAutoCommitPostgresqlCompatibilityTest.java b/db-scheduler/src/test/java/com/github/kagkarlsson/scheduler/compatibility/NoAutoCommitPostgresqlCompatibilityTest.java new file mode 100644 index 00000000..92435acd --- /dev/null +++ b/db-scheduler/src/test/java/com/github/kagkarlsson/scheduler/compatibility/NoAutoCommitPostgresqlCompatibilityTest.java @@ -0,0 +1,48 @@ +package com.github.kagkarlsson.scheduler.compatibility; + +import com.github.kagkarlsson.scheduler.DbUtils; +import com.github.kagkarlsson.scheduler.EmbeddedPostgresqlExtension; +import com.zaxxer.hikari.HikariConfig; +import com.zaxxer.hikari.HikariDataSource; +import com.zaxxer.hikari.util.DriverDataSource; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.extension.RegisterExtension; +import org.testcontainers.containers.MSSQLServerContainer; +import org.testcontainers.containers.PostgreSQLContainer; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; + +import javax.sql.DataSource; +import java.util.Properties; + +@Tag("compatibility") +@Testcontainers +public class NoAutoCommitPostgresqlCompatibilityTest extends CompatibilityTest { + + @Container + private static final PostgreSQLContainer POSTGRES = new PostgreSQLContainer(); + private static HikariDataSource pooledDatasource; + + @BeforeAll + private static void initSchema() { + final DriverDataSource datasource = new DriverDataSource(POSTGRES.getJdbcUrl(), "org.postgresql.Driver", + new Properties(), POSTGRES.getUsername(), POSTGRES.getPassword()); + + // init schema + DbUtils.runSqlResource("/postgresql_tables.sql").accept(datasource); + + + // Setup non auto-committing datasource + final HikariConfig hikariConfig = new HikariConfig(); + hikariConfig.setDataSource(datasource); + hikariConfig.setAutoCommit(false); + pooledDatasource = new HikariDataSource(hikariConfig); + + } + @Override + public DataSource getDataSource() { + return pooledDatasource; + } + +} diff --git a/db-scheduler/src/test/java/com/github/kagkarlsson/scheduler/functional/DeadExecutionTest.java b/db-scheduler/src/test/java/com/github/kagkarlsson/scheduler/functional/DeadExecutionTest.java index 548a8872..8f2dafc4 100644 --- a/db-scheduler/src/test/java/com/github/kagkarlsson/scheduler/functional/DeadExecutionTest.java +++ b/db-scheduler/src/test/java/com/github/kagkarlsson/scheduler/functional/DeadExecutionTest.java @@ -3,6 +3,7 @@ import com.github.kagkarlsson.scheduler.DbUtils; import com.github.kagkarlsson.scheduler.EmbeddedPostgresqlExtension; import com.github.kagkarlsson.scheduler.Scheduler; +import com.github.kagkarlsson.scheduler.SchedulerName; import com.github.kagkarlsson.scheduler.StopSchedulerExtension; import com.github.kagkarlsson.scheduler.helper.TestableRegistry; import com.github.kagkarlsson.scheduler.task.CompletionHandler; @@ -29,7 +30,7 @@ public class DeadExecutionTest { @Test public void test_dead_execution() { - Assertions.assertTimeoutPreemptively(Duration.ofSeconds(5), () -> { + Assertions.assertTimeoutPreemptively(Duration.ofSeconds(20), () -> { CustomTask customTask = Tasks.custom("custom-a", Void.class) .execute((taskInstance, executionContext) -> new CompletionHandler() { @Override @@ -45,6 +46,7 @@ public void complete(ExecutionComplete executionComplete, ExecutionOperations { + Assertions.assertTimeoutPreemptively(Duration.ofSeconds(20), () -> { Instant now = Instant.now(); OneTimeTask task = TestTasks.oneTime("onetime-a", Void.class, TestTasks.DO_NOTHING); @@ -69,6 +70,7 @@ private void testExecuteUntilNoneLeft(int pollingLimit, int threads, int executi .pollingLimit(pollingLimit) .threads(threads) .pollingInterval(Duration.ofMinutes(1)) + .schedulerName(new SchedulerName.Fixed("test")) .statsRegistry(registry) .build(); stopScheduler.register(scheduler); diff --git a/db-scheduler/src/test/java/com/github/kagkarlsson/scheduler/functional/ImmediateExecutionTest.java b/db-scheduler/src/test/java/com/github/kagkarlsson/scheduler/functional/ImmediateExecutionTest.java index 50dbbd1e..70337222 100644 --- a/db-scheduler/src/test/java/com/github/kagkarlsson/scheduler/functional/ImmediateExecutionTest.java +++ b/db-scheduler/src/test/java/com/github/kagkarlsson/scheduler/functional/ImmediateExecutionTest.java @@ -4,6 +4,7 @@ import com.github.kagkarlsson.scheduler.DbUtils; import com.github.kagkarlsson.scheduler.EmbeddedPostgresqlExtension; import com.github.kagkarlsson.scheduler.Scheduler; +import com.github.kagkarlsson.scheduler.SchedulerName; import com.github.kagkarlsson.scheduler.StopSchedulerExtension; import com.github.kagkarlsson.scheduler.TestTasks; import com.github.kagkarlsson.scheduler.helper.TestableRegistry; @@ -40,7 +41,7 @@ public void setUp() { @Test public void test_immediate_execution() { - Assertions.assertTimeoutPreemptively(Duration.ofSeconds(10), () -> { + Assertions.assertTimeoutPreemptively(Duration.ofSeconds(20), () -> { Instant now = Instant.now(); OneTimeTask task = TestTasks.oneTime("onetime-a", Void.class, TestTasks.DO_NOTHING); @@ -52,6 +53,7 @@ public void test_immediate_execution() { Scheduler scheduler = Scheduler.create(postgres.getDataSource(), task) .pollingInterval(Duration.ofMinutes(1)) .enableImmediateExecution() + .schedulerName(new SchedulerName.Fixed("test")) .statsRegistry(registry) .build(); stopScheduler.register(scheduler); diff --git a/pom.xml b/pom.xml index 6035e096..805fae7b 100644 --- a/pom.xml +++ b/pom.xml @@ -13,7 +13,7 @@ Gustav Karlsson - gustav80@gmail.com + kagkarlsson@gmail.com From 81e912088c6d35aacf439650dfd6e3ce0252d577 Mon Sep 17 00:00:00 2001 From: Gustav Karlsson Date: Sat, 23 May 2020 22:07:39 +0200 Subject: [PATCH 2/2] Restore test-timeouts --- .../java/com/github/kagkarlsson/scheduler/ClusterTest.java | 4 ++-- .../kagkarlsson/scheduler/functional/DeadExecutionTest.java | 2 +- .../kagkarlsson/scheduler/functional/ExecutorPoolTest.java | 2 +- .../scheduler/functional/ImmediateExecutionTest.java | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/db-scheduler/src/test/java/com/github/kagkarlsson/scheduler/ClusterTest.java b/db-scheduler/src/test/java/com/github/kagkarlsson/scheduler/ClusterTest.java index 45abed83..0e1915c5 100644 --- a/db-scheduler/src/test/java/com/github/kagkarlsson/scheduler/ClusterTest.java +++ b/db-scheduler/src/test/java/com/github/kagkarlsson/scheduler/ClusterTest.java @@ -38,7 +38,7 @@ public class ClusterTest { @Test public void test_concurrency() throws InterruptedException { - Assertions.assertTimeoutPreemptively(Duration.ofSeconds(20), () -> { + Assertions.assertTimeoutPreemptively(Duration.ofSeconds(10), () -> { final List ids = IntStream.range(1, 1001).mapToObj(String::valueOf).collect(toList()); @@ -75,7 +75,7 @@ public void test_concurrency() throws InterruptedException { @Test public void test_concurrency_recurring() throws InterruptedException { - Assertions.assertTimeoutPreemptively(Duration.ofSeconds(20), () -> { + Assertions.assertTimeoutPreemptively(Duration.ofSeconds(10), () -> { final RecurringTask task1 = Tasks.recurring("task1", Schedules.fixedDelay(Duration.ofMillis(0))) .execute((taskInstance, executionContext) -> { diff --git a/db-scheduler/src/test/java/com/github/kagkarlsson/scheduler/functional/DeadExecutionTest.java b/db-scheduler/src/test/java/com/github/kagkarlsson/scheduler/functional/DeadExecutionTest.java index 8f2dafc4..608d8c0a 100644 --- a/db-scheduler/src/test/java/com/github/kagkarlsson/scheduler/functional/DeadExecutionTest.java +++ b/db-scheduler/src/test/java/com/github/kagkarlsson/scheduler/functional/DeadExecutionTest.java @@ -30,7 +30,7 @@ public class DeadExecutionTest { @Test public void test_dead_execution() { - Assertions.assertTimeoutPreemptively(Duration.ofSeconds(20), () -> { + Assertions.assertTimeoutPreemptively(Duration.ofSeconds(5), () -> { CustomTask customTask = Tasks.custom("custom-a", Void.class) .execute((taskInstance, executionContext) -> new CompletionHandler() { @Override diff --git a/db-scheduler/src/test/java/com/github/kagkarlsson/scheduler/functional/ExecutorPoolTest.java b/db-scheduler/src/test/java/com/github/kagkarlsson/scheduler/functional/ExecutorPoolTest.java index de69fbdd..cc21feb4 100644 --- a/db-scheduler/src/test/java/com/github/kagkarlsson/scheduler/functional/ExecutorPoolTest.java +++ b/db-scheduler/src/test/java/com/github/kagkarlsson/scheduler/functional/ExecutorPoolTest.java @@ -59,7 +59,7 @@ public void test_execute_until_none_left_high_volume() { private void testExecuteUntilNoneLeft(int pollingLimit, int threads, int executionsToRun) { - Assertions.assertTimeoutPreemptively(Duration.ofSeconds(20), () -> { + Assertions.assertTimeoutPreemptively(Duration.ofSeconds(5), () -> { Instant now = Instant.now(); OneTimeTask task = TestTasks.oneTime("onetime-a", Void.class, TestTasks.DO_NOTHING); diff --git a/db-scheduler/src/test/java/com/github/kagkarlsson/scheduler/functional/ImmediateExecutionTest.java b/db-scheduler/src/test/java/com/github/kagkarlsson/scheduler/functional/ImmediateExecutionTest.java index 70337222..7813986d 100644 --- a/db-scheduler/src/test/java/com/github/kagkarlsson/scheduler/functional/ImmediateExecutionTest.java +++ b/db-scheduler/src/test/java/com/github/kagkarlsson/scheduler/functional/ImmediateExecutionTest.java @@ -41,7 +41,7 @@ public void setUp() { @Test public void test_immediate_execution() { - Assertions.assertTimeoutPreemptively(Duration.ofSeconds(20), () -> { + Assertions.assertTimeoutPreemptively(Duration.ofSeconds(10), () -> { Instant now = Instant.now(); OneTimeTask task = TestTasks.oneTime("onetime-a", Void.class, TestTasks.DO_NOTHING);