From 7bd565005033f83075a1ed250a6355034a9770e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sami=20M=C3=A4kinen?= Date: Mon, 18 Mar 2024 12:38:11 +0200 Subject: [PATCH 1/2] Make sure we don't write duplicated indicators --- .../statistics/plugins/DataSourceUpdater.java | 22 ++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/service-statistics-common/src/main/java/fi/nls/oskari/control/statistics/plugins/DataSourceUpdater.java b/service-statistics-common/src/main/java/fi/nls/oskari/control/statistics/plugins/DataSourceUpdater.java index a3aef1a8a5..3309520859 100755 --- a/service-statistics-common/src/main/java/fi/nls/oskari/control/statistics/plugins/DataSourceUpdater.java +++ b/service-statistics-common/src/main/java/fi/nls/oskari/control/statistics/plugins/DataSourceUpdater.java @@ -6,8 +6,13 @@ import fi.nls.oskari.control.statistics.data.StatisticalIndicator; import fi.nls.oskari.log.LogFactory; import fi.nls.oskari.log.Logger; +import net.opengis.ows10.SectionsType; +import java.util.ArrayList; +import java.util.HashSet; import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; /** * Used to preload and -process statistical indicator data from a datasource @@ -54,13 +59,28 @@ protected void storeIndicatorList(List indicators) { if (indicators.isEmpty()) { return; } + // Make sure we don't store duplicates of indicators + // This might happen when multiple nodes in cluster processes the list at the same time. + // One node might be faster and store an indicator while another still has it in it's workqueue. + // When the slower one saves, it combines the processed from redis + workqueue on its memory + // where processed already might contain indicators that are in the workqueue of the node that is saving/adding it's queue + List nonDuplicates = new ArrayList<>(indicators.size()); + Set indicatorIds = new HashSet<>(indicators.size()); + indicators.stream().forEach(ind -> { + if (indicatorIds.contains(ind.getId())) { + return; + } + nonDuplicates.add(ind); + indicatorIds.add(ind.getId()); + }); + final ObjectMapper listMapper = new ObjectMapper(); // skip f.ex. description and source when writing list listMapper.addMixIn(StatisticalIndicator.class, JacksonIndicatorListMixin.class); // write new indicator list try { - String result = listMapper.writeValueAsString(indicators); + String result = listMapper.writeValueAsString(nonDuplicates); JedisManager.setex(plugin.getIndicatorListKey(), JedisManager.EXPIRY_TIME_DAY * 7, result); } catch (JsonProcessingException ex) { LOG.error(ex, "Error updating indicator list"); From 083040d98883bb66c8779e511f74ff43faeb4edc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sami=20M=C3=A4kinen?= Date: Mon, 18 Mar 2024 12:43:45 +0200 Subject: [PATCH 2/2] Remove unnecessary imports --- .../oskari/control/statistics/plugins/DataSourceUpdater.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/service-statistics-common/src/main/java/fi/nls/oskari/control/statistics/plugins/DataSourceUpdater.java b/service-statistics-common/src/main/java/fi/nls/oskari/control/statistics/plugins/DataSourceUpdater.java index 3309520859..99ace4a372 100755 --- a/service-statistics-common/src/main/java/fi/nls/oskari/control/statistics/plugins/DataSourceUpdater.java +++ b/service-statistics-common/src/main/java/fi/nls/oskari/control/statistics/plugins/DataSourceUpdater.java @@ -6,13 +6,11 @@ import fi.nls.oskari.control.statistics.data.StatisticalIndicator; import fi.nls.oskari.log.LogFactory; import fi.nls.oskari.log.Logger; -import net.opengis.ows10.SectionsType; import java.util.ArrayList; import java.util.HashSet; import java.util.List; import java.util.Set; -import java.util.stream.Collectors; /** * Used to preload and -process statistical indicator data from a datasource