Merge branch 'release-1.145.1'

telstra · Oct 27, 2023 · 9a14314 · 9a14314
2 parents 78ab944 + 9502725
commit 9a14314
Show file tree

Hide file tree

Showing 254 changed files with 4,347 additions and 4,906 deletions.
diff --git a/.github/workflows/unittest.yml b/.github/workflows/unittest.yml
@@ -34,6 +34,7 @@ jobs:
         mkdir -p $HOME/.gradle
         echo "org.gradle.jvmargs=-Xmx4096M" >> $HOME/.gradle/gradle.properties
         echo "org.gradle.console=plain" >> $HOME/.gradle/gradle.properties
+        echo "org.gradle.parallel=true" >> $HOME/.gradle/gradle.properties
         rm -f $HOME/.gradle/caches/modules-2/modules-2.lock
         rm -fr $HOME/.gradle/caches/*/plugin-resolution/
     - name: Run checkstyle

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,5 +1,32 @@
 # Changelog
-## v1.145.0 (16/10/2023)
+
+## v1.145.1 (24/10/2023)
+
+### Bug Fixes:
+-  [#5445](https://github.com/telstra/open-kilda/pull/5445) Do not write false 'flow not found' log if monitoring is disabled
+-  [#5413](https://github.com/telstra/open-kilda/pull/5413) Victoria metrics url fixes for multicluster victoria configuration. [**gui**]
+
+### Improvements:
+-  [#5443](https://github.com/telstra/open-kilda/pull/5443) #5390: [TEST] Fix flaky Y-Flow Path Swap test (Issues: [#5390](https://github.com/telstra/open-kilda/issues/5390) [#5390](https://github.com/telstra/open-kilda/issues/5390)) [**tests**]
+-  [#5446](https://github.com/telstra/open-kilda/pull/5446) #5390: [TEST] Fix flaky Y-Flow reroute test (Issues: [#5390](https://github.com/telstra/open-kilda/issues/5390) [#5390](https://github.com/telstra/open-kilda/issues/5390)) [**tests**]
+-  [#5448](https://github.com/telstra/open-kilda/pull/5448) #5390: [TEST] Fix flaky Autoreroute test (Issues: [#5390](https://github.com/telstra/open-kilda/issues/5390) [#5390](https://github.com/telstra/open-kilda/issues/5390)) [**tests**]
+-  [#5319](https://github.com/telstra/open-kilda/pull/5319) Remove multi_table fields from DB objects (Issue: [#5313](https://github.com/telstra/open-kilda/issues/5313))
+-  [#4552](https://github.com/telstra/open-kilda/pull/4552) Multi AZ deployment for kilda [**docs**]
+-  [#4695](https://github.com/telstra/open-kilda/pull/4695) Make all storm topologies scalable across storm workers
+-  [#5434](https://github.com/telstra/open-kilda/pull/5434) [TEST]: Flaky tests: Ha-Flow: Update [**tests**]
+-  [#5436](https://github.com/telstra/open-kilda/pull/5436) Split StatsTopologyTest to run it in parallel [**tests**]
+-  [#5437](https://github.com/telstra/open-kilda/pull/5437) Ignore lock-keeper tests in docker directory
+-  [#5438](https://github.com/telstra/open-kilda/pull/5438) Run unit tests via github actions in parallel
+-  [#5439](https://github.com/telstra/open-kilda/pull/5439) [TEST]: Env setup: Parallel execution: Topology readiness [**tests**]
+
+For the complete list of changes, check out [the commit log](https://github.com/telstra/open-kilda/compare/v1.145.0...v1.145.1).
+
+### Upgrade notes:
+OrientDB schema have been changed in this release. You need to apply schema migration. Please follow [migration instructions](https://github.com/telstra/open-kilda/tree/develop/docker/db-migration/migrations).
+
+---
+
+## v1.145.0 (12/10/2023)
 
 ### Bug Fixes:
 -  [#5427](https://github.com/telstra/open-kilda/pull/5427) 5424 GUI: enable to get flow path if flow name contains a dot (Issue: [#5424](https://github.com/telstra/open-kilda/issues/5424)) [**gui**]

diff --git a/confd/templates/connecteddevices-topology/connecteddevices-topology.tmpl b/confd/templates/connecteddevices-topology/connecteddevices-topology.tmpl
@@ -4,7 +4,7 @@
 # topology configuration
 config:
   topology.parallelism: {{ getv "/kilda_storm_connected_devices_parallelism" }}
-  topology.workers: {{ getv "/kilda_storm_parallelism_workers_count" }}
+  topology.workers: {{ getv "/kilda_storm_connected_devices_workers_count" }}
   topology.spouts.parallelism: 1
 
 # spout definitions

diff --git a/confd/templates/floodlightrouter-topology/floodlightrouter-topology.tmpl b/confd/templates/floodlightrouter-topology/floodlightrouter-topology.tmpl
@@ -4,7 +4,7 @@
 # topology configuration
 config:
   topology.parallelism: {{ getv "/kilda_storm_floodlight_router_parallelism" }}
-  topology.workers: {{ getv "/kilda_storm_parallelism_workers_count" }}
+  topology.workers: {{ getv "/kilda_storm_floodlight_router_workers_count" }}
   topology.spouts.parallelism: {{ getv "/kilda_storm_parallelism_level" }}
 
 # spout definitions

diff --git a/confd/templates/flowmonitoring-topology/flowmonitoring-topology.tmpl b/confd/templates/flowmonitoring-topology/flowmonitoring-topology.tmpl
@@ -4,7 +4,7 @@
 # topology configuration
 config:
   topology.parallelism: {{ getv "/kilda_storm_flow_monitoring_parallelism" }}
-  topology.workers: {{ getv "/kilda_storm_parallelism_workers_count" }}
+  topology.workers: {{ getv "/kilda_storm_flow_monitoring_workers_count" }}
   topology.spouts.parallelism: {{ getv "/kilda_storm_spout_parallelism" }}
 
 # spout definitions

diff --git a/confd/templates/gui/application.properties.tmpl b/confd/templates/gui/application.properties.tmpl
@@ -11,6 +11,15 @@ server.ssl.key-alias = kilda
 server.ssl.key-store = classpath:keystore-kilda.jks
 server.ssl.key-store-password = openkilda
 
+#SSL Victoria configuration. Configure the SSL keystore file for Victoria Metrics in Java with the certificate required
+#for the HTTPS schema. Provide this file to enable REST API calls over HTTPS to the Victoria DB.
+victoria.trust.store=file:./victoria-keystore-kilda.jks
+{{- if exists "/victoria_keystore_password" }}
+victoria.trust.store.password = {{ getv "/victoria_keystore_password" }}
+{{- else }}
+victoria.trust.store.password =
+{{- end }}
+
 #Derby database configuration (In Memory)
 spring.jpa.database=default
 spring.jpa.properties.hibernate.dialect={{ getv "/kilda_gui_db_dialect" }}
@@ -40,7 +49,7 @@ opentsdb.base.url=http://{{ getv "/kilda_opentsdb_hosts" }}:{{ getv "/kilda_open
 opentsdb.metric.prefix = {{ getv "/kilda_opentsdb_metric_prefix" }}
 
 #VICTORIA METRICS Base URL and metric prefix
-victoria.base.url=http://{{ getv "/kilda_victoriametrics_host" }}:{{ getv "/kilda_victoriametrics_read_port" }}/prometheus
+victoria.base.url={{ getv "/kilda_victoria_read_url_schema" }}://{{ getv "/kilda_victoriametrics_host" }}:{{ getv "/kilda_victoriametrics_read_port" }}{{ getv "/kilda_victoriametrics_read_path" }}/prometheus
 
 #Kilda username and password
 kilda.username = {{ getv "/kilda_northbound_username" }}

diff --git a/confd/templates/history-topology/history-topology.tmpl b/confd/templates/history-topology/history-topology.tmpl
@@ -4,7 +4,7 @@
 # topology configuration
 config:
   topology.parallelism: {{ getv "/kilda_storm_history_parallelism" }}
-  topology.workers: {{ getv "/kilda_storm_parallelism_workers_count" }}
+  topology.workers: {{ getv "/kilda_storm_history_workers_count" }}
   topology.spouts.parallelism: {{ getv "/kilda_storm_spout_parallelism" }}
 
 # spout definitions

diff --git a/confd/templates/isllatency-topology/isllatency-topology.tmpl b/confd/templates/isllatency-topology/isllatency-topology.tmpl
@@ -4,7 +4,7 @@
 # topology configuration
 config:
   topology.parallelism: {{ getv "/kilda_storm_isl_latency_parallelism" }}
-  topology.workers: {{ getv "/kilda_storm_parallelism_workers_count" }}
+  topology.workers: {{ getv "/kilda_storm_isl_latency_workers_count" }}
   topology.spouts.parallelism: 1
 
 # spout definitions

diff --git a/confd/templates/nbworker-topology/nbworker-topology.tmpl b/confd/templates/nbworker-topology/nbworker-topology.tmpl
@@ -4,7 +4,7 @@
 # topology configuration
 config:
   topology.parallelism: {{ getv "/kilda_storm_nb_worker_parallelism" }}
-  topology.workers: {{ getv "/kilda_storm_parallelism_workers_count" }}
+  topology.workers: {{ getv "/kilda_storm_nbworker_workers_count" }}
   topology.spouts.parallelism: {{ getv "/kilda_storm_spout_parallelism" }}
 
 # spout definitions

diff --git a/confd/templates/network-topology/network-topology.tmpl b/confd/templates/network-topology/network-topology.tmpl
@@ -3,8 +3,8 @@
 
 # topology configuration
 config:
-  topology.parallelism: 2
-  topology.workers: {{ getv "/kilda_storm_parallelism_workers_count" }}
+  topology.parallelism: {{ getv "/kilda_storm_network_parallelism" }}
+  topology.workers: {{ getv "/kilda_storm_network_workers_count" }}
   topology.spouts.parallelism: {{ getv "/kilda_storm_spout_parallelism" }}
 
 # spout definitions

diff --git a/confd/templates/ping-topology/ping-topology.tmpl b/confd/templates/ping-topology/ping-topology.tmpl
@@ -3,8 +3,8 @@
 
 # topology configuration
 config:
-  topology.parallelism: 2
-  topology.workers: {{ getv "/kilda_storm_parallelism_workers_count" }}
+  topology.parallelism: {{ getv "/kilda_storm_ping_parallelism" }}
+  topology.workers: {{ getv "/kilda_storm_ping_workers_count" }}
   topology.spouts.parallelism: {{ getv "/kilda_storm_spout_parallelism" }}
 
 # spout definitions

diff --git a/confd/templates/portstate-topology/portstate-topology.tmpl b/confd/templates/portstate-topology/portstate-topology.tmpl
@@ -3,8 +3,8 @@
 
 # topology configuration
 config:
-  topology.parallelism: {{ getv "/kilda_storm_parallelism_level" }}
-  topology.workers: {{ getv "/kilda_storm_parallelism_workers_count" }}
+  topology.parallelism: {{ getv "/kilda_storm_port_state_parallelism" }}
+  topology.workers: {{ getv "/kilda_storm_port_state_workers_count" }}
   topology.spouts.parallelism: 1
 
 # spout definitions

diff --git a/confd/templates/reroute-topology/reroute-topology.tmpl b/confd/templates/reroute-topology/reroute-topology.tmpl
@@ -4,7 +4,7 @@
 # topology configuration
 config:
   topology.parallelism: {{ getv "/kilda_storm_reroute_parallelism" }}
-  topology.workers: {{ getv "/kilda_storm_parallelism_workers_count" }}
+  topology.workers: {{ getv "/kilda_storm_reroute_workers_count" }}
   topology.spouts.parallelism: {{ getv "/kilda_storm_spout_parallelism" }}
 
 # spout definitions

diff --git a/confd/templates/server42/server42-control-topology.tmpl b/confd/templates/server42/server42-control-topology.tmpl
@@ -4,7 +4,7 @@
 # topology configuration
 config:
   topology.parallelism: {{ getv "/kilda_storm_server42_control_parallelism" }}
-  topology.workers: {{ getv "/kilda_storm_parallelism_workers_count" }}
+  topology.workers: {{ getv "/kilda_storm_server42_control_count" }}
   topology.spouts.parallelism: {{ getv "/kilda_storm_spout_parallelism" }}
 
 # spout definitions

diff --git a/confd/templates/stats-topology/stats-topology.tmpl b/confd/templates/stats-topology/stats-topology.tmpl
@@ -4,7 +4,7 @@
 # topology configuration
 config:
   topology.parallelism: {{ getv "/kilda_storm_stats_parallelism" }}
-  topology.workers: {{ getv "/kilda_storm_parallelism_workers_count" }}
+  topology.workers: {{ getv "/kilda_storm_stats_workers_count" }}
   topology.spouts.parallelism: {{ getv "/kilda_storm_spout_parallelism" }}
 
 # spout definitions

diff --git a/confd/templates/storm/storm.yaml.tmpl b/confd/templates/storm/storm.yaml.tmpl
@@ -72,3 +72,5 @@ nimbus.childopts: "-Xmx128m"
 ui.childopts: "-Xmx128m"
 supervisor.childopts: "-Xmx128m"
 worker.childopts: "-Xmx280m"
+
+topology.testing.always.try.serialize: True
diff --git a/confd/templates/swmanager-topology/swmanager-topology.tmpl b/confd/templates/swmanager-topology/swmanager-topology.tmpl
@@ -4,7 +4,7 @@
 # topology configuration
 config:
   topology.parallelism: {{ getv "/kilda_storm_swmanager_parallelism" }}
-  topology.workers: {{ getv "/kilda_storm_parallelism_workers_count" }}
+  topology.workers: {{ getv "/kilda_storm_swmanager_workers_count" }}
   topology.spouts.parallelism: 1
 
 # spout definitions

diff --git a/confd/vars/main.yaml b/confd/vars/main.yaml
@@ -24,6 +24,9 @@ kilda_victoriametrics_host: "victoriametrics.pendev"
 kilda_victoriametrics_write_port: "4242"
 kilda_victoriametrics_read_port: "8428"
 kilda_victoriametrics_path: ""
+kilda_victoriametrics_read_path: ""
+kilda_victoria_read_url_schema: "http"
+victoria_keystore_password: ""
 kilda_storm_numbus_hosts: "nimbus.pendev"
 
 kilda_hibernate_url: jdbc:mysql://mysql.pendev:3306/kilda
@@ -95,7 +98,7 @@ kilda_opentsdb_num_datapointparserbolt: 1
 kilda_opentsdb_workers_datapointparserbolt: 1
 kilda_opentsdb_batch_size: 50
 kilda_opentsdb_flush_interval: 1
-kilda_opentsdb_workers: 1
+kilda_opentsdb_workers: 2
 kilda_opentsdb_parallelism: 2
 kilda_opentsdb_metric_prefix: "kilda."
 
@@ -172,41 +175,69 @@ kilda_latency_update_interval: 120
 kilda_latency_update_time_range: 600
 kilda_latency_discovery_interval_multiplier: 3
 
-kilda_storm_swmanager_parallelism: 2
-kilda_storm_swmanager_heavy_operation_parallelism: 2
-kilda_storm_isl_latency_parallelism: 4
 kilda_storm_parallelism_level_new: 2
 kilda_storm_parallelism_level: 1
 kilda_storm_parallelism_workers_count: 1
+kilda_storm_spout_parallelism: 2
+
+# SW manager
+kilda_storm_swmanager_parallelism: 2
+kilda_storm_swmanager_heavy_operation_parallelism: 2
+kilda_storm_swmanager_workers_count: 2
+
+# ISL latency
+kilda_storm_isl_latency_parallelism: 4
+kilda_storm_isl_latency_workers_count: 2
+
+# Stats
 kilda_storm_stats_parallelism: 2
+kilda_storm_stats_workers_count: 2
+
+# Network
+kilda_storm_network_parallelism: 2
+kilda_storm_network_workers_count: 2
 
 # Reroute
 kilda_storm_reroute_parallelism: 2
+kilda_storm_reroute_workers_count: 2
 
 # Floodlight router
 kilda_storm_floodlight_router_parallelism: 2
+kilda_storm_floodlight_router_workers_count: 2
 
 # NB worker
 kilda_storm_nb_worker_parallelism: 2
 kilda_storm_nb_worker_flow_operations_parallelism: 2
+kilda_storm_nbworker_workers_count: 2
 
 # Server42 control
 kilda_storm_server42_control_parallelism: 2
+kilda_storm_server42_control_count: 2
 
 # History
 kilda_storm_history_parallelism: 2
 kilda_storm_history_bolt_parallelism: 2
 kilda_storm_history_bolt_num_tasks: 2
+kilda_storm_history_workers_count: 2
 
 # Connected devices
 kilda_storm_connected_devices_parallelism: 2
+kilda_storm_connected_devices_workers_count: 2
 kilda_storm_connected_devices_packet_bolt_parallelism: 2
 
+# Ping
+kilda_storm_ping_parallelism: 2
+kilda_storm_ping_workers_count: 2
+
+# Port state
+kilda_storm_port_state_parallelism: 1
+kilda_storm_port_state_workers_count: 2
+
 # Flow HS
 kilda_storm_flow_hs_parallelism: 2
 kilda_storm_flow_hs_num_tasks: 2
 kilda_storm_flow_hs_reroute_hub_count_multiplier: 2
-kilda_storm_flowhs_workers: 1
+kilda_storm_flowhs_workers: 2
 kilda_storm_flow_hs_validation_worker_parallelism: 2
 kilda_storm_flow_hs_flow_create_hub_parallelism: 2
 kilda_storm_flow_hs_flow_update_hub_parallelism: 2
@@ -246,9 +277,9 @@ kilda_storm_flow_hs_ping_sender_parallelism: 2
 kilda_storm_flow_hs_monitoring_sender_parallelism: 2
 kilda_storm_flow_hs_speaker_request_sender_parallelism: 2
 
-kilda_storm_spout_parallelism: 2
-
+# Flow monitoring
 kilda_storm_flow_monitoring_parallelism: 2
+kilda_storm_flow_monitoring_workers_count: 2
 kilda_storm_flow_monitoring_flow_stats_bolt_parallelism: 2
 
 kilda_storm_disruptor_wait_timeout: 1000

diff --git a/docker/.gitignore b/docker/.gitignore
@@ -6,5 +6,6 @@ lab-service/log.json
 lock-keeper/README.md
 lock-keeper/*.py
 lock-keeper/cli
+lock-keeper/test
 grpc-stub/*.py
 kafka/kafka-conf/server.properties
diff --git a/docker/db-migration/migrations/030-ensure-all-objects-are-multi-table.yaml b/docker/db-migration/migrations/030-ensure-all-objects-are-multi-table.yaml
@@ -0,0 +1,41 @@
+databaseChangeLog:
+  - changeSet:
+      id: tag
+      author: snikitin
+      changes:
+        - tagDatabase:
+            tag: 030-ensure-all-objects-are-multi-table
+  - changeSet:
+      id: remove_multi_table_fields
+      author: snikitin
+      preConditions:
+        - onFail: HALT
+        - onFailMessage: "Failed preconditions for migration '030-ensure-all-objects-are-multi-table.yaml'.\n
+            \n
+            Current release removes multi table fields from Java classes. \n
+            This migration is required to be sure that all objects in DB are in multi table mode. \n
+            It means that values of all following fields must be 'true':\n
+             1. switch_properties.multi_table\n
+             2. flow_path.src_with_multi_table\n
+             3. flow_path.dst_with_multi_table\n
+             4. path_segment.src_with_multi_table\n
+             5. path_segment.dst_with_multi_table\n
+             6. flow_mirror_path.egress_with_multi_table\n"
+        - sqlCheck:
+            expectedResult: 0
+            sql: "SELECT COUNT(*) FROM switch_properties WHERE multi_table <> true"
+        - sqlCheck:
+            expectedResult: 0
+            sql: "SELECT COUNT(*) FROM flow_path WHERE src_with_multi_table <> true"
+        - sqlCheck:
+            expectedResult: 0
+            sql: "SELECT COUNT(*) FROM flow_path WHERE dst_with_multi_table <> true"
+        - sqlCheck:
+            expectedResult: 0
+            sql: "SELECT COUNT(*) FROM path_segment WHERE src_with_multi_table <> true"
+        - sqlCheck:
+            expectedResult: 0
+            sql: "SELECT COUNT(*) FROM path_segment WHERE dst_with_multi_table <> true"
+        - sqlCheck:
+            expectedResult: 0
+            sql: "SELECT COUNT(*) FROM flow_mirror_path WHERE egress_with_multi_table <> true"
diff --git a/docker/db-migration/migrations/root.yaml b/docker/db-migration/migrations/root.yaml
@@ -99,4 +99,6 @@ databaseChangeLog:
   - include:
       relativeToChangelogFile: true
       file: 029-add-ha-flow-history-event.yaml
-
+  - include:
+      relativeToChangelogFile: true
+      file: 030-ensure-all-objects-are-multi-table.yaml