Begin work to only target OpenSearch 2 as our analytics database.

- Upgrading development environment images to OpenSearch 2. - Some tweaks for OpenSearch 2 compatibility (eg, removing _type references). - Renaming all "elasticsearch" code to "opensearch" for better parity. - Starting to remove legacy code from older versions of Elasticsearch, since we're not trying to maintain this legacy compatibility.
NREL · Jan 27, 2024 · 4ab6626 · 4ab6626
1 parent 28d2196
commit 4ab6626
Show file tree

Hide file tree

Showing 56 changed files with 494 additions and 1,552 deletions.
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -10,7 +10,6 @@ env:
   IMAGE_NAME: nrel/api-umbrella
   DOCKER_BUILDKIT: 1
   TESTS_GLOB: "test/**/test_*.rb"
-  ELASTICSEARCH_TESTS_GLOB: "test/admin_ui/**/test_stats_*.rb test/admin_ui/**/test_stats_*.rb test/apis/admin/stats/**/test_*.rb test/apis/v0/test_analytics.rb test/apis/v1/analytics/**/test_*.rb test/proxy/logging/**/test_*.rb"
 
 jobs:
   build:
@@ -152,8 +151,8 @@ jobs:
           --health-interval 10s
           --health-timeout 5s
           --health-retries 5
-      elasticsearch:
-        image: opensearchproject/opensearch:1.3.8
+      opensearch:
+        image: public.ecr.aws/opensearchproject/opensearch:2.0.1
         env:
           OPENSEARCH_JAVA_OPTS: "-Xms32m -Xmx512m"
           DISABLE_INSTALL_DEMO_CONFIG: "true"
@@ -230,171 +229,10 @@ jobs:
           name: test-artifacts-matrix-${{ matrix.ci_node_index }}
           path: test/tmp/artifacts/
 
-  test_elasticsearch_v6:
-    needs: [build]
-    runs-on: ubuntu-latest
-    services:
-      postgres:
-        image: postgres:12
-        env:
-          POSTGRES_PASSWORD: dev_password
-        options: >-
-          --health-cmd pg_isready
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-      elasticsearch:
-        image: docker.elastic.co/elasticsearch/elasticsearch-oss:6.8.18
-        env:
-          ES_JAVA_OPTS: "-Xms32m -Xmx256m"
-          discovery.type: single-node
-          cluster.routing.allocation.disk.threshold_enabled: "false"
-        options: >-
-          --health-cmd "curl -fsS http://localhost:9200/_cluster/health"
-          --health-interval 10s
-          --health-timeout 90s
-          --health-retries 5
-    steps:
-      - uses: actions/checkout@v4
-
-      - name: dev-env container metadata
-        id: meta
-        uses: docker/metadata-action@v5
-        with:
-          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
-          flavor: |
-            suffix=-dev-env
-
-      - name: Pull
-        run: docker pull "$DOCKER_IMAGE_CACHE_FROM"
-        env:
-          DOCKER_IMAGE_CACHE_FROM: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.meta.outputs.version }}
-
-      - name: Test
-        run: |
-          shopt -s globstar
-          tests=(${{ env.ELASTICSEARCH_TESTS_GLOB }})
-          docker run --rm \
-            --network "${{ job.services.postgres.network }}" \
-            -v "$(pwd)/test/tmp/artifacts:/app/test/tmp/artifacts" \
-            -e ELASTICSEARCH_TEST_API_VERSION=6 \
-            -e ELASTICSEARCH_TEST_TEMPLATE_VERSION=2 \
-            -e CI=true \
-            -e TESTS="${tests[*]}" \
-            -e MAXMIND_LICENSE_KEY="${MAXMIND_LICENSE_KEY}" \
-            "$DOCKER_IMAGE_CACHE_FROM" \
-            make test
-        env:
-          DOCKER_IMAGE_CACHE_FROM: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.meta.outputs.version }}
-          MAXMIND_LICENSE_KEY: ${{ secrets.MAXMIND_LICENSE_KEY }}
-
-      - name: Artifact permissions
-        if: ${{ always() }}
-        run: sudo chmod -R a+rwX test test/tmp/artifacts
-
-      - name: Dump docker logs
-        if: ${{ always() }}
-        uses: jwalton/gh-docker-logs@v2
-        with:
-          dest: test/tmp/artifacts/docker-logs
-
-      - name: Compress artifacts
-        if: ${{ always() }}
-        run: gzip -r ./test/tmp/artifacts/log ./test/tmp/artifacts/docker-logs
-
-      - name: Archive artifacts
-        if: ${{ always() }}
-        uses: actions/upload-artifact@v3
-        with:
-          name: test-artifacts-elasticsearch-v6
-          path: test/tmp/artifacts/
-
-  test_elasticsearch_v7_daily:
-    needs: [build]
-    runs-on: ubuntu-latest
-    services:
-      postgres:
-        image: postgres:12
-        env:
-          POSTGRES_PASSWORD: dev_password
-        options: >-
-          --health-cmd pg_isready
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-      elasticsearch:
-        image: docker.elastic.co/elasticsearch/elasticsearch-oss:7.10.2
-        env:
-          ES_JAVA_OPTS: "-Xms32m -Xmx256m"
-          discovery.type: single-node
-          cluster.routing.allocation.disk.threshold_enabled: "false"
-        options: >-
-          --health-cmd "curl -fsS http://localhost:9200/_cluster/health"
-          --health-interval 10s
-          --health-timeout 90s
-          --health-retries 5
-    steps:
-      - uses: actions/checkout@v4
-
-      - name: dev-env container metadata
-        id: meta
-        uses: docker/metadata-action@v5
-        with:
-          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
-          flavor: |
-            suffix=-dev-env
-
-      - name: Pull
-        run: docker pull "$DOCKER_IMAGE_CACHE_FROM"
-        env:
-          DOCKER_IMAGE_CACHE_FROM: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.meta.outputs.version }}
-
-      - name: Test
-        run: |
-          shopt -s globstar
-          tests=(${{ env.ELASTICSEARCH_TESTS_GLOB }})
-          docker run --rm \
-            --network "${{ job.services.postgres.network }}" \
-            -v "$(pwd)/test/tmp/artifacts:/app/test/tmp/artifacts" \
-            -e ELASTICSEARCH_TEST_API_VERSION=7 \
-            -e ELASTICSEARCH_TEST_TEMPLATE_VERSION=2 \
-            -e ELASTICSEARCH_TEST_INDEX_PARTITION=daily \
-            -e CI=true \
-            -e TESTS="${tests[*]}" \
-            -e MAXMIND_LICENSE_KEY="${MAXMIND_LICENSE_KEY}" \
-            "$DOCKER_IMAGE_CACHE_FROM" \
-            make test
-        env:
-          DOCKER_IMAGE_CACHE_FROM: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.meta.outputs.version }}
-          MAXMIND_LICENSE_KEY: ${{ secrets.MAXMIND_LICENSE_KEY }}
-
-      - name: Artifact permissions
-        if: ${{ always() }}
-        run: sudo chmod -R a+rwX test/tmp/artifacts
-
-      - name: Dump docker logs
-        if: ${{ always() }}
-        uses: jwalton/gh-docker-logs@v2
-        with:
-          dest: test/tmp/artifacts/docker-logs
-
-      - name: Compress artifacts
-        if: ${{ always() }}
-        run: gzip -r ./test/tmp/artifacts/log ./test/tmp/artifacts/docker-logs
-
-      - name: Archive artifacts
-        if: ${{ always() }}
-        uses: actions/upload-artifact@v3
-        with:
-          name: test-artifacts-elasticsearch-v7-daily
-          path: test/tmp/artifacts/
-
   publish-test-results:
     runs-on: ubuntu-latest
     needs:
       - test
-      - test_elasticsearch_v6
-      - test_elasticsearch_v7_daily
     permissions:
       checks: write
     if: always()

diff --git a/Gemfile b/Gemfile
@@ -37,7 +37,8 @@ gem "oj", "~> 3.10"
 # Database libraries
 gem "activerecord", "~> 7.1.1", :require => "active_record"
 gem "pg", "~> 1.5.3"
-gem "elasticsearch", "~> 7.13.3"
+gem "opensearch-ruby", "~> 3.1.0"
+gem "faraday-typhoeus", "~> 1.1.0"
 gem "active_attr", "~> 0.16.0"
 
 # Factories for test database data

diff --git a/Gemfile.lock b/Gemfile.lock
@@ -66,14 +66,6 @@ GEM
     crass (1.0.6)
     drb (2.2.0)
       ruby2_keywords
-    elasticsearch (7.13.3)
-      elasticsearch-api (= 7.13.3)
-      elasticsearch-transport (= 7.13.3)
-    elasticsearch-api (7.13.3)
-      multi_json
-    elasticsearch-transport (7.13.3)
-      faraday (~> 1)
-      multi_json
     encryptor (3.0.0)
     erubi (1.12.0)
     ethon (0.16.0)
@@ -82,29 +74,13 @@ GEM
       activesupport (>= 5.0.0)
     faker (3.2.3)
       i18n (>= 1.8.11, < 2)
-    faraday (1.10.3)
-      faraday-em_http (~> 1.0)
-      faraday-em_synchrony (~> 1.0)
-      faraday-excon (~> 1.1)
-      faraday-httpclient (~> 1.0)
-      faraday-multipart (~> 1.0)
-      faraday-net_http (~> 1.0)
-      faraday-net_http_persistent (~> 1.0)
-      faraday-patron (~> 1.0)
-      faraday-rack (~> 1.0)
-      faraday-retry (~> 1.0)
-      ruby2_keywords (>= 0.0.4)
-    faraday-em_http (1.0.0)
-    faraday-em_synchrony (1.0.0)
-    faraday-excon (1.1.0)
-    faraday-httpclient (1.0.1)
-    faraday-multipart (1.0.4)
-      multipart-post (~> 2)
-    faraday-net_http (1.0.1)
-    faraday-net_http_persistent (1.2.0)
-    faraday-patron (1.0.0)
-    faraday-rack (1.0.0)
-    faraday-retry (1.0.3)
+    faraday (2.9.0)
+      faraday-net_http (>= 2.0, < 3.2)
+    faraday-net_http (3.1.0)
+      net-http
+    faraday-typhoeus (1.1.0)
+      faraday (~> 2.0)
+      typhoeus (~> 1.4)
     ffi (1.16.3)
     i18n (1.14.1)
       concurrent-ruby (~> 1.0)
@@ -132,8 +108,9 @@ GEM
     minitest-sprint (1.2.2)
       path_expander (~> 1.1)
     multi_json (1.15.0)
-    multipart-post (2.3.0)
     mutex_m (0.2.0)
+    net-http (0.4.1)
+      uri
     net-protocol (0.2.2)
       timeout
     net-smtp (0.4.0.1)
@@ -147,6 +124,9 @@ GEM
       racc (~> 1.4)
     oj (3.16.3)
       bigdecimal (>= 3.0)
+    opensearch-ruby (3.1.0)
+      faraday (>= 1.0, < 3)
+      multi_json (>= 1.0)
     parallel (1.24.0)
     parser (3.3.0.5)
       ast (~> 2.4.1)
@@ -171,7 +151,7 @@ GEM
     rainbow (3.1.1)
     regexp_parser (2.9.0)
     rexml (3.2.6)
-    rubocop (1.60.1)
+    rubocop (1.60.2)
       json (~> 2.3)
       language_server-protocol (>= 3.17.0)
       parallel (~> 1.10)
@@ -190,7 +170,8 @@ GEM
     ruby-progressbar (1.13.0)
     ruby2_keywords (0.0.5)
     rubyzip (2.3.2)
-    selenium-webdriver (4.16.0)
+    selenium-webdriver (4.17.0)
+      base64 (~> 0.2)
       rexml (~> 3.2, >= 3.2.5)
       rubyzip (>= 1.2.2, < 3.0)
       websocket (~> 1.0)
@@ -201,6 +182,7 @@ GEM
     tzinfo (2.0.6)
       concurrent-ruby (~> 1.0)
     unicode-display_width (2.5.0)
+    uri (0.13.0)
     websocket (1.2.10)
     xpath (3.2.0)
       nokogiri (~> 1.8)
@@ -223,10 +205,10 @@ DEPENDENCIES
   capybara-shadowdom (~> 0.3.0)
   childprocess
   concurrent-ruby (~> 1.2.0)
-  elasticsearch (~> 7.13.3)
   encryptor (~> 3.0.0)
   factory_bot (~> 6.4.0)
   faker (~> 3.0)
+  faraday-typhoeus (~> 1.1.0)
   ice_nine (~> 0.11.2)
   minitest (~> 5.21.2)
   minitest-ci (~> 3.4.0)
@@ -237,6 +219,7 @@ DEPENDENCIES
   net-smtp (~> 0.4.0)
   nokogiri (~> 1.14)
   oj (~> 3.10)
+  opensearch-ruby (~> 3.1.0)
   pg (~> 1.5.3)
   rails_compatible_cookies_utils (~> 0.1.0)
   rainbow (~> 3.1.1)

diff --git a/build/package/verify/spec/localhost/service_spec.rb b/build/package/verify/spec/localhost/service_spec.rb
@@ -214,11 +214,11 @@ def install_package(version)
     expect(command_result.exit_status).to eql(0)
     output = command_result.stdout
     [
-      "elasticsearch",
       "geoip-auto-updater",
       "mongod",
       "mora",
       "nginx",
+      "opensearch",
       "rsyslog",
       "trafficserver",
       "web-delayed-job",
@@ -233,7 +233,7 @@ def install_package(version)
   it "does not contain unexpected errors in logs" do
     logs = Dir.glob("/opt/api-umbrella/var/log/*/current")
     expect(logs).to eql([
-      "/opt/api-umbrella/var/log/elasticsearch/current",
+      "/opt/api-umbrella/var/log/opensearch/current",
       "/opt/api-umbrella/var/log/geoip-auto-updater/current",
       "/opt/api-umbrella/var/log/mongod/current",
       "/opt/api-umbrella/var/log/mora/current",

diff --git a/build/package_dependencies.sh b/build/package_dependencies.sh
@@ -34,10 +34,6 @@ if [[ "$ID_NORMALIZED" == "rhel" ]]; then
     # TrafficServer
     libxml2
 
-    # ElasticSearch
-    java-1.8.0-openjdk-headless
-    which
-
     # rsyslog omelasticsearch
     libcurl