Merge remote-tracking branch 'origin/main' into cluster-cli

Signed-off-by: Peter Nied <[email protected]>

.codecov.yml

@@ -1,3 +1,5 @@
+codecov:
+  max_report_age: off
 coverage:
   status:
     project:

.github/CODEOWNERS

@@ -1,2 +1,2 @@
 # This should match the owning team set up in https://github.com/orgs/opensearch-project/teams
-*   @andrekurait @chelma @gregschohn @lewijacn @mikaylathompson @okhasawn @peternied @sumobrian
+*   @AndreKurait @chelma @gregschohn @lewijacn @mikaylathompson @okhasawn @peternied @sumobrian

.github/workflows/CI.yml

@@ -156,8 +156,8 @@ jobs:
           docker exec $(docker ps --filter "name=migration-console" -q) pipenv run pytest /root/lib/integ_test/integ_test/replayer_tests.py --unique_id="testindex" -s
       - name: Clean up migrations docker images before caching
         run: |
-          docker stop $(docker ps -q) && docker rm $(docker ps -aq)
-          docker image ls --format '{{.Repository}}:{{.Tag}}' | grep '^migrations/' | grep -v '<none>' | xargs -I {} docker image rm {}
+          docker stop $(docker ps -q) && docker system prune --volumes -f
+          docker image ls --format '{{.Repository}}:{{.Tag}}' | grep '^migrations/' | xargs -I {} docker image rm {}
 
   cdk-tests:
     runs-on: ubuntu-latest

.github/workflows/release-drafter.yml

@@ -5,12 +5,30 @@ on:
   push:
     tags:
       - "*"
+env:
+  java-version: '11'
+  gradle-version: '8.0.2'
+
+permissions:
+  id-token: write
+  contents: read
+  issues: write
 
 jobs:
   draft-a-release:
     name: Draft a release
+    if: github.repository == 'opensearch-project/opensearch-migrations'
     runs-on: ubuntu-latest
+    outputs:
+      dockerhub-password: ${{ steps.retrieve-values.outputs.dockerhub-password }}
     steps:
+      - uses: actions/setup-java@v4
+        with:
+          java-version: ${{ env.java-version }}
+          distribution: 'corretto'
+      - uses: gradle/actions/setup-gradle@v3
+        with:
+          gradle-version: ${{ env.gradle-version }}
       - uses: actions/checkout@v4
       - id: get_data
         run: |
@@ -29,6 +47,48 @@ jobs:
         run: |
           wget https://github.com/opensearch-project/opensearch-migrations/archive/refs/tags/${{ steps.get_data.outputs.version }}.tar.gz -O artifacts.tar.gz
           ./gradlew publishMavenJavaPublicationToMavenRepository -Dbuild.snapshot=false -Dbuild.version=0.${{ steps.get_data.outputs.version }} && tar -C build -cvf traffic-capture-artifacts.tar.gz repository
+      - name: Build Docker Images
+        run: |
+          ./deployment/cdk/opensearch-service-migration/buildDockerImages.sh
+      - name: Tag Docker image
+        run: |
+          docker tag migrations/migration_console:latest opensearchstaging/opensearch-migrations-console:${{ steps.get_data.outputs.version }}
+          docker tag migrations/migration_console:latest opensearchstaging/opensearch-migrations-console:latest
+          docker tag migrations/traffic_replayer:latest opensearchstaging/opensearch-migrations-traffic-replayer:${{ steps.get_data.outputs.version }}
+          docker tag migrations/traffic_replayer:latest opensearchstaging/opensearch-migrations-traffic-replayer:latest
+          docker tag migrations/capture_proxy:latest opensearchstaging/opensearch-migrations-traffic-capture-proxy:${{ steps.get_data.outputs.version }}
+          docker tag migrations/capture_proxy:latest opensearchstaging/opensearch-migrations-traffic-capture-proxy:latest
+          docker tag migrations/reindex_from_snapshot:latest opensearchstaging/opensearch-migrations-reindex-from-snapshot:${{ steps.get_data.outputs.version }}
+          docker tag migrations/reindex_from_snapshot:latest opensearchstaging/opensearch-migrations-reindex-from-snapshot:latest
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          role-to-assume: ${{ secrets.MIGRATIONS_DOCKER_ROLE }}
+          aws-region: us-east-1
+      - name: Retrieve Values
+        id: retrieve-values
+        run: |
+          DOCKERHUB_PASSWORD=`aws secretsmanager get-secret-value --secret-id jenkins-staging-dockerhub-credential --query SecretString --output text`
+          echo "::add-mask::$DOCKERHUB_PASSWORD"
+          echo "dockerhub-password=$DOCKERHUB_PASSWORD" >> $GITHUB_OUTPUT
+      - name: Login to DockerHub
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.MIGRATIONS_DOCKER_USERNAME }}
+          password: ${{ steps.retrieve-values.outputs.dockerhub-password }}
+      - name: Push Docker image to Docker Hub
+        run: |
+          docker push opensearchstaging/opensearch-migrations-console:${{ steps.get_data.outputs.version }}
+          docker push opensearchstaging/opensearch-migrations-console:latest
+          docker push opensearchstaging/opensearch-migrations-traffic-replayer:${{ steps.get_data.outputs.version }}
+          docker push opensearchstaging/opensearch-migrations-traffic-replayer:latest
+          docker push opensearchstaging/opensearch-migrations-traffic-capture-proxy:${{ steps.get_data.outputs.version }}
+          docker push opensearchstaging/opensearch-migrations-traffic-capture-proxy:latest
+          docker push opensearchstaging/opensearch-migrations-reindex-from-snapshot:${{ steps.get_data.outputs.version }}
+          docker push opensearchstaging/opensearch-migrations-reindex-from-snapshot:latest
+      - name: Logout to DockerHub
+        if: always()
+        run: docker logout
       - name: Draft a release
         uses: softprops/action-gh-release@v2
         with:

CreateSnapshot/README.md

@@ -4,34 +4,50 @@
 
 This tool exposes the underlying Reindex-From-Snapshot (RFS) core library in a executable that will create a snapshot of a source cluster consistent with the expectations of RFS.  In brief, that means taking the snapshot with the global metadata included.  Currently, there are two may ways this tool can perform the snapshot:
 
-* On-disk: The tool kick off a snapshot on the source cluster tell the cluster to store files on disk.  Keep in mind that the disk in question here is the disk of the nodes in the cluster.  This is typically used when your nodes share a filesystem, either because have have a network file system attached or they're all running on the same host.
 * S3: The tool kick off a snapshot on the source cluster tell the cluster to store files to a specified S3 URI.
+* On-disk: The tool kick off a snapshot on the source cluster tell the cluster to store files on disk.  Keep in mind that the disk in question here is the disk of the nodes in the cluster.  This is typically used when your nodes share a filesystem, either because have have a network file system attached or they're all running on the same host.
 
 There's not much magic involved; the tool will reach out to the host specificed using the Elasticsearch REST API to register a new repository then create a snapshot.  This means you'll need a network path to the source cluster from where you're running the tool to the source cluster.
 
-## How do use the tool?
+## How to use the tool
 
-You can kick off the tool using Gradle.
+You can kick off the tool locally using Gradle.
 
-### On-Disk Snapshot
+### S3 Snapshot
 
 From the root directory of the repo, run a CLI command like so:
 
+```shell
+./gradlew CreateSnapshot:run --args='--snapshot-name reindex-from-snapshot --s3-repo-uri s3://your-s3-uri --s3-region us-fake-1 --source-host http://hostname:9200'
 ```
+
+In order for this succeed, you'll need to make sure:
+* You have valid AWS Credentials in your key ring (~/.aws/credentials) with permission to operate on the S3 URI specified
+* The S3 URI currently exists, and is in the region you specify
+* There are no other snapshots present in that S3 URI
+
+### On-Disk Snapshot
+
+From the root directory of the repo, run a CLI command like so:
+
+```shell
 ./gradlew CreateSnapshot:run --args='--snapshot-name reindex-from-snapshot --file-system-repo-path /snapshot --source-host http://hostname:9200'
 ```
 
 In order for this to succeed, you must first configure your Elasticsearch source cluster register the directory you want to use as a snapshot repository.  See [the docs for ES 7.10 here](https://www.elastic.co/guide/en/elasticsearch/reference/7.10/snapshots-register-repository.html#snapshots-filesystem-repository) for an example of how to do this.
 
-### S3 Snapshot
+### Handling Auth
 
-From the root directory of the repo, run a CLI command like so:
+If your source cluster has basic auth enabled, you can supply those credentials to the tool via the CLI:
 
-```
-./gradlew CreateSnapshot:run --args='--snapshot-name reindex-from-snapshot --s3-repo-uri s3://your-s3-uri --s3-region us-fake-1 --source-host http://hostname:9200'
+```shell
+./gradlew CreateSnapshot:run --args='--snapshot-name reindex-from-snapshot --s3-repo-uri s3://your-s3-uri --s3-region us-fake-1 --source-host http://hostname:9200 --source-username <user> --source-password <pass>'
 ```
 
-In order for this succeed, you'll need to make sure:
-* You have valid AWS Credentials in your key ring (~/.aws/credentials) with permission to operate on the S3 URI specified
-* The S3 URI currently exists, and is in the region you specify
-* There are no other snapshots present in that S3 URI
+### Snapshot Creation Rate
+
+You can tweak the maximum rate at which the source cluster's nodes will create the snapshot.  The tradeoff here is that increasing the speed at which the snapshot is created also increases the amount of resources the nodes are devoting to that creation instead of serving normal traffic.  If you don't specify a number for this, it will use the default for whatever version of source cluster you're using.  See [the docs for ES 7.10 here](https://www.elastic.co/guide/en/elasticsearch/reference/7.10/put-snapshot-repo-api.html#put-snapshot-repo-api-request-body) for more context.
+
+```shell
+./gradlew CreateSnapshot:run --args='--snapshot-name reindex-from-snapshot --s3-repo-uri s3://your-s3-uri --s3-region us-fake-1 --source-host http://hostname:9200 --max-snapshot-rate-mb-per-node 100'
+```

CreateSnapshot/src/main/java/com/rfs/CreateSnapshot.java

@@ -12,12 +12,12 @@
 import com.beust.jcommander.Parameter;
 import com.beust.jcommander.ParameterException;
 import com.beust.jcommander.ParametersDelegate;
-import com.rfs.common.ConnectionDetails;
 import com.rfs.common.FileSystemSnapshotCreator;
 import com.rfs.common.OpenSearchClient;
 import com.rfs.common.S3SnapshotCreator;
 import com.rfs.common.SnapshotCreator;
 import com.rfs.common.TryHandlePhaseFailure;
+import com.rfs.common.http.ConnectionContext;
 import com.rfs.worker.SnapshotRunner;
 import lombok.AllArgsConstructor;
 import lombok.Getter;
@@ -26,6 +26,9 @@
 @Slf4j
 public class CreateSnapshot {
     public static class Args {
+        @Parameter(names = {"--help", "-h"}, help = true, description = "Displays information about how to use this tool")
+        private boolean help;
+
         @Parameter(names = { "--snapshot-name" }, required = true, description = "The name of the snapshot to migrate")
         public String snapshotName;
 
@@ -42,7 +45,7 @@ public static class Args {
         public String s3Region;
 
         @ParametersDelegate
-        public ConnectionDetails.SourceArgs sourceArgs = new ConnectionDetails.SourceArgs();
+        public ConnectionContext.SourceArgs sourceArgs = new ConnectionContext.SourceArgs();
 
         @Parameter(names = {
             "--no-wait" }, description = "Optional.  If provided, the snapshot runner will not wait for completion")
@@ -52,6 +55,10 @@ public static class Args {
             "--max-snapshot-rate-mb-per-node" }, required = false, description = "The maximum snapshot rate in megabytes per second per node")
         public Integer maxSnapshotRateMBPerNode;
 
+        @Parameter(names = {
+            "--s3-role-arn" }, required = false, description = "The role ARN the cluster will assume to write a snapshot to S3")
+        public String s3RoleArn;
+
         @Parameter(required = false, names = {
             "--otel-collector-endpoint" }, arity = 1, description = "Endpoint (host:port) for the OpenTelemetry Collector to which metrics logs should be"
                 + "forwarded. If no value is provided, metrics will not be forwarded.")
@@ -66,9 +73,14 @@ public static class S3RepoInfo {
     }
 
     public static void main(String[] args) throws Exception {
-        // Grab out args
         Args arguments = new Args();
-        JCommander.newBuilder().addObject(arguments).build().parse(args);
+        JCommander jCommander = JCommander.newBuilder().addObject(arguments).build();
+        jCommander.parse(args);
+
+        if (arguments.help) {
+            jCommander.usage();
+            return;
+        }
 
         var rootContext = new RootSnapshotContext(
             RootOtelContext.initializeOpenTelemetryWithCollectorOrAsNoop(arguments.otelCollectorEndpoint, "rfs"),
@@ -100,9 +112,10 @@ public static void main(String[] args) throws Exception {
                     arguments.s3RepoUri,
                     arguments.s3Region,
                     arguments.maxSnapshotRateMBPerNode,
+                    arguments.s3RoleArn,
                     rootContext.createSnapshotCreateContext()
                 )),
-            new OpenSearchClient(new ConnectionDetails(arguments.sourceArgs)),
+            new OpenSearchClient(arguments.sourceArgs.toConnectionContext()),
             arguments.noWait
         );
     }