Experiment with k8s structure pattern

TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/services.yaml

@@ -1,11 +1,11 @@
 source_cluster:
-  endpoint: "https://capture-proxy:9200"
+  endpoint: "https://capture-proxy:9201"
   allow_insecure: true
   basic_auth:
     username: "admin"
     password: "admin"
 target_cluster:
-  endpoint: "https://opensearchtarget:9200"
+  endpoint: "https://opensearch-cluster-master:9200"
   allow_insecure: true
   basic_auth:
     username: "admin"
@@ -19,14 +19,14 @@ backfill:
 replay:
   docker:
 snapshot:
-  snapshot_name: "snapshot_2023_01_01"
+  snapshot_name: "rfs-snapshot"
   fs:
-    repo_path: "/snapshot/test-console"
-  otel_endpoint: "http://otel-collector:4317"
+    repo_path: "/storage/snapshot"
+  otel_endpoint: "http://localhost:4317"
 metadata_migration:
   from_snapshot:
   min_replicas: 0
-  otel_endpoint: "http://otel-collector:4317"
+  otel_endpoint: "http://localhost:4317"
 kafka:
-  broker_endpoints: "kafka:9092"
+  broker_endpoints: "kafka-cluster-kafka-bootstrap:9092"
   standard: ""

deployment/k8/.gitignore

@@ -0,0 +1 @@
+*.tgz

deployment/k8/README.md

@@ -0,0 +1,85 @@
+# Kubernetes Deployment
+
+## Prerequisites 
+
+#### Install kubectl
+Follow instructions [here](https://kubernetes.io/docs/tasks/tools/) to install the Kubernetes command-line tool. This will be the go-to tool for interacting with the Kubernetes cluster
+
+#### Install helm
+Follow instructions [here](https://helm.sh/docs/intro/install/) to install helm. helm will be used for deploying to the Kubernetes cluster
+
+#### Install docker
+Follow instructions [here](https://docs.docker.com/engine/install/) to set up Docker. Docker will be used to build Docker images as well as run a local Kubernetes cluster. Later versions are recommended.
+
+
+## Local Kubernetes Cluster
+Creating a local Kubernetes cluster is useful for testing and developing a given deployment. There are a few different tools for running a Kubernetes cluster locally. This documentation focuses on using [Minikube](https://github.com/kubernetes/minikube) to run the local Kubernetes cluster.
+
+### Install Minikube
+Follow instructions [here](https://minikube.sigs.k8s.io/docs/start/?arch=%2Fmacos%2Fx86-64%2Fstable%2Fbinary+download) to install Minikube
+
+### Loading Docker images into Minikube
+Since Minikube uses a different Docker registry than the normal host machine, the Docker images shown will differ from that on the host machine. The script `buildDockerImagesMini.sh` in this directory will configure the environment to use the Minikube Docker registry and build the Docker images into Minikube
+
+Show Docker images available to Minikube
+```shell
+minikube image ls
+```
+Build Docker images into Minikube
+```shell
+./buildDockerImagesMini.sh
+```
+
+### Start/Pause/Delete
+A convenience script `minikubeLocal.sh` is located in this directory which wraps the Minikube commands to start/pause/delete Minikube. This is useful for automatically handling items such as mounting the local repo and creating a tunnel to make localhost calls to containers
+```shell
+./miniKubeLocal.sh --start
+./miniKubeLocal.sh --pause
+./miniKubeLocal.sh --delete
+```
+
+
+## Deploying
+
+### Full environment
+Guide for deploying a complete environment helm chart comprised of many Migration service helm charts
+
+The full environment helm charts consists of:
+* Source cluster
+* Target cluster
+* Migration services
+
+**Note**: For first-time deployments and deployments after changes have been made to a dependent helm package, such as the `migration-console` chart, the following command is needed to update dependent charts
+```shell
+helm dependency update environments/full-environment
+```
+
+The full environment helm chart can be deployed with the helm command
+```shell
+helm install local environments/full-environment
+```
+
+### Specific services
+Guide for deploying an individual Migration service helm chart
+
+Most migration services have a dependency on Persistent Volumes that can be installed to the Kubernetes cluster using the following commands
+```shell
+helm install shared-logs shared/shared-logs-vol
+helm install snapshot-vol shared/snapshot-vol
+```
+
+A particular service could then be deployed with a command similar to the below.
+```shell
+helm install migration-console migration-console
+```
+
+## Uninstalling
+To show all helm deployments
+```shell
+helm list
+```
+
+To uninstall a particular helm deployment
+```shell
+helm uninstall <deployment_name>
+```

deployment/k8/aws/ack-resource-setup/.helmignore

@@ -0,0 +1,23 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*.orig
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/

deployment/k8/aws/ack-resource-setup/Chart.yaml

@@ -0,0 +1,9 @@
+apiVersion: v2
+name: ack-resource-setup
+description: A Helm chart for deploying required AWS resources for running the Migration Assistant
+version: 0.1.0
+appVersion: "3.5.0"
+dependencies:
+  - name: strimzi-kafka-operator
+    version: 0.43.0
+    repository: https://strimzi.io/charts/

deployment/k8/aws/ack-resource-setup/templates/kafka-single-node.yaml

@@ -0,0 +1,51 @@
+#https://github.com/strimzi/strimzi-kafka-operator/blob/release-0.43.x/examples/kafka/kraft/kafka-single-node.yaml
+apiVersion: kafka.strimzi.io/v1beta2
+kind: KafkaNodePool
+metadata:
+  name: dual-role
+  labels:
+    strimzi.io/cluster: kafka-cluster
+spec:
+  replicas: 1
+  roles:
+    - controller
+    - broker
+  storage:
+    type: jbod
+    volumes:
+      - id: 0
+        type: persistent-claim
+        size: 10Gi
+        deleteClaim: true
+        kraftMetadata: shared
+---
+
+apiVersion: kafka.strimzi.io/v1beta2
+kind: Kafka
+metadata:
+  name: kafka-cluster
+  annotations:
+    strimzi.io/node-pools: enabled
+    strimzi.io/kraft: enabled
+spec:
+  kafka:
+    version: 3.8.0
+    metadataVersion: 3.8-IV0
+    listeners:
+      - name: plain
+        port: 9092
+        type: internal
+        tls: false
+      - name: tls
+        port: 9093
+        type: internal
+        tls: true
+    config:
+      offsets.topic.replication.factor: 1
+      transaction.state.log.replication.factor: 1
+      transaction.state.log.min.isr: 1
+      default.replication.factor: 1
+      min.insync.replicas: 1
+  entityOperator:
+    topicOperator: {}
+    userOperator: {}

deployment/k8/aws/ack-resource-setup/values.yaml

@@ -0,0 +1,38 @@
+# Configuration for Strimzi Kafka Operator
+#strimzi-kafka-operator:
+# You can add operator-specific configurations here if needed
+# For example, enabling metrics, RBAC settings, etc.
+
+## Configuration for the Kafka Cluster
+#kafka:
+#  name: kafka-cluster
+#  version: "3.6.0" # Kafka version supporting KRaft
+#  replicas: 1
+#  config:
+#    processRoles: "broker,controller"
+#    nodeId: 1
+#    controllerQuorumVoters: "[email protected]:9093"
+#    interBrokerProtocolVersion: "3.5"
+#    logMessageFormatVersion: "3.5"
+#    autoCreateTopicsEnable: "true"
+#  storage:
+#    type: "ephemeral" # Use "persistent-claim" for persistent storage
+#    size: "20Gi"
+#    class: "managed-nfs-storage" # Set to empty string "" if not using a specific storage class
+#    deleteClaim: false
+#
+## Configuration for the Controller
+#controller:
+#  replicas: 1
+#  resources:
+#    limits:
+#      memory: "2Gi"
+#      cpu: "1000m"
+#    requests:
+#      memory: "1Gi"
+#      cpu: "500m"
+#  storage:
+#    type: "ephemeral" # Use "persistent-claim" for persistent storage
+#    size: "10Gi"
+#    class: "managed-nfs-storage" # Set to empty string "" if not using a specific storage class
+#    deleteClaim: false

deployment/k8/buildDockerImagesMini.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+# Allow executing this script from any dir
+script_abs_path=$(readlink -f "$0")
+script_dir_abs_path=$(dirname "$script_abs_path")
+cd "$script_dir_abs_path" || exit
+
+cd ../.. || exit
+
+eval $(minikube docker-env)
+./gradlew :buildDockerImages -x test

deployment/k8/migration-assistant/Chart.lock

@@ -0,0 +1,15 @@
+dependencies:
+- name: opensearch
+  repository: https://opensearch-project.github.io/helm-charts/
+  version: 2.23.1
+- name: kafka-op
+  repository: file://../services/kafka-op
+  version: 0.1.0
+- name: migration-console
+  repository: file://../services/migration-console
+  version: 0.1.0
+- name: capture-proxy
+  repository: file://../services/capture-proxy
+  version: 0.1.0
+digest: sha256:3dcd125d6e968ddfc212318128d5f4222822bf33d0593f061f3c2eed40cea144
+generated: "2024-10-24T15:36:04.42393-05:00"

deployment/k8/migration-assistant/Chart.yaml

@@ -0,0 +1,36 @@
+apiVersion: v2
+name: migration-assistant
+version: 0.1.0
+dependencies:
+#  - name: elasticsearch
+#    version: "0.1.0"
+#    repository: "file://../services/elasticsearch"
+  - name: opensearch
+    version: "2.23.1"
+    repository: "https://opensearch-project.github.io/helm-charts/"
+    alias: opensearch-helm
+    condition: opensearch-helm-enabled
+  - name: kafka-op
+    version: "0.1.0"
+    repository: "file://../services/kafka-op"
+    condition: kafka-op-enabled
+  - name: migration-console
+    version: "0.1.0"
+    repository: "file://../services/migration-console"
+    condition: migration-console-enabled
+#  - name: replayer
+#    version: "0.1.0"
+#    repository: "file://../services/replayer"
+  - name: capture-proxy
+    version: "0.1.0"
+    repository: "file://../services/capture-proxy"
+    condition: capture-proxy-enabled
+#  - name: reindex-from-snapshot
+#    version: "0.1.0"
+#    repository: "file://../services/reindex-from-snapshot"
+#  - name: shared-logs-vol
+#    version: "0.1.0"
+#    repository: "file://../services/shared/shared-logs-vol"
+#  - name: snapshot-vol
+#    version: "0.1.0"
+#    repository: "file://../services/shared/snapshot-vol"