sillsdev · rmunn · Oct 28, 2024
diff --git a/.github/workflows/deploy-branch.yaml b/.github/workflows/deploy-branch.yaml
@@ -41,10 +41,18 @@ jobs:
       version: ${{ needs.set-version.outputs.version }}
       label-latest: false
 
+  build-crdtmerge:
+    name: Build CrdtMerge
+    needs: [ set-version ]
+    uses: ./.github/workflows/lexbox-crdtmerge.yaml
+    with:
+      version: ${{ needs.set-version.outputs.version }}
+      label-latest: false
+
   deploy:
     name: Deploy Develop
     uses: ./.github/workflows/deploy.yaml
-    needs: [ build-api, build-ui, build-hgweb, set-version ]
+    needs: [ build-api, build-ui, build-hgweb, build-crdtmerge, set-version ]
     secrets: inherit
     with:
       version: ${{ needs.set-version.outputs.version }}

diff --git a/.github/workflows/deploy.yaml b/.github/workflows/deploy.yaml
@@ -50,6 +50,7 @@ jobs:
       url: https://${{ inputs.deploy-domain }}
     outputs:
       api-version: ${{ steps.get-api-version.outputs.result }}
+      crdtmerge-version: ${{ steps.get-crdtmerge-version.outputs.result }}
       ui-version: ${{ steps.get-ui-version.outputs.result }}
     steps:
       - name: Checkout lexbox repo
@@ -80,6 +81,11 @@ jobs:
         id: get-api-version
         with:
           cmd: yq '.images.[] | select(.name == "ghcr.io/sillsdev/lexbox-api").newTag' "fleet/${{ inputs.k8s-environment }}/kustomization.yaml"
+      - name: Get CrdtMerge version
+        uses: mikefarah/yq@0b34c9a00de1c575a34eea05af1d956a525c4fc1 # v4.34.2
+        id: get-crdtmerge-version
+        with:
+          cmd: yq '.images.[] | select(.name == "ghcr.io/sillsdev/lexbox-crdtmerge").newTag' "fleet/${{ inputs.k8s-environment }}/kustomization.yaml"
       - name: Get UI version
         uses: mikefarah/yq@0b34c9a00de1c575a34eea05af1d956a525c4fc1 # v4.34.2
         id: get-ui-version

diff --git a/.github/workflows/integration-test-gha.yaml b/.github/workflows/integration-test-gha.yaml
@@ -39,7 +39,16 @@ jobs:
         uses: mikefarah/yq@0b34c9a00de1c575a34eea05af1d956a525c4fc1 # v4.34.2
         with:
           cmd: yq eval -i '(.images.[] | select(.name == "ghcr.io/sillsdev/lexbox-api").newTag) = "${{ inputs.lexbox-api-tag }}"' "./deployment/gha/kustomization.yaml"
-      # It's also possible that hgweb and/or ui image may have changed; if so, pull them and update kustomization.yaml for them as well
+      # It's also possible that hgweb, crdtmerge, and/or ui image may have changed; if so, pull them and update kustomization.yaml for them as well
+      - name: Pull crdtmerge if updated
+        id: crdtmerge_image
+        continue-on-error: true
+        run: docker pull ghcr.io/sillsdev/lexbox-crdtmerge:${{ inputs.lexbox-api-tag }}
+      - name: Update image crdtmerge version
+        if: ${{ steps.crdtmerge_image.outcome == 'success' }}
+        uses: mikefarah/yq@0b34c9a00de1c575a34eea05af1d956a525c4fc1 # v4.34.2
+        with:
+          cmd: yq eval -i '(.images.[] | select(.name == "ghcr.io/sillsdev/lexbox-crdtmerge").newTag) = "${{ inputs.lexbox-api-tag }}"' "./deployment/gha/kustomization.yaml"
       - name: Pull hgweb if updated
         id: hgweb_image
         continue-on-error: true

diff --git a/.github/workflows/lexbox-crdtmerge.yaml b/.github/workflows/lexbox-crdtmerge.yaml
@@ -0,0 +1,98 @@
+name: Build CrdtMerge
+
+# https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions#on
+on:
+  workflow_call:
+    inputs:
+      version:
+        description: 'The version of the image to build'
+        required: true
+        type: string
+      label-latest:
+        description: 'The label to apply to the latest image'
+        type: boolean
+        default: false
+
+env:
+  IMAGE_NAME: ghcr.io/sillsdev/lexbox-crdtmerge
+
+
+jobs:
+  publish-api:
+    timeout-minutes: 60
+    runs-on: ubuntu-latest
+
+    # postgres db is for automated tests
+    # services:
+    #   postgres:
+    #     image: postgres:15-alpine
+    #     env:
+    #       POSTGRES_PASSWORD: 972b722e63f549938d07bd8c4ee5086c
+    #       POSTGRES_DB: lexbox-tests
+    #     # Set health checks to wait until postgres has started
+    #     options: >-
+    #       --health-cmd pg_isready
+    #       --health-interval 10s
+    #       --health-timeout 5s
+    #       --health-retries 5
+    #     ports:
+    #       # Maps tcp port 5432 on service container to the host
+    #       - 5433:5432
+
+    env:
+      # https://docs.docker.com/develop/develop-images/build_enhancements/
+      DOCKER_BUILDKIT: 1
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: true
+      - uses: actions/setup-dotnet@v4
+        with:
+          dotnet-version: '8.x'
+      - name: Dotnet build
+        run: dotnet build backend/CrdtMerge/CrdtMerge.csproj
+      # TODO: Write CrdtMerge unit tests, probably based on existing sync tests
+      # - name: Unit tests
+      #   run: dotnet test backend/CrdtMerge/CrdtMerge.csproj --logger:"xunit;LogFileName={assembly}.results.xml" --results-directory ./test-results --filter "Category!=Integration&Category!=FlakyIntegration" --blame-hang-timeout 10m
+      # - name: Publish unit test results
+      #   uses: EnricoMi/publish-unit-test-result-action@8885e273a4343cd7b48eaa72428dea0c3067ea98 # v2.14.0
+      #   if: always()
+      #   with:
+      #     check_name: C# Unit Tests
+      #     files: ./test-results/*.xml
+      # - name: Upload test results
+      #   if: always()
+      #   uses: actions/upload-artifact@v4
+      #   with:
+      #     name: dotnet-unit-test-results
+      #     path: ./test-results
+
+      - name: Docker meta
+        id: meta
+        if: ${{ !env.ACT }}
+        uses: docker/metadata-action@8e5442c4ef9f78752691e2d8f8d19755c6f78e81 # v5.5.1
+        with:
+          images: ${{ env.IMAGE_NAME }}
+          tags: |
+            type=ref,event=branch
+            type=ref,event=pr
+            type=raw,enable=${{ inputs.label-latest }},value=latest
+            type=raw,value=${{ inputs.version }}
+
+      - name: ghcr.io login
+        uses: docker/login-action@e92390c5fb421da1463c202d546fed0ec5c39f20 # v3.1.0
+        if: ${{ !env.ACT }}
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - uses: docker/build-push-action@2cdde995de11925a030ce8070c3d77a52ffcf1c0 # v5.3.0
+        with:
+          context: backend
+          build-args: |
+            APP_VERSION=${{ inputs.version }}
+          push: ${{ !env.ACT && github.repository == 'sillsdev/languageforge-lexbox' }}
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
diff --git a/.github/workflows/release-pipeline.yaml b/.github/workflows/release-pipeline.yaml
@@ -43,10 +43,18 @@ jobs:
       version: ${{ needs.set-version.outputs.version }}
       label-latest: true
 
+  build-crdtmerge:
+    name: Build crdtmerge
+    needs: [ set-version ]
+    uses: ./.github/workflows/lexbox-crdtmerge.yaml
+    with:
+      version: ${{ needs.set-version.outputs.version }}
+      label-latest: true
+
   deploy:
     name: Deploy Staging
     uses: ./.github/workflows/deploy.yaml
-    needs: [ build-api, build-ui, build-hgweb, set-version ]
+    needs: [ build-api, build-ui, build-hgweb, build-crdtmerge, set-version ]
     secrets: inherit
     with:
       version: ${{ needs.set-version.outputs.version }}

diff --git a/backend/CrdtMerge/Dockerfile b/backend/CrdtMerge/Dockerfile
@@ -0,0 +1,25 @@
+# syntax=docker/dockerfile:1
+FROM mcr.microsoft.com/dotnet/aspnet:9.0 AS base
+WORKDIR /app
+EXPOSE 80
+EXPOSE 443
+
+FROM mcr.microsoft.com/dotnet/sdk:9.0 AS build
+
+COPY . .
+RUN --mount=type=cache,target=/root/.nuget/packages dotnet restore "CrdtMerge/CrdtMerge.csproj"
+
+ARG APP_VERSION
+LABEL version=$APP_VERSION
+
+RUN --mount=type=cache,target=/root/.nuget/packages dotnet build /p:InformationalVersion=$APP_VERSION "CrdtMerge/CrdtMerge.csproj" -c Release -o /app/build
+
+FROM build AS publish
+RUN --mount=type=cache,target=/root/.nuget/packages dotnet publish /p:InformationalVersion=$APP_VERSION "CrdtMerge/CrdtMerge.csproj" -c Release -o /app/publish
+
+FROM base AS final
+RUN mkdir -p /var/lib/crdtmerge && chown -R www-data:www-data /var/lib/crdtmerge
+WORKDIR /app
+COPY --from=publish /app/publish .
+USER www-data:www-data
+ENTRYPOINT ["dotnet", "CrdtMerge.dll"]
diff --git a/backend/CrdtMerge/appsettings.json b/backend/CrdtMerge/appsettings.json
@@ -1,5 +1,5 @@
 {
-  "CrdtMergeConfig": {
+  "SendReceiveConfig": {
     "LexboxUsername": null
   },
   "Logging": {

diff --git a/deployment/base/crdtmerge-deployment.yaml b/deployment/base/crdtmerge-deployment.yaml
@@ -0,0 +1,144 @@
+# https://kubernetes.io/docs/concepts/services-networking/service/#defining-a-service
+apiVersion: v1
+kind: Service
+metadata:
+  name: crdtmerge
+  namespace: languagedepot
+  labels:
+    app: crdtmerge
+spec:
+  type: ClusterIP
+  clusterIP: None
+  selector:
+    app: crdtmerge
+  ports:
+  - name: http
+    protocol: TCP
+    port: 80
+
+---
+
+# https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#writing-a-deployment-spec
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: crdtmerge
+  namespace: languagedepot
+  labels:
+    app: crdtmerge
+spec:
+  selector:
+    matchLabels:
+      app: crdtmerge
+  strategy:
+    rollingUpdate:
+      maxSurge: 2
+      maxUnavailable: 0
+    type: RollingUpdate
+  template:
+    # https://kubernetes.io/docs/concepts/workloads/pods/#pod-templates
+    metadata:
+      labels:
+        app: crdtmerge
+    spec:
+      securityContext:
+        runAsGroup: 33
+        runAsUser: 33
+        runAsNonRoot: true
+      containers:
+      - name: crdtmerge
+        image: ghcr.io/sillsdev/lexbox-crdtmerge:develop
+        imagePullPolicy: IfNotPresent
+        # https://kubernetes.io/docs/concepts/configuration/manage-resources-containers
+        resources:
+          requests:
+            memory: 1500Mi
+          limits:
+            memory: 2400Mi
+        startupProbe:
+          httpGet:
+            port: 80
+            path: /api/healthz
+          failureThreshold: 30
+          periodSeconds: 10
+        ports:
+          - containerPort: 80
+
+        volumeMounts:
+        - name: crdtmerge
+          mountPath: /var/lib/crdtmerge
+
+        env:
+          - name: DOTNET_URLS
+            value: http://0.0.0.0:80
+          - name: ASPNETCORE_ENVIRONMENT
+            valueFrom:
+              configMapKeyRef:
+                name: app-config
+                key: environment-name
+          - name: K8S_NODE_NAME
+            valueFrom:
+              fieldRef:
+                fieldPath: spec.nodeName
+          - name: K8S_POD_NAME
+            valueFrom:
+              fieldRef:
+                fieldPath: metadata.name
+          - name: POSTGRES_DB
+            valueFrom:
+              secretKeyRef:
+                  key: POSTGRES_DB
+                  name: db
+          - name: POSTGRES_PASSWORD
+            valueFrom:
+              secretKeyRef:
+                  key: POSTGRES_PASSWORD
+                  name: db
+          - name: DbConfig__LexBoxConnectionString
+            value: Host=db;Port=5432;Username=postgres;Password=$(POSTGRES_PASSWORD);Database=$(POSTGRES_DB)
+          - name: SendReceiveConfig__ProjectStorageRoot
+            value: /var/lib/crdtmerge/projects
+          - name: HgConfig__SendReceiveDomain
+            valueFrom:
+              configMapKeyRef:
+                name: app-config
+                key: hg-domain
+          - name: SendReceiveConfig__LexboxUrl
+            value: http://lexbox:5158/
+          # - name: SendReceiveConfig__HgUrl
+          #   value: http://lexbox:5158/hg/
+          - name: SendReceiveConfig__LexboxUsername
+            valueFrom:
+              secretKeyRef:
+                key: CRDT_MERGE_SEND_RECEIVE_USERNAME
+                name: crdtmerge
+          - name: SendReceiveConfig__LexboxPassword
+            valueFrom:
+              secretKeyRef:
+                key: CRDT_MERGE_SEND_RECEIVE_PASSWORD
+                name: crdtmerge
+          - name: SendReceiveConfig__FdoDataModelVersion
+            value: "7000072"
+
+      initContainers:
+      - name: populate-crdt-project-storage
+        securityContext:
+          # Make sure we're authorized to set ownership
+          runAsUser: 0
+          runAsGroup: 0
+          runAsNonRoot: false
+        image: busybox:1.36.1
+        command:
+          - 'sh'
+          - '-c'
+          - |
+            mkdir -p /crdtmerge/projects
+            chown www-data:www-data /crdtmerge/projects
+        volumeMounts:
+          - name: crdtmerge
+            mountPath: /crdtmerge
+
+      volumes:
+      - name: crdtmerge
+        persistentVolumeClaim:
+          claimName: crdtmerge # established in pvc.yaml
diff --git a/deployment/base/pvc.yaml b/deployment/base/pvc.yaml
@@ -33,3 +33,21 @@ spec:
     requests:
       storage: 10Gi
   storageClassName: weekly-snapshots-retain-4 # provided by LTOps
+
+---
+
+# https://kubernetes.io/docs/concepts/storage/persistent-volumes/#persistentvolumeclaims
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: crdtmerge
+  namespace: languagedepot
+  labels:
+    app.kubernetes.io/part-of: languagedepot
+spec:
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: 10Gi
+  storageClassName: weekly-snapshots-retain-4 # provided by LTOps
diff --git a/deployment/base/secrets.yaml b/deployment/base/secrets.yaml
@@ -84,3 +84,13 @@ metadata:
 stringData:
   KOPIA_PASSWORD: ''
   kopia.config: ''
+---
+
+apiVersion: v1
+kind: Secret
+metadata:
+  name: crdtmerge
+  namespace: languagedepot
+stringData:
+  CRDT_MERGE_SEND_RECEIVE_USERNAME: ''
+  CRDT_MERGE_SEND_RECEIVE_PASSWORD: ''
diff --git a/deployment/local-dev/crdtmerge-secrets.yaml b/deployment/local-dev/crdtmerge-secrets.yaml
@@ -0,0 +1,8 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: crdtmerge
+  namespace: languagedepot
+stringData:
+  CRDT_MERGE_SEND_RECEIVE_USERNAME: 'admin'
+  CRDT_MERGE_SEND_RECEIVE_PASSWORD: 'pass'