diff --git a/.github/workflows/deploy-branch.yaml b/.github/workflows/deploy-branch.yaml index f8a199eff..8b1d9ca92 100644 --- a/.github/workflows/deploy-branch.yaml +++ b/.github/workflows/deploy-branch.yaml @@ -41,10 +41,18 @@ jobs: version: ${{ needs.set-version.outputs.version }} label-latest: false + build-crdtmerge: + name: Build CrdtMerge + needs: [ set-version ] + uses: ./.github/workflows/lexbox-crdtmerge.yaml + with: + version: ${{ needs.set-version.outputs.version }} + label-latest: false + deploy: name: Deploy Develop uses: ./.github/workflows/deploy.yaml - needs: [ build-api, build-ui, build-hgweb, set-version ] + needs: [ build-api, build-ui, build-hgweb, build-crdtmerge, set-version ] secrets: inherit with: version: ${{ needs.set-version.outputs.version }} diff --git a/.github/workflows/deploy.yaml b/.github/workflows/deploy.yaml index 0ab79e53e..f31011c4a 100644 --- a/.github/workflows/deploy.yaml +++ b/.github/workflows/deploy.yaml @@ -50,6 +50,7 @@ jobs: url: https://${{ inputs.deploy-domain }} outputs: api-version: ${{ steps.get-api-version.outputs.result }} + crdtmerge-version: ${{ steps.get-crdtmerge-version.outputs.result }} ui-version: ${{ steps.get-ui-version.outputs.result }} steps: - name: Checkout lexbox repo @@ -80,6 +81,11 @@ jobs: id: get-api-version with: cmd: yq '.images.[] | select(.name == "ghcr.io/sillsdev/lexbox-api").newTag' "fleet/${{ inputs.k8s-environment }}/kustomization.yaml" + - name: Get CrdtMerge version + uses: mikefarah/yq@0b34c9a00de1c575a34eea05af1d956a525c4fc1 # v4.34.2 + id: get-crdtmerge-version + with: + cmd: yq '.images.[] | select(.name == "ghcr.io/sillsdev/lexbox-crdtmerge").newTag' "fleet/${{ inputs.k8s-environment }}/kustomization.yaml" - name: Get UI version uses: mikefarah/yq@0b34c9a00de1c575a34eea05af1d956a525c4fc1 # v4.34.2 id: get-ui-version diff --git a/.github/workflows/integration-test-gha.yaml b/.github/workflows/integration-test-gha.yaml index 8161dcb22..c0ddb9636 100644 --- a/.github/workflows/integration-test-gha.yaml +++ b/.github/workflows/integration-test-gha.yaml @@ -39,7 +39,16 @@ jobs: uses: mikefarah/yq@0b34c9a00de1c575a34eea05af1d956a525c4fc1 # v4.34.2 with: cmd: yq eval -i '(.images.[] | select(.name == "ghcr.io/sillsdev/lexbox-api").newTag) = "${{ inputs.lexbox-api-tag }}"' "./deployment/gha/kustomization.yaml" - # It's also possible that hgweb and/or ui image may have changed; if so, pull them and update kustomization.yaml for them as well + # It's also possible that hgweb, crdtmerge, and/or ui image may have changed; if so, pull them and update kustomization.yaml for them as well + - name: Pull crdtmerge if updated + id: crdtmerge_image + continue-on-error: true + run: docker pull ghcr.io/sillsdev/lexbox-crdtmerge:${{ inputs.lexbox-api-tag }} + - name: Update image crdtmerge version + if: ${{ steps.crdtmerge_image.outcome == 'success' }} + uses: mikefarah/yq@0b34c9a00de1c575a34eea05af1d956a525c4fc1 # v4.34.2 + with: + cmd: yq eval -i '(.images.[] | select(.name == "ghcr.io/sillsdev/lexbox-crdtmerge").newTag) = "${{ inputs.lexbox-api-tag }}"' "./deployment/gha/kustomization.yaml" - name: Pull hgweb if updated id: hgweb_image continue-on-error: true diff --git a/.github/workflows/lexbox-crdtmerge.yaml b/.github/workflows/lexbox-crdtmerge.yaml new file mode 100644 index 000000000..a01a2163b --- /dev/null +++ b/.github/workflows/lexbox-crdtmerge.yaml @@ -0,0 +1,98 @@ +name: Build CrdtMerge + +# https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions#on +on: + workflow_call: + inputs: + version: + description: 'The version of the image to build' + required: true + type: string + label-latest: + description: 'The label to apply to the latest image' + type: boolean + default: false + +env: + IMAGE_NAME: ghcr.io/sillsdev/lexbox-crdtmerge + + +jobs: + publish-api: + timeout-minutes: 60 + runs-on: ubuntu-latest + + # postgres db is for automated tests + # services: + # postgres: + # image: postgres:15-alpine + # env: + # POSTGRES_PASSWORD: 972b722e63f549938d07bd8c4ee5086c + # POSTGRES_DB: lexbox-tests + # # Set health checks to wait until postgres has started + # options: >- + # --health-cmd pg_isready + # --health-interval 10s + # --health-timeout 5s + # --health-retries 5 + # ports: + # # Maps tcp port 5432 on service container to the host + # - 5433:5432 + + env: + # https://docs.docker.com/develop/develop-images/build_enhancements/ + DOCKER_BUILDKIT: 1 + + steps: + - uses: actions/checkout@v4 + with: + submodules: true + - uses: actions/setup-dotnet@v4 + with: + dotnet-version: '8.x' + - name: Dotnet build + run: dotnet build backend/CrdtMerge/CrdtMerge.csproj + # TODO: Write CrdtMerge unit tests, probably based on existing sync tests + # - name: Unit tests + # run: dotnet test backend/CrdtMerge/CrdtMerge.csproj --logger:"xunit;LogFileName={assembly}.results.xml" --results-directory ./test-results --filter "Category!=Integration&Category!=FlakyIntegration" --blame-hang-timeout 10m + # - name: Publish unit test results + # uses: EnricoMi/publish-unit-test-result-action@8885e273a4343cd7b48eaa72428dea0c3067ea98 # v2.14.0 + # if: always() + # with: + # check_name: C# Unit Tests + # files: ./test-results/*.xml + # - name: Upload test results + # if: always() + # uses: actions/upload-artifact@v4 + # with: + # name: dotnet-unit-test-results + # path: ./test-results + + - name: Docker meta + id: meta + if: ${{ !env.ACT }} + uses: docker/metadata-action@8e5442c4ef9f78752691e2d8f8d19755c6f78e81 # v5.5.1 + with: + images: ${{ env.IMAGE_NAME }} + tags: | + type=ref,event=branch + type=ref,event=pr + type=raw,enable=${{ inputs.label-latest }},value=latest + type=raw,value=${{ inputs.version }} + + - name: ghcr.io login + uses: docker/login-action@e92390c5fb421da1463c202d546fed0ec5c39f20 # v3.1.0 + if: ${{ !env.ACT }} + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - uses: docker/build-push-action@2cdde995de11925a030ce8070c3d77a52ffcf1c0 # v5.3.0 + with: + context: backend + build-args: | + APP_VERSION=${{ inputs.version }} + push: ${{ !env.ACT && github.repository == 'sillsdev/languageforge-lexbox' }} + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} diff --git a/.github/workflows/release-pipeline.yaml b/.github/workflows/release-pipeline.yaml index 92450db98..9f65e9e3c 100644 --- a/.github/workflows/release-pipeline.yaml +++ b/.github/workflows/release-pipeline.yaml @@ -43,10 +43,18 @@ jobs: version: ${{ needs.set-version.outputs.version }} label-latest: true + build-crdtmerge: + name: Build crdtmerge + needs: [ set-version ] + uses: ./.github/workflows/lexbox-crdtmerge.yaml + with: + version: ${{ needs.set-version.outputs.version }} + label-latest: true + deploy: name: Deploy Staging uses: ./.github/workflows/deploy.yaml - needs: [ build-api, build-ui, build-hgweb, set-version ] + needs: [ build-api, build-ui, build-hgweb, build-crdtmerge, set-version ] secrets: inherit with: version: ${{ needs.set-version.outputs.version }} diff --git a/backend/CrdtMerge/Dockerfile b/backend/CrdtMerge/Dockerfile new file mode 100644 index 000000000..0746776db --- /dev/null +++ b/backend/CrdtMerge/Dockerfile @@ -0,0 +1,25 @@ +# syntax=docker/dockerfile:1 +FROM mcr.microsoft.com/dotnet/aspnet:9.0 AS base +WORKDIR /app +EXPOSE 80 +EXPOSE 443 + +FROM mcr.microsoft.com/dotnet/sdk:9.0 AS build + +COPY . . +RUN --mount=type=cache,target=/root/.nuget/packages dotnet restore "CrdtMerge/CrdtMerge.csproj" + +ARG APP_VERSION +LABEL version=$APP_VERSION + +RUN --mount=type=cache,target=/root/.nuget/packages dotnet build /p:InformationalVersion=$APP_VERSION "CrdtMerge/CrdtMerge.csproj" -c Release -o /app/build + +FROM build AS publish +RUN --mount=type=cache,target=/root/.nuget/packages dotnet publish /p:InformationalVersion=$APP_VERSION "CrdtMerge/CrdtMerge.csproj" -c Release -o /app/publish + +FROM base AS final +RUN mkdir -p /var/lib/crdtmerge && chown -R www-data:www-data /var/lib/crdtmerge +WORKDIR /app +COPY --from=publish /app/publish . +USER www-data:www-data +ENTRYPOINT ["dotnet", "CrdtMerge.dll"] diff --git a/backend/CrdtMerge/appsettings.json b/backend/CrdtMerge/appsettings.json index d766d8006..38010f509 100644 --- a/backend/CrdtMerge/appsettings.json +++ b/backend/CrdtMerge/appsettings.json @@ -1,5 +1,5 @@ { - "CrdtMergeConfig": { + "SendReceiveConfig": { "LexboxUsername": null }, "Logging": { diff --git a/deployment/base/crdtmerge-deployment.yaml b/deployment/base/crdtmerge-deployment.yaml new file mode 100644 index 000000000..9a2e813ad --- /dev/null +++ b/deployment/base/crdtmerge-deployment.yaml @@ -0,0 +1,144 @@ +# https://kubernetes.io/docs/concepts/services-networking/service/#defining-a-service +apiVersion: v1 +kind: Service +metadata: + name: crdtmerge + namespace: languagedepot + labels: + app: crdtmerge +spec: + type: ClusterIP + clusterIP: None + selector: + app: crdtmerge + ports: + - name: http + protocol: TCP + port: 80 + +--- + +# https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#writing-a-deployment-spec +apiVersion: apps/v1 +kind: Deployment +metadata: + name: crdtmerge + namespace: languagedepot + labels: + app: crdtmerge +spec: + selector: + matchLabels: + app: crdtmerge + strategy: + rollingUpdate: + maxSurge: 2 + maxUnavailable: 0 + type: RollingUpdate + template: + # https://kubernetes.io/docs/concepts/workloads/pods/#pod-templates + metadata: + labels: + app: crdtmerge + spec: + securityContext: + runAsGroup: 33 + runAsUser: 33 + runAsNonRoot: true + containers: + - name: crdtmerge + image: ghcr.io/sillsdev/lexbox-crdtmerge:develop + imagePullPolicy: IfNotPresent + # https://kubernetes.io/docs/concepts/configuration/manage-resources-containers + resources: + requests: + memory: 1500Mi + limits: + memory: 2400Mi + startupProbe: + httpGet: + port: 80 + path: /api/healthz + failureThreshold: 30 + periodSeconds: 10 + ports: + - containerPort: 80 + + volumeMounts: + - name: crdtmerge + mountPath: /var/lib/crdtmerge + + env: + - name: DOTNET_URLS + value: http://0.0.0.0:80 + - name: ASPNETCORE_ENVIRONMENT + valueFrom: + configMapKeyRef: + name: app-config + key: environment-name + - name: K8S_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: K8S_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POSTGRES_DB + valueFrom: + secretKeyRef: + key: POSTGRES_DB + name: db + - name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + key: POSTGRES_PASSWORD + name: db + - name: DbConfig__LexBoxConnectionString + value: Host=db;Port=5432;Username=postgres;Password=$(POSTGRES_PASSWORD);Database=$(POSTGRES_DB) + - name: SendReceiveConfig__ProjectStorageRoot + value: /var/lib/crdtmerge/projects + - name: HgConfig__SendReceiveDomain + valueFrom: + configMapKeyRef: + name: app-config + key: hg-domain + - name: SendReceiveConfig__LexboxUrl + value: http://lexbox:5158/ + # - name: SendReceiveConfig__HgUrl + # value: http://lexbox:5158/hg/ + - name: SendReceiveConfig__LexboxUsername + valueFrom: + secretKeyRef: + key: CRDT_MERGE_SEND_RECEIVE_USERNAME + name: crdtmerge + - name: SendReceiveConfig__LexboxPassword + valueFrom: + secretKeyRef: + key: CRDT_MERGE_SEND_RECEIVE_PASSWORD + name: crdtmerge + - name: SendReceiveConfig__FdoDataModelVersion + value: "7000072" + + initContainers: + - name: populate-crdt-project-storage + securityContext: + # Make sure we're authorized to set ownership + runAsUser: 0 + runAsGroup: 0 + runAsNonRoot: false + image: busybox:1.36.1 + command: + - 'sh' + - '-c' + - | + mkdir -p /crdtmerge/projects + chown www-data:www-data /crdtmerge/projects + volumeMounts: + - name: crdtmerge + mountPath: /crdtmerge + + volumes: + - name: crdtmerge + persistentVolumeClaim: + claimName: crdtmerge # established in pvc.yaml diff --git a/deployment/base/pvc.yaml b/deployment/base/pvc.yaml index 8ffbdb558..e2e318625 100644 --- a/deployment/base/pvc.yaml +++ b/deployment/base/pvc.yaml @@ -33,3 +33,21 @@ spec: requests: storage: 10Gi storageClassName: weekly-snapshots-retain-4 # provided by LTOps + +--- + +# https://kubernetes.io/docs/concepts/storage/persistent-volumes/#persistentvolumeclaims +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: crdtmerge + namespace: languagedepot + labels: + app.kubernetes.io/part-of: languagedepot +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi + storageClassName: weekly-snapshots-retain-4 # provided by LTOps diff --git a/deployment/base/secrets.yaml b/deployment/base/secrets.yaml index 94236d32a..28b3817c8 100644 --- a/deployment/base/secrets.yaml +++ b/deployment/base/secrets.yaml @@ -84,3 +84,13 @@ metadata: stringData: KOPIA_PASSWORD: '' kopia.config: '' +--- + +apiVersion: v1 +kind: Secret +metadata: + name: crdtmerge + namespace: languagedepot +stringData: + CRDT_MERGE_SEND_RECEIVE_USERNAME: '' + CRDT_MERGE_SEND_RECEIVE_PASSWORD: '' diff --git a/deployment/local-dev/crdtmerge-secrets.yaml b/deployment/local-dev/crdtmerge-secrets.yaml new file mode 100644 index 000000000..180e85498 --- /dev/null +++ b/deployment/local-dev/crdtmerge-secrets.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: Secret +metadata: + name: crdtmerge + namespace: languagedepot +stringData: + CRDT_MERGE_SEND_RECEIVE_USERNAME: 'admin' + CRDT_MERGE_SEND_RECEIVE_PASSWORD: 'pass' diff --git a/deployment/local-dev/kustomization.yaml b/deployment/local-dev/kustomization.yaml index 02e526433..dae570b77 100644 --- a/deployment/local-dev/kustomization.yaml +++ b/deployment/local-dev/kustomization.yaml @@ -6,6 +6,7 @@ resources: - ../base/ #- secrets.yaml - ingress-deployment.yaml +- crdtmerge-secrets.yaml - db-secrets.yaml - lf-classic-secrets.yaml - self-signed-ssl.yaml diff --git a/skaffold.yaml b/skaffold.yaml index bb40c87d7..fdcea20fe 100644 --- a/skaffold.yaml +++ b/skaffold.yaml @@ -14,6 +14,10 @@ build: manual: - src: '**' dest: /src/backend + - image: ghcr.io/sillsdev/lexbox-crdtmerge + context: backend + docker: + dockerfile: CrdtMerge/Dockerfile - image: ghcr.io/sillsdev/lexbox-ui context: frontend docker: @@ -41,6 +45,12 @@ portForward: resourceName: lexbox namespace: languagedepot port: 5158 + # CrdtMerge + - resourceType: Service + resourceName: crdtmerge + namespace: languagedepot + port: 80 + localPort: 5275 # OpenTelemetry - resourceType: Service resourceName: lexbox @@ -89,6 +99,12 @@ profiles: namespace: languagedepot port: 5432 localPort: 5433 + # CrdtMerge + - resourceType: Service + resourceName: crdtmerge + namespace: languagedepot + port: 80 + localPort: 5275 # OpenTelemetry - resourceType: Service resourceName: lexbox @@ -141,6 +157,12 @@ profiles: resourceName: lexbox namespace: languagedepot port: 5158 + # CrdtMerge + - resourceType: Service + resourceName: crdtmerge + namespace: languagedepot + port: 80 + localPort: 5275 # OpenTelemetry - resourceType: Service resourceName: lexbox