Redesign Build Deployment Process (External)

.github/fetch_runID.py

@@ -0,0 +1,42 @@
+
+import json
+import logging
+import requests
+import sys
+
+if __name__ == '__main__':
+    logging.basicConfig(level=logging.DEBUG)
+
+    '''
+        Workflow "docker image" uses image_build_push.yml
+        From above commented out code, and checked via terminal as well,
+        workflow id for the "docker image" can be fetched using:
+        https://api.github.com/repos/MukuFlash03/e-mission-server/actions/workflows
+        https://api.github.com/repos/e-mission/e-mission-server/actions/workflows
+
+        For MukuFlash03: id = 75506902 
+        For e-mission-server: id = 35580278 
+    '''
+
+    download_url = "https://api.github.com/repos/e-mission/e-mission-server/actions/workflows/35580278/runs"
+    logging.debug("About to fetch workflow runs present in docker image workflow present in e-mission-server from %s" % download_url)
+    r = requests.get(download_url)
+    if r.status_code != 200:
+        logging.debug(f"Unable to fetch workflow runs, status code: {r.status_code}")
+        sys.exit(1)
+    else:
+        workflow_runs_json = json.loads(r.text)
+        logging.debug(f"Successfully fetched workflow runs")
+
+        workflow_runs = workflow_runs_json["workflow_runs"]
+        if workflow_runs:
+            successful_runs = [run for run in workflow_runs \
+                                if run["status"] == "completed" and \
+                                run["conclusion"] == "success" and \
+                                run["head_branch"] == "main"
+                               ]
+            if successful_runs:
+                sorted_runs = successful_runs.sort(reverse=True, key=lambda x: x["updated_at"])
+                sorted_runs = sorted(successful_runs, reverse=True, key=lambda x: x["updated_at"])
+                latest_run_id = sorted_runs[0]["id"]
+                print(f"::set-output name=run_id::{latest_run_id}")

.github/workflows/image_build_push.yml

@@ -0,0 +1,153 @@
+name: docker-image-push-public-dash
+
+on:
+  push:
+    branches: [ main ]
+
+  workflow_dispatch:
+    inputs:
+      docker_image_tag:
+        description: "Latest Docker image tags passed from e-mission-server repository on image build and push"
+        required: true
+
+env:
+  DOCKER_USER: ${{secrets.DOCKER_USER}}
+  DOCKER_PASSWORD: ${{secrets.DOCKER_PASSWORD}}
+
+jobs:
+  fetch_run_id:
+    runs-on: ubuntu-latest
+
+    outputs:
+      run_id: ${{ steps.get_run_id.outputs.run_id }}
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with: 
+          python-version: '3.x'
+
+      - name: Install Python dependencies
+        run: |
+         pip install requests
+
+      - name: Run Python script
+        id: run_script
+        run: |
+          echo "Fetching latest successful run ID from e-misison-server docker image workflow"
+          python .github/fetch_runID.py
+
+      - name: Get Run ID
+        id: get_run_id
+        run: echo "run_id=${{ steps.run_script.outputs.run_id }}" >> "$GITHUB_OUTPUT"
+
+  fetch_tag:
+      needs: fetch_run_id
+      runs-on: ubuntu-latest
+
+      env:
+        RUN_ID: ${{ needs.fetch_run_id.outputs.run_id }}
+
+      outputs:
+        docker_image_tag: ${{ steps.get_docker_tag.outputs.docker_image_tag }}
+
+      steps:
+      - uses: actions/checkout@v4
+
+      - name: Use Run ID from previous fetch_run_id job
+        run: echo Run ID from previous job ${{ env.RUN_ID }}
+
+      - name: Download artifact
+        uses: actions/download-artifact@v4
+        with:
+          # TODO: Create a token with basic repo permissions
+          name: docker-image-tag
+          github-token: ${{ secrets.GH_PAT_TAG }}
+          repository: e-mission/e-mission-server
+          run-id: ${{ env.RUN_ID }}
+
+      - name: Print artifact tag
+        id: get_docker_tag
+        run: |
+          cat tag_file.txt
+          docker_image_tag=$(cat tag_file.txt)
+          echo $docker_image_tag
+          echo "docker_image_tag=$(echo $docker_image_tag)" >> $GITHUB_OUTPUT
+  build:
+    needs: fetch_tag
+
+    runs-on: ubuntu-latest
+
+    env:
+      DOCKER_IMAGE_TAG_1: ${{ needs.fetch_tag.outputs.docker_image_tag }}
+      DOCKER_IMAGE_TAG_2: ${{ github.event.inputs.docker_image_tag }}
+
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Print input docker image tag
+      run: |
+        echo "Event name: ${{ github.event_name }}"
+        echo "Latest docker image tag (push): ${{ env.DOCKER_IMAGE_TAG_1 }}"
+        echo "Latest docker image tag (workflow_dispatch): ${{ env.DOCKER_IMAGE_TAG_2 }}"
+
+    - name: Update .env file 
+      run: |
+        if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then
+          echo "Workflow_dispatch: New server image built and pushed, Updating image tag in .env"
+          echo "DOCKER_IMAGE_TAG=$DOCKER_IMAGE_TAG_2" > .env
+        else
+          echo "Push event: Restoring latest server image tag in .env"
+          echo "DOCKER_IMAGE_TAG=$DOCKER_IMAGE_TAG_1" > .env
+        fi
+
+    - name: Add, Commit, Push changes to .env file
+      run: |
+        git config --local user.email "[email protected]"
+        git config --local user.name "GitHub Action"
+        if git diff --quiet; then
+          echo "Latest timestamp already present in .env file, no changes to commit"
+        else
+          git add .env
+          git commit -m "Updated docker image tag in .env file to the latest timestamp"
+          git push origin 
+        fi
+
+    - name: docker login
+      run: | # log into docker hub account
+        docker login -u $DOCKER_USER -p $DOCKER_PASSWORD
+
+    - name: Get current date # get the date of the build
+      id: date
+      run: echo "::set-output name=date::$(date +'%Y-%m-%d--%M-%S')"
+
+    - name: Run a one-line script
+      run: echo running in repo ${GITHUB_REPOSITORY#*/} branch ${GITHUB_REF##*/} on ${{ steps.date.outputs.date }}
+
+    - name: build docker image
+      run: |
+        if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then
+          SERVER_IMAGE_TAG=$DOCKER_IMAGE_TAG_2 FRONTEND_TAG=$DOCKER_USER/${GITHUB_REPOSITORY#*/}:${GITHUB_REF##*/}_${{ steps.date.outputs.date }} VIZ_TAG=$DOCKER_USER/${GITHUB_REPOSITORY#*/}_notebook:${GITHUB_REF##*/}_${{ steps.date.outputs.date }} docker compose -f docker-compose.yml build
+        else
+          SERVER_IMAGE_TAG=$DOCKER_IMAGE_TAG_1 FRONTEND_TAG=$DOCKER_USER/${GITHUB_REPOSITORY#*/}:${GITHUB_REF##*/}_${{ steps.date.outputs.date }} VIZ_TAG=$DOCKER_USER/${GITHUB_REPOSITORY#*/}_notebook:${GITHUB_REF##*/}_${{ steps.date.outputs.date }} docker compose -f docker-compose.yml build
+        fi
+        docker images
+
+    - name: push docker image
+      run: |
+        docker push $DOCKER_USER/${GITHUB_REPOSITORY#*/}:${GITHUB_REF##*/}_${{ steps.date.outputs.date }}
+        docker push $DOCKER_USER/${GITHUB_REPOSITORY#*/}_notebook:${GITHUB_REF##*/}_${{ steps.date.outputs.date }}
+
+    - name: Create a text file
+      run: |
+          echo ${{ steps.date.outputs.date }} > public_dash_tag_file.txt
+          echo "Created tag text file"
+
+    - name: Upload Artifact
+      uses: actions/upload-artifact@v4
+      with:
+          name: public-dash-image-tag
+          path: public_dash_tag_file.txt
+          overwrite: true

.gitignore

@@ -108,7 +108,6 @@ celerybeat.pid
 *.sage.py
 
 # Environments
-.env
 .venv
 env/
 venv/

docker-compose.dev.yml

@@ -18,8 +18,10 @@ services:
   notebook-server:
     image: em-pub-dash-dev/viz-scripts
     build:
-        context: viz_scripts
-        dockerfile: docker/Dockerfile.dev
+      context: viz_scripts
+      dockerfile: docker/Dockerfile.dev
+      args:
+        SERVER_IMAGE_TAG: ${SERVER_IMAGE_TAG}
     depends_on:
       - db
     environment:

docker-compose.yml

@@ -1,7 +1,7 @@
 version: "3"
 services:
   dashboard:
-    image: em-pub-dash-prod/frontend
+    image: ${FRONTEND_TAG}
     build: frontend
     depends_on:
       - db
@@ -13,8 +13,12 @@ services:
     networks:
        - emission
   notebook-server:
-    image: em-pub-dash-prod/viz-scripts
-    build: viz_scripts
+    image: ${VIZ_TAG}
+    build: 
+      context: viz_scripts
+      dockerfile: Dockerfile
+      args:
+        SERVER_IMAGE_TAG: ${SERVER_IMAGE_TAG}
     depends_on:
       - db
     environment:

viz_scripts/Dockerfile

@@ -1,5 +1,10 @@
 # python 3
-FROM shankari/e-mission-server:master_2024-05-06--36-33
+
+ARG SERVER_IMAGE_TAG
+FROM shankari/e-mission-server:master_${SERVER_IMAGE_TAG}
+
+ADD https://s3.amazonaws.com/rds-downloads/rds-combined-ca-bundle.pem /etc/ssl/certs/
+
 
 VOLUME /plots
 

viz_scripts/docker/Dockerfile.dev

@@ -1,5 +1,7 @@
 # python 3
-FROM shankari/e-mission-server:master_2024-05-06--36-33
+
+ARG SERVER_IMAGE_TAG
+FROM shankari/e-mission-server:master_${SERVER_IMAGE_TAG}
 
 VOLUME /plots
 

viz_scripts/docker/start_notebook.sh

@@ -5,9 +5,8 @@
 echo "DB host = "${DB_HOST}
 if [ -z ${DB_HOST} ] ; then
     local_host=`hostname -i`
-    sed "s-localhost-${local_host}_" conf/storage/db.conf.sample > conf/storage/db.conf
-else
-    sed "s-localhost-${DB_HOST}-" conf/storage/db.conf.sample > conf/storage/db.conf
+    export DB_HOST=$local_host
+    echo "Setting db host environment variable to localhost"
 fi
 
 ### configure the saved-notebooks directory for persistent notebooks