Merge pull request #2 from unicef/feature/update-charts-improved

Feature/update charts improved

README.md

@@ -1,2 +1,7 @@
-# magasin
+# _magasin_
 Open Source end-to-end data ML/AI platform
+
+
+# License
+
+Apache License Version 2.0. 

dev-scripts/update_helm_charts.sh

@@ -17,63 +17,92 @@
 #     helm/README.md
 #
 #
-
-DAGSTER_TAG="1.5.6"
-DASKHUB_TAG="main" # Tags not up to date...
-SUPERSET_TAG="superset-helm-chart-0.10.14"
-
 # Stop the script if there is any error
 set -e
 
 # Displays the commands that are executed. Useful for debug
 set -x
 
-# Create temporary folders
-temp_clone=../.base_repos
+# Define a temporary folder to keep all charts
 temp_charts=$(mktemp -d)
 
-# Location where the new files are copied:
-script_dir=$(dirname "$(realpath "$0")")
-default_output="$script_dir"/../helm/
-
-#TODO add command line option to select output directory
-output_dir=$default_output
-
-# Clones or updates a local repo.
-# If the repo exists in the target_folder(3), it updates it with the tag_name (1)
-# If the repo does not exist, it clones the repo from the repository_url(2)
-clone_or_update_repo() {
-    local tag_name=$1
-    local repository_url=$2
-    local target_folder=$3
-
-    if [ -d "$target_folder/.git" ]; then
-        echo "Updating existing repository $repository_url in $target_folder to tag $tag_name"
-        (cd "$target_folder" && git fetch && git checkout "$tag_name")
-    else
-        echo "Cloning repository $repository_url to $target_folder with tag $tag_name"
-        git clone --branch "$tag_name" --single-branch "$repository_url" "$target_folder"
-    fi
+#
+# Pulls from a repo a chart of a particular version within a specific folder
+#
+helm_pull() {
+    # URL of the helm repo to pull the chart
+    local repo_url=$1
+    # How we're calling the repo locally
+    local repo_name=$2
+    # Name of the cahrt to download
+    local chart_name=$3
+    # Version of the chart to download
+    local chart_version=$4
+    # Folder in which the repo will be downloaded
+    local output_dir=$5
+
+    echo "Adding helm repo $repo_name ($repor_url)..."
+    helm repo add $repo_name $repo_url
+    echo Updating helm repo $repo_name...
+    helm repo update $repo_name
+    echo "Pulling chart $repo_name/$chart_name:$chart_version..."
+    helm pull  $repo_name/$chart_name --version $chart_version --untar --untardir $output_dir
 }
 
-                    # Branch       # Repo url                             # Dest folder to update/clone
-clone_or_update_repo $DAGSTER_TAG  https://github.com/dagster-io/dagster/ $temp_clone/dagster
-clone_or_update_repo $DASKHUB_TAG  https://github.com/dask/helm-chart     $temp_clone/daskhub
-clone_or_update_repo $SUPERSET_TAG https://github.com/apache/superset     $temp_clone/superset
+#
+# Dagster
+#
+# To get the latest version number use the tag in the github repo
+# https://github.com/dagster-io/dagster/releases Example: 1.5.9
+helm_pull https://dagster-io.github.io/helm \
+        dagster \
+        dagster \
+        1.5.9   \
+        $temp_charts          
+
+#
+# Daskhub
+#
+# To get the version number use the release tag in the repo
+# https://github.com/dask/helm-chart/tags. Example 2023.1.0
+helm_pull https://helm.dask.org/ \
+        dask \
+        daskhub \
+        2023.01.0 \
+        $temp_charts          
 
-#git clone https://github.com/dask/helm-chart $temp_clone/daskhub
-#git clone https://github.com/apache/superset $temp_clone/superset
-#git clone --branch=DAGSTER_TAG https://github.com/dagster-io/dagster/ $temp_clone/dagster
+#
+# Apache Drill
+#
+# To get the version Use the latest release tag in the repo
+# https://github.com/magasin-drill/tags
+helm_pull https://unicef.github.io/magasin-drill \
+        drill \
+        drill \
+        0.6.1   \
+        $temp_charts          
 
 
+#
+# Superset 
+#
+# Search for superset-helm-chart version 
+# https://github.com/apache/superset/tags
+helm_pull https://apache.github.io/superset \
+        superset \
+        superset \
+        0.10.15   \
+        $temp_charts          
 
-cp -r $temp_clone/daskhub/daskhub $temp_charts/
-cp -r $temp_clone/superset/helm/superset $temp_charts/
-cp -rp $temp_clone/dagster/helm/dagster $temp_charts/dagster
+# Location where the new files are copied:
+# TODO add command line option to select output directory
+script_dir=$(dirname "$(realpath "$0")")
+default_output_dir="$script_dir"/../helm/
+output_dir=$default_output_dir
 
 rm -rf $output_dir
 cp -rp $temp_charts/ $output_dir
 
 
 # Clean temporary folders
-rm -rf $temp_charts
+#rm -rf $temp_charts

helm/dagster/Chart.lock

@@ -1,7 +1,7 @@
 dependencies:
 - name: dagster-user-deployments
   repository: ""
-  version: 0.0.1-dev
+  version: 1.5.9
 - name: postgresql
   repository: https://raw.githubusercontent.com/bitnami/charts/eb5f9a9513d987b519f0ecd732e7031241c50328/bitnami
   version: 8.1.0
@@ -11,5 +11,5 @@ dependencies:
 - name: redis
   repository: https://raw.githubusercontent.com/bitnami/charts/eb5f9a9513d987b519f0ecd732e7031241c50328/bitnami
   version: 12.7.4
-digest: sha256:ce5154e3bf81d92443f6ccac79c750df666a98ba266ffa9af2ae5be5b40a99ff
-generated: "2022-06-02T09:37:56.87999-07:00"
+digest: sha256:22d6a38819704be5d0983ff730e29b4ab94f102daf8c9f49ecc3b45749d059fe
+generated: "2023-11-16T22:53:05.499706409Z"

helm/dagster/Chart.yaml

@@ -1,37 +1,37 @@
----
 apiVersion: v2
-name: dagster
-version: 0.0.1-dev
-kubeVersion: ">= 1.18.0-0"
+appVersion: 1.5.9
+dependencies:
+- condition: dagster-user-deployments.enableSubchart
+  name: dagster-user-deployments
+  repository: ""
+  version: 1.5.9
+- condition: postgresql.enabled
+  name: postgresql
+  repository: https://raw.githubusercontent.com/bitnami/charts/eb5f9a9513d987b519f0ecd732e7031241c50328/bitnami
+  version: 8.1.0
+- condition: rabbitmq.enabled
+  name: rabbitmq
+  repository: https://raw.githubusercontent.com/bitnami/charts/eb5f9a9513d987b519f0ecd732e7031241c50328/bitnami
+  version: 6.16.3
+- condition: redis.internal
+  name: redis
+  repository: https://raw.githubusercontent.com/bitnami/charts/eb5f9a9513d987b519f0ecd732e7031241c50328/bitnami
+  version: 12.7.4
 description: The data orchestration platform built for productivity.
-type: application
+icon: https://dagster.io/images/brand/logos/dagster-primary-mark.png
 keywords:
-  - analytics
-  - data-orchestrator
-  - data-pipelines
-  - etl
-  - workflow
-sources:
-  - https://github.com/dagster-io/dagster/tree/master/helm/dagster
-dependencies:
-  - name: dagster-user-deployments
-    version: 0.0.1-dev
-    condition: dagster-user-deployments.enableSubchart
-  - name: postgresql
-    version: 8.1.0
-    repository: https://raw.githubusercontent.com/bitnami/charts/eb5f9a9513d987b519f0ecd732e7031241c50328/bitnami
-    condition: postgresql.enabled
-  - name: rabbitmq
-    version: 6.16.3
-    repository: https://raw.githubusercontent.com/bitnami/charts/eb5f9a9513d987b519f0ecd732e7031241c50328/bitnami
-    condition: rabbitmq.enabled
-  - name: redis
-    version: 12.7.4
-    repository: https://raw.githubusercontent.com/bitnami/charts/eb5f9a9513d987b519f0ecd732e7031241c50328/bitnami
-    condition: redis.internal
+- analytics
+- data-orchestrator
+- data-pipelines
+- etl
+- workflow
+kubeVersion: '>= 1.18.0-0'
 maintainers:
-  - name: Dagster Labs
-    email: [email protected]
-    url: https://dagster.io/about
-icon: https://dagster.io/images/brand/logos/dagster-primary-mark.png
-appVersion: dev
+- email: [email protected]
+  name: Dagster Labs
+  url: https://dagster.io/about
+name: dagster
+sources:
+- https://github.com/dagster-io/dagster/tree/master/helm/dagster
+type: application
+version: 1.5.9

helm/dagster/charts/dagster-user-deployments/Chart.yaml

@@ -1,21 +1,20 @@
----
 apiVersion: v2
-name: dagster-user-deployments
-version: 0.0.1-dev
-kubeVersion: ">= 1.18.0-0"
+appVersion: 1.5.9
 description: A chart to deploy code servers for your Dagster deployment.
-type: application
+icon: https://dagster.io/images/brand/logos/dagster-primary-mark.png
 keywords:
-  - analytics
-  - data-orchestrator
-  - data-pipelines
-  - etl
-  - workflow
-sources:
-  - https://github.com/dagster-io/dagster/tree/master/helm/dagster/charts/dagster-user-deployments
+- analytics
+- data-orchestrator
+- data-pipelines
+- etl
+- workflow
+kubeVersion: '>= 1.18.0-0'
 maintainers:
-  - name: Dagster Labs
-    email: [email protected]
-    url: https://dagster.io/about
-icon: https://dagster.io/images/brand/logos/dagster-primary-mark.png
-appVersion: dev
+- email: [email protected]
+  name: Dagster Labs
+  url: https://dagster.io/about
+name: dagster-user-deployments
+sources:
+- https://github.com/dagster-io/dagster/tree/master/helm/dagster/charts/dagster-user-deployments
+type: application
+version: 1.5.9

helm/dagster/charts/postgresql/.helmignore

@@ -0,0 +1,2 @@
+.git
+OWNERS

helm/dagster/charts/postgresql/Chart.yaml

@@ -0,0 +1,22 @@
+apiVersion: v1
+appVersion: 11.6.0
+description: Chart for PostgreSQL, an object-relational database management system
+  (ORDBMS) with an emphasis on extensibility and on standards-compliance.
+home: https://www.postgresql.org/
+icon: https://bitnami.com/assets/stacks/postgresql/img/postgresql-stack-110x117.png
+keywords:
+- postgresql
+- postgres
+- database
+- sql
+- replication
+- cluster
+maintainers:
+- email: [email protected]
+  name: Bitnami
+- email: [email protected]
+  name: desaintmartin
+name: postgresql
+sources:
+- https://github.com/bitnami/bitnami-docker-postgresql
+version: 8.1.0