From 0380bcb94b6c530de7c7c9147eb47faaee9d849f Mon Sep 17 00:00:00 2001 From: Zhijie Yang Date: Thu, 12 Sep 2024 11:54:48 +0200 Subject: [PATCH 1/4] feat: make temporal worker as a rock --- .../charms/temporal-worker/Makefile | 104 ++++++++++++++ .../temporal-worker/oci_factory/worker.py | 129 ++++++++++++++++++ .../charms/temporal-worker/pyproject.toml | 3 +- .../charms/temporal-worker/rockcraft.yaml | 75 ++++++++++ .../temporal-worker/scripts/start-worker.sh | 6 + 5 files changed, 315 insertions(+), 2 deletions(-) create mode 100644 tools/workflow-engine/charms/temporal-worker/Makefile create mode 100644 tools/workflow-engine/charms/temporal-worker/oci_factory/worker.py create mode 100644 tools/workflow-engine/charms/temporal-worker/rockcraft.yaml create mode 100644 tools/workflow-engine/charms/temporal-worker/scripts/start-worker.sh diff --git a/tools/workflow-engine/charms/temporal-worker/Makefile b/tools/workflow-engine/charms/temporal-worker/Makefile new file mode 100644 index 00000000..3fa79116 --- /dev/null +++ b/tools/workflow-engine/charms/temporal-worker/Makefile @@ -0,0 +1,104 @@ +# Copyright 2023 Canonical Ltd. +# See LICENSE file for licensing details. + +# Makefile to help automate tasks + +# The name of the python package/project +PY_PACKAGE := resource_sample + +# ROCK build parameters +ROCK_NAME := temporal-worker_1.0_amd64.rock +IMAGE_NAME := temporal-worker-rock:latest + +# build and dist folders +BUILD := build +DIST := dist + +# Paths to venv executables +POETRY := poetry +PY := python3 +PYTEST := pytest +BANDIT := bandit +BLACK := black +FLAKE8 := flake8 +ISORT := isort +MYPY := mypy +PYLINT := pylint +PYDOCSTYLE := pydocstyle + +.PHONY: help +help: ## Print help about available targets + @grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' + +.PHONY: run +run: ## Run the application + $(POETRY) run $(PY) -m $(PY_PACKAGE) + +.PHONY: install +install: + $(POETRY) install --only main --no-root + +# Development tools. +.PHONY: install-dev +install-dev: + $(POETRY) install --with dev --no-root + +.PHONY: lint +lint: ## Run linter + $(POETRY) run $(FLAKE8) $(PY_PACKAGE) tests + $(POETRY) run $(ISORT) --check $(PY_PACKAGE) tests + $(POETRY) run $(BLACK) --check $(PY_PACKAGE) tests + $(POETRY) run $(BANDIT) --configfile pyproject.toml --quiet --recursive $(PY_PACKAGE) tests + +.PHONY: fmt +fmt: ## Reformat code for linter + $(POETRY) run $(ISORT) $(PY_PACKAGE) tests + $(POETRY) run $(BLACK) $(PY_PACKAGE) tests + +.PHONY: test +test: ## Run tests + $(POETRY) run $(PYTEST) --cov=$(PY_PACKAGE) tests + +.PHONY: check +check: clean install-dev lint test ## Runs linter and tests from a clean directory + + +# Release management. + +.PHONY: changelog +changelog: ## Add a new entry to the Changelog and bump the package version + ./scripts/update_changelog.sh + +.PHONY: build +build: ## Create a Python source distribution and a wheel in dist + $(POETRY) build + +.PHONY: build_rock +build_rock: + rockcraft pack + rockcraft.skopeo --insecure-policy copy --dest-tls-verify=false oci-archive:$(ROCK_NAME) docker://localhost:32000/$(IMAGE_NAME) + +.PHONY: publish +publish: ## Publish the package to PYPI + $(POETRY) publish + +# Cleaning up. + +.PHONY: clean-dist +clean-dist: + rm -rf $(BUILD) $(DIST) + +.PHONY: clean-pyc +clean-pyc: + find . -name '*.pyc' -exec rm -f {} + + find . -name '*.pyo' -exec rm -f {} + + find . -name '*~' -exec rm -f {} + + rm -rf .pytest_cache .coverage .mypy_cache + +.PHONY: clean-venv +clean-venv: + $(POETRY) env remove --all + +.PHONY: clean +clean: clean-dist clean-pyc clean-venv ## Clean up the virtualenv, Python bytecode and docs + rm -rf *.egg-info diff --git a/tools/workflow-engine/charms/temporal-worker/oci_factory/worker.py b/tools/workflow-engine/charms/temporal-worker/oci_factory/worker.py new file mode 100644 index 00000000..70998fec --- /dev/null +++ b/tools/workflow-engine/charms/temporal-worker/oci_factory/worker.py @@ -0,0 +1,129 @@ +#!/usr/bin/env python3 +# Copyright 2023 Canonical Ltd. +# See LICENSE file for licensing details. + + +"""Temporal client worker.""" + +import asyncio +import logging +import os + +from activities.activity_consume_events import consume +from temporalio.runtime import PrometheusConfig, Runtime, TelemetryConfig +from temporallib.auth import ( + AuthOptions, + GoogleAuthOptions, + KeyPair, + MacaroonAuthOptions, +) +from temporallib.client import Client, Options +from temporallib.encryption import EncryptionOptions +from temporallib.worker import SentryOptions, Worker, WorkerOptions +from workflows.consume_events_workflow import ConsumeEventsWorkflow + +logger = logging.getLogger(__name__) + + +def _get_auth_header(): + """Get auth options based on provider. + + Returns: + AuthOptions object. + """ + if os.getenv("TWC_AUTH_PROVIDER") == "candid": + return MacaroonAuthOptions( + keys=KeyPair( + private=os.getenv("TWC_CANDID_PRIVATE_KEY"), + public=os.getenv("TWC_CANDID_PUBLIC_KEY"), + ), + macaroon_url=os.getenv("TWC_CANDID_URL"), + username=os.getenv("TWC_CANDID_USERNAME"), + ) + + if os.getenv("TWC_AUTH_PROVIDER") == "google": + return GoogleAuthOptions( + type="service_account", + project_id=os.getenv("TWC_OIDC_PROJECT_ID"), + private_key_id=os.getenv("TWC_OIDC_PRIVATE_KEY_ID"), + private_key=os.getenv("TWC_OIDC_PRIVATE_KEY"), + client_email=os.getenv("TWC_OIDC_CLIENT_EMAIL"), + client_id=os.getenv("TWC_OIDC_CLIENT_ID"), + auth_uri=os.getenv("TWC_OIDC_AUTH_URI"), + token_uri=os.getenv("TWC_OIDC_TOKEN_URI"), + auth_provider_x509_cert_url=os.getenv("TWC_OIDC_AUTH_CERT_URL"), + client_x509_cert_url=os.getenv("TWC_OIDC_CLIENT_CERT_URL"), + ) + + return None + + +def _init_runtime_with_prometheus(port: int) -> Runtime: + """Create runtime for use with Prometheus metrics. + + Args: + port: Port of prometheus. + + Returns: + Runtime for temporalio with prometheus. + """ + return Runtime( + telemetry=TelemetryConfig( + metrics=PrometheusConfig(bind_address=f"0.0.0.0:{port}") + ) + ) + + +async def run_worker(): + """Connect Temporal worker to Temporal server.""" + client_config = Options( + host=os.getenv("TWC_HOST"), + namespace=os.getenv("TWC_NAMESPACE"), + queue=os.getenv("TWC_QUEUE"), + ) + + if os.getenv("TWC_TLS_ROOT_CAS", "").strip() != "": + client_config.tls_root_cas = os.getenv("TWC_TLS_ROOT_CAS") + + if os.getenv("TWC_AUTH_PROVIDER", "").strip() != "": + client_config.auth = AuthOptions( + provider=os.getenv("TWC_AUTH_PROVIDER"), config=_get_auth_header() + ) + + if os.getenv("TWC_ENCRYPTION_KEY", "").strip() != "": + client_config.encryption = EncryptionOptions( + key=os.getenv("TWC_ENCRYPTION_KEY"), compress=True + ) + + worker_opt = None + dsn = os.getenv("TWC_SENTRY_DSN", "").strip() + if dsn != "": + sentry = SentryOptions( + dsn=dsn, + release=os.getenv("TWC_SENTRY_RELEASE", "").strip() or None, + environment=os.getenv("TWC_SENTRY_ENVIRONMENT", "").strip() or None, + redact_params=os.getenv("TWC_SENTRY_REDACT_PARAMS", False), + sample_rate=os.getenv("TWC_SENTRY_SAMPLE_RATE", 1.0), + ) + + worker_opt = WorkerOptions(sentry=sentry) + + runtime = None + if os.getenv("TWC_PROMETHEUS_PORT"): + runtime = _init_runtime_with_prometheus(int(os.getenv("TWC_PROMETHEUS_PORT"))) + + client = await Client.connect(client_config, runtime=runtime) + + worker = Worker( + client=client, + task_queue=os.getenv("TWC_QUEUE"), + workflows=[GreetingWorkflow, VaultWorkflow], + activities=[compose_greeting, vault_test], + worker_opt=worker_opt, + ) + + await worker.run() + + +if __name__ == "__main__": # pragma: nocover + asyncio.run(run_worker()) diff --git a/tools/workflow-engine/charms/temporal-worker/pyproject.toml b/tools/workflow-engine/charms/temporal-worker/pyproject.toml index c6d5d7f3..cdf6af4b 100644 --- a/tools/workflow-engine/charms/temporal-worker/pyproject.toml +++ b/tools/workflow-engine/charms/temporal-worker/pyproject.toml @@ -8,9 +8,8 @@ description = "Temporal workflows for supporting the OCI Factory processes" authors = ["cjdc "] packages = [ { include = "**/*.py", from = "." }, - { include = "**/ca.crt", from = "." } ] -include = ["**/*.avsc"] +include = ["**/*.avsc", "**/*.crt"] [tool.poetry.dependencies] python = "^3.8" diff --git a/tools/workflow-engine/charms/temporal-worker/rockcraft.yaml b/tools/workflow-engine/charms/temporal-worker/rockcraft.yaml new file mode 100644 index 00000000..cc4aba3e --- /dev/null +++ b/tools/workflow-engine/charms/temporal-worker/rockcraft.yaml @@ -0,0 +1,75 @@ +# Copyright 2023 Canonical Ltd. +# See LICENSE file for licensing details. +name: temporal-worker +summary: Temporal worker app +description: OCI image for Temporal worker app +version: "1.0" +base: ubuntu@22.04 +build-base: ubuntu@22.04 +license: Apache-2.0 + +services: + temporal-worker: + override: replace + summary: "temporal worker" + startup: disabled + command: "./app/scripts/start-worker.sh" + environment: + TWC_HOST: localhost:7233 + TWC_NAMESPACE: default + TWC_QUEUE: test-queue + TWC_PROMETHEUS_PORT: "9000" + TWC_TLS_ROOT_CAS: "" + TWC_AUTH_PROVIDER: "" # "google" or "candid" + TWC_ENCRYPTION_KEY: "" + TWC_SENTRY_DSN: "" + TWC_SENTRY_RELEASE: "" + TWC_SENTRY_ENVIRONMENT: "" + TWC_SENTRY_REDACT_PARAMS: "False" + TWC_SENTRY_SAMPLE_RATE: "1.0" + TWC_CANDID_URL: "" + TWC_CANDID_USERNAME: "" + TWC_CANDID_PRIVATE_KEY: "" + TWC_CANDID_PUBLIC_KEY: "" + TWC_OIDC_PROJECT_ID: "" + TWC_OIDC_PRIVATE_KEY_ID: "" + TWC_OIDC_PRIVATE_KEY: "" + TWC_OIDC_CLIENT_EMAIL: "" + TWC_OIDC_CLIENT_ID: "" + TWC_OIDC_AUTH_URI: "" + TWC_OIDC_TOKEN_URI: "" + TWC_OIDC_AUTH_CERT_URL: "" + TWC_OIDC_CLIENT_CERT_URL: "" + + # Vault variables + TWC_VAULT_ADDR: "" + TWC_VAULT_CACERT_BYTES: "" + TWC_VAULT_ROLE_ID: "" + TWC_VAULT_ROLE_SECRET_ID: "" + TWC_VAULT_MOUNT: "" + TWC_VAULT_CERT_PATH: "" + +platforms: + amd64: + +parts: + worker-dependencies: + plugin: python + source: . + build-packages: + - build-essential + # Uncomment if using a 'requirements.txt' file + # python-requirements: + # - requirements.txt + stage-packages: + - python3.10-venv + - coreutils + - bash + + worker-app: + plugin: dump + source: . + organize: + "*": app/ + stage: + - app diff --git a/tools/workflow-engine/charms/temporal-worker/scripts/start-worker.sh b/tools/workflow-engine/charms/temporal-worker/scripts/start-worker.sh new file mode 100644 index 00000000..ac763ff9 --- /dev/null +++ b/tools/workflow-engine/charms/temporal-worker/scripts/start-worker.sh @@ -0,0 +1,6 @@ +#!/bin/bash +# Copyright 2023 Canonical Ltd. +# See LICENSE file for licensing details. + +# update 'resource_sample' accordingly +python3 app/oci_factory/worker.py From 766c070a9bff5a54a1a1322f2f102f28e477f5d5 Mon Sep 17 00:00:00 2001 From: Zhijie Yang Date: Fri, 20 Sep 2024 12:16:38 +0200 Subject: [PATCH 2/4] feat: add environment template --- .../temporal-worker/config.yml.template | 11 -------- .../temporal-worker/environment.yml.template | 27 +++++++++++++++++++ .../temporal-worker/oci_factory/.env.template | 12 --------- .../temporal-worker/oci_factory/worker.py | 10 +++---- .../charms/temporal-worker/rockcraft.yaml | 1 + .../temporal-worker/scripts/start-worker.sh | 0 6 files changed, 31 insertions(+), 30 deletions(-) create mode 100644 tools/workflow-engine/charms/temporal-worker/environment.yml.template delete mode 100644 tools/workflow-engine/charms/temporal-worker/oci_factory/.env.template mode change 100644 => 100755 tools/workflow-engine/charms/temporal-worker/scripts/start-worker.sh diff --git a/tools/workflow-engine/charms/temporal-worker/config.yml.template b/tools/workflow-engine/charms/temporal-worker/config.yml.template index db603906..db6e10d9 100644 --- a/tools/workflow-engine/charms/temporal-worker/config.yml.template +++ b/tools/workflow-engine/charms/temporal-worker/config.yml.template @@ -46,14 +46,3 @@ temporal-worker-k8s: mRGunUHBcnWEvgJBQl9nJEiU0Zsnvgc/ubhPgXRR4Xq37Z0j4r7g1SgEEzwxA57d emyPxgcYxn/eR44/KJ4EBs+lVDR3veyJm+kXQ99b21/+jh5Xos1AnX5iItreGCc= -----END CERTIFICATE----- - # Only needed if deploying under a proxy - # http-proxy: "http://squid.internal:3128" - # https-proxy: "http://squid.internal:3128" - # no-proxy: ".canonical.com" - - # Charmed workflows - # The "workflows-file-name" must match the wheel file created with poetry - workflows-file-name: "oci_factory_workflows-0.0.1-py3-none-any.whl" - # To support all defined workflows and activities, use the 'all' keyword - supported-workflows: "all" - supported-activities: "all" diff --git a/tools/workflow-engine/charms/temporal-worker/environment.yml.template b/tools/workflow-engine/charms/temporal-worker/environment.yml.template new file mode 100644 index 00000000..d800a334 --- /dev/null +++ b/tools/workflow-engine/charms/temporal-worker/environment.yml.template @@ -0,0 +1,27 @@ +env: + - name: ROCKS_EVENTBUS_USERNAME + value: + - name: ROCKS_EVENTBUS_PASSWORD + value: + - name: ROCKS_EVENTBUS_KAFKA_ADDRESS + value: + - name: ROCKS_EVENTBUS_KARAPACE_URL + value: + - name: OS_AUTH_URL + value: + - name: OS_USERNAME + value: + - name: OS_PASSWORD + value: + - name: OS_PROJECT_NAME + value: + - name: OS_STORAGE_URL + value: + - name: GITHUB_TOKEN + value: + - name: MATTERMOST_TOKEN + value: + - name: MATTERMOST_SERVER + value: https://chat.canonical.com + - name: MATTERMOST_CHANNEL_ID + value: diff --git a/tools/workflow-engine/charms/temporal-worker/oci_factory/.env.template b/tools/workflow-engine/charms/temporal-worker/oci_factory/.env.template deleted file mode 100644 index 7294370b..00000000 --- a/tools/workflow-engine/charms/temporal-worker/oci_factory/.env.template +++ /dev/null @@ -1,12 +0,0 @@ -ROCKS_EVENTBUS_USERNAME= -ROCKS_EVENTBUS_PASSWORD= -ROCKS_EVENTBUS_KAFKA_ADDRESS="kafka.staging.cs.canonical.com:9094" -ROCKS_EVENTBUS_KARAPACE_URL="https://karapace.staging.cs.canonical.com" -OS_AUTH_URL= -OS_USERNAME= -OS_PASSWORD= -OS_PROJECT_NAME= -OS_STORAGE_URL= -MATTERMOST_SERVER= -MATTERMOST_TOKEN= -MATTERMOST_CHANNEL_ID= diff --git a/tools/workflow-engine/charms/temporal-worker/oci_factory/worker.py b/tools/workflow-engine/charms/temporal-worker/oci_factory/worker.py index 70998fec..4bfba5f2 100644 --- a/tools/workflow-engine/charms/temporal-worker/oci_factory/worker.py +++ b/tools/workflow-engine/charms/temporal-worker/oci_factory/worker.py @@ -108,17 +108,13 @@ async def run_worker(): worker_opt = WorkerOptions(sentry=sentry) - runtime = None - if os.getenv("TWC_PROMETHEUS_PORT"): - runtime = _init_runtime_with_prometheus(int(os.getenv("TWC_PROMETHEUS_PORT"))) - - client = await Client.connect(client_config, runtime=runtime) + client = await Client.connect(client_config) worker = Worker( client=client, task_queue=os.getenv("TWC_QUEUE"), - workflows=[GreetingWorkflow, VaultWorkflow], - activities=[compose_greeting, vault_test], + workflows=[ConsumeEventsWorkflow], + activities=[consume], worker_opt=worker_opt, ) diff --git a/tools/workflow-engine/charms/temporal-worker/rockcraft.yaml b/tools/workflow-engine/charms/temporal-worker/rockcraft.yaml index cc4aba3e..bb14d7ed 100644 --- a/tools/workflow-engine/charms/temporal-worker/rockcraft.yaml +++ b/tools/workflow-engine/charms/temporal-worker/rockcraft.yaml @@ -15,6 +15,7 @@ services: startup: disabled command: "./app/scripts/start-worker.sh" environment: + TWC_LOG_LEVEL: "DEBUG" TWC_HOST: localhost:7233 TWC_NAMESPACE: default TWC_QUEUE: test-queue diff --git a/tools/workflow-engine/charms/temporal-worker/scripts/start-worker.sh b/tools/workflow-engine/charms/temporal-worker/scripts/start-worker.sh old mode 100644 new mode 100755 From 9b24fcca73c36a38fc6ddfda39118d589c0ef049 Mon Sep 17 00:00:00 2001 From: Zhijie Yang Date: Fri, 20 Sep 2024 12:42:32 +0200 Subject: [PATCH 3/4] chore: add env.yml to gitignore --- .gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 90072ac8..cdded7c2 100644 --- a/.gitignore +++ b/.gitignore @@ -1,10 +1,11 @@ __pycache__ .env tools/workflow-engine/charms/temporal-worker/config.yml +tools/workflow-engine/charms/temporal-worker/environment.yml tools/workflow-engine/charms/temporal-worker/dist *.rock .terraform *terraform.tfstate* *.tfvars* .vscode -venv \ No newline at end of file +venv From 0fbbb2d5ca10a393294450d7f65e632c3c7f4fbd Mon Sep 17 00:00:00 2001 From: clay-lake Date: Wed, 25 Sep 2024 14:13:21 +0000 Subject: [PATCH 4/4] ci: automatically update oci/mock-rock/_releases.json, from https://github.com/canonical/oci-factory/actions/runs/11034664245 --- oci/mock-rock/_releases.json | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/oci/mock-rock/_releases.json b/oci/mock-rock/_releases.json index 7ef217c1..2a6668fe 100644 --- a/oci/mock-rock/_releases.json +++ b/oci/mock-rock/_releases.json @@ -13,13 +13,13 @@ }, "1.0-22.04": { "candidate": { - "target": "565" + "target": "568" }, "beta": { - "target": "565" + "target": "568" }, "edge": { - "target": "565" + "target": "568" }, "end-of-life": "2025-05-01T00:00:00Z" }, @@ -35,31 +35,31 @@ "1.1-22.04": { "end-of-life": "2025-05-01T00:00:00Z", "candidate": { - "target": "566" + "target": "569" }, "beta": { - "target": "566" + "target": "569" }, "edge": { - "target": "566" + "target": "569" } }, "1-22.04": { "end-of-life": "2025-05-01T00:00:00Z", "candidate": { - "target": "566" + "target": "569" }, "beta": { - "target": "566" + "target": "569" }, "edge": { - "target": "566" + "target": "569" } }, "1.2-22.04": { "end-of-life": "2025-05-01T00:00:00Z", "beta": { - "target": "567" + "target": "570" }, "edge": { "target": "1.2-22.04_beta"