From 24caa920d1d9b42d9875909adb0c4c3d05d908de Mon Sep 17 00:00:00 2001 From: shubhusion Date: Fri, 28 Jun 2024 13:51:01 +0530 Subject: [PATCH 01/33] Created Procedure for Setting Up a Scaler Signed-off-by: Shubham Sharma Signed-off-by: shubhusion --- content/docs/2.14/operate/setupscaler.md | 116 +++++++++++++++++++++++ content/docs/2.15/operate/setupscaler.md | 116 +++++++++++++++++++++++ 2 files changed, 232 insertions(+) create mode 100644 content/docs/2.14/operate/setupscaler.md create mode 100644 content/docs/2.15/operate/setupscaler.md diff --git a/content/docs/2.14/operate/setupscaler.md b/content/docs/2.14/operate/setupscaler.md new file mode 100644 index 000000000..7279354e3 --- /dev/null +++ b/content/docs/2.14/operate/setupscaler.md @@ -0,0 +1,116 @@ ++++ +title = "Setupscaler" +weight = 500 +description = "Procedure to Setup a Scaler in KEDA" ++++ + +## Prerequisites + +1. **Kubernetes Cluster**: + - Ensure you have a running Kubernetes cluster set up and accessible. + - If you don't have a cluster yet, follow the [official Kubernetes documentation](https://kubernetes.io/docs/setup/) to create a new cluster suitable for your environment (local machine, cloud provider, etc.). + +2. **KEDA Installation**: + - KEDA needs to be installed on your Kubernetes cluster before you can use it. + - Follow the [KEDA installation guide](https://keda.sh/docs/2.14/deploy/) carefully, including any prerequisites specific to your Kubernetes setup. + - The installation guide provides instructions for different installation methods (e.g., YAML, Helm charts, etc.). Choose the method that suits your needs. + +3. **kubectl**: + - The `kubectl` command-line tool is required to interact with your Kubernetes cluster. + - Follow the [official kubectl installation guide](https://kubernetes.io/docs/tasks/tools/#kubectl) to install `kubectl` on your operating system. + - Once installed, configure `kubectl` to communicate with your Kubernetes cluster by following the cluster-specific instructions provided by your Kubernetes setup. + +## Step 1: Identify the Scaler You Need + +KEDA supports various scalers that correspond to different event sources or triggers. Determining the right scaler is crucial for scaling your application based on the desired event source. + +1. Visit the [KEDA Scalers documentation](https://keda.sh/docs/2.14/scalers/) and browse through the list of available scalers. +2. Identify the scaler that matches the event source you want to use for scaling your application. For example: + - If you want to scale based on incoming HTTP traffic, you would need the **HTTP scaler**. + - If you want to scale based on messages in a RabbitMQ queue, you would need the **RabbitMQ scaler**. + - If you want to scale based on a cron schedule, you would need the **Cron scaler**. +3. Open the documentation page for your chosen scaler and familiarize yourself with its specific requirements and configuration options. + +## Step 2: Install the Required Scaler (if needed) + +Some scalers are part of the core KEDA installation, while others need to be installed separately as add-ons. + +1. Refer to the documentation of your chosen scaler to check if it needs to be installed separately. +2. If the scaler needs to be installed separately, follow the installation instructions provided in the scaler's documentation carefully. + - The installation process typically involves running a command (e.g., `helm install` for Helm charts) or applying YAML manifests using `kubectl`. +3. Verify that the scaler has been installed successfully by checking the output of the installation process or by running any provided verification commands. + +## Step 3: Create a ScaledObject Configuration File + +KEDA uses a custom resource called `ScaledObject` to define how your application should be scaled based on the chosen event source or trigger. + +1. Create a new file (e.g., `scaledobject.yaml`) in a text editor or using the command line. +2. Define the `ScaledObject` configuration in this file, following the structure and examples provided in the documentation of your chosen scaler. +3. Typically, the `ScaledObject` configuration includes the following sections: + - `metadata`: Specifies the name and namespace for the `ScaledObject`. + - `spec.scaleTargetRef`: Identifies the Kubernetes deployment or other resource that should be scaled. + - `spec.pollingInterval` (optional): Specifies how often KEDA should check for scaling events (defaults to 15 seconds). + - `spec.cooldownPeriod` (optional): Specifies the cool-down period in seconds after a scaling event (defaults to 300 seconds). + - `spec.maxReplicaCount` (optional): Specifies the maximum number of replicas to scale up to (defaults to 100). + - `spec.triggers`: Defines the specific configuration for your chosen scaler, including any required parameters or settings. +4. Refer to the scaler's documentation for detailed explanations and examples of the `triggers` section and any other required or optional configuration settings. +5. Save the `scaledobject.yaml` file after making the necessary modifications. + +## Step 4: Apply the ScaledObject Configuration + +Once you have created the `ScaledObject` configuration file, apply it to your Kubernetes cluster using `kubectl`: + +1. Open a terminal or command prompt and navigate to the directory containing the `scaledobject.yaml` file. +2. Run the following command to apply the `ScaledObject` configuration: + + ```bash + kubectl apply -f scaledobject.yaml + ``` + + ```plaintext + scaledobject.keda.sh/ created + ``` + +3. Verify that the `ScaledObject` has been created successfully by running: + + ```bash + kubectl get scaledobjects + ``` + + This should display the `ScaledObject` you just created. + + ```plaintext + NAME SCALETARGETKIND SCALETARGETNAME MIN MAX TRIGGERS AUTHENTICATION READY ACTIVE FALLBACK AGE + Deployment 1 10 cpu True False 10s + ``` + +After applying the `ScaledObject` configuration, KEDA will start monitoring the specified event source and scale your application accordingly, based on the configurations you provided. + +## Step 5: Monitor Scaling Events + +You can monitor the scaling events and logs generated by KEDA using the following commands: + +1. List all `ScaledObjects` in your cluster: + + ```bash + kubectl get scaledobjects + ``` + + This will show you the current state of your `ScaledObject` and the number of replicas. + + ```plaintext + NAME SCALETARGETKIND SCALETARGETNAME MIN MAX TRIGGERS AUTHENTICATION READY ACTIVE FALLBACK AGE + Deployment 1 10 cpu True False 10s + ``` + +2. View the logs of the KEDA operator: + + ```bash + kubectl logs -n keda -l app=keda-operator + ``` + + The KEDA operator logs will show you detailed information about scaling events, decisions made by KEDA based on the event source, and any errors or warnings. + + ```plaintext + {"level":"info","ts":,"logger":"scalehandler","msg":"Successfully scaled deployment","scaledobject.Namespace":"","scaledobject.Name":"","scaler":} + ``` diff --git a/content/docs/2.15/operate/setupscaler.md b/content/docs/2.15/operate/setupscaler.md new file mode 100644 index 000000000..7279354e3 --- /dev/null +++ b/content/docs/2.15/operate/setupscaler.md @@ -0,0 +1,116 @@ ++++ +title = "Setupscaler" +weight = 500 +description = "Procedure to Setup a Scaler in KEDA" ++++ + +## Prerequisites + +1. **Kubernetes Cluster**: + - Ensure you have a running Kubernetes cluster set up and accessible. + - If you don't have a cluster yet, follow the [official Kubernetes documentation](https://kubernetes.io/docs/setup/) to create a new cluster suitable for your environment (local machine, cloud provider, etc.). + +2. **KEDA Installation**: + - KEDA needs to be installed on your Kubernetes cluster before you can use it. + - Follow the [KEDA installation guide](https://keda.sh/docs/2.14/deploy/) carefully, including any prerequisites specific to your Kubernetes setup. + - The installation guide provides instructions for different installation methods (e.g., YAML, Helm charts, etc.). Choose the method that suits your needs. + +3. **kubectl**: + - The `kubectl` command-line tool is required to interact with your Kubernetes cluster. + - Follow the [official kubectl installation guide](https://kubernetes.io/docs/tasks/tools/#kubectl) to install `kubectl` on your operating system. + - Once installed, configure `kubectl` to communicate with your Kubernetes cluster by following the cluster-specific instructions provided by your Kubernetes setup. + +## Step 1: Identify the Scaler You Need + +KEDA supports various scalers that correspond to different event sources or triggers. Determining the right scaler is crucial for scaling your application based on the desired event source. + +1. Visit the [KEDA Scalers documentation](https://keda.sh/docs/2.14/scalers/) and browse through the list of available scalers. +2. Identify the scaler that matches the event source you want to use for scaling your application. For example: + - If you want to scale based on incoming HTTP traffic, you would need the **HTTP scaler**. + - If you want to scale based on messages in a RabbitMQ queue, you would need the **RabbitMQ scaler**. + - If you want to scale based on a cron schedule, you would need the **Cron scaler**. +3. Open the documentation page for your chosen scaler and familiarize yourself with its specific requirements and configuration options. + +## Step 2: Install the Required Scaler (if needed) + +Some scalers are part of the core KEDA installation, while others need to be installed separately as add-ons. + +1. Refer to the documentation of your chosen scaler to check if it needs to be installed separately. +2. If the scaler needs to be installed separately, follow the installation instructions provided in the scaler's documentation carefully. + - The installation process typically involves running a command (e.g., `helm install` for Helm charts) or applying YAML manifests using `kubectl`. +3. Verify that the scaler has been installed successfully by checking the output of the installation process or by running any provided verification commands. + +## Step 3: Create a ScaledObject Configuration File + +KEDA uses a custom resource called `ScaledObject` to define how your application should be scaled based on the chosen event source or trigger. + +1. Create a new file (e.g., `scaledobject.yaml`) in a text editor or using the command line. +2. Define the `ScaledObject` configuration in this file, following the structure and examples provided in the documentation of your chosen scaler. +3. Typically, the `ScaledObject` configuration includes the following sections: + - `metadata`: Specifies the name and namespace for the `ScaledObject`. + - `spec.scaleTargetRef`: Identifies the Kubernetes deployment or other resource that should be scaled. + - `spec.pollingInterval` (optional): Specifies how often KEDA should check for scaling events (defaults to 15 seconds). + - `spec.cooldownPeriod` (optional): Specifies the cool-down period in seconds after a scaling event (defaults to 300 seconds). + - `spec.maxReplicaCount` (optional): Specifies the maximum number of replicas to scale up to (defaults to 100). + - `spec.triggers`: Defines the specific configuration for your chosen scaler, including any required parameters or settings. +4. Refer to the scaler's documentation for detailed explanations and examples of the `triggers` section and any other required or optional configuration settings. +5. Save the `scaledobject.yaml` file after making the necessary modifications. + +## Step 4: Apply the ScaledObject Configuration + +Once you have created the `ScaledObject` configuration file, apply it to your Kubernetes cluster using `kubectl`: + +1. Open a terminal or command prompt and navigate to the directory containing the `scaledobject.yaml` file. +2. Run the following command to apply the `ScaledObject` configuration: + + ```bash + kubectl apply -f scaledobject.yaml + ``` + + ```plaintext + scaledobject.keda.sh/ created + ``` + +3. Verify that the `ScaledObject` has been created successfully by running: + + ```bash + kubectl get scaledobjects + ``` + + This should display the `ScaledObject` you just created. + + ```plaintext + NAME SCALETARGETKIND SCALETARGETNAME MIN MAX TRIGGERS AUTHENTICATION READY ACTIVE FALLBACK AGE + Deployment 1 10 cpu True False 10s + ``` + +After applying the `ScaledObject` configuration, KEDA will start monitoring the specified event source and scale your application accordingly, based on the configurations you provided. + +## Step 5: Monitor Scaling Events + +You can monitor the scaling events and logs generated by KEDA using the following commands: + +1. List all `ScaledObjects` in your cluster: + + ```bash + kubectl get scaledobjects + ``` + + This will show you the current state of your `ScaledObject` and the number of replicas. + + ```plaintext + NAME SCALETARGETKIND SCALETARGETNAME MIN MAX TRIGGERS AUTHENTICATION READY ACTIVE FALLBACK AGE + Deployment 1 10 cpu True False 10s + ``` + +2. View the logs of the KEDA operator: + + ```bash + kubectl logs -n keda -l app=keda-operator + ``` + + The KEDA operator logs will show you detailed information about scaling events, decisions made by KEDA based on the event source, and any errors or warnings. + + ```plaintext + {"level":"info","ts":,"logger":"scalehandler","msg":"Successfully scaled deployment","scaledobject.Namespace":"","scaledobject.Name":"","scaler":} + ``` From 3f00c4313545cb20b7ab565289595e86f4a888e9 Mon Sep 17 00:00:00 2001 From: shubhusion Date: Tue, 30 Jul 2024 20:55:56 +0530 Subject: [PATCH 02/33] Added a Note Signed-off-by: shubhusion --- content/docs/2.14/operate/setupscaler.md | 6 +++++- content/docs/2.15/operate/setupscaler.md | 6 +++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/content/docs/2.14/operate/setupscaler.md b/content/docs/2.14/operate/setupscaler.md index 7279354e3..a56feb769 100644 --- a/content/docs/2.14/operate/setupscaler.md +++ b/content/docs/2.14/operate/setupscaler.md @@ -26,7 +26,11 @@ KEDA supports various scalers that correspond to different event sources or trig 1. Visit the [KEDA Scalers documentation](https://keda.sh/docs/2.14/scalers/) and browse through the list of available scalers. 2. Identify the scaler that matches the event source you want to use for scaling your application. For example: - - If you want to scale based on incoming HTTP traffic, you would need the **HTTP scaler**. + - If you want to scale based on incoming HTTP traffic, you would need the **HTTP Add-on**. + + > **Note:** + > The HTTP Add-on is still in beta stage and may not provide the full functionality or stability expected in a production environment. + - If you want to scale based on messages in a RabbitMQ queue, you would need the **RabbitMQ scaler**. - If you want to scale based on a cron schedule, you would need the **Cron scaler**. 3. Open the documentation page for your chosen scaler and familiarize yourself with its specific requirements and configuration options. diff --git a/content/docs/2.15/operate/setupscaler.md b/content/docs/2.15/operate/setupscaler.md index 7279354e3..a56feb769 100644 --- a/content/docs/2.15/operate/setupscaler.md +++ b/content/docs/2.15/operate/setupscaler.md @@ -26,7 +26,11 @@ KEDA supports various scalers that correspond to different event sources or trig 1. Visit the [KEDA Scalers documentation](https://keda.sh/docs/2.14/scalers/) and browse through the list of available scalers. 2. Identify the scaler that matches the event source you want to use for scaling your application. For example: - - If you want to scale based on incoming HTTP traffic, you would need the **HTTP scaler**. + - If you want to scale based on incoming HTTP traffic, you would need the **HTTP Add-on**. + + > **Note:** + > The HTTP Add-on is still in beta stage and may not provide the full functionality or stability expected in a production environment. + - If you want to scale based on messages in a RabbitMQ queue, you would need the **RabbitMQ scaler**. - If you want to scale based on a cron schedule, you would need the **Cron scaler**. 3. Open the documentation page for your chosen scaler and familiarize yourself with its specific requirements and configuration options. From 68f0a47646a516378c3b9a90f9a29ce93219951f Mon Sep 17 00:00:00 2001 From: SHUBHAM SHARMA <96301987+shubhusion@users.noreply.github.com> Date: Mon, 5 Aug 2024 19:11:38 +0530 Subject: [PATCH 03/33] Update content/docs/2.14/operate/setupscaler.md Co-authored-by: Zbynek Roubalik Signed-off-by: SHUBHAM SHARMA <96301987+shubhusion@users.noreply.github.com> --- content/docs/2.14/operate/setupscaler.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/content/docs/2.14/operate/setupscaler.md b/content/docs/2.14/operate/setupscaler.md index a56feb769..ff5346a1c 100644 --- a/content/docs/2.14/operate/setupscaler.md +++ b/content/docs/2.14/operate/setupscaler.md @@ -1,7 +1,7 @@ +++ title = "Setupscaler" weight = 500 -description = "Procedure to Setup a Scaler in KEDA" +description = "Procedure to Setup a Autoscaling with KEDA" +++ ## Prerequisites From 98cda61b4d06ff7f0efe9e6b643956e9e7c0c4a6 Mon Sep 17 00:00:00 2001 From: SHUBHAM SHARMA <96301987+shubhusion@users.noreply.github.com> Date: Mon, 5 Aug 2024 19:11:54 +0530 Subject: [PATCH 04/33] Update content/docs/2.14/operate/setupscaler.md Co-authored-by: Zbynek Roubalik Signed-off-by: SHUBHAM SHARMA <96301987+shubhusion@users.noreply.github.com> --- content/docs/2.14/operate/setupscaler.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/content/docs/2.14/operate/setupscaler.md b/content/docs/2.14/operate/setupscaler.md index ff5346a1c..f24707dc2 100644 --- a/content/docs/2.14/operate/setupscaler.md +++ b/content/docs/2.14/operate/setupscaler.md @@ -1,5 +1,5 @@ +++ -title = "Setupscaler" +title = "Setup Autoscaling with KEDA" weight = 500 description = "Procedure to Setup a Autoscaling with KEDA" +++ From 2909cf4aab1d6c90796c2c1d9142851cf7020b84 Mon Sep 17 00:00:00 2001 From: SHUBHAM SHARMA <96301987+shubhusion@users.noreply.github.com> Date: Mon, 5 Aug 2024 19:12:12 +0530 Subject: [PATCH 05/33] Update content/docs/2.15/operate/setupscaler.md Co-authored-by: Zbynek Roubalik Signed-off-by: SHUBHAM SHARMA <96301987+shubhusion@users.noreply.github.com> --- content/docs/2.15/operate/setupscaler.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/content/docs/2.15/operate/setupscaler.md b/content/docs/2.15/operate/setupscaler.md index a56feb769..fa9d8d722 100644 --- a/content/docs/2.15/operate/setupscaler.md +++ b/content/docs/2.15/operate/setupscaler.md @@ -1,5 +1,5 @@ +++ -title = "Setupscaler" +title = "Setup Autoscaling with KEDA" weight = 500 description = "Procedure to Setup a Scaler in KEDA" +++ From 1b843b357c328417ec9579ea9f58f8cf4500fd4b Mon Sep 17 00:00:00 2001 From: SHUBHAM SHARMA <96301987+shubhusion@users.noreply.github.com> Date: Mon, 5 Aug 2024 19:12:28 +0530 Subject: [PATCH 06/33] Update content/docs/2.15/operate/setupscaler.md Co-authored-by: Zbynek Roubalik Signed-off-by: SHUBHAM SHARMA <96301987+shubhusion@users.noreply.github.com> --- content/docs/2.15/operate/setupscaler.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/content/docs/2.15/operate/setupscaler.md b/content/docs/2.15/operate/setupscaler.md index fa9d8d722..a8efd7078 100644 --- a/content/docs/2.15/operate/setupscaler.md +++ b/content/docs/2.15/operate/setupscaler.md @@ -26,7 +26,7 @@ KEDA supports various scalers that correspond to different event sources or trig 1. Visit the [KEDA Scalers documentation](https://keda.sh/docs/2.14/scalers/) and browse through the list of available scalers. 2. Identify the scaler that matches the event source you want to use for scaling your application. For example: - - If you want to scale based on incoming HTTP traffic, you would need the **HTTP Add-on**. + - If you want to scale based on incoming HTTP traffic, you would need the [HTTP Add-on](https://kedacore.github.io/http-add-on/). > **Note:** > The HTTP Add-on is still in beta stage and may not provide the full functionality or stability expected in a production environment. From e941483c23bcca7b8cf7424478d70d8147331b00 Mon Sep 17 00:00:00 2001 From: SHUBHAM SHARMA <96301987+shubhusion@users.noreply.github.com> Date: Mon, 5 Aug 2024 19:12:38 +0530 Subject: [PATCH 07/33] Update content/docs/2.14/operate/setupscaler.md Co-authored-by: Zbynek Roubalik Signed-off-by: SHUBHAM SHARMA <96301987+shubhusion@users.noreply.github.com> --- content/docs/2.14/operate/setupscaler.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/content/docs/2.14/operate/setupscaler.md b/content/docs/2.14/operate/setupscaler.md index f24707dc2..6bca2879c 100644 --- a/content/docs/2.14/operate/setupscaler.md +++ b/content/docs/2.14/operate/setupscaler.md @@ -26,7 +26,7 @@ KEDA supports various scalers that correspond to different event sources or trig 1. Visit the [KEDA Scalers documentation](https://keda.sh/docs/2.14/scalers/) and browse through the list of available scalers. 2. Identify the scaler that matches the event source you want to use for scaling your application. For example: - - If you want to scale based on incoming HTTP traffic, you would need the **HTTP Add-on**. + - If you want to scale based on incoming HTTP traffic, you would need the [HTTP Add-on](https://kedacore.github.io/http-add-on/). > **Note:** > The HTTP Add-on is still in beta stage and may not provide the full functionality or stability expected in a production environment. From 5de13765902be62b3a2f424621b310d2457f65e6 Mon Sep 17 00:00:00 2001 From: shubhusion Date: Mon, 5 Aug 2024 21:55:14 +0530 Subject: [PATCH 08/33] Changed File location Signed-off-by: shubhusion --- content/docs/2.15/{operate => }/setupscaler.md | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename content/docs/2.15/{operate => }/setupscaler.md (100%) diff --git a/content/docs/2.15/operate/setupscaler.md b/content/docs/2.15/setupscaler.md similarity index 100% rename from content/docs/2.15/operate/setupscaler.md rename to content/docs/2.15/setupscaler.md From 8c6ae103bf97edbba28045831d1b575324cd0ba8 Mon Sep 17 00:00:00 2001 From: shubhusion Date: Fri, 28 Jun 2024 13:51:01 +0530 Subject: [PATCH 09/33] Created Procedure for Setting Up a Scaler Signed-off-by: Shubham Sharma Signed-off-by: shubhusion --- content/docs/2.14/operate/setupscaler.md | 116 +++++++++++++++++++++++ content/docs/2.15/operate/setupscaler.md | 116 +++++++++++++++++++++++ 2 files changed, 232 insertions(+) create mode 100644 content/docs/2.14/operate/setupscaler.md create mode 100644 content/docs/2.15/operate/setupscaler.md diff --git a/content/docs/2.14/operate/setupscaler.md b/content/docs/2.14/operate/setupscaler.md new file mode 100644 index 000000000..7279354e3 --- /dev/null +++ b/content/docs/2.14/operate/setupscaler.md @@ -0,0 +1,116 @@ ++++ +title = "Setupscaler" +weight = 500 +description = "Procedure to Setup a Scaler in KEDA" ++++ + +## Prerequisites + +1. **Kubernetes Cluster**: + - Ensure you have a running Kubernetes cluster set up and accessible. + - If you don't have a cluster yet, follow the [official Kubernetes documentation](https://kubernetes.io/docs/setup/) to create a new cluster suitable for your environment (local machine, cloud provider, etc.). + +2. **KEDA Installation**: + - KEDA needs to be installed on your Kubernetes cluster before you can use it. + - Follow the [KEDA installation guide](https://keda.sh/docs/2.14/deploy/) carefully, including any prerequisites specific to your Kubernetes setup. + - The installation guide provides instructions for different installation methods (e.g., YAML, Helm charts, etc.). Choose the method that suits your needs. + +3. **kubectl**: + - The `kubectl` command-line tool is required to interact with your Kubernetes cluster. + - Follow the [official kubectl installation guide](https://kubernetes.io/docs/tasks/tools/#kubectl) to install `kubectl` on your operating system. + - Once installed, configure `kubectl` to communicate with your Kubernetes cluster by following the cluster-specific instructions provided by your Kubernetes setup. + +## Step 1: Identify the Scaler You Need + +KEDA supports various scalers that correspond to different event sources or triggers. Determining the right scaler is crucial for scaling your application based on the desired event source. + +1. Visit the [KEDA Scalers documentation](https://keda.sh/docs/2.14/scalers/) and browse through the list of available scalers. +2. Identify the scaler that matches the event source you want to use for scaling your application. For example: + - If you want to scale based on incoming HTTP traffic, you would need the **HTTP scaler**. + - If you want to scale based on messages in a RabbitMQ queue, you would need the **RabbitMQ scaler**. + - If you want to scale based on a cron schedule, you would need the **Cron scaler**. +3. Open the documentation page for your chosen scaler and familiarize yourself with its specific requirements and configuration options. + +## Step 2: Install the Required Scaler (if needed) + +Some scalers are part of the core KEDA installation, while others need to be installed separately as add-ons. + +1. Refer to the documentation of your chosen scaler to check if it needs to be installed separately. +2. If the scaler needs to be installed separately, follow the installation instructions provided in the scaler's documentation carefully. + - The installation process typically involves running a command (e.g., `helm install` for Helm charts) or applying YAML manifests using `kubectl`. +3. Verify that the scaler has been installed successfully by checking the output of the installation process or by running any provided verification commands. + +## Step 3: Create a ScaledObject Configuration File + +KEDA uses a custom resource called `ScaledObject` to define how your application should be scaled based on the chosen event source or trigger. + +1. Create a new file (e.g., `scaledobject.yaml`) in a text editor or using the command line. +2. Define the `ScaledObject` configuration in this file, following the structure and examples provided in the documentation of your chosen scaler. +3. Typically, the `ScaledObject` configuration includes the following sections: + - `metadata`: Specifies the name and namespace for the `ScaledObject`. + - `spec.scaleTargetRef`: Identifies the Kubernetes deployment or other resource that should be scaled. + - `spec.pollingInterval` (optional): Specifies how often KEDA should check for scaling events (defaults to 15 seconds). + - `spec.cooldownPeriod` (optional): Specifies the cool-down period in seconds after a scaling event (defaults to 300 seconds). + - `spec.maxReplicaCount` (optional): Specifies the maximum number of replicas to scale up to (defaults to 100). + - `spec.triggers`: Defines the specific configuration for your chosen scaler, including any required parameters or settings. +4. Refer to the scaler's documentation for detailed explanations and examples of the `triggers` section and any other required or optional configuration settings. +5. Save the `scaledobject.yaml` file after making the necessary modifications. + +## Step 4: Apply the ScaledObject Configuration + +Once you have created the `ScaledObject` configuration file, apply it to your Kubernetes cluster using `kubectl`: + +1. Open a terminal or command prompt and navigate to the directory containing the `scaledobject.yaml` file. +2. Run the following command to apply the `ScaledObject` configuration: + + ```bash + kubectl apply -f scaledobject.yaml + ``` + + ```plaintext + scaledobject.keda.sh/ created + ``` + +3. Verify that the `ScaledObject` has been created successfully by running: + + ```bash + kubectl get scaledobjects + ``` + + This should display the `ScaledObject` you just created. + + ```plaintext + NAME SCALETARGETKIND SCALETARGETNAME MIN MAX TRIGGERS AUTHENTICATION READY ACTIVE FALLBACK AGE + Deployment 1 10 cpu True False 10s + ``` + +After applying the `ScaledObject` configuration, KEDA will start monitoring the specified event source and scale your application accordingly, based on the configurations you provided. + +## Step 5: Monitor Scaling Events + +You can monitor the scaling events and logs generated by KEDA using the following commands: + +1. List all `ScaledObjects` in your cluster: + + ```bash + kubectl get scaledobjects + ``` + + This will show you the current state of your `ScaledObject` and the number of replicas. + + ```plaintext + NAME SCALETARGETKIND SCALETARGETNAME MIN MAX TRIGGERS AUTHENTICATION READY ACTIVE FALLBACK AGE + Deployment 1 10 cpu True False 10s + ``` + +2. View the logs of the KEDA operator: + + ```bash + kubectl logs -n keda -l app=keda-operator + ``` + + The KEDA operator logs will show you detailed information about scaling events, decisions made by KEDA based on the event source, and any errors or warnings. + + ```plaintext + {"level":"info","ts":,"logger":"scalehandler","msg":"Successfully scaled deployment","scaledobject.Namespace":"","scaledobject.Name":"","scaler":} + ``` diff --git a/content/docs/2.15/operate/setupscaler.md b/content/docs/2.15/operate/setupscaler.md new file mode 100644 index 000000000..7279354e3 --- /dev/null +++ b/content/docs/2.15/operate/setupscaler.md @@ -0,0 +1,116 @@ ++++ +title = "Setupscaler" +weight = 500 +description = "Procedure to Setup a Scaler in KEDA" ++++ + +## Prerequisites + +1. **Kubernetes Cluster**: + - Ensure you have a running Kubernetes cluster set up and accessible. + - If you don't have a cluster yet, follow the [official Kubernetes documentation](https://kubernetes.io/docs/setup/) to create a new cluster suitable for your environment (local machine, cloud provider, etc.). + +2. **KEDA Installation**: + - KEDA needs to be installed on your Kubernetes cluster before you can use it. + - Follow the [KEDA installation guide](https://keda.sh/docs/2.14/deploy/) carefully, including any prerequisites specific to your Kubernetes setup. + - The installation guide provides instructions for different installation methods (e.g., YAML, Helm charts, etc.). Choose the method that suits your needs. + +3. **kubectl**: + - The `kubectl` command-line tool is required to interact with your Kubernetes cluster. + - Follow the [official kubectl installation guide](https://kubernetes.io/docs/tasks/tools/#kubectl) to install `kubectl` on your operating system. + - Once installed, configure `kubectl` to communicate with your Kubernetes cluster by following the cluster-specific instructions provided by your Kubernetes setup. + +## Step 1: Identify the Scaler You Need + +KEDA supports various scalers that correspond to different event sources or triggers. Determining the right scaler is crucial for scaling your application based on the desired event source. + +1. Visit the [KEDA Scalers documentation](https://keda.sh/docs/2.14/scalers/) and browse through the list of available scalers. +2. Identify the scaler that matches the event source you want to use for scaling your application. For example: + - If you want to scale based on incoming HTTP traffic, you would need the **HTTP scaler**. + - If you want to scale based on messages in a RabbitMQ queue, you would need the **RabbitMQ scaler**. + - If you want to scale based on a cron schedule, you would need the **Cron scaler**. +3. Open the documentation page for your chosen scaler and familiarize yourself with its specific requirements and configuration options. + +## Step 2: Install the Required Scaler (if needed) + +Some scalers are part of the core KEDA installation, while others need to be installed separately as add-ons. + +1. Refer to the documentation of your chosen scaler to check if it needs to be installed separately. +2. If the scaler needs to be installed separately, follow the installation instructions provided in the scaler's documentation carefully. + - The installation process typically involves running a command (e.g., `helm install` for Helm charts) or applying YAML manifests using `kubectl`. +3. Verify that the scaler has been installed successfully by checking the output of the installation process or by running any provided verification commands. + +## Step 3: Create a ScaledObject Configuration File + +KEDA uses a custom resource called `ScaledObject` to define how your application should be scaled based on the chosen event source or trigger. + +1. Create a new file (e.g., `scaledobject.yaml`) in a text editor or using the command line. +2. Define the `ScaledObject` configuration in this file, following the structure and examples provided in the documentation of your chosen scaler. +3. Typically, the `ScaledObject` configuration includes the following sections: + - `metadata`: Specifies the name and namespace for the `ScaledObject`. + - `spec.scaleTargetRef`: Identifies the Kubernetes deployment or other resource that should be scaled. + - `spec.pollingInterval` (optional): Specifies how often KEDA should check for scaling events (defaults to 15 seconds). + - `spec.cooldownPeriod` (optional): Specifies the cool-down period in seconds after a scaling event (defaults to 300 seconds). + - `spec.maxReplicaCount` (optional): Specifies the maximum number of replicas to scale up to (defaults to 100). + - `spec.triggers`: Defines the specific configuration for your chosen scaler, including any required parameters or settings. +4. Refer to the scaler's documentation for detailed explanations and examples of the `triggers` section and any other required or optional configuration settings. +5. Save the `scaledobject.yaml` file after making the necessary modifications. + +## Step 4: Apply the ScaledObject Configuration + +Once you have created the `ScaledObject` configuration file, apply it to your Kubernetes cluster using `kubectl`: + +1. Open a terminal or command prompt and navigate to the directory containing the `scaledobject.yaml` file. +2. Run the following command to apply the `ScaledObject` configuration: + + ```bash + kubectl apply -f scaledobject.yaml + ``` + + ```plaintext + scaledobject.keda.sh/ created + ``` + +3. Verify that the `ScaledObject` has been created successfully by running: + + ```bash + kubectl get scaledobjects + ``` + + This should display the `ScaledObject` you just created. + + ```plaintext + NAME SCALETARGETKIND SCALETARGETNAME MIN MAX TRIGGERS AUTHENTICATION READY ACTIVE FALLBACK AGE + Deployment 1 10 cpu True False 10s + ``` + +After applying the `ScaledObject` configuration, KEDA will start monitoring the specified event source and scale your application accordingly, based on the configurations you provided. + +## Step 5: Monitor Scaling Events + +You can monitor the scaling events and logs generated by KEDA using the following commands: + +1. List all `ScaledObjects` in your cluster: + + ```bash + kubectl get scaledobjects + ``` + + This will show you the current state of your `ScaledObject` and the number of replicas. + + ```plaintext + NAME SCALETARGETKIND SCALETARGETNAME MIN MAX TRIGGERS AUTHENTICATION READY ACTIVE FALLBACK AGE + Deployment 1 10 cpu True False 10s + ``` + +2. View the logs of the KEDA operator: + + ```bash + kubectl logs -n keda -l app=keda-operator + ``` + + The KEDA operator logs will show you detailed information about scaling events, decisions made by KEDA based on the event source, and any errors or warnings. + + ```plaintext + {"level":"info","ts":,"logger":"scalehandler","msg":"Successfully scaled deployment","scaledobject.Namespace":"","scaledobject.Name":"","scaler":} + ``` From a5d80c0982602dc5d1460283a544b183d5a8dc95 Mon Sep 17 00:00:00 2001 From: Uchechukwu Obasi Date: Fri, 28 Jun 2024 14:04:43 +0100 Subject: [PATCH 10/33] chore: strip out raw HTML from docs (#1414) Signed-off-by: shubhusion --- content/docs/1.4/concepts/external-scalers.md | 172 ++++++++------- content/docs/1.5/concepts/external-scalers.md | 172 ++++++++------- content/docs/2.0/concepts/external-scalers.md | 205 ++++++++++-------- content/docs/2.0/migration.md | 165 +++++++------- content/docs/2.0/operate/cluster.md | 8 +- content/docs/2.1/concepts/external-scalers.md | 205 ++++++++++-------- content/docs/2.1/migration.md | 165 +++++++------- content/docs/2.1/operate/cluster.md | 8 +- .../docs/2.10/concepts/external-scalers.md | 179 ++++++++------- content/docs/2.10/migration.md | 165 +++++++------- content/docs/2.10/operate/cluster.md | 67 +++--- .../docs/2.11/concepts/external-scalers.md | 179 ++++++++------- content/docs/2.11/migration.md | 165 +++++++------- content/docs/2.11/operate/cluster.md | 67 +++--- .../docs/2.12/concepts/external-scalers.md | 179 ++++++++------- content/docs/2.12/migration.md | 165 +++++++------- content/docs/2.12/operate/cluster.md | 69 +++--- .../docs/2.13/concepts/external-scalers.md | 179 ++++++++------- content/docs/2.13/migration.md | 165 +++++++------- content/docs/2.13/operate/cluster.md | 71 +++--- .../docs/2.14/concepts/external-scalers.md | 179 ++++++++------- content/docs/2.14/migration.md | 165 +++++++------- content/docs/2.14/operate/cluster.md | 79 +++---- .../docs/2.15/concepts/external-scalers.md | 179 ++++++++------- content/docs/2.15/migration.md | 165 +++++++------- content/docs/2.15/operate/cluster.md | 81 +++---- content/docs/2.2/concepts/external-scalers.md | 205 ++++++++++-------- content/docs/2.2/migration.md | 165 +++++++------- content/docs/2.2/operate/cluster.md | 10 +- content/docs/2.3/concepts/external-scalers.md | 205 ++++++++++-------- content/docs/2.3/migration.md | 165 +++++++------- content/docs/2.3/operate/cluster.md | 12 +- content/docs/2.4/concepts/external-scalers.md | 205 ++++++++++-------- content/docs/2.4/migration.md | 165 +++++++------- content/docs/2.4/operate/cluster.md | 22 +- content/docs/2.5/concepts/external-scalers.md | 183 ++++++++-------- content/docs/2.5/migration.md | 165 +++++++------- content/docs/2.5/operate/cluster.md | 31 ++- content/docs/2.6/concepts/external-scalers.md | 179 ++++++++------- content/docs/2.6/migration.md | 165 +++++++------- content/docs/2.6/operate/cluster.md | 45 ++-- content/docs/2.7/concepts/external-scalers.md | 179 ++++++++------- content/docs/2.7/migration.md | 165 +++++++------- content/docs/2.7/operate/cluster.md | 45 ++-- content/docs/2.8/concepts/external-scalers.md | 179 ++++++++------- content/docs/2.8/migration.md | 165 +++++++------- content/docs/2.8/operate/cluster.md | 44 ++-- content/docs/2.9/concepts/external-scalers.md | 179 ++++++++------- content/docs/2.9/migration.md | 165 +++++++------- content/docs/2.9/operate/cluster.md | 78 +++---- 50 files changed, 3536 insertions(+), 3183 deletions(-) diff --git a/content/docs/1.4/concepts/external-scalers.md b/content/docs/1.4/concepts/external-scalers.md index 5bf6b7495..f02eeeb5d 100644 --- a/content/docs/1.4/concepts/external-scalers.md +++ b/content/docs/1.4/concepts/external-scalers.md @@ -27,11 +27,12 @@ type Scaler interface { ``` The `Scaler` interface defines 4 methods: + - `IsActive` is called on `pollingInterval` defined in the ScaledObject/ScaledJob CRDs and scaling to 1 happens if this returns true. - `Close` is called to allow the scaler to clean up connections or other resources. - `GetMetricSpec` returns the target value for the HPA definition for the scaler. For more details refer to [Implementing `GetMetricSpec`](#4-implementing-getmetricspec). - `GetMetrics` returns the value of the metric referred to from `GetMetricSpec`. For more details refer to [Implementing `GetMetrics`](#5-implementing-getmetrics). -> Refer to the [HPA docs](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale-walkthrough/) for how HPA calculates replicaCount based on metric value and target value. + > Refer to the [HPA docs](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale-walkthrough/) for how HPA calculates replicaCount based on metric value and target value. ### External Scaler GRPC interface @@ -51,6 +52,7 @@ service ExternalScaler { - `IsActive` maps to the `IsActive` method on the `Scaler` interface. Few things to notice: + - `IsActive`, `StreamIsActive`, and `GetMetricsSpec` are called with a `ScaledObjectRef` that contains the scaledObject name/namespace as well as the content of `metadata` defined in the trigger. For example the following `ScaledObject`: @@ -85,6 +87,7 @@ KEDA will attempt a connection to `service-address.svc.local:9090` and calls `Is } } ``` + ## Implementing KEDA external scaler GRPC interface ### Implementing an external scaler: @@ -110,6 +113,7 @@ go mod init example.com/external-scaler/sample mkdir externalscaler protoc externalscaler.proto --go_out=plugins=grpc:externalscaler ``` + {{< /collapsible >}} {{< collapsible "C#" >}} @@ -146,9 +150,8 @@ mkdir Services ```bash npm install --save grpc request ``` -{{< /collapsible >}} -
+{{< /collapsible >}} #### 3. Implementing `IsActive` @@ -178,6 +181,7 @@ spec: Full implementation can be found here: https://github.com/kedacore/external-scaler-samples `main.go` + ```golang func (e *ExternalScaler) IsActive(ctx context.Context, scaledObject *pb.ScaledObjectRef) (*pb.IsActiveResponse, error) { // request.Scalermetadata contains the `metadata` defined in the ScaledObject @@ -224,12 +228,14 @@ func (e *ExternalScaler) IsActive(ctx context.Context, scaledObject *pb.ScaledOb }, nil } ``` + {{< /collapsible >}} {{< collapsible "C#" >}} Full implementation can be found here: https://github.com/kedacore/external-scaler-samples `Services/ExternalScalerService.cs` + ```csharp public class ExternalScalerService : ExternalScaler.ExternalScalerBase { @@ -262,79 +268,84 @@ public class ExternalScalerService : ExternalScaler.ExternalScalerBase } } ``` + {{< /collapsible >}} {{< collapsible "Javascript" >}} `index.js` + ```js -const grpc = require('grpc') -const request = require('request') -const externalScalerProto = grpc.load('externalscaler.proto') +const grpc = require("grpc"); +const request = require("request"); +const externalScalerProto = grpc.load("externalscaler.proto"); -const server = new grpc.Server() +const server = new grpc.Server(); server.addService(externalScalerProto.externalscaler.ExternalScaler.service, { isActive: (call, callback) => { - const longitude = call.request.scalerMetadata.longitude - const latitude = call.request.scalerMetadata.latitude + const longitude = call.request.scalerMetadata.longitude; + const latitude = call.request.scalerMetadata.latitude; if (!longitude || !latitude) { callback({ code: grpc.status.INVALID_ARGUMENT, - details: 'longitude and latitude must be specified', - }) + details: "longitude and latitude must be specified", + }); } else { - const now = new Date() - const yesterday = new Date(new Date().setDate(new Date().getDate()-1)); - - const startTime = `${yesterday.getUTCFullYear()}-${yesterday.getUTCMonth()}-${yesterday.getUTCDay()}` - const endTime = `${now.getUTCFullYear()}-${now.getUTCMonth()}-${now.getUTCDay()}` - const radiusKm = 500 - const query = `format=geojson&starttime=${startTime}&endtime=${endTime}&longitude=${longitude}&latitude=${latitude}&maxradiuskm=${radiusKm}` - - request.get({ - url: `https://earthquake.usgs.gov/fdsnws/event/1/query?${query}`, - json: true, - }, (err, resp, data) => { - if (err) { - callback({ - code: grpc.status.INTERNAL, - details: err, - }) - } else if (resp.statusCode !== 200) { - callback({ - code: grpc.status.INTERNAL, - details: `expected status 200, got ${resp.statusCode}` - }) - } else { - // count how many earthquakes with mag > 1.0 - let count = 0 - data.features.forEach(i => { - if (i.properties.mag > 1.0) { - count++ - } - }) - callback(null, { - result: count > 2, - }) + const now = new Date(); + const yesterday = new Date(new Date().setDate(new Date().getDate() - 1)); + + const startTime = `${yesterday.getUTCFullYear()}-${yesterday.getUTCMonth()}-${yesterday.getUTCDay()}`; + const endTime = `${now.getUTCFullYear()}-${now.getUTCMonth()}-${now.getUTCDay()}`; + const radiusKm = 500; + const query = `format=geojson&starttime=${startTime}&endtime=${endTime}&longitude=${longitude}&latitude=${latitude}&maxradiuskm=${radiusKm}`; + + request.get( + { + url: `https://earthquake.usgs.gov/fdsnws/event/1/query?${query}`, + json: true, + }, + (err, resp, data) => { + if (err) { + callback({ + code: grpc.status.INTERNAL, + details: err, + }); + } else if (resp.statusCode !== 200) { + callback({ + code: grpc.status.INTERNAL, + details: `expected status 200, got ${resp.statusCode}`, + }); + } else { + // count how many earthquakes with mag > 1.0 + let count = 0; + data.features.forEach((i) => { + if (i.properties.mag > 1.0) { + count++; + } + }); + callback(null, { + result: count > 2, + }); + } } - }) + ); } - } -}) + }, +}); -server.bind('127.0.0.1:9090', grpc.ServerCredentials.createInsecure()) -console.log('Server listening on 127.0.0.1:9090') +server.bind("127.0.0.1:9090", grpc.ServerCredentials.createInsecure()); +console.log("Server listening on 127.0.0.1:9090"); -server.start() +server.start(); ``` -{{< /collapsible >}} -
+{{< /collapsible >}} #### 4. Implementing `GetMetricSpec` `GetMetricSpec` returns the `target` value for [the HPA definition for the scaler](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale-walkthrough/). This scaler will define a static target of 10, but the threshold value is often specified in the metadata for other scalers. {{< collapsible "Golang" >}} + ```golang func (e *ExternalScaler) GetMetricSpec(context.Context, *pb.ScaledObjectRef) (*pb.GetMetricSpecResponse, error) { return &pb.GetMetricSpecResponse{ @@ -345,9 +356,11 @@ func (e *ExternalScaler) GetMetricSpec(context.Context, *pb.ScaledObjectRef) (*p }, nil } ``` + {{< /collapsible >}} {{< collapsible "C#" >}} + ```csharp public override async Task GetMetricSpec(ScaledObjectRef request, ServerCallContext context) { @@ -362,31 +375,35 @@ public override async Task GetMetricSpec(ScaledObjectRef return Task.FromResult(resp); } ``` + {{< /collapsible >}} {{< collapsible "Javascript" >}} + ```js server.addService(externalScalerProto.externalscaler.ExternalScaler.service, { // ... getMetricSpec: (call, callback) => { callback(null, { - metricSpecs: [{ - metricName: 'earthquakeThreshold', - targetSize: 10, - }] - }) - } -}) + metricSpecs: [ + { + metricName: "earthquakeThreshold", + targetSize: 10, + }, + ], + }); + }, +}); ``` -{{< /collapsible >}} -
+{{< /collapsible >}} #### 5. Implementing `GetMetrics` `GetMetrics` returns the value of the metric referred to from `GetMetricSpec`, in this example it's `earthquakeThreshold`. {{< collapsible "Golang" >}} + ```golang func (e *ExternalScaler) GetMetrics(_ context.Context, metricRequest *pb.GetMetricsRequest) (*pb.GetMetricsResponse, error) { longitude := metricRequest.ScaledObjectRef.ScalerMetadata["longitude"] @@ -409,9 +426,11 @@ func (e *ExternalScaler) GetMetrics(_ context.Context, metricRequest *pb.GetMetr }, nil } ``` + {{< /collapsible >}} {{< collapsible "C#" >}} + ```csharp public override async Task GetMetrics(GetMetricsRequest request, ServerCallContext context) { @@ -436,38 +455,43 @@ public override async Task GetMetrics(GetMetricsRequest requ return resp; } ``` + {{< /collapsible >}} {{< collapsible "Javascript" >}} + ```js server.addService(externalScalerProto.externalscaler.ExternalScaler.service, { // ... getMetrics: (call, callback) => { - const longitude = call.request.scaledObjectRef.scalerMetadata.longitude - const latitude = call.request.scaledObjectRef.scalerMetadata.latitude + const longitude = call.request.scaledObjectRef.scalerMetadata.longitude; + const latitude = call.request.scaledObjectRef.scalerMetadata.latitude; if (!longitude || !latitude) { callback({ code: grpc.status.INVALID_ARGUMENT, - details: 'longitude and latitude must be specified', - }) + details: "longitude and latitude must be specified", + }); } else { getEarthquakeCount((err, count) => { if (err) { callback({ code: grpc.status.INTERNAL, details: err, - }) + }); } else { callback(null, { - metricValues: [{ - metricName: 'earthquakeThreshold', - metricValue: count, - }] - }) + metricValues: [ + { + metricName: "earthquakeThreshold", + metricValue: count, + }, + ], + }); } - }) + }); } - } -}) + }, +}); ``` + {{< /collapsible >}} diff --git a/content/docs/1.5/concepts/external-scalers.md b/content/docs/1.5/concepts/external-scalers.md index 271e5ef34..712967345 100644 --- a/content/docs/1.5/concepts/external-scalers.md +++ b/content/docs/1.5/concepts/external-scalers.md @@ -27,11 +27,12 @@ type Scaler interface { ``` The `Scaler` interface defines 4 methods: + - `IsActive` is called on `pollingInterval` defined in the ScaledObject/ScaledJob CRDs and scaling to 1 happens if this returns true. - `Close` is called to allow the scaler to clean up connections or other resources. - `GetMetricSpec` returns the target value for the HPA definition for the scaler. For more details refer to [Implementing `GetMetricSpec`](#4-implementing-getmetricspec). - `GetMetrics` returns the value of the metric referred to from `GetMetricSpec`. For more details refer to [Implementing `GetMetrics`](#5-implementing-getmetrics). -> Refer to the [HPA docs](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale-walkthrough/) for how HPA calculates replicaCount based on metric value and target value. + > Refer to the [HPA docs](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale-walkthrough/) for how HPA calculates replicaCount based on metric value and target value. ### External Scaler GRPC interface @@ -51,6 +52,7 @@ service ExternalScaler { - `IsActive` maps to the `IsActive` method on the `Scaler` interface. Few things to notice: + - `IsActive`, `StreamIsActive`, and `GetMetricsSpec` are called with a `ScaledObjectRef` that contains the scaledObject name/namespace as well as the content of `metadata` defined in the trigger. For example the following `ScaledObject`: @@ -85,6 +87,7 @@ KEDA will attempt a connection to `service-address.svc.local:9090` and calls `Is } } ``` + ## Implementing KEDA external scaler GRPC interface ### Implementing an external scaler: @@ -110,6 +113,7 @@ go mod init example.com/external-scaler/sample mkdir externalscaler protoc externalscaler.proto --go_out=plugins=grpc:externalscaler ``` + {{< /collapsible >}} {{< collapsible "C#" >}} @@ -146,9 +150,8 @@ mkdir Services ```bash npm install --save grpc request ``` -{{< /collapsible >}} -
+{{< /collapsible >}} #### 3. Implementing `IsActive` @@ -178,6 +181,7 @@ spec: Full implementation can be found here: https://github.com/kedacore/external-scaler-samples `main.go` + ```golang func (e *ExternalScaler) IsActive(ctx context.Context, scaledObject *pb.ScaledObjectRef) (*pb.IsActiveResponse, error) { // request.Scalermetadata contains the `metadata` defined in the ScaledObject @@ -224,12 +228,14 @@ func (e *ExternalScaler) IsActive(ctx context.Context, scaledObject *pb.ScaledOb }, nil } ``` + {{< /collapsible >}} {{< collapsible "C#" >}} Full implementation can be found here: https://github.com/kedacore/external-scaler-samples `Services/ExternalScalerService.cs` + ```csharp public class ExternalScalerService : ExternalScaler.ExternalScalerBase { @@ -262,79 +268,84 @@ public class ExternalScalerService : ExternalScaler.ExternalScalerBase } } ``` + {{< /collapsible >}} {{< collapsible "Javascript" >}} `index.js` + ```js -const grpc = require('grpc') -const request = require('request') -const externalScalerProto = grpc.load('externalscaler.proto') +const grpc = require("grpc"); +const request = require("request"); +const externalScalerProto = grpc.load("externalscaler.proto"); -const server = new grpc.Server() +const server = new grpc.Server(); server.addService(externalScalerProto.externalscaler.ExternalScaler.service, { isActive: (call, callback) => { - const longitude = call.request.scalerMetadata.longitude - const latitude = call.request.scalerMetadata.latitude + const longitude = call.request.scalerMetadata.longitude; + const latitude = call.request.scalerMetadata.latitude; if (!longitude || !latitude) { callback({ code: grpc.status.INVALID_ARGUMENT, - details: 'longitude and latitude must be specified', - }) + details: "longitude and latitude must be specified", + }); } else { - const now = new Date() - const yesterday = new Date(new Date().setDate(new Date().getDate()-1)); - - const startTime = `${yesterday.getUTCFullYear()}-${yesterday.getUTCMonth()}-${yesterday.getUTCDay()}` - const endTime = `${now.getUTCFullYear()}-${now.getUTCMonth()}-${now.getUTCDay()}` - const radiusKm = 500 - const query = `format=geojson&starttime=${startTime}&endtime=${endTime}&longitude=${longitude}&latitude=${latitude}&maxradiuskm=${radiusKm}` - - request.get({ - url: `https://earthquake.usgs.gov/fdsnws/event/1/query?${query}`, - json: true, - }, (err, resp, data) => { - if (err) { - callback({ - code: grpc.status.INTERNAL, - details: err, - }) - } else if (resp.statusCode !== 200) { - callback({ - code: grpc.status.INTERNAL, - details: `expected status 200, got ${resp.statusCode}` - }) - } else { - // count how many earthquakes with mag > 1.0 - let count = 0 - data.features.forEach(i => { - if (i.properties.mag > 1.0) { - count++ - } - }) - callback(null, { - result: count > 2, - }) + const now = new Date(); + const yesterday = new Date(new Date().setDate(new Date().getDate() - 1)); + + const startTime = `${yesterday.getUTCFullYear()}-${yesterday.getUTCMonth()}-${yesterday.getUTCDay()}`; + const endTime = `${now.getUTCFullYear()}-${now.getUTCMonth()}-${now.getUTCDay()}`; + const radiusKm = 500; + const query = `format=geojson&starttime=${startTime}&endtime=${endTime}&longitude=${longitude}&latitude=${latitude}&maxradiuskm=${radiusKm}`; + + request.get( + { + url: `https://earthquake.usgs.gov/fdsnws/event/1/query?${query}`, + json: true, + }, + (err, resp, data) => { + if (err) { + callback({ + code: grpc.status.INTERNAL, + details: err, + }); + } else if (resp.statusCode !== 200) { + callback({ + code: grpc.status.INTERNAL, + details: `expected status 200, got ${resp.statusCode}`, + }); + } else { + // count how many earthquakes with mag > 1.0 + let count = 0; + data.features.forEach((i) => { + if (i.properties.mag > 1.0) { + count++; + } + }); + callback(null, { + result: count > 2, + }); + } } - }) + ); } - } -}) + }, +}); -server.bind('127.0.0.1:9090', grpc.ServerCredentials.createInsecure()) -console.log('Server listening on 127.0.0.1:9090') +server.bind("127.0.0.1:9090", grpc.ServerCredentials.createInsecure()); +console.log("Server listening on 127.0.0.1:9090"); -server.start() +server.start(); ``` -{{< /collapsible >}} -
+{{< /collapsible >}} #### 4. Implementing `GetMetricSpec` `GetMetricSpec` returns the `target` value for [the HPA definition for the scaler](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale-walkthrough/). This scaler will define a static target of 10, but the threshold value is often specified in the metadata for other scalers. {{< collapsible "Golang" >}} + ```golang func (e *ExternalScaler) GetMetricSpec(context.Context, *pb.ScaledObjectRef) (*pb.GetMetricSpecResponse, error) { return &pb.GetMetricSpecResponse{ @@ -345,9 +356,11 @@ func (e *ExternalScaler) GetMetricSpec(context.Context, *pb.ScaledObjectRef) (*p }, nil } ``` + {{< /collapsible >}} {{< collapsible "C#" >}} + ```csharp public override async Task GetMetricSpec(ScaledObjectRef request, ServerCallContext context) { @@ -362,31 +375,35 @@ public override async Task GetMetricSpec(ScaledObjectRef return Task.FromResult(resp); } ``` + {{< /collapsible >}} {{< collapsible "Javascript" >}} + ```js server.addService(externalScalerProto.externalscaler.ExternalScaler.service, { // ... getMetricSpec: (call, callback) => { callback(null, { - metricSpecs: [{ - metricName: 'earthquakeThreshold', - targetSize: 10, - }] - }) - } -}) + metricSpecs: [ + { + metricName: "earthquakeThreshold", + targetSize: 10, + }, + ], + }); + }, +}); ``` -{{< /collapsible >}} -
+{{< /collapsible >}} #### 5. Implementing `GetMetrics` `GetMetrics` returns the value of the metric referred to from `GetMetricSpec`, in this example it's `earthquakeThreshold`. {{< collapsible "Golang" >}} + ```golang func (e *ExternalScaler) GetMetrics(_ context.Context, metricRequest *pb.GetMetricsRequest) (*pb.GetMetricsResponse, error) { longitude := metricRequest.ScaledObjectRef.ScalerMetadata["longitude"] @@ -409,9 +426,11 @@ func (e *ExternalScaler) GetMetrics(_ context.Context, metricRequest *pb.GetMetr }, nil } ``` + {{< /collapsible >}} {{< collapsible "C#" >}} + ```csharp public override async Task GetMetrics(GetMetricsRequest request, ServerCallContext context) { @@ -436,38 +455,43 @@ public override async Task GetMetrics(GetMetricsRequest requ return resp; } ``` + {{< /collapsible >}} {{< collapsible "Javascript" >}} + ```js server.addService(externalScalerProto.externalscaler.ExternalScaler.service, { // ... getMetrics: (call, callback) => { - const longitude = call.request.scaledObjectRef.scalerMetadata.longitude - const latitude = call.request.scaledObjectRef.scalerMetadata.latitude + const longitude = call.request.scaledObjectRef.scalerMetadata.longitude; + const latitude = call.request.scaledObjectRef.scalerMetadata.latitude; if (!longitude || !latitude) { callback({ code: grpc.status.INVALID_ARGUMENT, - details: 'longitude and latitude must be specified', - }) + details: "longitude and latitude must be specified", + }); } else { getEarthquakeCount((err, count) => { if (err) { callback({ code: grpc.status.INTERNAL, details: err, - }) + }); } else { callback(null, { - metricValues: [{ - metricName: 'earthquakeThreshold', - metricValue: count, - }] - }) + metricValues: [ + { + metricName: "earthquakeThreshold", + metricValue: count, + }, + ], + }); } - }) + }); } - } -}) + }, +}); ``` + {{< /collapsible >}} diff --git a/content/docs/2.0/concepts/external-scalers.md b/content/docs/2.0/concepts/external-scalers.md index 27ed9e0c6..1945394d3 100644 --- a/content/docs/2.0/concepts/external-scalers.md +++ b/content/docs/2.0/concepts/external-scalers.md @@ -33,15 +33,15 @@ type PushScaler interface { ``` The `Scaler` interface defines 4 methods: + - `IsActive` is called on `pollingInterval` defined in the ScaledObject/ScaledJob CRDs and scaling to 1 happens if this returns true. - `Close` is called to allow the scaler to clean up connections or other resources. - `GetMetricSpec` returns the target value for the HPA definition for the scaler. For more details refer to [Implementing `GetMetricSpec`](#5-implementing-getmetricspec). - `GetMetrics` returns the value of the metric referred to from `GetMetricSpec`. For more details refer to [Implementing `GetMetrics`](#6-implementing-getmetrics). -> Refer to the [HPA docs](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale-walkthrough/) for how HPA calculates `replicaCount` based on metric value and target value. KEDA uses the metric target type `AverageValue` for external metrics. This will cause the metric value returned by the external scaler to be divided by the number of replicas. + > Refer to the [HPA docs](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale-walkthrough/) for how HPA calculates `replicaCount` based on metric value and target value. KEDA uses the metric target type `AverageValue` for external metrics. This will cause the metric value returned by the external scaler to be divided by the number of replicas. The `PushScaler` interface adds `Run` method. The `Run` method receives a push channel (`active`), that the scaler can push `true` to any time to force scaling action independently from `pollingInterval`. - ### External Scaler GRPC interface KEDA comes with 2 external scalers [`external`](../scalers/external.md) and [`external-push`](../scalers/external-push.md). @@ -62,6 +62,7 @@ service ExternalScaler { - `StreamIsActive` maps to the `Run` method on the `PushScaler` interface. Few things to notice: + - Lack of `Close` method as the GRPC connection defines the lifetime of the scaler - `IsActive`, `StreamIsActive`, and `GetMetricsSpec` are called with a `ScaledObjectRef` that contains the scaledObject name/namespace as well as the content of `metadata` defined in the trigger. @@ -125,6 +126,7 @@ go mod init example.com/external-scaler/sample mkdir externalscaler protoc externalscaler.proto --go_out=plugins=grpc:externalscaler ``` + {{< /collapsible >}} {{< collapsible "C#" >}} @@ -161,9 +163,8 @@ mkdir Services ```bash npm install --save grpc request ``` -{{< /collapsible >}} -
+{{< /collapsible >}} #### 3. Implementing `IsActive` @@ -193,6 +194,7 @@ spec: Full implementation can be found here: https://github.com/kedacore/external-scaler-samples `main.go` + ```golang func (e *ExternalScaler) IsActive(ctx context.Context, scaledObject *pb.ScaledObjectRef) (*pb.IsActiveResponse, error) { // request.Scalermetadata contains the `metadata` defined in the ScaledObject @@ -239,12 +241,14 @@ func (e *ExternalScaler) IsActive(ctx context.Context, scaledObject *pb.ScaledOb }, nil } ``` + {{< /collapsible >}} {{< collapsible "C#" >}} Full implementation can be found here: https://github.com/kedacore/external-scaler-samples `Services/ExternalScalerService.cs` + ```csharp public class ExternalScalerService : ExternalScaler.ExternalScalerBase { @@ -277,73 +281,77 @@ public class ExternalScalerService : ExternalScaler.ExternalScalerBase } } ``` + {{< /collapsible >}} {{< collapsible "Javascript" >}} `index.js` + ```js -const grpc = require('grpc') -const request = require('request') -const externalScalerProto = grpc.load('externalscaler.proto') +const grpc = require("grpc"); +const request = require("request"); +const externalScalerProto = grpc.load("externalscaler.proto"); -const server = new grpc.Server() +const server = new grpc.Server(); server.addService(externalScalerProto.externalscaler.ExternalScaler.service, { isActive: (call, callback) => { - const longitude = call.request.scalerMetadata.longitude - const latitude = call.request.scalerMetadata.latitude + const longitude = call.request.scalerMetadata.longitude; + const latitude = call.request.scalerMetadata.latitude; if (!longitude || !latitude) { callback({ code: grpc.status.INVALID_ARGUMENT, - details: 'longitude and latitude must be specified', - }) + details: "longitude and latitude must be specified", + }); } else { - const now = new Date() - const yesterday = new Date(new Date().setDate(new Date().getDate()-1)); - - const startTime = `${yesterday.getUTCFullYear()}-${yesterday.getUTCMonth()}-${yesterday.getUTCDay()}` - const endTime = `${now.getUTCFullYear()}-${now.getUTCMonth()}-${now.getUTCDay()}` - const radiusKm = 500 - const query = `format=geojson&starttime=${startTime}&endtime=${endTime}&longitude=${longitude}&latitude=${latitude}&maxradiuskm=${radiusKm}` - - request.get({ - url: `https://earthquake.usgs.gov/fdsnws/event/1/query?${query}`, - json: true, - }, (err, resp, data) => { - if (err) { - callback({ - code: grpc.status.INTERNAL, - details: err, - }) - } else if (resp.statusCode !== 200) { - callback({ - code: grpc.status.INTERNAL, - details: `expected status 200, got ${resp.statusCode}` - }) - } else { - // count how many earthquakes with mag > 1.0 - let count = 0 - data.features.forEach(i => { - if (i.properties.mag > 1.0) { - count++ - } - }) - callback(null, { - result: count > 2, - }) + const now = new Date(); + const yesterday = new Date(new Date().setDate(new Date().getDate() - 1)); + + const startTime = `${yesterday.getUTCFullYear()}-${yesterday.getUTCMonth()}-${yesterday.getUTCDay()}`; + const endTime = `${now.getUTCFullYear()}-${now.getUTCMonth()}-${now.getUTCDay()}`; + const radiusKm = 500; + const query = `format=geojson&starttime=${startTime}&endtime=${endTime}&longitude=${longitude}&latitude=${latitude}&maxradiuskm=${radiusKm}`; + + request.get( + { + url: `https://earthquake.usgs.gov/fdsnws/event/1/query?${query}`, + json: true, + }, + (err, resp, data) => { + if (err) { + callback({ + code: grpc.status.INTERNAL, + details: err, + }); + } else if (resp.statusCode !== 200) { + callback({ + code: grpc.status.INTERNAL, + details: `expected status 200, got ${resp.statusCode}`, + }); + } else { + // count how many earthquakes with mag > 1.0 + let count = 0; + data.features.forEach((i) => { + if (i.properties.mag > 1.0) { + count++; + } + }); + callback(null, { + result: count > 2, + }); + } } - }) + ); } - } -}) + }, +}); -server.bind('127.0.0.1:9090', grpc.ServerCredentials.createInsecure()) -console.log('Server listening on 127.0.0.1:9090') +server.bind("127.0.0.1:9090", grpc.ServerCredentials.createInsecure()); +console.log("Server listening on 127.0.0.1:9090"); -server.start() +server.start(); ``` -{{< /collapsible >}} -
+{{< /collapsible >}} #### 4. Implementing `StreamIsActive` @@ -351,8 +359,8 @@ Unlike `IsActive`, `StreamIsActive` is called once when KEDA reconciles the `Sca This implementation creates a timer and queries USGS APIs on that timer, effectively ignoring `pollingInterval` set in the scaledObject. Alternatively any other asynchronous event can be used instead of a timer, like an HTTP request, or a network connection. - {{< collapsible "Golang" >}} + ```golang func (e *ExternalScaler) StreamIsActive(scaledObject *pb.ScaledObjectRef, epsServer pb.ExternalScaler_StreamIsActiveServer) error { longitude := scaledObject.ScalerMetadata["longitude"] @@ -380,9 +388,11 @@ func (e *ExternalScaler) StreamIsActive(scaledObject *pb.ScaledObjectRef, epsSer } } ``` + {{< /collapsible >}} {{< collapsible "C#" >}} + ```csharp public override async Task StreamIsActive(ScaledObjectRef request, IServerStreamWriter responseStream, ServerCallContext context) { @@ -410,49 +420,51 @@ public override async Task StreamIsActive(ScaledObjectRef request, IServerStream } } ``` + {{< /collapsible >}} {{< collapsible "Javascript" >}} + ```js server.addService(externalScalerProto.externalscaler.ExternalScaler.service, { // ... streamIsActive: (call, callback) => { - const longitude = call.request.scalerMetadata.longitude - const latitude = call.request.scalerMetadata.latitude + const longitude = call.request.scalerMetadata.longitude; + const latitude = call.request.scalerMetadata.latitude; if (!longitude || !latitude) { callback({ code: grpc.status.INVALID_ARGUMENT, - details: 'longitude and latitude must be specified', - }) + details: "longitude and latitude must be specified", + }); } else { const interval = setInterval(() => { getEarthquakeCount((err, count) => { if (err) { - console.error(err) + console.error(err); } else if (count > 2) { call.write({ result: true, - }) + }); } - }) + }); }, 1000 * 60 * 60); - call.on('end', () => { - clearInterval(interval) - }) + call.on("end", () => { + clearInterval(interval); + }); } - } -}) + }, +}); ``` -{{< /collapsible >}} -
+{{< /collapsible >}} #### 5. Implementing `GetMetricSpec` `GetMetricSpec` returns the `target` value for [the HPA definition for the scaler](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale-walkthrough/). This scaler will define a static target of 10, but the threshold value is often specified in the metadata for other scalers. {{< collapsible "Golang" >}} + ```golang func (e *ExternalScaler) GetMetricSpec(context.Context, *pb.ScaledObjectRef) (*pb.GetMetricSpecResponse, error) { return &pb.GetMetricSpecResponse{ @@ -463,9 +475,11 @@ func (e *ExternalScaler) GetMetricSpec(context.Context, *pb.ScaledObjectRef) (*p }, nil } ``` + {{< /collapsible >}} {{< collapsible "C#" >}} + ```csharp public override async Task GetMetricSpec(ScaledObjectRef request, ServerCallContext context) { @@ -480,31 +494,35 @@ public override async Task GetMetricSpec(ScaledObjectRef return Task.FromResult(resp); } ``` + {{< /collapsible >}} {{< collapsible "Javascript" >}} + ```js server.addService(externalScalerProto.externalscaler.ExternalScaler.service, { // ... getMetricSpec: (call, callback) => { callback(null, { - metricSpecs: [{ - metricName: 'earthquakeThreshold', - targetSize: 10, - }] - }) - } -}) + metricSpecs: [ + { + metricName: "earthquakeThreshold", + targetSize: 10, + }, + ], + }); + }, +}); ``` -{{< /collapsible >}} -
+{{< /collapsible >}} #### 6. Implementing `GetMetrics` `GetMetrics` returns the value of the metric referred to from `GetMetricSpec`, in this example it's `earthquakeThreshold`. {{< collapsible "Golang" >}} + ```golang func (e *ExternalScaler) GetMetrics(_ context.Context, metricRequest *pb.GetMetricsRequest) (*pb.GetMetricsResponse, error) { longitude := metricRequest.ScaledObjectRef.ScalerMetadata["longitude"] @@ -527,9 +545,11 @@ func (e *ExternalScaler) GetMetrics(_ context.Context, metricRequest *pb.GetMetr }, nil } ``` + {{< /collapsible >}} {{< collapsible "C#" >}} + ```csharp public override async Task GetMetrics(GetMetricsRequest request, ServerCallContext context) { @@ -554,38 +574,43 @@ public override async Task GetMetrics(GetMetricsRequest requ return resp; } ``` + {{< /collapsible >}} {{< collapsible "Javascript" >}} + ```js server.addService(externalScalerProto.externalscaler.ExternalScaler.service, { // ... getMetrics: (call, callback) => { - const longitude = call.request.scaledObjectRef.scalerMetadata.longitude - const latitude = call.request.scaledObjectRef.scalerMetadata.latitude + const longitude = call.request.scaledObjectRef.scalerMetadata.longitude; + const latitude = call.request.scaledObjectRef.scalerMetadata.latitude; if (!longitude || !latitude) { callback({ code: grpc.status.INVALID_ARGUMENT, - details: 'longitude and latitude must be specified', - }) + details: "longitude and latitude must be specified", + }); } else { getEarthquakeCount((err, count) => { if (err) { callback({ code: grpc.status.INTERNAL, details: err, - }) + }); } else { callback(null, { - metricValues: [{ - metricName: 'earthquakeThreshold', - metricValue: count, - }] - }) + metricValues: [ + { + metricName: "earthquakeThreshold", + metricValue: count, + }, + ], + }); } - }) + }); } - } -}) + }, +}); ``` + {{< /collapsible >}} diff --git a/content/docs/2.0/migration.md b/content/docs/2.0/migration.md index b7e2cbad5..fe54ad324 100644 --- a/content/docs/2.0/migration.md +++ b/content/docs/2.0/migration.md @@ -11,15 +11,17 @@ Please note that you **can not** run both KEDA v1 and v2 on the same Kubernetes KEDA v2 is using a new API namespace for it's Custom Resources Definitions (CRD): `keda.sh` instead of `keda.k8s.io` and introduces a new Custom Resource for scaling of Jobs. See full details on KEDA Custom Resources [here](../concepts/#custom-resources-crd). Here's an overview of what's changed: + - [Scaling of Deployments](#scaling-of-deployments) - [Scaling of Jobs](#scaling-of-jobs) - [Improved flexibility & usability of trigger metadata](#improved-flexibility--usability-of-trigger-metadata) - [Scalers](#scalers) - [TriggerAuthentication](#triggerauthentication) - ### Scaling of Deployments + In order to scale `Deployments` with KEDA v2, you need to do only a few modifications to existing v1 `ScaledObjects` definitions, so they comply with v2: + - Change the value of `apiVersion` property from `keda.k8s.io/v1alpha1` to `keda.sh/v1alpha1` - Rename property `spec.scaleTargetRef.deploymentName` to `spec.scaleTargetRef.name` - Rename property `spec.scaleTargetRef.containerName` to `spec.scaleTargetRef.envSourceContainerName` @@ -27,22 +29,23 @@ In order to scale `Deployments` with KEDA v2, you need to do only a few modifica Please see the examples below or refer to the full [v2 ScaledObject Specification](../concepts/scaling-deployments/#scaledobject-spec) - **Example of v1 ScaledObject** +**Example of v1 ScaledObject** + ```yaml apiVersion: keda.k8s.io/v1alpha1 kind: ScaledObject metadata: - name: {scaled-object-name} + name: { scaled-object-name } labels: - deploymentName: {deployment-name} + deploymentName: { deployment-name } spec: scaleTargetRef: - deploymentName: {deployment-name} - containerName: {container-name} - pollingInterval: 30 - cooldownPeriod: 300 - minReplicaCount: 0 - maxReplicaCount: 100 + deploymentName: { deployment-name } + containerName: { container-name } + pollingInterval: 30 + cooldownPeriod: 300 + minReplicaCount: 0 + maxReplicaCount: 100 triggers: # {list of triggers to activate the deployment} ``` @@ -50,18 +53,18 @@ spec: **Example of v2 ScaledObject** ```yaml -apiVersion: keda.sh/v1alpha1 # <--- Property value was changed +apiVersion: keda.sh/v1alpha1 # <--- Property value was changed kind: ScaledObject -metadata: # <--- labels.deploymentName is not needed - name: {scaled-object-name} +metadata: # <--- labels.deploymentName is not needed + name: { scaled-object-name } spec: scaleTargetRef: - name: {deployment-name} # <--- Property name was changed - envSourceContainerName: {container-name} # <--- Property name was changed - pollingInterval: 30 - cooldownPeriod: 300 - minReplicaCount: 0 - maxReplicaCount: 100 + name: { deployment-name } # <--- Property name was changed + envSourceContainerName: { container-name } # <--- Property name was changed + pollingInterval: 30 + cooldownPeriod: 300 + minReplicaCount: 0 + maxReplicaCount: 100 triggers: # {list of triggers to activate the deployment} ``` @@ -69,34 +72,36 @@ spec: ### Scaling of Jobs In order to scale `Jobs` with KEDA v2, you need to do only a few modifications to existing v1 `ScaledObjects` definitions, so they comply with v2: + - Change the value of `apiVersion` property from `keda.k8s.io/v1alpha1` to `keda.sh/v1alpha1` - Change the value of `kind` property from `ScaledObject` to `ScaledJob` - Remove property `spec.scaleType` -- Remove properties `spec.cooldownPeriod` and `spec.minReplicaCount` +- Remove properties `spec.cooldownPeriod` and `spec.minReplicaCount` You can configure `successfulJobsHistoryLimit` and `failedJobsHistoryLimit`. They will remove the old job histories automatically. Please see the examples below or refer to the full [v2 ScaledJob Specification](../concepts/scaling-jobs/#scaledjob-spec) **Example of v1 ScaledObject for Jobs scaling** + ```yaml apiVersion: keda.k8s.io/v1alpha1 kind: ScaledObject metadata: - name: {scaled-object-name} + name: { scaled-object-name } spec: scaleType: job jobTargetRef: - parallelism: 1 + parallelism: 1 completions: 1 activeDeadlineSeconds: 600 - backoffLimit: 6 + backoffLimit: 6 template: # {job template} - pollingInterval: 30 - cooldownPeriod: 300 - minReplicaCount: 0 - maxReplicaCount: 100 + pollingInterval: 30 + cooldownPeriod: 300 + minReplicaCount: 0 + maxReplicaCount: 100 triggers: # {list of triggers to create jobs} ``` @@ -104,21 +109,21 @@ spec: **Example of v2 ScaledJob** ```yaml -apiVersion: keda.sh/v1alpha1 # <--- Property value was changed -kind: ScaledJob # <--- Property value was changed +apiVersion: keda.sh/v1alpha1 # <--- Property value was changed +kind: ScaledJob # <--- Property value was changed metadata: - name: {scaled-job-name} -spec: # <--- spec.scaleType is not needed + name: { scaled-job-name } +spec: # <--- spec.scaleType is not needed jobTargetRef: - parallelism: 1 - completions: 1 + parallelism: 1 + completions: 1 activeDeadlineSeconds: 600 backoffLimit: 6 template: # {job template} - pollingInterval: 30 # <--- spec.cooldownPeriod and spec.minReplicaCount are not needed - successfulJobsHistoryLimit: 5 # <--- property is added - failedJobsHistoryLimit: 5 # <--- Property is added + pollingInterval: 30 # <--- spec.cooldownPeriod and spec.minReplicaCount are not needed + successfulJobsHistoryLimit: 5 # <--- property is added + failedJobsHistoryLimit: 5 # <--- Property is added maxReplicaCount: 100 triggers: # {list of triggers to create jobs} @@ -132,68 +137,74 @@ We've introduced more options to configure trigger metadata to give users more f Here's an overview: -| Scaler | 1.x | 2.0 | -|--------|--------|--------| -| `azure-blob` | `connection` (**Default**: `AzureWebJobsStorage`) | `connectionFromEnv` | -| `azure-monitor` | `activeDirectoryClientId`
`activeDirectoryClientPassword` | `activeDirectoryClientId`
`activeDirectoryClientIdFromEnv`
`activeDirectoryClientPasswordFromEnv` | -| `azure-queue` | `connection` (**Default**: AzureWebJobsStorage) | `connectionFromEnv` | -| `azure-servicebus` | `connection` | `connectionFromEnv` | -| `azure-eventhub` | `storageConnection` (**Default**: `AzureWebJobsStorage`)
`connection` (**Default**: `EventHub`) | `storageConnectionFromEnv`
`connectionFromEnv` | -| `aws-cloudwatch` | `awsAccessKeyID` (**Default**: `AWS_ACCESS_KEY_ID`)
`awsSecretAccessKey` (**Default**: `AWS_SECRET_ACCESS_KEY`) | `awsAccessKeyID`
`awsAccessKeyIDFromEnv`
`awsSecretAccessKeyFromEnv` | -| `aws-kinesis-stream` | `awsAccessKeyID` (**Default**: `AWS_ACCESS_KEY_ID`)
`awsSecretAccessKey` (**Default**: `AWS_SECRET_ACCESS_KEY`) | `awsAccessKeyID`
`awsAccessKeyIDFromEnv`
`awsSecretAccessKeyFromEnv` | -| `aws-sqs-queue` | `awsAccessKeyID` (**Default**: `AWS_ACCESS_KEY_ID`)
`awsSecretAccessKey` (**Default**: `AWS_SECRET_ACCESS_KEY`) | `awsAccessKeyID`
`awsAccessKeyIDFromEnv`
`awsSecretAccessKeyFromEnv` | -| `kafka` | _(none)_ | _(none)_ -| `rabbitmq` | `apiHost`
`host` | ~~`apiHost`~~
`host`
`hostFromEnv` | -| `prometheus` | _(none)_ | _(none)_ | -| `cron` | _(none)_ | _(none)_ | -| `redis` | `address`
`host`
`port`
`password` | `address`
`addressFromEnv`
`host`
`hostFromEnv`
~~`port`~~
`passwordFromEnv` -| `redis-streams` | `address`
`host`
`port`
`password` | `address`
`addressFromEnv`
`host`
`hostFromEnv`
~~`port`~~
`passwordFromEnv` -| `gcp-pubsub` | `credentials` | `credentialsFromEnv` | -| `external` | _(any matching value)_ | _(any matching value with `FromEnv` suffix)_ -| `liiklus` | _(none)_ | _(none)_ | -| `stan` | _(none)_ | _(none)_ | -| `huawei-cloudeye` | | _(none)_ | _(none)_ | -| `postgresql` | `connection`
`password` | `connectionFromEnv`
`passwordFromEnv` | -| `mysql` | `connectionString`
`password` | `connectionStringFromEnv`
`passwordFromEnv` | +| Scaler | 1.x | 2.0 | +| -------------------- | --------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------- | -------- | +| `azure-blob` | `connection` (**Default**: `AzureWebJobsStorage`) | `connectionFromEnv` | +| `azure-monitor` | `activeDirectoryClientId` `activeDirectoryClientPassword` | `activeDirectoryClientId` `activeDirectoryClientIdFromEnv` `activeDirectoryClientPasswordFromEnv` | +| `azure-queue` | `connection` (**Default**: AzureWebJobsStorage) | `connectionFromEnv` | +| `azure-servicebus` | `connection` | `connectionFromEnv` | +| `azure-eventhub` | `storageConnection` (**Default**: `AzureWebJobsStorage`) `connection` (**Default**: `EventHub`) | `storageConnectionFromEnv` `connectionFromEnv` | +| `aws-cloudwatch` | `awsAccessKeyID` (**Default**: `AWS_ACCESS_KEY_ID`) `awsSecretAccessKey` (**Default**: `AWS_SECRET_ACCESS_KEY`) | `awsAccessKeyID` `awsAccessKeyIDFromEnv` `awsSecretAccessKeyFromEnv` | +| `aws-kinesis-stream` | `awsAccessKeyID` (**Default**: `AWS_ACCESS_KEY_ID`) `awsSecretAccessKey` (**Default**: `AWS_SECRET_ACCESS_KEY`) | `awsAccessKeyID` `awsAccessKeyIDFromEnv` `awsSecretAccessKeyFromEnv` | +| `aws-sqs-queue` | `awsAccessKeyID` (**Default**: `AWS_ACCESS_KEY_ID`) `awsSecretAccessKey` (**Default**: `AWS_SECRET_ACCESS_KEY`) | `awsAccessKeyID` `awsAccessKeyIDFromEnv` `awsSecretAccessKeyFromEnv` | +| `kafka` | _(none)_ | _(none)_ | +| `rabbitmq` | `apiHost` `host` | ~~`apiHost`~~ `host` `hostFromEnv` | +| `prometheus` | _(none)_ | _(none)_ | +| `cron` | _(none)_ | _(none)_ | +| `redis` | `address` `host` `port` `password` | `address` `addressFromEnv` `host` `hostFromEnv` ~~`port`~~ `passwordFromEnv` | +| `redis-streams` | `address` `host` `port` `password` | `address` `addressFromEnv` `host` `hostFromEnv` ~~`port`~~ `passwordFromEnv` | +| `gcp-pubsub` | `credentials` | `credentialsFromEnv` | +| `external` | _(any matching value)_ | _(any matching value with `FromEnv` suffix)_ | +| `liiklus` | _(none)_ | _(none)_ | +| `stan` | _(none)_ | _(none)_ | +| `huawei-cloudeye` | | _(none)_ | _(none)_ | +| `postgresql` | `connection` `password` | `connectionFromEnv` `passwordFromEnv` | +| `mysql` | `connectionString` `password` | `connectionStringFromEnv` `passwordFromEnv` | ### Scalers **Azure Service Bus** - - `queueLength` was renamed to `messageCount` + +- `queueLength` was renamed to `messageCount` **Kafka** - - `authMode` property was replaced with `sasl` and `tls` properties. Please refer [documentation](../scalers/apache-kafka/#authentication-parameters) for Kafka Authentication Parameters details. + +- `authMode` property was replaced with `sasl` and `tls` properties. Please refer [documentation](../scalers/apache-kafka/#authentication-parameters) for Kafka Authentication Parameters details. **RabbitMQ** In KEDA 2.0 the RabbitMQ scaler has only `host` parameter, and the protocol for communication can be specified by -`protocol` (http or amqp). The default value is `amqp`. The behavior changes only for scalers that were using HTTP -protocol. +`protocol` (http or amqp). The default value is `amqp`. The behavior changes only for scalers that were using HTTP +protocol. Example of RabbitMQ trigger before 2.0: + ```yaml triggers: -- type: rabbitmq - metadata: - queueLength: '20' - queueName: testqueue - includeUnacked: 'true' - apiHost: 'https://guest:password@localhost:443/vhostname' -``` + - type: rabbitmq + metadata: + queueLength: "20" + queueName: testqueue + includeUnacked: "true" + apiHost: "https://guest:password@localhost:443/vhostname" +``` The same trigger in 2.0: + ```yaml triggers: -- type: rabbitmq - metadata: - queueLength: '20' - queueName: testqueue - protocol: 'http' - host: 'https://guest:password@localhost:443/vhostname' -``` + - type: rabbitmq + metadata: + queueLength: "20" + queueName: testqueue + protocol: "http" + host: "https://guest:password@localhost:443/vhostname" +``` ### TriggerAuthentication + In order to use Authentication via `TriggerAuthentication` with KEDA v2, you need to change: + - Change the value of `apiVersion` property from `keda.k8s.io/v1alpha1` to `keda.sh/v1alpha1` For more details please refer to the full diff --git a/content/docs/2.0/operate/cluster.md b/content/docs/2.0/operate/cluster.md index 96925a55e..3620e816a 100644 --- a/content/docs/2.0/operate/cluster.md +++ b/content/docs/2.0/operate/cluster.md @@ -15,7 +15,7 @@ KEDA is designed, tested and supported to be run on any Kubernetes cluster that The KEDA runtime require the following resources in a production-ready setup: | Deployment | CPU | Memory | -|----------------|-------------------------|-------------------------------| +| -------------- | ----------------------- | ----------------------------- | | Operator | Limit: 1, Request: 100m | Limit: 1000Mi, Request: 100Mi | | Metrics Server | Limit: 1, Request: 100m | Limit: 1000Mi, Request: 100Mi | @@ -40,7 +40,7 @@ KEDA does not provide full support for high-availability due to upstream limitat Here is an overview of all KEDA deployments and the HA notes: -| Deployment | Support Replicas | Note | -| -------------- | ---------------- |---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Deployment | Support Replicas | Note | +| -------------- | ---------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | Metrics Server | 1 | You can run multiple replicas of our metrics sever, and it is recommended to add the `--enable-aggregator-routing=true` CLI flag to the kube-apiserver so that requests sent to our metrics servers are load balanced. However, [you can only run one active metric server in a Kubernetes cluster serving external.metrics.k8s.io](https://github.com/kubernetes-sigs/custom-metrics-apiserver/issues/70) which has to be the KEDA metric server. | -| Operator | 1 | While you can run multiple replicas of our operator, only one operator instance will be active. The rest will be standing by, which may reduce downtime during a failure. Multiple replicas will not improve the performance of KEDA, it could only reduce a downtime during a failover.

This is only supported as of KEDA v2.6 if you are using our Helm chart. | +| Operator | 1 | While you can run multiple replicas of our operator, only one operator instance will be active. The rest will be standing by, which may reduce downtime during a failure. Multiple replicas will not improve the performance of KEDA, it could only reduce a downtime during a failover.This is only supported as of KEDA v2.6 if you are using our Helm chart. | diff --git a/content/docs/2.1/concepts/external-scalers.md b/content/docs/2.1/concepts/external-scalers.md index 27ed9e0c6..1945394d3 100644 --- a/content/docs/2.1/concepts/external-scalers.md +++ b/content/docs/2.1/concepts/external-scalers.md @@ -33,15 +33,15 @@ type PushScaler interface { ``` The `Scaler` interface defines 4 methods: + - `IsActive` is called on `pollingInterval` defined in the ScaledObject/ScaledJob CRDs and scaling to 1 happens if this returns true. - `Close` is called to allow the scaler to clean up connections or other resources. - `GetMetricSpec` returns the target value for the HPA definition for the scaler. For more details refer to [Implementing `GetMetricSpec`](#5-implementing-getmetricspec). - `GetMetrics` returns the value of the metric referred to from `GetMetricSpec`. For more details refer to [Implementing `GetMetrics`](#6-implementing-getmetrics). -> Refer to the [HPA docs](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale-walkthrough/) for how HPA calculates `replicaCount` based on metric value and target value. KEDA uses the metric target type `AverageValue` for external metrics. This will cause the metric value returned by the external scaler to be divided by the number of replicas. + > Refer to the [HPA docs](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale-walkthrough/) for how HPA calculates `replicaCount` based on metric value and target value. KEDA uses the metric target type `AverageValue` for external metrics. This will cause the metric value returned by the external scaler to be divided by the number of replicas. The `PushScaler` interface adds `Run` method. The `Run` method receives a push channel (`active`), that the scaler can push `true` to any time to force scaling action independently from `pollingInterval`. - ### External Scaler GRPC interface KEDA comes with 2 external scalers [`external`](../scalers/external.md) and [`external-push`](../scalers/external-push.md). @@ -62,6 +62,7 @@ service ExternalScaler { - `StreamIsActive` maps to the `Run` method on the `PushScaler` interface. Few things to notice: + - Lack of `Close` method as the GRPC connection defines the lifetime of the scaler - `IsActive`, `StreamIsActive`, and `GetMetricsSpec` are called with a `ScaledObjectRef` that contains the scaledObject name/namespace as well as the content of `metadata` defined in the trigger. @@ -125,6 +126,7 @@ go mod init example.com/external-scaler/sample mkdir externalscaler protoc externalscaler.proto --go_out=plugins=grpc:externalscaler ``` + {{< /collapsible >}} {{< collapsible "C#" >}} @@ -161,9 +163,8 @@ mkdir Services ```bash npm install --save grpc request ``` -{{< /collapsible >}} -
+{{< /collapsible >}} #### 3. Implementing `IsActive` @@ -193,6 +194,7 @@ spec: Full implementation can be found here: https://github.com/kedacore/external-scaler-samples `main.go` + ```golang func (e *ExternalScaler) IsActive(ctx context.Context, scaledObject *pb.ScaledObjectRef) (*pb.IsActiveResponse, error) { // request.Scalermetadata contains the `metadata` defined in the ScaledObject @@ -239,12 +241,14 @@ func (e *ExternalScaler) IsActive(ctx context.Context, scaledObject *pb.ScaledOb }, nil } ``` + {{< /collapsible >}} {{< collapsible "C#" >}} Full implementation can be found here: https://github.com/kedacore/external-scaler-samples `Services/ExternalScalerService.cs` + ```csharp public class ExternalScalerService : ExternalScaler.ExternalScalerBase { @@ -277,73 +281,77 @@ public class ExternalScalerService : ExternalScaler.ExternalScalerBase } } ``` + {{< /collapsible >}} {{< collapsible "Javascript" >}} `index.js` + ```js -const grpc = require('grpc') -const request = require('request') -const externalScalerProto = grpc.load('externalscaler.proto') +const grpc = require("grpc"); +const request = require("request"); +const externalScalerProto = grpc.load("externalscaler.proto"); -const server = new grpc.Server() +const server = new grpc.Server(); server.addService(externalScalerProto.externalscaler.ExternalScaler.service, { isActive: (call, callback) => { - const longitude = call.request.scalerMetadata.longitude - const latitude = call.request.scalerMetadata.latitude + const longitude = call.request.scalerMetadata.longitude; + const latitude = call.request.scalerMetadata.latitude; if (!longitude || !latitude) { callback({ code: grpc.status.INVALID_ARGUMENT, - details: 'longitude and latitude must be specified', - }) + details: "longitude and latitude must be specified", + }); } else { - const now = new Date() - const yesterday = new Date(new Date().setDate(new Date().getDate()-1)); - - const startTime = `${yesterday.getUTCFullYear()}-${yesterday.getUTCMonth()}-${yesterday.getUTCDay()}` - const endTime = `${now.getUTCFullYear()}-${now.getUTCMonth()}-${now.getUTCDay()}` - const radiusKm = 500 - const query = `format=geojson&starttime=${startTime}&endtime=${endTime}&longitude=${longitude}&latitude=${latitude}&maxradiuskm=${radiusKm}` - - request.get({ - url: `https://earthquake.usgs.gov/fdsnws/event/1/query?${query}`, - json: true, - }, (err, resp, data) => { - if (err) { - callback({ - code: grpc.status.INTERNAL, - details: err, - }) - } else if (resp.statusCode !== 200) { - callback({ - code: grpc.status.INTERNAL, - details: `expected status 200, got ${resp.statusCode}` - }) - } else { - // count how many earthquakes with mag > 1.0 - let count = 0 - data.features.forEach(i => { - if (i.properties.mag > 1.0) { - count++ - } - }) - callback(null, { - result: count > 2, - }) + const now = new Date(); + const yesterday = new Date(new Date().setDate(new Date().getDate() - 1)); + + const startTime = `${yesterday.getUTCFullYear()}-${yesterday.getUTCMonth()}-${yesterday.getUTCDay()}`; + const endTime = `${now.getUTCFullYear()}-${now.getUTCMonth()}-${now.getUTCDay()}`; + const radiusKm = 500; + const query = `format=geojson&starttime=${startTime}&endtime=${endTime}&longitude=${longitude}&latitude=${latitude}&maxradiuskm=${radiusKm}`; + + request.get( + { + url: `https://earthquake.usgs.gov/fdsnws/event/1/query?${query}`, + json: true, + }, + (err, resp, data) => { + if (err) { + callback({ + code: grpc.status.INTERNAL, + details: err, + }); + } else if (resp.statusCode !== 200) { + callback({ + code: grpc.status.INTERNAL, + details: `expected status 200, got ${resp.statusCode}`, + }); + } else { + // count how many earthquakes with mag > 1.0 + let count = 0; + data.features.forEach((i) => { + if (i.properties.mag > 1.0) { + count++; + } + }); + callback(null, { + result: count > 2, + }); + } } - }) + ); } - } -}) + }, +}); -server.bind('127.0.0.1:9090', grpc.ServerCredentials.createInsecure()) -console.log('Server listening on 127.0.0.1:9090') +server.bind("127.0.0.1:9090", grpc.ServerCredentials.createInsecure()); +console.log("Server listening on 127.0.0.1:9090"); -server.start() +server.start(); ``` -{{< /collapsible >}} -
+{{< /collapsible >}} #### 4. Implementing `StreamIsActive` @@ -351,8 +359,8 @@ Unlike `IsActive`, `StreamIsActive` is called once when KEDA reconciles the `Sca This implementation creates a timer and queries USGS APIs on that timer, effectively ignoring `pollingInterval` set in the scaledObject. Alternatively any other asynchronous event can be used instead of a timer, like an HTTP request, or a network connection. - {{< collapsible "Golang" >}} + ```golang func (e *ExternalScaler) StreamIsActive(scaledObject *pb.ScaledObjectRef, epsServer pb.ExternalScaler_StreamIsActiveServer) error { longitude := scaledObject.ScalerMetadata["longitude"] @@ -380,9 +388,11 @@ func (e *ExternalScaler) StreamIsActive(scaledObject *pb.ScaledObjectRef, epsSer } } ``` + {{< /collapsible >}} {{< collapsible "C#" >}} + ```csharp public override async Task StreamIsActive(ScaledObjectRef request, IServerStreamWriter responseStream, ServerCallContext context) { @@ -410,49 +420,51 @@ public override async Task StreamIsActive(ScaledObjectRef request, IServerStream } } ``` + {{< /collapsible >}} {{< collapsible "Javascript" >}} + ```js server.addService(externalScalerProto.externalscaler.ExternalScaler.service, { // ... streamIsActive: (call, callback) => { - const longitude = call.request.scalerMetadata.longitude - const latitude = call.request.scalerMetadata.latitude + const longitude = call.request.scalerMetadata.longitude; + const latitude = call.request.scalerMetadata.latitude; if (!longitude || !latitude) { callback({ code: grpc.status.INVALID_ARGUMENT, - details: 'longitude and latitude must be specified', - }) + details: "longitude and latitude must be specified", + }); } else { const interval = setInterval(() => { getEarthquakeCount((err, count) => { if (err) { - console.error(err) + console.error(err); } else if (count > 2) { call.write({ result: true, - }) + }); } - }) + }); }, 1000 * 60 * 60); - call.on('end', () => { - clearInterval(interval) - }) + call.on("end", () => { + clearInterval(interval); + }); } - } -}) + }, +}); ``` -{{< /collapsible >}} -
+{{< /collapsible >}} #### 5. Implementing `GetMetricSpec` `GetMetricSpec` returns the `target` value for [the HPA definition for the scaler](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale-walkthrough/). This scaler will define a static target of 10, but the threshold value is often specified in the metadata for other scalers. {{< collapsible "Golang" >}} + ```golang func (e *ExternalScaler) GetMetricSpec(context.Context, *pb.ScaledObjectRef) (*pb.GetMetricSpecResponse, error) { return &pb.GetMetricSpecResponse{ @@ -463,9 +475,11 @@ func (e *ExternalScaler) GetMetricSpec(context.Context, *pb.ScaledObjectRef) (*p }, nil } ``` + {{< /collapsible >}} {{< collapsible "C#" >}} + ```csharp public override async Task GetMetricSpec(ScaledObjectRef request, ServerCallContext context) { @@ -480,31 +494,35 @@ public override async Task GetMetricSpec(ScaledObjectRef return Task.FromResult(resp); } ``` + {{< /collapsible >}} {{< collapsible "Javascript" >}} + ```js server.addService(externalScalerProto.externalscaler.ExternalScaler.service, { // ... getMetricSpec: (call, callback) => { callback(null, { - metricSpecs: [{ - metricName: 'earthquakeThreshold', - targetSize: 10, - }] - }) - } -}) + metricSpecs: [ + { + metricName: "earthquakeThreshold", + targetSize: 10, + }, + ], + }); + }, +}); ``` -{{< /collapsible >}} -
+{{< /collapsible >}} #### 6. Implementing `GetMetrics` `GetMetrics` returns the value of the metric referred to from `GetMetricSpec`, in this example it's `earthquakeThreshold`. {{< collapsible "Golang" >}} + ```golang func (e *ExternalScaler) GetMetrics(_ context.Context, metricRequest *pb.GetMetricsRequest) (*pb.GetMetricsResponse, error) { longitude := metricRequest.ScaledObjectRef.ScalerMetadata["longitude"] @@ -527,9 +545,11 @@ func (e *ExternalScaler) GetMetrics(_ context.Context, metricRequest *pb.GetMetr }, nil } ``` + {{< /collapsible >}} {{< collapsible "C#" >}} + ```csharp public override async Task GetMetrics(GetMetricsRequest request, ServerCallContext context) { @@ -554,38 +574,43 @@ public override async Task GetMetrics(GetMetricsRequest requ return resp; } ``` + {{< /collapsible >}} {{< collapsible "Javascript" >}} + ```js server.addService(externalScalerProto.externalscaler.ExternalScaler.service, { // ... getMetrics: (call, callback) => { - const longitude = call.request.scaledObjectRef.scalerMetadata.longitude - const latitude = call.request.scaledObjectRef.scalerMetadata.latitude + const longitude = call.request.scaledObjectRef.scalerMetadata.longitude; + const latitude = call.request.scaledObjectRef.scalerMetadata.latitude; if (!longitude || !latitude) { callback({ code: grpc.status.INVALID_ARGUMENT, - details: 'longitude and latitude must be specified', - }) + details: "longitude and latitude must be specified", + }); } else { getEarthquakeCount((err, count) => { if (err) { callback({ code: grpc.status.INTERNAL, details: err, - }) + }); } else { callback(null, { - metricValues: [{ - metricName: 'earthquakeThreshold', - metricValue: count, - }] - }) + metricValues: [ + { + metricName: "earthquakeThreshold", + metricValue: count, + }, + ], + }); } - }) + }); } - } -}) + }, +}); ``` + {{< /collapsible >}} diff --git a/content/docs/2.1/migration.md b/content/docs/2.1/migration.md index 5ee8a6455..29e8408c8 100644 --- a/content/docs/2.1/migration.md +++ b/content/docs/2.1/migration.md @@ -11,15 +11,17 @@ Please note that you **can not** run both KEDA v1 and v2 on the same Kubernetes KEDA v2 is using a new API namespace for its Custom Resources Definitions (CRD): `keda.sh` instead of `keda.k8s.io` and introduces a new Custom Resource for scaling of Jobs. See full details on KEDA Custom Resources [here](../concepts/#custom-resources-crd). Here's an overview of what's changed: + - [Scaling of Deployments](#scaling-of-deployments) - [Scaling of Jobs](#scaling-of-jobs) - [Improved flexibility & usability of trigger metadata](#improved-flexibility--usability-of-trigger-metadata) - [Scalers](#scalers) - [TriggerAuthentication](#triggerauthentication) - ### Scaling of Deployments + In order to scale `Deployments` with KEDA v2, you need to do only a few modifications to existing v1 `ScaledObjects` definitions, so they comply with v2: + - Change the value of `apiVersion` property from `keda.k8s.io/v1alpha1` to `keda.sh/v1alpha1` - Rename property `spec.scaleTargetRef.deploymentName` to `spec.scaleTargetRef.name` - Rename property `spec.scaleTargetRef.containerName` to `spec.scaleTargetRef.envSourceContainerName` @@ -27,22 +29,23 @@ In order to scale `Deployments` with KEDA v2, you need to do only a few modifica Please see the examples below or refer to the full [v2 ScaledObject Specification](../concepts/scaling-deployments/#scaledobject-spec) - **Example of v1 ScaledObject** +**Example of v1 ScaledObject** + ```yaml apiVersion: keda.k8s.io/v1alpha1 kind: ScaledObject metadata: - name: {scaled-object-name} + name: { scaled-object-name } labels: - deploymentName: {deployment-name} + deploymentName: { deployment-name } spec: scaleTargetRef: - deploymentName: {deployment-name} - containerName: {container-name} - pollingInterval: 30 - cooldownPeriod: 300 - minReplicaCount: 0 - maxReplicaCount: 100 + deploymentName: { deployment-name } + containerName: { container-name } + pollingInterval: 30 + cooldownPeriod: 300 + minReplicaCount: 0 + maxReplicaCount: 100 triggers: # {list of triggers to activate the deployment} ``` @@ -50,18 +53,18 @@ spec: **Example of v2 ScaledObject** ```yaml -apiVersion: keda.sh/v1alpha1 # <--- Property value was changed +apiVersion: keda.sh/v1alpha1 # <--- Property value was changed kind: ScaledObject -metadata: # <--- labels.deploymentName is not needed - name: {scaled-object-name} +metadata: # <--- labels.deploymentName is not needed + name: { scaled-object-name } spec: scaleTargetRef: - name: {deployment-name} # <--- Property name was changed - envSourceContainerName: {container-name} # <--- Property name was changed - pollingInterval: 30 - cooldownPeriod: 300 - minReplicaCount: 0 - maxReplicaCount: 100 + name: { deployment-name } # <--- Property name was changed + envSourceContainerName: { container-name } # <--- Property name was changed + pollingInterval: 30 + cooldownPeriod: 300 + minReplicaCount: 0 + maxReplicaCount: 100 triggers: # {list of triggers to activate the deployment} ``` @@ -69,34 +72,36 @@ spec: ### Scaling of Jobs In order to scale `Jobs` with KEDA v2, you need to do only a few modifications to existing v1 `ScaledObjects` definitions, so they comply with v2: + - Change the value of `apiVersion` property from `keda.k8s.io/v1alpha1` to `keda.sh/v1alpha1` - Change the value of `kind` property from `ScaledObject` to `ScaledJob` - Remove property `spec.scaleType` -- Remove properties `spec.cooldownPeriod` and `spec.minReplicaCount` +- Remove properties `spec.cooldownPeriod` and `spec.minReplicaCount` You can configure `successfulJobsHistoryLimit` and `failedJobsHistoryLimit`. They will remove the old job histories automatically. Please see the examples below or refer to the full [v2 ScaledJob Specification](../concepts/scaling-jobs/#scaledjob-spec) **Example of v1 ScaledObject for Jobs scaling** + ```yaml apiVersion: keda.k8s.io/v1alpha1 kind: ScaledObject metadata: - name: {scaled-object-name} + name: { scaled-object-name } spec: scaleType: job jobTargetRef: - parallelism: 1 + parallelism: 1 completions: 1 activeDeadlineSeconds: 600 - backoffLimit: 6 + backoffLimit: 6 template: # {job template} - pollingInterval: 30 - cooldownPeriod: 300 - minReplicaCount: 0 - maxReplicaCount: 100 + pollingInterval: 30 + cooldownPeriod: 300 + minReplicaCount: 0 + maxReplicaCount: 100 triggers: # {list of triggers to create jobs} ``` @@ -104,21 +109,21 @@ spec: **Example of v2 ScaledJob** ```yaml -apiVersion: keda.sh/v1alpha1 # <--- Property value was changed -kind: ScaledJob # <--- Property value was changed +apiVersion: keda.sh/v1alpha1 # <--- Property value was changed +kind: ScaledJob # <--- Property value was changed metadata: - name: {scaled-job-name} -spec: # <--- spec.scaleType is not needed + name: { scaled-job-name } +spec: # <--- spec.scaleType is not needed jobTargetRef: - parallelism: 1 - completions: 1 + parallelism: 1 + completions: 1 activeDeadlineSeconds: 600 backoffLimit: 6 template: # {job template} - pollingInterval: 30 # <--- spec.cooldownPeriod and spec.minReplicaCount are not needed - successfulJobsHistoryLimit: 5 # <--- property is added - failedJobsHistoryLimit: 5 # <--- Property is added + pollingInterval: 30 # <--- spec.cooldownPeriod and spec.minReplicaCount are not needed + successfulJobsHistoryLimit: 5 # <--- property is added + failedJobsHistoryLimit: 5 # <--- Property is added maxReplicaCount: 100 triggers: # {list of triggers to create jobs} @@ -132,68 +137,74 @@ We've introduced more options to configure trigger metadata to give users more f Here's an overview: -| Scaler | 1.x | 2.0 | -|--------|--------|--------| -| `azure-blob` | `connection` (**Default**: `AzureWebJobsStorage`) | `connectionFromEnv` | -| `azure-monitor` | `activeDirectoryClientId`
`activeDirectoryClientPassword` | `activeDirectoryClientId`
`activeDirectoryClientIdFromEnv`
`activeDirectoryClientPasswordFromEnv` | -| `azure-queue` | `connection` (**Default**: AzureWebJobsStorage) | `connectionFromEnv` | -| `azure-servicebus` | `connection` | `connectionFromEnv` | -| `azure-eventhub` | `storageConnection` (**Default**: `AzureWebJobsStorage`)
`connection` (**Default**: `EventHub`) | `storageConnectionFromEnv`
`connectionFromEnv` | -| `aws-cloudwatch` | `awsAccessKeyID` (**Default**: `AWS_ACCESS_KEY_ID`)
`awsSecretAccessKey` (**Default**: `AWS_SECRET_ACCESS_KEY`) | `awsAccessKeyID`
`awsAccessKeyIDFromEnv`
`awsSecretAccessKeyFromEnv` | -| `aws-kinesis-stream` | `awsAccessKeyID` (**Default**: `AWS_ACCESS_KEY_ID`)
`awsSecretAccessKey` (**Default**: `AWS_SECRET_ACCESS_KEY`) | `awsAccessKeyID`
`awsAccessKeyIDFromEnv`
`awsSecretAccessKeyFromEnv` | -| `aws-sqs-queue` | `awsAccessKeyID` (**Default**: `AWS_ACCESS_KEY_ID`)
`awsSecretAccessKey` (**Default**: `AWS_SECRET_ACCESS_KEY`) | `awsAccessKeyID`
`awsAccessKeyIDFromEnv`
`awsSecretAccessKeyFromEnv` | -| `kafka` | _(none)_ | _(none)_ -| `rabbitmq` | `apiHost`
`host` | ~~`apiHost`~~
`host`
`hostFromEnv` | -| `prometheus` | _(none)_ | _(none)_ | -| `cron` | _(none)_ | _(none)_ | -| `redis` | `address`
`host`
`port`
`password` | `address`
`addressFromEnv`
`host`
`hostFromEnv`
~~`port`~~
`passwordFromEnv` -| `redis-streams` | `address`
`host`
`port`
`password` | `address`
`addressFromEnv`
`host`
`hostFromEnv`
~~`port`~~
`passwordFromEnv` -| `gcp-pubsub` | `credentials` | `credentialsFromEnv` | -| `external` | _(any matching value)_ | _(any matching value with `FromEnv` suffix)_ -| `liiklus` | _(none)_ | _(none)_ | -| `stan` | _(none)_ | _(none)_ | -| `huawei-cloudeye` | | _(none)_ | _(none)_ | -| `postgresql` | `connection`
`password` | `connectionFromEnv`
`passwordFromEnv` | -| `mysql` | `connectionString`
`password` | `connectionStringFromEnv`
`passwordFromEnv` | +| Scaler | 1.x | 2.0 | +| -------------------- | --------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------- | -------- | +| `azure-blob` | `connection` (**Default**: `AzureWebJobsStorage`) | `connectionFromEnv` | +| `azure-monitor` | `activeDirectoryClientId` `activeDirectoryClientPassword` | `activeDirectoryClientId` `activeDirectoryClientIdFromEnv` `activeDirectoryClientPasswordFromEnv` | +| `azure-queue` | `connection` (**Default**: AzureWebJobsStorage) | `connectionFromEnv` | +| `azure-servicebus` | `connection` | `connectionFromEnv` | +| `azure-eventhub` | `storageConnection` (**Default**: `AzureWebJobsStorage`) `connection` (**Default**: `EventHub`) | `storageConnectionFromEnv` `connectionFromEnv` | +| `aws-cloudwatch` | `awsAccessKeyID` (**Default**: `AWS_ACCESS_KEY_ID`) `awsSecretAccessKey` (**Default**: `AWS_SECRET_ACCESS_KEY`) | `awsAccessKeyID` `awsAccessKeyIDFromEnv` `awsSecretAccessKeyFromEnv` | +| `aws-kinesis-stream` | `awsAccessKeyID` (**Default**: `AWS_ACCESS_KEY_ID`) `awsSecretAccessKey` (**Default**: `AWS_SECRET_ACCESS_KEY`) | `awsAccessKeyID` `awsAccessKeyIDFromEnv` `awsSecretAccessKeyFromEnv` | +| `aws-sqs-queue` | `awsAccessKeyID` (**Default**: `AWS_ACCESS_KEY_ID`) `awsSecretAccessKey` (**Default**: `AWS_SECRET_ACCESS_KEY`) | `awsAccessKeyID` `awsAccessKeyIDFromEnv` `awsSecretAccessKeyFromEnv` | +| `kafka` | _(none)_ | _(none)_ | +| `rabbitmq` | `apiHost` `host` | ~~`apiHost`~~ `host` `hostFromEnv` | +| `prometheus` | _(none)_ | _(none)_ | +| `cron` | _(none)_ | _(none)_ | +| `redis` | `address` `host` `port` `password` | `address` `addressFromEnv` `host` `hostFromEnv` ~~`port`~~ `passwordFromEnv` | +| `redis-streams` | `address` `host` `port` `password` | `address` `addressFromEnv` `host` `hostFromEnv` ~~`port`~~ `passwordFromEnv` | +| `gcp-pubsub` | `credentials` | `credentialsFromEnv` | +| `external` | _(any matching value)_ | _(any matching value with `FromEnv` suffix)_ | +| `liiklus` | _(none)_ | _(none)_ | +| `stan` | _(none)_ | _(none)_ | +| `huawei-cloudeye` | | _(none)_ | _(none)_ | +| `postgresql` | `connection` `password` | `connectionFromEnv` `passwordFromEnv` | +| `mysql` | `connectionString` `password` | `connectionStringFromEnv` `passwordFromEnv` | ### Scalers **Azure Service Bus** - - `queueLength` was renamed to `messageCount` + +- `queueLength` was renamed to `messageCount` **Kafka** - - `authMode` property was replaced with `sasl` and `tls` properties. Please refer [documentation](../scalers/apache-kafka/#authentication-parameters) for Kafka Authentication Parameters details. + +- `authMode` property was replaced with `sasl` and `tls` properties. Please refer [documentation](../scalers/apache-kafka/#authentication-parameters) for Kafka Authentication Parameters details. **RabbitMQ** In KEDA 2.0 the RabbitMQ scaler has only `host` parameter, and the protocol for communication can be specified by -`protocol` (http or amqp). The default value is `amqp`. The behavior changes only for scalers that were using HTTP -protocol. +`protocol` (http or amqp). The default value is `amqp`. The behavior changes only for scalers that were using HTTP +protocol. Example of RabbitMQ trigger before 2.0: + ```yaml triggers: -- type: rabbitmq - metadata: - queueLength: '20' - queueName: testqueue - includeUnacked: 'true' - apiHost: 'https://guest:password@localhost:443/vhostname' -``` + - type: rabbitmq + metadata: + queueLength: "20" + queueName: testqueue + includeUnacked: "true" + apiHost: "https://guest:password@localhost:443/vhostname" +``` The same trigger in 2.0: + ```yaml triggers: -- type: rabbitmq - metadata: - queueLength: '20' - queueName: testqueue - protocol: 'http' - host: 'https://guest:password@localhost:443/vhostname' -``` + - type: rabbitmq + metadata: + queueLength: "20" + queueName: testqueue + protocol: "http" + host: "https://guest:password@localhost:443/vhostname" +``` ### TriggerAuthentication + In order to use Authentication via `TriggerAuthentication` with KEDA v2, you need to change: + - Change the value of `apiVersion` property from `keda.k8s.io/v1alpha1` to `keda.sh/v1alpha1` For more details please refer to the full diff --git a/content/docs/2.1/operate/cluster.md b/content/docs/2.1/operate/cluster.md index c4ccbb85f..fab8edc37 100644 --- a/content/docs/2.1/operate/cluster.md +++ b/content/docs/2.1/operate/cluster.md @@ -15,7 +15,7 @@ KEDA is designed, tested and supported to be run on any Kubernetes cluster that The KEDA runtime require the following resources in a production-ready setup: | Deployment | CPU | Memory | -|----------------|-------------------------|-------------------------------| +| -------------- | ----------------------- | ----------------------------- | | Operator | Limit: 1, Request: 100m | Limit: 1000Mi, Request: 100Mi | | Metrics Server | Limit: 1, Request: 100m | Limit: 1000Mi, Request: 100Mi | @@ -40,10 +40,10 @@ KEDA does not provide full support for high-availability due to upstream limitat Here is an overview of all KEDA deployments and the HA notes: -| Deployment | Support Replicas | Note | -| -------------- | ---------------- |---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Deployment | Support Replicas | Note | +| -------------- | ---------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | Metrics Server | 1 | You can run multiple replicas of our metrics sever, and it is recommended to add the `--enable-aggregator-routing=true` CLI flag to the kube-apiserver so that requests sent to our metrics servers are load balanced. However, [you can only run one active metric server in a Kubernetes cluster serving external.metrics.k8s.io](https://github.com/kubernetes-sigs/custom-metrics-apiserver/issues/70) which has to be the KEDA metric server. | -| Operator | 2 | While you can run multiple replicas of our operator, only one operator instance will be active. The rest will be standing by, which may reduce downtime during a failure. Multiple replicas will not improve the performance of KEDA, it could only reduce a downtime during a failover.

This is only supported as of KEDA v2.6 if you are using our Helm chart. | +| Operator | 2 | While you can run multiple replicas of our operator, only one operator instance will be active. The rest will be standing by, which may reduce downtime during a failure. Multiple replicas will not improve the performance of KEDA, it could only reduce a downtime during a failover.This is only supported as of KEDA v2.6 if you are using our Helm chart. | ## HTTP Timeouts diff --git a/content/docs/2.10/concepts/external-scalers.md b/content/docs/2.10/concepts/external-scalers.md index 5badde098..aa00a18a0 100644 --- a/content/docs/2.10/concepts/external-scalers.md +++ b/content/docs/2.10/concepts/external-scalers.md @@ -9,7 +9,7 @@ Built-in scalers run in the KEDA process/pod, while external scalers require an This document describes the external scaler interfaces and how to implement them in Go, Node, and .NET; however for more details on GRPC refer to [the official GRPC documentation](https://grpc.io/docs/) ->Want to learn about existing external scalers? Explore our [external scaler community](https://github.com/kedacore/external-scalers). +> Want to learn about existing external scalers? Explore our [external scaler community](https://github.com/kedacore/external-scalers). ## Overview @@ -45,7 +45,7 @@ The `Scaler` interface defines 3 methods: - `GetMetricSpecForScaling` returns the target value for the HPA definition for the scaler. For more details refer to [Implementing `GetMetricSpec`](#5-implementing-getmetricspec). - `GetMetricsAndActivity` is called on `pollingInterval` and. When activity returns `true`, KEDA will scale to what is returned by the metric limited by `maxReplicaCount` on the ScaledObject/ScaledJob. When `false` is returned, KEDA will scale to `minReplicaCount` or optionally `idleReplicaCount`. More details around the defaults and how these options work together can be found on the [ScaledObjectSpec](https://keda.sh/docs/latest/concepts/scaling-deployments/#scaledobject-spec). -> Refer to the [HPA docs](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale-walkthrough/) for how HPA calculates `replicaCount` based on metric value and target value. KEDA supports both `AverageValue` and `Value` metric target types for external metrics. When `AverageValue` (the default metric type) is used, the metric value returned by the external scaler will be divided by the number of replicas. + > Refer to the [HPA docs](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale-walkthrough/) for how HPA calculates `replicaCount` based on metric value and target value. KEDA supports both `AverageValue` and `Value` metric target types for external metrics. When `AverageValue` (the default metric type) is used, the metric value returned by the external scaler will be divided by the number of replicas. The `PushScaler` interface adds a `Run` method. This method receives a push channel (`active`), on which the scaler can send `true` at any time. The purpose of this mechanism is to initiate a scaling operation independently from `pollingInterval`. @@ -115,7 +115,7 @@ KEDA will attempt a GRPC connection to `service-address.svc.local:9090` immediat } ``` ->**Note**: KEDA will issue all of the above RPC calls except `StreamIsActive` if `spec.triggers.type` is `external`. It _must_ be `external-push` for `StreamIsActive` to be called. +> **Note**: KEDA will issue all of the above RPC calls except `StreamIsActive` if `spec.triggers.type` is `external`. It _must_ be `external-push` for `StreamIsActive` to be called. ## Implementing KEDA external scaler GRPC interface @@ -182,8 +182,6 @@ npm install --save grpc request {{< /collapsible >}} -
- #### 3. Implementing `IsActive` Just like `IsActive(ctx context.Context) (bool, error)` in the go interface, the `IsActive` method in the GRPC interface is called every `pollingInterval` with a `ScaledObjectRef` object that contains the scaledObject name, namespace, and scaler metadata. @@ -310,70 +308,71 @@ public class ExternalScalerService : ExternalScaler.ExternalScalerBase Put the following code into your `index.js` file: ```js -const grpc = require('grpc') -const request = require('request') -const externalScalerProto = grpc.load('externalscaler.proto') +const grpc = require("grpc"); +const request = require("request"); +const externalScalerProto = grpc.load("externalscaler.proto"); -const server = new grpc.Server() +const server = new grpc.Server(); server.addService(externalScalerProto.externalscaler.ExternalScaler.service, { isActive: (call, callback) => { - const longitude = call.request.scalerMetadata.longitude - const latitude = call.request.scalerMetadata.latitude + const longitude = call.request.scalerMetadata.longitude; + const latitude = call.request.scalerMetadata.latitude; if (!longitude || !latitude) { callback({ code: grpc.status.INVALID_ARGUMENT, - details: 'longitude and latitude must be specified', - }) + details: "longitude and latitude must be specified", + }); } else { - const now = new Date() - const yesterday = new Date(new Date().setDate(new Date().getDate()-1)); - - const startTime = `${yesterday.getUTCFullYear()}-${yesterday.getUTCMonth()}-${yesterday.getUTCDay()}` - const endTime = `${now.getUTCFullYear()}-${now.getUTCMonth()}-${now.getUTCDay()}` - const radiusKm = 500 - const query = `format=geojson&starttime=${startTime}&endtime=${endTime}&longitude=${longitude}&latitude=${latitude}&maxradiuskm=${radiusKm}` - - request.get({ - url: `https://earthquake.usgs.gov/fdsnws/event/1/query?${query}`, - json: true, - }, (err, resp, data) => { - if (err) { - callback({ - code: grpc.status.INTERNAL, - details: err, - }) - } else if (resp.statusCode !== 200) { - callback({ - code: grpc.status.INTERNAL, - details: `expected status 200, got ${resp.statusCode}` - }) - } else { - // count how many earthquakes with mag > 1.0 - let count = 0 - data.features.forEach(i => { - if (i.properties.mag > 1.0) { - count++ - } - }) - callback(null, { - result: count > 2, - }) + const now = new Date(); + const yesterday = new Date(new Date().setDate(new Date().getDate() - 1)); + + const startTime = `${yesterday.getUTCFullYear()}-${yesterday.getUTCMonth()}-${yesterday.getUTCDay()}`; + const endTime = `${now.getUTCFullYear()}-${now.getUTCMonth()}-${now.getUTCDay()}`; + const radiusKm = 500; + const query = `format=geojson&starttime=${startTime}&endtime=${endTime}&longitude=${longitude}&latitude=${latitude}&maxradiuskm=${radiusKm}`; + + request.get( + { + url: `https://earthquake.usgs.gov/fdsnws/event/1/query?${query}`, + json: true, + }, + (err, resp, data) => { + if (err) { + callback({ + code: grpc.status.INTERNAL, + details: err, + }); + } else if (resp.statusCode !== 200) { + callback({ + code: grpc.status.INTERNAL, + details: `expected status 200, got ${resp.statusCode}`, + }); + } else { + // count how many earthquakes with mag > 1.0 + let count = 0; + data.features.forEach((i) => { + if (i.properties.mag > 1.0) { + count++; + } + }); + callback(null, { + result: count > 2, + }); + } } - }) + ); } - } -}) + }, +}); -server.bind('127.0.0.1:9090', grpc.ServerCredentials.createInsecure()) -console.log('Server listening on 127.0.0.1:9090') +server.bind("127.0.0.1:9090", grpc.ServerCredentials.createInsecure()); +console.log("Server listening on 127.0.0.1:9090"); -server.start() +server.start(); ``` {{< /collapsible >}} -
- #### 4. Implementing `StreamIsActive` Unlike `IsActive`, `StreamIsActive` is called once when KEDA reconciles the `ScaledObject`, and expects the external scaler to maintain a long-lived connection and push `IsActiveResponse` objects whenever the scaler needs KEDA to activate the deployment. @@ -450,38 +449,36 @@ public override async Task StreamIsActive(ScaledObjectRef request, IServerStream server.addService(externalScalerProto.externalscaler.ExternalScaler.service, { // ... streamIsActive: (call, callback) => { - const longitude = call.request.scalerMetadata.longitude - const latitude = call.request.scalerMetadata.latitude + const longitude = call.request.scalerMetadata.longitude; + const latitude = call.request.scalerMetadata.latitude; if (!longitude || !latitude) { callback({ code: grpc.status.INVALID_ARGUMENT, - details: 'longitude and latitude must be specified', - }) + details: "longitude and latitude must be specified", + }); } else { const interval = setInterval(() => { getEarthquakeCount((err, count) => { if (err) { - console.error(err) + console.error(err); } else if (count > 2) { call.write({ result: true, - }) + }); } - }) + }); }, 1000 * 60 * 60); - call.on('end', () => { - clearInterval(interval) - }) + call.on("end", () => { + clearInterval(interval); + }); } - } -}) + }, +}); ``` {{< /collapsible >}} -
- #### 5. Implementing `GetMetricSpec` `GetMetricSpec` returns the `target` value for [the HPA definition for the scaler](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale-walkthrough/). This scaler will define a static target of 10, but the threshold value is often specified in the metadata for other scalers. @@ -527,19 +524,19 @@ server.addService(externalScalerProto.externalscaler.ExternalScaler.service, { // ... getMetricSpec: (call, callback) => { callback(null, { - metricSpecs: [{ - metricName: 'earthquakeThreshold', - targetSize: 10, - }] - }) - } -}) + metricSpecs: [ + { + metricName: "earthquakeThreshold", + targetSize: 10, + }, + ], + }); + }, +}); ``` {{< /collapsible >}} -
- #### 6. Implementing `GetMetrics` `GetMetrics` returns the value of the metric referred to from `GetMetricSpec`, in this example it's `earthquakeThreshold`. @@ -606,32 +603,34 @@ public override async Task GetMetrics(GetMetricsRequest requ server.addService(externalScalerProto.externalscaler.ExternalScaler.service, { // ... getMetrics: (call, callback) => { - const longitude = call.request.scaledObjectRef.scalerMetadata.longitude - const latitude = call.request.scaledObjectRef.scalerMetadata.latitude + const longitude = call.request.scaledObjectRef.scalerMetadata.longitude; + const latitude = call.request.scaledObjectRef.scalerMetadata.latitude; if (!longitude || !latitude) { callback({ code: grpc.status.INVALID_ARGUMENT, - details: 'longitude and latitude must be specified', - }) + details: "longitude and latitude must be specified", + }); } else { getEarthquakeCount((err, count) => { if (err) { callback({ code: grpc.status.INTERNAL, details: err, - }) + }); } else { callback(null, { - metricValues: [{ - metricName: 'earthquakeThreshold', - metricValue: count, - }] - }) + metricValues: [ + { + metricName: "earthquakeThreshold", + metricValue: count, + }, + ], + }); } - }) + }); } - } -}) + }, +}); ``` {{< /collapsible >}} diff --git a/content/docs/2.10/migration.md b/content/docs/2.10/migration.md index 5ee8a6455..29e8408c8 100644 --- a/content/docs/2.10/migration.md +++ b/content/docs/2.10/migration.md @@ -11,15 +11,17 @@ Please note that you **can not** run both KEDA v1 and v2 on the same Kubernetes KEDA v2 is using a new API namespace for its Custom Resources Definitions (CRD): `keda.sh` instead of `keda.k8s.io` and introduces a new Custom Resource for scaling of Jobs. See full details on KEDA Custom Resources [here](../concepts/#custom-resources-crd). Here's an overview of what's changed: + - [Scaling of Deployments](#scaling-of-deployments) - [Scaling of Jobs](#scaling-of-jobs) - [Improved flexibility & usability of trigger metadata](#improved-flexibility--usability-of-trigger-metadata) - [Scalers](#scalers) - [TriggerAuthentication](#triggerauthentication) - ### Scaling of Deployments + In order to scale `Deployments` with KEDA v2, you need to do only a few modifications to existing v1 `ScaledObjects` definitions, so they comply with v2: + - Change the value of `apiVersion` property from `keda.k8s.io/v1alpha1` to `keda.sh/v1alpha1` - Rename property `spec.scaleTargetRef.deploymentName` to `spec.scaleTargetRef.name` - Rename property `spec.scaleTargetRef.containerName` to `spec.scaleTargetRef.envSourceContainerName` @@ -27,22 +29,23 @@ In order to scale `Deployments` with KEDA v2, you need to do only a few modifica Please see the examples below or refer to the full [v2 ScaledObject Specification](../concepts/scaling-deployments/#scaledobject-spec) - **Example of v1 ScaledObject** +**Example of v1 ScaledObject** + ```yaml apiVersion: keda.k8s.io/v1alpha1 kind: ScaledObject metadata: - name: {scaled-object-name} + name: { scaled-object-name } labels: - deploymentName: {deployment-name} + deploymentName: { deployment-name } spec: scaleTargetRef: - deploymentName: {deployment-name} - containerName: {container-name} - pollingInterval: 30 - cooldownPeriod: 300 - minReplicaCount: 0 - maxReplicaCount: 100 + deploymentName: { deployment-name } + containerName: { container-name } + pollingInterval: 30 + cooldownPeriod: 300 + minReplicaCount: 0 + maxReplicaCount: 100 triggers: # {list of triggers to activate the deployment} ``` @@ -50,18 +53,18 @@ spec: **Example of v2 ScaledObject** ```yaml -apiVersion: keda.sh/v1alpha1 # <--- Property value was changed +apiVersion: keda.sh/v1alpha1 # <--- Property value was changed kind: ScaledObject -metadata: # <--- labels.deploymentName is not needed - name: {scaled-object-name} +metadata: # <--- labels.deploymentName is not needed + name: { scaled-object-name } spec: scaleTargetRef: - name: {deployment-name} # <--- Property name was changed - envSourceContainerName: {container-name} # <--- Property name was changed - pollingInterval: 30 - cooldownPeriod: 300 - minReplicaCount: 0 - maxReplicaCount: 100 + name: { deployment-name } # <--- Property name was changed + envSourceContainerName: { container-name } # <--- Property name was changed + pollingInterval: 30 + cooldownPeriod: 300 + minReplicaCount: 0 + maxReplicaCount: 100 triggers: # {list of triggers to activate the deployment} ``` @@ -69,34 +72,36 @@ spec: ### Scaling of Jobs In order to scale `Jobs` with KEDA v2, you need to do only a few modifications to existing v1 `ScaledObjects` definitions, so they comply with v2: + - Change the value of `apiVersion` property from `keda.k8s.io/v1alpha1` to `keda.sh/v1alpha1` - Change the value of `kind` property from `ScaledObject` to `ScaledJob` - Remove property `spec.scaleType` -- Remove properties `spec.cooldownPeriod` and `spec.minReplicaCount` +- Remove properties `spec.cooldownPeriod` and `spec.minReplicaCount` You can configure `successfulJobsHistoryLimit` and `failedJobsHistoryLimit`. They will remove the old job histories automatically. Please see the examples below or refer to the full [v2 ScaledJob Specification](../concepts/scaling-jobs/#scaledjob-spec) **Example of v1 ScaledObject for Jobs scaling** + ```yaml apiVersion: keda.k8s.io/v1alpha1 kind: ScaledObject metadata: - name: {scaled-object-name} + name: { scaled-object-name } spec: scaleType: job jobTargetRef: - parallelism: 1 + parallelism: 1 completions: 1 activeDeadlineSeconds: 600 - backoffLimit: 6 + backoffLimit: 6 template: # {job template} - pollingInterval: 30 - cooldownPeriod: 300 - minReplicaCount: 0 - maxReplicaCount: 100 + pollingInterval: 30 + cooldownPeriod: 300 + minReplicaCount: 0 + maxReplicaCount: 100 triggers: # {list of triggers to create jobs} ``` @@ -104,21 +109,21 @@ spec: **Example of v2 ScaledJob** ```yaml -apiVersion: keda.sh/v1alpha1 # <--- Property value was changed -kind: ScaledJob # <--- Property value was changed +apiVersion: keda.sh/v1alpha1 # <--- Property value was changed +kind: ScaledJob # <--- Property value was changed metadata: - name: {scaled-job-name} -spec: # <--- spec.scaleType is not needed + name: { scaled-job-name } +spec: # <--- spec.scaleType is not needed jobTargetRef: - parallelism: 1 - completions: 1 + parallelism: 1 + completions: 1 activeDeadlineSeconds: 600 backoffLimit: 6 template: # {job template} - pollingInterval: 30 # <--- spec.cooldownPeriod and spec.minReplicaCount are not needed - successfulJobsHistoryLimit: 5 # <--- property is added - failedJobsHistoryLimit: 5 # <--- Property is added + pollingInterval: 30 # <--- spec.cooldownPeriod and spec.minReplicaCount are not needed + successfulJobsHistoryLimit: 5 # <--- property is added + failedJobsHistoryLimit: 5 # <--- Property is added maxReplicaCount: 100 triggers: # {list of triggers to create jobs} @@ -132,68 +137,74 @@ We've introduced more options to configure trigger metadata to give users more f Here's an overview: -| Scaler | 1.x | 2.0 | -|--------|--------|--------| -| `azure-blob` | `connection` (**Default**: `AzureWebJobsStorage`) | `connectionFromEnv` | -| `azure-monitor` | `activeDirectoryClientId`
`activeDirectoryClientPassword` | `activeDirectoryClientId`
`activeDirectoryClientIdFromEnv`
`activeDirectoryClientPasswordFromEnv` | -| `azure-queue` | `connection` (**Default**: AzureWebJobsStorage) | `connectionFromEnv` | -| `azure-servicebus` | `connection` | `connectionFromEnv` | -| `azure-eventhub` | `storageConnection` (**Default**: `AzureWebJobsStorage`)
`connection` (**Default**: `EventHub`) | `storageConnectionFromEnv`
`connectionFromEnv` | -| `aws-cloudwatch` | `awsAccessKeyID` (**Default**: `AWS_ACCESS_KEY_ID`)
`awsSecretAccessKey` (**Default**: `AWS_SECRET_ACCESS_KEY`) | `awsAccessKeyID`
`awsAccessKeyIDFromEnv`
`awsSecretAccessKeyFromEnv` | -| `aws-kinesis-stream` | `awsAccessKeyID` (**Default**: `AWS_ACCESS_KEY_ID`)
`awsSecretAccessKey` (**Default**: `AWS_SECRET_ACCESS_KEY`) | `awsAccessKeyID`
`awsAccessKeyIDFromEnv`
`awsSecretAccessKeyFromEnv` | -| `aws-sqs-queue` | `awsAccessKeyID` (**Default**: `AWS_ACCESS_KEY_ID`)
`awsSecretAccessKey` (**Default**: `AWS_SECRET_ACCESS_KEY`) | `awsAccessKeyID`
`awsAccessKeyIDFromEnv`
`awsSecretAccessKeyFromEnv` | -| `kafka` | _(none)_ | _(none)_ -| `rabbitmq` | `apiHost`
`host` | ~~`apiHost`~~
`host`
`hostFromEnv` | -| `prometheus` | _(none)_ | _(none)_ | -| `cron` | _(none)_ | _(none)_ | -| `redis` | `address`
`host`
`port`
`password` | `address`
`addressFromEnv`
`host`
`hostFromEnv`
~~`port`~~
`passwordFromEnv` -| `redis-streams` | `address`
`host`
`port`
`password` | `address`
`addressFromEnv`
`host`
`hostFromEnv`
~~`port`~~
`passwordFromEnv` -| `gcp-pubsub` | `credentials` | `credentialsFromEnv` | -| `external` | _(any matching value)_ | _(any matching value with `FromEnv` suffix)_ -| `liiklus` | _(none)_ | _(none)_ | -| `stan` | _(none)_ | _(none)_ | -| `huawei-cloudeye` | | _(none)_ | _(none)_ | -| `postgresql` | `connection`
`password` | `connectionFromEnv`
`passwordFromEnv` | -| `mysql` | `connectionString`
`password` | `connectionStringFromEnv`
`passwordFromEnv` | +| Scaler | 1.x | 2.0 | +| -------------------- | --------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------- | -------- | +| `azure-blob` | `connection` (**Default**: `AzureWebJobsStorage`) | `connectionFromEnv` | +| `azure-monitor` | `activeDirectoryClientId` `activeDirectoryClientPassword` | `activeDirectoryClientId` `activeDirectoryClientIdFromEnv` `activeDirectoryClientPasswordFromEnv` | +| `azure-queue` | `connection` (**Default**: AzureWebJobsStorage) | `connectionFromEnv` | +| `azure-servicebus` | `connection` | `connectionFromEnv` | +| `azure-eventhub` | `storageConnection` (**Default**: `AzureWebJobsStorage`) `connection` (**Default**: `EventHub`) | `storageConnectionFromEnv` `connectionFromEnv` | +| `aws-cloudwatch` | `awsAccessKeyID` (**Default**: `AWS_ACCESS_KEY_ID`) `awsSecretAccessKey` (**Default**: `AWS_SECRET_ACCESS_KEY`) | `awsAccessKeyID` `awsAccessKeyIDFromEnv` `awsSecretAccessKeyFromEnv` | +| `aws-kinesis-stream` | `awsAccessKeyID` (**Default**: `AWS_ACCESS_KEY_ID`) `awsSecretAccessKey` (**Default**: `AWS_SECRET_ACCESS_KEY`) | `awsAccessKeyID` `awsAccessKeyIDFromEnv` `awsSecretAccessKeyFromEnv` | +| `aws-sqs-queue` | `awsAccessKeyID` (**Default**: `AWS_ACCESS_KEY_ID`) `awsSecretAccessKey` (**Default**: `AWS_SECRET_ACCESS_KEY`) | `awsAccessKeyID` `awsAccessKeyIDFromEnv` `awsSecretAccessKeyFromEnv` | +| `kafka` | _(none)_ | _(none)_ | +| `rabbitmq` | `apiHost` `host` | ~~`apiHost`~~ `host` `hostFromEnv` | +| `prometheus` | _(none)_ | _(none)_ | +| `cron` | _(none)_ | _(none)_ | +| `redis` | `address` `host` `port` `password` | `address` `addressFromEnv` `host` `hostFromEnv` ~~`port`~~ `passwordFromEnv` | +| `redis-streams` | `address` `host` `port` `password` | `address` `addressFromEnv` `host` `hostFromEnv` ~~`port`~~ `passwordFromEnv` | +| `gcp-pubsub` | `credentials` | `credentialsFromEnv` | +| `external` | _(any matching value)_ | _(any matching value with `FromEnv` suffix)_ | +| `liiklus` | _(none)_ | _(none)_ | +| `stan` | _(none)_ | _(none)_ | +| `huawei-cloudeye` | | _(none)_ | _(none)_ | +| `postgresql` | `connection` `password` | `connectionFromEnv` `passwordFromEnv` | +| `mysql` | `connectionString` `password` | `connectionStringFromEnv` `passwordFromEnv` | ### Scalers **Azure Service Bus** - - `queueLength` was renamed to `messageCount` + +- `queueLength` was renamed to `messageCount` **Kafka** - - `authMode` property was replaced with `sasl` and `tls` properties. Please refer [documentation](../scalers/apache-kafka/#authentication-parameters) for Kafka Authentication Parameters details. + +- `authMode` property was replaced with `sasl` and `tls` properties. Please refer [documentation](../scalers/apache-kafka/#authentication-parameters) for Kafka Authentication Parameters details. **RabbitMQ** In KEDA 2.0 the RabbitMQ scaler has only `host` parameter, and the protocol for communication can be specified by -`protocol` (http or amqp). The default value is `amqp`. The behavior changes only for scalers that were using HTTP -protocol. +`protocol` (http or amqp). The default value is `amqp`. The behavior changes only for scalers that were using HTTP +protocol. Example of RabbitMQ trigger before 2.0: + ```yaml triggers: -- type: rabbitmq - metadata: - queueLength: '20' - queueName: testqueue - includeUnacked: 'true' - apiHost: 'https://guest:password@localhost:443/vhostname' -``` + - type: rabbitmq + metadata: + queueLength: "20" + queueName: testqueue + includeUnacked: "true" + apiHost: "https://guest:password@localhost:443/vhostname" +``` The same trigger in 2.0: + ```yaml triggers: -- type: rabbitmq - metadata: - queueLength: '20' - queueName: testqueue - protocol: 'http' - host: 'https://guest:password@localhost:443/vhostname' -``` + - type: rabbitmq + metadata: + queueLength: "20" + queueName: testqueue + protocol: "http" + host: "https://guest:password@localhost:443/vhostname" +``` ### TriggerAuthentication + In order to use Authentication via `TriggerAuthentication` with KEDA v2, you need to change: + - Change the value of `apiVersion` property from `keda.k8s.io/v1alpha1` to `keda.sh/v1alpha1` For more details please refer to the full diff --git a/content/docs/2.10/operate/cluster.md b/content/docs/2.10/operate/cluster.md index fe08db931..b705893e8 100644 --- a/content/docs/2.10/operate/cluster.md +++ b/content/docs/2.10/operate/cluster.md @@ -14,12 +14,12 @@ However, maintainers can decide to extend this by supporting more minor versions As a reference, this compatibility matrix shows supported k8s versions per KEDA version: -| KEDA | Kubernetes | -|-----------|---------------| -| v2.10 | v1.24 - v1.26 | -| v2.9 | v1.23 - v1.25 | -| v2.8 | v1.17 - v1.25 | -| v2.7 | v1.17 - v1.25 | +| KEDA | Kubernetes | +| ----- | ------------- | +| v2.10 | v1.24 - v1.26 | +| v2.9 | v1.23 - v1.25 | +| v2.8 | v1.17 - v1.25 | +| v2.7 | v1.17 - v1.25 | ### Cluster Capacity @@ -41,12 +41,10 @@ KEDA requires to be accessible inside the cluster to be able to autoscale. Here is an overview of the required ports that need to be accessible for KEDA to work: - | Port | Why? | Remarks | | ------ | -------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `443` | Used by Kubernetes API server to get metrics | Required for all platforms because it uses Control Plane → port 443 on the Service IP range communication.

This is not applicable for Google Cloud. | +| `443` | Used by Kubernetes API server to get metrics | Required for all platforms because it uses Control Plane → port 443 on the Service IP range communication. This is not applicable for Google Cloud. | | `6443` | Used by Kubernetes API server to get metrics | Only required for Google Cloud because it uses Control Plane → port 6443 on the Pod IP range for communication | - ## High Availability @@ -54,10 +52,10 @@ KEDA does not provide full support for high-availability due to upstream limitat Here is an overview of all KEDA deployments and the HA notes: -| Deployment | Support Replicas | Note | -| -------------- | ---------------- |---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Deployment | Support Replicas | Note | +| -------------- | ---------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | Metrics Server | 1 | You can run multiple replicas of our metrics sever, and it is recommended to add the `--enable-aggregator-routing=true` CLI flag to the kube-apiserver so that requests sent to our metrics servers are load balanced. However, [you can only run one active metric server in a Kubernetes cluster serving external.metrics.k8s.io](https://github.com/kubernetes-sigs/custom-metrics-apiserver/issues/70) which has to be the KEDA metric server. | -| Operator | 2 | While you can run multiple replicas of our operator, only one operator instance will be active. The rest will be standing by, which may reduce downtime during a failure. Multiple replicas will not improve the performance of KEDA, it could only reduce a downtime during a failover. | +| Operator | 2 | While you can run multiple replicas of our operator, only one operator instance will be active. The rest will be standing by, which may reduce downtime during a failure. Multiple replicas will not improve the performance of KEDA, it could only reduce a downtime during a failover. | ## HTTP Timeouts @@ -79,7 +77,7 @@ All applicable scalers will use this timeout. Setting a per-scaler timeout is cu ## HTTP connection disable keep alive -Keep alive behaviour is enabled by default for every HTTP connection, this could stack a huge amount of connections (one per scaler) in some scenarios. +Keep alive behaviour is enabled by default for every HTTP connection, this could stack a huge amount of connections (one per scaler) in some scenarios. You can disable keep alive for every HTTP connection by adding the relevant environment variable to both the KEDA Operator, and KEDA Metrics Server deployments: @@ -108,6 +106,7 @@ Our industry has seen an evolution of TLS versions and some are more secure than By default, KEDA uses TLS1.2 as a minimum TLS version given it is the lowest version without vulnerabilities. However, if you need to support another version you can configure it by using the environment variable `KEDA_HTTP_MIN_TLS_VERSION`. For example: + ```yaml - env: KEDA_HTTP_MIN_TLS_VERSION: TLS13 @@ -119,17 +118,18 @@ The following values are allowed: `TLS13`, `TLS12`, `TLS11` and `TLS10`. The Kubernetes client config used within KEDA Operator and KEDA Metrics Adapter can be adjusted by passing the following command-line flags to the binary: -| Adapter Flag | Client Config Setting | Default Value | Description | -| ------------------- | ----------------------- | ------------- | -------------------------------------------------------------- | -| kube-api-qps | cfg.QPS | 20.0 | Set the QPS rate for throttling requests sent to the apiserver | -| kube-api-burst | cfg.Burst | 30 | Set the burst for throttling requests sent to the apiserver | -| disable-compression | cfg.DisableCompression | true | Disable compression for response in k8s restAPI in client-go, see [this Kubernetes issue](https://github.com/kubernetes/kubernetes/issues/112296) for details | +| Adapter Flag | Client Config Setting | Default Value | Description | +| ------------------- | ---------------------- | ------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| kube-api-qps | cfg.QPS | 20.0 | Set the QPS rate for throttling requests sent to the apiserver | +| kube-api-burst | cfg.Burst | 30 | Set the burst for throttling requests sent to the apiserver | +| disable-compression | cfg.DisableCompression | true | Disable compression for response in k8s restAPI in client-go, see [this Kubernetes issue](https://github.com/kubernetes/kubernetes/issues/112296) for details | ## Configure `MaxConcurrentReconciles` for Controllers To implement internal controllers KEDA uses the [controller-runtime project](https://github.com/kubernetes-sigs/controller-runtime), that enables configuration of [MaxConcurrentReconciles property](https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/controller#Options), ie. the maximum number of concurrent reconciles which can be run for a controller. KEDA Operator exposes properties for specifying `MaxConcurrentReconciles` for following controllers/reconcilers: + - `ScaledObjectReconciler` - responsible for watching and managing `ScaledObjects`, ie. validates input trigger specification, starts scaling logic and manages dependent HPA. - `ScaledJobReconciler` - responsible for watching and managing `ScaledJobs` and dependent Kubernetes Jobs @@ -137,15 +137,16 @@ KEDA Metrics Server exposes property for specifying `MaxConcurrentReconciles` fo To modify this properties you can set environment variables on both KEDA Operator and Metrics Server Deployments: -| Environment variable name | Deployment | Default Value | Affected reconciler | -| ------------------------------------- | -------------- | ------------- | -------------------------------------------------------------- | -| KEDA_SCALEDOBJECT_CTRL_MAX_RECONCILES | Operator | 5 | ScaledObjectReconciler | -| KEDA_SCALEDJOB_CTRL_MAX_RECONCILES | Operator | 1 | ScaledJobReconciler | -| KEDA_METRICS_CTRL_MAX_RECONCILES | Metrics Server | 1 | MetricsScaledObjectReconciler | +| Environment variable name | Deployment | Default Value | Affected reconciler | +| ------------------------------------- | -------------- | ------------- | ----------------------------- | +| KEDA_SCALEDOBJECT_CTRL_MAX_RECONCILES | Operator | 5 | ScaledObjectReconciler | +| KEDA_SCALEDJOB_CTRL_MAX_RECONCILES | Operator | 1 | ScaledJobReconciler | +| KEDA_METRICS_CTRL_MAX_RECONCILES | Metrics Server | 1 | MetricsScaledObjectReconciler | ## Configure Leader Election Like reconciliation, KEDA also uses the [controller-runtime project](https://github.com/kubernetes-sigs/controller-runtime) for electing the leader replica. The following properties can be configured for either the Operator and Metrics Server Deployment: + - [`LeaseDuration`](https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/manager#Options.LeaseDuration) - [`RenewDeadline`](https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/manager#Options.RenewDeadline) - [`RetryPeriod`](https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/manager#Options.RetryPeriod) @@ -168,25 +169,29 @@ To learn more please refer to [security section](./security#use-your-own-tls-cer ## Restrict Secret Access By default, KEDA requires adding `secrets` to the cluster role as following: + ```yaml - apiGroups: - - "" + - "" resources: - - external - - pods - - secrets - - services + - external + - pods + - secrets + - services verbs: - - get - - list - - watch + - get + - list + - watch ``` + However, this might lead to security risk (especially in production environment) since it will grant permission to read `secrets` from all namespaces. To restrict `secret` access and limited to KEDA namespace, you could add `KEDA_RESTRICT_SECRET_ACCESS` as environment variable to both KEDA Operator and KEDA Metrics Server: + ```yaml env: - name: KEDA_RESTRICT_SECRET_ACCESS value: "true" ``` + This allows you to omit `secrets` from the cluster role, which will disallow `TriggerAuthentication` to be used for your triggers if the `TriggerAuthentication` is using secrets. You can, however, still use `ClusterTriggerAuthentication`. diff --git a/content/docs/2.11/concepts/external-scalers.md b/content/docs/2.11/concepts/external-scalers.md index 5badde098..aa00a18a0 100644 --- a/content/docs/2.11/concepts/external-scalers.md +++ b/content/docs/2.11/concepts/external-scalers.md @@ -9,7 +9,7 @@ Built-in scalers run in the KEDA process/pod, while external scalers require an This document describes the external scaler interfaces and how to implement them in Go, Node, and .NET; however for more details on GRPC refer to [the official GRPC documentation](https://grpc.io/docs/) ->Want to learn about existing external scalers? Explore our [external scaler community](https://github.com/kedacore/external-scalers). +> Want to learn about existing external scalers? Explore our [external scaler community](https://github.com/kedacore/external-scalers). ## Overview @@ -45,7 +45,7 @@ The `Scaler` interface defines 3 methods: - `GetMetricSpecForScaling` returns the target value for the HPA definition for the scaler. For more details refer to [Implementing `GetMetricSpec`](#5-implementing-getmetricspec). - `GetMetricsAndActivity` is called on `pollingInterval` and. When activity returns `true`, KEDA will scale to what is returned by the metric limited by `maxReplicaCount` on the ScaledObject/ScaledJob. When `false` is returned, KEDA will scale to `minReplicaCount` or optionally `idleReplicaCount`. More details around the defaults and how these options work together can be found on the [ScaledObjectSpec](https://keda.sh/docs/latest/concepts/scaling-deployments/#scaledobject-spec). -> Refer to the [HPA docs](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale-walkthrough/) for how HPA calculates `replicaCount` based on metric value and target value. KEDA supports both `AverageValue` and `Value` metric target types for external metrics. When `AverageValue` (the default metric type) is used, the metric value returned by the external scaler will be divided by the number of replicas. + > Refer to the [HPA docs](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale-walkthrough/) for how HPA calculates `replicaCount` based on metric value and target value. KEDA supports both `AverageValue` and `Value` metric target types for external metrics. When `AverageValue` (the default metric type) is used, the metric value returned by the external scaler will be divided by the number of replicas. The `PushScaler` interface adds a `Run` method. This method receives a push channel (`active`), on which the scaler can send `true` at any time. The purpose of this mechanism is to initiate a scaling operation independently from `pollingInterval`. @@ -115,7 +115,7 @@ KEDA will attempt a GRPC connection to `service-address.svc.local:9090` immediat } ``` ->**Note**: KEDA will issue all of the above RPC calls except `StreamIsActive` if `spec.triggers.type` is `external`. It _must_ be `external-push` for `StreamIsActive` to be called. +> **Note**: KEDA will issue all of the above RPC calls except `StreamIsActive` if `spec.triggers.type` is `external`. It _must_ be `external-push` for `StreamIsActive` to be called. ## Implementing KEDA external scaler GRPC interface @@ -182,8 +182,6 @@ npm install --save grpc request {{< /collapsible >}} -
- #### 3. Implementing `IsActive` Just like `IsActive(ctx context.Context) (bool, error)` in the go interface, the `IsActive` method in the GRPC interface is called every `pollingInterval` with a `ScaledObjectRef` object that contains the scaledObject name, namespace, and scaler metadata. @@ -310,70 +308,71 @@ public class ExternalScalerService : ExternalScaler.ExternalScalerBase Put the following code into your `index.js` file: ```js -const grpc = require('grpc') -const request = require('request') -const externalScalerProto = grpc.load('externalscaler.proto') +const grpc = require("grpc"); +const request = require("request"); +const externalScalerProto = grpc.load("externalscaler.proto"); -const server = new grpc.Server() +const server = new grpc.Server(); server.addService(externalScalerProto.externalscaler.ExternalScaler.service, { isActive: (call, callback) => { - const longitude = call.request.scalerMetadata.longitude - const latitude = call.request.scalerMetadata.latitude + const longitude = call.request.scalerMetadata.longitude; + const latitude = call.request.scalerMetadata.latitude; if (!longitude || !latitude) { callback({ code: grpc.status.INVALID_ARGUMENT, - details: 'longitude and latitude must be specified', - }) + details: "longitude and latitude must be specified", + }); } else { - const now = new Date() - const yesterday = new Date(new Date().setDate(new Date().getDate()-1)); - - const startTime = `${yesterday.getUTCFullYear()}-${yesterday.getUTCMonth()}-${yesterday.getUTCDay()}` - const endTime = `${now.getUTCFullYear()}-${now.getUTCMonth()}-${now.getUTCDay()}` - const radiusKm = 500 - const query = `format=geojson&starttime=${startTime}&endtime=${endTime}&longitude=${longitude}&latitude=${latitude}&maxradiuskm=${radiusKm}` - - request.get({ - url: `https://earthquake.usgs.gov/fdsnws/event/1/query?${query}`, - json: true, - }, (err, resp, data) => { - if (err) { - callback({ - code: grpc.status.INTERNAL, - details: err, - }) - } else if (resp.statusCode !== 200) { - callback({ - code: grpc.status.INTERNAL, - details: `expected status 200, got ${resp.statusCode}` - }) - } else { - // count how many earthquakes with mag > 1.0 - let count = 0 - data.features.forEach(i => { - if (i.properties.mag > 1.0) { - count++ - } - }) - callback(null, { - result: count > 2, - }) + const now = new Date(); + const yesterday = new Date(new Date().setDate(new Date().getDate() - 1)); + + const startTime = `${yesterday.getUTCFullYear()}-${yesterday.getUTCMonth()}-${yesterday.getUTCDay()}`; + const endTime = `${now.getUTCFullYear()}-${now.getUTCMonth()}-${now.getUTCDay()}`; + const radiusKm = 500; + const query = `format=geojson&starttime=${startTime}&endtime=${endTime}&longitude=${longitude}&latitude=${latitude}&maxradiuskm=${radiusKm}`; + + request.get( + { + url: `https://earthquake.usgs.gov/fdsnws/event/1/query?${query}`, + json: true, + }, + (err, resp, data) => { + if (err) { + callback({ + code: grpc.status.INTERNAL, + details: err, + }); + } else if (resp.statusCode !== 200) { + callback({ + code: grpc.status.INTERNAL, + details: `expected status 200, got ${resp.statusCode}`, + }); + } else { + // count how many earthquakes with mag > 1.0 + let count = 0; + data.features.forEach((i) => { + if (i.properties.mag > 1.0) { + count++; + } + }); + callback(null, { + result: count > 2, + }); + } } - }) + ); } - } -}) + }, +}); -server.bind('127.0.0.1:9090', grpc.ServerCredentials.createInsecure()) -console.log('Server listening on 127.0.0.1:9090') +server.bind("127.0.0.1:9090", grpc.ServerCredentials.createInsecure()); +console.log("Server listening on 127.0.0.1:9090"); -server.start() +server.start(); ``` {{< /collapsible >}} -
- #### 4. Implementing `StreamIsActive` Unlike `IsActive`, `StreamIsActive` is called once when KEDA reconciles the `ScaledObject`, and expects the external scaler to maintain a long-lived connection and push `IsActiveResponse` objects whenever the scaler needs KEDA to activate the deployment. @@ -450,38 +449,36 @@ public override async Task StreamIsActive(ScaledObjectRef request, IServerStream server.addService(externalScalerProto.externalscaler.ExternalScaler.service, { // ... streamIsActive: (call, callback) => { - const longitude = call.request.scalerMetadata.longitude - const latitude = call.request.scalerMetadata.latitude + const longitude = call.request.scalerMetadata.longitude; + const latitude = call.request.scalerMetadata.latitude; if (!longitude || !latitude) { callback({ code: grpc.status.INVALID_ARGUMENT, - details: 'longitude and latitude must be specified', - }) + details: "longitude and latitude must be specified", + }); } else { const interval = setInterval(() => { getEarthquakeCount((err, count) => { if (err) { - console.error(err) + console.error(err); } else if (count > 2) { call.write({ result: true, - }) + }); } - }) + }); }, 1000 * 60 * 60); - call.on('end', () => { - clearInterval(interval) - }) + call.on("end", () => { + clearInterval(interval); + }); } - } -}) + }, +}); ``` {{< /collapsible >}} -
- #### 5. Implementing `GetMetricSpec` `GetMetricSpec` returns the `target` value for [the HPA definition for the scaler](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale-walkthrough/). This scaler will define a static target of 10, but the threshold value is often specified in the metadata for other scalers. @@ -527,19 +524,19 @@ server.addService(externalScalerProto.externalscaler.ExternalScaler.service, { // ... getMetricSpec: (call, callback) => { callback(null, { - metricSpecs: [{ - metricName: 'earthquakeThreshold', - targetSize: 10, - }] - }) - } -}) + metricSpecs: [ + { + metricName: "earthquakeThreshold", + targetSize: 10, + }, + ], + }); + }, +}); ``` {{< /collapsible >}} -
- #### 6. Implementing `GetMetrics` `GetMetrics` returns the value of the metric referred to from `GetMetricSpec`, in this example it's `earthquakeThreshold`. @@ -606,32 +603,34 @@ public override async Task GetMetrics(GetMetricsRequest requ server.addService(externalScalerProto.externalscaler.ExternalScaler.service, { // ... getMetrics: (call, callback) => { - const longitude = call.request.scaledObjectRef.scalerMetadata.longitude - const latitude = call.request.scaledObjectRef.scalerMetadata.latitude + const longitude = call.request.scaledObjectRef.scalerMetadata.longitude; + const latitude = call.request.scaledObjectRef.scalerMetadata.latitude; if (!longitude || !latitude) { callback({ code: grpc.status.INVALID_ARGUMENT, - details: 'longitude and latitude must be specified', - }) + details: "longitude and latitude must be specified", + }); } else { getEarthquakeCount((err, count) => { if (err) { callback({ code: grpc.status.INTERNAL, details: err, - }) + }); } else { callback(null, { - metricValues: [{ - metricName: 'earthquakeThreshold', - metricValue: count, - }] - }) + metricValues: [ + { + metricName: "earthquakeThreshold", + metricValue: count, + }, + ], + }); } - }) + }); } - } -}) + }, +}); ``` {{< /collapsible >}} diff --git a/content/docs/2.11/migration.md b/content/docs/2.11/migration.md index 5ee8a6455..29e8408c8 100644 --- a/content/docs/2.11/migration.md +++ b/content/docs/2.11/migration.md @@ -11,15 +11,17 @@ Please note that you **can not** run both KEDA v1 and v2 on the same Kubernetes KEDA v2 is using a new API namespace for its Custom Resources Definitions (CRD): `keda.sh` instead of `keda.k8s.io` and introduces a new Custom Resource for scaling of Jobs. See full details on KEDA Custom Resources [here](../concepts/#custom-resources-crd). Here's an overview of what's changed: + - [Scaling of Deployments](#scaling-of-deployments) - [Scaling of Jobs](#scaling-of-jobs) - [Improved flexibility & usability of trigger metadata](#improved-flexibility--usability-of-trigger-metadata) - [Scalers](#scalers) - [TriggerAuthentication](#triggerauthentication) - ### Scaling of Deployments + In order to scale `Deployments` with KEDA v2, you need to do only a few modifications to existing v1 `ScaledObjects` definitions, so they comply with v2: + - Change the value of `apiVersion` property from `keda.k8s.io/v1alpha1` to `keda.sh/v1alpha1` - Rename property `spec.scaleTargetRef.deploymentName` to `spec.scaleTargetRef.name` - Rename property `spec.scaleTargetRef.containerName` to `spec.scaleTargetRef.envSourceContainerName` @@ -27,22 +29,23 @@ In order to scale `Deployments` with KEDA v2, you need to do only a few modifica Please see the examples below or refer to the full [v2 ScaledObject Specification](../concepts/scaling-deployments/#scaledobject-spec) - **Example of v1 ScaledObject** +**Example of v1 ScaledObject** + ```yaml apiVersion: keda.k8s.io/v1alpha1 kind: ScaledObject metadata: - name: {scaled-object-name} + name: { scaled-object-name } labels: - deploymentName: {deployment-name} + deploymentName: { deployment-name } spec: scaleTargetRef: - deploymentName: {deployment-name} - containerName: {container-name} - pollingInterval: 30 - cooldownPeriod: 300 - minReplicaCount: 0 - maxReplicaCount: 100 + deploymentName: { deployment-name } + containerName: { container-name } + pollingInterval: 30 + cooldownPeriod: 300 + minReplicaCount: 0 + maxReplicaCount: 100 triggers: # {list of triggers to activate the deployment} ``` @@ -50,18 +53,18 @@ spec: **Example of v2 ScaledObject** ```yaml -apiVersion: keda.sh/v1alpha1 # <--- Property value was changed +apiVersion: keda.sh/v1alpha1 # <--- Property value was changed kind: ScaledObject -metadata: # <--- labels.deploymentName is not needed - name: {scaled-object-name} +metadata: # <--- labels.deploymentName is not needed + name: { scaled-object-name } spec: scaleTargetRef: - name: {deployment-name} # <--- Property name was changed - envSourceContainerName: {container-name} # <--- Property name was changed - pollingInterval: 30 - cooldownPeriod: 300 - minReplicaCount: 0 - maxReplicaCount: 100 + name: { deployment-name } # <--- Property name was changed + envSourceContainerName: { container-name } # <--- Property name was changed + pollingInterval: 30 + cooldownPeriod: 300 + minReplicaCount: 0 + maxReplicaCount: 100 triggers: # {list of triggers to activate the deployment} ``` @@ -69,34 +72,36 @@ spec: ### Scaling of Jobs In order to scale `Jobs` with KEDA v2, you need to do only a few modifications to existing v1 `ScaledObjects` definitions, so they comply with v2: + - Change the value of `apiVersion` property from `keda.k8s.io/v1alpha1` to `keda.sh/v1alpha1` - Change the value of `kind` property from `ScaledObject` to `ScaledJob` - Remove property `spec.scaleType` -- Remove properties `spec.cooldownPeriod` and `spec.minReplicaCount` +- Remove properties `spec.cooldownPeriod` and `spec.minReplicaCount` You can configure `successfulJobsHistoryLimit` and `failedJobsHistoryLimit`. They will remove the old job histories automatically. Please see the examples below or refer to the full [v2 ScaledJob Specification](../concepts/scaling-jobs/#scaledjob-spec) **Example of v1 ScaledObject for Jobs scaling** + ```yaml apiVersion: keda.k8s.io/v1alpha1 kind: ScaledObject metadata: - name: {scaled-object-name} + name: { scaled-object-name } spec: scaleType: job jobTargetRef: - parallelism: 1 + parallelism: 1 completions: 1 activeDeadlineSeconds: 600 - backoffLimit: 6 + backoffLimit: 6 template: # {job template} - pollingInterval: 30 - cooldownPeriod: 300 - minReplicaCount: 0 - maxReplicaCount: 100 + pollingInterval: 30 + cooldownPeriod: 300 + minReplicaCount: 0 + maxReplicaCount: 100 triggers: # {list of triggers to create jobs} ``` @@ -104,21 +109,21 @@ spec: **Example of v2 ScaledJob** ```yaml -apiVersion: keda.sh/v1alpha1 # <--- Property value was changed -kind: ScaledJob # <--- Property value was changed +apiVersion: keda.sh/v1alpha1 # <--- Property value was changed +kind: ScaledJob # <--- Property value was changed metadata: - name: {scaled-job-name} -spec: # <--- spec.scaleType is not needed + name: { scaled-job-name } +spec: # <--- spec.scaleType is not needed jobTargetRef: - parallelism: 1 - completions: 1 + parallelism: 1 + completions: 1 activeDeadlineSeconds: 600 backoffLimit: 6 template: # {job template} - pollingInterval: 30 # <--- spec.cooldownPeriod and spec.minReplicaCount are not needed - successfulJobsHistoryLimit: 5 # <--- property is added - failedJobsHistoryLimit: 5 # <--- Property is added + pollingInterval: 30 # <--- spec.cooldownPeriod and spec.minReplicaCount are not needed + successfulJobsHistoryLimit: 5 # <--- property is added + failedJobsHistoryLimit: 5 # <--- Property is added maxReplicaCount: 100 triggers: # {list of triggers to create jobs} @@ -132,68 +137,74 @@ We've introduced more options to configure trigger metadata to give users more f Here's an overview: -| Scaler | 1.x | 2.0 | -|--------|--------|--------| -| `azure-blob` | `connection` (**Default**: `AzureWebJobsStorage`) | `connectionFromEnv` | -| `azure-monitor` | `activeDirectoryClientId`
`activeDirectoryClientPassword` | `activeDirectoryClientId`
`activeDirectoryClientIdFromEnv`
`activeDirectoryClientPasswordFromEnv` | -| `azure-queue` | `connection` (**Default**: AzureWebJobsStorage) | `connectionFromEnv` | -| `azure-servicebus` | `connection` | `connectionFromEnv` | -| `azure-eventhub` | `storageConnection` (**Default**: `AzureWebJobsStorage`)
`connection` (**Default**: `EventHub`) | `storageConnectionFromEnv`
`connectionFromEnv` | -| `aws-cloudwatch` | `awsAccessKeyID` (**Default**: `AWS_ACCESS_KEY_ID`)
`awsSecretAccessKey` (**Default**: `AWS_SECRET_ACCESS_KEY`) | `awsAccessKeyID`
`awsAccessKeyIDFromEnv`
`awsSecretAccessKeyFromEnv` | -| `aws-kinesis-stream` | `awsAccessKeyID` (**Default**: `AWS_ACCESS_KEY_ID`)
`awsSecretAccessKey` (**Default**: `AWS_SECRET_ACCESS_KEY`) | `awsAccessKeyID`
`awsAccessKeyIDFromEnv`
`awsSecretAccessKeyFromEnv` | -| `aws-sqs-queue` | `awsAccessKeyID` (**Default**: `AWS_ACCESS_KEY_ID`)
`awsSecretAccessKey` (**Default**: `AWS_SECRET_ACCESS_KEY`) | `awsAccessKeyID`
`awsAccessKeyIDFromEnv`
`awsSecretAccessKeyFromEnv` | -| `kafka` | _(none)_ | _(none)_ -| `rabbitmq` | `apiHost`
`host` | ~~`apiHost`~~
`host`
`hostFromEnv` | -| `prometheus` | _(none)_ | _(none)_ | -| `cron` | _(none)_ | _(none)_ | -| `redis` | `address`
`host`
`port`
`password` | `address`
`addressFromEnv`
`host`
`hostFromEnv`
~~`port`~~
`passwordFromEnv` -| `redis-streams` | `address`
`host`
`port`
`password` | `address`
`addressFromEnv`
`host`
`hostFromEnv`
~~`port`~~
`passwordFromEnv` -| `gcp-pubsub` | `credentials` | `credentialsFromEnv` | -| `external` | _(any matching value)_ | _(any matching value with `FromEnv` suffix)_ -| `liiklus` | _(none)_ | _(none)_ | -| `stan` | _(none)_ | _(none)_ | -| `huawei-cloudeye` | | _(none)_ | _(none)_ | -| `postgresql` | `connection`
`password` | `connectionFromEnv`
`passwordFromEnv` | -| `mysql` | `connectionString`
`password` | `connectionStringFromEnv`
`passwordFromEnv` | +| Scaler | 1.x | 2.0 | +| -------------------- | --------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------- | -------- | +| `azure-blob` | `connection` (**Default**: `AzureWebJobsStorage`) | `connectionFromEnv` | +| `azure-monitor` | `activeDirectoryClientId` `activeDirectoryClientPassword` | `activeDirectoryClientId` `activeDirectoryClientIdFromEnv` `activeDirectoryClientPasswordFromEnv` | +| `azure-queue` | `connection` (**Default**: AzureWebJobsStorage) | `connectionFromEnv` | +| `azure-servicebus` | `connection` | `connectionFromEnv` | +| `azure-eventhub` | `storageConnection` (**Default**: `AzureWebJobsStorage`) `connection` (**Default**: `EventHub`) | `storageConnectionFromEnv` `connectionFromEnv` | +| `aws-cloudwatch` | `awsAccessKeyID` (**Default**: `AWS_ACCESS_KEY_ID`) `awsSecretAccessKey` (**Default**: `AWS_SECRET_ACCESS_KEY`) | `awsAccessKeyID` `awsAccessKeyIDFromEnv` `awsSecretAccessKeyFromEnv` | +| `aws-kinesis-stream` | `awsAccessKeyID` (**Default**: `AWS_ACCESS_KEY_ID`) `awsSecretAccessKey` (**Default**: `AWS_SECRET_ACCESS_KEY`) | `awsAccessKeyID` `awsAccessKeyIDFromEnv` `awsSecretAccessKeyFromEnv` | +| `aws-sqs-queue` | `awsAccessKeyID` (**Default**: `AWS_ACCESS_KEY_ID`) `awsSecretAccessKey` (**Default**: `AWS_SECRET_ACCESS_KEY`) | `awsAccessKeyID` `awsAccessKeyIDFromEnv` `awsSecretAccessKeyFromEnv` | +| `kafka` | _(none)_ | _(none)_ | +| `rabbitmq` | `apiHost` `host` | ~~`apiHost`~~ `host` `hostFromEnv` | +| `prometheus` | _(none)_ | _(none)_ | +| `cron` | _(none)_ | _(none)_ | +| `redis` | `address` `host` `port` `password` | `address` `addressFromEnv` `host` `hostFromEnv` ~~`port`~~ `passwordFromEnv` | +| `redis-streams` | `address` `host` `port` `password` | `address` `addressFromEnv` `host` `hostFromEnv` ~~`port`~~ `passwordFromEnv` | +| `gcp-pubsub` | `credentials` | `credentialsFromEnv` | +| `external` | _(any matching value)_ | _(any matching value with `FromEnv` suffix)_ | +| `liiklus` | _(none)_ | _(none)_ | +| `stan` | _(none)_ | _(none)_ | +| `huawei-cloudeye` | | _(none)_ | _(none)_ | +| `postgresql` | `connection` `password` | `connectionFromEnv` `passwordFromEnv` | +| `mysql` | `connectionString` `password` | `connectionStringFromEnv` `passwordFromEnv` | ### Scalers **Azure Service Bus** - - `queueLength` was renamed to `messageCount` + +- `queueLength` was renamed to `messageCount` **Kafka** - - `authMode` property was replaced with `sasl` and `tls` properties. Please refer [documentation](../scalers/apache-kafka/#authentication-parameters) for Kafka Authentication Parameters details. + +- `authMode` property was replaced with `sasl` and `tls` properties. Please refer [documentation](../scalers/apache-kafka/#authentication-parameters) for Kafka Authentication Parameters details. **RabbitMQ** In KEDA 2.0 the RabbitMQ scaler has only `host` parameter, and the protocol for communication can be specified by -`protocol` (http or amqp). The default value is `amqp`. The behavior changes only for scalers that were using HTTP -protocol. +`protocol` (http or amqp). The default value is `amqp`. The behavior changes only for scalers that were using HTTP +protocol. Example of RabbitMQ trigger before 2.0: + ```yaml triggers: -- type: rabbitmq - metadata: - queueLength: '20' - queueName: testqueue - includeUnacked: 'true' - apiHost: 'https://guest:password@localhost:443/vhostname' -``` + - type: rabbitmq + metadata: + queueLength: "20" + queueName: testqueue + includeUnacked: "true" + apiHost: "https://guest:password@localhost:443/vhostname" +``` The same trigger in 2.0: + ```yaml triggers: -- type: rabbitmq - metadata: - queueLength: '20' - queueName: testqueue - protocol: 'http' - host: 'https://guest:password@localhost:443/vhostname' -``` + - type: rabbitmq + metadata: + queueLength: "20" + queueName: testqueue + protocol: "http" + host: "https://guest:password@localhost:443/vhostname" +``` ### TriggerAuthentication + In order to use Authentication via `TriggerAuthentication` with KEDA v2, you need to change: + - Change the value of `apiVersion` property from `keda.k8s.io/v1alpha1` to `keda.sh/v1alpha1` For more details please refer to the full diff --git a/content/docs/2.11/operate/cluster.md b/content/docs/2.11/operate/cluster.md index 9cdadfe80..ffb15a12a 100644 --- a/content/docs/2.11/operate/cluster.md +++ b/content/docs/2.11/operate/cluster.md @@ -14,13 +14,13 @@ However, maintainers can decide to extend this by supporting more minor versions As a reference, this compatibility matrix shows supported k8s versions per KEDA version: -| KEDA | Kubernetes | -|-----------|---------------| -| v2.11 | v1.25 - v1.27 | -| v2.10 | v1.24 - v1.26 | -| v2.9 | v1.23 - v1.25 | -| v2.8 | v1.17 - v1.25 | -| v2.7 | v1.17 - v1.25 | +| KEDA | Kubernetes | +| ----- | ------------- | +| v2.11 | v1.25 - v1.27 | +| v2.10 | v1.24 - v1.26 | +| v2.9 | v1.23 - v1.25 | +| v2.8 | v1.17 - v1.25 | +| v2.7 | v1.17 - v1.25 | ### Cluster Capacity @@ -42,12 +42,10 @@ KEDA requires to be accessible inside the cluster to be able to autoscale. Here is an overview of the required ports that need to be accessible for KEDA to work: - | Port | Why? | Remarks | | ------ | -------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `443` | Used by Kubernetes API server to get metrics | Required for all platforms because it uses Control Plane → port 443 on the Service IP range communication.

This is not applicable for Google Cloud. | +| `443` | Used by Kubernetes API server to get metrics | Required for all platforms because it uses Control Plane → port 443 on the Service IP range communication. This is not applicable for Google Cloud. | | `6443` | Used by Kubernetes API server to get metrics | Only required for Google Cloud because it uses Control Plane → port 6443 on the Pod IP range for communication | - ## High Availability @@ -55,10 +53,10 @@ KEDA does not provide full support for high-availability due to upstream limitat Here is an overview of all KEDA deployments and the HA notes: -| Deployment | Support Replicas | Note | -| -------------- | ---------------- |---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Deployment | Support Replicas | Note | +| -------------- | ---------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | Metrics Server | 1 | You can run multiple replicas of our metrics sever, and it is recommended to add the `--enable-aggregator-routing=true` CLI flag to the kube-apiserver so that requests sent to our metrics servers are load balanced. However, [you can only run one active metric server in a Kubernetes cluster serving external.metrics.k8s.io](https://github.com/kubernetes-sigs/custom-metrics-apiserver/issues/70) which has to be the KEDA metric server. | -| Operator | 2 | While you can run multiple replicas of our operator, only one operator instance will be active. The rest will be standing by, which may reduce downtime during a failure. Multiple replicas will not improve the performance of KEDA, it could only reduce a downtime during a failover. | +| Operator | 2 | While you can run multiple replicas of our operator, only one operator instance will be active. The rest will be standing by, which may reduce downtime during a failure. Multiple replicas will not improve the performance of KEDA, it could only reduce a downtime during a failover. | ## HTTP Timeouts @@ -80,7 +78,7 @@ All applicable scalers will use this timeout. Setting a per-scaler timeout is cu ## HTTP connection disable keep alive -Keep alive behaviour is enabled by default for every HTTP connection, this could stack a huge amount of connections (one per scaler) in some scenarios. +Keep alive behaviour is enabled by default for every HTTP connection, this could stack a huge amount of connections (one per scaler) in some scenarios. You can disable keep alive for every HTTP connection by adding the relevant environment variable to both the KEDA Operator, and KEDA Metrics Server deployments: @@ -109,6 +107,7 @@ Our industry has seen an evolution of TLS versions and some are more secure than By default, KEDA uses TLS1.2 as a minimum TLS version given it is the lowest version without vulnerabilities. However, if you need to support another version you can configure it by using the environment variable `KEDA_HTTP_MIN_TLS_VERSION`. For example: + ```yaml - env: KEDA_HTTP_MIN_TLS_VERSION: TLS13 @@ -120,17 +119,18 @@ The following values are allowed: `TLS13`, `TLS12`, `TLS11` and `TLS10`. The Kubernetes client config used within KEDA Operator and KEDA Metrics Adapter can be adjusted by passing the following command-line flags to the binary: -| Adapter Flag | Client Config Setting | Default Value | Description | -| ------------------- | ----------------------- | ------------- | -------------------------------------------------------------- | -| kube-api-qps | cfg.QPS | 20.0 | Set the QPS rate for throttling requests sent to the apiserver | -| kube-api-burst | cfg.Burst | 30 | Set the burst for throttling requests sent to the apiserver | -| disable-compression | cfg.DisableCompression | true | Disable compression for response in k8s restAPI in client-go, see [this Kubernetes issue](https://github.com/kubernetes/kubernetes/issues/112296) for details | +| Adapter Flag | Client Config Setting | Default Value | Description | +| ------------------- | ---------------------- | ------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| kube-api-qps | cfg.QPS | 20.0 | Set the QPS rate for throttling requests sent to the apiserver | +| kube-api-burst | cfg.Burst | 30 | Set the burst for throttling requests sent to the apiserver | +| disable-compression | cfg.DisableCompression | true | Disable compression for response in k8s restAPI in client-go, see [this Kubernetes issue](https://github.com/kubernetes/kubernetes/issues/112296) for details | ## Configure `MaxConcurrentReconciles` for Controllers To implement internal controllers KEDA uses the [controller-runtime project](https://github.com/kubernetes-sigs/controller-runtime), that enables configuration of [MaxConcurrentReconciles property](https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/controller#Options), ie. the maximum number of concurrent reconciles which can be run for a controller. KEDA Operator exposes properties for specifying `MaxConcurrentReconciles` for following controllers/reconcilers: + - `ScaledObjectReconciler` - responsible for watching and managing `ScaledObjects`, ie. validates input trigger specification, starts scaling logic and manages dependent HPA. - `ScaledJobReconciler` - responsible for watching and managing `ScaledJobs` and dependent Kubernetes Jobs @@ -138,14 +138,15 @@ KEDA Metrics Server exposes property for specifying `MaxConcurrentReconciles` fo To modify this properties you can set environment variables on both KEDA Operator and Metrics Server Deployments: -| Environment variable name | Deployment | Default Value | Affected reconciler | -| ------------------------------------- | -------------- | ------------- | -------------------------------------------------------------- | -| KEDA_SCALEDOBJECT_CTRL_MAX_RECONCILES | Operator | 5 | ScaledObjectReconciler | -| KEDA_SCALEDJOB_CTRL_MAX_RECONCILES | Operator | 1 | ScaledJobReconciler | +| Environment variable name | Deployment | Default Value | Affected reconciler | +| ------------------------------------- | ---------- | ------------- | ---------------------- | +| KEDA_SCALEDOBJECT_CTRL_MAX_RECONCILES | Operator | 5 | ScaledObjectReconciler | +| KEDA_SCALEDJOB_CTRL_MAX_RECONCILES | Operator | 1 | ScaledJobReconciler | ## Configure Leader Election Like reconciliation, KEDA also uses the [controller-runtime project](https://github.com/kubernetes-sigs/controller-runtime) for electing the leader replica. The following properties can be configured for either the Operator and Metrics Server Deployment: + - [`LeaseDuration`](https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/manager#Options.LeaseDuration) - [`RenewDeadline`](https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/manager#Options.RenewDeadline) - [`RetryPeriod`](https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/manager#Options.RetryPeriod) @@ -168,25 +169,29 @@ To learn more please refer to [security section](./security#use-your-own-tls-cer ## Restrict Secret Access By default, KEDA requires adding `secrets` to the cluster role as following: + ```yaml - apiGroups: - - "" + - "" resources: - - external - - pods - - secrets - - services + - external + - pods + - secrets + - services verbs: - - get - - list - - watch + - get + - list + - watch ``` + However, this might lead to security risk (especially in production environment) since it will grant permission to read `secrets` from all namespaces. To restrict `secret` access and limited to KEDA namespace, you could add `KEDA_RESTRICT_SECRET_ACCESS` as environment variable to both KEDA Operator and KEDA Metrics Server: + ```yaml env: - name: KEDA_RESTRICT_SECRET_ACCESS value: "true" ``` + This allows you to omit `secrets` from the cluster role, which will disallow `TriggerAuthentication` to be used for your triggers if the `TriggerAuthentication` is using secrets. You can, however, still use `ClusterTriggerAuthentication`. diff --git a/content/docs/2.12/concepts/external-scalers.md b/content/docs/2.12/concepts/external-scalers.md index 5badde098..aa00a18a0 100644 --- a/content/docs/2.12/concepts/external-scalers.md +++ b/content/docs/2.12/concepts/external-scalers.md @@ -9,7 +9,7 @@ Built-in scalers run in the KEDA process/pod, while external scalers require an This document describes the external scaler interfaces and how to implement them in Go, Node, and .NET; however for more details on GRPC refer to [the official GRPC documentation](https://grpc.io/docs/) ->Want to learn about existing external scalers? Explore our [external scaler community](https://github.com/kedacore/external-scalers). +> Want to learn about existing external scalers? Explore our [external scaler community](https://github.com/kedacore/external-scalers). ## Overview @@ -45,7 +45,7 @@ The `Scaler` interface defines 3 methods: - `GetMetricSpecForScaling` returns the target value for the HPA definition for the scaler. For more details refer to [Implementing `GetMetricSpec`](#5-implementing-getmetricspec). - `GetMetricsAndActivity` is called on `pollingInterval` and. When activity returns `true`, KEDA will scale to what is returned by the metric limited by `maxReplicaCount` on the ScaledObject/ScaledJob. When `false` is returned, KEDA will scale to `minReplicaCount` or optionally `idleReplicaCount`. More details around the defaults and how these options work together can be found on the [ScaledObjectSpec](https://keda.sh/docs/latest/concepts/scaling-deployments/#scaledobject-spec). -> Refer to the [HPA docs](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale-walkthrough/) for how HPA calculates `replicaCount` based on metric value and target value. KEDA supports both `AverageValue` and `Value` metric target types for external metrics. When `AverageValue` (the default metric type) is used, the metric value returned by the external scaler will be divided by the number of replicas. + > Refer to the [HPA docs](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale-walkthrough/) for how HPA calculates `replicaCount` based on metric value and target value. KEDA supports both `AverageValue` and `Value` metric target types for external metrics. When `AverageValue` (the default metric type) is used, the metric value returned by the external scaler will be divided by the number of replicas. The `PushScaler` interface adds a `Run` method. This method receives a push channel (`active`), on which the scaler can send `true` at any time. The purpose of this mechanism is to initiate a scaling operation independently from `pollingInterval`. @@ -115,7 +115,7 @@ KEDA will attempt a GRPC connection to `service-address.svc.local:9090` immediat } ``` ->**Note**: KEDA will issue all of the above RPC calls except `StreamIsActive` if `spec.triggers.type` is `external`. It _must_ be `external-push` for `StreamIsActive` to be called. +> **Note**: KEDA will issue all of the above RPC calls except `StreamIsActive` if `spec.triggers.type` is `external`. It _must_ be `external-push` for `StreamIsActive` to be called. ## Implementing KEDA external scaler GRPC interface @@ -182,8 +182,6 @@ npm install --save grpc request {{< /collapsible >}} -
- #### 3. Implementing `IsActive` Just like `IsActive(ctx context.Context) (bool, error)` in the go interface, the `IsActive` method in the GRPC interface is called every `pollingInterval` with a `ScaledObjectRef` object that contains the scaledObject name, namespace, and scaler metadata. @@ -310,70 +308,71 @@ public class ExternalScalerService : ExternalScaler.ExternalScalerBase Put the following code into your `index.js` file: ```js -const grpc = require('grpc') -const request = require('request') -const externalScalerProto = grpc.load('externalscaler.proto') +const grpc = require("grpc"); +const request = require("request"); +const externalScalerProto = grpc.load("externalscaler.proto"); -const server = new grpc.Server() +const server = new grpc.Server(); server.addService(externalScalerProto.externalscaler.ExternalScaler.service, { isActive: (call, callback) => { - const longitude = call.request.scalerMetadata.longitude - const latitude = call.request.scalerMetadata.latitude + const longitude = call.request.scalerMetadata.longitude; + const latitude = call.request.scalerMetadata.latitude; if (!longitude || !latitude) { callback({ code: grpc.status.INVALID_ARGUMENT, - details: 'longitude and latitude must be specified', - }) + details: "longitude and latitude must be specified", + }); } else { - const now = new Date() - const yesterday = new Date(new Date().setDate(new Date().getDate()-1)); - - const startTime = `${yesterday.getUTCFullYear()}-${yesterday.getUTCMonth()}-${yesterday.getUTCDay()}` - const endTime = `${now.getUTCFullYear()}-${now.getUTCMonth()}-${now.getUTCDay()}` - const radiusKm = 500 - const query = `format=geojson&starttime=${startTime}&endtime=${endTime}&longitude=${longitude}&latitude=${latitude}&maxradiuskm=${radiusKm}` - - request.get({ - url: `https://earthquake.usgs.gov/fdsnws/event/1/query?${query}`, - json: true, - }, (err, resp, data) => { - if (err) { - callback({ - code: grpc.status.INTERNAL, - details: err, - }) - } else if (resp.statusCode !== 200) { - callback({ - code: grpc.status.INTERNAL, - details: `expected status 200, got ${resp.statusCode}` - }) - } else { - // count how many earthquakes with mag > 1.0 - let count = 0 - data.features.forEach(i => { - if (i.properties.mag > 1.0) { - count++ - } - }) - callback(null, { - result: count > 2, - }) + const now = new Date(); + const yesterday = new Date(new Date().setDate(new Date().getDate() - 1)); + + const startTime = `${yesterday.getUTCFullYear()}-${yesterday.getUTCMonth()}-${yesterday.getUTCDay()}`; + const endTime = `${now.getUTCFullYear()}-${now.getUTCMonth()}-${now.getUTCDay()}`; + const radiusKm = 500; + const query = `format=geojson&starttime=${startTime}&endtime=${endTime}&longitude=${longitude}&latitude=${latitude}&maxradiuskm=${radiusKm}`; + + request.get( + { + url: `https://earthquake.usgs.gov/fdsnws/event/1/query?${query}`, + json: true, + }, + (err, resp, data) => { + if (err) { + callback({ + code: grpc.status.INTERNAL, + details: err, + }); + } else if (resp.statusCode !== 200) { + callback({ + code: grpc.status.INTERNAL, + details: `expected status 200, got ${resp.statusCode}`, + }); + } else { + // count how many earthquakes with mag > 1.0 + let count = 0; + data.features.forEach((i) => { + if (i.properties.mag > 1.0) { + count++; + } + }); + callback(null, { + result: count > 2, + }); + } } - }) + ); } - } -}) + }, +}); -server.bind('127.0.0.1:9090', grpc.ServerCredentials.createInsecure()) -console.log('Server listening on 127.0.0.1:9090') +server.bind("127.0.0.1:9090", grpc.ServerCredentials.createInsecure()); +console.log("Server listening on 127.0.0.1:9090"); -server.start() +server.start(); ``` {{< /collapsible >}} -
- #### 4. Implementing `StreamIsActive` Unlike `IsActive`, `StreamIsActive` is called once when KEDA reconciles the `ScaledObject`, and expects the external scaler to maintain a long-lived connection and push `IsActiveResponse` objects whenever the scaler needs KEDA to activate the deployment. @@ -450,38 +449,36 @@ public override async Task StreamIsActive(ScaledObjectRef request, IServerStream server.addService(externalScalerProto.externalscaler.ExternalScaler.service, { // ... streamIsActive: (call, callback) => { - const longitude = call.request.scalerMetadata.longitude - const latitude = call.request.scalerMetadata.latitude + const longitude = call.request.scalerMetadata.longitude; + const latitude = call.request.scalerMetadata.latitude; if (!longitude || !latitude) { callback({ code: grpc.status.INVALID_ARGUMENT, - details: 'longitude and latitude must be specified', - }) + details: "longitude and latitude must be specified", + }); } else { const interval = setInterval(() => { getEarthquakeCount((err, count) => { if (err) { - console.error(err) + console.error(err); } else if (count > 2) { call.write({ result: true, - }) + }); } - }) + }); }, 1000 * 60 * 60); - call.on('end', () => { - clearInterval(interval) - }) + call.on("end", () => { + clearInterval(interval); + }); } - } -}) + }, +}); ``` {{< /collapsible >}} -
- #### 5. Implementing `GetMetricSpec` `GetMetricSpec` returns the `target` value for [the HPA definition for the scaler](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale-walkthrough/). This scaler will define a static target of 10, but the threshold value is often specified in the metadata for other scalers. @@ -527,19 +524,19 @@ server.addService(externalScalerProto.externalscaler.ExternalScaler.service, { // ... getMetricSpec: (call, callback) => { callback(null, { - metricSpecs: [{ - metricName: 'earthquakeThreshold', - targetSize: 10, - }] - }) - } -}) + metricSpecs: [ + { + metricName: "earthquakeThreshold", + targetSize: 10, + }, + ], + }); + }, +}); ``` {{< /collapsible >}} -
- #### 6. Implementing `GetMetrics` `GetMetrics` returns the value of the metric referred to from `GetMetricSpec`, in this example it's `earthquakeThreshold`. @@ -606,32 +603,34 @@ public override async Task GetMetrics(GetMetricsRequest requ server.addService(externalScalerProto.externalscaler.ExternalScaler.service, { // ... getMetrics: (call, callback) => { - const longitude = call.request.scaledObjectRef.scalerMetadata.longitude - const latitude = call.request.scaledObjectRef.scalerMetadata.latitude + const longitude = call.request.scaledObjectRef.scalerMetadata.longitude; + const latitude = call.request.scaledObjectRef.scalerMetadata.latitude; if (!longitude || !latitude) { callback({ code: grpc.status.INVALID_ARGUMENT, - details: 'longitude and latitude must be specified', - }) + details: "longitude and latitude must be specified", + }); } else { getEarthquakeCount((err, count) => { if (err) { callback({ code: grpc.status.INTERNAL, details: err, - }) + }); } else { callback(null, { - metricValues: [{ - metricName: 'earthquakeThreshold', - metricValue: count, - }] - }) + metricValues: [ + { + metricName: "earthquakeThreshold", + metricValue: count, + }, + ], + }); } - }) + }); } - } -}) + }, +}); ``` {{< /collapsible >}} diff --git a/content/docs/2.12/migration.md b/content/docs/2.12/migration.md index 5ee8a6455..29e8408c8 100644 --- a/content/docs/2.12/migration.md +++ b/content/docs/2.12/migration.md @@ -11,15 +11,17 @@ Please note that you **can not** run both KEDA v1 and v2 on the same Kubernetes KEDA v2 is using a new API namespace for its Custom Resources Definitions (CRD): `keda.sh` instead of `keda.k8s.io` and introduces a new Custom Resource for scaling of Jobs. See full details on KEDA Custom Resources [here](../concepts/#custom-resources-crd). Here's an overview of what's changed: + - [Scaling of Deployments](#scaling-of-deployments) - [Scaling of Jobs](#scaling-of-jobs) - [Improved flexibility & usability of trigger metadata](#improved-flexibility--usability-of-trigger-metadata) - [Scalers](#scalers) - [TriggerAuthentication](#triggerauthentication) - ### Scaling of Deployments + In order to scale `Deployments` with KEDA v2, you need to do only a few modifications to existing v1 `ScaledObjects` definitions, so they comply with v2: + - Change the value of `apiVersion` property from `keda.k8s.io/v1alpha1` to `keda.sh/v1alpha1` - Rename property `spec.scaleTargetRef.deploymentName` to `spec.scaleTargetRef.name` - Rename property `spec.scaleTargetRef.containerName` to `spec.scaleTargetRef.envSourceContainerName` @@ -27,22 +29,23 @@ In order to scale `Deployments` with KEDA v2, you need to do only a few modifica Please see the examples below or refer to the full [v2 ScaledObject Specification](../concepts/scaling-deployments/#scaledobject-spec) - **Example of v1 ScaledObject** +**Example of v1 ScaledObject** + ```yaml apiVersion: keda.k8s.io/v1alpha1 kind: ScaledObject metadata: - name: {scaled-object-name} + name: { scaled-object-name } labels: - deploymentName: {deployment-name} + deploymentName: { deployment-name } spec: scaleTargetRef: - deploymentName: {deployment-name} - containerName: {container-name} - pollingInterval: 30 - cooldownPeriod: 300 - minReplicaCount: 0 - maxReplicaCount: 100 + deploymentName: { deployment-name } + containerName: { container-name } + pollingInterval: 30 + cooldownPeriod: 300 + minReplicaCount: 0 + maxReplicaCount: 100 triggers: # {list of triggers to activate the deployment} ``` @@ -50,18 +53,18 @@ spec: **Example of v2 ScaledObject** ```yaml -apiVersion: keda.sh/v1alpha1 # <--- Property value was changed +apiVersion: keda.sh/v1alpha1 # <--- Property value was changed kind: ScaledObject -metadata: # <--- labels.deploymentName is not needed - name: {scaled-object-name} +metadata: # <--- labels.deploymentName is not needed + name: { scaled-object-name } spec: scaleTargetRef: - name: {deployment-name} # <--- Property name was changed - envSourceContainerName: {container-name} # <--- Property name was changed - pollingInterval: 30 - cooldownPeriod: 300 - minReplicaCount: 0 - maxReplicaCount: 100 + name: { deployment-name } # <--- Property name was changed + envSourceContainerName: { container-name } # <--- Property name was changed + pollingInterval: 30 + cooldownPeriod: 300 + minReplicaCount: 0 + maxReplicaCount: 100 triggers: # {list of triggers to activate the deployment} ``` @@ -69,34 +72,36 @@ spec: ### Scaling of Jobs In order to scale `Jobs` with KEDA v2, you need to do only a few modifications to existing v1 `ScaledObjects` definitions, so they comply with v2: + - Change the value of `apiVersion` property from `keda.k8s.io/v1alpha1` to `keda.sh/v1alpha1` - Change the value of `kind` property from `ScaledObject` to `ScaledJob` - Remove property `spec.scaleType` -- Remove properties `spec.cooldownPeriod` and `spec.minReplicaCount` +- Remove properties `spec.cooldownPeriod` and `spec.minReplicaCount` You can configure `successfulJobsHistoryLimit` and `failedJobsHistoryLimit`. They will remove the old job histories automatically. Please see the examples below or refer to the full [v2 ScaledJob Specification](../concepts/scaling-jobs/#scaledjob-spec) **Example of v1 ScaledObject for Jobs scaling** + ```yaml apiVersion: keda.k8s.io/v1alpha1 kind: ScaledObject metadata: - name: {scaled-object-name} + name: { scaled-object-name } spec: scaleType: job jobTargetRef: - parallelism: 1 + parallelism: 1 completions: 1 activeDeadlineSeconds: 600 - backoffLimit: 6 + backoffLimit: 6 template: # {job template} - pollingInterval: 30 - cooldownPeriod: 300 - minReplicaCount: 0 - maxReplicaCount: 100 + pollingInterval: 30 + cooldownPeriod: 300 + minReplicaCount: 0 + maxReplicaCount: 100 triggers: # {list of triggers to create jobs} ``` @@ -104,21 +109,21 @@ spec: **Example of v2 ScaledJob** ```yaml -apiVersion: keda.sh/v1alpha1 # <--- Property value was changed -kind: ScaledJob # <--- Property value was changed +apiVersion: keda.sh/v1alpha1 # <--- Property value was changed +kind: ScaledJob # <--- Property value was changed metadata: - name: {scaled-job-name} -spec: # <--- spec.scaleType is not needed + name: { scaled-job-name } +spec: # <--- spec.scaleType is not needed jobTargetRef: - parallelism: 1 - completions: 1 + parallelism: 1 + completions: 1 activeDeadlineSeconds: 600 backoffLimit: 6 template: # {job template} - pollingInterval: 30 # <--- spec.cooldownPeriod and spec.minReplicaCount are not needed - successfulJobsHistoryLimit: 5 # <--- property is added - failedJobsHistoryLimit: 5 # <--- Property is added + pollingInterval: 30 # <--- spec.cooldownPeriod and spec.minReplicaCount are not needed + successfulJobsHistoryLimit: 5 # <--- property is added + failedJobsHistoryLimit: 5 # <--- Property is added maxReplicaCount: 100 triggers: # {list of triggers to create jobs} @@ -132,68 +137,74 @@ We've introduced more options to configure trigger metadata to give users more f Here's an overview: -| Scaler | 1.x | 2.0 | -|--------|--------|--------| -| `azure-blob` | `connection` (**Default**: `AzureWebJobsStorage`) | `connectionFromEnv` | -| `azure-monitor` | `activeDirectoryClientId`
`activeDirectoryClientPassword` | `activeDirectoryClientId`
`activeDirectoryClientIdFromEnv`
`activeDirectoryClientPasswordFromEnv` | -| `azure-queue` | `connection` (**Default**: AzureWebJobsStorage) | `connectionFromEnv` | -| `azure-servicebus` | `connection` | `connectionFromEnv` | -| `azure-eventhub` | `storageConnection` (**Default**: `AzureWebJobsStorage`)
`connection` (**Default**: `EventHub`) | `storageConnectionFromEnv`
`connectionFromEnv` | -| `aws-cloudwatch` | `awsAccessKeyID` (**Default**: `AWS_ACCESS_KEY_ID`)
`awsSecretAccessKey` (**Default**: `AWS_SECRET_ACCESS_KEY`) | `awsAccessKeyID`
`awsAccessKeyIDFromEnv`
`awsSecretAccessKeyFromEnv` | -| `aws-kinesis-stream` | `awsAccessKeyID` (**Default**: `AWS_ACCESS_KEY_ID`)
`awsSecretAccessKey` (**Default**: `AWS_SECRET_ACCESS_KEY`) | `awsAccessKeyID`
`awsAccessKeyIDFromEnv`
`awsSecretAccessKeyFromEnv` | -| `aws-sqs-queue` | `awsAccessKeyID` (**Default**: `AWS_ACCESS_KEY_ID`)
`awsSecretAccessKey` (**Default**: `AWS_SECRET_ACCESS_KEY`) | `awsAccessKeyID`
`awsAccessKeyIDFromEnv`
`awsSecretAccessKeyFromEnv` | -| `kafka` | _(none)_ | _(none)_ -| `rabbitmq` | `apiHost`
`host` | ~~`apiHost`~~
`host`
`hostFromEnv` | -| `prometheus` | _(none)_ | _(none)_ | -| `cron` | _(none)_ | _(none)_ | -| `redis` | `address`
`host`
`port`
`password` | `address`
`addressFromEnv`
`host`
`hostFromEnv`
~~`port`~~
`passwordFromEnv` -| `redis-streams` | `address`
`host`
`port`
`password` | `address`
`addressFromEnv`
`host`
`hostFromEnv`
~~`port`~~
`passwordFromEnv` -| `gcp-pubsub` | `credentials` | `credentialsFromEnv` | -| `external` | _(any matching value)_ | _(any matching value with `FromEnv` suffix)_ -| `liiklus` | _(none)_ | _(none)_ | -| `stan` | _(none)_ | _(none)_ | -| `huawei-cloudeye` | | _(none)_ | _(none)_ | -| `postgresql` | `connection`
`password` | `connectionFromEnv`
`passwordFromEnv` | -| `mysql` | `connectionString`
`password` | `connectionStringFromEnv`
`passwordFromEnv` | +| Scaler | 1.x | 2.0 | +| -------------------- | --------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------- | -------- | +| `azure-blob` | `connection` (**Default**: `AzureWebJobsStorage`) | `connectionFromEnv` | +| `azure-monitor` | `activeDirectoryClientId` `activeDirectoryClientPassword` | `activeDirectoryClientId` `activeDirectoryClientIdFromEnv` `activeDirectoryClientPasswordFromEnv` | +| `azure-queue` | `connection` (**Default**: AzureWebJobsStorage) | `connectionFromEnv` | +| `azure-servicebus` | `connection` | `connectionFromEnv` | +| `azure-eventhub` | `storageConnection` (**Default**: `AzureWebJobsStorage`) `connection` (**Default**: `EventHub`) | `storageConnectionFromEnv` `connectionFromEnv` | +| `aws-cloudwatch` | `awsAccessKeyID` (**Default**: `AWS_ACCESS_KEY_ID`) `awsSecretAccessKey` (**Default**: `AWS_SECRET_ACCESS_KEY`) | `awsAccessKeyID` `awsAccessKeyIDFromEnv` `awsSecretAccessKeyFromEnv` | +| `aws-kinesis-stream` | `awsAccessKeyID` (**Default**: `AWS_ACCESS_KEY_ID`) `awsSecretAccessKey` (**Default**: `AWS_SECRET_ACCESS_KEY`) | `awsAccessKeyID` `awsAccessKeyIDFromEnv` `awsSecretAccessKeyFromEnv` | +| `aws-sqs-queue` | `awsAccessKeyID` (**Default**: `AWS_ACCESS_KEY_ID`) `awsSecretAccessKey` (**Default**: `AWS_SECRET_ACCESS_KEY`) | `awsAccessKeyID` `awsAccessKeyIDFromEnv` `awsSecretAccessKeyFromEnv` | +| `kafka` | _(none)_ | _(none)_ | +| `rabbitmq` | `apiHost` `host` | ~~`apiHost`~~ `host` `hostFromEnv` | +| `prometheus` | _(none)_ | _(none)_ | +| `cron` | _(none)_ | _(none)_ | +| `redis` | `address` `host` `port` `password` | `address` `addressFromEnv` `host` `hostFromEnv` ~~`port`~~ `passwordFromEnv` | +| `redis-streams` | `address` `host` `port` `password` | `address` `addressFromEnv` `host` `hostFromEnv` ~~`port`~~ `passwordFromEnv` | +| `gcp-pubsub` | `credentials` | `credentialsFromEnv` | +| `external` | _(any matching value)_ | _(any matching value with `FromEnv` suffix)_ | +| `liiklus` | _(none)_ | _(none)_ | +| `stan` | _(none)_ | _(none)_ | +| `huawei-cloudeye` | | _(none)_ | _(none)_ | +| `postgresql` | `connection` `password` | `connectionFromEnv` `passwordFromEnv` | +| `mysql` | `connectionString` `password` | `connectionStringFromEnv` `passwordFromEnv` | ### Scalers **Azure Service Bus** - - `queueLength` was renamed to `messageCount` + +- `queueLength` was renamed to `messageCount` **Kafka** - - `authMode` property was replaced with `sasl` and `tls` properties. Please refer [documentation](../scalers/apache-kafka/#authentication-parameters) for Kafka Authentication Parameters details. + +- `authMode` property was replaced with `sasl` and `tls` properties. Please refer [documentation](../scalers/apache-kafka/#authentication-parameters) for Kafka Authentication Parameters details. **RabbitMQ** In KEDA 2.0 the RabbitMQ scaler has only `host` parameter, and the protocol for communication can be specified by -`protocol` (http or amqp). The default value is `amqp`. The behavior changes only for scalers that were using HTTP -protocol. +`protocol` (http or amqp). The default value is `amqp`. The behavior changes only for scalers that were using HTTP +protocol. Example of RabbitMQ trigger before 2.0: + ```yaml triggers: -- type: rabbitmq - metadata: - queueLength: '20' - queueName: testqueue - includeUnacked: 'true' - apiHost: 'https://guest:password@localhost:443/vhostname' -``` + - type: rabbitmq + metadata: + queueLength: "20" + queueName: testqueue + includeUnacked: "true" + apiHost: "https://guest:password@localhost:443/vhostname" +``` The same trigger in 2.0: + ```yaml triggers: -- type: rabbitmq - metadata: - queueLength: '20' - queueName: testqueue - protocol: 'http' - host: 'https://guest:password@localhost:443/vhostname' -``` + - type: rabbitmq + metadata: + queueLength: "20" + queueName: testqueue + protocol: "http" + host: "https://guest:password@localhost:443/vhostname" +``` ### TriggerAuthentication + In order to use Authentication via `TriggerAuthentication` with KEDA v2, you need to change: + - Change the value of `apiVersion` property from `keda.k8s.io/v1alpha1` to `keda.sh/v1alpha1` For more details please refer to the full diff --git a/content/docs/2.12/operate/cluster.md b/content/docs/2.12/operate/cluster.md index e60759578..4f200e90a 100644 --- a/content/docs/2.12/operate/cluster.md +++ b/content/docs/2.12/operate/cluster.md @@ -14,14 +14,14 @@ However, maintainers can decide to extend this by supporting more minor versions As a reference, this compatibility matrix shows supported k8s versions per KEDA version: -| KEDA | Kubernetes | -|-----------|---------------| -| v2.12 | v1.26 - v1.28 | -| v2.11 | v1.25 - v1.27 | -| v2.10 | v1.24 - v1.26 | -| v2.9 | v1.23 - v1.25 | -| v2.8 | v1.17 - v1.25 | -| v2.7 | v1.17 - v1.25 | +| KEDA | Kubernetes | +| ----- | ------------- | +| v2.12 | v1.26 - v1.28 | +| v2.11 | v1.25 - v1.27 | +| v2.10 | v1.24 - v1.26 | +| v2.9 | v1.23 - v1.25 | +| v2.8 | v1.17 - v1.25 | +| v2.7 | v1.17 - v1.25 | ### Cluster Capacity @@ -43,12 +43,10 @@ KEDA requires to be accessible inside the cluster to be able to autoscale. Here is an overview of the required ports that need to be accessible for KEDA to work: - | Port | Why? | Remarks | | ------ | -------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `443` | Used by Kubernetes API server to get metrics | Required for all platforms because it uses Control Plane → port 443 on the Service IP range communication.

This is not applicable for Google Cloud. | +| `443` | Used by Kubernetes API server to get metrics | Required for all platforms because it uses Control Plane → port 443 on the Service IP range communication. This is not applicable for Google Cloud. | | `6443` | Used by Kubernetes API server to get metrics | Only required for Google Cloud because it uses Control Plane → port 6443 on the Pod IP range for communication | - ## High Availability @@ -56,10 +54,10 @@ KEDA does not provide full support for high-availability due to upstream limitat Here is an overview of all KEDA deployments and the HA notes: -| Deployment | Support Replicas | Note | -| -------------- | ---------------- |---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Deployment | Support Replicas | Note | +| -------------- | ---------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | Metrics Server | 1 | You can run multiple replicas of our metrics sever, and it is recommended to add the `--enable-aggregator-routing=true` CLI flag to the kube-apiserver so that requests sent to our metrics servers are load balanced. However, [you can only run one active metric server in a Kubernetes cluster serving external.metrics.k8s.io](https://github.com/kubernetes-sigs/custom-metrics-apiserver/issues/70) which has to be the KEDA metric server. | -| Operator | 2 | While you can run multiple replicas of our operator, only one operator instance will be active. The rest will be standing by, which may reduce downtime during a failure. Multiple replicas will not improve the performance of KEDA, it could only reduce a downtime during a failover. | +| Operator | 2 | While you can run multiple replicas of our operator, only one operator instance will be active. The rest will be standing by, which may reduce downtime during a failure. Multiple replicas will not improve the performance of KEDA, it could only reduce a downtime during a failover. | ## HTTP Timeouts @@ -81,7 +79,7 @@ All applicable scalers will use this timeout. Setting a per-scaler timeout is cu ## HTTP connection disable keep alive -Keep alive behaviour is enabled by default for every HTTP connection, this could stack a huge amount of connections (one per scaler) in some scenarios. +Keep alive behaviour is enabled by default for every HTTP connection, this could stack a huge amount of connections (one per scaler) in some scenarios. You can disable keep alive for every HTTP connection by adding the relevant environment variable to both the KEDA Operator, and KEDA Metrics Server deployments: @@ -110,6 +108,7 @@ Our industry has seen an evolution of TLS versions and some are more secure than By default, KEDA uses TLS1.2 as a minimum TLS version given it is the lowest version without vulnerabilities. However, if you need to support another version you can configure it by using the environment variable `KEDA_HTTP_MIN_TLS_VERSION`. For example: + ```yaml - env: KEDA_HTTP_MIN_TLS_VERSION: TLS13 @@ -121,17 +120,18 @@ The following values are allowed: `TLS13`, `TLS12`, `TLS11` and `TLS10`. The Kubernetes client config used within KEDA Operator and KEDA Metrics Adapter can be adjusted by passing the following command-line flags to the binary: -| Adapter Flag | Client Config Setting | Default Value | Description | -| ------------------- | ----------------------- | ------------- | -------------------------------------------------------------- | -| kube-api-qps | cfg.QPS | 20.0 | Set the QPS rate for throttling requests sent to the apiserver | -| kube-api-burst | cfg.Burst | 30 | Set the burst for throttling requests sent to the apiserver | -| disable-compression | cfg.DisableCompression | true | Disable compression for response in k8s restAPI in client-go, see [this Kubernetes issue](https://github.com/kubernetes/kubernetes/issues/112296) for details | +| Adapter Flag | Client Config Setting | Default Value | Description | +| ------------------- | ---------------------- | ------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| kube-api-qps | cfg.QPS | 20.0 | Set the QPS rate for throttling requests sent to the apiserver | +| kube-api-burst | cfg.Burst | 30 | Set the burst for throttling requests sent to the apiserver | +| disable-compression | cfg.DisableCompression | true | Disable compression for response in k8s restAPI in client-go, see [this Kubernetes issue](https://github.com/kubernetes/kubernetes/issues/112296) for details | ## Configure `MaxConcurrentReconciles` for Controllers To implement internal controllers KEDA uses the [controller-runtime project](https://github.com/kubernetes-sigs/controller-runtime), that enables configuration of [MaxConcurrentReconciles property](https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/controller#Options), ie. the maximum number of concurrent reconciles which can be run for a controller. KEDA Operator exposes properties for specifying `MaxConcurrentReconciles` for following controllers/reconcilers: + - `ScaledObjectReconciler` - responsible for watching and managing `ScaledObjects`, ie. validates input trigger specification, starts scaling logic and manages dependent HPA. - `ScaledJobReconciler` - responsible for watching and managing `ScaledJobs` and dependent Kubernetes Jobs @@ -139,14 +139,15 @@ KEDA Metrics Server exposes property for specifying `MaxConcurrentReconciles` fo To modify this properties you can set environment variables on both KEDA Operator and Metrics Server Deployments: -| Environment variable name | Deployment | Default Value | Affected reconciler | -| ------------------------------------- | -------------- | ------------- | -------------------------------------------------------------- | -| KEDA_SCALEDOBJECT_CTRL_MAX_RECONCILES | Operator | 5 | ScaledObjectReconciler | -| KEDA_SCALEDJOB_CTRL_MAX_RECONCILES | Operator | 1 | ScaledJobReconciler | +| Environment variable name | Deployment | Default Value | Affected reconciler | +| ------------------------------------- | ---------- | ------------- | ---------------------- | +| KEDA_SCALEDOBJECT_CTRL_MAX_RECONCILES | Operator | 5 | ScaledObjectReconciler | +| KEDA_SCALEDJOB_CTRL_MAX_RECONCILES | Operator | 1 | ScaledJobReconciler | ## Configure Leader Election Like reconciliation, KEDA also uses the [controller-runtime project](https://github.com/kubernetes-sigs/controller-runtime) for electing the leader replica. The following properties can be configured for either the Operator and Metrics Server Deployment: + - [`LeaseDuration`](https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/manager#Options.LeaseDuration) - [`RenewDeadline`](https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/manager#Options.RenewDeadline) - [`RetryPeriod`](https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/manager#Options.RetryPeriod) @@ -169,25 +170,29 @@ To learn more please refer to [security section](./security#use-your-own-tls-cer ## Restrict Secret Access By default, KEDA requires adding `secrets` to the cluster role as following: + ```yaml - apiGroups: - - "" + - "" resources: - - external - - pods - - secrets - - services + - external + - pods + - secrets + - services verbs: - - get - - list - - watch + - get + - list + - watch ``` + However, this might lead to security risk (especially in production environment) since it will grant permission to read `secrets` from all namespaces. To restrict `secret` access and limited to KEDA namespace, you could add `KEDA_RESTRICT_SECRET_ACCESS` as environment variable to both KEDA Operator and KEDA Metrics Server: + ```yaml env: - name: KEDA_RESTRICT_SECRET_ACCESS value: "true" ``` + This allows you to omit `secrets` from the cluster role, which will disallow `TriggerAuthentication` to be used for your triggers if the `TriggerAuthentication` is using secrets. You can, however, still use `ClusterTriggerAuthentication`. diff --git a/content/docs/2.13/concepts/external-scalers.md b/content/docs/2.13/concepts/external-scalers.md index 5badde098..aa00a18a0 100644 --- a/content/docs/2.13/concepts/external-scalers.md +++ b/content/docs/2.13/concepts/external-scalers.md @@ -9,7 +9,7 @@ Built-in scalers run in the KEDA process/pod, while external scalers require an This document describes the external scaler interfaces and how to implement them in Go, Node, and .NET; however for more details on GRPC refer to [the official GRPC documentation](https://grpc.io/docs/) ->Want to learn about existing external scalers? Explore our [external scaler community](https://github.com/kedacore/external-scalers). +> Want to learn about existing external scalers? Explore our [external scaler community](https://github.com/kedacore/external-scalers). ## Overview @@ -45,7 +45,7 @@ The `Scaler` interface defines 3 methods: - `GetMetricSpecForScaling` returns the target value for the HPA definition for the scaler. For more details refer to [Implementing `GetMetricSpec`](#5-implementing-getmetricspec). - `GetMetricsAndActivity` is called on `pollingInterval` and. When activity returns `true`, KEDA will scale to what is returned by the metric limited by `maxReplicaCount` on the ScaledObject/ScaledJob. When `false` is returned, KEDA will scale to `minReplicaCount` or optionally `idleReplicaCount`. More details around the defaults and how these options work together can be found on the [ScaledObjectSpec](https://keda.sh/docs/latest/concepts/scaling-deployments/#scaledobject-spec). -> Refer to the [HPA docs](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale-walkthrough/) for how HPA calculates `replicaCount` based on metric value and target value. KEDA supports both `AverageValue` and `Value` metric target types for external metrics. When `AverageValue` (the default metric type) is used, the metric value returned by the external scaler will be divided by the number of replicas. + > Refer to the [HPA docs](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale-walkthrough/) for how HPA calculates `replicaCount` based on metric value and target value. KEDA supports both `AverageValue` and `Value` metric target types for external metrics. When `AverageValue` (the default metric type) is used, the metric value returned by the external scaler will be divided by the number of replicas. The `PushScaler` interface adds a `Run` method. This method receives a push channel (`active`), on which the scaler can send `true` at any time. The purpose of this mechanism is to initiate a scaling operation independently from `pollingInterval`. @@ -115,7 +115,7 @@ KEDA will attempt a GRPC connection to `service-address.svc.local:9090` immediat } ``` ->**Note**: KEDA will issue all of the above RPC calls except `StreamIsActive` if `spec.triggers.type` is `external`. It _must_ be `external-push` for `StreamIsActive` to be called. +> **Note**: KEDA will issue all of the above RPC calls except `StreamIsActive` if `spec.triggers.type` is `external`. It _must_ be `external-push` for `StreamIsActive` to be called. ## Implementing KEDA external scaler GRPC interface @@ -182,8 +182,6 @@ npm install --save grpc request {{< /collapsible >}} -
- #### 3. Implementing `IsActive` Just like `IsActive(ctx context.Context) (bool, error)` in the go interface, the `IsActive` method in the GRPC interface is called every `pollingInterval` with a `ScaledObjectRef` object that contains the scaledObject name, namespace, and scaler metadata. @@ -310,70 +308,71 @@ public class ExternalScalerService : ExternalScaler.ExternalScalerBase Put the following code into your `index.js` file: ```js -const grpc = require('grpc') -const request = require('request') -const externalScalerProto = grpc.load('externalscaler.proto') +const grpc = require("grpc"); +const request = require("request"); +const externalScalerProto = grpc.load("externalscaler.proto"); -const server = new grpc.Server() +const server = new grpc.Server(); server.addService(externalScalerProto.externalscaler.ExternalScaler.service, { isActive: (call, callback) => { - const longitude = call.request.scalerMetadata.longitude - const latitude = call.request.scalerMetadata.latitude + const longitude = call.request.scalerMetadata.longitude; + const latitude = call.request.scalerMetadata.latitude; if (!longitude || !latitude) { callback({ code: grpc.status.INVALID_ARGUMENT, - details: 'longitude and latitude must be specified', - }) + details: "longitude and latitude must be specified", + }); } else { - const now = new Date() - const yesterday = new Date(new Date().setDate(new Date().getDate()-1)); - - const startTime = `${yesterday.getUTCFullYear()}-${yesterday.getUTCMonth()}-${yesterday.getUTCDay()}` - const endTime = `${now.getUTCFullYear()}-${now.getUTCMonth()}-${now.getUTCDay()}` - const radiusKm = 500 - const query = `format=geojson&starttime=${startTime}&endtime=${endTime}&longitude=${longitude}&latitude=${latitude}&maxradiuskm=${radiusKm}` - - request.get({ - url: `https://earthquake.usgs.gov/fdsnws/event/1/query?${query}`, - json: true, - }, (err, resp, data) => { - if (err) { - callback({ - code: grpc.status.INTERNAL, - details: err, - }) - } else if (resp.statusCode !== 200) { - callback({ - code: grpc.status.INTERNAL, - details: `expected status 200, got ${resp.statusCode}` - }) - } else { - // count how many earthquakes with mag > 1.0 - let count = 0 - data.features.forEach(i => { - if (i.properties.mag > 1.0) { - count++ - } - }) - callback(null, { - result: count > 2, - }) + const now = new Date(); + const yesterday = new Date(new Date().setDate(new Date().getDate() - 1)); + + const startTime = `${yesterday.getUTCFullYear()}-${yesterday.getUTCMonth()}-${yesterday.getUTCDay()}`; + const endTime = `${now.getUTCFullYear()}-${now.getUTCMonth()}-${now.getUTCDay()}`; + const radiusKm = 500; + const query = `format=geojson&starttime=${startTime}&endtime=${endTime}&longitude=${longitude}&latitude=${latitude}&maxradiuskm=${radiusKm}`; + + request.get( + { + url: `https://earthquake.usgs.gov/fdsnws/event/1/query?${query}`, + json: true, + }, + (err, resp, data) => { + if (err) { + callback({ + code: grpc.status.INTERNAL, + details: err, + }); + } else if (resp.statusCode !== 200) { + callback({ + code: grpc.status.INTERNAL, + details: `expected status 200, got ${resp.statusCode}`, + }); + } else { + // count how many earthquakes with mag > 1.0 + let count = 0; + data.features.forEach((i) => { + if (i.properties.mag > 1.0) { + count++; + } + }); + callback(null, { + result: count > 2, + }); + } } - }) + ); } - } -}) + }, +}); -server.bind('127.0.0.1:9090', grpc.ServerCredentials.createInsecure()) -console.log('Server listening on 127.0.0.1:9090') +server.bind("127.0.0.1:9090", grpc.ServerCredentials.createInsecure()); +console.log("Server listening on 127.0.0.1:9090"); -server.start() +server.start(); ``` {{< /collapsible >}} -
- #### 4. Implementing `StreamIsActive` Unlike `IsActive`, `StreamIsActive` is called once when KEDA reconciles the `ScaledObject`, and expects the external scaler to maintain a long-lived connection and push `IsActiveResponse` objects whenever the scaler needs KEDA to activate the deployment. @@ -450,38 +449,36 @@ public override async Task StreamIsActive(ScaledObjectRef request, IServerStream server.addService(externalScalerProto.externalscaler.ExternalScaler.service, { // ... streamIsActive: (call, callback) => { - const longitude = call.request.scalerMetadata.longitude - const latitude = call.request.scalerMetadata.latitude + const longitude = call.request.scalerMetadata.longitude; + const latitude = call.request.scalerMetadata.latitude; if (!longitude || !latitude) { callback({ code: grpc.status.INVALID_ARGUMENT, - details: 'longitude and latitude must be specified', - }) + details: "longitude and latitude must be specified", + }); } else { const interval = setInterval(() => { getEarthquakeCount((err, count) => { if (err) { - console.error(err) + console.error(err); } else if (count > 2) { call.write({ result: true, - }) + }); } - }) + }); }, 1000 * 60 * 60); - call.on('end', () => { - clearInterval(interval) - }) + call.on("end", () => { + clearInterval(interval); + }); } - } -}) + }, +}); ``` {{< /collapsible >}} -
- #### 5. Implementing `GetMetricSpec` `GetMetricSpec` returns the `target` value for [the HPA definition for the scaler](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale-walkthrough/). This scaler will define a static target of 10, but the threshold value is often specified in the metadata for other scalers. @@ -527,19 +524,19 @@ server.addService(externalScalerProto.externalscaler.ExternalScaler.service, { // ... getMetricSpec: (call, callback) => { callback(null, { - metricSpecs: [{ - metricName: 'earthquakeThreshold', - targetSize: 10, - }] - }) - } -}) + metricSpecs: [ + { + metricName: "earthquakeThreshold", + targetSize: 10, + }, + ], + }); + }, +}); ``` {{< /collapsible >}} -
- #### 6. Implementing `GetMetrics` `GetMetrics` returns the value of the metric referred to from `GetMetricSpec`, in this example it's `earthquakeThreshold`. @@ -606,32 +603,34 @@ public override async Task GetMetrics(GetMetricsRequest requ server.addService(externalScalerProto.externalscaler.ExternalScaler.service, { // ... getMetrics: (call, callback) => { - const longitude = call.request.scaledObjectRef.scalerMetadata.longitude - const latitude = call.request.scaledObjectRef.scalerMetadata.latitude + const longitude = call.request.scaledObjectRef.scalerMetadata.longitude; + const latitude = call.request.scaledObjectRef.scalerMetadata.latitude; if (!longitude || !latitude) { callback({ code: grpc.status.INVALID_ARGUMENT, - details: 'longitude and latitude must be specified', - }) + details: "longitude and latitude must be specified", + }); } else { getEarthquakeCount((err, count) => { if (err) { callback({ code: grpc.status.INTERNAL, details: err, - }) + }); } else { callback(null, { - metricValues: [{ - metricName: 'earthquakeThreshold', - metricValue: count, - }] - }) + metricValues: [ + { + metricName: "earthquakeThreshold", + metricValue: count, + }, + ], + }); } - }) + }); } - } -}) + }, +}); ``` {{< /collapsible >}} diff --git a/content/docs/2.13/migration.md b/content/docs/2.13/migration.md index 5ee8a6455..29e8408c8 100644 --- a/content/docs/2.13/migration.md +++ b/content/docs/2.13/migration.md @@ -11,15 +11,17 @@ Please note that you **can not** run both KEDA v1 and v2 on the same Kubernetes KEDA v2 is using a new API namespace for its Custom Resources Definitions (CRD): `keda.sh` instead of `keda.k8s.io` and introduces a new Custom Resource for scaling of Jobs. See full details on KEDA Custom Resources [here](../concepts/#custom-resources-crd). Here's an overview of what's changed: + - [Scaling of Deployments](#scaling-of-deployments) - [Scaling of Jobs](#scaling-of-jobs) - [Improved flexibility & usability of trigger metadata](#improved-flexibility--usability-of-trigger-metadata) - [Scalers](#scalers) - [TriggerAuthentication](#triggerauthentication) - ### Scaling of Deployments + In order to scale `Deployments` with KEDA v2, you need to do only a few modifications to existing v1 `ScaledObjects` definitions, so they comply with v2: + - Change the value of `apiVersion` property from `keda.k8s.io/v1alpha1` to `keda.sh/v1alpha1` - Rename property `spec.scaleTargetRef.deploymentName` to `spec.scaleTargetRef.name` - Rename property `spec.scaleTargetRef.containerName` to `spec.scaleTargetRef.envSourceContainerName` @@ -27,22 +29,23 @@ In order to scale `Deployments` with KEDA v2, you need to do only a few modifica Please see the examples below or refer to the full [v2 ScaledObject Specification](../concepts/scaling-deployments/#scaledobject-spec) - **Example of v1 ScaledObject** +**Example of v1 ScaledObject** + ```yaml apiVersion: keda.k8s.io/v1alpha1 kind: ScaledObject metadata: - name: {scaled-object-name} + name: { scaled-object-name } labels: - deploymentName: {deployment-name} + deploymentName: { deployment-name } spec: scaleTargetRef: - deploymentName: {deployment-name} - containerName: {container-name} - pollingInterval: 30 - cooldownPeriod: 300 - minReplicaCount: 0 - maxReplicaCount: 100 + deploymentName: { deployment-name } + containerName: { container-name } + pollingInterval: 30 + cooldownPeriod: 300 + minReplicaCount: 0 + maxReplicaCount: 100 triggers: # {list of triggers to activate the deployment} ``` @@ -50,18 +53,18 @@ spec: **Example of v2 ScaledObject** ```yaml -apiVersion: keda.sh/v1alpha1 # <--- Property value was changed +apiVersion: keda.sh/v1alpha1 # <--- Property value was changed kind: ScaledObject -metadata: # <--- labels.deploymentName is not needed - name: {scaled-object-name} +metadata: # <--- labels.deploymentName is not needed + name: { scaled-object-name } spec: scaleTargetRef: - name: {deployment-name} # <--- Property name was changed - envSourceContainerName: {container-name} # <--- Property name was changed - pollingInterval: 30 - cooldownPeriod: 300 - minReplicaCount: 0 - maxReplicaCount: 100 + name: { deployment-name } # <--- Property name was changed + envSourceContainerName: { container-name } # <--- Property name was changed + pollingInterval: 30 + cooldownPeriod: 300 + minReplicaCount: 0 + maxReplicaCount: 100 triggers: # {list of triggers to activate the deployment} ``` @@ -69,34 +72,36 @@ spec: ### Scaling of Jobs In order to scale `Jobs` with KEDA v2, you need to do only a few modifications to existing v1 `ScaledObjects` definitions, so they comply with v2: + - Change the value of `apiVersion` property from `keda.k8s.io/v1alpha1` to `keda.sh/v1alpha1` - Change the value of `kind` property from `ScaledObject` to `ScaledJob` - Remove property `spec.scaleType` -- Remove properties `spec.cooldownPeriod` and `spec.minReplicaCount` +- Remove properties `spec.cooldownPeriod` and `spec.minReplicaCount` You can configure `successfulJobsHistoryLimit` and `failedJobsHistoryLimit`. They will remove the old job histories automatically. Please see the examples below or refer to the full [v2 ScaledJob Specification](../concepts/scaling-jobs/#scaledjob-spec) **Example of v1 ScaledObject for Jobs scaling** + ```yaml apiVersion: keda.k8s.io/v1alpha1 kind: ScaledObject metadata: - name: {scaled-object-name} + name: { scaled-object-name } spec: scaleType: job jobTargetRef: - parallelism: 1 + parallelism: 1 completions: 1 activeDeadlineSeconds: 600 - backoffLimit: 6 + backoffLimit: 6 template: # {job template} - pollingInterval: 30 - cooldownPeriod: 300 - minReplicaCount: 0 - maxReplicaCount: 100 + pollingInterval: 30 + cooldownPeriod: 300 + minReplicaCount: 0 + maxReplicaCount: 100 triggers: # {list of triggers to create jobs} ``` @@ -104,21 +109,21 @@ spec: **Example of v2 ScaledJob** ```yaml -apiVersion: keda.sh/v1alpha1 # <--- Property value was changed -kind: ScaledJob # <--- Property value was changed +apiVersion: keda.sh/v1alpha1 # <--- Property value was changed +kind: ScaledJob # <--- Property value was changed metadata: - name: {scaled-job-name} -spec: # <--- spec.scaleType is not needed + name: { scaled-job-name } +spec: # <--- spec.scaleType is not needed jobTargetRef: - parallelism: 1 - completions: 1 + parallelism: 1 + completions: 1 activeDeadlineSeconds: 600 backoffLimit: 6 template: # {job template} - pollingInterval: 30 # <--- spec.cooldownPeriod and spec.minReplicaCount are not needed - successfulJobsHistoryLimit: 5 # <--- property is added - failedJobsHistoryLimit: 5 # <--- Property is added + pollingInterval: 30 # <--- spec.cooldownPeriod and spec.minReplicaCount are not needed + successfulJobsHistoryLimit: 5 # <--- property is added + failedJobsHistoryLimit: 5 # <--- Property is added maxReplicaCount: 100 triggers: # {list of triggers to create jobs} @@ -132,68 +137,74 @@ We've introduced more options to configure trigger metadata to give users more f Here's an overview: -| Scaler | 1.x | 2.0 | -|--------|--------|--------| -| `azure-blob` | `connection` (**Default**: `AzureWebJobsStorage`) | `connectionFromEnv` | -| `azure-monitor` | `activeDirectoryClientId`
`activeDirectoryClientPassword` | `activeDirectoryClientId`
`activeDirectoryClientIdFromEnv`
`activeDirectoryClientPasswordFromEnv` | -| `azure-queue` | `connection` (**Default**: AzureWebJobsStorage) | `connectionFromEnv` | -| `azure-servicebus` | `connection` | `connectionFromEnv` | -| `azure-eventhub` | `storageConnection` (**Default**: `AzureWebJobsStorage`)
`connection` (**Default**: `EventHub`) | `storageConnectionFromEnv`
`connectionFromEnv` | -| `aws-cloudwatch` | `awsAccessKeyID` (**Default**: `AWS_ACCESS_KEY_ID`)
`awsSecretAccessKey` (**Default**: `AWS_SECRET_ACCESS_KEY`) | `awsAccessKeyID`
`awsAccessKeyIDFromEnv`
`awsSecretAccessKeyFromEnv` | -| `aws-kinesis-stream` | `awsAccessKeyID` (**Default**: `AWS_ACCESS_KEY_ID`)
`awsSecretAccessKey` (**Default**: `AWS_SECRET_ACCESS_KEY`) | `awsAccessKeyID`
`awsAccessKeyIDFromEnv`
`awsSecretAccessKeyFromEnv` | -| `aws-sqs-queue` | `awsAccessKeyID` (**Default**: `AWS_ACCESS_KEY_ID`)
`awsSecretAccessKey` (**Default**: `AWS_SECRET_ACCESS_KEY`) | `awsAccessKeyID`
`awsAccessKeyIDFromEnv`
`awsSecretAccessKeyFromEnv` | -| `kafka` | _(none)_ | _(none)_ -| `rabbitmq` | `apiHost`
`host` | ~~`apiHost`~~
`host`
`hostFromEnv` | -| `prometheus` | _(none)_ | _(none)_ | -| `cron` | _(none)_ | _(none)_ | -| `redis` | `address`
`host`
`port`
`password` | `address`
`addressFromEnv`
`host`
`hostFromEnv`
~~`port`~~
`passwordFromEnv` -| `redis-streams` | `address`
`host`
`port`
`password` | `address`
`addressFromEnv`
`host`
`hostFromEnv`
~~`port`~~
`passwordFromEnv` -| `gcp-pubsub` | `credentials` | `credentialsFromEnv` | -| `external` | _(any matching value)_ | _(any matching value with `FromEnv` suffix)_ -| `liiklus` | _(none)_ | _(none)_ | -| `stan` | _(none)_ | _(none)_ | -| `huawei-cloudeye` | | _(none)_ | _(none)_ | -| `postgresql` | `connection`
`password` | `connectionFromEnv`
`passwordFromEnv` | -| `mysql` | `connectionString`
`password` | `connectionStringFromEnv`
`passwordFromEnv` | +| Scaler | 1.x | 2.0 | +| -------------------- | --------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------- | -------- | +| `azure-blob` | `connection` (**Default**: `AzureWebJobsStorage`) | `connectionFromEnv` | +| `azure-monitor` | `activeDirectoryClientId` `activeDirectoryClientPassword` | `activeDirectoryClientId` `activeDirectoryClientIdFromEnv` `activeDirectoryClientPasswordFromEnv` | +| `azure-queue` | `connection` (**Default**: AzureWebJobsStorage) | `connectionFromEnv` | +| `azure-servicebus` | `connection` | `connectionFromEnv` | +| `azure-eventhub` | `storageConnection` (**Default**: `AzureWebJobsStorage`) `connection` (**Default**: `EventHub`) | `storageConnectionFromEnv` `connectionFromEnv` | +| `aws-cloudwatch` | `awsAccessKeyID` (**Default**: `AWS_ACCESS_KEY_ID`) `awsSecretAccessKey` (**Default**: `AWS_SECRET_ACCESS_KEY`) | `awsAccessKeyID` `awsAccessKeyIDFromEnv` `awsSecretAccessKeyFromEnv` | +| `aws-kinesis-stream` | `awsAccessKeyID` (**Default**: `AWS_ACCESS_KEY_ID`) `awsSecretAccessKey` (**Default**: `AWS_SECRET_ACCESS_KEY`) | `awsAccessKeyID` `awsAccessKeyIDFromEnv` `awsSecretAccessKeyFromEnv` | +| `aws-sqs-queue` | `awsAccessKeyID` (**Default**: `AWS_ACCESS_KEY_ID`) `awsSecretAccessKey` (**Default**: `AWS_SECRET_ACCESS_KEY`) | `awsAccessKeyID` `awsAccessKeyIDFromEnv` `awsSecretAccessKeyFromEnv` | +| `kafka` | _(none)_ | _(none)_ | +| `rabbitmq` | `apiHost` `host` | ~~`apiHost`~~ `host` `hostFromEnv` | +| `prometheus` | _(none)_ | _(none)_ | +| `cron` | _(none)_ | _(none)_ | +| `redis` | `address` `host` `port` `password` | `address` `addressFromEnv` `host` `hostFromEnv` ~~`port`~~ `passwordFromEnv` | +| `redis-streams` | `address` `host` `port` `password` | `address` `addressFromEnv` `host` `hostFromEnv` ~~`port`~~ `passwordFromEnv` | +| `gcp-pubsub` | `credentials` | `credentialsFromEnv` | +| `external` | _(any matching value)_ | _(any matching value with `FromEnv` suffix)_ | +| `liiklus` | _(none)_ | _(none)_ | +| `stan` | _(none)_ | _(none)_ | +| `huawei-cloudeye` | | _(none)_ | _(none)_ | +| `postgresql` | `connection` `password` | `connectionFromEnv` `passwordFromEnv` | +| `mysql` | `connectionString` `password` | `connectionStringFromEnv` `passwordFromEnv` | ### Scalers **Azure Service Bus** - - `queueLength` was renamed to `messageCount` + +- `queueLength` was renamed to `messageCount` **Kafka** - - `authMode` property was replaced with `sasl` and `tls` properties. Please refer [documentation](../scalers/apache-kafka/#authentication-parameters) for Kafka Authentication Parameters details. + +- `authMode` property was replaced with `sasl` and `tls` properties. Please refer [documentation](../scalers/apache-kafka/#authentication-parameters) for Kafka Authentication Parameters details. **RabbitMQ** In KEDA 2.0 the RabbitMQ scaler has only `host` parameter, and the protocol for communication can be specified by -`protocol` (http or amqp). The default value is `amqp`. The behavior changes only for scalers that were using HTTP -protocol. +`protocol` (http or amqp). The default value is `amqp`. The behavior changes only for scalers that were using HTTP +protocol. Example of RabbitMQ trigger before 2.0: + ```yaml triggers: -- type: rabbitmq - metadata: - queueLength: '20' - queueName: testqueue - includeUnacked: 'true' - apiHost: 'https://guest:password@localhost:443/vhostname' -``` + - type: rabbitmq + metadata: + queueLength: "20" + queueName: testqueue + includeUnacked: "true" + apiHost: "https://guest:password@localhost:443/vhostname" +``` The same trigger in 2.0: + ```yaml triggers: -- type: rabbitmq - metadata: - queueLength: '20' - queueName: testqueue - protocol: 'http' - host: 'https://guest:password@localhost:443/vhostname' -``` + - type: rabbitmq + metadata: + queueLength: "20" + queueName: testqueue + protocol: "http" + host: "https://guest:password@localhost:443/vhostname" +``` ### TriggerAuthentication + In order to use Authentication via `TriggerAuthentication` with KEDA v2, you need to change: + - Change the value of `apiVersion` property from `keda.k8s.io/v1alpha1` to `keda.sh/v1alpha1` For more details please refer to the full diff --git a/content/docs/2.13/operate/cluster.md b/content/docs/2.13/operate/cluster.md index 70f84c8f4..4f9c61068 100644 --- a/content/docs/2.13/operate/cluster.md +++ b/content/docs/2.13/operate/cluster.md @@ -14,15 +14,15 @@ However, maintainers can decide to extend this by supporting more minor versions As a reference, this compatibility matrix shows supported k8s versions per KEDA version: -| KEDA | Kubernetes | -|-----------|---------------| -| v2.13 | v1.27 - v1.29 | -| v2.12 | v1.26 - v1.28 | -| v2.11 | v1.25 - v1.27 | -| v2.10 | v1.24 - v1.26 | -| v2.9 | v1.23 - v1.25 | -| v2.8 | v1.17 - v1.25 | -| v2.7 | v1.17 - v1.25 | +| KEDA | Kubernetes | +| ----- | ------------- | +| v2.13 | v1.27 - v1.29 | +| v2.12 | v1.26 - v1.28 | +| v2.11 | v1.25 - v1.27 | +| v2.10 | v1.24 - v1.26 | +| v2.9 | v1.23 - v1.25 | +| v2.8 | v1.17 - v1.25 | +| v2.7 | v1.17 - v1.25 | ### Cluster Capacity @@ -44,12 +44,10 @@ KEDA requires to be accessible inside the cluster to be able to autoscale. Here is an overview of the required ports that need to be accessible for KEDA to work: - | Port | Why? | Remarks | | ------ | -------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `443` | Used by Kubernetes API server to get metrics | Required for all platforms because it uses Control Plane → port 443 on the Service IP range communication.

This is not applicable for Google Cloud. | +| `443` | Used by Kubernetes API server to get metrics | Required for all platforms because it uses Control Plane → port 443 on the Service IP range communication. This is not applicable for Google Cloud. | | `6443` | Used by Kubernetes API server to get metrics | Only required for Google Cloud because it uses Control Plane → port 6443 on the Pod IP range for communication | - ## High Availability @@ -57,10 +55,10 @@ KEDA does not provide full support for high-availability due to upstream limitat Here is an overview of all KEDA deployments and the HA notes: -| Deployment | Support Replicas | Note | -| -------------- | ---------------- |---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Deployment | Support Replicas | Note | +| -------------- | ---------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | Metrics Server | 1 | You can run multiple replicas of our metrics sever, and it is recommended to add the `--enable-aggregator-routing=true` CLI flag to the kube-apiserver so that requests sent to our metrics servers are load balanced. However, [you can only run one active metric server in a Kubernetes cluster serving external.metrics.k8s.io](https://github.com/kubernetes-sigs/custom-metrics-apiserver/issues/70) which has to be the KEDA metric server. | -| Operator | 2 | While you can run multiple replicas of our operator, only one operator instance will be active. The rest will be standing by, which may reduce downtime during a failure. Multiple replicas will not improve the performance of KEDA, it could only reduce a downtime during a failover. | +| Operator | 2 | While you can run multiple replicas of our operator, only one operator instance will be active. The rest will be standing by, which may reduce downtime during a failure. Multiple replicas will not improve the performance of KEDA, it could only reduce a downtime during a failover. | ## HTTP Timeouts @@ -82,7 +80,7 @@ All applicable scalers will use this timeout. Setting a per-scaler timeout is cu ## HTTP connection disable keep alive -Keep alive behaviour is enabled by default for every HTTP connection, this could stack a huge amount of connections (one per scaler) in some scenarios. +Keep alive behaviour is enabled by default for every HTTP connection, this could stack a huge amount of connections (one per scaler) in some scenarios. You can disable keep alive for every HTTP connection by adding the relevant environment variable to both the KEDA Operator, and KEDA Metrics Server deployments: @@ -111,6 +109,7 @@ Our industry has seen an evolution of TLS versions and some are more secure than By default, KEDA uses TLS1.2 as a minimum TLS version given it is the lowest version without vulnerabilities. However, if you need to support another version you can configure it by using the environment variable `KEDA_HTTP_MIN_TLS_VERSION`. For example: + ```yaml - env: KEDA_HTTP_MIN_TLS_VERSION: TLS13 @@ -122,17 +121,18 @@ The following values are allowed: `TLS13`, `TLS12`, `TLS11` and `TLS10`. The Kubernetes client config used within KEDA Operator and KEDA Metrics Adapter can be adjusted by passing the following command-line flags to the binary: -| Adapter Flag | Client Config Setting | Default Value | Description | -| ------------------- | ----------------------- | ------------- | -------------------------------------------------------------- | -| kube-api-qps | cfg.QPS | 20.0 | Set the QPS rate for throttling requests sent to the apiserver | -| kube-api-burst | cfg.Burst | 30 | Set the burst for throttling requests sent to the apiserver | -| disable-compression | cfg.DisableCompression | true | Disable compression for response in k8s restAPI in client-go, see [this Kubernetes issue](https://github.com/kubernetes/kubernetes/issues/112296) for details | +| Adapter Flag | Client Config Setting | Default Value | Description | +| ------------------- | ---------------------- | ------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| kube-api-qps | cfg.QPS | 20.0 | Set the QPS rate for throttling requests sent to the apiserver | +| kube-api-burst | cfg.Burst | 30 | Set the burst for throttling requests sent to the apiserver | +| disable-compression | cfg.DisableCompression | true | Disable compression for response in k8s restAPI in client-go, see [this Kubernetes issue](https://github.com/kubernetes/kubernetes/issues/112296) for details | ## Configure `MaxConcurrentReconciles` for Controllers To implement internal controllers KEDA uses the [controller-runtime project](https://github.com/kubernetes-sigs/controller-runtime), that enables configuration of [MaxConcurrentReconciles property](https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/controller#Options), ie. the maximum number of concurrent reconciles which can be run for a controller. KEDA Operator exposes properties for specifying `MaxConcurrentReconciles` for following controllers/reconcilers: + - `ScaledObjectReconciler` - responsible for watching and managing `ScaledObjects`, ie. validates input trigger specification, starts scaling logic and manages dependent HPA. - `ScaledJobReconciler` - responsible for watching and managing `ScaledJobs` and dependent Kubernetes Jobs @@ -140,14 +140,15 @@ KEDA Metrics Server exposes property for specifying `MaxConcurrentReconciles` fo To modify this properties you can set environment variables on both KEDA Operator and Metrics Server Deployments: -| Environment variable name | Deployment | Default Value | Affected reconciler | -| ------------------------------------- | -------------- | ------------- | -------------------------------------------------------------- | -| KEDA_SCALEDOBJECT_CTRL_MAX_RECONCILES | Operator | 5 | ScaledObjectReconciler | -| KEDA_SCALEDJOB_CTRL_MAX_RECONCILES | Operator | 1 | ScaledJobReconciler | +| Environment variable name | Deployment | Default Value | Affected reconciler | +| ------------------------------------- | ---------- | ------------- | ---------------------- | +| KEDA_SCALEDOBJECT_CTRL_MAX_RECONCILES | Operator | 5 | ScaledObjectReconciler | +| KEDA_SCALEDJOB_CTRL_MAX_RECONCILES | Operator | 1 | ScaledJobReconciler | ## Configure Leader Election Like reconciliation, KEDA also uses the [controller-runtime project](https://github.com/kubernetes-sigs/controller-runtime) for electing the leader replica. The following properties can be configured for either the Operator and Metrics Server Deployment: + - [`LeaseDuration`](https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/manager#Options.LeaseDuration) - [`RenewDeadline`](https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/manager#Options.RenewDeadline) - [`RetryPeriod`](https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/manager#Options.RetryPeriod) @@ -170,25 +171,29 @@ To learn more please refer to [security section](./security#use-your-own-tls-cer ## Restrict Secret Access By default, KEDA requires adding `secrets` to the cluster role as following: + ```yaml - apiGroups: - - "" + - "" resources: - - external - - pods - - secrets - - services + - external + - pods + - secrets + - services verbs: - - get - - list - - watch + - get + - list + - watch ``` + However, this might lead to security risk (especially in production environment) since it will grant permission to read `secrets` from all namespaces. To restrict `secret` access and limited to KEDA namespace, you could add `KEDA_RESTRICT_SECRET_ACCESS` as environment variable to both KEDA Operator and KEDA Metrics Server: + ```yaml env: - name: KEDA_RESTRICT_SECRET_ACCESS value: "true" ``` + This allows you to omit `secrets` from the cluster role, which will disallow `TriggerAuthentication` to be used for your triggers if the `TriggerAuthentication` is using secrets. You can, however, still use `ClusterTriggerAuthentication`. diff --git a/content/docs/2.14/concepts/external-scalers.md b/content/docs/2.14/concepts/external-scalers.md index 5badde098..aa00a18a0 100644 --- a/content/docs/2.14/concepts/external-scalers.md +++ b/content/docs/2.14/concepts/external-scalers.md @@ -9,7 +9,7 @@ Built-in scalers run in the KEDA process/pod, while external scalers require an This document describes the external scaler interfaces and how to implement them in Go, Node, and .NET; however for more details on GRPC refer to [the official GRPC documentation](https://grpc.io/docs/) ->Want to learn about existing external scalers? Explore our [external scaler community](https://github.com/kedacore/external-scalers). +> Want to learn about existing external scalers? Explore our [external scaler community](https://github.com/kedacore/external-scalers). ## Overview @@ -45,7 +45,7 @@ The `Scaler` interface defines 3 methods: - `GetMetricSpecForScaling` returns the target value for the HPA definition for the scaler. For more details refer to [Implementing `GetMetricSpec`](#5-implementing-getmetricspec). - `GetMetricsAndActivity` is called on `pollingInterval` and. When activity returns `true`, KEDA will scale to what is returned by the metric limited by `maxReplicaCount` on the ScaledObject/ScaledJob. When `false` is returned, KEDA will scale to `minReplicaCount` or optionally `idleReplicaCount`. More details around the defaults and how these options work together can be found on the [ScaledObjectSpec](https://keda.sh/docs/latest/concepts/scaling-deployments/#scaledobject-spec). -> Refer to the [HPA docs](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale-walkthrough/) for how HPA calculates `replicaCount` based on metric value and target value. KEDA supports both `AverageValue` and `Value` metric target types for external metrics. When `AverageValue` (the default metric type) is used, the metric value returned by the external scaler will be divided by the number of replicas. + > Refer to the [HPA docs](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale-walkthrough/) for how HPA calculates `replicaCount` based on metric value and target value. KEDA supports both `AverageValue` and `Value` metric target types for external metrics. When `AverageValue` (the default metric type) is used, the metric value returned by the external scaler will be divided by the number of replicas. The `PushScaler` interface adds a `Run` method. This method receives a push channel (`active`), on which the scaler can send `true` at any time. The purpose of this mechanism is to initiate a scaling operation independently from `pollingInterval`. @@ -115,7 +115,7 @@ KEDA will attempt a GRPC connection to `service-address.svc.local:9090` immediat } ``` ->**Note**: KEDA will issue all of the above RPC calls except `StreamIsActive` if `spec.triggers.type` is `external`. It _must_ be `external-push` for `StreamIsActive` to be called. +> **Note**: KEDA will issue all of the above RPC calls except `StreamIsActive` if `spec.triggers.type` is `external`. It _must_ be `external-push` for `StreamIsActive` to be called. ## Implementing KEDA external scaler GRPC interface @@ -182,8 +182,6 @@ npm install --save grpc request {{< /collapsible >}} -
- #### 3. Implementing `IsActive` Just like `IsActive(ctx context.Context) (bool, error)` in the go interface, the `IsActive` method in the GRPC interface is called every `pollingInterval` with a `ScaledObjectRef` object that contains the scaledObject name, namespace, and scaler metadata. @@ -310,70 +308,71 @@ public class ExternalScalerService : ExternalScaler.ExternalScalerBase Put the following code into your `index.js` file: ```js -const grpc = require('grpc') -const request = require('request') -const externalScalerProto = grpc.load('externalscaler.proto') +const grpc = require("grpc"); +const request = require("request"); +const externalScalerProto = grpc.load("externalscaler.proto"); -const server = new grpc.Server() +const server = new grpc.Server(); server.addService(externalScalerProto.externalscaler.ExternalScaler.service, { isActive: (call, callback) => { - const longitude = call.request.scalerMetadata.longitude - const latitude = call.request.scalerMetadata.latitude + const longitude = call.request.scalerMetadata.longitude; + const latitude = call.request.scalerMetadata.latitude; if (!longitude || !latitude) { callback({ code: grpc.status.INVALID_ARGUMENT, - details: 'longitude and latitude must be specified', - }) + details: "longitude and latitude must be specified", + }); } else { - const now = new Date() - const yesterday = new Date(new Date().setDate(new Date().getDate()-1)); - - const startTime = `${yesterday.getUTCFullYear()}-${yesterday.getUTCMonth()}-${yesterday.getUTCDay()}` - const endTime = `${now.getUTCFullYear()}-${now.getUTCMonth()}-${now.getUTCDay()}` - const radiusKm = 500 - const query = `format=geojson&starttime=${startTime}&endtime=${endTime}&longitude=${longitude}&latitude=${latitude}&maxradiuskm=${radiusKm}` - - request.get({ - url: `https://earthquake.usgs.gov/fdsnws/event/1/query?${query}`, - json: true, - }, (err, resp, data) => { - if (err) { - callback({ - code: grpc.status.INTERNAL, - details: err, - }) - } else if (resp.statusCode !== 200) { - callback({ - code: grpc.status.INTERNAL, - details: `expected status 200, got ${resp.statusCode}` - }) - } else { - // count how many earthquakes with mag > 1.0 - let count = 0 - data.features.forEach(i => { - if (i.properties.mag > 1.0) { - count++ - } - }) - callback(null, { - result: count > 2, - }) + const now = new Date(); + const yesterday = new Date(new Date().setDate(new Date().getDate() - 1)); + + const startTime = `${yesterday.getUTCFullYear()}-${yesterday.getUTCMonth()}-${yesterday.getUTCDay()}`; + const endTime = `${now.getUTCFullYear()}-${now.getUTCMonth()}-${now.getUTCDay()}`; + const radiusKm = 500; + const query = `format=geojson&starttime=${startTime}&endtime=${endTime}&longitude=${longitude}&latitude=${latitude}&maxradiuskm=${radiusKm}`; + + request.get( + { + url: `https://earthquake.usgs.gov/fdsnws/event/1/query?${query}`, + json: true, + }, + (err, resp, data) => { + if (err) { + callback({ + code: grpc.status.INTERNAL, + details: err, + }); + } else if (resp.statusCode !== 200) { + callback({ + code: grpc.status.INTERNAL, + details: `expected status 200, got ${resp.statusCode}`, + }); + } else { + // count how many earthquakes with mag > 1.0 + let count = 0; + data.features.forEach((i) => { + if (i.properties.mag > 1.0) { + count++; + } + }); + callback(null, { + result: count > 2, + }); + } } - }) + ); } - } -}) + }, +}); -server.bind('127.0.0.1:9090', grpc.ServerCredentials.createInsecure()) -console.log('Server listening on 127.0.0.1:9090') +server.bind("127.0.0.1:9090", grpc.ServerCredentials.createInsecure()); +console.log("Server listening on 127.0.0.1:9090"); -server.start() +server.start(); ``` {{< /collapsible >}} -
- #### 4. Implementing `StreamIsActive` Unlike `IsActive`, `StreamIsActive` is called once when KEDA reconciles the `ScaledObject`, and expects the external scaler to maintain a long-lived connection and push `IsActiveResponse` objects whenever the scaler needs KEDA to activate the deployment. @@ -450,38 +449,36 @@ public override async Task StreamIsActive(ScaledObjectRef request, IServerStream server.addService(externalScalerProto.externalscaler.ExternalScaler.service, { // ... streamIsActive: (call, callback) => { - const longitude = call.request.scalerMetadata.longitude - const latitude = call.request.scalerMetadata.latitude + const longitude = call.request.scalerMetadata.longitude; + const latitude = call.request.scalerMetadata.latitude; if (!longitude || !latitude) { callback({ code: grpc.status.INVALID_ARGUMENT, - details: 'longitude and latitude must be specified', - }) + details: "longitude and latitude must be specified", + }); } else { const interval = setInterval(() => { getEarthquakeCount((err, count) => { if (err) { - console.error(err) + console.error(err); } else if (count > 2) { call.write({ result: true, - }) + }); } - }) + }); }, 1000 * 60 * 60); - call.on('end', () => { - clearInterval(interval) - }) + call.on("end", () => { + clearInterval(interval); + }); } - } -}) + }, +}); ``` {{< /collapsible >}} -
- #### 5. Implementing `GetMetricSpec` `GetMetricSpec` returns the `target` value for [the HPA definition for the scaler](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale-walkthrough/). This scaler will define a static target of 10, but the threshold value is often specified in the metadata for other scalers. @@ -527,19 +524,19 @@ server.addService(externalScalerProto.externalscaler.ExternalScaler.service, { // ... getMetricSpec: (call, callback) => { callback(null, { - metricSpecs: [{ - metricName: 'earthquakeThreshold', - targetSize: 10, - }] - }) - } -}) + metricSpecs: [ + { + metricName: "earthquakeThreshold", + targetSize: 10, + }, + ], + }); + }, +}); ``` {{< /collapsible >}} -
- #### 6. Implementing `GetMetrics` `GetMetrics` returns the value of the metric referred to from `GetMetricSpec`, in this example it's `earthquakeThreshold`. @@ -606,32 +603,34 @@ public override async Task GetMetrics(GetMetricsRequest requ server.addService(externalScalerProto.externalscaler.ExternalScaler.service, { // ... getMetrics: (call, callback) => { - const longitude = call.request.scaledObjectRef.scalerMetadata.longitude - const latitude = call.request.scaledObjectRef.scalerMetadata.latitude + const longitude = call.request.scaledObjectRef.scalerMetadata.longitude; + const latitude = call.request.scaledObjectRef.scalerMetadata.latitude; if (!longitude || !latitude) { callback({ code: grpc.status.INVALID_ARGUMENT, - details: 'longitude and latitude must be specified', - }) + details: "longitude and latitude must be specified", + }); } else { getEarthquakeCount((err, count) => { if (err) { callback({ code: grpc.status.INTERNAL, details: err, - }) + }); } else { callback(null, { - metricValues: [{ - metricName: 'earthquakeThreshold', - metricValue: count, - }] - }) + metricValues: [ + { + metricName: "earthquakeThreshold", + metricValue: count, + }, + ], + }); } - }) + }); } - } -}) + }, +}); ``` {{< /collapsible >}} diff --git a/content/docs/2.14/migration.md b/content/docs/2.14/migration.md index 5ee8a6455..29e8408c8 100644 --- a/content/docs/2.14/migration.md +++ b/content/docs/2.14/migration.md @@ -11,15 +11,17 @@ Please note that you **can not** run both KEDA v1 and v2 on the same Kubernetes KEDA v2 is using a new API namespace for its Custom Resources Definitions (CRD): `keda.sh` instead of `keda.k8s.io` and introduces a new Custom Resource for scaling of Jobs. See full details on KEDA Custom Resources [here](../concepts/#custom-resources-crd). Here's an overview of what's changed: + - [Scaling of Deployments](#scaling-of-deployments) - [Scaling of Jobs](#scaling-of-jobs) - [Improved flexibility & usability of trigger metadata](#improved-flexibility--usability-of-trigger-metadata) - [Scalers](#scalers) - [TriggerAuthentication](#triggerauthentication) - ### Scaling of Deployments + In order to scale `Deployments` with KEDA v2, you need to do only a few modifications to existing v1 `ScaledObjects` definitions, so they comply with v2: + - Change the value of `apiVersion` property from `keda.k8s.io/v1alpha1` to `keda.sh/v1alpha1` - Rename property `spec.scaleTargetRef.deploymentName` to `spec.scaleTargetRef.name` - Rename property `spec.scaleTargetRef.containerName` to `spec.scaleTargetRef.envSourceContainerName` @@ -27,22 +29,23 @@ In order to scale `Deployments` with KEDA v2, you need to do only a few modifica Please see the examples below or refer to the full [v2 ScaledObject Specification](../concepts/scaling-deployments/#scaledobject-spec) - **Example of v1 ScaledObject** +**Example of v1 ScaledObject** + ```yaml apiVersion: keda.k8s.io/v1alpha1 kind: ScaledObject metadata: - name: {scaled-object-name} + name: { scaled-object-name } labels: - deploymentName: {deployment-name} + deploymentName: { deployment-name } spec: scaleTargetRef: - deploymentName: {deployment-name} - containerName: {container-name} - pollingInterval: 30 - cooldownPeriod: 300 - minReplicaCount: 0 - maxReplicaCount: 100 + deploymentName: { deployment-name } + containerName: { container-name } + pollingInterval: 30 + cooldownPeriod: 300 + minReplicaCount: 0 + maxReplicaCount: 100 triggers: # {list of triggers to activate the deployment} ``` @@ -50,18 +53,18 @@ spec: **Example of v2 ScaledObject** ```yaml -apiVersion: keda.sh/v1alpha1 # <--- Property value was changed +apiVersion: keda.sh/v1alpha1 # <--- Property value was changed kind: ScaledObject -metadata: # <--- labels.deploymentName is not needed - name: {scaled-object-name} +metadata: # <--- labels.deploymentName is not needed + name: { scaled-object-name } spec: scaleTargetRef: - name: {deployment-name} # <--- Property name was changed - envSourceContainerName: {container-name} # <--- Property name was changed - pollingInterval: 30 - cooldownPeriod: 300 - minReplicaCount: 0 - maxReplicaCount: 100 + name: { deployment-name } # <--- Property name was changed + envSourceContainerName: { container-name } # <--- Property name was changed + pollingInterval: 30 + cooldownPeriod: 300 + minReplicaCount: 0 + maxReplicaCount: 100 triggers: # {list of triggers to activate the deployment} ``` @@ -69,34 +72,36 @@ spec: ### Scaling of Jobs In order to scale `Jobs` with KEDA v2, you need to do only a few modifications to existing v1 `ScaledObjects` definitions, so they comply with v2: + - Change the value of `apiVersion` property from `keda.k8s.io/v1alpha1` to `keda.sh/v1alpha1` - Change the value of `kind` property from `ScaledObject` to `ScaledJob` - Remove property `spec.scaleType` -- Remove properties `spec.cooldownPeriod` and `spec.minReplicaCount` +- Remove properties `spec.cooldownPeriod` and `spec.minReplicaCount` You can configure `successfulJobsHistoryLimit` and `failedJobsHistoryLimit`. They will remove the old job histories automatically. Please see the examples below or refer to the full [v2 ScaledJob Specification](../concepts/scaling-jobs/#scaledjob-spec) **Example of v1 ScaledObject for Jobs scaling** + ```yaml apiVersion: keda.k8s.io/v1alpha1 kind: ScaledObject metadata: - name: {scaled-object-name} + name: { scaled-object-name } spec: scaleType: job jobTargetRef: - parallelism: 1 + parallelism: 1 completions: 1 activeDeadlineSeconds: 600 - backoffLimit: 6 + backoffLimit: 6 template: # {job template} - pollingInterval: 30 - cooldownPeriod: 300 - minReplicaCount: 0 - maxReplicaCount: 100 + pollingInterval: 30 + cooldownPeriod: 300 + minReplicaCount: 0 + maxReplicaCount: 100 triggers: # {list of triggers to create jobs} ``` @@ -104,21 +109,21 @@ spec: **Example of v2 ScaledJob** ```yaml -apiVersion: keda.sh/v1alpha1 # <--- Property value was changed -kind: ScaledJob # <--- Property value was changed +apiVersion: keda.sh/v1alpha1 # <--- Property value was changed +kind: ScaledJob # <--- Property value was changed metadata: - name: {scaled-job-name} -spec: # <--- spec.scaleType is not needed + name: { scaled-job-name } +spec: # <--- spec.scaleType is not needed jobTargetRef: - parallelism: 1 - completions: 1 + parallelism: 1 + completions: 1 activeDeadlineSeconds: 600 backoffLimit: 6 template: # {job template} - pollingInterval: 30 # <--- spec.cooldownPeriod and spec.minReplicaCount are not needed - successfulJobsHistoryLimit: 5 # <--- property is added - failedJobsHistoryLimit: 5 # <--- Property is added + pollingInterval: 30 # <--- spec.cooldownPeriod and spec.minReplicaCount are not needed + successfulJobsHistoryLimit: 5 # <--- property is added + failedJobsHistoryLimit: 5 # <--- Property is added maxReplicaCount: 100 triggers: # {list of triggers to create jobs} @@ -132,68 +137,74 @@ We've introduced more options to configure trigger metadata to give users more f Here's an overview: -| Scaler | 1.x | 2.0 | -|--------|--------|--------| -| `azure-blob` | `connection` (**Default**: `AzureWebJobsStorage`) | `connectionFromEnv` | -| `azure-monitor` | `activeDirectoryClientId`
`activeDirectoryClientPassword` | `activeDirectoryClientId`
`activeDirectoryClientIdFromEnv`
`activeDirectoryClientPasswordFromEnv` | -| `azure-queue` | `connection` (**Default**: AzureWebJobsStorage) | `connectionFromEnv` | -| `azure-servicebus` | `connection` | `connectionFromEnv` | -| `azure-eventhub` | `storageConnection` (**Default**: `AzureWebJobsStorage`)
`connection` (**Default**: `EventHub`) | `storageConnectionFromEnv`
`connectionFromEnv` | -| `aws-cloudwatch` | `awsAccessKeyID` (**Default**: `AWS_ACCESS_KEY_ID`)
`awsSecretAccessKey` (**Default**: `AWS_SECRET_ACCESS_KEY`) | `awsAccessKeyID`
`awsAccessKeyIDFromEnv`
`awsSecretAccessKeyFromEnv` | -| `aws-kinesis-stream` | `awsAccessKeyID` (**Default**: `AWS_ACCESS_KEY_ID`)
`awsSecretAccessKey` (**Default**: `AWS_SECRET_ACCESS_KEY`) | `awsAccessKeyID`
`awsAccessKeyIDFromEnv`
`awsSecretAccessKeyFromEnv` | -| `aws-sqs-queue` | `awsAccessKeyID` (**Default**: `AWS_ACCESS_KEY_ID`)
`awsSecretAccessKey` (**Default**: `AWS_SECRET_ACCESS_KEY`) | `awsAccessKeyID`
`awsAccessKeyIDFromEnv`
`awsSecretAccessKeyFromEnv` | -| `kafka` | _(none)_ | _(none)_ -| `rabbitmq` | `apiHost`
`host` | ~~`apiHost`~~
`host`
`hostFromEnv` | -| `prometheus` | _(none)_ | _(none)_ | -| `cron` | _(none)_ | _(none)_ | -| `redis` | `address`
`host`
`port`
`password` | `address`
`addressFromEnv`
`host`
`hostFromEnv`
~~`port`~~
`passwordFromEnv` -| `redis-streams` | `address`
`host`
`port`
`password` | `address`
`addressFromEnv`
`host`
`hostFromEnv`
~~`port`~~
`passwordFromEnv` -| `gcp-pubsub` | `credentials` | `credentialsFromEnv` | -| `external` | _(any matching value)_ | _(any matching value with `FromEnv` suffix)_ -| `liiklus` | _(none)_ | _(none)_ | -| `stan` | _(none)_ | _(none)_ | -| `huawei-cloudeye` | | _(none)_ | _(none)_ | -| `postgresql` | `connection`
`password` | `connectionFromEnv`
`passwordFromEnv` | -| `mysql` | `connectionString`
`password` | `connectionStringFromEnv`
`passwordFromEnv` | +| Scaler | 1.x | 2.0 | +| -------------------- | --------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------- | -------- | +| `azure-blob` | `connection` (**Default**: `AzureWebJobsStorage`) | `connectionFromEnv` | +| `azure-monitor` | `activeDirectoryClientId` `activeDirectoryClientPassword` | `activeDirectoryClientId` `activeDirectoryClientIdFromEnv` `activeDirectoryClientPasswordFromEnv` | +| `azure-queue` | `connection` (**Default**: AzureWebJobsStorage) | `connectionFromEnv` | +| `azure-servicebus` | `connection` | `connectionFromEnv` | +| `azure-eventhub` | `storageConnection` (**Default**: `AzureWebJobsStorage`) `connection` (**Default**: `EventHub`) | `storageConnectionFromEnv` `connectionFromEnv` | +| `aws-cloudwatch` | `awsAccessKeyID` (**Default**: `AWS_ACCESS_KEY_ID`) `awsSecretAccessKey` (**Default**: `AWS_SECRET_ACCESS_KEY`) | `awsAccessKeyID` `awsAccessKeyIDFromEnv` `awsSecretAccessKeyFromEnv` | +| `aws-kinesis-stream` | `awsAccessKeyID` (**Default**: `AWS_ACCESS_KEY_ID`) `awsSecretAccessKey` (**Default**: `AWS_SECRET_ACCESS_KEY`) | `awsAccessKeyID` `awsAccessKeyIDFromEnv` `awsSecretAccessKeyFromEnv` | +| `aws-sqs-queue` | `awsAccessKeyID` (**Default**: `AWS_ACCESS_KEY_ID`) `awsSecretAccessKey` (**Default**: `AWS_SECRET_ACCESS_KEY`) | `awsAccessKeyID` `awsAccessKeyIDFromEnv` `awsSecretAccessKeyFromEnv` | +| `kafka` | _(none)_ | _(none)_ | +| `rabbitmq` | `apiHost` `host` | ~~`apiHost`~~ `host` `hostFromEnv` | +| `prometheus` | _(none)_ | _(none)_ | +| `cron` | _(none)_ | _(none)_ | +| `redis` | `address` `host` `port` `password` | `address` `addressFromEnv` `host` `hostFromEnv` ~~`port`~~ `passwordFromEnv` | +| `redis-streams` | `address` `host` `port` `password` | `address` `addressFromEnv` `host` `hostFromEnv` ~~`port`~~ `passwordFromEnv` | +| `gcp-pubsub` | `credentials` | `credentialsFromEnv` | +| `external` | _(any matching value)_ | _(any matching value with `FromEnv` suffix)_ | +| `liiklus` | _(none)_ | _(none)_ | +| `stan` | _(none)_ | _(none)_ | +| `huawei-cloudeye` | | _(none)_ | _(none)_ | +| `postgresql` | `connection` `password` | `connectionFromEnv` `passwordFromEnv` | +| `mysql` | `connectionString` `password` | `connectionStringFromEnv` `passwordFromEnv` | ### Scalers **Azure Service Bus** - - `queueLength` was renamed to `messageCount` + +- `queueLength` was renamed to `messageCount` **Kafka** - - `authMode` property was replaced with `sasl` and `tls` properties. Please refer [documentation](../scalers/apache-kafka/#authentication-parameters) for Kafka Authentication Parameters details. + +- `authMode` property was replaced with `sasl` and `tls` properties. Please refer [documentation](../scalers/apache-kafka/#authentication-parameters) for Kafka Authentication Parameters details. **RabbitMQ** In KEDA 2.0 the RabbitMQ scaler has only `host` parameter, and the protocol for communication can be specified by -`protocol` (http or amqp). The default value is `amqp`. The behavior changes only for scalers that were using HTTP -protocol. +`protocol` (http or amqp). The default value is `amqp`. The behavior changes only for scalers that were using HTTP +protocol. Example of RabbitMQ trigger before 2.0: + ```yaml triggers: -- type: rabbitmq - metadata: - queueLength: '20' - queueName: testqueue - includeUnacked: 'true' - apiHost: 'https://guest:password@localhost:443/vhostname' -``` + - type: rabbitmq + metadata: + queueLength: "20" + queueName: testqueue + includeUnacked: "true" + apiHost: "https://guest:password@localhost:443/vhostname" +``` The same trigger in 2.0: + ```yaml triggers: -- type: rabbitmq - metadata: - queueLength: '20' - queueName: testqueue - protocol: 'http' - host: 'https://guest:password@localhost:443/vhostname' -``` + - type: rabbitmq + metadata: + queueLength: "20" + queueName: testqueue + protocol: "http" + host: "https://guest:password@localhost:443/vhostname" +``` ### TriggerAuthentication + In order to use Authentication via `TriggerAuthentication` with KEDA v2, you need to change: + - Change the value of `apiVersion` property from `keda.k8s.io/v1alpha1` to `keda.sh/v1alpha1` For more details please refer to the full diff --git a/content/docs/2.14/operate/cluster.md b/content/docs/2.14/operate/cluster.md index bf0541bba..4ec1e1049 100644 --- a/content/docs/2.14/operate/cluster.md +++ b/content/docs/2.14/operate/cluster.md @@ -14,16 +14,16 @@ However, maintainers can decide to extend this by supporting more minor versions As a reference, this compatibility matrix shows supported k8s versions per KEDA version: -| KEDA | Kubernetes | -|-----------|---------------| -| v2.14 | v1.27 - v1.29 | -| v2.13 | v1.27 - v1.29 | -| v2.12 | v1.26 - v1.28 | -| v2.11 | v1.25 - v1.27 | -| v2.10 | v1.24 - v1.26 | -| v2.9 | v1.23 - v1.25 | -| v2.8 | v1.17 - v1.25 | -| v2.7 | v1.17 - v1.25 | +| KEDA | Kubernetes | +| ----- | ------------- | +| v2.14 | v1.27 - v1.29 | +| v2.13 | v1.27 - v1.29 | +| v2.12 | v1.26 - v1.28 | +| v2.11 | v1.25 - v1.27 | +| v2.10 | v1.24 - v1.26 | +| v2.9 | v1.23 - v1.25 | +| v2.8 | v1.17 - v1.25 | +| v2.7 | v1.17 - v1.25 | ### Cluster Capacity @@ -45,12 +45,10 @@ KEDA requires to be accessible inside the cluster to be able to autoscale. Here is an overview of the required ports that need to be accessible for KEDA to work: - | Port | Why? | Remarks | | ------ | -------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `443` | Used by Kubernetes API server to get metrics | Required for all platforms because it uses Control Plane → port 443 on the Service IP range communication.

This is not applicable for Google Cloud. | +| `443` | Used by Kubernetes API server to get metrics | Required for all platforms because it uses Control Plane → port 443 on the Service IP range communication. This is not applicable for Google Cloud. | | `6443` | Used by Kubernetes API server to get metrics | Only required for Google Cloud because it uses Control Plane → port 6443 on the Pod IP range for communication | - ## High Availability @@ -58,10 +56,10 @@ KEDA does not provide full support for high-availability due to upstream limitat Here is an overview of all KEDA deployments and the HA notes: -| Deployment | Support Replicas | Note | -| -------------- | ---------------- |---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Deployment | Support Replicas | Note | +| -------------- | ---------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | Metrics Server | 1 | You can run multiple replicas of our metrics sever, and it is recommended to add the `--enable-aggregator-routing=true` CLI flag to the kube-apiserver so that requests sent to our metrics servers are load balanced. However, [you can only run one active metric server in a Kubernetes cluster serving external.metrics.k8s.io](https://github.com/kubernetes-sigs/custom-metrics-apiserver/issues/70) which has to be the KEDA metric server. | -| Operator | 2 | While you can run multiple replicas of our operator, only one operator instance will be active. The rest will be standing by, which may reduce downtime during a failure. Multiple replicas will not improve the performance of KEDA, it could only reduce a downtime during a failover. | +| Operator | 2 | While you can run multiple replicas of our operator, only one operator instance will be active. The rest will be standing by, which may reduce downtime during a failure. Multiple replicas will not improve the performance of KEDA, it could only reduce a downtime during a failover. | ## HTTP Timeouts @@ -112,6 +110,7 @@ Our industry has seen an evolution of TLS versions and some are more secure than By default, KEDA uses TLS1.2 as a minimum TLS version given it is the lowest version without vulnerabilities. However, if you need to support another version you can configure it by using the environment variable `KEDA_HTTP_MIN_TLS_VERSION`. For example: + ```yaml - env: KEDA_HTTP_MIN_TLS_VERSION: TLS13 @@ -123,26 +122,27 @@ The following values are allowed: `TLS13`, `TLS12`, `TLS11` and `TLS10`. The Kubernetes client config used within KEDA Operator and KEDA Metrics Adapter can be adjusted by passing the following command-line flags to the binary: -| Adapter Flag | Client Config Setting | Default Value | Description | -| ------------------- | ----------------------- | ------------- | -------------------------------------------------------------- | -| kube-api-qps | cfg.QPS | 20.0 | Set the QPS rate for throttling requests sent to the apiserver | -| kube-api-burst | cfg.Burst | 30 | Set the burst for throttling requests sent to the apiserver | -| disable-compression | cfg.DisableCompression | true | Disable compression for response in k8s restAPI in client-go, see [this Kubernetes issue](https://github.com/kubernetes/kubernetes/issues/112296) for details | +| Adapter Flag | Client Config Setting | Default Value | Description | +| ------------------- | ---------------------- | ------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| kube-api-qps | cfg.QPS | 20.0 | Set the QPS rate for throttling requests sent to the apiserver | +| kube-api-burst | cfg.Burst | 30 | Set the burst for throttling requests sent to the apiserver | +| disable-compression | cfg.DisableCompression | true | Disable compression for response in k8s restAPI in client-go, see [this Kubernetes issue](https://github.com/kubernetes/kubernetes/issues/112296) for details | ## gRPC Metrics Service Parameters The gRPC Metrics Service is part of the KEDA Operator deployment and serves scaling events and metrics from the scalers over gRPC to the Metrics API Service, that in turn serves them to the Kubernetes API Server. The gRPC Metrics Service config used by the KEDA Metrics Adapter to connect to the KEDA Operator can be adjusted by passing the following command-line flags to the Adapter binary: -| Adapter Flag | Default Value | Description | -| ----------------------------------- | ------------------------------------------------- | ---------------------------------------------- | -| metrics-service-address | keda-operator.keda.svc.cluster.local:9666 | The address of the gRPC Metrics Service Server | -| metrics-service-grpc-authority | "" | Host Authority override for the Metrics Service if the Host Authority is not the same as the address used for the gRPC Metrics Service Server. This is required for mutual TLS when the identity of the adapter server as presented in its TLS certificate is not the same as the metrics-service-address | +| Adapter Flag | Default Value | Description | +| ------------------------------ | ----------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| metrics-service-address | keda-operator.keda.svc.cluster.local:9666 | The address of the gRPC Metrics Service Server | +| metrics-service-grpc-authority | "" | Host Authority override for the Metrics Service if the Host Authority is not the same as the address used for the gRPC Metrics Service Server. This is required for mutual TLS when the identity of the adapter server as presented in its TLS certificate is not the same as the metrics-service-address | ## Configure `MaxConcurrentReconciles` for Controllers To implement internal controllers KEDA uses the [controller-runtime project](https://github.com/kubernetes-sigs/controller-runtime), that enables configuration of [MaxConcurrentReconciles property](https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/controller#Options), ie. the maximum number of concurrent reconciles which can be run for a controller. KEDA Operator exposes properties for specifying `MaxConcurrentReconciles` for following controllers/reconcilers: + - `ScaledObjectReconciler` - responsible for watching and managing `ScaledObjects`, ie. validates input trigger specification, starts scaling logic and manages dependent HPA. - `ScaledJobReconciler` - responsible for watching and managing `ScaledJobs` and dependent Kubernetes Jobs @@ -150,14 +150,15 @@ KEDA Metrics Server exposes property for specifying `MaxConcurrentReconciles` fo To modify this properties you can set environment variables on both KEDA Operator and Metrics Server Deployments: -| Environment variable name | Deployment | Default Value | Affected reconciler | -| ------------------------------------- | -------------- | ------------- | -------------------------------------------------------------- | -| KEDA_SCALEDOBJECT_CTRL_MAX_RECONCILES | Operator | 5 | ScaledObjectReconciler | -| KEDA_SCALEDJOB_CTRL_MAX_RECONCILES | Operator | 1 | ScaledJobReconciler | +| Environment variable name | Deployment | Default Value | Affected reconciler | +| ------------------------------------- | ---------- | ------------- | ---------------------- | +| KEDA_SCALEDOBJECT_CTRL_MAX_RECONCILES | Operator | 5 | ScaledObjectReconciler | +| KEDA_SCALEDJOB_CTRL_MAX_RECONCILES | Operator | 1 | ScaledJobReconciler | ## Configure Leader Election Like reconciliation, KEDA also uses the [controller-runtime project](https://github.com/kubernetes-sigs/controller-runtime) for electing the leader replica. The following properties can be configured for either the Operator and Metrics Server Deployment: + - [`LeaseDuration`](https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/manager#Options.LeaseDuration) - [`RenewDeadline`](https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/manager#Options.RenewDeadline) - [`RetryPeriod`](https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/manager#Options.RetryPeriod) @@ -194,25 +195,29 @@ To learn more please refer to [security section](./security#use-your-own-tls-cer ## Restrict Secret Access By default, KEDA requires adding `secrets` to the cluster role as following: + ```yaml - apiGroups: - - "" + - "" resources: - - external - - pods - - secrets - - services + - external + - pods + - secrets + - services verbs: - - get - - list - - watch + - get + - list + - watch ``` + However, this might lead to security risk (especially in production environment) since it will grant permission to read `secrets` from all namespaces. To restrict `secret` access and limited to KEDA namespace, you could add `KEDA_RESTRICT_SECRET_ACCESS` as environment variable to both KEDA Operator and KEDA Metrics Server: + ```yaml env: - name: KEDA_RESTRICT_SECRET_ACCESS value: "true" ``` + This allows you to omit `secrets` from the cluster role, which will disallow `TriggerAuthentication` to be used for your triggers if the `TriggerAuthentication` is using secrets. You can, however, still use `ClusterTriggerAuthentication`. diff --git a/content/docs/2.15/concepts/external-scalers.md b/content/docs/2.15/concepts/external-scalers.md index 5badde098..aa00a18a0 100644 --- a/content/docs/2.15/concepts/external-scalers.md +++ b/content/docs/2.15/concepts/external-scalers.md @@ -9,7 +9,7 @@ Built-in scalers run in the KEDA process/pod, while external scalers require an This document describes the external scaler interfaces and how to implement them in Go, Node, and .NET; however for more details on GRPC refer to [the official GRPC documentation](https://grpc.io/docs/) ->Want to learn about existing external scalers? Explore our [external scaler community](https://github.com/kedacore/external-scalers). +> Want to learn about existing external scalers? Explore our [external scaler community](https://github.com/kedacore/external-scalers). ## Overview @@ -45,7 +45,7 @@ The `Scaler` interface defines 3 methods: - `GetMetricSpecForScaling` returns the target value for the HPA definition for the scaler. For more details refer to [Implementing `GetMetricSpec`](#5-implementing-getmetricspec). - `GetMetricsAndActivity` is called on `pollingInterval` and. When activity returns `true`, KEDA will scale to what is returned by the metric limited by `maxReplicaCount` on the ScaledObject/ScaledJob. When `false` is returned, KEDA will scale to `minReplicaCount` or optionally `idleReplicaCount`. More details around the defaults and how these options work together can be found on the [ScaledObjectSpec](https://keda.sh/docs/latest/concepts/scaling-deployments/#scaledobject-spec). -> Refer to the [HPA docs](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale-walkthrough/) for how HPA calculates `replicaCount` based on metric value and target value. KEDA supports both `AverageValue` and `Value` metric target types for external metrics. When `AverageValue` (the default metric type) is used, the metric value returned by the external scaler will be divided by the number of replicas. + > Refer to the [HPA docs](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale-walkthrough/) for how HPA calculates `replicaCount` based on metric value and target value. KEDA supports both `AverageValue` and `Value` metric target types for external metrics. When `AverageValue` (the default metric type) is used, the metric value returned by the external scaler will be divided by the number of replicas. The `PushScaler` interface adds a `Run` method. This method receives a push channel (`active`), on which the scaler can send `true` at any time. The purpose of this mechanism is to initiate a scaling operation independently from `pollingInterval`. @@ -115,7 +115,7 @@ KEDA will attempt a GRPC connection to `service-address.svc.local:9090` immediat } ``` ->**Note**: KEDA will issue all of the above RPC calls except `StreamIsActive` if `spec.triggers.type` is `external`. It _must_ be `external-push` for `StreamIsActive` to be called. +> **Note**: KEDA will issue all of the above RPC calls except `StreamIsActive` if `spec.triggers.type` is `external`. It _must_ be `external-push` for `StreamIsActive` to be called. ## Implementing KEDA external scaler GRPC interface @@ -182,8 +182,6 @@ npm install --save grpc request {{< /collapsible >}} -
- #### 3. Implementing `IsActive` Just like `IsActive(ctx context.Context) (bool, error)` in the go interface, the `IsActive` method in the GRPC interface is called every `pollingInterval` with a `ScaledObjectRef` object that contains the scaledObject name, namespace, and scaler metadata. @@ -310,70 +308,71 @@ public class ExternalScalerService : ExternalScaler.ExternalScalerBase Put the following code into your `index.js` file: ```js -const grpc = require('grpc') -const request = require('request') -const externalScalerProto = grpc.load('externalscaler.proto') +const grpc = require("grpc"); +const request = require("request"); +const externalScalerProto = grpc.load("externalscaler.proto"); -const server = new grpc.Server() +const server = new grpc.Server(); server.addService(externalScalerProto.externalscaler.ExternalScaler.service, { isActive: (call, callback) => { - const longitude = call.request.scalerMetadata.longitude - const latitude = call.request.scalerMetadata.latitude + const longitude = call.request.scalerMetadata.longitude; + const latitude = call.request.scalerMetadata.latitude; if (!longitude || !latitude) { callback({ code: grpc.status.INVALID_ARGUMENT, - details: 'longitude and latitude must be specified', - }) + details: "longitude and latitude must be specified", + }); } else { - const now = new Date() - const yesterday = new Date(new Date().setDate(new Date().getDate()-1)); - - const startTime = `${yesterday.getUTCFullYear()}-${yesterday.getUTCMonth()}-${yesterday.getUTCDay()}` - const endTime = `${now.getUTCFullYear()}-${now.getUTCMonth()}-${now.getUTCDay()}` - const radiusKm = 500 - const query = `format=geojson&starttime=${startTime}&endtime=${endTime}&longitude=${longitude}&latitude=${latitude}&maxradiuskm=${radiusKm}` - - request.get({ - url: `https://earthquake.usgs.gov/fdsnws/event/1/query?${query}`, - json: true, - }, (err, resp, data) => { - if (err) { - callback({ - code: grpc.status.INTERNAL, - details: err, - }) - } else if (resp.statusCode !== 200) { - callback({ - code: grpc.status.INTERNAL, - details: `expected status 200, got ${resp.statusCode}` - }) - } else { - // count how many earthquakes with mag > 1.0 - let count = 0 - data.features.forEach(i => { - if (i.properties.mag > 1.0) { - count++ - } - }) - callback(null, { - result: count > 2, - }) + const now = new Date(); + const yesterday = new Date(new Date().setDate(new Date().getDate() - 1)); + + const startTime = `${yesterday.getUTCFullYear()}-${yesterday.getUTCMonth()}-${yesterday.getUTCDay()}`; + const endTime = `${now.getUTCFullYear()}-${now.getUTCMonth()}-${now.getUTCDay()}`; + const radiusKm = 500; + const query = `format=geojson&starttime=${startTime}&endtime=${endTime}&longitude=${longitude}&latitude=${latitude}&maxradiuskm=${radiusKm}`; + + request.get( + { + url: `https://earthquake.usgs.gov/fdsnws/event/1/query?${query}`, + json: true, + }, + (err, resp, data) => { + if (err) { + callback({ + code: grpc.status.INTERNAL, + details: err, + }); + } else if (resp.statusCode !== 200) { + callback({ + code: grpc.status.INTERNAL, + details: `expected status 200, got ${resp.statusCode}`, + }); + } else { + // count how many earthquakes with mag > 1.0 + let count = 0; + data.features.forEach((i) => { + if (i.properties.mag > 1.0) { + count++; + } + }); + callback(null, { + result: count > 2, + }); + } } - }) + ); } - } -}) + }, +}); -server.bind('127.0.0.1:9090', grpc.ServerCredentials.createInsecure()) -console.log('Server listening on 127.0.0.1:9090') +server.bind("127.0.0.1:9090", grpc.ServerCredentials.createInsecure()); +console.log("Server listening on 127.0.0.1:9090"); -server.start() +server.start(); ``` {{< /collapsible >}} -
- #### 4. Implementing `StreamIsActive` Unlike `IsActive`, `StreamIsActive` is called once when KEDA reconciles the `ScaledObject`, and expects the external scaler to maintain a long-lived connection and push `IsActiveResponse` objects whenever the scaler needs KEDA to activate the deployment. @@ -450,38 +449,36 @@ public override async Task StreamIsActive(ScaledObjectRef request, IServerStream server.addService(externalScalerProto.externalscaler.ExternalScaler.service, { // ... streamIsActive: (call, callback) => { - const longitude = call.request.scalerMetadata.longitude - const latitude = call.request.scalerMetadata.latitude + const longitude = call.request.scalerMetadata.longitude; + const latitude = call.request.scalerMetadata.latitude; if (!longitude || !latitude) { callback({ code: grpc.status.INVALID_ARGUMENT, - details: 'longitude and latitude must be specified', - }) + details: "longitude and latitude must be specified", + }); } else { const interval = setInterval(() => { getEarthquakeCount((err, count) => { if (err) { - console.error(err) + console.error(err); } else if (count > 2) { call.write({ result: true, - }) + }); } - }) + }); }, 1000 * 60 * 60); - call.on('end', () => { - clearInterval(interval) - }) + call.on("end", () => { + clearInterval(interval); + }); } - } -}) + }, +}); ``` {{< /collapsible >}} -
- #### 5. Implementing `GetMetricSpec` `GetMetricSpec` returns the `target` value for [the HPA definition for the scaler](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale-walkthrough/). This scaler will define a static target of 10, but the threshold value is often specified in the metadata for other scalers. @@ -527,19 +524,19 @@ server.addService(externalScalerProto.externalscaler.ExternalScaler.service, { // ... getMetricSpec: (call, callback) => { callback(null, { - metricSpecs: [{ - metricName: 'earthquakeThreshold', - targetSize: 10, - }] - }) - } -}) + metricSpecs: [ + { + metricName: "earthquakeThreshold", + targetSize: 10, + }, + ], + }); + }, +}); ``` {{< /collapsible >}} -
- #### 6. Implementing `GetMetrics` `GetMetrics` returns the value of the metric referred to from `GetMetricSpec`, in this example it's `earthquakeThreshold`. @@ -606,32 +603,34 @@ public override async Task GetMetrics(GetMetricsRequest requ server.addService(externalScalerProto.externalscaler.ExternalScaler.service, { // ... getMetrics: (call, callback) => { - const longitude = call.request.scaledObjectRef.scalerMetadata.longitude - const latitude = call.request.scaledObjectRef.scalerMetadata.latitude + const longitude = call.request.scaledObjectRef.scalerMetadata.longitude; + const latitude = call.request.scaledObjectRef.scalerMetadata.latitude; if (!longitude || !latitude) { callback({ code: grpc.status.INVALID_ARGUMENT, - details: 'longitude and latitude must be specified', - }) + details: "longitude and latitude must be specified", + }); } else { getEarthquakeCount((err, count) => { if (err) { callback({ code: grpc.status.INTERNAL, details: err, - }) + }); } else { callback(null, { - metricValues: [{ - metricName: 'earthquakeThreshold', - metricValue: count, - }] - }) + metricValues: [ + { + metricName: "earthquakeThreshold", + metricValue: count, + }, + ], + }); } - }) + }); } - } -}) + }, +}); ``` {{< /collapsible >}} diff --git a/content/docs/2.15/migration.md b/content/docs/2.15/migration.md index 5ee8a6455..29e8408c8 100644 --- a/content/docs/2.15/migration.md +++ b/content/docs/2.15/migration.md @@ -11,15 +11,17 @@ Please note that you **can not** run both KEDA v1 and v2 on the same Kubernetes KEDA v2 is using a new API namespace for its Custom Resources Definitions (CRD): `keda.sh` instead of `keda.k8s.io` and introduces a new Custom Resource for scaling of Jobs. See full details on KEDA Custom Resources [here](../concepts/#custom-resources-crd). Here's an overview of what's changed: + - [Scaling of Deployments](#scaling-of-deployments) - [Scaling of Jobs](#scaling-of-jobs) - [Improved flexibility & usability of trigger metadata](#improved-flexibility--usability-of-trigger-metadata) - [Scalers](#scalers) - [TriggerAuthentication](#triggerauthentication) - ### Scaling of Deployments + In order to scale `Deployments` with KEDA v2, you need to do only a few modifications to existing v1 `ScaledObjects` definitions, so they comply with v2: + - Change the value of `apiVersion` property from `keda.k8s.io/v1alpha1` to `keda.sh/v1alpha1` - Rename property `spec.scaleTargetRef.deploymentName` to `spec.scaleTargetRef.name` - Rename property `spec.scaleTargetRef.containerName` to `spec.scaleTargetRef.envSourceContainerName` @@ -27,22 +29,23 @@ In order to scale `Deployments` with KEDA v2, you need to do only a few modifica Please see the examples below or refer to the full [v2 ScaledObject Specification](../concepts/scaling-deployments/#scaledobject-spec) - **Example of v1 ScaledObject** +**Example of v1 ScaledObject** + ```yaml apiVersion: keda.k8s.io/v1alpha1 kind: ScaledObject metadata: - name: {scaled-object-name} + name: { scaled-object-name } labels: - deploymentName: {deployment-name} + deploymentName: { deployment-name } spec: scaleTargetRef: - deploymentName: {deployment-name} - containerName: {container-name} - pollingInterval: 30 - cooldownPeriod: 300 - minReplicaCount: 0 - maxReplicaCount: 100 + deploymentName: { deployment-name } + containerName: { container-name } + pollingInterval: 30 + cooldownPeriod: 300 + minReplicaCount: 0 + maxReplicaCount: 100 triggers: # {list of triggers to activate the deployment} ``` @@ -50,18 +53,18 @@ spec: **Example of v2 ScaledObject** ```yaml -apiVersion: keda.sh/v1alpha1 # <--- Property value was changed +apiVersion: keda.sh/v1alpha1 # <--- Property value was changed kind: ScaledObject -metadata: # <--- labels.deploymentName is not needed - name: {scaled-object-name} +metadata: # <--- labels.deploymentName is not needed + name: { scaled-object-name } spec: scaleTargetRef: - name: {deployment-name} # <--- Property name was changed - envSourceContainerName: {container-name} # <--- Property name was changed - pollingInterval: 30 - cooldownPeriod: 300 - minReplicaCount: 0 - maxReplicaCount: 100 + name: { deployment-name } # <--- Property name was changed + envSourceContainerName: { container-name } # <--- Property name was changed + pollingInterval: 30 + cooldownPeriod: 300 + minReplicaCount: 0 + maxReplicaCount: 100 triggers: # {list of triggers to activate the deployment} ``` @@ -69,34 +72,36 @@ spec: ### Scaling of Jobs In order to scale `Jobs` with KEDA v2, you need to do only a few modifications to existing v1 `ScaledObjects` definitions, so they comply with v2: + - Change the value of `apiVersion` property from `keda.k8s.io/v1alpha1` to `keda.sh/v1alpha1` - Change the value of `kind` property from `ScaledObject` to `ScaledJob` - Remove property `spec.scaleType` -- Remove properties `spec.cooldownPeriod` and `spec.minReplicaCount` +- Remove properties `spec.cooldownPeriod` and `spec.minReplicaCount` You can configure `successfulJobsHistoryLimit` and `failedJobsHistoryLimit`. They will remove the old job histories automatically. Please see the examples below or refer to the full [v2 ScaledJob Specification](../concepts/scaling-jobs/#scaledjob-spec) **Example of v1 ScaledObject for Jobs scaling** + ```yaml apiVersion: keda.k8s.io/v1alpha1 kind: ScaledObject metadata: - name: {scaled-object-name} + name: { scaled-object-name } spec: scaleType: job jobTargetRef: - parallelism: 1 + parallelism: 1 completions: 1 activeDeadlineSeconds: 600 - backoffLimit: 6 + backoffLimit: 6 template: # {job template} - pollingInterval: 30 - cooldownPeriod: 300 - minReplicaCount: 0 - maxReplicaCount: 100 + pollingInterval: 30 + cooldownPeriod: 300 + minReplicaCount: 0 + maxReplicaCount: 100 triggers: # {list of triggers to create jobs} ``` @@ -104,21 +109,21 @@ spec: **Example of v2 ScaledJob** ```yaml -apiVersion: keda.sh/v1alpha1 # <--- Property value was changed -kind: ScaledJob # <--- Property value was changed +apiVersion: keda.sh/v1alpha1 # <--- Property value was changed +kind: ScaledJob # <--- Property value was changed metadata: - name: {scaled-job-name} -spec: # <--- spec.scaleType is not needed + name: { scaled-job-name } +spec: # <--- spec.scaleType is not needed jobTargetRef: - parallelism: 1 - completions: 1 + parallelism: 1 + completions: 1 activeDeadlineSeconds: 600 backoffLimit: 6 template: # {job template} - pollingInterval: 30 # <--- spec.cooldownPeriod and spec.minReplicaCount are not needed - successfulJobsHistoryLimit: 5 # <--- property is added - failedJobsHistoryLimit: 5 # <--- Property is added + pollingInterval: 30 # <--- spec.cooldownPeriod and spec.minReplicaCount are not needed + successfulJobsHistoryLimit: 5 # <--- property is added + failedJobsHistoryLimit: 5 # <--- Property is added maxReplicaCount: 100 triggers: # {list of triggers to create jobs} @@ -132,68 +137,74 @@ We've introduced more options to configure trigger metadata to give users more f Here's an overview: -| Scaler | 1.x | 2.0 | -|--------|--------|--------| -| `azure-blob` | `connection` (**Default**: `AzureWebJobsStorage`) | `connectionFromEnv` | -| `azure-monitor` | `activeDirectoryClientId`
`activeDirectoryClientPassword` | `activeDirectoryClientId`
`activeDirectoryClientIdFromEnv`
`activeDirectoryClientPasswordFromEnv` | -| `azure-queue` | `connection` (**Default**: AzureWebJobsStorage) | `connectionFromEnv` | -| `azure-servicebus` | `connection` | `connectionFromEnv` | -| `azure-eventhub` | `storageConnection` (**Default**: `AzureWebJobsStorage`)
`connection` (**Default**: `EventHub`) | `storageConnectionFromEnv`
`connectionFromEnv` | -| `aws-cloudwatch` | `awsAccessKeyID` (**Default**: `AWS_ACCESS_KEY_ID`)
`awsSecretAccessKey` (**Default**: `AWS_SECRET_ACCESS_KEY`) | `awsAccessKeyID`
`awsAccessKeyIDFromEnv`
`awsSecretAccessKeyFromEnv` | -| `aws-kinesis-stream` | `awsAccessKeyID` (**Default**: `AWS_ACCESS_KEY_ID`)
`awsSecretAccessKey` (**Default**: `AWS_SECRET_ACCESS_KEY`) | `awsAccessKeyID`
`awsAccessKeyIDFromEnv`
`awsSecretAccessKeyFromEnv` | -| `aws-sqs-queue` | `awsAccessKeyID` (**Default**: `AWS_ACCESS_KEY_ID`)
`awsSecretAccessKey` (**Default**: `AWS_SECRET_ACCESS_KEY`) | `awsAccessKeyID`
`awsAccessKeyIDFromEnv`
`awsSecretAccessKeyFromEnv` | -| `kafka` | _(none)_ | _(none)_ -| `rabbitmq` | `apiHost`
`host` | ~~`apiHost`~~
`host`
`hostFromEnv` | -| `prometheus` | _(none)_ | _(none)_ | -| `cron` | _(none)_ | _(none)_ | -| `redis` | `address`
`host`
`port`
`password` | `address`
`addressFromEnv`
`host`
`hostFromEnv`
~~`port`~~
`passwordFromEnv` -| `redis-streams` | `address`
`host`
`port`
`password` | `address`
`addressFromEnv`
`host`
`hostFromEnv`
~~`port`~~
`passwordFromEnv` -| `gcp-pubsub` | `credentials` | `credentialsFromEnv` | -| `external` | _(any matching value)_ | _(any matching value with `FromEnv` suffix)_ -| `liiklus` | _(none)_ | _(none)_ | -| `stan` | _(none)_ | _(none)_ | -| `huawei-cloudeye` | | _(none)_ | _(none)_ | -| `postgresql` | `connection`
`password` | `connectionFromEnv`
`passwordFromEnv` | -| `mysql` | `connectionString`
`password` | `connectionStringFromEnv`
`passwordFromEnv` | +| Scaler | 1.x | 2.0 | +| -------------------- | --------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------- | -------- | +| `azure-blob` | `connection` (**Default**: `AzureWebJobsStorage`) | `connectionFromEnv` | +| `azure-monitor` | `activeDirectoryClientId` `activeDirectoryClientPassword` | `activeDirectoryClientId` `activeDirectoryClientIdFromEnv` `activeDirectoryClientPasswordFromEnv` | +| `azure-queue` | `connection` (**Default**: AzureWebJobsStorage) | `connectionFromEnv` | +| `azure-servicebus` | `connection` | `connectionFromEnv` | +| `azure-eventhub` | `storageConnection` (**Default**: `AzureWebJobsStorage`) `connection` (**Default**: `EventHub`) | `storageConnectionFromEnv` `connectionFromEnv` | +| `aws-cloudwatch` | `awsAccessKeyID` (**Default**: `AWS_ACCESS_KEY_ID`) `awsSecretAccessKey` (**Default**: `AWS_SECRET_ACCESS_KEY`) | `awsAccessKeyID` `awsAccessKeyIDFromEnv` `awsSecretAccessKeyFromEnv` | +| `aws-kinesis-stream` | `awsAccessKeyID` (**Default**: `AWS_ACCESS_KEY_ID`) `awsSecretAccessKey` (**Default**: `AWS_SECRET_ACCESS_KEY`) | `awsAccessKeyID` `awsAccessKeyIDFromEnv` `awsSecretAccessKeyFromEnv` | +| `aws-sqs-queue` | `awsAccessKeyID` (**Default**: `AWS_ACCESS_KEY_ID`) `awsSecretAccessKey` (**Default**: `AWS_SECRET_ACCESS_KEY`) | `awsAccessKeyID` `awsAccessKeyIDFromEnv` `awsSecretAccessKeyFromEnv` | +| `kafka` | _(none)_ | _(none)_ | +| `rabbitmq` | `apiHost` `host` | ~~`apiHost`~~ `host` `hostFromEnv` | +| `prometheus` | _(none)_ | _(none)_ | +| `cron` | _(none)_ | _(none)_ | +| `redis` | `address` `host` `port` `password` | `address` `addressFromEnv` `host` `hostFromEnv` ~~`port`~~ `passwordFromEnv` | +| `redis-streams` | `address` `host` `port` `password` | `address` `addressFromEnv` `host` `hostFromEnv` ~~`port`~~ `passwordFromEnv` | +| `gcp-pubsub` | `credentials` | `credentialsFromEnv` | +| `external` | _(any matching value)_ | _(any matching value with `FromEnv` suffix)_ | +| `liiklus` | _(none)_ | _(none)_ | +| `stan` | _(none)_ | _(none)_ | +| `huawei-cloudeye` | | _(none)_ | _(none)_ | +| `postgresql` | `connection` `password` | `connectionFromEnv` `passwordFromEnv` | +| `mysql` | `connectionString` `password` | `connectionStringFromEnv` `passwordFromEnv` | ### Scalers **Azure Service Bus** - - `queueLength` was renamed to `messageCount` + +- `queueLength` was renamed to `messageCount` **Kafka** - - `authMode` property was replaced with `sasl` and `tls` properties. Please refer [documentation](../scalers/apache-kafka/#authentication-parameters) for Kafka Authentication Parameters details. + +- `authMode` property was replaced with `sasl` and `tls` properties. Please refer [documentation](../scalers/apache-kafka/#authentication-parameters) for Kafka Authentication Parameters details. **RabbitMQ** In KEDA 2.0 the RabbitMQ scaler has only `host` parameter, and the protocol for communication can be specified by -`protocol` (http or amqp). The default value is `amqp`. The behavior changes only for scalers that were using HTTP -protocol. +`protocol` (http or amqp). The default value is `amqp`. The behavior changes only for scalers that were using HTTP +protocol. Example of RabbitMQ trigger before 2.0: + ```yaml triggers: -- type: rabbitmq - metadata: - queueLength: '20' - queueName: testqueue - includeUnacked: 'true' - apiHost: 'https://guest:password@localhost:443/vhostname' -``` + - type: rabbitmq + metadata: + queueLength: "20" + queueName: testqueue + includeUnacked: "true" + apiHost: "https://guest:password@localhost:443/vhostname" +``` The same trigger in 2.0: + ```yaml triggers: -- type: rabbitmq - metadata: - queueLength: '20' - queueName: testqueue - protocol: 'http' - host: 'https://guest:password@localhost:443/vhostname' -``` + - type: rabbitmq + metadata: + queueLength: "20" + queueName: testqueue + protocol: "http" + host: "https://guest:password@localhost:443/vhostname" +``` ### TriggerAuthentication + In order to use Authentication via `TriggerAuthentication` with KEDA v2, you need to change: + - Change the value of `apiVersion` property from `keda.k8s.io/v1alpha1` to `keda.sh/v1alpha1` For more details please refer to the full diff --git a/content/docs/2.15/operate/cluster.md b/content/docs/2.15/operate/cluster.md index 905782f9f..4de7f51ed 100644 --- a/content/docs/2.15/operate/cluster.md +++ b/content/docs/2.15/operate/cluster.md @@ -14,17 +14,17 @@ However, maintainers can decide to extend this by supporting more minor versions As a reference, this compatibility matrix shows supported k8s versions per KEDA version: -| KEDA | Kubernetes | -|-----------|---------------| -| v2.15 | v1.28 - v1.30 | -| v2.14 | v1.27 - v1.29 | -| v2.13 | v1.27 - v1.29 | -| v2.12 | v1.26 - v1.28 | -| v2.11 | v1.25 - v1.27 | -| v2.10 | v1.24 - v1.26 | -| v2.9 | v1.23 - v1.25 | -| v2.8 | v1.17 - v1.25 | -| v2.7 | v1.17 - v1.25 | +| KEDA | Kubernetes | +| ----- | ------------- | +| v2.15 | v1.28 - v1.30 | +| v2.14 | v1.27 - v1.29 | +| v2.13 | v1.27 - v1.29 | +| v2.12 | v1.26 - v1.28 | +| v2.11 | v1.25 - v1.27 | +| v2.10 | v1.24 - v1.26 | +| v2.9 | v1.23 - v1.25 | +| v2.8 | v1.17 - v1.25 | +| v2.7 | v1.17 - v1.25 | ### Cluster Capacity @@ -46,12 +46,10 @@ KEDA requires to be accessible inside the cluster to be able to autoscale. Here is an overview of the required ports that need to be accessible for KEDA to work: - | Port | Why? | Remarks | | ------ | -------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `443` | Used by Kubernetes API server to get metrics | Required for all platforms because it uses Control Plane → port 443 on the Service IP range communication.

This is not applicable for Google Cloud. | +| `443` | Used by Kubernetes API server to get metrics | Required for all platforms because it uses Control Plane → port 443 on the Service IP range communication. This is not applicable for Google Cloud. | | `6443` | Used by Kubernetes API server to get metrics | Only required for Google Cloud because it uses Control Plane → port 6443 on the Pod IP range for communication | - ## High Availability @@ -59,10 +57,10 @@ KEDA does not provide full support for high-availability due to upstream limitat Here is an overview of all KEDA deployments and the HA notes: -| Deployment | Support Replicas | Note | -| -------------- | ---------------- |---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Deployment | Support Replicas | Note | +| -------------- | ---------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | Metrics Server | 1 | You can run multiple replicas of our metrics sever, and it is recommended to add the `--enable-aggregator-routing=true` CLI flag to the kube-apiserver so that requests sent to our metrics servers are load balanced. However, [you can only run one active metric server in a Kubernetes cluster serving external.metrics.k8s.io](https://github.com/kubernetes-sigs/custom-metrics-apiserver/issues/70) which has to be the KEDA metric server. | -| Operator | 2 | While you can run multiple replicas of our operator, only one operator instance will be active. The rest will be standing by, which may reduce downtime during a failure. Multiple replicas will not improve the performance of KEDA, it could only reduce a downtime during a failover. | +| Operator | 2 | While you can run multiple replicas of our operator, only one operator instance will be active. The rest will be standing by, which may reduce downtime during a failure. Multiple replicas will not improve the performance of KEDA, it could only reduce a downtime during a failover. | ## HTTP Timeouts @@ -113,6 +111,7 @@ Our industry has seen an evolution of TLS versions and some are more secure than By default, KEDA uses TLS1.2 as a minimum TLS version given it is the lowest version without vulnerabilities. However, if you need to support another version you can configure it by using the environment variable `KEDA_HTTP_MIN_TLS_VERSION`. For example: + ```yaml - env: KEDA_HTTP_MIN_TLS_VERSION: TLS13 @@ -124,26 +123,27 @@ The following values are allowed: `TLS13`, `TLS12`, `TLS11` and `TLS10`. The Kubernetes client config used within KEDA Operator and KEDA Metrics Adapter can be adjusted by passing the following command-line flags to the binary: -| Adapter Flag | Client Config Setting | Default Value | Description | -| ------------------- | ----------------------- | ------------- | -------------------------------------------------------------- | -| kube-api-qps | cfg.QPS | 20.0 | Set the QPS rate for throttling requests sent to the apiserver | -| kube-api-burst | cfg.Burst | 30 | Set the burst for throttling requests sent to the apiserver | -| disable-compression | cfg.DisableCompression | true | Disable compression for response in k8s restAPI in client-go, see [this Kubernetes issue](https://github.com/kubernetes/kubernetes/issues/112296) for details | +| Adapter Flag | Client Config Setting | Default Value | Description | +| ------------------- | ---------------------- | ------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| kube-api-qps | cfg.QPS | 20.0 | Set the QPS rate for throttling requests sent to the apiserver | +| kube-api-burst | cfg.Burst | 30 | Set the burst for throttling requests sent to the apiserver | +| disable-compression | cfg.DisableCompression | true | Disable compression for response in k8s restAPI in client-go, see [this Kubernetes issue](https://github.com/kubernetes/kubernetes/issues/112296) for details | ## gRPC Metrics Service Parameters The gRPC Metrics Service is part of the KEDA Operator deployment and serves scaling events and metrics from the scalers over gRPC to the Metrics API Service, that in turn serves them to the Kubernetes API Server. The gRPC Metrics Service config used by the KEDA Metrics Adapter to connect to the KEDA Operator can be adjusted by passing the following command-line flags to the Adapter binary: -| Adapter Flag | Default Value | Description | -| ----------------------------------- | ------------------------------------------------- | ---------------------------------------------- | -| metrics-service-address | keda-operator.keda.svc.cluster.local:9666 | The address of the gRPC Metrics Service Server | -| metrics-service-grpc-authority | "" | Host Authority override for the Metrics Service if the Host Authority is not the same as the address used for the gRPC Metrics Service Server. This is required for mutual TLS when the identity of the adapter server as presented in its TLS certificate is not the same as the metrics-service-address | +| Adapter Flag | Default Value | Description | +| ------------------------------ | ----------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| metrics-service-address | keda-operator.keda.svc.cluster.local:9666 | The address of the gRPC Metrics Service Server | +| metrics-service-grpc-authority | "" | Host Authority override for the Metrics Service if the Host Authority is not the same as the address used for the gRPC Metrics Service Server. This is required for mutual TLS when the identity of the adapter server as presented in its TLS certificate is not the same as the metrics-service-address | ## Configure `MaxConcurrentReconciles` for Controllers To implement internal controllers KEDA uses the [controller-runtime project](https://github.com/kubernetes-sigs/controller-runtime), that enables configuration of [MaxConcurrentReconciles property](https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/controller#Options), ie. the maximum number of concurrent reconciles which can be run for a controller. KEDA Operator exposes properties for specifying `MaxConcurrentReconciles` for following controllers/reconcilers: + - `ScaledObjectReconciler` - responsible for watching and managing `ScaledObjects`, ie. validates input trigger specification, starts scaling logic and manages dependent HPA. - `ScaledJobReconciler` - responsible for watching and managing `ScaledJobs` and dependent Kubernetes Jobs @@ -151,14 +151,15 @@ KEDA Metrics Server exposes property for specifying `MaxConcurrentReconciles` fo To modify this properties you can set environment variables on both KEDA Operator and Metrics Server Deployments: -| Environment variable name | Deployment | Default Value | Affected reconciler | -| ------------------------------------- | -------------- | ------------- | -------------------------------------------------------------- | -| KEDA_SCALEDOBJECT_CTRL_MAX_RECONCILES | Operator | 5 | ScaledObjectReconciler | -| KEDA_SCALEDJOB_CTRL_MAX_RECONCILES | Operator | 1 | ScaledJobReconciler | +| Environment variable name | Deployment | Default Value | Affected reconciler | +| ------------------------------------- | ---------- | ------------- | ---------------------- | +| KEDA_SCALEDOBJECT_CTRL_MAX_RECONCILES | Operator | 5 | ScaledObjectReconciler | +| KEDA_SCALEDJOB_CTRL_MAX_RECONCILES | Operator | 1 | ScaledJobReconciler | ## Configure Leader Election Like reconciliation, KEDA also uses the [controller-runtime project](https://github.com/kubernetes-sigs/controller-runtime) for electing the leader replica. The following properties can be configured for either the Operator and Metrics Server Deployment: + - [`LeaseDuration`](https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/manager#Options.LeaseDuration) - [`RenewDeadline`](https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/manager#Options.RenewDeadline) - [`RetryPeriod`](https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/manager#Options.RetryPeriod) @@ -195,25 +196,29 @@ To learn more please refer to [security section](./security#use-your-own-tls-cer ## Restrict Secret Access By default, KEDA requires adding `secrets` to the cluster role as following: + ```yaml - apiGroups: - - "" + - "" resources: - - external - - pods - - secrets - - services + - external + - pods + - secrets + - services verbs: - - get - - list - - watch + - get + - list + - watch ``` + However, this might lead to security risk (especially in production environment) since it will grant permission to read `secrets` from all namespaces. To restrict `secret` access and limited to KEDA namespace, you could add `KEDA_RESTRICT_SECRET_ACCESS` as environment variable to both KEDA Operator and KEDA Metrics Server: + ```yaml env: - name: KEDA_RESTRICT_SECRET_ACCESS value: "true" ``` + This allows you to omit `secrets` from the cluster role, which will disallow `TriggerAuthentication` to be used for your triggers if the `TriggerAuthentication` is using secrets. You can, however, still use `ClusterTriggerAuthentication`. diff --git a/content/docs/2.2/concepts/external-scalers.md b/content/docs/2.2/concepts/external-scalers.md index 27ed9e0c6..1945394d3 100644 --- a/content/docs/2.2/concepts/external-scalers.md +++ b/content/docs/2.2/concepts/external-scalers.md @@ -33,15 +33,15 @@ type PushScaler interface { ``` The `Scaler` interface defines 4 methods: + - `IsActive` is called on `pollingInterval` defined in the ScaledObject/ScaledJob CRDs and scaling to 1 happens if this returns true. - `Close` is called to allow the scaler to clean up connections or other resources. - `GetMetricSpec` returns the target value for the HPA definition for the scaler. For more details refer to [Implementing `GetMetricSpec`](#5-implementing-getmetricspec). - `GetMetrics` returns the value of the metric referred to from `GetMetricSpec`. For more details refer to [Implementing `GetMetrics`](#6-implementing-getmetrics). -> Refer to the [HPA docs](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale-walkthrough/) for how HPA calculates `replicaCount` based on metric value and target value. KEDA uses the metric target type `AverageValue` for external metrics. This will cause the metric value returned by the external scaler to be divided by the number of replicas. + > Refer to the [HPA docs](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale-walkthrough/) for how HPA calculates `replicaCount` based on metric value and target value. KEDA uses the metric target type `AverageValue` for external metrics. This will cause the metric value returned by the external scaler to be divided by the number of replicas. The `PushScaler` interface adds `Run` method. The `Run` method receives a push channel (`active`), that the scaler can push `true` to any time to force scaling action independently from `pollingInterval`. - ### External Scaler GRPC interface KEDA comes with 2 external scalers [`external`](../scalers/external.md) and [`external-push`](../scalers/external-push.md). @@ -62,6 +62,7 @@ service ExternalScaler { - `StreamIsActive` maps to the `Run` method on the `PushScaler` interface. Few things to notice: + - Lack of `Close` method as the GRPC connection defines the lifetime of the scaler - `IsActive`, `StreamIsActive`, and `GetMetricsSpec` are called with a `ScaledObjectRef` that contains the scaledObject name/namespace as well as the content of `metadata` defined in the trigger. @@ -125,6 +126,7 @@ go mod init example.com/external-scaler/sample mkdir externalscaler protoc externalscaler.proto --go_out=plugins=grpc:externalscaler ``` + {{< /collapsible >}} {{< collapsible "C#" >}} @@ -161,9 +163,8 @@ mkdir Services ```bash npm install --save grpc request ``` -{{< /collapsible >}} -
+{{< /collapsible >}} #### 3. Implementing `IsActive` @@ -193,6 +194,7 @@ spec: Full implementation can be found here: https://github.com/kedacore/external-scaler-samples `main.go` + ```golang func (e *ExternalScaler) IsActive(ctx context.Context, scaledObject *pb.ScaledObjectRef) (*pb.IsActiveResponse, error) { // request.Scalermetadata contains the `metadata` defined in the ScaledObject @@ -239,12 +241,14 @@ func (e *ExternalScaler) IsActive(ctx context.Context, scaledObject *pb.ScaledOb }, nil } ``` + {{< /collapsible >}} {{< collapsible "C#" >}} Full implementation can be found here: https://github.com/kedacore/external-scaler-samples `Services/ExternalScalerService.cs` + ```csharp public class ExternalScalerService : ExternalScaler.ExternalScalerBase { @@ -277,73 +281,77 @@ public class ExternalScalerService : ExternalScaler.ExternalScalerBase } } ``` + {{< /collapsible >}} {{< collapsible "Javascript" >}} `index.js` + ```js -const grpc = require('grpc') -const request = require('request') -const externalScalerProto = grpc.load('externalscaler.proto') +const grpc = require("grpc"); +const request = require("request"); +const externalScalerProto = grpc.load("externalscaler.proto"); -const server = new grpc.Server() +const server = new grpc.Server(); server.addService(externalScalerProto.externalscaler.ExternalScaler.service, { isActive: (call, callback) => { - const longitude = call.request.scalerMetadata.longitude - const latitude = call.request.scalerMetadata.latitude + const longitude = call.request.scalerMetadata.longitude; + const latitude = call.request.scalerMetadata.latitude; if (!longitude || !latitude) { callback({ code: grpc.status.INVALID_ARGUMENT, - details: 'longitude and latitude must be specified', - }) + details: "longitude and latitude must be specified", + }); } else { - const now = new Date() - const yesterday = new Date(new Date().setDate(new Date().getDate()-1)); - - const startTime = `${yesterday.getUTCFullYear()}-${yesterday.getUTCMonth()}-${yesterday.getUTCDay()}` - const endTime = `${now.getUTCFullYear()}-${now.getUTCMonth()}-${now.getUTCDay()}` - const radiusKm = 500 - const query = `format=geojson&starttime=${startTime}&endtime=${endTime}&longitude=${longitude}&latitude=${latitude}&maxradiuskm=${radiusKm}` - - request.get({ - url: `https://earthquake.usgs.gov/fdsnws/event/1/query?${query}`, - json: true, - }, (err, resp, data) => { - if (err) { - callback({ - code: grpc.status.INTERNAL, - details: err, - }) - } else if (resp.statusCode !== 200) { - callback({ - code: grpc.status.INTERNAL, - details: `expected status 200, got ${resp.statusCode}` - }) - } else { - // count how many earthquakes with mag > 1.0 - let count = 0 - data.features.forEach(i => { - if (i.properties.mag > 1.0) { - count++ - } - }) - callback(null, { - result: count > 2, - }) + const now = new Date(); + const yesterday = new Date(new Date().setDate(new Date().getDate() - 1)); + + const startTime = `${yesterday.getUTCFullYear()}-${yesterday.getUTCMonth()}-${yesterday.getUTCDay()}`; + const endTime = `${now.getUTCFullYear()}-${now.getUTCMonth()}-${now.getUTCDay()}`; + const radiusKm = 500; + const query = `format=geojson&starttime=${startTime}&endtime=${endTime}&longitude=${longitude}&latitude=${latitude}&maxradiuskm=${radiusKm}`; + + request.get( + { + url: `https://earthquake.usgs.gov/fdsnws/event/1/query?${query}`, + json: true, + }, + (err, resp, data) => { + if (err) { + callback({ + code: grpc.status.INTERNAL, + details: err, + }); + } else if (resp.statusCode !== 200) { + callback({ + code: grpc.status.INTERNAL, + details: `expected status 200, got ${resp.statusCode}`, + }); + } else { + // count how many earthquakes with mag > 1.0 + let count = 0; + data.features.forEach((i) => { + if (i.properties.mag > 1.0) { + count++; + } + }); + callback(null, { + result: count > 2, + }); + } } - }) + ); } - } -}) + }, +}); -server.bind('127.0.0.1:9090', grpc.ServerCredentials.createInsecure()) -console.log('Server listening on 127.0.0.1:9090') +server.bind("127.0.0.1:9090", grpc.ServerCredentials.createInsecure()); +console.log("Server listening on 127.0.0.1:9090"); -server.start() +server.start(); ``` -{{< /collapsible >}} -
+{{< /collapsible >}} #### 4. Implementing `StreamIsActive` @@ -351,8 +359,8 @@ Unlike `IsActive`, `StreamIsActive` is called once when KEDA reconciles the `Sca This implementation creates a timer and queries USGS APIs on that timer, effectively ignoring `pollingInterval` set in the scaledObject. Alternatively any other asynchronous event can be used instead of a timer, like an HTTP request, or a network connection. - {{< collapsible "Golang" >}} + ```golang func (e *ExternalScaler) StreamIsActive(scaledObject *pb.ScaledObjectRef, epsServer pb.ExternalScaler_StreamIsActiveServer) error { longitude := scaledObject.ScalerMetadata["longitude"] @@ -380,9 +388,11 @@ func (e *ExternalScaler) StreamIsActive(scaledObject *pb.ScaledObjectRef, epsSer } } ``` + {{< /collapsible >}} {{< collapsible "C#" >}} + ```csharp public override async Task StreamIsActive(ScaledObjectRef request, IServerStreamWriter responseStream, ServerCallContext context) { @@ -410,49 +420,51 @@ public override async Task StreamIsActive(ScaledObjectRef request, IServerStream } } ``` + {{< /collapsible >}} {{< collapsible "Javascript" >}} + ```js server.addService(externalScalerProto.externalscaler.ExternalScaler.service, { // ... streamIsActive: (call, callback) => { - const longitude = call.request.scalerMetadata.longitude - const latitude = call.request.scalerMetadata.latitude + const longitude = call.request.scalerMetadata.longitude; + const latitude = call.request.scalerMetadata.latitude; if (!longitude || !latitude) { callback({ code: grpc.status.INVALID_ARGUMENT, - details: 'longitude and latitude must be specified', - }) + details: "longitude and latitude must be specified", + }); } else { const interval = setInterval(() => { getEarthquakeCount((err, count) => { if (err) { - console.error(err) + console.error(err); } else if (count > 2) { call.write({ result: true, - }) + }); } - }) + }); }, 1000 * 60 * 60); - call.on('end', () => { - clearInterval(interval) - }) + call.on("end", () => { + clearInterval(interval); + }); } - } -}) + }, +}); ``` -{{< /collapsible >}} -
+{{< /collapsible >}} #### 5. Implementing `GetMetricSpec` `GetMetricSpec` returns the `target` value for [the HPA definition for the scaler](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale-walkthrough/). This scaler will define a static target of 10, but the threshold value is often specified in the metadata for other scalers. {{< collapsible "Golang" >}} + ```golang func (e *ExternalScaler) GetMetricSpec(context.Context, *pb.ScaledObjectRef) (*pb.GetMetricSpecResponse, error) { return &pb.GetMetricSpecResponse{ @@ -463,9 +475,11 @@ func (e *ExternalScaler) GetMetricSpec(context.Context, *pb.ScaledObjectRef) (*p }, nil } ``` + {{< /collapsible >}} {{< collapsible "C#" >}} + ```csharp public override async Task GetMetricSpec(ScaledObjectRef request, ServerCallContext context) { @@ -480,31 +494,35 @@ public override async Task GetMetricSpec(ScaledObjectRef return Task.FromResult(resp); } ``` + {{< /collapsible >}} {{< collapsible "Javascript" >}} + ```js server.addService(externalScalerProto.externalscaler.ExternalScaler.service, { // ... getMetricSpec: (call, callback) => { callback(null, { - metricSpecs: [{ - metricName: 'earthquakeThreshold', - targetSize: 10, - }] - }) - } -}) + metricSpecs: [ + { + metricName: "earthquakeThreshold", + targetSize: 10, + }, + ], + }); + }, +}); ``` -{{< /collapsible >}} -
+{{< /collapsible >}} #### 6. Implementing `GetMetrics` `GetMetrics` returns the value of the metric referred to from `GetMetricSpec`, in this example it's `earthquakeThreshold`. {{< collapsible "Golang" >}} + ```golang func (e *ExternalScaler) GetMetrics(_ context.Context, metricRequest *pb.GetMetricsRequest) (*pb.GetMetricsResponse, error) { longitude := metricRequest.ScaledObjectRef.ScalerMetadata["longitude"] @@ -527,9 +545,11 @@ func (e *ExternalScaler) GetMetrics(_ context.Context, metricRequest *pb.GetMetr }, nil } ``` + {{< /collapsible >}} {{< collapsible "C#" >}} + ```csharp public override async Task GetMetrics(GetMetricsRequest request, ServerCallContext context) { @@ -554,38 +574,43 @@ public override async Task GetMetrics(GetMetricsRequest requ return resp; } ``` + {{< /collapsible >}} {{< collapsible "Javascript" >}} + ```js server.addService(externalScalerProto.externalscaler.ExternalScaler.service, { // ... getMetrics: (call, callback) => { - const longitude = call.request.scaledObjectRef.scalerMetadata.longitude - const latitude = call.request.scaledObjectRef.scalerMetadata.latitude + const longitude = call.request.scaledObjectRef.scalerMetadata.longitude; + const latitude = call.request.scaledObjectRef.scalerMetadata.latitude; if (!longitude || !latitude) { callback({ code: grpc.status.INVALID_ARGUMENT, - details: 'longitude and latitude must be specified', - }) + details: "longitude and latitude must be specified", + }); } else { getEarthquakeCount((err, count) => { if (err) { callback({ code: grpc.status.INTERNAL, details: err, - }) + }); } else { callback(null, { - metricValues: [{ - metricName: 'earthquakeThreshold', - metricValue: count, - }] - }) + metricValues: [ + { + metricName: "earthquakeThreshold", + metricValue: count, + }, + ], + }); } - }) + }); } - } -}) + }, +}); ``` + {{< /collapsible >}} diff --git a/content/docs/2.2/migration.md b/content/docs/2.2/migration.md index 5ee8a6455..29e8408c8 100644 --- a/content/docs/2.2/migration.md +++ b/content/docs/2.2/migration.md @@ -11,15 +11,17 @@ Please note that you **can not** run both KEDA v1 and v2 on the same Kubernetes KEDA v2 is using a new API namespace for its Custom Resources Definitions (CRD): `keda.sh` instead of `keda.k8s.io` and introduces a new Custom Resource for scaling of Jobs. See full details on KEDA Custom Resources [here](../concepts/#custom-resources-crd). Here's an overview of what's changed: + - [Scaling of Deployments](#scaling-of-deployments) - [Scaling of Jobs](#scaling-of-jobs) - [Improved flexibility & usability of trigger metadata](#improved-flexibility--usability-of-trigger-metadata) - [Scalers](#scalers) - [TriggerAuthentication](#triggerauthentication) - ### Scaling of Deployments + In order to scale `Deployments` with KEDA v2, you need to do only a few modifications to existing v1 `ScaledObjects` definitions, so they comply with v2: + - Change the value of `apiVersion` property from `keda.k8s.io/v1alpha1` to `keda.sh/v1alpha1` - Rename property `spec.scaleTargetRef.deploymentName` to `spec.scaleTargetRef.name` - Rename property `spec.scaleTargetRef.containerName` to `spec.scaleTargetRef.envSourceContainerName` @@ -27,22 +29,23 @@ In order to scale `Deployments` with KEDA v2, you need to do only a few modifica Please see the examples below or refer to the full [v2 ScaledObject Specification](../concepts/scaling-deployments/#scaledobject-spec) - **Example of v1 ScaledObject** +**Example of v1 ScaledObject** + ```yaml apiVersion: keda.k8s.io/v1alpha1 kind: ScaledObject metadata: - name: {scaled-object-name} + name: { scaled-object-name } labels: - deploymentName: {deployment-name} + deploymentName: { deployment-name } spec: scaleTargetRef: - deploymentName: {deployment-name} - containerName: {container-name} - pollingInterval: 30 - cooldownPeriod: 300 - minReplicaCount: 0 - maxReplicaCount: 100 + deploymentName: { deployment-name } + containerName: { container-name } + pollingInterval: 30 + cooldownPeriod: 300 + minReplicaCount: 0 + maxReplicaCount: 100 triggers: # {list of triggers to activate the deployment} ``` @@ -50,18 +53,18 @@ spec: **Example of v2 ScaledObject** ```yaml -apiVersion: keda.sh/v1alpha1 # <--- Property value was changed +apiVersion: keda.sh/v1alpha1 # <--- Property value was changed kind: ScaledObject -metadata: # <--- labels.deploymentName is not needed - name: {scaled-object-name} +metadata: # <--- labels.deploymentName is not needed + name: { scaled-object-name } spec: scaleTargetRef: - name: {deployment-name} # <--- Property name was changed - envSourceContainerName: {container-name} # <--- Property name was changed - pollingInterval: 30 - cooldownPeriod: 300 - minReplicaCount: 0 - maxReplicaCount: 100 + name: { deployment-name } # <--- Property name was changed + envSourceContainerName: { container-name } # <--- Property name was changed + pollingInterval: 30 + cooldownPeriod: 300 + minReplicaCount: 0 + maxReplicaCount: 100 triggers: # {list of triggers to activate the deployment} ``` @@ -69,34 +72,36 @@ spec: ### Scaling of Jobs In order to scale `Jobs` with KEDA v2, you need to do only a few modifications to existing v1 `ScaledObjects` definitions, so they comply with v2: + - Change the value of `apiVersion` property from `keda.k8s.io/v1alpha1` to `keda.sh/v1alpha1` - Change the value of `kind` property from `ScaledObject` to `ScaledJob` - Remove property `spec.scaleType` -- Remove properties `spec.cooldownPeriod` and `spec.minReplicaCount` +- Remove properties `spec.cooldownPeriod` and `spec.minReplicaCount` You can configure `successfulJobsHistoryLimit` and `failedJobsHistoryLimit`. They will remove the old job histories automatically. Please see the examples below or refer to the full [v2 ScaledJob Specification](../concepts/scaling-jobs/#scaledjob-spec) **Example of v1 ScaledObject for Jobs scaling** + ```yaml apiVersion: keda.k8s.io/v1alpha1 kind: ScaledObject metadata: - name: {scaled-object-name} + name: { scaled-object-name } spec: scaleType: job jobTargetRef: - parallelism: 1 + parallelism: 1 completions: 1 activeDeadlineSeconds: 600 - backoffLimit: 6 + backoffLimit: 6 template: # {job template} - pollingInterval: 30 - cooldownPeriod: 300 - minReplicaCount: 0 - maxReplicaCount: 100 + pollingInterval: 30 + cooldownPeriod: 300 + minReplicaCount: 0 + maxReplicaCount: 100 triggers: # {list of triggers to create jobs} ``` @@ -104,21 +109,21 @@ spec: **Example of v2 ScaledJob** ```yaml -apiVersion: keda.sh/v1alpha1 # <--- Property value was changed -kind: ScaledJob # <--- Property value was changed +apiVersion: keda.sh/v1alpha1 # <--- Property value was changed +kind: ScaledJob # <--- Property value was changed metadata: - name: {scaled-job-name} -spec: # <--- spec.scaleType is not needed + name: { scaled-job-name } +spec: # <--- spec.scaleType is not needed jobTargetRef: - parallelism: 1 - completions: 1 + parallelism: 1 + completions: 1 activeDeadlineSeconds: 600 backoffLimit: 6 template: # {job template} - pollingInterval: 30 # <--- spec.cooldownPeriod and spec.minReplicaCount are not needed - successfulJobsHistoryLimit: 5 # <--- property is added - failedJobsHistoryLimit: 5 # <--- Property is added + pollingInterval: 30 # <--- spec.cooldownPeriod and spec.minReplicaCount are not needed + successfulJobsHistoryLimit: 5 # <--- property is added + failedJobsHistoryLimit: 5 # <--- Property is added maxReplicaCount: 100 triggers: # {list of triggers to create jobs} @@ -132,68 +137,74 @@ We've introduced more options to configure trigger metadata to give users more f Here's an overview: -| Scaler | 1.x | 2.0 | -|--------|--------|--------| -| `azure-blob` | `connection` (**Default**: `AzureWebJobsStorage`) | `connectionFromEnv` | -| `azure-monitor` | `activeDirectoryClientId`
`activeDirectoryClientPassword` | `activeDirectoryClientId`
`activeDirectoryClientIdFromEnv`
`activeDirectoryClientPasswordFromEnv` | -| `azure-queue` | `connection` (**Default**: AzureWebJobsStorage) | `connectionFromEnv` | -| `azure-servicebus` | `connection` | `connectionFromEnv` | -| `azure-eventhub` | `storageConnection` (**Default**: `AzureWebJobsStorage`)
`connection` (**Default**: `EventHub`) | `storageConnectionFromEnv`
`connectionFromEnv` | -| `aws-cloudwatch` | `awsAccessKeyID` (**Default**: `AWS_ACCESS_KEY_ID`)
`awsSecretAccessKey` (**Default**: `AWS_SECRET_ACCESS_KEY`) | `awsAccessKeyID`
`awsAccessKeyIDFromEnv`
`awsSecretAccessKeyFromEnv` | -| `aws-kinesis-stream` | `awsAccessKeyID` (**Default**: `AWS_ACCESS_KEY_ID`)
`awsSecretAccessKey` (**Default**: `AWS_SECRET_ACCESS_KEY`) | `awsAccessKeyID`
`awsAccessKeyIDFromEnv`
`awsSecretAccessKeyFromEnv` | -| `aws-sqs-queue` | `awsAccessKeyID` (**Default**: `AWS_ACCESS_KEY_ID`)
`awsSecretAccessKey` (**Default**: `AWS_SECRET_ACCESS_KEY`) | `awsAccessKeyID`
`awsAccessKeyIDFromEnv`
`awsSecretAccessKeyFromEnv` | -| `kafka` | _(none)_ | _(none)_ -| `rabbitmq` | `apiHost`
`host` | ~~`apiHost`~~
`host`
`hostFromEnv` | -| `prometheus` | _(none)_ | _(none)_ | -| `cron` | _(none)_ | _(none)_ | -| `redis` | `address`
`host`
`port`
`password` | `address`
`addressFromEnv`
`host`
`hostFromEnv`
~~`port`~~
`passwordFromEnv` -| `redis-streams` | `address`
`host`
`port`
`password` | `address`
`addressFromEnv`
`host`
`hostFromEnv`
~~`port`~~
`passwordFromEnv` -| `gcp-pubsub` | `credentials` | `credentialsFromEnv` | -| `external` | _(any matching value)_ | _(any matching value with `FromEnv` suffix)_ -| `liiklus` | _(none)_ | _(none)_ | -| `stan` | _(none)_ | _(none)_ | -| `huawei-cloudeye` | | _(none)_ | _(none)_ | -| `postgresql` | `connection`
`password` | `connectionFromEnv`
`passwordFromEnv` | -| `mysql` | `connectionString`
`password` | `connectionStringFromEnv`
`passwordFromEnv` | +| Scaler | 1.x | 2.0 | +| -------------------- | --------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------- | -------- | +| `azure-blob` | `connection` (**Default**: `AzureWebJobsStorage`) | `connectionFromEnv` | +| `azure-monitor` | `activeDirectoryClientId` `activeDirectoryClientPassword` | `activeDirectoryClientId` `activeDirectoryClientIdFromEnv` `activeDirectoryClientPasswordFromEnv` | +| `azure-queue` | `connection` (**Default**: AzureWebJobsStorage) | `connectionFromEnv` | +| `azure-servicebus` | `connection` | `connectionFromEnv` | +| `azure-eventhub` | `storageConnection` (**Default**: `AzureWebJobsStorage`) `connection` (**Default**: `EventHub`) | `storageConnectionFromEnv` `connectionFromEnv` | +| `aws-cloudwatch` | `awsAccessKeyID` (**Default**: `AWS_ACCESS_KEY_ID`) `awsSecretAccessKey` (**Default**: `AWS_SECRET_ACCESS_KEY`) | `awsAccessKeyID` `awsAccessKeyIDFromEnv` `awsSecretAccessKeyFromEnv` | +| `aws-kinesis-stream` | `awsAccessKeyID` (**Default**: `AWS_ACCESS_KEY_ID`) `awsSecretAccessKey` (**Default**: `AWS_SECRET_ACCESS_KEY`) | `awsAccessKeyID` `awsAccessKeyIDFromEnv` `awsSecretAccessKeyFromEnv` | +| `aws-sqs-queue` | `awsAccessKeyID` (**Default**: `AWS_ACCESS_KEY_ID`) `awsSecretAccessKey` (**Default**: `AWS_SECRET_ACCESS_KEY`) | `awsAccessKeyID` `awsAccessKeyIDFromEnv` `awsSecretAccessKeyFromEnv` | +| `kafka` | _(none)_ | _(none)_ | +| `rabbitmq` | `apiHost` `host` | ~~`apiHost`~~ `host` `hostFromEnv` | +| `prometheus` | _(none)_ | _(none)_ | +| `cron` | _(none)_ | _(none)_ | +| `redis` | `address` `host` `port` `password` | `address` `addressFromEnv` `host` `hostFromEnv` ~~`port`~~ `passwordFromEnv` | +| `redis-streams` | `address` `host` `port` `password` | `address` `addressFromEnv` `host` `hostFromEnv` ~~`port`~~ `passwordFromEnv` | +| `gcp-pubsub` | `credentials` | `credentialsFromEnv` | +| `external` | _(any matching value)_ | _(any matching value with `FromEnv` suffix)_ | +| `liiklus` | _(none)_ | _(none)_ | +| `stan` | _(none)_ | _(none)_ | +| `huawei-cloudeye` | | _(none)_ | _(none)_ | +| `postgresql` | `connection` `password` | `connectionFromEnv` `passwordFromEnv` | +| `mysql` | `connectionString` `password` | `connectionStringFromEnv` `passwordFromEnv` | ### Scalers **Azure Service Bus** - - `queueLength` was renamed to `messageCount` + +- `queueLength` was renamed to `messageCount` **Kafka** - - `authMode` property was replaced with `sasl` and `tls` properties. Please refer [documentation](../scalers/apache-kafka/#authentication-parameters) for Kafka Authentication Parameters details. + +- `authMode` property was replaced with `sasl` and `tls` properties. Please refer [documentation](../scalers/apache-kafka/#authentication-parameters) for Kafka Authentication Parameters details. **RabbitMQ** In KEDA 2.0 the RabbitMQ scaler has only `host` parameter, and the protocol for communication can be specified by -`protocol` (http or amqp). The default value is `amqp`. The behavior changes only for scalers that were using HTTP -protocol. +`protocol` (http or amqp). The default value is `amqp`. The behavior changes only for scalers that were using HTTP +protocol. Example of RabbitMQ trigger before 2.0: + ```yaml triggers: -- type: rabbitmq - metadata: - queueLength: '20' - queueName: testqueue - includeUnacked: 'true' - apiHost: 'https://guest:password@localhost:443/vhostname' -``` + - type: rabbitmq + metadata: + queueLength: "20" + queueName: testqueue + includeUnacked: "true" + apiHost: "https://guest:password@localhost:443/vhostname" +``` The same trigger in 2.0: + ```yaml triggers: -- type: rabbitmq - metadata: - queueLength: '20' - queueName: testqueue - protocol: 'http' - host: 'https://guest:password@localhost:443/vhostname' -``` + - type: rabbitmq + metadata: + queueLength: "20" + queueName: testqueue + protocol: "http" + host: "https://guest:password@localhost:443/vhostname" +``` ### TriggerAuthentication + In order to use Authentication via `TriggerAuthentication` with KEDA v2, you need to change: + - Change the value of `apiVersion` property from `keda.k8s.io/v1alpha1` to `keda.sh/v1alpha1` For more details please refer to the full diff --git a/content/docs/2.2/operate/cluster.md b/content/docs/2.2/operate/cluster.md index 862719278..f010cc0f2 100644 --- a/content/docs/2.2/operate/cluster.md +++ b/content/docs/2.2/operate/cluster.md @@ -29,12 +29,10 @@ KEDA requires to be accessible inside the cluster to be able to autoscale. Here is an overview of the required ports that need to be accessible for KEDA to work: - | Port | Why? | Remarks | | ------ | -------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `443` | Used by Kubernetes API server to get metrics | Required for all platforms because it uses Control Plane → port 443 on the Service IP range communication.

This is not applicable for Google Cloud. | +| `443` | Used by Kubernetes API server to get metrics | Required for all platforms because it uses Control Plane → port 443 on the Service IP range communication. This is not applicable for Google Cloud. | | `6443` | Used by Kubernetes API server to get metrics | Only required for Google Cloud because it uses Control Plane → port 6443 on the Pod IP range for communication | - ## High Availability @@ -42,10 +40,10 @@ KEDA does not provide full support for high-availability due to upstream limitat Here is an overview of all KEDA deployments and the HA notes: -| Deployment | Support Replicas | Note | -| -------------- | ---------------- |---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Deployment | Support Replicas | Note | +| -------------- | ---------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | Metrics Server | 1 | You can run multiple replicas of our metrics sever, and it is recommended to add the `--enable-aggregator-routing=true` CLI flag to the kube-apiserver so that requests sent to our metrics servers are load balanced. However, [you can only run one active metric server in a Kubernetes cluster serving external.metrics.k8s.io](https://github.com/kubernetes-sigs/custom-metrics-apiserver/issues/70) which has to be the KEDA metric server. | -| Operator | 2 | While you can run multiple replicas of our operator, only one operator instance will be active. The rest will be standing by, which may reduce downtime during a failure. Multiple replicas will not improve the performance of KEDA, it could only reduce a downtime during a failover.

This is only supported as of KEDA v2.6 if you are using our Helm chart. | +| Operator | 2 | While you can run multiple replicas of our operator, only one operator instance will be active. The rest will be standing by, which may reduce downtime during a failure. Multiple replicas will not improve the performance of KEDA, it could only reduce a downtime during a failover.This is only supported as of KEDA v2.6 if you are using our Helm chart. | ## HTTP Timeouts diff --git a/content/docs/2.3/concepts/external-scalers.md b/content/docs/2.3/concepts/external-scalers.md index 27ed9e0c6..1945394d3 100644 --- a/content/docs/2.3/concepts/external-scalers.md +++ b/content/docs/2.3/concepts/external-scalers.md @@ -33,15 +33,15 @@ type PushScaler interface { ``` The `Scaler` interface defines 4 methods: + - `IsActive` is called on `pollingInterval` defined in the ScaledObject/ScaledJob CRDs and scaling to 1 happens if this returns true. - `Close` is called to allow the scaler to clean up connections or other resources. - `GetMetricSpec` returns the target value for the HPA definition for the scaler. For more details refer to [Implementing `GetMetricSpec`](#5-implementing-getmetricspec). - `GetMetrics` returns the value of the metric referred to from `GetMetricSpec`. For more details refer to [Implementing `GetMetrics`](#6-implementing-getmetrics). -> Refer to the [HPA docs](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale-walkthrough/) for how HPA calculates `replicaCount` based on metric value and target value. KEDA uses the metric target type `AverageValue` for external metrics. This will cause the metric value returned by the external scaler to be divided by the number of replicas. + > Refer to the [HPA docs](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale-walkthrough/) for how HPA calculates `replicaCount` based on metric value and target value. KEDA uses the metric target type `AverageValue` for external metrics. This will cause the metric value returned by the external scaler to be divided by the number of replicas. The `PushScaler` interface adds `Run` method. The `Run` method receives a push channel (`active`), that the scaler can push `true` to any time to force scaling action independently from `pollingInterval`. - ### External Scaler GRPC interface KEDA comes with 2 external scalers [`external`](../scalers/external.md) and [`external-push`](../scalers/external-push.md). @@ -62,6 +62,7 @@ service ExternalScaler { - `StreamIsActive` maps to the `Run` method on the `PushScaler` interface. Few things to notice: + - Lack of `Close` method as the GRPC connection defines the lifetime of the scaler - `IsActive`, `StreamIsActive`, and `GetMetricsSpec` are called with a `ScaledObjectRef` that contains the scaledObject name/namespace as well as the content of `metadata` defined in the trigger. @@ -125,6 +126,7 @@ go mod init example.com/external-scaler/sample mkdir externalscaler protoc externalscaler.proto --go_out=plugins=grpc:externalscaler ``` + {{< /collapsible >}} {{< collapsible "C#" >}} @@ -161,9 +163,8 @@ mkdir Services ```bash npm install --save grpc request ``` -{{< /collapsible >}} -
+{{< /collapsible >}} #### 3. Implementing `IsActive` @@ -193,6 +194,7 @@ spec: Full implementation can be found here: https://github.com/kedacore/external-scaler-samples `main.go` + ```golang func (e *ExternalScaler) IsActive(ctx context.Context, scaledObject *pb.ScaledObjectRef) (*pb.IsActiveResponse, error) { // request.Scalermetadata contains the `metadata` defined in the ScaledObject @@ -239,12 +241,14 @@ func (e *ExternalScaler) IsActive(ctx context.Context, scaledObject *pb.ScaledOb }, nil } ``` + {{< /collapsible >}} {{< collapsible "C#" >}} Full implementation can be found here: https://github.com/kedacore/external-scaler-samples `Services/ExternalScalerService.cs` + ```csharp public class ExternalScalerService : ExternalScaler.ExternalScalerBase { @@ -277,73 +281,77 @@ public class ExternalScalerService : ExternalScaler.ExternalScalerBase } } ``` + {{< /collapsible >}} {{< collapsible "Javascript" >}} `index.js` + ```js -const grpc = require('grpc') -const request = require('request') -const externalScalerProto = grpc.load('externalscaler.proto') +const grpc = require("grpc"); +const request = require("request"); +const externalScalerProto = grpc.load("externalscaler.proto"); -const server = new grpc.Server() +const server = new grpc.Server(); server.addService(externalScalerProto.externalscaler.ExternalScaler.service, { isActive: (call, callback) => { - const longitude = call.request.scalerMetadata.longitude - const latitude = call.request.scalerMetadata.latitude + const longitude = call.request.scalerMetadata.longitude; + const latitude = call.request.scalerMetadata.latitude; if (!longitude || !latitude) { callback({ code: grpc.status.INVALID_ARGUMENT, - details: 'longitude and latitude must be specified', - }) + details: "longitude and latitude must be specified", + }); } else { - const now = new Date() - const yesterday = new Date(new Date().setDate(new Date().getDate()-1)); - - const startTime = `${yesterday.getUTCFullYear()}-${yesterday.getUTCMonth()}-${yesterday.getUTCDay()}` - const endTime = `${now.getUTCFullYear()}-${now.getUTCMonth()}-${now.getUTCDay()}` - const radiusKm = 500 - const query = `format=geojson&starttime=${startTime}&endtime=${endTime}&longitude=${longitude}&latitude=${latitude}&maxradiuskm=${radiusKm}` - - request.get({ - url: `https://earthquake.usgs.gov/fdsnws/event/1/query?${query}`, - json: true, - }, (err, resp, data) => { - if (err) { - callback({ - code: grpc.status.INTERNAL, - details: err, - }) - } else if (resp.statusCode !== 200) { - callback({ - code: grpc.status.INTERNAL, - details: `expected status 200, got ${resp.statusCode}` - }) - } else { - // count how many earthquakes with mag > 1.0 - let count = 0 - data.features.forEach(i => { - if (i.properties.mag > 1.0) { - count++ - } - }) - callback(null, { - result: count > 2, - }) + const now = new Date(); + const yesterday = new Date(new Date().setDate(new Date().getDate() - 1)); + + const startTime = `${yesterday.getUTCFullYear()}-${yesterday.getUTCMonth()}-${yesterday.getUTCDay()}`; + const endTime = `${now.getUTCFullYear()}-${now.getUTCMonth()}-${now.getUTCDay()}`; + const radiusKm = 500; + const query = `format=geojson&starttime=${startTime}&endtime=${endTime}&longitude=${longitude}&latitude=${latitude}&maxradiuskm=${radiusKm}`; + + request.get( + { + url: `https://earthquake.usgs.gov/fdsnws/event/1/query?${query}`, + json: true, + }, + (err, resp, data) => { + if (err) { + callback({ + code: grpc.status.INTERNAL, + details: err, + }); + } else if (resp.statusCode !== 200) { + callback({ + code: grpc.status.INTERNAL, + details: `expected status 200, got ${resp.statusCode}`, + }); + } else { + // count how many earthquakes with mag > 1.0 + let count = 0; + data.features.forEach((i) => { + if (i.properties.mag > 1.0) { + count++; + } + }); + callback(null, { + result: count > 2, + }); + } } - }) + ); } - } -}) + }, +}); -server.bind('127.0.0.1:9090', grpc.ServerCredentials.createInsecure()) -console.log('Server listening on 127.0.0.1:9090') +server.bind("127.0.0.1:9090", grpc.ServerCredentials.createInsecure()); +console.log("Server listening on 127.0.0.1:9090"); -server.start() +server.start(); ``` -{{< /collapsible >}} -
+{{< /collapsible >}} #### 4. Implementing `StreamIsActive` @@ -351,8 +359,8 @@ Unlike `IsActive`, `StreamIsActive` is called once when KEDA reconciles the `Sca This implementation creates a timer and queries USGS APIs on that timer, effectively ignoring `pollingInterval` set in the scaledObject. Alternatively any other asynchronous event can be used instead of a timer, like an HTTP request, or a network connection. - {{< collapsible "Golang" >}} + ```golang func (e *ExternalScaler) StreamIsActive(scaledObject *pb.ScaledObjectRef, epsServer pb.ExternalScaler_StreamIsActiveServer) error { longitude := scaledObject.ScalerMetadata["longitude"] @@ -380,9 +388,11 @@ func (e *ExternalScaler) StreamIsActive(scaledObject *pb.ScaledObjectRef, epsSer } } ``` + {{< /collapsible >}} {{< collapsible "C#" >}} + ```csharp public override async Task StreamIsActive(ScaledObjectRef request, IServerStreamWriter responseStream, ServerCallContext context) { @@ -410,49 +420,51 @@ public override async Task StreamIsActive(ScaledObjectRef request, IServerStream } } ``` + {{< /collapsible >}} {{< collapsible "Javascript" >}} + ```js server.addService(externalScalerProto.externalscaler.ExternalScaler.service, { // ... streamIsActive: (call, callback) => { - const longitude = call.request.scalerMetadata.longitude - const latitude = call.request.scalerMetadata.latitude + const longitude = call.request.scalerMetadata.longitude; + const latitude = call.request.scalerMetadata.latitude; if (!longitude || !latitude) { callback({ code: grpc.status.INVALID_ARGUMENT, - details: 'longitude and latitude must be specified', - }) + details: "longitude and latitude must be specified", + }); } else { const interval = setInterval(() => { getEarthquakeCount((err, count) => { if (err) { - console.error(err) + console.error(err); } else if (count > 2) { call.write({ result: true, - }) + }); } - }) + }); }, 1000 * 60 * 60); - call.on('end', () => { - clearInterval(interval) - }) + call.on("end", () => { + clearInterval(interval); + }); } - } -}) + }, +}); ``` -{{< /collapsible >}} -
+{{< /collapsible >}} #### 5. Implementing `GetMetricSpec` `GetMetricSpec` returns the `target` value for [the HPA definition for the scaler](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale-walkthrough/). This scaler will define a static target of 10, but the threshold value is often specified in the metadata for other scalers. {{< collapsible "Golang" >}} + ```golang func (e *ExternalScaler) GetMetricSpec(context.Context, *pb.ScaledObjectRef) (*pb.GetMetricSpecResponse, error) { return &pb.GetMetricSpecResponse{ @@ -463,9 +475,11 @@ func (e *ExternalScaler) GetMetricSpec(context.Context, *pb.ScaledObjectRef) (*p }, nil } ``` + {{< /collapsible >}} {{< collapsible "C#" >}} + ```csharp public override async Task GetMetricSpec(ScaledObjectRef request, ServerCallContext context) { @@ -480,31 +494,35 @@ public override async Task GetMetricSpec(ScaledObjectRef return Task.FromResult(resp); } ``` + {{< /collapsible >}} {{< collapsible "Javascript" >}} + ```js server.addService(externalScalerProto.externalscaler.ExternalScaler.service, { // ... getMetricSpec: (call, callback) => { callback(null, { - metricSpecs: [{ - metricName: 'earthquakeThreshold', - targetSize: 10, - }] - }) - } -}) + metricSpecs: [ + { + metricName: "earthquakeThreshold", + targetSize: 10, + }, + ], + }); + }, +}); ``` -{{< /collapsible >}} -
+{{< /collapsible >}} #### 6. Implementing `GetMetrics` `GetMetrics` returns the value of the metric referred to from `GetMetricSpec`, in this example it's `earthquakeThreshold`. {{< collapsible "Golang" >}} + ```golang func (e *ExternalScaler) GetMetrics(_ context.Context, metricRequest *pb.GetMetricsRequest) (*pb.GetMetricsResponse, error) { longitude := metricRequest.ScaledObjectRef.ScalerMetadata["longitude"] @@ -527,9 +545,11 @@ func (e *ExternalScaler) GetMetrics(_ context.Context, metricRequest *pb.GetMetr }, nil } ``` + {{< /collapsible >}} {{< collapsible "C#" >}} + ```csharp public override async Task GetMetrics(GetMetricsRequest request, ServerCallContext context) { @@ -554,38 +574,43 @@ public override async Task GetMetrics(GetMetricsRequest requ return resp; } ``` + {{< /collapsible >}} {{< collapsible "Javascript" >}} + ```js server.addService(externalScalerProto.externalscaler.ExternalScaler.service, { // ... getMetrics: (call, callback) => { - const longitude = call.request.scaledObjectRef.scalerMetadata.longitude - const latitude = call.request.scaledObjectRef.scalerMetadata.latitude + const longitude = call.request.scaledObjectRef.scalerMetadata.longitude; + const latitude = call.request.scaledObjectRef.scalerMetadata.latitude; if (!longitude || !latitude) { callback({ code: grpc.status.INVALID_ARGUMENT, - details: 'longitude and latitude must be specified', - }) + details: "longitude and latitude must be specified", + }); } else { getEarthquakeCount((err, count) => { if (err) { callback({ code: grpc.status.INTERNAL, details: err, - }) + }); } else { callback(null, { - metricValues: [{ - metricName: 'earthquakeThreshold', - metricValue: count, - }] - }) + metricValues: [ + { + metricName: "earthquakeThreshold", + metricValue: count, + }, + ], + }); } - }) + }); } - } -}) + }, +}); ``` + {{< /collapsible >}} diff --git a/content/docs/2.3/migration.md b/content/docs/2.3/migration.md index 5ee8a6455..29e8408c8 100644 --- a/content/docs/2.3/migration.md +++ b/content/docs/2.3/migration.md @@ -11,15 +11,17 @@ Please note that you **can not** run both KEDA v1 and v2 on the same Kubernetes KEDA v2 is using a new API namespace for its Custom Resources Definitions (CRD): `keda.sh` instead of `keda.k8s.io` and introduces a new Custom Resource for scaling of Jobs. See full details on KEDA Custom Resources [here](../concepts/#custom-resources-crd). Here's an overview of what's changed: + - [Scaling of Deployments](#scaling-of-deployments) - [Scaling of Jobs](#scaling-of-jobs) - [Improved flexibility & usability of trigger metadata](#improved-flexibility--usability-of-trigger-metadata) - [Scalers](#scalers) - [TriggerAuthentication](#triggerauthentication) - ### Scaling of Deployments + In order to scale `Deployments` with KEDA v2, you need to do only a few modifications to existing v1 `ScaledObjects` definitions, so they comply with v2: + - Change the value of `apiVersion` property from `keda.k8s.io/v1alpha1` to `keda.sh/v1alpha1` - Rename property `spec.scaleTargetRef.deploymentName` to `spec.scaleTargetRef.name` - Rename property `spec.scaleTargetRef.containerName` to `spec.scaleTargetRef.envSourceContainerName` @@ -27,22 +29,23 @@ In order to scale `Deployments` with KEDA v2, you need to do only a few modifica Please see the examples below or refer to the full [v2 ScaledObject Specification](../concepts/scaling-deployments/#scaledobject-spec) - **Example of v1 ScaledObject** +**Example of v1 ScaledObject** + ```yaml apiVersion: keda.k8s.io/v1alpha1 kind: ScaledObject metadata: - name: {scaled-object-name} + name: { scaled-object-name } labels: - deploymentName: {deployment-name} + deploymentName: { deployment-name } spec: scaleTargetRef: - deploymentName: {deployment-name} - containerName: {container-name} - pollingInterval: 30 - cooldownPeriod: 300 - minReplicaCount: 0 - maxReplicaCount: 100 + deploymentName: { deployment-name } + containerName: { container-name } + pollingInterval: 30 + cooldownPeriod: 300 + minReplicaCount: 0 + maxReplicaCount: 100 triggers: # {list of triggers to activate the deployment} ``` @@ -50,18 +53,18 @@ spec: **Example of v2 ScaledObject** ```yaml -apiVersion: keda.sh/v1alpha1 # <--- Property value was changed +apiVersion: keda.sh/v1alpha1 # <--- Property value was changed kind: ScaledObject -metadata: # <--- labels.deploymentName is not needed - name: {scaled-object-name} +metadata: # <--- labels.deploymentName is not needed + name: { scaled-object-name } spec: scaleTargetRef: - name: {deployment-name} # <--- Property name was changed - envSourceContainerName: {container-name} # <--- Property name was changed - pollingInterval: 30 - cooldownPeriod: 300 - minReplicaCount: 0 - maxReplicaCount: 100 + name: { deployment-name } # <--- Property name was changed + envSourceContainerName: { container-name } # <--- Property name was changed + pollingInterval: 30 + cooldownPeriod: 300 + minReplicaCount: 0 + maxReplicaCount: 100 triggers: # {list of triggers to activate the deployment} ``` @@ -69,34 +72,36 @@ spec: ### Scaling of Jobs In order to scale `Jobs` with KEDA v2, you need to do only a few modifications to existing v1 `ScaledObjects` definitions, so they comply with v2: + - Change the value of `apiVersion` property from `keda.k8s.io/v1alpha1` to `keda.sh/v1alpha1` - Change the value of `kind` property from `ScaledObject` to `ScaledJob` - Remove property `spec.scaleType` -- Remove properties `spec.cooldownPeriod` and `spec.minReplicaCount` +- Remove properties `spec.cooldownPeriod` and `spec.minReplicaCount` You can configure `successfulJobsHistoryLimit` and `failedJobsHistoryLimit`. They will remove the old job histories automatically. Please see the examples below or refer to the full [v2 ScaledJob Specification](../concepts/scaling-jobs/#scaledjob-spec) **Example of v1 ScaledObject for Jobs scaling** + ```yaml apiVersion: keda.k8s.io/v1alpha1 kind: ScaledObject metadata: - name: {scaled-object-name} + name: { scaled-object-name } spec: scaleType: job jobTargetRef: - parallelism: 1 + parallelism: 1 completions: 1 activeDeadlineSeconds: 600 - backoffLimit: 6 + backoffLimit: 6 template: # {job template} - pollingInterval: 30 - cooldownPeriod: 300 - minReplicaCount: 0 - maxReplicaCount: 100 + pollingInterval: 30 + cooldownPeriod: 300 + minReplicaCount: 0 + maxReplicaCount: 100 triggers: # {list of triggers to create jobs} ``` @@ -104,21 +109,21 @@ spec: **Example of v2 ScaledJob** ```yaml -apiVersion: keda.sh/v1alpha1 # <--- Property value was changed -kind: ScaledJob # <--- Property value was changed +apiVersion: keda.sh/v1alpha1 # <--- Property value was changed +kind: ScaledJob # <--- Property value was changed metadata: - name: {scaled-job-name} -spec: # <--- spec.scaleType is not needed + name: { scaled-job-name } +spec: # <--- spec.scaleType is not needed jobTargetRef: - parallelism: 1 - completions: 1 + parallelism: 1 + completions: 1 activeDeadlineSeconds: 600 backoffLimit: 6 template: # {job template} - pollingInterval: 30 # <--- spec.cooldownPeriod and spec.minReplicaCount are not needed - successfulJobsHistoryLimit: 5 # <--- property is added - failedJobsHistoryLimit: 5 # <--- Property is added + pollingInterval: 30 # <--- spec.cooldownPeriod and spec.minReplicaCount are not needed + successfulJobsHistoryLimit: 5 # <--- property is added + failedJobsHistoryLimit: 5 # <--- Property is added maxReplicaCount: 100 triggers: # {list of triggers to create jobs} @@ -132,68 +137,74 @@ We've introduced more options to configure trigger metadata to give users more f Here's an overview: -| Scaler | 1.x | 2.0 | -|--------|--------|--------| -| `azure-blob` | `connection` (**Default**: `AzureWebJobsStorage`) | `connectionFromEnv` | -| `azure-monitor` | `activeDirectoryClientId`
`activeDirectoryClientPassword` | `activeDirectoryClientId`
`activeDirectoryClientIdFromEnv`
`activeDirectoryClientPasswordFromEnv` | -| `azure-queue` | `connection` (**Default**: AzureWebJobsStorage) | `connectionFromEnv` | -| `azure-servicebus` | `connection` | `connectionFromEnv` | -| `azure-eventhub` | `storageConnection` (**Default**: `AzureWebJobsStorage`)
`connection` (**Default**: `EventHub`) | `storageConnectionFromEnv`
`connectionFromEnv` | -| `aws-cloudwatch` | `awsAccessKeyID` (**Default**: `AWS_ACCESS_KEY_ID`)
`awsSecretAccessKey` (**Default**: `AWS_SECRET_ACCESS_KEY`) | `awsAccessKeyID`
`awsAccessKeyIDFromEnv`
`awsSecretAccessKeyFromEnv` | -| `aws-kinesis-stream` | `awsAccessKeyID` (**Default**: `AWS_ACCESS_KEY_ID`)
`awsSecretAccessKey` (**Default**: `AWS_SECRET_ACCESS_KEY`) | `awsAccessKeyID`
`awsAccessKeyIDFromEnv`
`awsSecretAccessKeyFromEnv` | -| `aws-sqs-queue` | `awsAccessKeyID` (**Default**: `AWS_ACCESS_KEY_ID`)
`awsSecretAccessKey` (**Default**: `AWS_SECRET_ACCESS_KEY`) | `awsAccessKeyID`
`awsAccessKeyIDFromEnv`
`awsSecretAccessKeyFromEnv` | -| `kafka` | _(none)_ | _(none)_ -| `rabbitmq` | `apiHost`
`host` | ~~`apiHost`~~
`host`
`hostFromEnv` | -| `prometheus` | _(none)_ | _(none)_ | -| `cron` | _(none)_ | _(none)_ | -| `redis` | `address`
`host`
`port`
`password` | `address`
`addressFromEnv`
`host`
`hostFromEnv`
~~`port`~~
`passwordFromEnv` -| `redis-streams` | `address`
`host`
`port`
`password` | `address`
`addressFromEnv`
`host`
`hostFromEnv`
~~`port`~~
`passwordFromEnv` -| `gcp-pubsub` | `credentials` | `credentialsFromEnv` | -| `external` | _(any matching value)_ | _(any matching value with `FromEnv` suffix)_ -| `liiklus` | _(none)_ | _(none)_ | -| `stan` | _(none)_ | _(none)_ | -| `huawei-cloudeye` | | _(none)_ | _(none)_ | -| `postgresql` | `connection`
`password` | `connectionFromEnv`
`passwordFromEnv` | -| `mysql` | `connectionString`
`password` | `connectionStringFromEnv`
`passwordFromEnv` | +| Scaler | 1.x | 2.0 | +| -------------------- | --------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------- | -------- | +| `azure-blob` | `connection` (**Default**: `AzureWebJobsStorage`) | `connectionFromEnv` | +| `azure-monitor` | `activeDirectoryClientId` `activeDirectoryClientPassword` | `activeDirectoryClientId` `activeDirectoryClientIdFromEnv` `activeDirectoryClientPasswordFromEnv` | +| `azure-queue` | `connection` (**Default**: AzureWebJobsStorage) | `connectionFromEnv` | +| `azure-servicebus` | `connection` | `connectionFromEnv` | +| `azure-eventhub` | `storageConnection` (**Default**: `AzureWebJobsStorage`) `connection` (**Default**: `EventHub`) | `storageConnectionFromEnv` `connectionFromEnv` | +| `aws-cloudwatch` | `awsAccessKeyID` (**Default**: `AWS_ACCESS_KEY_ID`) `awsSecretAccessKey` (**Default**: `AWS_SECRET_ACCESS_KEY`) | `awsAccessKeyID` `awsAccessKeyIDFromEnv` `awsSecretAccessKeyFromEnv` | +| `aws-kinesis-stream` | `awsAccessKeyID` (**Default**: `AWS_ACCESS_KEY_ID`) `awsSecretAccessKey` (**Default**: `AWS_SECRET_ACCESS_KEY`) | `awsAccessKeyID` `awsAccessKeyIDFromEnv` `awsSecretAccessKeyFromEnv` | +| `aws-sqs-queue` | `awsAccessKeyID` (**Default**: `AWS_ACCESS_KEY_ID`) `awsSecretAccessKey` (**Default**: `AWS_SECRET_ACCESS_KEY`) | `awsAccessKeyID` `awsAccessKeyIDFromEnv` `awsSecretAccessKeyFromEnv` | +| `kafka` | _(none)_ | _(none)_ | +| `rabbitmq` | `apiHost` `host` | ~~`apiHost`~~ `host` `hostFromEnv` | +| `prometheus` | _(none)_ | _(none)_ | +| `cron` | _(none)_ | _(none)_ | +| `redis` | `address` `host` `port` `password` | `address` `addressFromEnv` `host` `hostFromEnv` ~~`port`~~ `passwordFromEnv` | +| `redis-streams` | `address` `host` `port` `password` | `address` `addressFromEnv` `host` `hostFromEnv` ~~`port`~~ `passwordFromEnv` | +| `gcp-pubsub` | `credentials` | `credentialsFromEnv` | +| `external` | _(any matching value)_ | _(any matching value with `FromEnv` suffix)_ | +| `liiklus` | _(none)_ | _(none)_ | +| `stan` | _(none)_ | _(none)_ | +| `huawei-cloudeye` | | _(none)_ | _(none)_ | +| `postgresql` | `connection` `password` | `connectionFromEnv` `passwordFromEnv` | +| `mysql` | `connectionString` `password` | `connectionStringFromEnv` `passwordFromEnv` | ### Scalers **Azure Service Bus** - - `queueLength` was renamed to `messageCount` + +- `queueLength` was renamed to `messageCount` **Kafka** - - `authMode` property was replaced with `sasl` and `tls` properties. Please refer [documentation](../scalers/apache-kafka/#authentication-parameters) for Kafka Authentication Parameters details. + +- `authMode` property was replaced with `sasl` and `tls` properties. Please refer [documentation](../scalers/apache-kafka/#authentication-parameters) for Kafka Authentication Parameters details. **RabbitMQ** In KEDA 2.0 the RabbitMQ scaler has only `host` parameter, and the protocol for communication can be specified by -`protocol` (http or amqp). The default value is `amqp`. The behavior changes only for scalers that were using HTTP -protocol. +`protocol` (http or amqp). The default value is `amqp`. The behavior changes only for scalers that were using HTTP +protocol. Example of RabbitMQ trigger before 2.0: + ```yaml triggers: -- type: rabbitmq - metadata: - queueLength: '20' - queueName: testqueue - includeUnacked: 'true' - apiHost: 'https://guest:password@localhost:443/vhostname' -``` + - type: rabbitmq + metadata: + queueLength: "20" + queueName: testqueue + includeUnacked: "true" + apiHost: "https://guest:password@localhost:443/vhostname" +``` The same trigger in 2.0: + ```yaml triggers: -- type: rabbitmq - metadata: - queueLength: '20' - queueName: testqueue - protocol: 'http' - host: 'https://guest:password@localhost:443/vhostname' -``` + - type: rabbitmq + metadata: + queueLength: "20" + queueName: testqueue + protocol: "http" + host: "https://guest:password@localhost:443/vhostname" +``` ### TriggerAuthentication + In order to use Authentication via `TriggerAuthentication` with KEDA v2, you need to change: + - Change the value of `apiVersion` property from `keda.k8s.io/v1alpha1` to `keda.sh/v1alpha1` For more details please refer to the full diff --git a/content/docs/2.3/operate/cluster.md b/content/docs/2.3/operate/cluster.md index 4caa62f3d..f010cc0f2 100644 --- a/content/docs/2.3/operate/cluster.md +++ b/content/docs/2.3/operate/cluster.md @@ -29,12 +29,10 @@ KEDA requires to be accessible inside the cluster to be able to autoscale. Here is an overview of the required ports that need to be accessible for KEDA to work: - | Port | Why? | Remarks | | ------ | -------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `443` | Used by Kubernetes API server to get metrics | Required for all platforms because it uses Control Plane → port 443 on the Service IP range communication.

This is not applicable for Google Cloud. | +| `443` | Used by Kubernetes API server to get metrics | Required for all platforms because it uses Control Plane → port 443 on the Service IP range communication. This is not applicable for Google Cloud. | | `6443` | Used by Kubernetes API server to get metrics | Only required for Google Cloud because it uses Control Plane → port 6443 on the Pod IP range for communication | - ## High Availability @@ -42,14 +40,14 @@ KEDA does not provide full support for high-availability due to upstream limitat Here is an overview of all KEDA deployments and the HA notes: -| Deployment | Support Replicas | Note | -| -------------- | ---------------- |---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Deployment | Support Replicas | Note | +| -------------- | ---------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | Metrics Server | 1 | You can run multiple replicas of our metrics sever, and it is recommended to add the `--enable-aggregator-routing=true` CLI flag to the kube-apiserver so that requests sent to our metrics servers are load balanced. However, [you can only run one active metric server in a Kubernetes cluster serving external.metrics.k8s.io](https://github.com/kubernetes-sigs/custom-metrics-apiserver/issues/70) which has to be the KEDA metric server. | -| Operator | 2 | While you can run multiple replicas of our operator, only one operator instance will be active. The rest will be standing by, which may reduce downtime during a failure. Multiple replicas will not improve the performance of KEDA, it could only reduce a downtime during a failover.

This is only supported as of KEDA v2.6 if you are using our Helm chart. | +| Operator | 2 | While you can run multiple replicas of our operator, only one operator instance will be active. The rest will be standing by, which may reduce downtime during a failure. Multiple replicas will not improve the performance of KEDA, it could only reduce a downtime during a failover.This is only supported as of KEDA v2.6 if you are using our Helm chart. | ## HTTP Timeouts -Some scalers issue HTTP requests to external servers (i.e. cloud services). Each applicable scaler uses its own dedicated HTTP client with its own connection pool, and by default each client is set to time out any HTTP request after 3 seconds. +Some scalers issue HTTP requests to external servers (i.e. cloud services). Each applicable scaler uses its own dedicated HTTP client with its own connection pool, and by default each client is set to time out any HTTP request after 3 seconds. You can override this default by setting the `KEDA_HTTP_DEFAULT_TIMEOUT` environment variable to your desired timeout in milliseconds. For example, on Linux/Mac/Windows WSL2 operating systems, you'd use this command to set to 1 second: diff --git a/content/docs/2.4/concepts/external-scalers.md b/content/docs/2.4/concepts/external-scalers.md index 27ed9e0c6..1945394d3 100644 --- a/content/docs/2.4/concepts/external-scalers.md +++ b/content/docs/2.4/concepts/external-scalers.md @@ -33,15 +33,15 @@ type PushScaler interface { ``` The `Scaler` interface defines 4 methods: + - `IsActive` is called on `pollingInterval` defined in the ScaledObject/ScaledJob CRDs and scaling to 1 happens if this returns true. - `Close` is called to allow the scaler to clean up connections or other resources. - `GetMetricSpec` returns the target value for the HPA definition for the scaler. For more details refer to [Implementing `GetMetricSpec`](#5-implementing-getmetricspec). - `GetMetrics` returns the value of the metric referred to from `GetMetricSpec`. For more details refer to [Implementing `GetMetrics`](#6-implementing-getmetrics). -> Refer to the [HPA docs](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale-walkthrough/) for how HPA calculates `replicaCount` based on metric value and target value. KEDA uses the metric target type `AverageValue` for external metrics. This will cause the metric value returned by the external scaler to be divided by the number of replicas. + > Refer to the [HPA docs](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale-walkthrough/) for how HPA calculates `replicaCount` based on metric value and target value. KEDA uses the metric target type `AverageValue` for external metrics. This will cause the metric value returned by the external scaler to be divided by the number of replicas. The `PushScaler` interface adds `Run` method. The `Run` method receives a push channel (`active`), that the scaler can push `true` to any time to force scaling action independently from `pollingInterval`. - ### External Scaler GRPC interface KEDA comes with 2 external scalers [`external`](../scalers/external.md) and [`external-push`](../scalers/external-push.md). @@ -62,6 +62,7 @@ service ExternalScaler { - `StreamIsActive` maps to the `Run` method on the `PushScaler` interface. Few things to notice: + - Lack of `Close` method as the GRPC connection defines the lifetime of the scaler - `IsActive`, `StreamIsActive`, and `GetMetricsSpec` are called with a `ScaledObjectRef` that contains the scaledObject name/namespace as well as the content of `metadata` defined in the trigger. @@ -125,6 +126,7 @@ go mod init example.com/external-scaler/sample mkdir externalscaler protoc externalscaler.proto --go_out=plugins=grpc:externalscaler ``` + {{< /collapsible >}} {{< collapsible "C#" >}} @@ -161,9 +163,8 @@ mkdir Services ```bash npm install --save grpc request ``` -{{< /collapsible >}} -
+{{< /collapsible >}} #### 3. Implementing `IsActive` @@ -193,6 +194,7 @@ spec: Full implementation can be found here: https://github.com/kedacore/external-scaler-samples `main.go` + ```golang func (e *ExternalScaler) IsActive(ctx context.Context, scaledObject *pb.ScaledObjectRef) (*pb.IsActiveResponse, error) { // request.Scalermetadata contains the `metadata` defined in the ScaledObject @@ -239,12 +241,14 @@ func (e *ExternalScaler) IsActive(ctx context.Context, scaledObject *pb.ScaledOb }, nil } ``` + {{< /collapsible >}} {{< collapsible "C#" >}} Full implementation can be found here: https://github.com/kedacore/external-scaler-samples `Services/ExternalScalerService.cs` + ```csharp public class ExternalScalerService : ExternalScaler.ExternalScalerBase { @@ -277,73 +281,77 @@ public class ExternalScalerService : ExternalScaler.ExternalScalerBase } } ``` + {{< /collapsible >}} {{< collapsible "Javascript" >}} `index.js` + ```js -const grpc = require('grpc') -const request = require('request') -const externalScalerProto = grpc.load('externalscaler.proto') +const grpc = require("grpc"); +const request = require("request"); +const externalScalerProto = grpc.load("externalscaler.proto"); -const server = new grpc.Server() +const server = new grpc.Server(); server.addService(externalScalerProto.externalscaler.ExternalScaler.service, { isActive: (call, callback) => { - const longitude = call.request.scalerMetadata.longitude - const latitude = call.request.scalerMetadata.latitude + const longitude = call.request.scalerMetadata.longitude; + const latitude = call.request.scalerMetadata.latitude; if (!longitude || !latitude) { callback({ code: grpc.status.INVALID_ARGUMENT, - details: 'longitude and latitude must be specified', - }) + details: "longitude and latitude must be specified", + }); } else { - const now = new Date() - const yesterday = new Date(new Date().setDate(new Date().getDate()-1)); - - const startTime = `${yesterday.getUTCFullYear()}-${yesterday.getUTCMonth()}-${yesterday.getUTCDay()}` - const endTime = `${now.getUTCFullYear()}-${now.getUTCMonth()}-${now.getUTCDay()}` - const radiusKm = 500 - const query = `format=geojson&starttime=${startTime}&endtime=${endTime}&longitude=${longitude}&latitude=${latitude}&maxradiuskm=${radiusKm}` - - request.get({ - url: `https://earthquake.usgs.gov/fdsnws/event/1/query?${query}`, - json: true, - }, (err, resp, data) => { - if (err) { - callback({ - code: grpc.status.INTERNAL, - details: err, - }) - } else if (resp.statusCode !== 200) { - callback({ - code: grpc.status.INTERNAL, - details: `expected status 200, got ${resp.statusCode}` - }) - } else { - // count how many earthquakes with mag > 1.0 - let count = 0 - data.features.forEach(i => { - if (i.properties.mag > 1.0) { - count++ - } - }) - callback(null, { - result: count > 2, - }) + const now = new Date(); + const yesterday = new Date(new Date().setDate(new Date().getDate() - 1)); + + const startTime = `${yesterday.getUTCFullYear()}-${yesterday.getUTCMonth()}-${yesterday.getUTCDay()}`; + const endTime = `${now.getUTCFullYear()}-${now.getUTCMonth()}-${now.getUTCDay()}`; + const radiusKm = 500; + const query = `format=geojson&starttime=${startTime}&endtime=${endTime}&longitude=${longitude}&latitude=${latitude}&maxradiuskm=${radiusKm}`; + + request.get( + { + url: `https://earthquake.usgs.gov/fdsnws/event/1/query?${query}`, + json: true, + }, + (err, resp, data) => { + if (err) { + callback({ + code: grpc.status.INTERNAL, + details: err, + }); + } else if (resp.statusCode !== 200) { + callback({ + code: grpc.status.INTERNAL, + details: `expected status 200, got ${resp.statusCode}`, + }); + } else { + // count how many earthquakes with mag > 1.0 + let count = 0; + data.features.forEach((i) => { + if (i.properties.mag > 1.0) { + count++; + } + }); + callback(null, { + result: count > 2, + }); + } } - }) + ); } - } -}) + }, +}); -server.bind('127.0.0.1:9090', grpc.ServerCredentials.createInsecure()) -console.log('Server listening on 127.0.0.1:9090') +server.bind("127.0.0.1:9090", grpc.ServerCredentials.createInsecure()); +console.log("Server listening on 127.0.0.1:9090"); -server.start() +server.start(); ``` -{{< /collapsible >}} -
+{{< /collapsible >}} #### 4. Implementing `StreamIsActive` @@ -351,8 +359,8 @@ Unlike `IsActive`, `StreamIsActive` is called once when KEDA reconciles the `Sca This implementation creates a timer and queries USGS APIs on that timer, effectively ignoring `pollingInterval` set in the scaledObject. Alternatively any other asynchronous event can be used instead of a timer, like an HTTP request, or a network connection. - {{< collapsible "Golang" >}} + ```golang func (e *ExternalScaler) StreamIsActive(scaledObject *pb.ScaledObjectRef, epsServer pb.ExternalScaler_StreamIsActiveServer) error { longitude := scaledObject.ScalerMetadata["longitude"] @@ -380,9 +388,11 @@ func (e *ExternalScaler) StreamIsActive(scaledObject *pb.ScaledObjectRef, epsSer } } ``` + {{< /collapsible >}} {{< collapsible "C#" >}} + ```csharp public override async Task StreamIsActive(ScaledObjectRef request, IServerStreamWriter responseStream, ServerCallContext context) { @@ -410,49 +420,51 @@ public override async Task StreamIsActive(ScaledObjectRef request, IServerStream } } ``` + {{< /collapsible >}} {{< collapsible "Javascript" >}} + ```js server.addService(externalScalerProto.externalscaler.ExternalScaler.service, { // ... streamIsActive: (call, callback) => { - const longitude = call.request.scalerMetadata.longitude - const latitude = call.request.scalerMetadata.latitude + const longitude = call.request.scalerMetadata.longitude; + const latitude = call.request.scalerMetadata.latitude; if (!longitude || !latitude) { callback({ code: grpc.status.INVALID_ARGUMENT, - details: 'longitude and latitude must be specified', - }) + details: "longitude and latitude must be specified", + }); } else { const interval = setInterval(() => { getEarthquakeCount((err, count) => { if (err) { - console.error(err) + console.error(err); } else if (count > 2) { call.write({ result: true, - }) + }); } - }) + }); }, 1000 * 60 * 60); - call.on('end', () => { - clearInterval(interval) - }) + call.on("end", () => { + clearInterval(interval); + }); } - } -}) + }, +}); ``` -{{< /collapsible >}} -
+{{< /collapsible >}} #### 5. Implementing `GetMetricSpec` `GetMetricSpec` returns the `target` value for [the HPA definition for the scaler](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale-walkthrough/). This scaler will define a static target of 10, but the threshold value is often specified in the metadata for other scalers. {{< collapsible "Golang" >}} + ```golang func (e *ExternalScaler) GetMetricSpec(context.Context, *pb.ScaledObjectRef) (*pb.GetMetricSpecResponse, error) { return &pb.GetMetricSpecResponse{ @@ -463,9 +475,11 @@ func (e *ExternalScaler) GetMetricSpec(context.Context, *pb.ScaledObjectRef) (*p }, nil } ``` + {{< /collapsible >}} {{< collapsible "C#" >}} + ```csharp public override async Task GetMetricSpec(ScaledObjectRef request, ServerCallContext context) { @@ -480,31 +494,35 @@ public override async Task GetMetricSpec(ScaledObjectRef return Task.FromResult(resp); } ``` + {{< /collapsible >}} {{< collapsible "Javascript" >}} + ```js server.addService(externalScalerProto.externalscaler.ExternalScaler.service, { // ... getMetricSpec: (call, callback) => { callback(null, { - metricSpecs: [{ - metricName: 'earthquakeThreshold', - targetSize: 10, - }] - }) - } -}) + metricSpecs: [ + { + metricName: "earthquakeThreshold", + targetSize: 10, + }, + ], + }); + }, +}); ``` -{{< /collapsible >}} -
+{{< /collapsible >}} #### 6. Implementing `GetMetrics` `GetMetrics` returns the value of the metric referred to from `GetMetricSpec`, in this example it's `earthquakeThreshold`. {{< collapsible "Golang" >}} + ```golang func (e *ExternalScaler) GetMetrics(_ context.Context, metricRequest *pb.GetMetricsRequest) (*pb.GetMetricsResponse, error) { longitude := metricRequest.ScaledObjectRef.ScalerMetadata["longitude"] @@ -527,9 +545,11 @@ func (e *ExternalScaler) GetMetrics(_ context.Context, metricRequest *pb.GetMetr }, nil } ``` + {{< /collapsible >}} {{< collapsible "C#" >}} + ```csharp public override async Task GetMetrics(GetMetricsRequest request, ServerCallContext context) { @@ -554,38 +574,43 @@ public override async Task GetMetrics(GetMetricsRequest requ return resp; } ``` + {{< /collapsible >}} {{< collapsible "Javascript" >}} + ```js server.addService(externalScalerProto.externalscaler.ExternalScaler.service, { // ... getMetrics: (call, callback) => { - const longitude = call.request.scaledObjectRef.scalerMetadata.longitude - const latitude = call.request.scaledObjectRef.scalerMetadata.latitude + const longitude = call.request.scaledObjectRef.scalerMetadata.longitude; + const latitude = call.request.scaledObjectRef.scalerMetadata.latitude; if (!longitude || !latitude) { callback({ code: grpc.status.INVALID_ARGUMENT, - details: 'longitude and latitude must be specified', - }) + details: "longitude and latitude must be specified", + }); } else { getEarthquakeCount((err, count) => { if (err) { callback({ code: grpc.status.INTERNAL, details: err, - }) + }); } else { callback(null, { - metricValues: [{ - metricName: 'earthquakeThreshold', - metricValue: count, - }] - }) + metricValues: [ + { + metricName: "earthquakeThreshold", + metricValue: count, + }, + ], + }); } - }) + }); } - } -}) + }, +}); ``` + {{< /collapsible >}} diff --git a/content/docs/2.4/migration.md b/content/docs/2.4/migration.md index 5ee8a6455..29e8408c8 100644 --- a/content/docs/2.4/migration.md +++ b/content/docs/2.4/migration.md @@ -11,15 +11,17 @@ Please note that you **can not** run both KEDA v1 and v2 on the same Kubernetes KEDA v2 is using a new API namespace for its Custom Resources Definitions (CRD): `keda.sh` instead of `keda.k8s.io` and introduces a new Custom Resource for scaling of Jobs. See full details on KEDA Custom Resources [here](../concepts/#custom-resources-crd). Here's an overview of what's changed: + - [Scaling of Deployments](#scaling-of-deployments) - [Scaling of Jobs](#scaling-of-jobs) - [Improved flexibility & usability of trigger metadata](#improved-flexibility--usability-of-trigger-metadata) - [Scalers](#scalers) - [TriggerAuthentication](#triggerauthentication) - ### Scaling of Deployments + In order to scale `Deployments` with KEDA v2, you need to do only a few modifications to existing v1 `ScaledObjects` definitions, so they comply with v2: + - Change the value of `apiVersion` property from `keda.k8s.io/v1alpha1` to `keda.sh/v1alpha1` - Rename property `spec.scaleTargetRef.deploymentName` to `spec.scaleTargetRef.name` - Rename property `spec.scaleTargetRef.containerName` to `spec.scaleTargetRef.envSourceContainerName` @@ -27,22 +29,23 @@ In order to scale `Deployments` with KEDA v2, you need to do only a few modifica Please see the examples below or refer to the full [v2 ScaledObject Specification](../concepts/scaling-deployments/#scaledobject-spec) - **Example of v1 ScaledObject** +**Example of v1 ScaledObject** + ```yaml apiVersion: keda.k8s.io/v1alpha1 kind: ScaledObject metadata: - name: {scaled-object-name} + name: { scaled-object-name } labels: - deploymentName: {deployment-name} + deploymentName: { deployment-name } spec: scaleTargetRef: - deploymentName: {deployment-name} - containerName: {container-name} - pollingInterval: 30 - cooldownPeriod: 300 - minReplicaCount: 0 - maxReplicaCount: 100 + deploymentName: { deployment-name } + containerName: { container-name } + pollingInterval: 30 + cooldownPeriod: 300 + minReplicaCount: 0 + maxReplicaCount: 100 triggers: # {list of triggers to activate the deployment} ``` @@ -50,18 +53,18 @@ spec: **Example of v2 ScaledObject** ```yaml -apiVersion: keda.sh/v1alpha1 # <--- Property value was changed +apiVersion: keda.sh/v1alpha1 # <--- Property value was changed kind: ScaledObject -metadata: # <--- labels.deploymentName is not needed - name: {scaled-object-name} +metadata: # <--- labels.deploymentName is not needed + name: { scaled-object-name } spec: scaleTargetRef: - name: {deployment-name} # <--- Property name was changed - envSourceContainerName: {container-name} # <--- Property name was changed - pollingInterval: 30 - cooldownPeriod: 300 - minReplicaCount: 0 - maxReplicaCount: 100 + name: { deployment-name } # <--- Property name was changed + envSourceContainerName: { container-name } # <--- Property name was changed + pollingInterval: 30 + cooldownPeriod: 300 + minReplicaCount: 0 + maxReplicaCount: 100 triggers: # {list of triggers to activate the deployment} ``` @@ -69,34 +72,36 @@ spec: ### Scaling of Jobs In order to scale `Jobs` with KEDA v2, you need to do only a few modifications to existing v1 `ScaledObjects` definitions, so they comply with v2: + - Change the value of `apiVersion` property from `keda.k8s.io/v1alpha1` to `keda.sh/v1alpha1` - Change the value of `kind` property from `ScaledObject` to `ScaledJob` - Remove property `spec.scaleType` -- Remove properties `spec.cooldownPeriod` and `spec.minReplicaCount` +- Remove properties `spec.cooldownPeriod` and `spec.minReplicaCount` You can configure `successfulJobsHistoryLimit` and `failedJobsHistoryLimit`. They will remove the old job histories automatically. Please see the examples below or refer to the full [v2 ScaledJob Specification](../concepts/scaling-jobs/#scaledjob-spec) **Example of v1 ScaledObject for Jobs scaling** + ```yaml apiVersion: keda.k8s.io/v1alpha1 kind: ScaledObject metadata: - name: {scaled-object-name} + name: { scaled-object-name } spec: scaleType: job jobTargetRef: - parallelism: 1 + parallelism: 1 completions: 1 activeDeadlineSeconds: 600 - backoffLimit: 6 + backoffLimit: 6 template: # {job template} - pollingInterval: 30 - cooldownPeriod: 300 - minReplicaCount: 0 - maxReplicaCount: 100 + pollingInterval: 30 + cooldownPeriod: 300 + minReplicaCount: 0 + maxReplicaCount: 100 triggers: # {list of triggers to create jobs} ``` @@ -104,21 +109,21 @@ spec: **Example of v2 ScaledJob** ```yaml -apiVersion: keda.sh/v1alpha1 # <--- Property value was changed -kind: ScaledJob # <--- Property value was changed +apiVersion: keda.sh/v1alpha1 # <--- Property value was changed +kind: ScaledJob # <--- Property value was changed metadata: - name: {scaled-job-name} -spec: # <--- spec.scaleType is not needed + name: { scaled-job-name } +spec: # <--- spec.scaleType is not needed jobTargetRef: - parallelism: 1 - completions: 1 + parallelism: 1 + completions: 1 activeDeadlineSeconds: 600 backoffLimit: 6 template: # {job template} - pollingInterval: 30 # <--- spec.cooldownPeriod and spec.minReplicaCount are not needed - successfulJobsHistoryLimit: 5 # <--- property is added - failedJobsHistoryLimit: 5 # <--- Property is added + pollingInterval: 30 # <--- spec.cooldownPeriod and spec.minReplicaCount are not needed + successfulJobsHistoryLimit: 5 # <--- property is added + failedJobsHistoryLimit: 5 # <--- Property is added maxReplicaCount: 100 triggers: # {list of triggers to create jobs} @@ -132,68 +137,74 @@ We've introduced more options to configure trigger metadata to give users more f Here's an overview: -| Scaler | 1.x | 2.0 | -|--------|--------|--------| -| `azure-blob` | `connection` (**Default**: `AzureWebJobsStorage`) | `connectionFromEnv` | -| `azure-monitor` | `activeDirectoryClientId`
`activeDirectoryClientPassword` | `activeDirectoryClientId`
`activeDirectoryClientIdFromEnv`
`activeDirectoryClientPasswordFromEnv` | -| `azure-queue` | `connection` (**Default**: AzureWebJobsStorage) | `connectionFromEnv` | -| `azure-servicebus` | `connection` | `connectionFromEnv` | -| `azure-eventhub` | `storageConnection` (**Default**: `AzureWebJobsStorage`)
`connection` (**Default**: `EventHub`) | `storageConnectionFromEnv`
`connectionFromEnv` | -| `aws-cloudwatch` | `awsAccessKeyID` (**Default**: `AWS_ACCESS_KEY_ID`)
`awsSecretAccessKey` (**Default**: `AWS_SECRET_ACCESS_KEY`) | `awsAccessKeyID`
`awsAccessKeyIDFromEnv`
`awsSecretAccessKeyFromEnv` | -| `aws-kinesis-stream` | `awsAccessKeyID` (**Default**: `AWS_ACCESS_KEY_ID`)
`awsSecretAccessKey` (**Default**: `AWS_SECRET_ACCESS_KEY`) | `awsAccessKeyID`
`awsAccessKeyIDFromEnv`
`awsSecretAccessKeyFromEnv` | -| `aws-sqs-queue` | `awsAccessKeyID` (**Default**: `AWS_ACCESS_KEY_ID`)
`awsSecretAccessKey` (**Default**: `AWS_SECRET_ACCESS_KEY`) | `awsAccessKeyID`
`awsAccessKeyIDFromEnv`
`awsSecretAccessKeyFromEnv` | -| `kafka` | _(none)_ | _(none)_ -| `rabbitmq` | `apiHost`
`host` | ~~`apiHost`~~
`host`
`hostFromEnv` | -| `prometheus` | _(none)_ | _(none)_ | -| `cron` | _(none)_ | _(none)_ | -| `redis` | `address`
`host`
`port`
`password` | `address`
`addressFromEnv`
`host`
`hostFromEnv`
~~`port`~~
`passwordFromEnv` -| `redis-streams` | `address`
`host`
`port`
`password` | `address`
`addressFromEnv`
`host`
`hostFromEnv`
~~`port`~~
`passwordFromEnv` -| `gcp-pubsub` | `credentials` | `credentialsFromEnv` | -| `external` | _(any matching value)_ | _(any matching value with `FromEnv` suffix)_ -| `liiklus` | _(none)_ | _(none)_ | -| `stan` | _(none)_ | _(none)_ | -| `huawei-cloudeye` | | _(none)_ | _(none)_ | -| `postgresql` | `connection`
`password` | `connectionFromEnv`
`passwordFromEnv` | -| `mysql` | `connectionString`
`password` | `connectionStringFromEnv`
`passwordFromEnv` | +| Scaler | 1.x | 2.0 | +| -------------------- | --------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------- | -------- | +| `azure-blob` | `connection` (**Default**: `AzureWebJobsStorage`) | `connectionFromEnv` | +| `azure-monitor` | `activeDirectoryClientId` `activeDirectoryClientPassword` | `activeDirectoryClientId` `activeDirectoryClientIdFromEnv` `activeDirectoryClientPasswordFromEnv` | +| `azure-queue` | `connection` (**Default**: AzureWebJobsStorage) | `connectionFromEnv` | +| `azure-servicebus` | `connection` | `connectionFromEnv` | +| `azure-eventhub` | `storageConnection` (**Default**: `AzureWebJobsStorage`) `connection` (**Default**: `EventHub`) | `storageConnectionFromEnv` `connectionFromEnv` | +| `aws-cloudwatch` | `awsAccessKeyID` (**Default**: `AWS_ACCESS_KEY_ID`) `awsSecretAccessKey` (**Default**: `AWS_SECRET_ACCESS_KEY`) | `awsAccessKeyID` `awsAccessKeyIDFromEnv` `awsSecretAccessKeyFromEnv` | +| `aws-kinesis-stream` | `awsAccessKeyID` (**Default**: `AWS_ACCESS_KEY_ID`) `awsSecretAccessKey` (**Default**: `AWS_SECRET_ACCESS_KEY`) | `awsAccessKeyID` `awsAccessKeyIDFromEnv` `awsSecretAccessKeyFromEnv` | +| `aws-sqs-queue` | `awsAccessKeyID` (**Default**: `AWS_ACCESS_KEY_ID`) `awsSecretAccessKey` (**Default**: `AWS_SECRET_ACCESS_KEY`) | `awsAccessKeyID` `awsAccessKeyIDFromEnv` `awsSecretAccessKeyFromEnv` | +| `kafka` | _(none)_ | _(none)_ | +| `rabbitmq` | `apiHost` `host` | ~~`apiHost`~~ `host` `hostFromEnv` | +| `prometheus` | _(none)_ | _(none)_ | +| `cron` | _(none)_ | _(none)_ | +| `redis` | `address` `host` `port` `password` | `address` `addressFromEnv` `host` `hostFromEnv` ~~`port`~~ `passwordFromEnv` | +| `redis-streams` | `address` `host` `port` `password` | `address` `addressFromEnv` `host` `hostFromEnv` ~~`port`~~ `passwordFromEnv` | +| `gcp-pubsub` | `credentials` | `credentialsFromEnv` | +| `external` | _(any matching value)_ | _(any matching value with `FromEnv` suffix)_ | +| `liiklus` | _(none)_ | _(none)_ | +| `stan` | _(none)_ | _(none)_ | +| `huawei-cloudeye` | | _(none)_ | _(none)_ | +| `postgresql` | `connection` `password` | `connectionFromEnv` `passwordFromEnv` | +| `mysql` | `connectionString` `password` | `connectionStringFromEnv` `passwordFromEnv` | ### Scalers **Azure Service Bus** - - `queueLength` was renamed to `messageCount` + +- `queueLength` was renamed to `messageCount` **Kafka** - - `authMode` property was replaced with `sasl` and `tls` properties. Please refer [documentation](../scalers/apache-kafka/#authentication-parameters) for Kafka Authentication Parameters details. + +- `authMode` property was replaced with `sasl` and `tls` properties. Please refer [documentation](../scalers/apache-kafka/#authentication-parameters) for Kafka Authentication Parameters details. **RabbitMQ** In KEDA 2.0 the RabbitMQ scaler has only `host` parameter, and the protocol for communication can be specified by -`protocol` (http or amqp). The default value is `amqp`. The behavior changes only for scalers that were using HTTP -protocol. +`protocol` (http or amqp). The default value is `amqp`. The behavior changes only for scalers that were using HTTP +protocol. Example of RabbitMQ trigger before 2.0: + ```yaml triggers: -- type: rabbitmq - metadata: - queueLength: '20' - queueName: testqueue - includeUnacked: 'true' - apiHost: 'https://guest:password@localhost:443/vhostname' -``` + - type: rabbitmq + metadata: + queueLength: "20" + queueName: testqueue + includeUnacked: "true" + apiHost: "https://guest:password@localhost:443/vhostname" +``` The same trigger in 2.0: + ```yaml triggers: -- type: rabbitmq - metadata: - queueLength: '20' - queueName: testqueue - protocol: 'http' - host: 'https://guest:password@localhost:443/vhostname' -``` + - type: rabbitmq + metadata: + queueLength: "20" + queueName: testqueue + protocol: "http" + host: "https://guest:password@localhost:443/vhostname" +``` ### TriggerAuthentication + In order to use Authentication via `TriggerAuthentication` with KEDA v2, you need to change: + - Change the value of `apiVersion` property from `keda.k8s.io/v1alpha1` to `keda.sh/v1alpha1` For more details please refer to the full diff --git a/content/docs/2.4/operate/cluster.md b/content/docs/2.4/operate/cluster.md index 9c5ee2895..8ace09018 100644 --- a/content/docs/2.4/operate/cluster.md +++ b/content/docs/2.4/operate/cluster.md @@ -29,12 +29,10 @@ KEDA requires to be accessible inside the cluster to be able to autoscale. Here is an overview of the required ports that need to be accessible for KEDA to work: - | Port | Why? | Remarks | | ------ | -------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `443` | Used by Kubernetes API server to get metrics | Required for all platforms because it uses Control Plane → port 443 on the Service IP range communication.

This is not applicable for Google Cloud. | +| `443` | Used by Kubernetes API server to get metrics | Required for all platforms because it uses Control Plane → port 443 on the Service IP range communication. This is not applicable for Google Cloud. | | `6443` | Used by Kubernetes API server to get metrics | Only required for Google Cloud because it uses Control Plane → port 6443 on the Pod IP range for communication | - ## High Availability @@ -42,14 +40,14 @@ KEDA does not provide full support for high-availability due to upstream limitat Here is an overview of all KEDA deployments and the HA notes: -| Deployment | Support Replicas | Note | -| -------------- | ---------------- |---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Deployment | Support Replicas | Note | +| -------------- | ---------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | Metrics Server | 1 | You can run multiple replicas of our metrics sever, and it is recommended to add the `--enable-aggregator-routing=true` CLI flag to the kube-apiserver so that requests sent to our metrics servers are load balanced. However, [you can only run one active metric server in a Kubernetes cluster serving external.metrics.k8s.io](https://github.com/kubernetes-sigs/custom-metrics-apiserver/issues/70) which has to be the KEDA metric server. | -| Operator | 2 | While you can run multiple replicas of our operator, only one operator instance will be active. The rest will be standing by, which may reduce downtime during a failure. Multiple replicas will not improve the performance of KEDA, it could only reduce a downtime during a failover.

This is only supported as of KEDA v2.6 if you are using our Helm chart. | +| Operator | 2 | While you can run multiple replicas of our operator, only one operator instance will be active. The rest will be standing by, which may reduce downtime during a failure. Multiple replicas will not improve the performance of KEDA, it could only reduce a downtime during a failover.This is only supported as of KEDA v2.6 if you are using our Helm chart. | ## HTTP Timeouts -Some scalers issue HTTP requests to external servers (i.e. cloud services). Each applicable scaler uses its own dedicated HTTP client with its own connection pool, and by default each client is set to time out any HTTP request after 3 seconds. +Some scalers issue HTTP requests to external servers (i.e. cloud services). Each applicable scaler uses its own dedicated HTTP client with its own connection pool, and by default each client is set to time out any HTTP request after 3 seconds. You can override this default by setting the `KEDA_HTTP_DEFAULT_TIMEOUT` environment variable to your desired timeout in milliseconds. For example, on Linux/Mac/Windows WSL2 operating systems, you'd use this command to set to 1 second: @@ -69,9 +67,7 @@ All applicable scalers will use this timeout. Setting a per-scaler timeout is cu The Kubernetes client config used within KEDA Metrics Adapter can be adjusted by passing the following command-line flags to the binary: -| Adapter Flag | Client Config Setting | Default Value | Description | -| -------------- | ----------------------- | ------------- | -------------------------------------------------------------- | -| kube-api-qps | cfg.QPS | 20.0 | Set the QPS rate for throttling requests sent to the apiserver | -| kube-api-burst | cfg.Burst | 30 | Set the burst for throttling requests sent to the apiserver | - - +| Adapter Flag | Client Config Setting | Default Value | Description | +| -------------- | --------------------- | ------------- | -------------------------------------------------------------- | +| kube-api-qps | cfg.QPS | 20.0 | Set the QPS rate for throttling requests sent to the apiserver | +| kube-api-burst | cfg.Burst | 30 | Set the burst for throttling requests sent to the apiserver | diff --git a/content/docs/2.5/concepts/external-scalers.md b/content/docs/2.5/concepts/external-scalers.md index 47c4a45a9..214107edc 100644 --- a/content/docs/2.5/concepts/external-scalers.md +++ b/content/docs/2.5/concepts/external-scalers.md @@ -9,7 +9,7 @@ Built-in scalers run in the KEDA process/pod, while external scalers require an This document describes the external scaler interfaces and how to implement them in Go, Node, and .NET; however for more details on GRPC refer to [the official GRPC documentation](https://grpc.io/docs/) ->Want to learn about existing external scalers? Explore our [external scaler community](https://github.com/kedacore/external-scalers). +> Want to learn about existing external scalers? Explore our [external scaler community](https://github.com/kedacore/external-scalers). ## Overview @@ -34,12 +34,12 @@ type PushScaler interface { The `Scaler` interface defines 4 methods: -- `IsActive` is called on `pollingInterval`. When `isActive` returns `true`, KEDA will scale to what is returned by `GetMetricSpec` limited by `maxReplicaCount` on the ScaledObject/ScaledJob. - When `false` is returned, KEDA will scale to `minReplicaCount` or optionally `idleReplicaCount`. More details around the defaults and how these options work together can be found on the [ScaledObjectSpec](https://keda.sh/docs/2.6/concepts/scaling-deployments/#scaledobject-spec). +- `IsActive` is called on `pollingInterval`. When `isActive` returns `true`, KEDA will scale to what is returned by `GetMetricSpec` limited by `maxReplicaCount` on the ScaledObject/ScaledJob. + When `false` is returned, KEDA will scale to `minReplicaCount` or optionally `idleReplicaCount`. More details around the defaults and how these options work together can be found on the [ScaledObjectSpec](https://keda.sh/docs/2.6/concepts/scaling-deployments/#scaledobject-spec). - `Close` is called to allow the scaler to clean up connections or other resources. - `GetMetricSpec` returns the target value for the HPA definition for the scaler. For more details refer to [Implementing `GetMetricSpec`](#5-implementing-getmetricspec). - `GetMetrics` returns the value of the metric referred to from `GetMetricSpec`. For more details refer to [Implementing `GetMetrics`](#6-implementing-getmetrics). -> Refer to the [HPA docs](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale-walkthrough/) for how HPA calculates `replicaCount` based on metric value and target value. KEDA uses the metric target type `AverageValue` for external metrics. This will cause the metric value returned by the external scaler to be divided by the number of replicas. + > Refer to the [HPA docs](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale-walkthrough/) for how HPA calculates `replicaCount` based on metric value and target value. KEDA uses the metric target type `AverageValue` for external metrics. This will cause the metric value returned by the external scaler to be divided by the number of replicas. The `PushScaler` interface adds a `Run` method. This method receives a push channel (`active`), on which the scaler can send `true` at any time. The purpose of this mechanism is to initiate a scaling operation independently from `pollingInterval`. @@ -109,7 +109,7 @@ KEDA will attempt a GRPC connection to `service-address.svc.local:9090` immediat } ``` ->**Note**: KEDA will issue all of the above RPC calls except `StreamIsActive` if `spec.triggers.type` is `external`. It _must_ be `external-push` for `StreamIsActive` to be called. +> **Note**: KEDA will issue all of the above RPC calls except `StreamIsActive` if `spec.triggers.type` is `external`. It _must_ be `external-push` for `StreamIsActive` to be called. ## Implementing KEDA external scaler GRPC interface @@ -176,8 +176,6 @@ npm install --save grpc request {{< /collapsible >}} -
- #### 3. Implementing `IsActive` Just like `IsActive(ctx context.Context) (bool, error)` in the go interface, the `IsActive` method in the GRPC interface is called every `pollingInterval` with a `ScaledObjectRef` object that contains the scaledObject name, namespace, and scaler metadata. @@ -304,70 +302,71 @@ public class ExternalScalerService : ExternalScaler.ExternalScalerBase Put the following code into your `index.js` file: ```js -const grpc = require('grpc') -const request = require('request') -const externalScalerProto = grpc.load('externalscaler.proto') +const grpc = require("grpc"); +const request = require("request"); +const externalScalerProto = grpc.load("externalscaler.proto"); -const server = new grpc.Server() +const server = new grpc.Server(); server.addService(externalScalerProto.externalscaler.ExternalScaler.service, { isActive: (call, callback) => { - const longitude = call.request.scalerMetadata.longitude - const latitude = call.request.scalerMetadata.latitude + const longitude = call.request.scalerMetadata.longitude; + const latitude = call.request.scalerMetadata.latitude; if (!longitude || !latitude) { callback({ code: grpc.status.INVALID_ARGUMENT, - details: 'longitude and latitude must be specified', - }) + details: "longitude and latitude must be specified", + }); } else { - const now = new Date() - const yesterday = new Date(new Date().setDate(new Date().getDate()-1)); - - const startTime = `${yesterday.getUTCFullYear()}-${yesterday.getUTCMonth()}-${yesterday.getUTCDay()}` - const endTime = `${now.getUTCFullYear()}-${now.getUTCMonth()}-${now.getUTCDay()}` - const radiusKm = 500 - const query = `format=geojson&starttime=${startTime}&endtime=${endTime}&longitude=${longitude}&latitude=${latitude}&maxradiuskm=${radiusKm}` - - request.get({ - url: `https://earthquake.usgs.gov/fdsnws/event/1/query?${query}`, - json: true, - }, (err, resp, data) => { - if (err) { - callback({ - code: grpc.status.INTERNAL, - details: err, - }) - } else if (resp.statusCode !== 200) { - callback({ - code: grpc.status.INTERNAL, - details: `expected status 200, got ${resp.statusCode}` - }) - } else { - // count how many earthquakes with mag > 1.0 - let count = 0 - data.features.forEach(i => { - if (i.properties.mag > 1.0) { - count++ - } - }) - callback(null, { - result: count > 2, - }) + const now = new Date(); + const yesterday = new Date(new Date().setDate(new Date().getDate() - 1)); + + const startTime = `${yesterday.getUTCFullYear()}-${yesterday.getUTCMonth()}-${yesterday.getUTCDay()}`; + const endTime = `${now.getUTCFullYear()}-${now.getUTCMonth()}-${now.getUTCDay()}`; + const radiusKm = 500; + const query = `format=geojson&starttime=${startTime}&endtime=${endTime}&longitude=${longitude}&latitude=${latitude}&maxradiuskm=${radiusKm}`; + + request.get( + { + url: `https://earthquake.usgs.gov/fdsnws/event/1/query?${query}`, + json: true, + }, + (err, resp, data) => { + if (err) { + callback({ + code: grpc.status.INTERNAL, + details: err, + }); + } else if (resp.statusCode !== 200) { + callback({ + code: grpc.status.INTERNAL, + details: `expected status 200, got ${resp.statusCode}`, + }); + } else { + // count how many earthquakes with mag > 1.0 + let count = 0; + data.features.forEach((i) => { + if (i.properties.mag > 1.0) { + count++; + } + }); + callback(null, { + result: count > 2, + }); + } } - }) + ); } - } -}) + }, +}); -server.bind('127.0.0.1:9090', grpc.ServerCredentials.createInsecure()) -console.log('Server listening on 127.0.0.1:9090') +server.bind("127.0.0.1:9090", grpc.ServerCredentials.createInsecure()); +console.log("Server listening on 127.0.0.1:9090"); -server.start() +server.start(); ``` {{< /collapsible >}} -
- #### 4. Implementing `StreamIsActive` Unlike `IsActive`, `StreamIsActive` is called once when KEDA reconciles the `ScaledObject`, and expects the external scaler to maintain a long-lived connection and push `IsActiveResponse` objects whenever the scaler needs KEDA to activate the deployment. @@ -444,38 +443,36 @@ public override async Task StreamIsActive(ScaledObjectRef request, IServerStream server.addService(externalScalerProto.externalscaler.ExternalScaler.service, { // ... streamIsActive: (call, callback) => { - const longitude = call.request.scalerMetadata.longitude - const latitude = call.request.scalerMetadata.latitude + const longitude = call.request.scalerMetadata.longitude; + const latitude = call.request.scalerMetadata.latitude; if (!longitude || !latitude) { callback({ code: grpc.status.INVALID_ARGUMENT, - details: 'longitude and latitude must be specified', - }) + details: "longitude and latitude must be specified", + }); } else { const interval = setInterval(() => { getEarthquakeCount((err, count) => { if (err) { - console.error(err) + console.error(err); } else if (count > 2) { call.write({ result: true, - }) + }); } - }) + }); }, 1000 * 60 * 60); - call.on('end', () => { - clearInterval(interval) - }) + call.on("end", () => { + clearInterval(interval); + }); } - } -}) + }, +}); ``` {{< /collapsible >}} -
- #### 5. Implementing `GetMetricSpec` `GetMetricSpec` returns the `target` value for [the HPA definition for the scaler](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale-walkthrough/). This scaler will define a static target of 10, but the threshold value is often specified in the metadata for other scalers. @@ -521,19 +518,19 @@ server.addService(externalScalerProto.externalscaler.ExternalScaler.service, { // ... getMetricSpec: (call, callback) => { callback(null, { - metricSpecs: [{ - metricName: 'earthquakeThreshold', - targetSize: 10, - }] - }) - } -}) + metricSpecs: [ + { + metricName: "earthquakeThreshold", + targetSize: 10, + }, + ], + }); + }, +}); ``` {{< /collapsible >}} -
- #### 6. Implementing `GetMetrics` `GetMetrics` returns the value of the metric referred to from `GetMetricSpec`, in this example it's `earthquakeThreshold`. @@ -600,32 +597,34 @@ public override async Task GetMetrics(GetMetricsRequest requ server.addService(externalScalerProto.externalscaler.ExternalScaler.service, { // ... getMetrics: (call, callback) => { - const longitude = call.request.scaledObjectRef.scalerMetadata.longitude - const latitude = call.request.scaledObjectRef.scalerMetadata.latitude + const longitude = call.request.scaledObjectRef.scalerMetadata.longitude; + const latitude = call.request.scaledObjectRef.scalerMetadata.latitude; if (!longitude || !latitude) { callback({ code: grpc.status.INVALID_ARGUMENT, - details: 'longitude and latitude must be specified', - }) + details: "longitude and latitude must be specified", + }); } else { getEarthquakeCount((err, count) => { if (err) { callback({ code: grpc.status.INTERNAL, details: err, - }) + }); } else { callback(null, { - metricValues: [{ - metricName: 'earthquakeThreshold', - metricValue: count, - }] - }) + metricValues: [ + { + metricName: "earthquakeThreshold", + metricValue: count, + }, + ], + }); } - }) + }); } - } -}) + }, +}); ``` {{< /collapsible >}} diff --git a/content/docs/2.5/migration.md b/content/docs/2.5/migration.md index 5ee8a6455..29e8408c8 100644 --- a/content/docs/2.5/migration.md +++ b/content/docs/2.5/migration.md @@ -11,15 +11,17 @@ Please note that you **can not** run both KEDA v1 and v2 on the same Kubernetes KEDA v2 is using a new API namespace for its Custom Resources Definitions (CRD): `keda.sh` instead of `keda.k8s.io` and introduces a new Custom Resource for scaling of Jobs. See full details on KEDA Custom Resources [here](../concepts/#custom-resources-crd). Here's an overview of what's changed: + - [Scaling of Deployments](#scaling-of-deployments) - [Scaling of Jobs](#scaling-of-jobs) - [Improved flexibility & usability of trigger metadata](#improved-flexibility--usability-of-trigger-metadata) - [Scalers](#scalers) - [TriggerAuthentication](#triggerauthentication) - ### Scaling of Deployments + In order to scale `Deployments` with KEDA v2, you need to do only a few modifications to existing v1 `ScaledObjects` definitions, so they comply with v2: + - Change the value of `apiVersion` property from `keda.k8s.io/v1alpha1` to `keda.sh/v1alpha1` - Rename property `spec.scaleTargetRef.deploymentName` to `spec.scaleTargetRef.name` - Rename property `spec.scaleTargetRef.containerName` to `spec.scaleTargetRef.envSourceContainerName` @@ -27,22 +29,23 @@ In order to scale `Deployments` with KEDA v2, you need to do only a few modifica Please see the examples below or refer to the full [v2 ScaledObject Specification](../concepts/scaling-deployments/#scaledobject-spec) - **Example of v1 ScaledObject** +**Example of v1 ScaledObject** + ```yaml apiVersion: keda.k8s.io/v1alpha1 kind: ScaledObject metadata: - name: {scaled-object-name} + name: { scaled-object-name } labels: - deploymentName: {deployment-name} + deploymentName: { deployment-name } spec: scaleTargetRef: - deploymentName: {deployment-name} - containerName: {container-name} - pollingInterval: 30 - cooldownPeriod: 300 - minReplicaCount: 0 - maxReplicaCount: 100 + deploymentName: { deployment-name } + containerName: { container-name } + pollingInterval: 30 + cooldownPeriod: 300 + minReplicaCount: 0 + maxReplicaCount: 100 triggers: # {list of triggers to activate the deployment} ``` @@ -50,18 +53,18 @@ spec: **Example of v2 ScaledObject** ```yaml -apiVersion: keda.sh/v1alpha1 # <--- Property value was changed +apiVersion: keda.sh/v1alpha1 # <--- Property value was changed kind: ScaledObject -metadata: # <--- labels.deploymentName is not needed - name: {scaled-object-name} +metadata: # <--- labels.deploymentName is not needed + name: { scaled-object-name } spec: scaleTargetRef: - name: {deployment-name} # <--- Property name was changed - envSourceContainerName: {container-name} # <--- Property name was changed - pollingInterval: 30 - cooldownPeriod: 300 - minReplicaCount: 0 - maxReplicaCount: 100 + name: { deployment-name } # <--- Property name was changed + envSourceContainerName: { container-name } # <--- Property name was changed + pollingInterval: 30 + cooldownPeriod: 300 + minReplicaCount: 0 + maxReplicaCount: 100 triggers: # {list of triggers to activate the deployment} ``` @@ -69,34 +72,36 @@ spec: ### Scaling of Jobs In order to scale `Jobs` with KEDA v2, you need to do only a few modifications to existing v1 `ScaledObjects` definitions, so they comply with v2: + - Change the value of `apiVersion` property from `keda.k8s.io/v1alpha1` to `keda.sh/v1alpha1` - Change the value of `kind` property from `ScaledObject` to `ScaledJob` - Remove property `spec.scaleType` -- Remove properties `spec.cooldownPeriod` and `spec.minReplicaCount` +- Remove properties `spec.cooldownPeriod` and `spec.minReplicaCount` You can configure `successfulJobsHistoryLimit` and `failedJobsHistoryLimit`. They will remove the old job histories automatically. Please see the examples below or refer to the full [v2 ScaledJob Specification](../concepts/scaling-jobs/#scaledjob-spec) **Example of v1 ScaledObject for Jobs scaling** + ```yaml apiVersion: keda.k8s.io/v1alpha1 kind: ScaledObject metadata: - name: {scaled-object-name} + name: { scaled-object-name } spec: scaleType: job jobTargetRef: - parallelism: 1 + parallelism: 1 completions: 1 activeDeadlineSeconds: 600 - backoffLimit: 6 + backoffLimit: 6 template: # {job template} - pollingInterval: 30 - cooldownPeriod: 300 - minReplicaCount: 0 - maxReplicaCount: 100 + pollingInterval: 30 + cooldownPeriod: 300 + minReplicaCount: 0 + maxReplicaCount: 100 triggers: # {list of triggers to create jobs} ``` @@ -104,21 +109,21 @@ spec: **Example of v2 ScaledJob** ```yaml -apiVersion: keda.sh/v1alpha1 # <--- Property value was changed -kind: ScaledJob # <--- Property value was changed +apiVersion: keda.sh/v1alpha1 # <--- Property value was changed +kind: ScaledJob # <--- Property value was changed metadata: - name: {scaled-job-name} -spec: # <--- spec.scaleType is not needed + name: { scaled-job-name } +spec: # <--- spec.scaleType is not needed jobTargetRef: - parallelism: 1 - completions: 1 + parallelism: 1 + completions: 1 activeDeadlineSeconds: 600 backoffLimit: 6 template: # {job template} - pollingInterval: 30 # <--- spec.cooldownPeriod and spec.minReplicaCount are not needed - successfulJobsHistoryLimit: 5 # <--- property is added - failedJobsHistoryLimit: 5 # <--- Property is added + pollingInterval: 30 # <--- spec.cooldownPeriod and spec.minReplicaCount are not needed + successfulJobsHistoryLimit: 5 # <--- property is added + failedJobsHistoryLimit: 5 # <--- Property is added maxReplicaCount: 100 triggers: # {list of triggers to create jobs} @@ -132,68 +137,74 @@ We've introduced more options to configure trigger metadata to give users more f Here's an overview: -| Scaler | 1.x | 2.0 | -|--------|--------|--------| -| `azure-blob` | `connection` (**Default**: `AzureWebJobsStorage`) | `connectionFromEnv` | -| `azure-monitor` | `activeDirectoryClientId`
`activeDirectoryClientPassword` | `activeDirectoryClientId`
`activeDirectoryClientIdFromEnv`
`activeDirectoryClientPasswordFromEnv` | -| `azure-queue` | `connection` (**Default**: AzureWebJobsStorage) | `connectionFromEnv` | -| `azure-servicebus` | `connection` | `connectionFromEnv` | -| `azure-eventhub` | `storageConnection` (**Default**: `AzureWebJobsStorage`)
`connection` (**Default**: `EventHub`) | `storageConnectionFromEnv`
`connectionFromEnv` | -| `aws-cloudwatch` | `awsAccessKeyID` (**Default**: `AWS_ACCESS_KEY_ID`)
`awsSecretAccessKey` (**Default**: `AWS_SECRET_ACCESS_KEY`) | `awsAccessKeyID`
`awsAccessKeyIDFromEnv`
`awsSecretAccessKeyFromEnv` | -| `aws-kinesis-stream` | `awsAccessKeyID` (**Default**: `AWS_ACCESS_KEY_ID`)
`awsSecretAccessKey` (**Default**: `AWS_SECRET_ACCESS_KEY`) | `awsAccessKeyID`
`awsAccessKeyIDFromEnv`
`awsSecretAccessKeyFromEnv` | -| `aws-sqs-queue` | `awsAccessKeyID` (**Default**: `AWS_ACCESS_KEY_ID`)
`awsSecretAccessKey` (**Default**: `AWS_SECRET_ACCESS_KEY`) | `awsAccessKeyID`
`awsAccessKeyIDFromEnv`
`awsSecretAccessKeyFromEnv` | -| `kafka` | _(none)_ | _(none)_ -| `rabbitmq` | `apiHost`
`host` | ~~`apiHost`~~
`host`
`hostFromEnv` | -| `prometheus` | _(none)_ | _(none)_ | -| `cron` | _(none)_ | _(none)_ | -| `redis` | `address`
`host`
`port`
`password` | `address`
`addressFromEnv`
`host`
`hostFromEnv`
~~`port`~~
`passwordFromEnv` -| `redis-streams` | `address`
`host`
`port`
`password` | `address`
`addressFromEnv`
`host`
`hostFromEnv`
~~`port`~~
`passwordFromEnv` -| `gcp-pubsub` | `credentials` | `credentialsFromEnv` | -| `external` | _(any matching value)_ | _(any matching value with `FromEnv` suffix)_ -| `liiklus` | _(none)_ | _(none)_ | -| `stan` | _(none)_ | _(none)_ | -| `huawei-cloudeye` | | _(none)_ | _(none)_ | -| `postgresql` | `connection`
`password` | `connectionFromEnv`
`passwordFromEnv` | -| `mysql` | `connectionString`
`password` | `connectionStringFromEnv`
`passwordFromEnv` | +| Scaler | 1.x | 2.0 | +| -------------------- | --------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------- | -------- | +| `azure-blob` | `connection` (**Default**: `AzureWebJobsStorage`) | `connectionFromEnv` | +| `azure-monitor` | `activeDirectoryClientId` `activeDirectoryClientPassword` | `activeDirectoryClientId` `activeDirectoryClientIdFromEnv` `activeDirectoryClientPasswordFromEnv` | +| `azure-queue` | `connection` (**Default**: AzureWebJobsStorage) | `connectionFromEnv` | +| `azure-servicebus` | `connection` | `connectionFromEnv` | +| `azure-eventhub` | `storageConnection` (**Default**: `AzureWebJobsStorage`) `connection` (**Default**: `EventHub`) | `storageConnectionFromEnv` `connectionFromEnv` | +| `aws-cloudwatch` | `awsAccessKeyID` (**Default**: `AWS_ACCESS_KEY_ID`) `awsSecretAccessKey` (**Default**: `AWS_SECRET_ACCESS_KEY`) | `awsAccessKeyID` `awsAccessKeyIDFromEnv` `awsSecretAccessKeyFromEnv` | +| `aws-kinesis-stream` | `awsAccessKeyID` (**Default**: `AWS_ACCESS_KEY_ID`) `awsSecretAccessKey` (**Default**: `AWS_SECRET_ACCESS_KEY`) | `awsAccessKeyID` `awsAccessKeyIDFromEnv` `awsSecretAccessKeyFromEnv` | +| `aws-sqs-queue` | `awsAccessKeyID` (**Default**: `AWS_ACCESS_KEY_ID`) `awsSecretAccessKey` (**Default**: `AWS_SECRET_ACCESS_KEY`) | `awsAccessKeyID` `awsAccessKeyIDFromEnv` `awsSecretAccessKeyFromEnv` | +| `kafka` | _(none)_ | _(none)_ | +| `rabbitmq` | `apiHost` `host` | ~~`apiHost`~~ `host` `hostFromEnv` | +| `prometheus` | _(none)_ | _(none)_ | +| `cron` | _(none)_ | _(none)_ | +| `redis` | `address` `host` `port` `password` | `address` `addressFromEnv` `host` `hostFromEnv` ~~`port`~~ `passwordFromEnv` | +| `redis-streams` | `address` `host` `port` `password` | `address` `addressFromEnv` `host` `hostFromEnv` ~~`port`~~ `passwordFromEnv` | +| `gcp-pubsub` | `credentials` | `credentialsFromEnv` | +| `external` | _(any matching value)_ | _(any matching value with `FromEnv` suffix)_ | +| `liiklus` | _(none)_ | _(none)_ | +| `stan` | _(none)_ | _(none)_ | +| `huawei-cloudeye` | | _(none)_ | _(none)_ | +| `postgresql` | `connection` `password` | `connectionFromEnv` `passwordFromEnv` | +| `mysql` | `connectionString` `password` | `connectionStringFromEnv` `passwordFromEnv` | ### Scalers **Azure Service Bus** - - `queueLength` was renamed to `messageCount` + +- `queueLength` was renamed to `messageCount` **Kafka** - - `authMode` property was replaced with `sasl` and `tls` properties. Please refer [documentation](../scalers/apache-kafka/#authentication-parameters) for Kafka Authentication Parameters details. + +- `authMode` property was replaced with `sasl` and `tls` properties. Please refer [documentation](../scalers/apache-kafka/#authentication-parameters) for Kafka Authentication Parameters details. **RabbitMQ** In KEDA 2.0 the RabbitMQ scaler has only `host` parameter, and the protocol for communication can be specified by -`protocol` (http or amqp). The default value is `amqp`. The behavior changes only for scalers that were using HTTP -protocol. +`protocol` (http or amqp). The default value is `amqp`. The behavior changes only for scalers that were using HTTP +protocol. Example of RabbitMQ trigger before 2.0: + ```yaml triggers: -- type: rabbitmq - metadata: - queueLength: '20' - queueName: testqueue - includeUnacked: 'true' - apiHost: 'https://guest:password@localhost:443/vhostname' -``` + - type: rabbitmq + metadata: + queueLength: "20" + queueName: testqueue + includeUnacked: "true" + apiHost: "https://guest:password@localhost:443/vhostname" +``` The same trigger in 2.0: + ```yaml triggers: -- type: rabbitmq - metadata: - queueLength: '20' - queueName: testqueue - protocol: 'http' - host: 'https://guest:password@localhost:443/vhostname' -``` + - type: rabbitmq + metadata: + queueLength: "20" + queueName: testqueue + protocol: "http" + host: "https://guest:password@localhost:443/vhostname" +``` ### TriggerAuthentication + In order to use Authentication via `TriggerAuthentication` with KEDA v2, you need to change: + - Change the value of `apiVersion` property from `keda.k8s.io/v1alpha1` to `keda.sh/v1alpha1` For more details please refer to the full diff --git a/content/docs/2.5/operate/cluster.md b/content/docs/2.5/operate/cluster.md index c685303b3..2c7b818ee 100644 --- a/content/docs/2.5/operate/cluster.md +++ b/content/docs/2.5/operate/cluster.md @@ -29,12 +29,10 @@ KEDA requires to be accessible inside the cluster to be able to autoscale. Here is an overview of the required ports that need to be accessible for KEDA to work: - | Port | Why? | Remarks | | ------ | -------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `443` | Used by Kubernetes API server to get metrics | Required for all platforms because it uses Control Plane → port 443 on the Service IP range communication.

This is not applicable for Google Cloud. | +| `443` | Used by Kubernetes API server to get metrics | Required for all platforms because it uses Control Plane → port 443 on the Service IP range communication. This is not applicable for Google Cloud. | | `6443` | Used by Kubernetes API server to get metrics | Only required for Google Cloud because it uses Control Plane → port 6443 on the Pod IP range for communication | - ## High Availability @@ -42,14 +40,14 @@ KEDA does not provide full support for high-availability due to upstream limitat Here is an overview of all KEDA deployments and the HA notes: -| Deployment | Support Replicas | Note | -| -------------- | ---------------- |---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Deployment | Support Replicas | Note | +| -------------- | ---------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | Metrics Server | 1 | You can run multiple replicas of our metrics sever, and it is recommended to add the `--enable-aggregator-routing=true` CLI flag to the kube-apiserver so that requests sent to our metrics servers are load balanced. However, [you can only run one active metric server in a Kubernetes cluster serving external.metrics.k8s.io](https://github.com/kubernetes-sigs/custom-metrics-apiserver/issues/70) which has to be the KEDA metric server. | -| Operator | 2 | While you can run multiple replicas of our operator, only one operator instance will be active. The rest will be standing by, which may reduce downtime during a failure. Multiple replicas will not improve the performance of KEDA, it could only reduce a downtime during a failover. | +| Operator | 2 | While you can run multiple replicas of our operator, only one operator instance will be active. The rest will be standing by, which may reduce downtime during a failure. Multiple replicas will not improve the performance of KEDA, it could only reduce a downtime during a failover. | ## HTTP Timeouts -Some scalers issue HTTP requests to external servers (i.e. cloud services). Each applicable scaler uses its own dedicated HTTP client with its own connection pool, and by default each client is set to time out any HTTP request after 3 seconds. +Some scalers issue HTTP requests to external servers (i.e. cloud services). Each applicable scaler uses its own dedicated HTTP client with its own connection pool, and by default each client is set to time out any HTTP request after 3 seconds. You can override this default by setting the `KEDA_HTTP_DEFAULT_TIMEOUT` environment variable to your desired timeout in milliseconds. For example, on Linux/Mac/Windows WSL2 operating systems, you'd use this command to set to 1 second: @@ -69,16 +67,17 @@ All applicable scalers will use this timeout. Setting a per-scaler timeout is cu The Kubernetes client config used within KEDA Metrics Adapter can be adjusted by passing the following command-line flags to the binary: -| Adapter Flag | Client Config Setting | Default Value | Description | -| -------------- | ----------------------- | ------------- | -------------------------------------------------------------- | -| kube-api-qps | cfg.QPS | 20.0 | Set the QPS rate for throttling requests sent to the apiserver | -| kube-api-burst | cfg.Burst | 30 | Set the burst for throttling requests sent to the apiserver | +| Adapter Flag | Client Config Setting | Default Value | Description | +| -------------- | --------------------- | ------------- | -------------------------------------------------------------- | +| kube-api-qps | cfg.QPS | 20.0 | Set the QPS rate for throttling requests sent to the apiserver | +| kube-api-burst | cfg.Burst | 30 | Set the burst for throttling requests sent to the apiserver | ## Configure `MaxConcurrentReconciles` for Controllers To implement internal controllers KEDA uses [controller-runtime project](https://github.com/kubernetes-sigs/controller-runtime), that enables configuration of [MaxConcurrentReconciles property](https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/controller#Options), ie. the maximum number of concurrent reconciles which can be run for a controller. KEDA Operator exposes properties for specifying `MaxConcurrentReconciles` for following controllers/reconcilers: + - `ScaledObjectReconciler` - responsible for watching and managing `ScaledObjects`, ie. validates input trigger specification, starts scaling logic and manages dependent HPA. - `ScaledJobReconciler` - responsible for watching and managing `ScaledJobs` and dependent Kubernetes Jobs @@ -86,8 +85,8 @@ KEDA Metrics Server exposes property for specifying `MaxConcurrentReconciles` fo To modify this properties you can set environment variables on both KEDA Operator and Metrics Server Deployments: -| Environment variable name | Deployment | Default Value | Affected reconciler | -| ------------------------------------- | -------------- | ------------- | -------------------------------------------------------------- | -| KEDA_SCALEDOBJECT_CTRL_MAX_RECONCILES | Operator | 5 | ScaledObjectReconciler | -| KEDA_SCALEDJOB_CTRL_MAX_RECONCILES | Operator | 1 | ScaledJobReconciler | -| KEDA_METRICS_CTRL_MAX_RECONCILES | Metrics Server | 1 | MetricsScaledObjectReconciler | +| Environment variable name | Deployment | Default Value | Affected reconciler | +| ------------------------------------- | -------------- | ------------- | ----------------------------- | +| KEDA_SCALEDOBJECT_CTRL_MAX_RECONCILES | Operator | 5 | ScaledObjectReconciler | +| KEDA_SCALEDJOB_CTRL_MAX_RECONCILES | Operator | 1 | ScaledJobReconciler | +| KEDA_METRICS_CTRL_MAX_RECONCILES | Metrics Server | 1 | MetricsScaledObjectReconciler | diff --git a/content/docs/2.6/concepts/external-scalers.md b/content/docs/2.6/concepts/external-scalers.md index 64d7bffb4..214107edc 100644 --- a/content/docs/2.6/concepts/external-scalers.md +++ b/content/docs/2.6/concepts/external-scalers.md @@ -9,7 +9,7 @@ Built-in scalers run in the KEDA process/pod, while external scalers require an This document describes the external scaler interfaces and how to implement them in Go, Node, and .NET; however for more details on GRPC refer to [the official GRPC documentation](https://grpc.io/docs/) ->Want to learn about existing external scalers? Explore our [external scaler community](https://github.com/kedacore/external-scalers). +> Want to learn about existing external scalers? Explore our [external scaler community](https://github.com/kedacore/external-scalers). ## Overview @@ -39,7 +39,7 @@ The `Scaler` interface defines 4 methods: - `Close` is called to allow the scaler to clean up connections or other resources. - `GetMetricSpec` returns the target value for the HPA definition for the scaler. For more details refer to [Implementing `GetMetricSpec`](#5-implementing-getmetricspec). - `GetMetrics` returns the value of the metric referred to from `GetMetricSpec`. For more details refer to [Implementing `GetMetrics`](#6-implementing-getmetrics). -> Refer to the [HPA docs](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale-walkthrough/) for how HPA calculates `replicaCount` based on metric value and target value. KEDA uses the metric target type `AverageValue` for external metrics. This will cause the metric value returned by the external scaler to be divided by the number of replicas. + > Refer to the [HPA docs](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale-walkthrough/) for how HPA calculates `replicaCount` based on metric value and target value. KEDA uses the metric target type `AverageValue` for external metrics. This will cause the metric value returned by the external scaler to be divided by the number of replicas. The `PushScaler` interface adds a `Run` method. This method receives a push channel (`active`), on which the scaler can send `true` at any time. The purpose of this mechanism is to initiate a scaling operation independently from `pollingInterval`. @@ -109,7 +109,7 @@ KEDA will attempt a GRPC connection to `service-address.svc.local:9090` immediat } ``` ->**Note**: KEDA will issue all of the above RPC calls except `StreamIsActive` if `spec.triggers.type` is `external`. It _must_ be `external-push` for `StreamIsActive` to be called. +> **Note**: KEDA will issue all of the above RPC calls except `StreamIsActive` if `spec.triggers.type` is `external`. It _must_ be `external-push` for `StreamIsActive` to be called. ## Implementing KEDA external scaler GRPC interface @@ -176,8 +176,6 @@ npm install --save grpc request {{< /collapsible >}} -
- #### 3. Implementing `IsActive` Just like `IsActive(ctx context.Context) (bool, error)` in the go interface, the `IsActive` method in the GRPC interface is called every `pollingInterval` with a `ScaledObjectRef` object that contains the scaledObject name, namespace, and scaler metadata. @@ -304,70 +302,71 @@ public class ExternalScalerService : ExternalScaler.ExternalScalerBase Put the following code into your `index.js` file: ```js -const grpc = require('grpc') -const request = require('request') -const externalScalerProto = grpc.load('externalscaler.proto') +const grpc = require("grpc"); +const request = require("request"); +const externalScalerProto = grpc.load("externalscaler.proto"); -const server = new grpc.Server() +const server = new grpc.Server(); server.addService(externalScalerProto.externalscaler.ExternalScaler.service, { isActive: (call, callback) => { - const longitude = call.request.scalerMetadata.longitude - const latitude = call.request.scalerMetadata.latitude + const longitude = call.request.scalerMetadata.longitude; + const latitude = call.request.scalerMetadata.latitude; if (!longitude || !latitude) { callback({ code: grpc.status.INVALID_ARGUMENT, - details: 'longitude and latitude must be specified', - }) + details: "longitude and latitude must be specified", + }); } else { - const now = new Date() - const yesterday = new Date(new Date().setDate(new Date().getDate()-1)); - - const startTime = `${yesterday.getUTCFullYear()}-${yesterday.getUTCMonth()}-${yesterday.getUTCDay()}` - const endTime = `${now.getUTCFullYear()}-${now.getUTCMonth()}-${now.getUTCDay()}` - const radiusKm = 500 - const query = `format=geojson&starttime=${startTime}&endtime=${endTime}&longitude=${longitude}&latitude=${latitude}&maxradiuskm=${radiusKm}` - - request.get({ - url: `https://earthquake.usgs.gov/fdsnws/event/1/query?${query}`, - json: true, - }, (err, resp, data) => { - if (err) { - callback({ - code: grpc.status.INTERNAL, - details: err, - }) - } else if (resp.statusCode !== 200) { - callback({ - code: grpc.status.INTERNAL, - details: `expected status 200, got ${resp.statusCode}` - }) - } else { - // count how many earthquakes with mag > 1.0 - let count = 0 - data.features.forEach(i => { - if (i.properties.mag > 1.0) { - count++ - } - }) - callback(null, { - result: count > 2, - }) + const now = new Date(); + const yesterday = new Date(new Date().setDate(new Date().getDate() - 1)); + + const startTime = `${yesterday.getUTCFullYear()}-${yesterday.getUTCMonth()}-${yesterday.getUTCDay()}`; + const endTime = `${now.getUTCFullYear()}-${now.getUTCMonth()}-${now.getUTCDay()}`; + const radiusKm = 500; + const query = `format=geojson&starttime=${startTime}&endtime=${endTime}&longitude=${longitude}&latitude=${latitude}&maxradiuskm=${radiusKm}`; + + request.get( + { + url: `https://earthquake.usgs.gov/fdsnws/event/1/query?${query}`, + json: true, + }, + (err, resp, data) => { + if (err) { + callback({ + code: grpc.status.INTERNAL, + details: err, + }); + } else if (resp.statusCode !== 200) { + callback({ + code: grpc.status.INTERNAL, + details: `expected status 200, got ${resp.statusCode}`, + }); + } else { + // count how many earthquakes with mag > 1.0 + let count = 0; + data.features.forEach((i) => { + if (i.properties.mag > 1.0) { + count++; + } + }); + callback(null, { + result: count > 2, + }); + } } - }) + ); } - } -}) + }, +}); -server.bind('127.0.0.1:9090', grpc.ServerCredentials.createInsecure()) -console.log('Server listening on 127.0.0.1:9090') +server.bind("127.0.0.1:9090", grpc.ServerCredentials.createInsecure()); +console.log("Server listening on 127.0.0.1:9090"); -server.start() +server.start(); ``` {{< /collapsible >}} -
- #### 4. Implementing `StreamIsActive` Unlike `IsActive`, `StreamIsActive` is called once when KEDA reconciles the `ScaledObject`, and expects the external scaler to maintain a long-lived connection and push `IsActiveResponse` objects whenever the scaler needs KEDA to activate the deployment. @@ -444,38 +443,36 @@ public override async Task StreamIsActive(ScaledObjectRef request, IServerStream server.addService(externalScalerProto.externalscaler.ExternalScaler.service, { // ... streamIsActive: (call, callback) => { - const longitude = call.request.scalerMetadata.longitude - const latitude = call.request.scalerMetadata.latitude + const longitude = call.request.scalerMetadata.longitude; + const latitude = call.request.scalerMetadata.latitude; if (!longitude || !latitude) { callback({ code: grpc.status.INVALID_ARGUMENT, - details: 'longitude and latitude must be specified', - }) + details: "longitude and latitude must be specified", + }); } else { const interval = setInterval(() => { getEarthquakeCount((err, count) => { if (err) { - console.error(err) + console.error(err); } else if (count > 2) { call.write({ result: true, - }) + }); } - }) + }); }, 1000 * 60 * 60); - call.on('end', () => { - clearInterval(interval) - }) + call.on("end", () => { + clearInterval(interval); + }); } - } -}) + }, +}); ``` {{< /collapsible >}} -
- #### 5. Implementing `GetMetricSpec` `GetMetricSpec` returns the `target` value for [the HPA definition for the scaler](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale-walkthrough/). This scaler will define a static target of 10, but the threshold value is often specified in the metadata for other scalers. @@ -521,19 +518,19 @@ server.addService(externalScalerProto.externalscaler.ExternalScaler.service, { // ... getMetricSpec: (call, callback) => { callback(null, { - metricSpecs: [{ - metricName: 'earthquakeThreshold', - targetSize: 10, - }] - }) - } -}) + metricSpecs: [ + { + metricName: "earthquakeThreshold", + targetSize: 10, + }, + ], + }); + }, +}); ``` {{< /collapsible >}} -
- #### 6. Implementing `GetMetrics` `GetMetrics` returns the value of the metric referred to from `GetMetricSpec`, in this example it's `earthquakeThreshold`. @@ -600,32 +597,34 @@ public override async Task GetMetrics(GetMetricsRequest requ server.addService(externalScalerProto.externalscaler.ExternalScaler.service, { // ... getMetrics: (call, callback) => { - const longitude = call.request.scaledObjectRef.scalerMetadata.longitude - const latitude = call.request.scaledObjectRef.scalerMetadata.latitude + const longitude = call.request.scaledObjectRef.scalerMetadata.longitude; + const latitude = call.request.scaledObjectRef.scalerMetadata.latitude; if (!longitude || !latitude) { callback({ code: grpc.status.INVALID_ARGUMENT, - details: 'longitude and latitude must be specified', - }) + details: "longitude and latitude must be specified", + }); } else { getEarthquakeCount((err, count) => { if (err) { callback({ code: grpc.status.INTERNAL, details: err, - }) + }); } else { callback(null, { - metricValues: [{ - metricName: 'earthquakeThreshold', - metricValue: count, - }] - }) + metricValues: [ + { + metricName: "earthquakeThreshold", + metricValue: count, + }, + ], + }); } - }) + }); } - } -}) + }, +}); ``` {{< /collapsible >}} diff --git a/content/docs/2.6/migration.md b/content/docs/2.6/migration.md index 5ee8a6455..29e8408c8 100644 --- a/content/docs/2.6/migration.md +++ b/content/docs/2.6/migration.md @@ -11,15 +11,17 @@ Please note that you **can not** run both KEDA v1 and v2 on the same Kubernetes KEDA v2 is using a new API namespace for its Custom Resources Definitions (CRD): `keda.sh` instead of `keda.k8s.io` and introduces a new Custom Resource for scaling of Jobs. See full details on KEDA Custom Resources [here](../concepts/#custom-resources-crd). Here's an overview of what's changed: + - [Scaling of Deployments](#scaling-of-deployments) - [Scaling of Jobs](#scaling-of-jobs) - [Improved flexibility & usability of trigger metadata](#improved-flexibility--usability-of-trigger-metadata) - [Scalers](#scalers) - [TriggerAuthentication](#triggerauthentication) - ### Scaling of Deployments + In order to scale `Deployments` with KEDA v2, you need to do only a few modifications to existing v1 `ScaledObjects` definitions, so they comply with v2: + - Change the value of `apiVersion` property from `keda.k8s.io/v1alpha1` to `keda.sh/v1alpha1` - Rename property `spec.scaleTargetRef.deploymentName` to `spec.scaleTargetRef.name` - Rename property `spec.scaleTargetRef.containerName` to `spec.scaleTargetRef.envSourceContainerName` @@ -27,22 +29,23 @@ In order to scale `Deployments` with KEDA v2, you need to do only a few modifica Please see the examples below or refer to the full [v2 ScaledObject Specification](../concepts/scaling-deployments/#scaledobject-spec) - **Example of v1 ScaledObject** +**Example of v1 ScaledObject** + ```yaml apiVersion: keda.k8s.io/v1alpha1 kind: ScaledObject metadata: - name: {scaled-object-name} + name: { scaled-object-name } labels: - deploymentName: {deployment-name} + deploymentName: { deployment-name } spec: scaleTargetRef: - deploymentName: {deployment-name} - containerName: {container-name} - pollingInterval: 30 - cooldownPeriod: 300 - minReplicaCount: 0 - maxReplicaCount: 100 + deploymentName: { deployment-name } + containerName: { container-name } + pollingInterval: 30 + cooldownPeriod: 300 + minReplicaCount: 0 + maxReplicaCount: 100 triggers: # {list of triggers to activate the deployment} ``` @@ -50,18 +53,18 @@ spec: **Example of v2 ScaledObject** ```yaml -apiVersion: keda.sh/v1alpha1 # <--- Property value was changed +apiVersion: keda.sh/v1alpha1 # <--- Property value was changed kind: ScaledObject -metadata: # <--- labels.deploymentName is not needed - name: {scaled-object-name} +metadata: # <--- labels.deploymentName is not needed + name: { scaled-object-name } spec: scaleTargetRef: - name: {deployment-name} # <--- Property name was changed - envSourceContainerName: {container-name} # <--- Property name was changed - pollingInterval: 30 - cooldownPeriod: 300 - minReplicaCount: 0 - maxReplicaCount: 100 + name: { deployment-name } # <--- Property name was changed + envSourceContainerName: { container-name } # <--- Property name was changed + pollingInterval: 30 + cooldownPeriod: 300 + minReplicaCount: 0 + maxReplicaCount: 100 triggers: # {list of triggers to activate the deployment} ``` @@ -69,34 +72,36 @@ spec: ### Scaling of Jobs In order to scale `Jobs` with KEDA v2, you need to do only a few modifications to existing v1 `ScaledObjects` definitions, so they comply with v2: + - Change the value of `apiVersion` property from `keda.k8s.io/v1alpha1` to `keda.sh/v1alpha1` - Change the value of `kind` property from `ScaledObject` to `ScaledJob` - Remove property `spec.scaleType` -- Remove properties `spec.cooldownPeriod` and `spec.minReplicaCount` +- Remove properties `spec.cooldownPeriod` and `spec.minReplicaCount` You can configure `successfulJobsHistoryLimit` and `failedJobsHistoryLimit`. They will remove the old job histories automatically. Please see the examples below or refer to the full [v2 ScaledJob Specification](../concepts/scaling-jobs/#scaledjob-spec) **Example of v1 ScaledObject for Jobs scaling** + ```yaml apiVersion: keda.k8s.io/v1alpha1 kind: ScaledObject metadata: - name: {scaled-object-name} + name: { scaled-object-name } spec: scaleType: job jobTargetRef: - parallelism: 1 + parallelism: 1 completions: 1 activeDeadlineSeconds: 600 - backoffLimit: 6 + backoffLimit: 6 template: # {job template} - pollingInterval: 30 - cooldownPeriod: 300 - minReplicaCount: 0 - maxReplicaCount: 100 + pollingInterval: 30 + cooldownPeriod: 300 + minReplicaCount: 0 + maxReplicaCount: 100 triggers: # {list of triggers to create jobs} ``` @@ -104,21 +109,21 @@ spec: **Example of v2 ScaledJob** ```yaml -apiVersion: keda.sh/v1alpha1 # <--- Property value was changed -kind: ScaledJob # <--- Property value was changed +apiVersion: keda.sh/v1alpha1 # <--- Property value was changed +kind: ScaledJob # <--- Property value was changed metadata: - name: {scaled-job-name} -spec: # <--- spec.scaleType is not needed + name: { scaled-job-name } +spec: # <--- spec.scaleType is not needed jobTargetRef: - parallelism: 1 - completions: 1 + parallelism: 1 + completions: 1 activeDeadlineSeconds: 600 backoffLimit: 6 template: # {job template} - pollingInterval: 30 # <--- spec.cooldownPeriod and spec.minReplicaCount are not needed - successfulJobsHistoryLimit: 5 # <--- property is added - failedJobsHistoryLimit: 5 # <--- Property is added + pollingInterval: 30 # <--- spec.cooldownPeriod and spec.minReplicaCount are not needed + successfulJobsHistoryLimit: 5 # <--- property is added + failedJobsHistoryLimit: 5 # <--- Property is added maxReplicaCount: 100 triggers: # {list of triggers to create jobs} @@ -132,68 +137,74 @@ We've introduced more options to configure trigger metadata to give users more f Here's an overview: -| Scaler | 1.x | 2.0 | -|--------|--------|--------| -| `azure-blob` | `connection` (**Default**: `AzureWebJobsStorage`) | `connectionFromEnv` | -| `azure-monitor` | `activeDirectoryClientId`
`activeDirectoryClientPassword` | `activeDirectoryClientId`
`activeDirectoryClientIdFromEnv`
`activeDirectoryClientPasswordFromEnv` | -| `azure-queue` | `connection` (**Default**: AzureWebJobsStorage) | `connectionFromEnv` | -| `azure-servicebus` | `connection` | `connectionFromEnv` | -| `azure-eventhub` | `storageConnection` (**Default**: `AzureWebJobsStorage`)
`connection` (**Default**: `EventHub`) | `storageConnectionFromEnv`
`connectionFromEnv` | -| `aws-cloudwatch` | `awsAccessKeyID` (**Default**: `AWS_ACCESS_KEY_ID`)
`awsSecretAccessKey` (**Default**: `AWS_SECRET_ACCESS_KEY`) | `awsAccessKeyID`
`awsAccessKeyIDFromEnv`
`awsSecretAccessKeyFromEnv` | -| `aws-kinesis-stream` | `awsAccessKeyID` (**Default**: `AWS_ACCESS_KEY_ID`)
`awsSecretAccessKey` (**Default**: `AWS_SECRET_ACCESS_KEY`) | `awsAccessKeyID`
`awsAccessKeyIDFromEnv`
`awsSecretAccessKeyFromEnv` | -| `aws-sqs-queue` | `awsAccessKeyID` (**Default**: `AWS_ACCESS_KEY_ID`)
`awsSecretAccessKey` (**Default**: `AWS_SECRET_ACCESS_KEY`) | `awsAccessKeyID`
`awsAccessKeyIDFromEnv`
`awsSecretAccessKeyFromEnv` | -| `kafka` | _(none)_ | _(none)_ -| `rabbitmq` | `apiHost`
`host` | ~~`apiHost`~~
`host`
`hostFromEnv` | -| `prometheus` | _(none)_ | _(none)_ | -| `cron` | _(none)_ | _(none)_ | -| `redis` | `address`
`host`
`port`
`password` | `address`
`addressFromEnv`
`host`
`hostFromEnv`
~~`port`~~
`passwordFromEnv` -| `redis-streams` | `address`
`host`
`port`
`password` | `address`
`addressFromEnv`
`host`
`hostFromEnv`
~~`port`~~
`passwordFromEnv` -| `gcp-pubsub` | `credentials` | `credentialsFromEnv` | -| `external` | _(any matching value)_ | _(any matching value with `FromEnv` suffix)_ -| `liiklus` | _(none)_ | _(none)_ | -| `stan` | _(none)_ | _(none)_ | -| `huawei-cloudeye` | | _(none)_ | _(none)_ | -| `postgresql` | `connection`
`password` | `connectionFromEnv`
`passwordFromEnv` | -| `mysql` | `connectionString`
`password` | `connectionStringFromEnv`
`passwordFromEnv` | +| Scaler | 1.x | 2.0 | +| -------------------- | --------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------- | -------- | +| `azure-blob` | `connection` (**Default**: `AzureWebJobsStorage`) | `connectionFromEnv` | +| `azure-monitor` | `activeDirectoryClientId` `activeDirectoryClientPassword` | `activeDirectoryClientId` `activeDirectoryClientIdFromEnv` `activeDirectoryClientPasswordFromEnv` | +| `azure-queue` | `connection` (**Default**: AzureWebJobsStorage) | `connectionFromEnv` | +| `azure-servicebus` | `connection` | `connectionFromEnv` | +| `azure-eventhub` | `storageConnection` (**Default**: `AzureWebJobsStorage`) `connection` (**Default**: `EventHub`) | `storageConnectionFromEnv` `connectionFromEnv` | +| `aws-cloudwatch` | `awsAccessKeyID` (**Default**: `AWS_ACCESS_KEY_ID`) `awsSecretAccessKey` (**Default**: `AWS_SECRET_ACCESS_KEY`) | `awsAccessKeyID` `awsAccessKeyIDFromEnv` `awsSecretAccessKeyFromEnv` | +| `aws-kinesis-stream` | `awsAccessKeyID` (**Default**: `AWS_ACCESS_KEY_ID`) `awsSecretAccessKey` (**Default**: `AWS_SECRET_ACCESS_KEY`) | `awsAccessKeyID` `awsAccessKeyIDFromEnv` `awsSecretAccessKeyFromEnv` | +| `aws-sqs-queue` | `awsAccessKeyID` (**Default**: `AWS_ACCESS_KEY_ID`) `awsSecretAccessKey` (**Default**: `AWS_SECRET_ACCESS_KEY`) | `awsAccessKeyID` `awsAccessKeyIDFromEnv` `awsSecretAccessKeyFromEnv` | +| `kafka` | _(none)_ | _(none)_ | +| `rabbitmq` | `apiHost` `host` | ~~`apiHost`~~ `host` `hostFromEnv` | +| `prometheus` | _(none)_ | _(none)_ | +| `cron` | _(none)_ | _(none)_ | +| `redis` | `address` `host` `port` `password` | `address` `addressFromEnv` `host` `hostFromEnv` ~~`port`~~ `passwordFromEnv` | +| `redis-streams` | `address` `host` `port` `password` | `address` `addressFromEnv` `host` `hostFromEnv` ~~`port`~~ `passwordFromEnv` | +| `gcp-pubsub` | `credentials` | `credentialsFromEnv` | +| `external` | _(any matching value)_ | _(any matching value with `FromEnv` suffix)_ | +| `liiklus` | _(none)_ | _(none)_ | +| `stan` | _(none)_ | _(none)_ | +| `huawei-cloudeye` | | _(none)_ | _(none)_ | +| `postgresql` | `connection` `password` | `connectionFromEnv` `passwordFromEnv` | +| `mysql` | `connectionString` `password` | `connectionStringFromEnv` `passwordFromEnv` | ### Scalers **Azure Service Bus** - - `queueLength` was renamed to `messageCount` + +- `queueLength` was renamed to `messageCount` **Kafka** - - `authMode` property was replaced with `sasl` and `tls` properties. Please refer [documentation](../scalers/apache-kafka/#authentication-parameters) for Kafka Authentication Parameters details. + +- `authMode` property was replaced with `sasl` and `tls` properties. Please refer [documentation](../scalers/apache-kafka/#authentication-parameters) for Kafka Authentication Parameters details. **RabbitMQ** In KEDA 2.0 the RabbitMQ scaler has only `host` parameter, and the protocol for communication can be specified by -`protocol` (http or amqp). The default value is `amqp`. The behavior changes only for scalers that were using HTTP -protocol. +`protocol` (http or amqp). The default value is `amqp`. The behavior changes only for scalers that were using HTTP +protocol. Example of RabbitMQ trigger before 2.0: + ```yaml triggers: -- type: rabbitmq - metadata: - queueLength: '20' - queueName: testqueue - includeUnacked: 'true' - apiHost: 'https://guest:password@localhost:443/vhostname' -``` + - type: rabbitmq + metadata: + queueLength: "20" + queueName: testqueue + includeUnacked: "true" + apiHost: "https://guest:password@localhost:443/vhostname" +``` The same trigger in 2.0: + ```yaml triggers: -- type: rabbitmq - metadata: - queueLength: '20' - queueName: testqueue - protocol: 'http' - host: 'https://guest:password@localhost:443/vhostname' -``` + - type: rabbitmq + metadata: + queueLength: "20" + queueName: testqueue + protocol: "http" + host: "https://guest:password@localhost:443/vhostname" +``` ### TriggerAuthentication + In order to use Authentication via `TriggerAuthentication` with KEDA v2, you need to change: + - Change the value of `apiVersion` property from `keda.k8s.io/v1alpha1` to `keda.sh/v1alpha1` For more details please refer to the full diff --git a/content/docs/2.6/operate/cluster.md b/content/docs/2.6/operate/cluster.md index 5932357b8..10e6a4043 100644 --- a/content/docs/2.6/operate/cluster.md +++ b/content/docs/2.6/operate/cluster.md @@ -29,12 +29,10 @@ KEDA requires to be accessible inside the cluster to be able to autoscale. Here is an overview of the required ports that need to be accessible for KEDA to work: - | Port | Why? | Remarks | | ------ | -------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `443` | Used by Kubernetes API server to get metrics | Required for all platforms because it uses Control Plane → port 443 on the Service IP range communication.

This is not applicable for Google Cloud. | +| `443` | Used by Kubernetes API server to get metrics | Required for all platforms because it uses Control Plane → port 443 on the Service IP range communication. This is not applicable for Google Cloud. | | `6443` | Used by Kubernetes API server to get metrics | Only required for Google Cloud because it uses Control Plane → port 6443 on the Pod IP range for communication | - ## High Availability @@ -42,14 +40,14 @@ KEDA does not provide full support for high-availability due to upstream limitat Here is an overview of all KEDA deployments and the HA notes: -| Deployment | Support Replicas | Note | -| -------------- | ---------------- |---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Deployment | Support Replicas | Note | +| -------------- | ---------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | Metrics Server | 1 | You can run multiple replicas of our metrics sever, and it is recommended to add the `--enable-aggregator-routing=true` CLI flag to the kube-apiserver so that requests sent to our metrics servers are load balanced. However, [you can only run one active metric server in a Kubernetes cluster serving external.metrics.k8s.io](https://github.com/kubernetes-sigs/custom-metrics-apiserver/issues/70) which has to be the KEDA metric server. | -| Operator | 2 | While you can run multiple replicas of our operator, only one operator instance will be active. The rest will be standing by, which may reduce downtime during a failure. Multiple replicas will not improve the performance of KEDA, it could only reduce a downtime during a failover. | +| Operator | 2 | While you can run multiple replicas of our operator, only one operator instance will be active. The rest will be standing by, which may reduce downtime during a failure. Multiple replicas will not improve the performance of KEDA, it could only reduce a downtime during a failover. | ## HTTP Timeouts -Some scalers issue HTTP requests to external servers (i.e. cloud services). Each applicable scaler uses its own dedicated HTTP client with its own connection pool, and by default each client is set to time out any HTTP request after 3 seconds. +Some scalers issue HTTP requests to external servers (i.e. cloud services). Each applicable scaler uses its own dedicated HTTP client with its own connection pool, and by default each client is set to time out any HTTP request after 3 seconds. You can override this default by setting the `KEDA_HTTP_DEFAULT_TIMEOUT` environment variable to your desired timeout in milliseconds. For example, on Linux/Mac/Windows WSL2 operating systems, you'd use this command to set to 1 second: @@ -69,16 +67,17 @@ All applicable scalers will use this timeout. Setting a per-scaler timeout is cu The Kubernetes client config used within KEDA Metrics Adapter can be adjusted by passing the following command-line flags to the binary: -| Adapter Flag | Client Config Setting | Default Value | Description | -| -------------- | ----------------------- | ------------- | -------------------------------------------------------------- | -| kube-api-qps | cfg.QPS | 20.0 | Set the QPS rate for throttling requests sent to the apiserver | -| kube-api-burst | cfg.Burst | 30 | Set the burst for throttling requests sent to the apiserver | +| Adapter Flag | Client Config Setting | Default Value | Description | +| -------------- | --------------------- | ------------- | -------------------------------------------------------------- | +| kube-api-qps | cfg.QPS | 20.0 | Set the QPS rate for throttling requests sent to the apiserver | +| kube-api-burst | cfg.Burst | 30 | Set the burst for throttling requests sent to the apiserver | ## Configure `MaxConcurrentReconciles` for Controllers To implement internal controllers KEDA uses [controller-runtime project](https://github.com/kubernetes-sigs/controller-runtime), that enables configuration of [MaxConcurrentReconciles property](https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/controller#Options), ie. the maximum number of concurrent reconciles which can be run for a controller. KEDA Operator exposes properties for specifying `MaxConcurrentReconciles` for following controllers/reconcilers: + - `ScaledObjectReconciler` - responsible for watching and managing `ScaledObjects`, ie. validates input trigger specification, starts scaling logic and manages dependent HPA. - `ScaledJobReconciler` - responsible for watching and managing `ScaledJobs` and dependent Kubernetes Jobs @@ -86,11 +85,11 @@ KEDA Metrics Server exposes property for specifying `MaxConcurrentReconciles` fo To modify this properties you can set environment variables on both KEDA Operator and Metrics Server Deployments: -| Environment variable name | Deployment | Default Value | Affected reconciler | -| ------------------------------------- | -------------- | ------------- | -------------------------------------------------------------- | -| KEDA_SCALEDOBJECT_CTRL_MAX_RECONCILES | Operator | 5 | ScaledObjectReconciler | -| KEDA_SCALEDJOB_CTRL_MAX_RECONCILES | Operator | 1 | ScaledJobReconciler | -| KEDA_METRICS_CTRL_MAX_RECONCILES | Metrics Server | 1 | MetricsScaledObjectReconciler | +| Environment variable name | Deployment | Default Value | Affected reconciler | +| ------------------------------------- | -------------- | ------------- | ----------------------------- | +| KEDA_SCALEDOBJECT_CTRL_MAX_RECONCILES | Operator | 5 | ScaledObjectReconciler | +| KEDA_SCALEDJOB_CTRL_MAX_RECONCILES | Operator | 1 | ScaledJobReconciler | +| KEDA_METRICS_CTRL_MAX_RECONCILES | Metrics Server | 1 | MetricsScaledObjectReconciler | ## Certificates used by KEDA Metrics Server @@ -99,20 +98,19 @@ By default, KEDA Metrics Server uses self-signed certificates while communicatin Certificates and CA bundle can be referenced in `args` section in KEDA Metrics Server Deployment: ```yaml -... +--- args: - - '--client-ca-file=/cabundle/service-ca.crt' - - '--tls-cert-file=/certs/tls.crt' - - '--tls-private-key-file=/certs/tls.key' -... + - "--client-ca-file=/cabundle/service-ca.crt" + - "--tls-cert-file=/certs/tls.crt" + - "--tls-private-key-file=/certs/tls.key" ``` -The custom CA bundle should be also referenced in the `v1beta1.external.metrics.k8s.io` [APIService](https://kubernetes.io/docs/reference/kubernetes-api/cluster-resources/api-service-v1/#APIServiceSpec) resource (which is created during the installation of KEDA). +The custom CA bundle should be also referenced in the `v1beta1.external.metrics.k8s.io` [APIService](https://kubernetes.io/docs/reference/kubernetes-api/cluster-resources/api-service-v1/#APIServiceSpec) resource (which is created during the installation of KEDA). You should also make sure that `insecureSkipTLSVerify` is not set to `true`. ```yaml -... +--- spec: service: namespace: keda @@ -124,5 +122,4 @@ spec: YOURCABUNDLE... groupPriorityMinimum: 100 versionPriority: 100 -... ``` diff --git a/content/docs/2.7/concepts/external-scalers.md b/content/docs/2.7/concepts/external-scalers.md index 16f721117..d360d39cd 100644 --- a/content/docs/2.7/concepts/external-scalers.md +++ b/content/docs/2.7/concepts/external-scalers.md @@ -9,7 +9,7 @@ Built-in scalers run in the KEDA process/pod, while external scalers require an This document describes the external scaler interfaces and how to implement them in Go, Node, and .NET; however for more details on GRPC refer to [the official GRPC documentation](https://grpc.io/docs/) ->Want to learn about existing external scalers? Explore our [external scaler community](https://github.com/kedacore/external-scalers). +> Want to learn about existing external scalers? Explore our [external scaler community](https://github.com/kedacore/external-scalers). ## Overview @@ -39,7 +39,7 @@ The `Scaler` interface defines 4 methods: - `Close` is called to allow the scaler to clean up connections or other resources. - `GetMetricSpec` returns the target value for the HPA definition for the scaler. For more details refer to [Implementing `GetMetricSpec`](#5-implementing-getmetricspec). - `GetMetrics` returns the value of the metric referred to from `GetMetricSpec`. For more details refer to [Implementing `GetMetrics`](#6-implementing-getmetrics). -> Refer to the [HPA docs](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale-walkthrough/) for how HPA calculates `replicaCount` based on metric value and target value. KEDA supports both `AverageValue` and `Value` metric target types for external metrics. When `AverageValue` (the default metric type) is used, the metric value returned by the external scaler will be divided by the number of replicas. + > Refer to the [HPA docs](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale-walkthrough/) for how HPA calculates `replicaCount` based on metric value and target value. KEDA supports both `AverageValue` and `Value` metric target types for external metrics. When `AverageValue` (the default metric type) is used, the metric value returned by the external scaler will be divided by the number of replicas. The `PushScaler` interface adds a `Run` method. This method receives a push channel (`active`), on which the scaler can send `true` at any time. The purpose of this mechanism is to initiate a scaling operation independently from `pollingInterval`. @@ -109,7 +109,7 @@ KEDA will attempt a GRPC connection to `service-address.svc.local:9090` immediat } ``` ->**Note**: KEDA will issue all of the above RPC calls except `StreamIsActive` if `spec.triggers.type` is `external`. It _must_ be `external-push` for `StreamIsActive` to be called. +> **Note**: KEDA will issue all of the above RPC calls except `StreamIsActive` if `spec.triggers.type` is `external`. It _must_ be `external-push` for `StreamIsActive` to be called. ## Implementing KEDA external scaler GRPC interface @@ -176,8 +176,6 @@ npm install --save grpc request {{< /collapsible >}} -
- #### 3. Implementing `IsActive` Just like `IsActive(ctx context.Context) (bool, error)` in the go interface, the `IsActive` method in the GRPC interface is called every `pollingInterval` with a `ScaledObjectRef` object that contains the scaledObject name, namespace, and scaler metadata. @@ -304,70 +302,71 @@ public class ExternalScalerService : ExternalScaler.ExternalScalerBase Put the following code into your `index.js` file: ```js -const grpc = require('grpc') -const request = require('request') -const externalScalerProto = grpc.load('externalscaler.proto') +const grpc = require("grpc"); +const request = require("request"); +const externalScalerProto = grpc.load("externalscaler.proto"); -const server = new grpc.Server() +const server = new grpc.Server(); server.addService(externalScalerProto.externalscaler.ExternalScaler.service, { isActive: (call, callback) => { - const longitude = call.request.scalerMetadata.longitude - const latitude = call.request.scalerMetadata.latitude + const longitude = call.request.scalerMetadata.longitude; + const latitude = call.request.scalerMetadata.latitude; if (!longitude || !latitude) { callback({ code: grpc.status.INVALID_ARGUMENT, - details: 'longitude and latitude must be specified', - }) + details: "longitude and latitude must be specified", + }); } else { - const now = new Date() - const yesterday = new Date(new Date().setDate(new Date().getDate()-1)); - - const startTime = `${yesterday.getUTCFullYear()}-${yesterday.getUTCMonth()}-${yesterday.getUTCDay()}` - const endTime = `${now.getUTCFullYear()}-${now.getUTCMonth()}-${now.getUTCDay()}` - const radiusKm = 500 - const query = `format=geojson&starttime=${startTime}&endtime=${endTime}&longitude=${longitude}&latitude=${latitude}&maxradiuskm=${radiusKm}` - - request.get({ - url: `https://earthquake.usgs.gov/fdsnws/event/1/query?${query}`, - json: true, - }, (err, resp, data) => { - if (err) { - callback({ - code: grpc.status.INTERNAL, - details: err, - }) - } else if (resp.statusCode !== 200) { - callback({ - code: grpc.status.INTERNAL, - details: `expected status 200, got ${resp.statusCode}` - }) - } else { - // count how many earthquakes with mag > 1.0 - let count = 0 - data.features.forEach(i => { - if (i.properties.mag > 1.0) { - count++ - } - }) - callback(null, { - result: count > 2, - }) + const now = new Date(); + const yesterday = new Date(new Date().setDate(new Date().getDate() - 1)); + + const startTime = `${yesterday.getUTCFullYear()}-${yesterday.getUTCMonth()}-${yesterday.getUTCDay()}`; + const endTime = `${now.getUTCFullYear()}-${now.getUTCMonth()}-${now.getUTCDay()}`; + const radiusKm = 500; + const query = `format=geojson&starttime=${startTime}&endtime=${endTime}&longitude=${longitude}&latitude=${latitude}&maxradiuskm=${radiusKm}`; + + request.get( + { + url: `https://earthquake.usgs.gov/fdsnws/event/1/query?${query}`, + json: true, + }, + (err, resp, data) => { + if (err) { + callback({ + code: grpc.status.INTERNAL, + details: err, + }); + } else if (resp.statusCode !== 200) { + callback({ + code: grpc.status.INTERNAL, + details: `expected status 200, got ${resp.statusCode}`, + }); + } else { + // count how many earthquakes with mag > 1.0 + let count = 0; + data.features.forEach((i) => { + if (i.properties.mag > 1.0) { + count++; + } + }); + callback(null, { + result: count > 2, + }); + } } - }) + ); } - } -}) + }, +}); -server.bind('127.0.0.1:9090', grpc.ServerCredentials.createInsecure()) -console.log('Server listening on 127.0.0.1:9090') +server.bind("127.0.0.1:9090", grpc.ServerCredentials.createInsecure()); +console.log("Server listening on 127.0.0.1:9090"); -server.start() +server.start(); ``` {{< /collapsible >}} -
- #### 4. Implementing `StreamIsActive` Unlike `IsActive`, `StreamIsActive` is called once when KEDA reconciles the `ScaledObject`, and expects the external scaler to maintain a long-lived connection and push `IsActiveResponse` objects whenever the scaler needs KEDA to activate the deployment. @@ -444,38 +443,36 @@ public override async Task StreamIsActive(ScaledObjectRef request, IServerStream server.addService(externalScalerProto.externalscaler.ExternalScaler.service, { // ... streamIsActive: (call, callback) => { - const longitude = call.request.scalerMetadata.longitude - const latitude = call.request.scalerMetadata.latitude + const longitude = call.request.scalerMetadata.longitude; + const latitude = call.request.scalerMetadata.latitude; if (!longitude || !latitude) { callback({ code: grpc.status.INVALID_ARGUMENT, - details: 'longitude and latitude must be specified', - }) + details: "longitude and latitude must be specified", + }); } else { const interval = setInterval(() => { getEarthquakeCount((err, count) => { if (err) { - console.error(err) + console.error(err); } else if (count > 2) { call.write({ result: true, - }) + }); } - }) + }); }, 1000 * 60 * 60); - call.on('end', () => { - clearInterval(interval) - }) + call.on("end", () => { + clearInterval(interval); + }); } - } -}) + }, +}); ``` {{< /collapsible >}} -
- #### 5. Implementing `GetMetricSpec` `GetMetricSpec` returns the `target` value for [the HPA definition for the scaler](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale-walkthrough/). This scaler will define a static target of 10, but the threshold value is often specified in the metadata for other scalers. @@ -521,19 +518,19 @@ server.addService(externalScalerProto.externalscaler.ExternalScaler.service, { // ... getMetricSpec: (call, callback) => { callback(null, { - metricSpecs: [{ - metricName: 'earthquakeThreshold', - targetSize: 10, - }] - }) - } -}) + metricSpecs: [ + { + metricName: "earthquakeThreshold", + targetSize: 10, + }, + ], + }); + }, +}); ``` {{< /collapsible >}} -
- #### 6. Implementing `GetMetrics` `GetMetrics` returns the value of the metric referred to from `GetMetricSpec`, in this example it's `earthquakeThreshold`. @@ -600,32 +597,34 @@ public override async Task GetMetrics(GetMetricsRequest requ server.addService(externalScalerProto.externalscaler.ExternalScaler.service, { // ... getMetrics: (call, callback) => { - const longitude = call.request.scaledObjectRef.scalerMetadata.longitude - const latitude = call.request.scaledObjectRef.scalerMetadata.latitude + const longitude = call.request.scaledObjectRef.scalerMetadata.longitude; + const latitude = call.request.scaledObjectRef.scalerMetadata.latitude; if (!longitude || !latitude) { callback({ code: grpc.status.INVALID_ARGUMENT, - details: 'longitude and latitude must be specified', - }) + details: "longitude and latitude must be specified", + }); } else { getEarthquakeCount((err, count) => { if (err) { callback({ code: grpc.status.INTERNAL, details: err, - }) + }); } else { callback(null, { - metricValues: [{ - metricName: 'earthquakeThreshold', - metricValue: count, - }] - }) + metricValues: [ + { + metricName: "earthquakeThreshold", + metricValue: count, + }, + ], + }); } - }) + }); } - } -}) + }, +}); ``` {{< /collapsible >}} diff --git a/content/docs/2.7/migration.md b/content/docs/2.7/migration.md index 5ee8a6455..29e8408c8 100644 --- a/content/docs/2.7/migration.md +++ b/content/docs/2.7/migration.md @@ -11,15 +11,17 @@ Please note that you **can not** run both KEDA v1 and v2 on the same Kubernetes KEDA v2 is using a new API namespace for its Custom Resources Definitions (CRD): `keda.sh` instead of `keda.k8s.io` and introduces a new Custom Resource for scaling of Jobs. See full details on KEDA Custom Resources [here](../concepts/#custom-resources-crd). Here's an overview of what's changed: + - [Scaling of Deployments](#scaling-of-deployments) - [Scaling of Jobs](#scaling-of-jobs) - [Improved flexibility & usability of trigger metadata](#improved-flexibility--usability-of-trigger-metadata) - [Scalers](#scalers) - [TriggerAuthentication](#triggerauthentication) - ### Scaling of Deployments + In order to scale `Deployments` with KEDA v2, you need to do only a few modifications to existing v1 `ScaledObjects` definitions, so they comply with v2: + - Change the value of `apiVersion` property from `keda.k8s.io/v1alpha1` to `keda.sh/v1alpha1` - Rename property `spec.scaleTargetRef.deploymentName` to `spec.scaleTargetRef.name` - Rename property `spec.scaleTargetRef.containerName` to `spec.scaleTargetRef.envSourceContainerName` @@ -27,22 +29,23 @@ In order to scale `Deployments` with KEDA v2, you need to do only a few modifica Please see the examples below or refer to the full [v2 ScaledObject Specification](../concepts/scaling-deployments/#scaledobject-spec) - **Example of v1 ScaledObject** +**Example of v1 ScaledObject** + ```yaml apiVersion: keda.k8s.io/v1alpha1 kind: ScaledObject metadata: - name: {scaled-object-name} + name: { scaled-object-name } labels: - deploymentName: {deployment-name} + deploymentName: { deployment-name } spec: scaleTargetRef: - deploymentName: {deployment-name} - containerName: {container-name} - pollingInterval: 30 - cooldownPeriod: 300 - minReplicaCount: 0 - maxReplicaCount: 100 + deploymentName: { deployment-name } + containerName: { container-name } + pollingInterval: 30 + cooldownPeriod: 300 + minReplicaCount: 0 + maxReplicaCount: 100 triggers: # {list of triggers to activate the deployment} ``` @@ -50,18 +53,18 @@ spec: **Example of v2 ScaledObject** ```yaml -apiVersion: keda.sh/v1alpha1 # <--- Property value was changed +apiVersion: keda.sh/v1alpha1 # <--- Property value was changed kind: ScaledObject -metadata: # <--- labels.deploymentName is not needed - name: {scaled-object-name} +metadata: # <--- labels.deploymentName is not needed + name: { scaled-object-name } spec: scaleTargetRef: - name: {deployment-name} # <--- Property name was changed - envSourceContainerName: {container-name} # <--- Property name was changed - pollingInterval: 30 - cooldownPeriod: 300 - minReplicaCount: 0 - maxReplicaCount: 100 + name: { deployment-name } # <--- Property name was changed + envSourceContainerName: { container-name } # <--- Property name was changed + pollingInterval: 30 + cooldownPeriod: 300 + minReplicaCount: 0 + maxReplicaCount: 100 triggers: # {list of triggers to activate the deployment} ``` @@ -69,34 +72,36 @@ spec: ### Scaling of Jobs In order to scale `Jobs` with KEDA v2, you need to do only a few modifications to existing v1 `ScaledObjects` definitions, so they comply with v2: + - Change the value of `apiVersion` property from `keda.k8s.io/v1alpha1` to `keda.sh/v1alpha1` - Change the value of `kind` property from `ScaledObject` to `ScaledJob` - Remove property `spec.scaleType` -- Remove properties `spec.cooldownPeriod` and `spec.minReplicaCount` +- Remove properties `spec.cooldownPeriod` and `spec.minReplicaCount` You can configure `successfulJobsHistoryLimit` and `failedJobsHistoryLimit`. They will remove the old job histories automatically. Please see the examples below or refer to the full [v2 ScaledJob Specification](../concepts/scaling-jobs/#scaledjob-spec) **Example of v1 ScaledObject for Jobs scaling** + ```yaml apiVersion: keda.k8s.io/v1alpha1 kind: ScaledObject metadata: - name: {scaled-object-name} + name: { scaled-object-name } spec: scaleType: job jobTargetRef: - parallelism: 1 + parallelism: 1 completions: 1 activeDeadlineSeconds: 600 - backoffLimit: 6 + backoffLimit: 6 template: # {job template} - pollingInterval: 30 - cooldownPeriod: 300 - minReplicaCount: 0 - maxReplicaCount: 100 + pollingInterval: 30 + cooldownPeriod: 300 + minReplicaCount: 0 + maxReplicaCount: 100 triggers: # {list of triggers to create jobs} ``` @@ -104,21 +109,21 @@ spec: **Example of v2 ScaledJob** ```yaml -apiVersion: keda.sh/v1alpha1 # <--- Property value was changed -kind: ScaledJob # <--- Property value was changed +apiVersion: keda.sh/v1alpha1 # <--- Property value was changed +kind: ScaledJob # <--- Property value was changed metadata: - name: {scaled-job-name} -spec: # <--- spec.scaleType is not needed + name: { scaled-job-name } +spec: # <--- spec.scaleType is not needed jobTargetRef: - parallelism: 1 - completions: 1 + parallelism: 1 + completions: 1 activeDeadlineSeconds: 600 backoffLimit: 6 template: # {job template} - pollingInterval: 30 # <--- spec.cooldownPeriod and spec.minReplicaCount are not needed - successfulJobsHistoryLimit: 5 # <--- property is added - failedJobsHistoryLimit: 5 # <--- Property is added + pollingInterval: 30 # <--- spec.cooldownPeriod and spec.minReplicaCount are not needed + successfulJobsHistoryLimit: 5 # <--- property is added + failedJobsHistoryLimit: 5 # <--- Property is added maxReplicaCount: 100 triggers: # {list of triggers to create jobs} @@ -132,68 +137,74 @@ We've introduced more options to configure trigger metadata to give users more f Here's an overview: -| Scaler | 1.x | 2.0 | -|--------|--------|--------| -| `azure-blob` | `connection` (**Default**: `AzureWebJobsStorage`) | `connectionFromEnv` | -| `azure-monitor` | `activeDirectoryClientId`
`activeDirectoryClientPassword` | `activeDirectoryClientId`
`activeDirectoryClientIdFromEnv`
`activeDirectoryClientPasswordFromEnv` | -| `azure-queue` | `connection` (**Default**: AzureWebJobsStorage) | `connectionFromEnv` | -| `azure-servicebus` | `connection` | `connectionFromEnv` | -| `azure-eventhub` | `storageConnection` (**Default**: `AzureWebJobsStorage`)
`connection` (**Default**: `EventHub`) | `storageConnectionFromEnv`
`connectionFromEnv` | -| `aws-cloudwatch` | `awsAccessKeyID` (**Default**: `AWS_ACCESS_KEY_ID`)
`awsSecretAccessKey` (**Default**: `AWS_SECRET_ACCESS_KEY`) | `awsAccessKeyID`
`awsAccessKeyIDFromEnv`
`awsSecretAccessKeyFromEnv` | -| `aws-kinesis-stream` | `awsAccessKeyID` (**Default**: `AWS_ACCESS_KEY_ID`)
`awsSecretAccessKey` (**Default**: `AWS_SECRET_ACCESS_KEY`) | `awsAccessKeyID`
`awsAccessKeyIDFromEnv`
`awsSecretAccessKeyFromEnv` | -| `aws-sqs-queue` | `awsAccessKeyID` (**Default**: `AWS_ACCESS_KEY_ID`)
`awsSecretAccessKey` (**Default**: `AWS_SECRET_ACCESS_KEY`) | `awsAccessKeyID`
`awsAccessKeyIDFromEnv`
`awsSecretAccessKeyFromEnv` | -| `kafka` | _(none)_ | _(none)_ -| `rabbitmq` | `apiHost`
`host` | ~~`apiHost`~~
`host`
`hostFromEnv` | -| `prometheus` | _(none)_ | _(none)_ | -| `cron` | _(none)_ | _(none)_ | -| `redis` | `address`
`host`
`port`
`password` | `address`
`addressFromEnv`
`host`
`hostFromEnv`
~~`port`~~
`passwordFromEnv` -| `redis-streams` | `address`
`host`
`port`
`password` | `address`
`addressFromEnv`
`host`
`hostFromEnv`
~~`port`~~
`passwordFromEnv` -| `gcp-pubsub` | `credentials` | `credentialsFromEnv` | -| `external` | _(any matching value)_ | _(any matching value with `FromEnv` suffix)_ -| `liiklus` | _(none)_ | _(none)_ | -| `stan` | _(none)_ | _(none)_ | -| `huawei-cloudeye` | | _(none)_ | _(none)_ | -| `postgresql` | `connection`
`password` | `connectionFromEnv`
`passwordFromEnv` | -| `mysql` | `connectionString`
`password` | `connectionStringFromEnv`
`passwordFromEnv` | +| Scaler | 1.x | 2.0 | +| -------------------- | --------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------- | -------- | +| `azure-blob` | `connection` (**Default**: `AzureWebJobsStorage`) | `connectionFromEnv` | +| `azure-monitor` | `activeDirectoryClientId` `activeDirectoryClientPassword` | `activeDirectoryClientId` `activeDirectoryClientIdFromEnv` `activeDirectoryClientPasswordFromEnv` | +| `azure-queue` | `connection` (**Default**: AzureWebJobsStorage) | `connectionFromEnv` | +| `azure-servicebus` | `connection` | `connectionFromEnv` | +| `azure-eventhub` | `storageConnection` (**Default**: `AzureWebJobsStorage`) `connection` (**Default**: `EventHub`) | `storageConnectionFromEnv` `connectionFromEnv` | +| `aws-cloudwatch` | `awsAccessKeyID` (**Default**: `AWS_ACCESS_KEY_ID`) `awsSecretAccessKey` (**Default**: `AWS_SECRET_ACCESS_KEY`) | `awsAccessKeyID` `awsAccessKeyIDFromEnv` `awsSecretAccessKeyFromEnv` | +| `aws-kinesis-stream` | `awsAccessKeyID` (**Default**: `AWS_ACCESS_KEY_ID`) `awsSecretAccessKey` (**Default**: `AWS_SECRET_ACCESS_KEY`) | `awsAccessKeyID` `awsAccessKeyIDFromEnv` `awsSecretAccessKeyFromEnv` | +| `aws-sqs-queue` | `awsAccessKeyID` (**Default**: `AWS_ACCESS_KEY_ID`) `awsSecretAccessKey` (**Default**: `AWS_SECRET_ACCESS_KEY`) | `awsAccessKeyID` `awsAccessKeyIDFromEnv` `awsSecretAccessKeyFromEnv` | +| `kafka` | _(none)_ | _(none)_ | +| `rabbitmq` | `apiHost` `host` | ~~`apiHost`~~ `host` `hostFromEnv` | +| `prometheus` | _(none)_ | _(none)_ | +| `cron` | _(none)_ | _(none)_ | +| `redis` | `address` `host` `port` `password` | `address` `addressFromEnv` `host` `hostFromEnv` ~~`port`~~ `passwordFromEnv` | +| `redis-streams` | `address` `host` `port` `password` | `address` `addressFromEnv` `host` `hostFromEnv` ~~`port`~~ `passwordFromEnv` | +| `gcp-pubsub` | `credentials` | `credentialsFromEnv` | +| `external` | _(any matching value)_ | _(any matching value with `FromEnv` suffix)_ | +| `liiklus` | _(none)_ | _(none)_ | +| `stan` | _(none)_ | _(none)_ | +| `huawei-cloudeye` | | _(none)_ | _(none)_ | +| `postgresql` | `connection` `password` | `connectionFromEnv` `passwordFromEnv` | +| `mysql` | `connectionString` `password` | `connectionStringFromEnv` `passwordFromEnv` | ### Scalers **Azure Service Bus** - - `queueLength` was renamed to `messageCount` + +- `queueLength` was renamed to `messageCount` **Kafka** - - `authMode` property was replaced with `sasl` and `tls` properties. Please refer [documentation](../scalers/apache-kafka/#authentication-parameters) for Kafka Authentication Parameters details. + +- `authMode` property was replaced with `sasl` and `tls` properties. Please refer [documentation](../scalers/apache-kafka/#authentication-parameters) for Kafka Authentication Parameters details. **RabbitMQ** In KEDA 2.0 the RabbitMQ scaler has only `host` parameter, and the protocol for communication can be specified by -`protocol` (http or amqp). The default value is `amqp`. The behavior changes only for scalers that were using HTTP -protocol. +`protocol` (http or amqp). The default value is `amqp`. The behavior changes only for scalers that were using HTTP +protocol. Example of RabbitMQ trigger before 2.0: + ```yaml triggers: -- type: rabbitmq - metadata: - queueLength: '20' - queueName: testqueue - includeUnacked: 'true' - apiHost: 'https://guest:password@localhost:443/vhostname' -``` + - type: rabbitmq + metadata: + queueLength: "20" + queueName: testqueue + includeUnacked: "true" + apiHost: "https://guest:password@localhost:443/vhostname" +``` The same trigger in 2.0: + ```yaml triggers: -- type: rabbitmq - metadata: - queueLength: '20' - queueName: testqueue - protocol: 'http' - host: 'https://guest:password@localhost:443/vhostname' -``` + - type: rabbitmq + metadata: + queueLength: "20" + queueName: testqueue + protocol: "http" + host: "https://guest:password@localhost:443/vhostname" +``` ### TriggerAuthentication + In order to use Authentication via `TriggerAuthentication` with KEDA v2, you need to change: + - Change the value of `apiVersion` property from `keda.k8s.io/v1alpha1` to `keda.sh/v1alpha1` For more details please refer to the full diff --git a/content/docs/2.7/operate/cluster.md b/content/docs/2.7/operate/cluster.md index dc1a201d6..dfb78df81 100644 --- a/content/docs/2.7/operate/cluster.md +++ b/content/docs/2.7/operate/cluster.md @@ -29,12 +29,10 @@ KEDA requires to be accessible inside the cluster to be able to autoscale. Here is an overview of the required ports that need to be accessible for KEDA to work: - | Port | Why? | Remarks | | ------ | -------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `443` | Used by Kubernetes API server to get metrics | Required for all platforms because it uses Control Plane → port 443 on the Service IP range communication.

This is not applicable for Google Cloud. | +| `443` | Used by Kubernetes API server to get metrics | Required for all platforms because it uses Control Plane → port 443 on the Service IP range communication. This is not applicable for Google Cloud. | | `6443` | Used by Kubernetes API server to get metrics | Only required for Google Cloud because it uses Control Plane → port 6443 on the Pod IP range for communication | - ## High Availability @@ -42,14 +40,14 @@ KEDA does not provide full support for high-availability due to upstream limitat Here is an overview of all KEDA deployments and the HA notes: -| Deployment | Support Replicas | Note | -| -------------- | ---------------- |---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Deployment | Support Replicas | Note | +| -------------- | ---------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | Metrics Server | 1 | You can run multiple replicas of our metrics sever, and it is recommended to add the `--enable-aggregator-routing=true` CLI flag to the kube-apiserver so that requests sent to our metrics servers are load balanced. However, [you can only run one active metric server in a Kubernetes cluster serving external.metrics.k8s.io](https://github.com/kubernetes-sigs/custom-metrics-apiserver/issues/70) which has to be the KEDA metric server. | -| Operator | 2 | While you can run multiple replicas of our operator, only one operator instance will be active. The rest will be standing by, which may reduce downtime during a failure. Multiple replicas will not improve the performance of KEDA, it could only reduce a downtime during a failover. | +| Operator | 2 | While you can run multiple replicas of our operator, only one operator instance will be active. The rest will be standing by, which may reduce downtime during a failure. Multiple replicas will not improve the performance of KEDA, it could only reduce a downtime during a failover. | ## HTTP Timeouts -Some scalers issue HTTP requests to external servers (i.e. cloud services). Each applicable scaler uses its own dedicated HTTP client with its own connection pool, and by default each client is set to time out any HTTP request after 3 seconds. +Some scalers issue HTTP requests to external servers (i.e. cloud services). Each applicable scaler uses its own dedicated HTTP client with its own connection pool, and by default each client is set to time out any HTTP request after 3 seconds. You can override this default by setting the `KEDA_HTTP_DEFAULT_TIMEOUT` environment variable to your desired timeout in milliseconds. For example, on Linux/Mac/Windows WSL2 operating systems, you'd use this command to set to 1 second: @@ -80,16 +78,17 @@ Some scalers issue HTTP requests to external servers (i.e. cloud services). As c The Kubernetes client config used within KEDA Metrics Adapter can be adjusted by passing the following command-line flags to the binary: -| Adapter Flag | Client Config Setting | Default Value | Description | -| -------------- | ----------------------- | ------------- | -------------------------------------------------------------- | -| kube-api-qps | cfg.QPS | 20.0 | Set the QPS rate for throttling requests sent to the apiserver | -| kube-api-burst | cfg.Burst | 30 | Set the burst for throttling requests sent to the apiserver | +| Adapter Flag | Client Config Setting | Default Value | Description | +| -------------- | --------------------- | ------------- | -------------------------------------------------------------- | +| kube-api-qps | cfg.QPS | 20.0 | Set the QPS rate for throttling requests sent to the apiserver | +| kube-api-burst | cfg.Burst | 30 | Set the burst for throttling requests sent to the apiserver | ## Configure `MaxConcurrentReconciles` for Controllers To implement internal controllers KEDA uses [controller-runtime project](https://github.com/kubernetes-sigs/controller-runtime), that enables configuration of [MaxConcurrentReconciles property](https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/controller#Options), ie. the maximum number of concurrent reconciles which can be run for a controller. KEDA Operator exposes properties for specifying `MaxConcurrentReconciles` for following controllers/reconcilers: + - `ScaledObjectReconciler` - responsible for watching and managing `ScaledObjects`, ie. validates input trigger specification, starts scaling logic and manages dependent HPA. - `ScaledJobReconciler` - responsible for watching and managing `ScaledJobs` and dependent Kubernetes Jobs @@ -97,11 +96,11 @@ KEDA Metrics Server exposes property for specifying `MaxConcurrentReconciles` fo To modify this properties you can set environment variables on both KEDA Operator and Metrics Server Deployments: -| Environment variable name | Deployment | Default Value | Affected reconciler | -| ------------------------------------- | -------------- | ------------- | -------------------------------------------------------------- | -| KEDA_SCALEDOBJECT_CTRL_MAX_RECONCILES | Operator | 5 | ScaledObjectReconciler | -| KEDA_SCALEDJOB_CTRL_MAX_RECONCILES | Operator | 1 | ScaledJobReconciler | -| KEDA_METRICS_CTRL_MAX_RECONCILES | Metrics Server | 1 | MetricsScaledObjectReconciler | +| Environment variable name | Deployment | Default Value | Affected reconciler | +| ------------------------------------- | -------------- | ------------- | ----------------------------- | +| KEDA_SCALEDOBJECT_CTRL_MAX_RECONCILES | Operator | 5 | ScaledObjectReconciler | +| KEDA_SCALEDJOB_CTRL_MAX_RECONCILES | Operator | 1 | ScaledJobReconciler | +| KEDA_METRICS_CTRL_MAX_RECONCILES | Metrics Server | 1 | MetricsScaledObjectReconciler | ## Certificates used by KEDA Metrics Server @@ -110,20 +109,19 @@ By default, KEDA Metrics Server uses self-signed certificates while communicatin Certificates and CA bundle can be referenced in `args` section in KEDA Metrics Server Deployment: ```yaml -... +--- args: - - '--client-ca-file=/cabundle/service-ca.crt' - - '--tls-cert-file=/certs/tls.crt' - - '--tls-private-key-file=/certs/tls.key' -... + - "--client-ca-file=/cabundle/service-ca.crt" + - "--tls-cert-file=/certs/tls.crt" + - "--tls-private-key-file=/certs/tls.key" ``` -The custom CA bundle should be also referenced in the `v1beta1.external.metrics.k8s.io` [APIService](https://kubernetes.io/docs/reference/kubernetes-api/cluster-resources/api-service-v1/#APIServiceSpec) resource (which is created during the installation of KEDA). +The custom CA bundle should be also referenced in the `v1beta1.external.metrics.k8s.io` [APIService](https://kubernetes.io/docs/reference/kubernetes-api/cluster-resources/api-service-v1/#APIServiceSpec) resource (which is created during the installation of KEDA). You should also make sure that `insecureSkipTLSVerify` is not set to `true`. ```yaml -... +--- spec: service: namespace: keda @@ -135,5 +133,4 @@ spec: YOURCABUNDLE... groupPriorityMinimum: 100 versionPriority: 100 -... ``` diff --git a/content/docs/2.8/concepts/external-scalers.md b/content/docs/2.8/concepts/external-scalers.md index 16f721117..d360d39cd 100644 --- a/content/docs/2.8/concepts/external-scalers.md +++ b/content/docs/2.8/concepts/external-scalers.md @@ -9,7 +9,7 @@ Built-in scalers run in the KEDA process/pod, while external scalers require an This document describes the external scaler interfaces and how to implement them in Go, Node, and .NET; however for more details on GRPC refer to [the official GRPC documentation](https://grpc.io/docs/) ->Want to learn about existing external scalers? Explore our [external scaler community](https://github.com/kedacore/external-scalers). +> Want to learn about existing external scalers? Explore our [external scaler community](https://github.com/kedacore/external-scalers). ## Overview @@ -39,7 +39,7 @@ The `Scaler` interface defines 4 methods: - `Close` is called to allow the scaler to clean up connections or other resources. - `GetMetricSpec` returns the target value for the HPA definition for the scaler. For more details refer to [Implementing `GetMetricSpec`](#5-implementing-getmetricspec). - `GetMetrics` returns the value of the metric referred to from `GetMetricSpec`. For more details refer to [Implementing `GetMetrics`](#6-implementing-getmetrics). -> Refer to the [HPA docs](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale-walkthrough/) for how HPA calculates `replicaCount` based on metric value and target value. KEDA supports both `AverageValue` and `Value` metric target types for external metrics. When `AverageValue` (the default metric type) is used, the metric value returned by the external scaler will be divided by the number of replicas. + > Refer to the [HPA docs](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale-walkthrough/) for how HPA calculates `replicaCount` based on metric value and target value. KEDA supports both `AverageValue` and `Value` metric target types for external metrics. When `AverageValue` (the default metric type) is used, the metric value returned by the external scaler will be divided by the number of replicas. The `PushScaler` interface adds a `Run` method. This method receives a push channel (`active`), on which the scaler can send `true` at any time. The purpose of this mechanism is to initiate a scaling operation independently from `pollingInterval`. @@ -109,7 +109,7 @@ KEDA will attempt a GRPC connection to `service-address.svc.local:9090` immediat } ``` ->**Note**: KEDA will issue all of the above RPC calls except `StreamIsActive` if `spec.triggers.type` is `external`. It _must_ be `external-push` for `StreamIsActive` to be called. +> **Note**: KEDA will issue all of the above RPC calls except `StreamIsActive` if `spec.triggers.type` is `external`. It _must_ be `external-push` for `StreamIsActive` to be called. ## Implementing KEDA external scaler GRPC interface @@ -176,8 +176,6 @@ npm install --save grpc request {{< /collapsible >}} -
- #### 3. Implementing `IsActive` Just like `IsActive(ctx context.Context) (bool, error)` in the go interface, the `IsActive` method in the GRPC interface is called every `pollingInterval` with a `ScaledObjectRef` object that contains the scaledObject name, namespace, and scaler metadata. @@ -304,70 +302,71 @@ public class ExternalScalerService : ExternalScaler.ExternalScalerBase Put the following code into your `index.js` file: ```js -const grpc = require('grpc') -const request = require('request') -const externalScalerProto = grpc.load('externalscaler.proto') +const grpc = require("grpc"); +const request = require("request"); +const externalScalerProto = grpc.load("externalscaler.proto"); -const server = new grpc.Server() +const server = new grpc.Server(); server.addService(externalScalerProto.externalscaler.ExternalScaler.service, { isActive: (call, callback) => { - const longitude = call.request.scalerMetadata.longitude - const latitude = call.request.scalerMetadata.latitude + const longitude = call.request.scalerMetadata.longitude; + const latitude = call.request.scalerMetadata.latitude; if (!longitude || !latitude) { callback({ code: grpc.status.INVALID_ARGUMENT, - details: 'longitude and latitude must be specified', - }) + details: "longitude and latitude must be specified", + }); } else { - const now = new Date() - const yesterday = new Date(new Date().setDate(new Date().getDate()-1)); - - const startTime = `${yesterday.getUTCFullYear()}-${yesterday.getUTCMonth()}-${yesterday.getUTCDay()}` - const endTime = `${now.getUTCFullYear()}-${now.getUTCMonth()}-${now.getUTCDay()}` - const radiusKm = 500 - const query = `format=geojson&starttime=${startTime}&endtime=${endTime}&longitude=${longitude}&latitude=${latitude}&maxradiuskm=${radiusKm}` - - request.get({ - url: `https://earthquake.usgs.gov/fdsnws/event/1/query?${query}`, - json: true, - }, (err, resp, data) => { - if (err) { - callback({ - code: grpc.status.INTERNAL, - details: err, - }) - } else if (resp.statusCode !== 200) { - callback({ - code: grpc.status.INTERNAL, - details: `expected status 200, got ${resp.statusCode}` - }) - } else { - // count how many earthquakes with mag > 1.0 - let count = 0 - data.features.forEach(i => { - if (i.properties.mag > 1.0) { - count++ - } - }) - callback(null, { - result: count > 2, - }) + const now = new Date(); + const yesterday = new Date(new Date().setDate(new Date().getDate() - 1)); + + const startTime = `${yesterday.getUTCFullYear()}-${yesterday.getUTCMonth()}-${yesterday.getUTCDay()}`; + const endTime = `${now.getUTCFullYear()}-${now.getUTCMonth()}-${now.getUTCDay()}`; + const radiusKm = 500; + const query = `format=geojson&starttime=${startTime}&endtime=${endTime}&longitude=${longitude}&latitude=${latitude}&maxradiuskm=${radiusKm}`; + + request.get( + { + url: `https://earthquake.usgs.gov/fdsnws/event/1/query?${query}`, + json: true, + }, + (err, resp, data) => { + if (err) { + callback({ + code: grpc.status.INTERNAL, + details: err, + }); + } else if (resp.statusCode !== 200) { + callback({ + code: grpc.status.INTERNAL, + details: `expected status 200, got ${resp.statusCode}`, + }); + } else { + // count how many earthquakes with mag > 1.0 + let count = 0; + data.features.forEach((i) => { + if (i.properties.mag > 1.0) { + count++; + } + }); + callback(null, { + result: count > 2, + }); + } } - }) + ); } - } -}) + }, +}); -server.bind('127.0.0.1:9090', grpc.ServerCredentials.createInsecure()) -console.log('Server listening on 127.0.0.1:9090') +server.bind("127.0.0.1:9090", grpc.ServerCredentials.createInsecure()); +console.log("Server listening on 127.0.0.1:9090"); -server.start() +server.start(); ``` {{< /collapsible >}} -
- #### 4. Implementing `StreamIsActive` Unlike `IsActive`, `StreamIsActive` is called once when KEDA reconciles the `ScaledObject`, and expects the external scaler to maintain a long-lived connection and push `IsActiveResponse` objects whenever the scaler needs KEDA to activate the deployment. @@ -444,38 +443,36 @@ public override async Task StreamIsActive(ScaledObjectRef request, IServerStream server.addService(externalScalerProto.externalscaler.ExternalScaler.service, { // ... streamIsActive: (call, callback) => { - const longitude = call.request.scalerMetadata.longitude - const latitude = call.request.scalerMetadata.latitude + const longitude = call.request.scalerMetadata.longitude; + const latitude = call.request.scalerMetadata.latitude; if (!longitude || !latitude) { callback({ code: grpc.status.INVALID_ARGUMENT, - details: 'longitude and latitude must be specified', - }) + details: "longitude and latitude must be specified", + }); } else { const interval = setInterval(() => { getEarthquakeCount((err, count) => { if (err) { - console.error(err) + console.error(err); } else if (count > 2) { call.write({ result: true, - }) + }); } - }) + }); }, 1000 * 60 * 60); - call.on('end', () => { - clearInterval(interval) - }) + call.on("end", () => { + clearInterval(interval); + }); } - } -}) + }, +}); ``` {{< /collapsible >}} -
- #### 5. Implementing `GetMetricSpec` `GetMetricSpec` returns the `target` value for [the HPA definition for the scaler](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale-walkthrough/). This scaler will define a static target of 10, but the threshold value is often specified in the metadata for other scalers. @@ -521,19 +518,19 @@ server.addService(externalScalerProto.externalscaler.ExternalScaler.service, { // ... getMetricSpec: (call, callback) => { callback(null, { - metricSpecs: [{ - metricName: 'earthquakeThreshold', - targetSize: 10, - }] - }) - } -}) + metricSpecs: [ + { + metricName: "earthquakeThreshold", + targetSize: 10, + }, + ], + }); + }, +}); ``` {{< /collapsible >}} -
- #### 6. Implementing `GetMetrics` `GetMetrics` returns the value of the metric referred to from `GetMetricSpec`, in this example it's `earthquakeThreshold`. @@ -600,32 +597,34 @@ public override async Task GetMetrics(GetMetricsRequest requ server.addService(externalScalerProto.externalscaler.ExternalScaler.service, { // ... getMetrics: (call, callback) => { - const longitude = call.request.scaledObjectRef.scalerMetadata.longitude - const latitude = call.request.scaledObjectRef.scalerMetadata.latitude + const longitude = call.request.scaledObjectRef.scalerMetadata.longitude; + const latitude = call.request.scaledObjectRef.scalerMetadata.latitude; if (!longitude || !latitude) { callback({ code: grpc.status.INVALID_ARGUMENT, - details: 'longitude and latitude must be specified', - }) + details: "longitude and latitude must be specified", + }); } else { getEarthquakeCount((err, count) => { if (err) { callback({ code: grpc.status.INTERNAL, details: err, - }) + }); } else { callback(null, { - metricValues: [{ - metricName: 'earthquakeThreshold', - metricValue: count, - }] - }) + metricValues: [ + { + metricName: "earthquakeThreshold", + metricValue: count, + }, + ], + }); } - }) + }); } - } -}) + }, +}); ``` {{< /collapsible >}} diff --git a/content/docs/2.8/migration.md b/content/docs/2.8/migration.md index 5ee8a6455..29e8408c8 100644 --- a/content/docs/2.8/migration.md +++ b/content/docs/2.8/migration.md @@ -11,15 +11,17 @@ Please note that you **can not** run both KEDA v1 and v2 on the same Kubernetes KEDA v2 is using a new API namespace for its Custom Resources Definitions (CRD): `keda.sh` instead of `keda.k8s.io` and introduces a new Custom Resource for scaling of Jobs. See full details on KEDA Custom Resources [here](../concepts/#custom-resources-crd). Here's an overview of what's changed: + - [Scaling of Deployments](#scaling-of-deployments) - [Scaling of Jobs](#scaling-of-jobs) - [Improved flexibility & usability of trigger metadata](#improved-flexibility--usability-of-trigger-metadata) - [Scalers](#scalers) - [TriggerAuthentication](#triggerauthentication) - ### Scaling of Deployments + In order to scale `Deployments` with KEDA v2, you need to do only a few modifications to existing v1 `ScaledObjects` definitions, so they comply with v2: + - Change the value of `apiVersion` property from `keda.k8s.io/v1alpha1` to `keda.sh/v1alpha1` - Rename property `spec.scaleTargetRef.deploymentName` to `spec.scaleTargetRef.name` - Rename property `spec.scaleTargetRef.containerName` to `spec.scaleTargetRef.envSourceContainerName` @@ -27,22 +29,23 @@ In order to scale `Deployments` with KEDA v2, you need to do only a few modifica Please see the examples below or refer to the full [v2 ScaledObject Specification](../concepts/scaling-deployments/#scaledobject-spec) - **Example of v1 ScaledObject** +**Example of v1 ScaledObject** + ```yaml apiVersion: keda.k8s.io/v1alpha1 kind: ScaledObject metadata: - name: {scaled-object-name} + name: { scaled-object-name } labels: - deploymentName: {deployment-name} + deploymentName: { deployment-name } spec: scaleTargetRef: - deploymentName: {deployment-name} - containerName: {container-name} - pollingInterval: 30 - cooldownPeriod: 300 - minReplicaCount: 0 - maxReplicaCount: 100 + deploymentName: { deployment-name } + containerName: { container-name } + pollingInterval: 30 + cooldownPeriod: 300 + minReplicaCount: 0 + maxReplicaCount: 100 triggers: # {list of triggers to activate the deployment} ``` @@ -50,18 +53,18 @@ spec: **Example of v2 ScaledObject** ```yaml -apiVersion: keda.sh/v1alpha1 # <--- Property value was changed +apiVersion: keda.sh/v1alpha1 # <--- Property value was changed kind: ScaledObject -metadata: # <--- labels.deploymentName is not needed - name: {scaled-object-name} +metadata: # <--- labels.deploymentName is not needed + name: { scaled-object-name } spec: scaleTargetRef: - name: {deployment-name} # <--- Property name was changed - envSourceContainerName: {container-name} # <--- Property name was changed - pollingInterval: 30 - cooldownPeriod: 300 - minReplicaCount: 0 - maxReplicaCount: 100 + name: { deployment-name } # <--- Property name was changed + envSourceContainerName: { container-name } # <--- Property name was changed + pollingInterval: 30 + cooldownPeriod: 300 + minReplicaCount: 0 + maxReplicaCount: 100 triggers: # {list of triggers to activate the deployment} ``` @@ -69,34 +72,36 @@ spec: ### Scaling of Jobs In order to scale `Jobs` with KEDA v2, you need to do only a few modifications to existing v1 `ScaledObjects` definitions, so they comply with v2: + - Change the value of `apiVersion` property from `keda.k8s.io/v1alpha1` to `keda.sh/v1alpha1` - Change the value of `kind` property from `ScaledObject` to `ScaledJob` - Remove property `spec.scaleType` -- Remove properties `spec.cooldownPeriod` and `spec.minReplicaCount` +- Remove properties `spec.cooldownPeriod` and `spec.minReplicaCount` You can configure `successfulJobsHistoryLimit` and `failedJobsHistoryLimit`. They will remove the old job histories automatically. Please see the examples below or refer to the full [v2 ScaledJob Specification](../concepts/scaling-jobs/#scaledjob-spec) **Example of v1 ScaledObject for Jobs scaling** + ```yaml apiVersion: keda.k8s.io/v1alpha1 kind: ScaledObject metadata: - name: {scaled-object-name} + name: { scaled-object-name } spec: scaleType: job jobTargetRef: - parallelism: 1 + parallelism: 1 completions: 1 activeDeadlineSeconds: 600 - backoffLimit: 6 + backoffLimit: 6 template: # {job template} - pollingInterval: 30 - cooldownPeriod: 300 - minReplicaCount: 0 - maxReplicaCount: 100 + pollingInterval: 30 + cooldownPeriod: 300 + minReplicaCount: 0 + maxReplicaCount: 100 triggers: # {list of triggers to create jobs} ``` @@ -104,21 +109,21 @@ spec: **Example of v2 ScaledJob** ```yaml -apiVersion: keda.sh/v1alpha1 # <--- Property value was changed -kind: ScaledJob # <--- Property value was changed +apiVersion: keda.sh/v1alpha1 # <--- Property value was changed +kind: ScaledJob # <--- Property value was changed metadata: - name: {scaled-job-name} -spec: # <--- spec.scaleType is not needed + name: { scaled-job-name } +spec: # <--- spec.scaleType is not needed jobTargetRef: - parallelism: 1 - completions: 1 + parallelism: 1 + completions: 1 activeDeadlineSeconds: 600 backoffLimit: 6 template: # {job template} - pollingInterval: 30 # <--- spec.cooldownPeriod and spec.minReplicaCount are not needed - successfulJobsHistoryLimit: 5 # <--- property is added - failedJobsHistoryLimit: 5 # <--- Property is added + pollingInterval: 30 # <--- spec.cooldownPeriod and spec.minReplicaCount are not needed + successfulJobsHistoryLimit: 5 # <--- property is added + failedJobsHistoryLimit: 5 # <--- Property is added maxReplicaCount: 100 triggers: # {list of triggers to create jobs} @@ -132,68 +137,74 @@ We've introduced more options to configure trigger metadata to give users more f Here's an overview: -| Scaler | 1.x | 2.0 | -|--------|--------|--------| -| `azure-blob` | `connection` (**Default**: `AzureWebJobsStorage`) | `connectionFromEnv` | -| `azure-monitor` | `activeDirectoryClientId`
`activeDirectoryClientPassword` | `activeDirectoryClientId`
`activeDirectoryClientIdFromEnv`
`activeDirectoryClientPasswordFromEnv` | -| `azure-queue` | `connection` (**Default**: AzureWebJobsStorage) | `connectionFromEnv` | -| `azure-servicebus` | `connection` | `connectionFromEnv` | -| `azure-eventhub` | `storageConnection` (**Default**: `AzureWebJobsStorage`)
`connection` (**Default**: `EventHub`) | `storageConnectionFromEnv`
`connectionFromEnv` | -| `aws-cloudwatch` | `awsAccessKeyID` (**Default**: `AWS_ACCESS_KEY_ID`)
`awsSecretAccessKey` (**Default**: `AWS_SECRET_ACCESS_KEY`) | `awsAccessKeyID`
`awsAccessKeyIDFromEnv`
`awsSecretAccessKeyFromEnv` | -| `aws-kinesis-stream` | `awsAccessKeyID` (**Default**: `AWS_ACCESS_KEY_ID`)
`awsSecretAccessKey` (**Default**: `AWS_SECRET_ACCESS_KEY`) | `awsAccessKeyID`
`awsAccessKeyIDFromEnv`
`awsSecretAccessKeyFromEnv` | -| `aws-sqs-queue` | `awsAccessKeyID` (**Default**: `AWS_ACCESS_KEY_ID`)
`awsSecretAccessKey` (**Default**: `AWS_SECRET_ACCESS_KEY`) | `awsAccessKeyID`
`awsAccessKeyIDFromEnv`
`awsSecretAccessKeyFromEnv` | -| `kafka` | _(none)_ | _(none)_ -| `rabbitmq` | `apiHost`
`host` | ~~`apiHost`~~
`host`
`hostFromEnv` | -| `prometheus` | _(none)_ | _(none)_ | -| `cron` | _(none)_ | _(none)_ | -| `redis` | `address`
`host`
`port`
`password` | `address`
`addressFromEnv`
`host`
`hostFromEnv`
~~`port`~~
`passwordFromEnv` -| `redis-streams` | `address`
`host`
`port`
`password` | `address`
`addressFromEnv`
`host`
`hostFromEnv`
~~`port`~~
`passwordFromEnv` -| `gcp-pubsub` | `credentials` | `credentialsFromEnv` | -| `external` | _(any matching value)_ | _(any matching value with `FromEnv` suffix)_ -| `liiklus` | _(none)_ | _(none)_ | -| `stan` | _(none)_ | _(none)_ | -| `huawei-cloudeye` | | _(none)_ | _(none)_ | -| `postgresql` | `connection`
`password` | `connectionFromEnv`
`passwordFromEnv` | -| `mysql` | `connectionString`
`password` | `connectionStringFromEnv`
`passwordFromEnv` | +| Scaler | 1.x | 2.0 | +| -------------------- | --------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------- | -------- | +| `azure-blob` | `connection` (**Default**: `AzureWebJobsStorage`) | `connectionFromEnv` | +| `azure-monitor` | `activeDirectoryClientId` `activeDirectoryClientPassword` | `activeDirectoryClientId` `activeDirectoryClientIdFromEnv` `activeDirectoryClientPasswordFromEnv` | +| `azure-queue` | `connection` (**Default**: AzureWebJobsStorage) | `connectionFromEnv` | +| `azure-servicebus` | `connection` | `connectionFromEnv` | +| `azure-eventhub` | `storageConnection` (**Default**: `AzureWebJobsStorage`) `connection` (**Default**: `EventHub`) | `storageConnectionFromEnv` `connectionFromEnv` | +| `aws-cloudwatch` | `awsAccessKeyID` (**Default**: `AWS_ACCESS_KEY_ID`) `awsSecretAccessKey` (**Default**: `AWS_SECRET_ACCESS_KEY`) | `awsAccessKeyID` `awsAccessKeyIDFromEnv` `awsSecretAccessKeyFromEnv` | +| `aws-kinesis-stream` | `awsAccessKeyID` (**Default**: `AWS_ACCESS_KEY_ID`) `awsSecretAccessKey` (**Default**: `AWS_SECRET_ACCESS_KEY`) | `awsAccessKeyID` `awsAccessKeyIDFromEnv` `awsSecretAccessKeyFromEnv` | +| `aws-sqs-queue` | `awsAccessKeyID` (**Default**: `AWS_ACCESS_KEY_ID`) `awsSecretAccessKey` (**Default**: `AWS_SECRET_ACCESS_KEY`) | `awsAccessKeyID` `awsAccessKeyIDFromEnv` `awsSecretAccessKeyFromEnv` | +| `kafka` | _(none)_ | _(none)_ | +| `rabbitmq` | `apiHost` `host` | ~~`apiHost`~~ `host` `hostFromEnv` | +| `prometheus` | _(none)_ | _(none)_ | +| `cron` | _(none)_ | _(none)_ | +| `redis` | `address` `host` `port` `password` | `address` `addressFromEnv` `host` `hostFromEnv` ~~`port`~~ `passwordFromEnv` | +| `redis-streams` | `address` `host` `port` `password` | `address` `addressFromEnv` `host` `hostFromEnv` ~~`port`~~ `passwordFromEnv` | +| `gcp-pubsub` | `credentials` | `credentialsFromEnv` | +| `external` | _(any matching value)_ | _(any matching value with `FromEnv` suffix)_ | +| `liiklus` | _(none)_ | _(none)_ | +| `stan` | _(none)_ | _(none)_ | +| `huawei-cloudeye` | | _(none)_ | _(none)_ | +| `postgresql` | `connection` `password` | `connectionFromEnv` `passwordFromEnv` | +| `mysql` | `connectionString` `password` | `connectionStringFromEnv` `passwordFromEnv` | ### Scalers **Azure Service Bus** - - `queueLength` was renamed to `messageCount` + +- `queueLength` was renamed to `messageCount` **Kafka** - - `authMode` property was replaced with `sasl` and `tls` properties. Please refer [documentation](../scalers/apache-kafka/#authentication-parameters) for Kafka Authentication Parameters details. + +- `authMode` property was replaced with `sasl` and `tls` properties. Please refer [documentation](../scalers/apache-kafka/#authentication-parameters) for Kafka Authentication Parameters details. **RabbitMQ** In KEDA 2.0 the RabbitMQ scaler has only `host` parameter, and the protocol for communication can be specified by -`protocol` (http or amqp). The default value is `amqp`. The behavior changes only for scalers that were using HTTP -protocol. +`protocol` (http or amqp). The default value is `amqp`. The behavior changes only for scalers that were using HTTP +protocol. Example of RabbitMQ trigger before 2.0: + ```yaml triggers: -- type: rabbitmq - metadata: - queueLength: '20' - queueName: testqueue - includeUnacked: 'true' - apiHost: 'https://guest:password@localhost:443/vhostname' -``` + - type: rabbitmq + metadata: + queueLength: "20" + queueName: testqueue + includeUnacked: "true" + apiHost: "https://guest:password@localhost:443/vhostname" +``` The same trigger in 2.0: + ```yaml triggers: -- type: rabbitmq - metadata: - queueLength: '20' - queueName: testqueue - protocol: 'http' - host: 'https://guest:password@localhost:443/vhostname' -``` + - type: rabbitmq + metadata: + queueLength: "20" + queueName: testqueue + protocol: "http" + host: "https://guest:password@localhost:443/vhostname" +``` ### TriggerAuthentication + In order to use Authentication via `TriggerAuthentication` with KEDA v2, you need to change: + - Change the value of `apiVersion` property from `keda.k8s.io/v1alpha1` to `keda.sh/v1alpha1` For more details please refer to the full diff --git a/content/docs/2.8/operate/cluster.md b/content/docs/2.8/operate/cluster.md index 47365ca41..01a4f368d 100644 --- a/content/docs/2.8/operate/cluster.md +++ b/content/docs/2.8/operate/cluster.md @@ -29,12 +29,10 @@ KEDA requires to be accessible inside the cluster to be able to autoscale. Here is an overview of the required ports that need to be accessible for KEDA to work: - | Port | Why? | Remarks | | ------ | -------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `443` | Used by Kubernetes API server to get metrics | Required for all platforms because it uses Control Plane → port 443 on the Service IP range communication.

This is not applicable for Google Cloud. | +| `443` | Used by Kubernetes API server to get metrics | Required for all platforms because it uses Control Plane → port 443 on the Service IP range communication. This is not applicable for Google Cloud. | | `6443` | Used by Kubernetes API server to get metrics | Only required for Google Cloud because it uses Control Plane → port 6443 on the Pod IP range for communication | - ## High Availability @@ -42,10 +40,10 @@ KEDA does not provide full support for high-availability due to upstream limitat Here is an overview of all KEDA deployments and the HA notes: -| Deployment | Support Replicas | Note | -| -------------- | ---------------- |---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Deployment | Support Replicas | Note | +| -------------- | ---------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | Metrics Server | 1 | You can run multiple replicas of our metrics sever, and it is recommended to add the `--enable-aggregator-routing=true` CLI flag to the kube-apiserver so that requests sent to our metrics servers are load balanced. However, [you can only run one active metric server in a Kubernetes cluster serving external.metrics.k8s.io](https://github.com/kubernetes-sigs/custom-metrics-apiserver/issues/70) which has to be the KEDA metric server. | -| Operator | 2 | While you can run multiple replicas of our operator, only one operator instance will be active. The rest will be standing by, which may reduce downtime during a failure. Multiple replicas will not improve the performance of KEDA, it could only reduce a downtime during a failover. | +| Operator | 2 | While you can run multiple replicas of our operator, only one operator instance will be active. The rest will be standing by, which may reduce downtime during a failure. Multiple replicas will not improve the performance of KEDA, it could only reduce a downtime during a failover. | ## HTTP Timeouts @@ -80,16 +78,17 @@ Some scalers issue HTTP requests to external servers (i.e. cloud services). As c The Kubernetes client config used within KEDA Metrics Adapter can be adjusted by passing the following command-line flags to the binary: -| Adapter Flag | Client Config Setting | Default Value | Description | -| -------------- | ----------------------- | ------------- | -------------------------------------------------------------- | -| kube-api-qps | cfg.QPS | 20.0 | Set the QPS rate for throttling requests sent to the apiserver | -| kube-api-burst | cfg.Burst | 30 | Set the burst for throttling requests sent to the apiserver | +| Adapter Flag | Client Config Setting | Default Value | Description | +| -------------- | --------------------- | ------------- | -------------------------------------------------------------- | +| kube-api-qps | cfg.QPS | 20.0 | Set the QPS rate for throttling requests sent to the apiserver | +| kube-api-burst | cfg.Burst | 30 | Set the burst for throttling requests sent to the apiserver | ## Configure `MaxConcurrentReconciles` for Controllers To implement internal controllers KEDA uses the [controller-runtime project](https://github.com/kubernetes-sigs/controller-runtime), that enables configuration of [MaxConcurrentReconciles property](https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/controller#Options), ie. the maximum number of concurrent reconciles which can be run for a controller. KEDA Operator exposes properties for specifying `MaxConcurrentReconciles` for following controllers/reconcilers: + - `ScaledObjectReconciler` - responsible for watching and managing `ScaledObjects`, ie. validates input trigger specification, starts scaling logic and manages dependent HPA. - `ScaledJobReconciler` - responsible for watching and managing `ScaledJobs` and dependent Kubernetes Jobs @@ -97,15 +96,16 @@ KEDA Metrics Server exposes property for specifying `MaxConcurrentReconciles` fo To modify this properties you can set environment variables on both KEDA Operator and Metrics Server Deployments: -| Environment variable name | Deployment | Default Value | Affected reconciler | -| ------------------------------------- | -------------- | ------------- | -------------------------------------------------------------- | -| KEDA_SCALEDOBJECT_CTRL_MAX_RECONCILES | Operator | 5 | ScaledObjectReconciler | -| KEDA_SCALEDJOB_CTRL_MAX_RECONCILES | Operator | 1 | ScaledJobReconciler | -| KEDA_METRICS_CTRL_MAX_RECONCILES | Metrics Server | 1 | MetricsScaledObjectReconciler | +| Environment variable name | Deployment | Default Value | Affected reconciler | +| ------------------------------------- | -------------- | ------------- | ----------------------------- | +| KEDA_SCALEDOBJECT_CTRL_MAX_RECONCILES | Operator | 5 | ScaledObjectReconciler | +| KEDA_SCALEDJOB_CTRL_MAX_RECONCILES | Operator | 1 | ScaledJobReconciler | +| KEDA_METRICS_CTRL_MAX_RECONCILES | Metrics Server | 1 | MetricsScaledObjectReconciler | ## Configure Leader Election Like reconciliation, KEDA also uses the [controller-runtime project](https://github.com/kubernetes-sigs/controller-runtime) for electing the leader replica. The following properties can be configured for either the Operator and Metrics Server Deployment: + - [`LeaseDuration`](https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/manager#Options.LeaseDuration) - [`RenewDeadline`](https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/manager#Options.RenewDeadline) - [`RetryPeriod`](https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/manager#Options.RetryPeriod) @@ -128,20 +128,19 @@ By default, KEDA Metrics Server uses self-signed certificates while communicatin Certificates and CA bundle can be referenced in `args` section in KEDA Metrics Server Deployment: ```yaml -... +--- args: - - '--client-ca-file=/cabundle/service-ca.crt' - - '--tls-cert-file=/certs/tls.crt' - - '--tls-private-key-file=/certs/tls.key' -... + - "--client-ca-file=/cabundle/service-ca.crt" + - "--tls-cert-file=/certs/tls.crt" + - "--tls-private-key-file=/certs/tls.key" ``` -The custom CA bundle should be also referenced in the `v1beta1.external.metrics.k8s.io` [APIService](https://kubernetes.io/docs/reference/kubernetes-api/cluster-resources/api-service-v1/#APIServiceSpec) resource (which is created during the installation of KEDA). +The custom CA bundle should be also referenced in the `v1beta1.external.metrics.k8s.io` [APIService](https://kubernetes.io/docs/reference/kubernetes-api/cluster-resources/api-service-v1/#APIServiceSpec) resource (which is created during the installation of KEDA). You should also make sure that `insecureSkipTLSVerify` is not set to `true`. ```yaml -... +--- spec: service: namespace: keda @@ -153,5 +152,4 @@ spec: YOURCABUNDLE... groupPriorityMinimum: 100 versionPriority: 100 -... ``` diff --git a/content/docs/2.9/concepts/external-scalers.md b/content/docs/2.9/concepts/external-scalers.md index 5badde098..aa00a18a0 100644 --- a/content/docs/2.9/concepts/external-scalers.md +++ b/content/docs/2.9/concepts/external-scalers.md @@ -9,7 +9,7 @@ Built-in scalers run in the KEDA process/pod, while external scalers require an This document describes the external scaler interfaces and how to implement them in Go, Node, and .NET; however for more details on GRPC refer to [the official GRPC documentation](https://grpc.io/docs/) ->Want to learn about existing external scalers? Explore our [external scaler community](https://github.com/kedacore/external-scalers). +> Want to learn about existing external scalers? Explore our [external scaler community](https://github.com/kedacore/external-scalers). ## Overview @@ -45,7 +45,7 @@ The `Scaler` interface defines 3 methods: - `GetMetricSpecForScaling` returns the target value for the HPA definition for the scaler. For more details refer to [Implementing `GetMetricSpec`](#5-implementing-getmetricspec). - `GetMetricsAndActivity` is called on `pollingInterval` and. When activity returns `true`, KEDA will scale to what is returned by the metric limited by `maxReplicaCount` on the ScaledObject/ScaledJob. When `false` is returned, KEDA will scale to `minReplicaCount` or optionally `idleReplicaCount`. More details around the defaults and how these options work together can be found on the [ScaledObjectSpec](https://keda.sh/docs/latest/concepts/scaling-deployments/#scaledobject-spec). -> Refer to the [HPA docs](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale-walkthrough/) for how HPA calculates `replicaCount` based on metric value and target value. KEDA supports both `AverageValue` and `Value` metric target types for external metrics. When `AverageValue` (the default metric type) is used, the metric value returned by the external scaler will be divided by the number of replicas. + > Refer to the [HPA docs](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale-walkthrough/) for how HPA calculates `replicaCount` based on metric value and target value. KEDA supports both `AverageValue` and `Value` metric target types for external metrics. When `AverageValue` (the default metric type) is used, the metric value returned by the external scaler will be divided by the number of replicas. The `PushScaler` interface adds a `Run` method. This method receives a push channel (`active`), on which the scaler can send `true` at any time. The purpose of this mechanism is to initiate a scaling operation independently from `pollingInterval`. @@ -115,7 +115,7 @@ KEDA will attempt a GRPC connection to `service-address.svc.local:9090` immediat } ``` ->**Note**: KEDA will issue all of the above RPC calls except `StreamIsActive` if `spec.triggers.type` is `external`. It _must_ be `external-push` for `StreamIsActive` to be called. +> **Note**: KEDA will issue all of the above RPC calls except `StreamIsActive` if `spec.triggers.type` is `external`. It _must_ be `external-push` for `StreamIsActive` to be called. ## Implementing KEDA external scaler GRPC interface @@ -182,8 +182,6 @@ npm install --save grpc request {{< /collapsible >}} -
- #### 3. Implementing `IsActive` Just like `IsActive(ctx context.Context) (bool, error)` in the go interface, the `IsActive` method in the GRPC interface is called every `pollingInterval` with a `ScaledObjectRef` object that contains the scaledObject name, namespace, and scaler metadata. @@ -310,70 +308,71 @@ public class ExternalScalerService : ExternalScaler.ExternalScalerBase Put the following code into your `index.js` file: ```js -const grpc = require('grpc') -const request = require('request') -const externalScalerProto = grpc.load('externalscaler.proto') +const grpc = require("grpc"); +const request = require("request"); +const externalScalerProto = grpc.load("externalscaler.proto"); -const server = new grpc.Server() +const server = new grpc.Server(); server.addService(externalScalerProto.externalscaler.ExternalScaler.service, { isActive: (call, callback) => { - const longitude = call.request.scalerMetadata.longitude - const latitude = call.request.scalerMetadata.latitude + const longitude = call.request.scalerMetadata.longitude; + const latitude = call.request.scalerMetadata.latitude; if (!longitude || !latitude) { callback({ code: grpc.status.INVALID_ARGUMENT, - details: 'longitude and latitude must be specified', - }) + details: "longitude and latitude must be specified", + }); } else { - const now = new Date() - const yesterday = new Date(new Date().setDate(new Date().getDate()-1)); - - const startTime = `${yesterday.getUTCFullYear()}-${yesterday.getUTCMonth()}-${yesterday.getUTCDay()}` - const endTime = `${now.getUTCFullYear()}-${now.getUTCMonth()}-${now.getUTCDay()}` - const radiusKm = 500 - const query = `format=geojson&starttime=${startTime}&endtime=${endTime}&longitude=${longitude}&latitude=${latitude}&maxradiuskm=${radiusKm}` - - request.get({ - url: `https://earthquake.usgs.gov/fdsnws/event/1/query?${query}`, - json: true, - }, (err, resp, data) => { - if (err) { - callback({ - code: grpc.status.INTERNAL, - details: err, - }) - } else if (resp.statusCode !== 200) { - callback({ - code: grpc.status.INTERNAL, - details: `expected status 200, got ${resp.statusCode}` - }) - } else { - // count how many earthquakes with mag > 1.0 - let count = 0 - data.features.forEach(i => { - if (i.properties.mag > 1.0) { - count++ - } - }) - callback(null, { - result: count > 2, - }) + const now = new Date(); + const yesterday = new Date(new Date().setDate(new Date().getDate() - 1)); + + const startTime = `${yesterday.getUTCFullYear()}-${yesterday.getUTCMonth()}-${yesterday.getUTCDay()}`; + const endTime = `${now.getUTCFullYear()}-${now.getUTCMonth()}-${now.getUTCDay()}`; + const radiusKm = 500; + const query = `format=geojson&starttime=${startTime}&endtime=${endTime}&longitude=${longitude}&latitude=${latitude}&maxradiuskm=${radiusKm}`; + + request.get( + { + url: `https://earthquake.usgs.gov/fdsnws/event/1/query?${query}`, + json: true, + }, + (err, resp, data) => { + if (err) { + callback({ + code: grpc.status.INTERNAL, + details: err, + }); + } else if (resp.statusCode !== 200) { + callback({ + code: grpc.status.INTERNAL, + details: `expected status 200, got ${resp.statusCode}`, + }); + } else { + // count how many earthquakes with mag > 1.0 + let count = 0; + data.features.forEach((i) => { + if (i.properties.mag > 1.0) { + count++; + } + }); + callback(null, { + result: count > 2, + }); + } } - }) + ); } - } -}) + }, +}); -server.bind('127.0.0.1:9090', grpc.ServerCredentials.createInsecure()) -console.log('Server listening on 127.0.0.1:9090') +server.bind("127.0.0.1:9090", grpc.ServerCredentials.createInsecure()); +console.log("Server listening on 127.0.0.1:9090"); -server.start() +server.start(); ``` {{< /collapsible >}} -
- #### 4. Implementing `StreamIsActive` Unlike `IsActive`, `StreamIsActive` is called once when KEDA reconciles the `ScaledObject`, and expects the external scaler to maintain a long-lived connection and push `IsActiveResponse` objects whenever the scaler needs KEDA to activate the deployment. @@ -450,38 +449,36 @@ public override async Task StreamIsActive(ScaledObjectRef request, IServerStream server.addService(externalScalerProto.externalscaler.ExternalScaler.service, { // ... streamIsActive: (call, callback) => { - const longitude = call.request.scalerMetadata.longitude - const latitude = call.request.scalerMetadata.latitude + const longitude = call.request.scalerMetadata.longitude; + const latitude = call.request.scalerMetadata.latitude; if (!longitude || !latitude) { callback({ code: grpc.status.INVALID_ARGUMENT, - details: 'longitude and latitude must be specified', - }) + details: "longitude and latitude must be specified", + }); } else { const interval = setInterval(() => { getEarthquakeCount((err, count) => { if (err) { - console.error(err) + console.error(err); } else if (count > 2) { call.write({ result: true, - }) + }); } - }) + }); }, 1000 * 60 * 60); - call.on('end', () => { - clearInterval(interval) - }) + call.on("end", () => { + clearInterval(interval); + }); } - } -}) + }, +}); ``` {{< /collapsible >}} -
- #### 5. Implementing `GetMetricSpec` `GetMetricSpec` returns the `target` value for [the HPA definition for the scaler](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale-walkthrough/). This scaler will define a static target of 10, but the threshold value is often specified in the metadata for other scalers. @@ -527,19 +524,19 @@ server.addService(externalScalerProto.externalscaler.ExternalScaler.service, { // ... getMetricSpec: (call, callback) => { callback(null, { - metricSpecs: [{ - metricName: 'earthquakeThreshold', - targetSize: 10, - }] - }) - } -}) + metricSpecs: [ + { + metricName: "earthquakeThreshold", + targetSize: 10, + }, + ], + }); + }, +}); ``` {{< /collapsible >}} -
- #### 6. Implementing `GetMetrics` `GetMetrics` returns the value of the metric referred to from `GetMetricSpec`, in this example it's `earthquakeThreshold`. @@ -606,32 +603,34 @@ public override async Task GetMetrics(GetMetricsRequest requ server.addService(externalScalerProto.externalscaler.ExternalScaler.service, { // ... getMetrics: (call, callback) => { - const longitude = call.request.scaledObjectRef.scalerMetadata.longitude - const latitude = call.request.scaledObjectRef.scalerMetadata.latitude + const longitude = call.request.scaledObjectRef.scalerMetadata.longitude; + const latitude = call.request.scaledObjectRef.scalerMetadata.latitude; if (!longitude || !latitude) { callback({ code: grpc.status.INVALID_ARGUMENT, - details: 'longitude and latitude must be specified', - }) + details: "longitude and latitude must be specified", + }); } else { getEarthquakeCount((err, count) => { if (err) { callback({ code: grpc.status.INTERNAL, details: err, - }) + }); } else { callback(null, { - metricValues: [{ - metricName: 'earthquakeThreshold', - metricValue: count, - }] - }) + metricValues: [ + { + metricName: "earthquakeThreshold", + metricValue: count, + }, + ], + }); } - }) + }); } - } -}) + }, +}); ``` {{< /collapsible >}} diff --git a/content/docs/2.9/migration.md b/content/docs/2.9/migration.md index 5ee8a6455..29e8408c8 100644 --- a/content/docs/2.9/migration.md +++ b/content/docs/2.9/migration.md @@ -11,15 +11,17 @@ Please note that you **can not** run both KEDA v1 and v2 on the same Kubernetes KEDA v2 is using a new API namespace for its Custom Resources Definitions (CRD): `keda.sh` instead of `keda.k8s.io` and introduces a new Custom Resource for scaling of Jobs. See full details on KEDA Custom Resources [here](../concepts/#custom-resources-crd). Here's an overview of what's changed: + - [Scaling of Deployments](#scaling-of-deployments) - [Scaling of Jobs](#scaling-of-jobs) - [Improved flexibility & usability of trigger metadata](#improved-flexibility--usability-of-trigger-metadata) - [Scalers](#scalers) - [TriggerAuthentication](#triggerauthentication) - ### Scaling of Deployments + In order to scale `Deployments` with KEDA v2, you need to do only a few modifications to existing v1 `ScaledObjects` definitions, so they comply with v2: + - Change the value of `apiVersion` property from `keda.k8s.io/v1alpha1` to `keda.sh/v1alpha1` - Rename property `spec.scaleTargetRef.deploymentName` to `spec.scaleTargetRef.name` - Rename property `spec.scaleTargetRef.containerName` to `spec.scaleTargetRef.envSourceContainerName` @@ -27,22 +29,23 @@ In order to scale `Deployments` with KEDA v2, you need to do only a few modifica Please see the examples below or refer to the full [v2 ScaledObject Specification](../concepts/scaling-deployments/#scaledobject-spec) - **Example of v1 ScaledObject** +**Example of v1 ScaledObject** + ```yaml apiVersion: keda.k8s.io/v1alpha1 kind: ScaledObject metadata: - name: {scaled-object-name} + name: { scaled-object-name } labels: - deploymentName: {deployment-name} + deploymentName: { deployment-name } spec: scaleTargetRef: - deploymentName: {deployment-name} - containerName: {container-name} - pollingInterval: 30 - cooldownPeriod: 300 - minReplicaCount: 0 - maxReplicaCount: 100 + deploymentName: { deployment-name } + containerName: { container-name } + pollingInterval: 30 + cooldownPeriod: 300 + minReplicaCount: 0 + maxReplicaCount: 100 triggers: # {list of triggers to activate the deployment} ``` @@ -50,18 +53,18 @@ spec: **Example of v2 ScaledObject** ```yaml -apiVersion: keda.sh/v1alpha1 # <--- Property value was changed +apiVersion: keda.sh/v1alpha1 # <--- Property value was changed kind: ScaledObject -metadata: # <--- labels.deploymentName is not needed - name: {scaled-object-name} +metadata: # <--- labels.deploymentName is not needed + name: { scaled-object-name } spec: scaleTargetRef: - name: {deployment-name} # <--- Property name was changed - envSourceContainerName: {container-name} # <--- Property name was changed - pollingInterval: 30 - cooldownPeriod: 300 - minReplicaCount: 0 - maxReplicaCount: 100 + name: { deployment-name } # <--- Property name was changed + envSourceContainerName: { container-name } # <--- Property name was changed + pollingInterval: 30 + cooldownPeriod: 300 + minReplicaCount: 0 + maxReplicaCount: 100 triggers: # {list of triggers to activate the deployment} ``` @@ -69,34 +72,36 @@ spec: ### Scaling of Jobs In order to scale `Jobs` with KEDA v2, you need to do only a few modifications to existing v1 `ScaledObjects` definitions, so they comply with v2: + - Change the value of `apiVersion` property from `keda.k8s.io/v1alpha1` to `keda.sh/v1alpha1` - Change the value of `kind` property from `ScaledObject` to `ScaledJob` - Remove property `spec.scaleType` -- Remove properties `spec.cooldownPeriod` and `spec.minReplicaCount` +- Remove properties `spec.cooldownPeriod` and `spec.minReplicaCount` You can configure `successfulJobsHistoryLimit` and `failedJobsHistoryLimit`. They will remove the old job histories automatically. Please see the examples below or refer to the full [v2 ScaledJob Specification](../concepts/scaling-jobs/#scaledjob-spec) **Example of v1 ScaledObject for Jobs scaling** + ```yaml apiVersion: keda.k8s.io/v1alpha1 kind: ScaledObject metadata: - name: {scaled-object-name} + name: { scaled-object-name } spec: scaleType: job jobTargetRef: - parallelism: 1 + parallelism: 1 completions: 1 activeDeadlineSeconds: 600 - backoffLimit: 6 + backoffLimit: 6 template: # {job template} - pollingInterval: 30 - cooldownPeriod: 300 - minReplicaCount: 0 - maxReplicaCount: 100 + pollingInterval: 30 + cooldownPeriod: 300 + minReplicaCount: 0 + maxReplicaCount: 100 triggers: # {list of triggers to create jobs} ``` @@ -104,21 +109,21 @@ spec: **Example of v2 ScaledJob** ```yaml -apiVersion: keda.sh/v1alpha1 # <--- Property value was changed -kind: ScaledJob # <--- Property value was changed +apiVersion: keda.sh/v1alpha1 # <--- Property value was changed +kind: ScaledJob # <--- Property value was changed metadata: - name: {scaled-job-name} -spec: # <--- spec.scaleType is not needed + name: { scaled-job-name } +spec: # <--- spec.scaleType is not needed jobTargetRef: - parallelism: 1 - completions: 1 + parallelism: 1 + completions: 1 activeDeadlineSeconds: 600 backoffLimit: 6 template: # {job template} - pollingInterval: 30 # <--- spec.cooldownPeriod and spec.minReplicaCount are not needed - successfulJobsHistoryLimit: 5 # <--- property is added - failedJobsHistoryLimit: 5 # <--- Property is added + pollingInterval: 30 # <--- spec.cooldownPeriod and spec.minReplicaCount are not needed + successfulJobsHistoryLimit: 5 # <--- property is added + failedJobsHistoryLimit: 5 # <--- Property is added maxReplicaCount: 100 triggers: # {list of triggers to create jobs} @@ -132,68 +137,74 @@ We've introduced more options to configure trigger metadata to give users more f Here's an overview: -| Scaler | 1.x | 2.0 | -|--------|--------|--------| -| `azure-blob` | `connection` (**Default**: `AzureWebJobsStorage`) | `connectionFromEnv` | -| `azure-monitor` | `activeDirectoryClientId`
`activeDirectoryClientPassword` | `activeDirectoryClientId`
`activeDirectoryClientIdFromEnv`
`activeDirectoryClientPasswordFromEnv` | -| `azure-queue` | `connection` (**Default**: AzureWebJobsStorage) | `connectionFromEnv` | -| `azure-servicebus` | `connection` | `connectionFromEnv` | -| `azure-eventhub` | `storageConnection` (**Default**: `AzureWebJobsStorage`)
`connection` (**Default**: `EventHub`) | `storageConnectionFromEnv`
`connectionFromEnv` | -| `aws-cloudwatch` | `awsAccessKeyID` (**Default**: `AWS_ACCESS_KEY_ID`)
`awsSecretAccessKey` (**Default**: `AWS_SECRET_ACCESS_KEY`) | `awsAccessKeyID`
`awsAccessKeyIDFromEnv`
`awsSecretAccessKeyFromEnv` | -| `aws-kinesis-stream` | `awsAccessKeyID` (**Default**: `AWS_ACCESS_KEY_ID`)
`awsSecretAccessKey` (**Default**: `AWS_SECRET_ACCESS_KEY`) | `awsAccessKeyID`
`awsAccessKeyIDFromEnv`
`awsSecretAccessKeyFromEnv` | -| `aws-sqs-queue` | `awsAccessKeyID` (**Default**: `AWS_ACCESS_KEY_ID`)
`awsSecretAccessKey` (**Default**: `AWS_SECRET_ACCESS_KEY`) | `awsAccessKeyID`
`awsAccessKeyIDFromEnv`
`awsSecretAccessKeyFromEnv` | -| `kafka` | _(none)_ | _(none)_ -| `rabbitmq` | `apiHost`
`host` | ~~`apiHost`~~
`host`
`hostFromEnv` | -| `prometheus` | _(none)_ | _(none)_ | -| `cron` | _(none)_ | _(none)_ | -| `redis` | `address`
`host`
`port`
`password` | `address`
`addressFromEnv`
`host`
`hostFromEnv`
~~`port`~~
`passwordFromEnv` -| `redis-streams` | `address`
`host`
`port`
`password` | `address`
`addressFromEnv`
`host`
`hostFromEnv`
~~`port`~~
`passwordFromEnv` -| `gcp-pubsub` | `credentials` | `credentialsFromEnv` | -| `external` | _(any matching value)_ | _(any matching value with `FromEnv` suffix)_ -| `liiklus` | _(none)_ | _(none)_ | -| `stan` | _(none)_ | _(none)_ | -| `huawei-cloudeye` | | _(none)_ | _(none)_ | -| `postgresql` | `connection`
`password` | `connectionFromEnv`
`passwordFromEnv` | -| `mysql` | `connectionString`
`password` | `connectionStringFromEnv`
`passwordFromEnv` | +| Scaler | 1.x | 2.0 | +| -------------------- | --------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------- | -------- | +| `azure-blob` | `connection` (**Default**: `AzureWebJobsStorage`) | `connectionFromEnv` | +| `azure-monitor` | `activeDirectoryClientId` `activeDirectoryClientPassword` | `activeDirectoryClientId` `activeDirectoryClientIdFromEnv` `activeDirectoryClientPasswordFromEnv` | +| `azure-queue` | `connection` (**Default**: AzureWebJobsStorage) | `connectionFromEnv` | +| `azure-servicebus` | `connection` | `connectionFromEnv` | +| `azure-eventhub` | `storageConnection` (**Default**: `AzureWebJobsStorage`) `connection` (**Default**: `EventHub`) | `storageConnectionFromEnv` `connectionFromEnv` | +| `aws-cloudwatch` | `awsAccessKeyID` (**Default**: `AWS_ACCESS_KEY_ID`) `awsSecretAccessKey` (**Default**: `AWS_SECRET_ACCESS_KEY`) | `awsAccessKeyID` `awsAccessKeyIDFromEnv` `awsSecretAccessKeyFromEnv` | +| `aws-kinesis-stream` | `awsAccessKeyID` (**Default**: `AWS_ACCESS_KEY_ID`) `awsSecretAccessKey` (**Default**: `AWS_SECRET_ACCESS_KEY`) | `awsAccessKeyID` `awsAccessKeyIDFromEnv` `awsSecretAccessKeyFromEnv` | +| `aws-sqs-queue` | `awsAccessKeyID` (**Default**: `AWS_ACCESS_KEY_ID`) `awsSecretAccessKey` (**Default**: `AWS_SECRET_ACCESS_KEY`) | `awsAccessKeyID` `awsAccessKeyIDFromEnv` `awsSecretAccessKeyFromEnv` | +| `kafka` | _(none)_ | _(none)_ | +| `rabbitmq` | `apiHost` `host` | ~~`apiHost`~~ `host` `hostFromEnv` | +| `prometheus` | _(none)_ | _(none)_ | +| `cron` | _(none)_ | _(none)_ | +| `redis` | `address` `host` `port` `password` | `address` `addressFromEnv` `host` `hostFromEnv` ~~`port`~~ `passwordFromEnv` | +| `redis-streams` | `address` `host` `port` `password` | `address` `addressFromEnv` `host` `hostFromEnv` ~~`port`~~ `passwordFromEnv` | +| `gcp-pubsub` | `credentials` | `credentialsFromEnv` | +| `external` | _(any matching value)_ | _(any matching value with `FromEnv` suffix)_ | +| `liiklus` | _(none)_ | _(none)_ | +| `stan` | _(none)_ | _(none)_ | +| `huawei-cloudeye` | | _(none)_ | _(none)_ | +| `postgresql` | `connection` `password` | `connectionFromEnv` `passwordFromEnv` | +| `mysql` | `connectionString` `password` | `connectionStringFromEnv` `passwordFromEnv` | ### Scalers **Azure Service Bus** - - `queueLength` was renamed to `messageCount` + +- `queueLength` was renamed to `messageCount` **Kafka** - - `authMode` property was replaced with `sasl` and `tls` properties. Please refer [documentation](../scalers/apache-kafka/#authentication-parameters) for Kafka Authentication Parameters details. + +- `authMode` property was replaced with `sasl` and `tls` properties. Please refer [documentation](../scalers/apache-kafka/#authentication-parameters) for Kafka Authentication Parameters details. **RabbitMQ** In KEDA 2.0 the RabbitMQ scaler has only `host` parameter, and the protocol for communication can be specified by -`protocol` (http or amqp). The default value is `amqp`. The behavior changes only for scalers that were using HTTP -protocol. +`protocol` (http or amqp). The default value is `amqp`. The behavior changes only for scalers that were using HTTP +protocol. Example of RabbitMQ trigger before 2.0: + ```yaml triggers: -- type: rabbitmq - metadata: - queueLength: '20' - queueName: testqueue - includeUnacked: 'true' - apiHost: 'https://guest:password@localhost:443/vhostname' -``` + - type: rabbitmq + metadata: + queueLength: "20" + queueName: testqueue + includeUnacked: "true" + apiHost: "https://guest:password@localhost:443/vhostname" +``` The same trigger in 2.0: + ```yaml triggers: -- type: rabbitmq - metadata: - queueLength: '20' - queueName: testqueue - protocol: 'http' - host: 'https://guest:password@localhost:443/vhostname' -``` + - type: rabbitmq + metadata: + queueLength: "20" + queueName: testqueue + protocol: "http" + host: "https://guest:password@localhost:443/vhostname" +``` ### TriggerAuthentication + In order to use Authentication via `TriggerAuthentication` with KEDA v2, you need to change: + - Change the value of `apiVersion` property from `keda.k8s.io/v1alpha1` to `keda.sh/v1alpha1` For more details please refer to the full diff --git a/content/docs/2.9/operate/cluster.md b/content/docs/2.9/operate/cluster.md index 5cc6024bb..18bca5f61 100644 --- a/content/docs/2.9/operate/cluster.md +++ b/content/docs/2.9/operate/cluster.md @@ -14,11 +14,11 @@ However, maintainers can decide to extend this by supporting more minor versions As a reference, this compatibility matrix shows supported k8s versions per KEDA version: -| KEDA | Kubernetes | -|-----------|---------------| -| v2.9 | v1.23 - v1.25 | -| v2.8 | v1.17 - v1.25 | -| v2.7 | v1.17 - v1.25 | +| KEDA | Kubernetes | +| ---- | ------------- | +| v2.9 | v1.23 - v1.25 | +| v2.8 | v1.17 - v1.25 | +| v2.7 | v1.17 - v1.25 | ### Cluster Capacity @@ -39,12 +39,10 @@ KEDA requires to be accessible inside the cluster to be able to autoscale. Here is an overview of the required ports that need to be accessible for KEDA to work: - | Port | Why? | Remarks | | ------ | -------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `443` | Used by Kubernetes API server to get metrics | Required for all platforms because it uses Control Plane → port 443 on the Service IP range communication.

This is not applicable for Google Cloud. | +| `443` | Used by Kubernetes API server to get metrics | Required for all platforms because it uses Control Plane → port 443 on the Service IP range communication. This is not applicable for Google Cloud. | | `6443` | Used by Kubernetes API server to get metrics | Only required for Google Cloud because it uses Control Plane → port 6443 on the Pod IP range for communication | - ## High Availability @@ -52,10 +50,10 @@ KEDA does not provide full support for high-availability due to upstream limitat Here is an overview of all KEDA deployments and the HA notes: -| Deployment | Support Replicas | Note | -| -------------- | ---------------- |---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Deployment | Support Replicas | Note | +| -------------- | ---------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | Metrics Server | 1 | You can run multiple replicas of our metrics sever, and it is recommended to add the `--enable-aggregator-routing=true` CLI flag to the kube-apiserver so that requests sent to our metrics servers are load balanced. However, [you can only run one active metric server in a Kubernetes cluster serving external.metrics.k8s.io](https://github.com/kubernetes-sigs/custom-metrics-apiserver/issues/70) which has to be the KEDA metric server. | -| Operator | 2 | While you can run multiple replicas of our operator, only one operator instance will be active. The rest will be standing by, which may reduce downtime during a failure. Multiple replicas will not improve the performance of KEDA, it could only reduce a downtime during a failover. | +| Operator | 2 | While you can run multiple replicas of our operator, only one operator instance will be active. The rest will be standing by, which may reduce downtime during a failure. Multiple replicas will not improve the performance of KEDA, it could only reduce a downtime during a failover. | ## HTTP Timeouts @@ -77,7 +75,7 @@ All applicable scalers will use this timeout. Setting a per-scaler timeout is cu ## HTTP connection disable keep alive -Keep alive behaviour is enabled by default for every HTTP connection, this could stack a huge amount of connections (one per scaler) in some scenarios. +Keep alive behaviour is enabled by default for every HTTP connection, this could stack a huge amount of connections (one per scaler) in some scenarios. You can disable keep alive for every HTTP connection by adding the relevant environment variable to both the KEDA Operator, and KEDA Metrics Server deployments: @@ -103,17 +101,18 @@ Some scalers issue HTTP requests to external servers (i.e. cloud services). As c The Kubernetes client config used within KEDA Operator and KEDA Metrics Adapter can be adjusted by passing the following command-line flags to the binary: -| Adapter Flag | Client Config Setting | Default Value | Description | -| ------------------- | ----------------------- | ------------- | -------------------------------------------------------------- | -| kube-api-qps | cfg.QPS | 20.0 | Set the QPS rate for throttling requests sent to the apiserver | -| kube-api-burst | cfg.Burst | 30 | Set the burst for throttling requests sent to the apiserver | -| disable-compression | cfg.DisableCompression | true | Disable compression for response in k8s restAPI in client-go, see [this Kubernetes issue](https://github.com/kubernetes/kubernetes/issues/112296) for details | +| Adapter Flag | Client Config Setting | Default Value | Description | +| ------------------- | ---------------------- | ------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| kube-api-qps | cfg.QPS | 20.0 | Set the QPS rate for throttling requests sent to the apiserver | +| kube-api-burst | cfg.Burst | 30 | Set the burst for throttling requests sent to the apiserver | +| disable-compression | cfg.DisableCompression | true | Disable compression for response in k8s restAPI in client-go, see [this Kubernetes issue](https://github.com/kubernetes/kubernetes/issues/112296) for details | ## Configure `MaxConcurrentReconciles` for Controllers To implement internal controllers KEDA uses the [controller-runtime project](https://github.com/kubernetes-sigs/controller-runtime), that enables configuration of [MaxConcurrentReconciles property](https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/controller#Options), ie. the maximum number of concurrent reconciles which can be run for a controller. KEDA Operator exposes properties for specifying `MaxConcurrentReconciles` for following controllers/reconcilers: + - `ScaledObjectReconciler` - responsible for watching and managing `ScaledObjects`, ie. validates input trigger specification, starts scaling logic and manages dependent HPA. - `ScaledJobReconciler` - responsible for watching and managing `ScaledJobs` and dependent Kubernetes Jobs @@ -121,15 +120,16 @@ KEDA Metrics Server exposes property for specifying `MaxConcurrentReconciles` fo To modify this properties you can set environment variables on both KEDA Operator and Metrics Server Deployments: -| Environment variable name | Deployment | Default Value | Affected reconciler | -| ------------------------------------- | -------------- | ------------- | -------------------------------------------------------------- | -| KEDA_SCALEDOBJECT_CTRL_MAX_RECONCILES | Operator | 5 | ScaledObjectReconciler | -| KEDA_SCALEDJOB_CTRL_MAX_RECONCILES | Operator | 1 | ScaledJobReconciler | -| KEDA_METRICS_CTRL_MAX_RECONCILES | Metrics Server | 1 | MetricsScaledObjectReconciler | +| Environment variable name | Deployment | Default Value | Affected reconciler | +| ------------------------------------- | -------------- | ------------- | ----------------------------- | +| KEDA_SCALEDOBJECT_CTRL_MAX_RECONCILES | Operator | 5 | ScaledObjectReconciler | +| KEDA_SCALEDJOB_CTRL_MAX_RECONCILES | Operator | 1 | ScaledJobReconciler | +| KEDA_METRICS_CTRL_MAX_RECONCILES | Metrics Server | 1 | MetricsScaledObjectReconciler | ## Configure Leader Election Like reconciliation, KEDA also uses the [controller-runtime project](https://github.com/kubernetes-sigs/controller-runtime) for electing the leader replica. The following properties can be configured for either the Operator and Metrics Server Deployment: + - [`LeaseDuration`](https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/manager#Options.LeaseDuration) - [`RenewDeadline`](https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/manager#Options.RenewDeadline) - [`RetryPeriod`](https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/manager#Options.RetryPeriod) @@ -152,20 +152,19 @@ By default KEDA Metrics Server uses self-signed certificates while communicating Certificates and CA bundle can be referenced in `args` section in KEDA Metrics Server Deployment: ```yaml -... +--- args: - - '--client-ca-file=/cabundle/service-ca.crt' - - '--tls-cert-file=/certs/tls.crt' - - '--tls-private-key-file=/certs/tls.key' -... + - "--client-ca-file=/cabundle/service-ca.crt" + - "--tls-cert-file=/certs/tls.crt" + - "--tls-private-key-file=/certs/tls.key" ``` -The custom CA bundle should be also referenced in the `v1beta1.external.metrics.k8s.io` [APIService](https://kubernetes.io/docs/reference/kubernetes-api/cluster-resources/api-service-v1/#APIServiceSpec) resource (which is created during the installation of KEDA). +The custom CA bundle should be also referenced in the `v1beta1.external.metrics.k8s.io` [APIService](https://kubernetes.io/docs/reference/kubernetes-api/cluster-resources/api-service-v1/#APIServiceSpec) resource (which is created during the installation of KEDA). You should also make sure that `insecureSkipTLSVerify` is not set to `true`. ```yaml -... +--- spec: service: namespace: keda @@ -177,31 +176,34 @@ spec: YOURCABUNDLE... groupPriorityMinimum: 100 versionPriority: 100 -... ``` ## Restrict Secret Access By default, KEDA requires adding `secrets` to the cluster role as following: + ```yaml - apiGroups: - - "" + - "" resources: - - external - - pods - - secrets - - services + - external + - pods + - secrets + - services verbs: - - get - - list - - watch + - get + - list + - watch ``` + However, this might lead to security risk (especially in production environment) since it will grant permission to read `secrets` from all namespaces. To restrict `secret` access and limited to KEDA namespace, you could add `KEDA_RESTRICT_SECRET_ACCESS` as environment variable to both KEDA Operator and KEDA Metrics Server: + ```yaml env: - name: KEDA_RESTRICT_SECRET_ACCESS value: "true" ``` + This allows you to omit `secrets` from the cluster role, which will disallow `TriggerAuthentication` to be used for your triggers if the `TriggerAuthentication` is using secrets. You can, however, still use `ClusterTriggerAuthentication`. From eba4f5343112e8f0b05a3b5328b826cf70fee7e1 Mon Sep 17 00:00:00 2001 From: Dave Welsch <116022979+dwelsch-esi@users.noreply.github.com> Date: Fri, 28 Jun 2024 06:07:32 -0700 Subject: [PATCH 11/33] Added Reference section, Glossary, removed non-inclusive language. (#1389) Co-authored-by: Dave Welsch <116022979+dwelsch-esi@users.noreply.github.com> Co-authored-by: Tom Kerkhove Co-authored-by: Nate W Co-authored-by: thisisobate Signed-off-by: shubhusion --- config.toml | 2 +- content/docs/2.14/operate/cloud-events.md | 116 +++++++ content/docs/2.14/reference/_index.md | 13 + content/docs/2.14/reference/events.md | 28 ++ content/docs/{2.15 => 2.14/reference}/faq.md | 1 + content/docs/2.14/reference/glossary.md | 88 ++++++ content/docs/2.14/reference/scaledjob-spec.md | 236 ++++++++++++++ .../docs/2.14/reference/scaledobject-spec.md | 254 +++++++++++++++ content/docs/2.15/_index.md | 17 +- .../docs/2.15/authentication-providers/aws.md | 6 +- .../docs/2.15/concepts/external-scalers.md | 4 +- .../docs/2.15/concepts/scaling-deployments.md | 293 ++---------------- content/docs/2.15/concepts/scaling-jobs.md | 247 +-------------- content/docs/2.15/deploy.md | 12 +- content/docs/2.15/migration.md | 4 +- content/docs/2.15/operate/_index.md | 6 +- content/docs/2.15/operate/cloud-events.md | 116 +++++++ content/docs/2.15/operate/events.md | 144 --------- content/docs/2.15/operate/metrics-server.md | 2 +- content/docs/2.15/reference/_index.md | 13 + content/docs/2.15/reference/events.md | 28 ++ content/docs/2.15/reference/faq.md | 6 + content/docs/2.15/reference/glossary.md | 88 ++++++ content/docs/2.15/reference/scaledjob-spec.md | 236 ++++++++++++++ .../docs/2.15/reference/scaledobject-spec.md | 254 +++++++++++++++ content/docs/2.15/scalers/aws-sqs.md | 4 +- content/docs/2.15/scalers/azure-pipelines.md | 2 +- .../docs/2.15/scalers/azure-service-bus.md | 4 +- content/docs/2.15/scalers/datadog.md | 4 +- .../docs/2.15/scalers/redis-sentinel-lists.md | 8 +- .../2.15/scalers/redis-sentinel-streams.md | 10 +- 31 files changed, 1570 insertions(+), 676 deletions(-) create mode 100644 content/docs/2.14/operate/cloud-events.md create mode 100644 content/docs/2.14/reference/_index.md create mode 100644 content/docs/2.14/reference/events.md rename content/docs/{2.15 => 2.14/reference}/faq.md (72%) create mode 100644 content/docs/2.14/reference/glossary.md create mode 100644 content/docs/2.14/reference/scaledjob-spec.md create mode 100644 content/docs/2.14/reference/scaledobject-spec.md create mode 100644 content/docs/2.15/operate/cloud-events.md delete mode 100644 content/docs/2.15/operate/events.md create mode 100644 content/docs/2.15/reference/_index.md create mode 100644 content/docs/2.15/reference/events.md create mode 100644 content/docs/2.15/reference/faq.md create mode 100644 content/docs/2.15/reference/glossary.md create mode 100644 content/docs/2.15/reference/scaledjob-spec.md create mode 100644 content/docs/2.15/reference/scaledobject-spec.md diff --git a/config.toml b/config.toml index 3407abf04..fe27d32f0 100644 --- a/config.toml +++ b/config.toml @@ -188,7 +188,7 @@ parent = "docs" weight = 5 [[menu.docs]] -url = "/docs/faq/" +url = "/docs/reference/faq/" name = "FAQ" parent = "docs" weight = 6 diff --git a/content/docs/2.14/operate/cloud-events.md b/content/docs/2.14/operate/cloud-events.md new file mode 100644 index 000000000..e07dc0505 --- /dev/null +++ b/content/docs/2.14/operate/cloud-events.md @@ -0,0 +1,116 @@ ++++ +title = "CloudEvent Support" +description = "Experimental support for cloud events" +weight = 100 ++++ + +## Subscribing to events with `CloudEventSource` +`CloudEventSource` resource can be used in KEDA for subscribing to events that are emitted to the user's defined CloudEvent sink. + +> 📝 Event will be emitted to both Kubernetes Events and CloudEvents Destination if CloudEventSource resource is created. + +Here is a the schema of the `CloudEventSource` CRD: + +```yaml +apiVersion: eventing.keda.sh/v1alpha1 +kind: CloudEventSource +metadata: + name: {cloud-event-name} +spec: + clusterName: {cluster-name} #Optional. Will be used in the source/subject to specify where the event comes from. The default value is 'kubernetes-default' and it can also be set during the installation of KEDA with --k8sClusterName. This one will overwrite others if set. + authenticationRef: + name: {trigger-authentication-name} #Optional. Used to reference a `TriggerAuthentication` for authentication. + kind: TriggerAuthentication # Optional. Used to choose the authentication scopes. https://keda.sh/docs/latest/concepts/authentication/#authentication-scopes-namespace-vs-cluster + destination: + http: + uri: http://foo.bar + azureEventGridTopic: + endpoint: https://my-topic.eastus-1.eventgrid.azure.net/api/events + + eventSubscription: #Optional. Submit included/excluded event types will filter events when emitting events. + includedEventTypes: #Optional. Only events in this section will be emitted. + - keda.scaledobject.failed.v1 + excludedEventTypes: #Optional. Events in this section will not be emitted. + - keda.scaledobject.ready.v1 +``` + +In general, an event emitted by KEDA would fundamentally come down to the following structure: +```json +{ + "specversion" : "1.0", + "type" : "com.cloudeventsource.keda", + "source" : "/{cluster-name}/{keda-namespace}/keda", + "subject" : "/{cluster-name}/{namespace}/{object-type}/{object-name}", + "id" : "", + "time" : "2018-04-05T17:31:00Z", + "datacontenttype" : "application/json", + "data" : { + "reason":"", + "message":"" + } +} +``` + +## Event Sinks + +There will be multiple types of destination to emit KEDA events to. + +Here is an overview of the supported destinations: + +- [HTTP endpoint](#http-endpoint). +- [Azure Event Grid endpoint](#azure-event-grid). + +### HTTP endpoint +```yaml + destination: + http: + uri: http://foo.bar #An http endpoint that can receive cloudevent +``` + +### Azure Event Grid + +```yaml + destination: + azureEventGrid: + endpoint: foo.bar #endpoint from AzureEventGrid Topic +``` + +Authentication information must be provided by using `authenticationRef` which allows you to provide the access key or managed identity for Azure Event Grid authentication by providing a `TriggerAuthentication`. + +Here is an overview of the supported authentication types: + +#### Connection String Authentication + +- `accessKey` - Access key string for the Azure Event Grid connection auth. + +#### Pod identity based authentication +[Azure AD Workload Identity](https://azure.github.io/azure-workload-identity/docs/) providers can be used. + +```yaml +apiVersion: keda.sh/v1alpha1 +kind: TriggerAuthentication +metadata: + name: nameOfTriggerAuth + namespace: default +spec: + podIdentity: + provider: azure-workload +``` + +## Event Filter + +You can include filter(s) to define what event types you are interested in, or want to ignore. This is done by using `includedEventTypes` or `excludedEventTypes` respectively for a given sink. + +```yaml +eventSubscription: #Optional. Submit included/excluded event types will filter events when emitting events. + includedEventTypes: #Optional. Only events in this section will be emitted. + - keda.scaledobject.failed.v1 + excludedEventTypes: #Optional. Events in this section will not be emitted. + - keda.scaledobject.ready.v1 +``` + +## Supported Event List +| Event Type | Scenario Description | +|-------------------------------|-----------------------------------------------------------------------------------------------------------------------------| +| `keda.scaledobject.ready.v1` | On the first time a ScaledObject is ready, or if the previous ready condition status of the object was `Unknown` or `False` | +| `keda.scaledobject.failed.v1` | If the check validation for a ScaledObject fails | diff --git a/content/docs/2.14/reference/_index.md b/content/docs/2.14/reference/_index.md new file mode 100644 index 000000000..987370371 --- /dev/null +++ b/content/docs/2.14/reference/_index.md @@ -0,0 +1,13 @@ ++++ +title = "Reference" +weight = 2 ++++ + +Reference information for the KEDA autoscaler. + +- [ScaledObject specification](./scaledobject-spec) +- [ScaledJob specification](./scaledjob-spec) +- [Events](./events) +- [Firewall requirements](../operate/cluster#firewall) +- [FAQ](./faq.md) +- [Glossary](./glossary.md) \ No newline at end of file diff --git a/content/docs/2.14/reference/events.md b/content/docs/2.14/reference/events.md new file mode 100644 index 000000000..aa264c0d8 --- /dev/null +++ b/content/docs/2.14/reference/events.md @@ -0,0 +1,28 @@ ++++ +title = "Events reference" +description = "Kubernetes Events emitted by KEDA" +weight = 2500 ++++ + +KEDA emits the following [Kubernetes Events](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.19/#event-v1-core): + +| Event | Type | Description | +|---------------------------------------|-----------|-----------------------------------------------------------------------------------------------------------------------------| +| `ScaledObjectReady` | `Normal` | On the first time a ScaledObject is ready, or if the previous ready condition status of the object was `Unknown` or `False` | +| `ScaledJobReady` | `Normal` | On the first time a ScaledJob is ready, or if the previous ready condition status of the object was `Unknown` or `False` | +| `ScaledObjectCheckFailed` | `Warning` | If the check validation for a ScaledObject fails | | +| `ScaledJobCheckFailed` | `Warning` | If the check validation for a ScaledJob fails | | +| `ScaledObjectDeleted` | `Normal` | When a ScaledObject is deleted and removed from KEDA watch | | +| `ScaledJobDeleted` | `Normal` | When a ScaledJob is deleted and removed from KEDA watch | | +| `KEDAScalersStarted` | `Normal` | When Scalers watch loop have started for a ScaledObject or ScaledJob | | +| `KEDAScalersStopped` | `Normal` | When Scalers watch loop have stopped for a ScaledObject or a ScaledJob | | +| `KEDAScalerFailed` | `Warning` | When a Scaler fails to create or check its event source | | +| `KEDAScaleTargetActivated` | `Normal` | When the scale target (Deployment, StatefulSet, etc) of a ScaledObject is scaled to 1, triggered by {scalers1;scalers2;...} | | +| `KEDAScaleTargetDeactivated` | `Normal` | When the scale target (Deployment, StatefulSet, etc) of a ScaledObject is scaled to 0 | | +| `KEDAScaleTargetActivationFailed` | `Warning` | When KEDA fails to scale the scale target of a ScaledObject to 1 | | +| `KEDAScaleTargetDeactivationFailed` | `Warning` | When KEDA fails to scale the scale target of a ScaledObject to 0 | | +| `KEDAJobsCreated` | `Normal` | When KEDA creates jobs for a ScaledJob | | +| `TriggerAuthenticationAdded` | `Normal` | When a new TriggerAuthentication is added | | +| `TriggerAuthenticationDeleted` | `Normal` | When a TriggerAuthentication is deleted | | +| `ClusterTriggerAuthenticationAdded` | `Normal` | When a new ClusterTriggerAuthentication is added | | +| `ClusterTriggerAuthenticationDeleted` | `Normal` | When a ClusterTriggerAuthentication is deleted | | diff --git a/content/docs/2.15/faq.md b/content/docs/2.14/reference/faq.md similarity index 72% rename from content/docs/2.15/faq.md rename to content/docs/2.14/reference/faq.md index d012d6767..df75bbe78 100644 --- a/content/docs/2.15/faq.md +++ b/content/docs/2.14/reference/faq.md @@ -1,5 +1,6 @@ +++ title = "FAQ" +weight = 2000 +++ {{< faq20 >}} diff --git a/content/docs/2.14/reference/glossary.md b/content/docs/2.14/reference/glossary.md new file mode 100644 index 000000000..fa8ecf689 --- /dev/null +++ b/content/docs/2.14/reference/glossary.md @@ -0,0 +1,88 @@ ++++ +title = "Glossary" +weight = 1000 ++++ + +This document defines the various terms needed to understand the documentation and set up and use KEDA. + +## Admission Webhook + +[In Kubernetes](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/), an HTTP callback that handle admission requests. KEDA uses an admission webhook to validate and mutate ScaledObject resources. + +## Agent + +A primary role held by the KEDA operator. The Agent activates and deactivates Kubernetes Deployments to scale to and from zero. + +## Cluster + +[In Kubernetes](https://kubernetes.io/docs/reference/glossary/?fundamental=true#term-cluster), a set of one or more nodes that run containerized applications. + +## CRD + +Custom Resource Definition. [In Kubernetes](https://kubernetes.io/docs/reference/glossary/?fundamental=true#term-CustomResourceDefinition), a custom resource that extends the Kubernetes API with custom resources like ScaledObjects that have custom fields and behavior. + +## Event + +A notable occurrence captured by an event source that KEDA may use as a trigger to scale a container or deployment. + +## Event Source + +An external system like Kafka, RabbitMQ, that generates events that KEDA can monitor using a scaler. + +## Grafana + +An open-source monitoring platform that can visualize metrics collected by KEDA. + +## GRPC + +Go Remote Procedure Call. An open-source remote procedure call framework used by KEDA components to communicate. + +## HPA + +Horizontal Pod Autoscaler. Kubernetes autoscaler. By default, scales based on CPU/memory usage. KEDA uses HPA to scale Kubernetes clusters and deployments. + +## KEDA + +Kubernetes Event-Driven Autoscaling. A single-purpose, lightweight autoscaler that can scale a Kubernetes workload based on event metrics. + +## Metric + +Measurement of an event source such as queue length or response lag that KEDA uses to determine scaling. + +## OpenTelemetry + +An observability framework used by KEDA to instrument applications and collect metrics. + +## Operator + +The core KEDA component that monitors metrics and scales workloads accordingly. + +## Prometheus + +An open-source monitoring system that can scrape and store metrics from KEDA. + +## Scaled Object + +A custom resource that defines how KEDA should scale a workload based on events. + +## Scaled Job + +A custom resource KEDA uses to scale an application. + +## Scaler + +A component that integrates KEDA with a specific event source to collect metrics. + +## Stateful Set + +A Kubernetes workload with persistent data. KEDA can scale stateful sets. + +## TLS + +Transport Layer Security. KEDA uses TLS to encrypt communications between KEDA components. + +## Webhook + +An HTTP callback used to notify KEDA of events from external sources. + +[In Kubernetes](https://kubernetes.io/docs/reference/access-authn-authz/webhook/), an HTTP callback used as an event notification mechanism. diff --git a/content/docs/2.14/reference/scaledjob-spec.md b/content/docs/2.14/reference/scaledjob-spec.md new file mode 100644 index 000000000..dbc33680b --- /dev/null +++ b/content/docs/2.14/reference/scaledjob-spec.md @@ -0,0 +1,236 @@ ++++ +title = "ScaledJob specification" +weight = 4000 ++++ + +## Overview + +This specification describes the `ScaledJob` custom resource definition that defines the triggers and scaling behaviors use by KEDA + +to scale jobs. The `.spec.ScaleTargetRef` section holds the reference to the job, defined in [_scaledjob_types.go_](https://github.com/kedacore/keda/blob/main/apis/keda/v1alpha1/scaledjob_types.go). + +```yaml +apiVersion: keda.sh/v1alpha1 +kind: ScaledJob +metadata: + name: {scaled-job-name} + labels: + my-label: {my-label-value} # Optional. ScaledJob labels are applied to child Jobs + annotations: + autoscaling.keda.sh/paused: true # Optional. Use to pause autoscaling of Jobs + my-annotation: {my-annotation-value} # Optional. ScaledJob annotations are applied to child Jobs +spec: + jobTargetRef: + parallelism: 1 # [max number of desired pods](https://kubernetes.io/docs/concepts/workloads/controllers/job/#controlling-parallelism) + completions: 1 # [desired number of successfully finished pods](https://kubernetes.io/docs/concepts/workloads/controllers/job/#controlling-parallelism) + activeDeadlineSeconds: 600 # Specifies the duration in seconds relative to the startTime that the job may be active before the system tries to terminate it; value must be positive integer + backoffLimit: 6 # Specifies the number of retries before marking this job failed. Defaults to 6 + template: + # describes the [job template](https://kubernetes.io/docs/concepts/workloads/controllers/job) + pollingInterval: 30 # Optional. Default: 30 seconds + successfulJobsHistoryLimit: 5 # Optional. Default: 100. How many completed jobs should be kept. + failedJobsHistoryLimit: 5 # Optional. Default: 100. How many failed jobs should be kept. + envSourceContainerName: {container-name} # Optional. Default: .spec.JobTargetRef.template.spec.containers[0] + minReplicaCount: 10 # Optional. Default: 0 + maxReplicaCount: 100 # Optional. Default: 100 + rolloutStrategy: gradual # Deprecated: Use rollout.strategy instead (see below). + rollout: + strategy: gradual # Optional. Default: default. Which Rollout Strategy KEDA will use. + propagationPolicy: foreground # Optional. Default: background. Kubernetes propagation policy for cleaning up existing jobs during rollout. + scalingStrategy: + strategy: "custom" # Optional. Default: default. Which Scaling Strategy to use. + customScalingQueueLengthDeduction: 1 # Optional. A parameter to optimize custom ScalingStrategy. + customScalingRunningJobPercentage: "0.5" # Optional. A parameter to optimize custom ScalingStrategy. + pendingPodConditions: # Optional. A parameter to calculate pending job count per the specified pod conditions + - "Ready" + - "PodScheduled" + - "AnyOtherCustomPodCondition" + multipleScalersCalculation : "max" # Optional. Default: max. Specifies how to calculate the target metrics when multiple scalers are defined. + triggers: + # {list of triggers to create jobs} +``` + +You can find all supported triggers [here](../scalers). + +## jobTargetRef + +```yaml + jobTargetRef: + parallelism: 1 # Optional. Max number of desired instances ([docs](https://kubernetes.io/docs/concepts/workloads/controllers/job/#controlling-parallelism)) + completions: 1 # Optional. Desired number of successfully finished instances ([docs](https://kubernetes.io/docs/concepts/workloads/controllers/job/#controlling-parallelism)) + activeDeadlineSeconds: 600 # Optional. Specifies the duration in seconds relative to the startTime that the job may be active before the system tries to terminate it; value must be positive integer + backoffLimit: 6 # Optional. Specifies the number of retries before marking this job failed. Defaults to 6 +``` + +The `jobTargetRef` is a batch/v1 `JobSpec` object; refer to the Kubernetes API for [more details](https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/job-v1/#JobSpec) about the fields. The `template` field is required. + + +## pollingInterval + +```yaml + pollingInterval: 30 # Optional. Default: 30 seconds +``` + +This is the interval to check each trigger on. By default, KEDA will check each trigger source on every ScaledJob every 30 seconds. + + +## successfulJobsHistoryLimit, failedJobsHistoryLimit + +```yaml + successfulJobsHistoryLimit: 5 # Optional. Default: 100. How many completed jobs should be kept. + failedJobsHistoryLimit: 5 # Optional. Default: 100. How many failed jobs should be kept. +``` + +The `successfulJobsHistoryLimit` and `failedJobsHistoryLimit` fields are optional. These fields specify how many completed and failed jobs should be kept. By default, they are set to 100. + +This concept is similar to [Jobs History Limits](https://kubernetes.io/docs/tasks/job/automated-tasks-with-cron-jobs/#jobs-history-limits) allowing you to learn what the outcomes of your jobs are. + +The actual number of jobs could exceed the limit in a short time. However, it is going to resolve in the cleanup period. Currently, the cleanup period is the same as the Polling interval. + + +## envSourceContainerName + +```yaml + envSourceContainerName: {container-name} # Optional. Default: .spec.JobTargetRef.template.spec.containers[0] +``` + +This optional property specifies the name of container in the Job, from which KEDA should try to get environment properties holding secrets etc. If it is not defined it, KEDA will try to get environment properties from the first Container, ie. from `.spec.JobTargetRef.template.spec.containers[0]`. + +___ +## minReplicaCount + +```yaml + minReplicaCount: 10 # Optional. Default: 0 +``` + +The min number of jobs that is created by default. This can be useful to avoid bootstrapping time of new jobs. If minReplicaCount is greater than maxReplicaCount, minReplicaCount will be set to maxReplicaCount. + +New messages may create new jobs - within the limits imposed by maxReplicaCount - in order to reach the state where minReplicaCount jobs are always running. For example, if one sets minReplicaCount to 2 then there will be 2 jobs running permanently. Using a targetValue of 1, if 3 new messages are sent, 2 of those messages will be processed on the already running jobs but another 3 jobs will be created in order to fulfill the desired state dictated by the minReplicaCount parameter that is set to 2. + +## maxReplicaCount + +```yaml + maxReplicaCount: 100 # Optional. Default: 100 +``` + +The max number of pods that is created within a single polling period. If there are running jobs, the number of running jobs will be deducted. This table is an example of the scaling logic. + +| Queue Length | Max Replica Count | Target Average Value | Running Job Count | Number of the Scale | +|--------------|-------------------|----------------------|-------------------|---------------------| +| 10 | 3 | 1 | 0 | 3 | +| 10 | 3 | 2 | 0 | 3 | +| 10 | 3 | 1 | 1 | 2 | +| 10 | 100 | 1 | 0 | 10 | +| 4 | 3 | 5 | 0 | 1 | + +* **Queue Length:** The number of items in the queue. +* **Target Average Value:** The number of messages that will be consumed on a job. It is defined on the scaler side. e.g. `queueLength` on `Azure Storage Queue` scaler. +* **Running Job Count:** How many jobs are running. +* **Number of the Scale:** The number of the job that is created. + + +## rollout + +```yaml + rollout: + strategy: gradual # Optional. Default: default. Which Rollout Strategy KEDA will use. + propagationPolicy: foreground # Optional. Default: background. Kubernetes propagation policy for cleaning up existing jobs during +``` + +The optional property rollout.strategy specifies the rollout strategy KEDA will use while updating an existing ScaledJob. +Possible values are `default` or `gradual`. \ +When using the `default` rolloutStrategy, KEDA will terminate existing Jobs whenever a ScaledJob is being updated. Then, it will recreate those Jobs with the latest specs. The order in which this termination happens can be configured via the rollout.propagationPolicy property. By default, the kubernetes background propagation is used. To change this behavior specify set propagationPolicy to `foreground`. For further information see [Kubernetes Documentation](https://kubernetes.io/docs/tasks/administer-cluster/use-cascading-deletion/#use-foreground-cascading-deletion). +On the `gradual` rolloutStartegy, whenever a ScaledJob is being updated, KEDA will not delete existing Jobs. Only new Jobs will be created with the latest specs. + + +## scalingStrategy + +```yaml +scalingStrategy: + strategy: "default" # Optional. Default: default. Which Scaling Strategy to use. +``` + +Select a Scaling Strategy. Possible values are `default`, `custom`, or `accurate`. The default value is `default`. + +> 💡 **NOTE:** +> +>`maxScale` is not the running Job count. It is measured as follows: + >```go + >maxScale = min(scaledJob.MaxReplicaCount(), divideWithCeil(queueLength, targetAverageValue)) + >``` + >That means it will use the value of `queueLength` divided by `targetAvarageValue` unless it is exceeding the `MaxReplicaCount`. +> +>`RunningJobCount` represents the number of jobs that are currently running or have not finished yet. +> +>It is measured as follows: +>```go +>if !e.isJobFinished(&job) { +> runningJobs++ +>} +>``` +>`PendingJobCount` provides an indication of the amount of jobs that are in pending state. Pending jobs can be calculated in two ways: +> - Default behavior - Job that have not finished yet **and** the underlying pod is either not running or has not been completed yet +> - Setting `pendingPodConditions` - Job that has not finished yet **and** all specified pod conditions of the underlying pod mark as `true` by kubernetes. +> +>It is measured as follows: +>```go +>if !e.isJobFinished(&job) { +> if len(scaledJob.Spec.ScalingStrategy.PendingPodConditions) > 0 { +> if !e.areAllPendingPodConditionsFulfilled(&job, scaledJob.Spec.ScalingStrategy.PendingPodConditions) { +> pendingJobs++ +> } +> } else { +> if !e.isAnyPodRunningOrCompleted(&job) { +> pendingJobs++ +> } +> } +>} +>``` + +**default** +This logic is the same as Job for V1. The number of the scale will be calculated as follows. + +_The number of the scale_ + +```go +maxScale - runningJobCount +``` + +**custom** +You can customize the default scale logic. You need to configure the following parameters. If you don't configure it, then the strategy will be `default.` + +```yaml +customScalingQueueLengthDeduction: 1 # Optional. A parameter to optimize custom ScalingStrategy. +customScalingRunningJobPercentage: "0.5" # Optional. A parameter to optimize custom ScalingStrategy. +``` + +_The number of the scale_ + +```go +min(maxScale-int64(*s.CustomScalingQueueLengthDeduction)-int64(float64(runningJobCount)*(*s.CustomScalingRunningJobPercentage)), maxReplicaCount) +``` + +**accurate** +If the scaler returns `queueLength` (number of items in the queue) that does not include the number of locked messages, this strategy is recommended. `Azure Storage Queue` is one example. You can use this strategy if you delete a message once your app consumes it. + +```go +if (maxScale + runningJobCount) > maxReplicaCount { + return maxReplicaCount - runningJobCount + } + return maxScale - pendingJobCount +``` +For more details, you can refer to [this PR](https://github.com/kedacore/keda/pull/1227). + + +### multipleScalersCalculation + +```yaml +scalingStrategy: + multipleScalersCalculation : "max" # Optional. Default: max. Specifies how to calculate the target metrics (`queueLength` and `maxScale`) when multiple scalers are defined. +``` +Select a behavior if you have multiple triggers. Possible values are `max`, `min`, `avg`, or `sum`. The default value is `max`. + +* **max:** - Use metrics from the scaler that has the max number of `queueLength`. (default) +* **min:** - Use metrics from the scaler that has the min number of `queueLength`. +* **avg:** - Sum up all the active scalers metrics and divide by the number of active scalers. +* **sum:** - Sum up all the active scalers metrics. diff --git a/content/docs/2.14/reference/scaledobject-spec.md b/content/docs/2.14/reference/scaledobject-spec.md new file mode 100644 index 000000000..391a968c8 --- /dev/null +++ b/content/docs/2.14/reference/scaledobject-spec.md @@ -0,0 +1,254 @@ + ++++ +title = "ScaledObject specification" +weight = 3000 ++++ + +## Overview + +This specification describes the `ScaledObject` Custom Resource definition that defines the triggers and scaling behaviors used by KEDA to scale `Deployment`, `StatefulSet` and `Custom Resource` target resources. The `.spec.ScaleTargetRef` section holds the reference to the target resource, defined in [_scaledobject_types.go_](https://github.com/kedacore/keda/blob/main/apis/keda/v1alpha1/scaledobject_types.go). + +```yaml +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: {scaled-object-name} + annotations: + scaledobject.keda.sh/transfer-hpa-ownership: "true" # Optional. Use to transfer an existing HPA ownership to this ScaledObject + validations.keda.sh/hpa-ownership: "true" # Optional. Use to disable HPA ownership validation on this ScaledObject + autoscaling.keda.sh/paused: "true" # Optional. Use to pause autoscaling of objects explicitly +spec: + scaleTargetRef: + apiVersion: {api-version-of-target-resource} # Optional. Default: apps/v1 + kind: {kind-of-target-resource} # Optional. Default: Deployment + name: {name-of-target-resource} # Mandatory. Must be in the same namespace as the ScaledObject + envSourceContainerName: {container-name} # Optional. Default: .spec.template.spec.containers[0] + pollingInterval: 30 # Optional. Default: 30 seconds + cooldownPeriod: 300 # Optional. Default: 300 seconds + idleReplicaCount: 0 # Optional. Default: ignored, must be less than minReplicaCount + minReplicaCount: 1 # Optional. Default: 0 + maxReplicaCount: 100 # Optional. Default: 100 + fallback: # Optional. Section to specify fallback options + failureThreshold: 3 # Mandatory if fallback section is included + replicas: 6 # Mandatory if fallback section is included + advanced: # Optional. Section to specify advanced options + restoreToOriginalReplicaCount: true/false # Optional. Default: false + horizontalPodAutoscalerConfig: # Optional. Section to specify HPA related options + name: {name-of-hpa-resource} # Optional. Default: keda-hpa-{scaled-object-name} + behavior: # Optional. Use to modify HPA's scaling behavior + scaleDown: + stabilizationWindowSeconds: 300 + policies: + - type: Percent + value: 100 + periodSeconds: 15 + triggers: + # {list of triggers to activate scaling of the target resource} +``` + +## scaleTargetRef + +```yaml + scaleTargetRef: + apiVersion: {api-version-of-target-resource} # Optional. Default: apps/v1 + kind: {kind-of-target-resource} # Optional. Default: Deployment + name: {name-of-target-resource} # Mandatory. Must be in the same namespace as the ScaledObject + envSourceContainerName: {container-name} # Optional. Default: .spec.template.spec.containers[0] +``` + +The reference to the resource this ScaledObject is configured for. This is the resource KEDA will scale up/down and set up an HPA for, based on the triggers defined in `triggers:`. + +To scale Kubernetes Deployments only `name` need be specified. To scale a different resource such as StatefulSet or Custom Resource (that defines `/scale` subresource), appropriate `apiVersion` (following standard Kubernetes convention, ie. `{api}/{version}`) and `kind` need to be specified. + +`envSourceContainerName` is an optional property that specifies the name of container in the target resource, from which KEDA should try to get environment properties holding secrets etc. If it is not defined, KEDA will try to get environment properties from the first Container, ie. from `.spec.template.spec.containers[0]`. + +**Assumptions:** Resource referenced by `name` (and `apiVersion`, `kind`) is in the same namespace as the ScaledObject + + +## pollingInterval +```yaml + pollingInterval: 30 # Optional. Default: 30 seconds +``` + +This is the interval to check each trigger on. By default, KEDA will check each trigger source on every ScaledObject every 30 seconds. + +**Example:** in a queue scenario, KEDA will check the queueLength every `pollingInterval`, and scale the resource up or down accordingly. + + +## cooldownPeriod +```yaml + cooldownPeriod: 300 # Optional. Default: 300 seconds +``` + +The period to wait after the last trigger reported active before scaling the resource back to 0, in seconds. By default, it's 300 (5 minutes). + +The `cooldownPeriod` only applies after a trigger occurs; when you first create your `Deployment` (or `StatefulSet`/`CustomResource`), KEDA will immediately scale it to `minReplicaCount`. Additionally, the KEDA `cooldownPeriod` only applies when scaling to 0; scaling from 1 to N replicas is handled by the [Kubernetes Horizontal Pod Autoscaler](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/../concepts/scaling-deployments.md#support-for-cooldowndelay). + +**Example:** wait 5 minutes after the last time KEDA checked the queue and it was empty. (this is obviously dependent on `pollingInterval`) + + +## initialCooldownPeriod +```yaml + InitialCooldownPeriod: 120 # Optional. Default: 0 seconds +``` +The delay before the `cooldownPeriod` starts after the initial creation of the `ScaledObject`, in seconds. By default, it's 0, meaning the `cooldownPeriod` begins immediately upon creation. If set to a value such as 120 seconds, the `cooldownPeriod` will only start after the `ScaledObject` has been active for that duration. + +This parameter is particularly useful for managing the scale-down behavior during the initial phase of a `ScaledObject`. For instance, if `InitialCooldownPeriod` is set to 120 seconds, KEDA will not scale the resource back to 0 until 120 seconds have passed since the `ScaledObject` creation, regardless of the activity triggers. This allows for a grace period in situations where immediate scaling down after creation is not desirable. + +**Example:** Wait 120 seconds after the `ScaledObject` is created before starting the `cooldownPeriod`. For instance, if the `InitialCooldownPeriod` is set to 120 seconds, KEDA will not initiate the cooldown process until 120 seconds have passed since the `ScaledObject` was first created, regardless of the triggers' activity. This ensures a buffer period where the resource won’t be scaled down immediately after creation. (Note: This setting is independent of the `pollingInterval`.) + + +## idleReplicaCount + +```yaml + idleReplicaCount: 0 # Optional. Default: ignored, must be less than minReplicaCount +``` + +> 💡 **NOTE:** Due to limitations in HPA controller the only supported value for this property is 0, it will not work correctly otherwise. See this [issue](https://github.com/kedacore/keda/issues/2314) for more details. +> +> In some cases, you always need at least `n` pod running. Thus, you can omit this property and set `minReplicaCount` to `n`. +> +> **Example** You set `minReplicaCount` to 1 and `maxReplicaCount` to 10. If there’s no activity on triggers, the target resource is scaled down to `minReplicaCount` (1). Once there are activities, the target resource will scale base on the HPA rule. If there’s no activity on triggers, the resource is again scaled down to `minReplicaCount` (1). + +If this property is set, KEDA will scale the resource down to this number of replicas. If there's some activity on target triggers KEDA will scale the target resource immediately to `minReplicaCount` and then will be scaling handled by HPA. When there is no activity, the target resource is again scaled down to `idleReplicaCount`. This setting must be less than `minReplicaCount`. + +**Example:** If there's no activity on triggers the target resource is scaled down to `idleReplicaCount` (0), once there is an activity the target resource is immediately scaled to `minReplicaCount` (10) and then up to `maxReplicaCount` (100) as needed. If there's no activity on triggers the resource is again scaled down to `idleReplicaCount` (0). + + +## minReplicaCount + +```yaml + minReplicaCount: 1 # Optional. Default: 0 +``` + +Minimum number of replicas KEDA will scale the resource down to. By default, it's scale to zero, but you can use it with some other value as well. + +## maxReplicaCount + +```yaml + maxReplicaCount: 100 # Optional. Default: 100 +``` +This setting is passed to the HPA definition that KEDA will create for a given resource and holds the maximum number of replicas of the target resource. + + +## fallback +```yaml + fallback: # Optional. Section to specify fallback options + failureThreshold: 3 # Mandatory if fallback section is included + replicas: 6 # Mandatory if fallback section is included +``` + +The `fallback` section is optional. It defines a number of replicas to fall back to if a scaler is in an error state. + +KEDA will keep track of the number of consecutive times each scaler has failed to get metrics from its source. Once that value passes the `failureThreshold`, instead of not propagating a metric to the HPA (the default error behaviour), the scaler will, instead, return a normalised metric using the formula: +``` +target metric value * fallback replicas +``` +Due to the HPA metric being of type `AverageValue` (see below), this will have the effect of the HPA scaling the deployment to the defined number of fallback replicas. + +**Example:** When my instance of prometheus is unavailable 3 consecutive times, KEDA will change the HPA metric such that the deployment will scale to 6 replicas. + +There are a few limitations to using a fallback: + - It only supports scalers whose target is an `AverageValue` metric. Thus, it is **not** supported by the CPU & memory scalers, or by scalers whose metric target type is `Value`. In these cases, it will assume that fallback is disabled. + - It is only supported by `ScaledObjects` **not** `ScaledJobs`. + + +## advanced + +### restoreToOriginalReplicaCount + +```yaml +advanced: + restoreToOriginalReplicaCount: true/false # Optional. Default: false +``` + +This property specifies whether the target resource (`Deployment`, `StatefulSet`,...) should be scaled back to original replicas count, after the `ScaledObject` is deleted. +Default behavior is to keep the replica count at the same number as it is in the moment of `ScaledObject's` deletion. + +For example a `Deployment` with `3 replicas` is created, then `ScaledObject` is created and the `Deployment` is scaled by KEDA to `10 replicas`. Then `ScaledObject` is deleted: + 1. if `restoreToOriginalReplicaCount = false` (default behavior) then `Deployment` replicas count is `10` + 2. if `restoreToOriginalReplicaCount = true` then `Deployment` replicas count is set back to `3` (the original value) + + +### horizontalPodAutoscalerConfig + +```yaml +advanced: + horizontalPodAutoscalerConfig: # Optional. Section to specify HPA related options + name: {name-of-hpa-resource} # Optional. Default: keda-hpa-{scaled-object-name} + behavior: # Optional. Use to modify HPA's scaling behavior + scaleDown: + stabilizationWindowSeconds: 300 + policies: + - type: Percent + value: 100 + periodSeconds: 15 +``` + +#### horizontalPodAutoscalerConfig.name + +The name of the HPA resource KEDA will create. By default, it's `keda-hpa-{scaled-object-name}` + +#### horizontalPodAutoscalerConfig.behavior + +Starting from Kubernetes v1.18 the autoscaling API allows scaling behavior to be configured through the HPA behavior field. This way one can directly affect scaling of 1<->N replicas, which is internally being handled by HPA. KEDA would feed values from this section directly to the HPA's `behavior` field. Please follow [Kubernetes documentation](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/../concepts/scaling-deployments.md#configurable-scaling-behavior) for details. + +**Assumptions:** KEDA must be running on Kubernetes cluster v1.18+, in order to be able to benefit from this setting. + + + +```yaml +advanced: + scalingModifiers: # Optional. Section to specify scaling modifiers + target: {target-value-to-scale-on} # Mandatory. New target if metrics are anyhow composed together + activationTarget: {activation-target-value-to-scale-on} # Optional. New activation target if metrics are anyhow composed together + metricType: {metric-tipe-for-the-modifier} # Optional. Metric type to be used if metrics are anyhow composed together + formula: {formula-for-fetched-metrics} # Mandatory. Formula for calculation +``` + +### scalingModifiers + +The `scalingModifiers` is optional and **experimental**. If defined, both `target` and `formula` are mandatory. Using this structure creates `composite-metric` for the HPA that will replace all requests for external metrics and handle them internally. With `scalingModifiers` each trigger used in the `formula` **must** have a name defined. + +#### scalingModifiers.target + +`target` defines new target value to scale on for the composed metric. + +#### scalingModifiers.activationTarget + +`activationTarget` defines a new [activation target value](../concepts/scaling-deployments.md#activating-and-scaling-thresholds) to scale on for the composed metric. (Default: `0`, Optional) + +#### scalingModifiers.metricType + +`metricType` defines metric type used for this new `composite-metric`. (Values: `AverageValue`, `Value`, Default: `AverageValue`, Optional) + +#### scalingModifiers.formula + + `formula` composes metrics together and allows them to be modified/manipulated. It accepts mathematical/conditional statements using [this external project](https://github.com/antonmedv/expr). If the `fallback` scaling feature is in effect, the `formula` will NOT modify its metrics (therefore it modifies metrics only when all of their triggers are healthy). Complete language definition of `expr` package can be found [here](https://expr.medv.io/docs/Language-Definition). Formula must return a single value (not boolean). + +For examples of this feature see section [Scaling Modifiers](../concepts/scaling-deployments.md#scaling-modifiers-experimental). + + +## triggers + +```yaml + triggers: + # {list of triggers to activate scaling of the target resource} +``` + +> 💡 **NOTE:** You can find all supported triggers [here](../scalers). + +Trigger fields: +- **type**: The type of trigger to use. (Mandatory) +- **metadata**: The configuration parameters that the trigger requires. (Mandatory) +- **name**: Name for this trigger. This value can be used to easily distinguish this specific trigger and its metrics when consuming [Prometheus metrics](../operate/prometheus.md). By default, the name is generated from the trigger type. (Optional) +- **useCachedMetrics**: Enables caching of metric values during polling interval (as specified in `.spec.pollingInterval`). For more information, see ["Caching Metrics"](../concepts/scaling-deployments.md#caching-metrics). (Values: `false`, `true`, Default: `false`, Optional) +- **authenticationRef**: A reference to the `TriggerAuthentication` or `ClusterTriggerAuthentication` object that is used to authenticate the scaler with the environment. + - More details can be found [here](../concepts/authentication). (Optional) +- **metricType**: The type of metric that should be used. (Values: `AverageValue`, `Value`, `Utilization`, Default: `AverageValue`, Optional) + - Learn more about how the [Horizontal Pod Autoscaler (HPA) calculates `replicaCount`](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale-walkthrough/) based on metric type and value. + - To show the differences between the metric types, let's assume we want to scale a deployment with 3 running replicas based on a queue of messages: + - With `AverageValue` metric type, we can control how many messages, on average, each replica will handle. If our metric is the queue size, the threshold is 5 messages, and the current message count in the queue is 20, HPA will scale the deployment to 20 / 5 = 4 replicas, regardless of the current replica count. + - The `Value` metric type, on the other hand, can be used when we don't want to take the average of the given metric across all replicas. For example, with the `Value` type, we can control the average time of messages in the queue. If our metric is average time in the queue, the threshold is 5 milliseconds, and the current average time is 20 milliseconds, HPA will scale the deployment to 3 * 20 / 5 = 12. + +> ⚠️ **NOTE:** All scalers, except CPU and Memory, support metric types `AverageValue` and `Value` while CPU and Memory scalers both support `AverageValue` and `Utilization`. diff --git a/content/docs/2.15/_index.md b/content/docs/2.15/_index.md index a3c18da2f..3dce1ff48 100644 --- a/content/docs/2.15/_index.md +++ b/content/docs/2.15/_index.md @@ -1,8 +1,19 @@ +++ -title = "The KEDA Documentation" +title = "Getting Started" weight = 1 +++ -Welcome to the documentation for **KEDA**, the Kubernetes Event-driven Autoscaler. Use the navigation to the left to learn more about how to use KEDA and its components. +Welcome to the documentation for **KEDA**, the Kubernetes Event-driven Autoscaler. -Additions and contributions to these docs are managed on [the keda-docs GitHub repo](https://github.com/kedacore/keda-docs). +Use the navigation bar on the left to learn more about KEDA's architecture and how to deploy and use KEDA. + +## Where to go + +What is your involvement with KEDA? + +| Role | Documentation | +|---------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------| +| User | This documentation is for users who want to deploy KEDA to scale Kubernetes. | +| Core Contributor | To contribute to the core KEDA project see the [KEDA GitHub repo](https://github.com/kedacore/keda). | +| Documentation Contributor | To add or contribute to these docs, or to build and serve the documentation locally, see the [keda-docs GitHub repo](https://github.com/kedacore/keda-docs). | +| Other Contributor | See the [KEDA project on GitHub](https://github.com/kedacore/) for other KEDA repos, including project governance, testing, and external scalers. | diff --git a/content/docs/2.15/authentication-providers/aws.md b/content/docs/2.15/authentication-providers/aws.md index c78d64b10..f47cc38c2 100644 --- a/content/docs/2.15/authentication-providers/aws.md +++ b/content/docs/2.15/authentication-providers/aws.md @@ -35,15 +35,15 @@ If you would like to use the same IAM credentials as your workload is currently ## AssumeRole or AssumeRoleWithWebIdentity? -This authentication uses automatically both, doing a fallback from [AssumeRoleWithWebIdentity](https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRoleWithWebIdentity.html) to [AssumeRole](https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRole.html) if the first one fails. This extends the capabilities because KEDA doesn't need `sts:AssumeRole` permission if you are already working with [WebIdentities](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_providers_oidc.html), you just need to add KEDA service account to the trusted relations of the role. +This authentication automatically uses both, falling back from [AssumeRoleWithWebIdentity](https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRoleWithWebIdentity.html) to [AssumeRole](https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRole.html) if the first one fails. This extends the capabilities because KEDA doesn't need `sts:AssumeRole` permission if you are already working with [WebIdentities](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_providers_oidc.html); in this case, you can add a KEDA service account to the trusted relations of the role. ## Setting up KEDA role and policy -The [official AWS docs](https://aws.amazon.com/es/blogs/opensource/introducing-fine-grained-iam-roles-service-accounts/) explain how to set up a a basic configuration for an IRSA role. The policy changes depend if you are using the KEDA role (`podIdentity.roleArn` is not set) or workload role (`podIdentity.roleArn` sets a RoleArn or `podIdentity.identityOwner` sets to `workload`). +The [official AWS docs](https://aws.amazon.com/es/blogs/opensource/introducing-fine-grained-iam-roles-service-accounts/) explain how to set up a basic configuration for an IRSA role. The policy changes depending on if you are using the KEDA role (`podIdentity.roleArn` is not set) or workload role (`podIdentity.roleArn` sets a RoleArn or `podIdentity.identityOwner` sets to `workload`). ### Using KEDA role to access infrastructure -This is the easiest case and you just need to attach to KEDA's role the desired policy/policies, granting the access permissions that you want to provide. For example, this could be a policy to use with SQS: +Attach the desired policies to KEDA's role, granting the access permissions that you want to provide. For example, this could be a policy to use with SQS: ```json { diff --git a/content/docs/2.15/concepts/external-scalers.md b/content/docs/2.15/concepts/external-scalers.md index aa00a18a0..d58ffe06d 100644 --- a/content/docs/2.15/concepts/external-scalers.md +++ b/content/docs/2.15/concepts/external-scalers.md @@ -44,8 +44,8 @@ The `Scaler` interface defines 3 methods: - `Close` is called to allow the scaler to clean up connections or other resources. - `GetMetricSpecForScaling` returns the target value for the HPA definition for the scaler. For more details refer to [Implementing `GetMetricSpec`](#5-implementing-getmetricspec). - `GetMetricsAndActivity` is called on `pollingInterval` and. When activity returns `true`, KEDA will scale to what is returned by the metric limited by `maxReplicaCount` on the ScaledObject/ScaledJob. - When `false` is returned, KEDA will scale to `minReplicaCount` or optionally `idleReplicaCount`. More details around the defaults and how these options work together can be found on the [ScaledObjectSpec](https://keda.sh/docs/latest/concepts/scaling-deployments/#scaledobject-spec). - > Refer to the [HPA docs](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale-walkthrough/) for how HPA calculates `replicaCount` based on metric value and target value. KEDA supports both `AverageValue` and `Value` metric target types for external metrics. When `AverageValue` (the default metric type) is used, the metric value returned by the external scaler will be divided by the number of replicas. + When `false` is returned, KEDA will scale to `minReplicaCount` or optionally `idleReplicaCount`. More details around the defaults and how these options work together can be found on the [ScaledObjectSpec](../reference/scaledobject-spec). +> Refer to the [HPA docs](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale-walkthrough/) for how HPA calculates `replicaCount` based on metric value and target value. KEDA supports both `AverageValue` and `Value` metric target types for external metrics. When `AverageValue` (the default metric type) is used, the metric value returned by the external scaler will be divided by the number of replicas. The `PushScaler` interface adds a `Run` method. This method receives a push channel (`active`), on which the scaler can send `true` at any time. The purpose of this mechanism is to initiate a scaling operation independently from `pollingInterval`. diff --git a/content/docs/2.15/concepts/scaling-deployments.md b/content/docs/2.15/concepts/scaling-deployments.md index 94594d562..6d0d90271 100644 --- a/content/docs/2.15/concepts/scaling-deployments.md +++ b/content/docs/2.15/concepts/scaling-deployments.md @@ -3,9 +3,15 @@ title = "Scaling Deployments, StatefulSets & Custom Resources" weight = 200 +++ -## Overview +This page describes the deployment scaling behavior of KEDA. -### Scaling of Deployments and StatefulSets +# Specification + +See the [Scaled Object specification](../reference/scaledobject-spec.md) for details on how to set the behaviors described below. + +# Scaling objects + +## Scaling Deployments and StatefulSets Deployments and StatefulSets are the most common way to scale workloads with KEDA. @@ -21,270 +27,30 @@ For example, if you wanted to use KEDA with an Apache Kafka topic as event sourc * As more messages arrive at the Kafka Topic, KEDA can feed this data to the HPA to drive scale out. * Each replica of the deployment is actively processing messages. Very likely, each replica is processing a batch of messages in a distributed manner. -### Scaling of Custom Resources +## Scaling Custom Resources With KEDA you can scale any workload defined as any `Custom Resource` (for example `ArgoRollout` [resource](https://argoproj.github.io/argo-rollouts/)). The scaling behaves the same way as scaling for arbitrary Kubernetes `Deployment` or `StatefulSet`. The only constraint is that the target `Custom Resource` must define `/scale` [subresource](https://kubernetes.io/docs/tasks/extend-kubernetes/custom-resources/custom-resource-definitions/#scale-subresource). -## ScaledObject spec - -This specification describes the `ScaledObject` Custom Resource definition which is used to define how KEDA should scale your application and what the triggers are. The `.spec.ScaleTargetRef` section holds the reference to the target resource, ie. `Deployment`, `StatefulSet` or `Custom Resource`. - -[`scaledobject_types.go`](https://github.com/kedacore/keda/blob/main/apis/keda/v1alpha1/scaledobject_types.go) - -```yaml -apiVersion: keda.sh/v1alpha1 -kind: ScaledObject -metadata: - name: {scaled-object-name} - annotations: - scaledobject.keda.sh/transfer-hpa-ownership: "true" # Optional. Use to transfer an existing HPA ownership to this ScaledObject - validations.keda.sh/hpa-ownership: "true" # Optional. Use to disable HPA ownership validation on this ScaledObject - autoscaling.keda.sh/paused: "true" # Optional. Use to pause autoscaling of objects explicitly -spec: - scaleTargetRef: - apiVersion: {api-version-of-target-resource} # Optional. Default: apps/v1 - kind: {kind-of-target-resource} # Optional. Default: Deployment - name: {name-of-target-resource} # Mandatory. Must be in the same namespace as the ScaledObject - envSourceContainerName: {container-name} # Optional. Default: .spec.template.spec.containers[0] - pollingInterval: 30 # Optional. Default: 30 seconds - cooldownPeriod: 300 # Optional. Default: 300 seconds - idleReplicaCount: 0 # Optional. Default: ignored, must be less than minReplicaCount - minReplicaCount: 1 # Optional. Default: 0 - maxReplicaCount: 100 # Optional. Default: 100 - fallback: # Optional. Section to specify fallback options - failureThreshold: 3 # Mandatory if fallback section is included - replicas: 6 # Mandatory if fallback section is included - advanced: # Optional. Section to specify advanced options - restoreToOriginalReplicaCount: true/false # Optional. Default: false - horizontalPodAutoscalerConfig: # Optional. Section to specify HPA related options - name: {name-of-hpa-resource} # Optional. Default: keda-hpa-{scaled-object-name} - behavior: # Optional. Use to modify HPA's scaling behavior - scaleDown: - stabilizationWindowSeconds: 300 - policies: - - type: Percent - value: 100 - periodSeconds: 15 - triggers: - # {list of triggers to activate scaling of the target resource} -``` - -### Details -```yaml - scaleTargetRef: - apiVersion: {api-version-of-target-resource} # Optional. Default: apps/v1 - kind: {kind-of-target-resource} # Optional. Default: Deployment - name: {name-of-target-resource} # Mandatory. Must be in the same namespace as the ScaledObject - envSourceContainerName: {container-name} # Optional. Default: .spec.template.spec.containers[0] -``` - -The reference to the resource this ScaledObject is configured for. This is the resource KEDA will scale up/down and setup an HPA for, based on the triggers defined in `triggers:`. - -To scale Kubernetes Deployments only `name` is needed to be specified, if one wants to scale a different resource such as StatefulSet or Custom Resource (that defines `/scale` subresource), appropriate `apiVersion` (following standard Kubernetes convention, ie. `{api}/{version}`) and `kind` need to be specified. - -`envSourceContainerName` is an optional property that specifies the name of container in the target resource, from which KEDA should try to get environment properties holding secrets etc. If it is not defined, KEDA will try to get environment properties from the first Container, ie. from `.spec.template.spec.containers[0]`. - -**Assumptions:** Resource referenced by `name` (and `apiVersion`, `kind`) is in the same namespace as the ScaledObject - ---- -#### pollingInterval -```yaml - pollingInterval: 30 # Optional. Default: 30 seconds -``` - -This is the interval to check each trigger on. By default, KEDA will check each trigger source on every ScaledObject every 30 seconds. - -**Example:** in a queue scenario, KEDA will check the queueLength every `pollingInterval`, and scale the resource up or down accordingly. - ---- -#### cooldownPeriod -```yaml - cooldownPeriod: 300 # Optional. Default: 300 seconds -``` - -The period to wait after the last trigger reported active before scaling the resource back to 0. By default, it's 5 minutes (300 seconds). - -The `cooldownPeriod` only applies after a trigger occurs; when you first create your `Deployment` (or `StatefulSet`/`CustomResource`), KEDA will immediately scale it to `minReplicaCount`. Additionally, the KEDA `cooldownPeriod` only applies when scaling to 0; scaling from 1 to N replicas is handled by the [Kubernetes Horizontal Pod Autoscaler](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/#support-for-cooldowndelay). - -**Example:** wait 5 minutes after the last time KEDA checked the queue and it was empty. (this is obviously dependent on `pollingInterval`) - ---- -#### initialCooldownPeriod -```yaml - initialCooldownPeriod: 120 # Optional. Default: 0 seconds -``` -The delay before the `cooldownPeriod` starts after the initial creation of the `ScaledObject`. By default, this is 0 seconds, meaning the `cooldownPeriod` begins immediately upon creation. If set to a value such as 120 seconds, the `cooldownPeriod` will only start after the `ScaledObject` has been active for that duration. - -This parameter is particularly useful for managing the scale-down behavior during the initial phase of a `ScaledObject`. For instance, if `initialCooldownPeriod` is set to 120 seconds, KEDA will not scale the resource back to 0 until 120 seconds have passed since the `ScaledObject` creation, regardless of the activity triggers. This allows for a grace period in situations where immediate scaling down after creation is not desirable. - -**Example:** Wait 120 seconds after the `ScaledObject` is created before starting the `cooldownPeriod`. For instance, if the `initialCooldownPeriod` is set to 120 seconds, KEDA will not initiate the cooldown process until 120 seconds have passed since the `ScaledObject` was first created, regardless of the triggers' activity. This ensures a buffer period where the resource won’t be scaled down immediately after creation. (Note: This setting is independent of the `pollingInterval`.) - ---- -#### idleReplicaCount - -```yaml - idleReplicaCount: 0 # Optional. Default: ignored, must be less than minReplicaCount -``` - -> 💡 **NOTE:** Due to limitations in HPA controller the only supported value for this property is 0, it will not work correctly otherwise. See this [issue](https://github.com/kedacore/keda/issues/2314) for more details. -> -> In some cases, you always need at least `n` pod running. Thus, you can omit this property and set `minReplicaCount` to `n`. -> -> **Example** You set `minReplicaCount` to 1 and `maxReplicaCount` to 10. If there’s no activity on triggers, the target resource is scaled down to `minReplicaCount` (1). Once there are activities, the target resource will scale base on the HPA rule. If there’s no activity on triggers, the resource is again scaled down to `minReplicaCount` (1). - -If this property is set, KEDA will scale the resource down to this number of replicas. If there's some activity on target triggers KEDA will scale the target resource immediately to `minReplicaCount` and then will be scaling handled by HPA. When there is no activity, the target resource is again scaled down to `idleReplicaCount`. This setting must be less than `minReplicaCount`. - -**Example:** If there's no activity on triggers the target resource is scaled down to `idleReplicaCount` (0), once there is an activity the target resource is immediately scaled to `minReplicaCount` (10) and then up to `maxReplicaCount` (100) as needed. If there's no activity on triggers the resource is again scaled down to `idleReplicaCount` (0). - ---- -#### minReplicaCount -```yaml - minReplicaCount: 1 # Optional. Default: 0 -``` - -Minimum number of replicas KEDA will scale the resource down to. By default, it's scale to zero, but you can use it with some other value as well. - ---- -#### maxReplicaCount -```yaml - maxReplicaCount: 100 # Optional. Default: 100 -``` - -This setting is passed to the HPA definition that KEDA will create for a given resource and holds the maximum number of replicas of the target resource. - ---- -#### fallback -```yaml - fallback: # Optional. Section to specify fallback options - failureThreshold: 3 # Mandatory if fallback section is included - replicas: 6 # Mandatory if fallback section is included -``` - -The `fallback` section is optional. It defines a number of replicas to fall back to if a scaler is in an error state. - -KEDA will keep track of the number of consecutive times each scaler has failed to get metrics from its source. Once that value passes the `failureThreshold`, instead of not propagating a metric to the HPA (the default error behaviour), the scaler will, instead, return a normalised metric using the formula: -``` -target metric value * fallback replicas -``` -Due to the HPA metric being of type `AverageValue` (see below), this will have the effect of the HPA scaling the deployment to the defined number of fallback replicas. - -**Example:** When my instance of prometheus is unavailable 3 consecutive times, KEDA will change the HPA metric such that the deployment will scale to 6 replicas. - -There are a few limitations to using a fallback: - - It only supports scalers whose target is an `AverageValue` metric. Thus, it is **not** supported by the CPU & memory scalers, or by scalers whose metric target type is `Value`. In these cases, it will assume that fallback is disabled. - - It is only supported by `ScaledObjects` **not** `ScaledJobs`. - ---- -#### advanced -```yaml -advanced: - restoreToOriginalReplicaCount: true/false # Optional. Default: false -``` - -This property specifies whether the target resource (`Deployment`, `StatefulSet`,...) should be scaled back to original replicas count, after the `ScaledObject` is deleted. -Default behavior is to keep the replica count at the same number as it is in the moment of `ScaledObject's` deletion. - -For example a `Deployment` with `3 replicas` is created, then `ScaledObject` is created and the `Deployment` is scaled by KEDA to `10 replicas`. Then `ScaledObject` is deleted: - 1. if `restoreToOriginalReplicaCount = false` (default behavior) then `Deployment` replicas count is `10` - 2. if `restoreToOriginalReplicaCount = true` then `Deployment` replicas count is set back to `3` (the original value) - ---- - -```yaml -advanced: - horizontalPodAutoscalerConfig: # Optional. Section to specify HPA related options - name: {name-of-hpa-resource} # Optional. Default: keda-hpa-{scaled-object-name} - behavior: # Optional. Use to modify HPA's scaling behavior - scaleDown: - stabilizationWindowSeconds: 300 - policies: - - type: Percent - value: 100 - periodSeconds: 15 -``` - -##### `horizontalPodAutoscalerConfig:` - -###### `horizontalPodAutoscalerConfig.name` - -The name of the HPA resource KEDA will create. By default, it's `keda-hpa-{scaled-object-name}` - -###### `horizontalPodAutoscalerConfig.behavior` - -Starting from Kubernetes v1.18 the autoscaling API allows scaling behavior to be configured through the HPA behavior field. This way one can directly affect scaling of 1<->N replicas, which is internally being handled by HPA. KEDA would feed values from this section directly to the HPA's `behavior` field. Please follow [Kubernetes documentation](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/#configurable-scaling-behavior) for details. - -**Assumptions:** KEDA must be running on Kubernetes cluster v1.18+, in order to be able to benefit from this setting. +# Features ---- +## Caching Metrics -```yaml -advanced: - scalingModifiers: # Optional. Section to specify scaling modifiers - target: {target-value-to-scale-on} # Mandatory. New target if metrics are anyhow composed together - activationTarget: {activation-target-value-to-scale-on} # Optional. New activation target if metrics are anyhow composed together - metricType: {metric-tipe-for-the-modifier} # Optional. Metric type to be used if metrics are anyhow composed together - formula: {formula-for-fetched-metrics} # Mandatory. Formula for calculation -``` - -##### `scalingModifiers` - -The `scalingModifiers` is optional and **experimental**. If defined, both `target` and `formula` are mandatory. Using this structure creates `composite-metric` for the HPA that will replace all requests for external metrics and handle them internally. With `scalingModifiers` each trigger used in the `formula` **must** have a name defined. - -###### `scalingModifiers.target` - -`target` defines new target value to scale on for the composed metric. - -###### `scalingModifiers.activationTarget` - -`activationTarget` defines new [activation target value](./scaling-deployments.md#activating-and-scaling-thresholds) to scale on for the composed metric. (Default: `0`, Optional) - -###### `scalingModifiers.metricType` - -`metricType` defines metric type used for this new `composite-metric`. (Values: `AverageValue`, `Value`, Default: `AverageValue`, Optional) - -###### `scalingModifiers.formula` - - `formula` composes metrics together and allows them to be modified/manipulated. It accepts mathematical/conditional statements using [this external project](https://github.com/antonmedv/expr). If the `fallback` scaling feature is in effect, the `formula` will NOT modify its metrics (therefore it modifies metrics only when all of their triggers are healthy). Complete language definition of `expr` package can be found [here](https://expr.medv.io/docs/Language-Definition). Formula must return a single value (not boolean). +This feature enables caching of metric values during polling interval (as specified in `.spec.pollingInterval`). Kubernetes (HPA controller) asks for a metric every few seconds (as defined by `--horizontal-pod-autoscaler-sync-period`, usually 15s), then this request is routed to KEDA Metrics Server, that by default queries the scaler and reads the metric values. Enabling this feature changes this behavior such that KEDA Metrics Server tries to read metric from the cache first. This cache is updated periodically during the polling interval. -For examples of this feature see section [Scaling Modifiers](#scaling-modifiers-experimental) below. - ---- -#### triggers -```yaml - triggers: - # {list of triggers to activate scaling of the target resource} -``` - -> 💡 **NOTE:** You can find all supported triggers [here](/scalers). - -Trigger fields: -- **type**: The type of trigger to use. (Mandatory) -- **metadata**: The configuration parameters that the trigger requires. (Mandatory) -- **name**: Name for this trigger. This value can be used to easily distinguish this specific trigger and its metrics when consuming [Prometheus metrics](../operate/prometheus.md). By default, the name is generated from the trigger type. (Optional) -- **useCachedMetrics**: Enables caching of metric values during polling interval (as specified in `.spec.pollingInterval`). For more information, see ["Caching Metrics"](#caching-metrics). (Values: `false`, `true`, Default: `false`, Optional) -- **authenticationRef**: A reference to the `TriggerAuthentication` or `ClusterTriggerAuthentication` object that is used to authenticate the scaler with the environment. - - More details can be found [here](./authentication). (Optional) -- **metricType**: The type of metric that should be used. (Values: `AverageValue`, `Value`, `Utilization`, Default: `AverageValue`, Optional) - - Learn more about how the [Horizontal Pod Autoscaler (HPA) calculates `replicaCount`](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale-walkthrough/) based on metric type and value. - - To show the differences between the metric types, let's assume we want to scale a deployment with 3 running replicas based on a queue of messages: - - With `AverageValue` metric type, we can control how many messages, on average, each replica will handle. If our metric is the queue size, the threshold is 5 messages, and the current message count in the queue is 20, HPA will scale the deployment to 20 / 5 = 4 replicas, regardless of the current replica count. - - The `Value` metric type, on the other hand, can be used when we don't want to take the average of the given metric across all replicas. For example, with the `Value` type, we can control the average time of messages in the queue. If our metric is average time in the queue, the threshold is 5 milliseconds, and the current average time is 20 milliseconds, HPA will scale the deployment to 3 * 20 / 5 = 12. - -> ⚠️ **NOTE:** All scalers, except CPU and Memory, support metric types `AverageValue` and `Value` while CPU and Memory scalers both support `AverageValue` and `Utilization`. +Enabling this feature can significantly reduce the load on the scaler service. -### Caching Metrics +This feature is not supported for `cpu`, `memory` or `cron` scaler. -This feature enables caching of metric values during polling interval (as specified in `.spec.pollingInterval`). Kubernetes (HPA controller) asks for a metric every few seconds (as defined by `--horizontal-pod-autoscaler-sync-period`, usually 15s), then this request is routed to KEDA Metrics Server, that by default queries the scaler and reads the metric values. Enabling this feature changes this behavior, KEDA Metrics Server tries to read metric from the cache first. This cache is being updated periodically during the polling interval. +## Pausing autoscaling -Enabling this feature can significantly reduce the load on the scaler service. +It can be useful to instruct KEDA to pause the autoscaling of objects, to do to cluster maintenance or to avoid resource starvation by removing non-mission-critical workloads. -This feature is not supported for `cpu`, `memory` or `cron` scaler. +This is preferable to deleting the resource because it removes the instances it is running from operation without touching the applications themselves. When ready, you can then reenable scaling. -### Pause autoscaling +You can pause autoscaling by adding this annotation to your `ScaledObject` definition: -It can be useful to instruct KEDA to pause autoscaling of objects, if you want to do to cluster maintenance or you want to avoid resource starvation by removing non-mission-critical workloads. You can enable this by adding the below annotation to your `ScaledObject` definition: ```yaml metadata: @@ -299,10 +65,10 @@ The annotation `autoscaling.keda.sh/paused` will pause scaling immediately and u Typically, either one or the other is being used given they serve a different purpose/scenario. However, if both `paused` and `paused-replicas` are set, KEDA will scale your current workload to the number specified count in `paused-replicas` and then pause autoscaling. -To enable/unpause autoscaling again, simply remove all paused annotations from the `ScaledObject` definition. If you paused with `autoscaling.keda.sh/paused`, you can also set the annotation to `false` to unpause. +To unpause (reenable) autoscaling again, remove all paused annotations from the `ScaledObject` definition. If you paused with `autoscaling.keda.sh/paused`, you can unpause by setting the annotation to `false`. -### Scaling Modifiers (Experimental) +## Scaling Modifiers (Experimental) **Example: compose average value** @@ -370,15 +136,14 @@ Conditions can be used within another condition as well. If value of `trig_one` is less than 2 AND `trig_one`+`trig_two` is at least 2 then return 5, if only the first is true return 10, if the first condition is false then return 0. Complete language definition of `expr` package can be found [here](https://expr.medv.io/docs/Language-Definition). Formula must return a single value (not boolean). All formulas are internally wrapped with float cast. -### Activating and Scaling thresholds -To give a consistent solution to this problem, KEDA has 2 different phases during the autoscaling process. +## Activating and Scaling thresholds + +KEDA has 2 different phases during the autoscaling process. - **Activation phase:** The activating (or deactivating) phase is the moment when KEDA (operator) has to decide if the workload should be scaled from/to zero. KEDA takes responsibility for this action based on the result of the scaler `IsActive` function and only applies to 0<->1 scaling. There are use-cases where the activating value (0-1 and 1-0) is totally different than 0, such as workloads scaled with the Prometheus scaler where the values go from -X to X. - **Scaling phase:** The scaling phase is the moment when KEDA has decided to scale out to 1 instance and now it is the HPA controller who takes the scaling decisions based on the configuration defined in the generated HPA (from ScaledObject data) and the metrics exposed by KEDA (metrics server). This phase applies the to 1<->N scaling. -#### Managing Activation & Scaling Thresholds - KEDA allows you to specify different values for each scenario: - **Activation:** Defines when the scaler is active or not and scales from/to 0 based on it. @@ -397,7 +162,7 @@ There are some important topics to take into account: > ⚠️ **NOTE:** If a scaler doesn't define "activation" parameter (a property that starts with `activation` prefix), then this specific scaler doesn't support configurable activation value and the activation value is always 0. -## Transfer ownership of an existing HPA +## Transferring ownership of an existing HPA If your environment already operates using kubernetes HPA, you can transfer the ownership of this resource to a new ScaledObject: @@ -413,7 +178,7 @@ spec: > ⚠️ **NOTE:** You need to specify a custom HPA name in your ScaledObject matching the existing HPA name you want it to manage. -## Disable validations on an existing HPA +## Disabling validations on an existing HPA You are allowed to disable admission webhooks validations with the following snippet. It grants you better flexibility but also brings vulnerabilities. Do it **at your own risk**. @@ -423,13 +188,13 @@ metadata: validations.keda.sh/hpa-ownership: "true" ``` -## Long-running executions +### Long-running executions One important consideration to make is how this pattern can work with long-running executions. Imagine a deployment triggers on a RabbitMQ queue message. Each message takes 3 hours to process. It's possible that if many queue messages arrive, KEDA will help drive scaling out to many replicas - let's say 4. Now the HPA makes a decision to scale down from 4 replicas to 2. There is no way to control which of the 2 replicas get terminated to scale down. That means the HPA may attempt to terminate a replica that is 2.9 hours into processing a 3 hour queue message. There are two main ways to handle this scenario. -### Leverage the container lifecycle +#### Leverage the container lifecycle Kubernetes provides a few [lifecycle hooks](https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/) that can be leveraged to delay termination. Imagine a replica is scheduled for termination and is 2.9 hours into processing a 3 hour message. Kubernetes will send a [`SIGTERM`](https://www.gnu.org/software/libc/manual/html_node/Termination-Signals.html) to signal the intent to terminate. Rather than immediately terminating, a deployment can delay termination until processing the current batch of messages has completed. Kubernetes will wait for a `SIGTERM` response or the `terminationGracePeriodSeconds` before killing the replica. @@ -437,6 +202,6 @@ Kubernetes provides a few [lifecycle hooks](https://kubernetes.io/docs/concepts/ Using this method can preserve a replica and enable long-running executions. However, one downside of this approach is while delaying termination, the pod phase will remain in the `Terminating` state. That means a pod that is delaying termination for a very long duration may show `Terminating` during that entire period of delay. -### Run as jobs +#### Run as jobs -The other alternative to handling long-running executions is by running the event driven code in Kubernetes Jobs instead of Deployments or Custom Resources. This approach is discussed [in the next section](../scaling-jobs). +The other alternative to handling long-running executions is by running the event driven code in Kubernetes Jobs instead of Deployments or Custom Resources. This approach is discussed [in the next section](./scaling-jobs). diff --git a/content/docs/2.15/concepts/scaling-jobs.md b/content/docs/2.15/concepts/scaling-jobs.md index 9a5c58595..2f9029332 100644 --- a/content/docs/2.15/concepts/scaling-jobs.md +++ b/content/docs/2.15/concepts/scaling-jobs.md @@ -3,10 +3,12 @@ title = "Scaling Jobs" weight = 300 +++ +This page describes the job scaling behavior of KEDA. See the [Scaled Job specification](../reference/scaledjob-spec.md) for details on how to set the behaviors described below. -## Overview -As an alternate to [scaling event-driven code as deployments](../scaling-deployments) you can also run and scale your code as Kubernetes Jobs. The primary reason to consider this option is to handle processing long-running executions. Rather than processing multiple events within a deployment, for each detected event a single Kubernetes Job is scheduled. That job will initialize, pull a single event from the message source, and process to completion and terminate. +# Overview + +As an alternate to [scaling event-driven code as deployments](./scaling-deployments) you can also run and scale your code as Kubernetes Jobs. The primary reason to consider this option is to handle processing long-running executions. Rather than processing multiple events within a deployment, for each detected event a single Kubernetes Job is scheduled. That job will initialize, pull a single event from the message source, and process to completion and terminate. For example, if you wanted to use KEDA to run a job for each message that lands on a RabbitMQ queue, the flow may be: @@ -16,250 +18,33 @@ For example, if you wanted to use KEDA to run a job for each message that lands 1. As additional messages arrive, additional jobs are created. Each job processes a single message to completion. 1. Periodically remove completed/failed job by the `SuccessfulJobsHistoryLimit` and `FailedJobsHistoryLimit.` -## ScaledJob spec - -This specification describes the `ScaledJob` custom resource definition which is used to define how KEDA should scale your application and what the triggers are. - -[`scaledjob_types.go`](https://github.com/kedacore/keda/blob/main/apis/keda/v1alpha1/scaledjob_types.go) - -```yaml -apiVersion: keda.sh/v1alpha1 -kind: ScaledJob -metadata: - name: {scaled-job-name} - labels: - my-label: {my-label-value} # Optional. ScaledJob labels are applied to child Jobs - annotations: - autoscaling.keda.sh/paused: true # Optional. Use to pause autoscaling of Jobs - my-annotation: {my-annotation-value} # Optional. ScaledJob annotations are applied to child Jobs -spec: - jobTargetRef: - parallelism: 1 # [max number of desired pods](https://kubernetes.io/docs/concepts/workloads/controllers/job/#controlling-parallelism) - completions: 1 # [desired number of successfully finished pods](https://kubernetes.io/docs/concepts/workloads/controllers/job/#controlling-parallelism) - activeDeadlineSeconds: 600 # Specifies the duration in seconds relative to the startTime that the job may be active before the system tries to terminate it; value must be positive integer - backoffLimit: 6 # Specifies the number of retries before marking this job failed. Defaults to 6 - template: - # describes the [job template](https://kubernetes.io/docs/concepts/workloads/controllers/job) - pollingInterval: 30 # Optional. Default: 30 seconds - successfulJobsHistoryLimit: 5 # Optional. Default: 100. How many completed jobs should be kept. - failedJobsHistoryLimit: 5 # Optional. Default: 100. How many failed jobs should be kept. - envSourceContainerName: {container-name} # Optional. Default: .spec.JobTargetRef.template.spec.containers[0] - minReplicaCount: 10 # Optional. Default: 0 - maxReplicaCount: 100 # Optional. Default: 100 - rolloutStrategy: gradual # Deprecated: Use rollout.strategy instead (see below). - rollout: - strategy: gradual # Optional. Default: default. Which Rollout Strategy KEDA will use. - propagationPolicy: foreground # Optional. Default: background. Kubernetes propagation policy for cleaning up existing jobs during rollout. - scalingStrategy: - strategy: "custom" # Optional. Default: default. Which Scaling Strategy to use. - customScalingQueueLengthDeduction: 1 # Optional. A parameter to optimize custom ScalingStrategy. - customScalingRunningJobPercentage: "0.5" # Optional. A parameter to optimize custom ScalingStrategy. - pendingPodConditions: # Optional. A parameter to calculate pending job count per the specified pod conditions - - "Ready" - - "PodScheduled" - - "AnyOtherCustomPodCondition" - multipleScalersCalculation : "max" # Optional. Default: max. Specifies how to calculate the target metrics when multiple scalers are defined. - triggers: - # {list of triggers to create jobs} -``` - -You can find all supported triggers [here](../scalers). - -## Details - -```yaml - jobTargetRef: - parallelism: 1 # Optional. Max number of desired instances ([docs](https://kubernetes.io/docs/concepts/workloads/controllers/job/#controlling-parallelism)) - completions: 1 # Optional. Desired number of successfully finished instances ([docs](https://kubernetes.io/docs/concepts/workloads/controllers/job/#controlling-parallelism)) - activeDeadlineSeconds: 600 # Optional. Specifies the duration in seconds relative to the startTime that the job may be active before the system tries to terminate it; value must be positive integer - backoffLimit: 6 # Optional. Specifies the number of retries before marking this job failed. Defaults to 6 -``` - -The `jobTargetRef` is a batch/v1 `JobSpec` object; refer to the Kubernetes API for [more details](https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/job-v1/#JobSpec) about the fields. The `template` field is required. - ---- - -```yaml - pollingInterval: 30 # Optional. Default: 30 seconds -``` - -This is the interval to check each trigger on. By default, KEDA will check each trigger source on every ScaledJob every 30 seconds. - ---- - -```yaml - successfulJobsHistoryLimit: 5 # Optional. Default: 100. How many completed jobs should be kept. - failedJobsHistoryLimit: 5 # Optional. Default: 100. How many failed jobs should be kept. -``` - -The `successfulJobsHistoryLimit` and `failedJobsHistoryLimit` fields are optional. These fields specify how many completed and failed jobs should be kept. By default, they are set to 100. - -This concept is similar to [Jobs History Limits](https://kubernetes.io/docs/tasks/job/automated-tasks-with-cron-jobs/#jobs-history-limits) allowing you to learn what the outcomes of your jobs are. - -The actual number of jobs could exceed the limit in a short time. However, it is going to resolve in the cleanup period. Currently, the cleanup period is the same as the Polling interval. - ---- - - -```yaml - envSourceContainerName: {container-name} # Optional. Default: .spec.JobTargetRef.template.spec.containers[0] -``` - -This optional property specifies the name of container in the Job, from which KEDA should try to get environment properties holding secrets etc. If it is not defined it, KEDA will try to get environment properties from the first Container, ie. from `.spec.JobTargetRef.template.spec.containers[0]`. - -___ -```yaml - minReplicaCount: 10 # Optional. Default: 0 -``` - -The min number of jobs that is created by default. This can be useful to avoid bootstrapping time of new jobs. If minReplicaCount is greater than maxReplicaCount, minReplicaCount will be set to maxReplicaCount. - -New messages may create new jobs - within the limits imposed by maxReplicaCount - in order to reach the state where minReplicaCount jobs are always running. For example, if one sets minReplicaCount to 2 then there will be 2 jobs running permanently. Using a targetValue of 1, if 3 new messages are sent, 2 of those messages will be processed on the already running jobs but another 3 jobs will be created in order to fulfill the desired state dictated by the minReplicaCount parameter that is set to 2. -___ - ---- - -```yaml - maxReplicaCount: 100 # Optional. Default: 100 -``` - -The max number of pods that is created within a single polling period. If there are running jobs, the number of running jobs will be deducted. This table is an example of the scaling logic. - -| Queue Length | Max Replica Count | Target Average Value | Running Job Count | Number of the Scale | -| ------- | ------ | ------- | ------ | ----- | -| 10 | 3 | 1 | 0 | 3 | -| 10 | 3 | 2 | 0 | 3 | -| 10 | 3 | 1 | 1 | 2 | -| 10 | 100 | 1 | 0 | 10 | -| 4 | 3 | 5 | 0 | 1 | - -* **Queue Length:** The number of items in the queue. -* **Target Average Value:** The number of messages that will be consumed on a job. It is defined on the scaler side. e.g. `queueLength` on `Azure Storage Queue` scaler. -* **Running Job Count:** How many jobs are running. -* **Number of the Scale:** The number of the job that is created. - ---- - -```yaml - rollout: - strategy: gradual # Optional. Default: default. Which Rollout Strategy KEDA will use. - propagationPolicy: foreground # Optional. Default: background. Kubernetes propagation policy for cleaning up existing jobs during -``` - -The optional property rollout.strategy specifies the rollout strategy KEDA will use while updating an existing ScaledJob. -Possible values are `default` or `gradual`. \ -When using the `default` rolloutStrategy, KEDA will terminate existing Jobs whenever a ScaledJob is being updated. Then, it will recreate those Jobs with the latest specs. The order in which this termination happens can be configured via the rollout.propagationPolicy property. By default, the kubernetes background propagation is used. To change this behavior specify set propagationPolicy to `foreground`. For further information see [Kubernetes Documentation](https://kubernetes.io/docs/tasks/administer-cluster/use-cascading-deletion/#use-foreground-cascading-deletion). -On the `gradual` rolloutStartegy, whenever a ScaledJob is being updated, KEDA will not delete existing Jobs. Only new Jobs will be created with the latest specs. - - ---- - -```yaml -scalingStrategy: - strategy: "default" # Optional. Default: default. Which Scaling Strategy to use. -``` -Select a Scaling Strategy. Possible values are `default`, `custom`, or `accurate`. The default value is `default`. +# Pausing autoscaling -> 💡 **NOTE:** -> ->`maxScale` is not the running Job count. It is measured as follows: - >```go - >maxScale = min(scaledJob.MaxReplicaCount(), divideWithCeil(queueLength, targetAverageValue)) - >``` - >That means it will use the value of `queueLength` divided by `targetAvarageValue` unless it is exceeding the `MaxReplicaCount`. -> ->`RunningJobCount` represents the number of jobs that are currently running or have not finished yet. -> ->It is measured as follows: ->```go ->if !e.isJobFinished(&job) { -> runningJobs++ ->} ->``` ->`PendingJobCount` provides an indication of the amount of jobs that are in pending state. Pending jobs can be calculated in two ways: -> - Default behavior - Job that have not finished yet **and** the underlying pod is either not running or has not been completed yet -> - Setting `pendingPodConditions` - Job that has not finished yet **and** all specified pod conditions of the underlying pod mark as `true` by kubernetes. -> ->It is measured as follows: ->```go ->if !e.isJobFinished(&job) { -> if len(scaledJob.Spec.ScalingStrategy.PendingPodConditions) > 0 { -> if !e.areAllPendingPodConditionsFulfilled(&job, scaledJob.Spec.ScalingStrategy.PendingPodConditions) { -> pendingJobs++ -> } -> } else { -> if !e.isAnyPodRunningOrCompleted(&job) { -> pendingJobs++ -> } -> } ->} ->``` +It can be useful to instruct KEDA to pause the autoscaling of objects, to do to cluster maintenance or to avoid resource starvation by removing non-mission-critical workloads. -**default** -This logic is the same as Job for V1. The number of the scale will be calculated as follows. +This is preferable to deleting the resource because it removes the instances it is running from operation without touching the applications themselves. When ready, you can then reenable scaling. -_The number of the scale_ - -```go -maxScale - runningJobCount -``` - -**custom** -You can customize the default scale logic. You need to configure the following parameters. If you don't configure it, then the strategy will be `default.` +You can pause autoscaling by adding this annotation to your `ScaledJob` definition: ```yaml -customScalingQueueLengthDeduction: 1 # Optional. A parameter to optimize custom ScalingStrategy. -customScalingRunningJobPercentage: "0.5" # Optional. A parameter to optimize custom ScalingStrategy. -``` - -_The number of the scale_ - -```go -min(maxScale-int64(*s.CustomScalingQueueLengthDeduction)-int64(float64(runningJobCount)*(*s.CustomScalingRunningJobPercentage)), maxReplicaCount) -``` - -**accurate** -If the scaler returns `queueLength` (number of items in the queue) that does not include the number of locked messages, this strategy is recommended. `Azure Storage Queue` is one example. You can use this strategy if you delete a message once your app consumes it. - -```go -if (maxScale + runningJobCount) > maxReplicaCount { - return maxReplicaCount - runningJobCount - } - return maxScale - pendingJobCount -``` -For more details, you can refer to [this PR](https://github.com/kedacore/keda/pull/1227). - ---- - -```yaml -scalingStrategy: - multipleScalersCalculation : "max" # Optional. Default: max. Specifies how to calculate the target metrics (`queueLength` and `maxScale`) when multiple scalers are defined. +metadata: + annotations: + autoscaling.keda.sh/paused: true ``` -Select a behavior if you have multiple triggers. Possible values are `max`, `min`, `avg`, or `sum`. The default value is `max`. - -* **max:** - Use metrics from the scaler that has the max number of `queueLength`. (default) -* **min:** - Use metrics from the scaler that has the min number of `queueLength`. -* **avg:** - Sum up all the active scalers metrics and divide by the number of active scalers. -* **sum:** - Sum up all the active scalers metrics. -### Pause autoscaling - -It can be useful to instruct KEDA to pause the autoscaling of objects, if you want to do to cluster maintenance or you want to avoid resource starvation by removing non-mission-critical workloads. - -This is a great alternative to deleting the resource, because we do not want to touch the applications themselves but simply remove the instances it is running from an operational perspective. Once everything is good to go, we can enable it to scale again. - -You can enable this by adding the below annotation to your `ScaledJob` definition: +To reenable autoscaling, remove the annotation from the `ScaledJob` definition or set the value to `false`. ```yaml metadata: annotations: - autoscaling.keda.sh/paused: true + autoscaling.keda.sh/paused: false ``` -The above annotation will pause autoscaling. To enable autoscaling again, simply remove the annotation from the `ScaledJob` definition or set the value to `false`. -# Sample +## Example + +An example configuration for autoscaling jobs using a RabbitMQ scaler is given below. ```yaml apiVersion: v1 diff --git a/content/docs/2.15/deploy.md b/content/docs/2.15/deploy.md index 0ebec231f..4e78035e4 100644 --- a/content/docs/2.15/deploy.md +++ b/content/docs/2.15/deploy.md @@ -4,9 +4,9 @@ title = "Deploying KEDA" We provide a few approaches to deploy KEDA runtime in your Kubernetes clusters: -- [Helm charts](#helm) -- [Operator Hub](#operatorhub) -- [YAML declarations](#yaml) +- Helm charts +- Operator Hub +- YAML declarations > 💡 **NOTE:** KEDA requires Kubernetes cluster version 1.27 and higher @@ -16,7 +16,7 @@ Don't see what you need? Feel free to [create an issue](https://github.com/kedac ### Install -Deploying KEDA with Helm is very simple: +To deploy KEDA with Helm: 1. Add Helm repo @@ -147,7 +147,7 @@ VERSION=2.15.0 make undeploy ### Install -If you want to try KEDA v2 on [MicroK8s](https://microk8s.io/) from `1.20` channel, KEDA is included into MicroK8s addons. +If you want to try KEDA v2 on [MicroK8s](https://microk8s.io/) from `1.20` channel, KEDA is included into MicroK8s add-ons. ```sh microk8s enable keda @@ -155,7 +155,7 @@ microk8s enable keda ### Uninstall -To uninstall KEDA in MicroK8s, simply disable the addon as shown below. +To uninstall KEDA in MicroK8s, disable the add-on as shown below. ```sh microk8s disable keda diff --git a/content/docs/2.15/migration.md b/content/docs/2.15/migration.md index 29e8408c8..3e52fd485 100644 --- a/content/docs/2.15/migration.md +++ b/content/docs/2.15/migration.md @@ -27,7 +27,7 @@ In order to scale `Deployments` with KEDA v2, you need to do only a few modifica - Rename property `spec.scaleTargetRef.containerName` to `spec.scaleTargetRef.envSourceContainerName` - Label `deploymentName` (in `metadata.labels.`) is no longer needed to be specified on v2 ScaledObject (it was mandatory on older versions of v1) -Please see the examples below or refer to the full [v2 ScaledObject Specification](../concepts/scaling-deployments/#scaledobject-spec) +Please see the examples below or refer to the full [v2 ScaledObject Specification](./reference/scaledobject-spec) **Example of v1 ScaledObject** @@ -80,7 +80,7 @@ In order to scale `Jobs` with KEDA v2, you need to do only a few modifications t You can configure `successfulJobsHistoryLimit` and `failedJobsHistoryLimit`. They will remove the old job histories automatically. -Please see the examples below or refer to the full [v2 ScaledJob Specification](../concepts/scaling-jobs/#scaledjob-spec) +Please see the examples below or refer to the full [v2 ScaledJob Specification](./reference/scaledjob-spec/) **Example of v1 ScaledObject for Jobs scaling** diff --git a/content/docs/2.15/operate/_index.md b/content/docs/2.15/operate/_index.md index e7604a0cf..dac7c02f2 100644 --- a/content/docs/2.15/operate/_index.md +++ b/content/docs/2.15/operate/_index.md @@ -1,14 +1,14 @@ +++ title = "Operate" -description = "Guidance & requirements for operating KEDA" +description = "Guidance and requirements for operating KEDA" weight = 1 +++ -We provide guidance & requirements around various areas to operate KEDA: +We provide guidance and requirements around various areas to operate KEDA: - Admission Webhooks ([link](./admission-webhooks)) - Cluster ([link](./cluster)) -- Kubernetes Events ([link](./events)) +- Kubernetes Events ([link](../reference/events)) - KEDA Metrics Server ([link](./metrics-server)) - Integrate with Prometheus ([link](./prometheus)) - Integrate with OpenTelemetry Collector (Experimental) ([link](./opentelemetry)) diff --git a/content/docs/2.15/operate/cloud-events.md b/content/docs/2.15/operate/cloud-events.md new file mode 100644 index 000000000..e07dc0505 --- /dev/null +++ b/content/docs/2.15/operate/cloud-events.md @@ -0,0 +1,116 @@ ++++ +title = "CloudEvent Support" +description = "Experimental support for cloud events" +weight = 100 ++++ + +## Subscribing to events with `CloudEventSource` +`CloudEventSource` resource can be used in KEDA for subscribing to events that are emitted to the user's defined CloudEvent sink. + +> 📝 Event will be emitted to both Kubernetes Events and CloudEvents Destination if CloudEventSource resource is created. + +Here is a the schema of the `CloudEventSource` CRD: + +```yaml +apiVersion: eventing.keda.sh/v1alpha1 +kind: CloudEventSource +metadata: + name: {cloud-event-name} +spec: + clusterName: {cluster-name} #Optional. Will be used in the source/subject to specify where the event comes from. The default value is 'kubernetes-default' and it can also be set during the installation of KEDA with --k8sClusterName. This one will overwrite others if set. + authenticationRef: + name: {trigger-authentication-name} #Optional. Used to reference a `TriggerAuthentication` for authentication. + kind: TriggerAuthentication # Optional. Used to choose the authentication scopes. https://keda.sh/docs/latest/concepts/authentication/#authentication-scopes-namespace-vs-cluster + destination: + http: + uri: http://foo.bar + azureEventGridTopic: + endpoint: https://my-topic.eastus-1.eventgrid.azure.net/api/events + + eventSubscription: #Optional. Submit included/excluded event types will filter events when emitting events. + includedEventTypes: #Optional. Only events in this section will be emitted. + - keda.scaledobject.failed.v1 + excludedEventTypes: #Optional. Events in this section will not be emitted. + - keda.scaledobject.ready.v1 +``` + +In general, an event emitted by KEDA would fundamentally come down to the following structure: +```json +{ + "specversion" : "1.0", + "type" : "com.cloudeventsource.keda", + "source" : "/{cluster-name}/{keda-namespace}/keda", + "subject" : "/{cluster-name}/{namespace}/{object-type}/{object-name}", + "id" : "", + "time" : "2018-04-05T17:31:00Z", + "datacontenttype" : "application/json", + "data" : { + "reason":"", + "message":"" + } +} +``` + +## Event Sinks + +There will be multiple types of destination to emit KEDA events to. + +Here is an overview of the supported destinations: + +- [HTTP endpoint](#http-endpoint). +- [Azure Event Grid endpoint](#azure-event-grid). + +### HTTP endpoint +```yaml + destination: + http: + uri: http://foo.bar #An http endpoint that can receive cloudevent +``` + +### Azure Event Grid + +```yaml + destination: + azureEventGrid: + endpoint: foo.bar #endpoint from AzureEventGrid Topic +``` + +Authentication information must be provided by using `authenticationRef` which allows you to provide the access key or managed identity for Azure Event Grid authentication by providing a `TriggerAuthentication`. + +Here is an overview of the supported authentication types: + +#### Connection String Authentication + +- `accessKey` - Access key string for the Azure Event Grid connection auth. + +#### Pod identity based authentication +[Azure AD Workload Identity](https://azure.github.io/azure-workload-identity/docs/) providers can be used. + +```yaml +apiVersion: keda.sh/v1alpha1 +kind: TriggerAuthentication +metadata: + name: nameOfTriggerAuth + namespace: default +spec: + podIdentity: + provider: azure-workload +``` + +## Event Filter + +You can include filter(s) to define what event types you are interested in, or want to ignore. This is done by using `includedEventTypes` or `excludedEventTypes` respectively for a given sink. + +```yaml +eventSubscription: #Optional. Submit included/excluded event types will filter events when emitting events. + includedEventTypes: #Optional. Only events in this section will be emitted. + - keda.scaledobject.failed.v1 + excludedEventTypes: #Optional. Events in this section will not be emitted. + - keda.scaledobject.ready.v1 +``` + +## Supported Event List +| Event Type | Scenario Description | +|-------------------------------|-----------------------------------------------------------------------------------------------------------------------------| +| `keda.scaledobject.ready.v1` | On the first time a ScaledObject is ready, or if the previous ready condition status of the object was `Unknown` or `False` | +| `keda.scaledobject.failed.v1` | If the check validation for a ScaledObject fails | diff --git a/content/docs/2.15/operate/events.md b/content/docs/2.15/operate/events.md deleted file mode 100644 index a4bf08ff1..000000000 --- a/content/docs/2.15/operate/events.md +++ /dev/null @@ -1,144 +0,0 @@ -+++ -title = "Events" -description = "Kubernetes Events emitted by KEDA" -weight = 100 -+++ - -## Kubernetes Events emitted by KEDA - -KEDA emits the following [Kubernetes Events](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.19/#event-v1-core): - -| Event | Type | Description | -| ------------------------------------- | --------- | --------------------------------------------------------------------------------------------------------------------------- | -| `ScaledObjectReady` | `Normal` | On the first time a ScaledObject is ready, or if the previous ready condition status of the object was `Unknown` or `False` | -| `ScaledJobReady` | `Normal` | On the first time a ScaledJob is ready, or if the previous ready condition status of the object was `Unknown` or `False` | -| `ScaledObjectCheckFailed` | `Warning` | If the check validation for a ScaledObject fails | | -| `ScaledJobCheckFailed` | `Warning` | If the check validation for a ScaledJob fails | | -| `ScaledObjectDeleted` | `Normal` | When a ScaledObject is deleted and removed from KEDA watch | | -| `ScaledJobDeleted` | `Normal` | When a ScaledJob is deleted and removed from KEDA watch | | -| `KEDAScalersStarted` | `Normal` | When Scalers watch loop have started for a ScaledObject or ScaledJob | | -| `KEDAScalersStopped` | `Normal` | When Scalers watch loop have stopped for a ScaledObject or a ScaledJob | | -| `KEDAScalerFailed` | `Warning` | When a Scaler fails to create or check its event source| | -| `KEDAScaleTargetActivated` | `Normal` | When the scale target (Deployment, StatefulSet, etc) of a ScaledObject is scaled to 1, triggered by {scalers1;scalers2;...}| | -| `KEDAScaleTargetDeactivated` | `Normal` | When the scale target (Deployment, StatefulSet, etc) of a ScaledObject is scaled to 0 | | -| `KEDAScaleTargetActivationFailed` | `Warning` | When KEDA fails to scale the scale target of a ScaledObject to 1| | -| `KEDAScaleTargetDeactivationFailed` | `Warning` | When KEDA fails to scale the scale target of a ScaledObject to 0| | -| `KEDAJobsCreated` | `Normal` | When KEDA creates jobs for a ScaledJob | | -| `TriggerAuthenticationAdded` | `Normal` | When a new TriggerAuthentication is added| | -| `TriggerAuthenticationDeleted` | `Normal` | When a TriggerAuthentication is deleted| | -| `ClusterTriggerAuthenticationAdded` | `Normal` | When a new ClusterTriggerAuthentication is added| | -| `ClusterTriggerAuthenticationDeleted` | `Normal` | When a ClusterTriggerAuthentication is deleted| | - - -## CloudEvent Support (Experimental) - -### Subscribing to events with `CloudEventSource` -`CloudEventSource` resource can be used in KEDA for subscribing to events that are emitted to the user's defined CloudEvent sink. - -> 📝 Event will be emitted to both Kubernetes Events and CloudEvents Destination if CloudEventSource resource is created. - -Here is a the schema of the `CloudEventSource` CRD: - -```yaml -apiVersion: eventing.keda.sh/v1alpha1 -kind: CloudEventSource -metadata: - name: {cloud-event-name} -spec: - clusterName: {cluster-name} #Optional. Will be used in the source/subject to specify where the event comes from. The default value is 'kubernetes-default' and it can also be set during the installation of KEDA with --k8sClusterName. This one will overwrite others if set. - authenticationRef: - name: {trigger-authentication-name} #Optional. Used to reference a `TriggerAuthentication` for authentication. - kind: TriggerAuthentication # Optional. Used to choose the authentication scopes. https://keda.sh/docs/latest/concepts/authentication/#authentication-scopes-namespace-vs-cluster - destination: - http: - uri: http://foo.bar - azureEventGridTopic: - endpoint: https://my-topic.eastus-1.eventgrid.azure.net/api/events - - eventSubscription: #Optional. Submit included/excluded event types will filter events when emitting events. - includedEventTypes: #Optional. Only events in this section will be emitted. - - keda.scaledobject.failed.v1 - excludedEventTypes: #Optional. Events in this section will not be emitted. - - keda.scaledobject.ready.v1 -``` - -In general, an event emitted by KEDA would fundamentally come down to the following structure: -```json -{ - "specversion" : "1.0", - "type" : "com.cloudeventsource.keda", - "source" : "/{cluster-name}/{keda-namespace}/keda", - "subject" : "/{cluster-name}/{namespace}/{object-type}/{object-name}", - "id" : "", - "time" : "2018-04-05T17:31:00Z", - "datacontenttype" : "application/json", - "data" : { - "reason":"", - "message":"" - } -} -``` - -### Event Sinks - -There will be multiple types of destination to emit KEDA events to. - -Here is an overview of the supported destinations: - -- [HTTP endpoint](#http-endpoint). -- [Azure Event Grid endpoint](#azure-event-grid). - -#### HTTP endpoint -```yaml - destination: - http: - uri: http://foo.bar #An http endpoint that can receive cloudevent -``` - -#### Azure Event Grid - -```yaml - destination: - azureEventGrid: - endpoint: foo.bar #endpoint from AzureEventGrid Topic -``` - -Authentication information must be provided by using `authenticationRef` which allows you to provide the access key or managed identity for Azure Event Grid authentication by providing a `TriggerAuthentication`. - -Here is an overview of the supported authentication types: - -##### Connection String Authentication - -- `accessKey` - Access key string for the Azure Event Grid connection auth. - -##### Pod identity based authentication -[Azure AD Workload Identity](https://azure.github.io/azure-workload-identity/docs/) providers can be used. - -```yaml -apiVersion: keda.sh/v1alpha1 -kind: TriggerAuthentication -metadata: - name: nameOfTriggerAuth - namespace: default -spec: - podIdentity: - provider: azure-workload -``` - -### Event Filter - -You can include filter(s) to define what event types you are interested in, or want to ignore. This is done by using `includedEventTypes` or `excludedEventTypes` respectively for a given sink. - -```yaml -eventSubscription: #Optional. Submit included/excluded event types will filter events when emitting events. - includedEventTypes: #Optional. Only events in this section will be emitted. - - keda.scaledobject.failed.v1 - excludedEventTypes: #Optional. Events in this section will not be emitted. - - keda.scaledobject.ready.v1 -``` - -### Supported Event List -| Event Type | Scenario Description | -| ------------------------------------- | --------------------------------------------------------------------------------------------------------------------------- | -| `keda.scaledobject.ready.v1` | On the first time a ScaledObject is ready, or if the previous ready condition status of the object was `Unknown` or `False` | -| `keda.scaledobject.failed.v1` | If the check validation for a ScaledObject fails | \ No newline at end of file diff --git a/content/docs/2.15/operate/metrics-server.md b/content/docs/2.15/operate/metrics-server.md index 6977e990e..45e9699fa 100644 --- a/content/docs/2.15/operate/metrics-server.md +++ b/content/docs/2.15/operate/metrics-server.md @@ -11,7 +11,7 @@ The metrics exposed by KEDA Metrics Server can be queried directly using `kubect kubectl get --raw "/apis/external.metrics.k8s.io/v1beta1" ``` -This will return a json with the list of metrics exposed by KEDA (just an external metric): +This will return a json with the list of metrics exposed by KEDA (external metrics only): ```json { "kind": "APIResourceList", diff --git a/content/docs/2.15/reference/_index.md b/content/docs/2.15/reference/_index.md new file mode 100644 index 000000000..82fc53448 --- /dev/null +++ b/content/docs/2.15/reference/_index.md @@ -0,0 +1,13 @@ ++++ +title = "Reference" +weight = 2 ++++ + +Reference information for the KEDA autoscaler. + +- [ScaledObject specification](./scaledobject-spec) +- [ScaledJob specification](./scaledjob-spec) +- [Kubernetes Events](./events) +- [Firewall requirements] +- [FAQ](./faq.md) +- [Glossary](./glossary.md) \ No newline at end of file diff --git a/content/docs/2.15/reference/events.md b/content/docs/2.15/reference/events.md new file mode 100644 index 000000000..aa264c0d8 --- /dev/null +++ b/content/docs/2.15/reference/events.md @@ -0,0 +1,28 @@ ++++ +title = "Events reference" +description = "Kubernetes Events emitted by KEDA" +weight = 2500 ++++ + +KEDA emits the following [Kubernetes Events](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.19/#event-v1-core): + +| Event | Type | Description | +|---------------------------------------|-----------|-----------------------------------------------------------------------------------------------------------------------------| +| `ScaledObjectReady` | `Normal` | On the first time a ScaledObject is ready, or if the previous ready condition status of the object was `Unknown` or `False` | +| `ScaledJobReady` | `Normal` | On the first time a ScaledJob is ready, or if the previous ready condition status of the object was `Unknown` or `False` | +| `ScaledObjectCheckFailed` | `Warning` | If the check validation for a ScaledObject fails | | +| `ScaledJobCheckFailed` | `Warning` | If the check validation for a ScaledJob fails | | +| `ScaledObjectDeleted` | `Normal` | When a ScaledObject is deleted and removed from KEDA watch | | +| `ScaledJobDeleted` | `Normal` | When a ScaledJob is deleted and removed from KEDA watch | | +| `KEDAScalersStarted` | `Normal` | When Scalers watch loop have started for a ScaledObject or ScaledJob | | +| `KEDAScalersStopped` | `Normal` | When Scalers watch loop have stopped for a ScaledObject or a ScaledJob | | +| `KEDAScalerFailed` | `Warning` | When a Scaler fails to create or check its event source | | +| `KEDAScaleTargetActivated` | `Normal` | When the scale target (Deployment, StatefulSet, etc) of a ScaledObject is scaled to 1, triggered by {scalers1;scalers2;...} | | +| `KEDAScaleTargetDeactivated` | `Normal` | When the scale target (Deployment, StatefulSet, etc) of a ScaledObject is scaled to 0 | | +| `KEDAScaleTargetActivationFailed` | `Warning` | When KEDA fails to scale the scale target of a ScaledObject to 1 | | +| `KEDAScaleTargetDeactivationFailed` | `Warning` | When KEDA fails to scale the scale target of a ScaledObject to 0 | | +| `KEDAJobsCreated` | `Normal` | When KEDA creates jobs for a ScaledJob | | +| `TriggerAuthenticationAdded` | `Normal` | When a new TriggerAuthentication is added | | +| `TriggerAuthenticationDeleted` | `Normal` | When a TriggerAuthentication is deleted | | +| `ClusterTriggerAuthenticationAdded` | `Normal` | When a new ClusterTriggerAuthentication is added | | +| `ClusterTriggerAuthenticationDeleted` | `Normal` | When a ClusterTriggerAuthentication is deleted | | diff --git a/content/docs/2.15/reference/faq.md b/content/docs/2.15/reference/faq.md new file mode 100644 index 000000000..df75bbe78 --- /dev/null +++ b/content/docs/2.15/reference/faq.md @@ -0,0 +1,6 @@ ++++ +title = "FAQ" +weight = 2000 ++++ + +{{< faq20 >}} diff --git a/content/docs/2.15/reference/glossary.md b/content/docs/2.15/reference/glossary.md new file mode 100644 index 000000000..2451000bf --- /dev/null +++ b/content/docs/2.15/reference/glossary.md @@ -0,0 +1,88 @@ ++++ +title = "Glossary" +weight = 1000 ++++ + +This document defines the various terms needed to understand the documentation and set up and use KEDA. + +## Admission Webhook + +[In Kubernetes](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/), an HTTP callback that handle admission requests. KEDA uses an admission webhook to validate and mutate ScaledObject resources. + +## Agent + +A primary role held by the KEDA operator. The Agent activates and deactivates Kubernetes Deployments to scale to and from zero. + +## Cluster + +[In Kubernetes](https://kubernetes.io/docs/reference/glossary/?fundamental=true#term-cluster), a set of one or more nodes that run containerized applications. + +## CRD + +Custom Resource Definition. [In Kubernetes](https://kubernetes.io/docs/reference/glossary/?fundamental=true#term-CustomResourceDefinition), a custom resource that extends the Kubernetes API with custom resources like ScaledObjects that have custom fields and behavior. + +## Event + +A notable occurrence captured by an event source that KEDA may use as a trigger to scale a container or deployment. + +## Event Source + +An external system like Kafka, RabbitMQ, that generates events that KEDA can monitor using a scaler. + +## Grafana + +An open-source monitoring platform that can visualize metrics collected by KEDA. + +## gRPC Remote Procedure Calls (gRPC) + +gRPC Remote Procedure Calls (gRPC). An open-source remote procedure call framework used by KEDA components to communicate. + +## HPA + +Horizontal Pod Autoscaler. Kubernetes autoscaler. By default, scales based on CPU/memory usage. KEDA uses HPA to scale Kubernetes clusters and deployments. + +## KEDA + +Kubernetes Event-Driven Autoscaling. A single-purpose, lightweight autoscaler that can scale a Kubernetes workload based on event metrics. + +## Metric + +Measurement of an event source such as queue length or response lag that KEDA uses to determine scaling. + +## OpenTelemetry + +An observability framework used by KEDA to instrument applications and collect metrics. + +## Operator + +The core KEDA component that monitors metrics and scales workloads accordingly. + +## Prometheus + +An open-source monitoring system that can scrape and store metrics from KEDA. + +## Scaled Object + +A custom resource that defines how KEDA should scale a workload based on events. + +## Scaled Job + +A custom resource KEDA uses to scale an application. + +## Scaler + +A component that integrates KEDA with a specific event source to collect metrics. + +## Stateful Set + +A Kubernetes workload with persistent data. KEDA can scale stateful sets. + +## TLS + +Transport Layer Security. KEDA uses TLS to encrypt communications between KEDA components. + +## Webhook + +An HTTP callback used to notify KEDA of events from external sources. + +[In Kubernetes](https://kubernetes.io/docs/reference/access-authn-authz/webhook/), an HTTP callback used as an event notification mechanism. diff --git a/content/docs/2.15/reference/scaledjob-spec.md b/content/docs/2.15/reference/scaledjob-spec.md new file mode 100644 index 000000000..dbc33680b --- /dev/null +++ b/content/docs/2.15/reference/scaledjob-spec.md @@ -0,0 +1,236 @@ ++++ +title = "ScaledJob specification" +weight = 4000 ++++ + +## Overview + +This specification describes the `ScaledJob` custom resource definition that defines the triggers and scaling behaviors use by KEDA + +to scale jobs. The `.spec.ScaleTargetRef` section holds the reference to the job, defined in [_scaledjob_types.go_](https://github.com/kedacore/keda/blob/main/apis/keda/v1alpha1/scaledjob_types.go). + +```yaml +apiVersion: keda.sh/v1alpha1 +kind: ScaledJob +metadata: + name: {scaled-job-name} + labels: + my-label: {my-label-value} # Optional. ScaledJob labels are applied to child Jobs + annotations: + autoscaling.keda.sh/paused: true # Optional. Use to pause autoscaling of Jobs + my-annotation: {my-annotation-value} # Optional. ScaledJob annotations are applied to child Jobs +spec: + jobTargetRef: + parallelism: 1 # [max number of desired pods](https://kubernetes.io/docs/concepts/workloads/controllers/job/#controlling-parallelism) + completions: 1 # [desired number of successfully finished pods](https://kubernetes.io/docs/concepts/workloads/controllers/job/#controlling-parallelism) + activeDeadlineSeconds: 600 # Specifies the duration in seconds relative to the startTime that the job may be active before the system tries to terminate it; value must be positive integer + backoffLimit: 6 # Specifies the number of retries before marking this job failed. Defaults to 6 + template: + # describes the [job template](https://kubernetes.io/docs/concepts/workloads/controllers/job) + pollingInterval: 30 # Optional. Default: 30 seconds + successfulJobsHistoryLimit: 5 # Optional. Default: 100. How many completed jobs should be kept. + failedJobsHistoryLimit: 5 # Optional. Default: 100. How many failed jobs should be kept. + envSourceContainerName: {container-name} # Optional. Default: .spec.JobTargetRef.template.spec.containers[0] + minReplicaCount: 10 # Optional. Default: 0 + maxReplicaCount: 100 # Optional. Default: 100 + rolloutStrategy: gradual # Deprecated: Use rollout.strategy instead (see below). + rollout: + strategy: gradual # Optional. Default: default. Which Rollout Strategy KEDA will use. + propagationPolicy: foreground # Optional. Default: background. Kubernetes propagation policy for cleaning up existing jobs during rollout. + scalingStrategy: + strategy: "custom" # Optional. Default: default. Which Scaling Strategy to use. + customScalingQueueLengthDeduction: 1 # Optional. A parameter to optimize custom ScalingStrategy. + customScalingRunningJobPercentage: "0.5" # Optional. A parameter to optimize custom ScalingStrategy. + pendingPodConditions: # Optional. A parameter to calculate pending job count per the specified pod conditions + - "Ready" + - "PodScheduled" + - "AnyOtherCustomPodCondition" + multipleScalersCalculation : "max" # Optional. Default: max. Specifies how to calculate the target metrics when multiple scalers are defined. + triggers: + # {list of triggers to create jobs} +``` + +You can find all supported triggers [here](../scalers). + +## jobTargetRef + +```yaml + jobTargetRef: + parallelism: 1 # Optional. Max number of desired instances ([docs](https://kubernetes.io/docs/concepts/workloads/controllers/job/#controlling-parallelism)) + completions: 1 # Optional. Desired number of successfully finished instances ([docs](https://kubernetes.io/docs/concepts/workloads/controllers/job/#controlling-parallelism)) + activeDeadlineSeconds: 600 # Optional. Specifies the duration in seconds relative to the startTime that the job may be active before the system tries to terminate it; value must be positive integer + backoffLimit: 6 # Optional. Specifies the number of retries before marking this job failed. Defaults to 6 +``` + +The `jobTargetRef` is a batch/v1 `JobSpec` object; refer to the Kubernetes API for [more details](https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/job-v1/#JobSpec) about the fields. The `template` field is required. + + +## pollingInterval + +```yaml + pollingInterval: 30 # Optional. Default: 30 seconds +``` + +This is the interval to check each trigger on. By default, KEDA will check each trigger source on every ScaledJob every 30 seconds. + + +## successfulJobsHistoryLimit, failedJobsHistoryLimit + +```yaml + successfulJobsHistoryLimit: 5 # Optional. Default: 100. How many completed jobs should be kept. + failedJobsHistoryLimit: 5 # Optional. Default: 100. How many failed jobs should be kept. +``` + +The `successfulJobsHistoryLimit` and `failedJobsHistoryLimit` fields are optional. These fields specify how many completed and failed jobs should be kept. By default, they are set to 100. + +This concept is similar to [Jobs History Limits](https://kubernetes.io/docs/tasks/job/automated-tasks-with-cron-jobs/#jobs-history-limits) allowing you to learn what the outcomes of your jobs are. + +The actual number of jobs could exceed the limit in a short time. However, it is going to resolve in the cleanup period. Currently, the cleanup period is the same as the Polling interval. + + +## envSourceContainerName + +```yaml + envSourceContainerName: {container-name} # Optional. Default: .spec.JobTargetRef.template.spec.containers[0] +``` + +This optional property specifies the name of container in the Job, from which KEDA should try to get environment properties holding secrets etc. If it is not defined it, KEDA will try to get environment properties from the first Container, ie. from `.spec.JobTargetRef.template.spec.containers[0]`. + +___ +## minReplicaCount + +```yaml + minReplicaCount: 10 # Optional. Default: 0 +``` + +The min number of jobs that is created by default. This can be useful to avoid bootstrapping time of new jobs. If minReplicaCount is greater than maxReplicaCount, minReplicaCount will be set to maxReplicaCount. + +New messages may create new jobs - within the limits imposed by maxReplicaCount - in order to reach the state where minReplicaCount jobs are always running. For example, if one sets minReplicaCount to 2 then there will be 2 jobs running permanently. Using a targetValue of 1, if 3 new messages are sent, 2 of those messages will be processed on the already running jobs but another 3 jobs will be created in order to fulfill the desired state dictated by the minReplicaCount parameter that is set to 2. + +## maxReplicaCount + +```yaml + maxReplicaCount: 100 # Optional. Default: 100 +``` + +The max number of pods that is created within a single polling period. If there are running jobs, the number of running jobs will be deducted. This table is an example of the scaling logic. + +| Queue Length | Max Replica Count | Target Average Value | Running Job Count | Number of the Scale | +|--------------|-------------------|----------------------|-------------------|---------------------| +| 10 | 3 | 1 | 0 | 3 | +| 10 | 3 | 2 | 0 | 3 | +| 10 | 3 | 1 | 1 | 2 | +| 10 | 100 | 1 | 0 | 10 | +| 4 | 3 | 5 | 0 | 1 | + +* **Queue Length:** The number of items in the queue. +* **Target Average Value:** The number of messages that will be consumed on a job. It is defined on the scaler side. e.g. `queueLength` on `Azure Storage Queue` scaler. +* **Running Job Count:** How many jobs are running. +* **Number of the Scale:** The number of the job that is created. + + +## rollout + +```yaml + rollout: + strategy: gradual # Optional. Default: default. Which Rollout Strategy KEDA will use. + propagationPolicy: foreground # Optional. Default: background. Kubernetes propagation policy for cleaning up existing jobs during +``` + +The optional property rollout.strategy specifies the rollout strategy KEDA will use while updating an existing ScaledJob. +Possible values are `default` or `gradual`. \ +When using the `default` rolloutStrategy, KEDA will terminate existing Jobs whenever a ScaledJob is being updated. Then, it will recreate those Jobs with the latest specs. The order in which this termination happens can be configured via the rollout.propagationPolicy property. By default, the kubernetes background propagation is used. To change this behavior specify set propagationPolicy to `foreground`. For further information see [Kubernetes Documentation](https://kubernetes.io/docs/tasks/administer-cluster/use-cascading-deletion/#use-foreground-cascading-deletion). +On the `gradual` rolloutStartegy, whenever a ScaledJob is being updated, KEDA will not delete existing Jobs. Only new Jobs will be created with the latest specs. + + +## scalingStrategy + +```yaml +scalingStrategy: + strategy: "default" # Optional. Default: default. Which Scaling Strategy to use. +``` + +Select a Scaling Strategy. Possible values are `default`, `custom`, or `accurate`. The default value is `default`. + +> 💡 **NOTE:** +> +>`maxScale` is not the running Job count. It is measured as follows: + >```go + >maxScale = min(scaledJob.MaxReplicaCount(), divideWithCeil(queueLength, targetAverageValue)) + >``` + >That means it will use the value of `queueLength` divided by `targetAvarageValue` unless it is exceeding the `MaxReplicaCount`. +> +>`RunningJobCount` represents the number of jobs that are currently running or have not finished yet. +> +>It is measured as follows: +>```go +>if !e.isJobFinished(&job) { +> runningJobs++ +>} +>``` +>`PendingJobCount` provides an indication of the amount of jobs that are in pending state. Pending jobs can be calculated in two ways: +> - Default behavior - Job that have not finished yet **and** the underlying pod is either not running or has not been completed yet +> - Setting `pendingPodConditions` - Job that has not finished yet **and** all specified pod conditions of the underlying pod mark as `true` by kubernetes. +> +>It is measured as follows: +>```go +>if !e.isJobFinished(&job) { +> if len(scaledJob.Spec.ScalingStrategy.PendingPodConditions) > 0 { +> if !e.areAllPendingPodConditionsFulfilled(&job, scaledJob.Spec.ScalingStrategy.PendingPodConditions) { +> pendingJobs++ +> } +> } else { +> if !e.isAnyPodRunningOrCompleted(&job) { +> pendingJobs++ +> } +> } +>} +>``` + +**default** +This logic is the same as Job for V1. The number of the scale will be calculated as follows. + +_The number of the scale_ + +```go +maxScale - runningJobCount +``` + +**custom** +You can customize the default scale logic. You need to configure the following parameters. If you don't configure it, then the strategy will be `default.` + +```yaml +customScalingQueueLengthDeduction: 1 # Optional. A parameter to optimize custom ScalingStrategy. +customScalingRunningJobPercentage: "0.5" # Optional. A parameter to optimize custom ScalingStrategy. +``` + +_The number of the scale_ + +```go +min(maxScale-int64(*s.CustomScalingQueueLengthDeduction)-int64(float64(runningJobCount)*(*s.CustomScalingRunningJobPercentage)), maxReplicaCount) +``` + +**accurate** +If the scaler returns `queueLength` (number of items in the queue) that does not include the number of locked messages, this strategy is recommended. `Azure Storage Queue` is one example. You can use this strategy if you delete a message once your app consumes it. + +```go +if (maxScale + runningJobCount) > maxReplicaCount { + return maxReplicaCount - runningJobCount + } + return maxScale - pendingJobCount +``` +For more details, you can refer to [this PR](https://github.com/kedacore/keda/pull/1227). + + +### multipleScalersCalculation + +```yaml +scalingStrategy: + multipleScalersCalculation : "max" # Optional. Default: max. Specifies how to calculate the target metrics (`queueLength` and `maxScale`) when multiple scalers are defined. +``` +Select a behavior if you have multiple triggers. Possible values are `max`, `min`, `avg`, or `sum`. The default value is `max`. + +* **max:** - Use metrics from the scaler that has the max number of `queueLength`. (default) +* **min:** - Use metrics from the scaler that has the min number of `queueLength`. +* **avg:** - Sum up all the active scalers metrics and divide by the number of active scalers. +* **sum:** - Sum up all the active scalers metrics. diff --git a/content/docs/2.15/reference/scaledobject-spec.md b/content/docs/2.15/reference/scaledobject-spec.md new file mode 100644 index 000000000..391a968c8 --- /dev/null +++ b/content/docs/2.15/reference/scaledobject-spec.md @@ -0,0 +1,254 @@ + ++++ +title = "ScaledObject specification" +weight = 3000 ++++ + +## Overview + +This specification describes the `ScaledObject` Custom Resource definition that defines the triggers and scaling behaviors used by KEDA to scale `Deployment`, `StatefulSet` and `Custom Resource` target resources. The `.spec.ScaleTargetRef` section holds the reference to the target resource, defined in [_scaledobject_types.go_](https://github.com/kedacore/keda/blob/main/apis/keda/v1alpha1/scaledobject_types.go). + +```yaml +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: {scaled-object-name} + annotations: + scaledobject.keda.sh/transfer-hpa-ownership: "true" # Optional. Use to transfer an existing HPA ownership to this ScaledObject + validations.keda.sh/hpa-ownership: "true" # Optional. Use to disable HPA ownership validation on this ScaledObject + autoscaling.keda.sh/paused: "true" # Optional. Use to pause autoscaling of objects explicitly +spec: + scaleTargetRef: + apiVersion: {api-version-of-target-resource} # Optional. Default: apps/v1 + kind: {kind-of-target-resource} # Optional. Default: Deployment + name: {name-of-target-resource} # Mandatory. Must be in the same namespace as the ScaledObject + envSourceContainerName: {container-name} # Optional. Default: .spec.template.spec.containers[0] + pollingInterval: 30 # Optional. Default: 30 seconds + cooldownPeriod: 300 # Optional. Default: 300 seconds + idleReplicaCount: 0 # Optional. Default: ignored, must be less than minReplicaCount + minReplicaCount: 1 # Optional. Default: 0 + maxReplicaCount: 100 # Optional. Default: 100 + fallback: # Optional. Section to specify fallback options + failureThreshold: 3 # Mandatory if fallback section is included + replicas: 6 # Mandatory if fallback section is included + advanced: # Optional. Section to specify advanced options + restoreToOriginalReplicaCount: true/false # Optional. Default: false + horizontalPodAutoscalerConfig: # Optional. Section to specify HPA related options + name: {name-of-hpa-resource} # Optional. Default: keda-hpa-{scaled-object-name} + behavior: # Optional. Use to modify HPA's scaling behavior + scaleDown: + stabilizationWindowSeconds: 300 + policies: + - type: Percent + value: 100 + periodSeconds: 15 + triggers: + # {list of triggers to activate scaling of the target resource} +``` + +## scaleTargetRef + +```yaml + scaleTargetRef: + apiVersion: {api-version-of-target-resource} # Optional. Default: apps/v1 + kind: {kind-of-target-resource} # Optional. Default: Deployment + name: {name-of-target-resource} # Mandatory. Must be in the same namespace as the ScaledObject + envSourceContainerName: {container-name} # Optional. Default: .spec.template.spec.containers[0] +``` + +The reference to the resource this ScaledObject is configured for. This is the resource KEDA will scale up/down and set up an HPA for, based on the triggers defined in `triggers:`. + +To scale Kubernetes Deployments only `name` need be specified. To scale a different resource such as StatefulSet or Custom Resource (that defines `/scale` subresource), appropriate `apiVersion` (following standard Kubernetes convention, ie. `{api}/{version}`) and `kind` need to be specified. + +`envSourceContainerName` is an optional property that specifies the name of container in the target resource, from which KEDA should try to get environment properties holding secrets etc. If it is not defined, KEDA will try to get environment properties from the first Container, ie. from `.spec.template.spec.containers[0]`. + +**Assumptions:** Resource referenced by `name` (and `apiVersion`, `kind`) is in the same namespace as the ScaledObject + + +## pollingInterval +```yaml + pollingInterval: 30 # Optional. Default: 30 seconds +``` + +This is the interval to check each trigger on. By default, KEDA will check each trigger source on every ScaledObject every 30 seconds. + +**Example:** in a queue scenario, KEDA will check the queueLength every `pollingInterval`, and scale the resource up or down accordingly. + + +## cooldownPeriod +```yaml + cooldownPeriod: 300 # Optional. Default: 300 seconds +``` + +The period to wait after the last trigger reported active before scaling the resource back to 0, in seconds. By default, it's 300 (5 minutes). + +The `cooldownPeriod` only applies after a trigger occurs; when you first create your `Deployment` (or `StatefulSet`/`CustomResource`), KEDA will immediately scale it to `minReplicaCount`. Additionally, the KEDA `cooldownPeriod` only applies when scaling to 0; scaling from 1 to N replicas is handled by the [Kubernetes Horizontal Pod Autoscaler](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/../concepts/scaling-deployments.md#support-for-cooldowndelay). + +**Example:** wait 5 minutes after the last time KEDA checked the queue and it was empty. (this is obviously dependent on `pollingInterval`) + + +## initialCooldownPeriod +```yaml + InitialCooldownPeriod: 120 # Optional. Default: 0 seconds +``` +The delay before the `cooldownPeriod` starts after the initial creation of the `ScaledObject`, in seconds. By default, it's 0, meaning the `cooldownPeriod` begins immediately upon creation. If set to a value such as 120 seconds, the `cooldownPeriod` will only start after the `ScaledObject` has been active for that duration. + +This parameter is particularly useful for managing the scale-down behavior during the initial phase of a `ScaledObject`. For instance, if `InitialCooldownPeriod` is set to 120 seconds, KEDA will not scale the resource back to 0 until 120 seconds have passed since the `ScaledObject` creation, regardless of the activity triggers. This allows for a grace period in situations where immediate scaling down after creation is not desirable. + +**Example:** Wait 120 seconds after the `ScaledObject` is created before starting the `cooldownPeriod`. For instance, if the `InitialCooldownPeriod` is set to 120 seconds, KEDA will not initiate the cooldown process until 120 seconds have passed since the `ScaledObject` was first created, regardless of the triggers' activity. This ensures a buffer period where the resource won’t be scaled down immediately after creation. (Note: This setting is independent of the `pollingInterval`.) + + +## idleReplicaCount + +```yaml + idleReplicaCount: 0 # Optional. Default: ignored, must be less than minReplicaCount +``` + +> 💡 **NOTE:** Due to limitations in HPA controller the only supported value for this property is 0, it will not work correctly otherwise. See this [issue](https://github.com/kedacore/keda/issues/2314) for more details. +> +> In some cases, you always need at least `n` pod running. Thus, you can omit this property and set `minReplicaCount` to `n`. +> +> **Example** You set `minReplicaCount` to 1 and `maxReplicaCount` to 10. If there’s no activity on triggers, the target resource is scaled down to `minReplicaCount` (1). Once there are activities, the target resource will scale base on the HPA rule. If there’s no activity on triggers, the resource is again scaled down to `minReplicaCount` (1). + +If this property is set, KEDA will scale the resource down to this number of replicas. If there's some activity on target triggers KEDA will scale the target resource immediately to `minReplicaCount` and then will be scaling handled by HPA. When there is no activity, the target resource is again scaled down to `idleReplicaCount`. This setting must be less than `minReplicaCount`. + +**Example:** If there's no activity on triggers the target resource is scaled down to `idleReplicaCount` (0), once there is an activity the target resource is immediately scaled to `minReplicaCount` (10) and then up to `maxReplicaCount` (100) as needed. If there's no activity on triggers the resource is again scaled down to `idleReplicaCount` (0). + + +## minReplicaCount + +```yaml + minReplicaCount: 1 # Optional. Default: 0 +``` + +Minimum number of replicas KEDA will scale the resource down to. By default, it's scale to zero, but you can use it with some other value as well. + +## maxReplicaCount + +```yaml + maxReplicaCount: 100 # Optional. Default: 100 +``` +This setting is passed to the HPA definition that KEDA will create for a given resource and holds the maximum number of replicas of the target resource. + + +## fallback +```yaml + fallback: # Optional. Section to specify fallback options + failureThreshold: 3 # Mandatory if fallback section is included + replicas: 6 # Mandatory if fallback section is included +``` + +The `fallback` section is optional. It defines a number of replicas to fall back to if a scaler is in an error state. + +KEDA will keep track of the number of consecutive times each scaler has failed to get metrics from its source. Once that value passes the `failureThreshold`, instead of not propagating a metric to the HPA (the default error behaviour), the scaler will, instead, return a normalised metric using the formula: +``` +target metric value * fallback replicas +``` +Due to the HPA metric being of type `AverageValue` (see below), this will have the effect of the HPA scaling the deployment to the defined number of fallback replicas. + +**Example:** When my instance of prometheus is unavailable 3 consecutive times, KEDA will change the HPA metric such that the deployment will scale to 6 replicas. + +There are a few limitations to using a fallback: + - It only supports scalers whose target is an `AverageValue` metric. Thus, it is **not** supported by the CPU & memory scalers, or by scalers whose metric target type is `Value`. In these cases, it will assume that fallback is disabled. + - It is only supported by `ScaledObjects` **not** `ScaledJobs`. + + +## advanced + +### restoreToOriginalReplicaCount + +```yaml +advanced: + restoreToOriginalReplicaCount: true/false # Optional. Default: false +``` + +This property specifies whether the target resource (`Deployment`, `StatefulSet`,...) should be scaled back to original replicas count, after the `ScaledObject` is deleted. +Default behavior is to keep the replica count at the same number as it is in the moment of `ScaledObject's` deletion. + +For example a `Deployment` with `3 replicas` is created, then `ScaledObject` is created and the `Deployment` is scaled by KEDA to `10 replicas`. Then `ScaledObject` is deleted: + 1. if `restoreToOriginalReplicaCount = false` (default behavior) then `Deployment` replicas count is `10` + 2. if `restoreToOriginalReplicaCount = true` then `Deployment` replicas count is set back to `3` (the original value) + + +### horizontalPodAutoscalerConfig + +```yaml +advanced: + horizontalPodAutoscalerConfig: # Optional. Section to specify HPA related options + name: {name-of-hpa-resource} # Optional. Default: keda-hpa-{scaled-object-name} + behavior: # Optional. Use to modify HPA's scaling behavior + scaleDown: + stabilizationWindowSeconds: 300 + policies: + - type: Percent + value: 100 + periodSeconds: 15 +``` + +#### horizontalPodAutoscalerConfig.name + +The name of the HPA resource KEDA will create. By default, it's `keda-hpa-{scaled-object-name}` + +#### horizontalPodAutoscalerConfig.behavior + +Starting from Kubernetes v1.18 the autoscaling API allows scaling behavior to be configured through the HPA behavior field. This way one can directly affect scaling of 1<->N replicas, which is internally being handled by HPA. KEDA would feed values from this section directly to the HPA's `behavior` field. Please follow [Kubernetes documentation](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/../concepts/scaling-deployments.md#configurable-scaling-behavior) for details. + +**Assumptions:** KEDA must be running on Kubernetes cluster v1.18+, in order to be able to benefit from this setting. + + + +```yaml +advanced: + scalingModifiers: # Optional. Section to specify scaling modifiers + target: {target-value-to-scale-on} # Mandatory. New target if metrics are anyhow composed together + activationTarget: {activation-target-value-to-scale-on} # Optional. New activation target if metrics are anyhow composed together + metricType: {metric-tipe-for-the-modifier} # Optional. Metric type to be used if metrics are anyhow composed together + formula: {formula-for-fetched-metrics} # Mandatory. Formula for calculation +``` + +### scalingModifiers + +The `scalingModifiers` is optional and **experimental**. If defined, both `target` and `formula` are mandatory. Using this structure creates `composite-metric` for the HPA that will replace all requests for external metrics and handle them internally. With `scalingModifiers` each trigger used in the `formula` **must** have a name defined. + +#### scalingModifiers.target + +`target` defines new target value to scale on for the composed metric. + +#### scalingModifiers.activationTarget + +`activationTarget` defines a new [activation target value](../concepts/scaling-deployments.md#activating-and-scaling-thresholds) to scale on for the composed metric. (Default: `0`, Optional) + +#### scalingModifiers.metricType + +`metricType` defines metric type used for this new `composite-metric`. (Values: `AverageValue`, `Value`, Default: `AverageValue`, Optional) + +#### scalingModifiers.formula + + `formula` composes metrics together and allows them to be modified/manipulated. It accepts mathematical/conditional statements using [this external project](https://github.com/antonmedv/expr). If the `fallback` scaling feature is in effect, the `formula` will NOT modify its metrics (therefore it modifies metrics only when all of their triggers are healthy). Complete language definition of `expr` package can be found [here](https://expr.medv.io/docs/Language-Definition). Formula must return a single value (not boolean). + +For examples of this feature see section [Scaling Modifiers](../concepts/scaling-deployments.md#scaling-modifiers-experimental). + + +## triggers + +```yaml + triggers: + # {list of triggers to activate scaling of the target resource} +``` + +> 💡 **NOTE:** You can find all supported triggers [here](../scalers). + +Trigger fields: +- **type**: The type of trigger to use. (Mandatory) +- **metadata**: The configuration parameters that the trigger requires. (Mandatory) +- **name**: Name for this trigger. This value can be used to easily distinguish this specific trigger and its metrics when consuming [Prometheus metrics](../operate/prometheus.md). By default, the name is generated from the trigger type. (Optional) +- **useCachedMetrics**: Enables caching of metric values during polling interval (as specified in `.spec.pollingInterval`). For more information, see ["Caching Metrics"](../concepts/scaling-deployments.md#caching-metrics). (Values: `false`, `true`, Default: `false`, Optional) +- **authenticationRef**: A reference to the `TriggerAuthentication` or `ClusterTriggerAuthentication` object that is used to authenticate the scaler with the environment. + - More details can be found [here](../concepts/authentication). (Optional) +- **metricType**: The type of metric that should be used. (Values: `AverageValue`, `Value`, `Utilization`, Default: `AverageValue`, Optional) + - Learn more about how the [Horizontal Pod Autoscaler (HPA) calculates `replicaCount`](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale-walkthrough/) based on metric type and value. + - To show the differences between the metric types, let's assume we want to scale a deployment with 3 running replicas based on a queue of messages: + - With `AverageValue` metric type, we can control how many messages, on average, each replica will handle. If our metric is the queue size, the threshold is 5 messages, and the current message count in the queue is 20, HPA will scale the deployment to 20 / 5 = 4 replicas, regardless of the current replica count. + - The `Value` metric type, on the other hand, can be used when we don't want to take the average of the given metric across all replicas. For example, with the `Value` type, we can control the average time of messages in the queue. If our metric is average time in the queue, the threshold is 5 milliseconds, and the current average time is 20 milliseconds, HPA will scale the deployment to 3 * 20 / 5 = 12. + +> ⚠️ **NOTE:** All scalers, except CPU and Memory, support metric types `AverageValue` and `Value` while CPU and Memory scalers both support `AverageValue` and `Utilization`. diff --git a/content/docs/2.15/scalers/aws-sqs.md b/content/docs/2.15/scalers/aws-sqs.md index 95eb2387c..ec059ba86 100644 --- a/content/docs/2.15/scalers/aws-sqs.md +++ b/content/docs/2.15/scalers/aws-sqs.md @@ -30,8 +30,8 @@ triggers: **Parameter list:** -- `queueURL` - Full URL for the SQS Queue. The simple name of the queue can be used in case there's no ambiguity. (Optional, You can use this instead of `queueURLFromEnv` parameter) -- `queueURLFromEnv` - Name of the environment variable on the scale target to read the queue URL from. (Optional, You can use this instead of `queueURL` parameter) +- `queueURL` - Full URL for the SQS Queue. The short name of the queue can be used if there's no ambiguity. (Optional. Only one of `queueURL` and `queueURLFromEnv` is required. If both are provided, `queueURL` is used.) +- `queueURLFromEnv` - Name of the environment variable on the scale target to read the queue URL from. (Optional. Only one of `queueURL` and `queueURLFromEnv` is required.) - `queueLength` - Target value for queue length passed to the scaler. Example: if one pod can handle 10 messages, set the queue length target to 10. If the actual messages in the SQS Queue is 30, the scaler scales to 3 pods. (default: 5) - `activationQueueLength` - Target value for activating the scaler. Learn more about activation [here](./../concepts/scaling-deployments.md#activating-and-scaling-thresholds). (Default: `0`, Optional) diff --git a/content/docs/2.15/scalers/azure-pipelines.md b/content/docs/2.15/scalers/azure-pipelines.md index 4933bfae4..3f8fa4373 100644 --- a/content/docs/2.15/scalers/azure-pipelines.md +++ b/content/docs/2.15/scalers/azure-pipelines.md @@ -79,7 +79,7 @@ Finally, it is also possible get the pool ID from the response of a HTTP request ### Supporting demands in agents -By default, if you do not wish to use demands in your agent scaler then it will scale based simply on the pool's queue length. +By default, if you do not use demands in your agent scaler then it scales based on the pool's queue length. Demands (Capabilities) are useful when you have multiple agents with different capabilities existing within the same pool, for instance in a kube cluster you may have an agent supporting dotnet5, dotnet6, java or maven; diff --git a/content/docs/2.15/scalers/azure-service-bus.md b/content/docs/2.15/scalers/azure-service-bus.md index ca498fbc9..e7b9c6bf3 100644 --- a/content/docs/2.15/scalers/azure-service-bus.md +++ b/content/docs/2.15/scalers/azure-service-bus.md @@ -36,7 +36,7 @@ triggers: **Parameter list:** - `messageCount` - Amount of active messages in your Azure Service Bus queue or topic to scale on. -- `activationMessageCount` - Target value for activating the scaler. Learn more about activation [here](./../concepts/scaling-deployments.md#activating-and-scaling-thresholds). (Default: `0`, Optional) +- `activationMessageCount` - Target value for activating the scaler. Learn more about activation [here](../concepts/scaling-deployments#activating-and-scaling-thresholds). (Default: `0`, Optional) - `queueName` - Name of the Azure Service Bus queue to scale on. (Optional) - `topicName` - Name of the Azure Service Bus topic to scale on. (Optional) - `subscriptionName` - Name of the Azure Service Bus queue to scale on. (Optional*, Required when `topicName` is specified) @@ -118,4 +118,4 @@ When KEDA logs show errors similar to `invalid queue runtime properties: no Coun Consider applying one of the following mitigations: - Scaling the Azure Service Bus namespace to a higher SKU, or use premium - Increase the polling interval of the ScaledObject/ScaledJob -- Use [caching of metrics](./../concepts/scaling-deployments/#caching-metrics) +- Use [caching of metrics](../concepts/scaling-deployments/#caching-metrics) diff --git a/content/docs/2.15/scalers/datadog.md b/content/docs/2.15/scalers/datadog.md index cf7e01d24..540fa96f5 100644 --- a/content/docs/2.15/scalers/datadog.md +++ b/content/docs/2.15/scalers/datadog.md @@ -37,7 +37,7 @@ triggers: - `query` - The Datadog query to run. - `queryValue` - Value to reach to start scaling (This value can be a float). -- `activationQueryValue` - Target value for activating the scaler. Learn more about activation [here](./../concepts/scaling-deployments.md#activating-and-scaling-thresholds).(Default: `0`, Optional, This value can be a float) +- `activationQueryValue` - Target value for activating the scaler. Learn more about activation [here](../concepts/scaling-deployments#activating-and-scaling-thresholds).(Default: `0`, Optional, This value can be a float) - `queryAggregator` - When `query` is multiple queries, comma-seperated, this sets how to aggregate the multiple results. (Values: `max`, `average`, Required only when `query` contains multiple queries) - `type` - Whether to start scaling based on the value or the average between pods. (Values: `average`, `global`, Default:`average`, Optional) - `age`: The time window (in seconds) to retrieve metrics from Datadog. (Default: `90`, Optional) @@ -133,7 +133,7 @@ often (per `ScaledObject`) we query Datadog for a metric. When scaling from 0 to 1, the polling interval is controlled by KEDA, using [the `spec.pollingInterval` parameter in the `ScaledObject` -definition](../concepts/scaling-deployments/#pollinginterval). For example, if +definition](../reference/scaledobject-spec/#pollinginterval). For example, if this parameter is set to `60`, KEDA will poll Datadog for a metric value every 60 seconds while the number of replicas is 0. diff --git a/content/docs/2.15/scalers/redis-sentinel-lists.md b/content/docs/2.15/scalers/redis-sentinel-lists.md index 07bf4b25e..1ce8b9805 100644 --- a/content/docs/2.15/scalers/redis-sentinel-lists.md +++ b/content/docs/2.15/scalers/redis-sentinel-lists.md @@ -42,7 +42,7 @@ triggers: - Both the hostname, username and password fields need to be set to the names of the environment variables in the target deployment that contain the host name, username and password respectively. - `sentinelUsernameFromEnv` - Environment variable to read the authentication username from to authenticate with the Redis Sentinel server. - `sentinelPasswordFromEnv` - Environment variable to read the authentication password from to authenticate with the Redis Sentinel server. -- `sentinelMaster` - The name of the master in Sentinel to get the Redis server address for. +- sentinelMaster - The name of the primary (still referred to as the 'master' in Sentinel) to get the Redis server address for. - `listName` - Name of the Redis List that you want to monitor. - `listLength` - Average target value to trigger scaling actions. - `activationListLength` - Target value for activating the scaler. Learn more about activation [here](./../concepts/scaling-deployments.md#activating-and-scaling-thresholds). (Default: `0`, Optional) @@ -55,7 +55,7 @@ Some parameters could be provided using environmental variables, instead of sett - `addressesFromEnv` - The hosts and their respective ports of the Redis Sentinel nodes, similar to `addresses`, but reads it from an environment variable on the scale target. - `hostsFromEnv` - The hosts of the Redis Sentinel nodes, similar to `hosts`, but reads it from an environment variable on the scale target. - `portsFromEnv` - The corresponding ports for the hosts of the Redis Sentinel nodes, similar to `ports`, but reads it from an environment variable on the scale target. -- `sentinelMasterFromEnv` - The name of the master in Sentinel to get the Redis server address for, similar to `sentinelMaster`, but reads it from an environment variable on the scale target. +- `sentinelMasterFromEnv` - The name of the primary (still referred to as the 'master' in Sentinel) to get the Redis server address for; similar to `sentinelMaster`, but reads it from an environment variable on the scale target. ### Authentication Parameters @@ -66,7 +66,7 @@ You can authenticate by using a password. - `addresses` - Comma separated list of host:port format. - `hosts` - Comma separated list of hostname of the Redis Sentinel nodes. If specified, the `ports` should also be specified. - `ports` - Comma separated list of ports of the Redis Sentinel nodes. If specified, the `hosts` should also be specified. -- `sentinelMaster` - The name of the master in Sentinel to get the Redis server address for. +- `sentinelMaster` - The name of the primary (still referred to as the 'master' in Sentinel) to get the Redis server address for. **Authentication:** @@ -134,7 +134,7 @@ spec: addresses: node1:26379, node2:26379, node3:26379 listName: mylist listLength: "10" - sentinelMaster: "mymaster" + sentinelMaster: "myprimary" authenticationRef: name: keda-trigger-auth-redis-secret ``` diff --git a/content/docs/2.15/scalers/redis-sentinel-streams.md b/content/docs/2.15/scalers/redis-sentinel-streams.md index 0070884d9..95db3210d 100644 --- a/content/docs/2.15/scalers/redis-sentinel-streams.md +++ b/content/docs/2.15/scalers/redis-sentinel-streams.md @@ -64,7 +64,7 @@ triggers: - `sentinelUsernameFromEnv` - Name of the environment variable your deployment uses to get the Redis Sentinel username. (Optional) - `sentinelPasswordFromEnv` - Name of the environment variable your deployment uses to get the Redis Sentinel password. (Optional) -- `sentinelMaster` - The name of the master in Sentinel to get the Redis server address for. +- `sentinelMaster` - The name of the primary (still referred to as the 'master' in Sentinel) in Sentinel to get the Redis server address for. - `stream` - Name of the Redis Stream. - `consumerGroup` - Name of the Consumer group associated with Redis Stream. > Setting the `consumerGroup` causes the scaler to operate on `pendingEntriesCount`. Lack of `consumerGroup` will cause the scaler to be based on `streamLength` @@ -81,7 +81,7 @@ Some parameters could be provided using environmental variables, instead of sett - `addressesFromEnv` - The hosts and corresponding ports of Redis Sentinel nodes, similar to `addresses`, but reads it from an environment variable on the scale target. Name of the environment variable your deployment uses to get the URLs of Redis Sentinel nodes. The resolved hosts should follow a format like `node1:26379, node2:26379, node3:26379 ...`. - `hostsFromEnv` - The hosts of the Redis Sentinel nodes, similar to `hosts`, but reads it from an environment variable on the scale target. - `portsFromEnv` - The corresponding ports for the hosts of Redis Sentinel nodes, similar to `ports`, but reads it from an environment variable on the scale target. -- `sentinelMasterFromEnv` - The name of the master in Sentinel to get the Redis server address for, similar to `sentinelMaster`, but reads it from an environment variable on the scale target. +- `sentinelMasterFromEnv` - The name of the primary (still referred to as the 'master' in Sentinel) in Sentinel to get the Redis server address for, similar to `sentinelMaster`, but reads it from an environment variable on the scale target. ### Authentication Parameters @@ -117,7 +117,7 @@ spec: stream: my-stream consumerGroup: consumer-group-1 pendingEntriesCount: "10" - sentinelMaster: "mymaster" + sentinelMaster: "myprimary" ``` #### Using `TriggerAuthentication` @@ -197,7 +197,7 @@ spec: stream: my-stream consumerGroup: consumer-group-1 pendingEntriesCount: "10" - sentinelMaster: "mymaster" + sentinelMaster: "myprimary" authenticationRef: name: keda-redis-stream-triggerauth # name of the TriggerAuthentication resource ``` @@ -227,7 +227,7 @@ spec: passwordFromEnv: REDIS_PASSWORD # name of the environment variable in the Deployment stream: my-stream streamLength: "50" - sentinelMaster: "mymaster" + sentinelMaster: "myprimary" ``` #### Using `lagCount` From 8f8bb08025569f73907f0616416022d42e3e0c9e Mon Sep 17 00:00:00 2001 From: shubhusion Date: Tue, 30 Jul 2024 20:55:56 +0530 Subject: [PATCH 12/33] Added a Note Signed-off-by: shubhusion --- content/docs/2.14/operate/setupscaler.md | 6 +++++- content/docs/2.15/operate/setupscaler.md | 6 +++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/content/docs/2.14/operate/setupscaler.md b/content/docs/2.14/operate/setupscaler.md index 7279354e3..a56feb769 100644 --- a/content/docs/2.14/operate/setupscaler.md +++ b/content/docs/2.14/operate/setupscaler.md @@ -26,7 +26,11 @@ KEDA supports various scalers that correspond to different event sources or trig 1. Visit the [KEDA Scalers documentation](https://keda.sh/docs/2.14/scalers/) and browse through the list of available scalers. 2. Identify the scaler that matches the event source you want to use for scaling your application. For example: - - If you want to scale based on incoming HTTP traffic, you would need the **HTTP scaler**. + - If you want to scale based on incoming HTTP traffic, you would need the **HTTP Add-on**. + + > **Note:** + > The HTTP Add-on is still in beta stage and may not provide the full functionality or stability expected in a production environment. + - If you want to scale based on messages in a RabbitMQ queue, you would need the **RabbitMQ scaler**. - If you want to scale based on a cron schedule, you would need the **Cron scaler**. 3. Open the documentation page for your chosen scaler and familiarize yourself with its specific requirements and configuration options. diff --git a/content/docs/2.15/operate/setupscaler.md b/content/docs/2.15/operate/setupscaler.md index 7279354e3..a56feb769 100644 --- a/content/docs/2.15/operate/setupscaler.md +++ b/content/docs/2.15/operate/setupscaler.md @@ -26,7 +26,11 @@ KEDA supports various scalers that correspond to different event sources or trig 1. Visit the [KEDA Scalers documentation](https://keda.sh/docs/2.14/scalers/) and browse through the list of available scalers. 2. Identify the scaler that matches the event source you want to use for scaling your application. For example: - - If you want to scale based on incoming HTTP traffic, you would need the **HTTP scaler**. + - If you want to scale based on incoming HTTP traffic, you would need the **HTTP Add-on**. + + > **Note:** + > The HTTP Add-on is still in beta stage and may not provide the full functionality or stability expected in a production environment. + - If you want to scale based on messages in a RabbitMQ queue, you would need the **RabbitMQ scaler**. - If you want to scale based on a cron schedule, you would need the **Cron scaler**. 3. Open the documentation page for your chosen scaler and familiarize yourself with its specific requirements and configuration options. From fc9e3a070fb76e4d2c91f39401d95c1c33a1c3ed Mon Sep 17 00:00:00 2001 From: rxg8255 <105524324+rxg8255@users.noreply.github.com> Date: Mon, 1 Jul 2024 06:54:48 -0500 Subject: [PATCH 13/33] Add support for cassandra tls under v2.15 (#1390) Signed-off-by: Ranjith Gopal Co-authored-by: Ranjith Gopal Signed-off-by: shubhusion --- content/docs/2.15/scalers/cassandra.md | 101 ++++++++++++++++++++++++- 1 file changed, 100 insertions(+), 1 deletion(-) diff --git a/content/docs/2.15/scalers/cassandra.md b/content/docs/2.15/scalers/cassandra.md index d662388d3..04c151c3f 100644 --- a/content/docs/2.15/scalers/cassandra.md +++ b/content/docs/2.15/scalers/cassandra.md @@ -45,8 +45,11 @@ You can authenticate by using a password via `TriggerAuthentication` configurati **Password Authentication:** - `password` - Password for configured user to log in to the Cassandra instance. +- `tls` - To enable SSL auth for Cassandra session, set this to enable. If not set, TLS for Cassandra is not used. (Values: enable, disable, Default: disable, Optional). +- `cert` - Certificate path for client authentication. Mandatory if tls enabled. (Optional) +- `key` - Key path for client authentication. Mandatory if tls enabled. (Optional) -### Example +### Example with no TLS auth ```yaml apiVersion: v1 @@ -87,3 +90,99 @@ spec: authenticationRef: name: keda-trigger-auth-cassandra-secret ``` + +### Example with TLS auth + +Since we are passing Cert and Key content as inputs to the scaler, we have to supply writable location for required GSSAPI configurations for the `keda-operator` container. + +##### `sasl/gssapi` in manager.yaml + +If you use YAML declarations to deploy KEDA, add below volume mount and volume to supply writable location for required GSSAPI configurations for the `keda-operator` container. + +``` + volumeMounts: + - mountPath: /tmp/cassandra + name: temp-cassandra-vol + readOnly: false + + volumes: + - name: temp-cassandra-vol + emptyDir: + medium: Memory +``` + +##### `sasl/gssapi` in keda-charts + +If you use Helm Charts to deploy KEDA, add below volume mount and volume to supply writable location for required gssapi configurations. + +``` +volumes.keda.extraVolumeMounts +- mountPath: /tmp/cassandra + name: temp-cassandra-vol + readOnly: false + +volumes.keda.extraVolumes +- name: temp-cassandra-vol + emptyDir: + medium: Memory +``` + +Once we have the writable mount path set up for the certificates and keys. + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: cassandra-secrets +type: Opaque +data: + cassandra_password: CASSANDRA_PASSWORD + tls: enable + cert: + key: + ## Optional parameter ca ## + ca: +--- +apiVersion: keda.sh/v1alpha1 +kind: TriggerAuthentication +metadata: + name: keda-trigger-auth-cassandra-secret +spec: + secretTargetRef: + - parameter: password + name: cassandra-secrets + key: cassandra_password + - parameter: tls + name: cassandra-secrets + key: tls + - parameter: cert + name: cassandra-secrets + key: cert + - parameter: key + name: cassandra-secrets + key: key + ## Optional parameter ca ## + - parameter: ca + name: cassandra-secrets + key: ca +--- +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: cassandra-scaledobject +spec: + scaleTargetRef: + name: nginx-deployment + triggers: + - type: cassandra + metadata: + username: "cassandra" + port: "9042" + clusterIPAddress: "cassandra.default" + consistency: "Quorum" + protocolVersion: "4" + query: "SELECT COUNT(*) FROM test_keyspace.test_table;" + targetQueryValue: "1" + authenticationRef: + name: keda-trigger-auth-cassandra-secret +``` From 0927db294a4e55bf7bd423289cbf835addee4068 Mon Sep 17 00:00:00 2001 From: Caleb Lemoine <21261388+circa10a@users.noreply.github.com> Date: Wed, 3 Jul 2024 00:53:55 -0700 Subject: [PATCH 14/33] feat: splunk scaler docs (#1413) Signed-off-by: shubhusion --- .htmltest.yml | 1 + content/docs/2.15/scalers/splunk.md | 292 ++++++++++++++++++++++++++++ 2 files changed, 293 insertions(+) create mode 100644 content/docs/2.15/scalers/splunk.md diff --git a/.htmltest.yml b/.htmltest.yml index afd8fd6cd..3628b12af 100644 --- a/.htmltest.yml +++ b/.htmltest.yml @@ -4,3 +4,4 @@ CheckExternal: false IgnoreAltMissing: true IgnoreEmptyHref: true IgnoreInternalURLs: + - /docs/2.14/scalers/splunk/ diff --git a/content/docs/2.15/scalers/splunk.md b/content/docs/2.15/scalers/splunk.md new file mode 100644 index 000000000..beee29ea8 --- /dev/null +++ b/content/docs/2.15/scalers/splunk.md @@ -0,0 +1,292 @@ ++++ +title = "Splunk" +availability = "v2.15+" +maintainer = "Community" +category = "Data & Storage" +description = "Scale applications based on Splunk saved search results." +go_file = "splunk_scaler" ++++ + +### Trigger Specification + +This specification describes the `splunk` trigger that scales based on the result of a [saved search](https://docs.splunk.com/Documentation/Splunk/9.2.1/Search/Savingsearches). + +The trigger always requires the following information: + +```yaml +triggers: + - type: splunk + metadata: + host: https://splunk.default.svc.cluster.local:8089 + targetValue: "1" + activationValue: "10" + savedSearchName: my-saved-search-name + valueField: count +``` + +**Parameter list:** + +- `host` - Search API host and port. Example: `https://localhost:8089`. +- `unsafeSsl` - Whether to trust invalid certificates or not. (Values: `"true"`, `"false"`, Default: `"false"`, Optional) +- `targetValue` - Value to reach to start scaling (This value can be a integer or float). +- `activationValue` - Target value for activating the scaler. Learn more about activation [here](./../concepts/scaling-deployments.md#activating-and-scaling-thresholds). +- `savedSearchName` - Name of saved search that returns metric data for scaling. +- `valueField` - The name of the field in the search results containing the metric value. Example: `index=_internal | tail | stats count`, the `valueField` is `count`. + +### Authentication Parameters + +You can authenticate by using a username/password or an API token. You will need to use `TriggerAuthentication` CRD to configure the authentication. + +> **Note:** +> +> `TriggerAuthentication` is required to use this scaler due to the hard requirement of providing a `username` for the Splunk API. + +**Parameter list:** + +- `username` - Splunk username authorized to access the search API. +- `apiToken` - Splunk API token for supplied `username`. Conflicts with `password`. +- `password` - Password for supplied `username`. Conflicts with `apiToken`. + +The user will need access to the saved search. + +### Examples + +### Username/password + +```yaml +--- +apiVersion: v1 +kind: Secret +metadata: + name: splunk-creds +data: + username: YWRtaW4= # "admin" + password: cGFzc3dvcmQ= # "password" +--- +apiVersion: keda.sh/v1alpha1 +kind: TriggerAuthentication +metadata: + name: splunk-auth +spec: + secretTargetRef: + - parameter: username + name: splunk-creds + key: username + - parameter: password + name: splunk-creds + key: password +--- +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: splunk-scaledobject +spec: + pollingInterval: 15 + minReplicaCount: 1 + maxReplicaCount: 3 + scaleTargetRef: + name: nginx + triggers: + - type: splunk + authenticationRef: + name: splunk-auth + metadata: + host: https://splunk.default.svc.cluster.local:8089 + targetValue: "11" + activationValue: "15" + savedSearchName: my-saved-search-name + valueField: count +``` + +### API Token + +```yaml +--- +apiVersion: v1 +kind: Secret +metadata: + name: splunk-creds +data: + username: YWRtaW4= # admin + apiToken: +--- +apiVersion: keda.sh/v1alpha1 +kind: TriggerAuthentication +metadata: + name: splunk-auth +spec: + secretTargetRef: + - parameter: username + name: splunk-creds + key: username + - parameter: apiToken + name: splunk-creds + key: apiToken +--- +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: splunk-scaledobject +spec: + pollingInterval: 15 + minReplicaCount: 1 + maxReplicaCount: 3 + scaleTargetRef: + name: nginx + triggers: + - type: splunk + authenticationRef: + name: splunk-auth + metadata: + host: https://splunk.default.svc.cluster.local:8089 + targetValue: "11" + activationValue: "15" + savedSearchName: my-saved-search-name + valueField: count +``` + +### Full example using Splunk deployment + +```yaml +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: splunkconf +data: + default.yml: | + splunk: + conf: + - key: savedsearches + value: + directory: /opt/splunk/etc/users/admin/search/local + content: + my-saved-search-name: + action.email.useNSSubject: 1 + action.webhook.enable_allowlist: 0 + alert.track: 0 + cron_schedule: '*/1 * * * *' + dispatch.earliest_time: -15m + dispatch.latest_time: now + display.general.type: statistics + display.page.search.tab: statistics + display.visualizations.show: 0 + enableSched: 1 + request.ui_dispatch_app: search + request.ui_dispatch_view: search + search: index=_internal | tail | stats count +--- +apiVersion: v1 +kind: Secret +metadata: + name: splunk-creds +data: + username: YWRtaW4= # "admin" + password: cGFzc3dvcmQ= # "password" +--- +apiVersion: keda.sh/v1alpha1 +kind: TriggerAuthentication +metadata: + name: splunk-auth +spec: + secretTargetRef: + - parameter: username + name: splunk-creds + key: username + - parameter: password + name: splunk-creds + key: password +--- +apiVersion: v1 +kind: Service +metadata: + name: splunk +spec: + ports: + - port: 8000 + targetPort: web + name: web-svc + - port: 8089 + targetPort: 8089 + name: api-svc + selector: + app: splunk +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: splunk + namespace: default +spec: + replicas: 1 + selector: + matchLabels: + app: splunk + template: + metadata: + labels: + app: splunk + spec: + containers: + - name: splunk + image: splunk/splunk:9.2 + ports: + - containerPort: 8000 + name: web + - containerPort: 8089 + name: api + env: + - name: SPLUNK_START_ARGS + value: --accept-license + - name: SPLUNK_PASSWORD + value: password + volumeMounts: + - name: splunkconf-volume + mountPath: /tmp/defaults + volumes: + - name: splunkconf-volume + configMap: + name: splunkconf +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: nginx +spec: + selector: + matchLabels: + app: nginx + replicas: 1 + template: + metadata: + labels: + app: nginx + spec: + containers: + - name: nginx + image: nginx + ports: + - containerPort: 8080 +--- +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: splunk-scaledobject +spec: + pollingInterval: 15 + minReplicaCount: 1 + maxReplicaCount: 3 + scaleTargetRef: + name: nginx + triggers: + - type: splunk + authenticationRef: + name: splunk-auth + metadata: + host: https://splunk.default.svc.cluster.local:8089 + unsafeSsl: "true" + targetValue: "5" + activationValue: "5" + savedSearchName: my-saved-search-name + valueField: count +``` From 704e541e501a596dfdb8f4d1790f260a02ae4fc1 Mon Sep 17 00:00:00 2001 From: Uchechukwu Obasi Date: Thu, 4 Jul 2024 14:45:17 +0100 Subject: [PATCH 15/33] Feat: version faq docs (#1416) Signed-off-by: shubhusion --- CONTRIBUTING.md | 113 ++++++++++----- content/docs/2.0/faq.md | 2 +- content/docs/2.1/faq.md | 2 +- content/docs/2.10/faq.md | 2 +- content/docs/2.11/faq.md | 2 +- content/docs/2.12/faq.md | 2 +- content/docs/2.13/faq.md | 2 +- content/docs/2.14/faq.md | 2 +- content/docs/2.14/reference/faq.md | 2 +- content/docs/2.15/reference/faq.md | 2 +- content/docs/2.2/faq.md | 2 +- content/docs/2.3/faq.md | 2 +- content/docs/2.4/faq.md | 2 +- content/docs/2.5/faq.md | 2 +- content/docs/2.6/faq.md | 2 +- content/docs/2.7/faq.md | 2 +- content/docs/2.8/faq.md | 2 +- content/docs/2.9/faq.md | 2 +- data/faq2_14.toml | 217 +++++++++++++++++++++++++++++ data/faq2_15.toml | 217 +++++++++++++++++++++++++++++ layouts/shortcodes/faq20.html | 8 +- 21 files changed, 534 insertions(+), 55 deletions(-) create mode 100644 data/faq2_14.toml create mode 100644 data/faq2_15.toml diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d909a47ea..8068d73d1 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -8,20 +8,32 @@ Our documentation is versioned so it's important to make the changes for the cor ## Getting Help -If you have a question about KEDA or how best to contribute, the [#KEDA](https://kubernetes.slack.com/archives/CKZJ36A5D) channel on the Kubernetes slack channel ([get an invite if you don't have one already](https://slack.k8s.io/)) is a good place to start. We also have regular [community stand-ups](https://github.com/kedacore/keda#community) to track ongoing work and discuss areas of contribution. For any issues with the product you can [create an issue](https://github.com/kedacore/keda/issues/new) in this repo. +If you have a question about KEDA or how best to contribute, the [#KEDA](https://kubernetes.slack.com/archives/CKZJ36A5D) channel on the Kubernetes slack channel ([get an invite if you don't have one already](https://slack.k8s.io/)) is a good place to start. We also have regular [community stand-ups](https://github.com/kedacore/keda#community) to track ongoing work and discuss areas of contribution. For any issues with the product you can [create an issue](https://github.com/kedacore/keda/issues/new) in this repo. ## Contributing New Documentation We provide easy ways to introduce new content: -- [Adding new blog post](#adding-blog-post) -- [Adding new Frequently Asked Question (FAQ)](#add-new-frequently-asked-question-faq) -- [Adding new scaler documentation](#adding-scaler-documentation) -- [Adding new troubleshooting guidance](#add-new-troubleshooting-guidance) -- [Become a listed KEDA user!](#become-a-listed-KEDA-user) -- [Become a listed KEDA commercial offering!](#become-a-listed-KEDA-commercial-offering) -- [Writing documentation for a scaler](#writing-documentation-for-a-new-authentication-provider) -- [Writing documentation for a scaler](#writing-documentation-for-a-scaler) +- [Contributing to KEDA](#contributing-to-keda) + - [Getting Help](#getting-help) + - [Contributing New Documentation](#contributing-new-documentation) + - [Become a listed KEDA user!](#become-a-listed-keda-user) + - [Become a listed KEDA commercial offering!](#become-a-listed-keda-commercial-offering) + - [Adding blog post](#adding-blog-post) + - [Adding scaler documentation](#adding-scaler-documentation) + - [Writing documentation for a new authentication provider](#writing-documentation-for-a-new-authentication-provider) + - [Add new Frequently Asked Question (FAQ)](#add-new-frequently-asked-question-faq) + - [Add new troubleshooting guidance](#add-new-troubleshooting-guidance) + - [Writing documentation for a scaler](#writing-documentation-for-a-scaler) + - [Working with documentation versions](#working-with-documentation-versions) + - [Preparing a new version](#preparing-a-new-version) + - [Publishing a new version](#publishing-a-new-version) + - [Developer Certificate of Origin: Signing your work](#developer-certificate-of-origin-signing-your-work) + - [Every commit needs to be signed](#every-commit-needs-to-be-signed) + - [I didn't sign my commit, now what?!](#i-didnt-sign-my-commit-now-what) + - [Changing the website](#changing-the-website) + - [Creating and building a local environment](#creating-and-building-a-local-environment) + - [Adding a new filter option](#adding-a-new-filter-option) Learn more how to [create and build a local environment](#creating-and-building-a-local-environment). @@ -30,6 +42,7 @@ Learn more how to [create and build a local environment](#creating-and-building- Are you using KEDA in production? Do you want to become a [listed user](https://keda.sh/community/#users)? Say no more! You can easily get listed by following these steps: + 1. Upload your logo to `static/img/logos/` _(350x180)_ 2. Configure your company as a new user in `config.toml` _(sorted alphabetically)_ @@ -46,6 +59,7 @@ Here's a good example of [Coralogix becoming a listed user](https://github.com/k Do you offer commercial support for KEDA and want to become a [listed commercial offering](https://keda.sh/support/#commercial-support)? Say no more! You can easily get listed by following these steps: + 1. Upload your logo to `static/img/logos/` _(350x180)_ 2. Configure your company as a new user in `config.toml` _(sorted alphabetically)_ @@ -66,9 +80,9 @@ $ hugo new blog/my-new-post.md This creates a boilerplate Markdown file in `content/blog/my-new-post.md` whose contents you can modify. The following fields are required: -* `title` -* `date` (in `YYYY-MM-DD` format) -* `author` +- `title` +- `date` (in `YYYY-MM-DD` format) +- `author` ### Adding scaler documentation @@ -82,10 +96,10 @@ This creates a boilerplate Markdown file in `content/docs//scalers/my-new-scaler.md` whose contents you can modify. Make sure to update the following metadata fields: -* `title` -* `availability` -* `maintainer` -* `description` +- `title` +- `availability` +- `maintainer` +- `description` ### Writing documentation for a new authentication provider @@ -99,7 +113,7 @@ This creates a boilerplate Markdown file in `content/docs//providers/my-new-provider.md` whose contents you can modify. Make sure to update the following metadata fields: -* `title` +- `title` ### Add new Frequently Asked Question (FAQ) @@ -143,13 +157,34 @@ Here are a few examples: ## Working with documentation versions The KEDA documentation is versioned. Each version has its own subdirectory under -[content/docs](content/docs). To add a new version, copy the directory for -the most recent version. Here's an example: +[content/docs](content/docs). To add a new version, follow these steps: + +1. Copy the directory for the most recent version. Here's an example: ```console $ cp -rf content/docs/ content/docs/ ``` +2. Copy the file for the most recent faq data in the `data` directory. Here's an example: + +```console +$ cp -rf data/faq data/faq +``` + +3. Navigate to the new faq file: + +```console +$ cd content/docs//reference/faq.md +``` + +4. Update the versionData option + +``` +{{< faq20 versionData="NEW_FAQ_FILE_NAME" >}} +``` + +Replace `NEW_FAQ_FILE_NAME` with the file name of the faq data for the new version. + By default, new documentation versions are not listed as available version so it's safe to make changes to them. After every release, the version will be published as new version. @@ -169,7 +204,7 @@ Ensure that compatibility matrix on `content/docs/{next-version}/operate/cluster Once a version is ready to be published, we must add the version to the `params.versions.docs` list in [config.toml](config.toml). -More recent versions should be placed first in the list (ordering *does* matter +More recent versions should be placed first in the list (ordering _does_ matter because the first element in that list is considered the latest version). > Note: Remember to [prepare the next version](#preparing-a-new-version). @@ -179,6 +214,7 @@ because the first element in that list is considered the latest version). ### Every commit needs to be signed The Developer Certificate of Origin (DCO) is a lightweight way for contributors to certify that they wrote or otherwise have the right to submit the code they are contributing to the project. Here is the full text of the DCO, reformatted for readability: + ``` By making a contribution to this project, I certify that: @@ -198,12 +234,14 @@ This is my commit message Signed-off-by: Random J Developer ``` + Git even has a `-s` command line option to append this automatically to your commit message: + ``` $ git commit -s -m 'This is my commit message' ``` -Each Pull Request is checked whether or not commits in a Pull Request do contain a valid Signed-off-by line. +Each Pull Request is checked whether or not commits in a Pull Request do contain a valid Signed-off-by line. ### I didn't sign my commit, now what?! @@ -259,7 +297,7 @@ FILTER_NAME = "filter_value" Replace FILTER_NAME with any desired name of your choice. Same applies to the value. 3. Navigate to the `list.lunr.json` file to edit: `cd layouts/_default/list.lunr.json`. -4. Open the file and go down to line 3. You will notice the format of the data represented in a key/value pair. Just before the closing parenthesis, append your new option like this: `"FILTER_NAME" $scalers.Params.FILTER_NAME`. +4. Open the file and go down to line 3. You will notice the format of the data represented in a key/value pair. Just before the closing parenthesis, append your new option like this: `"FILTER_NAME" $scalers.Params.FILTER_NAME`. Replace FILTER_NAME with the same name represented in the frontmatter (see step 2 above for reference). @@ -276,7 +314,7 @@ params = ["availability", "maintainer", "category", "type", "FILTER_NAME"] this.field("FILTER_NAME", { boost: 5, }); -``` +``` Replace FILTER_NAME with the same name represented in the frontmatter (see step 2 above for reference). @@ -291,7 +329,7 @@ parse[doc.title] = { availability: doc.availability, category: doc.category, type: doc.type, - FILTER_NAME: doc.FILTER_NAME + FILTER_NAME: doc.FILTER_NAME, }; ``` @@ -300,21 +338,22 @@ parse[doc.title] = { ```html
FILTER_NAME
- {{ $FILTER_NAME := slice }} - {{ range $scalers := where site.RegularPages ".CurrentSection.Title" "Scalers" }} - {{ with $scalers.Params.FILTER_NAME }} - {{ $FILTER_NAME = $categories | append ($scalers.Params.FILTER_NAME) }} - {{ $FILTER_NAME = uniq $FILTER_NAME }} - {{ end }} - {{ end }} - {{ range $FILTER_NAME }} - {{ $item := . }} + {{ $FILTER_NAME := slice }} {{ range $scalers := where site.RegularPages + ".CurrentSection.Title" "Scalers" }} {{ with $scalers.Params.FILTER_NAME }} {{ + $FILTER_NAME = $categories | append ($scalers.Params.FILTER_NAME) }} {{ + $FILTER_NAME = uniq $FILTER_NAME }} {{ end }} {{ end }} {{ range $FILTER_NAME + }} {{ $item := . }}
- - + +
{{ end }} -
+ ``` Replace FILTER_NAME with the same name represented in the frontmatter (see step 2 above for reference). @@ -324,4 +363,4 @@ Replace FILTER_NAME with the same name represented in the frontmatter (see step [localhost:8888]: http://localhost:8888 [LTS release]: https://nodejs.org/en/about/releases/ [Netlify]: https://netlify.com -[nvm]: https://github.com/nvm-sh/nvm/blob/master/README.md#installing-and-updating \ No newline at end of file +[nvm]: https://github.com/nvm-sh/nvm/blob/master/README.md#installing-and-updating diff --git a/content/docs/2.0/faq.md b/content/docs/2.0/faq.md index d012d6767..df84af22c 100644 --- a/content/docs/2.0/faq.md +++ b/content/docs/2.0/faq.md @@ -2,4 +2,4 @@ title = "FAQ" +++ -{{< faq20 >}} +{{< faq20 versionData="faq20" >}} diff --git a/content/docs/2.1/faq.md b/content/docs/2.1/faq.md index d012d6767..df84af22c 100644 --- a/content/docs/2.1/faq.md +++ b/content/docs/2.1/faq.md @@ -2,4 +2,4 @@ title = "FAQ" +++ -{{< faq20 >}} +{{< faq20 versionData="faq20" >}} diff --git a/content/docs/2.10/faq.md b/content/docs/2.10/faq.md index d012d6767..df84af22c 100644 --- a/content/docs/2.10/faq.md +++ b/content/docs/2.10/faq.md @@ -2,4 +2,4 @@ title = "FAQ" +++ -{{< faq20 >}} +{{< faq20 versionData="faq20" >}} diff --git a/content/docs/2.11/faq.md b/content/docs/2.11/faq.md index d012d6767..df84af22c 100644 --- a/content/docs/2.11/faq.md +++ b/content/docs/2.11/faq.md @@ -2,4 +2,4 @@ title = "FAQ" +++ -{{< faq20 >}} +{{< faq20 versionData="faq20" >}} diff --git a/content/docs/2.12/faq.md b/content/docs/2.12/faq.md index d012d6767..df84af22c 100644 --- a/content/docs/2.12/faq.md +++ b/content/docs/2.12/faq.md @@ -2,4 +2,4 @@ title = "FAQ" +++ -{{< faq20 >}} +{{< faq20 versionData="faq20" >}} diff --git a/content/docs/2.13/faq.md b/content/docs/2.13/faq.md index d012d6767..df84af22c 100644 --- a/content/docs/2.13/faq.md +++ b/content/docs/2.13/faq.md @@ -2,4 +2,4 @@ title = "FAQ" +++ -{{< faq20 >}} +{{< faq20 versionData="faq20" >}} diff --git a/content/docs/2.14/faq.md b/content/docs/2.14/faq.md index d012d6767..cffb0bd53 100644 --- a/content/docs/2.14/faq.md +++ b/content/docs/2.14/faq.md @@ -2,4 +2,4 @@ title = "FAQ" +++ -{{< faq20 >}} +{{< faq20 versionData="faq2_14" >}} diff --git a/content/docs/2.14/reference/faq.md b/content/docs/2.14/reference/faq.md index df75bbe78..995bfb24d 100644 --- a/content/docs/2.14/reference/faq.md +++ b/content/docs/2.14/reference/faq.md @@ -3,4 +3,4 @@ title = "FAQ" weight = 2000 +++ -{{< faq20 >}} +{{< faq20 versionData="faq2_14" >}} diff --git a/content/docs/2.15/reference/faq.md b/content/docs/2.15/reference/faq.md index df75bbe78..95365417f 100644 --- a/content/docs/2.15/reference/faq.md +++ b/content/docs/2.15/reference/faq.md @@ -3,4 +3,4 @@ title = "FAQ" weight = 2000 +++ -{{< faq20 >}} +{{< faq20 versionData="faq2_15" >}} diff --git a/content/docs/2.2/faq.md b/content/docs/2.2/faq.md index d012d6767..df84af22c 100644 --- a/content/docs/2.2/faq.md +++ b/content/docs/2.2/faq.md @@ -2,4 +2,4 @@ title = "FAQ" +++ -{{< faq20 >}} +{{< faq20 versionData="faq20" >}} diff --git a/content/docs/2.3/faq.md b/content/docs/2.3/faq.md index d012d6767..df84af22c 100644 --- a/content/docs/2.3/faq.md +++ b/content/docs/2.3/faq.md @@ -2,4 +2,4 @@ title = "FAQ" +++ -{{< faq20 >}} +{{< faq20 versionData="faq20" >}} diff --git a/content/docs/2.4/faq.md b/content/docs/2.4/faq.md index d012d6767..df84af22c 100644 --- a/content/docs/2.4/faq.md +++ b/content/docs/2.4/faq.md @@ -2,4 +2,4 @@ title = "FAQ" +++ -{{< faq20 >}} +{{< faq20 versionData="faq20" >}} diff --git a/content/docs/2.5/faq.md b/content/docs/2.5/faq.md index d012d6767..df84af22c 100644 --- a/content/docs/2.5/faq.md +++ b/content/docs/2.5/faq.md @@ -2,4 +2,4 @@ title = "FAQ" +++ -{{< faq20 >}} +{{< faq20 versionData="faq20" >}} diff --git a/content/docs/2.6/faq.md b/content/docs/2.6/faq.md index d012d6767..df84af22c 100644 --- a/content/docs/2.6/faq.md +++ b/content/docs/2.6/faq.md @@ -2,4 +2,4 @@ title = "FAQ" +++ -{{< faq20 >}} +{{< faq20 versionData="faq20" >}} diff --git a/content/docs/2.7/faq.md b/content/docs/2.7/faq.md index d012d6767..df84af22c 100644 --- a/content/docs/2.7/faq.md +++ b/content/docs/2.7/faq.md @@ -2,4 +2,4 @@ title = "FAQ" +++ -{{< faq20 >}} +{{< faq20 versionData="faq20" >}} diff --git a/content/docs/2.8/faq.md b/content/docs/2.8/faq.md index d012d6767..df84af22c 100644 --- a/content/docs/2.8/faq.md +++ b/content/docs/2.8/faq.md @@ -2,4 +2,4 @@ title = "FAQ" +++ -{{< faq20 >}} +{{< faq20 versionData="faq20" >}} diff --git a/content/docs/2.9/faq.md b/content/docs/2.9/faq.md index d012d6767..df84af22c 100644 --- a/content/docs/2.9/faq.md +++ b/content/docs/2.9/faq.md @@ -2,4 +2,4 @@ title = "FAQ" +++ -{{< faq20 >}} +{{< faq20 versionData="faq20" >}} diff --git a/data/faq2_14.toml b/data/faq2_14.toml new file mode 100644 index 000000000..95d1089e0 --- /dev/null +++ b/data/faq2_14.toml @@ -0,0 +1,217 @@ +[[qna]] +q = "What is KEDA and why is it useful?" +a = "KEDA stands for Kubernetes Event-driven Autoscaler. It is built to be able to activate a Kubernetes deployment (i.e. no pods to a single pod) and subsequently to more pods based on events from various event sources." +type = "General" + +[[qna]] +q = "What are the prerequisites for using KEDA?" +a = """ +KEDA is designed, tested and is supported to be run on any Kubernetes cluster that runs Kubernetes v1.17.0 or above. + +It uses a CRD (custom resource definition) and the Kubernetes metric server so you will have to use a Kubernetes version which supports these. + +> 💡 Kubernetes v1.16 is supported with KEDA v2.4.0 or below +""" +type = "General" + +[[qna]] +q = "Can KEDA be used in production?" +a = "Yes! KEDA v2.0 is suited for production workloads, but we still support v1.5 if you are running that as well.." +type = "General" + +[[qna]] +q = "What does it cost?" +a = "There is no charge for using KEDA itself, we just ask people to [become a listed user](https://github.com/kedacore/keda-docs#become-a-listed-keda-user) when possible." +type = "General" + +[[qna]] +q = "Can I scale HTTP workloads with KEDA and Kubernetes?" +a = """ +KEDA will scale a container using metrics from a scaler, but unfortunately there is no scaler today for HTTP workloads out-of-the-box. + +We do, however, provide some alternative approaches: +- Use our HTTP add-on scaler which is currently in experimental stage ([GitHub](https://github.com/kedacore/http-add-on)) +- Use [Prometheus scaler](/docs/latest/scalers/prometheus/) to create scale rule based on metrics around HTTP events + - Read [Anirudh Garg's blog post](https://dev.to/anirudhgarg_99/scale-up-and-down-a-http-triggered-function-app-in-kubernetes-using-keda-4m42) to learn more. +""" +type = "Features" + +[[qna]] +q = "Is short polling intervals a problem?" +a = "Polling interval really only impacts the time-to-activation (scaling from 0 to 1) but once scaled to one it's really up to the HPA (horizontal pod autoscaler) which polls KEDA." +type = "Features" + +[[qna]] +q = "Using multiple triggers for the same scale target" +a = """ +KEDA allows you to use multiple triggers as part of the same `ScaledObject` or `ScaledJob`. + +By doing this, your autoscaling becomes better: +- All your autoscaling rules are in one place +- You will not have multiple `ScaledObject`'s or `ScaledJob`'s interfering with each other + +KEDA will start scaling as soon as when one of the triggers meets the criteria. Horizontal Pod Autoscaler (HPA) will calculate metrics for every scaler and use the highest desired replica count to scale the workload to. +""" +type = "Best Practices" + +[[qna]] +q = "Don't combine `ScaledObject` with Horizontal Pod Autoscaler (HPA)" +a = """ +We recommend not to combine using KEDA's `ScaledObject` with a Horizontal Pod Autoscaler (HPA) to scale the same workload. + +They will compete with each other resulting given KEDA uses Horizontal Pod Autoscaler (HPA) under the hood and will result in odd scaling behavior. + +If you are using a Horizontal Pod Autoscaler (HPA) to scale on CPU and/or memory, we recommend using the [CPU scaler](/docs/latest/scalers/cpu/) & [Memory scaler](/docs/latest/scalers/memory/) scalers instead. +""" +type = "Best Practices" + +[[qna]] +q = "What does the target metric value in the Horizontal Pod Autoscaler (HPA) represent?" +a = """ +The target metric value is used by the Horizontal Pod Autoscaler (HPA) to make scaling decisions. + +The current target value on the Horizontal Pod Autoscaler (HPA) often does not match with the metrics on the system you are scaling on. This is because of how the Horizontal Pod Autoscaler's (HPA) [scaling algorithm](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/#algorithm-details) works. + +By default, KEDA scalers use average metrics (the `AverageValue` metric type). This means that the HPA will use the average value of the metric between the total amount of pods. As of KEDA v2.7, ScaledObjects also support the `Value` metric type. You can learn more about it [here](https://keda.sh/docs/latest/concepts/scaling-deployments/#triggers). +""" +type = "Kubernetes" + +[[qna]] +q = "Why does KEDA use external metrics and not custom metrics instead?" +a = """ +Kubernetes allows you to autoscale based on custom & external metrics which are fundamentally different: +- **Custom metrics** are metrics that come from applications solely running on the Kubernetes cluster (Prometheus) +- **External metrics** are metrics that represent the state of an application/service that is running outside of the Kubernetes cluster (AWS, Azure, GCP, Datadog, etc.) + +Because KEDA primarily serves metrics for metric sources outside of the Kubernetes cluster, it uses external metrics and not custom metrics. + +This is why KEDA registers the `v1beta1.external.metrics.k8s.io` namespace in the API service. However, this is just an implementation detail as both offer the same functionality. + +Read [about the different metric APIs](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/#support-for-metrics-apis) or [this article](https://cloud.google.com/kubernetes-engine/docs/concepts/custom-and-external-metrics) by Google Cloud to learn more. +""" +type = "Kubernetes" + +[[qna]] +q = "Can I run multiple metric servers serving external metrics in the same cluster?" +a = """ +Unfortunately, you cannot do that. + +Kubernetes currently only supports one metric server serving `external.metrics.k8s.io` metrics per cluster. This is because only one API Service can be registered to handle external metrics. + +If you want to know what external metric server is currently registered, you can use the following command: + +```shell +~ kubectl get APIService/v1beta1.external.metrics.k8s.io +NAME SERVICE AVAILABLE AGE +v1beta1.external.metrics.k8s.io keda-system/keda-operator-metrics-apiserver True 457d +``` + +Once a new metric server is installed, it will overwrite the existing API Server registration and take over the `v1beta1.external.metrics.k8s.io` namespace. This will cause the previously installed metric server to be ignored. + +There is an [open proposal](https://github.com/kubernetes-sigs/custom-metrics-apiserver/issues/70) to allow multiple metric servers in the same cluster, but it's not implemented yet. +""" +type = "Kubernetes" + +[[qna]] +q = "Can I run multiple installations of KEDA in the same cluster?" +a = """ +Unfortunately, you cannot do that. + +This is a limitation that is because Kubernetes does not allow you to run multiple metric servers in the same cluster that serve external metrics. + +Also, KEDA does not allow you to share a single metric server across multiple operator installations. + +Learn more in the "Can I run multiple metric servers serving external metrics in the same cluster?" FAQ entry. +""" +type = "Kubernetes" + +[[qna]] +q = "How can I get involved?" +a = """ +There are several ways to get involved. + +* Pick up an issue to work on. A good place to start might be issues which are marked as [Good First Issue](https://github.com/kedacore/keda/labels/good%20first%20issue) or [Help Wanted](https://github.com/kedacore/keda/labels/help%20wanted) +* We are always looking to add more scalers. +* We are always looking for more samples, documentation, etc. +* Please join us in our [weekly standup](https://github.com/kedacore/keda#community). +""" +type = "Community" + +[[qna]] +q = "Where can I get to the code for the Scalers?" +a = "All scalers have their code [here](https://github.com/kedacore/keda/tree/main/pkg/scalers)." +type = "Community" + +[[qna]] +q = "How do I access KEDA resources using `client-go`?" +a = """KEDA client-go is exported as part of the KEDA repository.""" + +[[qna]] +q = "How do I run KEDA with `readOnlyRootFilesystem=true`?" +a = """ +As default, KEDA v2.10 or above sets `readOnlyRootFilesystem=true` as default without any other manual intervention. + +If you are running KEDA v2.9 or below, you can't run KEDA with `readOnlyRootFilesystem=true` as default because Metrics adapter generates self-signed certificates during deployment and stores them on the root file system. +To overcome this, you can create a secret/configmap with a valid CA, cert and key and then mount it to the Metrics Deployment. +To use your certificate, you need to reference it in the container `args` section, e.g.: +``` +args: + - '--client-ca-file=/cabundle/service-ca.crt' + - '--tls-cert-file=/certs/tls.crt' + - '--tls-private-key-file=/certs/tls.key' +``` +It is also possible to run KEDA with `readOnlyRootFilesystem=true` by creating an emptyDir volume and mounting it to the path where, +by default, metrics server writes its generated cert. The corresponding helm command is: +``` +helm install keda kedacore/keda --namespace keda \ + --set 'volumes.metricsApiServer.extraVolumes[0].name=keda-volume' \ + --set 'volumes.metricsApiServer.extraVolumeMounts[0].name=keda-volume' \ + --set 'volumes.metricsApiServer.extraVolumeMounts[0].mountPath=/apiserver.local.config/certificates/' \ + --set 'securityContext.metricServer.readOnlyRootFilesystem=true' +``` +""" +type = "Features" + +[[qna]] +q = "How do I run KEDA with TLS v1.3 only?" +a = """ +By default, Keda listens on TLS v1.1 and TLSv1.2, with the default Golang ciphersuites. +In some environments, these ciphers may be considered less secure, for example CBC ciphers. + +As an alternative, you can configure the minimum TLS version to be v1.3 to increase security. +Since all modern clients support this version, there should be no impact in most scenarios. + +You can set this with args - e.g.: +``` +args: + - '--tls-min-version=VersionTLS13' +``` +""" +type = "Features" + +[[qna]] +q = "Does KEDA depend on any Azure service?" +a = "No, KEDA only takes a dependency on standard Kubernetes constructs and can run on any Kubernetes cluster whether in OpenShift, AKS, GKE, EKS or your own infrastructure." +type = "Azure" + +[[qna]] +q = "Does KEDA only work with Azure Functions?" +a = "No, KEDA can scale up/down any container that you specify in your deployment. There has been work done in the Azure Functions tooling to make it easy to scale an Azure Function container." +type = "Azure" + +[[qna]] +q = "Why should we use KEDA if we are already using Azure Functions in Azure?" +a = """ +There are a few reasons for this: + +* Run functions on-premises (potentially in something like an 'intelligent edge' architecture) +* Run functions alongside other Kubernetes apps (maybe in a restricted network, app mesh, custom environment, etc.) +* Run functions outside of Azure (no vendor lock-in) +* Specific need for more control (GPU enabled compute clusters, policies, etc.) +""" +type = "Azure" + +[[qna]] +q = "Does scaler search support wildcard search?" +a = "Yes. The search actually supports wildcard search. We've made our search to automatically perform wildcard filtering on the fly so you don't have to append special symbols within your search query." +type = "Website" diff --git a/data/faq2_15.toml b/data/faq2_15.toml new file mode 100644 index 000000000..95d1089e0 --- /dev/null +++ b/data/faq2_15.toml @@ -0,0 +1,217 @@ +[[qna]] +q = "What is KEDA and why is it useful?" +a = "KEDA stands for Kubernetes Event-driven Autoscaler. It is built to be able to activate a Kubernetes deployment (i.e. no pods to a single pod) and subsequently to more pods based on events from various event sources." +type = "General" + +[[qna]] +q = "What are the prerequisites for using KEDA?" +a = """ +KEDA is designed, tested and is supported to be run on any Kubernetes cluster that runs Kubernetes v1.17.0 or above. + +It uses a CRD (custom resource definition) and the Kubernetes metric server so you will have to use a Kubernetes version which supports these. + +> 💡 Kubernetes v1.16 is supported with KEDA v2.4.0 or below +""" +type = "General" + +[[qna]] +q = "Can KEDA be used in production?" +a = "Yes! KEDA v2.0 is suited for production workloads, but we still support v1.5 if you are running that as well.." +type = "General" + +[[qna]] +q = "What does it cost?" +a = "There is no charge for using KEDA itself, we just ask people to [become a listed user](https://github.com/kedacore/keda-docs#become-a-listed-keda-user) when possible." +type = "General" + +[[qna]] +q = "Can I scale HTTP workloads with KEDA and Kubernetes?" +a = """ +KEDA will scale a container using metrics from a scaler, but unfortunately there is no scaler today for HTTP workloads out-of-the-box. + +We do, however, provide some alternative approaches: +- Use our HTTP add-on scaler which is currently in experimental stage ([GitHub](https://github.com/kedacore/http-add-on)) +- Use [Prometheus scaler](/docs/latest/scalers/prometheus/) to create scale rule based on metrics around HTTP events + - Read [Anirudh Garg's blog post](https://dev.to/anirudhgarg_99/scale-up-and-down-a-http-triggered-function-app-in-kubernetes-using-keda-4m42) to learn more. +""" +type = "Features" + +[[qna]] +q = "Is short polling intervals a problem?" +a = "Polling interval really only impacts the time-to-activation (scaling from 0 to 1) but once scaled to one it's really up to the HPA (horizontal pod autoscaler) which polls KEDA." +type = "Features" + +[[qna]] +q = "Using multiple triggers for the same scale target" +a = """ +KEDA allows you to use multiple triggers as part of the same `ScaledObject` or `ScaledJob`. + +By doing this, your autoscaling becomes better: +- All your autoscaling rules are in one place +- You will not have multiple `ScaledObject`'s or `ScaledJob`'s interfering with each other + +KEDA will start scaling as soon as when one of the triggers meets the criteria. Horizontal Pod Autoscaler (HPA) will calculate metrics for every scaler and use the highest desired replica count to scale the workload to. +""" +type = "Best Practices" + +[[qna]] +q = "Don't combine `ScaledObject` with Horizontal Pod Autoscaler (HPA)" +a = """ +We recommend not to combine using KEDA's `ScaledObject` with a Horizontal Pod Autoscaler (HPA) to scale the same workload. + +They will compete with each other resulting given KEDA uses Horizontal Pod Autoscaler (HPA) under the hood and will result in odd scaling behavior. + +If you are using a Horizontal Pod Autoscaler (HPA) to scale on CPU and/or memory, we recommend using the [CPU scaler](/docs/latest/scalers/cpu/) & [Memory scaler](/docs/latest/scalers/memory/) scalers instead. +""" +type = "Best Practices" + +[[qna]] +q = "What does the target metric value in the Horizontal Pod Autoscaler (HPA) represent?" +a = """ +The target metric value is used by the Horizontal Pod Autoscaler (HPA) to make scaling decisions. + +The current target value on the Horizontal Pod Autoscaler (HPA) often does not match with the metrics on the system you are scaling on. This is because of how the Horizontal Pod Autoscaler's (HPA) [scaling algorithm](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/#algorithm-details) works. + +By default, KEDA scalers use average metrics (the `AverageValue` metric type). This means that the HPA will use the average value of the metric between the total amount of pods. As of KEDA v2.7, ScaledObjects also support the `Value` metric type. You can learn more about it [here](https://keda.sh/docs/latest/concepts/scaling-deployments/#triggers). +""" +type = "Kubernetes" + +[[qna]] +q = "Why does KEDA use external metrics and not custom metrics instead?" +a = """ +Kubernetes allows you to autoscale based on custom & external metrics which are fundamentally different: +- **Custom metrics** are metrics that come from applications solely running on the Kubernetes cluster (Prometheus) +- **External metrics** are metrics that represent the state of an application/service that is running outside of the Kubernetes cluster (AWS, Azure, GCP, Datadog, etc.) + +Because KEDA primarily serves metrics for metric sources outside of the Kubernetes cluster, it uses external metrics and not custom metrics. + +This is why KEDA registers the `v1beta1.external.metrics.k8s.io` namespace in the API service. However, this is just an implementation detail as both offer the same functionality. + +Read [about the different metric APIs](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/#support-for-metrics-apis) or [this article](https://cloud.google.com/kubernetes-engine/docs/concepts/custom-and-external-metrics) by Google Cloud to learn more. +""" +type = "Kubernetes" + +[[qna]] +q = "Can I run multiple metric servers serving external metrics in the same cluster?" +a = """ +Unfortunately, you cannot do that. + +Kubernetes currently only supports one metric server serving `external.metrics.k8s.io` metrics per cluster. This is because only one API Service can be registered to handle external metrics. + +If you want to know what external metric server is currently registered, you can use the following command: + +```shell +~ kubectl get APIService/v1beta1.external.metrics.k8s.io +NAME SERVICE AVAILABLE AGE +v1beta1.external.metrics.k8s.io keda-system/keda-operator-metrics-apiserver True 457d +``` + +Once a new metric server is installed, it will overwrite the existing API Server registration and take over the `v1beta1.external.metrics.k8s.io` namespace. This will cause the previously installed metric server to be ignored. + +There is an [open proposal](https://github.com/kubernetes-sigs/custom-metrics-apiserver/issues/70) to allow multiple metric servers in the same cluster, but it's not implemented yet. +""" +type = "Kubernetes" + +[[qna]] +q = "Can I run multiple installations of KEDA in the same cluster?" +a = """ +Unfortunately, you cannot do that. + +This is a limitation that is because Kubernetes does not allow you to run multiple metric servers in the same cluster that serve external metrics. + +Also, KEDA does not allow you to share a single metric server across multiple operator installations. + +Learn more in the "Can I run multiple metric servers serving external metrics in the same cluster?" FAQ entry. +""" +type = "Kubernetes" + +[[qna]] +q = "How can I get involved?" +a = """ +There are several ways to get involved. + +* Pick up an issue to work on. A good place to start might be issues which are marked as [Good First Issue](https://github.com/kedacore/keda/labels/good%20first%20issue) or [Help Wanted](https://github.com/kedacore/keda/labels/help%20wanted) +* We are always looking to add more scalers. +* We are always looking for more samples, documentation, etc. +* Please join us in our [weekly standup](https://github.com/kedacore/keda#community). +""" +type = "Community" + +[[qna]] +q = "Where can I get to the code for the Scalers?" +a = "All scalers have their code [here](https://github.com/kedacore/keda/tree/main/pkg/scalers)." +type = "Community" + +[[qna]] +q = "How do I access KEDA resources using `client-go`?" +a = """KEDA client-go is exported as part of the KEDA repository.""" + +[[qna]] +q = "How do I run KEDA with `readOnlyRootFilesystem=true`?" +a = """ +As default, KEDA v2.10 or above sets `readOnlyRootFilesystem=true` as default without any other manual intervention. + +If you are running KEDA v2.9 or below, you can't run KEDA with `readOnlyRootFilesystem=true` as default because Metrics adapter generates self-signed certificates during deployment and stores them on the root file system. +To overcome this, you can create a secret/configmap with a valid CA, cert and key and then mount it to the Metrics Deployment. +To use your certificate, you need to reference it in the container `args` section, e.g.: +``` +args: + - '--client-ca-file=/cabundle/service-ca.crt' + - '--tls-cert-file=/certs/tls.crt' + - '--tls-private-key-file=/certs/tls.key' +``` +It is also possible to run KEDA with `readOnlyRootFilesystem=true` by creating an emptyDir volume and mounting it to the path where, +by default, metrics server writes its generated cert. The corresponding helm command is: +``` +helm install keda kedacore/keda --namespace keda \ + --set 'volumes.metricsApiServer.extraVolumes[0].name=keda-volume' \ + --set 'volumes.metricsApiServer.extraVolumeMounts[0].name=keda-volume' \ + --set 'volumes.metricsApiServer.extraVolumeMounts[0].mountPath=/apiserver.local.config/certificates/' \ + --set 'securityContext.metricServer.readOnlyRootFilesystem=true' +``` +""" +type = "Features" + +[[qna]] +q = "How do I run KEDA with TLS v1.3 only?" +a = """ +By default, Keda listens on TLS v1.1 and TLSv1.2, with the default Golang ciphersuites. +In some environments, these ciphers may be considered less secure, for example CBC ciphers. + +As an alternative, you can configure the minimum TLS version to be v1.3 to increase security. +Since all modern clients support this version, there should be no impact in most scenarios. + +You can set this with args - e.g.: +``` +args: + - '--tls-min-version=VersionTLS13' +``` +""" +type = "Features" + +[[qna]] +q = "Does KEDA depend on any Azure service?" +a = "No, KEDA only takes a dependency on standard Kubernetes constructs and can run on any Kubernetes cluster whether in OpenShift, AKS, GKE, EKS or your own infrastructure." +type = "Azure" + +[[qna]] +q = "Does KEDA only work with Azure Functions?" +a = "No, KEDA can scale up/down any container that you specify in your deployment. There has been work done in the Azure Functions tooling to make it easy to scale an Azure Function container." +type = "Azure" + +[[qna]] +q = "Why should we use KEDA if we are already using Azure Functions in Azure?" +a = """ +There are a few reasons for this: + +* Run functions on-premises (potentially in something like an 'intelligent edge' architecture) +* Run functions alongside other Kubernetes apps (maybe in a restricted network, app mesh, custom environment, etc.) +* Run functions outside of Azure (no vendor lock-in) +* Specific need for more control (GPU enabled compute clusters, policies, etc.) +""" +type = "Azure" + +[[qna]] +q = "Does scaler search support wildcard search?" +a = "Yes. The search actually supports wildcard search. We've made our search to automatically perform wildcard filtering on the fly so you don't have to append special symbols within your search query." +type = "Website" diff --git a/layouts/shortcodes/faq20.html b/layouts/shortcodes/faq20.html index 75c294338..c26bdcd69 100644 --- a/layouts/shortcodes/faq20.html +++ b/layouts/shortcodes/faq20.html @@ -1,4 +1,10 @@ -{{ $faq20 := site.Data.faq20.qna }} +{{ $filename := .Get "versionData" }} +{{ range index .Site.Data $filename }} + {{ $.Scratch.Add "faq20" . }} +{{ end }} + +{{ $faq20 := $.Scratch.Get "faq20" }} +

General

From ddd894deaf34e797e51fd39fea77184f2479e3aa Mon Sep 17 00:00:00 2001 From: Zbynek Roubalik Date: Mon, 8 Jul 2024 19:32:07 +0200 Subject: [PATCH 16/33] Update Kedify link (#1421) Signed-off-by: Zbynek Roubalik Signed-off-by: shubhusion --- config.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config.toml b/config.toml index fe27d32f0..30e16a5af 100644 --- a/config.toml +++ b/config.toml @@ -643,5 +643,5 @@ logo = "vendors/azure-kubernetes-service.png" [[params.vendors]] name = "Kedify" -url = "https://kedify.io" +url = "https://kedify.io/?utm_source=keda_oss_site" logo = "vendors/kedify.png" From 7f49ce0ce2773a75b4582422ba7637e1f67bebe6 Mon Sep 17 00:00:00 2001 From: Tom Kerkhove Date: Tue, 9 Jul 2024 09:25:04 +0200 Subject: [PATCH 17/33] fix: Align Kedify links (#1422) Signed-off-by: Tom Kerkhove Signed-off-by: shubhusion --- config.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config.toml b/config.toml index 30e16a5af..4e9a32197 100644 --- a/config.toml +++ b/config.toml @@ -593,7 +593,7 @@ logo = "zapier.png" # "Partners" section on the Community page [[params.partners]] -url = "https://kedify.io" +url = "https://kedify.io/?utm_source=keda_oss_site" logo = "kedify.png" [[params.partners]] From eedd10ebd10cb6f4f4156f579854776b7368de98 Mon Sep 17 00:00:00 2001 From: Uchechukwu Obasi Date: Wed, 10 Jul 2024 13:39:57 +0100 Subject: [PATCH 18/33] chore: fix faq version dropdown in older versions (#1423) Signed-off-by: shubhusion --- content/docs/1.4/{ => reference}/faq.md | 0 content/docs/1.5/{ => reference}/faq.md | 0 content/docs/2.0/{ => reference}/faq.md | 0 content/docs/2.1/{ => reference}/faq.md | 0 content/docs/2.10/{ => reference}/faq.md | 0 content/docs/2.11/{ => reference}/faq.md | 0 content/docs/2.12/{ => reference}/faq.md | 0 content/docs/2.13/{ => reference}/faq.md | 0 content/docs/2.2/{ => reference}/faq.md | 0 content/docs/2.3/{ => reference}/faq.md | 0 content/docs/2.4/{ => reference}/faq.md | 0 content/docs/2.5/{ => reference}/faq.md | 0 content/docs/2.6/{ => reference}/faq.md | 0 content/docs/2.7/{ => reference}/faq.md | 0 content/docs/2.8/{ => reference}/faq.md | 0 content/docs/2.9/{ => reference}/faq.md | 0 layouts/shortcodes/faq20.html | 1 - 17 files changed, 1 deletion(-) rename content/docs/1.4/{ => reference}/faq.md (100%) rename content/docs/1.5/{ => reference}/faq.md (100%) rename content/docs/2.0/{ => reference}/faq.md (100%) rename content/docs/2.1/{ => reference}/faq.md (100%) rename content/docs/2.10/{ => reference}/faq.md (100%) rename content/docs/2.11/{ => reference}/faq.md (100%) rename content/docs/2.12/{ => reference}/faq.md (100%) rename content/docs/2.13/{ => reference}/faq.md (100%) rename content/docs/2.2/{ => reference}/faq.md (100%) rename content/docs/2.3/{ => reference}/faq.md (100%) rename content/docs/2.4/{ => reference}/faq.md (100%) rename content/docs/2.5/{ => reference}/faq.md (100%) rename content/docs/2.6/{ => reference}/faq.md (100%) rename content/docs/2.7/{ => reference}/faq.md (100%) rename content/docs/2.8/{ => reference}/faq.md (100%) rename content/docs/2.9/{ => reference}/faq.md (100%) diff --git a/content/docs/1.4/faq.md b/content/docs/1.4/reference/faq.md similarity index 100% rename from content/docs/1.4/faq.md rename to content/docs/1.4/reference/faq.md diff --git a/content/docs/1.5/faq.md b/content/docs/1.5/reference/faq.md similarity index 100% rename from content/docs/1.5/faq.md rename to content/docs/1.5/reference/faq.md diff --git a/content/docs/2.0/faq.md b/content/docs/2.0/reference/faq.md similarity index 100% rename from content/docs/2.0/faq.md rename to content/docs/2.0/reference/faq.md diff --git a/content/docs/2.1/faq.md b/content/docs/2.1/reference/faq.md similarity index 100% rename from content/docs/2.1/faq.md rename to content/docs/2.1/reference/faq.md diff --git a/content/docs/2.10/faq.md b/content/docs/2.10/reference/faq.md similarity index 100% rename from content/docs/2.10/faq.md rename to content/docs/2.10/reference/faq.md diff --git a/content/docs/2.11/faq.md b/content/docs/2.11/reference/faq.md similarity index 100% rename from content/docs/2.11/faq.md rename to content/docs/2.11/reference/faq.md diff --git a/content/docs/2.12/faq.md b/content/docs/2.12/reference/faq.md similarity index 100% rename from content/docs/2.12/faq.md rename to content/docs/2.12/reference/faq.md diff --git a/content/docs/2.13/faq.md b/content/docs/2.13/reference/faq.md similarity index 100% rename from content/docs/2.13/faq.md rename to content/docs/2.13/reference/faq.md diff --git a/content/docs/2.2/faq.md b/content/docs/2.2/reference/faq.md similarity index 100% rename from content/docs/2.2/faq.md rename to content/docs/2.2/reference/faq.md diff --git a/content/docs/2.3/faq.md b/content/docs/2.3/reference/faq.md similarity index 100% rename from content/docs/2.3/faq.md rename to content/docs/2.3/reference/faq.md diff --git a/content/docs/2.4/faq.md b/content/docs/2.4/reference/faq.md similarity index 100% rename from content/docs/2.4/faq.md rename to content/docs/2.4/reference/faq.md diff --git a/content/docs/2.5/faq.md b/content/docs/2.5/reference/faq.md similarity index 100% rename from content/docs/2.5/faq.md rename to content/docs/2.5/reference/faq.md diff --git a/content/docs/2.6/faq.md b/content/docs/2.6/reference/faq.md similarity index 100% rename from content/docs/2.6/faq.md rename to content/docs/2.6/reference/faq.md diff --git a/content/docs/2.7/faq.md b/content/docs/2.7/reference/faq.md similarity index 100% rename from content/docs/2.7/faq.md rename to content/docs/2.7/reference/faq.md diff --git a/content/docs/2.8/faq.md b/content/docs/2.8/reference/faq.md similarity index 100% rename from content/docs/2.8/faq.md rename to content/docs/2.8/reference/faq.md diff --git a/content/docs/2.9/faq.md b/content/docs/2.9/reference/faq.md similarity index 100% rename from content/docs/2.9/faq.md rename to content/docs/2.9/reference/faq.md diff --git a/layouts/shortcodes/faq20.html b/layouts/shortcodes/faq20.html index c26bdcd69..79be1a78a 100644 --- a/layouts/shortcodes/faq20.html +++ b/layouts/shortcodes/faq20.html @@ -5,7 +5,6 @@ {{ $faq20 := $.Scratch.Get "faq20" }} -

General

{{ range where $faq20 ".type" "==" "General" }} From cabfc599ec5439dd9064e4ca9d8fd352354fa97c Mon Sep 17 00:00:00 2001 From: SpiritZhou Date: Thu, 11 Jul 2024 14:39:46 +0800 Subject: [PATCH 19/33] Update (#1420) Signed-off-by: SpiritZhou Signed-off-by: shubhusion --- content/docs/2.15/operate/cloud-events.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/content/docs/2.15/operate/cloud-events.md b/content/docs/2.15/operate/cloud-events.md index e07dc0505..ad46c74f1 100644 --- a/content/docs/2.15/operate/cloud-events.md +++ b/content/docs/2.15/operate/cloud-events.md @@ -113,4 +113,5 @@ eventSubscription: #Optional. Submit included/excluded event types will filter e | Event Type | Scenario Description | |-------------------------------|-----------------------------------------------------------------------------------------------------------------------------| | `keda.scaledobject.ready.v1` | On the first time a ScaledObject is ready, or if the previous ready condition status of the object was `Unknown` or `False` | -| `keda.scaledobject.failed.v1` | If the check validation for a ScaledObject fails | +| `keda.scaledobject.failed.v1` | If the check validation for a ScaledObject fails | +| `keda.scaledobject.removed.v1`| When a ScaledObject is deleted | From c868e70e5647449848cfab3d6dd278f9dbc3b606 Mon Sep 17 00:00:00 2001 From: Zbynek Roubalik Date: Fri, 12 Jul 2024 11:13:40 +0200 Subject: [PATCH 20/33] Add lightweight Scarf analytics pixel to thewebsite (#1426) Signed-off-by: shubhusion --- layouts/index.html | 2 ++ 1 file changed, 2 insertions(+) diff --git a/layouts/index.html b/layouts/index.html index 66857cddc..b51911fde 100644 --- a/layouts/index.html +++ b/layouts/index.html @@ -17,4 +17,6 @@ {{ partial "home/get-involved.html" . }} {{ partial "home/cncf.html" . }} + + {{ end -}} From 4c84e5e7cf8c00a44d6f01de67d991954cbae4d4 Mon Sep 17 00:00:00 2001 From: Zbynek Roubalik Date: Fri, 12 Jul 2024 19:29:28 +0200 Subject: [PATCH 21/33] put tracking pixel into correct location (#1427) Signed-off-by: Zbynek Roubalik Signed-off-by: shubhusion --- layouts/partials/footer.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/layouts/partials/footer.html b/layouts/partials/footer.html index e29de702a..b76a955b5 100644 --- a/layouts/partials/footer.html +++ b/layouts/partials/footer.html @@ -40,7 +40,7 @@

© KEDA Authors 2014-{{ $year }} | Documentation Distributed under CC-BY-4.0

- +

From 4080c9174429f9a2790afb4a4c498125007dc763 Mon Sep 17 00:00:00 2001 From: Zbynek Roubalik Date: Tue, 16 Jul 2024 17:07:26 +0200 Subject: [PATCH 22/33] navbar + hero refactor (#1428) Signed-off-by: shubhusion --- config.toml | 25 +++++++++++++++++++++---- layouts/partials/navbar.html | 4 ++-- 2 files changed, 23 insertions(+), 6 deletions(-) diff --git a/config.toml b/config.toml index 4e9a32197..5ddbe7389 100644 --- a/config.toml +++ b/config.toml @@ -175,29 +175,41 @@ name = "Architecture" parent = "docs" weight = 3 +[[menu.docs]] +url = "/docs/scalers" +name = "Scalers" +parent = "docs" +weight = 4 + [[menu.docs]] url = "/docs/operate/" name = "Operate" parent = "docs" -weight = 4 +weight = 5 + +[[menu.docs]] +url = "/docs/authentication-providers" +name = "Auth providers" +parent = "docs" +weight = 6 [[menu.docs]] url = "/docs/migration/" name = "Migration Guide" parent = "docs" -weight = 5 +weight = 7 [[menu.docs]] url = "/docs/reference/faq/" name = "FAQ" parent = "docs" -weight = 6 +weight = 8 [[menu.docs]] url = "/docs/troubleshooting/" name = "Troubleshooting" parent = "docs" -weight = 7 +weight = 9 [[menu.main]] url = "/blog" @@ -328,6 +340,11 @@ name = "Merch" parent = "project" weight = 12 +[[menu.main]] +url = "/support/" +name = "Support" +weight = 5 + # "Features" section on the main page [[params.features]] title = "Autoscaling Made Simple" diff --git a/layouts/partials/navbar.html b/layouts/partials/navbar.html index 09372c3e2..2d1cc8dbc 100644 --- a/layouts/partials/navbar.html +++ b/layouts/partials/navbar.html @@ -33,13 +33,13 @@

-
+ {{ range $docs }} {{ $isExternal := hasPrefix .URL "http" }} From a8bb32f63a19440570f41a7b00e7a72ee36447d6 Mon Sep 17 00:00:00 2001 From: Indresh2410 Date: Mon, 22 Jul 2024 18:07:09 +0530 Subject: [PATCH 23/33] Mysql scaler username from env (#1419) * Add usernameFromEnv support for MYSQL Scaler Signed-off-by: Indresh2410 * Fix typo Signed-off-by: Indresh2410 * Update Readme Signed-off-by: Indresh2410 * Move readme to different folder Signed-off-by: Indresh2410 --------- Signed-off-by: Indresh2410 Signed-off-by: shubhusion --- content/docs/2.15/scalers/mysql.md | 1 + 1 file changed, 1 insertion(+) diff --git a/content/docs/2.15/scalers/mysql.md b/content/docs/2.15/scalers/mysql.md index 8598d34c8..0dd3947ab 100644 --- a/content/docs/2.15/scalers/mysql.md +++ b/content/docs/2.15/scalers/mysql.md @@ -35,6 +35,7 @@ Some parameters could be provided using environmental variables, instead of sett - `hostFromEnv` - The host of the MySQL server, similar to `host`, but reads it from an environment variable on the scale target. - `portFromEnv` - The port of the MySQL server, similar to `port`, but reads it from an environment variable on the scale target. +- `usernameFromEnv` - The username for connecting with host of the MySQL server, similar to `username`, but reads it from an environment variable on the scale target. ### Authentication Parameters From 785fe4e344d29f2b545b67f3e93aa031a710b1b8 Mon Sep 17 00:00:00 2001 From: rickbrouwer <75609067+rickbrouwer@users.noreply.github.com> Date: Thu, 25 Jul 2024 12:20:57 +0200 Subject: [PATCH 24/33] feat: update IBM MQ scaler docs for TLS support (#1433) Signed-off-by: rickbrouwer <75609067+rickbrouwer@users.noreply.github.com> Signed-off-by: shubhusion --- content/docs/2.15/scalers/ibm-mq.md | 69 +++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) diff --git a/content/docs/2.15/scalers/ibm-mq.md b/content/docs/2.15/scalers/ibm-mq.md index 786e66ee2..dd1e83934 100644 --- a/content/docs/2.15/scalers/ibm-mq.md +++ b/content/docs/2.15/scalers/ibm-mq.md @@ -23,6 +23,7 @@ triggers: activationQueueDepth: # OPTIONAL - Activation queue depth target. Default: 0 messages usernameFromEnv: # Optional: Provide admin username from env instead of as a secret passwordFromEnv: # Optional: Provide admin password from env instead of as a secret + unsafeSsl: # OPTIONAL - Set 'true' for unsafe SSL. Default: false ``` **Parameter list:** @@ -35,6 +36,7 @@ triggers: - `activationQueueDepth` - Target value for activating the scaler. Learn more about activation [here](./../concepts/scaling-deployments.md#activating-and-scaling-thresholds). (Default: `0`, Optional) - `usernameFromEnv` - Provide admin username from env instead of as a secret. (Optional) - `passwordFromEnv` - Provide admin password from env instead of as a secret. (Optional) +- `unsafeSsl` - Whether to allow unsafe SSL (Values: `true`, `false`, Default: `false` ) ### Authentication Parameters @@ -44,11 +46,17 @@ TriggerAuthentication CRD is used to connect and authenticate to IBM MQ: - `ADMIN_USER` - REQUIRED - The admin REST endpoint username for your MQ Queue Manager`. - `ADMIN_PASSWORD` - REQUIRED - The admin REST endpoint API key for your MQ Queue Manager. +- `ca` - Certificate authority file for TLS client authentication. (Optional) +- `cert` - Certificate for client authentication. (Optional) +- `key` - Key for client authentication. (Optional) +- `keyPassword` - If set the keyPassword is used to decrypt the provided key. (Optional) - `usernameFromEnv` - Provide admin username from env instead of as a secret. (Optional) - `passwordFromEnv` - Provide admin password from env instead of as a secret. (Optional) ### Example +Example with Basic Auth + ```yaml apiVersion: v1 kind: Secret @@ -98,3 +106,64 @@ spec: name: keda-ibmmq-secret key: ADMIN_PASSWORD ``` + +### Example + +Example with Basic Auth and TLS + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: keda-ibmmq-secret +data: + ADMIN_USER: # REQUIRED - Admin Username + ADMIN_PASSWORD: # REQUIRED - Admin Password + cert: + key: +--- +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: ibmmq-scaledobject + namespace: default + labels: + deploymentName: ibmmq-deployment +spec: + scaleTargetRef: + name: ibmmq-deployment + pollingInterval: 5 # OPTIONAL - Default: 30 seconds + cooldownPeriod: 30 # OPTIONAL - Default: 300 seconds + maxReplicaCount: 18 # OPTIONAL - Default: 100 + triggers: + - type: ibmmq + metadata: + host: # REQUIRED - IBM MQ Queue Manager Admin REST Endpoint + queueManager: # REQUIRED - Your queue manager + queueName: # REQUIRED - Your queue name + tlsDisabled: # OPTIONAL - Set 'true' to disable TLS. Default: false + queueDepth: # OPTIONAL - Queue depth target for HPA. Default: 5 messages + authenticationRef: + name: keda-ibmmq-trigger-auth +--- +apiVersion: keda.sh/v1alpha1 +kind: TriggerAuthentication +metadata: + name: keda-ibmmq-trigger-auth + namespace: default +spec: + secretTargetRef: + - parameter: username + name: keda-ibmmq-secret + key: ADMIN_USER + - parameter: password + name: keda-ibmmq-secret + key: ADMIN_PASSWORD + - parameter: cert + name: keda-ibmmq-secret + key: cert + - parameter: key + name: keda-ibmmq-secret + key: key +``` + From 95d3a31a3833dec6a86eaf5fdada6ca748e845b5 Mon Sep 17 00:00:00 2001 From: Ben Woodley Date: Thu, 25 Jul 2024 12:22:44 +0200 Subject: [PATCH 25/33] feat: add GCP Pub/Sub new optional valueIfNull trigger configuration (#1410) Signed-off-by: Benjamin Woodley Signed-off-by: shubhusion --- content/docs/2.15/scalers/gcp-pub-sub.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/content/docs/2.15/scalers/gcp-pub-sub.md b/content/docs/2.15/scalers/gcp-pub-sub.md index 801dbc2c3..9427d9bb9 100644 --- a/content/docs/2.15/scalers/gcp-pub-sub.md +++ b/content/docs/2.15/scalers/gcp-pub-sub.md @@ -19,6 +19,7 @@ triggers: mode: "SubscriptionSize" # Optional - Default is SubscriptionSize - SubscriptionSize or OldestUnackedMessageAge aggregation: "sum" # Optional - Only meaningful for distribution-valued metrics value: "5.5" # Optional - Default is 10 + valueIfNull: '0.0' # Optional - Default is "" activationValue: "10.5" # Optional - Default is 0 timeHorizon: "1m" # Optional - Default is 2m and with aggregation 5m # Exactly one of the subscription or topic name options is required @@ -42,6 +43,8 @@ The Google Cloud Platform (GCP) Pub/Sub trigger allows you to scale based on any - `timeHorizon` - Time range for which you want to retrieve the matrics. (Default: `2m` and Default with aggregation field: `5m`) +- `valueIfNull` - Value returned if request returns no timeseries.(Default: `""`, Optional, This value can be a float) + - `subscriptionName` defines the subscription that should be monitored. You can use different formulas: - Just the subscription name, in which case you will reference a subscription from the current project or the one specified in the credentials file used. - Use the full link provided by Google, so that you can reference a subscription that is hosted in another project Eg: `projects/myproject/subscriptions/mysubscription`. From fe0e494761d6fa2926b99cf21186bd0d203dacd6 Mon Sep 17 00:00:00 2001 From: Ferdinand de Baecque <45566171+Ferdinanddb@users.noreply.github.com> Date: Mon, 29 Jul 2024 11:07:10 +0200 Subject: [PATCH 26/33] Update postgres scaler doc for aad (#1437) * Update postgres scaler doc to include info about Azure access token auth Signed-off-by: Ferdinand de Baecque <45566171+Ferdinanddb@users.noreply.github.com> * minor changes to respect conventions Signed-off-by: Ferdinand de Baecque <45566171+Ferdinanddb@users.noreply.github.com> --------- Signed-off-by: Ferdinand de Baecque <45566171+Ferdinanddb@users.noreply.github.com> Signed-off-by: shubhusion --- content/docs/2.15/scalers/postgresql.md | 77 +++++++++++++++++++++++-- 1 file changed, 71 insertions(+), 6 deletions(-) diff --git a/content/docs/2.15/scalers/postgresql.md b/content/docs/2.15/scalers/postgresql.md index 9aa5b8f04..473fb8e4e 100644 --- a/content/docs/2.15/scalers/postgresql.md +++ b/content/docs/2.15/scalers/postgresql.md @@ -11,10 +11,10 @@ go_file = "postgresql_scaler" This specification describes the `postgresql` trigger that scales based on a PostgreSQL query -The PostgreSQL scaler allows for two connection options: +The PostgreSQL scaler allows for three connection options: A user can offer a full connection string -(often in the form of an environment variable secret) +(often in the form of an environment variable secret). - `connectionFromEnv` - PostgreSQL connection string that should point to environment variable with valid value. @@ -22,13 +22,16 @@ Alternatively, a user can specify individual arguments (host, userName, password, etc.), and the scaler will form a connection string internally. -- `host:` - Service URL to postgresql. Note that you should use a full svc URL as KEDA will need to contact postgresql from a different namespace. -- `userName:` - Username for postgresql user. +- `host` - Service URL to postgresql. Note that you should use a full svc URL as KEDA will need to contact postgresql from a different namespace. +- `userName` - Username for postgresql user. - `passwordFromEnv` Password for postgresql user. - `port` - Postgresql port. - `dbName` - Postgresql Database name. - `sslmode` - SSL policy for communicating with database. +It is also possible to leverage a `TriggerAuthentication` object having the `azure-workload`'s provider type to connect to an Azure Postgres Flexible Server resource through an UAMI Azure managed identity. +More details and an example are provided down below. + Finally, a user inserts a query that returns the desired value. - `query` - What query to poll postgresql with. Query must return an integer. @@ -70,7 +73,7 @@ triggers: ### Authentication Parameters -You can authenticate by using a password or store the password within the connectionString. +You can authenticate by using a password, or store the password within the connectionString, or leverage Azure Access Token authentication to connect to a Azure Postgres Flexible Server. **Connection String Authentication:** @@ -85,7 +88,7 @@ You can authenticate by using a password or store the password within the connec - `dbName` - PostgreSQL Database name. - `sslmode` - SSL policy for communicating with database. -### Example +#### Example ```yaml apiVersion: keda.sh/v1alpha1 @@ -105,3 +108,65 @@ spec: query: "SELECT ceil(COUNT(*)::decimal / 16) FROM task_instance WHERE state='running' OR state='queued';" targetQueryValue: 1 ``` + +**Azure Access Token authentication:** + +#### Prerequisites: +- The UAMI should be able to access the Azure Postgres Flexible Server, [refer to this link for more info](https://learn.microsoft.com/en-us/azure/postgresql/flexible-server/how-to-connect-with-managed-identity#create-an-azure-database-for-postgresql-flexible-server-user-for-your-managed-identity). +- The UAMI should be granted access to the table mentioned in the query performed by KEDA. +This can be achieved by: + - creating a group role to allow access to the particular schema where the table queried by KEDA is persisted, and then assign the newly created Postgres user identity from previous step to this group role. + - granting permission on the table queried by KEDA to the newly created Postgres user from previous step, via a query that looks like + `GRANT ALL ON TO "";`. + +Next, a user can specify individual arguments (host, userName, password, etc.), and the scaler will form a connection string internally. An access token, which will act as a password, will be retrieved each time KEDA performs its process. + - `host` - FQDN of the Azure Postgres Flexible Server. + - `userName` - Name of the UAMI Azure identity (``). + - `port` - Postgresql port (the default value is `"5432"`, please have a look at the `Remarks` down below). + - `dbName` - Postgresql Database name. + - `sslmode` - SSL policy for communicating with database (the value should be `require`). + + +#### Remarks + +- While the Azure Postgres Flexible Server resource provides a [`PGBouncer`](https://learn.microsoft.com/en-us/azure/postgresql/flexible-server/concepts-pgbouncer) feature which opens a port `6432` to interact with the server, this access token authentication's feature was not working properly while using the `PGBouncer` port, but it worked without issues while using the default server's port. Therefore, KEDA should use the Postgres server's default port, but the other applications (i.e. Airflow, ...) deployed on the same Kubernetes cluster can use the `PGBouncer` port. + + +#### Example + +```yaml +apiVersion: keda.sh/v1alpha1 +kind: TriggerAuthentication +metadata: + name: azure-pg-flex-auth +spec: + podIdentity: + provider: azure-workload + # Optional-> identityId: + # Optional-> identityTenantId: + +--- + +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: airflow-worker +spec: + scaleTargetRef: + name: airflow-worker + pollingInterval: 10 # Optional. Default: 30 seconds + cooldownPeriod: 30 # Optional. Default: 300 seconds + maxReplicaCount: 10 # Optional. Default: 100 + triggers: + - type: postgresql + authenticationRef: + name: azure-pg-flex-auth + metadata: + host: + port: "5432" + userName: + dbName: + sslmode: require + query: "SELECT ceil(COUNT(*)::decimal / 16) FROM task_instance WHERE state='running' OR state='queued';" + targetQueryValue: 1 +``` \ No newline at end of file From e6e63dd11e169b7c11536b88d3f435eb23765d10 Mon Sep 17 00:00:00 2001 From: Ara Pulido Date: Mon, 29 Jul 2024 11:44:28 +0200 Subject: [PATCH 27/33] Add Datadog scaler docs for the new option to use the Cluster Agent as proxy (#1310) * Add docs for Datadog scaler with Cluster Agent Signed-off-by: Ara Pulido * Add optional parameters for the Cluster Agent API server Signed-off-by: Ara Pulido * Move new DD docs to 2.14 Signed-off-by: Ara Pulido * Add missing cluster agent option Signed-off-by: Ara Pulido * Add option in the cluster agent to avoid autogenerating DatadogMetric objects Signed-off-by: Ara Pulido * fix typo in boolean string type Signed-off-by: Ara Pulido * Move Datadog cluster agent docs to 2.15 instead Signed-off-by: Ara Pulido * Mark using the Cluster Agent as proxy as experimental Signed-off-by: Ara Pulido * datadogMetricsService paramter is mandatory Signed-off-by: Ara Pulido * Establish 1.8.0 as the minimum version of the Datadog Operator required for this Signed-off-by: Ara Pulido --------- Signed-off-by: Ara Pulido Signed-off-by: shubhusion --- content/docs/2.15/scalers/datadog.md | 179 +++++++++++++++++++++++++-- 1 file changed, 171 insertions(+), 8 deletions(-) diff --git a/content/docs/2.15/scalers/datadog.md b/content/docs/2.15/scalers/datadog.md index 540fa96f5..9e7bd4a4d 100644 --- a/content/docs/2.15/scalers/datadog.md +++ b/content/docs/2.15/scalers/datadog.md @@ -13,15 +13,177 @@ polling interval. For more detailed information about polling intervals check [the Polling intervals and Datadog rate limiting section](#polling-intervals-and-datadog-rate-limiting). +There are two ways to poll Datadog for a query value using the Datadog scaler: using the REST API endpoints, or using the [Datadog Cluster Agent](https://docs.datadoghq.com/containers/cluster_agent/) as proxy. Using the Datadog Cluster Agent as proxy reduces the chance of reaching rate limits. As both types are different in terms of usage and authentication, this documentation handles them separately. + +## Using the Datadog Cluster Agent (Experimental) + +With this method, the Datadog scaler will be connecting to the Datadog Cluster Agent to retrieve the query values that will be used to drive the KEDA scaling events. This reduces the risk of reaching rate limits for the Datadog API, as the Cluster Agent retrieves metric values in batches. + +### Deploy the Datadog Cluster Agent with enabled external metrics + +First, deploy the Datadog Cluster Agent enabling the external metrics provider, but without registering it as an `APIService` (to avoid clashing with KEDA). + +If you are using Helm to deploy the Cluster Agent, set: + +* `clusterAgent.metricsProvider.enabled` to `true` +* `clusterAgent.metricsProvider.registerAPIService` to `false` +* `clusterAgent.metricsProvider.useDatadogMetrics` to `true` +* `clusterAgent.env` to `[{name: DD_EXTERNAL_METRICS_PROVIDER_ENABLE_DATADOGMETRIC_AUTOGEN, value: "false"}]` + +If you are using the Datadog Operator, add the following options to your `DatadogAgent` object: + +``` +apiVersion: datadoghq.com/v2alpha1 +kind: DatadogAgent +metadata: + name: datadog +spec: + features: + externalMetricsServer: + enabled: true + useDatadogMetrics: true + registerAPIService: false + override: + clusterAgent: + env: [{name: DD_EXTERNAL_METRICS_PROVIDER_ENABLE_DATADOGMETRIC_AUTOGEN, value: "false"}] +[...] +``` + +NOTE: Using the Datadog Operator for this purpose requires version 1.8.0 of the operator or later. + +### Create a DatadogMetric object for each scaling query + +To use the Datadog Cluster Agent to retrieve the query values from Datadog, first, create a [`DatadogMetric`](https://docs.datadoghq.com/containers/guide/cluster_agent_autoscaling_metrics/?tab=helm#create-the-datadogmetric-object) object with the query to drive your scaling events. You will need to add the `external-metrics.datadoghq.com/always-active: "true"` annotation, to ensure the Cluster Agent retrieves the query value. Example: + +```yaml +apiVersion: datadoghq.com/v1alpha1 +kind: DatadogMetric +metadata: + annotations: + external-metrics.datadoghq.com/always-active: "true" + name: nginx-hits +spec: + query: sum:nginx.net.request_per_s{kube_deployment:nginx} +``` + +### Trigger Specification + +This specification describes the `datadog` trigger that scales based on a Datadog query, using the Datadog Cluster Agent as proxy. + +```yaml +triggers: +- type: datadog + metricType: Value + metadata: + useClusterAgentProxy: "true" + datadogMetricName: "nginx-hits" + datadogMetricNamespace: "default" + targetValue: "7.75" + activationQueryValue: "1.1" + type: "global" # Deprecated in favor of trigger.metricType + metricUnavailableValue: "1.5" +``` + +**Parameter list:** + +- `useClusterAgentProxy` - Whether to use the Cluster Agent as proxy to get the query values. (Values: true, false, Default: false, Optional) +- `datadogMetricName` - The name of the `DatadogMetric` object to drive the scaling events. +- `datadogMetricNamespace` - The namespace of the `DatadogMetric` object to drive the scaling events. +- `targetValue` - Value to reach to start scaling (This value can be a float). +- `activationQueryValue` - Target value for activating the scaler. Learn more about activation [here](./../concepts/scaling-deployments.md#activating-and-scaling-thresholds).(Default: `0`, Optional, This value can be a float) +- `type` - Whether to start scaling based on the value or the average between pods. (Values: `average`, `global`, Default:`average`, Optional) +- `age`: The time window (in seconds) to retrieve metrics from Datadog. (Default: `90`, Optional) +- `lastAvailablePointOffset`: The offset to retrieve the X to last data point. The value of last data point of some queries might be inaccurate [because of the implicit rollup function](https://docs.datadoghq.com/dashboards/functions/rollup/#rollup-interval-enforced-vs-custom), try to adjust to `1` if you encounter this issue. (Default: `0`, Optional) +- `metricUnavailableValue`: The value of the metric to return to the HPA if Datadog doesn't find a metric value for the specified time window. If not set, an error will be returned to the HPA, which will log a warning. (Optional, This value can be a float) + +> 💡 **NOTE:** The `type` parameter is deprecated in favor of the global `metricType` and will be removed in a future release. Users are advised to use `metricType` instead. + +### Authentication + +The Datadog scaler with Cluster Agent supports one type of authentication - Bearer authentication. + +You can use `TriggerAuthentication` CRD to configure the authentication. Specify `authMode` and other trigger parameters along with secret credentials in `TriggerAuthentication` as mentioned below: + +**Common to all authentication types** +- `authMode` - The authentication mode to connect to the Cluster Agent. (Values: bearer, Default: bearer, Optional) +- `datadogNamespace` - The namespace where the Datadog Cluster Agent is deployed. +- `datadogMetricsService` - The service name for the Cluster Agent metrics server. To find the name of the service, check the available services in the Datadog namespace and look for the `*-cluster-agent-metrics*` name pattern. +- `datadogMetricsServicePort` - The port of the service for the Cluster Agent Metrics API. (Default: 8443, Optional) +- `unsafeSsl` - Skip certificate validation when connecting over HTTPS. (Values: true, false, Default: false, Optional) + +**Bearer authentication:** +- `token` - The ServiceAccount token to connect to the Datadog Cluster Agent. The service account needs to have permissions to `get`, `watch`, and `list` all `external.metrics.k8s.io` resources. + +### Example + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: datadog-config + namespace: my-project +type: Opaque +data: + datadogNamespace: # Required: base64 encoded value of the namespace where the Datadog Cluster Agent is deployed + datadogMetricsService: # Required: base64 encoded value of the Cluster Agent metrics server service + unsafeSsl: # Optional: base64 encoded value of `true` or `false` + authMode: # Required: base64 encoded value of the authentication mode (in this case, bearer) +--- +apiVersion: keda.sh/v1alpha1 +kind: TriggerAuthentication +metadata: + name: datadog-cluster-agent-creds + namespace: my-project +spec: + secretTargetRef: + - parameter: token + name: dd-cluster-agent-token + key: token + - parameter: datadogNamespace + name: datadog-config + key: namespace + - parameter: unsafeSsl + name: datadog-config + key: unsafeSsl + - parameter: authMode + name: datadog-config + key: authMode +--- +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: datadog-scaledobject + namespace: my-project +spec: + scaleTargetRef: + name: nginx + maxReplicaCount: 3 + minReplicaCount: 1 + pollingInterval: 60 + triggers: + - type: datadog + metadata: + useClusterAgentProxy: "true" + datadogMetricName: "nginx-hits" + datadogMetricNamespace: "default" + targetValue: "2" + type: "global" + authenticationRef: + name: datadog-cluster-agent-creds +``` + +## Using the Datadog REST API + ### Trigger Specification -This specification describes the `datadog` trigger that scales based on a Datadog metric. +This specification describes the `datadog` trigger that scales based on a Datadog query, using the Datadog REST API. ```yaml triggers: - type: datadog metricType: Value metadata: + useClusterAgentProxy: "false" query: "sum:trace.redis.command.hits{env:none,service:redis}.as_count()" queryValue: "7.75" activationQueryValue: "1.1" @@ -35,6 +197,7 @@ triggers: **Parameter list:** +- `useClusterAgentProxy` - Whether to use the Cluster Agent as proxy to get the query values. (Default: false) - `query` - The Datadog query to run. - `queryValue` - Value to reach to start scaling (This value can be a float). - `activationQueryValue` - Target value for activating the scaler. Learn more about activation [here](../concepts/scaling-deployments#activating-and-scaling-thresholds).(Default: `0`, Optional, This value can be a float) @@ -124,7 +287,7 @@ spec: name: keda-trigger-auth-datadog-secret ``` -## Polling intervals and Datadog rate limiting +### Polling intervals and Datadog rate limiting [API Datadog endpoints are rate limited](https://docs.datadoghq.com/api/latest/rate-limits/). Depending on the @@ -146,14 +309,14 @@ was started with `--horizontal-pod-autoscaler-sync-period=30`, the HPA will poll Datadog for a metric value every 30 seconds while the number of replicas is between 1 and N. -## Multi-Query Support +### Multi-Query Support To reduce issues with API rate limiting from Datadog, it is possible to send a single query, which contains multiple queries, comma-seperated. When doing this, the results from each query are aggregated based on the `queryAggregator` value (eg: `max` or `average`). > 💡 **NOTE:** Because the average/max aggregation operation happens at the scaler level, there won't be any validation or errors if the queries don't make sense to aggregate. Be sure to read and understand the two patterns below before using Multi-Query. -### Example 1 - Aggregating Similar Metrics +#### Example 1 - Aggregating Similar Metrics Simple aggregation works well, when wanting to scale on more than one metric with similar return values/scale (ie. where multiple metrics can use a single `queryValue` and still make sense). @@ -188,7 +351,7 @@ The example above looks at the `http.requests` value for a service; taking two v This works particularly well when scaling against the same metric, but with slightly different parameters, or methods like ```week_before()``` for example. -### Example 2 - Driving scale directly +#### Example 2 - Driving scale directly When wanting to scale on non-similar metrics, whilst still benefiting from reduced API calls with multi-query support, the easiest way to do this is to make each query directly return the desired scale (eg: number of pods), and then `max` or `average` the results to get the desired target scale. @@ -224,9 +387,9 @@ spec: Using the example above, if we assume that `http.requests` is currently returning `360`, dividing that by `180` in the query, results in a value of `2`; if `http.backlog` returns `90`, dividing that by `30` in the query, results in a value of `3`. With the `max` Aggregator set, the scaler will set the target scale to `3` as that is the higher value from all returned queries. -## Cases of unexpected metrics value in DataDog API response +### Cases of unexpected metrics value in DataDog API response -### Latest data point is unavailable +#### Latest data point is unavailable By default, Datadog scaler retrieves the metrics with time window from `now - metadata.age (in seconds)` to `now`, however, some kinds of queries need a small delay (usually 30 secs - 2 mins) before data is available when querying from the API. In this case, adjust `timeWindowOffset` to ensure that the latest point of your query is always available. @@ -256,7 +419,7 @@ spec: ``` Check [here](https://github.com/kedacore/keda/pull/3954#discussion_r1042820206) for the details of this issue -### The value of last data point is inaccurate +#### The value of last data point is inaccurate Datadog implicitly rolls up data points automatically with the `avg` method, effectively displaying the average of all data points within a time interval for a given metric. Essentially, there is a rollup for each point. The values at the end attempt to have the rollup applied. When this occurs, it looks at a X second bucket according to your time window, and will default average those values together. Since this is the last point in the query, there are no other values to average with in that X second bucket. This leads to the value of last data point that was not rolled up in the same fashion as the others, and leads to an inaccurate number. In these cases, adjust `lastAvailablePointOffset` to 1 to use the second to last points of an API response would be the most accurate. From 2a3835c753bfa7c2539745345c85f7f00b576722 Mon Sep 17 00:00:00 2001 From: Leonardo D'Ippolito Date: Tue, 30 Jul 2024 11:43:42 +0100 Subject: [PATCH 28/33] Add queueLengthStrategy option to azure-storage-queue scaler. (#1406) Signed-off-by: Leonardo D'Ippolito Signed-off-by: shubhusion --- content/docs/2.15/scalers/azure-storage-queue.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/content/docs/2.15/scalers/azure-storage-queue.md b/content/docs/2.15/scalers/azure-storage-queue.md index 76402c14a..a95618537 100644 --- a/content/docs/2.15/scalers/azure-storage-queue.md +++ b/content/docs/2.15/scalers/azure-storage-queue.md @@ -17,6 +17,7 @@ triggers: metadata: queueName: orders queueLength: '5' + queueLengthStrategy: all|visibleonly activationQueueLength: '50' connectionFromEnv: STORAGE_CONNECTIONSTRING_ENV_NAME accountName: storage-account-name @@ -27,6 +28,7 @@ triggers: - `queueName` - Name of the queue. - `queueLength` - Target value for queue length passed to the scaler. Example: if one pod can handle 10 messages, set the queue length target to 10. If the actual number of messages in the queue is 30, the scaler scales to 3 pods. (Default: `5`, Optional) +- `queueLengthStrategy` - `all` considers both visible and invisible messages, while `visibleonly` uses Peek to count only visible messages. In `visibleonly`, if the count of messages is 32 or higher, it falls back to the default `all` strategy, counting both visible and invisible messages. (Default: `all`, Optional) - `activationQueueLength` - Target value for activating the scaler. Learn more about activation [here](./../concepts/scaling-deployments.md#activating-and-scaling-thresholds). (Default: `0`, Optional) - `connectionFromEnv` - Name of the environment variable your deployment uses to get the connection string. - `accountName` - Name of the storage account that the queue belongs to. @@ -78,6 +80,8 @@ spec: # # Optional queueLength: "5" # default 5 + # Optional + queueLengthStrategy: "all" # or visibleonly. Default: all cloud: Private endpointSuffix: queue.local.azurestack.external # Required when cloud=Private authenticationRef: From b9dedbec1dc9e4936f0da9ffadceb814530e86e8 Mon Sep 17 00:00:00 2001 From: Zbynek Roubalik Date: Tue, 30 Jul 2024 15:22:25 +0200 Subject: [PATCH 29/33] add explanation fore eager scaling strategy (#1438) Signed-off-by: Zbynek Roubalik Signed-off-by: shubhusion --- content/docs/2.15/reference/scaledjob-spec.md | 55 ++++++++++++++++++- 1 file changed, 54 insertions(+), 1 deletion(-) diff --git a/content/docs/2.15/reference/scaledjob-spec.md b/content/docs/2.15/reference/scaledjob-spec.md index dbc33680b..43fe6a6b0 100644 --- a/content/docs/2.15/reference/scaledjob-spec.md +++ b/content/docs/2.15/reference/scaledjob-spec.md @@ -150,7 +150,7 @@ scalingStrategy: strategy: "default" # Optional. Default: default. Which Scaling Strategy to use. ``` -Select a Scaling Strategy. Possible values are `default`, `custom`, or `accurate`. The default value is `default`. +Select a Scaling Strategy. Possible values are `default`, `custom`, `accurate`, or `eager`. The default value is `default`. > 💡 **NOTE:** > @@ -221,6 +221,59 @@ if (maxScale + runningJobCount) > maxReplicaCount { ``` For more details, you can refer to [this PR](https://github.com/kedacore/keda/pull/1227). +**eager** +When adopting the **default** strategy, you are likely to come into a subtle case where messages need to be consumed by spawning jobs but remain in the queue, even when there are available slots between `runningJobCount` and `maxReplicaCount`. The **eager** strategy comes to the rescue. It addresses this issue by utilizing all available slots up to the maxReplicaCount, ensuring that waiting messages are processed as quickly as possible. + +For example, let's assume we configure a ScaledJob in a cluster as below: +```yaml + ### + # A job that runs for a minimum of 3 hours. + ### + pollingInterval: 10 # Optional. Default: 30 seconds + maxReplicaCount: 10 # Optional. Default: 100 + triggers: + - type: rabbitmq + metadata: + queueName: woker_queue + hostFromEnv: RABBITMQ_URL + mode: QueueLength + value: "1" +``` +We send 3 messages to the Rabbitmq and wait longer enough than the `pollingInterval`, then send another 3. + +With the `default` scaling strategy, we are supposed to see the metrics changes in the following table: + +| | initial | incoming 3 messages | after poll | incoming 3 messages | after poll | +|-------------|---------|---------------------|------------|---------------------|------------| +| queueLength | 0 | 3 | 3 | 6 | 6 | +| runningJobs | 0 | 0 | 3 | 3 | 3 | + + +If we switch to `eager`, the result becomes: + +| | initial | incoming 3 messages | after poll | incoming 3 messages | after poll | +|-------------|---------|---------------------|------------|---------------------|------------| +| queueLength | 0 | 3 | 3 | 6 | 6 | +| runningJobs | 0 | 0 | 3 | 3 | 6 | + +We can identify the difference in their final states. + + +You may also refer to [this original issue](https://github.com/kedacore/keda/issues/5114) for more information. + +--- + +```yaml +scalingStrategy: + multipleScalersCalculation : "max" # Optional. Default: max. Specifies how to calculate the target metrics (`queueLength` and `maxScale`) when multiple scalers are defined. +``` +Select a behavior if you have multiple triggers. Possible values are `max`, `min`, `avg`, or `sum`. The default value is `max`. + +* **max:** - Use metrics from the scaler that has the max number of `queueLength`. (default) +* **min:** - Use metrics from the scaler that has the min number of `queueLength`. +* **avg:** - Sum up all the active scalers metrics and divide by the number of active scalers. +* **sum:** - Sum up all the active scalers metrics. + ### multipleScalersCalculation From 824c5d3c1cec4e128ddb73f2339c02d6eea40dfe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Davide=20Benvegn=C3=B9=20aka=20CoderDave?= Date: Tue, 30 Jul 2024 21:26:58 +0800 Subject: [PATCH 30/33] Added Playstudios as listed user (#1436) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Davide Benvegnu Signed-off-by: Davide Benvegnù aka CoderDave Signed-off-by: shubhusion --- config.toml | 4 ++++ static/img/logos/playstudios.jpg | Bin 0 -> 225305 bytes 2 files changed, 4 insertions(+) create mode 100644 static/img/logos/playstudios.jpg diff --git a/config.toml b/config.toml index 5ddbe7389..b54fe8d23 100644 --- a/config.toml +++ b/config.toml @@ -523,6 +523,10 @@ logo = "nemlig.png" url = "https://opsworks.co" logo = "opsworks.png" +[[params.users]] +url = "https://www.playstudios.com" +logo = "playstudios.jpg" + [[params.users]] url = "https://www.pollinate.global" logo = "pollinate.gif" diff --git a/static/img/logos/playstudios.jpg b/static/img/logos/playstudios.jpg new file mode 100644 index 0000000000000000000000000000000000000000..bd8c608808a0697c8370873bef12848270520ca7 GIT binary patch literal 225305 zcmeFZc|6-`+c%s}XQ~^f(^9ogr!BR2v9EXQLRHaHd&$_BSc=+3=Bkz%ik70ajw(tJ zp-B*ADkVipZ3!aMh&_p2hzQRwX5Qz%?&tmMy`Ss;+}HcQkN(kLK0bcu@i}rF=lT79 zkK;J^NB6&izQ1i|VFo&M=o?TJ@DH?40hs`w{`Y_$_hYwy3`s?S|aiOCpj(qp+;ctN7dwvf(^zAp_96J2XH%AU1 z7dr9HS0T$E4ALp6;`oI8I%GIvDb znx@uG3n%Aab3R?YuKlR;9f_=Q%ibX(>eb~Nrsj6uK7NF7U=685hd>7={9hJ;Kte~4 z0Tvtwera+FFz}m0-yZrFSXk)0qldl&{{QB?BZq`epZVeFDKYVj+UWC_^pCkWI;+c6C3m_sw9i8Lrr$iJ!xS+=Ad|ks5bRr?cfQ4id!W64nW_?a6RE)Fe2` z{w*m;TT0&@?CMrs?&pz6e0)x&?`x&4ZjBWkak84-V253w-AIo*Jbmi7jD9=1=V$^P-DLb? zt;DrAd`3Yx=I?PGhXLpr!Gy5?na3 z^5mEUoQ@chK#sPzM{gzi+dO;1#SLn;deJp^Gov5QY;|>NZ95dpnDuV?={l}_rMLg| zjW#u>;wHDBwr7>FZj61ckEY#g>tvIwOlbU)bjKBwZQ+fq(anI)d6Tb$>_l#mrjamI z&eF|sF2dAgG@tmDWtB%gyLEj5gOMR{II<~=yTyd!&{2(3rr_~rb)0cYbT!WlLE&=F zwo!8sPYpNsLGF^YBE%76@5#t@T#G1l*IS`)FC$87`50n49-I6*@wPp&kqGfW6`P8&#e za@CSs0Z*B^x$1;+*VmUCB$e9rB}diZQ@~k@+BWcX>#NwN+GA!=-&Ypvv#%&olIK3& z-wx@r$QAaUg1oSe8LJ#6b}k6T#gnX-A((0#Q5IJ?HBI1o?eYFx1G>(O6)q( z>4eI$%M3lk&$`Q3=g9S=>moPmC!JDenC4zO^fL2qVJG?BsJge2oB7Lr$0Nfr^Y;-; zRw7!eu{DuB4-n?UU!|{OHm;K&E*T;Q919q%Zr5Y-xOjqQ6@52@G?A+nukNakMpiCQ zG%nz^?5npS8(k`49-pv4m}$o&iuknr(~R>*p&RK4iES&`1?q$@bQNjUE>@XxuYo_- zZ@;Uf6Z-2f>03Sl&)Xv~`=D&F0dZ+IfsGjs4U3Olyt}mz@?+TVe!q7ZdJ%2t5*GLU zwgwj5?ka4P1U4Te+jA!d?sDWCxAjl^Y&q`DLbW)IroBo07d|+9biI#vpJQyfmP5C>{d6Haf01ypdSzYbGI7H?7rRn*KXw@ z@XfM#F)A{nONG^dg_s4W=WyVAH18m*BX7c>zbM946=xB$W!2r3K zR&`sBjjSKFbg%tQ8S<{va}y^Ltjf&DPuf17WaFXquB7(bU7|ghq{ndC2Nfgv3e<7f(7k=o zg?r`Deb9zX@;AQIjJ@d79%MPLkNlDi$(-k2iIzlc1BXZu&14@`Qp;L+1Xkx8`_#3y zX76blmYm%O#m4CSZFucq92q+2==@+FBT11IC<0?X^69ZX*>o7BT_rzKRA%@rM5{H2d&h) zOY7}A3f!cPkGv~z6S$0Ia^NO`b4h=?>CgT3z)g*bv=qQi!2YU`Vj%Qye+EX#7FeH$ z(@^NTnLt~Ra=*Hgzh{nZ3g-hYx9|4oNNb44qGy3*86Fx5t)H zKXV0{GR0vEGKJU$-xg$wg&^kt$`rI@2@vywOlf{o%ChoXiIVSKLhwKD1hWZY{I-m=j1Jgm)5pz(B|1_HO2RR71Fi9%TaRFEEHAEfe@+P(8>9|RfZ;O+N8vN(3ZJZycl ztw=3xW*=0^LXk09u&&C+seRD0x=_&GGuW6Gb&vx41G$s?ICS@EiDPv8_C9D)vM)#G z3~VKYscXY~7Y`sFK8lqgKs>r(NWBBZ)1$XCuEzfp@q~E^9ieWd9UvZ$6X#}BUPc^w zH^W}FXjWI8P9OxRF<*v6N%a*@-hmcyt`~ebnX7I&eauI-R&fsYPwlE6pX0?&>d_=qAU`~GkC2=;1Hvzoy5Z6PEiS>GhFyG2g2etslH4?Pei;R6m(B zEgb z!tig~YBGYVhf*iC0?8dMXXkJ>{Mw~65q^z~oK2|!mJ_DrJmV^9aLF4JMU9VUO=Kwz z-kWf0etFL0BBe)4_k7n#cfgFx$ZHjOoJm~Xydj45H@CvQC+x*|+?1oDS+AW8zvK#Y zN82N26&bu6(?m*b-JR$tS?cLQJ^5w#4@PFJC7pt+E~Z8p%cySPN|)BS`=ED|uBldM zXmW92daQbC?XBHNC4AvND8KeCccNKXIC{E#VY(-2OKQQSX;-;B`o`Ji@jHofAN{K> zWp_mL%0yQm`1OWN6kA5kc8GR3PbMf16?io4db=gWe5&#tN%(hH|BF5y{urd1b7Mpd zvlyND#GGQivLna|cly6r}Y78GaD8PHmxhAZq(cY|?&b7}#Y+sMhFN ztCo+3YDL7XZK5{U1+fkp-csx1=WOn5*SH{LiqaP#cdKUc`VJm=M7OBhlTF`!P)B4& z6A3q01GCtQD%lu@eHsn`!}vlTzr1b+;;!i9Xe43J*#eur`C^Y;#ZHFr8Nw*qU-m(B z%y>Pwv%UKun=uF?^x{4!-NvtTA2jJMGRBwK2Q>s^%hCIwzk6+}eoNz?FsfUIuI__| zH4BvVF7JcBQc2+@TX(krFvEk4$q2wqIKwmY0L;9Mq`!CL{Rw8Kxczr}P!- zeSx0@5NKQArxk&B;Rk;DbKgAhQxFY92mItqtn8)}4}JL$fhJ{Bip|66Ewiq?r5HK< zo{8NDMNqOrg3f$~5?{IQXy(1@w{6_IZG|m}<(j$U)~{8e9AGo9Wm7Y1oaJ`IPIgHK zIwfK$-~x~;$(UEY4Fiw3@q@4vLd7TI5N+6XaLEcLvVHTj2hnTER4%Q?cp{`TiqX7h zsn6H0#Aeg?}=T44TmlF7Zoc;V1;E>%IP{pig)a{5yw+ zJQWqOa~e~%v=932LAjLbooG%19x+dS%INCEtWGgpl-AcHj>E9ZN~9kSuOH4ZoKYu) z>{Z7{e%^hA8Qe?3_Rj5<#fB(q*d;(0U{9ypl~5N}hv6a-dx7Zex$tQXO7bFybua2LT?cRN0Vna|E%Fd7= z7`Z(;dEIZ_Hc_eD{K_M=5zR>G(u_ewO5hANJiQ-hUX#+D*_hb%J$E*vmBYV%cOMj- zp4{QG8D!z=W~O|$b~~a272CM5`bgR@HQd?IiJ-RS3EO@ zYiP%T#nS8?}L74 zT{*3-Ah+IBAkwrC>Q&SZPi!zA-!!Zrfe{cq-yOMKeyE@28!l{try9wgE?KQ&ZR2>o zA1KCUavUqnBQ$MAC9(W>0MVcK8=)?@8}B4H`k0%_?Mk)u+wCpNa>RRZ!SNRaIHvzW zkagPuj>%}%O2hqyW1K#Gz(9XGz%j3GcEwr(IHp=RKEXD`vL{yV>bPagj8swbx}SEo zoG<^`?25^y-`G1ZOaDS7PFd)dXZG{M>wZrISLo!aR=`1F?NZeO2ek;$&w+ykenlNP z2;GJEP>n+!IH-WLs{=S_Y=di{`&IA%7F${+CeiANk+e|{3v@#54Y}yOIn2e0QN>rh zk}H@Ei2BvSa=)dQ1<_jTYtPzXvp#l#HT$3*jCp5|8=IC8pGGm32^N`~7&Kqs2MLE> z&GUJ)u(tJfPh8YMd=nfWJ&QA@hcKbcSNLMWM5^&wCs-3Tl}(7)&5hm(wTV{1?^?a- zQT62^@B^9KaM@#v!R;fOE6lIERMnDF8{K-+PI&k1f_A>tFzyB$=lK(BZbKo|4$w@0 zXuhrG@@kpIO#u&JPY}}6LK&T1@9pPiXV2_|q8#jNE(*;GNyqnqGpPLg9Omooh0t=o zXmlQtZ?-mJ>tC_W8{7w-=uWT?G-p&fNYCw#1S|Bm({=Z)N-8Fz-2sA9uoRVf8WjgR@ou2fnFC-<(th+Jj-R9;~JFU+I= z4|D$iBy;|cIUQ`2yyU*i#MO6X2XS2<8!hvRAr{T`c2~EIJVNJ5?q0gLiBv_crQo@l zb8-8i&pj0*K(yzCWTw5uQrfq-&Y~0b#$|}jZ!_oiBqzPpdQFFBmMwNwWMfnp+d0d9 ziLeeTiATPgVwl-{CQ+?^9~6caqh!o+RhBm)#nt#`afCD-~L9E&r3E#X05pNNLY*W&9sTb=wM;Q5o2rd&m!JM6b8^S(N6X z(9(i)-j&}asIcjkX2HR~$s;{2-r{OgtgLekYF()2FWZ#qro5yLC^4q?0s4V8KQZ3y zINzt)rQE0mx~pp{Gsg0Bwb7(}VKE89aRYc6;X+;kCdA93F6)jb(@Xd2v+VSO^vu`E z&%M+|)jHo6-mmdOFEhLuV2st!xuaWs_wC^%b(iYw+zg5mJX&?c%KLtw!D?@Ty)D&s z6XIUx+cxe9!G{#}WSjfewxGSb0yz+;6cu`L>&o1d`abbkf=Vo-_?%CMo8D#>XVJAZ zr0>LLjcwcQk0WY!n3FlR!x)!4rAU{)SY(I4$_=e^of_BZPhMCgJNaekdXXz_CZM8) z|1lo^RX#~HLPFR!Wpvx(am+ith_miMHsL3wGuy!v@3Znl28NZo^&bwV zpPRnX)o<3XWARacOxdtS41a4PJg2saYkJOXFeGBLG;Ub?a@5wdJ@sx*2Il;9I6iRW zVXwJ?e6FmqT{i+=53kQYrf35RZYjH{85h|M&!+So$0R-GpSfG5sm%X4y$^c1%Aal| z#7x-TFTMDMmZ{@e@SL*BQf0E4jDF=W_lE=237YL~ifpfqRbqjHOo;wbC3dCz0uP4^Sekfb=R<3KDf%^>xdq|k-BX9bg z3NpL!okSP0-s(Z~#gIeP#Au48Q(tFvuCi@E!Sbwz!`{{-{gVw@#SVI7B&$3pPh^Os z*g{`pDJggExYLY#T^Cw+b1|KlH17W0#CWqwuZBHqvIFO`u{B~ejM-s@Wh{y9IExEkcIUrYYcE8(~FC-jaIj|BbT%?Nz6dKG+5Y#(%o+Wj?Y%~(-2 z41&JCq0BYJWxB?x`%ZA?R)a4TIM$Hu*QnDCmU*2wc&|79{&|~-L48FVM-H0LA-9*G zMC#frjmC2l5=(l^t+r}y+jI=Zn|~Z0C@~lxRmVQ*stqFlwRpo311-BnqzFKC!+kpwVpYP z4)RsuMNQtdA>}QDKEkOaNMFY38=lAN`xQ&`PF_sCa}940)+^hlm&lk;`1-H1LH)ZO zC!JB4sLFwrDw|-}VHw>tL9sTo(MbyE!K6Gt+nWIaAafD|O3;tk@G2$6aAZDH5(%M|#9iYz`Lj3^Iof2>b}go>c-e0~ z_4e7HH?J@zi$2NM8qZYwZS`NO$&J6%6~NA-c@(0Y-X_4F7dKcM7v{t0?{6g8XxUof zEBjj=R9&xyF4tAo53~pm^6r@$T&=)bp@(AQpcPD?dJ2mj(^ix*IFtUADSOllAzDmr zmPcwQw_d$*HtSC7!%k@@Xgyv!&P~{%(!Q+p2;8EEys(**(2mC1-$u7T4K)~aF8E;m z_8~c?&ojU;k05JgQJ(c+k2{y^mUwQ(@ZZ0B5Qx}fToVtkA9T_rJt7e4gV3~du5U}E zKidA>_~GO2(5B}|J$!N<*i5s)Lum{%Pq|Jg893dGlF`1+zrC{ULc&h`=&8yIc0 zJRzezF-hSKeqpY4-HsVu@bl9$JfGWnE0o-<)sSL|EnM))No`^3(8+JKmqeqQ&IVP_ zXsb+;Yvc9!L3Uq;d2TdTSdV@)nd#~%;^v#$){No=;NRm&xdkVh81AGVwDZQ|YgNsK z>s8F)pM12fawEJ(vw4LUq_1wZ_P?*rbiAS7DkKLyj&{yWd3}$`5f{%M5q)VT(JJR~ z?#;_2#Sm~ zeNyLsS69W!D~;dftM|7zHWv><-fxRnf;&;Kp=DL5QM@c8D{_UF^&o^HJ?8U!+49}4 zHi%v~ih{E-Sn>e}8^w}}(Xx}WFZw0S=B|>ly$hH-uVxDJt$W-x6&0g}3wrfjb)=qX zc>hh$rljhL%AE&iSZBOLM8sf<3en!=C$Gyt0yMMioU(n-R&eZhY3nz@b++@dcXA%0 zqHdL8wV;ud7EMLlj?O}4sdWX;15xF8?m9ALa_?xI#}OGqXgkWQF;vpGxOhV*W87wBtL;G=nvLj}Jn!;0oa85a*+%5H(0}eQ#>p&cCQUXrH7a#up zA0qiAuZ*YL(z1Rjsn#zdp+m-Tv^}T1d;N2pe6+#D-`??FA7|EZ1IIQBoXk#Hv+lnB zo9n4xKbT!@WSw?Yvyl2F8eGH~BwY`4n@JsRL=JZLiQ(F)B6yXzl{p#c$ZQ{yM2&0m z)3}JB`N;C-j$BtXq73ys)ZEmcU+Op5hyb90iL|pbu7-EEfa0ov=@Nd>Z2E2R-!&# z#v0f#Wcej>m1_%jQ09GvwWwCzEiJ57I!;XXy0p`77VMmYl(v-Ib>Sbo+-2R;EOe@z z2*I?j?k-$NLB+LG%lYr%jy3Kmdj(Y)w7tWUYjysos{B|+o>otJ3uckjX4V(~zK)zs z)z6X7=yQz>B0L*mk&q(Jc*9sa@t8-x%6f*=I>F#tuXIZ!^B#K`bCa5Xmg!U;m-Aco zgUXO|%C1!*?_ZmEqVm_ho_uk79CSU^-v@u}r|gsUMCot>!n&ksM^5J-;^1H5@v1M0 zI$mPlQErF|z58fZt&Ny@V+XF6|B53+xE8&l+4ND)sdCRSz@If^B(Ay->ePr#;>w9- zDCyvz&q{BlJe1T4icR^EJkRV~+blyCX1ljB@@q~f>O0!%OBrU?7WFi{ZYViz&yp0N&Vz#ilDdTD7_SDN%^z4Bxf_9=6Bpd&|6Xqw4~Tqb88~I zR7zn9JB<%mvbzdkW$^PiYaCo}TK26fY(6aokKIeJg|4BF?}O0vbMI10GeQVdlynPzW%w-s9L{%CuKn5IKGF7=}vpf(?7_lVMK; z-dD8K70*szVG{>3Qk)s1(bt1{GVLYCD|@MseEvxoEKYO^O_PjoE9qCf?3Of)&y0)y zG5`HOD7UJFrjy&?%*s1giME6$hmUx^SlzoB*R8S_9ssE614o1n#w@f&O7wunq@ZqG zdrL9iY+l=vZF**gFhLz@JQ38$RXp2WUYK7xRPYQwlmJQR~p|DYhAbS4gF5<1>Rjw^SAP1 zuyhPpKG!0GH&^T&x$XDI#y`Z}VWasy|{C^2goDek}0nheLkx;4OGPBg~5qgc#M zq>YIkI~s;0Twa<3*9Je(jm%f#%#z0{MF26-#U)6Mksj}=H0C!{4Qb&nK&m$9x}3F@ zmj|hBxF=g*p{TOR_a+t9Ogd4v~AXY2oY5qycHR57>N*@)Rb=8eJR&m z;!Mp;wGQPj0}SD~5jZ~M`w#9%A9h^!NIQ8U)!-@+ z2L#z7z5DS|Y#PRw&(l(y#Ck%a)m-x}wb+zKc{`^@-;ZN{8XScJgPHt*h<*!iqRM)& zN&}OOSTaiK7ZmzYzq8{g)yaga8xxSaHrAd|aB4`aHl#lO+}aC~eb7npjec#fpSVX( zf9ZF!GK;lTmgZNF#TUj%=cR@9d%^Q8do;%Rvkjs0?x>w@tFQUD*M~U^R{UH|-ck5C zQYJY49AvU;mZ(QMPegPBm43%=oe+`?O`IB}5wDXF!_(uP!_Ot_YqEEuJ@&?V!U>!(EFA%Jc{Sd3r5t#hEkYFj`jKqWJBIQqptsH?egMs5iK)cPB7 zW15D8!~38=5WJAba=TAMAO$gIt7KG5$(HL{BQ>`g zx4CE~&2`Bi`7FPlzv=H@^U^&+--wu&urbkvodCKUYa+I8F_t2LTEF;of_{Kn-pc^A zq+y}aoi@nkdJ6GmltLAIuz&|)X|9dJJ@qJ(*G9{Ls%d16Rf;^tNhyad_G z?`{=%NkZV;kpnOFO7frBvmFIqvPTTBdpxcFGDn{JkF|b~Ai4WP>RP*hW;##MuB6y= z)XM|H6V}x&iip~$l#FI?wlIxc&`^2OsV8>WGxGPuKYCud_biC_?MQDHklf@s=zG+& z3CCDZlnUz+GQu=;IohPYS5a}stG38{EHYVHoL=f)NYRrC2zgkZVZX?Nc$K)n1CqP^ zt6LxZr9q>qdml@A6JZ0jV_U-N2og(laCz`sosE2#P5DvhW|6vmkY!8I(oA}q3q{vs z=natEi#b&sxh{t-Wx0V%ZG7RP!S-^688wl^_vIKjjanE0SBDy}aQ ztU76Vu{q&BklMGTiO+cP?kBvq+%6`@%Vlhv+vV8Srr`Vt3%f1%X>lXULrHz`EaAE# zuh35OX9RKMi?2wmP3$verauDKq%!1;rZ!U`pxdFz#O$1Mht-|Wp8 zb+hrHL{i|Kn$%ZbfvEUk_qAtIR%*Rq49R6}ha%(F!#l2JaN5E>^t+ZUEB8?0Uws~f zrC-O)rrv7g`6WATRPo%b$>xbrhh^vbA=ct{TTv7#rwF<;8ROL`T3pmbM~ddR4%nQ{ zGDvvdE8PTLAxgLK@n2%U{=+8!-~IT1So{BB?f4d=W=fkvThLL_4lmRSxsj2G5*X78#Aaa!f>w`Ds5NV{S2MmOwB$Fk54t`*ISGGE z<||t$6yv57!zeS35Wi*8T6~EtO2eqn^32@IBBKNXA01vBm>x9E>r_~M@uL0(a^=ea z%k%t{#QSHdsn7hWPlCK5M;1_CaO+4r-82Fkp%LdIGw-lWDTBzn^zJi+(`S{X~l;SmXnW{^x;vK+&B`L-7Db2M*j+S?sZe>-(T5EPhzSK8OfT ze5M8nbqiIZ8Skix#2t~gL&D)eTdRIYnYs+lAo{{ zVkI;(2I#S7F6KJ8d_a%2ruy#bcS2J`t}vTmhO-7oIPtP_pR^?PPbWZMG3TU*Lu2Tu zZ;fA!P%^q2y_g>1Bzwc1TtnT8o=&iE#;e{;xz9~PCzrlEsX0N&glwVc+_*vKj$5Gv zYgP`iI2lycC1^2btUu>LNZeWF_~-a^hEw@MUG7L&h9=4 zxAAZS_a>XYY_;nE2(r<&+nqAU@5a3b1lj3z=aN$#R&ew-CRkkvU~D>PF(x7FjK92H0H63fXd^+-ZFdM9LN z7T5M(DSQf}W8IQ)$8E3TE7V#Y`yMoevD7N-*)8l5o`dO~tIc|CXhf`;a7CUEg2Ggd zg$8HT%|M?@$sT))(0)v{kVj}If4ufVJ>yPmgy}_Rej{Eb&X!+S2`R%F!J}(N12~Zh z?)_NE?f!OtsKM2-6r%OS!_kyJcmK+?Zds#SJ{b>;w*4~P66x~>|1wP!o_)1glu_o> z+wp;uzb|}t19r-!O@~O$t1SHt@8z$MuwRXkHoI|y17kf2Dz5vWRDY(>nvt^f{jikt zz)+0HSB-~)p%}yfxj--!qbuJAOt<_w6ceQtFXyVCBN&Q70@E#8sxgp_o(J(FQh(@{ zJXD{nqff+pv>`DxSNxNY6;fIDwG!-nOZm49Iut%(y&iRMeW@MZ!5+8W$({hp|il`FBQ(;$$aH}m>$ z?Zfl9Frx--tokJpN+8ayicIJ4gIbe|fZpOzvru{@SUKM+w99FEu!KT<9{+MK`LkRq zCWRtJr<9b*h%^%FmwYK4eW($P&Obxe+>zMZ+mP!FZr9j~H!ifjixsL$k=ZhVXEGZV`|rJf^vB$XR13dFm|78W;yRp4*|_XZ%#_aHe`j!7)1i(6l2*iXry6s z`j?Qr@l-{6y5@K=x}bN6t?jCsO!I?uvRF}hDnaT_D<(2A2^@B}=5MW<^wguRSw4*9 z%TsuEvIJvrJ`kl>D|1xS#mX0FQv;vbM2%P5qc)UdXR|&}9*?|`+-d|lZE4W1v$|%b zk~OzQG-zP15dZ!UeaxUEL-Qf|q5U7@MHe)UQ{Slqn>F?LbXh+Cx_apBslD`gdh5$Z zXv6aix@_XTU+WVzs?%*twkw#iPQ7hdR&-eWYer+~P+Xja5p2qB)|UCmEzxKL)N`$4}jv)8UdB!aTRHZ< zt{>+ezuxylkd$)+A26F}eJ_#J$6aEW)3#+I4Xw{dYcF2AOS;iq9#yPWy~OBpN3CQ_ zeEKwcYjoS+pL6y5)srq}rlw{ZAst%RG?N~UBQrBSM>O&qJiDtcwEPDeEVP5D7Y9rY zPUEFxQ1(jpr4A@bP55B>?N&|2p{kRS)x&TX0}q4?DkiK$LivM;gz3qv9#6u|lXW~Y zbQnXGw@X5(SPqMZ^X;UNgRDYkdLt}+sn{s*9*B6*Xo1RX@yB()!bU5#*lv~AGsUq! zxo&Zre8Ly*AAhkL1G5g&dl`glX5cCoy{ zKX$~O>pk|4sOaNWf^t=l$UL028lw9-GJGM{tN`3fvYPHC;lRgQ0Wua>OjX1+bm2V2 z;vFv3BFj|cM-0a1fCB3#S5M@64_85W9FLu%)Fx;8iDt_AfxUDRRtUarIBS+M1?+*y z?Q~glCcCHhok*0vf^rGcOSq@n4E>4jUDRvkU-X-xbHOpQlt$6ddK$tW9_+z}46dJ> zwl@3F^`Xh4RbNr5vn2rEG!d55pu;tl?HY2A*Z`Xilhrv}*@-~sLKa4LI?O8YS%hW0 z!ECDR4ZH4oI0BByKDuxNV%O4kaU||h^LwCkAr52mm=B><2`kBbWXJ)H3yj83O%wIQ zxQkc5WFGY?$aGz0vRSIE3;nJ`!<2=~=ECi50gGiDtB4R;H2MWa(kS4v4BWJ^(kK!T zxt0e_hgAsmgSt%I-y6)-cCf~)^S$OCiYpD7dy14AiCfzUWk@#mZL`ZQR3KVL=(9?U&@ULTq5L4o_{!+QOQ%T@DdM)@!^?h1 z`a}x24FT+}8R&46#LO(nT@$X1eG`+l+1~%C*2jic={k`tT{E)o$b8o9iGKmyCGbCg z4!zv8UJ{t?%(jmcn9bgOd{XRBvrV&Yifg-n|I_SSb7$A@g$m535n>QXt1g*nQD9E` zmr0@*o_hpu?gMoW#7Q#D57S%*;1{!q{(MaOy4#LxkH)le_CBkYUec-x#J{o#b zBl)j?=A73Ta@l>Vn1|haVCQnXZ%BOC$b+_#16j_&kooBOj*r*3J420?l~#wgaJim? zJi9hS?9Qiwpw>tF+130g8n|R@FGDJC!xq}MfJ+@os0-P|NtBh|7PT|FvGkxAa(z#! zLN3T37S{eOdYmYaN5_YbEG+|_Z#944 zMYZm6Elop+u1y^c4fj>9L{+mgU#zZ=H2a=@j@Si+!D#wzGo44BUC z)`AQCRF$l);D{bszWm|+APdEV`XJP6ks}j``ZBK%8acBou2mODW>-d&oi43*=Vkc6 zO*2Nv_+;J&qlVd!qUT0JHVcG}`Vnbkc`UbK!kwR%0&^x=$i| zQ)nkwdl3_cLdE~{i~sL@=mchS1EF%q1+%#~hF-NC%;wfC26~M~|23Nn^(@)mEH4wx z<^qGRX9h^XaIS;SJBEnKtB6UNKm6&)S?MHj83M?ajIMC2xUG!9daG@=+bvcQ#|Ry* zG!y!z%4;Y&D9B}nyzsJ#0}SU!7<~mF4CmwHJ>>+$`4^I(%3aj@b2#6T1c4SRx(J5z zG0hsf4sc+mU!vhDe|@npqCS45D4{m-W?Jv~K1fZ?>Zgh&NTQs`@{Sg2t7lkwkMhL# zjch_brX(jdW`w8Sd+j%9)j;AD)H^iF{k9BxBoS+e!1I7UDxv(#NMNLk-uHp;^ZOD@oE>|a5Y&A*WZq%D@6zkm zGmGZJF$pC3+}MOL#&o{(CE_x0`jF`*hyQWlLb&n^RfAUaD@SfH5GHol73hC%5$iVB z-0^j%eZIh#+``8y`tp3v%XAp*mYw=}JHTGD+<4WBw1&)!j~-o`+PY5gGpX3v$O7E+ z5&OBEG~codo!odzX&fn9A-Qlv~n<#)nZvF9q+-O;8Cd zfIL5sO!I4BSl{s8Q^FM=&o61SdZK^SSUcKzd8i|3vwa%59oILTyz%8rT=r-lR*SFY zI>s=sHAB6MXxklOYJOo^^yr0uq~-YT+=SO z+wz9`yxyI5IsANWulx=yvrK#SK|!bOM52DAe7|Ui!@%ysL;=wIQs`#z`quwCK$W0> z$rBUyG~N|M{+(?edL9W zK?1*`(y_J72oYIJW^d-U0^BCRy1gSk^V@;Z6r$}!gku5TQ$oOP+;!go+-8iP;bti8 zKmqvB5wb<&0Uv5d6YQf7_|P?nf)C+up9}ahp*aKK5JYf4S(1PTtgoQues(Vfu;@O0cwYNmAn5&^+` zRowGR&8T9WebOVJQ2x+ zJ3fTW*$0P{qrXX(T=F8Ic;hR(Z2>Cui`w&CdAI>BV|+NPo#@J%yIrSkcp|6mlSdZ{ zp_$lh`tr+ss6(Jn(&!VG3}o$0h5Ir$%ql=x`;O}hHw89mk7#$GfjN}26Wp0fQ~GnTaLnep_p;T^6} zb9ihGal85z6j{pe+PO-D7!-(%v2k8iL-akl$vzu!d*<@|hg~%a!^WbK+k1)E$cqVE zwl4ghl(a@#=gp{!VGTnV$9uZ()_)j>HyD`{hz>KatZ(83G92xxq6B^Dth@4TowE+- ztlel7HENL$|EWS^qR5>Bq|%Wg%Z7tg0@A`U{2-ME9Sc5$yv-J*(m6p|0I39|#TY%; zP%)8ghZ>-j*kegS8azGV@*X!-xgR(Y14wZAxylfoAd^{yhF#}$rrgDA1U0#Y>jX92 z!wuP(fB5eIn;y-C-q8ZJ9o?ju^TFUg2jRL7FHbw=Z1eP zK-Ivv`I7m(oV6hmq6mUC%_)cpo^DRZ>X_Qy(`s*{kQfUQjT0wYeh7!`V%MdPHT9*t z8*1F`9dfNe*@f8!OJ8>#wQJ0n?R%y&nEL=?N3@YKs3P^d#>Yg7E8azjWf8rb`V4`q zmagZ*&tx84wdA*+ZL_QO>z`LGJ>rsq>j4B;Ee&%dwSlXa0xwr4>z4jMKRGK3|NlbZ z*G=O5{S7Txmehj80@Vc`j)A$1{xl>BToiGIyrb>T{o!7S!LQBozZvXK?9th`J2eY_ z*==W(_~@@vaOEM?heR3b2Dm+B6&R;POx{m5JY%dLr%sO{=Z;tyt(25dCxgE(l6BHw#@zE$QmyzE(DfNA) zf%~ns)xI;~bPn-k_`-Z3n&Kw2t<_>xjJIh76dkl4938pEjHPG2pFELsG5MoW;JW3v z?YdXjOjKU@ZQ%`&%pZ7jgTAkS-kROv`Uq$cYR_b%sONwN$y9Cx=du2zL4B63Tn&VP z2EmVj1vDtTg8Fes8=ygXykL}o2BmBrS%VzVAPLyh@!gOE8ib32g^CDhkmQ#{fCeRm zq=Y0kV#bNGxaRy4fCd3y$gdiAhXg!`2Z&r^Rb0CRB6RbmY(;{A2$B2*L};voLmJfg zGVi-|80Fvia;_g}kaF9()$;TeN7CFddPVD2C(Y4s&+wc->ynW;aEnN%eq_=yIWdls zc1K!-$akM(gjt#s_)L~YlsuqSJ9X&3n_R^UwCeX&m(qO+q>h;f!8_utxCpECcy}q8 zYiUw+d@O+K0F9etp#Pi+Xx!VptEBz|jXUrt>_Rn0=1+~gri!=01T^lP{PsY8Se}Yg zxY{585CqrMnkiV5d)*PrHOuH~%X0AFZW-PF#)ENrEbn`3@rW};sZwIPDGf;fvb>np zb4FdaHGP-CnhysY5N))2wZyaE~! ztc-uz_-Mh-%YEblqxj=BlfvtL$qBg2bOZDJkp-LFus)+bX=Q3H*f2*z0dv#vSy*R; zs@q=Eqqb_K)>RX7ZCh|EdL`@lx~bM_#}osVebAs)`Fm}1uoCY=V6A<`RlOazx7Wve z6<+jT{26W?Qw>)L^hQ7ELZd9kN>#E%PuIy0wyoBA#(Ky}R;qbXAPOF_(W-N9=;81T zkK#A3*b`?x%8aUyF)jSS@VtsL{rFVwCeuN+t*utL{PpSxL!@yex5Q<$dWPgO;0=Q{ zrJVW-{Y+U6kx8CX(p%i9z#SQnQ8jn;LZ#I)j_1cqJX#2JGqkee+xuS|_t7F!QkN~o zgBgR$*Vv`kN+14%>;HQmO%>qj=Ue;J%G~chhMvEAYxJEt{Myj5O9kZWsBH@k-}~9X zAZZucOTqU^*%fPRE77)NDrF1_{Tmif3ydmp&f`ayqj;HXM>jbt)Gy&NOusnU+_#yo z71Xt7KxLaVS`y?qv4v~%x+bu(pP`B63b68WzaHCZ2w=d&UKE-SlNN&Dc6H&iZ zc-boF*7VrYqk`kFzjRfEGX{szN`awkQs?ehDRkW~Qgub1d^G?J7*Aa{t^5aXQHABc zgb+ggCse*4snAF*%hhR5MFmy(#so-MG z=Pww4ek{@8@aU_YbrEq%>_OE1Nc#=RSjYy_CbY+=5e`Ro-C7t$klkD)qodGF3OxFR zy;c~cWueAg_}7)u%|^Lf*Z~7m(9UCrOANRE9A&s1<<=$@Kj|*#u^D_GqUcUcfFX?SBeb*>; zq9L?$d*?b#th?1-mB7H*27TSV6fDycKqWXy#3ds5Wc}dIa0RqK3eOzVpoM*ig)JiAz8X6VaEFuFy5 zCHPT+VRDHdz`27W9fO$5q;>qIfY!507Q(+KjLha9O9=OxPImr@xSSm8MY^;_eA{>J z!*bu|S}*swojPPUrCC3xph9&!LEmPcqCTb^ir@tgYAcW8%kb>t;n6=mX_O;HO@9e$5eXj59 zJ%2b{7cssi~*fqav*EDSyD94qNH;2@;!Qq>4}QLeP|@R<|z$P z6xNlJZFJ0c$V+-K54ezO*?tvzN@+zoIcn{*->c`C%0X7tts$aOj2r-iN(`fZe@<6+ z#VYWrz=G(h*XtrgXcJt5=)GdHxl@FS4>3oR6Dqdiw0ta#h5J(wPkS0k+w#Qig&g7C zR#i<80kw{VtplOSlR_&I;g<<((2k8=2qO38OyPX?=$Qip>i|urPGC)*ti2_D2~WQa zmsN=wV~`c>s^*>l^7maJj~y|J_rF@kxFReaJ|K)H{HVHP_0qbzQYn>qgA)7Zjy6fv zyWq6r1A7V87siq`4`2A2oP9*07U3f>{kvMmws$7Pr_D>Q{p?!s0@`3t@lxMPE3Pzz zyXZ%0M|ECM-$wFJ6+bI;CGb`o*dBkFZNGrinD2~ZtkO1l*XaW6kJL_O(LzX3hcMAF zKGrv=f)Uhn&STi#vlsER_i}k}pza&8Xy#Uqy4z#Xs>eESu34G>J^24aU-8(lPN{9p zSgGY+?~bv3Vh|d<*$UE{VvC|ZsxWHfn{U_!xA!4)+JoVsl&w}Exu7S_sy}O?7qa0s z=-=MbqT(=*{TH*^R)DIHT|Tez@*e9`OzheRh6^m;r2)0F=NHj>?t$1}rw>}DU8Yxb zz#+u>GhbSDVc=Qj4+83gb@S_ko7*tO)jeH7rVAFafTZkRDfLN8`sHs6@=eQB%ohO` zfWuN-FDT55V77F ze@t%6^GUG9*CTHkIAPT1oGs=T3X*1-^(kktcq|r0^lxP_r&pT^?fHp7R;v;?V4EXR zjiujSHf|7W+}#2Ct7)S7*nA1X_PI{TSkp!9u_xhmObT)|LiIwdde?{19yOncm`dJ%~egvIAAVS^Y|>xTB0Eg2PLgSsXbuEDdpe$HhSF>rAp ztYa%}A(GuX1Hd>n!l(@{#b1x;?Zz2{+te26L)_=RRVy$qV|pM*_kib3g#uVj8lRi* zOfHHS>vLbi2MfnC(hCO*ooNkEBO)oOcPV! z);`vS>d3jZ2gazvhV!_$_Jt_E{Nlz1&aJ%?r?U^ZwFf%;YC1%lql6LlKsO*yWttbK z8*m@E=`W*ky8+!1YL6*~mpR>l!0cj|i|WjzY`+=Zy_Y(i?a-@HDQ6U0 zc~v%K0AKfG7eV(}T0T6z zG3TLGb4+c_u+`X#XIdambI*HGKzSq8FL=g+9tb)5TtvEF3u!4zBZDnf02vZp}y}PQf-fSrqljW!u}stUPR;$8!-kxqH7SH_f5vb{f`kySLXnDetcaYMol3YT}(O@kgkur3Y@6nyTc zBq+r&`sT4Oxy=Ab>w!Tpe|Zl+nIR&=SW65#GCj(i`m}PZ$GDNO;^N4!+hNQWr&CRJK{-FKLC4Wg<1!Vz{n@2~J4CXW+)8Lod5%Jw zu}cx@qO6^;@Z$D2EB>grrEh#nYDkPh+GCPQM$tA(bY`@E{y7>MePoxyOQsTPAs{DWzq_I?S{9cAuZ%T0$M)M|8y3bckDo;@$=0Qe2wIkH}$m3V#dx z|J=VN5=?KY2B_W@_SAXNs@-xiC>3SlFM>@mO{)l(%}tj9a+xZII1Bg>Vnj2G6`7DWkU*=ADOoJ~< zDc)p86lnQxQzl*J7+GvSy!u$mcQXYu2J$f9Tul6QtU@_0xx;^vFij7|V=iplvLnmm zf?J8~jIT$U3M`x(X}AU?0BrT1-(jsEV1LP!6-RtT?`eymku&mfg>6-ZOMYdn)xh`B zXU19l&u%f_Z6Wji^eO5*S?b{Yc2X5_&lIceOsbFFs77G;XWp12Vw>8=SQ;M#v;LC0 zTO9BZ6H+gcTX1`jw!jJ_E%gGIjXR1AiJDD^gA-T~CYk%V6RiwqTatWS3Gld8Dk4T9 z&^!9P4U6(Dn0fm!Ox!An8>TH;K${z;!@-zF>|vN<2{VI8G$+iZFM*78Ak4@OWv|#Y^cYm zA$ZFAtiATgpj_lWZL~ZNk*!v4+hRnEib~Tn z)S5uRtV}91+LRB%>8A%u62Pnsx1m0cGbZlLHq`Sgr~nQ1ndEvHFe~$d zkzE-I<}}m;J@LS-47a2HEq7Li+fhGR&{3Pr?WpgKPBRbyUAVEv3rr!kc6$;$EhRXE z3J=^WU*(2y2NlkBqvmu!5O@y<6AbGeA?bCQfu9vv8;djI97wtAyz3owfo&UC$*l2b5r02=R2F|k8l>3$3K!3qYpF%?KhT`T-(Q~P1v<)zi`qF#+iNWiAGPF3 zO!`&ig-%-$#&g3b$kz9^sSWhwM@TeVrFt#YyH+zzEXFfMoq$v%e}|1j#dks zaF*tqRh102?WF3^UG%ueEq?*)D>>45BU@>|rk_`^o5TaBBP zBMo+fagXR1a~+(oKqKkyjoZweT;26z+M+=ntU=-%FnhDOyH$p+ zX9Uv-0chg`_zEi%`#5lLtuPT+9;|pbP&dnZw-;Qx?stRPOa-6HnGpXlhShKr#q<@d zdW~iUF;i$%peu1~uQNp(g#2Fb@aKipp5?75S#dXn8Qzk?8aJ9<&!LbCi)LoIvb>IB-TFGKgv^P(}|o#*(nvT>4u zw9Hg*sFl9^mYFtcak0Sy!KnA1M!3*q>lQee3rG!sqE@0DVlEyIxPA=^-x;$wy;0Qv))By#MoR zc!_RF*OMwWf&7l`P`CGO{Vq~vcIx&Hr{}F?Lc+9km4e1EZS2?tdCCO6ICVc<+ES$C zbUWs$?^&Vi0)orVGoW|6Vk61Hrh)B2Y=~DeVPW5rih zh|pVN@<`dELtj7{N`;L5*go2#*#JR-teggsl4K{5To2~Gn((T(xGw?K$=VuH7z%5L zfi`f=Y&JX^+l4(P$lkp@{ak0eX}YE+%^#+Tg3ABdK9A=T_iy0b+R`^arbgHAqIa$l z$$Sok2W5OYH_DSGGg4c|3m4q7yp!c?d8^puLDAQS@}{r!Rd1Q( zUMQf3hg|DdOm zCYtlpn5u&Cil94rOn+{b|KWt$G5Kt%=hiNZ$ME%;qoRZGV7YlDK6B)SSus*TqZ^MG zs161!)gzEC^h)URP(Ky z!+gI=U6D{uOU!r?HXEPro_6d+Gvu=Pe%a5O?)zy)slh`e4~vK6?S{a}nskCj(6&{B z@6$5E`*D;JP4*0_B)DkUrp{u_M=2YfjlbM`;uq9+L|3`n`uXSdr34of>?kRwXk8#R z2bX(3WsI#3{XI1w{lnB8F;XsY-!Ulw9f&~h3)9MYrMkbpOm6X7Y#+E8cwK=0A?}H7 zf6P0Dmtb7fcuTGB*CRgd#}UasLQp`faeMW|XL zK3T44BT9pme`47NC2k2HNfBZfwzkUDzFfIo`kv_3U1^3)p)4J=e3#?=h~UwKepXIh z37{_5;oY+YTQR#AKBJDdSQmAbO6jWpnftM^3m?{eIW3|;9VpwToASD4#hN_)COke> zenmwyUefJm&ppXS-K$Qh7d0l+cGA}c;nS^|x6Y9EYHMX&4P;&$XBkx!O6#y6WhntgFlqTmH5MO+Jq(VQFM#kRms05u z_vqp&5SM;&P$#)2?|5*jWOo0+nrooe9%(=>Y9g@{ZHi5?>nd*2RT`x_OhrJtc7ur- z)w9x#|2ZG}KX}T20~G#?0u+vx>F6q~D(c)HU3I^1H}Jsbb)SVK$|pCF_p;;Y;{06M zCkubbLt_E6H|DA3GWmSw?S?Gq>;0^z77H`@K;70-4ea`*l|kOjfp#>rxr|;b(5#0? z8s0E?N1P~mPaq{!sy}-CK@`SwFJc>&pju&_S+}&ABOCXDxVEE&Ba^3=7J_UDc7&k` zV&O-bog9%^2v%y*(PCi+VgZfx!bfdwiqZu=GGaDMT_$b3@4QeS-I<;G)1yCs{)62N zeCzJlBNgK98`P+9tGdwd8+ryzSETVS^d|Qt7N+CmlsO?eXEH`z1KUTCL@&PTiAe)` z+!};;BoZ_3bP6zt^B5iQ@!9oWWZERrPeY-d?|QF~?MB_k*b-#}0y6s^^YsWN{B`8@ z@~k?D<3_o5T;9idjKmM3Ufp$zqFvr$XBkoKa3R7VdCwPdU^?LF+^6eQzO%KW0x+V7 zyvUF7;>G?2J!6qNJ;4UwEyO7QnfucOz#W)l&CL^2Pd%kaMDnZs_QPeUR8`TUZvUo)E8kNwZJqoR}T0pkWy|PPn z+M6bbqaja3Y|8D>`2`*qO|wG7q`G9qE~Ql3Du0^5hCoy^r@Cv>Zb&p^%|uT%r`nU8 z61tve8oQ`h%{S*ZKT7Sg_O(aq@pRR@puK6(j`T6c;PnSS0bZqdGn>Vxj2^m*-_9Tm z9DLECZ@er}V`vZqTcxnCR#!!o_N8q>Cr3KokH@WCqKUiD4}SRbJtoRl19_D6^+5r#P?Z1`VFq%T2TZhS+SbYq=QyG z?3jya+zNfwYa}p;8AMyr;hRwJglW%Id0)xIm76NY!F1PvT;KKmI`j*(MHYN2TUh*q z3RcT42Gjq)Oxpi=L;Xj5_rJTW|EqRco2eCcVhm~5!#-u)Nxr#LZhddMv&Tn$&d^@A z^@m#JGG2W5fWMX7R3#?CLPyg6m)YyQ?pwi>38;!;CY*h46=C*vu{B(sx?NmOT57PS z;7({d4B!tkCe*Ix=vME*Wj6G>zEB)bUrK%;nK@(ckt1}u{q!PZcG&6(>^BYZb9TTf zrDV!SPCqkW(lD)JSQ?<#7CHH@Y-&h!kOjSJj{4amF;lHu_ zdc=|xlaW=w``+uLl3^vLY*^+@kFsHVfM^%`FM+Yvk=pp+W~gqL+nUU{ibGL1_AG)t z)LkEt8oV)W9&5LX@Ym{ZEOgV{)9cz1=8+eD+y1~kvx4zuUm)0#np+LImYfw%zk*YL z_*gdsf`HpK_s7@tBP0xkjj}6yebxx6T_M5FxJy$rU4G;qbwpcrR1%y)a{|>M zWh6^ZY9LaRl!=eM)+IXkSdcF3*7y5Q^xtN$|9i|{8zQGXehk(3AHzaOKP`>nF}o6% zRrggfkXk{Vw7tfJf@HIj>!M1@F)hahJqH(xSk|e@KP~yiVq$oWdmaiCBo~Z-sG%Wp!1y+bj_Zq0eYdtF&vzJ z_Kr?(4a_4p4kIg2j`M;E>YR0&%}VpB!f zcINAmJH(%)0!%@RL8@XwK&QJ!z&zU>R<|7OS_VvN8Wc@n_T=ZaEpK7Fy735Zgxf_L|dCbQBvbs@DJ z1(FuWk{|^rkc2ogqw`z^(xLK%t3WCr0>DTo$7VZ_oe{takz>@XIV;4~t8iC{v#!3} z6*@FBKshVqs7Bj+I5766W@$4bOBZX0>Zzh9ZL>=!qG8wnTQp9rVzC%z>f-;jw}- zErN9`8U!Ms6^{*Iag)0-<534EmVs*D>hn!@c1zVR8aB57Jv@4LJu-I9Eiz)Te11!R znFVsA18ohbBoxi?>*ST_GxpEMY|Q=_??hvEPB8nbU)~d-ePR}?GcBK`_-sp8Zw{+$ z2Flr+0d+zaBXq-`FGM_G6+ovq9N7nWLFXQNLCt2qad#}(U{~yb90?Ri3(Pi2B5G`~ zR|l9=A-!18b}ValejT^D2Wb^G(`R7$$=Y5MbgVO(;eWB^^G@vG2WUBzjdbjX5O#H7 zYEOs-om(6G#e1yH{|8IyzZGczS18cLue$n`@xDhiH>^baiD@#K1oX$lKm zK_UsjC|ktJN}#dBQ!`bei5gz9xcgBn*UQxo&I8oLy01rcAPln>0515h@Zesd5ioi8 zMeFo;{oOd$s#HfMXtT#;vY;Ym7j5RJ0^Xn=jAP(mkANJ3LI%NuxJio3@A39|OZa(y%xsTyEWjxQI@+|`_oEqa1n)n*Q-sV6{1k`o&Ed7g3@nrEMv(m7O~_to z^m#K`e|OS>*E0{gPO?63z&1E7;b9wTp+a&3&{pg@;c z3|=i4=7W5@58Sk0n&`Ixe{e3Kx!{B1KwiVeUn)9Wpej-bu`*&P>a=lcPJH|C6pdpU z4b!EX43!f(w65~|@ZkL$UBmUv)B3(DA5!}#oA2oy8&MQevED@3YX_(OFxK6kG@;>m zVc+CvaL31~U>-h^a^uAZ zOtGt7hQD@HI^Rm;edF6dGb&Mf!RtI~HLOEK?t_Tv6j|gxyr5n!JE$+gvK9-TTGANPe>fiza`-DA>dADSH#{kll%*yj6 z`wC~X(V9&g`zvN6+OpfdI<3~wiQXHE_ND2n+P9BMFuc~xE~hwqDhNTW4O}6DQ4{kQ z>zaLT7z|HaOEsWIof_bmAGeA<9O$x}EyyVqIW z=e{pFiE2Ps3}3ZKqO{~@hQ(JXC$v49-3+MmOf+BCQw=B&oownc;TM)1ThVW3;AdZ2 zr|4>RH0chnISNALV@DZ?qfRnA$u3R(Vm+#nlEcKVfS0aM{oZ$q2X)J#{c-8)dfIJ8 zj4KOiq%>ZGzD7)0iqSrWKS@#~;7*`ctmN}|XSz+<0V=1hH{sG-0|RQ5v=dho6*45d z{1%>kGl~9b*LW?7Ext6N1D@E@1#hLV?5qtiS>SKUap!qLHeA$*^uA4u*TAMQ6C^QP zzG$E|h)A}CjIp8d-YOtIP|JRQ0`4VKLXi)#%n;m2G3(qY7^KkADU!M%$F>5Lq&Mc7 zIsyjW4G8+E3cM$d74ON1wIl=e8ws@BLo*eo@ysQ+$$&G>cskpPNl;YB0n?QnQ!Vvj;vv6cUYX=C{59Jd#bO+q5Z zXU;b>(5t({6`@}!UFwrWyLLaM&!abkzIGeQPMV`>y!z2FxH+ppF|m^zw#QMjNItaj-xxCC5u0tebI2O z6t6*{=J_pjop8m0HU& zn{kk@M^>)0An5*sCJ-av7kSVIKiFaet-fn10J~}S>yZP@f8u|wX#L}k4h`*V7(Sv} zR@vrT&hK!YC$Q+xx03v34R|uUOpbb+ES(q9)mP^CO%3pHy0zJZ zTs-%-*U3ibnq2oG6F_S~p8z8QypERMo_ErE;V&JE%vZ7<9<2N1&`z+wLX97GLVKm^ z$;Y@Z0bPV$FD$$yUl|WiMs(!f#xb$KDqPgDTPNrCru!lvp%N^JQ*EX5{2`^o8c|-S zAAPYQTk1oktDk7gQR3{@lQrC^77yOFd*7z1;rSeO_Z5c)@>;FgNWM9c)WC1ktTk-U z)`9WDIswE_j(yPzy0{RtcA0?QI1qMulvKzd&f#^C;}Q3OyU3y_TtbPW4P;CP3&{2^W0we_l?Gs{sQr;_e$ zlP~gB1=JiDksM4TqoV~X;vH3HW>tfb?<)s3VHZO8-(f8Sbz;I7OS>Nyq(s3KzrQt{ z1*uKOq;|Ep)+;3%P0)(h6q6&Dy*!J~*>&;F&zR~^<_>)1))5KQ??g_htUHl4+CBm! zO{jERSAuU5ss)Ke@y8S=58$i~Iw{DbeX@@En&bXO|3+U$U!D9I)~0S*6*^TM*2=wK z5A!9JdukuvuMY@O2YDkn_v^XAxP0J#-HtMXv~!s8jwXQjn zYM8DnYmZb_H#z!Dszd!6^W$H*j)M+e?3g&2f)H-<0c+n&N&~PS^s@4jEhT zS4%O9Z$2}@MP3zkYu(Y)vV+)J>}?hnPIKK9=ld%~X)^}DaWRfwfD8+7ya;qz5I>m| zZbY(S@RhB$E;RCb0xYX=KQO#)9XGzi4pag9Gc=x5X(|$1dP!5Tx+MndvDjJ;x0~Dc zF-~eHCeB+|mp-SX2pUwhCGN(kYP+=7$t$61|K*?tG?4%e+H0$t#eA@xWMfDh z-)FU9?gQWQJu$9hl98lH;DPd?K%X@Ycpwz?1j!2u1NyU<8XDP+oH%KsZdh)d#*nd+ zi-&P~RE?DSg__I|fZm-<_8F@$j(X_mkJGIU}Q@hU|tZdTP zp3lX-^$RbmGUh5)>R!$P67%j3J~nUtlLY zdWs~JEl@JQflw##KiP100; zm%4jXLa1s^qJXCPm~GCD_k$zy`gwINQB)h*?V4YSxcH&F$^TYIx&&}i``>q@lmDHL zbney?GP7C|L$yj}{B_LAzW4b8X6B(9mC1^0N)osQVVI$P+{g0RCzAoSaxEah4!vPfF?BO~KU*H$ zWIRH`7RCxR;9yN!U?O50Xtsaoz0jnm4|rzadk7@pneo_6JoL;Q9C~Jaz34q~x(HP( zwq_H`^eSZsmw@4WPojBr5*PsN0<_*-0Nfwf%>EAGkL3dRKae_TFWN7982kpT>$l(d zD-4Hj88_M|f1DVzlTH|XV}Pzyez?%fxH<(TE>gV^z~G|sNR>vX$x17?gG$ybqeUN zMn1bExoKaD2MDg79kI;_M`0T!biT6}B1;r29(rl%Mc8`wjMI_s6J?;#bF?30gCwEOyD5`-KJ(ricVWVwbsrBcv8h*=W;gT_HWz5*$yy zy-gmkRGdw1I7y(*(LaQ+Z4dmGgPUAng5Z7X7Z#kH3oBHA-5cWbBg`q^@Iza{UT89iKl z1zV|Qb?dilV{AJrxUMzb1M*8+cOk&jt64l%B7DX){amrV4d8oIssWRe%N z@c2XsT#kWDf7w~!6h#pii;7@kbbgtygS`E?9Tj)yo~)o_V*Z-FG1Nvj2q&j*pN!tS z4nHNRr@i5-s{SZVa3PEL&7-ad3=gd4;@It)I%x-VXUAW?}7b3dinl3t|dI|Tp zj~mri_6!T#PM~6MjlsO$occ}$(Hfob!l~{BMH^iA3Y804UThJDI)_XTzNc&#ql~4g zE1{t(VZW4rml_I8+bld5^HW)INic&uYRPClqZ;x=x|I^dGhOQ(_YH=R`KVb(xxRf8 z)PdA@S-ok8rz!aR4hBS!mrVVOb?>MSK>5G)OAHqK`?HlZA=w`+9hb5kMy%81<(L0T z=zlp?D8fM2b|QKgl=iAn!3$yUmOd1v;r*>3;fXH{%cbP9i$#~?Oc(hdB>g;KoNFF_ zRYII$EoO;hJpL2%J9V2JNMN*? zzD>SO^L9H_rQELRso5g-KD%$O+60p13eH>l73ntUsEHM52{|1qW1H{i7sSlm%8owY zpZ7krPFYQ@ewA!l;xt?CITjW9vEtvDPLDVL8}x7Bf1m#Cv`(wCY4eK8bxnc3q?iM- z>*EhPHC|-Ip1pR|wf%~NP|{-^&$9Kg-Y;D>fur67{<*E^25y^a+<-3?CmNg|WJY&s zWxef9-Ylz^8*axrj#5g4J7pc%`O1|;$ZABc#-Snva^olL(W$s#j}%8Mz!35PP(ro* z+@s}z#l70l3Z=RXV2#wSG?=0%TEskUeZVM0O+4{X5rXgX`C?`aiI1QRBbLX#2+G2{ z=Z6V?(8087pOTs4<|iPQ(D92d}vUmfLFncaAseU95w`GL1#Ivr_nw zjtypOoH-F->H(J&&6#?j>qy3Nryi1#(#2pt?$pDhQuI#Q9nRFl{jeOJBqT8P0GNle z&yqmp2SVLw6I)Fr!)Dd&DC{F^zM(TKRozB_psBs7aZef8Tiy^cGM3W7LozuGq_`dZUhE*sB8XXY7$**xg9768CiAnnaXN}4)X|4=$|n@IAqzQiDtLQGp8?|hd6TL3$MZ%1 z@+;&&^b3FsBe&E1NS1UB({aqY5*Gn*VRYOxU5}mY0Ios4&-W|JNvLqxH_KCXpP6w6 z#ggJhis^FI5gv;))d{qi5XVr2HLob=8j3P@wxJwnJjYPPh4$(Lh9V^Z+8g8wKzoCz z0O(iCnX<U2>}e$d`m@t)zA=`Y^}zxj2HX~n_9d>n$t zq-*(?u@h&^;Pit{k#uEqHd_e$bS$QHbr*c22x|lj)CB>&L6MtrHt%45l>ipzMK}p0 z-qDm3NRSByQd;=7r6n=2h#CC}#$UfTISkc3hr7EX{!SD=S)x5w!ZK;q-MhPspCSS2 z<~2Z@WtnxcKbE7-dSCYxPMsQ0c)1@u95lx0ELO>r1~6V zt{;%a=91o=&<6%GGp%i8-Js17gOql#q0A>?mBKvLG`x;Kz*q6?a?{mYxkM`NZ z8>$ewVZRZ*a^P#u2|MTbn)I$=!(ins>(uKl>NtRHmmmhpag4pl_=iKk8+&~?#@_In zr&@NSbQ*zU?A_rSd!1mn7Fo9&wr>)*#PyIvwj9*9mxdJfchq)S5rEoObkwQ~Zu(+b z!$EEPb5Yx#aJ{GpBeDk{W`0`17I-MpVIQNQO?F0Zpp-u4gVbvFD;wsq#wl<6FqmM5 z$Xu~~^H20GWe03JLsx1Yyy98;;;4o#|f6g>;3s zWmg!pbUtY>s^ErV}W4dCi|%@cM9Eg2M3EZ@fWyTZ{t+W=@AHX8~8K-*sJzk{}! zT+sH<0BBpt8-bSvijLOJPop?z|4m)p?;Fk?p8dmDSvMNHXxy`Z@D8r(F>v-j)o8nF z?{T|letAa|EheZE$l;$OMRVmeO`x;BnaY(#5` zAxK)Mc4N-q93*slgH7B5c_Q2%D@wkJm`zl#V!9@%Z!_V)Z2bnjI{tJa)~rRNxvsLU z!q+4;B=%lKFZF$IOy5&Tz8fvo!ff??MXH{Nkp}C7mf-d>k48X-{|8xkqmjw&ybvYf zr5dS6pyl3X*4p!66SLD~tzJ=>b@&UbDVLg3=IG=^9VEQ!$#I7Ndhb-BOTDIDjGZs# z?X?&FGf!OQCjkG?sg3PdfXugN4?mL~oWvieS6|p8zjVWf#IE!y8$}ip%yRP{?UTPu zqrBuNr=50D;AD)|7qiO5r1u^uZQ(ZupZ@Z%UVp$TdzFSNT*$~Dvn}ljS#+8Cc&W|G z`EFk{kEFfw{_5{JmdvH8-mcL6mbq{1K@Mi#B)I# z96krs!iWpnkZGLURbY$mxMz*lcZ9BgH_``=0WOby~nVyCO-T7pK>xi6r>%052ujm)Rzm z{&muH&K|yV$I$`!S-E|Wo{JGtr2a=k! z1=xT8G&{*i7G8D5?J#?|WJgy?wfDc19jySeqw~t5X{9RrR45xDJEFN{$1GP4it9Fw z&ElZAR+Mcx8wL+-7?+KX-%(sip1t^X8VAL-!T>f*Hh04a0{Y)HHBCSjp+@LcTqL%} zA~(h7S7pvEq>$HXph7>ayY=0+Rdokn=f&>gdwcKS`PFlpfq+~nv4uKiMg8EWS}$9!e4y$Se#O-Br(IN*w^Q9rJdYB(<`cKILz z!Ub3SGbPm&fLFC{eapeC5>{0*Te*1E`u3*<-wku|s{f25sr#?|zf-H2aC)4#*^Bsn z?67JpWLAZh68?}v@9I?Dv&k2`*Gu=(uwDN=i@EYAwW*AEclQTo4oqUUKrL^!>SxT+ zoiswJ-VD1S$#z|Dk7qlTxZ~6LMep-4EpFdWMVBX5!r0k%F+^XFm_7J;#UyJA45+m3%Xe{Qb>D3;9g92QkB6$J_2u~!c12y+& zrjB6_Zk=EFdc;7BhaOMZ)S2q=1YMI^oOOr@6;Ygu7c1<`eY5xM_)M70qRQtM*uKhh zZQWHx9H+1!3XoUeSO)GIid)#f!g=6fVSj@_YXu(2DeUL(a=`GIrJgEKdjqGa(#&0x(&yNc%!=Kr+gs~++e_z1HT_ow?WHTmG^TD0R|2T zqq@T14FSOA|C!lbzz`5&a3R7YKN6Cx6Sd8dv1*8$0_rw)Em5>*9LtfA9H}9o@AaYO zD0hUI@3+pe9BEuj(*`U@)x9OAsPmXu`%<7T8h0L6uSkZb@+R?7{R^xOGwVu?@$-ByL^w5hq#s2-HRU z9biCRbS+-d-+!uJEq>UNWV53Y6hoNY+m45ZUeBTl2CXs~@dqGY}l8{vSDQ9D%`4s^VMdwP7 z^~}7rz&0tX6YBtQxO6ZE;Eg{VnEAxzjbrs)05<+LKgn$NnkfE|H*Pn+g7#aT;PA!= zOHPm5l+PkZ1W}#vD=Yu7xxSTN&b!J9yGpa{IVWt={!yajVc5^{1v>6&&WB;YYFAt+ z76ig>s!9gJ&WT~HN&pOEU37j6e1n~^`2m0l8|@kJ2ypgk zreML|!ARSFNTreoZc72yp#{_9Nl}^y=6oYDF12AHsZe@ri1p(@oNt){#H)Izro~n^ z|EZR4?rn%CAp!fKxXb&I8o(OEu9)od_U`XmN`$lE?^-hA-(B5fb(k&5Dvt99Rz)Y_ zpJsV&{A`YqvCetN2fM9bPM3~~4hL9c_`Nmybtsb`@Wrn3h!Spi19Ztx0063PZ)9o( zV{XVjN5Dp!z5~t?hzAGzKOT@{y#njwOkjRHph_z$!1F#6=vq=WW4OERb#T#gv#AIl zb1j)I&2HW2Ych@SYun3-h(%>4PPsLhKJ{>XZ~Mz~9dYd{84%vcU7EFvOeVywwLgCm zF!mfQy;Y*<6n*^qTFXPg?@Y?ufF-rZTY>-rFi9L*{@P!@D9xn@%Au)ErMmu(ghQMoy&LG zW`R03UA4nj6H6;3M23$Xbyc4Lw93USIVDe*O=YJSW04KZcajCZSuS~h4>^OFD+}i* zQvDk~cV1eSawe-Y^14E|vpVKTPCdG*)+2;LXF!@M9FIIsv>1Rq`(o|?MbBtpwvhb` zBW=#PD>*+W{92rRA3G|j-*jc&W@L3&Kt4+63Rz&qVZdj_Of^(8#W>`hi1sb?aNTLT z`A}08PKAlv9_(!iyw;CT))Fm$%DgC9y`N%7FmN0^ey80s$SKQT@HB#=cvB$HX6tC{xU1MY%)wGN73A(J z`B!XkI^co6$D9g0WUiyEmblDy(}Z0hJ?1jkb)lE1M&ttN_pBwC0`7Q{z|C50-ke(003EYNwxaft9Y9`g`(+mC;H27oeU7FeNt;N;GF(LtDAzA zVa{%2d9H+m4V@SQAj`G1V${+3N)}U5ktX3{r^>c3oRit`tLiGbI=r6R3>=_?lR zl!mW_J%TJ*O{ldSuK4LCo`3(I9*8o7o7#10=DZ0JAMywZ=>E`}-JfP?uv%>kI%rt7 zZYTI7T^%qN{Ror{Ij1hcjXE6%i=gpRd^YYul!<&4&<`Q+9Wr?1b1JQM4tKhupP+M` zA=D74iSB)ymm@TUfc)%;G?OmssSsGjeVlFb6lMogPus|M+>N*K>^SY7uW5ZVOKtj&xe|RmhRJ=IcKl<9SyECaNNa5Mu#U%tRZNMemZFDQ+;gsjVyrgv)ZLGQc z0u|n)mExRJ4pF^jXddnsCD~s*WmyKJt~>!^&;FvC-L<*)}n$b%lzW^tz!*-^=3sy#;Ma{ zWi40dV*q8Ref+#EApM(I5!cIU80mzAJ8Q0_#-`xo`MNl={ zIg^8k=iuI3GPsEN)Yj=!p^{ugd@q1|zXc%TUwHwDc+^t%zpwUlM6d?xBWBim+2XGg zwRT$11u%=aO7^wvem&+YIlpC3MAB`=??;JCwGYD16`TUPYaCSC;#1Ob*F+XS0vz^^ zrX5iZhn-5#;5b9=0B5NAp)(ZVu*c@Nj1S~-IP6p=;0%rCIztsWcwH|NbDx9PC3;tw zaPhh!KLS+VL%gn*aDNu!DT#yE9S5kq0A5#B6QXbvi;0yA2n9}&fyhkoAS!|hp4aJ~ zw7S+Wft#Ix-5ZblSoZYEL-)cKsakq98&u{Spt%Ncg6%!g=jl&K zFl#`AHXMNFn(3O^3y-AlEPH*7?M>_Vgnbz8pv4>8%L*7h8sW$RG^|kuoJ$};4gfo* z+|lQK=mrQwX74|621JH4cR;a)@uJu0SE?f^CSlom?~=C}Xncd~`|+`jXiZwNmN0XS zwT^R>&@m3`aFbAp{54-uGGhO7rp80&Y`#WlGXO0ukG#u)mLeyiTs>(S&Hw2s-4jJz zXz4%Glg8Zs9Q6;s9KLaf87*F1GHvbco07}=`E~0W?ZR!NQFM(k<=+pl$qWT7@7Fweo{-+&Q}pgv_T}D>wB2DEZFK||_0DlWbTX`d`oUtM zrEw`9>T8gh0KFLvOW9>30z&6nbbl?H$ghGfn>QpAWFx^l@QWg)lM_~tG%G>(rnTeb zQ?AyOq#RcyP0@C9w1OengxZ6Wc^xxbkEd_E+Zr2qXNs7ovdTQlx{6+BA}oLVho9sB z<1h7QD#X+V@Dt&#FWeF4RB$>#g6sd$eG(~kq-_) z6S0PvdgVBrz^rK2#9rKWSk(~2g)a-mMc3L$eV+SX+r(tf2;GvB@uU$;&XSsodm;dr} ze^c-X2Ebiii>ja%ODP2%MNENP4TicJ&x+c-o;*vR3A*eS3g>acynw22{f6P=(RTGAFF^XEOmvCM7NR^e1vG7&&<*tBw*<sAP3WsKuB@@u)wu3dx;;QMd|Q`y!yB0WVtk^A*e$#zvBlz~ z`A`s1wU!6eRr;;bV z83Auyo|F+Ft^b3)^NwmNU;8~XV;95*(ot#BrFS^WAO-{kgwPQJQbO;Y89`7=qCr5q z3{40jgx&%p2%!j(07>X#C_?C+1n_QfX5Mqodf&U&x%a+jt-J2cAMW6Chj$a!x|3(NsLFs4o-iy?8yRd8u`og`Q&mV7*QmtB=;c$7GXg zi5`TC0QSNGWm)0j9EZXA7=@<=_ovXz=^qGqNtkL_k}iM!!3JKeVIaJ06x!6HP716f zd&ma>uq_-kps)2H=Y$k77Vk5bWR}rp!}d15OVCEq9ww$vwu7mu{{u+)1~XCj8w0jy ze+8(zK;P0JF@IxtpwZXzjp4HM=}*AnQA-yuK&!jUerM#bh4lAdJgyn5Bu6Zi4yJoA zwkSh;PB(ptw9!a068TF)q1b1M1n8 zoSrmwGOIu>`OPMA=Iu&4qkacTdQ#xDz>Y#de}`~wdRec69t*!{!2M04_gdwv=DVt8 zvF8rC1(CM<;L<3g#2i zc&_?<9Qf|7^y!F$<)`EtYjxuQP&wmd0{|^%UKN}>dNd*bFyi(51yP>W- z6SkmYnVLMct4^9{r;a~cYsSCcSlRUiq92!LL+ip~qvm!?!*b3xcWqzMrtLU|Mrw}i zs3o3I!AnMwb|dhr`>8`~dqsO88Y!EAKfY>sj_vR&X?>Tk9WamgjI>{o^s3=O7g5*G zR@v}sU^WhH&K$emzhB#Fxdgc0vxhX}P_vjX+X5r@)|!8<&OcY_f8SAdrNk9Eibe)g54Vp8&aNA+8jU6|1O8!3Ud4=~aNgJYC*pNPnJG9-q zwuV;F?TEe&owas!-vpq87ARKRJ+#T84&bWDJvvs8G2-FR5(UBw8>(JGcvC0(@;BZa z^}dkGRL=}nSU8xOGIhwXd#2airY_^d6qdv5yM2p)2%6i-YAV_ev9%3^m|!Zo>UF%(@IX z0eAFST)s59n2EL{NPIXS=$j>0fdN}PaVc-!J9xiL`f(}Qs&FP_WDD(Ni;4^3$bmK< zz?8HDl@FkslM2)up@YZ6i{amiC2O zep*6tqJHBoSVbS?#M9SaHw0ZyC}0mZl^Zc#^pdtw?ZA3WX*4wh!Jd0lW6;47-+12o zWy-3-?n**(P_-yq2&-yDQBMJO1?6$xD>wk5i5#Y)g38w3&ETfr9(<&=%!Xx*4tZI8 zV<>Rp-yrS7`&7b8mfQ|#69&E%VqxrXv3=$tx0Uabe<(+`g=+f!irxG}X=(Y(^6DHM zY6(B9YO=n1WoVelS)Q1X_;_5t6ha|a>j{>=sc=Q;rOUb@^hOmxvW5mHR&hMy$vVl- zCa8OfuA9Y1XeJ%M1x;?(Y7ruw4N64WaCya?Th)xYq{(Pz$7MJLV}rmHH|Gg?KoaYq zQDJ2m6RZ*rQa00YKGa1G?Ehx`M$@+2)R^HrPn-P4&O0 z6U9q{4Tc($=qspi43D=B4I`>Fl1F&==->mDZtzcEi}s3vXKoSQc0v7CS#Fy6-~L=c z|8R7(HK$8OP?`2r!tWR$q7ZB5*7MBPwp z#r*VWf^Lkr1!;r~U2Z=hEopNuPjccxjcr5ZM;ivXy%QIw1P-|OZsWM};anB)QPSvI zI}gc(AJZ1W_62{|)lCFiBkL|#SSx-y?2~D1@I>%0KF;EuJTf)XX`1LgIe23tMwBn<}~jKsDj@g|N0HEZE-3ek^_ybOz48W;Lp-KbC?9NC4PL2K!rzU$%M1G`4MykOtOsg=l z9HTQUVXoKyKxd)=bY{g?c_UOL;sD5-4LCkIMrU?hJ7V+^Q^ zzcYGR9+ro~SdSS!K;BF^17P&X=!otJYtaEl50W=Y*RX7tpTf9l%)&gcQynPE4-voo2$voq7* zgqOd73fOn)ryl`ji$rQ}tsDboB|p!;2~NED9Vk1rVJjS8cm$LcTB-5^fU?u32CBtn z1IAVT6FJ}p;-VbqG10QB2x1%30uU`V=ZgPRqGhe7NzIVwtvl5Ow=H_&cM`TWsyx>T zAYm7-TK%UaY=f9E=07gCA9h4p>XC^E>_b$=9w$qQk6`UjywkEybG`Z+p6skWJrUox z@GmtxLuML(2Ha5nTfgSPTp>FP{(-d_dLF6Z)76!ui^2l{5%T~3rDej}SPaYIvp7g$` zIL?g)_sA0FG7(+b=5s0LjwH10SU7X$Ul-DU`E585WkS8hr^n1$3n)yaj%JL`r0g?J!jiN9-r)H>s!gbriI-iGF>4~mKl8NiR zr!m@gHaaPvH^7Kv`Z<4unLwQLp{-(moBA7oqBXeW`E|RYccLT`>{LWH8ljf%=H?Jv z(3RW`zPoPU$y+dkt82Q9IlR%BxmYUlR2Dz!mKDU%AAbb&MFJ8?E>A+W^i67lt}e2)G6t9axaQnxg{}M^=wqgZqb(9tCgKzPko1I-ymSrjA^L zy?c0q=3>ktsruN+vm4F79L>}Jbx{Jj@vzP6LD~jIEF*bm-eY32HFQ;dq`zV02F(^g zAPn!ew6NC;|wT#e0F@0soXsB2h9&sF6=BUZ2gUA z$el!{0oM+7wfIK_*RFWMV*cU0dtLnFm3~QDS62-fy}t}UDmHy)G3ZQ0>m-Y;4i@R> zOA(PqDCe~ptn1Je?zl+C?&|IKm`m+EY*R6c6u5qth+1R7%4lW8LB=&tYm8iDs(7zJ zM!?c=l5nYcrM+9_QFTnVPp@qK9DjvVO7KjIxj&V(!>c*>50~oAo}qy)kNiVAHIl6M z`jIV)c_twCFappI=&vXLP7^+K_58*#e@LRi<3!&*(S7(2C;#<>f@YCgM~%PhaFUZ^ zbH#$L=yY;}5dnO`O5VHgtla+O!}VkkQq)+m_-0Z{cj<`yFbBZ(NFI?Ov^+-jWxZU9 zn5zoS#^@3{24dXB?6P;X08eIwurejuHl48N+sY;n?A6$GS%cfbzNWXnP9WHC5ErYx zG{PD=3zrWw@-hDClG*fN0<5;K{ifvy(l#5J0$SP;`T8P$;o( zF59XbvsxzNb=jxdWmv(mz930Z3@vPeFi=3+aU6m^hRKX^7YVgpGcFo<@>KoKgaXE3L!LlYBFrNgiT|vwv$>f7y1GtBAkjoV`{w zz3%jLVNOR+wt@&SjL%`*@k8PWmn>mNv^;Su*AfbfZ@V?ie!AoTtjTl_Opkni=i>JWPb1dV{7`F0O#k@8y;I-0wexOk3|G--}FxX5twzv-- z)C%o29lG{EXI_ZhWAD@Gu_bbaKUPoq#*jUBZ6ANY-D%<|Ena>!IseH=U~=H6Y7gP< z4-HGf$zfQdWjf41ulTIlpuKR)$MU4G!yi-~ngY$F$P%ybuyu&>o4!}65$EgxgI_lG zcIWy?!^)s&2a2#~ztlo2-62ogij zml`C3oFXHp{c;v4m9JK0hj%pK#%P{7IEn7l2{o=27~jq)G!Z2ZI}*_;T&3Rcyb-zq zVkU60P3|Wt69nb4h-XS-KRVOpe*e4PoN5SwTg;DcXEopV&`F&=qioGH>7Z}Jo$v&_ z(02dbN4LwK&R3+lgw~AXBW&#p#Rmft#6}X*@0q`hJe61D+Mzc3WsL9DDoo2M)1^fK z-S4bm`M`sKHb(Oew&@q?CD-rHP&IfB?1U)L8G=|LEOGyHOd(N4dA~49NwZ27ow?TR zixc&-CQ;-$m+#~cIo{if4_BNRfu_)?u6Md_5%=in054q#!c#HDn7XxVmOfFaGs<`) z=iCdnhSB4W{BXM&{(4fyVOjjyx(#V2M3qGW@YQVT+uu>koa_odH04ofkAQ{QGPioq zh2gO?GQjV1*&3z^_0zeEGS+ROQ@dHGTD0eb)NOE!DqLu~4L*bSS`iHw|Ne6ld8aAc zS-$j}WV|1Rp%dMrz}xaz^$@ zIWk(q|BQnFyOIAt|MlE%NCR{-YRqBQFZ^em+$?fqxwYUbAs3uwL#9PQf8Nq$KjVd<#65ygSfhnpAb{t)!b2|J(72Q)O4|E?@2Ue&qaQK&6W}Ktx2De+V3niVaQj$pDqAJ<3 zD$OK@JVKJ|Jv^|=+7_i(%reAo#H+4ltEY(%7EDT!qF5xDN@u z`^?QZRZOb8eUu0tC}!5jI#hZDLEB)yx-sk~R-N)6{;~ak{q%z?EN!M@H^cyX1i+Teg{VYd7+mdJVv0l^GMYl|iKL-`U@?>maT(hi* zN%_?jOyxjxyF`6xW4L$$Y8DSLKPfd7^B#dlGZGwa^UAGiZxUl6_5@Rc9gx7hC$!VO zT*~GJ$>Z5G^)fwvXZwHp<*!aKoO(%n;TGACewe3e|^f|HY{2pnnnzsc5%)FRZv;mdBx+MCL6S&2SYZ!bq z1n{KT=TJJAYxBl#+CGnD>}m}k+0kI;vHF{!+9htl$jeoIy9GwiEVA1Nqvj89UTBHI z9wx?Zkhk{K60fb62k*CNd_nf0JLWn{3$<##H`g3YsK^ztSZMH-OtnW^M`1izcO6DV z0Gqiyb3UfKCTCSy-JUu(ciX=bnUPH|*=mt0;{z<#FC_c)78V}daB+8~YbH`JgZp?u zT|1#Aifx%`AQrRRc{pF#0Szch{)n{7R-Y#GSQTPmZS#=3GT@~%b68~xw+{vFerml(0CzDXxFTI1srtZ3xXba+f z^U06;V&r5}c~S~6#>SB^$=wAYb?Ha*uH(-gI$)Pc=u7G1ipS%@`fhW#v94*0&$#|& z7DP$P0wQ%E(O&4?r1;F!I43C%c?VRT_+HFh-SM7IX-SZp_F18|GRUmW>=H0*hVm>7NnwX zHx&H}q-7At3x@|5rL|jHLu|F7_VDXSqMQ2`Y(FE*T{j~<5rmq_F{k1?eaGxlOg6d; zLcPs%Pu->%lVbnrQab90-G$x=)OgfEr+efg*WT|tmV4G?IeVwy0Pp;ZMyFj`(z5_Z z+;?0GG`K|(<^}5a}gBd)FGjA!pq!gIPj{AoGcllnjv zdD{w|cM-9tY9k3$j|p-P3(YgP5D+hQI3Lf?t6ZSSL>}W3j=u)D7P(w; zhwv&vTXj|7?lKbNNeP2)2^pJ&c0$eN$$n4o)oQ16s!7qq1Ai}6*QSc?%|h%lhknBbDDwB*Dz61{Z+cmQ#aor5ZVSY|Zw%~nENCYit}<}3v1ULi z>pGmL<_5~g$cE#Yl;ayWT=a-&T!n{O0Y`kosGRwTM|P!FP`v-<&V>6(vt^sl zD#24J9W0nOU5Q47#k`<*kK}{=Je(a2t z9SV|AEQ~ZYh9V+`USV=;S@{ByUS20V%QWuYOZ?ux;i9KBD zwoW@DOZEuBt#9IHIB^W%wte4wdxPWpcK|nXw%Pca-VuP?VA6RC0C0DfW(RAj{Ofg$ zJ^%Y!wvmW*&zxPAmF}!z+F|cq@L1uZ~_pE=+Q;ZLcJ=3pb zp40P927X%~fHwdcGCaBwAS;vL0!LTwkmPS%pvOg|Si ztUiq8Ol{%ZT@-|gA`YewY1@Des>?y$*jG{Q>G6Txl^{GVlDU~~dG!!$yOmnKr5s_A zzvGu39z8seyQk6R#E2M(U2?zTK>qu!!9UN9=!xTf!3L{Ie#2ga!!{(glad})SABiv?`$z?N1|&xbLVS zFV940x!XbS=p=J^x8;CBG!ujErc)~_W4u6uK3c}5Mr{ecc4}W~JcLI%k@!6xAG7&T zlOn>rXgkb%6PcxdCt!(=(2F&z$Pm4x*YOTwsVW3hUcU8afj*PkD}nH}eOr)%_q647;dtb8H49SGJW%|=mHB7_JV z`|Dd+xKkUX{WIe6;)R{9iS_Lry9IhS?s}}&)CN#0%RF)&(-HoMt!Ctrt>(ygTTKj$ z?gKaX?|skq)Q%5J8qr_M6^Y`qqnef^V&h*Eygwi+z(FsZT^@0xk}^l$wKrOaGv zyN2z~5uLKRqDNj@GnyU0)s|m;>Knr^KC1i};x@^kfnXLdMxlbwWkxKCZcfKucmEEnzXj@*aSqzC(dp_8nD^)p*ef_SR25lfk;9` z>STP6w-5PkZIE>GkK*kjQ$5d&wXSX$t9sA9Shm&$)4j*e$wJM5DEiqMf8l{*#uRtU zYA3A-jx8iOmTNZq!-iAz!!}m-F0nL94BJeKwX*-R3dEz}!u0m`0bOq1u%J&d>nQAg zq$@C;gw7u4%39e(-(O%q%9TZXYsUb&GDQjrt>ecQH}=b)9pJ>yx6@|lTQ*|%OPZ4A z22O9_fhT5KWZvG7EeA%CH-kxg5E>3HE(^`${au|Kfw8P>#iB;3p8`r2;qdi~^7m7& zyhQISIBU4fws+MY)6tPmsDsU`+uXHxT5VlBi?Y=t28@vthDSM7>Nc&%M$*Z;s>s&v z9`330aZXhU**k?{G;W`UOLFyVOYpj_;v}+TknYv}lTpz8;YC@xGElz=DQR@Fji}9L z7uEY*Z*<>Ha~3UIu1iVc<(KYajSQ3N;b~y13BOGn!l>?@D-D}1z$ov3*f}jb%$Q|L zN>8JMA2%i64V^Ztjfx6~xyqmjS^XnE?+4y}$duDkthT2XNz)9LYz4Q)7DU|?J^DPm z@CgknYEOo`LUu-U)-4E&;U#O(t!-uhZYG-;4-ICE4L#$tc5NJKgB4sd3!@EV5F)Cs z56Vkh)B*PG#@*l(<6>=0OHlCDR!Sh99|T|h0?go}btL}nvk z4eRlZA@|4|b}nOs#x`?Y3!Z^*7zzh!!H>LQmpg!3aKIZDs0BC6kgNdM01LW+H|&-i zpuRdXhkeRR+F5TuGKbAQ9R)h}YCSTC6?X@`T7hj4;MMvRFoz9z`I6)JKbH){4^GT_ zdISn|t3v`Sy%bmD{x>>PPy?tru@BU)_c~Nt*2T%|Bl~Tm8?cKDNCUXn-0Av0#R5>b zxX3!lT-JPHg@98@dK1uESg|>@F|vweTPwcZ(QZ>yFmT6$J?Nn89qDQFBP2yvbu&3X zm3^HI%B}TONgC;!xH#Q1^md&IA(-Dx_U@GS30jM zqj3CV(omsiu_SxC1laJ2V%zu^fRvk`22iM)o+{S#mgr@x^hhGdhpdYNEAnhq*z`Jf zJCbxmh0Eb=Gu7&W^{kg4QKKsy1tMP8Zhq-eDu1elcXeB(a1%ik#t?SJ!*W`R=W%0{V z<2xH7ayiQbCQ!K#%J!<%ChZ9#m&+h59tlZ7qvGk(?CWb(Ww)&o1BC0A35Uo`M$LR( zyDHJGAUs%LtS#FY&|jS^poI>{QhJxjQeqng`i zCJ1L&hL9BVaYa_j>-&^I&qS|g+s?1rHkMNN#VN+}BYVOq z#wV@9Q%SdIGXrNKJ?IB(^=b8DLjN)5|Bi20G1y1^)mtN!yJ^<(X>nR5Z4Z-u*^tlj ztU#s(6PoHaYEF7-$|iATX(D-AR+Y|E_x6O+dtVx7BHGT@pGqNdPI~Mv&5QN~-l-jS z@%SFHL0J#K10YBLMaZUb(|c)LU-xbQy7DD*9Q^tKRpDrKyhq~-iLb9waZYC(v$@3V z7#RgfTFt<3QMU@-*0U~Jn(k||)(N7CoZRcr|1_1t z7aoUh{`Gf11!5Ijti}}%y}mIdgArl0$A`hIksFemk=McB7)lJL4S}U6wC2B>m+-@zXUm3|)LUP6!U0*?cU5=`vW4f;jFU+< zDghCa@lUF7>e67Xx|q_n$A8Sx4>x$yGYXpDE&5r9IF3#uRl;E$Yko=L0@2V=$nYtc zGof$ljWqCpS)9FWn!_{I$57Vr4!X(qx$hoN^QkQF4op+3u-ZNmMThr0fXh*_>w^$0U!*5KIwPk8!pONokbF+3 zD;&f5PsBL{2Tpa_OZwEVt;0TV*xa$TgShCrbnTe>2bOMC6G2tup7@vyF0V!y>s$0G zZ$CYQiG)MMwLt;7Xy$1+;5jGops@%+1vg#*NntV_&Z5 zQC4y?4ZX5dS#LiM7!Y&T%%o467k0Z9;F-B}k28{ISvPA751p67exB?K(#T6!>znzq z8)5Kq?_&mZGI6J=5QzORMTG7}%8Z2X?1O*(GBp=mx;RO=2VWomQV=-;Jetiuc?M=n zo=%rZb9y@kVfGTyXX+S)%8 zN5!eouutZprGe99)RM8$MG-xy7!7X|aVL9{v4Mmasb120lEBuMXkf$roaQur*@vgc ztIKT>x+v+2oSVo1Y7Gp_YMCB*&7XX(xXlr^*3rQ>iZQYv%^iopT!K#) z{cQbc`CBloz`Ik{+uV4C0#&|EXAs?ltc`WOB zJnR#1D7jw~&^7SLI+rgXHd2UvIr68msd98l%msrIR5{racmb)d8dXtr-AOY~Usf_j z*{Z3H_V;?KV`hw0*DJ4n(!1)b&AAf`$91b%;{EEWrS63}@k^!Dtu3kAtVWeAYpd-B zF{q}lnA{U; zCp{X|GcL>&%w{i(VShI{{dT%3pwnCwTJJ${$qQJ>{{q~k&js*EEd!VLT!7+ih&7R? ztQDOee(Lay7h)T+`Gzte%!d=pJ5n`!YgY|*iFr-NC|0%tzW5CX&xAFrxk=Mm`O|Ma zHq+=9^awVk)4bVkl-#CVo?yoWA4ixpaFfmkISjc&;y~~8`0Jclu;Ex(!Q!HJOl_{5irr0PcYYUlC(%JK}lgT2B zBBRBpe)qER@1Q4k%=^G314l_C^mbY4%~lm!IV*UM<~#cNVCS#Eh$D1kABptpL1qA$hS{8 zHnv9iO{Mx1T*>Q6DlS z)Fl^!@F2LYjrxt*7oMrNR+j>+^;77;%duwu^dks{Mf@<&vZ08ji4Qht67V_~Yons5 zDYiNVIpfsEULF^m*P);0*J>^`KcHTx(dn16OZ>(VjFap-+Bkbx60A}ReL$v`A;`*4 z7y-B0xU-e$%ui0Qm3mG{JMvnE%v}xkr^cwi+nV?u<_AWX(Z5c7-GXl9F1jeqX!X%Z zZfN2teJOPQUL68u>M(1Tdp#a3mL<{-et^&$(aV=!Pd{gj(|ol5tjKt2{}IPSU<_kf{W#|cbTMYC4+jiUsn^IB3NkidKp;@>9ngw1XI zm!n1TgOUyWYV6s2%i}NC&%(uRn%@ybYaDMUn(CgulsHpV_EJxVqB_eaWug>vH9Cg9 za75m^)iARtgY+=YzU^=7FZ?dsfwR|NY;Don)Dq>R%&U6}td<}G_m>v~8+Lu>Q@~I# zp0g>SH#^Ha4mEtcn8a0-9%C)))Iwsu4p#~LDgF)yLvR*du0J(k3^QFtz7Ci!DEqYd z1XrfmUkPC>_E8?Hn9r=4*UoBAgPc%Ur9Oeie}4Ptfy-L$^tlPGc4QXAm>^5b=vg_Iu|(8I|n?q3%gS`L2r%( z%6?fJ#Uqi_An&!1NF*Rr`Ou8RQdZwD>@RBegHnGv9s(=8!RU0&h5Vt~;n!?6ZyKp3 zTFoX^_WG9+Jp~*~J03LORlBBd(*TOo&o0UMIkPEXN`ANKX&=})AsO!zq`yGopUTvJ zWdmyh2kt}eG}N;^;D?o()})uDmTFs`Ey#8gos!+7T)2#veS;g|N$U}`R-PlB-RnRU zam;46z#bw#R0VxwxHUO1+~cV@T`R6E4-Mo+OZLA7=+bMRv%!ishM*clzm$czF zZy>MqvwFTa{I3Z#9<8)3g+Knr5KgzH2dmqA`nRO30a}HediWM=dmVS-p~?2ZVb1F0 zd@uA2*SWv_dE)qrz9dOO?;As=#*E2yj`|yo>@eQ_o&(-a79ehtw0EVirH7ZO8tjs1 z&31Tv65r$?UrFN5GTMe>90#FU5jeSq5|7Yj{mX86Bdorm zq~+wXY}dc25uxJe4ZOo#Tk;3gLDmK$xU`>%zx>hb^HMa+u@+hXyEm^;!g%|8zs%{a zH{pe5!{$&m*SbR0MS-&U7PWot*yU!R3J$+szR@HYg4}%7v|sDw@^C9WlBhr*=S@B0 z(7pIF-AsbF@uF4_VhO8Zi(yy|r(QVbbCl2_+Ryg+iuibJ^r5!=q>tc_aD$(tnYsb zy)Df#rQ<(r2P_>!ZzF&E1A3cn^PfX+mrfH{0p{&Y!7=l8_mKVv^Y%>Lf5E&p_3AGC zF5x<_5#j=!?96)juEn~P!mgL7aH0wZ8-FHT=bPYhxG`4+9wmf z&MQ)tCsg0w{5UGyfd;170#GVl`};L4OY7k=1BlV5`w^0{7#mgXBmxTi7J4DtiELzWu6JhS*ahbNZB^22?QrFBLZPl@w+!kQvX25EB zP6gXcvVNk7K`+pPjI*2Nsly0snwWj5lZWiIq zjtZ`IdTnn=?>>G)+?6?DDUpuHylHG#G8EFO@caM`K8f+o$j^`o1goR{YHG4_ySIWx zriFPLU@QkK zA0~4wHGX;XwRG&6b(pA^llH4Qf#&+2?=PAUCdv+kJW7^SWQx8A3%BV9;){i?s*sdHI=j;UELaGbmzmOlXyN9vBpK1{wQjA! zJN~JIYr6Y>$sH6N=EU8c*3KhEriQ}DQn_yQ1YYD7^X^6>$y<3s!X&Az9JOz2pFG1y5#C9us@;?PP$@9g0 z@M(`B2N*y#y+_o?EJ+57q}bT6TSlNS>AZ;t+-y}DX6jdPtl{C-+08Sa5{g?fk<95; zNnnJqFX^td&Yw&0U&>4zpL7nnGpF70H5+GchuoENf7FQatSG@e@b-XRl&Ud5Yg%zp zRm$-SGbd|@L@%@YNaHoGq?$$7pxl#q>26glra@6x$RMQ#!;Qb?WrE!!`5*E}J3nAetS9SMT6`?U+7s!EBfYox^Z%d?3 zdg)8Ac!dQyEbq^uI=7cpVha$FFuWa_lA)8_WbD6TIA}IHNXE~0s8>_A7Cq&>cMDR_ z7H&6tX-ix*>*nCy>MCiGN7h=_ecpr0(Jz2X>YaL+4VVtv-PN#uJY4zn_6ARyoD@b-ILNs`ti%Eo0wnvIeOM1-UM!G zX40+VV?=VUxdI_^xtL>O9@VqBj)ypkml)1@7Mi_h+7B821kYX85M%*-n}u_%ZA)dy zgtY^kQ88C1Pj(UQKu9x5Gxb3Q)}lFRoRZ>3A)P6)PCA$6-5vbSiBNUxwOYh+^G|;A zPgdCUMOANO_axKyUuYK(MA)_g63*m(^pCAqrkj|BccK6!IXHHI39O1kGVH}J9Kt%#7ZcjNlfF$2!6v%z*=CbaN@p>%A*WT*xFJ zPjLT1{7a9(=G2^#&2J2nn9B04@EoQ{fJv^Z9%;iY+151`KeLJ(&8Fx4IlBLzBis3^ z%^3Q%GL^Ez(koQaqK1Xu8>dgRML`K~ z=fruc{VKI6*(=U`)8plH>26!w@|6Z1@-_Z6T9&wzYu}u?rCK4J;}fBWQ{K(4uHfE% zS3AN3_pTVZf|wvj1Uo+r?{ikM#jlQ)%9I%X;I>bqo_kE|gvo8BrCwEa3^Fu1t&EcF zI;fhu z%b2jVJqNSHWww341V9xu*e3Ojp{)nf-Yh)5PYN17E+%y|zOW8_>pl)ZQGRTl(12zyGW4VX?+5Rq8(G0D?Z#Mtf%)UbQ?6R4DIP32sr=4Ux!c6O9**nBDi;USRV??G=bdLRT$^XS+oA=+Zcwc9K#eU4!(9o8 z!JPB^zGoKvT7{($INh}&F74H8;RykxHT>e*!pB|N{es_WuUZ1#OFLcQ+%^Bv!-N( zDhIrzb6`lZHtH#g`ymoN-f1D!QYhNzJZHq6ig_na@2&t{OVO)a8C-VNsWz{`wYYaK zzQxH_H;Y}1?t9`A(ku}lXoBGC=SN*&ieFI29^BdnKQMj-J#`R|{NOM)=>CdVgw?cG zza5&KeS@l+cK;@Q23daD8%0)uCY@d$zutGpv=r<$&#q9(1>Ig=@N(xdgf4S}qoWS}xZk#DVj z9t70JpSLJhRhVB2#Cr9+O1#(5WvN8ZyKS3@>po#`Fh?;;8vyH4`>M2M)rh>%U!(u$ zEc`bgh12RpE;Z2Imqe~guVl6nb3QbCG|DD(WOH|Xn?rXPWlKhsi#;Rq_N z$|En~buVkDP@k|}%-EK)jKi*ZljaoM;8y&KH^S0fz@f0%f-Gg^a zLgy6SIl6yn-x8q6Xmy|cjV>Jbv#}$KUlMV@qwb|PD_Pm3vLnS{Ud0~@o^V&=SR(=I z-*=Pqo{sXD9r;ZjlR%y2h$1V2@9N(oR`cHIBlT~lvKGPZOEw37%QL^Yda>rq!3!}v z1&%TAgKpeQ3X>^nt4?fxD-OBYjsh|$&M6}9nH507-6L{zG zmUF?#55u^o^|rvz{LhyE>-X(b1v`L9J8*M1;w25}6gQzu_;&H0R3J)E=bG4Z3vM2)`ZFYQT#sbw!7} zu3-};v_e!v+OH1#W5mU+pO@H?F1+f3Dte`x-RM!Gq^oD`_$z*62nhx1eF6kx!ap=T zkU~WVcM&E9h!d38{5|;I?(VzL?vHv-S+w4%ABK3aT3FAT6?|;Hbu$i_FLTkJrP;hYT%J&u1}Couj5lS* z%cnKOw2yK+Zu-83JB`ul860%h8fMX-*Jkrg626H%)u67D89;No8ia$O#*gn4b8wWos1wm9$w}-YRhw{cxsX&b z67s#iz`OIW`hvxR=^&7C+2yX+$uz_1tAB`ZzEsEFuZTTUl&;oIK3=##yz)C}RhkzFG#S(uUD*bNCUJ#6IJiyy z<_r+joD`S}I|^zxf$L9SJC128@gZ5TWrg2knnfAscJC z$4vhpC-Vnu1F!+{UK0H!;OgTkZA39$2BI7OoQ#@+0E z_g7KH4~-LX&Ns97uX@8b)l@t6phlL^?`d0Qv2Wk z&sOgA1IxzK^q0Vfm#nn$>vHXSW4*>Yv(#Z0Isd{yq@0S&H-?9bo?ATC29PV3zx>hC ze?5E(d3agEmwPlB0P%N)S=o*(a#Vl)R$f@a|tCry;9wZ;CS-Zn|-~w2x(B; zHVhN-KF?%AJAZziX#~Z#lY=aczmddXS*VJ;Lb6tksPxKYNTrbiZ45k-T`ZS0j8?c_T;2kP@}@uQHH|L}9mfiP z29T7jYU598#?6p=%ofBa>55`=Ou+)_%KuRz_?Y+pxoD_3sCuPXf-W2{(eC&#Y*@>3 zgX6XP%}QRGtBpTseaSoA7e#L}RKJ z==VS7>fd#^2&FRL_jsLx7-syO<1$BR;J_uO;l zz3-j>xij~?^Ul1CLo!J8*PiG3mQMjWDQzhVwIpVsLg&EeH&IH36s>>WP^-gn`fDMy z?@IDYb=?3zy0&PHtI5cUw_jNJe#3pkf!a0R{Dqp;dHkqmu7C{{ zA23E{Z*UXB-(fp*%i*{0Gx*ozi?f$jaVrG8)^q}n-aS-sF}N#>0Canvho|qX_x>D` ze^5DPLi-ys_ARw4k}Gfe`~P!xBf>wXr4@WM2RM|#gbsGPkG4O5J+V|!%6%yl#Xgx&pIA23{JkZRV)3j z!NfUDajUv^dUefg+^Wy(HTuGYbz6DE2SKRWvW-lhGIa6@Z+Xc`1hr4@Oy$W8e#94E zd8=op`l9rDnU%}j=wx+%guZUooaKJs>a~O9RoM&fiw!!;QIq^1dK>es%kqOBK|+$P zq;1QSdv~Qt8wQ{|6GeZyw~)1%7yV0HjV!bv7c~MC!V`clIl87!JI8>{Y|<)ehVM3+ zH{@MD0|oiG$k2oKj4-yqM_NBa^T#J~DY1FU^$oJkiP}Yh>~zzdGWcH@C~APvkrpnt zxzwR+LX`z781^e}1#D}tWnVx3zV!dDm(!87qA@|Aj{9)KP6vuHoLcn)oM7|@I_kHi zw|W?H`K{jP2DdSrwA*{(RyRw>Esd2SVE!iKK<|ufKeESoT6XPdgIY3H2p~d&!LJCW zeZ0gz68qbf1Ic+MDkXhpyt#l8=S_vk_1D+7Bht!7Z}NxLB#^M z!GTdcK)t-LP)|`8s>GdlJwXZ6{M`!hCICu!5?N}EDKJM`Z;R!9I$0aZY5-CSsi!&s zkWv}F+D<&;1XB7qKT!r|`W;g0pit~86M6zEeKh+Fy9Gc>mGo4?Hq2w5BP@YNOJC|U zGNaLQ*dfLHy62?PV!B|{53UX9KWVg_^h+3x7NB23R=hlGc!PmOEkTjc*;hQ&3w_ZZnh+wZ{~hKR*{K$P{L`LZN+b(^ zJvFmP5B+)yV83SykQ2Y2Di>IG0xVbL|MCTRsd8FK|9UFtI`gZwzkl+-5p{r+0@k(~ z)2cVZa~}u*m@5a92bYaCL{^&n3?7!GUbq%~J7Doml#H*;BbOy3dkajDOr@GlwMT7+ zPTrPPRH#J|-|KZg@(X{8_O z_=%otovlGsT)Rf=Yh(Q_qjO?7zwLp+0A~T)0+42V+EExa>1I9I#a6YN?E$iu3nkn> z#|SGWm(xnm$|}D4Hr@YDFn5XpH&}NB&}Xdxfe|*a6F!xLJ}8bYJXnupGD+Q3w^s)z z9rGSIdgKuY4~ji{J+|oNRRC@<6UkcaBM|nb7>?`{lqp_c`on$b6g_E#gj-GlKd4# zIY5hW1)*;h#Y};;b*m=UcN=J}7|qUgoU2E#JRQyw?d3aNjI>p$F`9!hL&9Dq*_7@d z-AGrb5!3BnSxP)^Tp>ifwsYGWpRD0E@;j^bA!PfHr*RKwWTkSlk3&rg!)k0_B|fL3 z6G6vC`NrbMIn|Pt5fo>P9-V>OVPg{r!sg zpMLX08sf}dgNG|dWp@)ClD(2UTieEN6!cfkabYQzwjT@RI2Jg)C&;BQOGzvGY8DMj zepkRrxo;+dm3t)*(tOL{{mLG^Ew3VQGjJaNX-*WpqO7^j zK!|9+m`(eJ5c%Y{&Pc;f5F%xV|JyW-XozIVe|NGdA_l|rv)lt~ZJzqCfQVhnvIOk1 zD4%~4u{+vwTbu(uT0W=O`I!6=d6OR^iSw%t83}I*82-h!ba(Uk@_JQw=W=_fGC+ZE z9P>h0ETWib<2|)c)+Ut%^EK;t@_8;}`o$LQn16<-6O(3b+!Dp+dLpC&&KW{8jzKwN z#OwYil(Pj!y#6HX8|BQ6)nYy|)rmnlW02JV%9*09QNfjuo{=hl{L^MK7@EIABK1bB z#&TR!G`z?C-fH^1sa1wp>Tj@J`Id3yTW}#juyRJm@l06{`4Y%DLh=T)+yI7VV^amd z(A0^~qM41a?Wx+axfKA-8c?i$)=GdjgRR0y8$#8Q$r^Z?#Yd^if6x;ZpOx)A$pC=yncZMb~hkw1V%T!ABnlu)l zv@EX1OCGdAq23nLwR-ib{@j!bSaW(-G}qc_O~dnt2X^)yH_SBdm+!mJwX5fQKv?oB zGp^Z2QnS#dV~>qQFT)C-EiLgxEw$jd#QQCym3T9@-6EiKbXNnVF%v*mw#w$`sp#R3mMxmL{5(x@Y^uZ|m~Zd&!SnXLDg!Qr z__y4$MhUM#x@~RhNG^q8DZ^ghckD-#W8o4(r(O3x(oW>n+Sl^bH^$e{O{vq(&mflL zI)IUrj9Uig-o>oMoUA0%DSL>2flomkZ{}h(?xuuKXKIy29l@a#D`8dS%`58bm(rlk zTWn*!go$SnX&$m{0Z76AW7^|^qpPiAq6*keME<^^Uk}LhEMqEd3x}%`sek-d6=9(=#5d9Wp z!SEeLn{NJ((P|{AEjl`c{=M~mv;*gBsBM7CD3Gk!p1n|&jJrTrwb4b#KocyJEi zB3q+$TMA(u$B1l6LRo=8Wc$SDU6HcW3J|uX@7gtAC7~Q`Dn@%thj^6r3ZlgW9J2UT zVhkg6BvpdyIQXEydwby2OVVa8EdzY3w8wfr+AjEiV5WUv`~T^eABzZ?46gJ+6MW^@ z*_?Uu-tS##(-W?=ti_OAQ0&%Gv7WO3!ULh#e6i^rwiWvOK$+zoYM!b1d?N^>Hc1@;w8Y5$Wb+X|0uIFST0We-QWHwJy{FF zMU&*V2qj;tTJCw=r^mZiqwK#++Ik41!z(yHoL8=DWY4#r=IAG1MPzvoRK$%1raB8v zYPzlaUP+nTwizWg#0m&(D6wV__Viw`v%A{J*{Y4jS#R{W;A|`2;-(U6Rj5VnM{yeW z!glHsEI^YU}fZ078 z&*PG^%1}`V$JZfxW(^!7g0p*#@5F+2uaSFMY^uhg1$?3?Sc?bHo7|5tFld;RT%B`~ zn6B-ub@IMeQR$*CI63FoS)y9#9Yh`x-`%IOXdJz&lf=#;ys=lPakq*J9j-`B{wW_M zDlt6&d`^roRnLD%2DlfFbL)hBEN83ge>gAG-bfueUmoRAfIKT+A$|AxbEN##J#@3a zT;_Jf3Y(#|=eglqPYlJkXG}FayR1Fx%0Gb=TY{K#) zcfqTYD6;~7kgT?&fWq%3Kzv!JEr4E@eCa$Y*=j}~dX(2ZA*pyMIQyc`wnDfjV}xJr zUQE;&aqn_Kdel881sB{v6e;*9#3;W}dt*2MRg&`OB4vlBNyiyqMLTFI9XkYm*V2fe z*wQ{%7wPO`5VE|9OXJ==s4)^M61$NEv|*S^kgNbpB1}QG3Q(M8jE|u>y~7wE5dQ*- z)4C_)`_JOvqwk(j|4(}D`}KeF@*~`_Dwt^!=gpq$(zfW}!xwCV_*lMXxuWDN)bEvd z`To6!$&c7|(th1}Rv|7#vZ_^q`2XB=>#=vWyRR&IWzWFMim(c^g7gIj7|WCr9Q%BT z1Io}lh3kM)hpIDsxULga>WoT2kI!-jv{G_;#PRtu`=u4s@X|Vfl)!(g<2yl0gqFU} zRMul4B`&C^t4L7?ZN4ET7!Zp~08+x@9huYIS#tZ{oE7IQ#<(>sIA?`nwPWOO`~@is za+JB3e1R1AygvNah+EQSCU`*qNR34hAVXX_LDapV)NjZZ@T6eAmcL7YU`vc#6a7YC zP^1yYm7#M&S#I#^HkOsGiJof@^HmQpXSp!(=137!D!TkKD#tN*XX}6!QX%{ath{(2 zsiXVcYd69GGB&8Z)@u&ZkKh!777UxPa^6meq9m$3%u|0vO1GcS;4geUKzPX2-Pxt+kUa{G3ti~R@1D&)e*tiElC{}tY(xmh2sU$`U zy`fO}h=v%Z+ch8nKz%%eiQ{brmMr$~x@FGeA#FJ+vrXu~S(y8k3> zuF=hFdnsMAF047BZF!p3@$PfJM!XUTIiQ%ejPn-{_Ykl%_gN(I%C64AqzpDOPU%S3 za_o>nRx4;3BA!>xdA-=nyfj1CMa==+YQJYwo0s}~b5qrOo?8xqTni*izDEv^@}%ss zH(Ks``UG67rjB9B#zPJ%Y#1=evHg>GZE#YByQ<>D@ALj~XeQ5gk0Sa&Tp9;b?lX5a zwozm+x!D*61`S3&^*b}B9W^26AJ2=q3$rVF*pMtFW=zx;XDMl)+4Mh)*mAVSwKT~aQe!kS#KEe15kKc zH#)o;9vWPd@8nW(6}QriQK3@8V`AE``G(vkRW2tj;>~U9Fi^lC9a2Veay@2RbdN_pqRBmN*cF^`Fa_Q5q z?w~72zx`B+P=-#SV#K&YE{rAmt{>D>O>PKgp4Q%A@;2F^NbzxoGRYZ~^?C{rGYs>K6XcXolOT2hCZ=XXX_t-P55?Noq%kE-+e;fozr zCsB0@36QC)t6MP;+vyxju>M;h2q#Z_Qz0785z+2-XOmwmS!N0q5 zeCpKyr3w0vQ^)d%J^~H)M#SZL6roxw_%FX&sGQC(CEk>ZN_%iKYqYx|$BWYYvDbI| zWp9Ahxu5P;4KpMaOx->liUfP zT^=3C$ej{Z-aPSXc%p8#Q45(B*tqELX%7KT^?;&U`?r2Vh44^?;A?&dw!dVn zyX7V+!4w5P|1QC14t0KE0%GyY_p9pLwFSsCrqRpoGLB|q6+GXcqiP+cEDJ2c$gG<0 z1Po$_{C=W)(9;&g>vK8r-#!4K;QmFdyHJ| z?%vrhyL=+8FTOa15HAI!^)qeUZksUI6PFQw&BHz+y`Y)UwQWcu^r) z(Z?wJQnw@1<3XG99Cg7Yb=e{HIuw6LL^OxEU6&vH0!IzQ4nyIaHW0%O!x?4^5A#3a z1cm7^%Q!RaFqECsY}`gGY^(P?8dwjdnu`rkm22byrw$dESgS!I;_h(hT0fg*#Il%5pLC9 z1bHCv=Huk*aOXID$GbW{{paB|dMsg4SP_7@K_DE8nny;_#Jt^IQcxsL;G3{L;v^B+5?; zKK)g&Ebo3{zN(4*dn?NQhQmHjqUN)CvQRtlu$e@?Z%{y%dk6z#yKt0nrS}R+`(Ar? zV1n57!M?}c3FaQ`mr$970pxg7BCee8cJcG|HHrg$8>n^y(2=u5kGAhbx?O}vmy0Ui zNOIfR>3qcy*W>K)(mu_XJA*c|B%FGODY^aoZSV9i5nWS68*d}iyB6mr3~fRrrXsK2 zES-HhR^r8x%k)iQi%;YL35`2Z*m7uvPU!&(TRLps8IulFXlwO3=DF!_)y{?ObwFW@ zQS@^-TZQ3a6#XKMQh#?}$P55L273Jdy#UAnnfZ8515dj#Vpp6OcuN;3Gn$V3N6-vT z%8bk4=f2fWC%6W&02sTvH(fDsFl?bm&pRmTW*x-mGiXY$iT=rt6WIJX%v2ig8QH!WW~o|Z zeWEEh#d@wB>Evo5&+BU8;S58JU^%P(LW18-f@K<5rt2GQ`y!JOrFrGif_+C7<0)iK zu6;VQuYvrwkr2gWZz?0_<#3UTvC{kxvGX61oiA`9+p4#=?>PqV#PRo)I&AMuaWk(4aFVbZcg9?`I;# zN7AP4)dIz)Cxzgv$B!^-eQmC)JT_<_F`m@X0cE}8DxM{DpbLWLhg?sWz#F*A(BaBQ znURU87UqZ*>RSq{G@s7~GWGt##ClNvPhn02Gi+`nV@=}TA-oZtifgDXNQM5m1V%(Y zFsU7ybKdBWf80&Hrt^vKa!xlxzTdit)nH}+=u=em#(SD$Y>^|P zZgt-$to^bf-`{5FZ8MHDs~S#$_=QVf+BIXH<_>@1JTXZeqU^*<115=FK%xO%f-Uae zsPsxref8%k|K*10CAk&XX%Eix5&~9Vtm~E^&ga8jXNxK9X& z5VS7YKwGS#m^+wy)isVXvWS%`ip+y}(@^@*pQ4qxWp;{z@>5oDG+`myCuqtM2G$g; zBtLBpBL`!N4YfI;vDY5QNn|UrKPuj0a~*MT=LWs-Rn?0ar$q;SnlxSf^5?W)fDJW_ zXuqu1L|5HO%yJle7jRWnrK9%fqxi0_^%2Wmg;4ThGM7rQ2wt2d@&wpoj)1OwC;Dp;~`HvCW2QhL@5F%FDao#kfO)yvt?{S3I=RbP%b- z`krnXOXQs!Bouc74Ui^Zu{_>#SSN)?>+Vs++X%XDsJH6y4%yWimfc|; z#A7Wpz)iTmz~=1d^Dh5FsN{R^YSBNwPe3Adr8gApyc#5B6i;5p1n;)Sr)R|h>fywP zzS-B#g9I136@s6b9(3kZX_YEJ&q$h!;zNM1@ovrdy{`8}-3E{)TdRk~3}lJRdv$Qi zuMA{~o%J}*FT~RC8?xl7PmRMU{sdV9ah3Of03b`;OR(Iu7dsxP1#CzYePyt&N*wTki^aI&-fc>l+y0Sk>kqAx0)HNlf8Xn0246k4$Hm`=1EUo130UN1O=HFEgYg7{c>_bi~UYa-x%Ky5HIsl%D?L zQ(8NsnNGB61GFlHAT>=?tA|}DS`}A3oK#*hv?{_L4QRFm^;lELqv~ER&T`*u*iX07 z{%W_7>)Qdlt9MpKdm?yLB63+=h?px>TFL3T?^((%m0ZAG-ivgIlP!H>Am`9i%X80t z0{zQyP{vfrj(IEmoZOOnC=(z%)=0eMS7VTUIT-#FdKL@+uf$(PX#_z0_37q}Zb7!H z<9I+axBiwZ&^%c-f9~xw+aiPyPQF=R$3E?ypwQ40M(k1X-O0XL58?g%?tIBA?w_{w zcA|Ti<$5=LL9uI>HR^LD^r&{-b;zckKbj3;BqSb2_hq2~Da*w24{^k0>g zIjym`*VXlc4%Ye7(nTR2xkxzsh*b#e1E`d!b>i95x0($@e&;X#HpzDL9 ztwq(65%AmDx3hml9(S$*Doy}WmzsGXEnFdb{I{>C`c!sk;Y2b3>-~CaTjw2KCui>* z&BA*T_$MS=f9l?u+L<~$_j+1$?h_!b3e+N6`ob_Dx;D;zJ#}7UJycH%by81oz!S$_^)4`gR_&B(zr6rS|A`7_CO$ z8}rNwl;rO+b4Ytkl6ai%w+!^;np^vOq3nZtHx-J!0rkey8Hu|+<%2s4o0HpiTofC* zz7EIiH=`!DT+PQSFsq_IE6}0=^ImXCj+iX2zY}4u`q;K-Ch!rU-l&nD8>&B3ACHwA z+e|v}f4Mo(+X;7@?)DB3XZhf14YT96l<6qqR+H69*^q}Qw|H`sClh8k?H~wJ8 zx;V8^d;E50TlTGtA1S-H=&evegz;r?reCKWH@oc0!$mY)C2EPksrO@Eb9G#BX;uh> zq4PFbUM8bct=f*h5z0#aK<<_2Ep{zKa^v+Fl!$jh>bfM`n#7m84cFV-AOaB z!7vy))_3^FLL~Ie(|02g4{r9mSbNOV)@H(O2GxP?J!B5|Vm6(>PKQ-PqBj!cBK|TM zc~h`5UKt3?T?;5684ls!7>oZr-B3I}E2oKSlPysPfU>FxYuh*5EPDKS`#OH}zpKae zkwgCS(lj7KIXL-S1{+OswjQ&*G@tMQnEm;F*U z)Diz_!OBsl(mn&eTN@|Xs?*WpC;(eU&1ZU**KlmNzT({`)%@B;?HZ;(ls8sSTkLVV z*gWFwfF^LQ(Uv6{bzgq10Lj3_^9y7@Ti{CL0vK{WpS{V#e=x@V@BL!z?wC*|-`knZ zk?Oj~r1jn+L}_V6EeANouiK92zl0@jWpo0p0#;ymQyHD(qhi-qr1(TXa2uf*(f|vO zMXdE}blI~?Ux{8Y>&xIYHEW}I*&$YC6A@r{L(QLuL?Wz6C>ODZ!0w8C>sEtPopJva zERp=@?oP%)?7MD+blo#L0uLPE5*lZ(e%y?PjmU|%?k#j|a4gSm&gRmckh|{=E}r{? z^723v6dw96e+bvsJ9S8hwXA~YhWFd)UC7-qQKBO*1F3}`K0>s8-gjN4(GhuFIYYc! zvujsZ9rD&WyC!#zW3&|aFrRldW2=YGi2znno+r*vfK?RW{InaTsP5R_G^}Wp{P`bu z?_5^L1Hlj$5r#PCwr6CAek^&j} zu)u8#NFcQVWtJRpJek~(-(hJ<77MDY`* zh<3*Ug0K0)!T!g64WZxlhgNF4W7ONb_U)_kaK3=?9Yr2AjQ`$;>yED-uJlw zT~P90yfy#Ge|R`M3zTY=-t5@V9{AH`INmj}o2MV?em&Kxvu5^|%GnS2HEn0mGxM3rJ`|BIL z&kgip+_5-30LX~49nO5F`OQ z9o3Vm6e9g+fTv7ok9FEae)w5(FyZ;%w+PUS_XO|%0v;$7rtwqnN=&!-NN;w|5)6T+ z>Jp&c^tYGsnW$~~k`GDgxlC(%*5HoYQQ}qDJL>6!S{N}u0INvT zVg)*r?Iiglx>h!RMT}CKmdnbvKjlkg#tI+sUWbSW9KWX1_5cvNJ74qk23e0tnBM== zJVX=FYC3cS^x|RE%n%M$x=88O&={6@W%I86p}uALfa%1If7FZrVP4S}4G`#?SbO$o z83g(!`%b}+K;p?D(9^+0V2wFW81&M)bLs$tKC%~J(A(_u!oq1N&V}|IAcAxJDN^I}RLQAl}1mg(S|)(6X@bMKE5k9v1IN zfSB@Uga}*?3<_9`XQz|8SCn&L*vZ14#zm-`@l*@9g=l{eomPF+S8~60BC|(NY%N!?_)USLtf@dzmIMhL{woksDL% zAmwZ&w$>l&+JQsn7}k}!_gyn1Hu>z3Q<^Gu3yqr>Pb*PK?s3PDDd%rj|DoK-!+q>r z)JJ~qtDwASt`^pBrfR#!+UFBN%vu@W?%NQA+TF_SHCugYN6MDNXtTmA#6F#k{O#|D z;@|e>$9BhB`NT_8BdOtc&!uR$9Mn)*98JrG$K&NVX@v8X7N~u3r>bU?Y>UWnDxz?BI@uI7$Q^Sk?vpX~M2Yveh=b z*T{md<-`+01bk1+yt9@vxK(9H5O}YZv%L)J*X5NP>c4pt78s#?6G6JZ$AEku|oeB=+CLvJ$l-VC1Yt*voC5V zU(NeGm*rP$iiwgWo~8JMqjbM0MBxTpNN0{hq4{k<&%NO1B>Ym5T3zL&>5MnE4Qfwk zmpFU7ce0wMQ9Gmuj~kyL?OqMG zFCq-NC0!no;!!*Z9PVQ3jh3%hwaSuj~8Qp`|~GC zk)z2}%xLWL#RH3Q(V_B-%f&$0Df{)5F6B%tt(5-DHjILzrt{l+;_hkv7IRHS8GnD` zd~WVphH0Ne|G+?h+c(p`U_Q;~&!b4coS63Mw}h+>R05`b!kjjSSCEp;2WwN+ccJ;X z8eExvRycIJ#m|#$QEAy2ZsdZ=zv32u+v*9gT((!)5Yv1*bHk9NKMLkptfQa3F$KN* z6vS|!Wzi2>(f;N>%LT$h9eD#!+-KRenXNql_t`Wq*KC_061IylLFGD@>ggPt6jk}4 z%AWY)cH49@Vn4SU$?VYL5ij(U{Z-Q)4Swe&FRe^V11_>~YRRfZecdHCsbyE+Yu&TO zl|$2C@N*p+Vqnyy|OrN?l1ls66}pvLByxY;gcjQuf~KvRdgo2 zaUO3m<)!@U*hB^)D z!>i!^n)jJ@={Iia-z6*wjfh)F4+{LCh|A&FiSDqVS+32J3R{z-L$ne$iX!p>qRi+) zSdeM2G}$6{+!1bOB5Yt8>KJvsBJd)Hu6m^5hDB`wzf?sh#*nAOgV;+CNXi!xZ(#I9 z)KDkHy2~9-Z%sAu-(MSpmTr0|4207!Qfv=ebs91jZ<63PggxLYt zRV8&4q4|+=ztu+)w7#+G9mKs3Z=FmtfY{=@1KnT)-py1JM`Tn?!tzhilFJ-+77<4G z3`+;bf|u$940vOVHX_14b_9>_%7-0Ccl|sO)R3wtSjOkzn-qabyFH1nud|d zZ1v(Kkf)+#jfH?fp2flkQx3>Md3moK%2jkY8UXRQ;^8qv#!9^;X^iKZ@;ciXDc*FE zFQ&1PeGz@Wp~$$Yw1MH50_fZ^lO_=GG3hUcwGmm z2RMvV!gt{FD4@+fSo`&qg*&xh1X<(YKJeb$zf~DKbe@>bw1^n7E+4ob_+&Mj&2Oz6 zqml!tm>+wtg>^&|eh=s|ZrVhkcE{hEr}5GZQj~Go{06ZS${wd79zqtXy^_i(#Vfe| z!Lyks%kh&-knt>olm9$7>kfmHU+aBdlt1GeCx1!oS9E1u&aH2p{HwCw^R*04{v^gu z*87d*Cwx~$$2dUDbq`}9zrZDVDqJQ#?rXXXushnpuAyne8Mh#4MF$@P11m?n@SUO3 z0NX*Ixg@559;U706+s}y+^cA2KS?pCb=m+rKMDY5QyhHxbz>NGe&rn)#~PGD=VvZW zE59@ch(M$T_BX{3JpJPzH}ZnJ--UR;&Z$WKr_623w|7#LL*MGCWLUJ07z=_dy_DT0 zK1hx?h*Wi2R10N>0PrB5w?tt%o%-|JN!HE?#gTDh)KCtP4qInu$7O8nG%g|v0&KHu zVIaTMaLHxnU$qNU=Hi&k2gR968f6=L`8~&*J+qdEWYgD{l$?3N2xYuuGr+8cqqI`J4lZ29c`{ncAz4CJk zndwk0am=Qx-&ik<9Pos<2Kc+w88o^hH7+$H!|g*LfNfU`McoRBF$o;UjUNI%)Sw=N zDYX&TA`5s#>x72@{2iQ_QR7dAC_5*ptP_fAJE3ilLwK(~p%@bb^zEIJM|O8PJ1H3X zj=;2p+W2r(YGEyQP^Nzcv~nOw5-Jpgc$GsgJa~n8hcGLpHm}WwfC<_4_xpa)EU54* z?+Rqnzbtz};pmUyb8AAEi}3f~1!i39M_nv=d0#f!LU$8>k)!&)A>lqz+U=biE+TUK z!(nA(gIQK;LV6r>52l@l@=%TyW)8XDkbr`qY#rgdJm-BZpzarP7hTeFknHlZc{qbM zhwQf@ChzRQaMh}&bsTO(Wo#Ct#;PmqBXZ<3Vwu+zu`Vb4veg<#o`-p3Kv!?~!H;M2 zyqdU>R!}L+hWqzf9{$I{ni(=#hc~aDT}jW%eb~ERKc4=26}M0<&0T)EmSaf=bFccN zeBAj*OL+X!b&J0D7&LpygG$s8+}2jpV)cgfs_1AUNN8P44H`i8-rh16bVkndqYFK1zmZQ7do%EGXx_Kno0+sgs){x8lhZO8hdmYS2B67pYc>dr|n zacp>p%>bx^vn|h?WvYgZxvNb({#XOae|BUx=98PGG{SM7M zb?USG`sUoWuSOjGlJFu4Yu z{oh)j7TK_qrknrc%>3KlDM)u@otNk`Q;L^2TeWL-v+xC5=$)|_niV5NYh47i9@M84 zLn@>Tkl0zp6a%4mevJ?)8$;-UGb~<#dcF+awK}9(!eHc`eItbhYI}x)WUJIwuKUHA~Rj{^9>0%mrzVXl@n>HFHhFy3hB#1z&5;T9@oXQ7Gu?Z z$G+3)PmHqqvOKV@j^hvgc&I6%mA>cX*)^=Wjh~;MO^xWf_Ro3WO zT!`|j2FwGKlQVr>g8WRKrydK4=EZ{M9G!ovFZukBdv`{oO@a1d_u}3xAdQB|SP}uO z8cP7H#>1C@E3S{--zLF}PQF?|0h)zgfc-mY|J14f-*@!>+O@6-IL@xbL;Ta=%W0($ zW@|SNUF5lezHSeBLeHvuEydSguUbShuQW3|IAP@eln77PlR3_FLYfJ1uB*7PdvB^4MQm78AkW?rvwTR zA>8Y&Ad5cW5p>)M9yxggWiI5LJc8cNaJF3_=zMW7s|_GI@DO?OP+eln?PYLi$MoO@leiOgXw2uuKpze$4T z8QI7@8W*1J@9%PG^f3f^1O)g^>NW-}N;}`LD{j!rEndbyT)%Y%YYH5qz1 zULYw0{#<|3erZlPIb>Cj!#3C@kI?cO%3ME3q_Ey8pKtX|zrLWxO@k^{by|XF8?qe) z6M(EWfeWxUTbADy%~vLmRXVEp5)hIK;@S$iZ<5u()F~Y-ZMl-1*V#l|)RZ2G)p%s7 z5N~&A0WtNn*Ejm`GZN^BT<5ryfip`dO87s1o+b?Z+PP|a}!MtiP0f=WOLt zL*1l@WrqPzbXkik+VA)!nFtto8fRa@`1LwC=(7vAfyerywwzjyk&}DcxoavRSZVXA zO)Sp)9m1X9t65bVeWL;f6)x!7y5%s)YP7#An@Ct*G$bul@5Aa z7qmF1snmVUJ50))mH0@B@fdux>>QQTFpmH#WT5TY_hB*Qjjs9jucss~s`YqeIRR?L z=+GoUtr&Ogdgnx~*eMeKLboSFQX81@))EaMG zbjRktx??&_-QEKTnag%NV`~Ai@29cpE#$BO>hbTy(wgMIGDdi9ubMrP=W5-gS$XCa z9to>9&W7tM=S1VZ>*_gFSD#oKKmu6YwtF-V*{fb$xy9Ef)pIbQq<4P_!`~W|th35I?;Vd!&&j{dy>1!sDS!e#1v7w8fy96K6xjWjJ_W}dQd6(} z;51Gjvkv|DilpgE`7G!d!q(6o<_>=r(Bmn2N0G~bdDtWVgSYB=7^L}(fGQdp^#_CG4qt7=@ zwoPi3qYWB?I?;;aRfWitI?)DbRnKgi-y3KI{I7MQje8X!?eHqBAJzQO7Sq?6jdK9F z)BG5gr=Dx0#YlJM6KB4Fu!^LcB?nQ$>C7~@bd!Xri+^S8n$Xu%BO&WE9e7ez{sHcz z?J8fek^slqq5%DSLv|7)Y>M~9)x2?MTsNcH9yo{k3y4beaLN148>{5E^BK+$Q|oCe zv#6T&=wpS{r%9U>W0@GZMvR89o0V$|cA}yYmiY;p7F<_nVB!=j3<_1%O_4$SHC_qI z+q>`cAhu>uOJoS%sAK^jUp4A$xz~xoBv_C2&9oUcREJu)0<~XB$wkNYr6GYdjDGZ^ z0WUx_F)Sb<-G+=`7+tdajWK~gzc}hdc-T?W3h-ZXHY))YRl7E-sWV6gQiWlFpwS%(NKA4LOKc=lD%mRVUu! zFm=ns7*f#1PY7*RYc?Gbm}G<>K97LTUV>n%Lu&dnv62&YiZ&RH-wtdb#rY8WdLO*QQuRPa%C_R~)A+#% zN_^L)f0*py6GSjpb@gwzUYd)QmD<3_L0dEY4_vNqOz!FcVz}oV`ed)QolP&io#NTe z^o1JWG7caOfX5hwPQ)B|i1j(dkMHFlm(c_=Gx{S~?6HeSMmOoBlReTXvg(5p4o@E= z#~KA9)J_vVm)gqLNz^Le^b_u_PjmtzFoALM23=f>@BHq&v1H2L!$VwkLD%yMDk8#8 z&ME=ZQq%7}$R}we!O5Haj`;y7Jn@JNnUpYQPFj{b)*0go%EsazgTDiBbBq zqOcNCCL%>)0v(lUce+rwD{TEQO3%dGxLOpeZDA#Jg@c$Cz4mOWMA-XA9*NCKn82p; znquMYVQg7lv6wKj0$zjngFb-9kJp$ew%sHS>QkkVg(kGVh1tY4!w9i*9JLPJau(sF zU?&?ZkX>zo(^$)I{UTF$qH^A)XSrA~*I>&OV%;smhHo+Rkpv7#CH`Ox{onm{#`tjJ z*VNLG*)VOLTL(z1rgALx!E=k&S{Vrb_NBdv%qy4%_un^r8+%`Dd%mwXP-5yU9cGw( z7o)`8f-y_`fJ1b|7I@`T7syLAAyvb1yF5YzKCw*J z@R6WJU{c#mZSwrG$Qb6pi5~B*e&!dk5zT-&T;+0vq%+>8PDudq?zNSLK;AK?Ho2CE zpdFLC^-)MI9UpAe#wW#UWz0Uto^Pe4yyo*Nk3?tRNnh-lLnSH$S zEGgB(6q4>aM@XGYNGjut&GEf4agu9rp-p9V6t~SXjEJ`+3xVvT2jlD7dO=7UGmB`YbsFa=@7c?wQQL;cX6`6BMBB>xy^S4L ziuKgi_MI%c?mC6Hqn~@ZzA}-g9Q+uicx{pxR8}nsq#Lv8&|5mA>ZPxZRpn#q+j?ZIr0ljUH`#B#}=_qtIXs4(cJXK}Z@%!t^42n&32f%Sq1ZU5I zbm+Fu{Du52eu0V4^k4oq2wq^s?^>dZvG-@B6X%pRV|ZpLMNx^3C>{FmJ273HOGi1A z2Xwnw*f8xx3}?4Zf{trgLZjDXhUPcJ1iz)-9&=7gbi`gKO8oPY$M}AXVESrk=SK13 zYD{k(%4vl*!yDyTyaLFG*GboHgdN?E=u*4F3;=Ao5UM;6n!9=qRUn(PQBVM&jw+JQAT_L`Ke-4?W*~xvY!rOUEs$3qkPoRzjZW0O#Wn^7A8Mkn_`6W1}1I6SUV(oF(udCX6PAhgc(RaF+v?UC(6 z&EiG)IE>agl+aa2KIE6ydU7JtpALxhpF40*rp8$GHxI!#V%k0^sY-`-uN)RX!%yH2 z9T9mb*lGkFe6CI_d6~FvuywGwHWWSd+sWv@eY-Fn0kfQYcz8!ZloQ<>?LVHSCj27y zI)G>pl*hV^{(5JlaW43U#~P4(%FxwLPd1|eploZ44i>fe=V)0+w5rlqu?Vw^EUC+v zMtv0_;qDoe9V~>-Y!&+D{-R4!!Fj2{V2&u~L>O#oK(k4t$R=3(3l0?PJgKH|%VY^w zvcywpS*67;Lu(W%sUY;98x-3djLjbMuWMLr$Q%vYjZ*F(X}_(*?!jT<^AE%R{`SZF zG>KuGTiyK%6KcZMUs44x9jcq^@=o7|Oi&DYA7#9>wz~N!quj#M7-CvhG&uZAf%u## zM;WK55IVG@ixBf4=-zUIakntKDNcJY9zSFmR%TGd*#mCCqA-? zzD)CG&D`APALLY-R;c$|Vysr;+eP~hW=lc6viXbQ zMM}spY5P@a$ZnLC!1Z{SMzTtJMRf^g!Y6kpDeLfo$cs+y9T_{Kf2v?0%)*AtRRzA1+DOxZk4K$oDAa zTrpb9paj6ZCX`HDs*`a;o&)0$6Qybop&o_CD7k6wo7a$>tp__3uQJkiJrZ?1oE)DL zppjFD4$dU2njGTvs;~W+j}*pAl4jx(qg7PG9DyH_uvqD}P1#3gnK<%>(q1mGQTTVZ zRX{#IK4L31rbKFlQC8=RPZ>tgN)vExB}@Bb@|iFdR-P0}UnkdCou8f3=K&wz!z+G& zQ~mkz|9V*d=YR487k|xzL{9$Zo+++7GErnFA8WnyHodn~jrGq~IHzYrJB>;zY|5_Z zUyHP(X@@g?sI|gOC2T>Yx!timqCVbWtl|@+VjnR-5Sfh6Po+d~`uWI`taxnK6bIlK zhZN^LpLY#ek!{J`OsiuRSi^FNc(zAkjeG>gaXR`Mg^5}?(pA<_SLeuY(7Ve~N!}Wm zu?SHsbgQznJuqTcUbf?5Zyf*ssC)0IrqX@?-*b+Rq5?Khx=K@NO79$pK|lnQ4gn${ zHS`W4FvkKTElQQH(u5E~2%&{Cf)D`#DM{!^2?(JDNPs}_w*%wMoO|#0u66HP_x$c< zvB=IJS!?In4|_k)=ktEQ-gQS)kz)&jH}O*S6^8Mo`rD4R?GLe+4@SpEZUERB&krU< zjAoDqFc)Hou|Msb@N`H{i+%{)9k$s!=Uw1Cb5YXi+CY$e{>?<|Po9C^hF7xY9=}AO z-H~|)xb3Z+j2XY6A*&&~&sL*uiW;6YtVqf%0-ZZo1CA%67y4~z%1ovrm#RgN4c3De z9UrEfYAw+UsFTh1A`LHwhsif85NLy8$6&gbiJQm-y&0SU((?ix%KrwDZjZLA`RL}2 zsYxxnTH`UPkLrkjHJbQgcgRX7 z4068BBijSv8Lz#fik06=#>V~!t%4-Y*i8$A3V*|3qYe*-u-*sF-U|=#N~cAtBa)Bv zh|F{X>ll3}L2XOba+T9kiad~^*j zsK8x#x9SEl{|+|Xti&?9P&MdEBGx7qq8qP3u|RPsv!bV4PrQkiyqS1y4S)I;o<$)xOCZWJ%FwX2h zF4P%03a3v3G`QbP&_G)8zlwa67?5aaHC#`-LYuK=4+fkWvt;$n1r_i<^=@ z?$xJJmEgBMNzS=KAW$i0wf3Swi<*G+B8@MH0JPXOQv(pqxVs6;Q@sU^2fcaRuyw1l zFwmPXAzpXs)USAQM>D1>?aSwYeckL-M&HPad_7C$pW`*<1MkO0&L%q$B^}xuD%Nc- z0ZskcWRCh&090jy0Hl?$;NdO+psvNHzt zn`l@pD0dowg%jvJ$K(L*#{o>tal}y4gas4B=^)aKQgo)zT1Et70cZJ&0AG?7%UO;O ziZ;g_I?Dt9Gb)#g6B>kzhyqxf%|@#0-CEa3EZ*!nSE0Mv>6#7xd*68pY9JObAxu}G z;YyENaSZ>Lq(4}jM)>@>mURheoLPO)4{(5ZN1^A(NJhsklSLUJe zz}i$Bb@dnKcmld|dnDJ`Kih9Uiumma4|O?I>sGYD)7-gcr}p6U+xy|AdGVM;k|Ae0 zFH@b5J%(IUw>^)|s{vWBBkZ+|W|Iv65^kpc$3OUyBe%86?7X><$v=TUJJYR-FRv2n z^f|9g*qce2GcAmcr9aQyxuyLp%&$^rR>?E21}wa+Vb>x#Bu3KMuL%lt7zy48;ujLg zZ3Ta%1uyNmQ0DUAq2JY%KWRE#W$9`n*?6Mf=Qu0Z_2`8v?{^F94UB{gtTc;o;uC- zCFbSnM z>cjIuZ!p*2KrGBP*V;MVzA?&gQ}fRYwEyc7M#T%FfC(3}`RKre8w&Yu!mWh;CBB~gR0=|<*2L0O+zkDDs1-QNBBxDMbkaMhk zZ*%(ldB9Fd-2r&G>k}N}jm-R5YOdiTLksQ4RNMkK%Jz9=8lhScg_&aQd+{;76u_<- zrhY4;>C$$tpj!v@J>=TAeU2r2JzKZ$jh&g*nEaS` zc51gQvNx3#e=NjC&SL0IM2a_RU&DtUNkzY)kk4%*k8 z2kI%LHDgOwz-;;w0SK6-?4v%)vjS$jR-xy+4gzLnET?ua0s*sDJK>Q4yx3{z@uxeb zl@WiwD1Yxo2LY(#upXS1FKaE4ZAghJ5wn(S$2wW|H%?^)DV=^kD;@D9Y`mt{wMPqy@vsZj<^4>QW^+y7ih@~4(6%9v_L5e)%!>=p z5uP0g!XVHdC*CxJ3ltX(h#nbTK%XjNAIy_aNl{D*J9KaU@lCV>A3aJLONo$|hc1LY z59t=Z`Zk{z5R8e7Fk8H=@OZRGW_fN?_<~uMyxXv-t%`4{+uK|Z*F<$D*A1Iy4=k@j znH}zT;s^-)pAtvFx#RUIH-?)dyTi@gEf1P5tW~Up?qSo#>R=ViYPzfet5~4v0<2-OHG(8QNtM=s-P0R7!YKAqgp~4OagnoRqla$?Z0>M3 zy?;a=cUd2tN|Nv9T+-^_RFx4IUnwsJSB#REED2P;+{8&sWDKf@O;IiJrE7q7)sS^- zxm^(LW)eZbD}stsG+ib?VdoCm+E~o=+JeW?^L;8v3slJ zTZ|!4D6Rv7S$r9Ii~Y_X9+Q|$@>Jto#=lp_NcvdpNIeOzopibHIO4eK0h|nwCv(6+ zwhl>_c>&iMNH0%vOfD`h>Jyg{s5NZ5{sogdI3pU+;(faw+o3etq!X@6wVPa~gAgyx zTJ#p%Anw5sqA>Dlf3G6fT4n?_hRrqep3?&S9ywV$CxFt(57jqKaFdVlf0`zmkKbf; zi>i3d^QVzZN(+zuCkfeP<10(78N%}Tnu+8tH=f?7)pDbDcM4o{K&zr@QUE zDjX4x^Bj5ChO>joWVNPK>O_@zt--0M^SXQ~09i=Q@4>fX{-4Oq` zYlx$Z2~L=KvEG8!n`v#2HeTvfrNpO6&)Gh#_t(=5Ns2PFVUvf=%_ePp> zTp|#`A0mt`f1KOpkfPvEAxd!X9AobZW(>DTSRey4H(zxmyjwCv0c!Qf8mrE=pX`Da z6_sk43C%*oL73D!kwQ_s)VR##_}(1SrW=|u z!f|X9KGtI{qNpOxwJnxuLtEO!R8c)c_L$);`T=!T#}(>@XkAxqLoSI zSN}j;&pFuR!qRLj-s6S28 z$O?gi!>^YagxFz5?eo|mnHSW2z z??F$Jt{~FREL`)vX22f$ZYd}{R_RMLM~

r}c7|ld+zm^7nc1+pDvNLJ?|oj(NU3 z=#w#^^@jBb;QW#jrXUr_K^>J-qDi;qZCTRr7w}W47HI)o@KnlA+SjjlT>FS`4Ef>9 z7`!$+G4ImM1t#(tssUptV>^>ymM<9UZv2K^mn*U6`e^yf42-M0|pNDcNrE04}`PGf&R`2jVI~>`n!2taCjyoZTY=ZZb6`> zT~NIu$ABy$p*7@T+rDf~{6<8XCwGA1=xYY|%S#lILtj+;lxqNQLZt(4XTrkoESp-Y ziJCD^ZgZzy&CU{B)5s2+)NF4|Ojlz1qjT&cE%Rg5F359a^oozJtHVZG zDtnBbhG)&H<)GHJmBPm8w@-=ku;$J-@2c&6Wm3xop7)%AD0J`fn1aa4EUP9gq8XO1HK`_FFx(P?r#SCpSFQ# z;l~35e%4bb6TpDqYbO?`FkQaAQu#K$#~xeM%k@a_3!wnw1}Pf2@nV&_nr-;RWlQZL3|tzD?;YreQ=>0RBB7Or_J0u`l%3>jYEWk_AH z`dqlD$So2Jv?P<0+V(3?lEinyL^sS9v^6uLXgQrxQe!Y<9u#z_7{VBF!V`tqn{cO9JKeog_Wo(S|7CCfa&*5Ud<$RcK3+yB zo1T<7u7+~8-_<00E3NPDNv7Uu`q2};HZS(=h@5RI-CnO`{hq}BEJP&ohIt6jDh=J< z05v-M=}Me~cpYlU5BYRo3-FB0csf-Lw2|96a@n7KJJPu-N9~2kcr?|JElx1ilJwsS z{luTuQ994B>W-x{pzI-Hr1!JiUS9h~)i%ga6Ov^?8x){L_`J_y_HnsjVNPB<;kibq zAN~~iBKTlj{^A$eI6mf?aPggwOtICli)8&`{uQSTc#l%r3NpJz?#@W&bL}rF5b;lC z9d|2|<2{~uUVW6M7T=Mt!|O6ERli)mZ}ebvh3liFr^K@KvT}P`Pe08Y8;)0;N1nJj zHj{P%-BQMVd1T7JvHk^~TYgzb!Kk7&MMWo?4|v4k+n_NAkGKSL0n2NLkGLk*)3M^W zN8GZpJIkM}`$Jn=r#!-70?WtMtjuntBLgppy*J;&(@w5#xh!7Rr&dZr5cbv)6OLy9N><| zOlL;fCpI;tE%ySdF{0QSDOz9szK&#nMC4iBTo_=Ld{aUIPk03Zf-ym2uU|!BDam5J z^*AQV!9_LY%Z7E*0l%3)(wfZTH>*5q<;ZaEzZznUNHDsEt3|Tifw{cW-RT|bTe(KO_xN$7C#P;dLp^(3}Wns4-fbj>y`ez`i>v8m=^*RzkQA_!;Zc?0Vg z#7F zzji?FUUXAYVi2X9WYGSoePzBZ?S@sSN4jGAji5n_C&VkLxc}Ze(rl2-Ov1;97?5D< zssSg+ph681q-<^XrV?pEENyZYkLRo7ur?df*KH@CA^s?v#r5Q5f%cLx!8bN>dMcyV zFKhU?w3+!}968&>+iol|q-VBQ6Vw^J`w9rXZpz##qdhTZFC~QcM|T?vs6*TrtlF#n zr&0*l-iy>3yIA}8KrBSk*8Dg|)})UKUh)&XUfkdPENn4oNE=kJo3UtJ)4xPOu6jGW z1_U=*_{OO_)J29)x1a3ONjyV9@mqyX+x>btA%6eM#=U4aYTPka*cfeS9MRzpl zi~rIle#n**nLj$E?P^FsNfjX<{u6XWyGtYlLgt6sr0X z^Y1pPxJvG58&#d{@`fsj;?*q545f8cL!^Hn@PmU`V@p0JWCz!iP!Mb73?xBeB+}wG}Zl2a?h}_q5;!T^~(T5a5f~JyhsC3!S!nmk_ zw1o4;I*$Tjsc&Y_{_YSsi#JKXY;1+jb@NT?J<%@YyE4$==UIjI8WXMfAk~o)M>lHY zQwxa?E^&u#?PE0zcR=421petz-&Mc(!$fJ)$r|%^Sd<@{Ck|yjO1^$qFJCFc0&ZA5eEnnh0y{M8>HFe>(zHm(b09KI-gu3m_(&qz$pd3J-4OrmS06Y%V%7Ov#IA#t%w01ig$X2VD`4Ez}(mF!da+_fYtj4^b z-h_pU&S;{NyJfH&%n(3uBe4hT1+}K5mW=;o<32u>9G{St@FF?`?j*x0-8S~9%q;89 z>dd1VaLoh6*PS1vlg$xC8Jkjss~3L(B1DStfGfEhsQl#Rj^C|W9L^8NLaWM2nT4If zQ;FT6AWwXXqJ%X{QKeNQY^qneDB4yo(Q2dXG_{Mrev`6kR^$Dwo&;F`?$RXAj28PB z^jp-JjJw~;Y|JEH={nfsI7rIrPF9!lJ4iUbbU8u zOIU)0IpmQVQD{*E-!MdnLRf?16+pCot_whuWSh9g*!oi%d2^8La|ml3!FU;>#LEEL z1?64%43$^p%C9ksVt+M-=FDv9>zK~HRs=|g($wV^(lj)A)-?XjY(HIG8kjgVasKYF zi=~p}whge)?8S#kQVtGGdXkJy1TD!hSYe3s#T_x-6U~_^4(sJ3b1iC}uq3YyN^R6% z=hPYW@D(9Eov-W69xOv*ieKJs=L%L`>*WY2sR?7ZG-X6o(AW)PcD(6q1S=%gwxtVw zs{3~2^fL3bqB{NsWz&3aC%kXCM;|po?Y3ZR7AP$dNXYFX_AJ|$hhJD?_;x2`+4C}G z)A{}qWI6!1bE`RP35%Ud>={A#zAc){^B>+&&EeKB^BF0!Jk04a=xr)IlbTvc5sR!Ce|9#xxkt3{>)Jc&ouf^DN?{2jV zV#{+!hw2Q)`Sm44Loj;UcV91iX3r7sNR3dwIt%yfWOAJIoikRFwkyJzWw-(+!`-E2Rzg$wt|{KLJ! zr=-%%Y@VjQ$C7Zj2gd`F-R4nBAj{?bk`PPmUy_G(xU#YeC8p%JofZ9V%fG@8NZ0C! z*{M2^@Uv?5*^t`}vmrBRDM78`;g#>vYv$KihZ3vL8yr;`>bWG>&M%sKX+61ILoQ7{ zoYu=%%vbsT&wZzVGx7gN-=5YOx~rY=@-3al6MvphvMt>Xluvolf9Vd8Ahr^q)?n&{yqM}4_gwq#jh=IDa_DlG?mSyuojyC-vWS@;kTIF6Ui_$8K>xUFYND<1JM! ze^7v_Y6eNbL}CmF9Dd*{^KVI18Od#6WUpA(m|P-vF6+CL5P8s9*D%Gme?cCQZH}C` z`$`P~f|HkttKwiFIGHzL0%vWiau}R!^TUGb0LRV4;N&;DA6+vOSi#9WEuk8a6GiuA zV)~@_)ujR^WHejw*2SozNs5AX5H`owsYG-|fYBeT8a-iu&bJ!u7a9OW4qDLe7E9p4y`26yXkmq=V?_7Cl5PbMm~)y{Je+6t!` zUXgHXPPmNCvxqAj-2$OEpdl{%qkQa7v5Ta{p7zaZ8PM8PPS$zXf!W@D!_FOX*^zD> zSq&Mz;i{A@>d)Bx-R0kU!eKP?fP$a(I)LJTTI_>-*_>^KdL>SCe-iY%gVXRpeLJ3Gxju;Pm_g zmYbRaKAq`o8GjmyG-UGGa@2>eA-RbV37kQc=jPTp#4A=V5J6lj$C!#is>BwM zxmQs1W+*^Nqf?U4J`#5ucCb5 z_s1Q4_^jJZ2u8{GldsFx2!t(Pf;e3{IDgi6_*eg!LP9B%eRrmZN2U}p304CrZe>02 zr(br@8%uaS8jG|_htew(b?`lw{_z>!%zW-LkrAyQ?`TL`)%$Eut!mAHivuF9+o*94 z>@Y!qzV~AcS(rMJXUSd{bv`(%ipw{Bowo-oqvOK6M#8fHvP5hx49oY9@u^c}7Hk z_cy49lK0B>oElPhU5?W+&JZWJ0%P)(Y!4}N*2iBpQc>^47IiuNJsCzDY^HbRPLQ+o z^*S8;d72=_BDjme=NCev#NZ8M-qN^}3-7U3nTvHwoDIefj1ei+Orl13o8#o= zH$^SQcZb_N2<0#O;MZ?qMZ*xy1T7&rblBTa#xN)$VMWX7!AFHg2`6v;>-v_?$24;B z*F4Ar1=vMD%5K>*8N%jaUP6SdI|-T#qmSjYdp`PJyg&Lw`1^qff#l-giyhGB%xfD} zlfSI(_a0vtp`VWuo!_zE(UM@?p+;!2H9uLK*h<-$)hJTwqoMO@JH{twn@GLuBka-e zLxe%K@+3f!R}6W1{7>u}w*VBOj1yfd z*aq155Z7j=H-XoUFy?a@9EzWMLf9FKd7CWrPjhgh2g``^F8@{8K~6$&*8(D5W*L#n z=;rJ!7xgt!c=}XH!6q)_l64#(PZjx9ovUM;6(jRyiGd^$wp12=6TT!4(actv^qWP7 zlOg=NvBvfWm;9xWf^{7M{G~*UZh4-8RO!Xw)YTdPh6+!HE66a_R@#-L9hzG4#u2w= zxoG;2V4|A*^+R#2B!;(=pLw&cr4u++;;b=(@+_xEaij3* z$k6rSUI+UK#@1kh1TlD(i*(;7d${n@29I1Arg_?a_&VQ*vr5rSEet0d5b*j z+h4s6mF9=!ByY! z)V#z5l?ZS!Di*(fPMni}XHd~TUi#8%A&dEG)ly@n;e|t|BG>fb0J?viL*pHin3$Q< zguPmlllyXBDvRXloOs^#!YF8KwlI}Kg;W+>jA0o|9XH3$1KkjmV92U0gRVW~ca;_gS zUZg1S?q(S#1HuIhb?3nU zUW0GEtR1I$8-154S`zq}I^}HH|_)?Lj)K=Js0Ls?RdQnNhr?-Wed+F!F9Y$nb-` z?Q6>X(EU9NxE4P*ng&^FvUtrR*aT`(qLM>twn8!5icQ7=4(Qp@6Y6;yNaJ=3ym?Vp zTS8(kdt~eS=$Lvkp?4HhwoO0_75l4*Kl~gW^7B9a8~zL5u(%CM>&+1I+`d5%Q~1F` zy3EB^Y7^i4NAvc3K81$Bcuo1lq4Y83AsTGUXT)z0pSj0JGB7CpaAe7ZiRj zj=a(S^@pz&-vv~Ey{TbU!Xc^Q_pJKnW5*q0(@;gyIZUhrZ*~g1QLB&?C?ZSO2y<6k zQI=k_8NGh?r7bF}hJ8lFUcS3b(P(IJvdrsfmgdm?F+ePVR^pr8lpGs!?k`uwj(DX) zGI5t$<%b_vj^swLktaWM@5NJo$x%tbH&JTRF9r$TS)mT z&zJk4jzrzpLZPe1yd(pWcM`l`(URO%XVSvR!0$naRs`)m{`@M!sLT{pBd_o#%~n0~ z_DA_KOGEUmdW;j;yog}g3CPabYR#AGkV9 zU>I&5uzP(f%MBx({*x9|Td}7FHl6n&f=f5z3#P3sX8EPtXSOt>uHiI zSlp=mlv4{U94}>9OpPh>2dno>MJdl(Z$D~R3lix{^oR`fe>|?rz{z%i!?w_lJyuIK z3aSz4Cw7i$#Si=&`iDM6Qka$-ny!lBE9a$KA5%Z6_mM@_{AM~7za2TX-D|R(gLKPB z=kHn~LHyWEP)?*uFB%!*Ra@wI3DNP~gP{J$aQ}Yfz6zsX{5oIC=f~i9-Jeo*yAb6# zY={J$SMxPUNk+WtAy>F1IyS|s*B-=ol3~DSnBX?uX?!t1r|wH`A>=Gs8Kz9l8&T_O zLFjX|s%9ocM`X@2w)gGxC>8oxl3}ndVkow$sePphVdtXn7uU7wV07vX2y1iCBgtvz z(ZRX1YPAc^sf;&gmDcS;7I>)vb)ii{32PIlh$CAwu+{7MkI?3qn&40LSIj(#FQIkY znviawRNJ>Nj;TWJ+T)jYnJYj>Bumyv^^Mjj7Tm()`*G2hr<@1l3$v)38XtvQCbgyqmOO(p!x{Fs5rgE^hQ%2HeQ8R~=q0~}L^Ft0uSA1C z&|#eC!l9>3iIaaqU3y?zLNy06#ugr~m9t!b9kd{KbNvYL~KD9+1= zR`cj*8Wvkczgf+zxtLNeod;I)yM&@>z-q1#I-O}4Co*gQ+m$65jX@^+;WjlI3W^x` zq-uUgcS}35Z@s9{GiB*zoyAoixwe$Yy-hZ4f8pVsdUB^uE3&VDX@=mj6TI>=9@^X9 z?7i%&(l^81()K)%8}%81fOm(_4d{~C=ZaF!1U?4L?%lcdhh}%~D~D!xVLyNnz=zs?I#L@;04LZ4>hH>^Dv~bBP^cc|Qr~76$__vDw z6mRGC^B>dlzeXAU<8!|so!O!s_g@RD&H;KjfGxSfXpBWu18m7Jlz(K=)aDOt$pB4_ z$g(ACO7y$`c{7y##`5+U1i(#N-_679MC^w$qjd38 zdBZ!T2BDvLWt>t25&`Rj)>%AlM8@DfKwonlB3H0>BAPPBv?$Ier1YV!hFmud@Uve| z$-OB!IK#ir)~ng?$I7puNhcMacrbEw%YQVC1nCYv3ws2)G`XRUlc`Jb@6q&cOUdqv zSsB7ci7hkMW!W5e2H<$#(Br&JwWofXGATCOo^G4G^%uV#xhRwvzpLk=vjeGsefcvb zqphS&--SiV5I(j0q@&q?UcDw|MzO_Xx94^huvRO}HCXc~s*#z#ZcIl*PBaUZEttfc zzd?<(6A?eV^FtuBHFiYI8PC`$Udal~) z8dt_v7`-E^Y2E_5un9_@7RD+UY}@g7W>=isPh97HGMBo#79D74bh(|Xfqk)1b6w9QNZG&w31l)RUs^{9{NkzGkH4I(Gm$HI z(JlcNBzPO_gQwehl1-FX@DPyW9L5fCQXWV)+1aJJZXvQQmO%^bC)Shs{4I0KUG%YE z4u;BV9^f2Jw~}}O&Jj^22eKS@z&Y|O>G=EL(k<#C=jdpGoRbVB>X37^gY%tWagIh> z9s3gVVEn7I!}ON@%=VVX=e3jK%{IBsf>=Ln`G6jQ(#S&MGZcf7Dazp@D zQH>2-eXDMP!uliykfoHZq_Pvdf&fO6y$lKy;E<-cjzEu5r0fV*?L0+YszXDCMM(S% z#w5>*8m;1ClgsbYNQ9n?1c~Z?ztsPmodk#H_+Jc$`2NNIc>O<~w8awlUTOLWhuI#oaQs=u^{ApAHS$eHZ(_LTIv9mG-v7@4hsO}L{z|^VdR2dd|n*ie8 z*YK2(5Pbs&K9llPwq z9;+B!I6Z6_Z$0);;@&?l#lPl=6;`3NnE-@UH15bm?``Q;n?{0LcQ|AB`u@6qeio?N z!gIdtOjyRuVb)@6R;`eYc`5lNL}v`1(S1ST9j zw7#vr78WU}B_Ab|t8x7N#6Akhst}{WR!Lz$WC8vQCOCk~$2-btIlMMR13u6?Pn)Mk z?B()AYMn0mBdVf>p=sM7kX(yhZQljTVr!N%=^L5{Wij78K49T^P!?Nn=v82q#d>3b zvKUdx^=n=jWj)uYy%C&8uHt3R1NIiiw<9P||2HgTNJG>7(mq%#R}0)n-FBr%1_7;C zfCUObHCQZA7F6Sa1~mQ51<+V3zQ5%HN-PHEvFnKx-YtXuZ1781#Zk7K=F}v zV?HQ89^RM_i;uFu-Ix!Hj|Vqq5UcnI+?Y=livXc-;xwJt$+{EX`Z9(v7)*)@Gx~PK zuD|9k8t<=W^Y!{tHIKHXbfY8M=f(Z8c^_oq!DHC?4HNG)LpD<>yPUBodv01`eYl+ z{{H(M<7-@KW6UiyZ^Z1CHYCpv9N*}D^6f}QRQ}Fio0%q2SHnmJiD3rO@;UZb+MRoU%Tyw|h6Ap zWH&_MXkAixl>(cY$eMNKq9pzjv;mRQ59AW^>np)Q;wo7EE~Q}j!YKnw_wgkktEBex zeoeP<;}6>TXWCV~jB^ClvV$3}7c%t0g;LB8zZU*4$%XGIu0;EaDarHZQDqwAjh3#J zZ|qffT06q-B-*c*8(xvD6RAos? z*>{8;>*W8%KgVvX8SD9#;{@s#J>K6!DmbG(`_K6Ly}DD#CuEA&lXz5AEJzeqpBcJk zQ6h7Cs=!hGN;W9SG||5=xd`|IogBZVmFtcnkqFwy#Ycuol*K^AGn)~woNRig zIz-z*5@CVL&0XfXQWB%LULqJuOuHl&tLHSD*z11+2rmugrm!a@R1HvJhF;Mv6BZja zt^o){GyFpx45c--v?n!@>}>SVz>eD02ztg0<8sf- z(^Hu5SJrt0KbJ{F!58D#r9>-v-fNNvh>D9kv7d6!-EIFeaqGi9nl#%)#C*2e@y-$% z*b8{;(NTSan!CoSa^b|Y>Q=76EspqDUCNiWg=Dq854VtqZSUWggV$)+6 zkXCqIu4I7!Vdg%}v*o!q>C3^nv;O>7Q^kM(;NijFK6s9%f1L+;3Ou7W%k7+ld)7th zW>5jl)Q*Ag}ZZQ6>RS)yasRO~lsiBcV5GKkg4#@Ay$8ZuhQ zl}K?rfd7mD@?-&9CTeOY7&pEK1$>-z1?)cH z`#kV*X1!m!eGt1D(G=Rh-7>d!aDHrao1yRYWeZkD7ZE_vMVH6B!AsSKmi!%}qI$oi zWu)4Ts3^WK^|VzOy4is*jcXgbF;$hz0g^Z5E^P#0AaCuY-pQ+Py>u0gtg6m}DW)}8 z&5MJqY(AUd!dVmu=@&EhR$-$25Z9;c`q?i*3zGL|_*Htdza6nvt506tKF$w-CuBUT zc9E=X2LL=#@=?0~jgfM|g0!i#df2|xHr2@ zf5>!l6DEdeM$a`hwS|U>UwY#**D9S>$y(8SjuD2wujq?WuUX_$r}j(a5|_|$LueUM zLHKL$3w_M87gH)H0earA{6PiM7@TN+s|MIfEcTC(zV9UJFds#b-t+phnR-aVW6wAF z{U@nEn-OYRqr_QfRcUBimP&znt+JGKFBSE8I0X)cbYaRhO~RW32SU2X`)b4{4jp#jhp?aou|k{Rmrib@xkMJp4kxkLcP-!!JX1rY7A#1mXhtkUt%3|$i`{Q!~)~*$K0A+g- z5FVZib^{|FQx`47O8j$bA0fR9>-(MAV~LRN#Phcu*(!RjzeztW5_?a!Ro$;kuu(N~ z=4-^4!wJV?P8IFD0nDk%j!>=qtTtKl^me52r=4C~%4EV;L(#V*Ld%iRohZ2x*w&sU z0DzwlE299=Dfs$$4lZg0sQy_^eyz!sIi5a27f(+JZ!(sTtZAhSh4ikW*Eg9y9Gdvt zQvh|UaZfkQ9pI!n(G#_r`obFaSse*!!{yLDSpdIAVu7TAjs$>R3*T>`v8Yp=eyAYi zE@YT@P@(Ov|BPeM(mZu7-YjUU%5L{sZ%TT|wqrX7J8psdnpSRxe+#! znQy<6q+?m4<^v^kjrbGT#R)9C8nb0~GHx9%9q%;C0NwX>C&LC6^2dJlKi~q^Yv>Npsn3_tcRg2F3~oZwkx~J zugt9;Qz8 zKa6U+F|1%yz=~=T22zRG&Hbr`o(%N3If3r#cjrT}Tvn3TCa;Z(_R}z3u6K&J)+J|a zNGeX9s!Vq`X`MvhDfJvLeGmsaFh<2S`V&mVU1}LRP~2>PcU!x?YVQdUU5-iS5iQZI zwpV!?r0Q_tRfkjI%55XtiY#T}1rrJD&GhRPv1|xI=kqoP=YOoIQ7s4$^g?(kozdIw zU6Y+lVUr6&FLuD)oc?wBk3fGrvItGt@0+lH zFx9vU*JWrm78I70H9>$W+)#3C>;)RGXP1yI4d_}!c+@R&ocoky+UZ9(kn`D z5_&$lBp;dn1i)*ymbU|fexTP%4UZ}VTDNQ(aA?Qe6XLcNQ0nEI1OugBxy3!82a2bS zPKNZdkd<~rWE2hN!@-iBz?Jfb;z*kjkvvRPPUs)S-Vr;Ix=83^bbQwKWbB&TvU`(E z#yqq+KOGgc+5w%A(8A0`Nk%lsEW1+oLjW)bwSmMZK4dt*q>;1C1bRFJw9PYL_w1vb z<{2M0s3-t6@?>T;QXD{y$kH0&pl+wO`Q(ec%(u*thQ<~EIXRF1C^oUvAP8j0{&IPC z0tN48Wym%c&hN}DhwlRb%XCtl`|}{qpAV+kky|cQQgb0p4TEj*dN{6G+qdNVqB-O2 z)|BAZHehfm|ftu;L+*Jf3|9pPmH3#=ZQc$=Lv zA9=pJ7qqSmCB^gUml|wmVVJM6*Vq*s&C7SOBNA;z6X*_B0Vm`e9NG00+GCAuN+X=Fpv`UY}v%(nxY?95D zLc1?{cN&~5MU?93EBNbKBve|sh6}fReJE1jB6L;T$$={umrxk$*>$v@q7X2X6x5k_ zZ)Gr(&$>T)FklkN@c^6XA<*1Mntrj%eXA8()zh>ZAf>lVN_tA5trHfC2v(U@rfro- z2#3DVJ@25F)LNI4XTycwHCD)hH`W%5Twhg@!D=73w|U15PAFRlI$k?o_VJdl@4T2ERaZjsQFBT^PxA)5)Vvz zlIzgS<%qVUAQ9S+2%)Z7hdCHKJ!+A#F=b!8M*sGQ(@y0&WpoQXLA<8x(+) zv4$k)ndocNt;;~x#=wf;lU|*4MDgOdK648|PE|m!y=%4+>RjgBVn3dI}$QpvhPM8-H)7_9JHB@8qXtx{l6Un zl3--_vxlY6c_nlHdib$F|2=w*(KL*k(pvMN@aahHXwh*>EcEeFcihv*aDJt6`zxF%RLn zp=)6GcE}KJ>yNW(3LT#{M+7aTn0t3TYszp^AG}4e*@JQVZEw`{?%ZQg1jHUUrNbBL z!%jXY>vX*ha02vt{(9#GKE<1z>HHX1!CC|Fl(0cc;(hHX&9sMuz4WIC-{4g71CO)y zjAWxX+R{6pG_E4-|HxK*z<%Ww>M|>AymZH+Ea|}rGNZ~nKR%_K`=fZzbqAb!l!0Z! z*FZpYpC(%PdJu9Tx@SeJm8Tqv?jQ3nxt_kn65XE;({zrlXj0{Y*xnrkUq~6gRp)Y_r|sN=ZudB3Aywp@ zbtZei_ne$!kKH>=L5AD6BYj4TWqWe3sAIzD{uNa6`+Bsrm2Q7|Pza-R&>hbK6Qw9x zT+K?D0H8AO`9pQ3jL-&(CWcRWP&8aY&2jmu2Dpj@Egw5gmrtF>tEi5)ObU%!H~Z^X zP`Gch_Qg`C|INNwn%rmdr`@faOi8LYcfdQP1(iGYri+2(a~&Y?$KV*ep=xdG;6>=N zlHCP+XzHlhW>g|m$?(?3Uc;HT4GSX>NvpKcP9VlPj0@)*R^~?Mhq@z@yW}I03pIgN zmBwqa_ipBq%SM+Vp=D0;8X9oLTbt<9=h_}=CRf+cm@6}>*8$^k18H5#f~a*t7HW2- zZ(vUC4?42XCy*Egh)w3j~D_%=#bFy-GQ0!Jm*{Q`~C4g-+Inkdu7M4 zVv^r)@4H<0eP7p|Yb&xAo{}(5G9p$6H)ZJ+T5wE zpkC#$jU1ItxKyBTG3B+sdC4{Xbm;yNW&2MtPb#{=AShAg_XN$4WA)5m{zJo&NXc>3 zO6-gHy*o+k0j&Ox)``pzA=3oL*>j*}ei|a^zgb-XLWF{^9UE=pKd(6}#MSI1W3~YS zY!VsUwz;dBT0pyM0yKCDW?xBd-z_l0Z}k%yngy(>oKGLdbxx8=QHoPCiIuBSe6fC< z05jL2wpTuz{7Qs|{9(Ib|3Sx<1T7MZ7E!qu^DFY5Zw}`lC)@HLdH8X5oEC`TnPy7a zY${G#@SRB6!0`Czxlc}|Y+$JDy+;nROevc%ub5*RbO_9M1BajV`L?DQ#T?p$*kVXi z%%MGxXJdsmsMDc6)?cFu^UR^WZlFe!Yvx?=D0uOk4-B&%<>0Yqsa;t=vMk{TB}PBS z)0{}-@M+aUW@<6Rdle_16keQQlG`#NpN{A60$8#YJIR0?l;nG z=Cw?~n0*Q#a2$YKrRK_$^7+;?x!>VcsHqwuSn1mSQpe8tJ@Vg~duK>4f=+)>+HM=l z%5xRG5=JuW6=hjd8MU|Z%ed6k{Wk4|lze&$N@JG9*xi=4!%k%hrJY_Xyj9f}8zMr{Sn_-BEcB>p7 zXuaB(!Y7IX9ivqa0YPtEGfO1xM9`bxcMY3zD(HO>?(}iyn)`{MH~KKLLum!K2`AeT zqy_}6>yx?@L9!ky|GM%N5=X=6_ofQwxArO@+|(}rerC>71*pBn*#yy*UuQgQ96}v5iKsLT>*Jk!wCw znni&KVacG?pdRNp5|s4%7BzIY+kSmA$G>9J*t5TxooiXoD6Vhkpdtv|rLv-D?TyAn zw!s1uY-hWAuHqBqoo{UEzsS3{7+>c)QW6v9`fO2Vqnh)B3UyDaKPq)2<~(8!Esrp) znolm+{kIMb*nXcM@_MW?-edNN82q-N<<=Z^}-YCKpA*l@K= zcfu~b48EjDUmpG(^qZ7(q<@B68n}7dsr)%#g|n=KpPtK1MdD3VM$4;1*Gl4m(RKZl zVu`iI< z$#0IGu#B3$D;NIQe7&ayi3qF7!IN6MCl8KM&C3C^L%VIqKuQ=c4$wQMVVNl*ruECN z>+=3FCc85W&!f+Bn0=Hf`cnRnhjHzz@a}~4M+N`~KMeaCntcYzSPEpqFcX9{`s{R}KFBc}iRnbO1l z{Sdb%gT_bN|N6YYp0NmxgHrcJK1V^EMdkNm%{ujS4$*fu$WmYJso;=H`rpr7?R$|H zCZN}0cn!6CAo>om)4mwzFQe>J`p1q*dhDCYYR|sF_T92u%DbUV=7@obDa2QM$} z1e)WhS%fmIOz-f~&*s>^xz>&H7vq$hu*zgW_&k=gC!}ysEk{-d1$-W4Wb2!E|A=$1 z_HT>h@8z_y;TlrOtLPGhrki7~?EtN@rgzu9*E+g@@bHImBJFJ%UHxVQwMQMyb|| zaA#jh4f@7VJRz1)SN)tQc}gt7vDKN<_7!(REJ53)zwiO4QTUC~NK%4EfTzrFVA-E@ z{@k*^7XE+qe0evr1wwu{Ww++q2suBLatxopM*c@?t)j5o+rIB-9vb1Jd|&A;kR8w# zLhr~Gd$hxB9Z$ZNM`Jw?BDibmZvhIlfnEIVB$;T%J%Oio*CRJ}*X$;OCgVO#j~zb2 zUOKiL=*UblQXR|3*+BQK(k@DkmQzQryn}=G*jW2^8|r<|c<>esHD2!6=TE-FG%6{lA~AkBsR^KE>j|$h@yU*M`_Q?VV&e zE#WOyz&eGYC}bZ8ltma;EUsA6icJmr-K|1I%@M)z3LjTLhdI^QDsI*BuZ@I-m=Gl* ztqEJ24$<;6-)iu(ulfZa)g*7(Imt#0XuXT5$6j*5tNM8(jre(oL3|U_nIrrM)Uj#2 zm$Ig4GBO4A)5I8_mF+-7jwWIT#i@2)YF(d=zw3W9k!35%{%%sqe36;cTfTd`b;Kpi z-}A2i%jVrlu&DcC)-0@W)`B*#~aCr#9&etjg z?RjS2cHN7nrm?8GXBlHLId(i+aqoAcYymE#g4aQL)jjwm?;{_cBZ5?z_&N~|xgHL| z0`T#zOm8jii-myP2ZfmPKv4Pbrx)YEk^7Nb3xvPRP*l8RPw$L~v znl`IEuGIx(td7fT0pFqk0P*+42x;uGK*5dCDV^F&zF(291cF+jx)qWK~zd> z&|x8(h&J5P&shFi8?2SIcQ!pLu{V4>hlgIa$#zFH_ckcw;oSIYXwKy~pLU4*F#-)^sn-Ir%f+E63k>70|CBI+lES=tdP*sv4^^ka0`iS zJWUjqAxp)d!RJnhB`X1Dv)E^^$Gu3h6p;z_R2a3cqvKgz^VfsCxSN!NpLp7Ps&PgN z0amnB1EI7MRp)%>!Q@&QG-NES_4qCio`MpxNx@@iRGO85 z%tsHDQGppM%U*u*1+-%GBf5~5i}|LP_(O)CmEm(ZpKxvqEC(Lin!Bpu|ugVzY6ee*%9J5YBO?LZ#bpn5cwj-IHnf^jbI z3L!qtTn5A=Kg{p|%*!1rn;Y00hy@0grEbx|czK7>*XmL5Sn=!>)f%K99S^}Q^JzcJ zo``kTmkVYqtuYZpWe-iug{~AiD6UEz7V2|uW!XX>$L#y2{pAUtT@pbKyxx4%Gf{ z&tnY+zCm{QmPkk+9-<4f<}LQ{x?XUD+H&2ZVnp*8P;19PG+1~~f5 zdn7z*8%HwL1|7pe6Zfo>0LFm;N#gELUk9VIPb`^Zo zGJE4q=whOi;LFveAmsV2S7i8yhVbJ8&(>!4N5Y6Hw4z06r;FMcNc!r6tI88RS72ti z>qq0J?LnIj8cD@6$nNf~?4^Pt8*$4YKv*{&qb>XEIY zf!F|#l7ZMtPEq#l(x>6Bsl0_|#dhov(?Yv;NsCIXS)STUoE_2-UD(4zue46&JCWIr ze~jhe(!UYTRM3WNs)fEkfUS@*9!CdCvNNqNkT6+rwvp1xWWAWVO})p5aZjeiwgxrZ zJ7mzn4KKabFI`6?p;omL2mi-R^8e<86g^u*$@rMZGl2!78;-2}gnoddel2V}n4a0S zOZNQe$%w{NQ-9skoI2JX+&Z+LQI7w5Yi%Z#Auzb}w?y>f{{AIQzS9LDsd5Qdy;Diq z(%IPQyuKK591%H#tDmj?x^8e6b|GqiEg~wah8}}=eX09yeeWnPapkHFMeEPRsFRr( z0ktw_VlutmDXZZgOmoc_MEF;CRb}nilF8)aoON<-|AA>4gU&e00eA50 zWBDeU3lqR*(-dgPpmqQ-qL9h2vLcIcDvf>kWM~wF!tk?-eC90H{o}8dHv4|YVGj*e2pVz4RAwV{|CcTN*S+0CfMYTufZ6yw)E&-jd=?x7bbpzR&nT14 z{iTz}XA(m*&J}2U{<3+fkgsfUbqBIF5XQ&(bkmp;Y-sP4 zJj`HlbZibNw1>bGs^cSYDS6ezLAkyq$kLwhqDY=F!n*`<+2VIh4aTg9Ua_higv={$ zbQy2FQ)!22*t7HwMO$JM+3bU!BIA@N4=pWy7^C!xN|Et8Bh{4ho@PfL?l4Gejg9mh z_nG6J)u`s?#TyRNKD!>(JrUm;Y!^}uL;-)^y@wy#BLkaGP6mBtu zUuM5|C4b7Ye@+9Be_cPZgOu$DnN2u|N#KUUZ|_s%$??MxHC!1Yge%WxnfDr%N zYh*arED-{=bJX$va(W3(UolD2jl%oU)x9r@kG!Opj&MduXFH2x!>!p>E~(lzEa#L> z4fmySSf^L?iRTb%QsPABwYXQ=M$%@J%SRULbp&s&DYGRFc69ICRdQk&E!bhw!*2#9 zB)jIaF>rRZ4fJUn-!s++T=d#}MPIv2H1|dhZ*1FV?NMMJClv;M(+_raa*BdDrV3UL z`&K{^hf#SMhEXxlhkld-gJlz3{gh{>;`+zw$XB+V=O0te#}WwVJLIPRx&!_r=O2uC z3HObh9I6#0GRB%;7ui)QUb2=;YO+)Okm2JnCr_miL&wvKZ7LQK6$+O9O7I&2h={83 zbE{7n^p-a4`WNBA)!!n`-?&o#OPS}ZN= zPBV=)C#WZDf3o{$YV*}D0;Mx16FjjT^{ zmM}(=TsumYa__89Y3ES)wAh`fhK*#LOV1A68vp*ewAQXc!Q3_vKE5fEUviF|1p)$n(Pb7awPbP^3x7Fag|cb)K}ksDK6B`cM9AtYsu>97XpEr&Q9s+~r>B?$;nwO(wpiOSL6FS9EJf9nu&(|Uo zz26LaFb%#Akz*JA2lFTVx6JjA4(f8_Nx;m$BzR>hI(;rH{o|{LZi`A=*4!nFrTJb$ z`QA-Tabux6hS9B+T!rUevn>Y7h+M?er>|c`RZcsqPQvxRZm1g2qAqpiP%N#bb>uz(f3gmhwX{+fW!obialaT})L+ye6WuaS>8{f|iS7l^nIGXi|b;$~~O-+?{*A4LHtX>IGO$}*$<)`Y8 zpQLM6DMl`R#+h4}q5ONSM3DE39JJS;46gLatjJiG8%pi0?7wtRXo77u^2ww3%c1uA z)=N>bB~cD3Z;CMJDcf=|&(QdoeHQ9iVb`pbA7p8ywA!!hWIkEIxBkyRPNz!&T@O;^ z-+f5jeSR^-4bDDA-|kFgbrWbgKn^vf86@c2a%)xnh2`t_+xLBM+v3{sN~RTBs$KEoKy{+qkxjTNjeCh+>0vO3Y@V4_^v{!TO-ZsLQkty_svlbp zsxv6K;^vxuq~sNPYd@t2#roy0r;}=Nt!vhb@W-EGWQg#K%Zs6^SP07gHq~*}DndZ% z$>IyfI^wy(J(?Nc`XVei<&0<3pEt#S=JdBqM-SCN3Pd0LxSt(v?)SOQ5+&pVCC$Px zD9&#}Bt7L=s38RCo)^6!N}+Y~a;jJ6p6V1Z_e=u%b@rY#?0cVJyK{4H@9ALj+qHJ`|L zavdGBIKGhRzhayH5caZ9DS|)=b4N?(0|ZQI#j@%nEQN1_tkskgxWoaZ7NXVKd=FP&U|S~qB4>@!){hF+Vu`{ZQc(m$J{#BS++C0`y8`p zoDG^M>fcV5iH%;y#6}4EFfu9}YHMZ!BRU9c8{^Uo$`ohB2V%(BM%i-za7U*zd$N@; z-kha!XgZ=e;rQ_c%?N@-9W5_&@~$>k5Lzth`&!T|GlQM4aNvt^q6*Nr$b{B313TG9 zC7wM90cqF%X`?>QU^c-oR^Dq_)b8ICB?oJ9>Q-AuD;VuT2o95-(YyzbOoFp zwth-3hY2p%qQ>+b2@%pXl%6BqM2X85#TFAQT`)IgjtHEtnlLsm9`j-Z?%ks|vTr5g zSzx{vI})HHG=pc)*Rq!1FNNFpGiDaA*64vP?faKpGgp1^Vg=jY@BNUxlSV6&dV|{`c6q=G+^=ccT`Zildb{M!^ij4v-v+DUh zJD~}O#rJ0l`gHmtOuRW^Ad7&Xf}Z=o3-@T@@+eGA$xYeh6>>vXQ?X6f{=5}wzui{7 zEiwKgc@^c;Ea<%_5a~b^U`C2D=Z`Ojz~QSeA}Avg!Z_(7I*dK~m3{-EPde>!_QU!v zu-$>((evqltms2`$*|oS-gdFr>aNlRHXYnT&ZRrw*7_1`UHRU)zKEUaAj54H!8Tvv zyz4kw?rw8ubJyy^>Ug(w=AHkeGChm_ing%UX%QE2lkEVlJs>a{ho4g4Pqt4zs6VV+ zbvn|XcN)nN?E=*~@BW*y`0+~}YZavfW_nWmfwgbw6t=yGu zIre5geaT|wqM~`u4G5*agtaeCdTqamtSU7v?aTy}X~|72wX*7QSFY|c6lwI*-xpI} zEYUXG`Rc4YJ26)tout{%tI%?GsUeZ-YTwVqTa@{#`AgZrKOSiW!l;J*R1F+G2wO%kxd)7 z4baR~Q>LOEPH1L&(3jb3PibcA@zTI?LNgy@q8-$-in;&Y`U9SdyJO??m6(PLIiw{o8&`fg^+YALZP_v$|<&2q9$w+4Q^C6t~% zm+s~|)A}sXUw_J2-{eJ5QY^8HG-F$mRUI@$G))Bt92+KVaX=ajIPe&ew4z{;!my5E zI!7Z%g+)Q?f;}yTv5+!N<$JeQj;drNak(tUXECzox*k>fpe^eV+5yfGoVX2zCK8Bc z37Kxy1jQJ^zMwG<4Gsa-k@?M)G=DAguZN{uL_Jpw08k7YvV?=onjH@%)+4zp|eXQRgp#LPl#b6X3L}#rd1}}@lhhOiJ!3@r=!cLj;yfC9DMp~aMH}r1~0fa59CcwdR)GgkGyo{_m(sO{{%j)ri3}S zzCB+O*KqUC#re@?A3vVFV-wIfxpX-?oi8it#RuycE~_Pp{EK(;FE#Cs_mYJ^ z(pzt_=Bhu3&2|}L5YG^KPhY1+Sx!6ZPsHlOX+{NT6ngeq-0%6ml)~0Rfe2`i>OC9~ zR6chSsqJ}LlL)gnF5j%)jC)l0UP9GJf(TfDl2mnSeJ65~*X5y*QQ=y84Q-Poo zFnYHcE#;z2qsI|$1Q=9m<*sS8nv%+ayIN$W zxE+H~EFKe$rppSs(a&~CO(GjvZ0oOo%}Gfi%uMW<{1#?yqD4*~uZ3GwL6P;NzJ*Ua z7Ocf043?;{Dh@yFaQm*y$4DibZM{`=V1B1TZL8stUGtr(K|=8A$m)}k&1}ME&q{12 zG$+P4ZG6Pf70cCL5z4Tzs2EV#?W+S$4fP+Ow$y&Pak8*V*^jH!f* zFRR>9o6BpSOK;ZaY+dq|Jo=i=9qB2`T6{@9RZ0zs^mR*}Bu#7pYD@bDe4hvSIK54P zNU69<%5us-cTx@Q+uiS5U<(z^*eo|&O)5^5ZBM&o6~|aB5UjD9j6{~rMZVhY{)=(SMO&V^ zBZ~_;8h10Lw=!q4r)ocUiVO`@jr)bRJsn{#vV5FvoUK4d>;YY$v3X{#)%RxuE z&+5xek)UqaJ=5Z-w}y^%TGAzBv!)Q+irCxWP5(pf1&u=~D#NTq_toa%6{i zVpXJ}jNErTTccxZo0+Q6Tqi(}i9)mEd84DCj%-(nDe&MHO?E0D3oSo41xv%CM+nWp z?F_~YRM1pQH4r_O$sm`XB3HFvnCaD1)dpYni5tm-d%``2y;ng?+eigdquNPY%Vd70 zU76VEs-Jb2v9Y1Q;e8Jeo$a;{tqZ}yb#?wz+K4nn?5w^?k%*w0skModW@tsVLu+tF zHD8mZUZ%OCe7|gn$8@Xpifw!3Kr+UE>_mJLF;EGs>N^#mv>nj=Z641QpWH&&$^+t) z{n9OTS_!+P#ak=D`&BZvy%i-!%2`gey`52Z=BkISy5?3|KF^ zC9~WDP{(SLV?C$sK3<2$dQR=Rjy8}>Z#Yh`uGy8$_)rw474N=>uD{n8T35nth2F9t7IGUKIU0F?YSx~PrEMCoY(CnFYCCd>F;U=)OMR~t{CMbdnno}) zTjx4zf8NPw<_D|O<0H+(fo#2`qombA-=$BL^B;d+&i|>?58Tt^-`E3ST>wGj#fNIu zRQTQ{abGTF=LHnr%*$)?t{skA+PfBD2FjgP8ceM=b@Yxt2& zRQ^zH@2(|lY+M@a7e#;Jq-}Hr#5p&8X>9kY8LyYgdwwkVB|#EeLUPTrtped@Q7m_-hIB^mm~qv% z7d|VllA&QNN9(Ml(-%w8tuc|ol`im&E`o{?@AB`6 zgzpJuss@OBCk>kQ60BEn% z7yfOtg-;szo8P$62TdSxd6DK$3iJ=@JqYVv+>DhA0jeo~rbg;&_dFAviYV_>z0czw=B17rlg`c?XWLn9Z7!wVXqQ^r1F8qyYoRutSr4hHB{q6{$U9jY z^980F+ z`03riE-vM(x(N~8vRr!RA!ZVHKEt_%#vxGf@5aCKo!&Q_m`FVnH;V-?(rYm#yW$LZ z5HKUn<=Gzw+QXkim@1kMh(9X07mBz-5gC)6FcdSWH&DU(iSQ@|XT9qD?A`pJz!|r9 zzN2Hrqyd!y8TH-nuL{v8Ndt|>z#;bZ!*yr=q)b{Ldj%kQFm3R*p2ci-Ip|oY`D<%l zm=I->mj^jm(W3;6N@r?{BpAP@CWHgW0t+Na{=#6ygbRZ@0mFQVmTH_FPuZKqgcbEiDG!C^TU~DrWlut zE#t|i5Kmp({3@8ql5)z9HGy()vGa$)8q#+DV54!9YsJ*Sg&gF#%D{_dkV^n-c;MI= z!J$W;E@h&mSUhD(SXO%NCE>vF_cLc$$V4ZiKB}PO$n>b~AL^^4-_IyS`F%g59k{k6 z6Rf?8-$XUX0enGk;*5Q>E-CdTemUKolgGbZI;daLQCRY3AZefi$`eZLeWhu%^RCzI zbjoA`+%@;ZV-ZUOT+jQSy~{EwtKwo?W@(@a%~Av4mA8ui_}Igv$1|C9xbQ?frV>>N z=w1i!J93LnfJE*QGJz?k{1XYwNA-)n-gG##1M zU6H95mko}w8_g)~6Af(<$>dWBro8wWx_AFARix6ou@q^kXDw@~_sP##@#A&*+v*O} zv7K&t>O$kJ_%LNn_bg8f5X~d;>WFV() z3k`ROz2-ku_+a_nEpEVT_dK{aI>Y+DX~BP&swLMWvEQ=a8R3!b*klmp8Tlx%XK=*Z zlTVn@$k7(G4PZGqmbMQXqLngz+LAn8_=q27Qve=7uVU3J?C7yaHYu zhY!QC9$N9SF&H81RuOD${U$+Yf^C`oK!kJVxYrx7ev8r$Ksgk! ze&Jr2=GhbLXWPH)KuTsgv3?7=X0!23>xZCo&BOgNG2R+5Yoaoww zs1-JD3h^O;a9lkOFqp?*d@`lO#^o$+yTA?_cv$#0_Muy%Z5_d?GI+6)kxpv0SFjRz54!v#bWeO-z`c1>k;w z4jM8&OTJ=bcB`uMiDv=O2`C%@-BcGHo&uhw<)shq+=6{T)0BVK?XNpA?J=k}bKN_S zggYxc1+Gy&Ez!sqh{&AFfRb#Vpn*t&m)p}7i51$)C`Ls6nA^2=lTssguaaU zk7gtO*q8@Pdn-JLA=zZol4`e;U6qowfs>Pd*4}(lu{U~*N|an*16o7xP7b{v5%l*C zYAt$(v|ph$jK`75CDW)govw($BwkkBIrQ>GHrlId`Of~m2 zDG{FVeE=oGO{_*RPPSd`h+Ss#tv`C7!`H0c@WTnQNCt8gWB2HkSmdOCj*Cew6938| z0U#Dpbsa$&Qo=lcw*rYO0PJmLLgF%ZKKgSac{NX_59f$g>l3Q!FCm)Q;Vscia6+=R z|JPNMVDqmN%7X%5wzzM&iX)q!kqb&?O8?gG`f9y`m$mZ(bDi45mwoR9T{(Mi@?@Ee zT>Q=|&Sv}M^zBoZbU7?AA=~KZAozkAGv=&V*p_0(oKB?+mN6$Wr(X=4^FsD%%xMF` z_KE@9!;CqZ+XKX$C)-n-YiU{Ye_s+5uv!Z%r$y$j%x|gnpe>McgFybLtFh}(=;Z#A zWKe=okzAxLRA=bKySbj_>oYP?_2%XpntIkakg=(JaHPyRvaEXH=9o2Mkq+IgkEG`} zhr+q`-I>()T&}qzV*rwaGAj7FIN~{{o9hV|3aD zu8IjC@4BX<#SF`|qO>rZDs_`&xNZ9;F$_tNMW=ocW(N+n;X>>xLqSf#k= zKpS(P9Z4E(XZ)r+_Rw~_%S#cM0h9NE%DcwD>5GqYsInpVr0|RV9-|SXVX=~D6Jafx z&_46U0hRUW?qWL3XG1d6l@{M{+bGd~wjfADmC8mAV6(-=NA#DE_GflV?d-g2vg|JV z%X6d&q=jAv`u$#$8HXY#{eIDD53Wphp7i@Y7=E?#B*dKKwBK)Ns<RZ|2^0iw3!GY*zgGD_bN=BUt(S=HPUA^AYSq0@4$j>Y3=(o^Adx(} z#=(Kre!B?cTL!dCN^NpEKI2N}a)@hd6vr=V%54o@F1LOL)K^!nAu3tW&~+bGJqQ=o z5ND(|(9!r%Y84vnnI212K$_T1X-bh3q@MuLAVkpYpQb>}<`({80;?F~{l1ZA|@qOjWs)q^fm+D|+^-s~E>uO#CaVmi_&P zyPhZM)e4=mXJBWDuo|p>#@jSaTD%PkioWq@TKK~cc}L$k=i+i^vSqIyCVoG|l`&?= zkmx)X{(gom%^+F0!L@f&zn_Wy3cJ)HIH$d9a2rT!g!e#RdqZ^IZR+loxF*exw@2@f z>lQ>Q78HOxLVLGQi(+y%Mtk|nu{VroxAyqU_cMkB`?w#d$vQ=XpiVQVnn?b(RsQd1 zHZJT}M0=*$XC1G{t;0KVLR^eHcL4g$Oqtj&`_2!7-$pT6%L3fZXEkrU3ID$bLw`tC zmZkpw&4;D2XKa`Zpj!-#*HDK`PFgErAQ0yfPNj%riCCrT-M4HbsfZLYaK zp}5VB2lFlDZVXOu^ye0&P0^wB*dbjOhr0;I;P&f<3_r)}MqAv9uS_@qQ^&Z$xt!rq z7*8Pi@}J^QJolw7t!WC8#rbu+z5O|=tsZp8p7OTxtw0b!%Ym+c5geJl55$260LGo| zUGy~Lw#>53Xt}1Ry605oj!#M9^_T8@cO0#cvaa%M#MdE^*R%UdKxA9-;s{svy{ADY z5t9$mqc7Zs|Hkf1p#amUM@e>ZlC6DY0sZL%f?WNDFHJCP&-kec5K4e z9||;Kf?3+qbj==B=&-fCcw<$5Ufy|U#R!}cVY62 zE+nwspzAbG#zt{UxDrnXyVgBVAFyqo*kfvcWk1_VHS;MMbpD<#Vk$GRvSZPR2h1w7 ze}x=ZF_9;^yz2>(8T-hv5-0gZv~lY{x2bUE3&*Ekt!LR|g1>{2c@gJLiSQz5J^J^L ztJctimh0(WKWhH**AMv5p2Jg~BfZoe8<*bJETn`sRw9z$N z%;VfsV!I?6Rr+Ap2X;FZ5%q)UUqfh{p?sbho5sB8#ISA*m{njcPmvVJJOt5M%-Sd{pzY3G-BMbu%0#QItty3 zdABy9U;JZD=95W|yCEs$hc&70w<~;w{Ep47179n?`sj#u`dGcq;C;nFd3#Jt9`T!w~+`aODW4XNJ|nWH2zn~}3`!t#0fUN*`Q?2!NR zCC~q}$p+ugJc`lv%Z@c{NA8HU(zG%SlGT~Ube8zY@-zQ2q z7Q6UlNlYdf5|_Y8vBzUbkG8RsVlE3%gA43>ScH|9$MMzDMoRxRKyV)z;Ehq9b-3R# z;ox1x1b2H4z-S9?*(rtEhQO6i$!{Xh23MRGWPb^I_siEA4m0-F2NvIY$9qSG{6@i@ z+D4=9g|1mJlsQ1hcc6r^zn^u|8Kuxz+=`R(%w_uaG-EDPai2O%m5kzZ+_39^ zSq+}Warmdlkysrwa)3D#Ju0zC(iJzrWH${cgEY2uShheX03K^roh_&DY>|{c>##u^ z&=GOt#s}AF$E6Ksb!7)!bp%vb6nI$A0;nppx`L}V#bw&`F{>-L(!3iG5 z4H>k{-p}Dr;rV0D_`?dk$Ch_o>3J$u>&_ek`A3b<3kUy-F4N-xlJg}Fa_Vu0TUA@d zW~PDCwIJ5{d8(F7rq>)fFzU^xsJw<`?|xV;xqQXBl4O;jmffxw1-n-(>cf6(@%x!) z=fnIB0=#|N4NaXo14f@oD`GKuHM4mrf>Fg+wNCR%y8Ohl?|^7~_16KzF(Q1hYpy+R zqqHkSWyz<)76Y^6+0UQaU5IBpck|EH)ed!YOP@*y6qtZ%g5Sahgm1ko6UcK@y^(RNDFW4lS(2SZI>aXe_Bbj(LEO~ zgk@F|*WtH;VX93WUnG6ec_uqDgh=P4Y9gCz4!bd92Z&|K4&4i*AAh8`DY~#9{Hzo3 z$vSDRj_Q=tqvtbaU%3|C@qYU`?B^vKKsvRJtDKEXnCBx4_HEe#{Jv9j)_3_=`VR=E1n+duvAuZ#CTeHKhGzNh4` zbf3pV^F^CRD`ya**36H$GbJr0cKQLTaE_}5bkYf3fy`?NMj)p!mcLk;C~Wl$#703S z$(0@{ON7u<~rLO2ls24ajQYTdg4W^MARCUDzMoK!EHi8;5n{MHtIlcAhmKv?n zWm-c6yJ;h6(Vy1vq@=;LhT#S`Uc2UV17$k3Z4_tlk?L}G@|Fj2q$$>AlqIG)jP4|K zK+F5#>5NO=(oxdyn0ZOfuyJ410QD9cAuF-d{N%~LR;w};nytC9?>Z6K`Du&oTI$@c zWn9XnTB>L=?~JOGwpLikfu}%R+GmK?HP1CVI)AuWT~Hbd;g(C)cNx zbE&(70j@wBJq2gik%+Lxp0?2^QdH057i0xf7p++>`!kN!`W2U#>H8O5ddn*R@#n7o z(88~}s#srX^(jZSo1SH6>m-64&O|f?&(^~O4N&v@iRl%m1DQ8+8!bJC8heSox{s=d zU>OJJ{6m)$qjoe4r|3zJnM%3x-mWY=Ex|?{YSN@YZh0bh%0H)AS8X!)l(C?izK%7pYpEf`+r_h-V5Ou&5`p`ffam5!` z2_v>lTPpbPcMSJT+FEXneJW7U+Y)CLDcr(wnr;S}A#EQ0qV--8+y*KtJnzr><`4t>_kY@!GFDHIbv3dH?&#J%Fpb}gf%tP|F!SQ`9;PN9r zHSodP&Dkti5h9BFj-9&p^#3-sfZK!R_hk7!50}|SDM1(Dk(og>BCn#XdG-J=db9u1 zlaD?yPk`LXwp7-}uF{38PGY3_baM1>66$%2I9^Oqm3qJ<6<52l)v@-p{KB&FRv!kg zW_|YD=1Xt!oSqaPrE;sb-k_T+*s!}+zs^W7^rhv-11 zgB+ux_|Kc_bJVB%0zEBxzSsbX*+e5u^OgGL0TDAAW0DOm-kMn*(b$@7EnF*Tk+}bwL$6$>0W2D9OH>tJwDWoWUSYCbQ?=a@{Wi&_Q7t9RD9{WCIjfeZ zXyZit;Fd1#we5RKXq0kL^)E&vtjm3c7{Qy}!`^Ccduc@(zAJ_4h`{kK!&`OX5(zdN zdI#QS1hy5%amo@=!l8J|U&4&49@u9`FtTh`y!d~nH5%RxQ=2wAdb`TL&jFb(3~M~B zn^w;dgAW)%r8iRI3CqM*oxZlNK)~s4+ zp}tf*?h)RE)%VFP@5*+c?AeS-&*wBbydcbgswEWNy3;HWh{2i(170_5*t-3#nOi@H z@!CV9)|5vDxmKVK3GZGIcA^h>==&p|LwkCl&^<@EtrVF&nOJg?cXSc$UL*CJ+xhe2 zx+E=>_?6kK^xb3gRrR4VG{(+sZQ;tXmGpLV@4b&591NAuh}+ALDReQ~3HZ2G4xh2B z^DgTHyKZH!VjV~zpL$GGd%YW2JV5yki3oPcU<6;6zsEh>pJ=R#J!=M1c=cYa>UEtD zf<;)E>*j@w+cspg^d;lZ)fS*Q(!dN-onjdzDV^H$fi{k(%yN~{uwJbU z(*gyO&H;M^@j~@PN13@-}wzH?J+8M z=Av8eITQb7^7x7VcE!f{*i2?SPrk2wy$hhfo%{Wa7&ST|VX0(#GmqG*UzOOGT3=He zuN6^8M~6#8(Q z?zKG~>jZ1P5VZ6MjZo0oz0Jo9yLX2&^HC({Qe|BF;?xYQYu&F%Yr1vabOI%>l0d}3 zIyxf;#SUybdyA0q|(8vGFF)X#ol{I zHJR>f|L#3AR=^5MpHYfb1?kPL3?d*PCA3fmDFG4!p@a^53kHX(p(#~_fDl3mkRXII z5+MQt(h^!g3`GdNcYja9%sKl#?^)-M^ZwRZbJp5xW%)$^iuZFr_jBLh?{$4H$-SZR zF$0D-t~9j_rWDR6+Ta}}l+@*WXn19w0^v@UNUCmt&Z}G)#M*U+hD^DmQYc&+`$fdw zMYem4O5beGok*Vs7#sRis$S?UD)_MBQKP#OhlnRw5Ct7QUsbLI8QsJeqUlR1TxFK1PQ!|e7VJ+}dj+hZYSeto`3 zFf;WzGu<3$Smxd@7@jLkH+W+#W;-F*5Yd2a-E2Yo{T`8hleY?6C$Logld5)#BO#;; zx`ag|NJZ#PH3+|fW$x8Y@=ubYr)~+K)!0?O*C|-E=%=IC+*9WAMfiHXWthZ$ea)oa z&9$fw4}V;-0RFZNZIA}W)}R)96LP#gi<=|dvI5^Y;*6Xhm7D2`+=9v2p$&;F^DHST z+T)XxiK;=59 z+O-SaVGLr8-RsPipFT$G^9bGn%>*Yv$*|J5<_g@xF5XA6sC=3>vUIfplJZh0zgNo<5#WnRY&E^IKoyN%>5FM0*3U?(Ic z#Z|?Bx^?TvUB^cbYqjdOwyJQ^HiNd?z6Hg-7tZEf#}?yr>p<4>j9QR2kG=Hk4ynwS zQyJ=2a0Yg>Y^=e(WuyjX44>-oH*g;yTe-GF`Cx-tM+SiGDr5POHrba`xgl=p4xt-; zr_-b#Vu6>&z^oa@Q6P1;AEhziWr-8HNypjMt4(n6evak3in>zKtX4o_zlspEeZ>AR*_w8D%skI>K6%Gzh zztPF}(5%Fcm}M~)v#Ft*C~ds}O5A(NaJ)7gp(L1O;ofTNKa5>wjn3Q1@3-}Z#@}in@ zg*ZM%^?+6b1*#Os=0KR+Ky8Vz!ng7UEVlIiH=m2HVG zLjjEokR)4u5nC5gK+wsIGcKpyFUZ{QX+fR0r1#UIZ{}?!ul`#-;!p4jId-yZvHNe> zFcL4oN`H9w&OnfC&<6v}U*3^VsyP%-Yx+z5NDEV1@oK?M^v#$sT1Imx^V&5L+1l@> z%{uOxT9y_^MrISkHxrZeo8Le;gudXbyu3D;czi*}{+bc|qoxpHmNz(P%_h^c)JKw) zT%%NL&giC<2dZd5od<7}$N_o*r^L0eJ+@we1wwVc1Zdc7y?}xRPK7K~*U8ZfIHRWH zcwPc}fy4zNVL4|Xhq`hMn&4V&tc@FC^`(I?1tqVZQ@Nk9?EgTk_33TT>6W${uN{19 zOCJPxuzqoCSnW=;lb?)au5C(n|AJCb@xUUH%zdv5A3sF*mKvmw;b%~q=XU16Nu1P@ zODiGQsBK(?j@WUom5%aPUZi2%)Jc^(WV9!m!3m5ySWIn%iaIrM7aaESWzA~0={fGa-KZ5XEc}gQsRb-Vt5bV4KO+J8AZrJ zzpcfP7js(b;yqJmBkQS}jP*8x>}8t5woj28nKsFY>nF2O_b8bZ3;L+(bB>6MX zKR-vexw5liVz8KxAEU244MxdNPPw@V&q<=wxp6L8ev5Vt2Ir!2EbaCE$E z(W!}a`|!<$fqW~#o(1j27}nDk?Q4_(ojKM{tBBy<`D-OdXU^ZD)H}o0nFAYfH6|Tz z`Wj;tv_Q+hON_56A4Fj^0)Ikxl9BUm3Cb)hJdZj@(0(!>C3_1Hv{Shh06}{I>v+S^ z#*dkSN_$otMcpeEh^C(e;lmaCO1_J&3lx2Yu1D7|NEdRJtS5Y}TNhJ8N(_n?MEVMW zMeJc_4G@_X8TrP?D3p4B!Epo-rEhvI)<)dmO7WIJ^Lup|CE0d!yjc>q>yt{m)KJFP zi9CIHd1G17mfiWm8<@;x1nLd<~p{|4TGVGf~*Ew7;;t^6uw4Tz~ z&wsdz24P1mXr-^Q8Pcz(Rf+|sPQ6SOE?;bNsx--YHjkPiXXOHbi&i|!Yu*{P6^6R9 zj<(fsiD`P$yW`u%zqv-qWdkmZtc|T~mK>8jC1@<GzUd^;c`QJho#!g}h3RsQ-^=It=TZNfaKF4O%u9eW z-igyC6l>o!37M!XEJ}dH`Hyk_b6h>65Ow|* z3v#BHEWt=;|T+RZgl)hA-0%kU7luZ)n%^tG0nuFeeLE4%{z?8 zO|Fmk#)r67gOm=~^(CiU1N3fgzAdfa>q}8*>QG=nb`kd_{a58dm!*Vd}o zA>#k8RusM?3@~A4flYdor*o!!ZJR#i-h;JPQlB+M_8QwF?iY|`LZB+$($|)$01bj_ z%(-no*RvA01QUCwgdCRuukTizhudsS&_)omS*vW0L?oaAmy>mys4YO50o+p095C7q z5q0S^EwNYUf4|A1b&4j27YudW1@5f>{9bn&jW8!9jow5jpnsWAec*a+T-oeM0B&NZ z2`e=bR5MV1Zt=rSs=Q8I`ocq^-G-nFP}g}R7_exWX-F%z!%5t-uCjZt?##CHafzcm zs@NSMq2&(XbxberhGUF2V4s@o8irM&o$OK&6=*f3yR-}PmeI3>IIX5^3tf4Bpw+~d zf+snI1`?cro}5?MV--$MPM~!bw8rVl-Ewmr>bea7y(f2W18YGPbYb`8?td71Nhbh3 zIZ(()ufPM9c}PDgD+@$JL?V!l#W|2i+-{ocx~T$Pwqk&#y5XrSN2c*8RI$*_YM@{!FmmOQlG2Kyx)rO&J5dQ6SljnzL-(PB4n*I)iJCv zXmKom=n$?l`V8_b>s1XsihgqAAPB5}TMk|D_kVkuzH4VJZfN^hec_jK|37q2;59a_$B(Kl&7TJwQB;J&yYivjrYDCiz@oE8xi~C zq~Ph@2A%RoD<8)yy8LsB=mP?U3jsJ&HT31UTc@t0jho7T_8s<@|JenPdih zw-aeP^!lNoiA8mv(+{leHb*<aneX^(b@T+1<@zhlSHNii;Eu8ZwZQbS?o*4Rf5sOBce zrsu<*R@FVUEIGX^`B=IO#9vt4^(Y$&81hrszT)*<$tT3kTn4b|JibE08`l$fzk#aOazT<0Qm{2*R7x#oJ7Z1>9?GgnTz?!G9-oZw=t#xp0=53D{S|~+_S7x zwia*-0+T8iR6DpTqDvqUM;Bmj02xl*w|V+!oiBIEp7`?4CzYfL`;ir~?4MI;7rcXAR0b+xMJ{63##hrTC!bC!2xpgVM${bF*ZY=NL zQ)AZ&4P5OCJ&&q#o831MCP#i(x%R3P`Yd)EWFHm%kjv|AvE& zLw&H7f!6vi@{XY93cmpCh|@@y>tw)u$F?TtW(39pmZs^vJ54~5hHXs_UT?p@(gB1s*FZe)RI9SwxVPr8o>Aig`r)|Rp)dYs)P*_ zfJf9Y(~YfYY6Yl$g$@+La?o;+^@f$tFaF1^K2<+kToQFCH7DOv`|a4yvmVX5dlui0 z753s42EVHL02b5)Cs_F8w`2QU(<6YcD}nxcAE`N4eAqe>(FO2!=L$@7hibkZ`v;2D zbUo>Q(+X>FiXZ53?(qjHH!V2+t^08I;Cj@E)(CMW>Oi%8`84tN%G%K2mtOrFhd;#a zg9Y-w9gE;gryrzHQ-YmPQGfsAhqYv7_@yR$M~gDW1FIEiOT02 zb)%v2syk9`H-zC2b1(SE4#0;QYTyuU+pKlAoC7mVlWG%e1G1 zE$#=Y=^}2VczmEOtb~_wM}8;O_I^?^nH;e+)i6qQ;>H{9=H4zRJc@7{jK;b-)(bl7 zS2pvcPbR5-W@!uCpPT3tw#tR}xJBVa&?7anInW8h5UdWmv18R=DPa7^a9_OB>tnoA zZ1Dh@;f7_~L|0Zp!XSkXeV%#vsFL^9VaIRBgwW+2=7X`&8}wjJW@;vUA+1$Wg-1CU z9Qbu!wRdCI$9js<-w*g^8i}|Y=g1Ri@4{YP-Z$l|8iNo?Ks+}|<*5&yxxWN%5c=dJ z4LE1esJYm!t+~t?C0PVdzcR!I|qGwH?r~JxvdRg^bM%i-m$RK!SD$hjEs4P|L z6PkY=30Rq^Q3jgFV{X;|@gMl}T!Hv`KH{r|(2-IUwdBqyw7g;dP{%tCUatzR2;*OU z&hzaUm(!gtRhO$f$1i^BvN66FVk9GUP&yQr9W0a)kLYs2W*D_A4IfNb`banucejl{ z1D%Qf%~|7vbmhI1*kKOC20B(U-K|0UsT<$+36X2vrUs`8kMl8LN?~f@!Fw8e{1l)p zp}epg?M^GV8 zkyM^$+i2JkRH#`y^>s-F02MNJX#Rz!FBLif$PxPn;rhc3n5+QoylS?HPeYtgoW@Y+ z_Mj_)8>T#(ZrBrm2RJ>C2E0Cjy9L&R96gHi6IALL1JJNqxYK`;JW$x+ zCVB2iZ`O>Wm&$f+C1hCyb7`e_3$ZpecbfMJ;XM?q(T}!XxQqS?;W6K}h)w3ddU+3F zFd_EiO39#HhKmlSMoWTL6~7h_JUo|AFGHn<+JzuPD{4%%32|OmOd*h5-LIR_AY)56 zO~YCSE?ds#&&GrvW_qXTFE#Z^WMvHut>&IFY%I=&Jc_`yzwHM0q+qe<)ZUhmf=tKM zK=7}X@wm*9>o-fxH(SFZf)4ysoaupO6$|ig@Z09fN=wEjBcpo5%sQ`6d_h7qQpnvs+QWBC)3K z(_>R%6)OS zE(b!fR{y%Pyks+*rq7vSBc zm>2p@#o8}8v{xgMR&Yi;*c=-B*M;u$3#3fp8<+a&m?UKuB?^-L5K_x%1It`+_6Y(C zK}Ah2g|vk>KK1oxj1a8N8f=#)^av{=qLy(aGB|eZpyxT-!q05^Q_m7+T6|n7AucY# zV`4uwJ7wz>(M43GrPQk~bccm8hxHV=ruj}am%NS(Daln^TemFnIwo{@gW(&&CNFsOwr4BIpb0Wi#a}Tk=UQwA{^5;;5k4pYW z6k%sh$)8OTW|#bd{ihIuMaM+%x&S(eH?G(bAOE*5Zi5R~s#5+@P&7o&+g`Q?Q&g|NBCEbZpzim;DV!rAbO6K-oDv7u!b^d`DbJa7 zt^Lh8P?J@s-~g5#ZapAxQNwikJ`q!mUF9s@Oy}5N>S)NwS%u7dySikOSms zO?NxvtV?WAY*hxDSeMQ+NZ^PvY>a<+DnVh3GM3R&H`y$B+0ejQNBOB;{~_he)1A0t z-~E<1SN8mt<{EXk=o;VsP~YMk2LPEjC2pHrMVH_l2~W>yjv zz0gwXu+q)~@8`H@sFx0}7i2b1kIn+-{>3o;TEN^d>#gSA+sZNbC)-|4DOWo(_j8E# z&TMl(C`kDcVDA43Ee!9l{%)&-PzPU{tuwl8w_EO6iq91g-VAo@RR;ta{*f1hCMfE~ zecdV{Gnd{v%QYz1x;EWaFUWGk4G&ox!|ebR-TQfOe>paa&OgXo=+P01uE+dtjcc+o zgoC2H*8bJEijAUkyK!z4I8G=UFBUDs0od`cR4Fz$0cZ?!xCzag!yP@$-?<5r*b3|< z2Pt>o9d#c-%F!AJjQpo=YXGi>8ZGqF<*7>z#!sB8D>^URPv%$WOZd5fp`}%+h`rD~ zM|BN-z{8x$Uyz$NC=COG%Z^)UY>BYPS>Vh_>tS}SWMG8jor;WQ zq08K{_hsh`XLB_jKHdzeb4X60ch$2a=wrD96LF&j`Q{(j-0b99aE=t~Cti#HZ8%%y ze-q9&0QeIBU*T-ppsUYeLwp~kl?6_1q|@CZVgTW6Xy-k29>kD*ed?`Y}hr^T;4o#DxvYB!y%zgJNB{PV-d|`mKLtvc> zPoLsIA|>dT7J$H(uohkh(wV$7W`#I^N;Az| z6BKr9eWz8^!P=>&fjeScr?nv|L-oLqkmC?(T@a^vNr_j2#VLN>!X^#d5f**)A* zY22|6+OWJO>;Bu<7cmW}*D*Auh^F=yc=h(78ohapDsaI_JZ%5*sP-*(#Vwa_wO`N6 zSeai2+IZe!5LyTK_HlOR9sOhR_>*0K(ytxMd|ux=h@^tQj5{kt&s33&|JHGQuC+E1 zRL4*5wyN;z;S_vRh8~pgmmkz;1+nh+`aU(k%9F3EyEn5=u*Ny$DJ%pUU17Ao-{USm z9ergDD`(G4ma6=s{3VtH1~&C6kCu|9!Zp=$bb|v%%Wci`6uHvI0i$^t(^^hpIAAoK7grSkMytdi z55Q(k{+gioVT7V+({`v0RoY6!mgc=tc?axYnxgf3lThH*0`*c6zG@q-S~7-(?|D%1|h`n5EmFQ&iy`^(+mkL&|Hx}>R4#`EKy z!^TSR32|_1*S)THx^pvLWGq#aKviIP`m@2BH+|MGcE)5>XB1N9-*SpHY>Z91v19Ka zMH-+nFbJJ>Z3^52wEqTew|e2rK<{9`Z{_gL_v%#Lkh@wd)FV!(3a{34ed2(@ADqt6 zz0ifkg?O<^M}QJ+6a?vaSvOGHv+&6NJaYrcfB&(0$uDwzD&J#i)nh96+%q{}2bEiw zp9Zyy9h%w7xXl$NdA7$d{p1?uA9WgV?LMdD{4rtp1`Ib5nyN zd*u^dH{k`H+=OwFl?8XvjqDoi2UckIn95rD=JKMi$TPD+TZX(lc?g9;==M~eUAJhD zRZ4Hu2WUX1)aoG2yP+;|Iy4?tU_rHq>L;W`pVVM6% zL|E3klLLwBBAX-=+E}}qcX_3BNU~7)+GTKQMQT*^_oAKcYBS%C73E^iUy8w?jbrS| zPwyL;#)m!D-tr+s*j9@S>-WUm@shcj>cOGRBddi1UN~!WiuLVSqRyofD%)ydRQW02 zr$0~V+Tq03Z4|Nz#7EEY);9GSp@7P9TudNaqatoX)vgzchuP3 z)oSWiBF$3>9U{-IW22OREy^rfjeNW3Zkvy(#k-r~}@OPt7Mq1N>#hY*pWn9;{A{QhvDNpp@0wk5u3TA1~bmH);9TW!4SI z57NKxyuat930-OR-yv zKow~Sjo_6AH^Bh}W0^z4zd5u#8HH=qN4C$W`MfE1% z_}lNx#PNy1TSYaC1I7j=B3rAD_4vKcnN7j%G;upgA2fzPmDO-+#$n)AxHS zDbZH=z4YLHg9c;f?)-!Op`k{qpU%=)O&XA^`a>(RNsXkH-lDaro48o94tPf#Ofhj$ z=s~wh6coXPpA77G1w7j;nO&8+t!I~RN2rZMJS6k z;3&&+h8egWJdMhJX-QCN-wy$d;=WeH_Ozj7W6x3^)T{Tb9RG`Kagl#4eAG4TSL~&} zm>!e6ZLZIZ*;FNUab`g?YdId7D4^f>ef2}i`Dt+Sn`)kH0OQd=U_ zoA#nKP5x`C?J}bS0OZrB&P2tb%X0@D>p*T`b%V=_AazNZkCY!5P9RSopwBEtxher# zxb<{-B1a3Sr34;bKGMQX?aQ-TP#i7XT9=eBKxc=Vw}mG7>#KBtt8>{{T6Q)lfTiVR zgL1I6E1Yc5BP{JvHYgiQ%gzQh1F*D;x@ALNSP($`-P3i#u!+BHiZX}zYxZ6)j<-4f zcj9jpz*1hJv5CKAQqo8RK>P)i?2=bHWJ9`-Peza@gVd;!#ekRh0E6Dz3mXb>I4P+Wr-SWLX4*O;1WgX?XEPKcy|E69S&lU zX;0aq>z*=pC0^ApjzZUCKG{5m-DHQZpSqgD1%MF!sKt(X+I49+27pW$uPqM-o|En_ zoJpFqO+HF@_h-kkCV!A(*|#7y&92$v4m02_FqOsM`wl}Zmi5Q zAG}p6uP{D5v%M0@{2KP2a%oa=pVWF;&9>HNNu+F|o`^o4Fs@J5-*Bd?3cL-}?x@<< z(Zkcna;ZOURLs6+P|EO$39c)aJ2sdmvXh{8)Mdw5JJm(dikEoYbe{v2*%`TXAhv3( zLW^jEVIzZ*U@Dh>#8D%%cbBUDVe>Oi*Y|65l$Q{@DvG$iRniS+l#>ZP!{<^p5iFuB z9~}g4ta3=%^hiD4m%5-te;0bFXq}!2%lNfWJ0ti8)I@GSo^RRLhd}*|^%l#k_B|*? z{u;|ui1i$bS@-Oko|4(cAZw@gmNBn$?agMd?&4|a(&%S+DJ`815 zIm2*otyT67B+rX&#AG>M=_JCkq+Yb0+*-l=8f1;xIV+q$B`zqmB}2?Ms;diD7J5xg z@WZ%Za@@zkV!T@#wYS=DLj|mgad45eB?{k7#}ap9W)(9A0I4SqTgrjZnh=Jg!*ImM zy(BsJMjHm6M(Q@JObXNRE(>=@ykB1J)vvpK_MPk|N& zFz(we*Dr+`+oRi*PZh>%i36nv{GbCZsXxdm zamy!hg4_!M*6-k7CWRtHD$s{X`#=Xpp)FmF3V;l^r&2(~r0Ww+Tz(VTAQfH}jX904 z_YD5_AMe#E^jCawIIwwhs2LnC;B4MjOXlK#b~)O-DVn~-j*7)e9c|v;CYIf=;cVVq zDdAEtR3XUv+opLB0C`6KbVPUGAO;8Mw(8)*q!Mu`*a%d}N(S$vo@C$5<>Pn4MBP7~ks;Mlw3i4v2X`!1YaF_SxV082Rv z>1e+09M1*piWLB%A5|nr0}??-E{#-09%NniaUdv@GvqqOvyclzDN<#&RgYj*oVui6 zr&8^VSS5rl^0N(O(f=u3>^~0rpXBdn&sPv}C!jQjrHZy857QxV39_;Vvk06ZJ46q5 zr#=nZezePjAoE}Ta5ER>*X(tx-PUGi!K!1F`P*bYn4S4k=V(^+`ymWZ=Vbom7S|a1 z{GR#qlm+wXU}ye>0OsL{NR#M<)kXsfvSM+Y#BweeqA;mL^uaDsErlVxPapa0CVdb( zMX;(*52U}Yx;q}rC1;hzlgA6u4C;k$*}>3?2rh`ch9`aW@WO6FXj0RSYKOi1>#vX( zJ_JV?CA9M zlzsZD{P)w>b3%PHT8ivArA%Rz1!&d?TGe~uJDdOWE}vLTNI++y*)ulchJySKXU6KEoh+Qp zd;9y01@+Op9i%xk78@(i3(VNg2P@P(zVa+iftC=lXDiR$WPclTd&nWkNiwHji2wY2cY>@q~*T*EPN32h{ zs-Yq}yOFvdwyUQcot*xppsB3@^0KpAqx1X0kB&c$U&o%TF7SJfqYD$dT6k~Vmi=xu+^4c^2Cuu46VgZ zRTT%h`8h=3OrU_QD1OV3PYnWJ<&+@ZYknLXN~tmQMqh5*nR4joT7lTFA>!FuSIj9(F)t!-J%b(kO>7^g`3~6EIx)+t9J{)mUZG^;;@@oO))-^fFs805n zYIkgmzEe9Tm=guK^7jOdTYIU7fHY$^*;XQ@Tpf~Y3p^rh&`h%t-@+hTx_Yifne4A? z$L~e~g4v^uTLMNzbUz@KH|)GY;br4Z5ZO9?t_VfI1_XRKR2u}n#Jt{bgbcUwvP3~| z>G2^g=OpjG(S~lb>PParM$$aAY;Wg*WBPIZjE2?i3|&V8DB8L+yhtAXILtMpGo<7m z7sLHsgOx^$ssq>}Om2OFs%gveS?btXMC3LV@*x!iC~}UOp8E`WYAbfek>hSdHAY}Y z3mf3|7dCKwugkj*tF(%jApJgKi+Cb+Vd0f2?5f!xk9jxw?r&`;pX`3R+->`9`8Tt_j3iKOUC!dTwt0mxD}s@7w^`FMkHRj zMF+arx0W)v7&BNKWsj61)0Y|s^F@{k^vE$|@D0~5Ql zy2ejRaw2`ssh?_~j#7%g)L}3CwYLQOz^dZHm3D3BnOZNv445y<#>1M$ZRDqy6mg3f zB;9L8qmhHIm$p&X#+V3^5ziuqUjs4JhUxgeU-^a6@vxlHv`Rdnrpg?hj)QJ$7{fVg zDrvLUk4Tc0r&%l5Y-kS`VL5OBDXEAnZBfXmoo)$`(^R}Gm1)&+Eq12@AvUBM9wMJF zU0;`w+^+blZIw_~@k>wNfZIHOn`s;L22}ITdorjPXKH5Bz*q!LCi}F{_eFDe^%|*L zq5Ew*4@q7ENmZ6OT2cBL{()$L=Z*WAkDS)DeO!&%uk0`*Xhj+Z$y@bHC{L(Ld0n%6O42ooJzirr8H%}?=!Xtxosa& zi)l|^j3oM(5cnWgf0vO%BIDOYj`fe#~rgv2Mg+>p8ccpX8>&MJgszR&s0U;n9Nz= zvb24cUgb6RH1k~8dv2?>`Z1@d*t4KP2Oo$V^il~7ckCj2nzS}?OM4Z{P?64!DYZ2+$TNTDAuwr2tI{ z%0_&pEhEXJF#a|v*e%%<6@K<_zu)ukyfte3a`K05C0SN}E;a>!(|Od@7LjK#ZT3dZ zfuhBrSlbef(qu_R&lde(PG7sM@EkYtrA3PM{c($DUiTj~6wb%WEly$i=j!PFD+hsb z%iijI)Kj|yNd{ukW&MHA^kVHvS;3x0X^Yi$Mpk8J>v$B^t(;XspK`yvxE203*+ZH} z@do%x&*rg%c_6BC?YrL5=TxDC9~-K3VY@D+eVw7Q9svzB8RS$7E@5MskzpK35uhRZ z?7V=Obp@a%`+gsYSs$Gkso|I4#H=e>Jgc-7yQs&BS^v6Rw||u#vp)3m&x(LNKOq6g zffI~I06f+9aF+xNHcyoum+s5qsZJ#36!mtg>9gC1Om7rz9B}7Ip@?(4E@V zK>hol_!;}BpC;f0gp~E~8Cd`!WiMzXhG!fnq%4nnw1%mC6jCNVy8vQgtk@xC{^AyB zKOm%x=Tp!x_@N&gw>EW+8J)KQf3uVI$Bn=>ie5uESDn6WgkqOd^J2XXL}~Ze>29{a zS)5ozl+AcURq}@s$p%b9!|Lc}v_DNM)Mj|O0he%V+{3q4Vim6W&b(aC!++mV^j8AWGFUdla^ z(feMJwFjR{rLH&Y#RsdY4owhtjfWNy(D zv4f8r>LPxM^=xVo?Gh^Jn}Ro#u0S_7H3Iihy`uS4(KQ z`rLxHp~lv7C3=a=F!%AS_}Pv9 zsALvvZhg1zK*p>9T;y;KrSsAaKzihcK$JEmv(#n5HIEFu?TNCqg-+#{m<+8NV2A17 zUZct0VXCN*b8rcq9cE~oQV!y1hsiOlpJ(qd5tbP9>%b0E;8P2q$eWF^{~d2PU_KoG z>%64hMwGiWY>xa>^!%l#5$~TM`=ep6!D z;|F7_eW#v2HMl-FQ|nOn#bKcD-J2=iGM6+NVOf^S?$D3;+JUxQvpt($qV|zkh`+VG#ICLqumg|2z2AIy4(cGq;YvDUgJ}`(dh-_NgT&n^6XYWN>o~v+U6Q{kVZ^v3S z1-zA?0?8U5%d zz1Djb@GL8Zm3h0O6srfe$_5gh0xAX)3t5Gk*NYzqyCe&Unq%yEp=Jek4bqlaig}L4 zGpZfVkXXes4)`aC$E_5syi$5Psx_~Z(`jR5ug1W<*4dO>nk4VEhB%2$kZpp(!GRS$ z^Ajt~dDcASZ0?nLdd)23|B!ds=CkD;SmRjJxT+lTtnwaC#eiQ8bt^+yiM1GYD&n5D zzEj4q{6S&pL(Nm}+yockcdo~u+%%6XYUUfCQVu#5Z7YOff%Z$6Or2|i@bq-=UOXo} zee=5CbrysZo~~R$_p_DJI|@$+Vej~8L4ok}*O{oB!>k|>j>glrwUxL923cW71P(+s zVi&;9RI#d<+vrS56Ar zE!d`9fzoh}LL8%c?s;7<8wgNd1~OFAZ3AZMfVu+%s5{01b%(&et2+$;OLd27o?e7j zHk4n292NDmg7q)0I-PdPXCx6-h-xZVw%cRAo#N+7PLz(}Dl>UuD?vr%rb_vE`dX>{ z^eFcGS-TtSJL`Rt7l~(;7qz8#R_)RiyOUSOsz6QFdSVv>XGP}rH^i$97sq8QW>t!= zOXQZn)*;L*q8(^~D?RStj`iQ}9}Op!jb0fTtHRw1ou%u>qcG@Ts~|K)+R-YqPdCQet%gR`RjJBh!Z0ENe|h z6^5{I+>2FI3siZUss_Ppsn1;c1lWT+L_(Z%)m<_ib0PsW#HSj&gEJ~@IQZ{q2sRx2 zzd%D^g2@@P-LCykSt_I)${OD*mxYd|(w(#8_FuX9)_60l=RrR-DVVVuJKQ3Dq~o^?!$-rLu75nR@!%V*2sAyYHRzs(YDL+ zx1(@nn`u&mJ20#buB@ps!$gyJL$7pt95qotbEDNiOjH_FBcd^|u7⩔gWLRYJPB= z|AR`Dy6=9J;ajhAd;AK`77u~oKk4%6>DiwsEi{+Qx;pg3y@ESBmBaJ+3U)c2!eg(2 z=;ALcjguyMa@e97QIZk|$R2_ME~l-H>?zXIk=hV(9N0yvY8>9CZfvS#C1R4&fF4kv zNTso)yh!X1OuF0%K@8+M5C=t0qpB4Z+5Y#K5NDe;gUXf7r<$SdNn!1hvaWi6n)- zb*{i-nqh0~wo^{#>JlUggVpFF^y77{g%bYiSm38KDC{k_w464(4gtZWl5JcJCH6G~}{KL4o^`jDwiI`E|h%_voZMT%VZqvP>c3@JRYblOUo^BDD4q zQNLU@bON7YX5rxE-`JFP-SXQpDQbAj=3VYGy(X9TM>V&sM7@X6 z^5(y^So>TV_=OgcdD~S?vSzY6{vZ3Thi!5^F|c)? zXp9Z*ySIOQYmLwDqxVDC29*6v(+@Kd(*}#oWtAOU2;4F(D$u0B+yc1p__T*q@ZQ6ts znct6FeGPWztwb7Wskd0JO@kIoJe%vUE=0G{_z_D#6g1qbK~d&wipNdDejpVNeVk`> z&^w`oCG+P!4}N5h+2R6ee!ZoHr}28^lhct#*~_q=N}JDdY7CrN*@cp7qph?FI|$-+H&}<;TdfB&dr*m|M7`6Ti2|@QdVi^$sco} zvXw($GVd4WYE8c`Ey+{+oas4{tUi+)f=Ifu(q-#U74rYUOWe4$TX3Kns=1StjFXcN zk_o4TeO!6@xM(+U^UGbjPj2qLM6+gt z$&}4c#%ZW6C@7P5LD56=%>DoVEjz9{0*<%%YIkX|mqC_DE(vULZc4cwJeVFF`ovL6 zC%9v{rT-(7^uX75Mm8edTLIwIXZuIHdz4Q+xcEUTZBB0^#Z%66ZJ6HY501<&zrUmK z?O5?am9RRf4*^H)q1U`Fi8fw#hTZVF(G)Y%J(2`^yR09kq601y%dCX597RRqiXp4tE+oS00-vK8|sn(~nuPGwYbK zpJv|d`@5$7u2kfQZ^ulWCcXpeL608&mhwM@y*;tht#y$ieyKy~CIv_d)mmi&UqsY_ z;tar)WC|geNg*BbAWfD@w?wCvj4VC)-xh0-it>6fK;~x~9{+?g_rl}t@XEt}Ndc!!W z6^ymxv$zcN?Q@7sPsgccJn?TeaPM$l_`D&_cv*bf)r`KP#`>PjF6p3~|9zLl! zb+s@_IXN-1`+36rCyFvdJ9vVxvxg$iXThZ~BmdX3$k$F*)Z%Z)Ec|ox14ASoK94>y z^Ba)dU3IG$b#fL*%*3QjTl<#LjN;?2e-fJLqP!v#l??veldgv`mix41ELN6kO2%$&`9emN>dGFkJ&wbxL=f3~_f8U#r zZ)f%}d+)W@%&b{!X3bj3x+2qt_ow@0tJisrmr^9eb*-lOrs`0Xdgfhndb&{|te+6L zugG73Y&v2Ga6RV-Ain+A$Adc^L2{GI_QZZBaPh(9#!_{|0(C`K9rxb5NX75XKzo`W zj#*>R17x|=StA*upF}tr-Fx$vCoVp%!z`2qv13}<8v0cF6W~%kIxlrSb`N`gLnL^g zgO^PHzYfyB@FZ)@vuASmW%ugfEAH;vPID@KeP5cNBn5c-vk7}_@8V)0s_P}!DdWeQACY2=opH^dA1eYN9-kxh_fLvBcds;F!MpLE}| zQ5u}jGJ?LS%xId^qH;pU!1ZCbZqi}b-H*W~v16Ck)YoLQ!zyaUFHWZ_4ld^hNgd=J z1uISr<@sX4g65C})!o*HzMf9W`vr;8^%+JDa{_O(RxLOp z-W^+T(lIU^Hy&6OMB+_X~BaknCvsm1Bo4neDgF zEx{frCfS3SVXds4E=nPsOwCeUNq9-__=7PGs;GMMk7KFXty9ip9(@||7QGIyiT(}> zXtb}V+O-x%3(QC=>MO|jI+EVeE|<4ktV@#XjMLh5j?-4?$f?XeCp^gTLD)4CGN92} zDMBxf)jpKfW#0n>NdmuYS`r4hS9?%$5vRVyaN_PdXg3WR_@e*qs3cw95YbP7i^Paw z3asazW@qmahfT!0SN}qT{_j;UbTT%uR)|{%2cIM!MOt1W{+HA-S4m)sbrp4b*X|c! z{kh?-p#S}u|DHQ~Ww^Aa`ZF&?wR8gmU({@G_K38xeRpSH$s+xfFJGWSPD4#(Jg~@6 zN;UF8n@j|n@AT9Cjh9{@Q;lwv?>7nj@ms$szxT%}P5v~K(Tn%)l#cr*7BO4`-tTB1 zS$FT5;GZgHH;nOXi{9wAm^~ZzI1%!|i!&Z#BkyqZ;Zm61YQ7>OMkjO3A#c_(5lp%n zoXt>8nfeM)AuuM3|5z(~7oTAiTPQN9W$rxG{bXHI4ZC1IpbAl$;Nx;B8{V9gD}1uV z$_Ggbs;?kmXOv6y{Wl^4g(&g`BT^+xqKbmZrO*MpCSiNpUF z;QKf7b7IkdF_B(%nGC9%OD67ZY-+9WMO`Pphq*G48etV&Eb#Mn{x3j{;7is2EA99a z*x!EojCn*3v2Kdm=lDLJ3ac;R9!aPY#*$d$*dR+uA0248>a#@qgRl+>*sBg>x5x_$ z-ewlm0BtHBd2BT4I3G-i^odv9j^69^+>|EOeDrel0HPD?hfU$eMH*1i-@|}J=vzRP zx3#-OtO{;9Rf6&|aX#h|YnPAx=#O#1NaFl`P2d**xNs;I{29K-PK*rOxYu(uy4Q0I zCOQ>VhRO2>fzN1z&knHX-oF5I!E|T$Z=e1%B&CoiTswOr-E+#iZ+0Z$btz2Z_fr>f zxr#{q!SvR*GxTo0{)LgB0H&YkO^EY|oNPqvIWpLe0H9X*^M144AC|uWLy<&6i~j=9 z0nGj{gbWDE%n}7d9Y23Qr!KNd~{AU_E;uc~R@rpU?i+E>u*t z`=*Fdqw~djXhkQYiqMOw>NBcLRo6@Oi8ym41u?5{uaW?J%ede@!JKiqTN z{TWph=Oo#uKgN*Q2CGT8eaPvSS^M_tg<$l@SfYd>aNfOe3ce8hPoYCRzF1eX8KSSAt`?Q)$iw0MzaOKh}s|y9?+iQrM1YzhhrXO0lt9@om zVY~toP5quXQcEKaAo&H53zI+nfjGGxyzNH3`)AUH;`5JYC7eEpH|Hqg`OeJ`Zpc56 zfp>qTodunkoU%5bdpdlQG6h`9VVxifWW&0njR(P#TN^ANF1~yDyYBx>Y!R1CP&__A z_w@GO>tEGaIE{&Q+lSAzqxBL_eHvB8w5b<WWb@j`PunDi|5Tz&5G;$K0|nsLK|$7lPkhw_lqADCC3ZGg)n z$7Oy@1Zm7p@N>hHht;h&RRZ(xZB{3<)p@~<^)tTIwYOIwQgwPl4|LXhvYVXhP37rl zm+0PUP>=g|735guNj6?hJO5rhf;v7WvZP~OeAP+WtN)II*8=BBIT6{0djV&%!JqTa zCdrYf_vMuqvV%2KD!gnC5yPCTbUO=J*Dnk(kdLYBs+j-Ex$(+r?yU!(4i}hlIk)~P zX8)%@{<-)b%~8{j*<5sLxq1D?w1-%sSUB6lc_k67g678dr2HsOv7%Cjed zvvS|g-MqO&vw>eLj9^fj>EehX?KfGGDZmyGcK>DBYPIK*h%Q=wOK|YHJCYaCPE_oE z0ph6sjq?4kvGz;bUsRCYweIZPM_E=7kt5K|5h<6j6^37ODF*1aI%NMLBmjaY?!cB= z#xm_9S!0*b0AGD$0kCSDGb>~DLm{ep-b|Fe zKFH!tAoGu~+~h5@H$TMXNbIokQ(|Ah%UWn)K$p%|y=_qcBS^eQf1|og#{i9q7-g1C zgfH%FMY1$T0-Ji!20_d3Tv+`(03^b4#R6tQQtA3NenFn!@ew9@ZBGx47i`BDa9#85 zFUkddA4cmma((V}=y8D+SdI^kCa9Xpo9Ho{KN=Prhtb$rv@oHC(W1h{(5on!-(IiL zGR?m;Ile?aoVZwn;%&y|co1)idE(#cW1?ppz<>MX^mh#Z%8_onEw+l`TcIs{M)gX> zz4s=A%Z!aKVRyJ|@#U32gp>6Szb65;`_UKt4b!eX$xRMjzR;unC>PxQ8I zeE$kG}*85-N_jD{Vc^{O1P0^3xjWb(LgWPGG-h^9l|Z^coVP&}W(Q>#{(Zqk`*xl)b1C@Rw)%{> zT%AE9iF`DlU8O#Ma`eQ7XO%{YD>Kx``pvQ|{qJj_&;9C;T;FBKI;<|kNSnWW>d)Uk zy&IxwKxP2AQnEkV)~@oSf+{+8M)s)6GughEzvB)MxKVOy!?CuDhFP#c`@42Riz!=X zaSFEpv(Bq@pE=%k!mtL5knXDzPLH7aISoTEL4zNu`sIwOC@*TKE~oulwo=Tpz&L}f zfWvkZ9UaZZm*GCSOg5vEtwl{~>MTCkWg*aZzYK}<2?gP?-vE@v_Q!*%-WKkr6PlXu z6%!dW24?qBp+w)vzSz*g6CQqy0^3jAy0RSfT6!Uxi6;y!a0vUN81!W`)?NHTvZ?d@ z#J-kY3+OpeJ21R$-)r!BQdgF*mOx97UHqwRRU_rCCDyLDOm`g^03)L+% zjk1-a>i2Okppfd8(yoA}!ItA0&-DnQFSfDay^{AUEKqbAAj@vgU?%G@Vi?U==%2V# zAXPPcYE`Z-YdPAC39f-~E)?hUYOLSF_>*dvZs?DgqUFDxq5`sXHS&m~xHJU87#zFf zqq_y!;!U)^L4L>U65hV@3lE*?RJbfy7LotOU5d z4X|_?kBj%p(4ZG}Th)sn;7*>SD$o3wq3vsg&?ba^*`%pRh!jgld$WGd1)eWi)d%QVWvHADkGq=p3%FnF-&bB+C3ws^BJ5!mQo&_}M!B`CLSsg~ zEGSECNE#&kVMoKJOvBSt6-Y05cc8*StgIHa9TGKSQzY+w@4CP1*zTCEzl4c%{d~~) zoN-g%^P|xC`5ESV6H5I-8k)uEXF{F5Rm>4(;;tEK%H3k0pJ$5WczyS8om^5YTv;in z@`%q#P0Ebx6B+3{Z9VYN@=)cMTx6^DBmh&fB1FD^yDM1|Nhyho@>zN%XLv{4Y_lCz zI(PR#-?=@rf~5f^Gb255Fas1F4Hz>aRkL~%v0m;_YMq%fm}Q&byELTU_ypXHRZK+P zNSwP#0Qhwi7WG*ZrE#r%+Ut5txnXk&+WzOVqh_r+kXWF}Fr8153z}^Qd$}o^7=$E( z$^5Nj7&tFXl_W$eolw2if_nEuZjGRBx}l;er~!wqyv|4-rxWX2BJNdi?5DeVdzDVJ z+T_lr@><%8qF52Uw4e2wwAg6Lz4rtr6h_s@Kq?XE@7I}RAx`S0Z)}bkAw@F=E_765 zNOr-PSFh57+BiP>cGB-GI(bDWY)0z&pi&+0=Baa}sW$;0$=mtgsg!y@;f$UdY^uxi zTY83X6+_m4yf_z$qRzjz?18>jH4ai!Rd^;>2V(g?jxe%h7re=`57m@~O{uK(mu=sG zO&mDProAPK3*!$65r!lKEUxqrYn5ndE1*%*#Zjgj4-*Y-YA5+v7$v=1x&-`>0-q z<5;IwochN{vB~Ym+ohkrXjvN+_Nw=P>QR~DWqP#2(bTEw!63Wb)!;nej}+53co(XP zXw~1VJ^w%)Q0I|S_;bQz`G37YOT5US^cMguYv9Ry@UmsH%fbLwZ}MKSyc|POe_Qv} z$(L)S^7*_QF!E&9zwQTSIhC$btTs^EFRo8uW-H9A*GyqG`i>jU_yPCZrfGozb5ip_9`RnKhFlL!)`Q8q z#o})mH52d14oemWl7Bz-{KH)I4+WPX#}^19o{?j*b$X^nUb_n38Zn>SkkczZHR}{! zh4<}fB)FX&G*JSQ9_i&!+C^a+x}2LoIv<%7Lbb%9l4%lII$e@XG@p3EL}czk+7U0P z3US3tQ)M}0>(_(6%v3u_I{X5t{Gi=vXy}efuG4 z0#giRl7K)xRMr>}D2(GBBYq4gvx_E5ZT!DCP>537rK8uWc7d-KjNoExj?8n7Uj5O6 z%2%4d?rZda%yeShsnpnZO%8I9W^-3yAICE3;#h8Bt$n+qe0Al!*|aqLc8)uLgx(m6 zT_c~g_ncV@u@|1vxAET&<0`7U_AQLJG0UdgDMR9{?LinBM{eCr4C8}-JNJly%Tr!e zc$KZ8R?u5->pG6t>(Ra)hC7>@jnUysPT{lG$Zy5%@;53L%obxOSRzT0@+CO<=|e$2Ybz7gPLw&OEdDB1Gd`d4egRCHgLBAm8Sa^C@4olk7?P)i zpsA7(J@{qGJ`#yOUb+4ALy%Nx^$$j!(whO}bQwWk@+TR8ZeGDXg+HC;PIQ*U{ahSS zifG$^c3%}CLtH889GK_LC1!T4NdIznqmN#rBaBw~fmS;_HxrUWs}J(sC-jZYx+>m0 zgi2{<5DaeOUHt3RqYNAtCwRpDkVg9K+I`tE-NR#T`JiiaHb&fFNr`D^G2@<;&yeOx zj*65MtFi2aVY^z@_Ka%7_1jwd@o8CEY4M z%fx;TI#_Kdz}ArYxKp8DfJK2P>m*C(H(7srpC%uZm*HXJJ?HP`HG|XTMb3k#TTq?5 z!K&5rWZUxM%ZCZ)!M%s;r$6Bb|0%!x&p9eZ@v1mReDz>~f5d^mz=2tI)^7zlcl-8@ zWZ(Aqs=6%MdvBW5$_ee$i95iU&RNckMjf#>!j&VWK2~pVr$R@YEpr8ybg};v0EV`c zt61tk1P2Umxhk$sZYsJCwz$3IQnBr_O!H*q_9 zPRdncO__{NTP2^m)p}mW;)cS12crLg`NhxkSvep5%*J*rZU&ZoO^*JkJyAPBZp~gN zHXmB{7+jrPEbTI+(Yd{csyM$*PryLJp3`V{=O^nf$$>>2Q<*H|Tz&pbGM8_97q7|g zRsM+VizrO>!FKEq2n~T`#^!7t8S88<3o@|l&9ODcO=;m6jQ0TZZkN5GFKot9nCYi` zEX@IVJ*eZJ-saap$GP&xiU+sHTwtqn53f|JH#j@Y3X91O`(hhe!zq@DVHd~8lP1Y{ zcaU4!QFUI&vedhSh-)pnQ7jfW)SJ(VC&*>j;B6b(CvzB^P5nN_Df{p^1~OXJpljbF z^-e63?;AgMAt^b>r!wL5N!Z4JygPL=zuo?xtV zF1C|z&SuVli>*KNNu~@~fE>rCje)sF~iW|8OUXBI1j{E~qQWM!H z<67~)inFZ@d8>5I77N#N*uEpIVu9kIm&28k5?yjMINHH&oScu6oFJJx9zFIR&pDy& zBpI;@L^*w`?_(vN^S3LZ7#}gA7Fx=B5p3Ixl$qSv){RBdOoX;6wi`1wA23^|((k@A zqRW#va;a>V4=wZyC#*Wl5!ME$+va8c`dyYPjkp?@I??4)5uer!EXO|@Zig>qMzj-y z8%tvYbe^^4QtW`-nm1pqL&G=p;csUHJ!bSw(B)1N*lz32H*)=c!u{$|0Zpa6#t%_V zsz26u7s>qgX6%k8y}>pkIUsg#M$MCpE%&PJ-w_PZxk z{~z$d|D02jllpo1g{}tc4%I2`=RegK3sp8nf%%eBzRvj3mR|7a)X&;zl@uo@y2xof zd~P`b9J=!eD19pis147Y`tUP!*y9Q6B!rqV@dVW(t{z=A1YmbxaXHNyleEd0vvRxMVB|vtoM(nKOK7sGVlNILr`Y#xt zcb$RHtNfEk_q+;`nuMSW#0Y@X^4bpX7^(Z6e387e#ZYJT5ZW7s6+}g)CcL#b)unOo zd3LpWX}c>fTo$&Lyr$loYNAm%e?B$CC~URUqnr{9LW~s5)3WyqKwj0a5**&BdpB=O+YC~Egkoo_ceybD+FnNFs$a(70Ck(G z3kQXQy|JfA&P9Kr`Peld(*05j3=qsK#&%hM!=Kk=adXf|Wv0$#Q6CFfmpJ@T$hFX-h}d3`SXU z{_Lsr-Ou7V%9QC2_!I^XzE0u^W>Sjecf;#mv!+g%E>>3#a2>X|KBLbqmaMXY`?9~O z_X=TZP0HBXB^zEn9vCqxtG`=QNsD25i8ZBjk1WTnSP8jD;N3e>7FCiz%i7$E0<3dc z-_fU0{YWTl1hRhxDjFy~r%|$k_lfENWts*>b#y7+2SsnAS%kY}B$s$SmZEBKF+NLi zV#=lm@=bp6iuxGYs`biRUF#G4lhNnu(-bK^3y zb`P+rGHEIY!s0IG+G$Y5xM^Jdzg zT#GeXOyyg$bQZd$BUOatYFRD8v;ob`2i3}#-s<+mG7){qzsjo9SM5~g4=Jq-S^ZYa z^B{+ttyayl*S;(v4U%$=qv(!%&Yi_n30$lUaSGi=Grc|o04}{1cOp(p{pY^`2b{aK zy;lGLlDeRnoUzg(S&i~`zm%kHhwPS-eIu00DW->Z6B?HPxp%0r!&sEMM<&Tx-Lm1{ z)W?0R+AUrcjxew-`XMi~61Fh!!~FuBt<}<^`8~^d)bjz9tTQvsZ5pO2<%C-v9#qsy z`B(QTZh}T9DpOS9`h8q2Ya9u_)}OT1#~If)!}C>){0>&lx|#dC(F38i2Uk$EZ+paO zRY_`}^!o{=@?-2!{qL;9d>&M9Y8Vvp<~i7!mCoKbL4svdDNPR5w%EYbTtlIMH+1~03g+JrI6B~W|#^|~=ZC}1ff&!rD@n1t! zR*8Gdb9}ssalWN-noB&QE<^_tnamwzRN+{u)_BtNRP##D<&C#(iXk%AsbsRTK6CxS zP&13PbD8sdo?B{EumiWXR7{1y8ie7*+mUJ9`W6x{dNsxQK$JOmDpJJ6$*91B*SukX zod1ynY&8w~GxJ_6n7z{BkBS25IMPWfKwLvRglE z?JRRtnq3p#Lps;lAD9%?1q`qyTwi&KWo(H|d)cc5ODVl#_y-NQu#IrM#+2E@`SQ8N zGooAz?50FGrHAtjs!wjA(L;OQqjN%aDl4n#ff|Gmk4b1X8Wy7!B4tHez@06*mUJ z!YNyhDhu5g^=LZl2P_6nHzO~5r0t@)pY|!@S0Y!Y0-W~SGW#Z~Dg9;Pn@ERNbZz!V z&V(!ZIIgJ@KCn~^?}tQtlwfvd`MvP3c2)-@U}^yOblb zOs;vX9jIs0|KntB9Jb+z=k$>37)3^5$@O~qPcBt_uR-$)x<8qM#K_{K0z9~P`}G1d z_LXWY=zAZJ^D`U^(g?qh(%Tzt4Yb+XObA&g1?~s%qdur(yYBEZ*rIrp?)N8JF@4e5 zrvObzJlEYAwr_TUrMqT*WQMyxG);P+5hiN4ge0MWISc@pHQ6NFwKCL?sU+=HBMW*V4K=X1>VCnJU9c zrP!Pg8Gqi4it^Bnc;;}1a7f$zqF@IAfE9jBZdMxEwzu*Wr`QKGWX=pi+mfGThU#Ij zduyiH@;Sa!ZM@@-vN4k`pK~h6x+&o71V<)f)sv_QcUxUTY6@#y88p6|5h>wTo5mJlXp@(|M2y+>$?b?Bypcb@ z=Z&imEHgnB3rah>8ZL7kw)cUv1u>ZQ;o>!)|e9JmzdM`DWk`TsTpbpI-4@b~UKK9C7IHuz2(KfLQk|+PF9)YiBX5plmwl zVN!%0M}{UIscLIZIt-!kkZ1Xb)B?o zo`~vII%Ys!L45qHGFd{JnhsCKG}Pdc`uj=xV z5WT5oD2{2#_(CXL!Ch-SWbJZtP9amA#BKY{Bo7Dcl`jGFC9QTPR@0>KS0pCu@%$c` ze-cI48eQ;7^4HdII+sEyorQfmP0sy&3)+jubYsRkB~}Yrg3b8SWox_C%jdl%4|>I< z_EJ>y!|CrOfZdTYBpdAl=QSp(^(F?QbfwPs#TDnwct_Y{E$cJMe3|)pvCn;6-bBAw z0}WshM!eVn?a!DH3QT`tA~N!eYTX23oVz^E#3tRdc-mjY{S!F})qL9b7MWZ=^ib7?u!;$b zID0bVSLTc)S9_xL{Kejd{fXlAo1T{!0>u#x6CYXIDBhgLP6MD)08(l*BlPsbn)vBU zSJsX0lA9sFE2%&2j__G>WmyqvgEX^OcpaN~E~ z3mE(=#aVP6;`=54;Hvwt;6HyC-exHYg17VSIiTm{+>)6&BJ>eoj&|g|Rt4#ub zU~E>h+REQzUz*PK?v7zNlSM8Cw6Krm!TO1PXDM5Z76`rE8{ak7_QQzxT0iXI?d{*- z5!3wqYrlBt!l=4Cd^THu%C&4L6_nr|9Vmc$pAD+N$M7%*JkcP^!>KVjD2k=Y{>b_W zqcUUd$Qfg%xBf`T!MBZ6LxToSF@Z1;Q0dq<6W!wbA>9!>8q`%gKNp;o`s^vP%En~X zvBkM)F?;z9O@-flZ!J%$L6i8)8KzwgA5eq4{uL_AL`vmPzoGdL%x3k-@%m>eMpTEt z0GQl${PYLOKdk_eY7e}JsA)(&eeCK=SbjnK$5>Q`WoTMzLG5e#iMp%lnjadh^L-+t z!BpV(aD7>O5FdicKbD=tI!n|7Bj&I%V~iG%$?vHN-3+?QR8i@l&Iln zfEiZ3T%9$f0&R-dM31_ z?;Y7+_??PI2>Tp=K(9I}nsfp&k|lb%%(-})lVoN{3!kZ_8RT4ZHm>?{G7uT&ej_{M zBRn7{HPf>}qQUuCx3qFa$JRi`14kVIdn?_%#^PdDxnm=3G+}}9`CwAiZvLwAahH)p zV%)x8PYmB%tRK=e(OUG9EIHQ{XHDXbk!@IWY*J*rO)dYVilAGovU9R36wj5AhWiru z>JJ)MX$WtQUmd5eKDn_HhMQRktV@hkBDhx3kd3bkRW=0VaMx6C@_1;l&)`c$XKW+C ze92lW21!KN< zRAJSNUalN|soiGb_{j`&6j5RqrB`^|k>Ds=6}Vp0qh#jDNFwy4GtDw`LV#fh22}Vk zQ5cS)7)HO~6wTIZrdBR)&r0trFqYEqsfK%%L9gROw{50!YGzzsyo((&0`(FDb*r~2 z1D$xjs64o2p#QVgi%p+gW!&g5avxKK#{W zzo|uLu~$HNx8YjQJJ-2TWJ}$++NzjueZUId5YpV9@Mn*Cm^bVz*0l7dywvz+BK3{l zOXiR3kHnhcwv2KkgSsnEYwk0fFF$`+L_Zq0a{=e)GJS)ml{OU%j!P{^)kBT=wI^NTh_L+AHsUgJs42f+01d~J zFwAu0wR6M-!#RT8p~ZwxN8>vWp=D1YU`|C%O+^ZQ`Gre}1mw{DaJg)4ITN~Cwy2jd zAPMo?3{*9?9Md$yj8~TQf$9#FjmkF}%mV__C}O9;w~gNtS2!i}aH%wB7E4n-YMa%{ zNldDuKvn0Mf;TwrdLu*N%iqNNE4eNdiWYtGEzvQTsgq-; zlRJp8_(2&6R>gOWfsb5dLS$TNUHzbZ_HGM#Ns9UGf%-<7>FMY(+}H6vYs$n)q{*mQ zAh&u}VB0*V%4-rYa|_RYxSK#J9G&RA+<^Y64Ucsv@##_ALB8sH^%sHf?yf!Bn9i2m zi{22eWij|(l^M}7T498fp;;tV8`>Pm(qG3bOa21jjMbj3wTZEp38I{QJ>E=SrdAMJ zv+QbEL~;QD@^I)TbJLT2MOx3*kv?&!sTNTAB2oA25qa^1db>|aAN0INgDnxsJBB8P z`eX6H#Yj@lb|lG4Ie*R7;)i+pORUuFmr+)P$4#?{U$yRPM?m`=8!%>^c8f7OS3mky zYxMTCn(>Z*r+8~@D%3;!r6^6s?&kY7Zp=w7cv&_%Sm{R=V_c3NW)`P`d<9749ghc%{{m2ODJEy`u%Ro zDl-+=b?3rgfE$c6XTH=VzZGo~gTP?9ykpyw}OWR;D5r>A|`lp7LRx z3SDt7WGr*m2 z71z1u2_DbF1q~DHQsLb_XynO=~wnjUt`}!$$=39Lm0!e`6 zWlP%!5Nqk1G;CiU{T6ZmqZ0}L3PrBO_=OyYdu5I)SHT0r*5|qw?`D>iD{-e&g$U7Y zlzR5%4ds_oU%ft|@K7>5EkB`(RSJ|u0hOjHDz65xKA(U@4!k+aRDcGr( zR!aQ>n5LO*mf@A;Os2<-{OJAP38vR|^wAUcpLO@8_?kcP%C?xIri0v#etI$Y>Vd@b zr!0?uqw-fGzj!PV_&6|SudqP@kGh>cxH!`HK1Q$E?!79LL2|x1+wPZ?rXT4bWwLyo zT8`SGf`u_U&-wRIkHvjqijV75({tu;VDfq=mLH_y`Kxp9@S44Eru;hA21P+A#U`!g zyVx2B%s}$91&Wk_FF^j}UjV_&yfnTAM*ThIdEoMJE(xcW&o!@zz>pXQZ+^4a| zl$>FmVph^>mS1cx)S-I5s==RR9Pg4AY+i3%0l7s>&X?c^?2Khwv9)CfOD7~J#d(&+ zu*e5sy*2f9MQIIWc5rbrlYb)$@#liGmf}_!m7k{q{IGS7>cY)v z_JE4@IZ*YtX~WekV4kRvO&dM@nTb0vlCU^S@{8@Chc2_F-!xRZ(V1|{v zbZoDL}OIe2$pWYu^2fDAMO9i3xgui@1^KOGAofpR?=$dyjz+?B*QHcs~M+{~rfHwZG zx2CK3o4R6S#u8HPDfO08B2G%(Cgzl*rgs|1eiBmoFABIC?VrKcW9?^zV%;X7KNh-{ z9U9BVFweepZK7o)c36#WA*#uF zj&;IO^|}|PV{mn7*^b$4Dv-qf*+Lldqk%j(>7tl5{CPKK5N~ARq5h7Ff?%Tt~>mga3s!D>F;q z_1U(7qou47r{Tzl;wJEeQQ>^8^4e zT9*0I&C9M&!8UkCgGM&eaB1K2eN7-ApcVbBY{H-x7NbO(i%*R6nOi4no}b3(XE%7E zT@(y_KNny1SheES;(qwK#bu$-wR6+x3vURxGWAX=QgR7-ujpC<%nSI$GfBx)Z;(CS z;CdpET^E%_t*W=X#5AfcvyHp{6CERjlITa8Cx zM_=(DUm;un1u!d>t;82?;!TkXH1@_w!&$F_oep`*)M!`7x8AgG@t2!H2pM&QoXT-fVtlkw*Xm4=ggSWSzvz4Zdr4~4C zf0G7?q;7N6OVDp3cGf*5NK>`CcDt6F_Yf=h!k!8g>X3j4XSyF}Eb$j=q*C$~{!L~X z;%54OqTu9%`}IIMvub0>MCBm7wad(%u8f%0=p@Hj`8W0vg<`FC>8`y2{^^PvSJt-C z^n9`3sV#}R(AQ)4V@sr0AjoggDE)n}vB{4<_7iFNh)IJ>+OFwnY+gaxx>e>`TO)+= z<#YnWKAdfM1uW{tD6~zJ2nhp;f~Bc1{B+ijz_Qd>w#;%YIRd!@uJ`1PTBbu2uliiW z2FZ5U&CbHGJD;z(*JwvG%LQyWZL7BG(GX?NE=uCrQsnW7U)JoIo^*TgJb!M#$OuN9 z$_r=AHMHX~RiUfXr6SPn*_xKDhq{tV@fH`g>1}Z5XMUSifw4i8R~EYs3GD9s7k|;? z&-IEcJ{a7{>D(C9f^T?T(C@?yuPdj(W7KAiK-NPJxR4ktcC5*6EZ-u`>qov~hqb#INBV@~Y~+l2vd?8@O`2 zUa>Ufi@@zGc{7St*gPCP%@Zx$CN0iRhX z6I=BQpHFge&&n?qegklm5MV(lKGM_RHT1sM#tWvuo?$lah+CMPQ9r*c-gvyjMNV`A zaOLAceybnPD7yTFyskYoM~6F7$tHvMUcRqQMcJ8Ke2){4(QzyTVTP@+BE`V2B#~pP z*=0DAV@DWHx%AC~-6}ntSxZ-5m$KKd5c&)f?AqLx*(=hw$6tUEq|Dk?9<5_%KH1=* z33IeYQnKZ^W!k3TjSHGm8wQHIkm=&F_7pd7-JJcm7+nxziIS_H$sLie8QlCTgEIBo zsGd_+giS3JDW0_hiQf%pFXKe%bqhE08m^ZNNy$5jan9*R5dhO8KI=}9xvXhv>&3|hfFd$e670CvY8o9 zRvy`5h{$z&b0SSteSFtNg@pRdRa+(%ABNxSvGw0#{y62_SKGL`S})hKAzSZ~+A>#O zd&j-A_O>pi5J+*lvFqnkDGo_03x3KwDOZtCpVpe+q@Szv=lH@wpZ0Iaa)Fm6_B^q* z7z3-j+2Z#er{_f>%{4=3-e~$<(EB#*#}LqKsnIUusNik;@k%*llIZmH`mXj~h^tM3 zo3no96#D~(8d8wdXpx_Po@k?S%bBLF;#f;rM{*DhFd;K5FvL_2pOQ>Zr#b4I#U%9b zukH~Ny!IXB(lg@&?{7R$agSq?Z#6$2g&G|E0yN8~H9UIs8@daLd*ewn+3%)Aon`X~ zJzb6)&TBEedTboVl7Pn#HY;@yby4k#nRq zGE1mWPO{pFZD;BFZdX(jQl@EHoVbGUXy_-t==FNf6C+X!k%o%7y?S>@uV{OzQiW+= z2KTCr4shAgk;y6irViLG1N8CX9h52u?86+&K&8ngqBK;gT}3X&1@8Q%?<{n6!pmzs zJ2OInwP95HqEQh3;|5hTR_SMKRE@ljtED4JHg9Ecp~CMt+C(#Mr)n~Y)#*5GuSr*d%c@cx7HOf^mXo4f zTo{|J%KGH(Hp(uPKPC!))4^)RFw|x`^mUc$fkqX_oXLh)Y38-A_udkmw@X`Zb7j&S z)hCN<(rHMvYz}G^SYKN>{FX;HV6K#7V)tdz{(!%$lt>Q1BE<~VcxEjEan^U*Q}g;2 zaVh&ha2;~(aiED@)z;o1NmUW2*@i^zWNSq2NZ+*obslu5W^M_4Emm#5um|Oel$ffh z=)9w0oVkN750M(a`eWkx{Nwd4ZvgJM(*RkoYp;E$y`|<`#*VuY#f;%jS$yIAW0rKntdx;i zJAF}(iZ!Zioyv~^mtLGa_{yKKe-_yrn6cIoIlIZyxb0g#=Y?@6OhuU4l5HokpIzar zqQNn-Bgrf_uY6{}QzNKrjNcyNSe_Qm5s$b&+7n%j4%uGCG~q--yWY-r#0<~WKGPwt zq%0dCskVL8&XGNwnQ~DJIT9yrC;HO9wv>{Kyt^?@#r%vUl>ToFv0+3szj5GqH5t4TCHEVe&V>63GTn{hIw8TO@|;FLI}%&g|v_@B%1Zi6>4cZtma zAXP@>F%m#qRKydvCPbYg>w@=J0OZnfkDuT%=~gsrlc+=$L+EUV8Phb-C3Si#=I5Bx zM;L7|f?B7iRb+inTJr?Z z9&L2nI>tC@pwKpwJYLx|lG)-sT|l5J?0u-1rwGNpH=!PjQ1{fSO$Prm$v#*d9{L+R zz=hLs5XUNa%UaQIMF7Ry;8T?FkTX`Yq^!(#NP;87$8v~hnGd!MlYyjww~2ecy*l96bK49w40wEovm&5l%-JqV~RI1j~e=yrC`Y41ygQTE@s&9i;MD zBe{8UK$nAF0Z7&4^KeD<_?RNs%FU^~QBC9&->aCK2`Z*KEzYZJ(T`wboL1iptOqkkIZ97%-yQu#}6vT9y#k%J-6=R>`=f$-rd!IOyQZv!|b*5__d>f)LS}f-&4;uvU$Cv9qrzhY6d? zbYVPl^WTVZs1G)ZXbZ-oDUlud3!SZ!n{+4pM`& z1Y*exYa#=5z3v&8WAvW$UI<3$)SIx()HcZG(I!9sXkfk3$m12gBNzv486&4DDB>v0 zZ9-i(s)@Cjvn7Y+nC+N{%UJe;c4K71*<8)g#;{Zv?Q!SFO6KQ4KVV@o+l^7`VhQfg zE1lWusWp`c$r}+RAGXLzA|wN*_3%}PxNGbht{21x*LA%~M?RZ=`#Ui%x^`qP%s76H zoxk$=TzLZ^%?!8{Ay8qv;YwgUeb%#tz7-{>NUb%*m>`?+4hBY#6?`KYj$(Wp(BbA4B2ml zbo3QodOm6f$`x{HeYVr!n>F@Ks93qm<4Diwp-tO#uEH3sqzTAzH1J$_Cq=1|oHxx~ za}(_b{RIe=e$wa z>oFDnrN7z#(f*T@515yK$NI0_Kc2ZKdE8O&I%WOg7a;ZwbUxHxb}8-d@4A;@6C;c# zjP}9ZSN4hPW=a=qzX5_I@6aEw>Fhj_nz5@-$4C6@wM+&`}Ik!23vqU z-vrJGxnf)fjWXdyYJ(j;T+t5^Hnp~QE=i*r)%>;fvePrJm(iXI?N2^4)GM3GYKr(D zw7qv!Q(w0>8b!qhSO7tqg%;Quv3Rt^@+e$f*fKG4x zPdbv4J~s4A8I7%JtY8Er<*2-_3!}1TAu0(C{R0yI{((iWvKNr-lbSs;a8aNQBn9_v z+?09yUnYTs)rS6nz92V;gR2t((VY$A1j#i z=@JUoRevf7tWU1?XIqgzUJy|)=C*i~+&=9x*psDBS>XM#yz?_{Xp5#lw;0)q!UQ$l z1Ed~KOgp_x)>qL%*OH?)-+`UR%BH@{mgB0n>kq+H8Wzs+85-kFUuR(A<1as z(Hz72#n5k%+>cH3PF?=qQKzl_JGyLa+-dM~HEC+2(1-Fa*;h5VYb9|VcfLIbYOo~? zR+n}*JoZ1kca*v5aeb&0dR508dj}@cNm)w66ryTTqC+jyIx2Wv>LSQ;?||b|@!4vCt0*(I{b7)_vyHo}c?1o&_eW z8Krtna^$~Hc@7z?BnI8MbJ)7p-07VU^Wf$-L;=NZhh}aYF za_$U8dxs*Jx155F|52jc?QQhzjgk`yn`(Xj9!tT}l-r#Y@Ry0hvXmz`(Y_LTLcIdj zBdeE)KLeKh<5owSW!Q5d@nm2H()#(=e1MYe{_jk>wBeLTAxdpJo?2as$VSM-lNb@O z-~4nj!%T5yoDEw)DXiSCf5&0PYpHvI*6S^ZXkBN`wdRDJq^_Lv_!?R#HKKCN>M5<| zs3N7u-JN^R_XheC@E>R-qhoESadJ0yyQ~{z1bOk1056 zrdVri4bFVq8xME#y%fTsbKq`KXSfJge5x~E0Z*N`UBpmxL>k|;h^DsarwUp8>*B}3 ztJgOkoZj?j@ztw*lN-Pe&_e+vMq7R(jai%D;+F_WiB-EWRtT+YmzT2U3U`^a`55AhXbh1KU2!M7vv8@1dno zF)hQ9*`p=cxufkv&N`}d@qkC3+$$`)gqCNkJch@m5N%|`G)2Km)7fyfR--frT{trn zsRQLyT#rDbN@we2&2+{LeXqFFs-6R{3}@h#ZIT?Kl-bc*Bor&QGOBVr81sZoc|V4u z&i+VUt^EVabxn-JdjYJ(?O&p$Q(C-Cx}zr^u&4=T9PZ;`xb1Y);_Zb4xV2Z}_TG6A z*N7ezHv87rX1dEpIYpV7l&n5H(`!ld?StfCg}iCcCwA4Sb@pFiCK@}3tiyCDPc+r) znLQL+oDZj>FTDwm8JJ$VVfax|)RBA8IMeiCr({3dt@ByD0rxW$ZNqjU`NQ{oHj6;L zcb7=#$*0{+k&{NVE^#LYeS>9yO2RYlO|h&~M*Lf-azY=qNXC2?CJuKMSWWJ$dK3%N z^mCY@*w}gpR`K7wEqcL>7Q!_24 zW-OB=;GwR);>BHoj*>cxd2n4bxKaT^k^K&2GBbO3gh5Uy5ix)fn02sI*3|Ch6kM#Q z^RF5S_d4N7Bq~WK-ymQ-)csy3V0^Jq`tE$|&-uQ;EFytF{jHJ;>D7 zO4$NqP)UYbLQ&l9@`#>t_JVK?F0(aGD(-WpQ3b1n8y+H!k+E{?PMz5I{O!uYcY(!M z0dg}Z=mu*Qfh|Zr3r-Cni3^jdnBx`Zy7USz(lJz~p|p#@raT&85fOJHR&C4k{7J{1 zr1j`z40~)FxP=@SJRKglt*F^i#pKCCBlQ7p&#;A#5aTSfN;HcDYp;+BfhC)6mY% zAQl>&TG5)O`k=RqX==#J+iGoE{`t2C6Waez%;~=W-po1H=zYMJ5ZR8D^`-G;5Usku zos?Z*!R+?C=D_-UpJgxhh?iJq!UJpDh7yjx4$NXdeinlQ7jM4vx1Fu(ll}u@IIcd+ z@YN!I_8hnjB8HUb2p7`PomE8BuJTy5q`!->DlL?qTCk)Gt zrAftt=zeL=xBzs~3Ng=}|I-%W_rMZq?+{$^rPwGMiPb5Yp<0b>g=LvA^oCCjxZ5GM~3)QK|Tp% z?1xcjm_GXm>aGqrkxn>Zis?i0;H~9_fi%m$QC|w?s_S|2Rb=&Pdo$vg+g>;pQm3eL zR7JamH*Q0d5~g2|g%x~uq%qekwoM);QVrHOT)2Yi;+*ks8Kt0j>(CX z+(#A2Eqzg>ImT4cWtM!z__pTA5U;yL$D`eGC8@^`&n~+caeHkK`QfH*vt!FUqwC+~ z_xEX+R`6@v3EaZpO-RDh8Z0Y}>oGU(q!p}KhY39FmXn0#)8@@(cP%%_NrF3HZTA6* zulaW=$J=8q9h2yhvg7^FEBDgwJJ^d|rf9O)V0rJPM;N)Mhpmq>YEr*;_l{F#lo49j zDwnqA`H(WxDTwy*e-DWI^wLqz)$ow7+m+i@ z!A~~YYn~4lYRk(`h^JIduHTC?<=Em?amGg*$}1_hSa+t1yP_JJR&f2gK-I+-UG&+# z7N*+S9E|#$dlT0e7+v^;{P!teo|+V46`8fxpiuW^`w12u_0AEnKgoRg8?7B z$|P{x8lvrnvHB95kvMvZFsgdL?1*KX5WP|w!8-p z$vH{x>M=8ywl{_qfA`Z-0#Z!qlTg3TL4Jq0kr8v0j_+H&i;(L@BXL}>*Aj^F6g{aD z#f<9*i@_@=oa{Ok1oZ@*6W=9v&?W@~WSH7AuwNS#3X(Kvou#z7-V$^D+}=+jw%PFF zP%1}_fMEL=70F-rss;A%hkaUJyrjj{1ds1k50f34A5IT0O%sdyYJZid^3O3_7=alJGuB+ARTkbqGxn zXK-6f!8{T5A9-m`ausF1i@Y?wGlt$9NKNAqUPR&Lr4Qr-=a2ST*6@b&X5>+0T-+&Z zTs)}S0|AWM+OTuO;M4Kq1D1%KRTA^5*9=YVj1@Yzxg1?U{SmFx75%p$Vwu3K{HF2A znm-_mCK8+EL(5w2cjc!0{KPl&6Fvx@Em|Mvp`%RT;%2Gp?n+qO^ucEQhHE-v&=o_C zi=oQ@YV`iXr&))659j=(BKYL%c8vgI8Le|)tSII!QGESK8t1XxXt__d zM;`m*OT|sU?^8#!<%hCX8fZdTwA92aP?7My6t=>v4eCZNnvp=%mTwM#E#CQi*1pt& zNRB%mH5iWh;+z(Wa~V}T@gEDR7sxxyeCJA9H&Xm7Ta1j`Nxht6EW#y1>6&Uod)c!- z$(~;H08Vg-bnY$S3jh)ztn0IhmGCN&`aL3!eAvfNK9E;Jt(BY}9&0viJW+LJ3uJ}-Cp8o1#lb>c>RDotNyeB6S(1SGZqs@L_O z4BJ8%&YvMI0ico2!6+cIZ z^pn(`9TCQrzE|E3?G&2oevJk|nn_Y}$_0P^oRQ)DQH&nXnbGuHfDmje@L-wjC`iJ* zZQs{}=nu%X{SOHHS!(|Jemqbgu|7_Ah9L%^iNK$Jd^)Q12L!1%kvbG-IBU96>qCAl z=9A&xF%px9t@d{7aIsU-e9;6o=N3+5G*`TnbfEDt!KA&)`BAaA=Pi@8FdO-U8@oa5 zuDu)az?KVzaQP+r@qZ;&dRw!lo{qolSk1w)C485eT!O3A;Y9V!}0~E?9?JW!bS}3fmt_zX^2F0T%PT+jpFv#i>m9{7_bG za|)&!?>;D_Z}n>8O8s2aI~OADxv3nO9c&6~5S^Lc=NjMDH=kRG9u!a?(zY zt(R2&r6ml4UMePss8)0cWNNbyP_ERBi+4QmNR0}sE~qngY;E;<;3L0};7ucO|G9oi z;CHrL{1T>qW&N z(52LcT-}61mbGY9wbKpgi7j=)Czw#7YM%~Tt z1CtZ3X$KC#IrZyJ6Q4r=#{idFi93OLZJDmCb8=e!^2l)WQfft~faAo?@0u`Yw01Gu z*h8T*heU(N!O4y^&tDF4>u%$o1?5a8A5Q-w$2v`S$la>(2tq>z^>j82M3e*!g`J|k znX7JP<%tx;cNa!00M+TZRBs%d8rNm$in?_A_r_839vk`o!spQ!>VHcyN#1fNa~yIU zJvs3t1*-%bsH7aL3bdq6^rh46K4z$X_OZz2cW)sJjed)Mx-(qQ^_wEGp%ci-XqMk$ z4cX_aJDWGnGD=cj$vne2boIVp{7+)`rfNiKS5@yKj#>1d&ky!Q2o;$G$Mm^!Te9uk60(mRaZzc~{Y)$osI8Vu)*vC8;91%9CaSs# z8W8(QI$M{0?;5x7gL2J8{|Na}c(1@sTA}f~)|PYG5Mh76PEs#SUlJlq^UK49!`OPv z(Co5}_Fn0r?mD!chRke}a;3Dg&vN{JPfZ#WtmPlU_NvC6TjA<7uQ4w3NpNYH^+!$k zC|2y1KA-nf-O@25@GUwZSn&^({ts-Ah5SXv?a|vcEiyt{*Z3$%>q`+O10vmmhuMft z4s%j6ZjX>tJc_m8M30d2X~{lLPmotp$+`>8kG>(gtDQg^cj2bL?Gi?bAi9T-D$}dk zEDULQ?|>E~bWanFFW@G&l6klWVmB8A&O2j!u2lDQ^eC2ajv^5WK1J8->@`LQN-Om` z7OQ@-S_6eEEGqT}M&a1wa*x;33*R>L4+L4PhUw>vx_3q~FRElGO_CRNe&@AP<+G4u zId>G4$xpoc*88FZNuO^F<=1_%^0ZYkXkt@ZSW2s}ML)IHO2@~0Th3PpmUQX=g7h&~ zp*gE`*!8kf%n{x!7_e(U^#vhOx4Fr`8}W|}EYGx~^CTL{Zls}O6VEBqD$RJDB{)W9 z745vNQDb$6P80ixrya(B!oFI&eKrSh%cYOH+PJmVVXrz{zkS&Nu#^R^nCxws0?+{U z<%};CcHRBU=bvQ1J79w$D<=c@4Fta@g!u5~oY8A)3yYN7gu};g@_NSH9gIsI)M0)( z<8T6x4le2bPsf9_HdiP1W*sMIKIUV?$sckHLlk%u8Z`xMklg^3hQVz-CE>pB`Z$5U+j_VI#=-?KB3SQ|2x30lk;cKcvEAs@>a14ozLvf zXo1|mGX`vd5qCz>0zt@g+DqT_M`jzS59^0R>ssJ8EQz@<=4x|)3HnAF?jVia>6;Yl zT1H79vY;fzlr0L~KYHWQlfd?&uTKHI2vKU(!@Z5#0#L4cd;V^1=VUW!qojShS$t-G z8zEhucGin-11e*puG(B&vZL!1N|9jq5N;U3{+6bU4ul->a`Fjny=|)D{K~SyGZ1Z- zWVyaZt+C#%_p4FEawG|3XbN{}<>5YYXwEl!GpNfPtqY3&t{TNMhrpsZ7(=fOE1FvD zvc-p7g8TvoPIP99umRA(7I5l0ICu_0?HcW61$<$nbr(?=ue&H(Trw??_Sy#C9=-cX zJg~dEl@D1?j7Iw8*479o6w6hXKKCTezyH{|%Y`n8wKf{!JMaaItSdBmGe8?3SWy7u zQd_rO7_jNp1S>P!YhLn{Z1!j?nXV~d@*@cS9uey=PURYLWh&+VDU|et-wnZ@*NdWl zxNl?ITt|hEAvJ};-YiZOA7!n_-!hB$yY@FNe{s)%(*pge;{f_WY>tpY$sCd>Z~0Kr?Q4LnxniYVY50O z(W@Vw@&bic?OVO&M+MERC7(Qe#o1-FF{!>6Em&};%|JGaN)Xl_Y`ONS${|7)Hu)sY zvh<;l8o5CoY#P|tURws}AD@Klnxrbs0V>zgBHO4-$H4vWonfshO?zEQSdV#KGF#wz zJ=sa97utW553ODOLP{;s_?6mokx-JsU1s%aVW^u?l|;3by(Dq z%x07L#T--4@wWjBi+ZCNG`6s?tTkt)i^|z0r^P2bEP^y+=RrGl!|w-tLJ}@zHa$hh zZ{m@M0#14V5wG|kgq$yt(zih?T!&&50)D}k6)2uM983pxMUO?)8N&zVvBK|z?HzyA zyg0Q8W$o1>B#hEj%Q*LKDzWz~vV@1@g1keiO)l;M@ty(`Q0;x4F{`g6@4`>!Sw+C( z9q<-QD79r)xK%QhX$Y+=uv_R70YYorU+L5()5)au^YfRAiVyHrdk0l5i<8<2#;4i& zcV*`Nt+MrWx8t=Mt`%h;@J?#Rk+x){S)iDVI&&7gTHZoS6m^ov?$o{oKKYWoV-e3! zoyxk}vrPc+p_R9lmi3PEU}TcZMki{H*}x}?a+V@bXiq>BC^0GP&XE%~(%9~>JT>zv zs`nMerE@v4FWQqjT?_(5mP^9RW|SPB862?6ZcC~V`CUAlqT1m=NP~A!&1|;XnA8hR ztUUcWB7|Uy0W#Wur*;g;lY3dtlIoBpH^XTCz998Z>-wM@4H`&OP`w#sUwM3Ggl6rs z2y@mr%Wsdh$$+b`jv!Jtx@$vtZf{hq&@a+^(_Oh7rM{k%Q7+Z5#<`xa1#IQAcQWuy znwLT>Cm$tRCnMr{&gv#A7HP$YH9QaIQ1S70{zqF6e_hyYWmN5<=DbhjOHl2xzU5ma zgdHAbEODnHGLLtO=5U$#?v|rm;~ULq#{;eW)*4@~XG;Goo2`IafvVoie0nhb5|9rW z(p()}&a~wiNtJUk*WE)_C_d>D)jLZFWAS&b|6%pbx|Bs{LTZqmP{*y+{yn&$v@wxW z+qo(V7dNhh7g^eUR#Qu3^7_EvQixE>Jwg|%k}5}h=U`GW{vlQm?vUlhp1NKAlg#`{ zW_GG#Uf#`Gk1}e2A8Gq9IZ|lZ#VCOG58Ox~(hNQLsI&?-?Lrp)Cjgb5mG)^j$1YN1J?e%yT$&jYqc-QuE4>uUBt z5tb=gSlKZQ9~l+^cJb@} z{02i3gyptjV#kbAB*9J_*6CmY=zn^L^ropSc3lTGc=5^Q@KKo(?#_Gr^}r3b_W6=?%qK$pb$jw#nTghr?A}!gV;$W!u>WB z6?P+A56Ku>owDJ~14K)rx$!XMDVyQAj-h6z8YKJ4bRvhq?Jt1-|} z(*i)5ObtAhtNP~p`BavFmM4xLcRhvyl~7mHZMR0iG<$MXE=8YmD`~xN{wg!9wxsm? z&szc_lY$kEIl3DNKAFAK->%UJS+cGBNKFgYk#szVn;YrGtsOM^@aWXOtF5 z!+x&w+D@k3J=&t>L)MQLwx%B(RzVFj(H)_wJT6B2IJO76=BWv_<7{{vt&*d;czFxl zq;2C%-ARl1>$_Kn^tqWFB8qF( zH40cTCup3qKPqBz;2r+gP-#&67jsPCN z{jI_NV|}dw@M)7NeoZOnSsk>XAH_!Byxc z)M&O{L&saXqTiI-hX`1FT{^%~IZ_bFr0R_qCL=d{7+wzLfZHKGvic$FGD#TmV0J(k z$Z>Rc45EA>B`;>(9<9Vzso_BewU5^oLPb8E)@0F~@Y#{$`F{h>N$U0oWH7U-0r7J)EA8f4&wVxQ z3ncaG+8w| z!UZutJLA3%SDQEWo8HdU*9g60OkiYqxpwQc`0&V<#EgAphP~FEn1|~U*s-?8Wnh=u z7ELc(aCXJH_R%(op!#==zpJ19JP`5L-w$CI^0@3FxF@hc+ecXyyZdY>O52Quf`jTe z(}a9icNNsb@l&(3W!Yp&vF8oe_>{SB=aK!By;l4nu8nftT`|Tl6Y5DrQBMYC5b5z$ zkR8g)r5DT??UOI=GI|9suX?i68t-7(l&p`aBO5Iv-f7(nLwDLe#w@LH;fzJ4Z zSl##Os}&uSj`>A#0!6(pH?}diSZ7p5mnzEo$TqQ-OU+HiW*s3VJKw=S|Mtpte=qCU zRrXM6m}PIh@>fF2V8+v!x7d3yME%@%?NE0WDIcudSSCv*_FL3^&Kzf&uv9&hhWZ9F zB0SEv*=_R3oX+6uMNbT9a}B5X0`7Rb&rRN0RUD?_VVv=wc7wv$5q=gyVTB=v!08Cr z55{3cE{;YRkTdf5?U`7rkHc|-eG61Mq>DjzUF-F%{?%g3qc1e~%XxM*##?eKRJg%1 zNT{nvc&)~hl!c-gDb5av`HXY4M&WMk3$Ke$CMpshm}}jc3`mG}SHhpKW$KIFt#K!? z?(=I3Vzqc~v^(f}PWew*h1{N*@SvDZfbsU-uGFWSoeH6Hi7)<8@)lRT|73r9)9}n3 z8S|izfs)u^>dDDn9rtiQ#`J2gk!xLZR(^2cmZsp?c1|C6j|h$i5L`CRoW+~Sx1BGe5zVasJuC( zH!G`v9d2EeM}1}4{baRjA3mU4VIhWGhK2=2bqZjm4~VsN^7+w>M*!DXpu?CUfh&qD zzIm}r5uDT{;d_LOW@}UknS>p<-&e8SRhBV9v=>fmE*Fe)Elq zkpeZHX7Ioen>VdP0B=7ZmV?IUz>Lsoq!ZFOSIjy$xw17LtL6Kr0W z7U$EX_11h)Z|jmq1d;XnZSl>Hjm6IzuICK5Pjrf=F8-*R_hshzWra3^JC(-Sc6vmu zkFnx(sBm*%CCsQ(W@u3>zW+@&7Y}`EFH5xEe0Vf31i{;Pg4uLACE~d7*gvH2g@uRm zWxh<5ow#A8B`p%)bC0c+r55NLya`P>m+78;3`RDpli3?;%S6er^1-e;&Xn>~b+9(v%(tv7?oG5*sj}PN^Yu?zoC)s{*ukId z8ECG>e*8=zeWtJ~!n{o$r{H(*>(gcsNYvtt+~Dqy+59of{e<*H$U|cL1YrWhBXohwnAl1;*Md%AeCRV7Cd2VVCF;=2+PB<-K(h+X+ zYl=Jl6`Zc@xcv;3UJY#BaFNa^dJ%kjRUZOSp7y#BoJNS?V!o4Z(8&!(Hep8`>h^ae zvN#QF=io!!t>ULP;q5+E&iK4X@?PI+YnL;W5H^YkpFZp zaO?70%g-K07wvl|j4P+TNl@)}C#bE5DrLweA_Jp< zNOzTXQPDZL^l*s+2qXSOq8mh%G&V9ekR&>=`eL?9@S|Uf9rv1+bK6$a4UXVzX{jNJ z+M4zm72#t8*A|T?(G~i%jPiVqRf)v8lfGSEhy=2zSveDq#B(485krU zy9C;A}Ml2^sA+F|Jh^!rljFjY}}V32}%m<4r%A$LM#k}-1~ zRsM~<3C?Zx<=1aPAg=a3Pq((c{Z%HXb&6;<;Y5cNJNKomDG|Auml^1WyPtA?zFwq| z9CqDeD>=vrV-a_$ssx@zxAH=6mIc?f4+8^vj2(;ktac{nFCbhNRe&;MX<)xzG4qhF zympiO(3Wt8(U`xqLT$T)p0SQZ_QhNYE}gwdWkS3$>vw+uzlGBekzV%r(h=>egBQe| z0&p2azQ4qsPU=jWn8?4xo$34?k|GzVFo3%g0scj{1PuLeBb#7&bk4M^Ad-)H%J@k{ z0ej>6AqSsd&m{F$7OB3NeX0YEh>Wr`P$aK;6<`PE>)-~I!Jem+ru)mzl=|~((DT(# z6qe4Fzj1lVFc4~W$JiB;UhPmu{q*67GMim!wQ-a*DZk~UWMSoJ-8$PmHby{VgKs+s z*9y!5T~B3S&^%@EbnmNrRgb4r`g|x;J|i`=jX4J&?fK*?+gM?=yJPWe!NI4!+~NnKTBj;Sg=XQ#>HlJuJ$=>e zGll!hEIU_~j!PcTGY*w{>+=>EJki^~;mpgJHhItWgq-tVK%H**GtzRCzmEA|fh4+k z9XAPG9^=u=h%(g1w1wmiq&_gNSDKMFzCq0%k5prxePg(j9oW*<2gO(6qaXD3?eOBd zpl#wG%q}bj(ZO1iNDHe7jyaOqP2y2o8)rpzP%#piwt3@DV|Y^hd`xL-9!}C*Q>3=G zlfB;3e3t$`Y*?RI0a0qg^;I#Fxs@oo@#XOxNV6NkPJ2DQ*Ce!s zsjCdD5$98NczrCAchNz=+%9>PGKGToq{6lv&AGyB9YsS$5DRBsNvKLNdWtm_J=7 zhoCt9faD;}-ST9uqYC_BMdCUtc(gnP(cH4$SlA62-~JI?=0Xa4d(q9ULw`~vk2vGm zD>p*Do<(q;gzJP8XQL+KnVb-}(7OI4fxHoq0QX?-3FK63l0L)JKb#9sFYK~$igmkr zIo4PYlfQuYF>nkS7Bg;Vdkh|{1rLd#W#!qC{Q~>*z9}T0R-S+&WDvksmBz+%1;2Kx zgYKGgSW)&9!E-#Ko%=hKix;99f_-s~z)0Z-g4$ZdK_RkpvCBD`&fckFpj3Tg8xG!1 zxJ6n1Kb0Bhm$(;L>zrJK)hPgAvjYgQ;%n~8Q`_c3dq2(~y z`)2i#Y67l$A`&g~_7p-TRa1;*R6o|N{nL8=L=85~%c%tB3ZTYAX{)w-*iFLZMr&$A z^D6utq4~|esD5aCx#)DSd@hEq>dVFzKG{$+_0{m4{)fz8TbNhCO#QH|!v3lQawp0s zz!T%Y3%cE3-0K*KZ}FMgDZsrJhUQ5_Y}{$JmKE_D-DYjd8y-mh6JDI#&Uj_$~HKi=$OTf$E2IS542 zf7%|j7VaQ5Fkf5;_m(+1AA(&DS@@{&aI*UgH-Tl7SUToQku!v%P2+f$l_mTxe}-(j z3WnMO&^0nKonXH87(ok^ria^am3+?9Gdnxb6E%qZ16t?+@qGQKVf~{{9Y?}GRLlcA zKvR9a-evBJ@r>x|z_ENumO_h7$nn_b7Hso2AL;?umS0cW4|Rjcp-!x9zX<9>tdclU z+%gCRGgd2NojtK1>3te$ur#wi-om7AOzh1QtUOQ!Yx=?0(1@-->L=N0-!SA#2;4f# z?>EdnzaRO0WAeY;1ahrCo?Ef5oSMm`xlajZo*ydlT<6l`_O?F|ll@>j>iUrv73n-T zJI#RTcEWK(Fw9fiETIw6H@9}vzlA>nT^iV24ms2D?Ep3dV}X77Am|>@ImCGN{qhW5 z?_Sa5f$f?s&q28c)N~^J$w$FC1!BQTH{Vg;AR=P6?Sy9C=8-{44q=qAiUSi&YqE*` zX_w8D#R?Z50KV6_tY*jQPrqShfHZ@24sN=IjFuYYW>d4Ka>KOB3@z?fvWsw*fq#Vv zv)|5u7Xd6vjx$00H`y~q3PZq7bf{I2qV)Rc#5g;I{Zz21T!Jo>9fcl)@uG9* zn8u6YBIem(R#X(C4wLST7oY@y)i^{TDG_jtA_kbt^c{X}&%|h!O&weo(xkQ(ThDl0 zzOR``j$yuKTdErD?<|uTqr_P?S0(hxb`rv3ZHl4VcyzlRX)tS{Gtsb71B$O>zTk(&JrV|m9Rt$=T^d539Y*IfO zY4ww{qbnA|h4wZ(<#Q5fo~WM+bMS`MXl7A=;C=pIcm0OWUhjU)MtQFP?~<9|wua2% zL5-;CHTtGmy0%Zlo*@Xlq)WkfZ(1(JI`(Kim%&Xn4Q@N)8v?3U__=$oSGGipk0Cyd z?q(wiN&#^-FwA09g04dkN$1#!a*j*dQyt|-wcB2L9_v+d9D=0UV=W()-pKMg-jz({ z;P)B4hG78fhTc#D%%Q#X^};Ah|Kpc!und+%phk39{jIz2#YGTkK7yckVsIw=2NX$y zJ8MX~{I>+)hUA_mkitsO?79|xJ|%N$5|{~~f13)IDEbd*xI<{Zoe(-pyrC{bK{HV^ z%q;Tw@GM2RWWQczbdoE%5#jCI|L}VjGNuFZY=;~&*)DMYgZDbiU!y#=mN)9{^F{Rl z0_%bNppYs8edrgHAz&{q;8JM2T63?V6hlhyk{y|3hH%j7!F5s^Gf}8xllXhwiERG% z0D4+If`tl9&cJ8BXobo%FUp;XuaP%vC@vswxSAK*DN%eY$|~Xr@nTFB(xpnOHdsFy z=#Y9nH~vFJ9Mg;!h1np#C8-09|^AcuINEFBo_S z$L-qpfpQn0X%_7@j}RB0yYjmuJ)uYD^50|)h;;fqe)w8b$Kc`6XyOE;h6aD$w3{zW zM8KWx-3ncI?KeG^(A#OX7C{A#l}f!L3hp#pD_u9|kt2rU>Yg{>zkdr100eqoK0No* zE?`e~qv_`7teB!WpMc-@nz*LK)YFRKC!_Z?Zf#8?Lrz^%_tz|OQw`3W!qK+By~VM` z37jIoM2`M;!VPRp)Q+Z4*4+&j{L52TPE!7W!cGHE>UTz$M=EEZ9-K9wOHu!>?m7)P z4VXv6LMF8E#fhrxG1 z#np9ASQ*NR;CKq`j|=-rY1kv)CqO{B2*S%1rrI+mq$W?vQtYRiWsC;-;kWA zG*K*}=_g4xc=TWwxb??p-*wZCN-D43hIaV~U)H6h6GLwC_mY3jzi`9Fw|~#>IK@|L zWtK&AuqDdhdJV5~M?GDRA|;HQ(PQS*$3;Vp-F*sxBn8|j{k^U)eQN~XL(fqxY&|Jj zC@6hMFXllf?K3tJMU7qORa_%8tdsDxv5nZXb-5$tLul6NWNQ5(e#AC@dfxMJvFJv> zx11jPP?WA3Wbv+%kY2Fd#{&4{jJN-dqvDd;Y%bBkdN| z6Vo|>(qm@BcRd0i5KyQwgxx=0KHLr*&BRNvlHwozmzl&;vt{-je(65OHJxrQZb1@* zPSr5&j^yk0?wMNmU`++v$?VJ#AE=QkS6xw&l<4Bf`+c0TkjNMB{%?=A0BZT}m4XQ> z8xJn?d5cfT$iyhWAO}5pIVp#1NQsdUgf`x31Z$x!{p}_$-Zm%C1|K7`h@lO=I}4CJ zbRP%!yz}rZ>aFpMvNDF*5FfYW&dJpk{H0xGZoy097B_UaB7;k8_$#)D9P}5mSjTZo zX1j~R%7&j~unXg3arKY17 z+^dVUr)GrRn>`j~6F6Ag+Y(oiYOarivFr8MW=|tn)UvB8CJjw7b1_`=6eo?ii;PvH zgig6}jt)m+Q@W-ud+m#B^yT{R&7yVrLd(C()rW_KTSa?YA-K$NsY2GXKurLrh8!th zAGa4C@p2Qmv1GwE#X63HX;-;dj6c+Aam7}-BUSW!sCsSN%*s{x^hMv0y3or!xVsa_ z7q>yn7aQXlUol>@Zr6j}Dja+!jh?`-KsJnNYE~^?+lJO(~UZ$abbx1`Epy~Y$`0ihg2<9U<&9%95 zHbLh*NsrXqDOc-egZL@qtjZfU+O}l*btg0OsfTZBsNZsg63%HL@xqjoS zlfJ}isa}9dN%wPoef}4s?#Z+vn{PqHAYhXMuw7*a#BO3)6wbZ^jPc|qbVgtQn)$y( z>x#?XXO(M-+(&L^Tdfb(?9i$(5$sK20q#w{<3a$QI9z3okU!y2S!l zD2!$|#fak&;ua`zCu;e>qasrAm!+?X+&kQaMFuD8Rm7~6Tg|k1;KVf?MzU3yyr6wa zBYg;774o}kg4!5vFVt;bK%*2F4VugP`~UwVt_biGf5;zD-B{)9xrzJiZ#ucl4bLxn zC0#;f5EyX)Q9|~u&qa2h+f=u1kl9}vFG%)~VgLVs{!DsKRr*{4-9xv#PLp;Mrzl;PnyFdP0uo)wAx*&&1da*>c_cnhS>>jaix--{oZA%pID1k?)IbclA2;=JKcL{tcW6vliwH8_}pc^ zOT7v$xo0TkbWCZHUhARrhW%8lCYu1HOU zl$2I!-aiwac8#r+2pa3&O_=SQ*iP>1atJq|t0J=-AI&D+joLTmpGvCMF<|Dt<`JQ}F}Admx2HRYy8KR) zjxnwFA>+Qmn6XEvDqVFMAQ=XI`XNN0ZdHY{gK^YnAYWu5e4KIa&f4D?*1rY^RK^@< zI<9S=*F(`df;z=r!<2)=*9diZ+G_NornKEP9dd+aTE6_!N>VX+{M<{etA}}==JQGs zPn`SV6(V5T%QAd9M^tmS90IT}IJMIZy3uW2f(dN)&C9a(VH_;g__ldVzqe<+7ffiv z95vbT@tmSPfE=>$yYo(S14!iMe;XWV-E}$l#LQ8n<=D_`y zO0I;Hsm>8WnE~-5nX6hd4~b0#B}Qdlwd||U@iZ}CJ-$_via00HasI|80!Y|#(t5lF z5iza!_KS8n+uA2(-gFIVo^Vs@H+%}HtMvv$XG7#$rP9Hq)J-D6<;`f)+1Vufzy%^dHbPdw8WMmhT^gkG-}NE=w~_u&4`}>9i7RlutI?hI=V! z3D!LQs36d=C@Cog6U4`OM()f#TA}^x<4gBuzMOwN&p6M6Wh!`(Db>Ei+SPUMz~A1>TT+p#rqJ)I>k5$GD6S~Tq~)Bj@$_8PQH8fxb&p|xt(bqh zY9#h_ko4Eyv8lZ3Pxowh>oKaPt;y~+3I8`iCCbGZFfQ7?IQ)l8p?)epRf#!m6<{J| zqLBWk(9k1GFnw$LcftHS7K4tJEBj>egmSMXA5GU(O*-kR@qNvbTA$7Ir^R$ZSuf>( zXe~Z`)9Nj3_4~YF&?l5N;Z=gCNd>Mn4(d1f-XZ4k@FNgN1elRP<2>054VVyB+W-V{ z3bH|~2NxGXmpUJeRs(EHQGY;GG^i72_A4fDfy?*&mjOcDGq6-73Ef#kI8p6|QU7+9 znV3Snkw zGMrtyV)1@t@CVIe4DC)Co7UO^SmD~#ZWs4soVi9{W<9spg2;aau`W)40d4+e=VViV zK%#;sbUdK{AE9Y&V_knX{qo{N{vxJuFIZWrjaU)&JX)x^Ea8UAyRt@z;FdNJ(U`Nn z-0idtyQ6vgZkN>iMwxJ>5(>k=gYbB_cMrRE=$T!fk8VuMnRYfNbQ-({4 zjs=h|CG=heLg>ASRHZ5)C7~!CLNB3%fDoyX5_%B=LMRniDWMdeDir_U<()0fA&)10i8(iVu+-1E-6jmca|g`2Vb_rPqqI`+$v;$ zVvAuCvDs4g!ehj%*B6@H_)|R9c4{-o1V!f_ecJloJ5(|t@~8vR8n76)XE0H~QVK8X zV~jA{y`nmZJ%1=$-117-&|M2`FVi(%QZ?YUd!pd`Xf`Cqi*pZsTV>6t_q{g0#~LY} z_f?_w!^GNN$XnjdIG^P)(mxtbRn(nk8SLE^X8p>o?N6)u+xiPOxMZ@-rVRZZRzfn> zf+lJ1@I)f3d~>oIN62f9g*5m(rG2r%sK3Jw9IuSj{oE&zyq*Yxnfhr|C_99C(L%+!|l*-;FiR~s%v+2ghFl2Fm-fDmFYnB1TwTGmb5c}pThXVw=iQ;pG9@K~Bh>WUixFm3| zAZ#=N4$Kdx@7w6>k1tIwf?$n}mtM&au#CMZwSLB^qb*XgAp6*e4~&xi z>-~S+^RIlfT8mm|a5~2eP3rDwNS|y*M~?>vgqh7^0dd`1`Mt`!=xNkuS6lZ6x6H28 zRF~>8`s=RFfv!uA_0yIw|ILm$XgFT8_0yNbk(}9-HS5hgNBs4k0qgRz%G%^DvEH^{ zEkCA!r_wKO7MGLarKstvMYpKW%p2Dy>LIYA-M0BJE+^Gv1x3j!q_P;w-CILC>B*%A z_m;GlZxY-8<6ztqW#k4VBZ9s$l z$yGKA4EN1!Js0b7>T`a`w5o+^Ut&quJ6-vAauZLM5@L4;cpet-I?9x(BeDD~>HS74 zQ9DtEcJWuTf){c4oaq>;(zSx=^FAuBnp$Qd8_7|>E~wsu&xPk^8YBW4o)Cs&tm-0$ zyBDv2+@Nf0L34-}Kxx#(xJxqc-!%tanfL`-onMUzWZlogt1&xpzKV^mzYku|AA_ut z8OkawG^BN_J?~bk*1Tpcl#Fn+^|)nSt!Ii2V?#hI(9%9wO0Am>|8^Z+#r1ye(okrL zNRNq6dz%2wQKBu zLE?XXUt@o8zHj{8aRYije3wjMK!;USl}gB0XR^FoVkgv#SRtoygLeXlCO$t_nktxM z&*Gz6@YEilF1$v6!2NTIAd8ui`Y@Ej*Wwk#LXy-K0}fg{)LEyIw#YPtLVp}2L3l}% zx!HRI3Oc0JE&_NvRuNH3`B9r8t>>Bh%UjN>G@htel3C5d>qd%UeCbQb z7nWhBgJBAKrm95=&8~6cUnl4GQf@uSl#9Ealkg@l_;=9g^qy~=>X~^%Kh0Cx>^i#- zpGv-~>Xz8kH?4q5n30^nk06*mW^_fzm79pB%s6?OXC;MnG|iC*%IgqB3}Gh@F;T3&LZ2BrwZ(c z=Vdx4zY1v|>;FA~{{|KrIyB{e>kz0ao0jD_-Q&Ch+q6r~ap}kH z>JnXtvY}Db6MxeMEx93m8h7$vjD#P ztn56u-Nr>DGw5tYQH&unjY6^VuqKLw+ti{d$5@*f*L%^hd^uD9JK+Vd#;rfoU5#4W4elO-v2 zPCbi_ZS*`0-2VgfI8%&?6Jq4I_Vy4ezRN~X?>316?}{?jHKfQg(OB2niE!mBmkd;o)I%80;e}!d?;?a2^ht!RO}v2@AV33T*y;y=ek{- z0ymFafBS?=%AZrscEmRlLMvO?G7~iRY~v=0H(?JrZrd!$j2T(L#iJ=+0B^^U;y#{C zp@zB&n$As(^`6yM6{mvOOt-Imf9Ic92FvlLn-3V&=hxa49PjotXy}aLyf4OVCn41G z4A9O~hsHtn3-RSawTJf{{zR_!Pjkl86_Hoo&$y7CT0L)$GpE*lOXud;9vSG?G4cjgsbz>T-5-S%!j@$@%qqA;A4eTcuIvv;? zoxX;);nq=n+SoG|TC2BDBF$yrszDD2Ut{9-X7O@%O-VL-74*F~_LeSWw;r*c`~sEb z*5X1Beu3V!>+uqDBj=B}%19~BTvUQXnY#knGb>mz}O^j@O??*6lgcdgL2fl$A9=X;IyHWE%!llKHptU_V+HuT>1GH;laOy#-qJo z=_S8qCS$cvNgqFI177!Zw)9+?kKl6cCUdPG8?V>=u*`VeC6Z+1p!q6R94NnlDVZ8H z$-wBHS4;QxkSwKQHm_fwCLTvyUA>%2d%0(EKg2c<^U5{ro)rmOLx*5va+(|J;_ji5 zKi3Du(2|?qPE4dF&J0@XIIeYEM5`X5(k)AeqAgHWRROzqdw+N9^6=+70{J00{zc%U zQ+_8quGt9GEV%XrQ^a74y~f;F2h~5OKU+#Nlg4$VztHwyKSOka{wsq8w9vk$I7VoEHy<}gXiA;2v?M58Ol(t8`*3t{ zE|)Xun+)I;iyojPTtWn47h1IsqgaB1Iy4@uY@ydrk4YZ_8=y(A<*{%1aUb-T zgHG*?4>+V$%qm?Il$Kg^X9AYfn`mn)`#$QaBWmui3syrZJIU^7+qCCID7$7!v(bS6 zJ#HjKQ$ZcWjqtIJ<;lRP1!4O}DZLq_POm||TGenWX2IlJWw~NuIX2Fn+Fm$K?=yo{ zoI^`$qUFG~Q3H?dZ{lVfGKo$zO;T1)z#Dm=DYA5Tz6zSq7U6T2f!s6yP(85(Bv}Tk z0h8|(<6d8<){~V%cd`rh?ex9h29l9~3Wxw*>8W0;{W)l&M3Ap{iW*gpsde^P7+t@1 zHSE1M@xiHxUGsD2N2*MAXlcnL)72;jR*jg~B!-`>u?$$$bi_T~R{(m|?woEG(+Bm) zaoX=5;$NC-%&L&6)-SXIr2PT2Ux)`NhO|3zv^xOB5XI>Fh2=jIZxHCLJhdv(dRJ5lZa@qqvb1gH$BpWf zvep|s(~Y(}h#1NzeR_``W%DtVWZLs2`{w7dHgD@&m()xwZF`hu;hID)M!X)1*`Q>a$ght=k?QQNTX_iQZi6f;So1<07r>M^XHk z&%#txy{U@W``deus-kQ*UmP1?(X{UIjMtKauLd{Q7#&F%h#NeYHW2qE`ryXfGOD*T zk804&sgYS(EPnn{8U_6ba1}PnF(-}@9%v9&P|%OnqpgOBDmpgCZ(-nACq+P zb_zrKHk~z2@KrdBiwu07$CC!S-6>*NS*ItIT+-$iCtVgQVS{;=uG>jcPd3-$w$5C* z0gb`T#ta>J)sceh)&%OfJn)c&Lm^9`SU%!S^JtPi<*1L!Pm_p%e@?&VcJK#^iu=|t z5as-1m&_}$;L8v&>YclN1`cL{&nF}dsFGZ#oZw<{uL6I8Y zv4+NX#<}cL=uHhgzv$m3F1Y-6A#zjErc%=ig3S6rsg57}*C7qS2nQ1k}tdBCKJhV9T;<6nDO`}_H_#*}C zGW?IZ+oq~P6V>AO36&>390&Lzz2jnhH+5~%jSUq&wkN0iFmrBWvBb2=|?5aNv zCt4&lvxY50Eyw~KNx5d}qc(ja>UC^JJa1Y=AI`8wucVc=&x9F82$FT$UUq}{gdjj* z7?Mq8GH5(d^z5y5A;9%sPURTXeSi#))QdVMN8wC!b>Hiyk00om#h6VM+;!M!(JGnJ z`m)g!N0;3*;TpPrXmwEF?0+43D$v9s_jrR8?UB?tHD3_*P5`=ap#OmB`YjANWl*+G zFT#n*uteT}ZGpn{T}+IUYJz>Fbj0N&4+Q#GQcEhMleUvFyvNB)PUJ$4o3|LoWV!4h z5l1@Iq*=t(A71sY+UWoyLTf>q6|Z$HRMwl*EkAq@|cas`DIwx zUzesdwn+6#!c0qEf>bD+9j2xK2C0xP#zVr*{tcwkTqB$#B1iHYqr#g1qBP_ZqvAA^ z8y%LU0y+!^!f>Ph09#8RntSd(&Gcz*9WmXqj95l%=-hx&m^@^JRwu z#wEj*#i$fzUB%gyn)nJHNeVmN)FgXog5H4kZqe9#zjj1U9WDFJrW8pzPAoB&Y=m1f zq-(|&X<4MW#*tCJ$2T4M7=Qi)50B)fXonS>pn45`Yv$FKls;=+w@t2C z2}{oh{0Kg+G@uqC_L^_cpdqmH5tt5tl$Sn9OC)jIQyKlFcJkHutD6pay>J;`z1lI3 z)N(p8)5NxTC_QTY=X6VnwrstiRW^Xet)ihEWo^3gY-4bF4&Lp8dx2hkPg@K2uJcg z0IZ?p+bRV4MjUH#BLh5@%1<)H-WIkvkrc0TM=L$zE(v>9YsGVBTL^vu0qvP22UlHJ;&OR7Z$MB@pdKu1tN}XAyfJqGWcrPp-1h7 zAH2S$3gaNUJvtVcGIw&WkmYuK)G*@pvA7}8xn&s9*f-HDmeh*lE@?G$UaTtS95mRa z|Ni^r@Ny*@Sh zXn*rzr8Z_VY+K%uO2HLRT;SU7Q+kyUaRNC1=r4=W8{tb10INueRD4jiej7FTRAVCnM;97)7`5IA;aFiCd!a)>J0PdWlF8M>2W7$joz+ z8v474C&pAUw=;sT{<)QLSm4)L1f3oUT&Lfxz({UVjS{LBY<)X*Neqx0Jc$mNYdhfm zucHYw<{h_M$S`N?XVnoO{;WhVWgs02UDF6Y1nTOO9Pa{$0~k2$V^G2t?a7UVJmr!o zbP>HJ46(HK=T?AcI#R*GekN>oWo$C1?!(5WDE^jaTyktz{$C3Emm|7bRySPzLO1JY zEXI42{^0=Tpr>sU&(R5iOMuxA$G(;-O8E{3xb}Tyey(dZQ{#>TEESYd7V*-HSs?wl zq}K?lYtBsA{k@GgJ%-25>{gQ}(4L4E2wr&D`9#0SB^A#i017w^fCuuXv%~lp>J_CE4XzJqGXjFVIw&@b54W(Pznmy$E)%tu)`nnJ}WFQ&?wdTOGeGH;_RmyisDKglR%g6T*LaI z=IHD=Xh*mTyi0;m&*B|nu3Tq{bfvmwdsz|wj!Kvcy9;cj)xEc$ohYy;cw22A)Fwl zHfx;AYGzw!xEaQ^=0!Za9)t$0}SURq~Lp3|rF>HKGbBsTD`%FlK z*@ujlLB()Jc(8xMFVaGu^LijACEB#9u>&z^TtwlRH`rFD|DdXm#|0-xW+*bnXm?=M zk#5l);EJMiA^IE5V2E8^Jc+ap6wNgvJgeR0q25e1Dq(Cc)1Ta4v+Ts2l5`f7;W&QW zn-cWpv>WsCRebkt*50)@Acid-?wKd(FMNP9ZXkNeYf>wM-1r3cD*My|_A*5_eMP1h znR04FctjA_v`b!7W_0&nX7=Q)zvX*#?2kTTusAPYD#7 zhGK2j2Ol+FuEG3+z^TaTT=7@4iUzq=C7%{(*`s#H9G|!GVw|t)lvO5G0`SI&mpkz$ zHc`froaCm?CK>P6#%m2UYxf*))UKKqrF-q^I{LC#+M7>Mlik&flyZcKn(vJIA>}7m zmP?gu5>s|+Xcf0$LOv(9ww?TJRBONI(`oCk6NLHZp@-iHhJJ;t-nGdKV^|(4s4!ysNPNkbJlD?H)&C$v7)IOOG%o4v z*28aCep{TAWxY$6aA!ypVzh9+NHXyl5~}bGd%pI9Jp#z>d*VTX!62)QraI5euraaN zfd;+`NxYt;!+LnxmS?r}LXt?(EdhSlzB(?~sA#^bG&i{Z(aMX)#N-by!+f0Z9*$VU z62|JLRl;9~{vLjB#-#*CW8b_QYUGx0PAXev5@OGE9QPs>eKH~o8DWYns7=%*!wXcl zie#5r?7G7wcX?}sZs|0ssH*ghelv+5+FD~<6D#JC_ck!dW%b*ciPO&yYmGvxny1UQ zC%nRlussw87e7NeyIJD5U4DV0r1I`eRPY*o+pSSur&RDdT(J1-cPz;?&Gdrea#rW{ zdf-%?C63omua1{$HbYBTk5#U{Y?g$zIRLaxLspnlRYH0^#+6!vPmToh!C$a>`Xi+$ z7t%P~?6Muw534mQs~7X0y0-+RbrWFgak1~>6-6i%ubaJCKWP$uw^Y?&nV?_T*n>@o zvWfBm^!WhEE*a?NcT_r8$4GFO8 zR0-?Qmx;o-B37b`dd)Z2@z<>oYB@Ytdck0@OEIFX_;N`USM}l&qKb>t++k70`2owY zD9%h{T&C~dAfQ@!saLp%zW4eu0Mv`#H}<&cfNYp&&hf5ihxcq;PrT6lSe^AJN8Vp9PF|Ecni*)JUQ7RoOoGzG9`*SiUAid|?I+G34qs6D6>1TO- z43;9ke)?wo^X2-n!Hu>wp2vcGyFKDFu1n&xK&;5#v?*lWOvN3Y4jvME*GZw<$sffNeu_qNt>`%u&!mP(39acjs={MVWXz;&-4iqQ3 zWXxp1HS@VRaDf6k@RL;T@9e!7Hn3yvrlJ=#Z3 zPKi_9xcQB1Gvwid)1P;4<|W0Kv^c;EOS^3yzIvbiZ7>0F(}(?E=j|b zQTCx;v9M1A^-6CqSHd6fv$PM(SNvEX&aSo(Pa|J!PLn`gnP4-XO0q8Qh1?0R0PdZG zpX&Pa1gO5YtW((_R>8BbZye_mEW0P*oh_fj_R1v&DKf$ zG);OjwhW$yuF<3wb*Z~0+iBMKaHqCTuU{B|0n^$ z78=I99kP*b*J*b%w{0=&N;;5-8v4u41eJOqgZJzXa{L`L?yEUO_15`%Wj!frUV95T zp`u3CAVhBBOgk5jC3`|R=iiI4Vc zWSB}$^aA<_lYJ<<$LM=7;u@mQ5pktax2;rYbk~-Yc4{te6)XzY<;;0t>Gao3qVwa| z@w~eO>$h)a{A^Zr;If9tNOLCZzOE#qUFgizdlt!cJy5D?%@erFjjqzQ&U#TV*F5-a z@@TjEStqs_)>^o%s-u$fan@1O=BAnzZy#8~x+PjXl3H(lTd6szZ{}|2HtFrZ!@7BO z)h3&XkTMdQYbdH2*){P>#%{EfPv93QB;6-vGAhY4?jrt2opcdu!6P*jcY5&{cVTvX zS24bw=W5`WE8lLffqxps#rWkxjep!1h#8s9Fo5N5-1tKRJ14KT8?K09U(Gi<>&{TI! zt|#_U-8G;R=z9K9gU4M1xaR;6#5jo`?C#I~R!()_bZU)!9y@W9SqbbM_ z@rd(~%{mLw>*|^!2}#*|73>aEo0=lSJ=X;i$305$N$*&;Z}CJhwgj88y?0rqY-kr# z7tndI^!J$mCC)5}i2pTWZ=?rpe2Tgem@y;9z2@B9b{O0CJo){@xQ4*eFrJn0LoK9=F{Wj8rtasLKdt zpzr3cZDA;y&fQUyHZ$3${{AuhezMZepoDcMvaNkS5W{#OUIa^rtSH^jq}zsrADfXF z-Po=_nf8;4(c>0bqjA)Q?`WfG56Nq9zbZ4`i(@RFPYc^3%UB#IIYv{?xe_H5V}Wv5 z7qdI=T8DHKFQLv^iZLKYOv}%{jNllqtywMAz1=$Cl*ag2o4+&dMhz@Rx9VNYdwFKx z9j*7(Q`l|K?%R!!<2Jg=mt0CSLFb1=ZR7g~8c^?+X;-vB4~}YLvOq1zUI&TZRZNK~ ztRcQJq*Lsu7a;1`dFEK5xz=loA-ao@^~k!tOCRt%dRGEpXK#(n`ra1ZC2&mq>0rTN zYP(m?X^Ws$TiMpZt;*}8FGJ(R~qZc^t|1M zue|9VWxPO9@8igq*VnBfV&WF@J+yYCFanGbieyhQ<=YwAnX>sl$&gYo)uU0#zPk&OU=V6c4&3 z{Jn4Q!CDj(Ij z_Rlm6E91tL%HiAZo!<94n1~8A8}Mx6oNje1Ho#1ZpX3AsOXymrjFi%rf(M8SxC-S+KG5ges0$6C(R@mc3)5!Oz5e}Wt%Y?P4Ciobca-f!-KyDwL1O0xJ41x*kz;ra!1{o&VUE1a~=A zo?4mN2^mg+x_3&8c)r=9Q7Gn5bT~QmQZ-Y}rZ)Y3X@H8$OS4ADKQ&&Va>N6wkIVMw zk(-f6qdduwnX`+m@8kk!nqGSr{a+iE@~V!R%Wom=5%}HA=g$+!s3x>Twyx)S^)t45 zUCd8TpW2xm%HoZ6 zX3xM7HOds2+w6^=Z~7(VfD@i|^76HhDOm{#iDoQ;Q|Y^O_5ah#5eV>*8eLs=z>aLS zQWzN-=?ccK0TLoC>KnW+x$cmi&|FA(#*ZziLph2$uyEa~V9BL%6q_KQYvpIsD%f_j zGtNPqQRvphzmU@m@qf=*%PU%^EsoSWf64b@f8%|Z_|#a<#@F2mfr9y|?8zqLBa^** z1=6BLbK(Pha+~>hVJ@!{y=9{~a2nEaaQf*UFH=nrruu-)sKt`d$T!*istRh@^lDjJ z7rI;><~9I0)y4U6MfKRh911ce7LxS=p~lF|LL>0_6nAX_`5A_>VM`KmlUp5v+y+gFpw1O2xq zwlp`tPBn2(Mth^D3*>SfKi0Q03FbIT8tIYIgpZ%>Ug@lBBhri9)dRLw-F9_t)%wFD zdN>5El9N9A(3@ZO^R(D}57kr?+9`ZZf#2^04c zNSYUi)4k?5nDvUb?nBxkszJ()h3v4lTYoHj|xZUdFkbY%vLxG=FbZ z6poBX(TwPU8}MevkUQ>7%hQm4C6>!NZQ!GKX?|yLiNc!(k0jnM*SoUt{&_qKffLd0 z42qWxoU*_4z(581cznRuUC?oYxHv%Aa)}f>o>?@PH}yH|PUBu%OBdp+W!wvnYpBX5 z8^aUY(<$rVujLkUDowS_=Z=z4X02^d-N8+wm<0xK(w3Pj z9Px|~7rME^pmY3(0~t2&8`crEO%}%NRVWD+lf@n(NL7zrJoW&0UrP4ispg9Qh4guY zxJQ9>5MV8_bF@qJ;c^Oqt1~}dcivmOEl>r2TbG5MrS{N9Qt(IQbY{?CO^E+Mlf$=D zAB!xm0EL?JspY`3pnj_DmRj8>($xl!H@QF{N}7{NKgheF3z6~7jx(MCTzx;+hqYO3?>Ww@w*qmVfq(H!+9K^ zGv5Ws1ien*|0If8ONT4_F=9JUmJ8lqxf$N-9-; zZU4*&nNKIyZu`oC)G=9aBQ1KF20&Li#k6R(JOJ(qxVfm2YF8c?D50-=-8d5R1q3Q< z8*N*Ciyze+Ik82ReiBGtkJq72pMia#F5S_!oHXd*X7m~?wMsecEv#*Apza8w3ia4? z=l0Ci$=2#&%WCMmxAbohnE%wde~ESny3@7Rwz{QCkodVdw^j=mpJ;kMLb-8bi+h;R zn^S%W)@;HQCZ``e#X((S4fNvI#xm)15e1Y@ydM&crBjq_-nv>$!CKFAk>Ep(4au(` z`K|E`9_Q2q?Gz2lT*LwFD?IgGVVyAt>}^q^-T<`*5$Q*R9DsGyCqE7;R$gHy{!b$T zz5>8Fls0nS=TxZRq&#O~H+jN8WTv~m-Se(XF4&fZSt!uH*?5)29?ASx`t(xj?#CVs5&br$Gn{6;d(m?N$ z%_fpA?t=6MOtDMy4IUNuaVB4~*=%5G%K)2g(0C7h|LiXy03hmx6R}3*Dbz1z#{b?< zWCM6jA+f6GGv@&=G2K+&$zr*)@i3~sq&T*tx~1aW zzKI)9Gn>ikH=?DzY!P(DLN+8&O}0>m+L1`rx0?1KQMJ=@Ab}oTd8TZ%WU&BWPNu6Y z*m#&gBKn+KNeFktrGJIP2`tVA7gozqd06xaV~$ibkgef+vu^7;be=wRiY?Xs*ivHF zqsP$cEMq&egS$3ngj=92g7Cd=?PK3@h$!|wxF6x;>&PGF!$-kh67w=N7j6t@#~%lGF@!u%U7#9&m7IqRnNsB zM-79uYqy6P$Vhfoh@lU#PO8*P_tg&D!!b)>8QVZz zG;M|~N=|MliQ6=Vuh&rW)3_-ZLIPgST{bx}cKt%t0nFO}Qb!lTqNPp=5m{8JQ zN$(FY2nN!@e`5RfPehMEAjR>b!-C8rg7oT2?^^57fn~-Z>qoe**^Xu6s*FJ$_r{#N zT8qNN0>($#yrVt`x6sU!ws9Oq+DM08XZ>aF&hCY7jv5o|nOX69BmUBi;KIf{CnvC_ zyJ^*P-oxUaIdRwMw@su$ib)B|Py_(&PA6D;mNAN?hP}5wC$FPw?SFO!h||sH{jrlb zGS35rD#G*d12J)n_$5=N&%Z!KfwSouW$|~I`eMK?5lyK|dq&Vo0BGb3SManCD`sIO z7)TTc?4G&C-}l`76lmEBZz_}JcAJQNl^l=L z?!n#4Ez^eO-+>FeJ1p3}mOhpt zalSpXfzi9Zi?tjg zxYF|A0aLlf(m4_imPn|8!5*75grX-P#czOwqQ~xq`KM>|c0on+ia`xR@14!&#`yYX z=&Z76wNG%*;Whby^emcm4`n`Gnd=ZfRanlr*y@^6q&NUra#qD@Z$aR}Mb3pRW8|}Q z-HIBHa~WFq{4C0rHNhcuUc>qj)Am;DYzHymnuH%s^oA4BQBwBFN|b1482vC_O!)49 zlZbd5_qd2Dd&I6;KYx=(g^|!YZIj?FWL0CxzQjY#c^tX6)U;~Q&|pWlv}~POp6MtR za6G)3&Yi5xQ+9cmjBE!c{J+n)cXy4nr{{+@V~)?^Hnc?rg;xJfVh z&6wbK+^T*0bH$nOvFpnbjk~y*qjF`H&&hzgMqweNQm$1lD;vx6J^^1kPTHM*k0eO6 zP1*~^Z-Qo`Xxdg=+_iLJpP>$AF}KMM`;w5=m4y0B8ICVOKtT?4HD`XvWEiLdGkL9h z`<3iuj;`!A3Jue@bRy@i!aR}1xOe_F{VL_o@m_q`N)CoEvKq7Kjl@wOVE0sU<8TA{^^2jZZ!L}`_T*p) zVr-SKR{<~5ryNGn%_t799J~5HXnDq8ssgTc+$1J5>~85KlV<%MX|ky+<}3X%9iE`Q z1-2196Oc!U7A{5;7b2BJqa)rq^s=(UA9eH6v(kPCy73>M>w6c`TN--9y659-+Ckw5 zKk%BnfqwW+4Y*ymM#=6`SMLVC-;T@Qz&xFVsNWnlnTX%6GajB*|_lh=ckrS3d9eqi2J7<16j!a5Q&Y|ss6sIg|8j&Pc( zG1QXD`4(q#FPh-~L;S;Li&$mjPMw!}^mJ|AM<#d?H2D_kV&f^ZkFIE^;RjQJ3u45< zHhh9G`m8T1$+0)aCWlct(Y$Isqt4*3?tcu&=@+P=4o?``TKoknl^gVX$u#)amFPog z#%C`Y;Jx`yO}4DJF;wQY)0p?$be)CknV8KAXT%<7wdVmRoQ{OH%|L&W(slW(`~I}7 z(;y#1LGt~tDy=4xlQP#fx_8!0F;7%W8bf{_(961`b|y{iohy_rLtnqs2&cvpH?>GU zrS8Qbl~-28(x_OiU7u=(j(xlFwWtThSUKPJA?Rp{FEL|90$~a;4@gxUzByjXTg5W2J22mcsZf z?K=Dd`?I(49f7~b`zD=)t^mniH^0)BZ#fsVPuWEHc!2gK^NRtee}2mXqtW9yZ3Y?M z9NMFkesaOjI`{t3sek_@f)-F0ejNu z5`?OcGfVvnhq(Y(6v@&^?w8xMAk5`s#t*7e0yRyTtf5B?A>fj^?UmX zlee=YknhRCB^u9!91CBe&*v*Wl0tJV!(mTFM!dG}^LoQ|p2Z0DWYO(1mPlEgjCkoC z0;bh+smRBdccuUQp*cRWY0X%*b%Py_=CmZ0(9S=x zqE&a?K5SOB+io(HM#@FXMJVwkE7e1@OEF5uot?q|IRO71yA8;BSjmgVbGSU3Fxt*) zdW%7FV6O&Qf3b3RXynZbCvLyQ_lgj`kpaE{>vRPtQBhwH)}eOtm{J4on@2~VssTjH z9&jG=TB9=c8J?FST+J%0c#(Vuz6O)f7-x79XVGNcH-+cVQ6d^~I&LQlPe#0)7z$7l zB@5Ogs5rO2IMAlV>M2HCHr@kKa;_1wTs&dVtD;}5@uEs235GVsI3^G#Ls?%{t2A@3YusweqVTFAj<<1udT-I*yiPLgjT39a&RO% z!URHX8r?v3I3U$@utxQ2=_H~W`sMaD zvf(AwUX%T0WOvBEw;16~i&t@`o;3RVF&tkU&iPeh^1*3E*roF|CmEFB80R>tQO(n* zLr$6AoDN78b4;5MwLzMJ{M~lNH0&3k;D>V_0JA1B$`MDX{!9xkIGo|cmaqr;&rhpEN@1H#g zK!23TX=i1R%xlVkP}Lbeh_e(|$#J6(JB~YQK3ZmetNCjUFxXh(!>k`or+J!F!@8xG zvwYO1npnJu$c?yM-uIiVp&yMRHtEGpa%!j@R*QSGf}BWGZ3{DILAwHvs^c2Xncp;+M=vdtjSl$AL|zKIin&UxjvVP5ZoSSJ^M6f zUId6$-hMvuRM(+4wY%c`p8_PB*f5yF*DgVa_Wx7dcSkk3wfW-lC@KmTsx*~el};dl zjoze}gd#}qy$L8)T9A%}CcTEK!i|36G9I~T0juIub!D%bMJh2=FFP!uA4tX z*5ZXc&(6+%pS_=7Nq8-Os^J*q4T!S&9lzap1F&xHY&sbolG?k6^$`#!h4S+vqT}Vw zWe|cZ?S5?QnEi-rzmmubgU>C|D_7+PtV-R=+?3?51&B%*gWQd)pFl6hV#*40S9T8K zHX%5XjMi>!g&69pB7dtBG<6}`VXi*RyCTDEpp^C7%Nw=aoZ?TCs>#yn7EfU}YdY(G zwDI>Q1~EPUcS8ca{GXvSNiLkbAul~bi3t2srQro?Y46{vG%jd{Dm3i3Dov(fFHlzl z8+e#Eph^QT91T2JM6F$g^ah;VQuN{bSdhnz^CG@2l`q+TCXecC(wISfp18RgKj>v> zL#RvgFfS937x9qT;JQx@GsLjL36*)-|1Jf6o@Pl1MXVQYo|9B8}u+T`OyU zFw}DEDj}1jH|v(!XvNJ#mVNL1zG?XH1_8k9U}RJu=KR1EHCR44rElKo5B$_n?%9WetKYdUpvmLyARe7I zsC%&F>vRW|m+z3)+B;d_pZ?&mT_o^wdb4VVahj4WsouAL#H^{4w+=|p$ao3R#2bos zWbKGIQZsf+?&g{j-5ysC&TK9653M9o6FcifcooT+5fUckOJ>4V*D_jUA^CRSMk848Mm zJ7(&h2<3zONY!5^r58;ilX^XUsc>29e1Oul0akIb%Qlium`V9$5CU7n@qTPCibl?L5cT z%Ba_LX9P7Vx<~b|EN`m#PO`5EeKlxOjLZ>PsoB zc&B7r@T#Q5ZSp;I1go|-Paoi3!gT)5Y0RosrsK@(N3-)^`!XO(?VltYhiemz@dryW zn}dct94#Dq4ZM!%m8o_|6RD9F#?K#GR{7K(VwO3G zZv7#^mC<7lrB}gfelNAztpqV*yPVMLnO62Z$umGcG`VV2+SXBNZ4%V&SyLzlUnd`p z$1)aU^;AUyUv8J-q_K>|&r>W~T2qU@N!I4@{_eT@oQ_DYNn6+bTMSK|mOhf5crU(@ z8`i%=`RfTN6);%yy7q$8rsPDzrK_KDG1{htm-Z25jBY9I&I2uZLc{^R5T%v_| z4%P*c-+EO&5a)a-F)LaRpBN|>PFBwpdSkh}-y2&{hSlYv>{C+s70w?~nXSyq7ai)@ zT(?OJq>QOMs+gBqET?ggC`XMm zTUqD0ChR|Aln4qljoEx{X>WxnobvF-D$p(AyzT!Tvuz{t;JbJ5M zT#H!w3nS@3iZ*E(CX&6czXydraDD6K%-!#}gP^{jnMmsP9=Mw+s26#yXP8t8PQVUN zytPy7=O}jL>YQNpNLC!EU_EirkxAL!Ul(R88Vkb>VFx#3+CLsWn6Q#Wa(OdoPHtya zcZ7o?*68|cXWDErWPH%7iKR@Wy54GT=5y696}d)@^l0r??I6DS9@Hbx$a3QB4Kk*#@d4bo`N??G05=rLR&4vtKek7hMp9(mFw;E4+vfx58iG(CQ`?(wy( z5?z-NtDzI$ZI6@iociQieVBFP(;~8NVP5HQ?HT0q>H0I2N)5uXuWyDy|F8u_n#w|B zF#l-y#y7bp&Y&Us&mtWHmgC`$K5!BG0KN3&t`AI9|5lEXwvRQ}m~U@|zZk`Z#aT|# z^PdTix2fi0a~OvVc-{EVw>2&3XuCzAe92s>luE$W7Z*7G$#2}*0;e= z18==|WH&bltk<6MJqE_*e`1#a{5kU~_>tE21T6*te&LD)N}%XU02Q|i6hD9-Uo=nx z`QUl~lH~JVjYYoaoF#Z2WKR(HO9O4r9^KEjJ{6f$D|E_`UDod(Z*OZsGwJ&9<*_lA z^w1dL(#w17B(ewjhu(_7)05vM=kgr9?zzarkty; zv=mi(0VCHL`FYB$(Qx<}|L~Dh)43Ej*X&@F4jGJ!n|$2v z%*xgcF6D33#kO>n;`6q#Fe&(F35B8p>k-F26$I}K^Zpk^_+}zpT=%{_eG)$&;h;5Y z6slFD!GCDLadX_)qyMsIg)Krw7`^F5C1nh^?J;IjLo8KF3$rC(3)UH}Re+&lh zXaD-R@KWYEaQ?MZd0r|;n-S;gWHutj9-OR27Pjp}7kLLlA>Mn9<-X!@+#bq%+@Q`_ z%$2_J+qsN1Phj(x}$hY!LID`F=%inam zwk+^#A+FeoZU3weVK-o`RWIB;wbH?VL|)lB0Cq^$80mkQP+IP(R_f*163sAKz^7_H z_1u)<`1EY;#)ifEBGJ*r!a5?RxX^6WxkPg3Nqyo}+Vs0xZ?v=p-&HxeeCP2kEOJw^ z>y)~}0EXHQ;I&UI0OvJ#l4buMJ7&XcmtCK^g$x(woF{vj+(D#a z=^ToA=%f8Hl`SmEq8S)_b+FbxZK3&cl$?7%y6*Nfk>XKK+?dOX10DE-`NK!^A1t2g zJpP@-KdPDkqvrLmOYctO5ZQ+nExyU-^2C%Eqh8dFDd@hQ2Lotn)_eKR>ca0Uv72!l zHPu?YgSR!b4JS)Rs)VkJ8lh!`k{xHo^f|>gln)#nXFV5@)n}rj4`1;>uYefp zQWrLu;#MV7PYL2i{68H?PAZUylMna!a3h z=Y@R?SEYHtkC{Qo8L_PhiJsQO7cS*?m1UmsX2pJp%bPO>0XfoxfTSfe%^RTsc*H9DLqec5~%Mfh_)wNqbj~W(xa!E zQ@MzT^eo3MhD{BnsA=1wSK4@TpX27ka6R-Rt7N5@LAS-q67ZLV{Cl}bK|n75$R+$& ziWj@KZaApTq)nJ^@I{cvguSh`5zX-N406GU=L%^xb+#iW%_-5$JC)rBDpB=wE6iaw zdxdlh7&?3}n3Qa2bZwQ~% z(LwKxKbpu+x9U~eKU6hZnw4wvxqahgbuuNAb%CORlsqXqXS>gmhka*OYYJBJ#QtvX zLvwNgLu75bML(kPnhg<)#W&AVS6M-}VhxD}DOwznk6={>Jf)HoL=G;OFr@{}srLqG zyJ@ew$cViLky7Ep*#1;gd`RNz=Ec8Un}IwAnuUNRu_h%fZ31!PX5om&4>;>xLsS(E z&E$Wvi*L+Lj1Kk|RhQ;wyCl?$PDw!JL3m}Tj9O%QT`_N()C(GhYnmQ9eXUn7jyYo# zQc`E%qw~POP=7i_{Jctg7Ue?kF?AiAmOY$WoWj6**e0cXfLt*E9K74)MmSjRs*JJb^m^X=T?S3FIJt(OU9{Ak#(%$_IQg*UUwuU z@m9Z_Du#=L81`3Zjf5n+;)S{bd64giC*0Gucb~GIozc#*@Jx?~rwPa4b!B;lI+L$I zj2omL>ZGUdin3zV?U0AsIHVX^L3Q`svf4gr+BM0@R@hmqFY_`2C+e$}ROEv0%Q0k* z&IKQFo5Sa)>@L?=rt>As9flfvfrAS%4xTdlqUuz)5N<&F(fg9SS7EMtNfuAk=jVZujzzMpk`Rc&f+|MowdwX;78mI5tlEb-grs$|4@>5*0?s#eg~EQx}Mg6XL;Zx@TEL4!j)k~>bZ2*yrzko6&BmDC3e5D)?F`kx8tR-BnYzI z@sz6m_3|E#_e$BF2EwJAPVm4u&D{$$CJTF!wZYYs8cy(;pgWKM`leX;^W})Cn^z3q zt$DJM@wSmDypg&s<~FN^zYUUbVGy=;%@>1-d^eoTb= zGk?Xp!-2a!y0gO3OB=Ft2XL!h^Awb}y=^PCsWKp3 z!#*OQIjqT-Xmy{CgK1YJO(r&A*dnd%)jAbK!oK7ke_8}rGB^~}rYZd%(cmdEnC0Z_ zKbd{Tz3Mm!zaFyv8am)f6jnK7w$$ukO6CVcTFJE?;fW zJFx2-dHahC#%B<5fjRqk7I)C=O$TcyFyE+_w{^XCR*Y_hrG&8l)_K20LCp%9`!~{3 zT^CmBKbA%LwGTPxLZSP&iR{v%@3v(d1mY*wE(tEbLguC9d*$dUw{dLHxfPW1*x%Ta zx}E6plcd^@D$#s4$y8B$+KS_n?m|ldon0hZ;nuEuhICyDcU$L`A&32mtK!L;lYi>_ zn=#5Gy-#d(PwB}hYo=4w>PJOBpzGJ1*SqfKI`PpXoCY7sUw@shu`-=mTq)xo3(6Z3 ztv*D4216-*KZESv0O|ZsT;G3>M)Zz`x^JU5boy`P@6$rX$FCW|aIWzFfQH8|weS=!hVMLT70(%N;?~ z8&Fm;i>0Wl0;jbr=YoUkW?E2Xh2=Z$r~~M(E^-bzakoEvn;d9}DUPR?V&=L%(zrOv zJqnYti8G=}@$nt1k09F?4=ZpLJw6X;2y2)^8r3t9NXK1lv9ECpBCSoff_ge_a#gCP z{?7R1?y-W2==Fv;^DOlIvX#oBpyyo8pQVMI4i}Q4gKbyb(`fG&hmnaAmMV`ur?!RyNvx}Pq_~rasdqO1zUG@A|ouG@OF@ExeZ9A zNr>!4PxPki*dDUFS#L35jjH=|3h0(-D}N^efEo$wDFuPPQjg12zxrm3pGDNN?7d%+ z(EO2ZLN%*lXe2GJF(m3=Y^C`i{Q5#ofUDcCnpL~QHmz&4HsmPs!PJYkl>GVOom^qy z{BZ7&so^tG-dvuy-%#f9a8hC{;m5@#LDdmOqm|6Mi2AKDI7pQtF#?q@`o4j-EiyB> z^|M6pBmQvXipScfFH6Q0#C^NX0Bk9F8?-`BF!b%)&?x%^&OUGJccb+3-QB%HveeNm zlMPRDty8|U3A0jFa|$3# zs9@rU`i-L`MKbeE%3c6?k+Q-*9}X(WwYa76eD4}@2a^LSrq0i7CpbBaA<}eul=(xX zM~XC$67N9weh8{2u-%)$-YK8NW9T>{s|B&K@km~64KJUP}xzeaoVCgE{(C5A!ryM9wTHb2($h40ii{PABtwx%RUC zBX@>7H&(?$0$jEH<%(V%I%>KmMY%y{R_}z9Q_6?mW*9w``Y3V>1s3q?5l~WW+#N$Q zyzid4R<9!ZlSJ?A7Gp)SQN3hti@#5{D2CF<8&UCbGo!VcZh7lFZM~CSuLAoHWz({W z6&tjAIj~CFt+S&g!}+!DgQfB9Nwm{e%OtE{K|}+b{^|87b^1iL&0-;F1>Tqj z_1g+cX$q%h8FD$*NY-EIpE5tcBV6N1EwYhS7%tx|L%(~bu|dolLAY6c_M4V9D;gW> zN(Fj1E>X(VgPAKPW(I&V1W}(&#yTE(ebhk38!wWCel+vxU#2zqF0MwuQAcNDf>LC{ zNM+o>qJ(Oh%0NeORfI}M1z$ZVONrqJ`apZDynCzZ)d2B-^}S5i8XO=c>8AZ8IWL%y z+5EzstT;=J1MNLIWA;)a1;dcTtXC7MBUA;OD4WOz2J=wZbXF>_oJ z0mJi-Y8KX6*TD}i^Q>fX`B=+N;a}lI{KFW2lDr9+Vl@^+M{7TEA4M`WUfUp=uhSmH z%^{do?37j(H_G46DS_|@d{q;vMJ3+>h^vSZwtgv~*KsUXCJ=0XM>D=y=vl}}cL85~ z_w%tUTGJiRRz6$SG)T~NB(_|mJEwaTGTp4?nqFhld8vca2*;HIAE!896~B)m*C>+r ze+PoV3@vj3le`Rbw}k)sJd*n)uK^RNbwHN%J4* zSngM6tm9~Eo3v*9FXVYx=;@p)?#7!0>k>-`w&zi+voJCX_&|xk44}}h3Z}!Op~;@q z*EZC#gmoUQo`6s@p;)+OYwx}s0&ee5EFGM-RZ~sjS{b($$=FilwlI9zj9OeEJt#@C z(cKP2e%#{f0h!$qjpPzjWWA{=tEev%Z|VNQif|BKmq@ZKYG>?PZma=mOqpM(}wdM@ehE;f#9 z3$ar{r3IZWuc@`J%e3|aN^-m|=SMJa{P|<)uYY}AuDBi!9SVmYG_R@kOz&!{c+X4T zpfM~gcP1BC?80~^!9YXe5WSa~+|G-n`be{reSsy8DAH}SMIN#y`|UyI>J2CX36cw5bHbX*!$M#)!_g7(VI>-jt40mogKa?{EjVQ1!&9t>jhKA!w zF53{o!rMJ#18opZc9*@|JGQry)W$`$$GX*T)RuR-q;qS!MsoCm>q5giaq;`H`=>8p z?_M3AMq?dkWd+EW{k?{3VbyC6ViW%MSmeibJ}c$1L#UAKpU}qee7(uXqH*!N35skh zY>noLdDuCZqdc}usQ|nP14u^&Dflnz1<6JHB+nP>K1HRm5^|m~*$iydJOT#3%ab;o1)5LT)T$?mt6!peW+;V@7950XuCnebK8`#~ zIj}e>)P_oEhF3Mm)Uybnne<(5RacBj{;|Ff*RI|-+cre7--?{vs|h_;Z7WgaO%u|R zVWEj^Fw3@?IU1=W#F8(tnxBMTv{hBnMvFO1#S!j!uI-1tUVvT0TTdT0K$j3Q3tG$6 z{R=$|=hv3fvLauTcAYQxPMG{8d6ISBl$jaWdXw4m&brYdD#^Lz;fWuap{lU|4C5eC zxwN#wM5g09*u4tjsC=oOZ^e<8pUw;#)6{`6E9XuZS380 zGPoiG<1)3JtSIBaMoAn7n!HCrXqW!Tdugv56?tdwXVEn()+N7+hPHZGp8GHTfE~jo|V@z za_0#MmfG?~w9tA}V}pI2!*AzR%TPz~jL#1rbfj7tY&Qk>{~LKo)9aqiM6(s^?`Iwovx>xp%8vel%+RBuTOY ziJ`it_T!_tGJPyk?kE21WCMOuAlc~X1-n1L`s_xhKjDoJF8a&%_tnrgK4(i<_EcZ)qg_S=r7vV=hwx*S2d1La-%Zb1=c z1p(*x??{TRp$?UcimDG7Q$`)?7?0eL+Gd{8pNb}HkfDYgN8p$y+rA>|CX1-h6y)Xc zCUTHtH)cM~QTx@@q|H{;y3XO}5FJ*$L<*$gYYwb((0AvR#OM^OfK$AFUUlQNXSdPo z1plrND-$8N44?wC#!Pt_X^qYo`&l!MCjd!yO$czU7s6tWf-XM72E>~fN}vqM+(&Ue1{*bLq3F_1fF4xTg2UcCfbjfJ6pk}$`D{|sIa%z4&8 zyaX?vxd^MUSnke>M;xtwK-oRq)1#rScoIG%BmCQ~;HE{RJhl8UZ&gxT-KzdL30- z`ey^KdTyx|A8}Q-+5kH4Szue7{3PSezfE1z!PKzuDp{G}N?Ey~r%H7O(RpAbb+wD8 zO1k+FSxq3*AsY^CThFi`G&u7hN68%XYaY2BU${J5cw*niENQs%)Yh)#%ZVob{*GG) zviuH3ZS`6aV3lDoBpeB78rsUMQ$fBN$yfWXpSF&l?hV>k83261TiZ?_Pc;?6?mHDm z$CI`7w97|=Z)R|oH8sXO;h>kXwHvbvjT=D0aotBl-Je?-Muld$wL5~B*89ZLKYyU= z8&s_)&1qD$-j>9~a6i%(veaO4ABtO=NT20Pdx48GU4)02_?Lfii0m&|t^l`cOzSi* zz@9A1G!`GKo_9!GI8QvKI^W2)Fyt6!n_a(@W?thLIA$32dKaJ4Kg?3U(rr*5pbl^( zpYu|-L4s$zZH_B?l*L~H#;^+&;$|t*Ru|YL#MYuW&OFc@{xRG2JGk|n0-olGQQwn< zJUYN#!+$`(>|~6kb)FNGe%wu?T(>AdDEPsXpz{FAofZ>>%#b<&A2h6CC!?q%Hq_eC zrbx3{IX^8cbC!mBNgMY5g7!Nvh>X#D$WUnro2EKHRD(_@UL6Kf{qvSG7~97d@^2rZ zGu8|m!7GP>chDX-z%H0?Vs!ZhS@^kT=F^_CGW@l+zzs2&`= z7x;aZsyIT9Y+#((0`sG>Zz7L{xiIAj)r8~^yvfrsiOD651UhpPl{L;YCF%qlT1$1) zCSDbL<$v9Rx~}f{8_mQ`|KXwQU7FGobp?{)!}^ni{H9~B=j2z|eR4mmHNUCoBxk(r z(MgnrJlJipOl@j}`kF+!+k%L6&o-~{>0DPQ5mXupUD=fw@ZtifAI{xSxH-CVnDB5o zbffAzfpWn!MTu`d?Da|=@a+CE zTUuu%Qy(|D9=HOA9%CM}?v+pLhAkD-U`d8=)^f<+i@QDR7* zI>@haFWJ^5GRxX2Am@as96@ci@9+C&!VFTF@Or5f7bT0D)b$Dn2k^tI8fWTnSY7#1 zxq3Rj!=Ul%SPK@gOq+k~|KZOOkCt_&gNWgh%EU~gJ@0N#y*pE`Y-H6|)DTmM8zergXu{|aL43p-k$;o_;{S75+ zbtSx7b%urUir{z$XiV&)u5u=&p2#xmcQ4j;j~S5rx)nLK&f|ek|2paaXFIO%M9BKX zgOGW|D@!iQ8^vR2%AiOIow<-H{%;j4NBn_If-4EN_hSC>GSP|M*rR&bp14Z!~(82!VOaGo;8x z-3o(RBA}v39P-jrp?Zqpb;3oJqdh=8_?c?)uq$QW~ zG$ajM@uF$il$+1s!-QP5p|KOUf{OK@ZIH& zl4#7yJ&~wObW)oYuOE8m)ttJ+kgHI0)nLTFA&oZ4GJ&j)y*NrUb_j?qY`r}L&7!}4Al>>MR zh(bn>yn^N}_4#0&4c+(BQ@yQyH?^)v{Lr>%VAax2?M5Q2?C7ZOqT0cx!J@Y|BAN!%@@QX(Rs+oh6>~obNA*Sa#r=oVGOeTha6wB#R zsk)OzJ@#{6sRRp@g-w=VS?i0C@iz=k_(9`umGR_ylTFg8xs)v~U~ZWo(4h1@JPA@? zGAp^IwtvMBK*u2c?=oeO+25G1ZBwXcAm`RPa5FFx`AGHNrbl04>oi1=VN==NFw*=O zpL7SRT&UY^?=nd0C>lI>Q;+67*}@xl*T0$YKmRiFo3sCcuqz2Y;R@&j^qWl z7CCsk(HulI$4eBbiaP7}-Btfo;q^r-?;Rm3QjV;4 zzbcEpHGF*kE(-qY*)PR`|38nv&#L*m@lh$AbT%-}jWDH2NKE9{7xayaejwi`DD9u} zSF{?e-Py@zT;m^9lnC|_(oT$sb0c4oW9K@3*PXtCvZ{*PPJG5TC93gg1z;1=st0r& zENuNG`AL#jzxtD8bCmSQOR1BeB-ro<;7~GtIty6+UkEt%Zg!X--7(hR8v5ioTYyg2 zKqmV!+Zk_f`E<4$tb~CZYSfk3hBz&AKCr}!2V7k$@Ojw-UQPjr-3D^{Z}OdU=9N2g zl(!>o4ke4H(vmeY>NYRwu-9(56Q1v;eDT%Q6CL5pVHVZ`&h@7tEFml_+X_4>x08G~ zUT7C8bPc-sMY&Ud{kuc?&%f?(f!{Ewz@cw9K?5i zT+)pfWLUB+V<|`BxNR-QV6};ZTW!kZ5PY6j8JDd7X|G3ZV=xjEiU{5R^mhaQ=U?~# zU{Td3Bj}zb8&VwqBpJ)vB;c|^LBHa@{p3rQqT|U)g8rB2jq!I?`dr7-3yH*)deeL% z#W7H)iOsy=>Ft--A1pBo{1DXWwk*m>Ud++Af^*|L8u-P@V=`j{RT^~s#Y}V?GB^C_ zacXnji*rx;37{KYcS~QOU4nQlGIR43zv0DnIgh-Tql_{f7wp&>?oDkO-hETOrzT^Z6<@%Flbz4htVEA={!_BQ` z_1lhHV1|NOY@)LX!>(c%YB%!3W!CfMpClb7<2x^82t;6M*#Z=7Q_jxO{SL5lMkb&7As%8zb^heNEP5XIh|NUf zCVg{;?*lwd){Tl{0woBTPo2TRbGx#URfc5Cu-E(H7h1FbevTw%v13kSr$_c}VBztm z3BiM7gGS?30=l$N&){m&TKkfFf0C6R! z(G5@qiGbA3pUYTNoM5X0PjxPw!B04d`REXOjzGc@U84c(qg6@W9z3~_9$2e4y{T7w z83QOHLETco<^WIV_#7Yy{dlA3nR@cHO;+r|h*H%&BX4kIi1YAOJ^md@)KZwVD+QHP z`fFFI*RnMjq-grrc76+%vsSl-=X;QJ|MsU^!i8VH$CA@^Y+42EX&^ur$P+~3NIL5R zRQgc7m)(mnHuY+Hx&Jj=vVM6vpxo;Nc)9LJ>K9e22Wm;Dsw*%RwMJ!O9-};;5^mRc z-Qa7>t=_MBzhL>@wD8!$g_&LltL}nw{|1T9t7}>1@^2qebCwsW)aJll{^MFX84aZD z>E{WaN_z-^o&3}*^e!|!#xnU|u#+vcL_Q59B^ literal 0 HcmV?d00001 From b751c5de4bc50fbb59ae279f073e4d4367aa8b1b Mon Sep 17 00:00:00 2001 From: Paul Yu Date: Tue, 30 Jul 2024 06:27:32 -0700 Subject: [PATCH 31/33] Adding a note about Prometheus metrics being emitted scalers are deployed (#1425) Signed-off-by: Paul Yu Signed-off-by: shubhusion --- content/docs/2.15/operate/prometheus.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/content/docs/2.15/operate/prometheus.md b/content/docs/2.15/operate/prometheus.md index 72cca70f8..b6dac773a 100644 --- a/content/docs/2.15/operate/prometheus.md +++ b/content/docs/2.15/operate/prometheus.md @@ -17,7 +17,7 @@ The KEDA Operator exposes Prometheus metrics which can be scraped on port `8080` - `keda_scaler_metrics_latency_seconds` - The latency of retrieving current metric from each scaler. - `keda_scaler_detail_errors_total` - The number of errors encountered for each scaler. - `keda_scaled_object_errors_total` - The number of errors that have occurred for each ScaledObject. -- `keda_scaled_job_errors` - The number of errors that have occurred for each ScaledJob. +- `keda_scaled_job_errors_total` - The number of errors that have occurred for each ScaledJob. - `keda_resource_registered_total` - Total number of KEDA custom resources per namespace for each custom resource type (CRD) handled by the operator. - `keda_trigger_registered_total` - Total number of triggers per trigger type handled by the operator. - `keda_internal_scale_loop_latency_seconds` - Total deviation (in seconds) between the expected execution time and the actual execution time for the scaling loop. This latency could be produced due to accumulated scalers latencies or high load. This is an internal metric. @@ -30,6 +30,8 @@ The KEDA Operator exposes Prometheus metrics which can be scraped on port `8080` - `keda_internal_metricsservice_grpc_server_handling_seconds` - Histogram of response latency (seconds) of gRPC that had been application-level handled by the server. - Metrics exposed by the `Operator SDK` framework as explained [here](https://sdk.operatorframework.io/docs/building-operators/golang/advanced-topics/#metrics). +> Note: When you deploy the KEDA Operator without any scalers deployed, the only metric you will see is `keda_build_info`. As you deploy scalers, you will start to see some of the metrics listed above but it is dependant on the types of scalers you have deployed. + #### Deprecated metrics The following metrics are exposed as well, but are deprecated and will be removed in KEDA v2.16. @@ -38,6 +40,7 @@ The following metrics are exposed as well, but are deprecated and will be remove - `keda_scaler_errors` - The number of errors that have occurred for each scaler. Replaced by `keda_scaler_detail_errors_total`. - `keda_scaler_errors_total` - The total number of errors encountered for all scalers. Replaced by `keda_scaler_detail_errors_total`. - `keda_scaled_object_errors` - The number of errors that have occurred for each ScaledObject. Replaced by `keda_scaled_object_errors_total`. +- `keda_scaled_job_errors` - The number of errors that have occurred for each ScaledJob. Replace by `keda_scaled_job_errors_total`. - `keda_resource_totals` - Total number of KEDA custom resources per namespace for each custom resource type (CRD). Replaced by `keda_resource_registered_total`. - `keda_trigger_totals` - Total number of triggers per trigger type. Replaced by `keda_trigger_registered_total`. - `keda_internal_scale_loop_latency` - Total deviation (in milliseconds) between the expected execution time and the actual execution time for the scaling loop. This latency could be produced due to accumulated scalers latencies or high load. This is an internal metric. Replaced by `keda_internal_scale_loop_latency_seconds`. From a618868389203e6358feb58df0ac2acc5e30ecdd Mon Sep 17 00:00:00 2001 From: Ivan Tuzhilkin <1it@users.noreply.github.com> Date: Tue, 30 Jul 2024 14:28:34 +0100 Subject: [PATCH 32/33] add: Istio operation doc (#1418) Signed-off-by: ivan.tuzhilkin Signed-off-by: shubhusion --- .../docs/2.14/operate/istio-integration.md | 63 +++++++++++++++++++ .../docs/2.15/operate/istio-integration.md | 63 +++++++++++++++++++ 2 files changed, 126 insertions(+) create mode 100644 content/docs/2.14/operate/istio-integration.md create mode 100644 content/docs/2.15/operate/istio-integration.md diff --git a/content/docs/2.14/operate/istio-integration.md b/content/docs/2.14/operate/istio-integration.md new file mode 100644 index 000000000..78427324c --- /dev/null +++ b/content/docs/2.14/operate/istio-integration.md @@ -0,0 +1,63 @@ ++++ +title = "KEDA Integration with Istio" +description = "Guidance for running KEDA along with Istio in your cluster" +weight = 100 ++++ + +## Overview + +Integrating KEDA with Istio can present challenges, particularly in environments with enforced mTLS. This document provides guidance on how to configure KEDA to work within an Istio service mesh without disabling Istio sidecar injection. This solution allows KEDA components to communicate securely and effectively while maintaining compliance with security requirements. + +This can be considered as workaround, however it's perfectly valid from the security standpoint. +Keda is still using own mTLS certificates for secure communication between it's components and at the same time it's able to communicate with Istio Mesh services (like Prometheus) through Istio sidecar proxies. + +## Background + +In some scenarios, users might face issues with KEDA components failing discovery checks when Istio sidecar injection is enabled. The current [troubleshooting guide](../../../troubleshooting/istio-keda-faileddiscoverycheck) suggests disabling Istio sidecar injection in the KEDA namespace. However, if this is not feasible due to security policies, the following workaround can be applied. + + +### Requirements + +- Istio version >= 1.18.* +- Kubernetes cluster with KEDA installed + +### Example configuration + +`values.yaml` fragment for the [helm chart](https://github.com/kedacore/charts/blob/main/keda/values.yaml) +``` +... + +podAnnotations: + # -- Pod annotations for KEDA operator + keda: + traffic.sidecar.istio.io/excludeInboundPorts: "9666" + traffic.sidecar.istio.io/excludeOutboundPorts: "9443,6443" + # -- Pod annotations for KEDA Metrics Adapter + metricsAdapter: + traffic.sidecar.istio.io/excludeInboundPorts: "6443" + traffic.sidecar.istio.io/excludeOutboundPorts: "9666,9443" + # -- Pod annotations for KEDA Admission webhooks + webhooks: + traffic.sidecar.istio.io/excludeInboundPorts: "9443" + traffic.sidecar.istio.io/excludeOutboundPorts: "9666,6443" + +... + +``` + +*Check your respective ports set correctly for each component.* + +### Applying the Annotations +- Annotate the KEDA Components: Update the deployment manifests for the KEDA operator, Metrics Adapter, and Admission Webhooks to include the specified pod annotations. +- Deploy Updated Manifests: Apply the updated manifests to your Kubernetes cluster. +- Verify Communication: Ensure that KEDA components can communicate internally and with external mesh services without failing discovery checks. + + +### References +For more information on the annotations used, refer to the Istio documentation on traffic management. +Existing troubleshooting guide for KEDA with Istio. + +### Conclusion +By applying these annotations, you can ensure that KEDA integrates seamlessly with Istio while adhering to security requirements. This configuration allows KEDA to maintain internal mTLS communication and interact properly with other mesh services. + +If you encounter any issues or have further questions, please refer to the KEDA and Istio documentation or reach out to the community for support. \ No newline at end of file diff --git a/content/docs/2.15/operate/istio-integration.md b/content/docs/2.15/operate/istio-integration.md new file mode 100644 index 000000000..78427324c --- /dev/null +++ b/content/docs/2.15/operate/istio-integration.md @@ -0,0 +1,63 @@ ++++ +title = "KEDA Integration with Istio" +description = "Guidance for running KEDA along with Istio in your cluster" +weight = 100 ++++ + +## Overview + +Integrating KEDA with Istio can present challenges, particularly in environments with enforced mTLS. This document provides guidance on how to configure KEDA to work within an Istio service mesh without disabling Istio sidecar injection. This solution allows KEDA components to communicate securely and effectively while maintaining compliance with security requirements. + +This can be considered as workaround, however it's perfectly valid from the security standpoint. +Keda is still using own mTLS certificates for secure communication between it's components and at the same time it's able to communicate with Istio Mesh services (like Prometheus) through Istio sidecar proxies. + +## Background + +In some scenarios, users might face issues with KEDA components failing discovery checks when Istio sidecar injection is enabled. The current [troubleshooting guide](../../../troubleshooting/istio-keda-faileddiscoverycheck) suggests disabling Istio sidecar injection in the KEDA namespace. However, if this is not feasible due to security policies, the following workaround can be applied. + + +### Requirements + +- Istio version >= 1.18.* +- Kubernetes cluster with KEDA installed + +### Example configuration + +`values.yaml` fragment for the [helm chart](https://github.com/kedacore/charts/blob/main/keda/values.yaml) +``` +... + +podAnnotations: + # -- Pod annotations for KEDA operator + keda: + traffic.sidecar.istio.io/excludeInboundPorts: "9666" + traffic.sidecar.istio.io/excludeOutboundPorts: "9443,6443" + # -- Pod annotations for KEDA Metrics Adapter + metricsAdapter: + traffic.sidecar.istio.io/excludeInboundPorts: "6443" + traffic.sidecar.istio.io/excludeOutboundPorts: "9666,9443" + # -- Pod annotations for KEDA Admission webhooks + webhooks: + traffic.sidecar.istio.io/excludeInboundPorts: "9443" + traffic.sidecar.istio.io/excludeOutboundPorts: "9666,6443" + +... + +``` + +*Check your respective ports set correctly for each component.* + +### Applying the Annotations +- Annotate the KEDA Components: Update the deployment manifests for the KEDA operator, Metrics Adapter, and Admission Webhooks to include the specified pod annotations. +- Deploy Updated Manifests: Apply the updated manifests to your Kubernetes cluster. +- Verify Communication: Ensure that KEDA components can communicate internally and with external mesh services without failing discovery checks. + + +### References +For more information on the annotations used, refer to the Istio documentation on traffic management. +Existing troubleshooting guide for KEDA with Istio. + +### Conclusion +By applying these annotations, you can ensure that KEDA integrates seamlessly with Istio while adhering to security requirements. This configuration allows KEDA to maintain internal mTLS communication and interact properly with other mesh services. + +If you encounter any issues or have further questions, please refer to the KEDA and Istio documentation or reach out to the community for support. \ No newline at end of file From 34b0e4e9c642521bc430ed9c4091d3f25339568b Mon Sep 17 00:00:00 2001 From: Zbynek Roubalik Date: Tue, 30 Jul 2024 15:37:19 +0200 Subject: [PATCH 33/33] fix: `initialCooldownPeriod` is missing in the SO Spec example (#1411) Signed-off-by: Zbynek Roubalik Signed-off-by: shubhusion --- content/docs/2.14/concepts/scaling-deployments.md | 11 ++++++----- content/docs/2.15/reference/scaledobject-spec.md | 1 + 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/content/docs/2.14/concepts/scaling-deployments.md b/content/docs/2.14/concepts/scaling-deployments.md index 94594d562..cc8f3ffd9 100644 --- a/content/docs/2.14/concepts/scaling-deployments.md +++ b/content/docs/2.14/concepts/scaling-deployments.md @@ -48,11 +48,12 @@ spec: kind: {kind-of-target-resource} # Optional. Default: Deployment name: {name-of-target-resource} # Mandatory. Must be in the same namespace as the ScaledObject envSourceContainerName: {container-name} # Optional. Default: .spec.template.spec.containers[0] - pollingInterval: 30 # Optional. Default: 30 seconds - cooldownPeriod: 300 # Optional. Default: 300 seconds - idleReplicaCount: 0 # Optional. Default: ignored, must be less than minReplicaCount - minReplicaCount: 1 # Optional. Default: 0 - maxReplicaCount: 100 # Optional. Default: 100 + pollingInterval: 30 # Optional. Default: 30 seconds + initialCooldownPeriod: 0 # Optional. Default: 0 seconds + cooldownPeriod: 300 # Optional. Default: 300 seconds + idleReplicaCount: 0 # Optional. Default: ignored, must be less than minReplicaCount + minReplicaCount: 1 # Optional. Default: 0 + maxReplicaCount: 100 # Optional. Default: 100 fallback: # Optional. Section to specify fallback options failureThreshold: 3 # Mandatory if fallback section is included replicas: 6 # Mandatory if fallback section is included diff --git a/content/docs/2.15/reference/scaledobject-spec.md b/content/docs/2.15/reference/scaledobject-spec.md index 391a968c8..59dbb3105 100644 --- a/content/docs/2.15/reference/scaledobject-spec.md +++ b/content/docs/2.15/reference/scaledobject-spec.md @@ -25,6 +25,7 @@ spec: envSourceContainerName: {container-name} # Optional. Default: .spec.template.spec.containers[0] pollingInterval: 30 # Optional. Default: 30 seconds cooldownPeriod: 300 # Optional. Default: 300 seconds + initialCooldownPeriod: 0 # Optional. Default: 0 seconds idleReplicaCount: 0 # Optional. Default: ignored, must be less than minReplicaCount minReplicaCount: 1 # Optional. Default: 0 maxReplicaCount: 100 # Optional. Default: 100