Unified prometheus support (#121)

* initial refactoring initial changes working version aws test added eks managed prom support coralogix prom support and fix prom bug fixed bug added all aws config changes added docs and fixed requirements copying connect changes from robusta fixing bad typing reformatting added prometrix updated prometrix version * rebase fixes * black formatting * refactored check connection * Fixing victoria metrics check * updating poetry file after rebase
robusta-dev · Aug 8, 2023 · 3c0615d · 3c0615d
1 parent c503daa
commit 3c0615d
Show file tree

Hide file tree

Showing 19 changed files with 294 additions and 200 deletions.
diff --git a/README.md b/README.md
@@ -377,6 +377,38 @@ Than run the following command with PROMETHEUS_URL substituted for your Azure Ma
 ```sh
 python krr.py simple --namespace default -p PROMETHEUS_URL --prometheus-auth-header "Bearer $AZURE_BEARER"
 ```
+<p ><a href="#scanning-with-a-centralized-prometheus">See here about configuring labels for centralized prometheus</a></p>
+
+<p align="right">(<a href="#readme-top">back to top</a>)</p>
+
+## EKS managed Prometheus
+
+For EKS managed Prometheus you need to add your prometheus link and the flag --eks-managed-prom and krr will automatically use your aws credentials 
+
+```sh
+python krr.py simple -p "https://aps-workspaces.REGION.amazonaws.com/workspaces/..." --eks-managed-prom
+```
+Additional optional parameters are:
+```sh
+--eks-profile-name PROFILE_NAME_HERE # to specify the profile to use from your config
+--eks-access-key ACCESS_KEY # to specify your access key
+--eks-secret-key SECRET_KEY # to specify your secret key
+--eks-service-name SERVICE_NAME # to use a specific service name in the signature
+--eks-managed-prom-region REGION_NAME # to specify the region the prometheus is in
+```
+<p ><a href="#scanning-with-a-centralized-prometheus">See here about configuring labels for centralized prometheus</a></p>
+
+<p align="right">(<a href="#readme-top">back to top</a>)</p>
+
+## Coralogix managed Prometheus
+
+For Coralogix managed Prometheus you need to specify your prometheus link and add the flag coralogix_token with your Logs Query Key
+
+```sh
+python krr.py simple -p "https://prom-api.coralogix..." --coralogix_token
+```
+
+<p ><a href="#scanning-with-a-centralized-prometheus">See here about configuring labels for centralized prometheus</a></p>
 
 <p align="right">(<a href="#readme-top">back to top</a>)</p>
 

diff --git a/poetry.lock b/poetry.lock
diff --git a/pyproject.toml b/pyproject.toml
@@ -27,9 +27,11 @@ python = ">=3.9,<3.12"
 typer = {extras = ["all"], version = "^0.7.0"}
 pydantic = "1.10.7"
 kubernetes = "^26.1.0"
-prometheus-api-client = "^0.5.3"
 numpy = "^1.24.2"
 alive-progress = "^3.1.2"
+botocore = "^1.31.10"
+boto3 = "^1.28.10"
+prometrix = "^0.1.10"
 
 
 [tool.poetry.group.dev.dependencies]

diff --git a/requirements.txt b/requirements.txt
@@ -1,52 +1,54 @@
 about-time==4.2.1 ; python_version >= "3.9" and python_version < "3.12"
-aiostream==0.4.5 ; python_version >= "3.9" and python_version < "3.12"
-alive-progress==3.1.2 ; python_version >= "3.9" and python_version < "3.12"
-cachetools==5.3.0 ; python_version >= "3.9" and python_version < "3.12"
-certifi==2022.12.7 ; python_version >= "3.9" and python_version < "3.12"
-charset-normalizer==3.0.1 ; python_version >= "3.9" and python_version < "3.12"
-click==8.1.3 ; python_version >= "3.9" and python_version < "3.12"
-colorama==0.4.6 ; python_version >= "3.9" and python_version < "3.12"
+alive-progress==3.1.4 ; python_version >= "3.9" and python_version < "3.12"
+boto3==1.28.15 ; python_version >= "3.9" and python_version < "3.12"
+botocore==1.31.15 ; python_version >= "3.9" and python_version < "3.12"
+cachetools==5.3.1 ; python_version >= "3.9" and python_version < "3.12"
+certifi==2023.7.22 ; python_version >= "3.9" and python_version < "3.12"
+charset-normalizer==3.2.0 ; python_version >= "3.9" and python_version < "3.12"
+click==8.1.6 ; python_version >= "3.9" and python_version < "3.12"
+colorama==0.4.6 ; python_version >= "3.9" and python_version < "3.12" and platform_system == "Windows"
 commonmark==0.9.1 ; python_version >= "3.9" and python_version < "3.12"
-contourpy==1.0.7 ; python_version >= "3.9" and python_version < "3.12"
+contourpy==1.1.0 ; python_version >= "3.9" and python_version < "3.12"
 cycler==0.11.0 ; python_version >= "3.9" and python_version < "3.12"
-dateparser==1.1.7 ; python_version >= "3.9" and python_version < "3.12"
-fonttools==4.39.0 ; python_version >= "3.9" and python_version < "3.12"
-google-auth==2.16.2 ; python_version >= "3.9" and python_version < "3.12"
+dateparser==1.1.8 ; python_version >= "3.9" and python_version < "3.12"
+fonttools==4.42.0 ; python_version >= "3.9" and python_version < "3.12"
+google-auth==2.17.3 ; python_version >= "3.9" and python_version < "3.12"
 grapheme==0.6.0 ; python_version >= "3.9" and python_version < "3.12"
 httmock==1.4.0 ; python_version >= "3.9" and python_version < "3.12"
 idna==3.4 ; python_version >= "3.9" and python_version < "3.12"
-importlib-resources==5.12.0 ; python_version >= "3.9" and python_version < "3.10"
+importlib-resources==6.0.0 ; python_version >= "3.9" and python_version < "3.10"
+jmespath==1.0.1 ; python_version >= "3.9" and python_version < "3.12"
 kiwisolver==1.4.4 ; python_version >= "3.9" and python_version < "3.12"
 kubernetes==26.1.0 ; python_version >= "3.9" and python_version < "3.12"
-matplotlib==3.7.1 ; python_version >= "3.9" and python_version < "3.12"
-numpy==1.24.2 ; python_version >= "3.9" and python_version < "3.12"
+matplotlib==3.7.2 ; python_version >= "3.9" and python_version < "3.12"
+numpy==1.25.2 ; python_version >= "3.9" and python_version < "3.12"
 oauthlib==3.2.2 ; python_version >= "3.9" and python_version < "3.12"
-packaging==23.0 ; python_version >= "3.9" and python_version < "3.12"
-pandas==1.5.3 ; python_version >= "3.9" and python_version < "3.12"
-pillow==9.4.0 ; python_version >= "3.9" and python_version < "3.12"
+packaging==23.1 ; python_version >= "3.9" and python_version < "3.12"
+pandas==2.0.3 ; python_version >= "3.9" and python_version < "3.12"
+pillow==10.0.0 ; python_version >= "3.9" and python_version < "3.12"
 prometheus-api-client==0.5.3 ; python_version >= "3.9" and python_version < "3.12"
-pyasn1-modules==0.2.8 ; python_version >= "3.9" and python_version < "3.12"
-pyasn1==0.4.8 ; python_version >= "3.9" and python_version < "3.12"
+prometrix==0.1.10 ; python_version >= "3.9" and python_version < "3.12"
+pyasn1-modules==0.3.0 ; python_version >= "3.9" and python_version < "3.12"
+pyasn1==0.5.0 ; python_version >= "3.9" and python_version < "3.12"
 pydantic==1.10.7 ; python_version >= "3.9" and python_version < "3.12"
-pygments==2.14.0 ; python_version >= "3.9" and python_version < "3.12"
+pygments==2.15.1 ; python_version >= "3.9" and python_version < "3.12"
 pyparsing==3.0.9 ; python_version >= "3.9" and python_version < "3.12"
 python-dateutil==2.8.2 ; python_version >= "3.9" and python_version < "3.12"
-pytz-deprecation-shim==0.1.0.post0 ; python_version >= "3.9" and python_version < "3.12"
-pytz==2022.7.1 ; python_version >= "3.9" and python_version < "3.12"
-pyyaml==6.0 ; python_version >= "3.9" and python_version < "3.12"
-regex==2022.10.31 ; python_version >= "3.9" and python_version < "3.12"
+pytz==2023.3 ; python_version >= "3.9" and python_version < "3.12"
+pyyaml==6.0.1 ; python_version >= "3.9" and python_version < "3.12"
+regex==2023.6.3 ; python_version >= "3.9" and python_version < "3.12"
 requests-oauthlib==1.3.1 ; python_version >= "3.9" and python_version < "3.12"
-requests==2.28.2 ; python_version >= "3.9" and python_version < "3.12"
+requests==2.31.0 ; python_version >= "3.9" and python_version < "3.12"
 rich==12.6.0 ; python_version >= "3.9" and python_version < "3.12"
 rsa==4.9 ; python_version >= "3.9" and python_version < "3.12"
-setuptools==67.4.0 ; python_version >= "3.9" and python_version < "3.12"
+s3transfer==0.6.1 ; python_version >= "3.9" and python_version < "3.12"
+setuptools==68.0.0 ; python_version >= "3.9" and python_version < "3.12"
 shellingham==1.5.0.post1 ; python_version >= "3.9" and python_version < "3.12"
 six==1.16.0 ; python_version >= "3.9" and python_version < "3.12"
 typer[all]==0.7.0 ; python_version >= "3.9" and python_version < "3.12"
-typing-extensions==4.5.0 ; python_version >= "3.9" and python_version < "3.12"
-tzdata==2022.7 ; python_version >= "3.9" and python_version < "3.12"
-tzlocal==4.2 ; python_version >= "3.9" and python_version < "3.12"
-urllib3==1.26.14 ; python_version >= "3.9" and python_version < "3.12"
-websocket-client==1.5.1 ; python_version >= "3.9" and python_version < "3.12"
-zipp==3.15.0 ; python_version >= "3.9" and python_version < "3.10"
-slack-sdk==3.21.3 ; python_version >= "3.9" and python_version < "3.12"
+typing-extensions==4.7.1 ; python_version >= "3.9" and python_version < "3.12"
+tzdata==2023.3 ; python_version >= "3.9" and python_version < "3.12"
+tzlocal==5.0.1 ; python_version >= "3.9" and python_version < "3.12"
+urllib3==1.26.16 ; python_version >= "3.9" and python_version < "3.12"
+websocket-client==1.6.1 ; python_version >= "3.9" and python_version < "3.12"
+zipp==3.16.2 ; python_version >= "3.9" and python_version < "3.10"
diff --git a/robusta_krr/core/integrations/prometheus/__init__.py b/robusta_krr/core/integrations/prometheus/__init__.py
@@ -1,3 +1,3 @@
 from .loader import PrometheusMetricsLoader
 from .metrics_service.prometheus_metrics_service import PrometheusDiscovery, PrometheusNotFound
-from .prometheus_client import CustomPrometheusConnect, ClusterNotSpecifiedException
+from .prometheus_utils import ClusterNotSpecifiedException
diff --git a/robusta_krr/core/integrations/prometheus/loader.py b/robusta_krr/core/integrations/prometheus/loader.py
@@ -2,21 +2,21 @@
 
 import datetime
 from concurrent.futures import ThreadPoolExecutor
-from typing import Optional, TYPE_CHECKING
+from typing import TYPE_CHECKING, Optional
 
 from kubernetes import config as k8s_config
 from kubernetes.client.api_client import ApiClient
+from prometrix import MetricsNotFound, PrometheusNotFound
 
 from robusta_krr.core.models.objects import K8sObjectData
 from robusta_krr.utils.configurable import Configurable
 
-from .metrics_service.base_metric_service import MetricsNotFound
 from .metrics_service.prometheus_metrics_service import PrometheusMetricsService, PrometheusNotFound
 from .metrics_service.thanos_metrics_service import ThanosMetricsService
 from .metrics_service.victoria_metrics_service import VictoriaMetricsService
 
 if TYPE_CHECKING:
-    from robusta_krr.core.abstract.strategies import MetricsPodData, BaseStrategy
+    from robusta_krr.core.abstract.strategies import BaseStrategy, MetricsPodData
     from robusta_krr.core.models.config import Config
 
 METRICS_SERVICES = {

diff --git a/robusta_krr/core/integrations/prometheus/metrics/__init__.py b/robusta_krr/core/integrations/prometheus/metrics/__init__.py
@@ -1,3 +1,3 @@
-from .cpu import CPULoader, MaxCPULoader, PercentileCPULoader
-from .memory import MemoryLoader, MaxMemoryLoader, PercentileMemoryLoader
 from .base import PrometheusMetric
+from .cpu import CPULoader, MaxCPULoader, PercentileCPULoader
+from .memory import MaxMemoryLoader, MemoryLoader, PercentileMemoryLoader
diff --git a/robusta_krr/core/integrations/prometheus/metrics/base.py b/robusta_krr/core/integrations/prometheus/metrics/base.py
@@ -5,7 +5,7 @@
 import datetime
 import enum
 from concurrent.futures import ThreadPoolExecutor
-from typing import Any, TYPE_CHECKING, Optional
+from typing import TYPE_CHECKING, Any, Optional
 
 import numpy as np
 import pydantic as pd

diff --git a/robusta_krr/core/integrations/prometheus/metrics/cpu.py b/robusta_krr/core/integrations/prometheus/metrics/cpu.py
@@ -1,6 +1,6 @@
 from robusta_krr.core.models.objects import K8sObjectData
 
-from .base import QueryMetric, QueryRangeMetric, FilterMetric
+from .base import FilterMetric, QueryMetric, QueryRangeMetric
 
 
 class CPULoader(QueryRangeMetric, FilterMetric):

diff --git a/robusta_krr/core/integrations/prometheus/metrics/memory.py b/robusta_krr/core/integrations/prometheus/metrics/memory.py
@@ -1,6 +1,6 @@
 from robusta_krr.core.models.objects import K8sObjectData
 
-from .base import QueryMetric, QueryRangeMetric, FilterMetric
+from .base import FilterMetric, QueryMetric, QueryRangeMetric
 
 
 class MemoryLoader(QueryRangeMetric, FilterMetric):

diff --git a/robusta_krr/core/integrations/prometheus/metrics_service/base_metric_service.py b/robusta_krr/core/integrations/prometheus/metrics_service/base_metric_service.py
@@ -13,14 +13,6 @@
 from ..metrics import PrometheusMetric
 
 
-class MetricsNotFound(Exception):
-    """
-    An exception raised when Metrics service is not found.
-    """
-
-    pass
-
-
 class MetricsService(Configurable, abc.ABC):
     def __init__(
         self,