refactor: improve readability, error handling, and performance in Dat…

CONTRIBUTING.md

@@ -94,7 +94,6 @@ pip install -r dev-requirements.txt
 cd tests
 py.test -vvv --target <your_target>
 ```
-
 ## Pre-commit hooks
 
 We use several pre-commit hooks in order to ensure code quality. These will also
@@ -104,4 +103,5 @@ installing the dev-requirements as shown in the previous section):
 ```bash
 # Run in the root directory of the repository
 pre-commit install
-```
+
+

Dockerfile

@@ -22,3 +22,9 @@ RUN pip install --upgrade pip
 RUN pip install --no-cache-dir "/app[all]"
 
 ENTRYPOINT ["edr"]
+Those features will allow Elementary to get all required info for computing the data lineage graph.
+
+### Connecting Looker to Elementary
+
+Navigate to the **Account settings > Environments** and choose the environment to which you would like to connect Elementary.
+Choose the Power BI connection and provide the following details to validate and complete the integration.

README.md

@@ -183,7 +183,29 @@ For additional information and help, you can use one of these channels:
 <a href="https://github.com/geo909"><img src="https://avatars.githubusercontent.com/u/7989227?v=4" width="50" height="50" alt=""/></a>
 <a href="https://github.com/dmitrii-khr"><img src="https://avatars.githubusercontent.com/u/20114683?v=4" width="50" height="50" alt=""/></a>
 
+ Installation
+
+1. Clone the repository:
+```bash
+git clone https://github.com/user/repo.git
+Install the dependencies:
+pip install -r requirements.txt
+Usage
+
+python script.py --option argument
+For detailed instructions on usage, refer to the documentation.
+Q: How do I install X?
+A: Run the following command:
+pip install X
+Q: I'm getting an error when I run the script.
+A: Make sure you've installed all the required dependencies by running:
+pip install -r requirements.txt
+Feel free to replace placeholder text like `"https://github.com/user/repo.git"` and `"link-t
+
 <!-- markdownlint-restore -->
 <!-- prettier-ignore-end -->
 
 <!-- ALL-CONTRIBUTORS-LIST:END -->
+Images & Interactive Links: Ensure the links and images (such as GitHub stars, Slack badges, and demo GIFs) are properly formatted and hosted at accessible URLs.
+Centering Elements: You have centered the main logo and content, which is great for a clean, focused layout. Just make sure the styling matches the platform you're using (like GitHub or a website).
+Badges and Descriptions: You might want to double-check the badge link for downloads as it contains an extra "&left_text=Downloads" that might not work as intended.s

elementary/monitor/data_monitoring/data_monitoring.py

@@ -1,5 +1,5 @@
 import json
-from typing import Any, Dict, Optional, cast
+from typing import Any, Dict, Optional, Union, cast
 
 from packaging import version
 
@@ -58,21 +58,17 @@ def __init__(
         self.selector_filter = selector_filter
 
     def _init_internal_dbt_runner(self):
-        internal_dbt_runner = create_dbt_runner(
+        return create_dbt_runner(
             dbt_project_utils.CLI_DBT_PROJECT_PATH,
             self.config.profiles_dir,
             self.config.profile_target,
             env_vars=self.config.env_vars,
             run_deps_if_needed=self.config.run_dbt_deps_if_needed,
             force_dbt_deps=self.force_update_dbt_package,
         )
-        return internal_dbt_runner
 
     def properties(self):
-        data_monitoring_properties = {
-            "data_monitoring_properties": self.execution_properties
-        }
-        return data_monitoring_properties
+        return {"data_monitoring_properties": self.execution_properties}
 
     def get_elementary_database_and_schema(self):
         try:
@@ -82,7 +78,7 @@ def get_elementary_database_and_schema(self):
             logger.info(f"Elementary's database and schema: '{relation}'")
             return relation
         except Exception as ex:
-            logger.error("Failed to parse Elementary's database and schema.")
+            logger.error(f"Failed to parse Elementary's database and schema: {str(ex)}")
             if self.tracking:
                 self.tracking.record_internal_exception(ex)
             return "<elementary_database>.<elementary_schema>"
@@ -94,7 +90,7 @@ def get_latest_invocation(self) -> Dict[str, Any]:
             )[0]
             return json.loads(latest_invocation)[0] if latest_invocation else {}
         except Exception as err:
-            logger.error(f"Unable to get the latest invocation: {err}")
+            logger.error(f"Unable to get the latest invocation: {str(err)}")
             if self.tracking:
                 self.tracking.record_internal_exception(err)
             return {}
@@ -106,33 +102,16 @@ def _check_dbt_package_compatibility(dbt_pkg_ver_str: str) -> None:
             logger.warning("Could not get package version!")
             return
 
-        dbt_pkg_ver = cast(version.Version, version.parse(dbt_pkg_ver_str))
-        py_pkg_ver = cast(version.Version, version.parse(py_pkg_ver_str))
-        if dbt_pkg_ver.major > py_pkg_ver.major or (
-            dbt_pkg_ver.major == py_pkg_ver.major
-            and dbt_pkg_ver.minor > py_pkg_ver.minor
-        ):
-            logger.warning(
-                f"You are using incompatible versions between edr ({py_pkg_ver}) and Elementary's dbt package ({dbt_pkg_ver}).\n "
-                "To fix please run:\n"
-                "pip install --upgrade elementary-data\n",
-            )
-            return
+        dbt_pkg_ver, py_pkg_ver = map(version.parse, [dbt_pkg_ver_str, py_pkg_ver_str])
 
-        if dbt_pkg_ver.major < py_pkg_ver.major or (
-            dbt_pkg_ver.major == py_pkg_ver.major
-            and dbt_pkg_ver.minor < py_pkg_ver.minor
-        ):
+        if dbt_pkg_ver != py_pkg_ver:
+            action = "upgrade" if dbt_pkg_ver > py_pkg_ver else "update"
             logger.warning(
-                f"You are using incompatible versions between edr ({py_pkg_ver}) and Elementary's dbt package ({dbt_pkg_ver}).\n "
-                "To fix please update your packages.yml, and run:\n"
-                "dbt deps && dbt run --select elementary\n",
+                f"Incompatible versions: edr ({py_pkg_ver}) vs dbt package ({dbt_pkg_ver}). "
+                f"Please {action} the package."
             )
-            return
-
-        logger.info(
-            f"edr ({py_pkg_ver}) and Elementary's dbt package ({dbt_pkg_ver}) are compatible."
-        )
+        else:
+            logger.info(f"Versions are compatible: {py_pkg_ver}")
 
     def _get_warehouse_info(self, hash_id: bool = False) -> Optional[WarehouseInfo]:
         try:
@@ -145,6 +124,6 @@ def _get_warehouse_info(self, hash_id: bool = False) -> Optional[WarehouseInfo]:
                 id=warehouse_unique_id if not hash_id else hash(warehouse_unique_id),
                 type=warehouse_type,
             )
-        except Exception:
-            logger.debug("Could not get warehouse info.", exc_info=True)
+        except Exception as ex:
+            logger.debug(f"Could not get warehouse info: {str(ex)}", exc_info=True)
             return None