diff --git a/longitude/core/common/config.py b/longitude/core/common/config.py index 6a21a19..d7d2e88 100644 --- a/longitude/core/common/config.py +++ b/longitude/core/common/config.py @@ -5,36 +5,78 @@ class EnvironmentConfiguration: + prefix = 'LONGITUDE' + separator = '__' + config = None - def __init__(self, d): - self._original_config = d - self._parsed_config = dict(d) + @classmethod + def _load_environment_variables(cls): + """ + It loads environment variables into the internal dictionary. + + Load is done by grouping and nesting environment variables following this convention: + 1. Only variables starting with the prefix are taken (i.e. LONGITUDE) + 2. For each separator used, a new nested object is created inside its parent (i.e. SEPARATOR is '__') + 3. The prefix indicates the root object (i.e. LONGITUDE__ is the default root dictionary) + + :return: None + """ + cls.config = {} + for v in [k for k in os.environ.keys() if k.startswith(cls.prefix)]: + value_path = v.split(cls.separator)[1:] + cls._append_value(os.environ.get(v), value_path, cls.config) - self._parse_env_vars_references(self._parsed_config) + @classmethod + def get(cls, key=None): + """ + Returns a nested config value from the configuration. It allows getting values as a series of joined keys using + dot ('.') as separator. This will search for keys in nested dictionaries until a final value is found. - def __getitem__(self, key): - return self._parsed_config[key] + :param key: String in the form of 'parent.child.value...'. It must replicate the configuration nested structure. + :return: It returns an integer, a string or a nested dictionary. If none of these is found, it returns None. + """ + + # We do a lazy load in the first access + if cls.config is None: + cls._load_environment_variables() + + if key is not None: + return cls._get_nested_key(key, cls.config) + else: + return cls.config @staticmethod - def _parse_env_vars_references(dictionary): + def _get_nested_key(key, d): """ - Modifies a dictionary like this: - * Recursively - * If a value is a string starting with '=', it gets substituted by the corresponding environment variable - :param dictionary: Dictionary that will be modified. - :return: Nothing + + :param key: + :param d: + :return: """ + key_path = key.split('.') + root_key = key_path[0] - for k in dictionary.keys(): - if isinstance(dictionary[k], dict): - EnvironmentConfiguration._parse_env_vars_references(dictionary[k]) - elif isinstance(dictionary[k], str) and dictionary[k].startswith('='): - env_var = dictionary[k][1:] # We remove the '=' - value = os.environ.get(env_var) - if value: - dictionary[k] = value - else: - dictionary[k] += ' [NOT FOUND]' + if root_key in d.keys(): + if len(key_path) == 1: + return d[root_key] # If a single node is in the path, it is the final one + # If there are more than one nodes left, keep digging... + return EnvironmentConfiguration._get_nested_key('.'.join(key_path[1:]), d[root_key]) + else: + return None # Nested key was not found in the config + + @staticmethod + def _append_value(value, value_path, d): + root_path = value_path[0].lower() + if len(value_path) == 1: + + try: + d[root_path] = int(value) + except ValueError: + d[root_path] = value + else: + if root_path not in d.keys(): + d[root_path] = {} + EnvironmentConfiguration._append_value(value, value_path[1:], d[root_path]) class LongitudeConfigurable: diff --git a/longitude/core/tests/test_config.py b/longitude/core/tests/test_configurable.py similarity index 100% rename from longitude/core/tests/test_config.py rename to longitude/core/tests/test_configurable.py diff --git a/longitude/core/tests/test_environment_configuration_dictionary.py b/longitude/core/tests/test_environment_configuration_dictionary.py index b51da8f..29f0fd7 100644 --- a/longitude/core/tests/test_environment_configuration_dictionary.py +++ b/longitude/core/tests/test_environment_configuration_dictionary.py @@ -1,32 +1,38 @@ from unittest import TestCase, mock -from longitude.core.common.config import EnvironmentConfiguration +from longitude.core.common.config import EnvironmentConfiguration as Config fake_environment = { - 'PATATUELA_RULES': 'my_root_value' + 'LONGITUDE__PARENT__CHILD__VALUE_A': '42', + 'LONGITUDE__PARENT__CHILD__VALUE_B': 'wut', + 'LONGITUDE__VALUE_A': '8008' } +@mock.patch.dict('longitude.core.common.config.os.environ', fake_environment) class TestConfigurationDictionary(TestCase): - @mock.patch.dict('longitude.core.common.config.os.environ', fake_environment) - def test_base(self): - d = EnvironmentConfiguration({ - 'root_patatuela': '=PATATUELA_RULES', - 'patata': 'patata value', - 'potato': 'potato value', - 'potatoes': [ - 'potato A', 'poteito B' - ], - 'potato_sack': { - 'colour': 'meh', - 'taste': 'buah', - 'texture': { - 'external': 'oh no', - 'internal': 'omg', - 'bumpiness': '=SOME_VALUE_FOR_BUMPINESS' - } - } - }) + def test_existing_values_return_strings_or_integers(self): + self.assertEqual(42, Config.get('parent.child.value_a')) + self.assertEqual('wut', Config.get('parent.child.value_b')) + self.assertEqual(8008, Config.get('value_a')) - self.assertEqual('my_root_value', d['root_patatuela']) - self.assertEqual('=SOME_VALUE_FOR_BUMPINESS [NOT FOUND]', d['potato_sack']['texture']['bumpiness']) + def test_non_existing_values_return_none(self): + self.assertEqual(None, Config.get('wrong_value')) + self.assertEqual(None, Config.get('wrong_parent.child.value')) + self.assertEqual(None, Config.get('parent.wrong_child.value')) + self.assertEqual(None, Config.get('parent.child.wrong_value')) + self.assertEqual(None, Config.get('parent.wrong_child')) + + def test_existing_nested_values_return_dictionaries(self): + fake_config = { + 'parent': + {'child': + { + 'value_a': 42, + 'value_b': 'wut' + } + }, + 'value_a': 8008 + } + self.assertEqual(fake_config, Config.get()) + self.assertEqual(fake_config['parent']['child'], Config.get('parent.child')) diff --git a/longitude/samples/mixed_datasources.py b/longitude/samples/mixed_datasources.py index 6dfd402..af31b36 100644 --- a/longitude/samples/mixed_datasources.py +++ b/longitude/samples/mixed_datasources.py @@ -28,7 +28,7 @@ from longitude.core.caches.redis import RedisCache from longitude.core.data_sources.postgres.default import DefaultPostgresDataSource from longitude.core.data_sources.carto import CartoDataSource -from longitude.core.common.config import EnvironmentConfiguration +from longitude.core.common.config import EnvironmentConfiguration as Config def import_table_values_from_carto(limit): @@ -59,33 +59,15 @@ def import_table_values_from_carto(limit): params=params, needs_commit=True) + res = postgres.query('select * from county_population') + print(res.rows) + if __name__ == "__main__": - # This is the global config object - # We are going to retrieve some values from a table in Carto, create a local table and copy the values - # doing simple inserts (to show how to do queries) - - config = EnvironmentConfiguration({ - 'carto_main': { - 'api_key': "=CARTO_API_KEY", - 'user': "=CARTO_USER", - - 'cache': { - 'password': '=REDIS_PASSWORD' - } - }, - 'postgres_main': { - 'host': "=POSTGRES_HOST", - 'port': "=POSTGRES_PORT", - 'db': "=POSTGRES_DB", - 'user': "=POSTGRES_USER", - 'password': "=POSTGRES_PASS" - } - }) - - carto = CartoDataSource(config['carto_main'], cache_class=RedisCache) - postgres = DefaultPostgresDataSource(config['postgres_main']) + print('REDIS password is %s' % Config.get('carto_main.cache.password')) + carto = CartoDataSource(Config.get('carto_main'), cache_class=RedisCache) + postgres = DefaultPostgresDataSource(Config.get('postgres_main')) carto.setup() postgres.setup()