Config reworked as in ch31846.

EnvironmentConfiguration is now a domain class that exposes a single get(key) method.
It will parse environment variables in the form LONGITUDE__PARENT_OBJECT__CHILD_OBJECT__VALUE=42 as {'parent_object': {'child_object': {'value': 42 } }
It also allows to recover the values using nested keys ('.' is the joiner): Config.get('parent_object.child_object.value') returns 42 (as integer).
Also, if a value can be parsed as integer, it will be parsed.

longitude/core/common/config.py

@@ -5,36 +5,78 @@
 
 
 class EnvironmentConfiguration:
+    prefix = 'LONGITUDE'
+    separator = '__'
+    config = None
 
-    def __init__(self, d):
-        self._original_config = d
-        self._parsed_config = dict(d)
+    @classmethod
+    def _load_environment_variables(cls):
+        """
+        It loads environment variables into the internal dictionary.
+
+        Load is done by grouping and nesting environment variables following this convention:
+        1. Only variables starting with the prefix are taken (i.e. LONGITUDE)
+        2. For each separator used, a new nested object is created inside its parent (i.e. SEPARATOR is '__')
+        3. The prefix indicates the root object (i.e. LONGITUDE__ is the default root dictionary)
+
+        :return: None
+        """
+        cls.config = {}
+        for v in [k for k in os.environ.keys() if k.startswith(cls.prefix)]:
+            value_path = v.split(cls.separator)[1:]
+            cls._append_value(os.environ.get(v), value_path, cls.config)
 
-        self._parse_env_vars_references(self._parsed_config)
+    @classmethod
+    def get(cls, key=None):
+        """
+        Returns a nested config value from the configuration. It allows getting values as a series of joined keys using
+        dot ('.') as separator. This will search for keys in nested dictionaries until a final value is found.
 
-    def __getitem__(self, key):
-        return self._parsed_config[key]
+        :param key: String in the form of 'parent.child.value...'. It must replicate the configuration nested structure.
+        :return: It returns an integer, a string or a nested dictionary. If none of these is found, it returns None.
+        """
+
+        # We do a lazy load in the first access
+        if cls.config is None:
+            cls._load_environment_variables()
+
+        if key is not None:
+            return cls._get_nested_key(key, cls.config)
+        else:
+            return cls.config
 
     @staticmethod
-    def _parse_env_vars_references(dictionary):
+    def _get_nested_key(key, d):
         """
-        Modifies a dictionary like this:
-          * Recursively
-          * If a value is a string starting with '=', it gets substituted by the corresponding environment variable
-        :param dictionary: Dictionary that will be modified.
-        :return: Nothing
+
+        :param key:
+        :param d:
+        :return:
         """
+        key_path = key.split('.')
+        root_key = key_path[0]
 
-        for k in dictionary.keys():
-            if isinstance(dictionary[k], dict):
-                EnvironmentConfiguration._parse_env_vars_references(dictionary[k])
-            elif isinstance(dictionary[k], str) and dictionary[k].startswith('='):
-                env_var = dictionary[k][1:]  # We remove the '='
-                value = os.environ.get(env_var)
-                if value:
-                    dictionary[k] = value
-                else:
-                    dictionary[k] += ' [NOT FOUND]'
+        if root_key in d.keys():
+            if len(key_path) == 1:
+                return d[root_key]  # If a single node is in the path, it is the final one
+            # If there are more than one nodes left, keep digging...
+            return EnvironmentConfiguration._get_nested_key('.'.join(key_path[1:]), d[root_key])
+        else:
+            return None  # Nested key was not found in the config
+
+    @staticmethod
+    def _append_value(value, value_path, d):
+        root_path = value_path[0].lower()
+        if len(value_path) == 1:
+
+            try:
+                d[root_path] = int(value)
+            except ValueError:
+                d[root_path] = value
+        else:
+            if root_path not in d.keys():
+                d[root_path] = {}
+            EnvironmentConfiguration._append_value(value, value_path[1:], d[root_path])
 
 
 class LongitudeConfigurable:

longitude/core/tests/test_environment_configuration_dictionary.py

@@ -1,32 +1,38 @@
 from unittest import TestCase, mock
-from longitude.core.common.config import EnvironmentConfiguration
+from longitude.core.common.config import EnvironmentConfiguration as Config
 
 fake_environment = {
-    'PATATUELA_RULES': 'my_root_value'
+    'LONGITUDE__PARENT__CHILD__VALUE_A': '42',
+    'LONGITUDE__PARENT__CHILD__VALUE_B': 'wut',
+    'LONGITUDE__VALUE_A': '8008'
 }
 
 
+@mock.patch.dict('longitude.core.common.config.os.environ', fake_environment)
 class TestConfigurationDictionary(TestCase):
 
-    @mock.patch.dict('longitude.core.common.config.os.environ', fake_environment)
-    def test_base(self):
-        d = EnvironmentConfiguration({
-            'root_patatuela': '=PATATUELA_RULES',
-            'patata': 'patata value',
-            'potato': 'potato value',
-            'potatoes': [
-                'potato A', 'poteito B'
-            ],
-            'potato_sack': {
-                'colour': 'meh',
-                'taste': 'buah',
-                'texture': {
-                    'external': 'oh no',
-                    'internal': 'omg',
-                    'bumpiness': '=SOME_VALUE_FOR_BUMPINESS'
-                }
-            }
-        })
+    def test_existing_values_return_strings_or_integers(self):
+        self.assertEqual(42, Config.get('parent.child.value_a'))
+        self.assertEqual('wut', Config.get('parent.child.value_b'))
+        self.assertEqual(8008, Config.get('value_a'))
 
-        self.assertEqual('my_root_value', d['root_patatuela'])
-        self.assertEqual('=SOME_VALUE_FOR_BUMPINESS [NOT FOUND]', d['potato_sack']['texture']['bumpiness'])
+    def test_non_existing_values_return_none(self):
+        self.assertEqual(None, Config.get('wrong_value'))
+        self.assertEqual(None, Config.get('wrong_parent.child.value'))
+        self.assertEqual(None, Config.get('parent.wrong_child.value'))
+        self.assertEqual(None, Config.get('parent.child.wrong_value'))
+        self.assertEqual(None, Config.get('parent.wrong_child'))
+
+    def test_existing_nested_values_return_dictionaries(self):
+        fake_config = {
+            'parent':
+                {'child':
+                    {
+                        'value_a': 42,
+                        'value_b': 'wut'
+                    }
+                },
+            'value_a': 8008
+        }
+        self.assertEqual(fake_config, Config.get())
+        self.assertEqual(fake_config['parent']['child'], Config.get('parent.child'))

longitude/samples/mixed_datasources.py

@@ -28,7 +28,7 @@
 from longitude.core.caches.redis import RedisCache
 from longitude.core.data_sources.postgres.default import DefaultPostgresDataSource
 from longitude.core.data_sources.carto import CartoDataSource
-from longitude.core.common.config import EnvironmentConfiguration
+from longitude.core.common.config import EnvironmentConfiguration as Config
 
 
 def import_table_values_from_carto(limit):
@@ -59,33 +59,15 @@ def import_table_values_from_carto(limit):
         params=params,
         needs_commit=True)
 
+    res = postgres.query('select * from county_population')
+    print(res.rows)
+
 
 if __name__ == "__main__":
 
-    # This is the global config object
-    # We are going to retrieve some values from a table in Carto, create a local table and copy the values
-    # doing simple inserts (to show how to do queries)
-
-    config = EnvironmentConfiguration({
-        'carto_main': {
-            'api_key': "=CARTO_API_KEY",
-            'user': "=CARTO_USER",
-
-            'cache': {
-                'password': '=REDIS_PASSWORD'
-            }
-        },
-        'postgres_main': {
-            'host': "=POSTGRES_HOST",
-            'port': "=POSTGRES_PORT",
-            'db': "=POSTGRES_DB",
-            'user': "=POSTGRES_USER",
-            'password': "=POSTGRES_PASS"
-        }
-    })
-
-    carto = CartoDataSource(config['carto_main'], cache_class=RedisCache)
-    postgres = DefaultPostgresDataSource(config['postgres_main'])
+    print('REDIS password is %s' % Config.get('carto_main.cache.password'))
+    carto = CartoDataSource(Config.get('carto_main'), cache_class=RedisCache)
+    postgres = DefaultPostgresDataSource(Config.get('postgres_main'))
     carto.setup()
     postgres.setup()