service: Support null (#79)

Until now, null values received from OData servers get substituted by
type specific default values.

Users could not differentiate between those substitutes and actual
values retrieved by the server, which might coincidentally equal the
default ones inserted by pyodata.

This commit allows the user to disable the substitution of null values,
retrieving a None object instead.

Example for Edm.Binary:

| Server | pyodata old behavior | pyodata new (retain_null=True) |
|--------+----------------------+--------------------------------|
|      0 |                    0 |                              0 |
|   null |                    0 |                           None |

CHANGELOG.md

@@ -6,6 +6,9 @@ and this project adheres to [Semantic Versioning](http://semver.org/).
 
 ## [Unreleased]
 
+### Added
+- Prevent substitution of missing, nullable values - Reto Schneider
+
 ### Fixed
 - Fix Increased robustness when schema with empty properties is returned
 - Use valid default value for Edm.DateTimeOffset - Reto Schneider

docs/usage/initialization.rst

@@ -164,6 +164,30 @@ Additionally, Schema class has Boolean atribute 'is_valid' that returns if the p
 
     northwind.schema.is_valid
 
+Prevent substitution by default values
+--------------------------------------
+
+Per default, missing properties get filled in by type specific default values. While convenient, this throws away
+the knowledge of whether a value was missing in the first place.
+To prevent this, the class config mentioned in the section above takes an additional parameter, `retain_null`.
+
+.. code-block:: python
+
+    import pyodata
+    import requests
+
+    SERVICE_URL = 'http://services.odata.org/V2/Northwind/Northwind.svc/'
+
+    northwind = pyodata.Client(SERVICE_URL, requests.Session(), config=pyodata.v2.model.Config(retain_null=True))
+
+    unknown_shipped_date = northwind.entity_sets.Orders_Qries.get_entity(OrderID=11058,
+                                                                         CompanyName='Blauer See Delikatessen').execute()
+
+    print(
+        f'Shipped date: {"unknown" if unknown_shipped_date.ShippedDate is None else unknown_shipped_date.ShippedDate}')
+
+Changing `retain_null` to `False` will print `Shipped date: 1753-01-01 00:00:00+00:00`.
+
 Set custom namespaces (Deprecated - use config instead)
 -------------------------------------------------------
 
@@ -183,4 +207,3 @@ hosted on private urls such as *customEdmxUrl.com* and *customEdmUrl.com*:
     }
 
     northwind = pyodata.Client(SERVICE_URL, requests.Session(), namespaces=namespaces)
-

pyodata/client.py

@@ -69,7 +69,7 @@ def __new__(cls, url, connection, odata_version=ODATA_VERSION_2, namespaces=None
 
             # create service instance based on model we have
             logger.info('Creating OData Service (version: %d)', odata_version)
-            service = pyodata.v2.service.Service(url, schema, connection)
+            service = pyodata.v2.service.Service(url, schema, connection, config=config)
 
             return service
 

pyodata/v2/model.py

@@ -91,7 +91,8 @@ class Config:
     def __init__(self,
                  custom_error_policies=None,
                  default_error_policy=None,
-                 xml_namespaces=None):
+                 xml_namespaces=None,
+                 retain_null=False):
 
         """
         :param custom_error_policies: {ParserError: ErrorPolicy} (default None)
@@ -102,6 +103,9 @@ def __init__(self,
                                      If custom policy is not specified for the tag, the default policy will be used.
 
         :param xml_namespaces: {str: str} (default None)
+
+        :param retain_null: bool (default False)
+                            If true, do not substitute missing (and null-able) values with default value.
         """
 
         self._custom_error_policy = custom_error_policies
@@ -116,6 +120,8 @@ def __init__(self,
 
         self._namespaces = xml_namespaces
 
+        self._retain_null = retain_null
+
     def err_policy(self, error: ParserError):
         if self._custom_error_policy is None:
             return self._default_error_policy
@@ -137,6 +143,10 @@ def namespaces(self):
     def namespaces(self, value: dict):
         self._namespaces = value
 
+    @property
+    def retain_null(self):
+        return self._retain_null
+
 
 class Identifier:
     def __init__(self, name):

pyodata/v2/service.py

@@ -734,7 +734,7 @@ class EntityProxy:
        named values), and links (references to other entities).
     """
 
-    # pylint: disable=too-many-branches,too-many-nested-blocks
+    # pylint: disable=too-many-branches,too-many-nested-blocks,too-many-statements
 
     def __init__(self, service, entity_set, entity_type, proprties=None, entity_key=None, etag=None):
         self._logger = logging.getLogger(LOGGER_NAME)
@@ -761,11 +761,20 @@ def __init__(self, service, entity_set, entity_type, proprties=None, entity_key=
             # first, cache values of direct properties
             for type_proprty in self._entity_type.proprties():
                 if type_proprty.name in proprties:
+                    # Property value available
                     if proprties[type_proprty.name] is not None:
                         self._cache[type_proprty.name] = type_proprty.from_json(proprties[type_proprty.name])
-                    else:
+                        continue
+                    # Property value missing and user wants a type specific default value filled in
+                    if not self._service.retain_null:
                         # null value is in literal form for now, convert it to python representation
                         self._cache[type_proprty.name] = type_proprty.from_literal(type_proprty.typ.null_value)
+                        continue
+                    # Property is nullable - save it as such
+                    if type_proprty.nullable:
+                        self._cache[type_proprty.name] = None
+                        continue
+                    raise PyODataException(f'Value of non-nullable Property {type_proprty.name} is null')
 
             # then, assign all navigation properties
             for prop in self._entity_type.nav_proprties:
@@ -1581,10 +1590,11 @@ def function_import_handler(fimport, response):
 class Service:
     """OData service"""
 
-    def __init__(self, url, schema, connection):
+    def __init__(self, url, schema, connection, config=None):
         self._url = url
         self._schema = schema
         self._connection = connection
+        self._retain_null = config.retain_null if config else False
         self._entity_container = EntityContainer(self)
         self._function_container = FunctionContainer(self)
 
@@ -1608,6 +1618,12 @@ def connection(self):
 
         return self._connection
 
+    @property
+    def retain_null(self):
+        """Whether to respect null-ed values or to substitute them with type specific default values"""
+
+        return self._retain_null
+
     @property
     def entity_sets(self):
         """EntitySet proxy"""

tests/test_service_v2.py

@@ -9,6 +9,7 @@
 import pyodata.v2.model
 import pyodata.v2.service
 from pyodata.exceptions import PyODataException, HttpError, ExpressionError, ProgramError, PyODataModelError
+from pyodata.v2 import model
 from pyodata.v2.service import EntityKey, EntityProxy, GetEntitySetFilter, ODataHttpResponse, HTTP_CODE_OK
 
 from tests.conftest import assert_request_contains_header, contents_of_fixtures_file
@@ -24,6 +25,13 @@ def service(schema):
     return pyodata.v2.service.Service(URL_ROOT, schema, requests)
 
 
+@pytest.fixture
+def service_retain_null(schema):
+    """Service fixture which keeps null values as such"""
+    assert schema.namespaces
+    return pyodata.v2.service.Service(URL_ROOT, schema, requests, model.Config(retain_null=True))
+
+
 @responses.activate
 def test_create_entity(service):
     """Basic test on creating entity"""
@@ -885,6 +893,88 @@ def test_get_entities(service):
     assert empls[0].NameFirst == 'Yennefer'
     assert empls[0].NameLast == 'De Vengerberg'
 
+
+@responses.activate
+def test_get_null_value_from_null_preserving_service(service_retain_null):
+    """Get entity with missing property value as None type"""
+
+    # pylint: disable=redefined-outer-name
+
+    responses.add(
+        responses.GET,
+        f"{service_retain_null.url}/Employees",
+        json={'d': {
+            'results': [
+                {
+                    'ID': 1337,
+                    'NameFirst': 'Neo',
+                    'NameLast': None
+                }
+            ]
+        }},
+        status=200)
+
+    request = service_retain_null.entity_sets.Employees.get_entities()
+
+    the_ones = request.execute()
+    assert the_ones[0].ID == 1337
+    assert the_ones[0].NameFirst == 'Neo'
+    assert the_ones[0].NameLast is None
+
+
+@responses.activate
+def test_get_null_value_from_non_null_preserving_service(service):
+    """Get entity with missing property value as default type"""
+
+    # pylint: disable=redefined-outer-name
+
+    responses.add(
+        responses.GET,
+        f"{service.url}/Employees",
+        json={'d': {
+            'results': [
+                {
+                    'ID': 1337,
+                    'NameFirst': 'Neo',
+                    'NameLast': None
+                }
+            ]
+        }},
+        status=200)
+
+    request = service.entity_sets.Employees.get_entities()
+
+    the_ones = request.execute()
+    assert the_ones[0].ID == 1337
+    assert the_ones[0].NameFirst == 'Neo'
+    assert the_ones[0].NameLast == ''
+
+
+@responses.activate
+def test_get_non_nullable_value(service_retain_null):
+    """Get error when receiving a null value for a non-nullable property"""
+
+    # pylint: disable=redefined-outer-name
+
+    responses.add(
+        responses.GET,
+        f"{service_retain_null.url}/Employees",
+        json={'d': {
+            'results': [
+                {
+                    'ID': None,
+                    'NameFirst': 'Neo',
+                }
+            ]
+        }},
+        status=200)
+
+    with pytest.raises(PyODataException) as e_info:
+        service_retain_null.entity_sets.Employees.get_entities().execute()
+
+    assert str(e_info.value) == 'Value of non-nullable Property ID is null'
+
+
 @responses.activate
 def test_navigation_multi(service):
     """Get entities via navigation property"""