diff --git a/.circleci/config.yml b/.circleci/config.yml index 040010f..6a02bc5 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -14,7 +14,7 @@ jobs: source /usr/local/share/virtualenvs/tap-s3-csv/bin/activate pip install . pip install pylint - pylint tap_s3_csv -d duplicate-code,consider-using-f-string,logging-format-interpolation,missing-docstring,invalid-name,line-too-long,too-many-locals,too-few-public-methods,fixme,stop-iteration-return,broad-except,bare-except,unused-variable,unnecessary-comprehension,no-member,deprecated-method,protected-access + pylint tap_s3_csv -d duplicate-code,consider-using-f-string,logging-format-interpolation,missing-docstring,invalid-name,line-too-long,too-many-locals,too-few-public-methods,fixme,stop-iteration-return,broad-except,bare-except,unused-variable,unnecessary-comprehension,no-member,deprecated-method,protected-access,broad-exception-raised - run: name: 'Unit Tests' command: | diff --git a/CHANGELOG.md b/CHANGELOG.md index 384fb7a..b4b79a3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog +## 1.3.8 + * Add Missing Type Information in JSON Schema + * [#55](https://github.com/singer-io/tap-s3-csv/pull/62) + ## 1.3.7 * Remove Backoff for Access Denied errors * Add unitttest for access denied giveup diff --git a/setup.py b/setup.py index c951d34..3dd26d6 100644 --- a/setup.py +++ b/setup.py @@ -3,7 +3,7 @@ from setuptools import setup setup(name='tap-s3-csv', - version='1.3.7', + version='1.3.8', description='Singer.io tap for extracting CSV files from S3', author='Stitch', url='https://singer.io', diff --git a/tap_s3_csv/conversion.py b/tap_s3_csv/conversion.py index 3805545..fb15633 100644 --- a/tap_s3_csv/conversion.py +++ b/tap_s3_csv/conversion.py @@ -110,7 +110,7 @@ def generate_schema(samples, table_spec): elif datatype == 'list': counts[key] = { 'anyOf': [ - {'type': 'array', 'items': ['null', 'string']}, + {'type': 'array', 'items': {'type': ['null', 'string']}}, {'type': ['null', 'string']} ] } diff --git a/tests/unittests/test_conversion.py b/tests/unittests/test_conversion.py index 680fa8d..77c19ee 100644 --- a/tests/unittests/test_conversion.py +++ b/tests/unittests/test_conversion.py @@ -102,5 +102,5 @@ def test_generate_schema(self,mock_count_sample): samples = [{'name': 'test', 'id': 3, 'marks': [45.85, 25.38], 'students': {'no': 5, 'col': 6}, 'created_at': '20-05-2021', 'tota': []}] table_spec = {'search_prefix': '', 'search_pattern': 'test\\/.*\\.jsonl', 'table_name': 'jsonl_table', 'key_properties': ['id'],'date_overrides': ['created_at'], 'delimiter': ','} res = conversion.generate_schema(samples, table_spec) - expected_result = {'name': {'type': ['null', 'string']}, 'id': {'type': ['null', 'integer', 'string']}, 'marks': {'anyOf': [{'type': 'array', 'items': {'type': ['null', 'number', 'string']}}, {'type': ['null', 'string']}]}, 'students': {'anyOf': [{'type': 'object', 'properties': {}}, {'type': ['null', 'string']}]}, 'created_at': {'anyOf': [{'type': ['null', 'string'], 'format': 'date-time'}, {'type': ['null', 'string']}]}, 'tota': {'anyOf': [{'type': 'array', 'items': ['null', 'string']}, {'type': ['null', 'string']}]}} + expected_result = {'name': {'type': ['null', 'string']}, 'id': {'type': ['null', 'integer', 'string']}, 'marks': {'anyOf': [{'type': 'array', 'items': {'type': ['null', 'number', 'string']}}, {'type': ['null', 'string']}]}, 'students': {'anyOf': [{'type': 'object', 'properties': {}}, {'type': ['null', 'string']}]}, 'created_at': {'anyOf': [{'type': ['null', 'string'], 'format': 'date-time'}, {'type': ['null', 'string']}]}, 'tota': {'anyOf': [{'type': 'array', 'items': {'type': ['null', 'string']}}, {'type': ['null', 'string']}]}} self.assertEqual(res, expected_result)