[MIG] connector_importer_source_sftp: Migration to 16.0

connector_importer_source_sftp/README.rst

@@ -0,0 +1,100 @@
+==============================
+Connector Importer Source SFTP
+==============================
+
+.. 
+   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+   !! This file is generated by oca-gen-addon-readme !!
+   !! changes will be overwritten.                   !!
+   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+   !! source digest: sha256:1346fb2319de5d481cf4fa1d6df1311103df47eaf86f246405f45bdfcf54a513
+   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+
+.. |badge1| image:: https://img.shields.io/badge/maturity-Beta-yellow.png
+    :target: https://odoo-community.org/page/development-status
+    :alt: Beta
+.. |badge2| image:: https://img.shields.io/badge/licence-AGPL--3-blue.png
+    :target: http://www.gnu.org/licenses/agpl-3.0-standalone.html
+    :alt: License: AGPL-3
+.. |badge3| image:: https://img.shields.io/badge/github-OCA%2Fconnector--interfaces-lightgray.png?logo=github
+    :target: https://github.com/OCA/connector-interfaces/tree/16.0/connector_importer_source_sftp
+    :alt: OCA/connector-interfaces
+.. |badge4| image:: https://img.shields.io/badge/weblate-Translate%20me-F47D42.png
+    :target: https://translation.odoo-community.org/projects/connector-interfaces-16-0/connector-interfaces-16-0-connector_importer_source_sftp
+    :alt: Translate me on Weblate
+.. |badge5| image:: https://img.shields.io/badge/runboat-Try%20me-875A7B.png
+    :target: https://runboat.odoo-community.org/builds?repo=OCA/connector-interfaces&target_branch=16.0
+    :alt: Try me on Runboat
+
+|badge1| |badge2| |badge3| |badge4| |badge5|
+
+Add import source capable of loading files from SFTP.
+
+Source type provided at the moment: CSV.
+
+Special feature: move files to another path when the import is finished.
+
+You can configure input, error, success path.
+
+Files are searched into input folder and if the flag `move_file_after_import`
+is enabled, the file will be moved to error or success path
+depending on the result of the import process.
+
+**Table of contents**
+
+.. contents::
+   :local:
+
+Bug Tracker
+===========
+
+Bugs are tracked on `GitHub Issues <https://github.com/OCA/connector-interfaces/issues>`_.
+In case of trouble, please check there if your issue has already been reported.
+If you spotted it first, help us to smash it by providing a detailed and welcomed
+`feedback <https://github.com/OCA/connector-interfaces/issues/new?body=module:%20connector_importer_source_sftp%0Aversion:%2016.0%0A%0A**Steps%20to%20reproduce**%0A-%20...%0A%0A**Current%20behavior**%0A%0A**Expected%20behavior**>`_.
+
+Do not contact contributors directly about support or help with technical issues.
+
+Credits
+=======
+
+Authors
+~~~~~~~
+
+* Camptocamp
+* ACSONE
+
+Contributors
+~~~~~~~~~~~~
+
+* Simone Orsi <[email protected]>
+* Sébastien Alix <[email protected]>
+* Daniel Haag <[email protected]>
+
+Maintainers
+~~~~~~~~~~~
+
+This module is maintained by the OCA.
+
+.. image:: https://odoo-community.org/logo.png
+   :alt: Odoo Community Association
+   :target: https://odoo-community.org
+
+OCA, or the Odoo Community Association, is a nonprofit organization whose
+mission is to support the collaborative development of Odoo features and
+promote its widespread use.
+
+.. |maintainer-simahawk| image:: https://github.com/simahawk.png?size=40px
+    :target: https://github.com/simahawk
+    :alt: simahawk
+.. |maintainer-sebalix| image:: https://github.com/sebalix.png?size=40px
+    :target: https://github.com/sebalix
+    :alt: sebalix
+
+Current `maintainers <https://odoo-community.org/page/maintainer-role>`__:
+
+|maintainer-simahawk| |maintainer-sebalix| 
+
+This module is part of the `OCA/connector-interfaces <https://github.com/OCA/connector-interfaces/tree/16.0/connector_importer_source_sftp>`_ project on GitHub.
+
+You are welcome to contribute. To learn how please visit https://odoo-community.org/page/Contribute.

connector_importer_source_sftp/__init__.py

@@ -0,0 +1,2 @@
+from . import components
+from . import models

connector_importer_source_sftp/__manifest__.py

@@ -0,0 +1,32 @@
+# Copyright 2019 Camptocamp SA
+# Copyright 2020 ACSONE SA/NV (<http://acsone.eu>)
+# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl).
+
+{
+    "name": "Connector Importer Source SFTP",
+    "summary": """Add import source capable of loading files from SFTP.""",
+    "version": "16.0.1.0.0",
+    "license": "AGPL-3",
+    "author": "Camptocamp,ACSONE,Odoo Community Association (OCA)",
+    "maintainers": ["simahawk", "sebalix"],
+    "website": "https://github.com/OCA/connector-interfaces",
+    # fmt: off
+    "depends": [
+        "connector_importer",
+        "component_event",
+        "storage_backend_sftp",
+    ],
+    # fmt: on
+    # Place an upper bound on cryptography version to be compatible with
+    # pyopenssl 19 mentioned in Odoo 15's requirements.txt. If we don't do
+    # this, installing this module will try to upgrade cryptography to the latest
+    # version because the minimum required version in pyopenssl (>=3.1) is greater than
+    # version 2.6 (from Odoo's requirement.txt). Since cryptography/pyopenssl don't
+    # declare minimum supported versions, this lead to inconsistencies.
+    # https://github.com/OCA/server-auth/issues/424
+    # https://github.com/OCA/storage/pull/247
+    # https://github.com/OCA/connector-interfaces/pull/94
+    "external_dependencies": {"python": ["cryptography<39"]},
+    "data": ["views/source_csv_sftp.xml", "security/ir.model.access.csv"],
+    "demo": ["demo/storage_backend_demo.xml", "demo/import_source_demo.xml"],
+}

connector_importer_source_sftp/components/__init__.py

@@ -0,0 +1 @@
+from . import event_listeners

connector_importer_source_sftp/components/event_listeners.py

@@ -0,0 +1,94 @@
+# Copyright 2019 Camptocamp SA
+# @author: Sebastien Alix <[email protected]>
+# Copyright 2020 ACSONE SA/NV (<http://acsone.eu>)
+# @author: Simone Orsi <[email protected]>
+# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html).
+
+import functools
+
+from odoo.addons.component.core import Component
+from odoo.addons.component_event import skip_if
+from odoo.addons.connector_importer.log import logger
+
+
+class SFTPSourceImportRecordsetEventListener(Component):
+    _name = "sftp.source.import.recorset.event.listener"
+    _inherit = "base.connector.listener"
+    _apply_on = ["import.recordset"]
+
+    @skip_if(lambda self, importer, record: self._skip_move_file(importer, record))
+    def on_last_record_import_finished(self, importer, record):
+        self._move_file(importer)
+        self._report_errors(importer)
+
+    def _skip_move_file(self, importer, record):
+        """Check if the file used for the import should be moved.
+
+        :param importer: importer component instance
+        :param record: import.record instance (the last one processed)
+        """
+        if record.env.context.get("_sftp_skip_move_file"):
+            return True
+        source = importer.recordset.get_source()
+        if source._name != "import.source.csv.sftp":
+            return True
+        return not source.move_file_after_import if source else True
+
+    def _move_file(self, importer):
+        """Move file on the SFTP.
+
+        :param importer: importer component instance
+        """
+        source = importer.recordset.get_source()
+        storage = source.storage_id
+        sftp_filepath = source._sftp_filepath()
+        sftp_destination_path = False
+
+        if self._move_file_to_error(importer):
+            sftp_destination_path = source.sftp_path_error
+            logger.info(
+                "Errors occurred during import, moving the file %s to %s...",
+                sftp_filepath,
+                sftp_destination_path,
+            )
+        elif self._move_file_to_success(importer):
+            sftp_destination_path = source.sftp_path_success
+            logger.info(
+                "File imported, moving the file %s to %s...",
+                sftp_filepath,
+                sftp_destination_path,
+            )
+
+        if sftp_destination_path:
+            self._add_after_commit_hook(
+                storage.move_files, sftp_filepath, sftp_destination_path
+            )
+
+    # TODO: make it configurable on the source via code snippet
+    # when it should go to errored or success.
+    def _move_file_to_error(self, importer):
+        """State if file should be moved to error folder."""
+        counters = importer.tracker.get_counters()
+        return counters["errored"]
+
+    def _move_file_to_success(self, importer):
+        """State if file should be moved to success folder."""
+        counters = importer.tracker.get_counters()
+        return counters["created"] or counters["updated"] or counters["skipped"]
+
+    def _add_after_commit_hook(self, move_func, sftp_filepath, sftp_destination_path):
+        """Add hook after commit to move the file when transaction is over."""
+        self.env.cr.postcommit.add(
+            functools.partial(move_func, [sftp_filepath], sftp_destination_path),
+        )
+
+    def _report_errors(self, importer):
+        recordset = importer.recordset
+        source = recordset.get_source()
+        if not source.send_back_error_report or not self._move_file_to_error(importer):
+            return
+        recordset.generate_report()
+        csv_report = recordset.report_file
+        if csv_report:
+            filepath = source._sftp_filepath("error").replace(".csv", ".report.csv")
+            source.storage_id.add(filepath, csv_report, binary=False)

connector_importer_source_sftp/demo/import_source_demo.xml

@@ -0,0 +1,16 @@
+<?xml version="1.0" encoding="utf-8" ?>
+<!-- Copyright 2020 ACSONE SA/NV (<http://acsone.eu>)
+     License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl). -->
+<odoo>
+    <record id="demo_import_source_csv_sftp" model="import.source.csv.sftp">
+        <field name="name">demo_source_sftp_csv</field>
+        <field name="csv_delimiter">,</field>
+        <field name="csv_encoding">utf-8</field>
+        <field name="storage_id" ref="demo_storage_backend_sftp" />
+        <field name="sftp_filename_pattern">.*\.csv$</field>
+        <field name="move_file_after_import" eval="False" />
+        <field name="sftp_path_input">input</field>
+        <field name="sftp_path_error">error</field>
+        <field name="sftp_path_success">success</field>
+    </record>
+</odoo>

connector_importer_source_sftp/demo/storage_backend_demo.xml

@@ -0,0 +1,12 @@
+<?xml version="1.0" encoding="utf-8" ?>
+<odoo noupdate="1">
+    <record id="demo_storage_backend_sftp" model="storage.backend">
+        <field name="name">SFTP Backend</field>
+        <field name="backend_type">sftp</field>
+        <field name="sftp_login">foo</field>
+        <field name="sftp_password">pass</field>
+        <field name="sftp_server">localhost</field>
+        <field name="sftp_port">2222</field>
+        <field name="directory_path">upload</field>
+    </record>
+</odoo>