[WIP][MIG] connector_jira: Migration to 17.0

connector_jira/__init__.py

@@ -4,4 +4,5 @@
 from . import components
 from . import controllers
 from . import models
+from . import reports
 from . import wizards

connector_jira/__manifest__.py

@@ -2,18 +2,23 @@
 
 {
     "name": "JIRA Connector",
-    "version": "15.0.2.0.0",
+    "version": "17.0.1.0.0",
     "author": "Camptocamp,Odoo Community Association (OCA)",
     "license": "AGPL-3",
     "category": "Connector",
     "depends": [
-        "connector",
+        # Odoo community
         "project",
         "hr_timesheet",
-        "queue_job",
         "web",
-        "web_widget_url_advanced",
+        # OCA/connector
+        "connector",
+        # OCA/queue
+        "queue_job",
+        # OCA/server-ux
         "multi_step_wizard",
+        # OCA/web
+        "web_widget_url_advanced",
     ],
     "external_dependencies": {
         "python": [
@@ -24,26 +29,36 @@
             "requests-toolbelt>=0.9.1",
             "requests-jwt>=0.6.0",
             "PyJWT>=1.7.1,<2.9.0",
-            "cryptography<37",
+            "cryptography>=38,<39",  # Compatibility w/ Odoo 17.0 requirements
             "atlassian_jwt>=3.0.0",
         ],
     },
     "website": "https://github.com/OCA/connector-jira",
     "data": [
-        "views/jira_menus.xml",
-        "views/project_link_jira_views.xml",
-        "views/task_link_jira_views.xml",
-        "views/jira_backend_views.xml",
-        "views/jira_backend_report_templates.xml",
-        "views/project_project_views.xml",
-        "views/project_task_views.xml",
-        "views/res_users_views.xml",
-        "views/jira_issue_type_views.xml",
-        "views/timesheet_account_analytic_line.xml",
-        "wizards/jira_account_analytic_line_import_views.xml",
+        # SECURITY
         "security/ir.model.access.csv",
+        # DATA
         "data/cron.xml",
-        "data/queue_job_data.xml",
+        "data/queue_job_channel.xml",
+        "data/queue_job_function.xml",
+        # VIEWS
+        # This file contains the root menu, import it first
+        "views/jira_menus.xml",
+        # Views, actions, menus
+        "views/account_analytic_line.xml",
+        "views/jira_backend.xml",
+        "views/jira_backend_report_templates.xml",
+        "views/jira_issue_type.xml",
+        "views/jira_project_project.xml",
+        "views/jira_project_task.xml",
+        "views/jira_res_users.xml",
+        "views/project_project.xml",
+        "views/project_task.xml",
+        "views/res_users.xml",
+        # Wizard views
+        "wizards/jira_account_analytic_line_import.xml",
+        "wizards/project_link_jira.xml",
+        "wizards/task_link_jira.xml",
     ],
     "demo": ["demo/jira_backend_demo.xml"],
     "installable": True,

connector_jira/components/__init__.py

@@ -1,8 +1,53 @@
 # License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl.html).
 
-from . import base
-from . import backend_adapter
-from . import binder
-from . import exporter
-from . import importer
-from . import mapper
+# ⚠️⚠️⚠️
+# 1) in order to ease readability and maintainability, components have been split into
+#    multiple files, each containing exactly 1 component
+# 2) components' import is sorted so that no dependency issue should arise
+# 3) next to each import, a comment will describe the components' dependencies
+# 4) when adding new components, please make sure it inherits (directly or indirectly)
+#    from ``jira.base``
+# ⚠️⚠️⚠️
+
+# Base abstract component
+from . import jira_base  # base.connector
+
+# Inheriting abstract components
+from . import jira_base_exporter  # base.exporter, jira.base
+from . import jira_batch_importer  # base.importer, jira.base
+from . import jira_delayed_batch_importer  # jira.batch.importer
+from . import jira_direct_batch_importer  # jira.batch.importer
+from . import jira_import_mapper  # base.import.mapper, jira.base
+from . import jira_timestamp_batch_importer  # base.importer, jira.base
+
+# Generic components
+from . import jira_binder  # base.binder, jira.base
+from . import jira_deleter  # base.deleter, jira.base
+from . import jira_exporter  # jira.base.exporter
+from . import jira_importer  # base.importer, jira.base
+from . import jira_webservice_adapter  # base.backend.adapter.crud, jira.base
+
+# Specific components
+from . import jira_analytic_line_batch_importer  # jira.timestamp.batch.importer
+from . import jira_analytic_line_importer  # jira.importer
+from . import jira_analytic_line_mapper  # jira.import.mapper
+from . import jira_analytic_line_timestamp_batch_deleter  # base.synchronizer, jira.base
+from . import jira_backend_adapter  # jira.webservice.adapter
+from . import jira_issue_type_adapter  # jira.webservice.adapter
+from . import jira_issue_type_batch_importer  # jira.direct.batch.importer
+from . import jira_issue_type_mapper  # jira.import.mapper
+from . import jira_mapper_from_attrs  # jira.base
+from . import jira_model_binder  # base.binder, jira.base
+from . import jira_project_adapter  # jira.webservice.adapter
+from . import jira_project_binder  # jira.binder
+from . import jira_project_project_listener  # base.connector.listener, jira.base
+from . import jira_project_project_exporter  # jira.exporter
+from . import jira_project_task_adapter  # jira.webservice.adapter
+from . import jira_project_task_batch_importer  # jira.timestamp.batch.importer
+from . import jira_project_task_importer  # jira.importer
+from . import jira_project_task_mapper  # jira.import.mapper
+from . import jira_res_users_adapter  # jira.webservice.adapter
+from . import jira_res_users_importer  # jira.importer
+from . import jira_task_project_matcher  # jira.base
+from . import jira_worklog_adapter  # jira.webservice.adapter
+from . import project_project_listener  # base.connector.listener, jira.base

connector_jira/components/mapper.py → connector_jira/components/common.py

@@ -1,4 +1,4 @@
-# Copyright 2016-2019 Camptocamp SA
+# Copyright 2016 Camptocamp SA
 # License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl.html).
 
 from datetime import datetime
@@ -8,20 +8,14 @@
 
 from odoo import fields
 
-from odoo.addons.component.core import AbstractComponent, Component
-from odoo.addons.connector.components.mapper import mapping
-
-
-class JiraImportMapper(AbstractComponent):
-    """Base Import Mapper for Jira"""
-
-    _name = "jira.import.mapper"
-    _inherit = ["base.import.mapper", "jira.base"]
-
-    @mapping
-    def jira_updated_at(self, record):
-        if self.options.external_updated_at:
-            return {"jira_updated_at": self.options.external_updated_at}
+JIRA_JQL_DATETIME_FORMAT = "%Y-%m-%d %H:%M"  # no seconds :-(
+RETRY_ON_ADVISORY_LOCK = 1  # seconds
+RETRY_WHEN_CONCURRENT_DETECTED = 1  # seconds
+# when we import using JQL, we always import tasks from
+# slightly before the last batch import, because Jira
+# does not send the results from the past minute and
+# maybe sometimes more
+IMPORT_DELTA = 300  # seconds
 
 
 def iso8601_to_utc_datetime(isodate):
@@ -33,9 +27,8 @@ def iso8601_to_utc_datetime(isodate):
     parsed = parser.parse(isodate)
     if not parsed.tzinfo:
         return parsed
-    utc = pytz.timezone("UTC")
     # set as UTC and then remove the tzinfo so the date becomes naive
-    return parsed.astimezone(utc).replace(tzinfo=None)
+    return parsed.astimezone(pytz.UTC).replace(tzinfo=None)
 
 
 def utc_datetime_to_iso8601(dt):
@@ -44,8 +37,7 @@ def utc_datetime_to_iso8601(dt):
     Example: 2013-11-04 12:52:01 → 2013-11-04T12:52:01+0000
 
     """
-    utc = pytz.timezone("UTC")
-    utc_dt = utc.localize(dt, is_dst=False)  # UTC = no DST
+    utc_dt = pytz.UTC.localize(dt, is_dst=False)  # UTC = no DST
     return utc_dt.isoformat()
 
 
@@ -85,8 +77,7 @@ def iso8601_to_naive_date(isodate):
     Example with 2014-10-07T00:34:59+0200: we want 2014-10-07 and not
     2014-10-06 that we would have using the timestamp converted to UTC.
     """
-    naive_date = isodate[:10]
-    return datetime.strptime(naive_date, "%Y-%m-%d").date()
+    return datetime.strptime(isodate[:10], "%Y-%m-%d").date()
 
 
 def iso8601_naive_date(field):
@@ -168,17 +159,3 @@ def modifier(self, record, to_attr):
         return value
 
     return modifier
-
-
-class FromFields(Component):
-    _name = "jira.mapper.from.attrs"
-    _inherit = ["jira.base"]
-    _usage = "map.from.attrs"
-
-    def values(self, record, mapper_):
-        values = {}
-        from_fields_mappings = getattr(mapper_, "from_fields", [])
-        fields_values = record.get("fields", {})
-        for source, target in from_fields_mappings:
-            values[target] = mapper_._map_direct(fields_values, source, target)
-        return values

connector_jira/components/jira_analytic_line_batch_importer.py

@@ -0,0 +1,78 @@
+# Copyright 2016 Camptocamp SA
+# Copyright 2019 Brainbean Apps (https://brainbeanapps.com)
+# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl.html).
+
+from odoo.addons.component.core import Component
+
+from ..fields import MilliDatetime
+
+
+class JiraAnalyticLineBatchImporter(Component):
+    """Import the Jira worklogs
+
+    For every ID in the list, a delayed job is created.
+    Import is executed starting from a given date.
+    """
+
+    _name = "jira.analytic.line.batch.importer"
+    _inherit = "jira.timestamp.batch.importer"
+    _apply_on = ["jira.account.analytic.line"]
+
+    def _search(self, timestamp):
+        unix_timestamp = MilliDatetime.to_timestamp(timestamp.last_timestamp)
+        result = self.backend_adapter.updated_since(since=unix_timestamp)
+        worklog_ids = self._filter_update(result.updated_worklogs)
+        # We need issue_id + worklog_id for the worklog importer (the jira
+        # "read" method for worklogs asks both), get it from yield_read.
+        # TODO we might consider to optimize the import process here:
+        #  yield_read reads worklogs data, then the individual
+        #  import will do a request again (and 2 with the tempo module)
+        next_timestamp = MilliDatetime.from_timestamp(result.until)
+        return next_timestamp, self.backend_adapter.yield_read(worklog_ids)
+
+    def _handle_records(self, records, force=False):
+        number = 0  # Cannot use ``len(records)`` cause ``records`` is a generator
+        for worklog in records:
+            number += 1
+            self._import_record(worklog["issueId"], worklog["id"], force=force)
+        return number
+
+    def _filter_update(self, updated_worklogs):
+        """Filter only the worklogs needing an update
+
+        The result from Jira contains the worklog id and
+        the last update on Jira. So we keep only the worklog
+        ids with a sync_date before the Jira last update.
+        """
+        if not updated_worklogs:
+            return []
+        self.env.cr.execute(
+            """
+            SELECT external_id, jira_updated_at
+            FROM jira_account_analytic_line
+            WHERE external_id IN %s
+            """,
+            (tuple(str(r.worklog_id) for r in updated_worklogs),),
+        )
+        bindings = dict(self.env.cr.fetchall())
+        td, ft = MilliDatetime.to_datetime, MilliDatetime.from_timestamp
+        worklog_ids = []
+        for worklog in updated_worklogs:
+            worklog_id = worklog.worklog_id
+            # we store the latest "updated_at" value on the binding
+            # so we can check if we already know the latest value,
+            # for instance because we imported the record from a
+            # webhook before, we can skip the import
+            binding_updated_at = bindings.get(str(worklog_id))
+            if not binding_updated_at or td(binding_updated_at) < ft(worklog.updated):
+                worklog_ids.append(worklog_id)
+        return worklog_ids
+
+    def _import_record(self, issue_id, worklog_id, force=False, **kwargs):
+        """Delay the import of the records"""
+        self.model.with_delay(**kwargs).import_record(
+            self.backend_record,
+            issue_id,
+            worklog_id,
+            force=force,
+        )