diff --git a/connector_importer/components/dynamicmapper.py b/connector_importer/components/dynamicmapper.py index ea4be5d9..4d118b6f 100644 --- a/connector_importer/components/dynamicmapper.py +++ b/connector_importer/components/dynamicmapper.py @@ -42,6 +42,11 @@ def dynamic_fields(self, record): # Eg: prefix all supplier fields w/ `supplier.` fname = fname[len(prefix) :] clean_record[fname] = clean_record.pop(prefix + fname) + final_fname = self._get_field_name(fname, clean_record) + if final_fname != fname: + clean_record[final_fname] = clean_record.pop(fname) + fname = final_fname + if available_fields.get(fname): fspec = available_fields.get(fname) ftype = fspec["type"] @@ -113,6 +118,18 @@ def _source_key_empty_skip(self): def _source_key_prefix(self): return self.work.options.mapper.get("source_key_prefix", "") + @property + def _source_key_rename(self): + return self.work.options.mapper.get("source_key_rename", {}) + + def _get_field_name(self, fname, clean_record): + """Return final field name. + + Field names can be manipulated via mapper option `source_key_rename` + which must be a dictionary w/ source name -> destination name. + """ + return self._source_key_rename.get(fname, fname) + def _is_xmlid_key(self, fname, ftype): return fname.startswith("xid::") and ftype in ( "many2one", diff --git a/connector_importer/models/record.py b/connector_importer/models/record.py index c61337a5..eac11471 100644 --- a/connector_importer/models/record.py +++ b/connector_importer/models/record.py @@ -72,6 +72,16 @@ def debug_mode(self): self.ensure_one() return self.backend_id.debug_mode or os.environ.get("IMPORTER_DEBUG_MODE") + def _should_use_jobs(self): + self.ensure_one() + debug_mode = self.debug_mode() + if debug_mode: + logger.warning("### DEBUG MODE ACTIVE: WILL NOT USE QUEUE ###") + use_job = self.recordset_id.import_type_id.use_job + if debug_mode: + use_job = False + return use_job + def import_record(self, importer_config): """This job will import a record. @@ -88,22 +98,18 @@ def import_record(self, importer_config): def run_import(self): """Queue a job for importing data stored in to self""" self.ensure_one() - use_job = self.recordset_id.import_type_id.use_job - # TODO: use ctx key to disable job instead - job_method = self.with_delay().import_record - if self.debug_mode(): - logger.warning("### DEBUG MODE ACTIVE: WILL NOT USE QUEUE ###") - if self.debug_mode() or not use_job: - job_method = self.import_record - result = self._run_import(job_method, use_job) + result = self._run_import(use_job=self._should_use_jobs()) return result - def _run_import(self, job_method, use_job): + def _run_import(self, use_job=True): res = {} # we create a record and a job for each model name # that needs to be imported + new_self = self.with_context(queue_job__no_delay=not use_job) for config in self.recordset_id.available_importers(): - result = job_method(config) + result = new_self.with_delay( + **self._run_import_job_params(config) + ).import_record(config) res[config.model] = result if self.debug_mode() or not use_job: # debug mode, no job here: reset it! @@ -114,3 +120,11 @@ def _run_import(self, job_method, use_job): # we keep the reference on w/ the last job. self.write({"job_id": result.db_record().id}) return res + + def _run_import_job_params(self, config): + params = { + "description": ( + f"recordset {self.recordset_id.name}: import {config['model']}" + ) + } + return params diff --git a/connector_importer/models/recordset.py b/connector_importer/models/recordset.py index f061279f..2d198bbb 100644 --- a/connector_importer/models/recordset.py +++ b/connector_importer/models/recordset.py @@ -8,6 +8,7 @@ from odoo import api, fields, models from odoo.addons.base_sparse_field.models.fields import Serialized +from odoo.addons.component.utils import is_component_registry_ready from odoo.addons.queue_job.job import DONE, STATES from ..log import logger @@ -342,7 +343,11 @@ def _get_importers(self): @api.depends("import_type_id") def _compute_docs_html(self): - template = self.env.ref("connector_importer.recordset_docs") + if not is_component_registry_ready(self.env.cr.dbname): + # We cannot render anything if we cannot load components + self.docs_html = False + return + qweb = self.env["ir.qweb"].sudo() for item in self: item.docs_html = False if isinstance(item.id, models.NewId) or not item.backend_id: @@ -352,7 +357,7 @@ def _compute_docs_html(self): continue importers = item._get_importers() data = {"recordset": item, "importers": importers} - item.docs_html = template._render(data) + item.docs_html = qweb._render("connector_importer.recordset_docs", data) # TODO diff --git a/connector_importer/tests/test_mapper.py b/connector_importer/tests/test_mapper.py index 1a82a00a..75de7a39 100644 --- a/connector_importer/tests/test_mapper.py +++ b/connector_importer/tests/test_mapper.py @@ -185,3 +185,16 @@ def test_rel_create_if_missing(self): self.assertEqual(cat.name, "New category") self.assertEqual(res["parent_id"], parent.id) self.assertEqual(res["category_id"], [(6, 0, [cat.id])]) + + def test_dynamic_mapper_rename_keys(self): + rec = { + "another_name": "John Doe", + } + # Whitelist + expected = { + "name": "John Doe", + } + mapper = self._get_dynamyc_mapper( + options=dict(source_key_rename={"another_name": "name"}) + ) + self.assertEqual(mapper.dynamic_fields(rec), expected) diff --git a/connector_importer/tests/test_record_importer.py b/connector_importer/tests/test_record_importer.py index f1f588e4..ba446d0f 100644 --- a/connector_importer/tests/test_record_importer.py +++ b/connector_importer/tests/test_record_importer.py @@ -42,7 +42,8 @@ def test_importer_create(self): expected = { model: {"created": 10, "errored": 0, "updated": 0, "skipped": 0}, } - self.assertEqual(res, expected) + delayable = res[model] + self.assertEqual(delayable.result, expected[model]) for k, v in expected[model].items(): self.assertEqual(len(report[model][k]), v) self.assertEqual(self.env[model].search_count([("ref", "like", "id_%")]), 10) @@ -60,7 +61,8 @@ def test_importer_skip(self): report = self.recordset.get_report() model = "res.partner" expected = {model: {"created": 8, "errored": 0, "updated": 0, "skipped": 2}} - self.assertEqual(res, expected) + delayable = res[model] + self.assertEqual(delayable.result, expected[model]) for k, v in expected[model].items(): self.assertEqual(len(report[model][k]), v) skipped_msg1 = report[model]["skipped"][0]["message"] @@ -80,7 +82,8 @@ def test_importer_update(self): report = self.recordset.get_report() model = "res.partner" expected = {model: {"created": 10, "errored": 0, "updated": 0, "skipped": 0}} - self.assertEqual(res, expected) + delayable = res[model] + self.assertEqual(delayable.result, expected[model]) for k, v in expected[model].items(): self.assertEqual(len(report[model][k]), v) # now run it a second time @@ -90,7 +93,8 @@ def test_importer_update(self): res = self.record.run_import() report = self.recordset.get_report() expected = {model: {"created": 0, "errored": 0, "updated": 10, "skipped": 0}} - self.assertEqual(res, expected) + delayable = res[model] + self.assertEqual(delayable.result, expected[model]) for k, v in expected[model].items(): self.assertEqual(len(report[model][k]), v) # now run it a second time @@ -100,7 +104,8 @@ def test_importer_update(self): res = self.record.run_import() report = self.recordset.get_report() expected = {model: {"created": 0, "errored": 0, "updated": 0, "skipped": 10}} - self.assertEqual(res, expected) + delayable = res[model] + self.assertEqual(delayable.result, expected[model]) for k, v in expected[model].items(): self.assertEqual(len(report[model][k]), v) skipped_msg1 = report[model]["skipped"][0]["message"] diff --git a/connector_importer/tests/test_record_importer_xmlid.py b/connector_importer/tests/test_record_importer_xmlid.py index f21515b7..14092c97 100644 --- a/connector_importer/tests/test_record_importer_xmlid.py +++ b/connector_importer/tests/test_record_importer_xmlid.py @@ -46,7 +46,8 @@ def test_importer_create(self): report = self.recordset.get_report() model = "res.partner" expected = {model: {"created": 10, "errored": 0, "updated": 0, "skipped": 0}} - self.assertEqual(res, expected) + delayable = res[model] + self.assertEqual(delayable.result, expected[model]) for k, v in expected[model].items(): self.assertEqual(len(report[model][k]), v) self.assertEqual(self.env[model].search_count([("ref", "like", "id_%")]), 10)