From 81c830d7d710375b5e5a7390dc07ef143ff32362 Mon Sep 17 00:00:00 2001 From: Simone Orsi Date: Wed, 4 Sep 2024 12:12:19 +0200 Subject: [PATCH] connector_importer: add some docs :) --- .../components/dynamicmapper.py | 41 +++- connector_importer/readme/CONFIGURE.rst | 181 ++++++++++++++++++ connector_importer/readme/DESCRIPTION.rst | 9 +- connector_importer/readme/ROADMAP.rst | 2 - 4 files changed, 228 insertions(+), 5 deletions(-) create mode 100644 connector_importer/readme/CONFIGURE.rst diff --git a/connector_importer/components/dynamicmapper.py b/connector_importer/components/dynamicmapper.py index 4d118b6fb..adaeb7009 100644 --- a/connector_importer/components/dynamicmapper.py +++ b/connector_importer/components/dynamicmapper.py @@ -9,7 +9,43 @@ class DynamicMapper(Component): - """A mapper that dynamically converts input data to odoo fields values.""" + """A mapper that dynamically converts input data to odoo fields values. + + The behavior is affected by the options provided to the mapper work ctx. + Normally these options are provided by the importer component + that will load them from the import type yaml conf: + + options: + mapper: + source_key_whitelist: [] + source_key_blacklist: [] + source_key_empty_skip: [] + source_key_prefix: "" + source_key_rename: {} + converter: {} + + `source_key_whitelist` and `source_key_blacklist` are used to filter the keys. + `source_key_empty_skip` is used to skip keys when empty or no value is computed for them. + `source_key_prefix` is to consider only keys that start with the given prefix. + + It's a sort of whitelist but it allows to filter keys dynamically + which is very handy when importing more than one model per import type. + + `source_key_rename` is used to rename source keys to destination key (the real odoo field). + `converter` is used to define custom converter options for specific fields. + + The value must be a dict containing the params to propagate to the converter function. + Eg: for a m2o field `partner_id` that needs to be converted to a res.partner record + the converter option could be: + + converter: + partner_id: + create_missing: true + search_field: "ref" + + The options are in fact the args that the converter functions accept. + Have a look at the `convert` function in `mapper_utils.py` for more details. + """ _name = "importer.mapper.dynamic" _inherit = "importer.base.mapper" @@ -19,7 +55,8 @@ class DynamicMapper(Component): def dynamic_fields(self, record): """Resolve values for non mapped keys. - Source keys = destination keys. + :param record: a dictionary of key/value pairs coming from the source data + already prepared by the importer. """ # TODO: add tests! model = self.work.model_name diff --git a/connector_importer/readme/CONFIGURE.rst b/connector_importer/readme/CONFIGURE.rst new file mode 100644 index 000000000..df833e946 --- /dev/null +++ b/connector_importer/readme/CONFIGURE.rst @@ -0,0 +1,181 @@ + +Import type +~~~~~~~~~~~ + +Import types are the main configuration of the import. +They describe which models you want to import and how to import them. + +Exaple of configuration:: + + + Import Product - all in one + product_product_all_in_one + + + - model: product.product + options: + importer: + odoo_unique_key: barcode + mapper: + name: product.product.mapper + + - model: res.partner + options: + importer: + odoo_unique_key: name + override_existing: false + mapper: + name: importer.mapper.dynamic + source_key_prefix: supplier. + source_key_whitelist: supplier.name + default_keys: + supplier_rank: 1 + + - model: product.supplierinfo + options: + importer: + odoo_unique_key: name + mapper: + name: product.supplierinfo.mapper + source_key_prefix: supplier. + + + + + +In this example we have 3 models to import one after the other using the same source file: + +* product.product +* res.partner +* product.supplierinfo + +The import will run in the order of the configuration: first product.product, then res.partner and finally product.supplierinfo. +For each model we have a configuration that describes how to import the data. +With the ``options`` key we can define the configuration of the import for each component: ``importer``, ``mapper``, ``record_handler``, ``tracking_handler``. + +The are 4 main components in the import configuration: + +* importer +* mapper +* record_handler +* tracking_handler + +Each of them is responsible for a specific part of the import. + +The importer +~~~~~~~~~~~~ + +``importer`` is the main component that will import the data. It will use the ``mapper`` to map the data from the source to the destination model. +If no ``name`` is defined the importer will use the default importer for the model which is capable of importing any model. +Most of the time you don't need a specific importer. + +As the importer is the main component of the import if you want to customize it you'll have to declare it at an higher level, next to the ``options`` key:: + + - model: product.product + importer: + name: product.product.importer + options: + mapper: + name: product.product.mapper + +The importer accepts the following options: + +* ``odoo_unique_key``: the field that will be used to find the record in Odoo. If the record is found it will be updated, otherwise it will be created. + + NOTE: the value in the column declared as ``odoo_unique_key`` will be treated as xid only if the name of the column is ``ìd`` or if it starts with ``xid::``. + +* ``break_on_error``: if set to True the import will stop if an error occurs. Default is False. +* ``override_existing``: if set to True the existing records will be updated. Default is True. +* ``translation_key_sep``: the separator used to split the translation key. Default is ``:``. See below for information about translation keys. +* ``translation_use_regional_lang``: if set to True the importer will use the regional language, eg: `fr_CH` vs `fr`. +* ``ctx``: a dictionary of values to inject in the context of the import. +* ``write_only``: if set to True the importer will not create new records, it will only update existing ones. Default is False. + + +The mapper +~~~~~~~~~~ + +The mapper is the component that will map the data from the source to the destination model. + +The most flexible mapper is the ``importer.mapper.dynamic`` that will map the data based on the model introspection and some options that you can define. +The dynamic mapper accepts the following options: + +* ``name``: the name of the mapper to use. If no name is defined the default mapper for the model will be used. +* ``source_key_prefix``: a prefix to add to the source key. This is useful when you want to map the same source key to different destination fields. +* ``source_key_whitelist``: a list of source keys to import. If not defined all the keys will be imported. +* ``source_key_blacklist``: a list of source keys to exclude from the import. +* ``source_key_rename``: a dictionary of source keys to rename. The key is the source key and the value is the new key. +* ``default_keys``: a dictionary of default values to set on the destination record. The key is the field name and the value is the default value. +* ``translation_keys``: a list of keys that will be used to translate the data. See below for information about translation keys. +* ``required_keys``: a list of keys that are required. If one of the keys is missing the record will be skipped. Please refer to the documentation of the mapper to see advanced options. + +Considering the example above:: + + - model: product.product + options: + mapper: + name: importer.mapper.dynamic + source_key_prefix: supplier. + source_key_whitelist: supplier.name + default_keys: + supplier_rank: 1 + +The mapper will: + +* import only keys starting with ``supplier.`` ignoring the rest +* import only the key ``supplier.name`` +* set the default value of ``supplier_rank`` to 1 + +The record_handler +~~~~~~~~~~~~~~~~~~ + +The record handler is the component that will handle the record create or update in Odoo. +This component is responsible for: + +* finding the record in Odoo +* creating the record if not found +* updating the record if found +* handling the translations + +If no ``name`` is defined the importer will use the default record handler for the model which is capable of handling any model. +If you want to customize the record handler you'll have to declare it at an higher level, next to the ``options`` key:: + + - model: product.product + options: + record_handler: + name: product.product.record_handler + +To find the record in Odoo the record handler will use the ``odoo_unique_key`` if defined in the importer otherwise it will fallback to the matching domain. See below. + +The record handler accepts the following options: + +* ``name``: the name of the record handler to use. If no name is defined the default record handler for the model will be used. +* ``match_domain``: a domain to match the record in Odoo. When no odoo_unique_key is provided by the importer you must provide a match_domain. + + This key accepts a snippet returning a domain. The snippet will be evaluated in the context of the import and will receive: + + * ``orig_values``: the values from the source + * ``values``: values computed by the mapper for the record + * ``env`` + * ``user`` + * ``datetime`` + * ``dateutil`` + * ``time`` + * ``ref_id``: a function to get a record ID from a reference + * ``ref``: a function to get a record from a reference + + Example:: + + match_domain: | + [('name', '=', values.get('name'))] + +* ``must_generate_xmlid``: if set to True the importer will generate an XML ID for the record. Default is True if the unique key is an xmlid. +* ``skip_fields_unchanged``: if set to True the importer will skip the fields that are unchanged. Default is False. + + +Translations +~~~~~~~~~~~~ + +The importer can translate the data using the translation keys. The translation keys are a list of keys (column) that will be handled as translatable. +Whenever a key is found in the translation keys the importer will look for a column with the same name suffixed by the language code (eg: name:fr_CH). +If the column is found the importer will translate the data using the language code as context. diff --git a/connector_importer/readme/DESCRIPTION.rst b/connector_importer/readme/DESCRIPTION.rst index 6d877e2ee..a27290ed5 100644 --- a/connector_importer/readme/DESCRIPTION.rst +++ b/connector_importer/readme/DESCRIPTION.rst @@ -1,2 +1,9 @@ This module allows to import / update records from files using the connector -framework (i.e. mappers) and job queues. +framework and job queue. + +To run an import you need at least: + +* a backend, hosts the global configuration of the import. +* a recordset, hosts the configuration of the import for specific models and source +* a source, provides the data to import +* an import type, describes which models you want to import and how to import them diff --git a/connector_importer/readme/ROADMAP.rst b/connector_importer/readme/ROADMAP.rst index b74ce95c5..b1b96f055 100644 --- a/connector_importer/readme/ROADMAP.rst +++ b/connector_importer/readme/ROADMAP.rst @@ -4,8 +4,6 @@ The job is automatically retried a second time (without concurrency errors). For small files it's not a big issue, but for files with a huge amount of lines it takes time to process them two times. -* refactor the `recordset.full_report_url` field to return a QWeb report - instead of a home-made HTML document + display it on the recordset form. * move generic functions from `utils.mapper_utils` to the `connector` module * unit tests for record handler and tracker * add more test coverage for mapper utils and dynamic mapper