From b18ef151126234b49fa1d0bf317edeed804033ad Mon Sep 17 00:00:00 2001 From: Youssef AbouEgla <57115457+YoussefEgla@users.noreply.github.com> Date: Fri, 17 Apr 2026 16:10:09 +0200 Subject: [PATCH] [19.0][MIG] base_import_async: migration to 19.0 (#1) * [19.0][MIG] base_import_async: migration to 19.0 * [IMP] base_import_async: add contributor credit * [FIX] base_import_async: sync generated files --- .pre-commit-config.yaml | 1 - base_import_async/README.rst | 18 +- base_import_async/__manifest__.py | 4 +- .../models/base_import_import.py | 22 +- base_import_async/models/queue_job.py | 8 +- base_import_async/readme/CONTRIBUTORS.md | 2 + .../static/description/index.html | 39 ++-- .../tests/test_base_import_import.py | 221 ++++++++++++------ 8 files changed, 214 insertions(+), 101 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 4cefd00cac..af6b972525 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,7 +1,6 @@ exclude: | (?x) # NOT INSTALLABLE ADDONS - ^base_import_async/| ^queue_job_batch/| ^queue_job_cron/| ^queue_job_cron_jobrunner/| diff --git a/base_import_async/README.rst b/base_import_async/README.rst index 030c371758..62b734c9af 100644 --- a/base_import_async/README.rst +++ b/base_import_async/README.rst @@ -1,3 +1,7 @@ +.. image:: https://odoo-community.org/readme-banner-image + :target: https://odoo-community.org/get-involved?utm_source=readme + :alt: Odoo Community Association + =================== Asynchronous Import =================== @@ -13,17 +17,17 @@ Asynchronous Import .. |badge1| image:: https://img.shields.io/badge/maturity-Production%2FStable-green.png :target: https://odoo-community.org/page/development-status :alt: Production/Stable -.. |badge2| image:: https://img.shields.io/badge/licence-AGPL--3-blue.png +.. |badge2| image:: https://img.shields.io/badge/license-AGPL--3-blue.png :target: http://www.gnu.org/licenses/agpl-3.0-standalone.html :alt: License: AGPL-3 .. |badge3| image:: https://img.shields.io/badge/github-OCA%2Fqueue-lightgray.png?logo=github - :target: https://github.com/OCA/queue/tree/18.0/base_import_async + :target: https://github.com/OCA/queue/tree/19.0/base_import_async :alt: OCA/queue .. |badge4| image:: https://img.shields.io/badge/weblate-Translate%20me-F47D42.png - :target: https://translation.odoo-community.org/projects/queue-18-0/queue-18-0-base_import_async + :target: https://translation.odoo-community.org/projects/queue-19-0/queue-19-0-base_import_async :alt: Translate me on Weblate .. |badge5| image:: https://img.shields.io/badge/runboat-Try%20me-875A7B.png - :target: https://runboat.odoo-community.org/builds?repo=OCA/queue&target_branch=18.0 + :target: https://runboat.odoo-community.org/builds?repo=OCA/queue&target_branch=19.0 :alt: Try me on Runboat |badge1| |badge2| |badge3| |badge4| |badge5| @@ -87,7 +91,7 @@ Bug Tracker Bugs are tracked on `GitHub Issues `_. In case of trouble, please check there if your issue has already been reported. If you spotted it first, help us to smash it by providing a detailed and welcomed -`feedback `_. +`feedback `_. Do not contact contributors directly about support or help with technical issues. @@ -130,6 +134,8 @@ Other contributors include: - Daniel Duque (FactorLibre) +- Youssef Egla + Other credits ------------- @@ -149,6 +155,6 @@ OCA, or the Odoo Community Association, is a nonprofit organization whose mission is to support the collaborative development of Odoo features and promote its widespread use. -This module is part of the `OCA/queue `_ project on GitHub. +This module is part of the `OCA/queue `_ project on GitHub. You are welcome to contribute. To learn how please visit https://odoo-community.org/page/Contribute. diff --git a/base_import_async/__manifest__.py b/base_import_async/__manifest__.py index 5432d7c5ca..58dd6eb8b0 100644 --- a/base_import_async/__manifest__.py +++ b/base_import_async/__manifest__.py @@ -5,7 +5,7 @@ { "name": "Asynchronous Import", "summary": "Import CSV files in the background", - "version": "18.0.1.0.0", + "version": "19.0.1.0.0", "author": "Akretion, ACSONE SA/NV, Odoo Community Association (OCA)", "license": "AGPL-3", "website": "https://github.com/OCA/queue", @@ -20,6 +20,6 @@ "base_import_async/static/src/xml/import_data_sidepanel.xml", ], }, - "installable": False, + "installable": True, "development_status": "Production/Stable", } diff --git a/base_import_async/models/base_import_import.py b/base_import_async/models/base_import_import.py index 856828a593..e9a0dede3c 100644 --- a/base_import_async/models/base_import_import.py +++ b/base_import_async/models/base_import_import.py @@ -9,7 +9,7 @@ from io import BytesIO, StringIO, TextIOWrapper from os.path import splitext -from odoo import _, api, models +from odoo import _, models from odoo.models import fix_import_export_id_paths from odoo.addons.base_import.models.base_import import ImportValidationError @@ -34,6 +34,7 @@ class BaseImportImport(models.TransientModel): _inherit = "base_import.import" def execute_import(self, fields, columns, options, dryrun=False): + self.ensure_one() if dryrun or not options.get(OPT_USE_QUEUE): # normal import return super().execute_import(fields, columns, options, dryrun=dryrun) @@ -41,8 +42,15 @@ def execute_import(self, fields, columns, options, dryrun=False): # asynchronous import try: data, import_fields = self._convert_import_data(fields, options) + if errors := self._parse_datetime_data(import_fields, data): + return {"messages": errors} # Parse date and float field data = self._parse_import_data(data, import_fields, options) + import_fields, data = self._handle_multi_mapping(import_fields, data) + if options.get("fallback_values"): + data = self._handle_fallback_values( + import_fields, data, options["fallback_values"] + ) except (ImportValidationError, ValueError) as e: return {"messages": [e.__dict__]} @@ -78,7 +86,6 @@ def _link_attachment_to_job(self, delayed_job, attachment): ) attachment.write({"res_model": "queue.job", "res_id": queue_job.id}) - @api.returns("ir.attachment") def _create_csv_attachment(self, fields, data, options, file_name): # write csv f = StringIO() @@ -182,7 +189,16 @@ def _split_file( priority += 1 def _import_one_chunk(self, model_name, attachment, options): - model_obj = self.env[model_name] + load_context = { + "import_file": True, + "tracking_disable": options.get("tracking_disable"), + "name_create_enabled_fields": options.get( + "name_create_enabled_fields", {} + ), + "import_set_empty_fields": options.get("import_set_empty_fields", []), + "import_skip_records": options.get("import_skip_records", []), + } + model_obj = self.env[model_name].with_context(**load_context) fields, data = self._read_csv_attachment(attachment, options) result = model_obj.load(fields, data) error_message = [ diff --git a/base_import_async/models/queue_job.py b/base_import_async/models/queue_job.py index b7313505f3..dbf0465365 100644 --- a/base_import_async/models/queue_job.py +++ b/base_import_async/models/queue_job.py @@ -10,10 +10,16 @@ class QueueJob(models.Model): _inherit = "queue.job" def _related_action_attachment(self): + attachment = self.env["ir.attachment"].search( + [("res_model", "=", "queue.job"), ("res_id", "=", self.id)], + limit=1, + ) + if not attachment: + return None return { "name": _("Attachment"), "type": "ir.actions.act_window", "res_model": "ir.attachment", "view_mode": "form", - "res_id": self.kwargs.get("att_id"), + "res_id": attachment.id, } diff --git a/base_import_async/readme/CONTRIBUTORS.md b/base_import_async/readme/CONTRIBUTORS.md index 729ae17015..8f66996913 100644 --- a/base_import_async/readme/CONTRIBUTORS.md +++ b/base_import_async/readme/CONTRIBUTORS.md @@ -23,3 +23,5 @@ Other contributors include: - Do Anh Duy \<\> - Daniel Duque (FactorLibre) + +- Youssef Egla diff --git a/base_import_async/static/description/index.html b/base_import_async/static/description/index.html index 2994cf9efa..af24627627 100644 --- a/base_import_async/static/description/index.html +++ b/base_import_async/static/description/index.html @@ -3,7 +3,7 @@ -Asynchronous Import +README.rst -
-

Asynchronous Import

+
+ + +Odoo Community Association + +
+

Asynchronous Import

-

Production/Stable License: AGPL-3 OCA/queue Translate me on Weblate Try me on Runboat

+

Production/Stable License: AGPL-3 OCA/queue Translate me on Weblate Try me on Runboat

This module extends the standard CSV import functionality to import files in the background using the OCA/queue framework.

Table of contents

@@ -392,7 +397,7 @@

Asynchronous Import

-

Usage

+

Usage

The user is presented with a new checkbox in the import screen. When selected, the import is delayed in a background job.

This job in turn splits the CSV file in chunks of minimum 100 lines (or @@ -416,7 +421,7 @@

Usage

-

Known issues / Roadmap

+

Known issues / Roadmap

  • There is currently no user interface to control the chunk size, which is currently 100 by default. Should this proves to be an issue, it is @@ -425,33 +430,33 @@

    Known issues / Roadmap

-

Changelog

+

Changelog

-

13.0.1.0.0 (2019-12-20)

+

13.0.1.0.0 (2019-12-20)

  • [MIGRATION] from 12.0 branched at rev. a7f8031
-

Bug Tracker

+

Bug Tracker

Bugs are tracked on GitHub Issues. In case of trouble, please check there if your issue has already been reported. If you spotted it first, help us to smash it by providing a detailed and welcomed -feedback.

+feedback.

Do not contact contributors directly about support or help with technical issues.

-

Credits

+

Credits

-

Authors

+

Authors

  • Akretion
  • ACSONE SA/NV
-

Contributors

+

Contributors

Sébastien Beau (Akretion) authored the initial prototype.

Stéphane Bidoul (ACSONE) extended it to version 1.0 to support multi-line records, store data to import as attachments and let the user @@ -470,15 +475,16 @@

Contributors

  • Daniel Duque (FactorLibre)
    • +
  • Youssef Egla
    • -

    Other credits

    +

    Other credits

    The migration of this module from 17.0 to 18.0 was financially supported by Camptocamp

    -

    Maintainers

    +

    Maintainers

    This module is maintained by the OCA.

    Odoo Community Association @@ -486,10 +492,11 @@

    Maintainers

    OCA, or the Odoo Community Association, is a nonprofit organization whose mission is to support the collaborative development of Odoo features and promote its widespread use.

    -

    This module is part of the OCA/queue project on GitHub.

    +

    This module is part of the OCA/queue project on GitHub.

    You are welcome to contribute. To learn how please visit https://odoo-community.org/page/Contribute.

    +
    diff --git a/base_import_async/tests/test_base_import_import.py b/base_import_async/tests/test_base_import_import.py index 4a815d6707..0234753c28 100644 --- a/base_import_async/tests/test_base_import_import.py +++ b/base_import_async/tests/test_base_import_import.py @@ -1,6 +1,12 @@ # Copyright 2024 Camptocamp # License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). +import base64 +import datetime +import uuid +from unittest import mock + +from odoo.addons.queue_job.tests.common import trap_jobs from odoo.tests.common import RecordCapturer, TransactionCase from ..models.base_import_import import OPT_USE_QUEUE @@ -11,89 +17,160 @@ class TestBaseImportImport(TransactionCase): def setUpClass(cls): super().setUpClass() cls.res_partners = cls.env["res.partner"] - cls.import_wizard = cls.env["base_import.import"] + cls.base_import = cls.env["base_import.import"] + cls.queue_job = cls.env["queue.job"] + + def _create_import_wizard(self, rows, file_name="partners.csv"): + csv_content = "\n".join(";".join(row) for row in rows) + return self.base_import.create( + { + "res_model": self.res_partners._name, + "file": csv_content.encode(), + "file_name": file_name, + "file_type": "text/csv", + } + ) + + def _get_import_preview(self, import_wizard, options): + preview = import_wizard.parse_preview(options) + self.assertIsNone(preview.get("error"), preview.get("error")) + return preview + + def _get_import_fields(self, import_wizard, options): + preview = self._get_import_preview(import_wizard, options) + return ["/".join(field_names) for field_names in preview["matches"].values()], ( + preview["headers"] or [] + ) def test_normal_import_res_partners(self): - values = [ - [ - "name", - "email", - "is_company", - ], + import_wizard = self._create_import_wizard( [ - "partner 1", - "partner1@example.com", - "1", - ], - [ - "partner 2", - "partner2@example.com", - "0", - ], - ] - import_vals = { - "res_model": self.res_partners._name, - "file": "\n".join([";".join(values) for values in values]), - "file_type": "text/csv", - } - self.import_wizard |= self.import_wizard.create(import_vals) - opts = {"quoting": '"', "separator": ";", "has_headers": True} - preview = self.import_wizard.parse_preview(opts) - self.assertEqual( - preview["matches"], - { - 0: ["name"], - 1: ["email"], - 2: ["is_company"], - }, + ["name", "email", "is_company"], + ["partner 1", "partner1@example.com", "1"], + ["partner 2", "partner2@example.com", "0"], + ] ) + options = {"quoting": '"', "separator": ";", "has_headers": True} + import_fields, columns = self._get_import_fields(import_wizard, options) + with RecordCapturer(self.res_partners, []) as capture: - results = self.import_wizard.execute_import( - [fnames[0] for fnames in preview["matches"].values()], - [], - opts, - ) - # if result is empty, no import error - self.assertItemsEqual(results["messages"], []) - records_created = capture.records - self.assertEqual(len(records_created), 2) - self.assertIn("partner1", records_created[0].email) - - def test_wrong_import_res_partners(self): - values = [ - [ - "name", - "email", - "date", # Adding date field to trigger parsing error - ], + result = import_wizard.execute_import(import_fields, columns, options) + + self.assertCountEqual(result["messages"], []) + self.assertEqual(len(capture.records), 2) + self.assertCountEqual(capture.records.mapped("email"), [ + "partner1@example.com", + "partner2@example.com", + ]) + + def test_async_import_schedules_and_imports_records(self): + import_wizard = self._create_import_wizard( [ - "partner 1", - "partner1@example.com", - "21-13-2024", - ], + ["name", "email", "is_company"], + ["async partner 1", "async1@example.com", "1"], + ["async partner 2", "async2@example.com", "0"], + ] + ) + options = { + "quoting": '"', + "separator": ";", + "has_headers": True, + OPT_USE_QUEUE: True, + } + import_fields, columns = self._get_import_fields(import_wizard, options) + + with trap_jobs() as trap: + result = import_wizard.execute_import(import_fields, columns, options) + self.assertEqual(result, []) + trap.assert_jobs_count(1, only=import_wizard._split_file) + + with RecordCapturer(self.res_partners, []) as capture: + trap.perform_enqueued_jobs() + trap.assert_jobs_count(1, only=import_wizard._import_one_chunk) + trap.perform_enqueued_jobs() + + self.assertEqual(len(capture.records), 2) + self.assertCountEqual(capture.records.mapped("name"), [ + "async partner 1", + "async partner 2", + ]) + + def test_async_import_uses_datetime_prevalidation(self): + import_wizard = self.base_import.create({"res_model": self.res_partners._name}) + + with trap_jobs() as trap: + with mock.patch.object( + type(import_wizard), + "_convert_import_data", + return_value=([ + [datetime.date(2026, 4, 17)], + ], ["name"]), + ): + result = import_wizard.execute_import( + ["name"], + ["name"], + {OPT_USE_QUEUE: True}, + ) + + self.assertEqual(trap.jobs_count(), 0) + self.assertEqual(len(result["messages"]), 1) + self.assertIn("does not accept date/time values", result["messages"][0]["message"]) + + def test_async_import_applies_fallback_values(self): + import_wizard = self._create_import_wizard( [ - "partner 2", - "partner2@example.com", - "2024-13-45", + ["name", "company_type"], + ["fallback partner", "Unknown value"], ], - ] - opts = { + file_name="fallback.csv", + ) + options = { "quoting": '"', "separator": ";", "has_headers": True, - "date_format": "%Y-%m-%d", # Set specific date format OPT_USE_QUEUE: True, + "fallback_values": { + "company_type": { + "fallback_value": "person", + "field_model": "res.partner", + "field_type": "selection", + } + }, } - import_vals = { - "res_model": self.res_partners._name, - "file": "\n".join([";".join(row) for row in values]), - "file_type": "text/csv", - } - import_wizard = self.import_wizard.create(import_vals) - preview = import_wizard.parse_preview(opts) - results = import_wizard.execute_import( - [field[0] for field in preview["matches"].values()], - ["name", "email", "date"], # Include date in fields to import - opts, + import_fields, columns = self._get_import_fields(import_wizard, options) + + with trap_jobs() as trap: + result = import_wizard.execute_import(import_fields, columns, options) + self.assertEqual(result, []) + trap.perform_enqueued_jobs() + trap.perform_enqueued_jobs() + + partner = self.res_partners.search([("name", "=", "fallback partner")], limit=1) + self.assertTrue(partner) + self.assertEqual(partner.company_type, "person") + + def test_related_action_attachment_returns_linked_attachment(self): + queue_job = self.queue_job.with_context( + _job_edit_sentinel=self.queue_job.EDIT_SENTINEL + ).create( + { + "uuid": str(uuid.uuid4()), + "state": "done", + "user_id": self.env.user.id, + "company_id": self.env.company.id, + "kwargs": {}, + } + ) + attachment = self.env["ir.attachment"].create( + { + "name": "chunk.csv", + "datas": base64.b64encode(b"name\nlinked attachment"), + "res_model": "queue.job", + "res_id": queue_job.id, + } ) - self.assertTrue(any(msg["type"] == "error" for msg in results["messages"])) + + action = queue_job._related_action_attachment() + + self.assertEqual(action["res_model"], "ir.attachment") + self.assertEqual(action["res_id"], attachment.id)