From a6e92a71bfc431142654fd5c81449484beb2b12b Mon Sep 17 00:00:00 2001 From: "Laurent Mignon (ACSONE)" Date: Mon, 10 Aug 2026 14:33:42 +0200 Subject: [PATCH 01/15] [REF] odoo_project: Move the module import logic to 'odoo.project' The logic mapping a module name to a project module was only reachable through the import wizard, while it is the natural entry point for any other way of feeding a project with its modules. --- odoo_project/models/odoo_project.py | 53 ++++++++++++ odoo_project/tests/test_import_modules.py | 9 ++- .../wizards/odoo_project_import_modules.py | 81 +++---------------- 3 files changed, 70 insertions(+), 73 deletions(-) diff --git a/odoo_project/models/odoo_project.py b/odoo_project/models/odoo_project.py index 6f613e6e..63a89ceb 100644 --- a/odoo_project/models/odoo_project.py +++ b/odoo_project/models/odoo_project.py @@ -1,4 +1,5 @@ # Copyright 2023 Camptocamp SA +# Copyright 2026 ACSONE SA/NV () # License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl) import ast @@ -166,6 +167,58 @@ def action_find_unknown_modules(self): for module in self.unknown_module_ids: module.action_find_pr_url() + def _get_module_branch(self, module): + """Return the `odoo.module.branch` matching `module` for this project. + + If it doesn't exist it'll be automatically created as an orphaned + module. + """ + self.ensure_one() + module_branch = self.env["odoo.module.branch"]._find_or_create( + self.odoo_version_id, module, self.repository_id + ) + if not module_branch.repository_branch_id and not module_branch.specific: + # If the module hasn't been found in existing repositories content, + # it could be available somewhere on GitHub as a PR that could help + # to identity its repository + module_branch.with_delay().action_find_pr_url() + return module_branch + + def _get_project_module(self, module_branch, version=False): + """Return the `odoo.project.module` of this project for `module_branch`. + + If it doesn't exist it'll be automatically created, otherwise its + installed version is updated. + """ + self.ensure_one() + project_module_model = self.env["odoo.project.module"] + domain = [ + ("module_branch_id", "=", module_branch.id), + ("odoo_project_id", "=", self.id), + ] + project_module = project_module_model.search(domain) + values = { + "module_branch_id": module_branch.id, + "odoo_project_id": self.id, + "installed_version": version, + } + if project_module: + project_module.sudo().write(values) + else: + # Create the module to make it available for the project + project_module = project_module_model.sudo().create(values) + return project_module + + def _import_missing_dependencies(self, project_modules): + """Complete the project with the dependencies of `project_modules`.""" + self.ensure_one() + branch_modules = project_modules.module_branch_id + all_dependencies = branch_modules._get_recursive_dependencies() + missing_dependencies = all_dependencies - branch_modules + for missing_dependency in missing_dependencies: + self._get_project_module(missing_dependency, missing_dependency.version) + return True + def _get_repositories_to_scan(self): """Return the repositories to scan.""" domain = self.env["odoo.repository"]._cron_scanner_domain() diff --git a/odoo_project/tests/test_import_modules.py b/odoo_project/tests/test_import_modules.py index 2f3167bf..3b2387a7 100644 --- a/odoo_project/tests/test_import_modules.py +++ b/odoo_project/tests/test_import_modules.py @@ -1,4 +1,5 @@ # Copyright 2024 Camptocamp SA +# Copyright 2026 ACSONE SA/NV () # License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl) from .common import ProjectCommon @@ -62,7 +63,7 @@ def test_import_modules_names_versions(self): def test_match_blacklisted_module(self): mod1 = "test1" mod2 = "test2" - mod1_blacklisted = self.wiz_import_modules_model._get_module(mod1) + mod1_blacklisted = self.module_branch_model._get_module(mod1) mod1_blacklisted.blacklisted = True # Import them through the wizard modules_list_text = f"{mod1}\n{mod2}" @@ -82,7 +83,7 @@ def test_match_blacklisted_module(self): def test_match_orphaned_module(self): mod1 = "test1" mod2 = "test2" - mod1_orphaned = self.wiz_import_modules_model._get_module(mod1) + mod1_orphaned = self.module_branch_model._get_module(mod1) mod1_branch_orphaned = self.module_branch_model._create_orphaned_module_branch( self.branch, mod1_orphaned ) @@ -106,7 +107,7 @@ def test_match_orphaned_module(self): def test_match_generic_module(self): mod1 = "test1" mod2 = "test2" - mod1_generic = self.wiz_import_modules_model._get_module(mod1) + mod1_generic = self.module_branch_model._get_module(mod1) repo_branch = self._create_odoo_repository_branch( self.odoo_repository, self.branch ) @@ -138,7 +139,7 @@ def test_match_project_repo_module(self): self.project.odoo_version_id = self.branch mod1 = "test1" mod2 = "test2" - mod1_in_repo = self.wiz_import_modules_model._get_module(mod1) + mod1_in_repo = self.module_branch_model._get_module(mod1) repo_branch = self._create_odoo_repository_branch( self.odoo_repository, self.branch ) diff --git a/odoo_project/wizards/odoo_project_import_modules.py b/odoo_project/wizards/odoo_project_import_modules.py index 42858cc7..e9f48b51 100644 --- a/odoo_project/wizards/odoo_project_import_modules.py +++ b/odoo_project/wizards/odoo_project_import_modules.py @@ -1,4 +1,5 @@ # Copyright 2023 Camptocamp SA +# Copyright 2026 ACSONE SA/NV () # License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl) import re @@ -49,11 +50,13 @@ def action_import(self): project_module_ids = self._action_import_modules_list() project_module_ids.extend(self._action_import_additional_modules()) if self.import_missing_dependencies: - self._action_import_missing_dependencies(project_module_ids) + project_modules = self.env["odoo.project.module"].browse(project_module_ids) + self.odoo_project_id._import_missing_dependencies(project_modules) def _action_import_modules_list(self): """Import a fresh list of installed modules into the project.""" - self.odoo_project_id.sudo().project_module_ids = False + project = self.odoo_project_id + project.sudo().project_module_ids = False module_lines = list(filter(None, self.modules_list.split("\n"))) project_module_ids = [] for line in module_lines: @@ -66,81 +69,21 @@ def _action_import_modules_list(self): else: module_name, version = data[0], False # for module_name in module_names: - module = self._get_module(module_name) + module = self.env["odoo.module.branch"]._get_module(module_name) if module.blacklisted: continue - module_branch = self._get_module_branch(module) - project_module = self._get_project_module(module_branch, version) + module_branch = project._get_module_branch(module) + project_module = project._get_project_module(module_branch, version) project_module_ids.append(project_module.id) - self.odoo_project_id.sudo().project_module_ids = project_module_ids + project.sudo().project_module_ids = project_module_ids return project_module_ids def _action_import_additional_modules(self): """Import additional modules into the project.""" project_module_ids = [] for module_branch in self.additional_module_ids: - project_module = self._get_project_module(module_branch, version=False) + project_module = self.odoo_project_id._get_project_module( + module_branch, version=False + ) project_module_ids.append(project_module.id) return project_module_ids - - def _action_import_missing_dependencies(self, project_module_ids): - """Complete list of modules by adding all dependencies.""" - project_modules = self.env["odoo.project.module"].browse(project_module_ids) - branch_modules = project_modules.module_branch_id - all_dependencies = branch_modules._get_recursive_dependencies() - missing_dependencies = all_dependencies - branch_modules - for missing_dependency in missing_dependencies: - self._get_project_module(missing_dependency, missing_dependency.version) - return True - - def _get_module(self, module_name): - """Return a `odoo.module` record. - - If it doesn't exist it'll be automatically created. - """ - module_model = self.env["odoo.module"] - module = module_model.search([("name", "=", module_name)]) - if not module: - module = module_model.sudo().create({"name": module_name}) - return module - - def _get_module_branch(self, module): - """Return a `odoo.module.branch` record. - - If it doesn't exist it'll be automatically created. - """ - module_branch_model = self.env["odoo.module.branch"] - module_branch = False - branch = self.odoo_project_id.odoo_version_id - module_branch = module_branch_model._find_or_create( - branch, module, self.odoo_project_id.repository_id - ) - if not module_branch.repository_branch_id and not module_branch.specific: - # If the module hasn't been found in existing repositories content, - # it could be available somewhere on GitHub as a PR that could help - # to identity its repository - module_branch.with_delay().action_find_pr_url() - return module_branch - - def _get_project_module(self, module_branch, version): - """Return a `odoo.project.module` record for the project. - - If it doesn't exist it'll be automatically created. - """ - project_module_model = self.env["odoo.project.module"] - domain = [ - ("module_branch_id", "=", module_branch.id), - ("odoo_project_id", "=", self.odoo_project_id.id), - ] - project_module = project_module_model.search(domain) - values = { - "module_branch_id": module_branch.id, - "odoo_project_id": self.odoo_project_id.id, - "installed_version": version, - } - if project_module: - project_module.sudo().write(values) - else: - # Create the module to make it available for the project - project_module = project_module_model.sudo().create(values) - return project_module From f2ab1227524e8d02e6e5ccae2c2ef00b4c248a1e Mon Sep 17 00:00:00 2001 From: "Laurent Mignon (ACSONE)" Date: Mon, 10 Aug 2026 14:44:25 +0200 Subject: [PATCH 02/15] [IMP] odoo_project: Look up a project module in a given repository '_get_module_branch()' always searched the repository of the project, which is the right default but not the only case: a module pinned on a fork belongs to the repository the fork originates from, and that repository is only known by the caller. Accept an optional 'repository' argument, keeping the project repository as the default. --- odoo_project/models/odoo_project.py | 11 +++++---- odoo_project/tests/test_import_modules.py | 29 +++++++++++++++++++++++ 2 files changed, 36 insertions(+), 4 deletions(-) diff --git a/odoo_project/models/odoo_project.py b/odoo_project/models/odoo_project.py index 63a89ceb..7cdf8a2d 100644 --- a/odoo_project/models/odoo_project.py +++ b/odoo_project/models/odoo_project.py @@ -167,15 +167,18 @@ def action_find_unknown_modules(self): for module in self.unknown_module_ids: module.action_find_pr_url() - def _get_module_branch(self, module): + def _get_module_branch(self, module, repository=None): """Return the `odoo.module.branch` matching `module` for this project. - If it doesn't exist it'll be automatically created as an orphaned - module. + The module is looked up in `repository` first, defaulting to the + repository of the project. If it doesn't exist it'll be automatically + created as an orphaned module. """ self.ensure_one() + if repository is None: + repository = self.repository_id module_branch = self.env["odoo.module.branch"]._find_or_create( - self.odoo_version_id, module, self.repository_id + self.odoo_version_id, module, repository ) if not module_branch.repository_branch_id and not module_branch.specific: # If the module hasn't been found in existing repositories content, diff --git a/odoo_project/tests/test_import_modules.py b/odoo_project/tests/test_import_modules.py index 3b2387a7..7a19baf2 100644 --- a/odoo_project/tests/test_import_modules.py +++ b/odoo_project/tests/test_import_modules.py @@ -164,3 +164,32 @@ def test_match_project_repo_module(self): self.assertIn(mod1_branch_in_repo, existing_mods) # Project modules are also created self.assertEqual(len(existing_mods.odoo_project_module_ids), 2) + + def test_get_module_branch_in_given_repository(self): + """A module can be looked up in a repository other than the project one.""" + module = self.module_branch_model._get_module("test1") + # The very same module lives in two scanned repositories + other_org = self.env["odoo.repository.org"].create({"name": "other-org"}) + other_repository = self.env["odoo.repository"].create( + { + "org_id": other_org.id, + "name": self.odoo_repository.name, + "repo_url": "https://github.com/other-org/repo", + "repo_type": "github", + } + ) + module_branches = {} + for repository in (self.odoo_repository, other_repository): + repo_branch = self._create_odoo_repository_branch(repository, self.branch) + module_branches[repository] = self._create_odoo_module_branch( + module, + self.branch, + specific=False, + repository_branch_id=repo_branch.id, + ) + for repository, module_branch in module_branches.items(): + with self.subTest(repository=repository.display_name): + self.assertEqual( + self.project._get_module_branch(module, repository=repository), + module_branch, + ) From 8bb8c292148e5fafa8947993451d3b7f6612d396 Mon Sep 17 00:00:00 2001 From: "Laurent Mignon (ACSONE)" Date: Mon, 10 Aug 2026 16:08:28 +0200 Subject: [PATCH 03/15] [FIX] odoo_project: Find the repository branch whenever the repository gets one '_compute_repository_branch_id' reads 'repository_id.branch_ids' but only depends on 'repository_id', so it never runs again once the repository is set. A project is commonly created before its repository has been scanned, hence before the matching branch exists: 'repository_branch_id' then stays empty for good, and 'module_not_installed_ids', computed from it, silently reports nothing. Depend on the branches of the repository so that the branch is picked up as soon as a scan creates it. --- odoo_project/models/odoo_project.py | 2 +- odoo_project/tests/test_import_modules.py | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/odoo_project/models/odoo_project.py b/odoo_project/models/odoo_project.py index 7cdf8a2d..d5f12f3c 100644 --- a/odoo_project/models/odoo_project.py +++ b/odoo_project/models/odoo_project.py @@ -92,7 +92,7 @@ def _compute_available_odoo_version_ids(self): if rec.repository_id: rec.available_odoo_version_ids = rec.repository_id.branch_ids.branch_id - @api.depends("repository_id", "odoo_version_id") + @api.depends("repository_id.branch_ids.branch_id", "odoo_version_id") def _compute_repository_branch_id(self): for rec in self: rec.repository_branch_id = False diff --git a/odoo_project/tests/test_import_modules.py b/odoo_project/tests/test_import_modules.py index 7a19baf2..858d0eeb 100644 --- a/odoo_project/tests/test_import_modules.py +++ b/odoo_project/tests/test_import_modules.py @@ -193,3 +193,21 @@ def test_get_module_branch_in_given_repository(self): self.project._get_module_branch(module, repository=repository), module_branch, ) + + def test_repository_branch_id_follows_the_repository_branches(self): + """The branch of a project is found whenever its repository gets one. + + A project is commonly created before its repository has been scanned, + so the matching branch does not exist yet at that point. + """ + self.project.write( + { + "repository_id": self.odoo_repository.id, + "odoo_version_id": self.branch.id, + } + ) + self.assertFalse(self.project.repository_branch_id) + repository_branch = self._create_odoo_repository_branch( + self.odoo_repository, self.branch + ) + self.assertEqual(self.project.repository_branch_id, repository_branch) From b254656f9c9c2750498ddaa489b4fe7b55e5c41a Mon Sep 17 00:00:00 2001 From: "Laurent Mignon (ACSONE)" Date: Mon, 10 Aug 2026 10:50:50 +0200 Subject: [PATCH 04/15] [FIX] odoo_repository: Avoid cache corruption @tools.ormcache is a process-wide cache that is never invalidated by a SQL ROLLBACK. If a newly created record's id were cached from within the caller's (still uncommitted) transaction, and that transaction was later rolled back for an unrelated reason (e.g. a conflicting concurrent queue job), the cache would keep returning an id that no longer exists in the database. This commit add a new method creating and committing the record, independently of the caller's transaction. That guarantees the returned id always stays valid, whatever happens to the caller's job afterwards. --- odoo_repository/models/__init__.py | 1 + odoo_repository/models/odoo_module_branch.py | 67 ++++++++------- odoo_repository/models/odoo_ref_data_mixin.py | 84 +++++++++++++++++++ odoo_repository/models/odoo_repository.py | 62 +++++++------- odoo_repository/readme/CONTRIBUTORS.md | 2 + 5 files changed, 156 insertions(+), 60 deletions(-) create mode 100644 odoo_repository/models/odoo_ref_data_mixin.py diff --git a/odoo_repository/models/__init__.py b/odoo_repository/models/__init__.py index 6c8e744d..b428a870 100644 --- a/odoo_repository/models/__init__.py +++ b/odoo_repository/models/__init__.py @@ -1,5 +1,6 @@ from . import authentication_token from . import ssh_key +from . import odoo_ref_data_mixin from . import odoo_author from . import odoo_branch from . import odoo_license diff --git a/odoo_repository/models/odoo_module_branch.py b/odoo_repository/models/odoo_module_branch.py index 2229d0f4..3d68de71 100644 --- a/odoo_repository/models/odoo_module_branch.py +++ b/odoo_repository/models/odoo_module_branch.py @@ -18,6 +18,7 @@ class OdooModuleBranch(models.Model): _name = "odoo.module.branch" + _inherit = "odoo.ref.data.mixin" _description = "Odoo Module Branch" _order = "repository_sequence, module_name, branch_name" @@ -644,13 +645,13 @@ def _get_module_category_id(self, category_name): rec = self.env["odoo.module.category"].search( [("name", "=", category_name)], limit=1 ) - if not rec: - rec = ( - self.env["odoo.module.category"] - .sudo() - .create({"name": category_name}) - ) - return rec.id + if rec: + return rec.id + return self._create_ref_data( + "odoo.module.category", + [("name", "=", category_name)], + {"name": category_name}, + ) return False @tools.ormcache("names") @@ -661,14 +662,14 @@ def _get_author_ids(self, names): names = [name.strip() for name in names.split(",")] authors = self.env["odoo.author"].search([("name", "in", names)]) missing_author_names = set(names) - set(authors.mapped("name")) - missing_authors = self.env["odoo.author"] + created_ids = [] if missing_author_names: - missing_authors = ( - self.env["odoo.author"] - .sudo() - .create([{"name": name} for name in missing_author_names]) + created_ids = self._create_ref_data_multi( + "odoo.author", + "name", + [{"name": name} for name in missing_author_names], ) - return (authors | missing_authors).ids + return authors.ids + created_ids return [] @tools.ormcache("names") @@ -676,12 +677,14 @@ def _get_maintainer_ids(self, names): if names: maintainers = self.env["odoo.maintainer"].search([("name", "in", names)]) missing_maintainer_names = set(names) - set(maintainers.mapped("name")) - created = self.env["odoo.maintainer"] + created_ids = [] if missing_maintainer_names: - created = created.sudo().create( - [{"name": name} for name in missing_maintainer_names] + created_ids = self._create_ref_data_multi( + "odoo.maintainer", + "name", + [{"name": name} for name in missing_maintainer_names], ) - return (maintainers | created).ids + return maintainers.ids + created_ids return [] @tools.ormcache("name") @@ -690,9 +693,11 @@ def _get_dev_status_id(self, name): rec = self.env["odoo.module.dev.status"].search( [("name", "=", name)], limit=1 ) - if not rec: - rec = self.env["odoo.module.dev.status"].sudo().create({"name": name}) - return rec.id + if rec: + return rec.id + return self._create_ref_data( + "odoo.module.dev.status", [("name", "=", name)], {"name": name} + ) return False @api.model @@ -759,13 +764,15 @@ def _get_python_dependency_ids(self, packages): dependencies = self.env["odoo.python.dependency"].search( [("name", "in", packages)] ) - missing_dependencies = set(packages) - set(dependencies.mapped("name")) - created = self.env["odoo.python.dependency"] - if missing_dependencies: - created = created.sudo().create( - [{"name": package} for package in missing_dependencies] + missing_names = set(packages) - set(dependencies.mapped("name")) + created_ids = [] + if missing_names: + created_ids = self._create_ref_data_multi( + "odoo.python.dependency", + "name", + [{"name": name} for name in missing_names], ) - return (dependencies | created).ids + return dependencies.ids + created_ids return [] @tools.ormcache("license_name") @@ -773,9 +780,11 @@ def _get_license_id(self, license_name): if license_name: license_model = self.env["odoo.license"] rec = license_model.search([("name", "=", license_name)], limit=1) - if not rec: - rec = license_model.sudo().create({"name": license_name}) - return rec.id + if rec: + return rec.id + return self._create_ref_data( + "odoo.license", [("name", "=", license_name)], {"name": license_name} + ) return False def _get_module(self, name): diff --git a/odoo_repository/models/odoo_ref_data_mixin.py b/odoo_repository/models/odoo_ref_data_mixin.py new file mode 100644 index 00000000..9b4262e2 --- /dev/null +++ b/odoo_repository/models/odoo_ref_data_mixin.py @@ -0,0 +1,84 @@ +# Copyright 2026 ACSONE SA/NV () +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl) + +import threading +from contextlib import contextmanager + +import psycopg2.errors + +from odoo import api, models + + +class OdooRefDataMixin(models.AbstractModel): + """Shared helpers to safely create reference data cached by ``@tools.ormcache``.""" + + _name = "odoo.ref.data.mixin" + _description = "Odoo Ref Data Mixin" + + @contextmanager + def _ref_data_cursor(self): + """Yield the cursor to use for an isolated reference-data creation. + + This method is a context manager that yields a cursor to use for creating + reference data in a separate transaction. If the current environment is in + test mode, it yields the current cursor instead, since in test mode we + don't want to create a new transaction. + """ + in_test_mode = self.env.registry.in_test_mode() or getattr( + threading.current_thread(), "testing", False + ) + if in_test_mode: + yield self.env.cr + else: + with self.pool.cursor() as new_cr: + yield new_cr + + def _create_ref_data(self, model_name, domain, values): + """Create a single get-or-create reference record in its own + dedicated, immediately-committed transaction, and return its id. + + ``@tools.ormcache`` is a process-wide cache that is never invalidated + by a SQL ROLLBACK. If a newly created record's id were cached from + within the caller's (still uncommitted) transaction, and that + transaction was later rolled back for an unrelated reason (e.g. a + conflicting concurrent queue job), the cache would keep returning an + id that no longer exists in the database. Creating and committing the + record here, independently of the caller's transaction, guarantees + the returned id always stays valid, whatever happens to the caller's + job afterwards. + + A concurrent job may create the same record at the same time: the + unique SQL constraint on the target model then raises + ``UniqueViolation`` for the loser, which simply looks up the row the + winner just committed. + """ + with self._ref_data_cursor() as new_cr: + env = api.Environment(new_cr, self.env.uid, self.env.context) + model = env[model_name].sudo() + try: + with new_cr.savepoint(): + return model.create(values).id + except psycopg2.errors.UniqueViolation: + return model.search(domain, limit=1).id + + def _create_ref_data_multi(self, model_name, search_field, values_list): + """Same as ``_create_ref_data``, but for a batch of records sharing + the same natural-key field (e.g. several new authors discovered at + once). Each record is created individually, in its own savepoint, so + that a conflict on one of them does not abort the creation of the + others. + """ + ids = [] + with self._ref_data_cursor() as new_cr: + env = api.Environment(new_cr, self.env.uid, self.env.context) + model = env[model_name].sudo() + for values in values_list: + try: + with new_cr.savepoint(): + ids.append(model.create(values).id) + except psycopg2.errors.UniqueViolation: + existing = model.search( + [(search_field, "=", values[search_field])], limit=1 + ) + ids.append(existing.id) + return ids diff --git a/odoo_repository/models/odoo_repository.py b/odoo_repository/models/odoo_repository.py index 71bbeba1..2118db3e 100644 --- a/odoo_repository/models/odoo_repository.py +++ b/odoo_repository/models/odoo_repository.py @@ -25,6 +25,7 @@ class OdooRepository(models.Model): _name = "odoo.repository" + _inherit = "odoo.ref.data.mixin" _description = "Odoo Modules Repository" _order = "sequence, display_name" @@ -501,12 +502,18 @@ def _import_data(self, data): def _prepare_module_branch_values(self, data): # Get branch, repository and technical module branch = self.env["odoo.branch"].search([("name", "=", data["branch"])]) - org = self._get_repository_org(data["repository"]["org"]) - repository = self._get_repository( - org.id, data["repository"]["name"], data["repository"] + org_id = self._get_repository_org_id(data["repository"]["org"]) + repository_id = self._get_repository_id( + org_id, data["repository"]["name"], data["repository"] ) - repository_branch = self._get_repository_branch( - org.id, repository.id, branch.id, data["repository"] + repository_branch_id = self._get_repository_branch_id( + org_id, repository_id, branch.id, data["repository"] + ) + # `_get_repository_branch_id` returns a plain id (see `_create_ref_data` + # docstring), so browse it here to get a recordset bound to this + # transaction's cursor. + repository_branch = self.env["odoo.repository.branch"].browse( + repository_branch_id ) mb_model = self.env["odoo.module.branch"] @@ -605,21 +612,18 @@ def _post_create_or_update_module_branch(self, rec, values, raw_data): """Hook executed after the creation or update of `rec`.""" @tools.ormcache("name") - def _get_repository_org(self, name): + def _get_repository_org_id(self, name): rec = self.env["odoo.repository.org"].search([("name", "=", name)], limit=1) - if not rec: - rec = self.env["odoo.repository.org"].sudo().create({"name": name}) - return rec + if rec: + return rec.id + return self._create_ref_data( + "odoo.repository.org", [("name", "=", name)], {"name": name} + ) @tools.ormcache("org_id", "name") - def _get_repository(self, org_id, name, data): - rec = self.env["odoo.repository"].search( - [ - ("org_id", "=", org_id), - ("name", "=", name), - ], - limit=1, - ) + def _get_repository_id(self, org_id, name, data): + domain = [("org_id", "=", org_id), ("name", "=", name)] + rec = self.env["odoo.repository"].search(domain, limit=1) values = { "org_id": org_id, "name": name, @@ -629,19 +633,16 @@ def _get_repository(self, org_id, name, data): } if rec: rec.sudo().write(values) - else: - rec = self.env["odoo.repository"].sudo().create(values) - return rec + return rec.id + return self._create_ref_data("odoo.repository", domain, values) @tools.ormcache("org_id", "repository_id", "branch_id") - def _get_repository_branch(self, org_id, repository_id, branch_id, data): - rec = self.env["odoo.repository.branch"].search( - [ - ("repository_id", "=", repository_id), - ("branch_id", "=", branch_id), - ], - limit=1, - ) + def _get_repository_branch_id(self, org_id, repository_id, branch_id, data): + domain = [ + ("repository_id", "=", repository_id), + ("branch_id", "=", branch_id), + ] + rec = self.env["odoo.repository.branch"].search(domain, limit=1) values = { "repository_id": repository_id, "branch_id": branch_id, @@ -649,9 +650,8 @@ def _get_repository_branch(self, org_id, repository_id, branch_id, data): } if rec: rec.sudo().write(values) - else: - rec = self.env["odoo.repository.branch"].sudo().create(values) - return rec + return rec.id + return self._create_ref_data("odoo.repository.branch", domain, values) def _get_resource_url(self, branch, path): self.ensure_one() diff --git a/odoo_repository/readme/CONTRIBUTORS.md b/odoo_repository/readme/CONTRIBUTORS.md index 17752927..e6aa7475 100644 --- a/odoo_repository/readme/CONTRIBUTORS.md +++ b/odoo_repository/readme/CONTRIBUTORS.md @@ -1,2 +1,4 @@ - Camptocamp - Sébastien Alix \ +- ACSONE SA/NV + - Laurent Mignon \ From 0824723f275e5d2f500a6d912acb0871a8ad3b33 Mon Sep 17 00:00:00 2001 From: "Laurent Mignon (ACSONE)" Date: Thu, 13 Aug 2026 09:54:21 +0200 Subject: [PATCH 05/15] [FIX] odoo_repository: GitPython 3.1.59 compatibility GitPython 3.1.59, released on 2026-08-10, added '--separate-git-dir' to the list of unsafe options. Allow unsafe options on clone created with workaround_fs_errors=True since they are the only once using this option. --- odoo_repository/lib/scanner.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/odoo_repository/lib/scanner.py b/odoo_repository/lib/scanner.py index d06dfd67..742f88d0 100644 --- a/odoo_repository/lib/scanner.py +++ b/odoo_repository/lib/scanner.py @@ -247,6 +247,9 @@ def _clone(self): extra = {"env": git_env} if self.workaround_fs_errors: extra["separate_git_dir"] = str(tmp_git_dir_path) + # GitPython blocks '--separate-git-dir' since 3.1.59 + # We must allow unsafe options to be able to use it, + extra["allow_unsafe_options"] = True params = self._clone_params(**extra) try: git.Repo.clone_from(**params) From c976bc51a7ab6bc1e58a72400b650be18a4ac9ba Mon Sep 17 00:00:00 2001 From: "Laurent Mignon (ACSONE)" Date: Mon, 10 Aug 2026 13:16:03 +0200 Subject: [PATCH 06/15] [FIX] odoo_repository: Singleton error on _get_recursive_dependencies This commit fix '_get_recursive_dependencies' to properly work on a recordset --- odoo_repository/models/odoo_module_branch.py | 5 +++-- ..._odoo_module_branch_recursive_dependencies.py | 16 ++++++++++++++++ 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/odoo_repository/models/odoo_module_branch.py b/odoo_repository/models/odoo_module_branch.py index 3d68de71..8e4b6208 100644 --- a/odoo_repository/models/odoo_module_branch.py +++ b/odoo_repository/models/odoo_module_branch.py @@ -308,9 +308,10 @@ def _get_recursive_dependencies(self, domain=None, _visited=None): domain = [] if _visited is None: _visited = set() - if self.id in _visited: + current_ids = set(self.ids) + if current_ids.issubset(_visited): return self.browse() - _visited.add(self.id) + _visited |= current_ids # Apply domain and exclude self dependencies = (self.dependency_ids - self).filtered_domain(domain) dep_ids = set(dependencies.ids) diff --git a/odoo_repository/tests/test_odoo_module_branch_recursive_dependencies.py b/odoo_repository/tests/test_odoo_module_branch_recursive_dependencies.py index f6b30940..6915b317 100644 --- a/odoo_repository/tests/test_odoo_module_branch_recursive_dependencies.py +++ b/odoo_repository/tests/test_odoo_module_branch_recursive_dependencies.py @@ -188,6 +188,22 @@ def test_get_recursive_dependencies_empty(self): deps = mod_alone_branch._get_recursive_dependencies() self.assertEqual(len(deps), 0) + def test_get_recursive_dependencies_multi_record_self(self): + """Test _get_recursive_dependencies called on a non-singleton recordset.""" + # This mirrors odoo.project.import.modules._action_import_missing_dependencies, + # which calls the method on the module_branch_id of several project + # modules at once. The previous implementation relied on `self.id`, + # raising `ValueError: Expected singleton` for more than one record. + self.mod_a_branch.dependency_ids = self.mod_base_branch + self.mod_b_branch.dependency_ids = self.mod_c_branch + modules = self.mod_a_branch + self.mod_b_branch + deps = modules._get_recursive_dependencies() + self.assertIn(self.mod_base_branch, deps) + self.assertIn(self.mod_c_branch, deps) + self.assertNotIn(self.mod_a_branch, deps) + self.assertNotIn(self.mod_b_branch, deps) + self.assertEqual(len(deps), 2) # base, module_c + def test_get_recursive_dependencies_self_exclusion(self): """Test that _get_recursive_dependencies excludes self.""" # Create modules From ed5a4b9a9c0a1355ebdd41a93617408240aadfe0 Mon Sep 17 00:00:00 2001 From: "Laurent Mignon (ACSONE)" Date: Tue, 11 Aug 2026 13:36:29 +0200 Subject: [PATCH 07/15] [IMP] odoo_repository: Prioritize the organizations holding a module A module can be hosted by several organizations, a fork of an OCA repository being the usual case. '_find()' then picks one of them with 'fields.first()', which follows the sequence of the repositories: setting it one by one to say that OCA comes before a fork is tedious, and says nothing about the ones not listed yet. Give an organization a sequence of its own, in front of the module branch '_order' so that it decides first. The uniform default keeps the current ordering unchanged until a sequence is actually set. Add '_parse_clone_url()' and '_find_from_org_and_name()' along the way, which locating the repository a clone URL points at needs. --- odoo_repository/models/odoo_module_branch.py | 9 +- odoo_repository/models/odoo_repository.py | 39 +++++++- odoo_repository/models/odoo_repository_org.py | 10 ++ odoo_repository/tests/__init__.py | 1 + .../tests/test_odoo_repository_lookup.py | 92 +++++++++++++++++++ odoo_repository/views/odoo_repository_org.xml | 3 + 6 files changed, 152 insertions(+), 2 deletions(-) create mode 100644 odoo_repository/tests/test_odoo_repository_lookup.py diff --git a/odoo_repository/models/odoo_module_branch.py b/odoo_repository/models/odoo_module_branch.py index 8e4b6208..fba45edd 100644 --- a/odoo_repository/models/odoo_module_branch.py +++ b/odoo_repository/models/odoo_module_branch.py @@ -1,4 +1,5 @@ # Copyright 2023 Camptocamp SA +# Copyright 2026 ACSONE SA/NV () # License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl) import pathlib @@ -20,7 +21,7 @@ class OdooModuleBranch(models.Model): _name = "odoo.module.branch" _inherit = "odoo.ref.data.mixin" _description = "Odoo Module Branch" - _order = "repository_sequence, module_name, branch_name" + _order = "org_sequence, repository_sequence, module_name, branch_name" module_id = fields.Many2one( comodel_name="odoo.module", @@ -55,6 +56,12 @@ class OdooModuleBranch(models.Model): store=True, string="Organization", ) + org_sequence = fields.Integer( + related="repository_branch_id.repository_id.org_id.sequence", + store=True, + index=True, + string="Organization Sequence", + ) branch_id = fields.Many2one( # NOTE: not a related on 'repository_branch_id' as we need to create # modules without knowing in advance what is their repo (orphaned modules). diff --git a/odoo_repository/models/odoo_repository.py b/odoo_repository/models/odoo_repository.py index 2118db3e..884c07bc 100644 --- a/odoo_repository/models/odoo_repository.py +++ b/odoo_repository/models/odoo_repository.py @@ -1,12 +1,13 @@ # Copyright 2023 Camptocamp SA # Copyright 2026 Sébastien Alix +# Copyright 2026 ACSONE SA/NV () # License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl) import json import logging import os import pathlib -from urllib.parse import urljoin +from urllib.parse import urljoin, urlparse import requests @@ -659,6 +660,42 @@ def _get_resource_url(self, branch, path): url = "/".join(["tree", branch, path]) return urljoin(self.repo_url + "/", url) + @staticmethod + def _parse_clone_url(clone_url): + """Return the ``(host, org, name)`` parts of a clone URL. + + Supports both HTTP(S) and SCP-like (``git@host:org/name.git``) syntax. + Returns ``(None, None, None)`` if the URL cannot be parsed. + """ + if not clone_url: + return (None, None, None) + url = clone_url.strip() + if "://" in url: + parts = urlparse(url) + host, path = parts.hostname, parts.path + elif ":" in url: + # SCP-like syntax: [user@]host:path + host, _, path = url.partition(":") + host = host.rpartition("@")[2] + else: + return (None, None, None) + path = path.strip("/") + if path.endswith(".git"): + path = path[: -len(".git")] + # NOTE: 'org' keeps every leading segment to support nested namespaces + # (GitLab sub-groups). + org, _, name = path.rpartition("/") + if not host or not org or not name: + return (None, None, None) + return (host, org, name) + + @api.model + def _find_from_org_and_name(self, org, name): + """Return the repository matching an organization and a name.""" + return self.with_context(active_test=False).search( + [("org_id.name", "=", org), ("name", "=", name)], limit=1 + ) + def unlink(self): # There is no deletion on cascade policy by default, but for specific # repositories we want to remove specific modules anyway. diff --git a/odoo_repository/models/odoo_repository_org.py b/odoo_repository/models/odoo_repository_org.py index 38605aa2..3842c2bc 100644 --- a/odoo_repository/models/odoo_repository_org.py +++ b/odoo_repository/models/odoo_repository_org.py @@ -1,4 +1,5 @@ # Copyright 2023 Camptocamp SA +# Copyright 2026 ACSONE SA/NV () # License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl) from odoo import api, fields, models @@ -9,8 +10,17 @@ class OdooRepositoryOrg(models.Model): _name = "odoo.repository.org" _description = "Odoo Repository Organization" + _order = "sequence, name" name = fields.Char(required=True, index=True) + sequence = fields.Integer( + default=100, + help=( + "Priority of this organization when looking for the origin of a " + "module shared by several organizations (forks). " + "The lowest sequence wins." + ), + ) github_url = fields.Char(string="GitHub URL", compute="_compute_github_url") @api.depends("name") diff --git a/odoo_repository/tests/__init__.py b/odoo_repository/tests/__init__.py index ac60db36..caa4f928 100644 --- a/odoo_repository/tests/__init__.py +++ b/odoo_repository/tests/__init__.py @@ -7,3 +7,4 @@ from . import test_odoo_branch from . import test_oca_repository_synchronizer from . import test_odoo_module_branch_recursive_dependencies +from . import test_odoo_repository_lookup diff --git a/odoo_repository/tests/test_odoo_repository_lookup.py b/odoo_repository/tests/test_odoo_repository_lookup.py new file mode 100644 index 00000000..9dc8354f --- /dev/null +++ b/odoo_repository/tests/test_odoo_repository_lookup.py @@ -0,0 +1,92 @@ +# Copyright 2026 ACSONE SA/NV () +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl) + +from .common import Common + + +class TestOdooRepositoryLookup(Common): + def setUp(self): + super().setUp() + self.repository_model = self.env["odoo.repository"] + + # -- _parse_clone_url -------------------------------------------------- + + def test_parse_clone_url_https(self): + self.assertEqual( + self.repository_model._parse_clone_url( + "https://github.com/acsone/account-invoicing.git" + ), + ("github.com", "acsone", "account-invoicing"), + ) + + def test_parse_clone_url_without_git_suffix(self): + self.assertEqual( + self.repository_model._parse_clone_url( + "https://github.com/OCA/account-invoicing" + ), + ("github.com", "OCA", "account-invoicing"), + ) + + def test_parse_clone_url_scp_syntax(self): + self.assertEqual( + self.repository_model._parse_clone_url( + "git@github.com:acsone/account-invoicing.git" + ), + ("github.com", "acsone", "account-invoicing"), + ) + + def test_parse_clone_url_nested_namespace(self): + """GitLab sub-groups are kept as part of the organization.""" + self.assertEqual( + self.repository_model._parse_clone_url( + "https://gitlab.com/acsone/odoo/addons.git" + ), + ("gitlab.com", "acsone/odoo", "addons"), + ) + + def test_parse_clone_url_invalid(self): + for url in (None, "", "not-an-url", "https://github.com/lonely-segment"): + with self.subTest(url=url): + self.assertEqual( + self.repository_model._parse_clone_url(url), (None, None, None) + ) + + # -- organization sequence --------------------------------------------- + + def _create_repository(self, org_name, repo_name, **values): + org = self.env["odoo.repository.org"].search([("name", "=", org_name)]) + if not org: + org = self.env["odoo.repository.org"].create({"name": org_name}) + vals = { + "org_id": org.id, + "name": repo_name, + "repo_url": f"https://github.com/{org_name}/{repo_name}", + "clone_url": f"https://github.com/{org_name}/{repo_name}.git", + "repo_type": "github", + } + vals.update(values) + return self.env["odoo.repository"].create(vals) + + def test_find_module_branch_honours_org_sequence(self): + """A module shared by several organizations resolves to the first one.""" + module = self._create_odoo_module("account_invoice_triple_discount") + module_branches = {} + for org_name, sequence in (("OCA", 10), ("acsone", 20)): + repository = self._create_repository(org_name, "account-invoicing") + repository.org_id.sequence = sequence + repository_branch = self._create_odoo_repository_branch( + repository, self.branch + ) + module_branches[org_name] = self._create_odoo_module_branch( + module, + self.branch, + repository_branch_id=repository_branch.id, + specific=False, + ) + module_branch_model = self.env["odoo.module.branch"] + found = module_branch_model._find(self.branch, module, repo=False) + self.assertEqual(found, module_branches["OCA"]) + # Reversing the priority of the organizations reverses the result + module_branches["acsone"].repository_id.org_id.sequence = 1 + found = module_branch_model._find(self.branch, module, repo=False) + self.assertEqual(found, module_branches["acsone"]) diff --git a/odoo_repository/views/odoo_repository_org.xml b/odoo_repository/views/odoo_repository_org.xml index fe540983..d6ac8514 100644 --- a/odoo_repository/views/odoo_repository_org.xml +++ b/odoo_repository/views/odoo_repository_org.xml @@ -1,5 +1,6 @@ @@ -10,6 +11,7 @@ + @@ -21,6 +23,7 @@ odoo.repository.org + From 39e428f783f3a0f27ff9889bef464c5cdf17080c Mon Sep 17 00:00:00 2001 From: "Laurent Mignon (ACSONE)" Date: Mon, 10 Aug 2026 14:36:38 +0200 Subject: [PATCH 08/15] [IMP] odoo_repository: Expose the local clone path Expose the local clone path of a repository so callers can access files at a specific revision without having to know how the scanner stores its clones. --- odoo_repository/models/odoo_repository.py | 18 ++++++++++++++++ .../tests/test_odoo_repository_lookup.py | 21 ++++++++++++++++++- 2 files changed, 38 insertions(+), 1 deletion(-) diff --git a/odoo_repository/models/odoo_repository.py b/odoo_repository/models/odoo_repository.py index 884c07bc..7a5beb9b 100644 --- a/odoo_repository/models/odoo_repository.py +++ b/odoo_repository/models/odoo_repository.py @@ -19,6 +19,7 @@ from odoo.addons.queue_job.exception import RetryableJobError from odoo.addons.queue_job.job import identity_exact +from ..lib.scanner import BaseScanner from ..utils.scanner import RepositoryScannerOdooEnv _logger = logging.getLogger(__name__) @@ -660,6 +661,23 @@ def _get_resource_url(self, branch, path): url = "/".join(["tree", branch, path]) return urljoin(self.repo_url + "/", url) + def _get_local_clone_path(self): + """Return the path of the local clone of this repository. + + The clone is the one maintained by the scanner, so it only exists + once the repository has been scanned at least once. + """ + self.ensure_one() + repositories_path = ( + self.env["ir.config_parameter"] + .sudo() + .get_param(self._repositories_path_key) + ) + # NOTE: delegate to the scanner to build the very same layout + return BaseScanner._prepare_repositories_path(repositories_path).joinpath( + self.org_id.name, self.clone_name or self.name + ) + @staticmethod def _parse_clone_url(clone_url): """Return the ``(host, org, name)`` parts of a clone URL. diff --git a/odoo_repository/tests/test_odoo_repository_lookup.py b/odoo_repository/tests/test_odoo_repository_lookup.py index 9dc8354f..b4612c3d 100644 --- a/odoo_repository/tests/test_odoo_repository_lookup.py +++ b/odoo_repository/tests/test_odoo_repository_lookup.py @@ -1,6 +1,7 @@ +import pathlib + # Copyright 2026 ACSONE SA/NV () # License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl) - from .common import Common @@ -90,3 +91,21 @@ def test_find_module_branch_honours_org_sequence(self): module_branches["acsone"].repository_id.org_id.sequence = 1 found = module_branch_model._find(self.branch, module, repo=False) self.assertEqual(found, module_branches["acsone"]) + + def test_get_local_clone_path(self): + """The clone path matches the layout built by the scanner.""" + repository = self._create_repository("OCA", "account-invoicing") + self.assertEqual( + repository._get_local_clone_path(), + pathlib.Path(self.repositories_path, "OCA", "account-invoicing"), + ) + + def test_get_local_clone_path_with_clone_name(self): + """A forced clone name overrides the repository name on disk.""" + repository = self._create_repository( + "OCA", "account-invoicing", clone_name="oca-account-invoicing" + ) + self.assertEqual( + repository._get_local_clone_path(), + pathlib.Path(self.repositories_path, "OCA", "oca-account-invoicing"), + ) From a26b4a1b1d454a39f9b0313ffe0865c7422d7457 Mon Sep 17 00:00:00 2001 From: "Laurent Mignon (ACSONE)" Date: Mon, 10 Aug 2026 17:12:08 +0200 Subject: [PATCH 09/15] [IMP] odoo_repository: Expose scan job creation Separate the creation of repository scan jobs from their execution. Previously, 'action_scan' created and immediately delayed the scan jobs, making them unavailable to callers that needed to build a larger job graph around the repository scan. Expose '_create_scan_jobs()' to let callers create and access the scan jobs without delaying them. They can then orchestrate these jobs together with other jobs or processes before starting the execution. 'action_scan' keeps the existing behaviour by creating and delaying the jobs returned by '_create_scan_jobs()'. --- odoo_repository/models/odoo_repository.py | 23 +++++++++++++++---- .../tests/test_odoo_repository_scan.py | 16 +++++++++++++ 2 files changed, 35 insertions(+), 4 deletions(-) diff --git a/odoo_repository/models/odoo_repository.py b/odoo_repository/models/odoo_repository.py index 7a5beb9b..b53fbc65 100644 --- a/odoo_repository/models/odoo_repository.py +++ b/odoo_repository/models/odoo_repository.py @@ -252,7 +252,21 @@ def _check_existing_jobs(self, raise_exc=True): def action_scan(self, branch_ids=None, force=False, raise_exc=True): """Scan the whole repository.""" + for job in self._create_scan_jobs( + branch_ids=branch_ids, force=force, raise_exc=raise_exc + ): + job.delay() + return True + + def _create_scan_jobs(self, branch_ids=None, force=False, raise_exc=True): + """Return the jobs scanning these repositories, without delaying them. + + Callers wanting something to happen once the scan is over can build + their own graph out of them, rather than delaying them right away as + `action_scan` does. + """ self._check_config() + jobs = [] for rec in self: if not rec.to_scan: continue @@ -286,11 +300,12 @@ def action_scan(self, branch_ids=None, force=False, raise_exc=True): # on the next branch version_branch = versions_branches[0] next_versions_branches = versions_branches[1:] - job = rec._create_job_detect_modules_to_scan_on_branch( - version_branch, next_versions_branches, versions_branches + jobs.append( + rec._create_job_detect_modules_to_scan_on_branch( + version_branch, next_versions_branches, versions_branches + ) ) - job.delay() - return True + return jobs def _create_job_detect_modules_to_scan_on_branch( self, version_branch, next_versions_branches, all_versions_branches diff --git a/odoo_repository/tests/test_odoo_repository_scan.py b/odoo_repository/tests/test_odoo_repository_scan.py index 12c06ba0..e350119f 100644 --- a/odoo_repository/tests/test_odoo_repository_scan.py +++ b/odoo_repository/tests/test_odoo_repository_scan.py @@ -8,6 +8,22 @@ class TestOdooRepositoryScan(Common): + def test_create_scan_jobs_does_not_delay_them(self): + """The jobs are returned so that callers can build a graph out of them. + + 'action_scan' delays them right away, but something meant to run once + the scan is over has to connect to them first. + """ + before = self.env["queue.job"].search_count([]) + jobs = self.odoo_repository._create_scan_jobs(branch_ids=self.branch.ids) + self.assertTrue(jobs) + self.assertEqual( + [job._job_method.__name__ for job in jobs], + ["_detect_modules_to_scan_on_branch"], + ) + self.env.flush_all() + self.assertEqual(self.env["queue.job"].search_count([]), before) + def test_check_config(self): self.odoo_repository._check_config() From 2ff49f5ced726bf4950cd786ec605df0fac39c9a Mon Sep 17 00:00:00 2001 From: "Laurent Mignon (ACSONE)" Date: Mon, 10 Aug 2026 17:28:16 +0200 Subject: [PATCH 10/15] [FIX] odoo_repository: Actually detect an ongoing scan '_check_existing_jobs()' was supposed to prevent multiple scans of the same repository from running concurrently, but its search domain could never match because 'queue.job.records' is a jsonb field containing an escaped JSON string. Search the ongoing jobs and identify their repository in Python, as the field cannot be reliably searched using a domain. --- odoo_repository/models/odoo_repository.py | 11 +++-- .../tests/test_odoo_repository_scan.py | 47 +++++++++++++++++++ 2 files changed, 54 insertions(+), 4 deletions(-) diff --git a/odoo_repository/models/odoo_repository.py b/odoo_repository/models/odoo_repository.py index b53fbc65..e67ce183 100644 --- a/odoo_repository/models/odoo_repository.py +++ b/odoo_repository/models/odoo_repository.py @@ -221,13 +221,12 @@ def _check_config(self): def _check_existing_jobs(self, raise_exc=True): """Check if a scan is already triggered for this repository.""" self.ensure_one() - existing_job = ( + ongoing_jobs = ( self.env["queue.job"] .sudo() .search( [ ("model_name", "=", self._name), - ("records", "ilike", f'%"ids": [{self.id}]%'), ( "state", "in", @@ -238,10 +237,14 @@ def _check_existing_jobs(self, raise_exc=True): "started", ], ), - ], - limit=1, + ] ) ) + # NOTE: 'records' cannot take part in the domain. It is a jsonb holding + # an escaped JSON string, so a pattern such as '"ids": [42]' is never + # found in it, and searching on it silently matched nothing. Which + # repository the ongoing jobs are about is therefore told here instead. + existing_job = ongoing_jobs.filtered(lambda job: self in job.records) if existing_job: msg = _("A scan is already ongoing for repository %s") % self.display_name if raise_exc: diff --git a/odoo_repository/tests/test_odoo_repository_scan.py b/odoo_repository/tests/test_odoo_repository_scan.py index e350119f..7f3d54cf 100644 --- a/odoo_repository/tests/test_odoo_repository_scan.py +++ b/odoo_repository/tests/test_odoo_repository_scan.py @@ -2,12 +2,31 @@ # Copyright 2026 Sébastien Alix # License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl) +import contextlib +import gc + from odoo import fields +from odoo.exceptions import UserError +from odoo.tools import mute_logger from .common import Common class TestOdooRepositoryScan(Common): + @classmethod + def setUpClass(cls): + super().setUpClass() + muted = contextlib.ExitStack() + cls.addClassCleanup(muted.close) + muted.enter_context(mute_logger("odoo.addons.queue_job.delay")) + + def tearDown(self): + super().tearDown() + # A delayable warns when collected without having been delayed, which + # is what these tests do on purpose. Collect here, while + # the logger above is still muted, rather than after 'tearDownClass'. + gc.collect() + def test_create_scan_jobs_does_not_delay_them(self): """The jobs are returned so that callers can build a graph out of them. @@ -24,6 +43,34 @@ def test_create_scan_jobs_does_not_delay_them(self): self.env.flush_all() self.assertEqual(self.env["queue.job"].search_count([]), before) + def test_check_existing_jobs(self): + """A scan is not started again while one is still ongoing.""" + self.assertFalse(self.odoo_repository._check_existing_jobs(raise_exc=False)) + for job in self.odoo_repository._create_scan_jobs(branch_ids=self.branch.ids): + job.delay() + self.env.flush_all() + with mute_logger("odoo.addons.odoo_repository.models.odoo_repository"): + self.assertTrue(self.odoo_repository._check_existing_jobs(raise_exc=False)) + with self.assertRaisesRegex(UserError, "already ongoing"): + self.odoo_repository._check_existing_jobs() + + def test_check_existing_jobs_ignores_other_repositories(self): + """The ongoing scan of another repository does not block this one.""" + other = self.env["odoo.repository"].create( + { + "org_id": self.org.id, + "name": "other-repo", + "repo_url": "https://github.com/ORG/other-repo", + "repo_type": "github", + } + ) + for job in other._create_scan_jobs(branch_ids=self.branch.ids): + job.delay() + self.env.flush_all() + with mute_logger("odoo.addons.odoo_repository.models.odoo_repository"): + self.assertTrue(other._check_existing_jobs(raise_exc=False)) + self.assertFalse(self.odoo_repository._check_existing_jobs(raise_exc=False)) + def test_check_config(self): self.odoo_repository._check_config() From 7b9957703e4895aa8eaf214f751b2aa05d0d1c01 Mon Sep 17 00:00:00 2001 From: "Laurent Mignon (ACSONE)" Date: Wed, 12 Aug 2026 11:24:30 +0200 Subject: [PATCH 11/15] [REF] odoo_repository: Share common scanner parameters Some scanners only need the basic information required to access a repository, while '_prepare_scanner_parameters' also includes parameters specific to branch scanning. Extract the parameters needed by every repository scanner into '_prepare_base_scanner_parameters', so they can be reused by scanners with different needs. --- odoo_repository/models/odoo_repository.py | 32 +++++++++++++------ .../tests/test_odoo_repository_lookup.py | 18 +++++++++++ 2 files changed, 40 insertions(+), 10 deletions(-) diff --git a/odoo_repository/models/odoo_repository.py b/odoo_repository/models/odoo_repository.py index e67ce183..4d629b30 100644 --- a/odoo_repository/models/odoo_repository.py +++ b/odoo_repository/models/odoo_repository.py @@ -445,13 +445,33 @@ def _get_token(self): or os.environ.get("GITHUB_TOKEN") ) - def _prepare_scanner_parameters(self, version, branch): + def _prepare_base_scanner_parameters(self): + """Return the parameters every scanner of this repository needs. + + They are the ones telling where the clone lives and how to reach the + remote, which any scanner working on this repository has to share, + whatever it scans. + """ ir_config = self.env["ir.config_parameter"] - repositories_path = ir_config.sudo().get_param(self._repositories_path_key) return { "org": self.org_id.name, "name": self.name, "clone_url": self.clone_url, + "repositories_path": ir_config.sudo().get_param( + self._repositories_path_key + ), + "repo_type": self.repo_type, + "ssh_key": self.ssh_key_id.private_key, + "token": self._get_token(), + "workaround_fs_errors": ( + self.env.company.config_odoo_repository_workaround_fs_errors + ), + "clone_name": self.clone_name, + } + + def _prepare_scanner_parameters(self, version, branch): + return { + **self._prepare_base_scanner_parameters(), "version": version, "branch": branch, "addons_paths_data": self.addons_path_ids.read( @@ -462,14 +482,6 @@ def _prepare_scanner_parameters(self, version, branch): "is_community", ] ), - "repositories_path": repositories_path, - "repo_type": self.repo_type, - "ssh_key": self.ssh_key_id.private_key, - "token": self._get_token(), - "workaround_fs_errors": ( - self.env.company.config_odoo_repository_workaround_fs_errors - ), - "clone_name": self.clone_name, "env": self.env, } diff --git a/odoo_repository/tests/test_odoo_repository_lookup.py b/odoo_repository/tests/test_odoo_repository_lookup.py index b4612c3d..c79ae407 100644 --- a/odoo_repository/tests/test_odoo_repository_lookup.py +++ b/odoo_repository/tests/test_odoo_repository_lookup.py @@ -109,3 +109,21 @@ def test_get_local_clone_path_with_clone_name(self): repository._get_local_clone_path(), pathlib.Path(self.repositories_path, "OCA", "oca-account-invoicing"), ) + + def test_prepare_base_scanner_parameters(self): + """Every scanner of a repository gets the same way to reach it.""" + token = self.env["authentication.token"].create( + {"name": "OCA", "token": "s3cr3t"} + ) + repository = self._create_repository("OCA", "account-invoicing") + repository.token_id = token + params = repository._prepare_base_scanner_parameters() + self.assertEqual(params["org"], "OCA") + self.assertEqual(params["name"], "account-invoicing") + self.assertEqual(params["token"], "s3cr3t") + self.assertEqual( + params["clone_url"], "https://github.com/OCA/account-invoicing.git" + ) + # The parameters of a repository scanner build on them + scanner_params = repository._prepare_scanner_parameters("18.0", "18.0") + self.assertLessEqual(params.items(), scanner_params.items()) From 0d0617d94af9ee1e809233a2211611c062f48e0a Mon Sep 17 00:00:00 2001 From: "Laurent Mignon (ACSONE)" Date: Wed, 12 Aug 2026 15:27:32 +0200 Subject: [PATCH 12/15] [REF] odoo_repository: Look dependencies up from a branch and a repository '_get_dependency_ids' took the repository branch being scanned to know where to look the dependencies of a module up. A module read outside of a scan has none: one added by a pull request not merged yet belongs to no repository at all, and its branch is only told by the module branch itself. Take the branch and the repository, which every caller has at hand. --- odoo_repository/models/odoo_module_branch.py | 15 ++++++++++----- odoo_repository/models/odoo_repository.py | 4 +++- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/odoo_repository/models/odoo_module_branch.py b/odoo_repository/models/odoo_module_branch.py index fba45edd..619c686a 100644 --- a/odoo_repository/models/odoo_module_branch.py +++ b/odoo_repository/models/odoo_module_branch.py @@ -438,7 +438,8 @@ def _prepare_module_branch_values(self, repo_branch, module, data): python_dependency_ids = [] if manifest.get("installable", True): dependency_ids = self._get_dependency_ids( - repo_branch, + repo_branch.branch_id, + repo_branch.repository_id, # Set at least a dependency on "base" if not defined manifest.get("depends") or ["base"], ) @@ -756,13 +757,17 @@ def _find_or_create(self, branch, module, repo, domain=None): module_branch = self.sudo()._create_orphaned_module_branch(branch, module) return module_branch - def _get_dependency_ids(self, repo_branch, depends: list): + def _get_dependency_ids(self, branch, repository, depends: list): + """Return the modules `depends` refers to, on `branch`. + + They are looked up in `repository` first. Both are those of the module + depending on them, which a repository branch does not always tell: a + module read outside of a scan can belong to no repository at all. + """ dependency_ids = [] for depend in depends: module = self._get_module(depend) - dependency = self._find_or_create( - repo_branch.branch_id, module, repo_branch.repository_id - ) + dependency = self._find_or_create(branch, module, repository) dependency_ids.append(dependency.id) return dependency_ids diff --git a/odoo_repository/models/odoo_repository.py b/odoo_repository/models/odoo_repository.py index 4d629b30..977e0519 100644 --- a/odoo_repository/models/odoo_repository.py +++ b/odoo_repository/models/odoo_repository.py @@ -556,7 +556,9 @@ def _prepare_module_branch_values(self, data): maintainer_ids = mb_model._get_maintainer_ids(tuple(data["maintainers"])) dev_status_id = mb_model._get_dev_status_id(data["development_status"]) dependency_ids = mb_model._get_dependency_ids( - repository_branch, data["depends"] + repository_branch.branch_id, + repository_branch.repository_id, + data["depends"], ) external_dependencies = data["external_dependencies"] python_dependency_ids = mb_model._get_python_dependency_ids( From 5dd38f69020727bdbe03b6d6b18fa334132ee6ee Mon Sep 17 00:00:00 2001 From: "Laurent Mignon (ACSONE)" Date: Wed, 12 Aug 2026 15:28:47 +0200 Subject: [PATCH 13/15] [REF] odoo_repository: Extract what a manifest and a code analysis carry '_prepare_module_branch_values' mixes three things: what the scan of a repository knows about a module, what its manifest declares, and what the analysis of its code counted. Only the first one is about being scanned, the other two hold for a module read from anywhere. Extract '_prepare_manifest_values' and '_prepare_code_analysis_values', so that reading a module outside of a scan does not have to spell the same mapping again. --- odoo_repository/models/odoo_module_branch.py | 120 ++++++++++--------- odoo_repository/models/odoo_repository.py | 12 +- 2 files changed, 74 insertions(+), 58 deletions(-) diff --git a/odoo_repository/models/odoo_module_branch.py b/odoo_repository/models/odoo_module_branch.py index 619c686a..61e08103 100644 --- a/odoo_repository/models/odoo_module_branch.py +++ b/odoo_repository/models/odoo_module_branch.py @@ -425,59 +425,14 @@ def _prepare_module_branch_values(self, repo_branch, module, data): "pr_url": False, } if manifest: - category_id = self._get_module_category_id(manifest.get("category", "")) - author_ids = self._get_author_ids(manifest.get("author", "")) - maintainer_ids = self._get_maintainer_ids( - tuple(manifest.get("maintainers", [])) - ) - dev_status_id = self._get_dev_status_id( - manifest.get("development_status", "") - ) - dependency_ids = [] - external_dependencies = {} - python_dependency_ids = [] - if manifest.get("installable", True): - dependency_ids = self._get_dependency_ids( - repo_branch.branch_id, - repo_branch.repository_id, - # Set at least a dependency on "base" if not defined - manifest.get("depends") or ["base"], - ) - external_dependencies = manifest.get("external_dependencies", {}) - python_dependency_ids = self._get_python_dependency_ids( - tuple(external_dependencies.get("python", [])) - ) - license_id = self._get_license_id(manifest.get("license", "")) values.update( - { - "title": manifest.get("name", False), - "summary": manifest.get( - "summary", manifest.get("description", False) - ), - "category_id": category_id, - "author_ids": [(6, 0, author_ids)], - "maintainer_ids": [(6, 0, maintainer_ids)], - "dependency_ids": [(6, 0, dependency_ids)], - "external_dependencies": external_dependencies, - "python_dependency_ids": [(6, 0, python_dependency_ids)], - "license_id": license_id, - "version": manifest.get("version", False), - "development_status_id": dev_status_id, - "application": manifest.get("application", False), - "installable": manifest.get("installable", True), - "auto_install": manifest.get("auto_install", False), - } + self._prepare_manifest_values( + manifest, repo_branch.branch_id, repo_branch.repository_id + ) ) if data.get("last_scanned_commit"): - values.update( - { - "removed": False, - "sloc_python": data["code"]["Python"], - "sloc_xml": data["code"]["XML"], - "sloc_js": data["code"]["JavaScript"], - "sloc_css": data["code"]["CSS"], - } - ) + values["removed"] = False + values.update(self._prepare_code_analysis_values(data["code"])) # Handle module removal elif module_branch: values.update( @@ -508,6 +463,66 @@ def _prepare_module_branch_values(self, repo_branch, module, data): values["version_ids"] = versions return values + def _prepare_manifest_values(self, manifest, branch, repository): + """Return the `odoo.module.branch` values a manifest carries. + + `branch` and `repository` are where the dependencies it declares are + looked up, those of the module the manifest belongs to. + """ + dependency_ids = [] + external_dependencies = {} + python_dependency_ids = [] + if manifest.get("installable", True): + dependency_ids = self._get_dependency_ids( + branch, + repository, + # Set at least a dependency on "base" if not defined + manifest.get("depends") or ["base"], + ) + external_dependencies = manifest.get("external_dependencies", {}) + python_dependency_ids = self._get_python_dependency_ids( + tuple(external_dependencies.get("python", [])) + ) + return { + "title": manifest.get("name", False), + "summary": manifest.get("summary", manifest.get("description", False)), + "category_id": self._get_module_category_id(manifest.get("category", "")), + "author_ids": [(6, 0, self._get_author_ids(manifest.get("author", "")))], + "maintainer_ids": [ + (6, 0, self._get_maintainer_ids(tuple(manifest.get("maintainers", [])))) + ], + "dependency_ids": [(6, 0, dependency_ids)], + "external_dependencies": external_dependencies, + "python_dependency_ids": [(6, 0, python_dependency_ids)], + "license_id": self._get_license_id(manifest.get("license", "")), + "version": manifest.get("version", False), + "development_status_id": self._get_dev_status_id( + manifest.get("development_status", "") + ), + "application": manifest.get("application", False), + "installable": manifest.get("installable", True), + "auto_install": manifest.get("auto_install", False), + } + + def _get_sloc_fields(self): + """Return the field holding the count of each analysed language. + + Counting one more language is adding it here, and having the scanner + analyse it. + """ + return { + "Python": "sloc_python", + "XML": "sloc_xml", + "JavaScript": "sloc_js", + "CSS": "sloc_css", + } + + def _prepare_code_analysis_values(self, code): + """Return the `odoo.module.branch` values a code analysis carries.""" + return { + field: code[language] for language, field in self._get_sloc_fields().items() + } + def _create_or_update(self, repo_branch, module, values): """Create or update a `odoo.module.branch` record from scanned module. @@ -926,10 +941,7 @@ def _to_dict(self): "is_standard": self.is_standard, "is_enterprise": self.is_enterprise, "is_community": self.is_community, - "sloc_python": self.sloc_python, - "sloc_xml": self.sloc_xml, - "sloc_js": self.sloc_js, - "sloc_css": self.sloc_css, + **{field: self[field] for field in self._get_sloc_fields().values()}, "last_scanned_commit": self.last_scanned_commit, "addons_path": self.addons_path, "pr_url": self.pr_url, diff --git a/odoo_repository/models/odoo_repository.py b/odoo_repository/models/odoo_repository.py index 977e0519..ba06c875 100644 --- a/odoo_repository/models/odoo_repository.py +++ b/odoo_repository/models/odoo_repository.py @@ -590,10 +590,14 @@ def _prepare_module_branch_values(self, data): "is_standard": data["is_standard"], "is_enterprise": data["is_enterprise"], "is_community": data["is_community"], - "sloc_python": data["sloc_python"], - "sloc_xml": data["sloc_xml"], - "sloc_js": data["sloc_js"], - "sloc_css": data["sloc_css"], + # The exchange format spells the code analysis with the very names + # of the fields holding it. A node counting one more language than + # the one it imports from simply gets nothing for it. + **{ + field: data[field] + for field in mb_model._get_sloc_fields().values() + if field in data + }, "last_scanned_commit": data["last_scanned_commit"], "pr_url": data["pr_url"], } From f1581763a91aef744831c7c4ade80ec61f0f3591 Mon Sep 17 00:00:00 2001 From: "Laurent Mignon (ACSONE)" Date: Wed, 12 Aug 2026 15:13:18 +0200 Subject: [PATCH 14/15] [ADD] odoo_repository_fork A module is not always installed from the repository it belongs to. Freezing one built from unmerged pull requests is commonly done by pinning a revision of a fork, and that fork was nowhere in the database: its ancestry had to be asked to GitHub over and over, and nothing held the credentials a private one needs to be read. Give a fork a record of its own, an 'odoo.repository' pointing at the one it originates from through 'upstream_repository_id'. It is registered on the fly whenever the ancestry of an unknown clone URL turns out to reach a known repository, and it can be created by hand as well. A fork is never scanned, a constraint enforcing it: it hosts the very same modules as its origin, and a second scanned repository holding them would make '_find()' pick one of the two by sequence, silently corrupting the dependency graph of every project. Its credentials stay editable nonetheless, being precisely what reading a private fork needs. '_find_from_clone_url()' returns what a URL points at, fork included, while '_find_upstream_repository()' returns the repository holding the modules. --- odoo_repository_fork/README.rst | 101 ++++ odoo_repository_fork/__init__.py | 1 + odoo_repository_fork/__manifest__.py | 18 + odoo_repository_fork/models/__init__.py | 1 + .../models/odoo_repository.py | 150 ++++++ odoo_repository_fork/pyproject.toml | 3 + odoo_repository_fork/readme/CONTRIBUTORS.md | 2 + odoo_repository_fork/readme/DESCRIPTION.md | 8 + odoo_repository_fork/readme/USAGE.md | 11 + .../static/description/index.html | 446 ++++++++++++++++++ odoo_repository_fork/tests/__init__.py | 1 + .../tests/test_odoo_repository_fork.py | 202 ++++++++ .../views/odoo_repository.xml | 37 ++ 13 files changed, 981 insertions(+) create mode 100644 odoo_repository_fork/README.rst create mode 100644 odoo_repository_fork/__init__.py create mode 100644 odoo_repository_fork/__manifest__.py create mode 100644 odoo_repository_fork/models/__init__.py create mode 100644 odoo_repository_fork/models/odoo_repository.py create mode 100644 odoo_repository_fork/pyproject.toml create mode 100644 odoo_repository_fork/readme/CONTRIBUTORS.md create mode 100644 odoo_repository_fork/readme/DESCRIPTION.md create mode 100644 odoo_repository_fork/readme/USAGE.md create mode 100644 odoo_repository_fork/static/description/index.html create mode 100644 odoo_repository_fork/tests/__init__.py create mode 100644 odoo_repository_fork/tests/test_odoo_repository_fork.py create mode 100644 odoo_repository_fork/views/odoo_repository.xml diff --git a/odoo_repository_fork/README.rst b/odoo_repository_fork/README.rst new file mode 100644 index 00000000..1b43b495 --- /dev/null +++ b/odoo_repository_fork/README.rst @@ -0,0 +1,101 @@ +================ +Odoo MCA - Forks +================ + +.. + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + !! This file is generated by oca-gen-addon-readme !! + !! changes will be overwritten. !! + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + !! source digest: sha256:373cd18ccca83c45fa3516d480eb32693d0a4e52fa7ef657346cfdce3261cb6a + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + +.. |badge1| image:: https://img.shields.io/badge/maturity-Beta-yellow.png + :target: https://odoo-community.org/page/development-status + :alt: Beta +.. |badge2| image:: https://img.shields.io/badge/licence-AGPL--3-blue.png + :target: http://www.gnu.org/licenses/agpl-3.0-standalone.html + :alt: License: AGPL-3 +.. |badge3| image:: https://img.shields.io/badge/github-OCA%2Fmodule--composition--analysis-lightgray.png?logo=github + :target: https://github.com/OCA/module-composition-analysis/tree/18.0/odoo_repository_fork + :alt: OCA/module-composition-analysis +.. |badge4| image:: https://img.shields.io/badge/weblate-Translate%20me-F47D42.png + :target: https://translation.odoo-community.org/projects/module-composition-analysis-18-0/module-composition-analysis-18-0-odoo_repository_fork + :alt: Translate me on Weblate +.. |badge5| image:: https://img.shields.io/badge/runboat-Try%20me-875A7B.png + :target: https://runboat.odoo-community.org/builds?repo=OCA/module-composition-analysis&target_branch=18.0 + :alt: Try me on Runboat + +|badge1| |badge2| |badge3| |badge4| |badge5| + +A module is not always installed from the repository it belongs to. +Freezing one built from unmerged pull requests is commonly done by +pinning a revision of a fork, and that fork is nowhere in Odoo MCA: its +ancestry has to be asked to GitHub over and over, and nothing holds the +credentials a private one needs to be read. + +This module registers a fork as a repository of its own, pointing at the +one it originates from. + +**Table of contents** + +.. contents:: + :local: + +Usage +===== + +A fork is registered on the fly, whenever something looks up the +repository a clone URL points at and GitHub answers that it is a fork of +a repository already known. + +It can be created by hand too, by setting *Fork Of* on a repository. +Either way, a fork is never scanned: it hosts the very same modules as +the repository it comes from, and a second scanned repository holding +them would make the module of a dependency ambiguous for every project. +A constraint enforces it. + +Its credentials remain editable though, and they are what reading a +private fork needs. The forks of a repository are listed on it. + +Bug Tracker +=========== + +Bugs are tracked on `GitHub Issues `_. +In case of trouble, please check there if your issue has already been reported. +If you spotted it first, help us to smash it by providing a detailed and welcomed +`feedback `_. + +Do not contact contributors directly about support or help with technical issues. + +Credits +======= + +Authors +------- + +* ACSONE SA/NV + +Contributors +------------ + +- ACSONE SA/NV + + - Laurent Mignon + +Maintainers +----------- + +This module is maintained by the OCA. + +.. image:: https://odoo-community.org/logo.png + :alt: Odoo Community Association + :target: https://odoo-community.org + +OCA, or the Odoo Community Association, is a nonprofit organization whose +mission is to support the collaborative development of Odoo features and +promote its widespread use. + +This module is part of the `OCA/module-composition-analysis `_ project on GitHub. + +You are welcome to contribute. To learn how please visit https://odoo-community.org/page/Contribute. diff --git a/odoo_repository_fork/__init__.py b/odoo_repository_fork/__init__.py new file mode 100644 index 00000000..0650744f --- /dev/null +++ b/odoo_repository_fork/__init__.py @@ -0,0 +1 @@ +from . import models diff --git a/odoo_repository_fork/__manifest__.py b/odoo_repository_fork/__manifest__.py new file mode 100644 index 00000000..4ef38011 --- /dev/null +++ b/odoo_repository_fork/__manifest__.py @@ -0,0 +1,18 @@ +# Copyright 2026 ACSONE SA/NV () +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl) +{ + "name": "Odoo MCA - Forks", + "summary": "Register the forks of a repository and where they come from.", + "version": "18.0.1.0.0", + "category": "Tools", + "author": "ACSONE SA/NV, Odoo Community Association (OCA)", + "website": "https://github.com/OCA/module-composition-analysis", + "data": [ + "views/odoo_repository.xml", + ], + "installable": True, + "depends": [ + "odoo_repository", + ], + "license": "AGPL-3", +} diff --git a/odoo_repository_fork/models/__init__.py b/odoo_repository_fork/models/__init__.py new file mode 100644 index 00000000..708af8c2 --- /dev/null +++ b/odoo_repository_fork/models/__init__.py @@ -0,0 +1 @@ +from . import odoo_repository diff --git a/odoo_repository_fork/models/odoo_repository.py b/odoo_repository_fork/models/odoo_repository.py new file mode 100644 index 00000000..80788dce --- /dev/null +++ b/odoo_repository_fork/models/odoo_repository.py @@ -0,0 +1,150 @@ +# Copyright 2026 ACSONE SA/NV () +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl) + +import logging + +from odoo import _, api, fields, models, tools +from odoo.exceptions import ValidationError + +from odoo.addons.odoo_repository.utils import github + +_logger = logging.getLogger(__name__) + + +class OdooRepository(models.Model): + _inherit = "odoo.repository" + + upstream_repository_id = fields.Many2one( + comodel_name="odoo.repository", + string="Fork Of", + ondelete="cascade", + index=True, + help=( + "Repository this one is a fork of. A fork hosts the very same " + "modules, so it is never scanned: its modules are those of the " + "repository it originates from. It is registered to document where " + "a project installs them from, and to hold the credentials needed " + "to read it." + ), + ) + fork_ids = fields.One2many( + comodel_name="odoo.repository", + inverse_name="upstream_repository_id", + string="Forks", + ) + + @api.constrains("upstream_repository_id", "to_scan") + def _check_fork_not_scanned(self): + for rec in self: + if rec.upstream_repository_id and rec.to_scan: + raise ValidationError( + _( + "Repository %(fork)s cannot be scanned, it is a fork of " + "%(upstream)s and hosts the very same modules. Scanning " + "it would make the module of a dependency ambiguous for " + "every project." + ) + % { + "fork": rec.display_name, + "upstream": rec.upstream_repository_id.display_name, + } + ) + + @api.constrains("upstream_repository_id") + def _check_upstream_repository_cycle(self): + if self._has_cycle("upstream_repository_id"): + raise ValidationError(_("A repository cannot be a fork of itself.")) + + @api.model + @tools.ormcache("org", "name") + def _fetch_github_fork_parents(self, org, name): + """Return the ancestors of a forked GitHub repository. + + The result is a tuple of ``org/name`` strings, from the direct parent + to the root of the fork chain, empty if the repository is not a fork. + """ + data = github.request(self.env, f"repos/{org}/{name}") + if not data.get("fork"): + return () + full_names = ( + (data.get("parent") or {}).get("full_name"), + (data.get("source") or {}).get("full_name"), + ) + # dict.fromkeys() deduplicates while keeping the parent first, as both + # entries are equal as soon as the fork chain has a single level. + return tuple(dict.fromkeys(filter(None, full_names))) + + @api.model + def _find_from_clone_url(self, clone_url): + """Return the repository a clone URL points at, fork or not. + + Lookup order: + 1. the repository matching the URL itself + 2. for a GitHub fork whose origin is known, the fork itself, which + is registered on the fly to document where the modules are + installed from and to hold the credentials reading it needs + + Returns an empty recordset when the origin cannot be told, letting the + caller fall back to its own heuristics. + """ + host, org, name = self._parse_clone_url(clone_url) + if not org: + return self.browse() + repository = self._find_from_org_and_name(org, name) + if repository: + return repository + if host not in ("github.com", "www.github.com"): + # Only GitHub exposes the fork ancestry through its API + return self.browse() + try: + full_names = self._fetch_github_fork_parents(org, name) + except RuntimeError: + # Do not let a GitHub outage or rate limit break the caller. The + # failure is not cached, so the next call will try again. + _logger.warning( + "Unable to get the fork ancestry of %s", clone_url, exc_info=True + ) + return self.browse() + for full_name in full_names: + parent_org, __, parent_name = full_name.rpartition("/") + upstream = self._find_from_org_and_name(parent_org, parent_name) + if upstream: + return self._create_fork(host, org, name, clone_url, upstream) + return self.browse() + + @api.model + def _create_fork(self, host, org, name, clone_url, upstream): + """Register the fork of `upstream` hosted at `host`, as `org`/`name`.""" + org_record = self.env["odoo.repository.org"].search([("name", "=", org)]) + if not org_record: + org_record = self.env["odoo.repository.org"].sudo().create({"name": org}) + _logger.info( + "Registering %s/%s as a fork of %s", org, name, upstream.display_name + ) + return ( + self.sudo() + .create( + { + "org_id": org_record.id, + "name": name, + "repo_url": f"https://{host}/{org}/{name}", + "clone_url": clone_url, + # A fork is hosted next to the repository it originates + # from, whose ancestry is what told us it is one. + "repo_type": upstream.repo_type, + "to_scan": False, + "upstream_repository_id": upstream.id, + } + ) + .with_env(self.env) + ) + + @api.model + def _find_upstream_repository(self, clone_url): + """Return the repository the modules of a clone URL belong to. + + The modules of a fork are those of the repository it originates from: + a fork is never scanned, so it hosts none of its own. + """ + repository = self._find_from_clone_url(clone_url) + return repository.upstream_repository_id or repository diff --git a/odoo_repository_fork/pyproject.toml b/odoo_repository_fork/pyproject.toml new file mode 100644 index 00000000..4231d0cc --- /dev/null +++ b/odoo_repository_fork/pyproject.toml @@ -0,0 +1,3 @@ +[build-system] +requires = ["whool"] +build-backend = "whool.buildapi" diff --git a/odoo_repository_fork/readme/CONTRIBUTORS.md b/odoo_repository_fork/readme/CONTRIBUTORS.md new file mode 100644 index 00000000..c4904374 --- /dev/null +++ b/odoo_repository_fork/readme/CONTRIBUTORS.md @@ -0,0 +1,2 @@ +- ACSONE SA/NV + - Laurent Mignon \ diff --git a/odoo_repository_fork/readme/DESCRIPTION.md b/odoo_repository_fork/readme/DESCRIPTION.md new file mode 100644 index 00000000..d92184c6 --- /dev/null +++ b/odoo_repository_fork/readme/DESCRIPTION.md @@ -0,0 +1,8 @@ +A module is not always installed from the repository it belongs to. Freezing +one built from unmerged pull requests is commonly done by pinning a revision +of a fork, and that fork is nowhere in Odoo MCA: its ancestry has to be asked +to GitHub over and over, and nothing holds the credentials a private one needs +to be read. + +This module registers a fork as a repository of its own, pointing at the one +it originates from. diff --git a/odoo_repository_fork/readme/USAGE.md b/odoo_repository_fork/readme/USAGE.md new file mode 100644 index 00000000..500243e7 --- /dev/null +++ b/odoo_repository_fork/readme/USAGE.md @@ -0,0 +1,11 @@ +A fork is registered on the fly, whenever something looks up the repository a +clone URL points at and GitHub answers that it is a fork of a repository +already known. + +It can be created by hand too, by setting *Fork Of* on a repository. Either +way, a fork is never scanned: it hosts the very same modules as the repository +it comes from, and a second scanned repository holding them would make the +module of a dependency ambiguous for every project. A constraint enforces it. + +Its credentials remain editable though, and they are what reading a private +fork needs. The forks of a repository are listed on it. diff --git a/odoo_repository_fork/static/description/index.html b/odoo_repository_fork/static/description/index.html new file mode 100644 index 00000000..623f8d98 --- /dev/null +++ b/odoo_repository_fork/static/description/index.html @@ -0,0 +1,446 @@ + + + + + +Odoo MCA - Forks + + + +
+

Odoo MCA - Forks

+ + +

Beta License: AGPL-3 OCA/module-composition-analysis Translate me on Weblate Try me on Runboat

+

A module is not always installed from the repository it belongs to. +Freezing one built from unmerged pull requests is commonly done by +pinning a revision of a fork, and that fork is nowhere in Odoo MCA: its +ancestry has to be asked to GitHub over and over, and nothing holds the +credentials a private one needs to be read.

+

This module registers a fork as a repository of its own, pointing at the +one it originates from.

+

Table of contents

+ +
+

Usage

+

A fork is registered on the fly, whenever something looks up the +repository a clone URL points at and GitHub answers that it is a fork of +a repository already known.

+

It can be created by hand too, by setting Fork Of on a repository. +Either way, a fork is never scanned: it hosts the very same modules as +the repository it comes from, and a second scanned repository holding +them would make the module of a dependency ambiguous for every project. +A constraint enforces it.

+

Its credentials remain editable though, and they are what reading a +private fork needs. The forks of a repository are listed on it.

+
+
+

Bug Tracker

+

Bugs are tracked on GitHub Issues. +In case of trouble, please check there if your issue has already been reported. +If you spotted it first, help us to smash it by providing a detailed and welcomed +feedback.

+

Do not contact contributors directly about support or help with technical issues.

+
+
+

Credits

+
+

Authors

+
    +
  • ACSONE SA/NV
  • +
+
+
+

Contributors

+ +
+
+

Maintainers

+

This module is maintained by the OCA.

+ +Odoo Community Association + +

OCA, or the Odoo Community Association, is a nonprofit organization whose +mission is to support the collaborative development of Odoo features and +promote its widespread use.

+

This module is part of the OCA/module-composition-analysis project on GitHub.

+

You are welcome to contribute. To learn how please visit https://odoo-community.org/page/Contribute.

+
+
+
+ + diff --git a/odoo_repository_fork/tests/__init__.py b/odoo_repository_fork/tests/__init__.py new file mode 100644 index 00000000..4eb89124 --- /dev/null +++ b/odoo_repository_fork/tests/__init__.py @@ -0,0 +1 @@ +from . import test_odoo_repository_fork diff --git a/odoo_repository_fork/tests/test_odoo_repository_fork.py b/odoo_repository_fork/tests/test_odoo_repository_fork.py new file mode 100644 index 00000000..29921995 --- /dev/null +++ b/odoo_repository_fork/tests/test_odoo_repository_fork.py @@ -0,0 +1,202 @@ +# Copyright 2026 ACSONE SA/NV () +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl) + +from unittest.mock import patch + +from odoo.exceptions import ValidationError +from odoo.tools import mute_logger + +from odoo.addons.odoo_repository.tests.common import Common + +GITHUB_REQUEST = ( + "odoo.addons.odoo_repository_fork.models.odoo_repository.github.request" +) + + +class TestOdooRepositoryFork(Common): + def setUp(self): + super().setUp() + # '_fetch_github_fork_parents' is an ormcache, make sure a test never + # observes what a previous one stored. + self.env.registry.clear_cache() + self.repository_model = self.env["odoo.repository"] + + def _create_repository(self, org_name, repo_name, **values): + org = self.env["odoo.repository.org"].search([("name", "=", org_name)]) + if not org: + org = self.env["odoo.repository.org"].create({"name": org_name}) + vals = { + "org_id": org.id, + "name": repo_name, + "repo_url": f"https://github.com/{org_name}/{repo_name}", + "clone_url": f"https://github.com/{org_name}/{repo_name}.git", + "repo_type": "github", + } + if values.get("upstream_repository_id"): + vals["to_scan"] = False + vals.update(values) + return self.env["odoo.repository"].create(vals) + + def test_find_upstream_known_repository(self): + """A URL pointing to a known repository resolves without calling GitHub.""" + repository = self._create_repository("OCA", "account-invoicing") + with patch(GITHUB_REQUEST) as request: + found = self.repository_model._find_upstream_repository( + "https://github.com/OCA/account-invoicing.git" + ) + self.assertEqual(found, repository) + request.assert_not_called() + + def test_find_upstream_from_github_fork(self): + """An unknown fork resolves to the repository it originates from.""" + upstream = self._create_repository("OCA", "account-invoicing") + payload = { + "fork": True, + "parent": {"full_name": "OCA/account-invoicing"}, + "source": {"full_name": "OCA/account-invoicing"}, + } + with patch(GITHUB_REQUEST, return_value=payload) as request: + found = self.repository_model._find_upstream_repository( + "https://github.com/acsone/account-invoicing.git" + ) + self.assertEqual(found, upstream) + request.assert_called_once() + self.assertIn("repos/acsone/account-invoicing", request.call_args[0]) + + def test_find_upstream_prefers_parent_over_source(self): + """On a fork chain, the closest known ancestor wins.""" + self._create_repository("OCA", "account-invoicing") + intermediate = self._create_repository("camptocamp", "account-invoicing") + payload = { + "fork": True, + "parent": {"full_name": "camptocamp/account-invoicing"}, + "source": {"full_name": "OCA/account-invoicing"}, + } + with patch(GITHUB_REQUEST, return_value=payload): + found = self.repository_model._find_upstream_repository( + "https://github.com/acsone/account-invoicing.git" + ) + self.assertEqual(found, intermediate) + + def test_find_upstream_falls_back_to_source(self): + """An unknown intermediate fork does not hide the root of the chain.""" + root = self._create_repository("OCA", "account-invoicing") + payload = { + "fork": True, + "parent": {"full_name": "unknown-org/account-invoicing"}, + "source": {"full_name": "OCA/account-invoicing"}, + } + with patch(GITHUB_REQUEST, return_value=payload): + found = self.repository_model._find_upstream_repository( + "https://github.com/acsone/account-invoicing.git" + ) + self.assertEqual(found, root) + + def test_find_upstream_not_a_fork(self): + """A standalone repository has no upstream to resolve.""" + self._create_repository("OCA", "account-invoicing") + with patch(GITHUB_REQUEST, return_value={"fork": False}): + found = self.repository_model._find_upstream_repository( + "https://github.com/acsone/vendored-addons.git" + ) + self.assertFalse(found) + + def test_find_upstream_non_github_host(self): + """Only GitHub exposes the fork ancestry: no API call is attempted.""" + with patch(GITHUB_REQUEST) as request: + found = self.repository_model._find_upstream_repository( + "https://gitlab.com/acsone/account-invoicing.git" + ) + self.assertFalse(found) + request.assert_not_called() + + def test_find_upstream_github_error_is_not_cached(self): + """A GitHub outage degrades gracefully and does not poison the cache.""" + upstream = self._create_repository("OCA", "account-invoicing") + url = "https://github.com/acsone/account-invoicing.git" + with ( + patch(GITHUB_REQUEST, side_effect=RuntimeError("API rate limit")), + mute_logger("odoo.addons.odoo_repository_fork.models.odoo_repository"), + ): + self.assertFalse(self.repository_model._find_upstream_repository(url)) + payload = { + "fork": True, + "parent": {"full_name": "OCA/account-invoicing"}, + "source": {"full_name": "OCA/account-invoicing"}, + } + with patch(GITHUB_REQUEST, return_value=payload): + self.assertEqual( + self.repository_model._find_upstream_repository(url), upstream + ) + + # -- registering a fork ------------------------------------------------- + + def test_fork_is_registered_on_the_fly(self): + """Discovering a fork documents it, so it can carry its credentials.""" + upstream = self._create_repository("OCA", "account-invoicing") + payload = { + "fork": True, + "parent": {"full_name": "OCA/account-invoicing"}, + "source": {"full_name": "OCA/account-invoicing"}, + } + url = "https://github.com/acsone/account-invoicing.git" + with patch(GITHUB_REQUEST, return_value=payload): + fork = self.repository_model._find_from_clone_url(url) + self.assertEqual(fork.upstream_repository_id, upstream) + self.assertEqual(fork.org_id.name, "acsone") + self.assertEqual(fork.name, "account-invoicing") + self.assertEqual(fork.clone_url, url) + self.assertEqual(fork.repo_url, "https://github.com/acsone/account-invoicing") + self.assertEqual(fork.repo_type, upstream.repo_type) + # A fork hosts the modules of its origin, scanning it would duplicate them + self.assertFalse(fork.to_scan) + self.assertEqual(upstream.fork_ids, fork) + + def test_fork_is_registered_once(self): + """A known fork is found by its URL, without asking GitHub again.""" + self._create_repository("OCA", "account-invoicing") + payload = { + "fork": True, + "parent": {"full_name": "OCA/account-invoicing"}, + "source": {"full_name": "OCA/account-invoicing"}, + } + url = "https://github.com/acsone/account-invoicing.git" + with patch(GITHUB_REQUEST, return_value=payload): + fork = self.repository_model._find_from_clone_url(url) + with patch(GITHUB_REQUEST) as request: + found = self.repository_model._find_from_clone_url(url) + self.assertEqual(found, fork) + request.assert_not_called() + + def test_fork_cannot_be_scanned(self): + """Scanning a fork would make the module of a dependency ambiguous.""" + upstream = self._create_repository("OCA", "account-invoicing") + fork = self._create_repository( + "acsone", "account-invoicing", upstream_repository_id=upstream.id + ) + self.assertFalse(fork.to_scan) + with self.assertRaisesRegex(ValidationError, "cannot be scanned"): + fork.to_scan = True + + def test_fork_cannot_be_its_own_upstream(self): + repository = self._create_repository("OCA", "account-invoicing") + with self.assertRaises(ValidationError): + repository.upstream_repository_id = repository + + def test_fork_is_reached_with_its_own_credentials(self): + """A private fork is read with the token registered on it.""" + upstream = self._create_repository("OCA", "account-invoicing") + token = self.env["authentication.token"].create( + {"name": "acsone", "token": "s3cr3t"} + ) + fork = self._create_repository( + "acsone", + "account-invoicing", + upstream_repository_id=upstream.id, + token_id=token.id, + ) + url = "https://github.com/acsone/account-invoicing.git" + self.assertEqual( + fork._get_authenticated_url(url), + "https://oauth2:s3cr3t@github.com/acsone/account-invoicing.git", + ) diff --git a/odoo_repository_fork/views/odoo_repository.xml b/odoo_repository_fork/views/odoo_repository.xml new file mode 100644 index 00000000..53d196d8 --- /dev/null +++ b/odoo_repository_fork/views/odoo_repository.xml @@ -0,0 +1,37 @@ + + + + + odoo.repository.form.fork + odoo.repository + + + + + + + + not to_scan and not upstream_repository_id + + + + + + + + + + + + + + + From 822d0b0deea2d1e0dff2d0545fefa1c6d8ebc183 Mon Sep 17 00:00:00 2001 From: "Laurent Mignon (ACSONE)" Date: Mon, 10 Aug 2026 15:00:47 +0200 Subject: [PATCH 15/15] [ADD] odoo_project_dependency_resolver Some projects pin the versions of their dependencies in a dedicated file, such as the 'requirements.txt' produced by pip. Keeping a second, manually maintained list of these dependencies in the project can easily lead to inconsistencies when dependencies are updated. This module introduces an automatic dependency resolution mechanism for projects. Instead of maintaining this information separately, the project's dependencies are resolved from the dependency file stored in its repository at each scan. The resolution mechanism is based on a format-agnostic description of a dependency, containing the information needed to identify a module, its version and its origin. The resolution logic therefore does not depend on the format used to declare the dependencies. The dependency file parsing is implemented through pluggable components, allowing support for additional formats to be added independently. The first implementation supports 'requirements.txt', with other formats such as 'pylock.toml' or 'uv.lock' being possible in the future. This provides a single dependency resolution mechanism while allowing each project to use the dependency format that best fits its environment. --- odoo_project/views/odoo_project_module.xml | 21 + odoo_project_dependency_resolver/README.rst | 151 +++++ odoo_project_dependency_resolver/__init__.py | 2 + .../__manifest__.py | 30 + .../components/__init__.py | 2 + .../components/dependency_resolver.py | 43 ++ .../components/requirements_txt_resolver.py | 151 +++++ .../data/queue_job.xml | 24 + .../lib/__init__.py | 0 .../lib/scanner.py | 106 ++++ .../models/__init__.py | 4 + .../models/odoo_module_branch.py | 47 ++ .../models/odoo_project.py | 280 +++++++++ .../models/odoo_project_module.py | 143 +++++ .../models/odoo_repository.py | 40 ++ .../pyproject.toml | 3 + .../readme/CONTRIBUTORS.md | 2 + .../readme/DESCRIPTION.md | 7 + .../readme/USAGE.md | 55 ++ .../static/description/index.html | 493 +++++++++++++++ .../tests/__init__.py | 2 + .../tests/test_requirements_txt_resolver.py | 163 +++++ .../tests/test_resolve_dependencies.py | 576 ++++++++++++++++++ .../views/odoo_project.xml | 73 +++ .../views/odoo_project_module.xml | 65 ++ odoo_project_migration/__manifest__.py | 1 + .../views/odoo_project_module.xml | 27 + .../tests/test_odoo_repository_fork.py | 9 +- 28 files changed, 2515 insertions(+), 5 deletions(-) create mode 100644 odoo_project_dependency_resolver/README.rst create mode 100644 odoo_project_dependency_resolver/__init__.py create mode 100644 odoo_project_dependency_resolver/__manifest__.py create mode 100644 odoo_project_dependency_resolver/components/__init__.py create mode 100644 odoo_project_dependency_resolver/components/dependency_resolver.py create mode 100644 odoo_project_dependency_resolver/components/requirements_txt_resolver.py create mode 100644 odoo_project_dependency_resolver/data/queue_job.xml create mode 100644 odoo_project_dependency_resolver/lib/__init__.py create mode 100644 odoo_project_dependency_resolver/lib/scanner.py create mode 100644 odoo_project_dependency_resolver/models/__init__.py create mode 100644 odoo_project_dependency_resolver/models/odoo_module_branch.py create mode 100644 odoo_project_dependency_resolver/models/odoo_project.py create mode 100644 odoo_project_dependency_resolver/models/odoo_project_module.py create mode 100644 odoo_project_dependency_resolver/models/odoo_repository.py create mode 100644 odoo_project_dependency_resolver/pyproject.toml create mode 100644 odoo_project_dependency_resolver/readme/CONTRIBUTORS.md create mode 100644 odoo_project_dependency_resolver/readme/DESCRIPTION.md create mode 100644 odoo_project_dependency_resolver/readme/USAGE.md create mode 100644 odoo_project_dependency_resolver/static/description/index.html create mode 100644 odoo_project_dependency_resolver/tests/__init__.py create mode 100644 odoo_project_dependency_resolver/tests/test_requirements_txt_resolver.py create mode 100644 odoo_project_dependency_resolver/tests/test_resolve_dependencies.py create mode 100644 odoo_project_dependency_resolver/views/odoo_project.xml create mode 100644 odoo_project_dependency_resolver/views/odoo_project_module.xml create mode 100644 odoo_project_migration/views/odoo_project_module.xml diff --git a/odoo_project/views/odoo_project_module.xml b/odoo_project/views/odoo_project_module.xml index 91c80751..4f5a41a5 100644 --- a/odoo_project/views/odoo_project_module.xml +++ b/odoo_project/views/odoo_project_module.xml @@ -7,7 +7,28 @@ odoo.project.module primary + + 99 + +
+
+ +
+
diff --git a/odoo_project_dependency_resolver/README.rst b/odoo_project_dependency_resolver/README.rst new file mode 100644 index 00000000..2ef20642 --- /dev/null +++ b/odoo_project_dependency_resolver/README.rst @@ -0,0 +1,151 @@ +================================== +Odoo Project - Dependency Resolver +================================== + +.. + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + !! This file is generated by oca-gen-addon-readme !! + !! changes will be overwritten. !! + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + !! source digest: sha256:b48258207df2f746b919569e9951023638da491aaa1e5787d44e370cb356ca94 + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + +.. |badge1| image:: https://img.shields.io/badge/maturity-Beta-yellow.png + :target: https://odoo-community.org/page/development-status + :alt: Beta +.. |badge2| image:: https://img.shields.io/badge/licence-AGPL--3-blue.png + :target: http://www.gnu.org/licenses/agpl-3.0-standalone.html + :alt: License: AGPL-3 +.. |badge3| image:: https://img.shields.io/badge/github-OCA%2Fmodule--composition--analysis-lightgray.png?logo=github + :target: https://github.com/OCA/module-composition-analysis/tree/18.0/odoo_project_dependency_resolver + :alt: OCA/module-composition-analysis +.. |badge4| image:: https://img.shields.io/badge/weblate-Translate%20me-F47D42.png + :target: https://translation.odoo-community.org/projects/module-composition-analysis-18-0/module-composition-analysis-18-0-odoo_project_dependency_resolver + :alt: Translate me on Weblate +.. |badge5| image:: https://img.shields.io/badge/runboat-Try%20me-875A7B.png + :target: https://runboat.odoo-community.org/builds?repo=OCA/module-composition-analysis&target_branch=18.0 + :alt: Try me on Runboat + +|badge1| |badge2| |badge3| |badge4| |badge5| + +Projects usually pin the exact version of every module they install in a +dependency file, such as the ``requirements.txt`` produced by pip. +Maintaining the very same list a second time by hand in Odoo MCA is +tedious and drifts away from reality as soon as a dependency is bumped. + +This module reads that file straight from the repository of the project +and rebuilds its modules from it, every time the project is scanned. + +**Table of contents** + +.. contents:: + :local: + +Usage +===== + +On the project, set *Dependency Management* to *Resolved*, pick the +format of the dependency file and, if it is not at the root of the +repository under its usual name, adjust its path. + +There is nothing else to declare. A dependency file only holds packaged +addons, so the modules of a resolved project are the union of three +sources: + +- the addons of the dependency file, with the versions it pins; +- the modules hosted by the project repository, which are the code of + the project itself and are shipped with it rather than installed as a + package; +- their missing dependencies, pulled from the dependency graph, which is + where Odoo standard modules come from. + +*Additional Modules* is there for what none of the three reaches, Odoo +Enterprise modules typically. + +The modules are refreshed at each scan, and by the *Resolve +Dependencies* button. Whatever the resolution could not do is reported +on the project, in the *Resolution Log* field. + +What a scan does +---------------- + +Scanning a resolved project only scans the repository of the project: +the repositories its modules come from are derived from the dependency +file, so none of them is known before that file has been read. + +The resolution is chained to that scan, so it reads a file that has just +been fetched, and it then spawns the scan of the repositories it +discovered. A second resolution runs once they have all been scanned, +completing what the first one could not know: the modules of a +repository that had never been scanned, and the dependency graph the +resolved modules are walked through. + +Note that the second resolution waits for the detection of the modules +to scan in every repository, not for every module to have been scanned: +the scanner spawns the latter as it goes. A module scanned late is +therefore accounted for by the next scan of the project. + +Modules pinned on a fork +------------------------ + +A module frozen on a commit of a fork, such as: + +:: + + odoo-addon-account-invoice-triple-discount @ + git+https://github.com/acsone/account-invoicing.git@3e4c54b + #subdirectory=setup/account_invoice_triple_discount + +is not treated as a new module. It stays attached to the module of the +repository the fork originates from, so that dependencies, migration +scripts and changelogs keep working, and the fork is recorded on the +project module as *Installed From*. Its version is read from the +manifest at the pinned revision. + +A module added by a pull request not merged yet is a special case: it +exists in no repository known to Odoo MCA, so it stays an orphaned +module and shows up among the unknown modules of the project. *Installed +From* is then the only way to reach a repository from it, and the *Fork +Of* of that fork tells where the module is heading. + +Bug Tracker +=========== + +Bugs are tracked on `GitHub Issues `_. +In case of trouble, please check there if your issue has already been reported. +If you spotted it first, help us to smash it by providing a detailed and welcomed +`feedback `_. + +Do not contact contributors directly about support or help with technical issues. + +Credits +======= + +Authors +------- + +* ACSONE SA/NV + +Contributors +------------ + +- ACSONE SA/NV + + - Laurent Mignon + +Maintainers +----------- + +This module is maintained by the OCA. + +.. image:: https://odoo-community.org/logo.png + :alt: Odoo Community Association + :target: https://odoo-community.org + +OCA, or the Odoo Community Association, is a nonprofit organization whose +mission is to support the collaborative development of Odoo features and +promote its widespread use. + +This module is part of the `OCA/module-composition-analysis `_ project on GitHub. + +You are welcome to contribute. To learn how please visit https://odoo-community.org/page/Contribute. diff --git a/odoo_project_dependency_resolver/__init__.py b/odoo_project_dependency_resolver/__init__.py new file mode 100644 index 00000000..f24d3e24 --- /dev/null +++ b/odoo_project_dependency_resolver/__init__.py @@ -0,0 +1,2 @@ +from . import components +from . import models diff --git a/odoo_project_dependency_resolver/__manifest__.py b/odoo_project_dependency_resolver/__manifest__.py new file mode 100644 index 00000000..aa4bcd44 --- /dev/null +++ b/odoo_project_dependency_resolver/__manifest__.py @@ -0,0 +1,30 @@ +# Copyright 2026 ACSONE SA/NV () +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl) +{ + "name": "Odoo Project - Dependency Resolver", + "summary": "Resolve the modules of a project from its dependency file.", + "version": "18.0.1.0.0", + "category": "Tools", + "author": "ACSONE SA/NV, Odoo Community Association (OCA)", + "website": "https://github.com/OCA/module-composition-analysis", + "data": [ + "data/queue_job.xml", + "views/odoo_project.xml", + "views/odoo_project_module.xml", + ], + "installable": True, + "depends": [ + # OCA/queue + "queue_job", + # OCA/module-composition-analysis + "odoo_project", + "odoo_repository_fork", + ], + "external_dependencies": { + "python": [ + "gitpython", + "odoo-addons-parser", + ], + }, + "license": "AGPL-3", +} diff --git a/odoo_project_dependency_resolver/components/__init__.py b/odoo_project_dependency_resolver/components/__init__.py new file mode 100644 index 00000000..2db60093 --- /dev/null +++ b/odoo_project_dependency_resolver/components/__init__.py @@ -0,0 +1,2 @@ +from . import dependency_resolver +from . import requirements_txt_resolver diff --git a/odoo_project_dependency_resolver/components/dependency_resolver.py b/odoo_project_dependency_resolver/components/dependency_resolver.py new file mode 100644 index 00000000..238f743a --- /dev/null +++ b/odoo_project_dependency_resolver/components/dependency_resolver.py @@ -0,0 +1,43 @@ +# Copyright 2026 ACSONE SA/NV () +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl) + +import dataclasses + +from odoo.addons.component.core import AbstractComponent + + +@dataclasses.dataclass(frozen=True) +class ResolvedDependency: + """A module declared by a project, as read from a dependency file.""" + + module_name: str + version: str = None + clone_url: str = None + ref: str = None + subdirectory: str = None + + @property + def is_pinned(self): + """Whether the module is frozen on a specific revision.""" + return bool(self.clone_url and self.ref) + + +class DependencyResolver(AbstractComponent): + """Turn the content of a dependency file into a list of modules. + + Concrete resolvers only deal with text: they parse what they are given + and return what they found, without ever touching the ORM. Mapping the + result onto modules of the database is the job of `odoo.project`, and is + shared by every format. + + Adding support for another format (pip.lock, uv.lock...) is therefore a + matter of adding a component named after the format, and an entry in the + `dependency_resolver` selection of `odoo.project`. + """ + + _name = "project.dependency.resolver" + _collection = "odoo.mca.backend" + + def resolve(self, content): + """Return the `ResolvedDependency` list declared in `content`.""" + raise NotImplementedError diff --git a/odoo_project_dependency_resolver/components/requirements_txt_resolver.py b/odoo_project_dependency_resolver/components/requirements_txt_resolver.py new file mode 100644 index 00000000..c48b48c3 --- /dev/null +++ b/odoo_project_dependency_resolver/components/requirements_txt_resolver.py @@ -0,0 +1,151 @@ +# Copyright 2026 ACSONE SA/NV () +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl) + +import re +from urllib.parse import parse_qs, urlparse, urlunparse + +from packaging.requirements import InvalidRequirement, Requirement + +from odoo.addons.component.core import Component + +from .dependency_resolver import ResolvedDependency + +# Odoo addons are distributed as 'odoo-addon-' since 15.0, and as +# 'odoo-addon-' before. Note that this excludes packages such +# as 'odoo-addons-parser', which are not addons. +ADDON_PREFIX_RE = re.compile(r"^odoo(?:\d+)?-addon-", re.IGNORECASE) +# Comments, but not the '#subdirectory=' fragment of a URL +COMMENT_RE = re.compile(r"(^|\s)#.*$") +HASH_RE = re.compile(r"\s--hash=\S+") +# An Odoo manifest version holds 5 segments ('....'). +# setuptools-odoo appends a serial to it when only the packaging changed, +# e.g. addon '18.0.1.2.0' published as wheel '18.0.1.2.0.3'. +MANIFEST_VERSION_SEGMENTS = 5 + + +class RequirementsTxtResolver(Component): + """Resolve the modules of a project from a pip requirements file.""" + + _name = "project.dependency.resolver.requirements_txt" + _inherit = "project.dependency.resolver" + _usage = "project.dependency.resolver.requirements_txt" + + def resolve(self, content): + dependencies = [] + for requirement in self._iter_requirements(content): + dependency = self._parse_requirement(requirement) + if dependency: + dependencies.append(dependency) + return dependencies + + def _iter_requirements(self, content): + """Yield the requirement lines.""" + buffer = "" + for raw_line in content.splitlines(): + line = COMMENT_RE.sub("", raw_line).strip() + if not line: + continue + if line.endswith("\\"): + buffer += line[:-1].strip() + " " + continue + line, buffer = buffer + line, "" + if line.startswith("-"): + # Options ('-r', '-c', '--index-url'...) + continue + line = HASH_RE.sub("", line) + # Remove environment markers ('; python_version < "3.8"') + # and trailing whitespace + line = line.split(";", 1)[0].strip() + if line: + yield line + + def _parse_requirement(self, requirement): + """Return the `ResolvedDependency` of a requirement, if it is an addon.""" + requirement = requirement.strip() + if not requirement: + return None + + try: + req = Requirement(requirement) + except InvalidRequirement: + # Fallback for non-PEP 508 lines that pip still accepts in practice. + # vcs legacy line ex: + # git+https://github.com/OCA/project.git@v1.2.3#egg=odoo-addon-project + name = re.split(r"[<>=!~\[\s]", requirement, maxsplit=1)[0] + module_name = self._get_module_name(name) + return ResolvedDependency(module_name) if module_name else None + + name = req.name + module_name = self._get_module_name(name) + if not module_name: + return None + if req.url: + return self._parse_direct_reference(name, req.url) + + version = str(req.specifier).strip() + if version.startswith("=="): + return ResolvedDependency( + module_name, + version=self._to_manifest_version(version[2:].strip()), + ) + + # Anything else (ranges, unpinned...) carries no usable version. + return ResolvedDependency(module_name) + + def _parse_direct_reference(self, name, url): + """Return the `ResolvedDependency` of a module pinned on a repository.""" + module_name = self._get_module_name(name) + if not module_name: + return None + url, __, fragment = url.partition("#") + subdirectory = self._get_subdirectory(fragment) + clone_url, ref = self._split_vcs_url(url) + return ResolvedDependency( + module_name, + clone_url=clone_url, + ref=ref, + subdirectory=subdirectory, + ) + + def _get_module_name(self, distribution_name): + """Return the technical name of the module packaged as `distribution_name`. + + Returns nothing when the distribution is not an Odoo addon. + """ + if not ADDON_PREFIX_RE.match(distribution_name): + return None + # Odoo module names never hold a dash, so undoing the normalization + # performed by the packaging is unambiguous + return ADDON_PREFIX_RE.sub("", distribution_name).replace("-", "_") + + @staticmethod + def _get_subdirectory(fragment): + """Return the 'subdirectory' held by the fragment of a requirement URL.""" + values = parse_qs(fragment).get("subdirectory") + return values[0] if values else None + + @staticmethod + def _split_vcs_url(url): + """Return the `(clone_url, ref)` parts of a VCS requirement URL. + + The revision is split off the path rather than off the whole URL, so + that credentials held by the network location ('https://user@host/...') + are not mistaken for one. + """ + url = re.sub(r"^\w+\+", "", url) + parts = urlparse(url) + path, __, ref = parts.path.rpartition("@") + if not path: + # No revision pinned + return (url, None) + return (urlunparse(parts._replace(path=path)), ref) + + @staticmethod + def _to_manifest_version(version): + """Return the manifest version a distribution version was built from.""" + # Drop the PEP 440 local and post segments added by the packaging + version = re.split(r"[+!]", version)[0] + segments = version.split(".") + if len(segments) > MANIFEST_VERSION_SEGMENTS: + return ".".join(segments[:MANIFEST_VERSION_SEGMENTS]) + return version diff --git a/odoo_project_dependency_resolver/data/queue_job.xml b/odoo_project_dependency_resolver/data/queue_job.xml new file mode 100644 index 00000000..4bca4cd8 --- /dev/null +++ b/odoo_project_dependency_resolver/data/queue_job.xml @@ -0,0 +1,24 @@ + + + + + + _resolve_dependencies_job + + + + + + + _analyse_pinned_revision + + + + diff --git a/odoo_project_dependency_resolver/lib/__init__.py b/odoo_project_dependency_resolver/lib/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/odoo_project_dependency_resolver/lib/scanner.py b/odoo_project_dependency_resolver/lib/scanner.py new file mode 100644 index 00000000..48c4a309 --- /dev/null +++ b/odoo_project_dependency_resolver/lib/scanner.py @@ -0,0 +1,106 @@ +# Copyright 2026 ACSONE SA/NV () +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl) + +import logging +import pathlib +import tarfile +import tempfile + +import git +from odoo_addons_parser import ModuleParser + +from odoo.addons.odoo_repository.lib.scanner import BaseScanner + +_logger = logging.getLogger(__name__) + + +class PinnedModuleScanner(BaseScanner): + """Analyse a module at the revision a project is pinned on. + + A pinned revision is the head of a pull request more often than of a + branch, so the standard scan of a repository, which walks branches, never + reaches it. Neither does it reach a module the pull request adds, which + exists in no branch at all. + + Nothing is cloned: only the revisions a project pins are fetched, without + the history leading to them nor the branches around them. What a full + clone would bring is of no use here, a module being read from a single + commit. + """ + + # The revisions are kept out of the way of the clones the repository + # scanner maintains: shallow and branchless, they would be of no use to a + # repository scan, and the repository holding them can be one it scans. + _revisions_dirname = "pinned-revisions" + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.path = self.repositories_path.joinpath( + self._revisions_dirname, self.org, self.clone_name or self.name + ) + + def scan_module_at_revision(self, module_path, revision): + """Return the analysis of `module_path` at `revision`, `None` if absent.""" + self._apply_git_global_config() + self.path.mkdir(parents=True, exist_ok=True) + repo = git.Repo.init(self.path) + try: + self._apply_git_config(repo) + self._set_git_remote_url(repo, "origin", self.clone_url) + self._fetch_revision(repo, revision) + with tempfile.TemporaryDirectory() as tmp_dir: + module_dir = self._extract_module(repo, revision, module_path, tmp_dir) + if module_dir is None: + return None + return ModuleParser(module_dir, scan_models=False).to_dict() + finally: + repo.close() + + def _fetch_revision(self, repo, revision): + """Fetch `revision` alone, without the history leading to it. + + Only a revision the repository does not hold yet is fetched: a pinned + one never moves, so it is worth fetching once and never again. The + ones fetched before are kept, as the revisions of a repository share + most of their content. + """ + if self._has_revision(repo, revision): + return + _logger.info("%s: fetch revision %s", self.full_name, revision) + with self._get_git_env() as env, repo.git.custom_environment(**env): + repo.git.fetch("origin", revision, depth=1) + + @staticmethod + def _has_revision(repo, revision): + """Return whether the local clone already holds `revision`.""" + try: + repo.commit(revision) + except (git.exc.BadName, ValueError): + return False + return True + + def _extract_module(self, repo, revision, module_path, target_dir): + """Extract `module_path` at `revision` into `target_dir`. + + Returns the path of the extracted module, `None` when the revision + holds no such module. + + The module is extracted rather than checked out: the working tree of + the clone belongs to the scan jobs, which check branches out in it. + """ + archive = pathlib.Path(target_dir, "module.tar") + try: + repo.git.archive(revision, module_path, output=str(archive)) + except git.exc.GitCommandError: + _logger.warning( + "%s: no %s at revision %s", + self.full_name, + module_path, + revision, + exc_info=True, + ) + return None + with tarfile.open(archive) as tar: + tar.extractall(target_dir, filter="data") + archive.unlink() + return pathlib.Path(target_dir, module_path) diff --git a/odoo_project_dependency_resolver/models/__init__.py b/odoo_project_dependency_resolver/models/__init__.py new file mode 100644 index 00000000..e26962c5 --- /dev/null +++ b/odoo_project_dependency_resolver/models/__init__.py @@ -0,0 +1,4 @@ +from . import odoo_repository +from . import odoo_module_branch +from . import odoo_project +from . import odoo_project_module diff --git a/odoo_project_dependency_resolver/models/odoo_module_branch.py b/odoo_project_dependency_resolver/models/odoo_module_branch.py new file mode 100644 index 00000000..128624d2 --- /dev/null +++ b/odoo_project_dependency_resolver/models/odoo_module_branch.py @@ -0,0 +1,47 @@ +# Copyright 2026 ACSONE SA/NV () +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl) + +from odoo import models + + +class OdooModuleBranch(models.Model): + _inherit = "odoo.module.branch" + + def _update_from_pinned_revision(self, data): + """Fill this module branch with a module analysed at a pinned revision. + + Meant for a module living in a pull request alone: it is in no branch + of any repository, so no scan ever reaches it and nothing else ever + fills it. + """ + self.ensure_one() + values = self._prepare_pinned_revision_values(data) + if values: + self.sudo().write(values) + return True + + def _prepare_pinned_revision_values(self, data): + """Return the values a module analysed at a pinned revision carries. + + Only what the revision itself holds: what its manifest declares, and + what the analysis of its code counted. What a scan knows on top of + that, which repository hosts the module and the history of its + versions, a revision cannot tell. + """ + values = {} + manifest = data.get("manifest") or {} + if manifest: + # The dependencies are looked up from this very module branch: the + # revision knows of no repository branch, and a module living in a + # pull request alone belongs to no repository at all. + values.update( + self._prepare_manifest_values( + manifest, self.branch_id, self.repository_id + ) + ) + code = data.get("code") or {} + if code: + values.update(self._prepare_code_analysis_values(code)) + if values: + values["removed"] = False + return values diff --git a/odoo_project_dependency_resolver/models/odoo_project.py b/odoo_project_dependency_resolver/models/odoo_project.py new file mode 100644 index 00000000..db652b3a --- /dev/null +++ b/odoo_project_dependency_resolver/models/odoo_project.py @@ -0,0 +1,280 @@ +# Copyright 2026 ACSONE SA/NV () +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl) + +import logging + +import git + +from odoo import _, fields, models +from odoo.exceptions import UserError + +from odoo.addons.queue_job.delay import group +from odoo.addons.queue_job.job import identity_exact + +_logger = logging.getLogger(__name__) + + +class OdooProject(models.Model): + _inherit = "odoo.project" + + dependency_management = fields.Selection( + selection=[("manual", "Manual"), ("resolved", "Resolved")], + default="manual", + required=True, + help=( + "Manual: the list of modules is maintained by hand.\n" + "Resolved: it is rebuilt from a dependency file of the repository " + "every time the project is scanned." + ), + ) + dependency_resolver = fields.Selection( + selection=[("requirements_txt", "requirements.txt")], + default="requirements_txt", + help="Format of the dependency file to read.", + ) + dependency_source_path = fields.Char( + default="requirements.txt", + help="Path of the dependency file, relative to the root of the repository.", + ) + app_module_ids = fields.Many2many( + comodel_name="odoo.module.branch", + relation="odoo_project_app_module_rel", + column1="odoo_project_id", + column2="module_branch_id", + string="Additional Modules", + help=( + "Modules installed from somewhere the dependency file does not " + "reach, Odoo Enterprise typically. The modules hosted by the " + "project repository are added on their own, and Odoo standard " + "modules are pulled from the dependency graph: neither of them " + "has to be declared here." + ), + ) + last_resolution_date = fields.Datetime(readonly=True) + resolution_log = fields.Text( + readonly=True, + help="What the last resolution could not do, and what it had to guess.", + ) + + def action_scan(self, force=False): + # Resolved projects are scanned in two passes: + # 1) the project repository is scanned first so the dependency file is read + # from a fresh clone, and + # 2) the dependencies are resolved, then their originating repositories are + # scanned to complete the dependency graph. + # Other projects keep the standard super() flow. + # For resolved projects, we trigger the repository scan directly here because + # the repository override of _create_subsequent_jobs schedules dependency + # resolution as a follow-up job of the repository scan. + resolved = self.filtered(lambda p: p.dependency_management == "resolved") + res = super(OdooProject, self - resolved).action_scan(force=force) + for project in resolved: + if not project.repository_id: + raise UserError( + _("Define the repository of project %s to resolve its modules.") + % project.name + ) + project.repository_id.action_scan(force=force, raise_exc=True) + return res + + def action_resolve_dependencies(self): + """Rebuild the list of modules from the dependency file of the project.""" + for project in self: + project._resolve_dependencies() + return True + + def _create_job_resolve_dependencies(self, scan_repositories=False): + """Return the job resolving the modules of this project.""" + self.ensure_one() + delayable = self.delayable( + description=f"Resolve the modules of {self.display_name}", + identity_key=identity_exact, + ) + return delayable._resolve_dependencies_job(scan_repositories=scan_repositories) + + def _resolve_dependencies_job(self, scan_repositories=False): + """Resolve the project dependencies from a job. + + Record expected resolution errors on the project instead of failing the job. + """ + self.ensure_one() + try: + self._resolve_dependencies() + except UserError as exc: + _logger.warning( + "Cannot resolve the modules of %s: %s", self.display_name, exc + ) + self.sudo().write( + { + "last_resolution_date": fields.Datetime.now(), + "resolution_log": str(exc), + } + ) + return True + if scan_repositories: + self._scan_resolved_repositories() + return True + + def _scan_resolved_repositories(self): + """ + Second pass: scan the repositories discovered from the resolved dependencies. + """ + self.ensure_one() + repositories = self._get_repositories_to_scan() - self.repository_id + branches = self._get_branches_to_scan() + if not repositories or not branches: + return False + # NOTE: the scan stays incremental whether the first pass was forced or + # not. Forcing the scan of a project is about the project, not about + # re-collecting the whole history of every repository it depends on. + scan_jobs = repositories.with_context( + strict_branches_scan=True + )._create_scan_jobs(branch_ids=branches.ids, raise_exc=False) + if not scan_jobs: + return False + group(*scan_jobs).on_done(self._create_job_resolve_dependencies()).delay() + return True + + def _resolve_dependencies(self): + """Refresh the modules of this project from its dependency file.""" + self.ensure_one() + content = self._read_dependency_source() + log = [] + backend = self.env.ref("odoo_repository.mca_backend") + usage = f"project.dependency.resolver.{self.dependency_resolver}" + with backend.work_on(self._name) as work: + dependencies = work.component(usage=usage).resolve(content) + project_modules = self.env["odoo.project.module"] + for dependency in dependencies: + project_modules |= self._apply_resolved_dependency(dependency, log) + project_modules |= self._resolve_undeclared_modules(log) + project_modules |= self._resolve_missing_dependencies(project_modules, log) + self._remove_unresolved_modules(project_modules, log) + self.sudo().write( + { + "last_resolution_date": fields.Datetime.now(), + "resolution_log": "\n".join(log), + } + ) + return project_modules + + def _read_dependency_source(self): + """Return the content of the dependency file of this project.""" + self.ensure_one() + if not self.repository_id: + raise UserError( + _("Define the repository of project %s to resolve its modules.") + % self.name + ) + clone_path = self.repository_id._get_local_clone_path() + if not clone_path.joinpath(".git").exists(): + raise UserError( + _("Repository %s has not been cloned yet, scan it first.") + % self.repository_id.display_name + ) + branch = self.repository_branch_id.cloned_branch or self.odoo_version_id.name + revision = f"remotes/origin/{branch}:{self.dependency_source_path}" + # NOTE: read the file straight from the object database. Checking the + # branch out would fight with the scan jobs, which check branches out + # in this very clone. + try: + with git.Repo(clone_path) as repo: + return repo.git.show(revision) + except git.exc.GitError as exc: + raise UserError( + _("Cannot read %(revision)s in %(repository)s: %(error)s") + % { + "revision": revision, + "repository": self.repository_id.display_name, + "error": exc, + } + ) from exc + + def _apply_resolved_dependency(self, dependency, log): + """Turn a resolved dependency into a module of this project.""" + self.ensure_one() + module = self.env["odoo.module.branch"]._get_module(dependency.module_name) + if module.blacklisted: + return self.env["odoo.project.module"] + source, upstream = self._get_dependency_repositories(dependency, log) + module_branch = self._get_module_branch(module, repository=upstream) + project_module = self._get_project_module(module_branch, dependency.version) + project_module.sudo().write( + { + "source_repository_id": source.id, + "source_clone_url": dependency.clone_url, + "source_ref": dependency.ref, + "source_subdirectory": dependency.subdirectory, + } + ) + # A pinned revision is read from its own repository, which has to be + # cloned first: too long to be done here, one dependency among + # hundreds, and worth retrying on its own when the network fails. + if dependency.is_pinned and project_module.analysed_ref != dependency.ref: + project_module._create_job_analyse_pinned_revision().delay() + return project_module + + def _get_dependency_repositories(self, dependency, log): + """Return the source and upstream repositories of a resolved dependency.""" + self.ensure_one() + empty = self.env["odoo.repository"] + if not dependency.clone_url: + return (empty, None) + repository = empty._find_from_clone_url(dependency.clone_url) + if not repository: + log.append( + _("%(module)s: unknown origin for %(url)s") + % {"module": dependency.module_name, "url": dependency.clone_url} + ) + return (empty, None) + return (repository, repository.upstream_repository_id or repository) + + def _resolve_undeclared_modules(self, log): + """ + Return modules not declared in the dependency file but belonging to the project. + """ + self.ensure_one() + hosted_modules = self.repository_branch_id.module_ids + resolved = self.env["odoo.project.module"] + for module_branch in hosted_modules | self.app_module_ids: + resolved |= self._get_project_module(module_branch, module_branch.version) + if hosted_modules: + log.append( + _("Added %s module(s) hosted by the project repository.") + % len(hosted_modules) + ) + return resolved + + def _resolve_missing_dependencies(self, project_modules, log): + """Return the missing dependencies of the resolved modules. + + Odoo standard modules are shipped with Odoo itself, so they show up in + no dependency file: they are pulled from the dependency graph. + """ + self.ensure_one() + module_branches = project_modules.module_branch_id + missing = module_branches._get_recursive_dependencies() - module_branches + resolved = self.env["odoo.project.module"] + for module_branch in missing: + resolved |= self._get_project_module(module_branch, module_branch.version) + if missing: + log.append( + _("Added %s module(s) pulled from the dependency graph.") % len(missing) + ) + return resolved + + def _remove_unresolved_modules(self, project_modules, log): + """Drop the modules of this project that are no longer declared.""" + self.ensure_one() + outdated = self.project_module_ids - project_modules + if not outdated: + return self.env["odoo.project.module"] + log.append( + _("Removed %(count)s module(s) no longer declared: %(modules)s") + % { + "count": len(outdated), + "modules": ", ".join(sorted(outdated.mapped("module_name"))), + } + ) + outdated.sudo().unlink() + return outdated diff --git a/odoo_project_dependency_resolver/models/odoo_project_module.py b/odoo_project_dependency_resolver/models/odoo_project_module.py new file mode 100644 index 00000000..edae4b22 --- /dev/null +++ b/odoo_project_dependency_resolver/models/odoo_project_module.py @@ -0,0 +1,143 @@ +# Copyright 2026 ACSONE SA/NV () +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl) + +import logging + +import git + +from odoo import _, api, fields, models + +from odoo.addons.queue_job.exception import RetryableJobError +from odoo.addons.queue_job.job import identity_exact + +from ..lib.scanner import PinnedModuleScanner + +_logger = logging.getLogger(__name__) + + +class OdooProjectModule(models.Model): + _inherit = "odoo.project.module" + + source_repository_id = fields.Many2one( + comodel_name="odoo.repository", + string="Installed From", + help=( + "Fork the module is installed from, when it differs from the " + "repository it belongs to. Its own 'Fork Of' tells where the " + "module comes from, which is the only way to reach it for a module " + "living in the fork alone." + ), + ) + source_clone_url = fields.Char( + string="Source URL", + help=( + "Repository the module is installed from, as the dependency file " + "spells it. Kept even when it matches no known repository, which " + "is then the only trace of where the module comes from." + ), + ) + source_ref = fields.Char( + string="Source Revision", + help="Revision the module is frozen on: a commit, a tag or a PR ref.", + ) + source_subdirectory = fields.Char( + help="Path of the module within its source repository.", + ) + is_pinned = fields.Boolean( + compute="_compute_is_pinned", + store=True, + help="The module is frozen on a revision instead of a published version.", + ) + analysed_ref = fields.Char( + string="Analysed Revision", + readonly=True, + help=( + "Revision the code of this module was last analysed at. A pinned " + "revision never moves, so it is worth analysing once and never " + "again." + ), + ) + + @api.depends("source_clone_url", "source_ref") + def _compute_is_pinned(self): + for rec in self: + rec.is_pinned = bool(rec.source_clone_url and rec.source_ref) + + @api.depends("version", "installed_version", "is_pinned") + def _compute_to_upgrade(self): + # Without an installed version the base computation falls back on the + # upstream one, which reads as up to date. A module pinned on a + # revision whose version could not be read is precisely the opposite: + # nothing is known about it. + res = super()._compute_to_upgrade() + for rec in self: + if rec.is_pinned and not rec.installed_version: + rec.to_upgrade = True + return res + + def action_analyse_pinned_revision(self): + """Analyse the code of the pinned modules of this selection.""" + for rec in self.filtered("is_pinned"): + rec._create_job_analyse_pinned_revision().delay() + return True + + def _create_job_analyse_pinned_revision(self): + """Return the job analysing this module at its pinned revision.""" + self.ensure_one() + delayable = self.delayable( + description=( + f"Analyse {self.module_name} at {self.source_ref} " + f"in {self.source_repository_id.display_name}" + ), + identity_key=identity_exact, + ) + return delayable._analyse_pinned_revision() + + def _analyse_pinned_revision(self): + """Analyse the code of this module at the revision it is pinned on. + + A pinned revision is the head of a pull request more often than of a + branch, so no repository scan ever reaches it: neither the version of + the module nor the analysis of its code can come from anywhere else. + """ + self.ensure_one() + repository = self.source_repository_id + if not repository or not self.source_ref: + return False + module_branch = self.module_branch_id + scanner = PinnedModuleScanner( + branches=[], **repository._prepare_base_scanner_parameters() + ) + try: + data = scanner.scan_module_at_revision( + module_branch.full_path, self.source_ref + ) + except git.exc.GitError as exc: + raise RetryableJobError( + _("Cannot read %(module)s at %(ref)s in %(repository)s") + % { + "module": module_branch.full_path, + "ref": self.source_ref, + "repository": repository.display_name, + } + ) from exc + if data is None: + _logger.warning( + "%s: %s holds no %s at %s", + self.display_name, + repository.display_name, + module_branch.full_path, + self.source_ref, + ) + return False + values = {"analysed_ref": self.source_ref} + version = (data.get("manifest") or {}).get("version") + if version: + values["installed_version"] = version + self.sudo().write(values) + # The module branch is shared with the repository the modules belong + # to, whose scan is authoritative on it. Only one no scan ever reaches, + # living in a pull request alone, takes its data from here. + if not module_branch.last_scanned_commit: + module_branch._update_from_pinned_revision(data) + return True diff --git a/odoo_project_dependency_resolver/models/odoo_repository.py b/odoo_project_dependency_resolver/models/odoo_repository.py new file mode 100644 index 00000000..5f2a16d7 --- /dev/null +++ b/odoo_project_dependency_resolver/models/odoo_repository.py @@ -0,0 +1,40 @@ +# Copyright 2026 ACSONE SA/NV () +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl) + +from odoo import models + + +class OdooRepository(models.Model): + _inherit = "odoo.repository" + + def _create_subsequent_jobs( + self, version_branch, next_versions_branches, all_versions_branches, data + ): + # Appending the resolution to the chain of the scan jobs is what makes + # it read an up to date dependency file: the branch has just been + # fetched by the job detecting the modules to scan. It also runs after + # the modules have been scanned, so the dependency graph it walks to + # complete the application modules is the fresh one. + jobs = super()._create_subsequent_jobs( + version_branch, next_versions_branches, all_versions_branches, data + ) + if not data: + # The branch does not exist, there is nothing to read + return jobs + version, __ = version_branch + for project in self._get_projects_to_resolve(version): + jobs.append( + project._create_job_resolve_dependencies(scan_repositories=True) + ) + return jobs + + def _get_projects_to_resolve(self, version): + """Return the projects whose dependency file this repository holds.""" + self.ensure_one() + return self.env["odoo.project"].search( + [ + ("dependency_management", "=", "resolved"), + ("odoo_version_id.name", "=", version), + ("repository_id", "=", self.id), + ] + ) diff --git a/odoo_project_dependency_resolver/pyproject.toml b/odoo_project_dependency_resolver/pyproject.toml new file mode 100644 index 00000000..4231d0cc --- /dev/null +++ b/odoo_project_dependency_resolver/pyproject.toml @@ -0,0 +1,3 @@ +[build-system] +requires = ["whool"] +build-backend = "whool.buildapi" diff --git a/odoo_project_dependency_resolver/readme/CONTRIBUTORS.md b/odoo_project_dependency_resolver/readme/CONTRIBUTORS.md new file mode 100644 index 00000000..c4904374 --- /dev/null +++ b/odoo_project_dependency_resolver/readme/CONTRIBUTORS.md @@ -0,0 +1,2 @@ +- ACSONE SA/NV + - Laurent Mignon \ diff --git a/odoo_project_dependency_resolver/readme/DESCRIPTION.md b/odoo_project_dependency_resolver/readme/DESCRIPTION.md new file mode 100644 index 00000000..f76f710b --- /dev/null +++ b/odoo_project_dependency_resolver/readme/DESCRIPTION.md @@ -0,0 +1,7 @@ +Projects usually pin the exact version of every module they install in a +dependency file, such as the `requirements.txt` produced by pip. Maintaining +the very same list a second time by hand in Odoo MCA is tedious and drifts +away from reality as soon as a dependency is bumped. + +This module reads that file straight from the repository of the project and +rebuilds its modules from it, every time the project is scanned. diff --git a/odoo_project_dependency_resolver/readme/USAGE.md b/odoo_project_dependency_resolver/readme/USAGE.md new file mode 100644 index 00000000..838a0e5d --- /dev/null +++ b/odoo_project_dependency_resolver/readme/USAGE.md @@ -0,0 +1,55 @@ +On the project, set *Dependency Management* to *Resolved*, pick the format of +the dependency file and, if it is not at the root of the repository under its +usual name, adjust its path. + +There is nothing else to declare. A dependency file only holds packaged +addons, so the modules of a resolved project are the union of three sources: + +- the addons of the dependency file, with the versions it pins; +- the modules hosted by the project repository, which are the code of the + project itself and are shipped with it rather than installed as a package; +- their missing dependencies, pulled from the dependency graph, which is + where Odoo standard modules come from. + +*Additional Modules* is there for what none of the three reaches, Odoo +Enterprise modules typically. + +The modules are refreshed at each scan, and by the *Resolve Dependencies* +button. Whatever the resolution could not do is reported on the project, in +the *Resolution Log* field. + +## What a scan does + +Scanning a resolved project only scans the repository of the project: the +repositories its modules come from are derived from the dependency file, so +none of them is known before that file has been read. + +The resolution is chained to that scan, so it reads a file that has just been +fetched, and it then spawns the scan of the repositories it discovered. A +second resolution runs once they have all been scanned, completing what the +first one could not know: the modules of a repository that had never been +scanned, and the dependency graph the resolved modules are walked through. + +Note that the second resolution waits for the detection of the modules to +scan in every repository, not for every module to have been scanned: the +scanner spawns the latter as it goes. A module scanned late is therefore +accounted for by the next scan of the project. + +## Modules pinned on a fork + +A module frozen on a commit of a fork, such as: + + odoo-addon-account-invoice-triple-discount @ + git+https://github.com/acsone/account-invoicing.git@3e4c54b + #subdirectory=setup/account_invoice_triple_discount + +is not treated as a new module. It stays attached to the module of the +repository the fork originates from, so that dependencies, migration scripts +and changelogs keep working, and the fork is recorded on the project module as +*Installed From*. Its version is read from the manifest at the pinned revision. + +A module added by a pull request not merged yet is a special case: it exists +in no repository known to Odoo MCA, so it stays an orphaned module and shows +up among the unknown modules of the project. *Installed From* is then the only +way to reach a repository from it, and the *Fork Of* of that fork tells where +the module is heading. diff --git a/odoo_project_dependency_resolver/static/description/index.html b/odoo_project_dependency_resolver/static/description/index.html new file mode 100644 index 00000000..ca8b92bc --- /dev/null +++ b/odoo_project_dependency_resolver/static/description/index.html @@ -0,0 +1,493 @@ + + + + + +Odoo Project - Dependency Resolver + + + +
+

Odoo Project - Dependency Resolver

+ + +

Beta License: AGPL-3 OCA/module-composition-analysis Translate me on Weblate Try me on Runboat

+

Projects usually pin the exact version of every module they install in a +dependency file, such as the requirements.txt produced by pip. +Maintaining the very same list a second time by hand in Odoo MCA is +tedious and drifts away from reality as soon as a dependency is bumped.

+

This module reads that file straight from the repository of the project +and rebuilds its modules from it, every time the project is scanned.

+

Table of contents

+ +
+

Usage

+

On the project, set Dependency Management to Resolved, pick the +format of the dependency file and, if it is not at the root of the +repository under its usual name, adjust its path.

+

There is nothing else to declare. A dependency file only holds packaged +addons, so the modules of a resolved project are the union of three +sources:

+
    +
  • the addons of the dependency file, with the versions it pins;
  • +
  • the modules hosted by the project repository, which are the code of +the project itself and are shipped with it rather than installed as a +package;
  • +
  • their missing dependencies, pulled from the dependency graph, which is +where Odoo standard modules come from.
  • +
+

Additional Modules is there for what none of the three reaches, Odoo +Enterprise modules typically.

+

The modules are refreshed at each scan, and by the Resolve +Dependencies button. Whatever the resolution could not do is reported +on the project, in the Resolution Log field.

+
+

What a scan does

+

Scanning a resolved project only scans the repository of the project: +the repositories its modules come from are derived from the dependency +file, so none of them is known before that file has been read.

+

The resolution is chained to that scan, so it reads a file that has just +been fetched, and it then spawns the scan of the repositories it +discovered. A second resolution runs once they have all been scanned, +completing what the first one could not know: the modules of a +repository that had never been scanned, and the dependency graph the +resolved modules are walked through.

+

Note that the second resolution waits for the detection of the modules +to scan in every repository, not for every module to have been scanned: +the scanner spawns the latter as it goes. A module scanned late is +therefore accounted for by the next scan of the project.

+
+
+

Modules pinned on a fork

+

A module frozen on a commit of a fork, such as:

+
+odoo-addon-account-invoice-triple-discount @
+    git+https://github.com/acsone/account-invoicing.git@3e4c54b
+    #subdirectory=setup/account_invoice_triple_discount
+
+

is not treated as a new module. It stays attached to the module of the +repository the fork originates from, so that dependencies, migration +scripts and changelogs keep working, and the fork is recorded on the +project module as Installed From. Its version is read from the +manifest at the pinned revision.

+

A module added by a pull request not merged yet is a special case: it +exists in no repository known to Odoo MCA, so it stays an orphaned +module and shows up among the unknown modules of the project. Installed +From is then the only way to reach a repository from it, and the Fork +Of of that fork tells where the module is heading.

+
+
+
+

Bug Tracker

+

Bugs are tracked on GitHub Issues. +In case of trouble, please check there if your issue has already been reported. +If you spotted it first, help us to smash it by providing a detailed and welcomed +feedback.

+

Do not contact contributors directly about support or help with technical issues.

+
+
+

Credits

+
+

Authors

+
    +
  • ACSONE SA/NV
  • +
+
+
+

Contributors

+ +
+
+

Maintainers

+

This module is maintained by the OCA.

+ +Odoo Community Association + +

OCA, or the Odoo Community Association, is a nonprofit organization whose +mission is to support the collaborative development of Odoo features and +promote its widespread use.

+

This module is part of the OCA/module-composition-analysis project on GitHub.

+

You are welcome to contribute. To learn how please visit https://odoo-community.org/page/Contribute.

+
+
+
+ + diff --git a/odoo_project_dependency_resolver/tests/__init__.py b/odoo_project_dependency_resolver/tests/__init__.py new file mode 100644 index 00000000..bbc6a685 --- /dev/null +++ b/odoo_project_dependency_resolver/tests/__init__.py @@ -0,0 +1,2 @@ +from . import test_requirements_txt_resolver +from . import test_resolve_dependencies diff --git a/odoo_project_dependency_resolver/tests/test_requirements_txt_resolver.py b/odoo_project_dependency_resolver/tests/test_requirements_txt_resolver.py new file mode 100644 index 00000000..8afdf2c2 --- /dev/null +++ b/odoo_project_dependency_resolver/tests/test_requirements_txt_resolver.py @@ -0,0 +1,163 @@ +# Copyright 2026 ACSONE SA/NV () +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl) + +from odoo.addons.component.tests.common import TransactionComponentCase + +# Excerpt of a 'requirements.txt' frozen by pip-deepfreeze +REQUIREMENTS = """\ +# frozen requirements generated by pip-deepfreeze +aiohttp==3.14.3 +click-odoo==1.8.0 +gitpython==3.1.58 +odoo @ git+https://github.com/acsone/odoo.git@8cc9bd3c893dfd8b43b4f1af287e5e9f +odoo-addon-base-time-window==18.0.1.1.1 +odoo-addon-component==18.0.1.0.5.1 +odoo-addon-web-responsive==18.0.1.0.7 +odoo-addons-parser==0.8 +""" + + +class TestRequirementsTxtResolver(TransactionComponentCase): + @classmethod + def setUpClass(cls): + super().setUpClass() + cls.backend = cls.env.ref("odoo_repository.mca_backend") + + def _resolve(self, content): + with self.backend.work_on("odoo.project") as work: + resolver = work.component( + usage="project.dependency.resolver.requirements_txt" + ) + return resolver.resolve(content) + + def _resolve_one(self, requirement): + dependencies = self._resolve(requirement) + self.assertEqual(len(dependencies), 1, requirement) + return dependencies[0] + + def test_resolve_keeps_addons_only(self): + """Only the addons of a requirements file are resolved.""" + dependencies = self._resolve(REQUIREMENTS) + self.assertEqual( + [dependency.module_name for dependency in dependencies], + ["base_time_window", "component", "web_responsive"], + ) + + def test_resolve_odoo_itself_is_not_an_addon(self): + """The 'odoo' requirement pins the core, not a module.""" + self.assertFalse( + self._resolve("odoo @ git+https://github.com/odoo/odoo.git@a1") + ) + + def test_resolve_lookalike_distribution(self): + """'odoo-addons-parser' is a library, not an addon.""" + self.assertFalse(self._resolve("odoo-addons-parser==0.8")) + + def test_resolve_version(self): + """A pinned version is the version of the module.""" + dependency = self._resolve_one("odoo-addon-web-responsive==18.0.1.0.7") + self.assertEqual(dependency.version, "18.0.1.0.7") + self.assertFalse(dependency.is_pinned) + + def test_resolve_version_drops_the_packaging_serial(self): + """A serial appended by the packaging is not part of the manifest version.""" + dependency = self._resolve_one("odoo-addon-odoo-repository==18.0.1.2.0.3") + self.assertEqual(dependency.module_name, "odoo_repository") + self.assertEqual(dependency.version, "18.0.1.2.0") + + def test_resolve_version_drops_the_local_segment(self): + dependency = self._resolve_one("odoo-addon-component==18.0.1.0.5+local") + self.assertEqual(dependency.version, "18.0.1.0.5") + + def test_resolve_unpinned_requirement(self): + """A requirement without an exact version yields no version.""" + dependency = self._resolve_one("odoo-addon-component>=18.0.1.0.5") + self.assertEqual(dependency.module_name, "component") + self.assertFalse(dependency.version) + + def test_resolve_legacy_prefix(self): + """Addons were distributed as 'odoo-addon-' before 15.0.""" + dependency = self._resolve_one("odoo14-addon-web-responsive==14.0.1.0.1") + self.assertEqual(dependency.module_name, "web_responsive") + + def test_resolve_pinned_on_a_fork(self): + """A fork commit is recorded as the origin of the module.""" + dependency = self._resolve_one( + "odoo-addon-account-invoice-triple-discount @ " + "git+https://github.com/acsone/account-invoicing.git" + "@3e4c54bb1a3a34b4b031b3dec9ccf6b2997d1295" + "#subdirectory=setup/account_invoice_triple_discount" + ) + self.assertEqual(dependency.module_name, "account_invoice_triple_discount") + self.assertEqual( + dependency.clone_url, "https://github.com/acsone/account-invoicing.git" + ) + self.assertEqual(dependency.ref, "3e4c54bb1a3a34b4b031b3dec9ccf6b2997d1295") + self.assertEqual( + dependency.subdirectory, "setup/account_invoice_triple_discount" + ) + self.assertTrue(dependency.is_pinned) + # A revision carries no version + self.assertFalse(dependency.version) + + def test_resolve_pinned_on_a_pull_request(self): + """A PR ref holds slashes, which must not be mistaken for a path.""" + dependency = self._resolve_one( + "odoo-addon-account-invoice-triple-discount @ " + "git+https://github.com/OCA/account-invoicing.git" + "@refs/pull/2279/head" + "#subdirectory=setup/account_invoice_triple_discount" + ) + self.assertEqual( + dependency.clone_url, "https://github.com/OCA/account-invoicing.git" + ) + self.assertEqual(dependency.ref, "refs/pull/2279/head") + + def test_resolve_subdirectory_wins_over_the_distribution_name(self): + """The subdirectory names the module, the distribution name is normalized.""" + dependency = self._resolve_one( + "odoo-addon-l10n-be-coa-multilang @ " + "git+https://github.com/OCA/l10n-belgium.git@18.0" + "#subdirectory=setup/l10n_be_coa_multilang" + ) + self.assertEqual(dependency.module_name, "l10n_be_coa_multilang") + + def test_resolve_url_with_credentials(self): + """Credentials in the URL are not mistaken for a revision separator.""" + dependency = self._resolve_one( + "odoo-addon-component @ " + "git+https://user@example.com/acsone/addons.git@18.0" + "#subdirectory=setup/component" + ) + self.assertEqual( + dependency.clone_url, "https://user@example.com/acsone/addons.git" + ) + self.assertEqual(dependency.ref, "18.0") + + def test_resolve_ignores_comments_and_options(self): + content = ( + "# a comment\n" + "-r base.txt\n" + "--index-url https://example.com/simple\n" + "odoo-addon-component==18.0.1.0.5 # trailing comment\n" + ) + dependency = self._resolve_one(content) + self.assertEqual(dependency.module_name, "component") + self.assertEqual(dependency.version, "18.0.1.0.5") + + def test_resolve_ignores_environment_markers(self): + dependency = self._resolve_one( + 'odoo-addon-component==18.0.1.0.5 ; python_version >= "3.10"' + ) + self.assertEqual(dependency.version, "18.0.1.0.5") + + def test_resolve_joins_continuation_lines(self): + content = ( + "odoo-addon-component==18.0.1.0.5 \\\n" + " --hash=sha256:0000000000000000000000000000000000000000\n" + ) + dependency = self._resolve_one(content) + self.assertEqual(dependency.version, "18.0.1.0.5") + + def test_resolve_empty_content(self): + self.assertEqual(self._resolve(""), []) diff --git a/odoo_project_dependency_resolver/tests/test_resolve_dependencies.py b/odoo_project_dependency_resolver/tests/test_resolve_dependencies.py new file mode 100644 index 00000000..64273b92 --- /dev/null +++ b/odoo_project_dependency_resolver/tests/test_resolve_dependencies.py @@ -0,0 +1,576 @@ +# Copyright 2026 ACSONE SA/NV () +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl) + +import contextlib +import gc +import pathlib +import tempfile +from unittest.mock import patch + +import git + +from odoo.exceptions import UserError +from odoo.tools import mute_logger + +from odoo.addons.odoo_project.tests.common import ProjectCommon + +READ_SOURCE = ( + "odoo.addons.odoo_project_dependency_resolver.models.odoo_project." + "OdooProject._read_dependency_source" +) + + +class TestResolveDependencies(ProjectCommon): + @classmethod + def setUpClass(cls): + super().setUpClass() + cls.project.write( + { + "repository_id": cls.odoo_repository.id, + "dependency_management": "resolved", + } + ) + cls.project_module_model = cls.env["odoo.project.module"] + + def _resolve(self, content): + with patch(READ_SOURCE, return_value=content): + self.project._resolve_dependencies() + self.env.flush_all() + return self.project.project_module_ids + + def _module_versions(self): + return { + project_module.module_name: project_module.installed_version + for project_module in self.project.project_module_ids + } + + # -- mapping onto project modules -------------------------------------- + + def test_resolve_creates_project_modules(self): + modules = self._resolve( + "odoo-addon-test1==15.0.1.0.0\nodoo-addon-test2==15.0.2.0.0\n" + ) + self.assertEqual(len(modules), 2) + self.assertEqual( + self._module_versions(), + {"test1": "15.0.1.0.0", "test2": "15.0.2.0.0"}, + ) + + def test_resolve_updates_the_installed_version(self): + """Resolving again bumps the version instead of duplicating the module.""" + self._resolve("odoo-addon-test1==15.0.1.0.0\n") + modules = self._resolve("odoo-addon-test1==15.0.1.1.0\n") + self.assertEqual(len(modules), 1) + self.assertEqual(self._module_versions(), {"test1": "15.0.1.1.0"}) + + def test_resolve_removes_undeclared_modules(self): + """A module dropped from the dependency file leaves the project.""" + self._resolve("odoo-addon-test1==15.0.1.0.0\nodoo-addon-test2==15.0.2.0.0\n") + self._resolve("odoo-addon-test1==15.0.1.0.0\n") + self.assertEqual(list(self._module_versions()), ["test1"]) + self.assertIn("test2", self.project.resolution_log) + + def test_resolve_skips_blacklisted_modules(self): + module = self.module_branch_model._get_module("test1") + module.blacklisted = True + self._resolve("odoo-addon-test1==15.0.1.0.0\nodoo-addon-test2==15.0.2.0.0\n") + self.assertEqual(list(self._module_versions()), ["test2"]) + + def test_resolve_pulls_the_missing_dependencies(self): + """Odoo standard modules show up in no dependency file.""" + dependency = self._create_odoo_module_branch( + self.module_branch_model._get_module("base"), + self.branch, + is_standard=True, + ) + app_module = self._create_odoo_module_branch( + self.module_branch_model._get_module("my_app"), + self.branch, + dependency_ids=[(4, dependency.id)], + ) + self.project.app_module_ids = app_module + self._resolve("odoo-addon-test1==15.0.1.0.0\n") + self.assertEqual(sorted(self._module_versions()), ["base", "my_app", "test1"]) + + def test_resolve_keeps_the_additional_modules(self): + """An additional module is never purged, the file does not hold it.""" + app_module = self._create_odoo_module_branch( + self.module_branch_model._get_module("my_app"), self.branch + ) + self.project.app_module_ids = app_module + self._resolve("odoo-addon-test1==15.0.1.0.0\n") + self.assertIn("my_app", self._module_versions()) + + # -- modules hosted by the project repository --------------------------- + + def _create_specific_module(self, name="my_specific_module"): + """Add a module to the repository of the project.""" + repository_branch = self.env["odoo.repository.branch"].search( + [ + ("repository_id", "=", self.odoo_repository.id), + ("branch_id", "=", self.branch.id), + ] + ) + if not repository_branch: + repository_branch = self._create_odoo_repository_branch( + self.odoo_repository, self.branch + ) + return self._create_odoo_module_branch( + self.module_branch_model._get_module(name), + self.branch, + specific=True, + repository_branch_id=repository_branch.id, + version="15.0.1.0.0", + ) + + def test_resolve_adds_the_modules_of_the_project_repository(self): + """The code of the project is shipped with it, not as a package.""" + specific_module = self._create_specific_module() + modules = self._resolve("odoo-addon-test1==15.0.1.0.0\n") + self.assertEqual( + sorted(self._module_versions()), ["my_specific_module", "test1"] + ) + # Their version is the one of the repository, nothing pins them + project_module = modules.filtered( + lambda m: m.module_branch_id == specific_module + ) + self.assertEqual(project_module.installed_version, "15.0.1.0.0") + self.assertFalse(project_module.to_upgrade) + + def test_resolve_does_not_purge_the_modules_of_the_project_repository(self): + """Resolving twice must not drop the code of the project.""" + self._create_specific_module() + self._resolve("odoo-addon-test1==15.0.1.0.0\n") + self._resolve("odoo-addon-test1==15.0.1.0.0\n") + self.assertIn("my_specific_module", self._module_versions()) + + # -- modules pinned on a fork ------------------------------------------ + + def _create_upstream_repository(self, with_module=True): + org_model = self.env["odoo.repository.org"].with_context(active_test=False) + org = org_model.search([("name", "=", "OCA")]) + if not org: + org = org_model.create({"name": "OCA"}) + repository = self.env["odoo.repository"].create( + { + "org_id": org.id, + "name": "account-invoicing", + "repo_url": "https://github.com/OCA/account-invoicing", + "clone_url": "https://github.com/OCA/account-invoicing.git", + "repo_type": "github", + } + ) + repository_branch = self._create_odoo_repository_branch(repository, self.branch) + module_branch = self.module_branch_model.browse() + if with_module: + module_branch = self._create_odoo_module_branch( + self.module_branch_model._get_module("account_invoice_triple_discount"), + self.branch, + specific=False, + repository_branch_id=repository_branch.id, + version="15.0.1.0.0", + # The scan of the upstream repository is authoritative on it + last_scanned_commit="c0ffee", + ) + return repository, module_branch + + def _create_fork(self, module_name, version, on_pull_request=False): + """Return the ``(path, revision)`` of a fork holding `module_name`. + + With `on_pull_request` the module is added on a branch of its own, as + a pull request does: the revision then belongs to no branch the clone + of the fork tracks, and has to be fetched on its own. + """ + fork_path = pathlib.Path(tempfile.mkdtemp()) + fork = git.Repo.init(fork_path) + fork.config_writer().set_value("user", "name", "test").release() + fork.config_writer().set_value("user", "email", "test@example.com").release() + # Serving a bare revision is what GitHub does, but not the local default + fork.config_writer().set_value( + "uploadpack", "allowAnySHA1InWant", "true" + ).release() + self.addCleanup(fork.close) + readme = fork_path.joinpath("README.md") + readme.write_text("A fork\n") + fork.index.add([str(readme)]) + fork.index.commit("Initial commit") + default_branch = fork.active_branch.name + if on_pull_request: + fork.git.checkout("-b", "add-module") + module_path = fork_path.joinpath(module_name) + module_path.mkdir() + manifest = module_path.joinpath("__manifest__.py") + manifest.write_text( + "{" + f'"name": "Test", "version": "{version}", ' + '"license": "AGPL-3", "depends": ["base"]' + "}" + ) + code = module_path.joinpath("models.py") + code.write_text( + "from odoo import fields, models\n" + "\n" + "\n" + "class Foo(models.Model):\n" + ' _name = "foo"\n' + "\n" + " name = fields.Char()\n" + ) + fork.index.add([str(manifest), str(code)]) + commit = fork.index.commit(f"Add {module_name} {version}") + if on_pull_request: + fork.git.checkout(default_branch) + return fork_path, commit.hexsha + + def test_resolve_fork_keeps_the_upstream_module(self): + """A fork commit does not create a module of its own.""" + __, upstream_module = self._create_upstream_repository() + content = ( + "odoo-addon-account-invoice-triple-discount @ " + "git+https://github.com/acsone/account-invoicing.git@3e4c54b" + "#subdirectory=setup/account_invoice_triple_discount\n" + ) + payload = { + "fork": True, + "parent": {"full_name": "OCA/account-invoicing"}, + "source": {"full_name": "OCA/account-invoicing"}, + } + github_request = ( + "odoo.addons.odoo_repository_fork.models.odoo_repository.github.request" + ) + with patch(github_request, return_value=payload): + modules = self._resolve(content) + self.assertEqual(len(modules), 1) + self.assertEqual(modules.module_branch_id, upstream_module) + self.assertTrue(modules.is_pinned) + self.assertEqual( + modules.source_clone_url, + "https://github.com/acsone/account-invoicing.git", + ) + self.assertEqual(modules.source_ref, "3e4c54b") + # The fork it is installed from, and through it the origin + fork = modules.source_repository_id + self.assertEqual(fork.display_name, "acsone/account-invoicing") + self.assertEqual(fork.upstream_repository_id, upstream_module.repository_id) + + def test_resolve_unreadable_pin_is_reported_as_to_upgrade(self): + """An unresolved pin must not read as an up to date module.""" + self._create_upstream_repository() + content = ( + "odoo-addon-account-invoice-triple-discount @ " + "git+https://github.com/acsone/account-invoicing.git@3e4c54b" + "#subdirectory=setup/account_invoice_triple_discount\n" + ) + with ( + patch( + "odoo.addons.odoo_repository_fork.models.odoo_repository.github.request", + side_effect=RuntimeError("API rate limit"), + ), + mute_logger("odoo.addons.odoo_repository_fork.models.odoo_repository"), + ): + modules = self._resolve(content) + self.assertFalse(modules.installed_version) + self.assertTrue(modules.to_upgrade) + self.assertIn("account_invoice_triple_discount", self.project.resolution_log) + + # -- reading the dependency file --------------------------------------- + + def _init_clone(self, repository, content=""): + """Create the local clone of `repository`, holding a requirements file.""" + clone_path = repository._get_local_clone_path() + clone_path.mkdir(parents=True, exist_ok=True) + repo = git.Repo.init(clone_path) + repo.config_writer().set_value("user", "name", "test").release() + repo.config_writer().set_value("user", "email", "test@example.com").release() + self.addCleanup(repo.close) + source = clone_path.joinpath("requirements.txt") + source.write_text(content) + repo.index.add([str(source)]) + commit = repo.index.commit("Add requirements") + # The scanner clones without checking out, tracking branches only + repo.git.update_ref(f"refs/remotes/origin/{self.branch.name}", commit.hexsha) + return repo + + def test_read_dependency_source(self): + """The file is read from the tracking branch, without checking it out.""" + self._init_clone(self.odoo_repository, "odoo-addon-test1==15.0.1.0.0\n") + self.assertEqual( + self.project._read_dependency_source(), "odoo-addon-test1==15.0.1.0.0" + ) + + def test_read_dependency_source_without_clone(self): + with self.assertRaisesRegex(UserError, "has not been cloned"): + self.project._read_dependency_source() + + def test_read_dependency_source_missing_file(self): + self._init_clone(self.odoo_repository, "odoo-addon-test1==15.0.1.0.0\n") + self.project.dependency_source_path = "does-not-exist.txt" + with self.assertRaisesRegex(UserError, "Cannot read"): + self.project._read_dependency_source() + + def test_read_dependency_source_without_repository(self): + self.project.repository_id = False + with self.assertRaisesRegex(UserError, "Define the repository"): + self.project._read_dependency_source() + + # -- analysis of a pinned revision ------------------------------------- + + def _setup_pinned(self, module_name, version, on_pull_request=False): + """Return the `(fork, content)` of a dependency pinned on a fork.""" + upstream, __ = self._create_upstream_repository( + with_module=module_name == "account_invoice_triple_discount" + ) + fork_path, revision = self._create_fork( + module_name, version, on_pull_request=on_pull_request + ) + fork = self.env["odoo.repository"].create( + { + "org_id": upstream.org_id.id, + "name": "account-invoicing-fork", + "repo_url": f"file://{fork_path}", + "clone_url": f"file://{fork_path}", + "repo_type": "github", + "to_scan": False, + "upstream_repository_id": upstream.id, + } + ) + content = ( + f"odoo-addon-{module_name.replace('_', '-')} @ " + f"git+file://{fork_path}@{revision}" + f"#subdirectory=setup/{module_name}\n" + ) + return fork, content + + def _resolve_pinned(self, fork, content): + """Resolve `content`, standing in for the lookup of the fork. + + The origin of a 'file://' URL cannot be told from the URL itself, and + that lookup is covered on its own. + """ + with patch.object( + self.env.registry["odoo.repository"], + "_find_from_clone_url", + autospec=True, + return_value=fork, + ): + return self._resolve(content) + + def test_analyse_pinned_revision_reads_the_version_of_the_fork(self): + """The version comes from the manifest at the pinned revision.""" + fork, content = self._setup_pinned( + "account_invoice_triple_discount", "15.0.1.1.0" + ) + modules = self._resolve_pinned(fork, content) + # The module branch stays the one of the upstream repository + upstream_module = modules.module_branch_id + self.assertEqual(upstream_module.version, "15.0.1.0.0") + modules._analyse_pinned_revision() + self.assertEqual(modules.installed_version, "15.0.1.1.0") + self.assertEqual(modules.analysed_ref, modules.source_ref) + # A module the upstream repository hosts is not rewritten from a fork + self.assertEqual(upstream_module.version, "15.0.1.0.0") + + def test_analyse_pinned_revision_of_a_module_absent_upstream(self): + """A module living only in a pull request gets its code analysed. + + It is in no branch of any repository, so no scan ever reaches it: the + pinned revision is the only place it can be read from. + """ + fork, content = self._setup_pinned( + "my_pull_request_module", "15.0.1.1.0", on_pull_request=True + ) + modules = self._resolve_pinned(fork, content) + module_branch = modules.module_branch_id + # The module belongs to no repository, it exists only in the fork... + self.assertFalse(module_branch.repository_id) + # ... but the project module says where it is installed from + self.assertEqual(modules.source_repository_id, fork) + modules._analyse_pinned_revision() + self.assertEqual(modules.installed_version, "15.0.1.1.0") + self.assertEqual(module_branch.version, "15.0.1.1.0") + self.assertEqual(module_branch.title, "Test") + self.assertEqual(module_branch.license_id.name, "AGPL-3") + self.assertEqual(module_branch.dependency_ids.mapped("module_name"), ["base"]) + self.assertTrue(module_branch.sloc_python) + + def test_analyse_pinned_revision_of_an_unknown_module(self): + """A revision holding no such module is reported, not raised.""" + fork, content = self._setup_pinned("my_pull_request_module", "15.0.1.1.0") + with mute_logger( + "odoo.addons.odoo_project_dependency_resolver.models.odoo_project_module", + ): + modules = self._resolve_pinned(fork, content) + modules.module_branch_id.addons_path = "does/not/exist" + with mute_logger( + "odoo.addons.odoo_project_dependency_resolver.lib.scanner", + "odoo.addons.odoo_project_dependency_resolver.models.odoo_project_module", + ): + self.assertFalse(modules._analyse_pinned_revision()) + self.assertFalse(modules.analysed_ref) + + def test_resolve_delays_the_analysis_of_a_pinned_revision(self): + """The analysis is a job of its own, and runs once per revision.""" + job_model = self.env["queue.job"] + domain = [("method_name", "=", "_analyse_pinned_revision")] + fork, content = self._setup_pinned( + "account_invoice_triple_discount", "15.0.1.1.0" + ) + modules = self._resolve_pinned(fork, content) + self.assertEqual(job_model.search_count(domain), 1) + # Once analysed, resolving the very same revision queues nothing more + modules.analysed_ref = modules.source_ref + job_model.search(domain).unlink() + self._resolve_pinned(fork, content) + self.assertEqual(job_model.search_count(domain), 0) + + +class TestResolveDependenciesChaining(ProjectCommon): + """The resolution is appended to the chain of the scan jobs.""" + + @classmethod + def setUpClass(cls): + super().setUpClass() + # We want to test the chaining of jobs, not the resolution itself, so + # mute the logger of the delayable to avoid a warning on the job being + # collected without having been delayed. The resolution itself is tested + # elsewhere. + # we could use 'enterClassContext(mute_logger("odoo.addons.queue_job.delay"))' + # in Python>=3.11, where Odoo 18 runs on 3.10. + muted = contextlib.ExitStack() + cls.addClassCleanup(muted.close) + muted.enter_context(mute_logger("odoo.addons.queue_job.delay")) + cls.project.write( + { + "repository_id": cls.odoo_repository.id, + "dependency_management": "resolved", + } + ) + + def tearDown(self): + super().tearDown() + # A delayable warns when collected without having been delayed, which + # is what these tests do on purpose. Collect here, while + # the logger above is still muted, rather than after 'tearDownClass'. + gc.collect() + + def _create_subsequent_jobs(self, data=None, version=None, repository=None): + repository = repository or self.odoo_repository + repository_branch = self.env["odoo.repository.branch"].search( + [("repository_id", "=", repository.id), ("branch_id", "=", self.branch.id)] + ) + if not repository_branch: + repository_branch = self._create_odoo_repository_branch( + repository, self.branch + ) + if data is None: + data = { + "repo_branch_id": repository_branch.id, + "last_fetched_commit": "c0ffee", + "last_scanned_commit": "c0ffee", + "addons_paths": {}, + } + version = version or self.branch.name + return repository._create_subsequent_jobs((version, version), [], [], data) + + def _resolution_jobs(self, jobs): + return [ + job + for job in jobs + if job._job_method.__name__ == "_resolve_dependencies_job" + ] + + def test_resolution_is_chained_after_the_scan(self): + jobs = self._create_subsequent_jobs() + resolution_jobs = self._resolution_jobs(jobs) + self.assertEqual(len(resolution_jobs), 1) + self.assertEqual(resolution_jobs[0].recordset, self.project) + # It runs last, once the modules of the branch have been scanned + self.assertEqual(jobs[-1], resolution_jobs[0]) + # Scanning the repository of the project is the first pass: it is the + # one discovering the repositories the modules come from + self.assertEqual(resolution_jobs[0]._job_kwargs, {"scan_repositories": True}) + + def _create_dependency_repository(self): + """Return a repository the modules of the project come from.""" + org_model = self.env["odoo.repository.org"].with_context(active_test=False) + org = org_model.search([("name", "=", "OCA")]) or org_model.create( + {"name": "OCA"} + ) + repository = self.env["odoo.repository"].create( + { + "org_id": org.id, + "name": "server-tools", + "repo_url": "https://github.com/OCA/server-tools", + "repo_type": "github", + } + ) + repository_branch = self._create_odoo_repository_branch(repository, self.branch) + module_branch = self._create_odoo_module_branch( + self.module_branch_model._get_module("test1"), + self.branch, + repository_branch_id=repository_branch.id, + ) + self.project._get_project_module(module_branch) + return repository + + def test_no_resolution_chained_to_a_dependency_repository(self): + """Only the repository holding the dependency file resolves. + + Chaining one resolution per repository the modules come from would + create as many as the project has dependencies. + """ + repository = self._create_dependency_repository() + jobs = self._create_subsequent_jobs(repository=repository) + self.assertFalse(self._resolution_jobs(jobs)) + + def test_second_pass_scans_the_resolved_repositories(self): + """The first pass scans what the resolution discovered.""" + repository = self._create_dependency_repository() + repository_model = self.env.registry["odoo.repository"] + with patch.object( + repository_model, "_create_scan_jobs", autospec=True, return_value=[] + ) as create_scan_jobs: + self.project._scan_resolved_repositories() + create_scan_jobs.assert_called_once() + scanned = create_scan_jobs.call_args_list[0].args[0] + self.assertEqual(scanned, repository) + self.assertTrue(scanned.env.context.get("strict_branches_scan")) + self.assertEqual( + create_scan_jobs.call_args_list[0].kwargs["branch_ids"], self.branch.ids + ) + + def test_second_pass_is_joined_after_the_scans(self): + """One resolution waits for every scan of the group to be done.""" + self._create_dependency_repository() + self.project._scan_resolved_repositories() + self.env.flush_all() + resolution = self.env["queue.job"].search( + [("method_name", "=", "_resolve_dependencies_job")] + ) + self.assertEqual(len(resolution), 1) + self.assertEqual(resolution.state, "wait_dependencies") + self.assertFalse(resolution.kwargs.get("scan_repositories")) + + def test_no_resolution_for_a_manual_project(self): + self.project.dependency_management = "manual" + self.assertFalse(self._resolution_jobs(self._create_subsequent_jobs())) + + def test_no_resolution_for_another_version(self): + """A project is resolved on its own Odoo version only.""" + self.assertFalse( + self._resolution_jobs( + self._create_subsequent_jobs(version=self.branch2.name) + ) + ) + + def test_no_resolution_when_the_branch_does_not_exist(self): + self.assertFalse(self._resolution_jobs(self._create_subsequent_jobs(data={}))) + + def test_job_records_an_expected_failure(self): + """An unreadable dependency file must not break the chain of jobs.""" + with mute_logger( + "odoo.addons.odoo_project_dependency_resolver.models.odoo_project" + ): + self.project._resolve_dependencies_job() + self.assertIn("has not been cloned", self.project.resolution_log) + self.assertTrue(self.project.last_resolution_date) diff --git a/odoo_project_dependency_resolver/views/odoo_project.xml b/odoo_project_dependency_resolver/views/odoo_project.xml new file mode 100644 index 00000000..f42cf552 --- /dev/null +++ b/odoo_project_dependency_resolver/views/odoo_project.xml @@ -0,0 +1,73 @@ + + + + + odoo.project.form.dependency.resolver + odoo.project + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/odoo_project_dependency_resolver/views/odoo_project_module.xml b/odoo_project_dependency_resolver/views/odoo_project_module.xml new file mode 100644 index 00000000..80cd921b --- /dev/null +++ b/odoo_project_dependency_resolver/views/odoo_project_module.xml @@ -0,0 +1,65 @@ + + + + + odoo.project.module.form.dependency.resolver + odoo.project.module + + + +
+ +
+ + + + + + + + + +
+ + + odoo.project.module.list.dependency.resolver + odoo.project.module + + + + + + + + + + + + odoo.project.module.search.dependency.resolver + odoo.project.module + + + + + + + +
diff --git a/odoo_project_migration/__manifest__.py b/odoo_project_migration/__manifest__.py index c362249b..33832237 100644 --- a/odoo_project_migration/__manifest__.py +++ b/odoo_project_migration/__manifest__.py @@ -14,6 +14,7 @@ "views/odoo_module_branch_timeline.xml", "views/odoo_module_branch_migration.xml", "views/odoo_project.xml", + "views/odoo_project_module.xml", "views/odoo_project_module_migration.xml", "wizards/generate_migration_data.xml", "wizards/export_migration_report.xml", diff --git a/odoo_project_migration/views/odoo_project_module.xml b/odoo_project_migration/views/odoo_project_module.xml new file mode 100644 index 00000000..2b1e3003 --- /dev/null +++ b/odoo_project_migration/views/odoo_project_module.xml @@ -0,0 +1,27 @@ + + + + + odoo.project.module.form.migration + odoo.project.module + + + +
+ + +
+
+
+
diff --git a/odoo_repository_fork/tests/test_odoo_repository_fork.py b/odoo_repository_fork/tests/test_odoo_repository_fork.py index 29921995..a264aa1d 100644 --- a/odoo_repository_fork/tests/test_odoo_repository_fork.py +++ b/odoo_repository_fork/tests/test_odoo_repository_fork.py @@ -195,8 +195,7 @@ def test_fork_is_reached_with_its_own_credentials(self): upstream_repository_id=upstream.id, token_id=token.id, ) - url = "https://github.com/acsone/account-invoicing.git" - self.assertEqual( - fork._get_authenticated_url(url), - "https://oauth2:s3cr3t@github.com/acsone/account-invoicing.git", - ) + params = fork._prepare_base_scanner_parameters() + self.assertEqual(params["org"], "acsone") + self.assertEqual(params["token"], "s3cr3t") + self.assertNotEqual(params["token"], upstream._get_token())