diff --git a/odoo_project/models/odoo_project.py b/odoo_project/models/odoo_project.py index 6f613e6e..d5f12f3c 100644 --- a/odoo_project/models/odoo_project.py +++ b/odoo_project/models/odoo_project.py @@ -1,4 +1,5 @@ # Copyright 2023 Camptocamp SA +# Copyright 2026 ACSONE SA/NV () # License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl) import ast @@ -91,7 +92,7 @@ def _compute_available_odoo_version_ids(self): if rec.repository_id: rec.available_odoo_version_ids = rec.repository_id.branch_ids.branch_id - @api.depends("repository_id", "odoo_version_id") + @api.depends("repository_id.branch_ids.branch_id", "odoo_version_id") def _compute_repository_branch_id(self): for rec in self: rec.repository_branch_id = False @@ -166,6 +167,61 @@ def action_find_unknown_modules(self): for module in self.unknown_module_ids: module.action_find_pr_url() + def _get_module_branch(self, module, repository=None): + """Return the `odoo.module.branch` matching `module` for this project. + + The module is looked up in `repository` first, defaulting to the + repository of the project. If it doesn't exist it'll be automatically + created as an orphaned module. + """ + self.ensure_one() + if repository is None: + repository = self.repository_id + module_branch = self.env["odoo.module.branch"]._find_or_create( + self.odoo_version_id, module, repository + ) + if not module_branch.repository_branch_id and not module_branch.specific: + # If the module hasn't been found in existing repositories content, + # it could be available somewhere on GitHub as a PR that could help + # to identity its repository + module_branch.with_delay().action_find_pr_url() + return module_branch + + def _get_project_module(self, module_branch, version=False): + """Return the `odoo.project.module` of this project for `module_branch`. + + If it doesn't exist it'll be automatically created, otherwise its + installed version is updated. + """ + self.ensure_one() + project_module_model = self.env["odoo.project.module"] + domain = [ + ("module_branch_id", "=", module_branch.id), + ("odoo_project_id", "=", self.id), + ] + project_module = project_module_model.search(domain) + values = { + "module_branch_id": module_branch.id, + "odoo_project_id": self.id, + "installed_version": version, + } + if project_module: + project_module.sudo().write(values) + else: + # Create the module to make it available for the project + project_module = project_module_model.sudo().create(values) + return project_module + + def _import_missing_dependencies(self, project_modules): + """Complete the project with the dependencies of `project_modules`.""" + self.ensure_one() + branch_modules = project_modules.module_branch_id + all_dependencies = branch_modules._get_recursive_dependencies() + missing_dependencies = all_dependencies - branch_modules + for missing_dependency in missing_dependencies: + self._get_project_module(missing_dependency, missing_dependency.version) + return True + def _get_repositories_to_scan(self): """Return the repositories to scan.""" domain = self.env["odoo.repository"]._cron_scanner_domain() diff --git a/odoo_project/tests/test_import_modules.py b/odoo_project/tests/test_import_modules.py index 2f3167bf..858d0eeb 100644 --- a/odoo_project/tests/test_import_modules.py +++ b/odoo_project/tests/test_import_modules.py @@ -1,4 +1,5 @@ # Copyright 2024 Camptocamp SA +# Copyright 2026 ACSONE SA/NV () # License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl) from .common import ProjectCommon @@ -62,7 +63,7 @@ def test_import_modules_names_versions(self): def test_match_blacklisted_module(self): mod1 = "test1" mod2 = "test2" - mod1_blacklisted = self.wiz_import_modules_model._get_module(mod1) + mod1_blacklisted = self.module_branch_model._get_module(mod1) mod1_blacklisted.blacklisted = True # Import them through the wizard modules_list_text = f"{mod1}\n{mod2}" @@ -82,7 +83,7 @@ def test_match_blacklisted_module(self): def test_match_orphaned_module(self): mod1 = "test1" mod2 = "test2" - mod1_orphaned = self.wiz_import_modules_model._get_module(mod1) + mod1_orphaned = self.module_branch_model._get_module(mod1) mod1_branch_orphaned = self.module_branch_model._create_orphaned_module_branch( self.branch, mod1_orphaned ) @@ -106,7 +107,7 @@ def test_match_orphaned_module(self): def test_match_generic_module(self): mod1 = "test1" mod2 = "test2" - mod1_generic = self.wiz_import_modules_model._get_module(mod1) + mod1_generic = self.module_branch_model._get_module(mod1) repo_branch = self._create_odoo_repository_branch( self.odoo_repository, self.branch ) @@ -138,7 +139,7 @@ def test_match_project_repo_module(self): self.project.odoo_version_id = self.branch mod1 = "test1" mod2 = "test2" - mod1_in_repo = self.wiz_import_modules_model._get_module(mod1) + mod1_in_repo = self.module_branch_model._get_module(mod1) repo_branch = self._create_odoo_repository_branch( self.odoo_repository, self.branch ) @@ -163,3 +164,50 @@ def test_match_project_repo_module(self): self.assertIn(mod1_branch_in_repo, existing_mods) # Project modules are also created self.assertEqual(len(existing_mods.odoo_project_module_ids), 2) + + def test_get_module_branch_in_given_repository(self): + """A module can be looked up in a repository other than the project one.""" + module = self.module_branch_model._get_module("test1") + # The very same module lives in two scanned repositories + other_org = self.env["odoo.repository.org"].create({"name": "other-org"}) + other_repository = self.env["odoo.repository"].create( + { + "org_id": other_org.id, + "name": self.odoo_repository.name, + "repo_url": "https://github.com/other-org/repo", + "repo_type": "github", + } + ) + module_branches = {} + for repository in (self.odoo_repository, other_repository): + repo_branch = self._create_odoo_repository_branch(repository, self.branch) + module_branches[repository] = self._create_odoo_module_branch( + module, + self.branch, + specific=False, + repository_branch_id=repo_branch.id, + ) + for repository, module_branch in module_branches.items(): + with self.subTest(repository=repository.display_name): + self.assertEqual( + self.project._get_module_branch(module, repository=repository), + module_branch, + ) + + def test_repository_branch_id_follows_the_repository_branches(self): + """The branch of a project is found whenever its repository gets one. + + A project is commonly created before its repository has been scanned, + so the matching branch does not exist yet at that point. + """ + self.project.write( + { + "repository_id": self.odoo_repository.id, + "odoo_version_id": self.branch.id, + } + ) + self.assertFalse(self.project.repository_branch_id) + repository_branch = self._create_odoo_repository_branch( + self.odoo_repository, self.branch + ) + self.assertEqual(self.project.repository_branch_id, repository_branch) diff --git a/odoo_project/views/odoo_project_module.xml b/odoo_project/views/odoo_project_module.xml index 91c80751..4f5a41a5 100644 --- a/odoo_project/views/odoo_project_module.xml +++ b/odoo_project/views/odoo_project_module.xml @@ -7,7 +7,28 @@ odoo.project.module primary + + 99 + +
+
+ +
+
diff --git a/odoo_project/wizards/odoo_project_import_modules.py b/odoo_project/wizards/odoo_project_import_modules.py index 42858cc7..e9f48b51 100644 --- a/odoo_project/wizards/odoo_project_import_modules.py +++ b/odoo_project/wizards/odoo_project_import_modules.py @@ -1,4 +1,5 @@ # Copyright 2023 Camptocamp SA +# Copyright 2026 ACSONE SA/NV () # License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl) import re @@ -49,11 +50,13 @@ def action_import(self): project_module_ids = self._action_import_modules_list() project_module_ids.extend(self._action_import_additional_modules()) if self.import_missing_dependencies: - self._action_import_missing_dependencies(project_module_ids) + project_modules = self.env["odoo.project.module"].browse(project_module_ids) + self.odoo_project_id._import_missing_dependencies(project_modules) def _action_import_modules_list(self): """Import a fresh list of installed modules into the project.""" - self.odoo_project_id.sudo().project_module_ids = False + project = self.odoo_project_id + project.sudo().project_module_ids = False module_lines = list(filter(None, self.modules_list.split("\n"))) project_module_ids = [] for line in module_lines: @@ -66,81 +69,21 @@ def _action_import_modules_list(self): else: module_name, version = data[0], False # for module_name in module_names: - module = self._get_module(module_name) + module = self.env["odoo.module.branch"]._get_module(module_name) if module.blacklisted: continue - module_branch = self._get_module_branch(module) - project_module = self._get_project_module(module_branch, version) + module_branch = project._get_module_branch(module) + project_module = project._get_project_module(module_branch, version) project_module_ids.append(project_module.id) - self.odoo_project_id.sudo().project_module_ids = project_module_ids + project.sudo().project_module_ids = project_module_ids return project_module_ids def _action_import_additional_modules(self): """Import additional modules into the project.""" project_module_ids = [] for module_branch in self.additional_module_ids: - project_module = self._get_project_module(module_branch, version=False) + project_module = self.odoo_project_id._get_project_module( + module_branch, version=False + ) project_module_ids.append(project_module.id) return project_module_ids - - def _action_import_missing_dependencies(self, project_module_ids): - """Complete list of modules by adding all dependencies.""" - project_modules = self.env["odoo.project.module"].browse(project_module_ids) - branch_modules = project_modules.module_branch_id - all_dependencies = branch_modules._get_recursive_dependencies() - missing_dependencies = all_dependencies - branch_modules - for missing_dependency in missing_dependencies: - self._get_project_module(missing_dependency, missing_dependency.version) - return True - - def _get_module(self, module_name): - """Return a `odoo.module` record. - - If it doesn't exist it'll be automatically created. - """ - module_model = self.env["odoo.module"] - module = module_model.search([("name", "=", module_name)]) - if not module: - module = module_model.sudo().create({"name": module_name}) - return module - - def _get_module_branch(self, module): - """Return a `odoo.module.branch` record. - - If it doesn't exist it'll be automatically created. - """ - module_branch_model = self.env["odoo.module.branch"] - module_branch = False - branch = self.odoo_project_id.odoo_version_id - module_branch = module_branch_model._find_or_create( - branch, module, self.odoo_project_id.repository_id - ) - if not module_branch.repository_branch_id and not module_branch.specific: - # If the module hasn't been found in existing repositories content, - # it could be available somewhere on GitHub as a PR that could help - # to identity its repository - module_branch.with_delay().action_find_pr_url() - return module_branch - - def _get_project_module(self, module_branch, version): - """Return a `odoo.project.module` record for the project. - - If it doesn't exist it'll be automatically created. - """ - project_module_model = self.env["odoo.project.module"] - domain = [ - ("module_branch_id", "=", module_branch.id), - ("odoo_project_id", "=", self.odoo_project_id.id), - ] - project_module = project_module_model.search(domain) - values = { - "module_branch_id": module_branch.id, - "odoo_project_id": self.odoo_project_id.id, - "installed_version": version, - } - if project_module: - project_module.sudo().write(values) - else: - # Create the module to make it available for the project - project_module = project_module_model.sudo().create(values) - return project_module diff --git a/odoo_project_dependency_resolver/README.rst b/odoo_project_dependency_resolver/README.rst new file mode 100644 index 00000000..2ef20642 --- /dev/null +++ b/odoo_project_dependency_resolver/README.rst @@ -0,0 +1,151 @@ +================================== +Odoo Project - Dependency Resolver +================================== + +.. + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + !! This file is generated by oca-gen-addon-readme !! + !! changes will be overwritten. !! + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + !! source digest: sha256:b48258207df2f746b919569e9951023638da491aaa1e5787d44e370cb356ca94 + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + +.. |badge1| image:: https://img.shields.io/badge/maturity-Beta-yellow.png + :target: https://odoo-community.org/page/development-status + :alt: Beta +.. |badge2| image:: https://img.shields.io/badge/licence-AGPL--3-blue.png + :target: http://www.gnu.org/licenses/agpl-3.0-standalone.html + :alt: License: AGPL-3 +.. |badge3| image:: https://img.shields.io/badge/github-OCA%2Fmodule--composition--analysis-lightgray.png?logo=github + :target: https://github.com/OCA/module-composition-analysis/tree/18.0/odoo_project_dependency_resolver + :alt: OCA/module-composition-analysis +.. |badge4| image:: https://img.shields.io/badge/weblate-Translate%20me-F47D42.png + :target: https://translation.odoo-community.org/projects/module-composition-analysis-18-0/module-composition-analysis-18-0-odoo_project_dependency_resolver + :alt: Translate me on Weblate +.. |badge5| image:: https://img.shields.io/badge/runboat-Try%20me-875A7B.png + :target: https://runboat.odoo-community.org/builds?repo=OCA/module-composition-analysis&target_branch=18.0 + :alt: Try me on Runboat + +|badge1| |badge2| |badge3| |badge4| |badge5| + +Projects usually pin the exact version of every module they install in a +dependency file, such as the ``requirements.txt`` produced by pip. +Maintaining the very same list a second time by hand in Odoo MCA is +tedious and drifts away from reality as soon as a dependency is bumped. + +This module reads that file straight from the repository of the project +and rebuilds its modules from it, every time the project is scanned. + +**Table of contents** + +.. contents:: + :local: + +Usage +===== + +On the project, set *Dependency Management* to *Resolved*, pick the +format of the dependency file and, if it is not at the root of the +repository under its usual name, adjust its path. + +There is nothing else to declare. A dependency file only holds packaged +addons, so the modules of a resolved project are the union of three +sources: + +- the addons of the dependency file, with the versions it pins; +- the modules hosted by the project repository, which are the code of + the project itself and are shipped with it rather than installed as a + package; +- their missing dependencies, pulled from the dependency graph, which is + where Odoo standard modules come from. + +*Additional Modules* is there for what none of the three reaches, Odoo +Enterprise modules typically. + +The modules are refreshed at each scan, and by the *Resolve +Dependencies* button. Whatever the resolution could not do is reported +on the project, in the *Resolution Log* field. + +What a scan does +---------------- + +Scanning a resolved project only scans the repository of the project: +the repositories its modules come from are derived from the dependency +file, so none of them is known before that file has been read. + +The resolution is chained to that scan, so it reads a file that has just +been fetched, and it then spawns the scan of the repositories it +discovered. A second resolution runs once they have all been scanned, +completing what the first one could not know: the modules of a +repository that had never been scanned, and the dependency graph the +resolved modules are walked through. + +Note that the second resolution waits for the detection of the modules +to scan in every repository, not for every module to have been scanned: +the scanner spawns the latter as it goes. A module scanned late is +therefore accounted for by the next scan of the project. + +Modules pinned on a fork +------------------------ + +A module frozen on a commit of a fork, such as: + +:: + + odoo-addon-account-invoice-triple-discount @ + git+https://github.com/acsone/account-invoicing.git@3e4c54b + #subdirectory=setup/account_invoice_triple_discount + +is not treated as a new module. It stays attached to the module of the +repository the fork originates from, so that dependencies, migration +scripts and changelogs keep working, and the fork is recorded on the +project module as *Installed From*. Its version is read from the +manifest at the pinned revision. + +A module added by a pull request not merged yet is a special case: it +exists in no repository known to Odoo MCA, so it stays an orphaned +module and shows up among the unknown modules of the project. *Installed +From* is then the only way to reach a repository from it, and the *Fork +Of* of that fork tells where the module is heading. + +Bug Tracker +=========== + +Bugs are tracked on `GitHub Issues `_. +In case of trouble, please check there if your issue has already been reported. +If you spotted it first, help us to smash it by providing a detailed and welcomed +`feedback `_. + +Do not contact contributors directly about support or help with technical issues. + +Credits +======= + +Authors +------- + +* ACSONE SA/NV + +Contributors +------------ + +- ACSONE SA/NV + + - Laurent Mignon + +Maintainers +----------- + +This module is maintained by the OCA. + +.. image:: https://odoo-community.org/logo.png + :alt: Odoo Community Association + :target: https://odoo-community.org + +OCA, or the Odoo Community Association, is a nonprofit organization whose +mission is to support the collaborative development of Odoo features and +promote its widespread use. + +This module is part of the `OCA/module-composition-analysis `_ project on GitHub. + +You are welcome to contribute. To learn how please visit https://odoo-community.org/page/Contribute. diff --git a/odoo_project_dependency_resolver/__init__.py b/odoo_project_dependency_resolver/__init__.py new file mode 100644 index 00000000..f24d3e24 --- /dev/null +++ b/odoo_project_dependency_resolver/__init__.py @@ -0,0 +1,2 @@ +from . import components +from . import models diff --git a/odoo_project_dependency_resolver/__manifest__.py b/odoo_project_dependency_resolver/__manifest__.py new file mode 100644 index 00000000..aa4bcd44 --- /dev/null +++ b/odoo_project_dependency_resolver/__manifest__.py @@ -0,0 +1,30 @@ +# Copyright 2026 ACSONE SA/NV () +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl) +{ + "name": "Odoo Project - Dependency Resolver", + "summary": "Resolve the modules of a project from its dependency file.", + "version": "18.0.1.0.0", + "category": "Tools", + "author": "ACSONE SA/NV, Odoo Community Association (OCA)", + "website": "https://github.com/OCA/module-composition-analysis", + "data": [ + "data/queue_job.xml", + "views/odoo_project.xml", + "views/odoo_project_module.xml", + ], + "installable": True, + "depends": [ + # OCA/queue + "queue_job", + # OCA/module-composition-analysis + "odoo_project", + "odoo_repository_fork", + ], + "external_dependencies": { + "python": [ + "gitpython", + "odoo-addons-parser", + ], + }, + "license": "AGPL-3", +} diff --git a/odoo_project_dependency_resolver/components/__init__.py b/odoo_project_dependency_resolver/components/__init__.py new file mode 100644 index 00000000..2db60093 --- /dev/null +++ b/odoo_project_dependency_resolver/components/__init__.py @@ -0,0 +1,2 @@ +from . import dependency_resolver +from . import requirements_txt_resolver diff --git a/odoo_project_dependency_resolver/components/dependency_resolver.py b/odoo_project_dependency_resolver/components/dependency_resolver.py new file mode 100644 index 00000000..238f743a --- /dev/null +++ b/odoo_project_dependency_resolver/components/dependency_resolver.py @@ -0,0 +1,43 @@ +# Copyright 2026 ACSONE SA/NV () +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl) + +import dataclasses + +from odoo.addons.component.core import AbstractComponent + + +@dataclasses.dataclass(frozen=True) +class ResolvedDependency: + """A module declared by a project, as read from a dependency file.""" + + module_name: str + version: str = None + clone_url: str = None + ref: str = None + subdirectory: str = None + + @property + def is_pinned(self): + """Whether the module is frozen on a specific revision.""" + return bool(self.clone_url and self.ref) + + +class DependencyResolver(AbstractComponent): + """Turn the content of a dependency file into a list of modules. + + Concrete resolvers only deal with text: they parse what they are given + and return what they found, without ever touching the ORM. Mapping the + result onto modules of the database is the job of `odoo.project`, and is + shared by every format. + + Adding support for another format (pip.lock, uv.lock...) is therefore a + matter of adding a component named after the format, and an entry in the + `dependency_resolver` selection of `odoo.project`. + """ + + _name = "project.dependency.resolver" + _collection = "odoo.mca.backend" + + def resolve(self, content): + """Return the `ResolvedDependency` list declared in `content`.""" + raise NotImplementedError diff --git a/odoo_project_dependency_resolver/components/requirements_txt_resolver.py b/odoo_project_dependency_resolver/components/requirements_txt_resolver.py new file mode 100644 index 00000000..c48b48c3 --- /dev/null +++ b/odoo_project_dependency_resolver/components/requirements_txt_resolver.py @@ -0,0 +1,151 @@ +# Copyright 2026 ACSONE SA/NV () +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl) + +import re +from urllib.parse import parse_qs, urlparse, urlunparse + +from packaging.requirements import InvalidRequirement, Requirement + +from odoo.addons.component.core import Component + +from .dependency_resolver import ResolvedDependency + +# Odoo addons are distributed as 'odoo-addon-' since 15.0, and as +# 'odoo-addon-' before. Note that this excludes packages such +# as 'odoo-addons-parser', which are not addons. +ADDON_PREFIX_RE = re.compile(r"^odoo(?:\d+)?-addon-", re.IGNORECASE) +# Comments, but not the '#subdirectory=' fragment of a URL +COMMENT_RE = re.compile(r"(^|\s)#.*$") +HASH_RE = re.compile(r"\s--hash=\S+") +# An Odoo manifest version holds 5 segments ('....'). +# setuptools-odoo appends a serial to it when only the packaging changed, +# e.g. addon '18.0.1.2.0' published as wheel '18.0.1.2.0.3'. +MANIFEST_VERSION_SEGMENTS = 5 + + +class RequirementsTxtResolver(Component): + """Resolve the modules of a project from a pip requirements file.""" + + _name = "project.dependency.resolver.requirements_txt" + _inherit = "project.dependency.resolver" + _usage = "project.dependency.resolver.requirements_txt" + + def resolve(self, content): + dependencies = [] + for requirement in self._iter_requirements(content): + dependency = self._parse_requirement(requirement) + if dependency: + dependencies.append(dependency) + return dependencies + + def _iter_requirements(self, content): + """Yield the requirement lines.""" + buffer = "" + for raw_line in content.splitlines(): + line = COMMENT_RE.sub("", raw_line).strip() + if not line: + continue + if line.endswith("\\"): + buffer += line[:-1].strip() + " " + continue + line, buffer = buffer + line, "" + if line.startswith("-"): + # Options ('-r', '-c', '--index-url'...) + continue + line = HASH_RE.sub("", line) + # Remove environment markers ('; python_version < "3.8"') + # and trailing whitespace + line = line.split(";", 1)[0].strip() + if line: + yield line + + def _parse_requirement(self, requirement): + """Return the `ResolvedDependency` of a requirement, if it is an addon.""" + requirement = requirement.strip() + if not requirement: + return None + + try: + req = Requirement(requirement) + except InvalidRequirement: + # Fallback for non-PEP 508 lines that pip still accepts in practice. + # vcs legacy line ex: + # git+https://github.com/OCA/project.git@v1.2.3#egg=odoo-addon-project + name = re.split(r"[<>=!~\[\s]", requirement, maxsplit=1)[0] + module_name = self._get_module_name(name) + return ResolvedDependency(module_name) if module_name else None + + name = req.name + module_name = self._get_module_name(name) + if not module_name: + return None + if req.url: + return self._parse_direct_reference(name, req.url) + + version = str(req.specifier).strip() + if version.startswith("=="): + return ResolvedDependency( + module_name, + version=self._to_manifest_version(version[2:].strip()), + ) + + # Anything else (ranges, unpinned...) carries no usable version. + return ResolvedDependency(module_name) + + def _parse_direct_reference(self, name, url): + """Return the `ResolvedDependency` of a module pinned on a repository.""" + module_name = self._get_module_name(name) + if not module_name: + return None + url, __, fragment = url.partition("#") + subdirectory = self._get_subdirectory(fragment) + clone_url, ref = self._split_vcs_url(url) + return ResolvedDependency( + module_name, + clone_url=clone_url, + ref=ref, + subdirectory=subdirectory, + ) + + def _get_module_name(self, distribution_name): + """Return the technical name of the module packaged as `distribution_name`. + + Returns nothing when the distribution is not an Odoo addon. + """ + if not ADDON_PREFIX_RE.match(distribution_name): + return None + # Odoo module names never hold a dash, so undoing the normalization + # performed by the packaging is unambiguous + return ADDON_PREFIX_RE.sub("", distribution_name).replace("-", "_") + + @staticmethod + def _get_subdirectory(fragment): + """Return the 'subdirectory' held by the fragment of a requirement URL.""" + values = parse_qs(fragment).get("subdirectory") + return values[0] if values else None + + @staticmethod + def _split_vcs_url(url): + """Return the `(clone_url, ref)` parts of a VCS requirement URL. + + The revision is split off the path rather than off the whole URL, so + that credentials held by the network location ('https://user@host/...') + are not mistaken for one. + """ + url = re.sub(r"^\w+\+", "", url) + parts = urlparse(url) + path, __, ref = parts.path.rpartition("@") + if not path: + # No revision pinned + return (url, None) + return (urlunparse(parts._replace(path=path)), ref) + + @staticmethod + def _to_manifest_version(version): + """Return the manifest version a distribution version was built from.""" + # Drop the PEP 440 local and post segments added by the packaging + version = re.split(r"[+!]", version)[0] + segments = version.split(".") + if len(segments) > MANIFEST_VERSION_SEGMENTS: + return ".".join(segments[:MANIFEST_VERSION_SEGMENTS]) + return version diff --git a/odoo_project_dependency_resolver/data/queue_job.xml b/odoo_project_dependency_resolver/data/queue_job.xml new file mode 100644 index 00000000..4bca4cd8 --- /dev/null +++ b/odoo_project_dependency_resolver/data/queue_job.xml @@ -0,0 +1,24 @@ + + + + + + _resolve_dependencies_job + + + + + + + _analyse_pinned_revision + + + + diff --git a/odoo_project_dependency_resolver/lib/__init__.py b/odoo_project_dependency_resolver/lib/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/odoo_project_dependency_resolver/lib/scanner.py b/odoo_project_dependency_resolver/lib/scanner.py new file mode 100644 index 00000000..48c4a309 --- /dev/null +++ b/odoo_project_dependency_resolver/lib/scanner.py @@ -0,0 +1,106 @@ +# Copyright 2026 ACSONE SA/NV () +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl) + +import logging +import pathlib +import tarfile +import tempfile + +import git +from odoo_addons_parser import ModuleParser + +from odoo.addons.odoo_repository.lib.scanner import BaseScanner + +_logger = logging.getLogger(__name__) + + +class PinnedModuleScanner(BaseScanner): + """Analyse a module at the revision a project is pinned on. + + A pinned revision is the head of a pull request more often than of a + branch, so the standard scan of a repository, which walks branches, never + reaches it. Neither does it reach a module the pull request adds, which + exists in no branch at all. + + Nothing is cloned: only the revisions a project pins are fetched, without + the history leading to them nor the branches around them. What a full + clone would bring is of no use here, a module being read from a single + commit. + """ + + # The revisions are kept out of the way of the clones the repository + # scanner maintains: shallow and branchless, they would be of no use to a + # repository scan, and the repository holding them can be one it scans. + _revisions_dirname = "pinned-revisions" + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.path = self.repositories_path.joinpath( + self._revisions_dirname, self.org, self.clone_name or self.name + ) + + def scan_module_at_revision(self, module_path, revision): + """Return the analysis of `module_path` at `revision`, `None` if absent.""" + self._apply_git_global_config() + self.path.mkdir(parents=True, exist_ok=True) + repo = git.Repo.init(self.path) + try: + self._apply_git_config(repo) + self._set_git_remote_url(repo, "origin", self.clone_url) + self._fetch_revision(repo, revision) + with tempfile.TemporaryDirectory() as tmp_dir: + module_dir = self._extract_module(repo, revision, module_path, tmp_dir) + if module_dir is None: + return None + return ModuleParser(module_dir, scan_models=False).to_dict() + finally: + repo.close() + + def _fetch_revision(self, repo, revision): + """Fetch `revision` alone, without the history leading to it. + + Only a revision the repository does not hold yet is fetched: a pinned + one never moves, so it is worth fetching once and never again. The + ones fetched before are kept, as the revisions of a repository share + most of their content. + """ + if self._has_revision(repo, revision): + return + _logger.info("%s: fetch revision %s", self.full_name, revision) + with self._get_git_env() as env, repo.git.custom_environment(**env): + repo.git.fetch("origin", revision, depth=1) + + @staticmethod + def _has_revision(repo, revision): + """Return whether the local clone already holds `revision`.""" + try: + repo.commit(revision) + except (git.exc.BadName, ValueError): + return False + return True + + def _extract_module(self, repo, revision, module_path, target_dir): + """Extract `module_path` at `revision` into `target_dir`. + + Returns the path of the extracted module, `None` when the revision + holds no such module. + + The module is extracted rather than checked out: the working tree of + the clone belongs to the scan jobs, which check branches out in it. + """ + archive = pathlib.Path(target_dir, "module.tar") + try: + repo.git.archive(revision, module_path, output=str(archive)) + except git.exc.GitCommandError: + _logger.warning( + "%s: no %s at revision %s", + self.full_name, + module_path, + revision, + exc_info=True, + ) + return None + with tarfile.open(archive) as tar: + tar.extractall(target_dir, filter="data") + archive.unlink() + return pathlib.Path(target_dir, module_path) diff --git a/odoo_project_dependency_resolver/models/__init__.py b/odoo_project_dependency_resolver/models/__init__.py new file mode 100644 index 00000000..e26962c5 --- /dev/null +++ b/odoo_project_dependency_resolver/models/__init__.py @@ -0,0 +1,4 @@ +from . import odoo_repository +from . import odoo_module_branch +from . import odoo_project +from . import odoo_project_module diff --git a/odoo_project_dependency_resolver/models/odoo_module_branch.py b/odoo_project_dependency_resolver/models/odoo_module_branch.py new file mode 100644 index 00000000..128624d2 --- /dev/null +++ b/odoo_project_dependency_resolver/models/odoo_module_branch.py @@ -0,0 +1,47 @@ +# Copyright 2026 ACSONE SA/NV () +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl) + +from odoo import models + + +class OdooModuleBranch(models.Model): + _inherit = "odoo.module.branch" + + def _update_from_pinned_revision(self, data): + """Fill this module branch with a module analysed at a pinned revision. + + Meant for a module living in a pull request alone: it is in no branch + of any repository, so no scan ever reaches it and nothing else ever + fills it. + """ + self.ensure_one() + values = self._prepare_pinned_revision_values(data) + if values: + self.sudo().write(values) + return True + + def _prepare_pinned_revision_values(self, data): + """Return the values a module analysed at a pinned revision carries. + + Only what the revision itself holds: what its manifest declares, and + what the analysis of its code counted. What a scan knows on top of + that, which repository hosts the module and the history of its + versions, a revision cannot tell. + """ + values = {} + manifest = data.get("manifest") or {} + if manifest: + # The dependencies are looked up from this very module branch: the + # revision knows of no repository branch, and a module living in a + # pull request alone belongs to no repository at all. + values.update( + self._prepare_manifest_values( + manifest, self.branch_id, self.repository_id + ) + ) + code = data.get("code") or {} + if code: + values.update(self._prepare_code_analysis_values(code)) + if values: + values["removed"] = False + return values diff --git a/odoo_project_dependency_resolver/models/odoo_project.py b/odoo_project_dependency_resolver/models/odoo_project.py new file mode 100644 index 00000000..db652b3a --- /dev/null +++ b/odoo_project_dependency_resolver/models/odoo_project.py @@ -0,0 +1,280 @@ +# Copyright 2026 ACSONE SA/NV () +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl) + +import logging + +import git + +from odoo import _, fields, models +from odoo.exceptions import UserError + +from odoo.addons.queue_job.delay import group +from odoo.addons.queue_job.job import identity_exact + +_logger = logging.getLogger(__name__) + + +class OdooProject(models.Model): + _inherit = "odoo.project" + + dependency_management = fields.Selection( + selection=[("manual", "Manual"), ("resolved", "Resolved")], + default="manual", + required=True, + help=( + "Manual: the list of modules is maintained by hand.\n" + "Resolved: it is rebuilt from a dependency file of the repository " + "every time the project is scanned." + ), + ) + dependency_resolver = fields.Selection( + selection=[("requirements_txt", "requirements.txt")], + default="requirements_txt", + help="Format of the dependency file to read.", + ) + dependency_source_path = fields.Char( + default="requirements.txt", + help="Path of the dependency file, relative to the root of the repository.", + ) + app_module_ids = fields.Many2many( + comodel_name="odoo.module.branch", + relation="odoo_project_app_module_rel", + column1="odoo_project_id", + column2="module_branch_id", + string="Additional Modules", + help=( + "Modules installed from somewhere the dependency file does not " + "reach, Odoo Enterprise typically. The modules hosted by the " + "project repository are added on their own, and Odoo standard " + "modules are pulled from the dependency graph: neither of them " + "has to be declared here." + ), + ) + last_resolution_date = fields.Datetime(readonly=True) + resolution_log = fields.Text( + readonly=True, + help="What the last resolution could not do, and what it had to guess.", + ) + + def action_scan(self, force=False): + # Resolved projects are scanned in two passes: + # 1) the project repository is scanned first so the dependency file is read + # from a fresh clone, and + # 2) the dependencies are resolved, then their originating repositories are + # scanned to complete the dependency graph. + # Other projects keep the standard super() flow. + # For resolved projects, we trigger the repository scan directly here because + # the repository override of _create_subsequent_jobs schedules dependency + # resolution as a follow-up job of the repository scan. + resolved = self.filtered(lambda p: p.dependency_management == "resolved") + res = super(OdooProject, self - resolved).action_scan(force=force) + for project in resolved: + if not project.repository_id: + raise UserError( + _("Define the repository of project %s to resolve its modules.") + % project.name + ) + project.repository_id.action_scan(force=force, raise_exc=True) + return res + + def action_resolve_dependencies(self): + """Rebuild the list of modules from the dependency file of the project.""" + for project in self: + project._resolve_dependencies() + return True + + def _create_job_resolve_dependencies(self, scan_repositories=False): + """Return the job resolving the modules of this project.""" + self.ensure_one() + delayable = self.delayable( + description=f"Resolve the modules of {self.display_name}", + identity_key=identity_exact, + ) + return delayable._resolve_dependencies_job(scan_repositories=scan_repositories) + + def _resolve_dependencies_job(self, scan_repositories=False): + """Resolve the project dependencies from a job. + + Record expected resolution errors on the project instead of failing the job. + """ + self.ensure_one() + try: + self._resolve_dependencies() + except UserError as exc: + _logger.warning( + "Cannot resolve the modules of %s: %s", self.display_name, exc + ) + self.sudo().write( + { + "last_resolution_date": fields.Datetime.now(), + "resolution_log": str(exc), + } + ) + return True + if scan_repositories: + self._scan_resolved_repositories() + return True + + def _scan_resolved_repositories(self): + """ + Second pass: scan the repositories discovered from the resolved dependencies. + """ + self.ensure_one() + repositories = self._get_repositories_to_scan() - self.repository_id + branches = self._get_branches_to_scan() + if not repositories or not branches: + return False + # NOTE: the scan stays incremental whether the first pass was forced or + # not. Forcing the scan of a project is about the project, not about + # re-collecting the whole history of every repository it depends on. + scan_jobs = repositories.with_context( + strict_branches_scan=True + )._create_scan_jobs(branch_ids=branches.ids, raise_exc=False) + if not scan_jobs: + return False + group(*scan_jobs).on_done(self._create_job_resolve_dependencies()).delay() + return True + + def _resolve_dependencies(self): + """Refresh the modules of this project from its dependency file.""" + self.ensure_one() + content = self._read_dependency_source() + log = [] + backend = self.env.ref("odoo_repository.mca_backend") + usage = f"project.dependency.resolver.{self.dependency_resolver}" + with backend.work_on(self._name) as work: + dependencies = work.component(usage=usage).resolve(content) + project_modules = self.env["odoo.project.module"] + for dependency in dependencies: + project_modules |= self._apply_resolved_dependency(dependency, log) + project_modules |= self._resolve_undeclared_modules(log) + project_modules |= self._resolve_missing_dependencies(project_modules, log) + self._remove_unresolved_modules(project_modules, log) + self.sudo().write( + { + "last_resolution_date": fields.Datetime.now(), + "resolution_log": "\n".join(log), + } + ) + return project_modules + + def _read_dependency_source(self): + """Return the content of the dependency file of this project.""" + self.ensure_one() + if not self.repository_id: + raise UserError( + _("Define the repository of project %s to resolve its modules.") + % self.name + ) + clone_path = self.repository_id._get_local_clone_path() + if not clone_path.joinpath(".git").exists(): + raise UserError( + _("Repository %s has not been cloned yet, scan it first.") + % self.repository_id.display_name + ) + branch = self.repository_branch_id.cloned_branch or self.odoo_version_id.name + revision = f"remotes/origin/{branch}:{self.dependency_source_path}" + # NOTE: read the file straight from the object database. Checking the + # branch out would fight with the scan jobs, which check branches out + # in this very clone. + try: + with git.Repo(clone_path) as repo: + return repo.git.show(revision) + except git.exc.GitError as exc: + raise UserError( + _("Cannot read %(revision)s in %(repository)s: %(error)s") + % { + "revision": revision, + "repository": self.repository_id.display_name, + "error": exc, + } + ) from exc + + def _apply_resolved_dependency(self, dependency, log): + """Turn a resolved dependency into a module of this project.""" + self.ensure_one() + module = self.env["odoo.module.branch"]._get_module(dependency.module_name) + if module.blacklisted: + return self.env["odoo.project.module"] + source, upstream = self._get_dependency_repositories(dependency, log) + module_branch = self._get_module_branch(module, repository=upstream) + project_module = self._get_project_module(module_branch, dependency.version) + project_module.sudo().write( + { + "source_repository_id": source.id, + "source_clone_url": dependency.clone_url, + "source_ref": dependency.ref, + "source_subdirectory": dependency.subdirectory, + } + ) + # A pinned revision is read from its own repository, which has to be + # cloned first: too long to be done here, one dependency among + # hundreds, and worth retrying on its own when the network fails. + if dependency.is_pinned and project_module.analysed_ref != dependency.ref: + project_module._create_job_analyse_pinned_revision().delay() + return project_module + + def _get_dependency_repositories(self, dependency, log): + """Return the source and upstream repositories of a resolved dependency.""" + self.ensure_one() + empty = self.env["odoo.repository"] + if not dependency.clone_url: + return (empty, None) + repository = empty._find_from_clone_url(dependency.clone_url) + if not repository: + log.append( + _("%(module)s: unknown origin for %(url)s") + % {"module": dependency.module_name, "url": dependency.clone_url} + ) + return (empty, None) + return (repository, repository.upstream_repository_id or repository) + + def _resolve_undeclared_modules(self, log): + """ + Return modules not declared in the dependency file but belonging to the project. + """ + self.ensure_one() + hosted_modules = self.repository_branch_id.module_ids + resolved = self.env["odoo.project.module"] + for module_branch in hosted_modules | self.app_module_ids: + resolved |= self._get_project_module(module_branch, module_branch.version) + if hosted_modules: + log.append( + _("Added %s module(s) hosted by the project repository.") + % len(hosted_modules) + ) + return resolved + + def _resolve_missing_dependencies(self, project_modules, log): + """Return the missing dependencies of the resolved modules. + + Odoo standard modules are shipped with Odoo itself, so they show up in + no dependency file: they are pulled from the dependency graph. + """ + self.ensure_one() + module_branches = project_modules.module_branch_id + missing = module_branches._get_recursive_dependencies() - module_branches + resolved = self.env["odoo.project.module"] + for module_branch in missing: + resolved |= self._get_project_module(module_branch, module_branch.version) + if missing: + log.append( + _("Added %s module(s) pulled from the dependency graph.") % len(missing) + ) + return resolved + + def _remove_unresolved_modules(self, project_modules, log): + """Drop the modules of this project that are no longer declared.""" + self.ensure_one() + outdated = self.project_module_ids - project_modules + if not outdated: + return self.env["odoo.project.module"] + log.append( + _("Removed %(count)s module(s) no longer declared: %(modules)s") + % { + "count": len(outdated), + "modules": ", ".join(sorted(outdated.mapped("module_name"))), + } + ) + outdated.sudo().unlink() + return outdated diff --git a/odoo_project_dependency_resolver/models/odoo_project_module.py b/odoo_project_dependency_resolver/models/odoo_project_module.py new file mode 100644 index 00000000..edae4b22 --- /dev/null +++ b/odoo_project_dependency_resolver/models/odoo_project_module.py @@ -0,0 +1,143 @@ +# Copyright 2026 ACSONE SA/NV () +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl) + +import logging + +import git + +from odoo import _, api, fields, models + +from odoo.addons.queue_job.exception import RetryableJobError +from odoo.addons.queue_job.job import identity_exact + +from ..lib.scanner import PinnedModuleScanner + +_logger = logging.getLogger(__name__) + + +class OdooProjectModule(models.Model): + _inherit = "odoo.project.module" + + source_repository_id = fields.Many2one( + comodel_name="odoo.repository", + string="Installed From", + help=( + "Fork the module is installed from, when it differs from the " + "repository it belongs to. Its own 'Fork Of' tells where the " + "module comes from, which is the only way to reach it for a module " + "living in the fork alone." + ), + ) + source_clone_url = fields.Char( + string="Source URL", + help=( + "Repository the module is installed from, as the dependency file " + "spells it. Kept even when it matches no known repository, which " + "is then the only trace of where the module comes from." + ), + ) + source_ref = fields.Char( + string="Source Revision", + help="Revision the module is frozen on: a commit, a tag or a PR ref.", + ) + source_subdirectory = fields.Char( + help="Path of the module within its source repository.", + ) + is_pinned = fields.Boolean( + compute="_compute_is_pinned", + store=True, + help="The module is frozen on a revision instead of a published version.", + ) + analysed_ref = fields.Char( + string="Analysed Revision", + readonly=True, + help=( + "Revision the code of this module was last analysed at. A pinned " + "revision never moves, so it is worth analysing once and never " + "again." + ), + ) + + @api.depends("source_clone_url", "source_ref") + def _compute_is_pinned(self): + for rec in self: + rec.is_pinned = bool(rec.source_clone_url and rec.source_ref) + + @api.depends("version", "installed_version", "is_pinned") + def _compute_to_upgrade(self): + # Without an installed version the base computation falls back on the + # upstream one, which reads as up to date. A module pinned on a + # revision whose version could not be read is precisely the opposite: + # nothing is known about it. + res = super()._compute_to_upgrade() + for rec in self: + if rec.is_pinned and not rec.installed_version: + rec.to_upgrade = True + return res + + def action_analyse_pinned_revision(self): + """Analyse the code of the pinned modules of this selection.""" + for rec in self.filtered("is_pinned"): + rec._create_job_analyse_pinned_revision().delay() + return True + + def _create_job_analyse_pinned_revision(self): + """Return the job analysing this module at its pinned revision.""" + self.ensure_one() + delayable = self.delayable( + description=( + f"Analyse {self.module_name} at {self.source_ref} " + f"in {self.source_repository_id.display_name}" + ), + identity_key=identity_exact, + ) + return delayable._analyse_pinned_revision() + + def _analyse_pinned_revision(self): + """Analyse the code of this module at the revision it is pinned on. + + A pinned revision is the head of a pull request more often than of a + branch, so no repository scan ever reaches it: neither the version of + the module nor the analysis of its code can come from anywhere else. + """ + self.ensure_one() + repository = self.source_repository_id + if not repository or not self.source_ref: + return False + module_branch = self.module_branch_id + scanner = PinnedModuleScanner( + branches=[], **repository._prepare_base_scanner_parameters() + ) + try: + data = scanner.scan_module_at_revision( + module_branch.full_path, self.source_ref + ) + except git.exc.GitError as exc: + raise RetryableJobError( + _("Cannot read %(module)s at %(ref)s in %(repository)s") + % { + "module": module_branch.full_path, + "ref": self.source_ref, + "repository": repository.display_name, + } + ) from exc + if data is None: + _logger.warning( + "%s: %s holds no %s at %s", + self.display_name, + repository.display_name, + module_branch.full_path, + self.source_ref, + ) + return False + values = {"analysed_ref": self.source_ref} + version = (data.get("manifest") or {}).get("version") + if version: + values["installed_version"] = version + self.sudo().write(values) + # The module branch is shared with the repository the modules belong + # to, whose scan is authoritative on it. Only one no scan ever reaches, + # living in a pull request alone, takes its data from here. + if not module_branch.last_scanned_commit: + module_branch._update_from_pinned_revision(data) + return True diff --git a/odoo_project_dependency_resolver/models/odoo_repository.py b/odoo_project_dependency_resolver/models/odoo_repository.py new file mode 100644 index 00000000..5f2a16d7 --- /dev/null +++ b/odoo_project_dependency_resolver/models/odoo_repository.py @@ -0,0 +1,40 @@ +# Copyright 2026 ACSONE SA/NV () +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl) + +from odoo import models + + +class OdooRepository(models.Model): + _inherit = "odoo.repository" + + def _create_subsequent_jobs( + self, version_branch, next_versions_branches, all_versions_branches, data + ): + # Appending the resolution to the chain of the scan jobs is what makes + # it read an up to date dependency file: the branch has just been + # fetched by the job detecting the modules to scan. It also runs after + # the modules have been scanned, so the dependency graph it walks to + # complete the application modules is the fresh one. + jobs = super()._create_subsequent_jobs( + version_branch, next_versions_branches, all_versions_branches, data + ) + if not data: + # The branch does not exist, there is nothing to read + return jobs + version, __ = version_branch + for project in self._get_projects_to_resolve(version): + jobs.append( + project._create_job_resolve_dependencies(scan_repositories=True) + ) + return jobs + + def _get_projects_to_resolve(self, version): + """Return the projects whose dependency file this repository holds.""" + self.ensure_one() + return self.env["odoo.project"].search( + [ + ("dependency_management", "=", "resolved"), + ("odoo_version_id.name", "=", version), + ("repository_id", "=", self.id), + ] + ) diff --git a/odoo_project_dependency_resolver/pyproject.toml b/odoo_project_dependency_resolver/pyproject.toml new file mode 100644 index 00000000..4231d0cc --- /dev/null +++ b/odoo_project_dependency_resolver/pyproject.toml @@ -0,0 +1,3 @@ +[build-system] +requires = ["whool"] +build-backend = "whool.buildapi" diff --git a/odoo_project_dependency_resolver/readme/CONTRIBUTORS.md b/odoo_project_dependency_resolver/readme/CONTRIBUTORS.md new file mode 100644 index 00000000..c4904374 --- /dev/null +++ b/odoo_project_dependency_resolver/readme/CONTRIBUTORS.md @@ -0,0 +1,2 @@ +- ACSONE SA/NV + - Laurent Mignon \ diff --git a/odoo_project_dependency_resolver/readme/DESCRIPTION.md b/odoo_project_dependency_resolver/readme/DESCRIPTION.md new file mode 100644 index 00000000..f76f710b --- /dev/null +++ b/odoo_project_dependency_resolver/readme/DESCRIPTION.md @@ -0,0 +1,7 @@ +Projects usually pin the exact version of every module they install in a +dependency file, such as the `requirements.txt` produced by pip. Maintaining +the very same list a second time by hand in Odoo MCA is tedious and drifts +away from reality as soon as a dependency is bumped. + +This module reads that file straight from the repository of the project and +rebuilds its modules from it, every time the project is scanned. diff --git a/odoo_project_dependency_resolver/readme/USAGE.md b/odoo_project_dependency_resolver/readme/USAGE.md new file mode 100644 index 00000000..838a0e5d --- /dev/null +++ b/odoo_project_dependency_resolver/readme/USAGE.md @@ -0,0 +1,55 @@ +On the project, set *Dependency Management* to *Resolved*, pick the format of +the dependency file and, if it is not at the root of the repository under its +usual name, adjust its path. + +There is nothing else to declare. A dependency file only holds packaged +addons, so the modules of a resolved project are the union of three sources: + +- the addons of the dependency file, with the versions it pins; +- the modules hosted by the project repository, which are the code of the + project itself and are shipped with it rather than installed as a package; +- their missing dependencies, pulled from the dependency graph, which is + where Odoo standard modules come from. + +*Additional Modules* is there for what none of the three reaches, Odoo +Enterprise modules typically. + +The modules are refreshed at each scan, and by the *Resolve Dependencies* +button. Whatever the resolution could not do is reported on the project, in +the *Resolution Log* field. + +## What a scan does + +Scanning a resolved project only scans the repository of the project: the +repositories its modules come from are derived from the dependency file, so +none of them is known before that file has been read. + +The resolution is chained to that scan, so it reads a file that has just been +fetched, and it then spawns the scan of the repositories it discovered. A +second resolution runs once they have all been scanned, completing what the +first one could not know: the modules of a repository that had never been +scanned, and the dependency graph the resolved modules are walked through. + +Note that the second resolution waits for the detection of the modules to +scan in every repository, not for every module to have been scanned: the +scanner spawns the latter as it goes. A module scanned late is therefore +accounted for by the next scan of the project. + +## Modules pinned on a fork + +A module frozen on a commit of a fork, such as: + + odoo-addon-account-invoice-triple-discount @ + git+https://github.com/acsone/account-invoicing.git@3e4c54b + #subdirectory=setup/account_invoice_triple_discount + +is not treated as a new module. It stays attached to the module of the +repository the fork originates from, so that dependencies, migration scripts +and changelogs keep working, and the fork is recorded on the project module as +*Installed From*. Its version is read from the manifest at the pinned revision. + +A module added by a pull request not merged yet is a special case: it exists +in no repository known to Odoo MCA, so it stays an orphaned module and shows +up among the unknown modules of the project. *Installed From* is then the only +way to reach a repository from it, and the *Fork Of* of that fork tells where +the module is heading. diff --git a/odoo_project_dependency_resolver/static/description/index.html b/odoo_project_dependency_resolver/static/description/index.html new file mode 100644 index 00000000..ca8b92bc --- /dev/null +++ b/odoo_project_dependency_resolver/static/description/index.html @@ -0,0 +1,493 @@ + + + + + +Odoo Project - Dependency Resolver + + + +
+

Odoo Project - Dependency Resolver

+ + +

Beta License: AGPL-3 OCA/module-composition-analysis Translate me on Weblate Try me on Runboat

+

Projects usually pin the exact version of every module they install in a +dependency file, such as the requirements.txt produced by pip. +Maintaining the very same list a second time by hand in Odoo MCA is +tedious and drifts away from reality as soon as a dependency is bumped.

+

This module reads that file straight from the repository of the project +and rebuilds its modules from it, every time the project is scanned.

+

Table of contents

+ +
+

Usage

+

On the project, set Dependency Management to Resolved, pick the +format of the dependency file and, if it is not at the root of the +repository under its usual name, adjust its path.

+

There is nothing else to declare. A dependency file only holds packaged +addons, so the modules of a resolved project are the union of three +sources:

+
    +
  • the addons of the dependency file, with the versions it pins;
    • +
  • the modules hosted by the project repository, which are the code of +the project itself and are shipped with it rather than installed as a +package;
    • +
  • their missing dependencies, pulled from the dependency graph, which is +where Odoo standard modules come from.
    • +
+

Additional Modules is there for what none of the three reaches, Odoo +Enterprise modules typically.

+

The modules are refreshed at each scan, and by the Resolve +Dependencies button. Whatever the resolution could not do is reported +on the project, in the Resolution Log field.

+
+

What a scan does

+

Scanning a resolved project only scans the repository of the project: +the repositories its modules come from are derived from the dependency +file, so none of them is known before that file has been read.

+

The resolution is chained to that scan, so it reads a file that has just +been fetched, and it then spawns the scan of the repositories it +discovered. A second resolution runs once they have all been scanned, +completing what the first one could not know: the modules of a +repository that had never been scanned, and the dependency graph the +resolved modules are walked through.

+

Note that the second resolution waits for the detection of the modules +to scan in every repository, not for every module to have been scanned: +the scanner spawns the latter as it goes. A module scanned late is +therefore accounted for by the next scan of the project.

+
+
+

Modules pinned on a fork

+

A module frozen on a commit of a fork, such as:

+
+odoo-addon-account-invoice-triple-discount @
+    git+https://github.com/acsone/account-invoicing.git@3e4c54b
+    #subdirectory=setup/account_invoice_triple_discount
+
+

is not treated as a new module. It stays attached to the module of the +repository the fork originates from, so that dependencies, migration +scripts and changelogs keep working, and the fork is recorded on the +project module as Installed From. Its version is read from the +manifest at the pinned revision.

+

A module added by a pull request not merged yet is a special case: it +exists in no repository known to Odoo MCA, so it stays an orphaned +module and shows up among the unknown modules of the project. Installed +From is then the only way to reach a repository from it, and the Fork +Of of that fork tells where the module is heading.

+
+
+
+

Bug Tracker

+

Bugs are tracked on GitHub Issues. +In case of trouble, please check there if your issue has already been reported. +If you spotted it first, help us to smash it by providing a detailed and welcomed +feedback.

+

Do not contact contributors directly about support or help with technical issues.

+
+
+

Credits

+
+

Authors

+
    +
  • ACSONE SA/NV
    • +
+
+
+

Contributors

+ +
+
+

Maintainers

+

This module is maintained by the OCA.

+ +Odoo Community Association + +

OCA, or the Odoo Community Association, is a nonprofit organization whose +mission is to support the collaborative development of Odoo features and +promote its widespread use.

+

This module is part of the OCA/module-composition-analysis project on GitHub.

+

You are welcome to contribute. To learn how please visit https://odoo-community.org/page/Contribute.

+
+
+
+ + diff --git a/odoo_project_dependency_resolver/tests/__init__.py b/odoo_project_dependency_resolver/tests/__init__.py new file mode 100644 index 00000000..bbc6a685 --- /dev/null +++ b/odoo_project_dependency_resolver/tests/__init__.py @@ -0,0 +1,2 @@ +from . import test_requirements_txt_resolver +from . import test_resolve_dependencies diff --git a/odoo_project_dependency_resolver/tests/test_requirements_txt_resolver.py b/odoo_project_dependency_resolver/tests/test_requirements_txt_resolver.py new file mode 100644 index 00000000..8afdf2c2 --- /dev/null +++ b/odoo_project_dependency_resolver/tests/test_requirements_txt_resolver.py @@ -0,0 +1,163 @@ +# Copyright 2026 ACSONE SA/NV () +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl) + +from odoo.addons.component.tests.common import TransactionComponentCase + +# Excerpt of a 'requirements.txt' frozen by pip-deepfreeze +REQUIREMENTS = """\ +# frozen requirements generated by pip-deepfreeze +aiohttp==3.14.3 +click-odoo==1.8.0 +gitpython==3.1.58 +odoo @ git+https://github.com/acsone/odoo.git@8cc9bd3c893dfd8b43b4f1af287e5e9f +odoo-addon-base-time-window==18.0.1.1.1 +odoo-addon-component==18.0.1.0.5.1 +odoo-addon-web-responsive==18.0.1.0.7 +odoo-addons-parser==0.8 +""" + + +class TestRequirementsTxtResolver(TransactionComponentCase): + @classmethod + def setUpClass(cls): + super().setUpClass() + cls.backend = cls.env.ref("odoo_repository.mca_backend") + + def _resolve(self, content): + with self.backend.work_on("odoo.project") as work: + resolver = work.component( + usage="project.dependency.resolver.requirements_txt" + ) + return resolver.resolve(content) + + def _resolve_one(self, requirement): + dependencies = self._resolve(requirement) + self.assertEqual(len(dependencies), 1, requirement) + return dependencies[0] + + def test_resolve_keeps_addons_only(self): + """Only the addons of a requirements file are resolved.""" + dependencies = self._resolve(REQUIREMENTS) + self.assertEqual( + [dependency.module_name for dependency in dependencies], + ["base_time_window", "component", "web_responsive"], + ) + + def test_resolve_odoo_itself_is_not_an_addon(self): + """The 'odoo' requirement pins the core, not a module.""" + self.assertFalse( + self._resolve("odoo @ git+https://github.com/odoo/odoo.git@a1") + ) + + def test_resolve_lookalike_distribution(self): + """'odoo-addons-parser' is a library, not an addon.""" + self.assertFalse(self._resolve("odoo-addons-parser==0.8")) + + def test_resolve_version(self): + """A pinned version is the version of the module.""" + dependency = self._resolve_one("odoo-addon-web-responsive==18.0.1.0.7") + self.assertEqual(dependency.version, "18.0.1.0.7") + self.assertFalse(dependency.is_pinned) + + def test_resolve_version_drops_the_packaging_serial(self): + """A serial appended by the packaging is not part of the manifest version.""" + dependency = self._resolve_one("odoo-addon-odoo-repository==18.0.1.2.0.3") + self.assertEqual(dependency.module_name, "odoo_repository") + self.assertEqual(dependency.version, "18.0.1.2.0") + + def test_resolve_version_drops_the_local_segment(self): + dependency = self._resolve_one("odoo-addon-component==18.0.1.0.5+local") + self.assertEqual(dependency.version, "18.0.1.0.5") + + def test_resolve_unpinned_requirement(self): + """A requirement without an exact version yields no version.""" + dependency = self._resolve_one("odoo-addon-component>=18.0.1.0.5") + self.assertEqual(dependency.module_name, "component") + self.assertFalse(dependency.version) + + def test_resolve_legacy_prefix(self): + """Addons were distributed as 'odoo-addon-' before 15.0.""" + dependency = self._resolve_one("odoo14-addon-web-responsive==14.0.1.0.1") + self.assertEqual(dependency.module_name, "web_responsive") + + def test_resolve_pinned_on_a_fork(self): + """A fork commit is recorded as the origin of the module.""" + dependency = self._resolve_one( + "odoo-addon-account-invoice-triple-discount @ " + "git+https://github.com/acsone/account-invoicing.git" + "@3e4c54bb1a3a34b4b031b3dec9ccf6b2997d1295" + "#subdirectory=setup/account_invoice_triple_discount" + ) + self.assertEqual(dependency.module_name, "account_invoice_triple_discount") + self.assertEqual( + dependency.clone_url, "https://github.com/acsone/account-invoicing.git" + ) + self.assertEqual(dependency.ref, "3e4c54bb1a3a34b4b031b3dec9ccf6b2997d1295") + self.assertEqual( + dependency.subdirectory, "setup/account_invoice_triple_discount" + ) + self.assertTrue(dependency.is_pinned) + # A revision carries no version + self.assertFalse(dependency.version) + + def test_resolve_pinned_on_a_pull_request(self): + """A PR ref holds slashes, which must not be mistaken for a path.""" + dependency = self._resolve_one( + "odoo-addon-account-invoice-triple-discount @ " + "git+https://github.com/OCA/account-invoicing.git" + "@refs/pull/2279/head" + "#subdirectory=setup/account_invoice_triple_discount" + ) + self.assertEqual( + dependency.clone_url, "https://github.com/OCA/account-invoicing.git" + ) + self.assertEqual(dependency.ref, "refs/pull/2279/head") + + def test_resolve_subdirectory_wins_over_the_distribution_name(self): + """The subdirectory names the module, the distribution name is normalized.""" + dependency = self._resolve_one( + "odoo-addon-l10n-be-coa-multilang @ " + "git+https://github.com/OCA/l10n-belgium.git@18.0" + "#subdirectory=setup/l10n_be_coa_multilang" + ) + self.assertEqual(dependency.module_name, "l10n_be_coa_multilang") + + def test_resolve_url_with_credentials(self): + """Credentials in the URL are not mistaken for a revision separator.""" + dependency = self._resolve_one( + "odoo-addon-component @ " + "git+https://user@example.com/acsone/addons.git@18.0" + "#subdirectory=setup/component" + ) + self.assertEqual( + dependency.clone_url, "https://user@example.com/acsone/addons.git" + ) + self.assertEqual(dependency.ref, "18.0") + + def test_resolve_ignores_comments_and_options(self): + content = ( + "# a comment\n" + "-r base.txt\n" + "--index-url https://example.com/simple\n" + "odoo-addon-component==18.0.1.0.5 # trailing comment\n" + ) + dependency = self._resolve_one(content) + self.assertEqual(dependency.module_name, "component") + self.assertEqual(dependency.version, "18.0.1.0.5") + + def test_resolve_ignores_environment_markers(self): + dependency = self._resolve_one( + 'odoo-addon-component==18.0.1.0.5 ; python_version >= "3.10"' + ) + self.assertEqual(dependency.version, "18.0.1.0.5") + + def test_resolve_joins_continuation_lines(self): + content = ( + "odoo-addon-component==18.0.1.0.5 \\\n" + " --hash=sha256:0000000000000000000000000000000000000000\n" + ) + dependency = self._resolve_one(content) + self.assertEqual(dependency.version, "18.0.1.0.5") + + def test_resolve_empty_content(self): + self.assertEqual(self._resolve(""), []) diff --git a/odoo_project_dependency_resolver/tests/test_resolve_dependencies.py b/odoo_project_dependency_resolver/tests/test_resolve_dependencies.py new file mode 100644 index 00000000..64273b92 --- /dev/null +++ b/odoo_project_dependency_resolver/tests/test_resolve_dependencies.py @@ -0,0 +1,576 @@ +# Copyright 2026 ACSONE SA/NV () +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl) + +import contextlib +import gc +import pathlib +import tempfile +from unittest.mock import patch + +import git + +from odoo.exceptions import UserError +from odoo.tools import mute_logger + +from odoo.addons.odoo_project.tests.common import ProjectCommon + +READ_SOURCE = ( + "odoo.addons.odoo_project_dependency_resolver.models.odoo_project." + "OdooProject._read_dependency_source" +) + + +class TestResolveDependencies(ProjectCommon): + @classmethod + def setUpClass(cls): + super().setUpClass() + cls.project.write( + { + "repository_id": cls.odoo_repository.id, + "dependency_management": "resolved", + } + ) + cls.project_module_model = cls.env["odoo.project.module"] + + def _resolve(self, content): + with patch(READ_SOURCE, return_value=content): + self.project._resolve_dependencies() + self.env.flush_all() + return self.project.project_module_ids + + def _module_versions(self): + return { + project_module.module_name: project_module.installed_version + for project_module in self.project.project_module_ids + } + + # -- mapping onto project modules -------------------------------------- + + def test_resolve_creates_project_modules(self): + modules = self._resolve( + "odoo-addon-test1==15.0.1.0.0\nodoo-addon-test2==15.0.2.0.0\n" + ) + self.assertEqual(len(modules), 2) + self.assertEqual( + self._module_versions(), + {"test1": "15.0.1.0.0", "test2": "15.0.2.0.0"}, + ) + + def test_resolve_updates_the_installed_version(self): + """Resolving again bumps the version instead of duplicating the module.""" + self._resolve("odoo-addon-test1==15.0.1.0.0\n") + modules = self._resolve("odoo-addon-test1==15.0.1.1.0\n") + self.assertEqual(len(modules), 1) + self.assertEqual(self._module_versions(), {"test1": "15.0.1.1.0"}) + + def test_resolve_removes_undeclared_modules(self): + """A module dropped from the dependency file leaves the project.""" + self._resolve("odoo-addon-test1==15.0.1.0.0\nodoo-addon-test2==15.0.2.0.0\n") + self._resolve("odoo-addon-test1==15.0.1.0.0\n") + self.assertEqual(list(self._module_versions()), ["test1"]) + self.assertIn("test2", self.project.resolution_log) + + def test_resolve_skips_blacklisted_modules(self): + module = self.module_branch_model._get_module("test1") + module.blacklisted = True + self._resolve("odoo-addon-test1==15.0.1.0.0\nodoo-addon-test2==15.0.2.0.0\n") + self.assertEqual(list(self._module_versions()), ["test2"]) + + def test_resolve_pulls_the_missing_dependencies(self): + """Odoo standard modules show up in no dependency file.""" + dependency = self._create_odoo_module_branch( + self.module_branch_model._get_module("base"), + self.branch, + is_standard=True, + ) + app_module = self._create_odoo_module_branch( + self.module_branch_model._get_module("my_app"), + self.branch, + dependency_ids=[(4, dependency.id)], + ) + self.project.app_module_ids = app_module + self._resolve("odoo-addon-test1==15.0.1.0.0\n") + self.assertEqual(sorted(self._module_versions()), ["base", "my_app", "test1"]) + + def test_resolve_keeps_the_additional_modules(self): + """An additional module is never purged, the file does not hold it.""" + app_module = self._create_odoo_module_branch( + self.module_branch_model._get_module("my_app"), self.branch + ) + self.project.app_module_ids = app_module + self._resolve("odoo-addon-test1==15.0.1.0.0\n") + self.assertIn("my_app", self._module_versions()) + + # -- modules hosted by the project repository --------------------------- + + def _create_specific_module(self, name="my_specific_module"): + """Add a module to the repository of the project.""" + repository_branch = self.env["odoo.repository.branch"].search( + [ + ("repository_id", "=", self.odoo_repository.id), + ("branch_id", "=", self.branch.id), + ] + ) + if not repository_branch: + repository_branch = self._create_odoo_repository_branch( + self.odoo_repository, self.branch + ) + return self._create_odoo_module_branch( + self.module_branch_model._get_module(name), + self.branch, + specific=True, + repository_branch_id=repository_branch.id, + version="15.0.1.0.0", + ) + + def test_resolve_adds_the_modules_of_the_project_repository(self): + """The code of the project is shipped with it, not as a package.""" + specific_module = self._create_specific_module() + modules = self._resolve("odoo-addon-test1==15.0.1.0.0\n") + self.assertEqual( + sorted(self._module_versions()), ["my_specific_module", "test1"] + ) + # Their version is the one of the repository, nothing pins them + project_module = modules.filtered( + lambda m: m.module_branch_id == specific_module + ) + self.assertEqual(project_module.installed_version, "15.0.1.0.0") + self.assertFalse(project_module.to_upgrade) + + def test_resolve_does_not_purge_the_modules_of_the_project_repository(self): + """Resolving twice must not drop the code of the project.""" + self._create_specific_module() + self._resolve("odoo-addon-test1==15.0.1.0.0\n") + self._resolve("odoo-addon-test1==15.0.1.0.0\n") + self.assertIn("my_specific_module", self._module_versions()) + + # -- modules pinned on a fork ------------------------------------------ + + def _create_upstream_repository(self, with_module=True): + org_model = self.env["odoo.repository.org"].with_context(active_test=False) + org = org_model.search([("name", "=", "OCA")]) + if not org: + org = org_model.create({"name": "OCA"}) + repository = self.env["odoo.repository"].create( + { + "org_id": org.id, + "name": "account-invoicing", + "repo_url": "https://github.com/OCA/account-invoicing", + "clone_url": "https://github.com/OCA/account-invoicing.git", + "repo_type": "github", + } + ) + repository_branch = self._create_odoo_repository_branch(repository, self.branch) + module_branch = self.module_branch_model.browse() + if with_module: + module_branch = self._create_odoo_module_branch( + self.module_branch_model._get_module("account_invoice_triple_discount"), + self.branch, + specific=False, + repository_branch_id=repository_branch.id, + version="15.0.1.0.0", + # The scan of the upstream repository is authoritative on it + last_scanned_commit="c0ffee", + ) + return repository, module_branch + + def _create_fork(self, module_name, version, on_pull_request=False): + """Return the ``(path, revision)`` of a fork holding `module_name`. + + With `on_pull_request` the module is added on a branch of its own, as + a pull request does: the revision then belongs to no branch the clone + of the fork tracks, and has to be fetched on its own. + """ + fork_path = pathlib.Path(tempfile.mkdtemp()) + fork = git.Repo.init(fork_path) + fork.config_writer().set_value("user", "name", "test").release() + fork.config_writer().set_value("user", "email", "test@example.com").release() + # Serving a bare revision is what GitHub does, but not the local default + fork.config_writer().set_value( + "uploadpack", "allowAnySHA1InWant", "true" + ).release() + self.addCleanup(fork.close) + readme = fork_path.joinpath("README.md") + readme.write_text("A fork\n") + fork.index.add([str(readme)]) + fork.index.commit("Initial commit") + default_branch = fork.active_branch.name + if on_pull_request: + fork.git.checkout("-b", "add-module") + module_path = fork_path.joinpath(module_name) + module_path.mkdir() + manifest = module_path.joinpath("__manifest__.py") + manifest.write_text( + "{" + f'"name": "Test", "version": "{version}", ' + '"license": "AGPL-3", "depends": ["base"]' + "}" + ) + code = module_path.joinpath("models.py") + code.write_text( + "from odoo import fields, models\n" + "\n" + "\n" + "class Foo(models.Model):\n" + ' _name = "foo"\n' + "\n" + " name = fields.Char()\n" + ) + fork.index.add([str(manifest), str(code)]) + commit = fork.index.commit(f"Add {module_name} {version}") + if on_pull_request: + fork.git.checkout(default_branch) + return fork_path, commit.hexsha + + def test_resolve_fork_keeps_the_upstream_module(self): + """A fork commit does not create a module of its own.""" + __, upstream_module = self._create_upstream_repository() + content = ( + "odoo-addon-account-invoice-triple-discount @ " + "git+https://github.com/acsone/account-invoicing.git@3e4c54b" + "#subdirectory=setup/account_invoice_triple_discount\n" + ) + payload = { + "fork": True, + "parent": {"full_name": "OCA/account-invoicing"}, + "source": {"full_name": "OCA/account-invoicing"}, + } + github_request = ( + "odoo.addons.odoo_repository_fork.models.odoo_repository.github.request" + ) + with patch(github_request, return_value=payload): + modules = self._resolve(content) + self.assertEqual(len(modules), 1) + self.assertEqual(modules.module_branch_id, upstream_module) + self.assertTrue(modules.is_pinned) + self.assertEqual( + modules.source_clone_url, + "https://github.com/acsone/account-invoicing.git", + ) + self.assertEqual(modules.source_ref, "3e4c54b") + # The fork it is installed from, and through it the origin + fork = modules.source_repository_id + self.assertEqual(fork.display_name, "acsone/account-invoicing") + self.assertEqual(fork.upstream_repository_id, upstream_module.repository_id) + + def test_resolve_unreadable_pin_is_reported_as_to_upgrade(self): + """An unresolved pin must not read as an up to date module.""" + self._create_upstream_repository() + content = ( + "odoo-addon-account-invoice-triple-discount @ " + "git+https://github.com/acsone/account-invoicing.git@3e4c54b" + "#subdirectory=setup/account_invoice_triple_discount\n" + ) + with ( + patch( + "odoo.addons.odoo_repository_fork.models.odoo_repository.github.request", + side_effect=RuntimeError("API rate limit"), + ), + mute_logger("odoo.addons.odoo_repository_fork.models.odoo_repository"), + ): + modules = self._resolve(content) + self.assertFalse(modules.installed_version) + self.assertTrue(modules.to_upgrade) + self.assertIn("account_invoice_triple_discount", self.project.resolution_log) + + # -- reading the dependency file --------------------------------------- + + def _init_clone(self, repository, content=""): + """Create the local clone of `repository`, holding a requirements file.""" + clone_path = repository._get_local_clone_path() + clone_path.mkdir(parents=True, exist_ok=True) + repo = git.Repo.init(clone_path) + repo.config_writer().set_value("user", "name", "test").release() + repo.config_writer().set_value("user", "email", "test@example.com").release() + self.addCleanup(repo.close) + source = clone_path.joinpath("requirements.txt") + source.write_text(content) + repo.index.add([str(source)]) + commit = repo.index.commit("Add requirements") + # The scanner clones without checking out, tracking branches only + repo.git.update_ref(f"refs/remotes/origin/{self.branch.name}", commit.hexsha) + return repo + + def test_read_dependency_source(self): + """The file is read from the tracking branch, without checking it out.""" + self._init_clone(self.odoo_repository, "odoo-addon-test1==15.0.1.0.0\n") + self.assertEqual( + self.project._read_dependency_source(), "odoo-addon-test1==15.0.1.0.0" + ) + + def test_read_dependency_source_without_clone(self): + with self.assertRaisesRegex(UserError, "has not been cloned"): + self.project._read_dependency_source() + + def test_read_dependency_source_missing_file(self): + self._init_clone(self.odoo_repository, "odoo-addon-test1==15.0.1.0.0\n") + self.project.dependency_source_path = "does-not-exist.txt" + with self.assertRaisesRegex(UserError, "Cannot read"): + self.project._read_dependency_source() + + def test_read_dependency_source_without_repository(self): + self.project.repository_id = False + with self.assertRaisesRegex(UserError, "Define the repository"): + self.project._read_dependency_source() + + # -- analysis of a pinned revision ------------------------------------- + + def _setup_pinned(self, module_name, version, on_pull_request=False): + """Return the `(fork, content)` of a dependency pinned on a fork.""" + upstream, __ = self._create_upstream_repository( + with_module=module_name == "account_invoice_triple_discount" + ) + fork_path, revision = self._create_fork( + module_name, version, on_pull_request=on_pull_request + ) + fork = self.env["odoo.repository"].create( + { + "org_id": upstream.org_id.id, + "name": "account-invoicing-fork", + "repo_url": f"file://{fork_path}", + "clone_url": f"file://{fork_path}", + "repo_type": "github", + "to_scan": False, + "upstream_repository_id": upstream.id, + } + ) + content = ( + f"odoo-addon-{module_name.replace('_', '-')} @ " + f"git+file://{fork_path}@{revision}" + f"#subdirectory=setup/{module_name}\n" + ) + return fork, content + + def _resolve_pinned(self, fork, content): + """Resolve `content`, standing in for the lookup of the fork. + + The origin of a 'file://' URL cannot be told from the URL itself, and + that lookup is covered on its own. + """ + with patch.object( + self.env.registry["odoo.repository"], + "_find_from_clone_url", + autospec=True, + return_value=fork, + ): + return self._resolve(content) + + def test_analyse_pinned_revision_reads_the_version_of_the_fork(self): + """The version comes from the manifest at the pinned revision.""" + fork, content = self._setup_pinned( + "account_invoice_triple_discount", "15.0.1.1.0" + ) + modules = self._resolve_pinned(fork, content) + # The module branch stays the one of the upstream repository + upstream_module = modules.module_branch_id + self.assertEqual(upstream_module.version, "15.0.1.0.0") + modules._analyse_pinned_revision() + self.assertEqual(modules.installed_version, "15.0.1.1.0") + self.assertEqual(modules.analysed_ref, modules.source_ref) + # A module the upstream repository hosts is not rewritten from a fork + self.assertEqual(upstream_module.version, "15.0.1.0.0") + + def test_analyse_pinned_revision_of_a_module_absent_upstream(self): + """A module living only in a pull request gets its code analysed. + + It is in no branch of any repository, so no scan ever reaches it: the + pinned revision is the only place it can be read from. + """ + fork, content = self._setup_pinned( + "my_pull_request_module", "15.0.1.1.0", on_pull_request=True + ) + modules = self._resolve_pinned(fork, content) + module_branch = modules.module_branch_id + # The module belongs to no repository, it exists only in the fork... + self.assertFalse(module_branch.repository_id) + # ... but the project module says where it is installed from + self.assertEqual(modules.source_repository_id, fork) + modules._analyse_pinned_revision() + self.assertEqual(modules.installed_version, "15.0.1.1.0") + self.assertEqual(module_branch.version, "15.0.1.1.0") + self.assertEqual(module_branch.title, "Test") + self.assertEqual(module_branch.license_id.name, "AGPL-3") + self.assertEqual(module_branch.dependency_ids.mapped("module_name"), ["base"]) + self.assertTrue(module_branch.sloc_python) + + def test_analyse_pinned_revision_of_an_unknown_module(self): + """A revision holding no such module is reported, not raised.""" + fork, content = self._setup_pinned("my_pull_request_module", "15.0.1.1.0") + with mute_logger( + "odoo.addons.odoo_project_dependency_resolver.models.odoo_project_module", + ): + modules = self._resolve_pinned(fork, content) + modules.module_branch_id.addons_path = "does/not/exist" + with mute_logger( + "odoo.addons.odoo_project_dependency_resolver.lib.scanner", + "odoo.addons.odoo_project_dependency_resolver.models.odoo_project_module", + ): + self.assertFalse(modules._analyse_pinned_revision()) + self.assertFalse(modules.analysed_ref) + + def test_resolve_delays_the_analysis_of_a_pinned_revision(self): + """The analysis is a job of its own, and runs once per revision.""" + job_model = self.env["queue.job"] + domain = [("method_name", "=", "_analyse_pinned_revision")] + fork, content = self._setup_pinned( + "account_invoice_triple_discount", "15.0.1.1.0" + ) + modules = self._resolve_pinned(fork, content) + self.assertEqual(job_model.search_count(domain), 1) + # Once analysed, resolving the very same revision queues nothing more + modules.analysed_ref = modules.source_ref + job_model.search(domain).unlink() + self._resolve_pinned(fork, content) + self.assertEqual(job_model.search_count(domain), 0) + + +class TestResolveDependenciesChaining(ProjectCommon): + """The resolution is appended to the chain of the scan jobs.""" + + @classmethod + def setUpClass(cls): + super().setUpClass() + # We want to test the chaining of jobs, not the resolution itself, so + # mute the logger of the delayable to avoid a warning on the job being + # collected without having been delayed. The resolution itself is tested + # elsewhere. + # we could use 'enterClassContext(mute_logger("odoo.addons.queue_job.delay"))' + # in Python>=3.11, where Odoo 18 runs on 3.10. + muted = contextlib.ExitStack() + cls.addClassCleanup(muted.close) + muted.enter_context(mute_logger("odoo.addons.queue_job.delay")) + cls.project.write( + { + "repository_id": cls.odoo_repository.id, + "dependency_management": "resolved", + } + ) + + def tearDown(self): + super().tearDown() + # A delayable warns when collected without having been delayed, which + # is what these tests do on purpose. Collect here, while + # the logger above is still muted, rather than after 'tearDownClass'. + gc.collect() + + def _create_subsequent_jobs(self, data=None, version=None, repository=None): + repository = repository or self.odoo_repository + repository_branch = self.env["odoo.repository.branch"].search( + [("repository_id", "=", repository.id), ("branch_id", "=", self.branch.id)] + ) + if not repository_branch: + repository_branch = self._create_odoo_repository_branch( + repository, self.branch + ) + if data is None: + data = { + "repo_branch_id": repository_branch.id, + "last_fetched_commit": "c0ffee", + "last_scanned_commit": "c0ffee", + "addons_paths": {}, + } + version = version or self.branch.name + return repository._create_subsequent_jobs((version, version), [], [], data) + + def _resolution_jobs(self, jobs): + return [ + job + for job in jobs + if job._job_method.__name__ == "_resolve_dependencies_job" + ] + + def test_resolution_is_chained_after_the_scan(self): + jobs = self._create_subsequent_jobs() + resolution_jobs = self._resolution_jobs(jobs) + self.assertEqual(len(resolution_jobs), 1) + self.assertEqual(resolution_jobs[0].recordset, self.project) + # It runs last, once the modules of the branch have been scanned + self.assertEqual(jobs[-1], resolution_jobs[0]) + # Scanning the repository of the project is the first pass: it is the + # one discovering the repositories the modules come from + self.assertEqual(resolution_jobs[0]._job_kwargs, {"scan_repositories": True}) + + def _create_dependency_repository(self): + """Return a repository the modules of the project come from.""" + org_model = self.env["odoo.repository.org"].with_context(active_test=False) + org = org_model.search([("name", "=", "OCA")]) or org_model.create( + {"name": "OCA"} + ) + repository = self.env["odoo.repository"].create( + { + "org_id": org.id, + "name": "server-tools", + "repo_url": "https://github.com/OCA/server-tools", + "repo_type": "github", + } + ) + repository_branch = self._create_odoo_repository_branch(repository, self.branch) + module_branch = self._create_odoo_module_branch( + self.module_branch_model._get_module("test1"), + self.branch, + repository_branch_id=repository_branch.id, + ) + self.project._get_project_module(module_branch) + return repository + + def test_no_resolution_chained_to_a_dependency_repository(self): + """Only the repository holding the dependency file resolves. + + Chaining one resolution per repository the modules come from would + create as many as the project has dependencies. + """ + repository = self._create_dependency_repository() + jobs = self._create_subsequent_jobs(repository=repository) + self.assertFalse(self._resolution_jobs(jobs)) + + def test_second_pass_scans_the_resolved_repositories(self): + """The first pass scans what the resolution discovered.""" + repository = self._create_dependency_repository() + repository_model = self.env.registry["odoo.repository"] + with patch.object( + repository_model, "_create_scan_jobs", autospec=True, return_value=[] + ) as create_scan_jobs: + self.project._scan_resolved_repositories() + create_scan_jobs.assert_called_once() + scanned = create_scan_jobs.call_args_list[0].args[0] + self.assertEqual(scanned, repository) + self.assertTrue(scanned.env.context.get("strict_branches_scan")) + self.assertEqual( + create_scan_jobs.call_args_list[0].kwargs["branch_ids"], self.branch.ids + ) + + def test_second_pass_is_joined_after_the_scans(self): + """One resolution waits for every scan of the group to be done.""" + self._create_dependency_repository() + self.project._scan_resolved_repositories() + self.env.flush_all() + resolution = self.env["queue.job"].search( + [("method_name", "=", "_resolve_dependencies_job")] + ) + self.assertEqual(len(resolution), 1) + self.assertEqual(resolution.state, "wait_dependencies") + self.assertFalse(resolution.kwargs.get("scan_repositories")) + + def test_no_resolution_for_a_manual_project(self): + self.project.dependency_management = "manual" + self.assertFalse(self._resolution_jobs(self._create_subsequent_jobs())) + + def test_no_resolution_for_another_version(self): + """A project is resolved on its own Odoo version only.""" + self.assertFalse( + self._resolution_jobs( + self._create_subsequent_jobs(version=self.branch2.name) + ) + ) + + def test_no_resolution_when_the_branch_does_not_exist(self): + self.assertFalse(self._resolution_jobs(self._create_subsequent_jobs(data={}))) + + def test_job_records_an_expected_failure(self): + """An unreadable dependency file must not break the chain of jobs.""" + with mute_logger( + "odoo.addons.odoo_project_dependency_resolver.models.odoo_project" + ): + self.project._resolve_dependencies_job() + self.assertIn("has not been cloned", self.project.resolution_log) + self.assertTrue(self.project.last_resolution_date) diff --git a/odoo_project_dependency_resolver/views/odoo_project.xml b/odoo_project_dependency_resolver/views/odoo_project.xml new file mode 100644 index 00000000..f42cf552 --- /dev/null +++ b/odoo_project_dependency_resolver/views/odoo_project.xml @@ -0,0 +1,73 @@ + + + + + odoo.project.form.dependency.resolver + odoo.project + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/odoo_project_dependency_resolver/views/odoo_project_module.xml b/odoo_project_dependency_resolver/views/odoo_project_module.xml new file mode 100644 index 00000000..80cd921b --- /dev/null +++ b/odoo_project_dependency_resolver/views/odoo_project_module.xml @@ -0,0 +1,65 @@ + + + + + odoo.project.module.form.dependency.resolver + odoo.project.module + + + +
+ +
+ + + + + + + + + + + + + odoo.project.module.list.dependency.resolver + odoo.project.module + + + + + + + + + + + + odoo.project.module.search.dependency.resolver + odoo.project.module + + + + + + + + diff --git a/odoo_project_migration/__manifest__.py b/odoo_project_migration/__manifest__.py index c362249b..33832237 100644 --- a/odoo_project_migration/__manifest__.py +++ b/odoo_project_migration/__manifest__.py @@ -14,6 +14,7 @@ "views/odoo_module_branch_timeline.xml", "views/odoo_module_branch_migration.xml", "views/odoo_project.xml", + "views/odoo_project_module.xml", "views/odoo_project_module_migration.xml", "wizards/generate_migration_data.xml", "wizards/export_migration_report.xml", diff --git a/odoo_project_migration/views/odoo_project_module.xml b/odoo_project_migration/views/odoo_project_module.xml new file mode 100644 index 00000000..2b1e3003 --- /dev/null +++ b/odoo_project_migration/views/odoo_project_module.xml @@ -0,0 +1,27 @@ + + + + + odoo.project.module.form.migration + odoo.project.module + + + +
+ + +
+ + + diff --git a/odoo_repository/lib/scanner.py b/odoo_repository/lib/scanner.py index d06dfd67..742f88d0 100644 --- a/odoo_repository/lib/scanner.py +++ b/odoo_repository/lib/scanner.py @@ -247,6 +247,9 @@ def _clone(self): extra = {"env": git_env} if self.workaround_fs_errors: extra["separate_git_dir"] = str(tmp_git_dir_path) + # GitPython blocks '--separate-git-dir' since 3.1.59 + # We must allow unsafe options to be able to use it, + extra["allow_unsafe_options"] = True params = self._clone_params(**extra) try: git.Repo.clone_from(**params) diff --git a/odoo_repository/models/__init__.py b/odoo_repository/models/__init__.py index 6c8e744d..b428a870 100644 --- a/odoo_repository/models/__init__.py +++ b/odoo_repository/models/__init__.py @@ -1,5 +1,6 @@ from . import authentication_token from . import ssh_key +from . import odoo_ref_data_mixin from . import odoo_author from . import odoo_branch from . import odoo_license diff --git a/odoo_repository/models/odoo_module_branch.py b/odoo_repository/models/odoo_module_branch.py index 2229d0f4..61e08103 100644 --- a/odoo_repository/models/odoo_module_branch.py +++ b/odoo_repository/models/odoo_module_branch.py @@ -1,4 +1,5 @@ # Copyright 2023 Camptocamp SA +# Copyright 2026 ACSONE SA/NV () # License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl) import pathlib @@ -18,8 +19,9 @@ class OdooModuleBranch(models.Model): _name = "odoo.module.branch" + _inherit = "odoo.ref.data.mixin" _description = "Odoo Module Branch" - _order = "repository_sequence, module_name, branch_name" + _order = "org_sequence, repository_sequence, module_name, branch_name" module_id = fields.Many2one( comodel_name="odoo.module", @@ -54,6 +56,12 @@ class OdooModuleBranch(models.Model): store=True, string="Organization", ) + org_sequence = fields.Integer( + related="repository_branch_id.repository_id.org_id.sequence", + store=True, + index=True, + string="Organization Sequence", + ) branch_id = fields.Many2one( # NOTE: not a related on 'repository_branch_id' as we need to create # modules without knowing in advance what is their repo (orphaned modules). @@ -307,9 +315,10 @@ def _get_recursive_dependencies(self, domain=None, _visited=None): domain = [] if _visited is None: _visited = set() - if self.id in _visited: + current_ids = set(self.ids) + if current_ids.issubset(_visited): return self.browse() - _visited.add(self.id) + _visited |= current_ids # Apply domain and exclude self dependencies = (self.dependency_ids - self).filtered_domain(domain) dep_ids = set(dependencies.ids) @@ -416,58 +425,14 @@ def _prepare_module_branch_values(self, repo_branch, module, data): "pr_url": False, } if manifest: - category_id = self._get_module_category_id(manifest.get("category", "")) - author_ids = self._get_author_ids(manifest.get("author", "")) - maintainer_ids = self._get_maintainer_ids( - tuple(manifest.get("maintainers", [])) - ) - dev_status_id = self._get_dev_status_id( - manifest.get("development_status", "") - ) - dependency_ids = [] - external_dependencies = {} - python_dependency_ids = [] - if manifest.get("installable", True): - dependency_ids = self._get_dependency_ids( - repo_branch, - # Set at least a dependency on "base" if not defined - manifest.get("depends") or ["base"], - ) - external_dependencies = manifest.get("external_dependencies", {}) - python_dependency_ids = self._get_python_dependency_ids( - tuple(external_dependencies.get("python", [])) - ) - license_id = self._get_license_id(manifest.get("license", "")) values.update( - { - "title": manifest.get("name", False), - "summary": manifest.get( - "summary", manifest.get("description", False) - ), - "category_id": category_id, - "author_ids": [(6, 0, author_ids)], - "maintainer_ids": [(6, 0, maintainer_ids)], - "dependency_ids": [(6, 0, dependency_ids)], - "external_dependencies": external_dependencies, - "python_dependency_ids": [(6, 0, python_dependency_ids)], - "license_id": license_id, - "version": manifest.get("version", False), - "development_status_id": dev_status_id, - "application": manifest.get("application", False), - "installable": manifest.get("installable", True), - "auto_install": manifest.get("auto_install", False), - } + self._prepare_manifest_values( + manifest, repo_branch.branch_id, repo_branch.repository_id + ) ) if data.get("last_scanned_commit"): - values.update( - { - "removed": False, - "sloc_python": data["code"]["Python"], - "sloc_xml": data["code"]["XML"], - "sloc_js": data["code"]["JavaScript"], - "sloc_css": data["code"]["CSS"], - } - ) + values["removed"] = False + values.update(self._prepare_code_analysis_values(data["code"])) # Handle module removal elif module_branch: values.update( @@ -498,6 +463,66 @@ def _prepare_module_branch_values(self, repo_branch, module, data): values["version_ids"] = versions return values + def _prepare_manifest_values(self, manifest, branch, repository): + """Return the `odoo.module.branch` values a manifest carries. + + `branch` and `repository` are where the dependencies it declares are + looked up, those of the module the manifest belongs to. + """ + dependency_ids = [] + external_dependencies = {} + python_dependency_ids = [] + if manifest.get("installable", True): + dependency_ids = self._get_dependency_ids( + branch, + repository, + # Set at least a dependency on "base" if not defined + manifest.get("depends") or ["base"], + ) + external_dependencies = manifest.get("external_dependencies", {}) + python_dependency_ids = self._get_python_dependency_ids( + tuple(external_dependencies.get("python", [])) + ) + return { + "title": manifest.get("name", False), + "summary": manifest.get("summary", manifest.get("description", False)), + "category_id": self._get_module_category_id(manifest.get("category", "")), + "author_ids": [(6, 0, self._get_author_ids(manifest.get("author", "")))], + "maintainer_ids": [ + (6, 0, self._get_maintainer_ids(tuple(manifest.get("maintainers", [])))) + ], + "dependency_ids": [(6, 0, dependency_ids)], + "external_dependencies": external_dependencies, + "python_dependency_ids": [(6, 0, python_dependency_ids)], + "license_id": self._get_license_id(manifest.get("license", "")), + "version": manifest.get("version", False), + "development_status_id": self._get_dev_status_id( + manifest.get("development_status", "") + ), + "application": manifest.get("application", False), + "installable": manifest.get("installable", True), + "auto_install": manifest.get("auto_install", False), + } + + def _get_sloc_fields(self): + """Return the field holding the count of each analysed language. + + Counting one more language is adding it here, and having the scanner + analyse it. + """ + return { + "Python": "sloc_python", + "XML": "sloc_xml", + "JavaScript": "sloc_js", + "CSS": "sloc_css", + } + + def _prepare_code_analysis_values(self, code): + """Return the `odoo.module.branch` values a code analysis carries.""" + return { + field: code[language] for language, field in self._get_sloc_fields().items() + } + def _create_or_update(self, repo_branch, module, values): """Create or update a `odoo.module.branch` record from scanned module. @@ -644,13 +669,13 @@ def _get_module_category_id(self, category_name): rec = self.env["odoo.module.category"].search( [("name", "=", category_name)], limit=1 ) - if not rec: - rec = ( - self.env["odoo.module.category"] - .sudo() - .create({"name": category_name}) - ) - return rec.id + if rec: + return rec.id + return self._create_ref_data( + "odoo.module.category", + [("name", "=", category_name)], + {"name": category_name}, + ) return False @tools.ormcache("names") @@ -661,14 +686,14 @@ def _get_author_ids(self, names): names = [name.strip() for name in names.split(",")] authors = self.env["odoo.author"].search([("name", "in", names)]) missing_author_names = set(names) - set(authors.mapped("name")) - missing_authors = self.env["odoo.author"] + created_ids = [] if missing_author_names: - missing_authors = ( - self.env["odoo.author"] - .sudo() - .create([{"name": name} for name in missing_author_names]) + created_ids = self._create_ref_data_multi( + "odoo.author", + "name", + [{"name": name} for name in missing_author_names], ) - return (authors | missing_authors).ids + return authors.ids + created_ids return [] @tools.ormcache("names") @@ -676,12 +701,14 @@ def _get_maintainer_ids(self, names): if names: maintainers = self.env["odoo.maintainer"].search([("name", "in", names)]) missing_maintainer_names = set(names) - set(maintainers.mapped("name")) - created = self.env["odoo.maintainer"] + created_ids = [] if missing_maintainer_names: - created = created.sudo().create( - [{"name": name} for name in missing_maintainer_names] + created_ids = self._create_ref_data_multi( + "odoo.maintainer", + "name", + [{"name": name} for name in missing_maintainer_names], ) - return (maintainers | created).ids + return maintainers.ids + created_ids return [] @tools.ormcache("name") @@ -690,9 +717,11 @@ def _get_dev_status_id(self, name): rec = self.env["odoo.module.dev.status"].search( [("name", "=", name)], limit=1 ) - if not rec: - rec = self.env["odoo.module.dev.status"].sudo().create({"name": name}) - return rec.id + if rec: + return rec.id + return self._create_ref_data( + "odoo.module.dev.status", [("name", "=", name)], {"name": name} + ) return False @api.model @@ -743,13 +772,17 @@ def _find_or_create(self, branch, module, repo, domain=None): module_branch = self.sudo()._create_orphaned_module_branch(branch, module) return module_branch - def _get_dependency_ids(self, repo_branch, depends: list): + def _get_dependency_ids(self, branch, repository, depends: list): + """Return the modules `depends` refers to, on `branch`. + + They are looked up in `repository` first. Both are those of the module + depending on them, which a repository branch does not always tell: a + module read outside of a scan can belong to no repository at all. + """ dependency_ids = [] for depend in depends: module = self._get_module(depend) - dependency = self._find_or_create( - repo_branch.branch_id, module, repo_branch.repository_id - ) + dependency = self._find_or_create(branch, module, repository) dependency_ids.append(dependency.id) return dependency_ids @@ -759,13 +792,15 @@ def _get_python_dependency_ids(self, packages): dependencies = self.env["odoo.python.dependency"].search( [("name", "in", packages)] ) - missing_dependencies = set(packages) - set(dependencies.mapped("name")) - created = self.env["odoo.python.dependency"] - if missing_dependencies: - created = created.sudo().create( - [{"name": package} for package in missing_dependencies] + missing_names = set(packages) - set(dependencies.mapped("name")) + created_ids = [] + if missing_names: + created_ids = self._create_ref_data_multi( + "odoo.python.dependency", + "name", + [{"name": name} for name in missing_names], ) - return (dependencies | created).ids + return dependencies.ids + created_ids return [] @tools.ormcache("license_name") @@ -773,9 +808,11 @@ def _get_license_id(self, license_name): if license_name: license_model = self.env["odoo.license"] rec = license_model.search([("name", "=", license_name)], limit=1) - if not rec: - rec = license_model.sudo().create({"name": license_name}) - return rec.id + if rec: + return rec.id + return self._create_ref_data( + "odoo.license", [("name", "=", license_name)], {"name": license_name} + ) return False def _get_module(self, name): @@ -904,10 +941,7 @@ def _to_dict(self): "is_standard": self.is_standard, "is_enterprise": self.is_enterprise, "is_community": self.is_community, - "sloc_python": self.sloc_python, - "sloc_xml": self.sloc_xml, - "sloc_js": self.sloc_js, - "sloc_css": self.sloc_css, + **{field: self[field] for field in self._get_sloc_fields().values()}, "last_scanned_commit": self.last_scanned_commit, "addons_path": self.addons_path, "pr_url": self.pr_url, diff --git a/odoo_repository/models/odoo_ref_data_mixin.py b/odoo_repository/models/odoo_ref_data_mixin.py new file mode 100644 index 00000000..9b4262e2 --- /dev/null +++ b/odoo_repository/models/odoo_ref_data_mixin.py @@ -0,0 +1,84 @@ +# Copyright 2026 ACSONE SA/NV () +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl) + +import threading +from contextlib import contextmanager + +import psycopg2.errors + +from odoo import api, models + + +class OdooRefDataMixin(models.AbstractModel): + """Shared helpers to safely create reference data cached by ``@tools.ormcache``.""" + + _name = "odoo.ref.data.mixin" + _description = "Odoo Ref Data Mixin" + + @contextmanager + def _ref_data_cursor(self): + """Yield the cursor to use for an isolated reference-data creation. + + This method is a context manager that yields a cursor to use for creating + reference data in a separate transaction. If the current environment is in + test mode, it yields the current cursor instead, since in test mode we + don't want to create a new transaction. + """ + in_test_mode = self.env.registry.in_test_mode() or getattr( + threading.current_thread(), "testing", False + ) + if in_test_mode: + yield self.env.cr + else: + with self.pool.cursor() as new_cr: + yield new_cr + + def _create_ref_data(self, model_name, domain, values): + """Create a single get-or-create reference record in its own + dedicated, immediately-committed transaction, and return its id. + + ``@tools.ormcache`` is a process-wide cache that is never invalidated + by a SQL ROLLBACK. If a newly created record's id were cached from + within the caller's (still uncommitted) transaction, and that + transaction was later rolled back for an unrelated reason (e.g. a + conflicting concurrent queue job), the cache would keep returning an + id that no longer exists in the database. Creating and committing the + record here, independently of the caller's transaction, guarantees + the returned id always stays valid, whatever happens to the caller's + job afterwards. + + A concurrent job may create the same record at the same time: the + unique SQL constraint on the target model then raises + ``UniqueViolation`` for the loser, which simply looks up the row the + winner just committed. + """ + with self._ref_data_cursor() as new_cr: + env = api.Environment(new_cr, self.env.uid, self.env.context) + model = env[model_name].sudo() + try: + with new_cr.savepoint(): + return model.create(values).id + except psycopg2.errors.UniqueViolation: + return model.search(domain, limit=1).id + + def _create_ref_data_multi(self, model_name, search_field, values_list): + """Same as ``_create_ref_data``, but for a batch of records sharing + the same natural-key field (e.g. several new authors discovered at + once). Each record is created individually, in its own savepoint, so + that a conflict on one of them does not abort the creation of the + others. + """ + ids = [] + with self._ref_data_cursor() as new_cr: + env = api.Environment(new_cr, self.env.uid, self.env.context) + model = env[model_name].sudo() + for values in values_list: + try: + with new_cr.savepoint(): + ids.append(model.create(values).id) + except psycopg2.errors.UniqueViolation: + existing = model.search( + [(search_field, "=", values[search_field])], limit=1 + ) + ids.append(existing.id) + return ids diff --git a/odoo_repository/models/odoo_repository.py b/odoo_repository/models/odoo_repository.py index 71bbeba1..ba06c875 100644 --- a/odoo_repository/models/odoo_repository.py +++ b/odoo_repository/models/odoo_repository.py @@ -1,12 +1,13 @@ # Copyright 2023 Camptocamp SA # Copyright 2026 Sébastien Alix +# Copyright 2026 ACSONE SA/NV () # License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl) import json import logging import os import pathlib -from urllib.parse import urljoin +from urllib.parse import urljoin, urlparse import requests @@ -18,6 +19,7 @@ from odoo.addons.queue_job.exception import RetryableJobError from odoo.addons.queue_job.job import identity_exact +from ..lib.scanner import BaseScanner from ..utils.scanner import RepositoryScannerOdooEnv _logger = logging.getLogger(__name__) @@ -25,6 +27,7 @@ class OdooRepository(models.Model): _name = "odoo.repository" + _inherit = "odoo.ref.data.mixin" _description = "Odoo Modules Repository" _order = "sequence, display_name" @@ -218,13 +221,12 @@ def _check_config(self): def _check_existing_jobs(self, raise_exc=True): """Check if a scan is already triggered for this repository.""" self.ensure_one() - existing_job = ( + ongoing_jobs = ( self.env["queue.job"] .sudo() .search( [ ("model_name", "=", self._name), - ("records", "ilike", f'%"ids": [{self.id}]%'), ( "state", "in", @@ -235,10 +237,14 @@ def _check_existing_jobs(self, raise_exc=True): "started", ], ), - ], - limit=1, + ] ) ) + # NOTE: 'records' cannot take part in the domain. It is a jsonb holding + # an escaped JSON string, so a pattern such as '"ids": [42]' is never + # found in it, and searching on it silently matched nothing. Which + # repository the ongoing jobs are about is therefore told here instead. + existing_job = ongoing_jobs.filtered(lambda job: self in job.records) if existing_job: msg = _("A scan is already ongoing for repository %s") % self.display_name if raise_exc: @@ -249,7 +255,21 @@ def _check_existing_jobs(self, raise_exc=True): def action_scan(self, branch_ids=None, force=False, raise_exc=True): """Scan the whole repository.""" + for job in self._create_scan_jobs( + branch_ids=branch_ids, force=force, raise_exc=raise_exc + ): + job.delay() + return True + + def _create_scan_jobs(self, branch_ids=None, force=False, raise_exc=True): + """Return the jobs scanning these repositories, without delaying them. + + Callers wanting something to happen once the scan is over can build + their own graph out of them, rather than delaying them right away as + `action_scan` does. + """ self._check_config() + jobs = [] for rec in self: if not rec.to_scan: continue @@ -283,11 +303,12 @@ def action_scan(self, branch_ids=None, force=False, raise_exc=True): # on the next branch version_branch = versions_branches[0] next_versions_branches = versions_branches[1:] - job = rec._create_job_detect_modules_to_scan_on_branch( - version_branch, next_versions_branches, versions_branches + jobs.append( + rec._create_job_detect_modules_to_scan_on_branch( + version_branch, next_versions_branches, versions_branches + ) ) - job.delay() - return True + return jobs def _create_job_detect_modules_to_scan_on_branch( self, version_branch, next_versions_branches, all_versions_branches @@ -424,13 +445,33 @@ def _get_token(self): or os.environ.get("GITHUB_TOKEN") ) - def _prepare_scanner_parameters(self, version, branch): + def _prepare_base_scanner_parameters(self): + """Return the parameters every scanner of this repository needs. + + They are the ones telling where the clone lives and how to reach the + remote, which any scanner working on this repository has to share, + whatever it scans. + """ ir_config = self.env["ir.config_parameter"] - repositories_path = ir_config.sudo().get_param(self._repositories_path_key) return { "org": self.org_id.name, "name": self.name, "clone_url": self.clone_url, + "repositories_path": ir_config.sudo().get_param( + self._repositories_path_key + ), + "repo_type": self.repo_type, + "ssh_key": self.ssh_key_id.private_key, + "token": self._get_token(), + "workaround_fs_errors": ( + self.env.company.config_odoo_repository_workaround_fs_errors + ), + "clone_name": self.clone_name, + } + + def _prepare_scanner_parameters(self, version, branch): + return { + **self._prepare_base_scanner_parameters(), "version": version, "branch": branch, "addons_paths_data": self.addons_path_ids.read( @@ -441,14 +482,6 @@ def _prepare_scanner_parameters(self, version, branch): "is_community", ] ), - "repositories_path": repositories_path, - "repo_type": self.repo_type, - "ssh_key": self.ssh_key_id.private_key, - "token": self._get_token(), - "workaround_fs_errors": ( - self.env.company.config_odoo_repository_workaround_fs_errors - ), - "clone_name": self.clone_name, "env": self.env, } @@ -501,12 +534,18 @@ def _import_data(self, data): def _prepare_module_branch_values(self, data): # Get branch, repository and technical module branch = self.env["odoo.branch"].search([("name", "=", data["branch"])]) - org = self._get_repository_org(data["repository"]["org"]) - repository = self._get_repository( - org.id, data["repository"]["name"], data["repository"] + org_id = self._get_repository_org_id(data["repository"]["org"]) + repository_id = self._get_repository_id( + org_id, data["repository"]["name"], data["repository"] + ) + repository_branch_id = self._get_repository_branch_id( + org_id, repository_id, branch.id, data["repository"] ) - repository_branch = self._get_repository_branch( - org.id, repository.id, branch.id, data["repository"] + # `_get_repository_branch_id` returns a plain id (see `_create_ref_data` + # docstring), so browse it here to get a recordset bound to this + # transaction's cursor. + repository_branch = self.env["odoo.repository.branch"].browse( + repository_branch_id ) mb_model = self.env["odoo.module.branch"] @@ -517,7 +556,9 @@ def _prepare_module_branch_values(self, data): maintainer_ids = mb_model._get_maintainer_ids(tuple(data["maintainers"])) dev_status_id = mb_model._get_dev_status_id(data["development_status"]) dependency_ids = mb_model._get_dependency_ids( - repository_branch, data["depends"] + repository_branch.branch_id, + repository_branch.repository_id, + data["depends"], ) external_dependencies = data["external_dependencies"] python_dependency_ids = mb_model._get_python_dependency_ids( @@ -549,10 +590,14 @@ def _prepare_module_branch_values(self, data): "is_standard": data["is_standard"], "is_enterprise": data["is_enterprise"], "is_community": data["is_community"], - "sloc_python": data["sloc_python"], - "sloc_xml": data["sloc_xml"], - "sloc_js": data["sloc_js"], - "sloc_css": data["sloc_css"], + # The exchange format spells the code analysis with the very names + # of the fields holding it. A node counting one more language than + # the one it imports from simply gets nothing for it. + **{ + field: data[field] + for field in mb_model._get_sloc_fields().values() + if field in data + }, "last_scanned_commit": data["last_scanned_commit"], "pr_url": data["pr_url"], } @@ -605,21 +650,18 @@ def _post_create_or_update_module_branch(self, rec, values, raw_data): """Hook executed after the creation or update of `rec`.""" @tools.ormcache("name") - def _get_repository_org(self, name): + def _get_repository_org_id(self, name): rec = self.env["odoo.repository.org"].search([("name", "=", name)], limit=1) - if not rec: - rec = self.env["odoo.repository.org"].sudo().create({"name": name}) - return rec + if rec: + return rec.id + return self._create_ref_data( + "odoo.repository.org", [("name", "=", name)], {"name": name} + ) @tools.ormcache("org_id", "name") - def _get_repository(self, org_id, name, data): - rec = self.env["odoo.repository"].search( - [ - ("org_id", "=", org_id), - ("name", "=", name), - ], - limit=1, - ) + def _get_repository_id(self, org_id, name, data): + domain = [("org_id", "=", org_id), ("name", "=", name)] + rec = self.env["odoo.repository"].search(domain, limit=1) values = { "org_id": org_id, "name": name, @@ -629,19 +671,16 @@ def _get_repository(self, org_id, name, data): } if rec: rec.sudo().write(values) - else: - rec = self.env["odoo.repository"].sudo().create(values) - return rec + return rec.id + return self._create_ref_data("odoo.repository", domain, values) @tools.ormcache("org_id", "repository_id", "branch_id") - def _get_repository_branch(self, org_id, repository_id, branch_id, data): - rec = self.env["odoo.repository.branch"].search( - [ - ("repository_id", "=", repository_id), - ("branch_id", "=", branch_id), - ], - limit=1, - ) + def _get_repository_branch_id(self, org_id, repository_id, branch_id, data): + domain = [ + ("repository_id", "=", repository_id), + ("branch_id", "=", branch_id), + ] + rec = self.env["odoo.repository.branch"].search(domain, limit=1) values = { "repository_id": repository_id, "branch_id": branch_id, @@ -649,9 +688,8 @@ def _get_repository_branch(self, org_id, repository_id, branch_id, data): } if rec: rec.sudo().write(values) - else: - rec = self.env["odoo.repository.branch"].sudo().create(values) - return rec + return rec.id + return self._create_ref_data("odoo.repository.branch", domain, values) def _get_resource_url(self, branch, path): self.ensure_one() @@ -659,6 +697,59 @@ def _get_resource_url(self, branch, path): url = "/".join(["tree", branch, path]) return urljoin(self.repo_url + "/", url) + def _get_local_clone_path(self): + """Return the path of the local clone of this repository. + + The clone is the one maintained by the scanner, so it only exists + once the repository has been scanned at least once. + """ + self.ensure_one() + repositories_path = ( + self.env["ir.config_parameter"] + .sudo() + .get_param(self._repositories_path_key) + ) + # NOTE: delegate to the scanner to build the very same layout + return BaseScanner._prepare_repositories_path(repositories_path).joinpath( + self.org_id.name, self.clone_name or self.name + ) + + @staticmethod + def _parse_clone_url(clone_url): + """Return the ``(host, org, name)`` parts of a clone URL. + + Supports both HTTP(S) and SCP-like (``git@host:org/name.git``) syntax. + Returns ``(None, None, None)`` if the URL cannot be parsed. + """ + if not clone_url: + return (None, None, None) + url = clone_url.strip() + if "://" in url: + parts = urlparse(url) + host, path = parts.hostname, parts.path + elif ":" in url: + # SCP-like syntax: [user@]host:path + host, _, path = url.partition(":") + host = host.rpartition("@")[2] + else: + return (None, None, None) + path = path.strip("/") + if path.endswith(".git"): + path = path[: -len(".git")] + # NOTE: 'org' keeps every leading segment to support nested namespaces + # (GitLab sub-groups). + org, _, name = path.rpartition("/") + if not host or not org or not name: + return (None, None, None) + return (host, org, name) + + @api.model + def _find_from_org_and_name(self, org, name): + """Return the repository matching an organization and a name.""" + return self.with_context(active_test=False).search( + [("org_id.name", "=", org), ("name", "=", name)], limit=1 + ) + def unlink(self): # There is no deletion on cascade policy by default, but for specific # repositories we want to remove specific modules anyway. diff --git a/odoo_repository/models/odoo_repository_org.py b/odoo_repository/models/odoo_repository_org.py index 38605aa2..3842c2bc 100644 --- a/odoo_repository/models/odoo_repository_org.py +++ b/odoo_repository/models/odoo_repository_org.py @@ -1,4 +1,5 @@ # Copyright 2023 Camptocamp SA +# Copyright 2026 ACSONE SA/NV () # License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl) from odoo import api, fields, models @@ -9,8 +10,17 @@ class OdooRepositoryOrg(models.Model): _name = "odoo.repository.org" _description = "Odoo Repository Organization" + _order = "sequence, name" name = fields.Char(required=True, index=True) + sequence = fields.Integer( + default=100, + help=( + "Priority of this organization when looking for the origin of a " + "module shared by several organizations (forks). " + "The lowest sequence wins." + ), + ) github_url = fields.Char(string="GitHub URL", compute="_compute_github_url") @api.depends("name") diff --git a/odoo_repository/readme/CONTRIBUTORS.md b/odoo_repository/readme/CONTRIBUTORS.md index 17752927..e6aa7475 100644 --- a/odoo_repository/readme/CONTRIBUTORS.md +++ b/odoo_repository/readme/CONTRIBUTORS.md @@ -1,2 +1,4 @@ - Camptocamp - Sébastien Alix \ +- ACSONE SA/NV + - Laurent Mignon \ diff --git a/odoo_repository/tests/__init__.py b/odoo_repository/tests/__init__.py index ac60db36..caa4f928 100644 --- a/odoo_repository/tests/__init__.py +++ b/odoo_repository/tests/__init__.py @@ -7,3 +7,4 @@ from . import test_odoo_branch from . import test_oca_repository_synchronizer from . import test_odoo_module_branch_recursive_dependencies +from . import test_odoo_repository_lookup diff --git a/odoo_repository/tests/test_odoo_module_branch_recursive_dependencies.py b/odoo_repository/tests/test_odoo_module_branch_recursive_dependencies.py index f6b30940..6915b317 100644 --- a/odoo_repository/tests/test_odoo_module_branch_recursive_dependencies.py +++ b/odoo_repository/tests/test_odoo_module_branch_recursive_dependencies.py @@ -188,6 +188,22 @@ def test_get_recursive_dependencies_empty(self): deps = mod_alone_branch._get_recursive_dependencies() self.assertEqual(len(deps), 0) + def test_get_recursive_dependencies_multi_record_self(self): + """Test _get_recursive_dependencies called on a non-singleton recordset.""" + # This mirrors odoo.project.import.modules._action_import_missing_dependencies, + # which calls the method on the module_branch_id of several project + # modules at once. The previous implementation relied on `self.id`, + # raising `ValueError: Expected singleton` for more than one record. + self.mod_a_branch.dependency_ids = self.mod_base_branch + self.mod_b_branch.dependency_ids = self.mod_c_branch + modules = self.mod_a_branch + self.mod_b_branch + deps = modules._get_recursive_dependencies() + self.assertIn(self.mod_base_branch, deps) + self.assertIn(self.mod_c_branch, deps) + self.assertNotIn(self.mod_a_branch, deps) + self.assertNotIn(self.mod_b_branch, deps) + self.assertEqual(len(deps), 2) # base, module_c + def test_get_recursive_dependencies_self_exclusion(self): """Test that _get_recursive_dependencies excludes self.""" # Create modules diff --git a/odoo_repository/tests/test_odoo_repository_lookup.py b/odoo_repository/tests/test_odoo_repository_lookup.py new file mode 100644 index 00000000..c79ae407 --- /dev/null +++ b/odoo_repository/tests/test_odoo_repository_lookup.py @@ -0,0 +1,129 @@ +import pathlib + +# Copyright 2026 ACSONE SA/NV () +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl) +from .common import Common + + +class TestOdooRepositoryLookup(Common): + def setUp(self): + super().setUp() + self.repository_model = self.env["odoo.repository"] + + # -- _parse_clone_url -------------------------------------------------- + + def test_parse_clone_url_https(self): + self.assertEqual( + self.repository_model._parse_clone_url( + "https://github.com/acsone/account-invoicing.git" + ), + ("github.com", "acsone", "account-invoicing"), + ) + + def test_parse_clone_url_without_git_suffix(self): + self.assertEqual( + self.repository_model._parse_clone_url( + "https://github.com/OCA/account-invoicing" + ), + ("github.com", "OCA", "account-invoicing"), + ) + + def test_parse_clone_url_scp_syntax(self): + self.assertEqual( + self.repository_model._parse_clone_url( + "git@github.com:acsone/account-invoicing.git" + ), + ("github.com", "acsone", "account-invoicing"), + ) + + def test_parse_clone_url_nested_namespace(self): + """GitLab sub-groups are kept as part of the organization.""" + self.assertEqual( + self.repository_model._parse_clone_url( + "https://gitlab.com/acsone/odoo/addons.git" + ), + ("gitlab.com", "acsone/odoo", "addons"), + ) + + def test_parse_clone_url_invalid(self): + for url in (None, "", "not-an-url", "https://github.com/lonely-segment"): + with self.subTest(url=url): + self.assertEqual( + self.repository_model._parse_clone_url(url), (None, None, None) + ) + + # -- organization sequence --------------------------------------------- + + def _create_repository(self, org_name, repo_name, **values): + org = self.env["odoo.repository.org"].search([("name", "=", org_name)]) + if not org: + org = self.env["odoo.repository.org"].create({"name": org_name}) + vals = { + "org_id": org.id, + "name": repo_name, + "repo_url": f"https://github.com/{org_name}/{repo_name}", + "clone_url": f"https://github.com/{org_name}/{repo_name}.git", + "repo_type": "github", + } + vals.update(values) + return self.env["odoo.repository"].create(vals) + + def test_find_module_branch_honours_org_sequence(self): + """A module shared by several organizations resolves to the first one.""" + module = self._create_odoo_module("account_invoice_triple_discount") + module_branches = {} + for org_name, sequence in (("OCA", 10), ("acsone", 20)): + repository = self._create_repository(org_name, "account-invoicing") + repository.org_id.sequence = sequence + repository_branch = self._create_odoo_repository_branch( + repository, self.branch + ) + module_branches[org_name] = self._create_odoo_module_branch( + module, + self.branch, + repository_branch_id=repository_branch.id, + specific=False, + ) + module_branch_model = self.env["odoo.module.branch"] + found = module_branch_model._find(self.branch, module, repo=False) + self.assertEqual(found, module_branches["OCA"]) + # Reversing the priority of the organizations reverses the result + module_branches["acsone"].repository_id.org_id.sequence = 1 + found = module_branch_model._find(self.branch, module, repo=False) + self.assertEqual(found, module_branches["acsone"]) + + def test_get_local_clone_path(self): + """The clone path matches the layout built by the scanner.""" + repository = self._create_repository("OCA", "account-invoicing") + self.assertEqual( + repository._get_local_clone_path(), + pathlib.Path(self.repositories_path, "OCA", "account-invoicing"), + ) + + def test_get_local_clone_path_with_clone_name(self): + """A forced clone name overrides the repository name on disk.""" + repository = self._create_repository( + "OCA", "account-invoicing", clone_name="oca-account-invoicing" + ) + self.assertEqual( + repository._get_local_clone_path(), + pathlib.Path(self.repositories_path, "OCA", "oca-account-invoicing"), + ) + + def test_prepare_base_scanner_parameters(self): + """Every scanner of a repository gets the same way to reach it.""" + token = self.env["authentication.token"].create( + {"name": "OCA", "token": "s3cr3t"} + ) + repository = self._create_repository("OCA", "account-invoicing") + repository.token_id = token + params = repository._prepare_base_scanner_parameters() + self.assertEqual(params["org"], "OCA") + self.assertEqual(params["name"], "account-invoicing") + self.assertEqual(params["token"], "s3cr3t") + self.assertEqual( + params["clone_url"], "https://github.com/OCA/account-invoicing.git" + ) + # The parameters of a repository scanner build on them + scanner_params = repository._prepare_scanner_parameters("18.0", "18.0") + self.assertLessEqual(params.items(), scanner_params.items()) diff --git a/odoo_repository/tests/test_odoo_repository_scan.py b/odoo_repository/tests/test_odoo_repository_scan.py index 12c06ba0..7f3d54cf 100644 --- a/odoo_repository/tests/test_odoo_repository_scan.py +++ b/odoo_repository/tests/test_odoo_repository_scan.py @@ -2,12 +2,75 @@ # Copyright 2026 Sébastien Alix # License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl) +import contextlib +import gc + from odoo import fields +from odoo.exceptions import UserError +from odoo.tools import mute_logger from .common import Common class TestOdooRepositoryScan(Common): + @classmethod + def setUpClass(cls): + super().setUpClass() + muted = contextlib.ExitStack() + cls.addClassCleanup(muted.close) + muted.enter_context(mute_logger("odoo.addons.queue_job.delay")) + + def tearDown(self): + super().tearDown() + # A delayable warns when collected without having been delayed, which + # is what these tests do on purpose. Collect here, while + # the logger above is still muted, rather than after 'tearDownClass'. + gc.collect() + + def test_create_scan_jobs_does_not_delay_them(self): + """The jobs are returned so that callers can build a graph out of them. + + 'action_scan' delays them right away, but something meant to run once + the scan is over has to connect to them first. + """ + before = self.env["queue.job"].search_count([]) + jobs = self.odoo_repository._create_scan_jobs(branch_ids=self.branch.ids) + self.assertTrue(jobs) + self.assertEqual( + [job._job_method.__name__ for job in jobs], + ["_detect_modules_to_scan_on_branch"], + ) + self.env.flush_all() + self.assertEqual(self.env["queue.job"].search_count([]), before) + + def test_check_existing_jobs(self): + """A scan is not started again while one is still ongoing.""" + self.assertFalse(self.odoo_repository._check_existing_jobs(raise_exc=False)) + for job in self.odoo_repository._create_scan_jobs(branch_ids=self.branch.ids): + job.delay() + self.env.flush_all() + with mute_logger("odoo.addons.odoo_repository.models.odoo_repository"): + self.assertTrue(self.odoo_repository._check_existing_jobs(raise_exc=False)) + with self.assertRaisesRegex(UserError, "already ongoing"): + self.odoo_repository._check_existing_jobs() + + def test_check_existing_jobs_ignores_other_repositories(self): + """The ongoing scan of another repository does not block this one.""" + other = self.env["odoo.repository"].create( + { + "org_id": self.org.id, + "name": "other-repo", + "repo_url": "https://github.com/ORG/other-repo", + "repo_type": "github", + } + ) + for job in other._create_scan_jobs(branch_ids=self.branch.ids): + job.delay() + self.env.flush_all() + with mute_logger("odoo.addons.odoo_repository.models.odoo_repository"): + self.assertTrue(other._check_existing_jobs(raise_exc=False)) + self.assertFalse(self.odoo_repository._check_existing_jobs(raise_exc=False)) + def test_check_config(self): self.odoo_repository._check_config() diff --git a/odoo_repository/views/odoo_repository_org.xml b/odoo_repository/views/odoo_repository_org.xml index fe540983..d6ac8514 100644 --- a/odoo_repository/views/odoo_repository_org.xml +++ b/odoo_repository/views/odoo_repository_org.xml @@ -1,5 +1,6 @@ @@ -10,6 +11,7 @@ + @@ -21,6 +23,7 @@ odoo.repository.org + diff --git a/odoo_repository_fork/README.rst b/odoo_repository_fork/README.rst new file mode 100644 index 00000000..1b43b495 --- /dev/null +++ b/odoo_repository_fork/README.rst @@ -0,0 +1,101 @@ +================ +Odoo MCA - Forks +================ + +.. + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + !! This file is generated by oca-gen-addon-readme !! + !! changes will be overwritten. !! + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + !! source digest: sha256:373cd18ccca83c45fa3516d480eb32693d0a4e52fa7ef657346cfdce3261cb6a + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + +.. |badge1| image:: https://img.shields.io/badge/maturity-Beta-yellow.png + :target: https://odoo-community.org/page/development-status + :alt: Beta +.. |badge2| image:: https://img.shields.io/badge/licence-AGPL--3-blue.png + :target: http://www.gnu.org/licenses/agpl-3.0-standalone.html + :alt: License: AGPL-3 +.. |badge3| image:: https://img.shields.io/badge/github-OCA%2Fmodule--composition--analysis-lightgray.png?logo=github + :target: https://github.com/OCA/module-composition-analysis/tree/18.0/odoo_repository_fork + :alt: OCA/module-composition-analysis +.. |badge4| image:: https://img.shields.io/badge/weblate-Translate%20me-F47D42.png + :target: https://translation.odoo-community.org/projects/module-composition-analysis-18-0/module-composition-analysis-18-0-odoo_repository_fork + :alt: Translate me on Weblate +.. |badge5| image:: https://img.shields.io/badge/runboat-Try%20me-875A7B.png + :target: https://runboat.odoo-community.org/builds?repo=OCA/module-composition-analysis&target_branch=18.0 + :alt: Try me on Runboat + +|badge1| |badge2| |badge3| |badge4| |badge5| + +A module is not always installed from the repository it belongs to. +Freezing one built from unmerged pull requests is commonly done by +pinning a revision of a fork, and that fork is nowhere in Odoo MCA: its +ancestry has to be asked to GitHub over and over, and nothing holds the +credentials a private one needs to be read. + +This module registers a fork as a repository of its own, pointing at the +one it originates from. + +**Table of contents** + +.. contents:: + :local: + +Usage +===== + +A fork is registered on the fly, whenever something looks up the +repository a clone URL points at and GitHub answers that it is a fork of +a repository already known. + +It can be created by hand too, by setting *Fork Of* on a repository. +Either way, a fork is never scanned: it hosts the very same modules as +the repository it comes from, and a second scanned repository holding +them would make the module of a dependency ambiguous for every project. +A constraint enforces it. + +Its credentials remain editable though, and they are what reading a +private fork needs. The forks of a repository are listed on it. + +Bug Tracker +=========== + +Bugs are tracked on `GitHub Issues `_. +In case of trouble, please check there if your issue has already been reported. +If you spotted it first, help us to smash it by providing a detailed and welcomed +`feedback `_. + +Do not contact contributors directly about support or help with technical issues. + +Credits +======= + +Authors +------- + +* ACSONE SA/NV + +Contributors +------------ + +- ACSONE SA/NV + + - Laurent Mignon + +Maintainers +----------- + +This module is maintained by the OCA. + +.. image:: https://odoo-community.org/logo.png + :alt: Odoo Community Association + :target: https://odoo-community.org + +OCA, or the Odoo Community Association, is a nonprofit organization whose +mission is to support the collaborative development of Odoo features and +promote its widespread use. + +This module is part of the `OCA/module-composition-analysis `_ project on GitHub. + +You are welcome to contribute. To learn how please visit https://odoo-community.org/page/Contribute. diff --git a/odoo_repository_fork/__init__.py b/odoo_repository_fork/__init__.py new file mode 100644 index 00000000..0650744f --- /dev/null +++ b/odoo_repository_fork/__init__.py @@ -0,0 +1 @@ +from . import models diff --git a/odoo_repository_fork/__manifest__.py b/odoo_repository_fork/__manifest__.py new file mode 100644 index 00000000..4ef38011 --- /dev/null +++ b/odoo_repository_fork/__manifest__.py @@ -0,0 +1,18 @@ +# Copyright 2026 ACSONE SA/NV () +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl) +{ + "name": "Odoo MCA - Forks", + "summary": "Register the forks of a repository and where they come from.", + "version": "18.0.1.0.0", + "category": "Tools", + "author": "ACSONE SA/NV, Odoo Community Association (OCA)", + "website": "https://github.com/OCA/module-composition-analysis", + "data": [ + "views/odoo_repository.xml", + ], + "installable": True, + "depends": [ + "odoo_repository", + ], + "license": "AGPL-3", +} diff --git a/odoo_repository_fork/models/__init__.py b/odoo_repository_fork/models/__init__.py new file mode 100644 index 00000000..708af8c2 --- /dev/null +++ b/odoo_repository_fork/models/__init__.py @@ -0,0 +1 @@ +from . import odoo_repository diff --git a/odoo_repository_fork/models/odoo_repository.py b/odoo_repository_fork/models/odoo_repository.py new file mode 100644 index 00000000..80788dce --- /dev/null +++ b/odoo_repository_fork/models/odoo_repository.py @@ -0,0 +1,150 @@ +# Copyright 2026 ACSONE SA/NV () +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl) + +import logging + +from odoo import _, api, fields, models, tools +from odoo.exceptions import ValidationError + +from odoo.addons.odoo_repository.utils import github + +_logger = logging.getLogger(__name__) + + +class OdooRepository(models.Model): + _inherit = "odoo.repository" + + upstream_repository_id = fields.Many2one( + comodel_name="odoo.repository", + string="Fork Of", + ondelete="cascade", + index=True, + help=( + "Repository this one is a fork of. A fork hosts the very same " + "modules, so it is never scanned: its modules are those of the " + "repository it originates from. It is registered to document where " + "a project installs them from, and to hold the credentials needed " + "to read it." + ), + ) + fork_ids = fields.One2many( + comodel_name="odoo.repository", + inverse_name="upstream_repository_id", + string="Forks", + ) + + @api.constrains("upstream_repository_id", "to_scan") + def _check_fork_not_scanned(self): + for rec in self: + if rec.upstream_repository_id and rec.to_scan: + raise ValidationError( + _( + "Repository %(fork)s cannot be scanned, it is a fork of " + "%(upstream)s and hosts the very same modules. Scanning " + "it would make the module of a dependency ambiguous for " + "every project." + ) + % { + "fork": rec.display_name, + "upstream": rec.upstream_repository_id.display_name, + } + ) + + @api.constrains("upstream_repository_id") + def _check_upstream_repository_cycle(self): + if self._has_cycle("upstream_repository_id"): + raise ValidationError(_("A repository cannot be a fork of itself.")) + + @api.model + @tools.ormcache("org", "name") + def _fetch_github_fork_parents(self, org, name): + """Return the ancestors of a forked GitHub repository. + + The result is a tuple of ``org/name`` strings, from the direct parent + to the root of the fork chain, empty if the repository is not a fork. + """ + data = github.request(self.env, f"repos/{org}/{name}") + if not data.get("fork"): + return () + full_names = ( + (data.get("parent") or {}).get("full_name"), + (data.get("source") or {}).get("full_name"), + ) + # dict.fromkeys() deduplicates while keeping the parent first, as both + # entries are equal as soon as the fork chain has a single level. + return tuple(dict.fromkeys(filter(None, full_names))) + + @api.model + def _find_from_clone_url(self, clone_url): + """Return the repository a clone URL points at, fork or not. + + Lookup order: + 1. the repository matching the URL itself + 2. for a GitHub fork whose origin is known, the fork itself, which + is registered on the fly to document where the modules are + installed from and to hold the credentials reading it needs + + Returns an empty recordset when the origin cannot be told, letting the + caller fall back to its own heuristics. + """ + host, org, name = self._parse_clone_url(clone_url) + if not org: + return self.browse() + repository = self._find_from_org_and_name(org, name) + if repository: + return repository + if host not in ("github.com", "www.github.com"): + # Only GitHub exposes the fork ancestry through its API + return self.browse() + try: + full_names = self._fetch_github_fork_parents(org, name) + except RuntimeError: + # Do not let a GitHub outage or rate limit break the caller. The + # failure is not cached, so the next call will try again. + _logger.warning( + "Unable to get the fork ancestry of %s", clone_url, exc_info=True + ) + return self.browse() + for full_name in full_names: + parent_org, __, parent_name = full_name.rpartition("/") + upstream = self._find_from_org_and_name(parent_org, parent_name) + if upstream: + return self._create_fork(host, org, name, clone_url, upstream) + return self.browse() + + @api.model + def _create_fork(self, host, org, name, clone_url, upstream): + """Register the fork of `upstream` hosted at `host`, as `org`/`name`.""" + org_record = self.env["odoo.repository.org"].search([("name", "=", org)]) + if not org_record: + org_record = self.env["odoo.repository.org"].sudo().create({"name": org}) + _logger.info( + "Registering %s/%s as a fork of %s", org, name, upstream.display_name + ) + return ( + self.sudo() + .create( + { + "org_id": org_record.id, + "name": name, + "repo_url": f"https://{host}/{org}/{name}", + "clone_url": clone_url, + # A fork is hosted next to the repository it originates + # from, whose ancestry is what told us it is one. + "repo_type": upstream.repo_type, + "to_scan": False, + "upstream_repository_id": upstream.id, + } + ) + .with_env(self.env) + ) + + @api.model + def _find_upstream_repository(self, clone_url): + """Return the repository the modules of a clone URL belong to. + + The modules of a fork are those of the repository it originates from: + a fork is never scanned, so it hosts none of its own. + """ + repository = self._find_from_clone_url(clone_url) + return repository.upstream_repository_id or repository diff --git a/odoo_repository_fork/pyproject.toml b/odoo_repository_fork/pyproject.toml new file mode 100644 index 00000000..4231d0cc --- /dev/null +++ b/odoo_repository_fork/pyproject.toml @@ -0,0 +1,3 @@ +[build-system] +requires = ["whool"] +build-backend = "whool.buildapi" diff --git a/odoo_repository_fork/readme/CONTRIBUTORS.md b/odoo_repository_fork/readme/CONTRIBUTORS.md new file mode 100644 index 00000000..c4904374 --- /dev/null +++ b/odoo_repository_fork/readme/CONTRIBUTORS.md @@ -0,0 +1,2 @@ +- ACSONE SA/NV + - Laurent Mignon \ diff --git a/odoo_repository_fork/readme/DESCRIPTION.md b/odoo_repository_fork/readme/DESCRIPTION.md new file mode 100644 index 00000000..d92184c6 --- /dev/null +++ b/odoo_repository_fork/readme/DESCRIPTION.md @@ -0,0 +1,8 @@ +A module is not always installed from the repository it belongs to. Freezing +one built from unmerged pull requests is commonly done by pinning a revision +of a fork, and that fork is nowhere in Odoo MCA: its ancestry has to be asked +to GitHub over and over, and nothing holds the credentials a private one needs +to be read. + +This module registers a fork as a repository of its own, pointing at the one +it originates from. diff --git a/odoo_repository_fork/readme/USAGE.md b/odoo_repository_fork/readme/USAGE.md new file mode 100644 index 00000000..500243e7 --- /dev/null +++ b/odoo_repository_fork/readme/USAGE.md @@ -0,0 +1,11 @@ +A fork is registered on the fly, whenever something looks up the repository a +clone URL points at and GitHub answers that it is a fork of a repository +already known. + +It can be created by hand too, by setting *Fork Of* on a repository. Either +way, a fork is never scanned: it hosts the very same modules as the repository +it comes from, and a second scanned repository holding them would make the +module of a dependency ambiguous for every project. A constraint enforces it. + +Its credentials remain editable though, and they are what reading a private +fork needs. The forks of a repository are listed on it. diff --git a/odoo_repository_fork/static/description/index.html b/odoo_repository_fork/static/description/index.html new file mode 100644 index 00000000..623f8d98 --- /dev/null +++ b/odoo_repository_fork/static/description/index.html @@ -0,0 +1,446 @@ + + + + + +Odoo MCA - Forks + + + +
+

Odoo MCA - Forks

+ + +

Beta License: AGPL-3 OCA/module-composition-analysis Translate me on Weblate Try me on Runboat

+

A module is not always installed from the repository it belongs to. +Freezing one built from unmerged pull requests is commonly done by +pinning a revision of a fork, and that fork is nowhere in Odoo MCA: its +ancestry has to be asked to GitHub over and over, and nothing holds the +credentials a private one needs to be read.

+

This module registers a fork as a repository of its own, pointing at the +one it originates from.

+

Table of contents

+ +
+

Usage

+

A fork is registered on the fly, whenever something looks up the +repository a clone URL points at and GitHub answers that it is a fork of +a repository already known.

+

It can be created by hand too, by setting Fork Of on a repository. +Either way, a fork is never scanned: it hosts the very same modules as +the repository it comes from, and a second scanned repository holding +them would make the module of a dependency ambiguous for every project. +A constraint enforces it.

+

Its credentials remain editable though, and they are what reading a +private fork needs. The forks of a repository are listed on it.

+
+
+

Bug Tracker

+

Bugs are tracked on GitHub Issues. +In case of trouble, please check there if your issue has already been reported. +If you spotted it first, help us to smash it by providing a detailed and welcomed +feedback.

+

Do not contact contributors directly about support or help with technical issues.

+
+
+

Credits

+
+

Authors

+
    +
  • ACSONE SA/NV
    • +
+
+
+

Contributors

+ +
+
+

Maintainers

+

This module is maintained by the OCA.

+ +Odoo Community Association + +

OCA, or the Odoo Community Association, is a nonprofit organization whose +mission is to support the collaborative development of Odoo features and +promote its widespread use.

+

This module is part of the OCA/module-composition-analysis project on GitHub.

+

You are welcome to contribute. To learn how please visit https://odoo-community.org/page/Contribute.

+
+
+
+ + diff --git a/odoo_repository_fork/tests/__init__.py b/odoo_repository_fork/tests/__init__.py new file mode 100644 index 00000000..4eb89124 --- /dev/null +++ b/odoo_repository_fork/tests/__init__.py @@ -0,0 +1 @@ +from . import test_odoo_repository_fork diff --git a/odoo_repository_fork/tests/test_odoo_repository_fork.py b/odoo_repository_fork/tests/test_odoo_repository_fork.py new file mode 100644 index 00000000..a264aa1d --- /dev/null +++ b/odoo_repository_fork/tests/test_odoo_repository_fork.py @@ -0,0 +1,201 @@ +# Copyright 2026 ACSONE SA/NV () +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl) + +from unittest.mock import patch + +from odoo.exceptions import ValidationError +from odoo.tools import mute_logger + +from odoo.addons.odoo_repository.tests.common import Common + +GITHUB_REQUEST = ( + "odoo.addons.odoo_repository_fork.models.odoo_repository.github.request" +) + + +class TestOdooRepositoryFork(Common): + def setUp(self): + super().setUp() + # '_fetch_github_fork_parents' is an ormcache, make sure a test never + # observes what a previous one stored. + self.env.registry.clear_cache() + self.repository_model = self.env["odoo.repository"] + + def _create_repository(self, org_name, repo_name, **values): + org = self.env["odoo.repository.org"].search([("name", "=", org_name)]) + if not org: + org = self.env["odoo.repository.org"].create({"name": org_name}) + vals = { + "org_id": org.id, + "name": repo_name, + "repo_url": f"https://github.com/{org_name}/{repo_name}", + "clone_url": f"https://github.com/{org_name}/{repo_name}.git", + "repo_type": "github", + } + if values.get("upstream_repository_id"): + vals["to_scan"] = False + vals.update(values) + return self.env["odoo.repository"].create(vals) + + def test_find_upstream_known_repository(self): + """A URL pointing to a known repository resolves without calling GitHub.""" + repository = self._create_repository("OCA", "account-invoicing") + with patch(GITHUB_REQUEST) as request: + found = self.repository_model._find_upstream_repository( + "https://github.com/OCA/account-invoicing.git" + ) + self.assertEqual(found, repository) + request.assert_not_called() + + def test_find_upstream_from_github_fork(self): + """An unknown fork resolves to the repository it originates from.""" + upstream = self._create_repository("OCA", "account-invoicing") + payload = { + "fork": True, + "parent": {"full_name": "OCA/account-invoicing"}, + "source": {"full_name": "OCA/account-invoicing"}, + } + with patch(GITHUB_REQUEST, return_value=payload) as request: + found = self.repository_model._find_upstream_repository( + "https://github.com/acsone/account-invoicing.git" + ) + self.assertEqual(found, upstream) + request.assert_called_once() + self.assertIn("repos/acsone/account-invoicing", request.call_args[0]) + + def test_find_upstream_prefers_parent_over_source(self): + """On a fork chain, the closest known ancestor wins.""" + self._create_repository("OCA", "account-invoicing") + intermediate = self._create_repository("camptocamp", "account-invoicing") + payload = { + "fork": True, + "parent": {"full_name": "camptocamp/account-invoicing"}, + "source": {"full_name": "OCA/account-invoicing"}, + } + with patch(GITHUB_REQUEST, return_value=payload): + found = self.repository_model._find_upstream_repository( + "https://github.com/acsone/account-invoicing.git" + ) + self.assertEqual(found, intermediate) + + def test_find_upstream_falls_back_to_source(self): + """An unknown intermediate fork does not hide the root of the chain.""" + root = self._create_repository("OCA", "account-invoicing") + payload = { + "fork": True, + "parent": {"full_name": "unknown-org/account-invoicing"}, + "source": {"full_name": "OCA/account-invoicing"}, + } + with patch(GITHUB_REQUEST, return_value=payload): + found = self.repository_model._find_upstream_repository( + "https://github.com/acsone/account-invoicing.git" + ) + self.assertEqual(found, root) + + def test_find_upstream_not_a_fork(self): + """A standalone repository has no upstream to resolve.""" + self._create_repository("OCA", "account-invoicing") + with patch(GITHUB_REQUEST, return_value={"fork": False}): + found = self.repository_model._find_upstream_repository( + "https://github.com/acsone/vendored-addons.git" + ) + self.assertFalse(found) + + def test_find_upstream_non_github_host(self): + """Only GitHub exposes the fork ancestry: no API call is attempted.""" + with patch(GITHUB_REQUEST) as request: + found = self.repository_model._find_upstream_repository( + "https://gitlab.com/acsone/account-invoicing.git" + ) + self.assertFalse(found) + request.assert_not_called() + + def test_find_upstream_github_error_is_not_cached(self): + """A GitHub outage degrades gracefully and does not poison the cache.""" + upstream = self._create_repository("OCA", "account-invoicing") + url = "https://github.com/acsone/account-invoicing.git" + with ( + patch(GITHUB_REQUEST, side_effect=RuntimeError("API rate limit")), + mute_logger("odoo.addons.odoo_repository_fork.models.odoo_repository"), + ): + self.assertFalse(self.repository_model._find_upstream_repository(url)) + payload = { + "fork": True, + "parent": {"full_name": "OCA/account-invoicing"}, + "source": {"full_name": "OCA/account-invoicing"}, + } + with patch(GITHUB_REQUEST, return_value=payload): + self.assertEqual( + self.repository_model._find_upstream_repository(url), upstream + ) + + # -- registering a fork ------------------------------------------------- + + def test_fork_is_registered_on_the_fly(self): + """Discovering a fork documents it, so it can carry its credentials.""" + upstream = self._create_repository("OCA", "account-invoicing") + payload = { + "fork": True, + "parent": {"full_name": "OCA/account-invoicing"}, + "source": {"full_name": "OCA/account-invoicing"}, + } + url = "https://github.com/acsone/account-invoicing.git" + with patch(GITHUB_REQUEST, return_value=payload): + fork = self.repository_model._find_from_clone_url(url) + self.assertEqual(fork.upstream_repository_id, upstream) + self.assertEqual(fork.org_id.name, "acsone") + self.assertEqual(fork.name, "account-invoicing") + self.assertEqual(fork.clone_url, url) + self.assertEqual(fork.repo_url, "https://github.com/acsone/account-invoicing") + self.assertEqual(fork.repo_type, upstream.repo_type) + # A fork hosts the modules of its origin, scanning it would duplicate them + self.assertFalse(fork.to_scan) + self.assertEqual(upstream.fork_ids, fork) + + def test_fork_is_registered_once(self): + """A known fork is found by its URL, without asking GitHub again.""" + self._create_repository("OCA", "account-invoicing") + payload = { + "fork": True, + "parent": {"full_name": "OCA/account-invoicing"}, + "source": {"full_name": "OCA/account-invoicing"}, + } + url = "https://github.com/acsone/account-invoicing.git" + with patch(GITHUB_REQUEST, return_value=payload): + fork = self.repository_model._find_from_clone_url(url) + with patch(GITHUB_REQUEST) as request: + found = self.repository_model._find_from_clone_url(url) + self.assertEqual(found, fork) + request.assert_not_called() + + def test_fork_cannot_be_scanned(self): + """Scanning a fork would make the module of a dependency ambiguous.""" + upstream = self._create_repository("OCA", "account-invoicing") + fork = self._create_repository( + "acsone", "account-invoicing", upstream_repository_id=upstream.id + ) + self.assertFalse(fork.to_scan) + with self.assertRaisesRegex(ValidationError, "cannot be scanned"): + fork.to_scan = True + + def test_fork_cannot_be_its_own_upstream(self): + repository = self._create_repository("OCA", "account-invoicing") + with self.assertRaises(ValidationError): + repository.upstream_repository_id = repository + + def test_fork_is_reached_with_its_own_credentials(self): + """A private fork is read with the token registered on it.""" + upstream = self._create_repository("OCA", "account-invoicing") + token = self.env["authentication.token"].create( + {"name": "acsone", "token": "s3cr3t"} + ) + fork = self._create_repository( + "acsone", + "account-invoicing", + upstream_repository_id=upstream.id, + token_id=token.id, + ) + params = fork._prepare_base_scanner_parameters() + self.assertEqual(params["org"], "acsone") + self.assertEqual(params["token"], "s3cr3t") + self.assertNotEqual(params["token"], upstream._get_token()) diff --git a/odoo_repository_fork/views/odoo_repository.xml b/odoo_repository_fork/views/odoo_repository.xml new file mode 100644 index 00000000..53d196d8 --- /dev/null +++ b/odoo_repository_fork/views/odoo_repository.xml @@ -0,0 +1,37 @@ + + + + + odoo.repository.form.fork + odoo.repository + + + + + + + + not to_scan and not upstream_repository_id + + + + + + + + + + + + + + +