build: Modernize xapi-db-load: migrate to uv, pyproject.toml, PEP 735 dependency groups

.github/workflows/ci.yml

@@ -1,39 +1,41 @@
-name: Run all tests & checks
+name: CI
 
 on:
   push:
-    branches: [ main ]
+    branches: [main]
   pull_request:
-    branches: [ main ]
+  workflow_call:
 
 jobs:
   run_tests:
-    name: tests
+    name: ${{ matrix.toxenv }}
     runs-on: ubuntu-latest
     strategy:
+      fail-fast: true
       matrix:
-        python-version: [ '3.12' ]
-    permissions:
-      # Gives the action the necessary permissions for publishing new
-      # comments in pull requests.
-      pull-requests: write
-      # Gives the action the necessary permissions for pushing data to the
-      # python-coverage-comment-action branch, and for editing existing
-      # comments (to avoid publishing multiple comments in the same PR)
-      contents: write
+        python-version: ["3.12"]
+        toxenv: [py, quality, docs]
 
     steps:
-      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
-      - name: setup python
-        uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
+      - name: Checkout repository
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v8.3.2
         with:
-          python-version: ${{ matrix.python-version }}
+          enable-cache: true
+          python-version: "${{ matrix.python-version }}"
 
-      - name: Install pip
-        run: pip install -r requirements/pip.txt
+      - name: Install CI dependencies
+        run: uv sync --group ci
 
-      - name: Install Dependencies
-        run: pip install -r requirements/ci.txt
+      - name: Run tox
+        run: uv run tox -e ${{ matrix.toxenv }}
 
-      - name: Run Tests
-        run: tox
+      - name: Upload coverage to Codecov
+        if: matrix.toxenv == 'py' && matrix.python-version == '3.12'
+        uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v6.0.2
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          flags: unittests
+          fail_ci_if_error: true

MANIFEST.in

@@ -1,6 +1,4 @@
 include CHANGELOG.rst
 include LICENSE.txt
 include README.rst
-include requirements/base.in
-include requirements/constraints.txt
-recursive-include xapi-db-load *.html *.png *.gif *.js *.css *.jpg *.jpeg *.svg *.csv
+recursive-include xapi_db_load *.html *.png *.gif *.js *.css *.jpg *.jpeg *.svg *.csv

Makefile

@@ -29,40 +29,24 @@ docs: ## generate Sphinx HTML documentation, including API docs
 	tox -e docs
 	$(BROWSER)docs/_build/html/index.html
 
-# Define PIP_COMPILE_OPTS=-v to get more information during make upgrade.
-PIP_COMPILE = pip-compile --upgrade $(PIP_COMPILE_OPTS)
-
-upgrade: export CUSTOM_COMPILE_COMMAND=make upgrade
-upgrade: ## update the requirements/*.txt files with the latest packages satisfying requirements/*.in
-	pip install -r requirements/pip-tools.txt
-	pip install -qr requirements/pip.txt
-	# Make sure to compile files after any other files they include!
-	$(PIP_COMPILE) --allow-unsafe --rebuild -o requirements/pip.txt requirements/pip.in
-	$(PIP_COMPILE) -o requirements/pip-tools.txt requirements/pip-tools.in
-	pip install -qr requirements/pip.txt
-	pip install -r requirements/pip-tools.txt
-	$(PIP_COMPILE) -o requirements/base.txt requirements/base.in
-	$(PIP_COMPILE) -o requirements/test.txt requirements/test.in
-	$(PIP_COMPILE) -o requirements/doc.txt requirements/doc.in
-	$(PIP_COMPILE) -o requirements/quality.txt requirements/quality.in
-	$(PIP_COMPILE) -o requirements/ci.txt requirements/ci.in
-	$(PIP_COMPILE) -o requirements/dev.txt requirements/dev.in
+upgrade: ## update the uv.lock file with the latest packages satisfying pyproject.toml
+	uv run --with edx-lint edx_lint write_uv_constraints pyproject.toml
+	uv lock --upgrade
 
 quality: ## check coding style with pycodestyle and pylint
-	pylint xapi_db_load *.py
-	pycodestyle xapi_db_load  *.py
-	pydocstyle xapi_db_load *.py
+	pylint xapi_db_load
+	pycodestyle xapi_db_load
+	pydocstyle xapi_db_load
 	mypy xapi_db_load
-	isort --check-only --diff --recursive xapi_db_load *.py test_settings.py
-	python setup.py bdist_wheel
+	isort --check-only --diff xapi_db_load
+	uv run python -m build
 	twine check dist/*
 	make selfcheck
 
 
 requirements: ## install development environment requirements
-	pip install -r requirements/pip.txt
-	pip install -r requirements/pip-tools.txt
-	pip-sync requirements/dev.txt requirements/private.*
+	uv sync --group dev
+	uv tool install tox --with tox-uv
 
 test: clean ## run tests in the current virtualenv
 	pytest

docs/conf.py

@@ -12,32 +12,15 @@
 serve to show the default.
 """
 import os
-import re
 import sys
 from datetime import datetime, UTC
+from importlib.metadata import PackageNotFoundError, version
 from subprocess import check_call
 
-
-def get_version(*file_paths):
-    """
-    Extract the version string from the file.
-
-    Input:
-     - file_paths: relative path fragments to file with
-                   version string
-    """
-    filename = os.path.join(os.path.dirname(__file__), *file_paths)
-    version_file = open(filename, encoding="utf8").read()
-    version_match = re.search(r"^__version__ = ['\"]([^'\"]*)['\"]", version_file, re.M)
-    if version_match:
-        return version_match.group(1)
-    raise RuntimeError("Unable to find version string.")
-
-
-REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
-sys.path.append(REPO_ROOT)
-
-VERSION = get_version("../xapi_db_load", "__init__.py")
+try:
+    VERSION = version("xapi-db-load")
+except PackageNotFoundError:
+    VERSION = "unknown"
 
 
 # If extensions (or modules to document with autodoc) are in another directory,
@@ -70,6 +53,9 @@ def get_version(*file_paths):
     "image.nonlocal_uri",
 ]
 
+# Mock imports that fail at module level in the docs environment.
+autodoc_mock_imports = ["chdb"]
+
 # Add any paths that contain templates here, relative to this directory.
 templates_path = ["_templates"]
 
@@ -187,7 +173,6 @@ def get_version(*file_paths):
     "repository_url": "https://github.com/openedx/xapi-db-load",
     "repository_branch": "main",
     "path_to_docs": "docs/",
-    "logo_only": True,
     "home_page_in_toc": True,
     "use_repository_button": True,
     "use_issues_button": True,
@@ -538,7 +523,7 @@ def on_init(app):  # pylint: disable=unused-argument
         # If we are, assemble the path manually
         bin_path = os.path.abspath(os.path.join(sys.prefix, "bin"))
         apidoc_path = os.path.join(bin_path, apidoc_path)
-    check_call([apidoc_path, "-o", docs_path, os.path.join(root_path, "xapi_db_load")])
+    check_call([apidoc_path, "-f", "-o", docs_path, os.path.join(root_path, "xapi_db_load")])
 
 
 def setup(app):

docs/xapi_db_load.backends.rst

@@ -4,42 +4,58 @@ xapi\_db\_load.backends package
 Submodules
 ----------
 
-xapi\_db\_load.backends.citus\_lake module
-------------------------------------------
+xapi\_db\_load.backends.base\_async\_backend module
+---------------------------------------------------
 
-.. automodule:: xapi_db_load.backends.citus_lake
+.. automodule:: xapi_db_load.backends.base_async_backend
    :members:
-   :undoc-members:
    :show-inheritance:
+   :undoc-members:
 
-xapi\_db\_load.backends.clickhouse\_lake module
------------------------------------------------
+xapi\_db\_load.backends.chdb module
+-----------------------------------
 
-.. automodule:: xapi_db_load.backends.clickhouse_lake
+.. automodule:: xapi_db_load.backends.chdb
    :members:
-   :undoc-members:
    :show-inheritance:
+   :undoc-members:
 
-xapi\_db\_load.backends.mongo\_lake module
-------------------------------------------
+xapi\_db\_load.backends.clickhouse module
+-----------------------------------------
 
-.. automodule:: xapi_db_load.backends.mongo_lake
+.. automodule:: xapi_db_load.backends.clickhouse
    :members:
+   :show-inheritance:
    :undoc-members:
+
+xapi\_db\_load.backends.csv module
+----------------------------------
+
+.. automodule:: xapi_db_load.backends.csv
+   :members:
    :show-inheritance:
+   :undoc-members:
 
-xapi\_db\_load.backends.ralph\_lrs module
------------------------------------------
+xapi\_db\_load.backends.ralph module
+------------------------------------
 
-.. automodule:: xapi_db_load.backends.ralph_lrs
+.. automodule:: xapi_db_load.backends.ralph
    :members:
+   :show-inheritance:
    :undoc-members:
+
+xapi\_db\_load.backends.vector module
+-------------------------------------
+
+.. automodule:: xapi_db_load.backends.vector
+   :members:
    :show-inheritance:
+   :undoc-members:
 
 Module contents
 ---------------
 
 .. automodule:: xapi_db_load.backends
    :members:
-   :undoc-members:
    :show-inheritance:
+   :undoc-members:

docs/xapi_db_load.fixtures.rst

@@ -0,0 +1,21 @@
+xapi\_db\_load.fixtures package
+===============================
+
+Submodules
+----------
+
+xapi\_db\_load.fixtures.music\_tags module
+------------------------------------------
+
+.. automodule:: xapi_db_load.fixtures.music_tags
+   :members:
+   :show-inheritance:
+   :undoc-members:
+
+Module contents
+---------------
+
+.. automodule:: xapi_db_load.fixtures
+   :members:
+   :show-inheritance:
+   :undoc-members:

docs/xapi_db_load.rst

@@ -8,39 +8,74 @@ Subpackages
    :maxdepth: 4
 
    xapi_db_load.backends
+   xapi_db_load.fixtures
+   xapi_db_load.tests
+   xapi_db_load.ui
    xapi_db_load.xapi
 
 Submodules
 ----------
 
+xapi\_db\_load.async\_app module
+--------------------------------
+
+.. automodule:: xapi_db_load.async_app
+   :members:
+   :show-inheritance:
+   :undoc-members:
+
+xapi\_db\_load.constants module
+-------------------------------
+
+.. automodule:: xapi_db_load.constants
+   :members:
+   :show-inheritance:
+   :undoc-members:
+
 xapi\_db\_load.course\_configs module
 -------------------------------------
 
 .. automodule:: xapi_db_load.course_configs
    :members:
-   :undoc-members:
    :show-inheritance:
+   :undoc-members:
 
-xapi\_db\_load.generate\_load module
-------------------------------------
+xapi\_db\_load.generate\_load\_async module
+-------------------------------------------
 
-.. automodule:: xapi_db_load.generate_load
+.. automodule:: xapi_db_load.generate_load_async
    :members:
-   :undoc-members:
    :show-inheritance:
+   :undoc-members:
 
 xapi\_db\_load.main module
 --------------------------
 
 .. automodule:: xapi_db_load.main
    :members:
+   :show-inheritance:
    :undoc-members:
+
+xapi\_db\_load.runner module
+----------------------------
+
+.. automodule:: xapi_db_load.runner
+   :members:
    :show-inheritance:
+   :undoc-members:
+
+xapi\_db\_load.waiter module
+----------------------------
+
+.. automodule:: xapi_db_load.waiter
+   :members:
+   :show-inheritance:
+   :undoc-members:
 
 Module contents
 ---------------
 
 .. automodule:: xapi_db_load
    :members:
-   :undoc-members:
    :show-inheritance:
+   :undoc-members: