diff --git a/.nanvix/NANVIX.md b/.nanvix/NANVIX.md index 33ab81dbd0e37c..f9ce7c115da8f8 100644 --- a/.nanvix/NANVIX.md +++ b/.nanvix/NANVIX.md @@ -28,9 +28,6 @@ This document describes the port of [CPython](https://www.python.org/) interpret - OpenSSL 3.5.0 - SQLite 3.49.0 - libffi 3.4.6 -- libxml2 2.12.9 -- libxslt 1.1.42 -- lxml 5.3.0 - xz 5.2.5 --- @@ -233,7 +230,7 @@ cd .nanvix/out/test && \ ### Test Suite Status -The `./z test` target runs **160 CPython stdlib test modules** on Nanvix +The `./z test` target runs **159 CPython stdlib test modules** on Nanvix (i686, microvm, 256 MB RAM). Tests are split into batches of 4 modules per VM invocation to stay within per-process memory limits. @@ -244,7 +241,7 @@ runner (`python -m test`). A `/tmp` directory is created on the ramfs so | Metric | Value | |--------|-------| -| **Modules enabled** | 160 | +| **Modules enabled** | 159 | | **Tests passed** | All supported tests | | **Tests skipped** | Unsupported Nanvix facilities use explicit skip reasons | | **Tests failed** | 0 | diff --git a/.nanvix/_docker.py b/.nanvix/_docker.py index 29f577a5c8a560..24deb16d272230 100644 --- a/.nanvix/_docker.py +++ b/.nanvix/_docker.py @@ -21,6 +21,7 @@ import build as build_mod import config +import setup_local as setup_local_mod def _workspace_id(workspace: Path) -> str: @@ -228,21 +229,17 @@ def docker_build( def _generate_setup_local_cmd() -> str: - """Shell command to generate Modules/Setup.local inside the container.""" + """Generate Modules/Setup.local inside the SDK container.""" buildroot = config.DOCKER_BUILDROOT_PATH ws = config.DOCKER_WORKSPACE_PATH - return ( - f"printf '%s\\n' " - f"'# Auto-generated by .nanvix/docker.py -- do not edit manually.' " - f"'# Statically-linked extension modules for Nanvix builds.' " - f"'#' " - f"'# Nanvix OS interface module (snapshot, host-mount).' " - f"'_nanvix _nanvixmodule.c' " - f"'# lxml C extension modules (statically linked via pre-built archives).' " - f"'_lxml_etree lxml_etree_builtin.c -L{buildroot}/lib -llxml_etree -lxslt -lexslt -lxml2 -lz' " - f"'_lxml_elementpath lxml_elementpath_builtin.c -L{buildroot}/lib -llxml_elementpath -lxml2 -lz' " - f"> {ws}/Modules/Setup.local" + rendered = setup_local_mod.render_setup_local( + buildroot=buildroot, + header_comment="Auto-generated by .nanvix/_docker.py -- do not edit manually.", + ) + quoted_lines = " ".join( + "'" + line.replace("'", "'\"'\"'") + "'" for line in rendered.splitlines() ) + return f"printf '%s\\n' {quoted_lines} > {ws}/Modules/Setup.local" def _copy_outputs_cmd() -> str: diff --git a/.nanvix/_test.py b/.nanvix/_test.py index 2830baee5dad81..acb195b0a4e56b 100644 --- a/.nanvix/_test.py +++ b/.nanvix/_test.py @@ -26,9 +26,96 @@ import build as build_mod import config -import lxml as lxml_mod import ramfs as ramfs_mod +# --------------------------------------------------------------------------- +# Shared-extension smoke checks +# --------------------------------------------------------------------------- + +_SO_MODULE_SANITY_CHECKS: tuple[tuple[str, tuple[tuple[str, str], ...]], ...] = ( + ( + "CPYTHON_TEST_DATA_PRIMITIVES", + ( + ("_bisect", "m.bisect_left([1, 3, 5], 4) == 2"), + ("_heapq", "m.heappush([], 1) is None"), + ("_struct", "m.pack('i', 42) == b'\\x2a\\x00\\x00\\x00'"), + ("_random", "hasattr(m, 'Random')"), + ("_opcode", "hasattr(m, 'stack_effect')"), + ("_queue", "hasattr(m, 'SimpleQueue')"), + ("_csv", "hasattr(m, 'reader')"), + ("binascii", "m.hexlify(b'\\xab') == b'ab'"), + ("_json", "hasattr(m, 'encode_basestring_ascii')"), + ("_pickle", "hasattr(m, 'Pickler')"), + ("_zoneinfo", "hasattr(m, 'ZoneInfo')"), + ), + ), + ( + "CPYTHON_TEST_MATH", + ( + ("math", "abs(m.sqrt(4.0) - 2.0) < 1e-9"), + ("cmath", "abs(m.sqrt(complex(-1)) - complex(0, 1)) < 1e-9"), + ("_statistics", "hasattr(m, '_normal_dist_inv_cdf')"), + ("mmap", "hasattr(m, 'mmap')"), + ("_contextvars", "hasattr(m, 'ContextVar')"), + ), + ), + ( + "CPYTHON_TEST_CODECS", + ( + ("unicodedata", "m.lookup('LATIN SMALL LETTER A') == 'a'"), + ("_codecs_cn", "hasattr(m, 'getcodec')"), + ("_codecs_hk", "hasattr(m, 'getcodec')"), + ("_codecs_iso2022", "hasattr(m, 'getcodec')"), + ("_codecs_jp", "hasattr(m, 'getcodec')"), + ("_codecs_kr", "hasattr(m, 'getcodec')"), + ("_codecs_tw", "hasattr(m, 'getcodec')"), + ), + ), + ( + "CPYTHON_TEST_BUNDLED_DEPS", + ( + ("_asyncio", "hasattr(m, 'Future')"), + ("_decimal", "m.Decimal('1.1') + m.Decimal('2.2') == m.Decimal('3.3')"), + ("_elementtree", "hasattr(m, 'XMLParser')"), + ("_md5", "hasattr(m, 'md5')"), + ("_sha1", "hasattr(m, 'sha1')"), + ("_sha2", "hasattr(m, 'sha256')"), + ("_sha3", "hasattr(m, 'sha3_256')"), + ("_blake2", "hasattr(m, 'blake2b')"), + ("select", "hasattr(m, 'select')"), + ("_socket", "hasattr(m, 'socket')"), + ("_posixsubprocess", "hasattr(m, 'fork_exec')"), + ("fcntl", "hasattr(m, 'fcntl')"), + ("termios", "hasattr(m, 'tcgetattr')"), + ), + ), +) + + +def _render_so_sanity_snippets( + checks: tuple[ + tuple[str, tuple[tuple[str, str], ...]], ... + ] = _SO_MODULE_SANITY_CHECKS, +) -> str: + """Render imports that prove each migrated module loads through dlopen.""" + snippets: list[str] = [] + for log_tag, modules in checks: + items = ",\n".join( + f" ({name!r}, lambda m: {check})" for name, check in modules + ) + snippets.append( + f"_so_checks = [\n{items},\n]\n" + "for _name, _check in _so_checks:\n" + " _mod = __import__(_name)\n" + " assert _name not in sys.builtin_module_names, " + "f'{_name} still built-in!'\n" + " assert _check(_mod), f'{_name} sanity check failed'\n" + f" print(f'{log_tag}: " + "{_name} loaded via dlopen from {_mod.__file__}')\n" + ) + return "".join(snippets) + + # --------------------------------------------------------------------------- # Initrd creation helper (standalone mode) # --------------------------------------------------------------------------- @@ -237,25 +324,34 @@ def stage(args: build_mod.MakeArgs) -> None: shutil.copy2(scdata_src, scdata_dst) print(f" Copied {scdata_name} from build dir (make install missed it)") - # Hello-world test script. The lxml import is exercised against the - # in-memory FAT ramfs VFS via xmlInitParser(). - lxml_snippet = ( - "try:\n" - " import lxml.etree\n" - " doc = lxml.etree.fromstring(b'lxml OK')\n" - " assert doc.tag == 'root'\n" - " assert doc[0].text == 'lxml OK'\n" - " print('CPYTHON_TEST_LXML: lxml.etree import and parse OK')\n" - "except ImportError as e:\n" - " print(f'CPYTHON_TEST_LXML_SKIP: {e}')\n" - "except Exception as e:\n" - " print(f'CPYTHON_TEST_LXML_FAIL: {e}')\n" - " sys.exit(1)\n" + # Hello-world test script. The array check proves that the first + # stdlib module migrated to a shared extension is loaded through dlopen. + array_snippet = ( + "import array\n" + "assert 'array' not in sys.builtin_module_names, 'array still built-in!'\n" + "_array = array.array('i', [1, 2, 3])\n" + "assert _array.tolist() == [1, 2, 3]\n" + "print(f'CPYTHON_TEST_ARRAY_SO: array loaded via dlopen from {array.__file__}')\n" + ) + nested_import_snippet = ( + "import xml.etree.ElementTree as _elementtree_api\n" + "assert _elementtree_api.fromstring('').tag == 'root'\n" + "assert '_elementtree' not in sys.builtin_module_names\n" + "assert 'pyexpat' in sys.builtin_module_names\n" + "import encodings.gb2312\n" + "assert '\\u4e2d\\u6587'.encode('gb2312') == b'\\xd6\\xd0\\xce\\xc4'\n" + "assert '_codecs_cn' not in sys.builtin_module_names\n" + "assert '_multibytecodec' in sys.builtin_module_names\n" + "print('CPYTHON_TEST_NESTED_IMPORTS: static C API anchors OK')\n" ) + (staging / "test_hello.py").write_text( "import sys\n" "print('CPYTHON_TEST_HELLO: Hello from Python', sys.version_info[:2])\n" - "print('CPYTHON_TEST_PLATFORM:', sys.platform)\n" + lxml_snippet + "print('CPYTHON_TEST_PLATFORM:', sys.platform)\n" + + array_snippet + + nested_import_snippet + + _render_so_sanity_snippets() ) # HTTP server smoke-test script must be present in the sysroot before @@ -472,26 +568,18 @@ def run_hello( # Validate output. found_hello = False - found_lxml = False for line in output.splitlines(): if line.startswith("CPYTHON_TEST_"): tag = line.split(":")[0].replace("CPYTHON_TEST_", "") print(f" {tag}: {line.strip()}") if tag == "HELLO": found_hello = True - elif tag in ("LXML", "LXML_SKIP"): - found_lxml = True if not found_hello: print(" FAIL: Hello test did not produce expected output") print(output) raise RuntimeError("Hello test did not produce expected output") - if not found_lxml: - # lxml staging is best-effort — if the runtime package was not - # available (e.g. release asset missing), the test is non-fatal. - print(" WARNING: lxml import/parse test did not produce expected output") - print(" PASS") @@ -774,8 +862,6 @@ def run_all( stage(args) stage_ramfs(args) - lxml_mod.stage_lxml_runtime(staging) - # Hello test. run_hello( staging, diff --git a/.nanvix/build.py b/.nanvix/build.py index 53456c5ba3f7f8..a4a85fefe69ec8 100644 --- a/.nanvix/build.py +++ b/.nanvix/build.py @@ -22,7 +22,7 @@ import _docker as docker_mod import _test as test_mod import config -import lxml as lxml_mod +import setup_local as setup_local_mod @dataclass @@ -100,7 +100,7 @@ def build( buildroot_for_setup = ( Path(config.DOCKER_BUILDROOT_PATH) if _args.docker else args.buildroot ) - lxml_mod.generate_setup_local(paths.repo_root(), buildroot_for_setup) + setup_local_mod.generate_setup_local(paths.repo_root(), buildroot_for_setup) _args.run(cwd=paths.repo_root()) install(dest_dir, args) diff --git a/.nanvix/config.py b/.nanvix/config.py index a60db3ad40947a..749a1174039590 100644 --- a/.nanvix/config.py +++ b/.nanvix/config.py @@ -239,8 +239,6 @@ def _manifest_sdk_image() -> str: # #323 wave 8 — regex and plistlib "test_re", "test_plistlib", - # #600 — lxml built-in smoke test - "test_nanvix_lxml", # #526 — _lzma stdlib enablement "test_lzma", # #327 — network and protocol tests (IPv4 only; IPv6 disabled) @@ -337,8 +335,8 @@ def _manifest_sdk_image() -> str: "lib/pkgconfig", ] -# site-packages is no longer trimmed because lxml runtime files may be -# installed there by downstream packaging. When the directory is empty +# site-packages is not trimmed because downstream third-party packages may +# install runtime files there. When the directory is empty # it remains harmlessly on disk (ramfs.trim_sysroot only removes empty # bin/). To force-trim site-packages for minimal images, add the path # back into SYSROOT_TRIM_DIRS above. diff --git a/.nanvix/lxml.py b/.nanvix/lxml.py deleted file mode 100644 index e7656679480e11..00000000000000 --- a/.nanvix/lxml.py +++ /dev/null @@ -1,87 +0,0 @@ -"""lxml build helpers and runtime staging for Nanvix CPython.""" - -from __future__ import annotations - -import shutil -from pathlib import Path - -from nanvix_zutil import paths - -import config - -_SETUP_LOCAL_TEMPLATE = """\ -# Auto-generated by .nanvix/lxml.py -- do not edit manually. -# Statically-linked extension modules for Nanvix builds. -# -# Nanvix OS interface module (snapshot, host-mount). -_nanvix _nanvixmodule.c -# lxml C extension modules (statically linked via pre-built archives). -_lxml_etree lxml_etree_builtin.c -L{sysroot}/lib -llxml_etree -lxslt -lexslt -lxml2 -lz -_lxml_elementpath lxml_elementpath_builtin.c -L{sysroot}/lib -llxml_elementpath -lxml2 -lz -""" - - -def generate_setup_local(repo_root: Path, buildroot: Path) -> None: - """Generate Modules/Setup.local with statically-linked module definitions. - - Includes both the _nanvix OS interface module and lxml C extensions. - """ - setup_local = repo_root / "Modules" / "Setup.local" - content = _SETUP_LOCAL_TEMPLATE.format(sysroot=buildroot) - setup_local.write_text(content, encoding="utf-8") - print(f"[lxml] Generated {setup_local}") - - -def clear_setup_local(repo_root: Path) -> None: - """Remove the generated Modules/Setup.local.""" - setup_local = repo_root / "Modules" / "Setup.local" - if setup_local.exists(): - setup_local.unlink() - print(f"[lxml] Removed {setup_local}") - - -def stage_lxml_runtime(pkg_root: Path) -> None: - """Copy lxml Python files from buildroot into the test/package sysroot. - - Looks for lxml in ``.nanvix/buildroot/python-packages/lxml/``. - Skips gracefully when the python-packages directory is not available. - """ - lxml_src = paths.nanvix_root() / "buildroot" / "python-packages" / "lxml" - if not lxml_src.is_dir(): - print( - f"[lxml] Staged lxml package not found at {lxml_src}; " - "skipping runtime staging." - ) - return - - py_lib = pkg_root / "lib" / config.PYTHON_LIB_DIR - if not py_lib.is_dir(): - raise RuntimeError(f"Python runtime library directory is missing: {py_lib}") - - dst = py_lib / "lxml" - if dst.exists(): - shutil.rmtree(dst) - shutil.copytree(lxml_src, dst) - - # Ensure the etree.py shim explicitly exports names that are not in - # lxml.etree.__all__ but are expected by downstream packages (e.g. - # openpyxl imports xmlfile). The star-import only picks up names - # listed in __all__; xmlfile/htmlfile are cdef classes omitted from - # that list. - _write_etree_shim(dst / "etree.py") - - print(f"[lxml] Staged {lxml_src} -> {dst}") - - -# The content of the etree.py shim that bridges the built-in _lxml_etree -# C extension to the expected lxml.etree import path. -_ETREE_SHIM = """\ -from _lxml_etree import * -from _lxml_etree import _Element, _ElementTree, _Comment, _ProcessingInstruction, ElementBase, QName, _Attrib -from _lxml_etree import xmlfile, htmlfile -""" - - -def _write_etree_shim(path: Path) -> None: - """Write (or overwrite) the lxml/etree.py shim with correct exports.""" - path.write_text(_ETREE_SHIM, encoding="utf-8") diff --git a/.nanvix/nanvix.lock b/.nanvix/nanvix.lock index e513314e050eba..87cd674d872de3 100644 --- a/.nanvix/nanvix.lock +++ b/.nanvix/nanvix.lock @@ -1,7 +1,7 @@ # nanvix.lock - auto-generated by nanvix_zutil. DO NOT EDIT. [metadata] -manifest-hash = "sha256:b55a24d9397cc0923444716aa0555d4a3696010850f39ab12ac723a701ae7547" +manifest-hash = "sha256:cea42a1c44c8c70e4d36517b7a56f395157dd73fcab0f7090ef4eb274852edda" nanvix-zutil-version = "0.17.0" [metadata.sdk] @@ -79,9 +79,6 @@ release-id = 354630018 dependencies = [] required-by = [ "sqlite", - "libxml2", - "libxslt", - "lxml", ] [[package.assets]] @@ -200,95 +197,6 @@ name = "libffi-windows-x86-microvm-standalone-256mb-dev.zip" url = "https://github.com/nanvix/libffi/releases/download/3.4.6-nanvix-0.20.9-sdk.1/libffi-windows-x86-microvm-standalone-256mb-dev.zip" id = 478241301 -[[package]] -name = "libxml2" -repo = "nanvix/libxml2" -kind = "dependency" -ref-kind = "version" -ref-value = "2.12.9-nanvix-0.20.9-sdk.1" -resolved-tag = "2.12.9-nanvix-0.20.9-sdk.1" -resolved-commitish = "nanvix/v2.12.9" -release-id = 354639992 -dependencies = [ - "zlib", -] -required-by = [ - "libxslt", - "lxml", -] - -[[package.assets]] -name = "libxml2-linux-x86-microvm-standalone-256mb-dev.tar.gz" -url = "https://github.com/nanvix/libxml2/releases/download/2.12.9-nanvix-0.20.9-sdk.1/libxml2-linux-x86-microvm-standalone-256mb-dev.tar.gz" -id = 478260873 - -[[package.assets]] -name = "libxml2-linux-x86-microvm-standalone-256mb.tar.gz" -url = "https://github.com/nanvix/libxml2/releases/download/2.12.9-nanvix-0.20.9-sdk.1/libxml2-linux-x86-microvm-standalone-256mb.tar.gz" -id = 478260875 - -[[package.assets]] -name = "libxml2-windows-x86-microvm-standalone-256mb-dev.zip" -url = "https://github.com/nanvix/libxml2/releases/download/2.12.9-nanvix-0.20.9-sdk.1/libxml2-windows-x86-microvm-standalone-256mb-dev.zip" -id = 478260871 - -[[package.assets]] -name = "libxml2-windows-x86-microvm-standalone-256mb.zip" -url = "https://github.com/nanvix/libxml2/releases/download/2.12.9-nanvix-0.20.9-sdk.1/libxml2-windows-x86-microvm-standalone-256mb.zip" -id = 478260872 - -[[package]] -name = "libxslt" -repo = "nanvix/libxslt" -kind = "dependency" -ref-kind = "version" -ref-value = "1.1.42-nanvix-0.20.9-sdk.1" -resolved-tag = "1.1.42-nanvix-0.20.9-sdk.1" -resolved-commitish = "nanvix/v1.1.42" -release-id = 354653236 -dependencies = [ - "libxml2", - "zlib", -] -required-by = [ - "lxml", -] - -[[package.assets]] -name = "libxslt-linux-x86-microvm-standalone-256mb-dev.tar.gz" -url = "https://github.com/nanvix/libxslt/releases/download/1.1.42-nanvix-0.20.9-sdk.1/libxslt-linux-x86-microvm-standalone-256mb-dev.tar.gz" -id = 478283663 - -[[package.assets]] -name = "libxslt-windows-x86-microvm-standalone-256mb-dev.zip" -url = "https://github.com/nanvix/libxslt/releases/download/1.1.42-nanvix-0.20.9-sdk.1/libxslt-windows-x86-microvm-standalone-256mb-dev.zip" -id = 478283666 - -[[package]] -name = "lxml" -repo = "nanvix/lxml" -kind = "dependency" -ref-kind = "version" -ref-value = "5.3.0-nanvix-0.20.9-sdk.1" -resolved-tag = "5.3.0-nanvix-0.20.9-sdk.1" -resolved-commitish = "nanvix/v5.3.0" -release-id = 354658318 -dependencies = [ - "zlib", - "libxml2", - "libxslt", -] - -[[package.assets]] -name = "lxml-linux-x86-microvm-standalone-256mb-dev.tar.gz" -url = "https://github.com/nanvix/lxml/releases/download/5.3.0-nanvix-0.20.9-sdk.1/lxml-linux-x86-microvm-standalone-256mb-dev.tar.gz" -id = 478290062 - -[[package.assets]] -name = "lxml-windows-x86-microvm-standalone-256mb-dev.zip" -url = "https://github.com/nanvix/lxml/releases/download/5.3.0-nanvix-0.20.9-sdk.1/lxml-windows-x86-microvm-standalone-256mb-dev.zip" -id = 478290065 - [[package]] name = "xz" repo = "nanvix/xz" diff --git a/.nanvix/nanvix.toml b/.nanvix/nanvix.toml index b538e6cc3bedd1..deace6fca73a9b 100644 --- a/.nanvix/nanvix.toml +++ b/.nanvix/nanvix.toml @@ -15,9 +15,6 @@ sqlite = "3.49.0" openssl = "3.5.0" bzip2 = "1.0.8" libffi = "3.4.6" -libxml2 = "2.12.9" -libxslt = "1.1.42" -lxml = "5.3.0" xz = "5.2.5" [toolchain] diff --git a/.nanvix/setup_local.py b/.nanvix/setup_local.py new file mode 100644 index 00000000000000..00da0a5aefdca3 --- /dev/null +++ b/.nanvix/setup_local.py @@ -0,0 +1,244 @@ +# Copyright(c) The Maintainers of Nanvix. +# Licensed under the MIT License. + +"""Single source of truth for Modules/Setup.local on Nanvix builds. + +The host build path in ``generate_setup_local()`` and the Docker build path in +``_docker._generate_setup_local_cmd()`` both render this table. + +Module ordering matters: makesetup applies "first rule wins" semantics when +the same module is declared both here and in ``Modules/Setup.stdlib``. +Declaring a shared duplicate before the upstream static default switches that +module to a runtime-loaded extension. +""" + +from __future__ import annotations + +from enum import Enum +from pathlib import Path +from typing import Iterable, NamedTuple, Sequence + + +class Linkage(Enum): + STATIC = "*static*" + SHARED = "*shared*" + + +class SetupEntry(NamedTuple): + """One Modules/Setup.local entry with optional surrounding comments.""" + + name: str + linkage: Linkage + tokens: Sequence[str] = () + comment: str = "" + section_header: str = "" + + +SETUP_LOCAL_ENTRIES: tuple[SetupEntry, ...] = ( + SetupEntry( + name="_nanvix", + linkage=Linkage.STATIC, + tokens=("_nanvixmodule.c",), + comment="Nanvix OS interface module (snapshot, host-mount).", + ), + SetupEntry( + name="array", + linkage=Linkage.SHARED, + tokens=("arraymodule.c",), + section_header=( + "`array` is the proof-of-concept shared module. It appears before " + "Setup.stdlib so makesetup selects this shared definition." + ), + ), + *( + SetupEntry( + name=name, + linkage=Linkage.SHARED, + tokens=(source,), + section_header=header if index == 0 else "", + ) + for index, (name, source) in enumerate( + ( + ("_bisect", "_bisectmodule.c"), + ("_heapq", "_heapqmodule.c"), + ("_struct", "_struct.c"), + ("_random", "_randommodule.c"), + ("_opcode", "_opcode.c"), + ("_queue", "_queuemodule.c"), + ("_csv", "_csv.c"), + ("binascii", "binascii.c"), + ("_json", "_json.c"), + ("_pickle", "_pickle.c"), + ("_zoneinfo", "_zoneinfo.c"), + ) + ) + for header in ("Data primitives without external dependencies.",) + ), + *( + SetupEntry( + name=name, + linkage=Linkage.SHARED, + tokens=(source,), + section_header=header if index == 0 else "", + ) + for index, (name, source) in enumerate( + ( + ("math", "mathmodule.c"), + ("cmath", "cmathmodule.c"), + ("_statistics", "_statisticsmodule.c"), + ("mmap", "mmapmodule.c"), + ("_contextvars", "_contextvarsmodule.c"), + ) + ) + for header in ("Math and memory modules using the SDK shared runtime.",) + ), + *( + SetupEntry( + name=name, + linkage=Linkage.SHARED, + tokens=(source,), + section_header=header if index == 0 else "", + ) + for index, (name, source) in enumerate( + ( + ("unicodedata", "unicodedata.c"), + ("_codecs_cn", "cjkcodecs/_codecs_cn.c"), + ("_codecs_hk", "cjkcodecs/_codecs_hk.c"), + ("_codecs_iso2022", "cjkcodecs/_codecs_iso2022.c"), + ("_codecs_jp", "cjkcodecs/_codecs_jp.c"), + ("_codecs_kr", "cjkcodecs/_codecs_kr.c"), + ("_codecs_tw", "cjkcodecs/_codecs_tw.c"), + ) + ) + for header in ("Text codecs without external dependencies.",) + ), + # Keep _datetime, pyexpat, and _multibytecodec static: migrated consumers + # import their C APIs while a shared object initializes, which Nanvix cannot + # nest safely yet. + SetupEntry( + name="_asyncio", + linkage=Linkage.SHARED, + tokens=("_asynciomodule.c",), + section_header=( + "Modules with bundled CPython dependencies. Each extension keeps " + "its vendored archive, matching the upstream build." + ), + ), + SetupEntry( + name="_decimal", linkage=Linkage.SHARED, tokens=("_decimal/_decimal.c",) + ), + SetupEntry(name="_elementtree", linkage=Linkage.SHARED, tokens=("_elementtree.c",)), + SetupEntry( + name="_md5", + linkage=Linkage.SHARED, + tokens=( + "md5module.c", + "-I$(srcdir)/Modules/_hacl/include", + "_hacl/Hacl_Hash_MD5.c", + "-D_BSD_SOURCE", + "-D_DEFAULT_SOURCE", + ), + ), + SetupEntry( + name="_sha1", + linkage=Linkage.SHARED, + tokens=( + "sha1module.c", + "-I$(srcdir)/Modules/_hacl/include", + "_hacl/Hacl_Hash_SHA1.c", + "-D_BSD_SOURCE", + "-D_DEFAULT_SOURCE", + ), + ), + SetupEntry( + name="_sha2", + linkage=Linkage.SHARED, + tokens=( + "sha2module.c", + "-I$(srcdir)/Modules/_hacl/include", + "Modules/_hacl/libHacl_Hash_SHA2.a", + ), + ), + SetupEntry( + name="_sha3", + linkage=Linkage.SHARED, + tokens=( + "sha3module.c", + "-I$(srcdir)/Modules/_hacl/include", + "_hacl/Hacl_Hash_SHA3.c", + "-D_BSD_SOURCE", + "-D_DEFAULT_SOURCE", + "$(COMPILER_RT_BUILTINS)", + ), + ), + SetupEntry( + name="_blake2", + linkage=Linkage.SHARED, + tokens=( + "_blake2/blake2module.c", + "_blake2/blake2b_impl.c", + "_blake2/blake2s_impl.c", + ), + ), + SetupEntry(name="select", linkage=Linkage.SHARED, tokens=("selectmodule.c",)), + SetupEntry(name="_socket", linkage=Linkage.SHARED, tokens=("socketmodule.c",)), + SetupEntry( + name="_posixsubprocess", linkage=Linkage.SHARED, tokens=("_posixsubprocess.c",) + ), + SetupEntry(name="fcntl", linkage=Linkage.SHARED, tokens=("fcntlmodule.c",)), + SetupEntry(name="termios", linkage=Linkage.SHARED, tokens=("termios.c", "-lc")), +) + + +def _wrap_comment(text: str, *, width: int = 78) -> list[str]: + """Return wrapped Setup.local comment lines.""" + if not text: + return [] + + import textwrap + + return [f"# {line}" for line in textwrap.wrap(text, width=width - 2)] + + +def render_setup_local( + entries: Iterable[SetupEntry] = SETUP_LOCAL_ENTRIES, + *, + buildroot: str, + header_comment: str, +) -> str: + """Render entries to a complete Modules/Setup.local file.""" + lines = [f"# {header_comment}", ""] + current_linkage: Linkage | None = None + + for entry in entries: + if entry.linkage is not current_linkage: + if current_linkage is not None: + lines.append("") + if entry.section_header: + lines.extend(_wrap_comment(entry.section_header)) + elif current_linkage is None and entry.linkage is Linkage.STATIC: + lines.append("# Statically-linked extension modules for Nanvix builds.") + lines.append(entry.linkage.value) + current_linkage = entry.linkage + elif entry.section_header: + lines.append("") + lines.extend(_wrap_comment(entry.section_header)) + + if entry.comment: + lines.extend(_wrap_comment(entry.comment)) + tokens = " ".join(entry.tokens).replace("{buildroot}", buildroot) + lines.append(f"{entry.name} {tokens}".rstrip()) + + lines.append("") + return "\n".join(lines) + + +def generate_setup_local(repo_root: Path, buildroot: Path) -> None: + """Generate Modules/Setup.local for the native-host build path.""" + setup_local = repo_root / "Modules" / "Setup.local" + content = render_setup_local( + buildroot=str(buildroot), + header_comment="Auto-generated by .nanvix/setup_local.py -- do not edit manually.", + ) + setup_local.write_text(content, encoding="utf-8") + print(f"[setup_local] Generated {setup_local}") diff --git a/.nanvix/z.py b/.nanvix/z.py index e2fabab47e832c..a879456d6c8039 100644 --- a/.nanvix/z.py +++ b/.nanvix/z.py @@ -22,15 +22,12 @@ import os import shutil -import tarfile -import zipfile from pathlib import Path from nanvix_zutil import paths import _test as test_mod import build as build_mod -import lxml as lxml_mod import config import package as package_mod import ramfs as ramfs_mod @@ -43,7 +40,6 @@ log, run, ) -from nanvix_zutil.paths import nanvix_root # --------------------------------------------------------------------------- # Path helpers @@ -210,7 +206,6 @@ def setup(self) -> bool: if path.is_dir(): shutil.rmtree(path) - self._install_lxml_runtime_payload() self._overlay_local_nanvix() self.config.save() return used_fallback @@ -223,7 +218,6 @@ def build(self) -> None: build_mod.clean(preserve_nanvix_root=False, preserve_cache=True) args = self._make_args(release=True, with_docker=True) build_mod.build(args) - lxml_mod.stage_lxml_runtime(package_mod.sysroot_pkg()) package_mod.stage() ramfs_mod.build_image( package_mod.sysroot_pkg(), @@ -235,7 +229,6 @@ def build(self) -> None: build_mod.clean(preserve_nanvix_root=True, preserve_cache=True) args = self._make_args(release=False, with_docker=True) build_mod.build(args) - lxml_mod.stage_lxml_runtime(paths.test_out()) test_mod.stage_ramfs(args) def test(self) -> None: @@ -262,71 +255,6 @@ def clean(self) -> None: """Remove build artifacts.""" build_mod.clean() - @staticmethod - def _python_package_path(member_name: str) -> Path | None: - """Return a safe path below an archive's ``python-packages`` directory.""" - parts = Path(member_name).parts - try: - package_index = parts.index("python-packages") - except ValueError: - return None - relative = Path(*parts[package_index + 1 :]) - if not relative.parts or relative.is_absolute() or ".." in relative.parts: - return None - return relative - - def _install_lxml_runtime_payload(self) -> None: - """Install the exact lxml release's Python payload into the buildroot.""" - cache_dir = nanvix_root() / "cache" - # Magic-path naming: lxml-{host}-{arch}-{machine}-{mode}-{mem}-dev.{ext}. - # Match any host/arch pair for the current machine + memory + mode. - pattern = ( - f"lxml-*-{self.config.machine}-" - f"{self.config.deployment_mode}-{self.config.memory_size}-dev.*" - ) - candidates = list(cache_dir.glob(pattern)) if cache_dir.is_dir() else [] - if not candidates: - raise FileNotFoundError( - "lxml release archive is missing from .nanvix/cache" - ) - - archive = max(candidates, key=lambda path: path.stat().st_mtime_ns) - destination = nanvix_root() / "buildroot" / "python-packages" - if destination.is_dir(): - shutil.rmtree(destination) - destination.mkdir(parents=True) - - installed = 0 - if zipfile.is_zipfile(archive): - with zipfile.ZipFile(archive) as source: - for member in source.infolist(): - relative = self._python_package_path(member.filename) - if relative is None or member.is_dir(): - continue - output = destination / relative - output.parent.mkdir(parents=True, exist_ok=True) - with source.open(member) as src, output.open("wb") as dst: - shutil.copyfileobj(src, dst) - installed += 1 - else: - with tarfile.open(archive, "r:*") as source: - for member in source.getmembers(): - relative = self._python_package_path(member.name) - if relative is None or not member.isfile(): - continue - extracted = source.extractfile(member) - if extracted is None: - continue - output = destination / relative - output.parent.mkdir(parents=True, exist_ok=True) - with extracted, output.open("wb") as dst: - shutil.copyfileobj(extracted, dst) - installed += 1 - - if installed == 0: - raise RuntimeError(f"{archive.name} contains no python-packages payload") - log.info(f"Installed {installed} lxml runtime file(s) from {archive.name}") - if __name__ == "__main__": CPythonBuild.main() diff --git a/Lib/test/test_nanvix_lxml.py b/Lib/test/test_nanvix_lxml.py deleted file mode 100644 index 28c62b312b953c..00000000000000 --- a/Lib/test/test_nanvix_lxml.py +++ /dev/null @@ -1,53 +0,0 @@ -"""Smoke tests for lxml built-in on NanVix.""" - -import sys -import unittest -from test.support import is_nanvix -from test.support.import_helper import import_module - -if not is_nanvix: - raise unittest.SkipTest("lxml built-in is Nanvix-specific") - -etree = import_module("lxml.etree") - - -class NanvixLxmlTests(unittest.TestCase): - - def test_import_lxml_etree(self): - self.assertTrue(hasattr(etree, "fromstring")) - self.assertTrue(hasattr(etree, "_Element")) - - def test_parse_xml(self): - root = etree.fromstring(b'text') - self.assertEqual(root.tag, "root") - child = root.find("child") - self.assertIsNotNone(child) - self.assertEqual(child.text, "text") - self.assertEqual(child.get("key"), "val") - - def test_element_creation(self): - root = etree.Element("doc") - etree.SubElement(root, "item").text = "hello" - xml = etree.tostring(root, encoding="unicode") - self.assertIn("hello", xml) - - def test_elementpath(self): - root = etree.fromstring(b"12") - results = root.findall("b") - self.assertEqual(len(results), 2) - self.assertEqual(results[0].text, "1") - - def test_xmlfile_available(self): - # xmlfile must be importable from lxml.etree for openpyxl compat. - from lxml.etree import xmlfile - - self.assertTrue(callable(xmlfile)) - - def test_htmlfile_available(self): - from lxml.etree import htmlfile - - self.assertTrue(callable(htmlfile)) - - -if __name__ == "__main__": - unittest.main() diff --git a/Makefile.nanvix b/Makefile.nanvix index 84bac275f706cf..446d0e6fe6cec4 100644 --- a/Makefile.nanvix +++ b/Makefile.nanvix @@ -41,6 +41,7 @@ TARGET_CXX := $(SDK_BIN)/clang++ --target=$(NANVIX_TARGET_TRIPLE) TARGET_AR := $(SDK_BIN)/llvm-ar TARGET_RANLIB := $(SDK_BIN)/llvm-ranlib TARGET_STRIP := $(SDK_BIN)/llvm-strip +COMPILER_RT_BUILTINS := $(shell $(TARGET_CC) --print-libgcc-file-name) DEPENDENCY_CPPFLAGS := -I$(NANVIX_BUILDROOT)/include DEPENDENCY_LDFLAGS := -L$(NANVIX_BUILDROOT)/lib @@ -57,9 +58,12 @@ CONFIGURE_ENV = \ RANLIB="$(TARGET_RANLIB)" \ STRIP="$(TARGET_STRIP)" \ CPPFLAGS="$(DEPENDENCY_CPPFLAGS)" \ - CFLAGS="-O3 -fomit-frame-pointer -fno-unwind-tables -fno-asynchronous-unwind-tables" \ + CFLAGS="-O3 -fPIE -fomit-frame-pointer -fno-unwind-tables -fno-asynchronous-unwind-tables" \ CFLAGS_NODIST="-fno-semantic-interposition" \ - LDFLAGS="$(DEPENDENCY_LDFLAGS) -static -Wl,--export-dynamic" \ + LDFLAGS="$(DEPENDENCY_LDFLAGS) -Wl,--export-dynamic" \ + LINKFORSHARED="-Wl,-pie,--export-dynamic,--no-dynamic-linker,-z,notext,-z,norelro,--hash-style=sysv" \ + LDSHARED="$(TARGET_CC) -shared -nostdlib -Wl,-z,now,--no-dynamic-linker" \ + LDCXXSHARED="$(TARGET_CXX) -shared -nostdlib -Wl,-z,now,--no-dynamic-linker" \ LIBS="$(DEPENDENCY_LIBS)" \ LIBSQLITE3_LIBS="$(DEPENDENCY_LDFLAGS) -lsqlite3" \ LIBSQLITE3_CFLAGS="$(DEPENDENCY_CPPFLAGS)" \ @@ -131,13 +135,15 @@ $(CONFIGURED_MARKER): touch $@ build: $(CONFIGURED_MARKER) - $(MAKE) -j$$(nproc) all + $(MAKE) -j$$(nproc) COMPILER_RT_BUILTINS="$(COMPILER_RT_BUILTINS)" all @if [ -f python ]; then cp -f python python$(EXE); fi @test -f python$(EXE) $(TARGET_STRIP) --strip-all python$(EXE) install: build - $(MAKE) install DESTDIR="$(DESTDIR)" + $(MAKE) COMPILER_RT_BUILTINS="$(COMPILER_RT_BUILTINS)" install DESTDIR="$(DESTDIR)" + find "$(DESTDIR)$(INSTALL_PREFIX)/lib/python3.12/lib-dynload" \ + -type f -name '*.so' -exec $(TARGET_STRIP) --strip-all {} + clean: find . -path './.nanvix' -prune -o -type d -name '__pycache__' \ diff --git a/Modules/lxml_elementpath_builtin.c b/Modules/lxml_elementpath_builtin.c deleted file mode 100644 index 6a1758e1224659..00000000000000 --- a/Modules/lxml_elementpath_builtin.c +++ /dev/null @@ -1,16 +0,0 @@ -/* - * lxml_elementpath_builtin.c - Shim for lxml._elementpath built-in module. - * - * Registers the Cython _elementpath extension under the flat name - * "_lxml_elementpath" for Modules/Setup.local. - */ - -#include "Python.h" - -extern PyObject* PyInit__elementpath(void); - -PyMODINIT_FUNC -PyInit__lxml_elementpath(void) -{ - return PyInit__elementpath(); -} diff --git a/Modules/lxml_etree_builtin.c b/Modules/lxml_etree_builtin.c deleted file mode 100644 index d028054d5b08f2..00000000000000 --- a/Modules/lxml_etree_builtin.c +++ /dev/null @@ -1,21 +0,0 @@ -/* - * lxml_etree_builtin.c - Shim to register lxml.etree as a CPython built-in. - * - * makesetup does not support dotted module names, so the Cython extension - * is registered under the flat name "_lxml_etree". A pure-Python shim at - * lxml/etree.py re-exports everything via `from _lxml_etree import *`. - * - * The Cython-generated code in liblxml_etree.a exports PyInit_etree. - * This wrapper provides PyInit__lxml_etree so the name matches the - * entry in Modules/Setup.local. - */ - -#include "Python.h" - -extern PyObject* PyInit_etree(void); - -PyMODINIT_FUNC -PyInit__lxml_etree(void) -{ - return PyInit_etree(); -}