From 125d9e4003363677b720d809bcbdd62d757ee54a Mon Sep 17 00:00:00 2001 From: Enrique Saurez Date: Wed, 15 Jul 2026 16:28:41 -0700 Subject: [PATCH 1/4] [nanvix] E: Build array as .so with the Nanvix SDK Use the SDK-supported static-PIE and dynamic loader contract instead of the retired GCC wrapper and manual Newlib runtime composition. Shared extensions bind to the SDK libc.so/libm.so through /lib and the install flow stages those runtime libraries. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 23312a7e-62f5-477a-a83f-65fd99522fe3 --- .nanvix/_docker.py | 8 +++++++- .nanvix/_test.py | 16 +++++++++++++--- .nanvix/lxml.py | 8 +++++++- Makefile.nanvix | 10 ++++++++-- 4 files changed, 35 insertions(+), 7 deletions(-) diff --git a/.nanvix/_docker.py b/.nanvix/_docker.py index 29f577a5c8a560..918a3e2bec6eda 100644 --- a/.nanvix/_docker.py +++ b/.nanvix/_docker.py @@ -234,13 +234,19 @@ def _generate_setup_local_cmd() -> str: return ( f"printf '%s\\n' " f"'# Auto-generated by .nanvix/docker.py -- do not edit manually.' " + f"'' " f"'# Statically-linked extension modules for Nanvix builds.' " - f"'#' " + f"'*static*' " f"'# Nanvix OS interface module (snapshot, host-mount).' " f"'_nanvix _nanvixmodule.c' " f"'# lxml C extension modules (statically linked via pre-built archives).' " f"'_lxml_etree lxml_etree_builtin.c -L{buildroot}/lib -llxml_etree -lxslt -lexslt -lxml2 -lz' " f"'_lxml_elementpath lxml_elementpath_builtin.c -L{buildroot}/lib -llxml_elementpath -lxml2 -lz' " + f"'' " + f"'# Phase 0 of the .a -> .so migration: array as a proof-of-concept shared module.' " + f"'# Listed before Setup.stdlib static declaration so makesetup first rule wins.' " + f"'*shared*' " + f"'array arraymodule.c' " f"> {ws}/Modules/Setup.local" ) diff --git a/.nanvix/_test.py b/.nanvix/_test.py index 2830baee5dad81..c70f8de7b286f1 100644 --- a/.nanvix/_test.py +++ b/.nanvix/_test.py @@ -237,8 +237,18 @@ def stage(args: build_mod.MakeArgs) -> None: shutil.copy2(scdata_src, scdata_dst) print(f" Copied {scdata_name} from build dir (make install missed it)") - # Hello-world test script. The lxml import is exercised against the - # in-memory FAT ramfs VFS via xmlInitParser(). + # Hello-world test script. The array check proves that the first + # stdlib module migrated to a shared extension is loaded through dlopen. + array_snippet = ( + "import array\n" + "assert 'array' not in sys.builtin_module_names, 'array still built-in!'\n" + "_array = array.array('i', [1, 2, 3])\n" + "assert _array.tolist() == [1, 2, 3]\n" + "print(f'CPYTHON_TEST_ARRAY_SO: array loaded via dlopen from {array.__file__}')\n" + ) + + # The lxml import is exercised against the in-memory FAT ramfs VFS via + # xmlInitParser(). lxml_snippet = ( "try:\n" " import lxml.etree\n" @@ -255,7 +265,7 @@ def stage(args: build_mod.MakeArgs) -> None: (staging / "test_hello.py").write_text( "import sys\n" "print('CPYTHON_TEST_HELLO: Hello from Python', sys.version_info[:2])\n" - "print('CPYTHON_TEST_PLATFORM:', sys.platform)\n" + lxml_snippet + "print('CPYTHON_TEST_PLATFORM:', sys.platform)\n" + array_snippet + lxml_snippet ) # HTTP server smoke-test script must be present in the sysroot before diff --git a/.nanvix/lxml.py b/.nanvix/lxml.py index e7656679480e11..9231d408af40f2 100644 --- a/.nanvix/lxml.py +++ b/.nanvix/lxml.py @@ -11,13 +11,19 @@ _SETUP_LOCAL_TEMPLATE = """\ # Auto-generated by .nanvix/lxml.py -- do not edit manually. + # Statically-linked extension modules for Nanvix builds. -# +*static* # Nanvix OS interface module (snapshot, host-mount). _nanvix _nanvixmodule.c # lxml C extension modules (statically linked via pre-built archives). _lxml_etree lxml_etree_builtin.c -L{sysroot}/lib -llxml_etree -lxslt -lexslt -lxml2 -lz _lxml_elementpath lxml_elementpath_builtin.c -L{sysroot}/lib -llxml_elementpath -lxml2 -lz + +# Phase 0 of the .a -> .so migration: array as a proof-of-concept shared module. +# Listed before Setup.stdlib's static declaration so makesetup's first rule wins. +*shared* +array arraymodule.c """ diff --git a/Makefile.nanvix b/Makefile.nanvix index 84bac275f706cf..735501d18c9d6b 100644 --- a/Makefile.nanvix +++ b/Makefile.nanvix @@ -57,9 +57,12 @@ CONFIGURE_ENV = \ RANLIB="$(TARGET_RANLIB)" \ STRIP="$(TARGET_STRIP)" \ CPPFLAGS="$(DEPENDENCY_CPPFLAGS)" \ - CFLAGS="-O3 -fomit-frame-pointer -fno-unwind-tables -fno-asynchronous-unwind-tables" \ + CFLAGS="-O3 -fPIE -fomit-frame-pointer -fno-unwind-tables -fno-asynchronous-unwind-tables" \ CFLAGS_NODIST="-fno-semantic-interposition" \ - LDFLAGS="$(DEPENDENCY_LDFLAGS) -static -Wl,--export-dynamic" \ + LDFLAGS="$(DEPENDENCY_LDFLAGS) -Wl,--export-dynamic" \ + LINKFORSHARED="-Wl,-pie,--export-dynamic,--no-dynamic-linker,-z,notext,-z,norelro,--hash-style=sysv" \ + LDSHARED="$(TARGET_CC) -shared -Wl,--no-as-needed,-z,now,-rpath,/lib,-Bdynamic -L$(NANVIX_SDK_ROOT)/lib -l:libc.so -l:libm.so -Wl,-Bstatic" \ + LDCXXSHARED="$(TARGET_CXX) -shared -Wl,--no-as-needed,-z,now,-rpath,/lib,-Bdynamic -L$(NANVIX_SDK_ROOT)/lib -l:libc.so -l:libm.so -Wl,-Bstatic" \ LIBS="$(DEPENDENCY_LIBS)" \ LIBSQLITE3_LIBS="$(DEPENDENCY_LDFLAGS) -lsqlite3" \ LIBSQLITE3_CFLAGS="$(DEPENDENCY_CPPFLAGS)" \ @@ -138,6 +141,9 @@ build: $(CONFIGURED_MARKER) install: build $(MAKE) install DESTDIR="$(DESTDIR)" + mkdir -p "$(DESTDIR)$(INSTALL_PREFIX)/lib" + cp -f "$(NANVIX_SDK_ROOT)/lib/libc.so" "$(DESTDIR)$(INSTALL_PREFIX)/lib/libc.so" + cp -f "$(NANVIX_SDK_ROOT)/lib/libm.so" "$(DESTDIR)$(INSTALL_PREFIX)/lib/libm.so" clean: find . -path './.nanvix' -prune -o -type d -name '__pycache__' \ From 44512fcaebbd249df8ce1ac80037f18452d44c22 Mon Sep 17 00:00:00 2001 From: Enrique Saurez Date: Wed, 15 Jul 2026 17:36:20 -0700 Subject: [PATCH 2/4] [nanvix] E: Build stdlib extensions as .so with the SDK Generate one Setup.local for native and Docker builds, link migrated extensions against the exported Nanvix SDK PIE without duplicating libc state, and retain static C-API anchors required by the current loader. Strip installed modules and exercise real nested-import and dlopen paths. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 23312a7e-62f5-477a-a83f-65fd99522fe3 --- .nanvix/_docker.py | 27 ++--- .nanvix/_test.py | 105 ++++++++++++++++- .nanvix/lxml.py | 28 +---- .nanvix/setup_local.py | 257 +++++++++++++++++++++++++++++++++++++++++ Makefile.nanvix | 14 +-- 5 files changed, 383 insertions(+), 48 deletions(-) create mode 100644 .nanvix/setup_local.py diff --git a/.nanvix/_docker.py b/.nanvix/_docker.py index 918a3e2bec6eda..24deb16d272230 100644 --- a/.nanvix/_docker.py +++ b/.nanvix/_docker.py @@ -21,6 +21,7 @@ import build as build_mod import config +import setup_local as setup_local_mod def _workspace_id(workspace: Path) -> str: @@ -228,27 +229,17 @@ def docker_build( def _generate_setup_local_cmd() -> str: - """Shell command to generate Modules/Setup.local inside the container.""" + """Generate Modules/Setup.local inside the SDK container.""" buildroot = config.DOCKER_BUILDROOT_PATH ws = config.DOCKER_WORKSPACE_PATH - return ( - f"printf '%s\\n' " - f"'# Auto-generated by .nanvix/docker.py -- do not edit manually.' " - f"'' " - f"'# Statically-linked extension modules for Nanvix builds.' " - f"'*static*' " - f"'# Nanvix OS interface module (snapshot, host-mount).' " - f"'_nanvix _nanvixmodule.c' " - f"'# lxml C extension modules (statically linked via pre-built archives).' " - f"'_lxml_etree lxml_etree_builtin.c -L{buildroot}/lib -llxml_etree -lxslt -lexslt -lxml2 -lz' " - f"'_lxml_elementpath lxml_elementpath_builtin.c -L{buildroot}/lib -llxml_elementpath -lxml2 -lz' " - f"'' " - f"'# Phase 0 of the .a -> .so migration: array as a proof-of-concept shared module.' " - f"'# Listed before Setup.stdlib static declaration so makesetup first rule wins.' " - f"'*shared*' " - f"'array arraymodule.c' " - f"> {ws}/Modules/Setup.local" + rendered = setup_local_mod.render_setup_local( + buildroot=buildroot, + header_comment="Auto-generated by .nanvix/_docker.py -- do not edit manually.", + ) + quoted_lines = " ".join( + "'" + line.replace("'", "'\"'\"'") + "'" for line in rendered.splitlines() ) + return f"printf '%s\\n' {quoted_lines} > {ws}/Modules/Setup.local" def _copy_outputs_cmd() -> str: diff --git a/.nanvix/_test.py b/.nanvix/_test.py index c70f8de7b286f1..714f057c5e6dc5 100644 --- a/.nanvix/_test.py +++ b/.nanvix/_test.py @@ -29,6 +29,94 @@ import lxml as lxml_mod import ramfs as ramfs_mod +# --------------------------------------------------------------------------- +# Shared-extension smoke checks +# --------------------------------------------------------------------------- + +_SO_MODULE_SANITY_CHECKS: tuple[tuple[str, tuple[tuple[str, str], ...]], ...] = ( + ( + "CPYTHON_TEST_DATA_PRIMITIVES", + ( + ("_bisect", "m.bisect_left([1, 3, 5], 4) == 2"), + ("_heapq", "m.heappush([], 1) is None"), + ("_struct", "m.pack('i', 42) == b'\\x2a\\x00\\x00\\x00'"), + ("_random", "hasattr(m, 'Random')"), + ("_opcode", "hasattr(m, 'stack_effect')"), + ("_queue", "hasattr(m, 'SimpleQueue')"), + ("_csv", "hasattr(m, 'reader')"), + ("binascii", "m.hexlify(b'\\xab') == b'ab'"), + ("_json", "hasattr(m, 'encode_basestring_ascii')"), + ("_pickle", "hasattr(m, 'Pickler')"), + ("_zoneinfo", "hasattr(m, 'ZoneInfo')"), + ), + ), + ( + "CPYTHON_TEST_MATH", + ( + ("math", "abs(m.sqrt(4.0) - 2.0) < 1e-9"), + ("cmath", "abs(m.sqrt(complex(-1)) - complex(0, 1)) < 1e-9"), + ("_statistics", "hasattr(m, '_normal_dist_inv_cdf')"), + ("mmap", "hasattr(m, 'mmap')"), + ("_contextvars", "hasattr(m, 'ContextVar')"), + ), + ), + ( + "CPYTHON_TEST_CODECS", + ( + ("unicodedata", "m.lookup('LATIN SMALL LETTER A') == 'a'"), + ("_codecs_cn", "hasattr(m, 'getcodec')"), + ("_codecs_hk", "hasattr(m, 'getcodec')"), + ("_codecs_iso2022", "hasattr(m, 'getcodec')"), + ("_codecs_jp", "hasattr(m, 'getcodec')"), + ("_codecs_kr", "hasattr(m, 'getcodec')"), + ("_codecs_tw", "hasattr(m, 'getcodec')"), + ), + ), + ( + "CPYTHON_TEST_BUNDLED_DEPS", + ( + ("_asyncio", "hasattr(m, 'Future')"), + ("_decimal", "m.Decimal('1.1') + m.Decimal('2.2') == m.Decimal('3.3')"), + ("_elementtree", "hasattr(m, 'XMLParser')"), + ("_md5", "hasattr(m, 'md5')"), + ("_sha1", "hasattr(m, 'sha1')"), + ("_sha2", "hasattr(m, 'sha256')"), + ("_sha3", "hasattr(m, 'sha3_256')"), + ("_blake2", "hasattr(m, 'blake2b')"), + ("select", "hasattr(m, 'select')"), + ("_socket", "hasattr(m, 'socket')"), + ("_posixsubprocess", "hasattr(m, 'fork_exec')"), + ("fcntl", "hasattr(m, 'fcntl')"), + ("termios", "hasattr(m, 'tcgetattr')"), + ), + ), +) + + +def _render_so_sanity_snippets( + checks: tuple[ + tuple[str, tuple[tuple[str, str], ...]], ... + ] = _SO_MODULE_SANITY_CHECKS, +) -> str: + """Render imports that prove each migrated module loads through dlopen.""" + snippets: list[str] = [] + for log_tag, modules in checks: + items = ",\n".join( + f" ({name!r}, lambda m: {check})" for name, check in modules + ) + snippets.append( + f"_so_checks = [\n{items},\n]\n" + "for _name, _check in _so_checks:\n" + " _mod = __import__(_name)\n" + " assert _name not in sys.builtin_module_names, " + "f'{_name} still built-in!'\n" + " assert _check(_mod), f'{_name} sanity check failed'\n" + f" print(f'{log_tag}: " + "{_name} loaded via dlopen from {_mod.__file__}')\n" + ) + return "".join(snippets) + + # --------------------------------------------------------------------------- # Initrd creation helper (standalone mode) # --------------------------------------------------------------------------- @@ -246,6 +334,17 @@ def stage(args: build_mod.MakeArgs) -> None: "assert _array.tolist() == [1, 2, 3]\n" "print(f'CPYTHON_TEST_ARRAY_SO: array loaded via dlopen from {array.__file__}')\n" ) + nested_import_snippet = ( + "import xml.etree.ElementTree as _elementtree_api\n" + "assert _elementtree_api.fromstring('').tag == 'root'\n" + "assert '_elementtree' not in sys.builtin_module_names\n" + "assert 'pyexpat' in sys.builtin_module_names\n" + "import encodings.gb2312\n" + "assert '中文'.encode('gb2312') == b'\\xd6\\xd0\\xce\\xc4'\n" + "assert '_codecs_cn' not in sys.builtin_module_names\n" + "assert '_multibytecodec' in sys.builtin_module_names\n" + "print('CPYTHON_TEST_NESTED_IMPORTS: static C API anchors OK')\n" + ) # The lxml import is exercised against the in-memory FAT ramfs VFS via # xmlInitParser(). @@ -265,7 +364,11 @@ def stage(args: build_mod.MakeArgs) -> None: (staging / "test_hello.py").write_text( "import sys\n" "print('CPYTHON_TEST_HELLO: Hello from Python', sys.version_info[:2])\n" - "print('CPYTHON_TEST_PLATFORM:', sys.platform)\n" + array_snippet + lxml_snippet + "print('CPYTHON_TEST_PLATFORM:', sys.platform)\n" + + array_snippet + + nested_import_snippet + + _render_so_sanity_snippets() + + lxml_snippet ) # HTTP server smoke-test script must be present in the sysroot before diff --git a/.nanvix/lxml.py b/.nanvix/lxml.py index 9231d408af40f2..cb4180178eb384 100644 --- a/.nanvix/lxml.py +++ b/.nanvix/lxml.py @@ -8,32 +8,16 @@ from nanvix_zutil import paths import config - -_SETUP_LOCAL_TEMPLATE = """\ -# Auto-generated by .nanvix/lxml.py -- do not edit manually. - -# Statically-linked extension modules for Nanvix builds. -*static* -# Nanvix OS interface module (snapshot, host-mount). -_nanvix _nanvixmodule.c -# lxml C extension modules (statically linked via pre-built archives). -_lxml_etree lxml_etree_builtin.c -L{sysroot}/lib -llxml_etree -lxslt -lexslt -lxml2 -lz -_lxml_elementpath lxml_elementpath_builtin.c -L{sysroot}/lib -llxml_elementpath -lxml2 -lz - -# Phase 0 of the .a -> .so migration: array as a proof-of-concept shared module. -# Listed before Setup.stdlib's static declaration so makesetup's first rule wins. -*shared* -array arraymodule.c -""" +import setup_local as setup_local_mod def generate_setup_local(repo_root: Path, buildroot: Path) -> None: - """Generate Modules/Setup.local with statically-linked module definitions. - - Includes both the _nanvix OS interface module and lxml C extensions. - """ + """Generate Modules/Setup.local for the native-host build path.""" setup_local = repo_root / "Modules" / "Setup.local" - content = _SETUP_LOCAL_TEMPLATE.format(sysroot=buildroot) + content = setup_local_mod.render_setup_local( + buildroot=str(buildroot), + header_comment="Auto-generated by .nanvix/lxml.py -- do not edit manually.", + ) setup_local.write_text(content, encoding="utf-8") print(f"[lxml] Generated {setup_local}") diff --git a/.nanvix/setup_local.py b/.nanvix/setup_local.py new file mode 100644 index 00000000000000..96b7a89c4d70cc --- /dev/null +++ b/.nanvix/setup_local.py @@ -0,0 +1,257 @@ +# Copyright(c) The Maintainers of Nanvix. +# Licensed under the MIT License. + +"""Single source of truth for Modules/Setup.local on Nanvix builds. + +The host build path in ``lxml.generate_setup_local()`` and the Docker build +path in ``_docker._generate_setup_local_cmd()`` both render this table. + +Module ordering matters: makesetup applies "first rule wins" semantics when +the same module is declared both here and in ``Modules/Setup.stdlib``. +Declaring a shared duplicate before the upstream static default switches that +module to a runtime-loaded extension. +""" + +from __future__ import annotations + +from enum import Enum +from typing import Iterable, NamedTuple, Sequence + + +class Linkage(Enum): + STATIC = "*static*" + SHARED = "*shared*" + + +class SetupEntry(NamedTuple): + """One Modules/Setup.local entry with optional surrounding comments.""" + + name: str + linkage: Linkage + tokens: Sequence[str] = () + comment: str = "" + section_header: str = "" + + +SETUP_LOCAL_ENTRIES: tuple[SetupEntry, ...] = ( + SetupEntry( + name="_nanvix", + linkage=Linkage.STATIC, + tokens=("_nanvixmodule.c",), + comment="Nanvix OS interface module (snapshot, host-mount).", + ), + SetupEntry( + name="_lxml_etree", + linkage=Linkage.STATIC, + tokens=( + "lxml_etree_builtin.c", + "-L{buildroot}/lib", + "-llxml_etree", + "-lxslt", + "-lexslt", + "-lxml2", + "-lz", + ), + comment="lxml C extensions linked with their pre-built archives.", + ), + SetupEntry( + name="_lxml_elementpath", + linkage=Linkage.STATIC, + tokens=( + "lxml_elementpath_builtin.c", + "-L{buildroot}/lib", + "-llxml_elementpath", + "-lxml2", + "-lz", + ), + ), + SetupEntry( + name="array", + linkage=Linkage.SHARED, + tokens=("arraymodule.c",), + section_header=( + "`array` is the proof-of-concept shared module. It appears before " + "Setup.stdlib so makesetup selects this shared definition." + ), + ), + *( + SetupEntry( + name=name, + linkage=Linkage.SHARED, + tokens=(source,), + section_header=header if index == 0 else "", + ) + for index, (name, source) in enumerate( + ( + ("_bisect", "_bisectmodule.c"), + ("_heapq", "_heapqmodule.c"), + ("_struct", "_struct.c"), + ("_random", "_randommodule.c"), + ("_opcode", "_opcode.c"), + ("_queue", "_queuemodule.c"), + ("_csv", "_csv.c"), + ("binascii", "binascii.c"), + ("_json", "_json.c"), + ("_pickle", "_pickle.c"), + ("_zoneinfo", "_zoneinfo.c"), + ) + ) + for header in ("Data primitives without external dependencies.",) + ), + *( + SetupEntry( + name=name, + linkage=Linkage.SHARED, + tokens=(source,), + section_header=header if index == 0 else "", + ) + for index, (name, source) in enumerate( + ( + ("math", "mathmodule.c"), + ("cmath", "cmathmodule.c"), + ("_statistics", "_statisticsmodule.c"), + ("mmap", "mmapmodule.c"), + ("_contextvars", "_contextvarsmodule.c"), + ) + ) + for header in ("Math and memory modules using the SDK shared runtime.",) + ), + *( + SetupEntry( + name=name, + linkage=Linkage.SHARED, + tokens=(source,), + section_header=header if index == 0 else "", + ) + for index, (name, source) in enumerate( + ( + ("unicodedata", "unicodedata.c"), + ("_codecs_cn", "cjkcodecs/_codecs_cn.c"), + ("_codecs_hk", "cjkcodecs/_codecs_hk.c"), + ("_codecs_iso2022", "cjkcodecs/_codecs_iso2022.c"), + ("_codecs_jp", "cjkcodecs/_codecs_jp.c"), + ("_codecs_kr", "cjkcodecs/_codecs_kr.c"), + ("_codecs_tw", "cjkcodecs/_codecs_tw.c"), + ) + ) + for header in ("Text codecs without external dependencies.",) + ), + # Keep _datetime, pyexpat, and _multibytecodec static: migrated consumers + # import their C APIs while a shared object initializes, which Nanvix cannot + # nest safely yet. + SetupEntry( + name="_asyncio", + linkage=Linkage.SHARED, + tokens=("_asynciomodule.c",), + section_header=( + "Modules with bundled CPython dependencies. Each extension keeps " + "its vendored archive, matching the upstream build." + ), + ), + SetupEntry( + name="_decimal", linkage=Linkage.SHARED, tokens=("_decimal/_decimal.c",) + ), + SetupEntry(name="_elementtree", linkage=Linkage.SHARED, tokens=("_elementtree.c",)), + SetupEntry( + name="_md5", + linkage=Linkage.SHARED, + tokens=( + "md5module.c", + "-I$(srcdir)/Modules/_hacl/include", + "_hacl/Hacl_Hash_MD5.c", + "-D_BSD_SOURCE", + "-D_DEFAULT_SOURCE", + ), + ), + SetupEntry( + name="_sha1", + linkage=Linkage.SHARED, + tokens=( + "sha1module.c", + "-I$(srcdir)/Modules/_hacl/include", + "_hacl/Hacl_Hash_SHA1.c", + "-D_BSD_SOURCE", + "-D_DEFAULT_SOURCE", + ), + ), + SetupEntry( + name="_sha2", + linkage=Linkage.SHARED, + tokens=( + "sha2module.c", + "-I$(srcdir)/Modules/_hacl/include", + "Modules/_hacl/libHacl_Hash_SHA2.a", + ), + ), + SetupEntry( + name="_sha3", + linkage=Linkage.SHARED, + tokens=( + "sha3module.c", + "-I$(srcdir)/Modules/_hacl/include", + "_hacl/Hacl_Hash_SHA3.c", + "-D_BSD_SOURCE", + "-D_DEFAULT_SOURCE", + "$(COMPILER_RT_BUILTINS)", + ), + ), + SetupEntry( + name="_blake2", + linkage=Linkage.SHARED, + tokens=( + "_blake2/blake2module.c", + "_blake2/blake2b_impl.c", + "_blake2/blake2s_impl.c", + ), + ), + SetupEntry(name="select", linkage=Linkage.SHARED, tokens=("selectmodule.c",)), + SetupEntry(name="_socket", linkage=Linkage.SHARED, tokens=("socketmodule.c",)), + SetupEntry( + name="_posixsubprocess", linkage=Linkage.SHARED, tokens=("_posixsubprocess.c",) + ), + SetupEntry(name="fcntl", linkage=Linkage.SHARED, tokens=("fcntlmodule.c",)), + SetupEntry(name="termios", linkage=Linkage.SHARED, tokens=("termios.c", "-lc")), +) + + +def _wrap_comment(text: str, *, width: int = 78) -> list[str]: + """Return wrapped Setup.local comment lines.""" + if not text: + return [] + + import textwrap + + return [f"# {line}" for line in textwrap.wrap(text, width=width - 2)] + + +def render_setup_local( + entries: Iterable[SetupEntry] = SETUP_LOCAL_ENTRIES, + *, + buildroot: str, + header_comment: str, +) -> str: + """Render entries to a complete Modules/Setup.local file.""" + lines = [f"# {header_comment}", ""] + current_linkage: Linkage | None = None + + for entry in entries: + if entry.linkage is not current_linkage: + if current_linkage is not None: + lines.append("") + if entry.section_header: + lines.extend(_wrap_comment(entry.section_header)) + elif current_linkage is None and entry.linkage is Linkage.STATIC: + lines.append("# Statically-linked extension modules for Nanvix builds.") + lines.append(entry.linkage.value) + current_linkage = entry.linkage + elif entry.section_header: + lines.append("") + lines.extend(_wrap_comment(entry.section_header)) + + if entry.comment: + lines.extend(_wrap_comment(entry.comment)) + tokens = " ".join(entry.tokens).replace("{buildroot}", buildroot) + lines.append(f"{entry.name} {tokens}".rstrip()) + + lines.append("") + return "\n".join(lines) diff --git a/Makefile.nanvix b/Makefile.nanvix index 735501d18c9d6b..446d0e6fe6cec4 100644 --- a/Makefile.nanvix +++ b/Makefile.nanvix @@ -41,6 +41,7 @@ TARGET_CXX := $(SDK_BIN)/clang++ --target=$(NANVIX_TARGET_TRIPLE) TARGET_AR := $(SDK_BIN)/llvm-ar TARGET_RANLIB := $(SDK_BIN)/llvm-ranlib TARGET_STRIP := $(SDK_BIN)/llvm-strip +COMPILER_RT_BUILTINS := $(shell $(TARGET_CC) --print-libgcc-file-name) DEPENDENCY_CPPFLAGS := -I$(NANVIX_BUILDROOT)/include DEPENDENCY_LDFLAGS := -L$(NANVIX_BUILDROOT)/lib @@ -61,8 +62,8 @@ CONFIGURE_ENV = \ CFLAGS_NODIST="-fno-semantic-interposition" \ LDFLAGS="$(DEPENDENCY_LDFLAGS) -Wl,--export-dynamic" \ LINKFORSHARED="-Wl,-pie,--export-dynamic,--no-dynamic-linker,-z,notext,-z,norelro,--hash-style=sysv" \ - LDSHARED="$(TARGET_CC) -shared -Wl,--no-as-needed,-z,now,-rpath,/lib,-Bdynamic -L$(NANVIX_SDK_ROOT)/lib -l:libc.so -l:libm.so -Wl,-Bstatic" \ - LDCXXSHARED="$(TARGET_CXX) -shared -Wl,--no-as-needed,-z,now,-rpath,/lib,-Bdynamic -L$(NANVIX_SDK_ROOT)/lib -l:libc.so -l:libm.so -Wl,-Bstatic" \ + LDSHARED="$(TARGET_CC) -shared -nostdlib -Wl,-z,now,--no-dynamic-linker" \ + LDCXXSHARED="$(TARGET_CXX) -shared -nostdlib -Wl,-z,now,--no-dynamic-linker" \ LIBS="$(DEPENDENCY_LIBS)" \ LIBSQLITE3_LIBS="$(DEPENDENCY_LDFLAGS) -lsqlite3" \ LIBSQLITE3_CFLAGS="$(DEPENDENCY_CPPFLAGS)" \ @@ -134,16 +135,15 @@ $(CONFIGURED_MARKER): touch $@ build: $(CONFIGURED_MARKER) - $(MAKE) -j$$(nproc) all + $(MAKE) -j$$(nproc) COMPILER_RT_BUILTINS="$(COMPILER_RT_BUILTINS)" all @if [ -f python ]; then cp -f python python$(EXE); fi @test -f python$(EXE) $(TARGET_STRIP) --strip-all python$(EXE) install: build - $(MAKE) install DESTDIR="$(DESTDIR)" - mkdir -p "$(DESTDIR)$(INSTALL_PREFIX)/lib" - cp -f "$(NANVIX_SDK_ROOT)/lib/libc.so" "$(DESTDIR)$(INSTALL_PREFIX)/lib/libc.so" - cp -f "$(NANVIX_SDK_ROOT)/lib/libm.so" "$(DESTDIR)$(INSTALL_PREFIX)/lib/libm.so" + $(MAKE) COMPILER_RT_BUILTINS="$(COMPILER_RT_BUILTINS)" install DESTDIR="$(DESTDIR)" + find "$(DESTDIR)$(INSTALL_PREFIX)/lib/python3.12/lib-dynload" \ + -type f -name '*.so' -exec $(TARGET_STRIP) --strip-all {} + clean: find . -path './.nanvix' -prune -o -type d -name '__pycache__' \ From 9def47b90b54b5b2373433497a6ed27edb490606 Mon Sep 17 00:00:00 2001 From: Pedro Henrique Penna Date: Wed, 15 Jul 2026 18:42:10 -0700 Subject: [PATCH 3/4] Make shared-module smoke source ASCII-safe Emit the gb2312 fixture through Unicode escapes so Windows runners using CP1252 can write test_hello.py without changing the guest assertion. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 23312a7e-62f5-477a-a83f-65fd99522fe3 --- .nanvix/_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.nanvix/_test.py b/.nanvix/_test.py index 714f057c5e6dc5..53c101fffb89fb 100644 --- a/.nanvix/_test.py +++ b/.nanvix/_test.py @@ -340,7 +340,7 @@ def stage(args: build_mod.MakeArgs) -> None: "assert '_elementtree' not in sys.builtin_module_names\n" "assert 'pyexpat' in sys.builtin_module_names\n" "import encodings.gb2312\n" - "assert '中文'.encode('gb2312') == b'\\xd6\\xd0\\xce\\xc4'\n" + "assert '\\u4e2d\\u6587'.encode('gb2312') == b'\\xd6\\xd0\\xce\\xc4'\n" "assert '_codecs_cn' not in sys.builtin_module_names\n" "assert '_multibytecodec' in sys.builtin_module_names\n" "print('CPYTHON_TEST_NESTED_IMPORTS: static C API anchors OK')\n" From 2807df859459356a2ab68487e473444bd94fa9b9 Mon Sep 17 00:00:00 2001 From: Enrique Saurez Date: Wed, 15 Jul 2026 17:54:20 -0700 Subject: [PATCH 4/4] [nanvix] Remove the built-in lxml shim from CPython Keep third-party lxml outside CPython, remove its SDK dependency graph and payload staging, and retain Setup.local generation for the shared stdlib modules introduced by #732. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 23312a7e-62f5-477a-a83f-65fd99522fe3 --- .nanvix/NANVIX.md | 7 +-- .nanvix/_test.py | 27 --------- .nanvix/build.py | 4 +- .nanvix/config.py | 6 +- .nanvix/lxml.py | 77 ------------------------ .nanvix/nanvix.lock | 94 +----------------------------- .nanvix/nanvix.toml | 3 - .nanvix/setup_local.py | 41 +++++-------- .nanvix/z.py | 72 ----------------------- Lib/test/test_nanvix_lxml.py | 53 ----------------- Modules/lxml_elementpath_builtin.c | 16 ----- Modules/lxml_etree_builtin.c | 21 ------- 12 files changed, 21 insertions(+), 400 deletions(-) delete mode 100644 .nanvix/lxml.py delete mode 100644 Lib/test/test_nanvix_lxml.py delete mode 100644 Modules/lxml_elementpath_builtin.c delete mode 100644 Modules/lxml_etree_builtin.c diff --git a/.nanvix/NANVIX.md b/.nanvix/NANVIX.md index 33ab81dbd0e37c..f9ce7c115da8f8 100644 --- a/.nanvix/NANVIX.md +++ b/.nanvix/NANVIX.md @@ -28,9 +28,6 @@ This document describes the port of [CPython](https://www.python.org/) interpret - OpenSSL 3.5.0 - SQLite 3.49.0 - libffi 3.4.6 -- libxml2 2.12.9 -- libxslt 1.1.42 -- lxml 5.3.0 - xz 5.2.5 --- @@ -233,7 +230,7 @@ cd .nanvix/out/test && \ ### Test Suite Status -The `./z test` target runs **160 CPython stdlib test modules** on Nanvix +The `./z test` target runs **159 CPython stdlib test modules** on Nanvix (i686, microvm, 256 MB RAM). Tests are split into batches of 4 modules per VM invocation to stay within per-process memory limits. @@ -244,7 +241,7 @@ runner (`python -m test`). A `/tmp` directory is created on the ramfs so | Metric | Value | |--------|-------| -| **Modules enabled** | 160 | +| **Modules enabled** | 159 | | **Tests passed** | All supported tests | | **Tests skipped** | Unsupported Nanvix facilities use explicit skip reasons | | **Tests failed** | 0 | diff --git a/.nanvix/_test.py b/.nanvix/_test.py index 53c101fffb89fb..acb195b0a4e56b 100644 --- a/.nanvix/_test.py +++ b/.nanvix/_test.py @@ -26,7 +26,6 @@ import build as build_mod import config -import lxml as lxml_mod import ramfs as ramfs_mod # --------------------------------------------------------------------------- @@ -346,21 +345,6 @@ def stage(args: build_mod.MakeArgs) -> None: "print('CPYTHON_TEST_NESTED_IMPORTS: static C API anchors OK')\n" ) - # The lxml import is exercised against the in-memory FAT ramfs VFS via - # xmlInitParser(). - lxml_snippet = ( - "try:\n" - " import lxml.etree\n" - " doc = lxml.etree.fromstring(b'lxml OK')\n" - " assert doc.tag == 'root'\n" - " assert doc[0].text == 'lxml OK'\n" - " print('CPYTHON_TEST_LXML: lxml.etree import and parse OK')\n" - "except ImportError as e:\n" - " print(f'CPYTHON_TEST_LXML_SKIP: {e}')\n" - "except Exception as e:\n" - " print(f'CPYTHON_TEST_LXML_FAIL: {e}')\n" - " sys.exit(1)\n" - ) (staging / "test_hello.py").write_text( "import sys\n" "print('CPYTHON_TEST_HELLO: Hello from Python', sys.version_info[:2])\n" @@ -368,7 +352,6 @@ def stage(args: build_mod.MakeArgs) -> None: + array_snippet + nested_import_snippet + _render_so_sanity_snippets() - + lxml_snippet ) # HTTP server smoke-test script must be present in the sysroot before @@ -585,26 +568,18 @@ def run_hello( # Validate output. found_hello = False - found_lxml = False for line in output.splitlines(): if line.startswith("CPYTHON_TEST_"): tag = line.split(":")[0].replace("CPYTHON_TEST_", "") print(f" {tag}: {line.strip()}") if tag == "HELLO": found_hello = True - elif tag in ("LXML", "LXML_SKIP"): - found_lxml = True if not found_hello: print(" FAIL: Hello test did not produce expected output") print(output) raise RuntimeError("Hello test did not produce expected output") - if not found_lxml: - # lxml staging is best-effort — if the runtime package was not - # available (e.g. release asset missing), the test is non-fatal. - print(" WARNING: lxml import/parse test did not produce expected output") - print(" PASS") @@ -887,8 +862,6 @@ def run_all( stage(args) stage_ramfs(args) - lxml_mod.stage_lxml_runtime(staging) - # Hello test. run_hello( staging, diff --git a/.nanvix/build.py b/.nanvix/build.py index 53456c5ba3f7f8..a4a85fefe69ec8 100644 --- a/.nanvix/build.py +++ b/.nanvix/build.py @@ -22,7 +22,7 @@ import _docker as docker_mod import _test as test_mod import config -import lxml as lxml_mod +import setup_local as setup_local_mod @dataclass @@ -100,7 +100,7 @@ def build( buildroot_for_setup = ( Path(config.DOCKER_BUILDROOT_PATH) if _args.docker else args.buildroot ) - lxml_mod.generate_setup_local(paths.repo_root(), buildroot_for_setup) + setup_local_mod.generate_setup_local(paths.repo_root(), buildroot_for_setup) _args.run(cwd=paths.repo_root()) install(dest_dir, args) diff --git a/.nanvix/config.py b/.nanvix/config.py index a60db3ad40947a..749a1174039590 100644 --- a/.nanvix/config.py +++ b/.nanvix/config.py @@ -239,8 +239,6 @@ def _manifest_sdk_image() -> str: # #323 wave 8 — regex and plistlib "test_re", "test_plistlib", - # #600 — lxml built-in smoke test - "test_nanvix_lxml", # #526 — _lzma stdlib enablement "test_lzma", # #327 — network and protocol tests (IPv4 only; IPv6 disabled) @@ -337,8 +335,8 @@ def _manifest_sdk_image() -> str: "lib/pkgconfig", ] -# site-packages is no longer trimmed because lxml runtime files may be -# installed there by downstream packaging. When the directory is empty +# site-packages is not trimmed because downstream third-party packages may +# install runtime files there. When the directory is empty # it remains harmlessly on disk (ramfs.trim_sysroot only removes empty # bin/). To force-trim site-packages for minimal images, add the path # back into SYSROOT_TRIM_DIRS above. diff --git a/.nanvix/lxml.py b/.nanvix/lxml.py deleted file mode 100644 index cb4180178eb384..00000000000000 --- a/.nanvix/lxml.py +++ /dev/null @@ -1,77 +0,0 @@ -"""lxml build helpers and runtime staging for Nanvix CPython.""" - -from __future__ import annotations - -import shutil -from pathlib import Path - -from nanvix_zutil import paths - -import config -import setup_local as setup_local_mod - - -def generate_setup_local(repo_root: Path, buildroot: Path) -> None: - """Generate Modules/Setup.local for the native-host build path.""" - setup_local = repo_root / "Modules" / "Setup.local" - content = setup_local_mod.render_setup_local( - buildroot=str(buildroot), - header_comment="Auto-generated by .nanvix/lxml.py -- do not edit manually.", - ) - setup_local.write_text(content, encoding="utf-8") - print(f"[lxml] Generated {setup_local}") - - -def clear_setup_local(repo_root: Path) -> None: - """Remove the generated Modules/Setup.local.""" - setup_local = repo_root / "Modules" / "Setup.local" - if setup_local.exists(): - setup_local.unlink() - print(f"[lxml] Removed {setup_local}") - - -def stage_lxml_runtime(pkg_root: Path) -> None: - """Copy lxml Python files from buildroot into the test/package sysroot. - - Looks for lxml in ``.nanvix/buildroot/python-packages/lxml/``. - Skips gracefully when the python-packages directory is not available. - """ - lxml_src = paths.nanvix_root() / "buildroot" / "python-packages" / "lxml" - if not lxml_src.is_dir(): - print( - f"[lxml] Staged lxml package not found at {lxml_src}; " - "skipping runtime staging." - ) - return - - py_lib = pkg_root / "lib" / config.PYTHON_LIB_DIR - if not py_lib.is_dir(): - raise RuntimeError(f"Python runtime library directory is missing: {py_lib}") - - dst = py_lib / "lxml" - if dst.exists(): - shutil.rmtree(dst) - shutil.copytree(lxml_src, dst) - - # Ensure the etree.py shim explicitly exports names that are not in - # lxml.etree.__all__ but are expected by downstream packages (e.g. - # openpyxl imports xmlfile). The star-import only picks up names - # listed in __all__; xmlfile/htmlfile are cdef classes omitted from - # that list. - _write_etree_shim(dst / "etree.py") - - print(f"[lxml] Staged {lxml_src} -> {dst}") - - -# The content of the etree.py shim that bridges the built-in _lxml_etree -# C extension to the expected lxml.etree import path. -_ETREE_SHIM = """\ -from _lxml_etree import * -from _lxml_etree import _Element, _ElementTree, _Comment, _ProcessingInstruction, ElementBase, QName, _Attrib -from _lxml_etree import xmlfile, htmlfile -""" - - -def _write_etree_shim(path: Path) -> None: - """Write (or overwrite) the lxml/etree.py shim with correct exports.""" - path.write_text(_ETREE_SHIM, encoding="utf-8") diff --git a/.nanvix/nanvix.lock b/.nanvix/nanvix.lock index e513314e050eba..87cd674d872de3 100644 --- a/.nanvix/nanvix.lock +++ b/.nanvix/nanvix.lock @@ -1,7 +1,7 @@ # nanvix.lock - auto-generated by nanvix_zutil. DO NOT EDIT. [metadata] -manifest-hash = "sha256:b55a24d9397cc0923444716aa0555d4a3696010850f39ab12ac723a701ae7547" +manifest-hash = "sha256:cea42a1c44c8c70e4d36517b7a56f395157dd73fcab0f7090ef4eb274852edda" nanvix-zutil-version = "0.17.0" [metadata.sdk] @@ -79,9 +79,6 @@ release-id = 354630018 dependencies = [] required-by = [ "sqlite", - "libxml2", - "libxslt", - "lxml", ] [[package.assets]] @@ -200,95 +197,6 @@ name = "libffi-windows-x86-microvm-standalone-256mb-dev.zip" url = "https://github.com/nanvix/libffi/releases/download/3.4.6-nanvix-0.20.9-sdk.1/libffi-windows-x86-microvm-standalone-256mb-dev.zip" id = 478241301 -[[package]] -name = "libxml2" -repo = "nanvix/libxml2" -kind = "dependency" -ref-kind = "version" -ref-value = "2.12.9-nanvix-0.20.9-sdk.1" -resolved-tag = "2.12.9-nanvix-0.20.9-sdk.1" -resolved-commitish = "nanvix/v2.12.9" -release-id = 354639992 -dependencies = [ - "zlib", -] -required-by = [ - "libxslt", - "lxml", -] - -[[package.assets]] -name = "libxml2-linux-x86-microvm-standalone-256mb-dev.tar.gz" -url = "https://github.com/nanvix/libxml2/releases/download/2.12.9-nanvix-0.20.9-sdk.1/libxml2-linux-x86-microvm-standalone-256mb-dev.tar.gz" -id = 478260873 - -[[package.assets]] -name = "libxml2-linux-x86-microvm-standalone-256mb.tar.gz" -url = "https://github.com/nanvix/libxml2/releases/download/2.12.9-nanvix-0.20.9-sdk.1/libxml2-linux-x86-microvm-standalone-256mb.tar.gz" -id = 478260875 - -[[package.assets]] -name = "libxml2-windows-x86-microvm-standalone-256mb-dev.zip" -url = "https://github.com/nanvix/libxml2/releases/download/2.12.9-nanvix-0.20.9-sdk.1/libxml2-windows-x86-microvm-standalone-256mb-dev.zip" -id = 478260871 - -[[package.assets]] -name = "libxml2-windows-x86-microvm-standalone-256mb.zip" -url = "https://github.com/nanvix/libxml2/releases/download/2.12.9-nanvix-0.20.9-sdk.1/libxml2-windows-x86-microvm-standalone-256mb.zip" -id = 478260872 - -[[package]] -name = "libxslt" -repo = "nanvix/libxslt" -kind = "dependency" -ref-kind = "version" -ref-value = "1.1.42-nanvix-0.20.9-sdk.1" -resolved-tag = "1.1.42-nanvix-0.20.9-sdk.1" -resolved-commitish = "nanvix/v1.1.42" -release-id = 354653236 -dependencies = [ - "libxml2", - "zlib", -] -required-by = [ - "lxml", -] - -[[package.assets]] -name = "libxslt-linux-x86-microvm-standalone-256mb-dev.tar.gz" -url = "https://github.com/nanvix/libxslt/releases/download/1.1.42-nanvix-0.20.9-sdk.1/libxslt-linux-x86-microvm-standalone-256mb-dev.tar.gz" -id = 478283663 - -[[package.assets]] -name = "libxslt-windows-x86-microvm-standalone-256mb-dev.zip" -url = "https://github.com/nanvix/libxslt/releases/download/1.1.42-nanvix-0.20.9-sdk.1/libxslt-windows-x86-microvm-standalone-256mb-dev.zip" -id = 478283666 - -[[package]] -name = "lxml" -repo = "nanvix/lxml" -kind = "dependency" -ref-kind = "version" -ref-value = "5.3.0-nanvix-0.20.9-sdk.1" -resolved-tag = "5.3.0-nanvix-0.20.9-sdk.1" -resolved-commitish = "nanvix/v5.3.0" -release-id = 354658318 -dependencies = [ - "zlib", - "libxml2", - "libxslt", -] - -[[package.assets]] -name = "lxml-linux-x86-microvm-standalone-256mb-dev.tar.gz" -url = "https://github.com/nanvix/lxml/releases/download/5.3.0-nanvix-0.20.9-sdk.1/lxml-linux-x86-microvm-standalone-256mb-dev.tar.gz" -id = 478290062 - -[[package.assets]] -name = "lxml-windows-x86-microvm-standalone-256mb-dev.zip" -url = "https://github.com/nanvix/lxml/releases/download/5.3.0-nanvix-0.20.9-sdk.1/lxml-windows-x86-microvm-standalone-256mb-dev.zip" -id = 478290065 - [[package]] name = "xz" repo = "nanvix/xz" diff --git a/.nanvix/nanvix.toml b/.nanvix/nanvix.toml index b538e6cc3bedd1..deace6fca73a9b 100644 --- a/.nanvix/nanvix.toml +++ b/.nanvix/nanvix.toml @@ -15,9 +15,6 @@ sqlite = "3.49.0" openssl = "3.5.0" bzip2 = "1.0.8" libffi = "3.4.6" -libxml2 = "2.12.9" -libxslt = "1.1.42" -lxml = "5.3.0" xz = "5.2.5" [toolchain] diff --git a/.nanvix/setup_local.py b/.nanvix/setup_local.py index 96b7a89c4d70cc..00da0a5aefdca3 100644 --- a/.nanvix/setup_local.py +++ b/.nanvix/setup_local.py @@ -3,8 +3,8 @@ """Single source of truth for Modules/Setup.local on Nanvix builds. -The host build path in ``lxml.generate_setup_local()`` and the Docker build -path in ``_docker._generate_setup_local_cmd()`` both render this table. +The host build path in ``generate_setup_local()`` and the Docker build path in +``_docker._generate_setup_local_cmd()`` both render this table. Module ordering matters: makesetup applies "first rule wins" semantics when the same module is declared both here and in ``Modules/Setup.stdlib``. @@ -15,6 +15,7 @@ from __future__ import annotations from enum import Enum +from pathlib import Path from typing import Iterable, NamedTuple, Sequence @@ -40,31 +41,6 @@ class SetupEntry(NamedTuple): tokens=("_nanvixmodule.c",), comment="Nanvix OS interface module (snapshot, host-mount).", ), - SetupEntry( - name="_lxml_etree", - linkage=Linkage.STATIC, - tokens=( - "lxml_etree_builtin.c", - "-L{buildroot}/lib", - "-llxml_etree", - "-lxslt", - "-lexslt", - "-lxml2", - "-lz", - ), - comment="lxml C extensions linked with their pre-built archives.", - ), - SetupEntry( - name="_lxml_elementpath", - linkage=Linkage.STATIC, - tokens=( - "lxml_elementpath_builtin.c", - "-L{buildroot}/lib", - "-llxml_elementpath", - "-lxml2", - "-lz", - ), - ), SetupEntry( name="array", linkage=Linkage.SHARED, @@ -255,3 +231,14 @@ def render_setup_local( lines.append("") return "\n".join(lines) + + +def generate_setup_local(repo_root: Path, buildroot: Path) -> None: + """Generate Modules/Setup.local for the native-host build path.""" + setup_local = repo_root / "Modules" / "Setup.local" + content = render_setup_local( + buildroot=str(buildroot), + header_comment="Auto-generated by .nanvix/setup_local.py -- do not edit manually.", + ) + setup_local.write_text(content, encoding="utf-8") + print(f"[setup_local] Generated {setup_local}") diff --git a/.nanvix/z.py b/.nanvix/z.py index e2fabab47e832c..a879456d6c8039 100644 --- a/.nanvix/z.py +++ b/.nanvix/z.py @@ -22,15 +22,12 @@ import os import shutil -import tarfile -import zipfile from pathlib import Path from nanvix_zutil import paths import _test as test_mod import build as build_mod -import lxml as lxml_mod import config import package as package_mod import ramfs as ramfs_mod @@ -43,7 +40,6 @@ log, run, ) -from nanvix_zutil.paths import nanvix_root # --------------------------------------------------------------------------- # Path helpers @@ -210,7 +206,6 @@ def setup(self) -> bool: if path.is_dir(): shutil.rmtree(path) - self._install_lxml_runtime_payload() self._overlay_local_nanvix() self.config.save() return used_fallback @@ -223,7 +218,6 @@ def build(self) -> None: build_mod.clean(preserve_nanvix_root=False, preserve_cache=True) args = self._make_args(release=True, with_docker=True) build_mod.build(args) - lxml_mod.stage_lxml_runtime(package_mod.sysroot_pkg()) package_mod.stage() ramfs_mod.build_image( package_mod.sysroot_pkg(), @@ -235,7 +229,6 @@ def build(self) -> None: build_mod.clean(preserve_nanvix_root=True, preserve_cache=True) args = self._make_args(release=False, with_docker=True) build_mod.build(args) - lxml_mod.stage_lxml_runtime(paths.test_out()) test_mod.stage_ramfs(args) def test(self) -> None: @@ -262,71 +255,6 @@ def clean(self) -> None: """Remove build artifacts.""" build_mod.clean() - @staticmethod - def _python_package_path(member_name: str) -> Path | None: - """Return a safe path below an archive's ``python-packages`` directory.""" - parts = Path(member_name).parts - try: - package_index = parts.index("python-packages") - except ValueError: - return None - relative = Path(*parts[package_index + 1 :]) - if not relative.parts or relative.is_absolute() or ".." in relative.parts: - return None - return relative - - def _install_lxml_runtime_payload(self) -> None: - """Install the exact lxml release's Python payload into the buildroot.""" - cache_dir = nanvix_root() / "cache" - # Magic-path naming: lxml-{host}-{arch}-{machine}-{mode}-{mem}-dev.{ext}. - # Match any host/arch pair for the current machine + memory + mode. - pattern = ( - f"lxml-*-{self.config.machine}-" - f"{self.config.deployment_mode}-{self.config.memory_size}-dev.*" - ) - candidates = list(cache_dir.glob(pattern)) if cache_dir.is_dir() else [] - if not candidates: - raise FileNotFoundError( - "lxml release archive is missing from .nanvix/cache" - ) - - archive = max(candidates, key=lambda path: path.stat().st_mtime_ns) - destination = nanvix_root() / "buildroot" / "python-packages" - if destination.is_dir(): - shutil.rmtree(destination) - destination.mkdir(parents=True) - - installed = 0 - if zipfile.is_zipfile(archive): - with zipfile.ZipFile(archive) as source: - for member in source.infolist(): - relative = self._python_package_path(member.filename) - if relative is None or member.is_dir(): - continue - output = destination / relative - output.parent.mkdir(parents=True, exist_ok=True) - with source.open(member) as src, output.open("wb") as dst: - shutil.copyfileobj(src, dst) - installed += 1 - else: - with tarfile.open(archive, "r:*") as source: - for member in source.getmembers(): - relative = self._python_package_path(member.name) - if relative is None or not member.isfile(): - continue - extracted = source.extractfile(member) - if extracted is None: - continue - output = destination / relative - output.parent.mkdir(parents=True, exist_ok=True) - with extracted, output.open("wb") as dst: - shutil.copyfileobj(extracted, dst) - installed += 1 - - if installed == 0: - raise RuntimeError(f"{archive.name} contains no python-packages payload") - log.info(f"Installed {installed} lxml runtime file(s) from {archive.name}") - if __name__ == "__main__": CPythonBuild.main() diff --git a/Lib/test/test_nanvix_lxml.py b/Lib/test/test_nanvix_lxml.py deleted file mode 100644 index 28c62b312b953c..00000000000000 --- a/Lib/test/test_nanvix_lxml.py +++ /dev/null @@ -1,53 +0,0 @@ -"""Smoke tests for lxml built-in on NanVix.""" - -import sys -import unittest -from test.support import is_nanvix -from test.support.import_helper import import_module - -if not is_nanvix: - raise unittest.SkipTest("lxml built-in is Nanvix-specific") - -etree = import_module("lxml.etree") - - -class NanvixLxmlTests(unittest.TestCase): - - def test_import_lxml_etree(self): - self.assertTrue(hasattr(etree, "fromstring")) - self.assertTrue(hasattr(etree, "_Element")) - - def test_parse_xml(self): - root = etree.fromstring(b'text') - self.assertEqual(root.tag, "root") - child = root.find("child") - self.assertIsNotNone(child) - self.assertEqual(child.text, "text") - self.assertEqual(child.get("key"), "val") - - def test_element_creation(self): - root = etree.Element("doc") - etree.SubElement(root, "item").text = "hello" - xml = etree.tostring(root, encoding="unicode") - self.assertIn("hello", xml) - - def test_elementpath(self): - root = etree.fromstring(b"12") - results = root.findall("b") - self.assertEqual(len(results), 2) - self.assertEqual(results[0].text, "1") - - def test_xmlfile_available(self): - # xmlfile must be importable from lxml.etree for openpyxl compat. - from lxml.etree import xmlfile - - self.assertTrue(callable(xmlfile)) - - def test_htmlfile_available(self): - from lxml.etree import htmlfile - - self.assertTrue(callable(htmlfile)) - - -if __name__ == "__main__": - unittest.main() diff --git a/Modules/lxml_elementpath_builtin.c b/Modules/lxml_elementpath_builtin.c deleted file mode 100644 index 6a1758e1224659..00000000000000 --- a/Modules/lxml_elementpath_builtin.c +++ /dev/null @@ -1,16 +0,0 @@ -/* - * lxml_elementpath_builtin.c - Shim for lxml._elementpath built-in module. - * - * Registers the Cython _elementpath extension under the flat name - * "_lxml_elementpath" for Modules/Setup.local. - */ - -#include "Python.h" - -extern PyObject* PyInit__elementpath(void); - -PyMODINIT_FUNC -PyInit__lxml_elementpath(void) -{ - return PyInit__elementpath(); -} diff --git a/Modules/lxml_etree_builtin.c b/Modules/lxml_etree_builtin.c deleted file mode 100644 index d028054d5b08f2..00000000000000 --- a/Modules/lxml_etree_builtin.c +++ /dev/null @@ -1,21 +0,0 @@ -/* - * lxml_etree_builtin.c - Shim to register lxml.etree as a CPython built-in. - * - * makesetup does not support dotted module names, so the Cython extension - * is registered under the flat name "_lxml_etree". A pure-Python shim at - * lxml/etree.py re-exports everything via `from _lxml_etree import *`. - * - * The Cython-generated code in liblxml_etree.a exports PyInit_etree. - * This wrapper provides PyInit__lxml_etree so the name matches the - * entry in Modules/Setup.local. - */ - -#include "Python.h" - -extern PyObject* PyInit_etree(void); - -PyMODINIT_FUNC -PyInit__lxml_etree(void) -{ - return PyInit_etree(); -}