From 5a6ea47e1b492270413f673b2b15af0312f3874d Mon Sep 17 00:00:00 2001 From: Enrique Saurez Date: Wed, 15 Jul 2026 17:36:20 -0700 Subject: [PATCH 1/2] [nanvix] E: Build stdlib extensions as .so with the SDK Generate one Setup.local for native and Docker builds, link migrated extensions against the exported Nanvix SDK PIE without duplicating libc state, and retain static C-API anchors required by the current loader. Strip installed modules and exercise real nested-import and dlopen paths. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 23312a7e-62f5-477a-a83f-65fd99522fe3 --- .nanvix/_docker.py | 27 ++--- .nanvix/_test.py | 105 ++++++++++++++++- .nanvix/lxml.py | 28 +---- .nanvix/setup_local.py | 257 +++++++++++++++++++++++++++++++++++++++++ Makefile.nanvix | 14 +-- 5 files changed, 383 insertions(+), 48 deletions(-) create mode 100644 .nanvix/setup_local.py diff --git a/.nanvix/_docker.py b/.nanvix/_docker.py index 7759258b8cb338d..be58e73ef8506bd 100644 --- a/.nanvix/_docker.py +++ b/.nanvix/_docker.py @@ -22,6 +22,7 @@ import build as build_mod import config +import setup_local as setup_local_mod def _workspace_id(workspace: Path) -> str: @@ -232,27 +233,17 @@ def docker_build( def _generate_setup_local_cmd() -> str: - """Shell command to generate Modules/Setup.local inside the container.""" + """Generate Modules/Setup.local inside the SDK container.""" buildroot = config.DOCKER_BUILDROOT_PATH ws = config.DOCKER_WORKSPACE_PATH - return ( - f"printf '%s\\n' " - f"'# Auto-generated by .nanvix/docker.py -- do not edit manually.' " - f"'' " - f"'# Statically-linked extension modules for Nanvix builds.' " - f"'*static*' " - f"'# Nanvix OS interface module (snapshot, host-mount).' " - f"'_nanvix _nanvixmodule.c' " - f"'# lxml C extension modules (statically linked via pre-built archives).' " - f"'_lxml_etree lxml_etree_builtin.c -L{buildroot}/lib -llxml_etree -lxslt -lexslt -lxml2 -lz' " - f"'_lxml_elementpath lxml_elementpath_builtin.c -L{buildroot}/lib -llxml_elementpath -lxml2 -lz' " - f"'' " - f"'# Phase 0 of the .a -> .so migration: array as a proof-of-concept shared module.' " - f"'# Listed before Setup.stdlib static declaration so makesetup first rule wins.' " - f"'*shared*' " - f"'array arraymodule.c' " - f"> {ws}/Modules/Setup.local" + rendered = setup_local_mod.render_setup_local( + buildroot=buildroot, + header_comment="Auto-generated by .nanvix/_docker.py -- do not edit manually.", + ) + quoted_lines = " ".join( + "'" + line.replace("'", "'\"'\"'") + "'" for line in rendered.splitlines() ) + return f"printf '%s\\n' {quoted_lines} > {ws}/Modules/Setup.local" def _copy_outputs_cmd() -> str: diff --git a/.nanvix/_test.py b/.nanvix/_test.py index f36b8eb20d98197..2b2c8f6501e208d 100644 --- a/.nanvix/_test.py +++ b/.nanvix/_test.py @@ -31,6 +31,94 @@ _DOWNLOADED_RELEASE_MARKER = ".downloaded-release" +# --------------------------------------------------------------------------- +# Shared-extension smoke checks +# --------------------------------------------------------------------------- + +_SO_MODULE_SANITY_CHECKS: tuple[tuple[str, tuple[tuple[str, str], ...]], ...] = ( + ( + "CPYTHON_TEST_DATA_PRIMITIVES", + ( + ("_bisect", "m.bisect_left([1, 3, 5], 4) == 2"), + ("_heapq", "m.heappush([], 1) is None"), + ("_struct", "m.pack('i', 42) == b'\\x2a\\x00\\x00\\x00'"), + ("_random", "hasattr(m, 'Random')"), + ("_opcode", "hasattr(m, 'stack_effect')"), + ("_queue", "hasattr(m, 'SimpleQueue')"), + ("_csv", "hasattr(m, 'reader')"), + ("binascii", "m.hexlify(b'\\xab') == b'ab'"), + ("_json", "hasattr(m, 'encode_basestring_ascii')"), + ("_pickle", "hasattr(m, 'Pickler')"), + ("_zoneinfo", "hasattr(m, 'ZoneInfo')"), + ), + ), + ( + "CPYTHON_TEST_MATH", + ( + ("math", "abs(m.sqrt(4.0) - 2.0) < 1e-9"), + ("cmath", "abs(m.sqrt(complex(-1)) - complex(0, 1)) < 1e-9"), + ("_statistics", "hasattr(m, '_normal_dist_inv_cdf')"), + ("mmap", "hasattr(m, 'mmap')"), + ("_contextvars", "hasattr(m, 'ContextVar')"), + ), + ), + ( + "CPYTHON_TEST_CODECS", + ( + ("unicodedata", "m.lookup('LATIN SMALL LETTER A') == 'a'"), + ("_codecs_cn", "hasattr(m, 'getcodec')"), + ("_codecs_hk", "hasattr(m, 'getcodec')"), + ("_codecs_iso2022", "hasattr(m, 'getcodec')"), + ("_codecs_jp", "hasattr(m, 'getcodec')"), + ("_codecs_kr", "hasattr(m, 'getcodec')"), + ("_codecs_tw", "hasattr(m, 'getcodec')"), + ), + ), + ( + "CPYTHON_TEST_BUNDLED_DEPS", + ( + ("_asyncio", "hasattr(m, 'Future')"), + ("_decimal", "m.Decimal('1.1') + m.Decimal('2.2') == m.Decimal('3.3')"), + ("_elementtree", "hasattr(m, 'XMLParser')"), + ("_md5", "hasattr(m, 'md5')"), + ("_sha1", "hasattr(m, 'sha1')"), + ("_sha2", "hasattr(m, 'sha256')"), + ("_sha3", "hasattr(m, 'sha3_256')"), + ("_blake2", "hasattr(m, 'blake2b')"), + ("select", "hasattr(m, 'select')"), + ("_socket", "hasattr(m, 'socket')"), + ("_posixsubprocess", "hasattr(m, 'fork_exec')"), + ("fcntl", "hasattr(m, 'fcntl')"), + ("termios", "hasattr(m, 'tcgetattr')"), + ), + ), +) + + +def _render_so_sanity_snippets( + checks: tuple[ + tuple[str, tuple[tuple[str, str], ...]], ... + ] = _SO_MODULE_SANITY_CHECKS, +) -> str: + """Render imports that prove each migrated module loads through dlopen.""" + snippets: list[str] = [] + for log_tag, modules in checks: + items = ",\n".join( + f" ({name!r}, lambda m: {check})" for name, check in modules + ) + snippets.append( + f"_so_checks = [\n{items},\n]\n" + "for _name, _check in _so_checks:\n" + " _mod = __import__(_name)\n" + " assert _name not in sys.builtin_module_names, " + "f'{_name} still built-in!'\n" + " assert _check(_mod), f'{_name} sanity check failed'\n" + f" print(f'{log_tag}: " + "{_name} loaded via dlopen from {_mod.__file__}')\n" + ) + return "".join(snippets) + + # --------------------------------------------------------------------------- # Initrd creation helper (standalone mode) # --------------------------------------------------------------------------- @@ -268,6 +356,17 @@ def stage( "assert 'array' not in sys.builtin_module_names, 'array still built-in!'\n" "print(f'CPYTHON_TEST_ARRAY_SO: array loaded via dlopen from {array.__file__}')\n" ) + nested_import_snippet = ( + "import xml.etree.ElementTree as _elementtree_api\n" + "assert _elementtree_api.fromstring('').tag == 'root'\n" + "assert '_elementtree' not in sys.builtin_module_names\n" + "assert 'pyexpat' in sys.builtin_module_names\n" + "import encodings.gb2312\n" + "assert '中文'.encode('gb2312') == b'\\xd6\\xd0\\xce\\xc4'\n" + "assert '_codecs_cn' not in sys.builtin_module_names\n" + "assert '_multibytecodec' in sys.builtin_module_names\n" + "print('CPYTHON_TEST_NESTED_IMPORTS: static C API anchors OK')\n" + ) # The lxml import is exercised against the in-memory FAT ramfs VFS via # xmlInitParser(). @@ -287,7 +386,11 @@ def stage( (staging / "test_hello.py").write_text( "import sys\n" "print('CPYTHON_TEST_HELLO: Hello from Python', sys.version_info[:2])\n" - "print('CPYTHON_TEST_PLATFORM:', sys.platform)\n" + array_snippet + lxml_snippet + "print('CPYTHON_TEST_PLATFORM:', sys.platform)\n" + + array_snippet + + nested_import_snippet + + _render_so_sanity_snippets() + + lxml_snippet ) # HTTP server smoke-test script must be present in the sysroot before diff --git a/.nanvix/lxml.py b/.nanvix/lxml.py index 9231d408af40f20..cb4180178eb3848 100644 --- a/.nanvix/lxml.py +++ b/.nanvix/lxml.py @@ -8,32 +8,16 @@ from nanvix_zutil import paths import config - -_SETUP_LOCAL_TEMPLATE = """\ -# Auto-generated by .nanvix/lxml.py -- do not edit manually. - -# Statically-linked extension modules for Nanvix builds. -*static* -# Nanvix OS interface module (snapshot, host-mount). -_nanvix _nanvixmodule.c -# lxml C extension modules (statically linked via pre-built archives). -_lxml_etree lxml_etree_builtin.c -L{sysroot}/lib -llxml_etree -lxslt -lexslt -lxml2 -lz -_lxml_elementpath lxml_elementpath_builtin.c -L{sysroot}/lib -llxml_elementpath -lxml2 -lz - -# Phase 0 of the .a -> .so migration: array as a proof-of-concept shared module. -# Listed before Setup.stdlib's static declaration so makesetup's first rule wins. -*shared* -array arraymodule.c -""" +import setup_local as setup_local_mod def generate_setup_local(repo_root: Path, buildroot: Path) -> None: - """Generate Modules/Setup.local with statically-linked module definitions. - - Includes both the _nanvix OS interface module and lxml C extensions. - """ + """Generate Modules/Setup.local for the native-host build path.""" setup_local = repo_root / "Modules" / "Setup.local" - content = _SETUP_LOCAL_TEMPLATE.format(sysroot=buildroot) + content = setup_local_mod.render_setup_local( + buildroot=str(buildroot), + header_comment="Auto-generated by .nanvix/lxml.py -- do not edit manually.", + ) setup_local.write_text(content, encoding="utf-8") print(f"[lxml] Generated {setup_local}") diff --git a/.nanvix/setup_local.py b/.nanvix/setup_local.py new file mode 100644 index 000000000000000..96b7a89c4d70cc9 --- /dev/null +++ b/.nanvix/setup_local.py @@ -0,0 +1,257 @@ +# Copyright(c) The Maintainers of Nanvix. +# Licensed under the MIT License. + +"""Single source of truth for Modules/Setup.local on Nanvix builds. + +The host build path in ``lxml.generate_setup_local()`` and the Docker build +path in ``_docker._generate_setup_local_cmd()`` both render this table. + +Module ordering matters: makesetup applies "first rule wins" semantics when +the same module is declared both here and in ``Modules/Setup.stdlib``. +Declaring a shared duplicate before the upstream static default switches that +module to a runtime-loaded extension. +""" + +from __future__ import annotations + +from enum import Enum +from typing import Iterable, NamedTuple, Sequence + + +class Linkage(Enum): + STATIC = "*static*" + SHARED = "*shared*" + + +class SetupEntry(NamedTuple): + """One Modules/Setup.local entry with optional surrounding comments.""" + + name: str + linkage: Linkage + tokens: Sequence[str] = () + comment: str = "" + section_header: str = "" + + +SETUP_LOCAL_ENTRIES: tuple[SetupEntry, ...] = ( + SetupEntry( + name="_nanvix", + linkage=Linkage.STATIC, + tokens=("_nanvixmodule.c",), + comment="Nanvix OS interface module (snapshot, host-mount).", + ), + SetupEntry( + name="_lxml_etree", + linkage=Linkage.STATIC, + tokens=( + "lxml_etree_builtin.c", + "-L{buildroot}/lib", + "-llxml_etree", + "-lxslt", + "-lexslt", + "-lxml2", + "-lz", + ), + comment="lxml C extensions linked with their pre-built archives.", + ), + SetupEntry( + name="_lxml_elementpath", + linkage=Linkage.STATIC, + tokens=( + "lxml_elementpath_builtin.c", + "-L{buildroot}/lib", + "-llxml_elementpath", + "-lxml2", + "-lz", + ), + ), + SetupEntry( + name="array", + linkage=Linkage.SHARED, + tokens=("arraymodule.c",), + section_header=( + "`array` is the proof-of-concept shared module. It appears before " + "Setup.stdlib so makesetup selects this shared definition." + ), + ), + *( + SetupEntry( + name=name, + linkage=Linkage.SHARED, + tokens=(source,), + section_header=header if index == 0 else "", + ) + for index, (name, source) in enumerate( + ( + ("_bisect", "_bisectmodule.c"), + ("_heapq", "_heapqmodule.c"), + ("_struct", "_struct.c"), + ("_random", "_randommodule.c"), + ("_opcode", "_opcode.c"), + ("_queue", "_queuemodule.c"), + ("_csv", "_csv.c"), + ("binascii", "binascii.c"), + ("_json", "_json.c"), + ("_pickle", "_pickle.c"), + ("_zoneinfo", "_zoneinfo.c"), + ) + ) + for header in ("Data primitives without external dependencies.",) + ), + *( + SetupEntry( + name=name, + linkage=Linkage.SHARED, + tokens=(source,), + section_header=header if index == 0 else "", + ) + for index, (name, source) in enumerate( + ( + ("math", "mathmodule.c"), + ("cmath", "cmathmodule.c"), + ("_statistics", "_statisticsmodule.c"), + ("mmap", "mmapmodule.c"), + ("_contextvars", "_contextvarsmodule.c"), + ) + ) + for header in ("Math and memory modules using the SDK shared runtime.",) + ), + *( + SetupEntry( + name=name, + linkage=Linkage.SHARED, + tokens=(source,), + section_header=header if index == 0 else "", + ) + for index, (name, source) in enumerate( + ( + ("unicodedata", "unicodedata.c"), + ("_codecs_cn", "cjkcodecs/_codecs_cn.c"), + ("_codecs_hk", "cjkcodecs/_codecs_hk.c"), + ("_codecs_iso2022", "cjkcodecs/_codecs_iso2022.c"), + ("_codecs_jp", "cjkcodecs/_codecs_jp.c"), + ("_codecs_kr", "cjkcodecs/_codecs_kr.c"), + ("_codecs_tw", "cjkcodecs/_codecs_tw.c"), + ) + ) + for header in ("Text codecs without external dependencies.",) + ), + # Keep _datetime, pyexpat, and _multibytecodec static: migrated consumers + # import their C APIs while a shared object initializes, which Nanvix cannot + # nest safely yet. + SetupEntry( + name="_asyncio", + linkage=Linkage.SHARED, + tokens=("_asynciomodule.c",), + section_header=( + "Modules with bundled CPython dependencies. Each extension keeps " + "its vendored archive, matching the upstream build." + ), + ), + SetupEntry( + name="_decimal", linkage=Linkage.SHARED, tokens=("_decimal/_decimal.c",) + ), + SetupEntry(name="_elementtree", linkage=Linkage.SHARED, tokens=("_elementtree.c",)), + SetupEntry( + name="_md5", + linkage=Linkage.SHARED, + tokens=( + "md5module.c", + "-I$(srcdir)/Modules/_hacl/include", + "_hacl/Hacl_Hash_MD5.c", + "-D_BSD_SOURCE", + "-D_DEFAULT_SOURCE", + ), + ), + SetupEntry( + name="_sha1", + linkage=Linkage.SHARED, + tokens=( + "sha1module.c", + "-I$(srcdir)/Modules/_hacl/include", + "_hacl/Hacl_Hash_SHA1.c", + "-D_BSD_SOURCE", + "-D_DEFAULT_SOURCE", + ), + ), + SetupEntry( + name="_sha2", + linkage=Linkage.SHARED, + tokens=( + "sha2module.c", + "-I$(srcdir)/Modules/_hacl/include", + "Modules/_hacl/libHacl_Hash_SHA2.a", + ), + ), + SetupEntry( + name="_sha3", + linkage=Linkage.SHARED, + tokens=( + "sha3module.c", + "-I$(srcdir)/Modules/_hacl/include", + "_hacl/Hacl_Hash_SHA3.c", + "-D_BSD_SOURCE", + "-D_DEFAULT_SOURCE", + "$(COMPILER_RT_BUILTINS)", + ), + ), + SetupEntry( + name="_blake2", + linkage=Linkage.SHARED, + tokens=( + "_blake2/blake2module.c", + "_blake2/blake2b_impl.c", + "_blake2/blake2s_impl.c", + ), + ), + SetupEntry(name="select", linkage=Linkage.SHARED, tokens=("selectmodule.c",)), + SetupEntry(name="_socket", linkage=Linkage.SHARED, tokens=("socketmodule.c",)), + SetupEntry( + name="_posixsubprocess", linkage=Linkage.SHARED, tokens=("_posixsubprocess.c",) + ), + SetupEntry(name="fcntl", linkage=Linkage.SHARED, tokens=("fcntlmodule.c",)), + SetupEntry(name="termios", linkage=Linkage.SHARED, tokens=("termios.c", "-lc")), +) + + +def _wrap_comment(text: str, *, width: int = 78) -> list[str]: + """Return wrapped Setup.local comment lines.""" + if not text: + return [] + + import textwrap + + return [f"# {line}" for line in textwrap.wrap(text, width=width - 2)] + + +def render_setup_local( + entries: Iterable[SetupEntry] = SETUP_LOCAL_ENTRIES, + *, + buildroot: str, + header_comment: str, +) -> str: + """Render entries to a complete Modules/Setup.local file.""" + lines = [f"# {header_comment}", ""] + current_linkage: Linkage | None = None + + for entry in entries: + if entry.linkage is not current_linkage: + if current_linkage is not None: + lines.append("") + if entry.section_header: + lines.extend(_wrap_comment(entry.section_header)) + elif current_linkage is None and entry.linkage is Linkage.STATIC: + lines.append("# Statically-linked extension modules for Nanvix builds.") + lines.append(entry.linkage.value) + current_linkage = entry.linkage + elif entry.section_header: + lines.append("") + lines.extend(_wrap_comment(entry.section_header)) + + if entry.comment: + lines.extend(_wrap_comment(entry.comment)) + tokens = " ".join(entry.tokens).replace("{buildroot}", buildroot) + lines.append(f"{entry.name} {tokens}".rstrip()) + + lines.append("") + return "\n".join(lines) diff --git a/Makefile.nanvix b/Makefile.nanvix index f4928c02191c9ca..7ada017b5599845 100644 --- a/Makefile.nanvix +++ b/Makefile.nanvix @@ -41,6 +41,7 @@ TARGET_CXX := $(SDK_BIN)/clang++ --target=$(NANVIX_TARGET_TRIPLE) TARGET_AR := $(SDK_BIN)/llvm-ar TARGET_RANLIB := $(SDK_BIN)/llvm-ranlib TARGET_STRIP := $(SDK_BIN)/llvm-strip +COMPILER_RT_BUILTINS := $(shell $(TARGET_CC) --print-libgcc-file-name) DEPENDENCY_CPPFLAGS := -I$(NANVIX_BUILDROOT)/include DEPENDENCY_LDFLAGS := -L$(NANVIX_BUILDROOT)/lib @@ -61,8 +62,8 @@ CONFIGURE_ENV = \ CFLAGS_NODIST="-fno-semantic-interposition" \ LDFLAGS="$(DEPENDENCY_LDFLAGS) -Wl,--export-dynamic" \ LINKFORSHARED="-Wl,-pie,--export-dynamic,--no-dynamic-linker,-z,notext,-z,norelro,--hash-style=sysv" \ - LDSHARED="$(TARGET_CC) -shared -Wl,--no-as-needed,-z,now,-rpath,/lib,-Bdynamic -L$(NANVIX_SDK_ROOT)/lib -l:libc.so -l:libm.so -Wl,-Bstatic" \ - LDCXXSHARED="$(TARGET_CXX) -shared -Wl,--no-as-needed,-z,now,-rpath,/lib,-Bdynamic -L$(NANVIX_SDK_ROOT)/lib -l:libc.so -l:libm.so -Wl,-Bstatic" \ + LDSHARED="$(TARGET_CC) -shared -nostdlib -Wl,-z,now,--no-dynamic-linker" \ + LDCXXSHARED="$(TARGET_CXX) -shared -nostdlib -Wl,-z,now,--no-dynamic-linker" \ LIBS="$(DEPENDENCY_LIBS)" \ LIBSQLITE3_LIBS="$(DEPENDENCY_LDFLAGS) -lsqlite3" \ LIBSQLITE3_CFLAGS="$(DEPENDENCY_CPPFLAGS)" \ @@ -135,16 +136,15 @@ $(CONFIGURED_MARKER): Makefile.nanvix touch $@ build: $(CONFIGURED_MARKER) - $(MAKE) -j$$(nproc) all + $(MAKE) -j$$(nproc) COMPILER_RT_BUILTINS="$(COMPILER_RT_BUILTINS)" all @if [ -f python ]; then cp -f python python$(EXE); fi @test -f python$(EXE) $(TARGET_STRIP) --strip-all python$(EXE) install: build - $(MAKE) install DESTDIR="$(DESTDIR)" - mkdir -p "$(DESTDIR)/lib" - cp -f "$(NANVIX_SDK_ROOT)/lib/libc.so" "$(DESTDIR)/lib/libc.so" - cp -f "$(NANVIX_SDK_ROOT)/lib/libm.so" "$(DESTDIR)/lib/libm.so" + $(MAKE) COMPILER_RT_BUILTINS="$(COMPILER_RT_BUILTINS)" install DESTDIR="$(DESTDIR)" + find "$(DESTDIR)$(INSTALL_PREFIX)/lib/python3.12/lib-dynload" \ + -type f -name '*.so' -exec $(TARGET_STRIP) --strip-all {} + clean: find . -path './.nanvix' -prune -o -type d -name '__pycache__' \ From 96fc2a7b07b4372f6cddd361533a815e5c723847 Mon Sep 17 00:00:00 2001 From: Pedro Henrique Penna Date: Wed, 15 Jul 2026 18:42:10 -0700 Subject: [PATCH 2/2] Make shared-module smoke source ASCII-safe Emit the gb2312 fixture through Unicode escapes so Windows runners using CP1252 can write test_hello.py without changing the guest assertion. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 23312a7e-62f5-477a-a83f-65fd99522fe3 --- .nanvix/_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.nanvix/_test.py b/.nanvix/_test.py index 2b2c8f6501e208d..b0113c4653a4eb4 100644 --- a/.nanvix/_test.py +++ b/.nanvix/_test.py @@ -362,7 +362,7 @@ def stage( "assert '_elementtree' not in sys.builtin_module_names\n" "assert 'pyexpat' in sys.builtin_module_names\n" "import encodings.gb2312\n" - "assert '中文'.encode('gb2312') == b'\\xd6\\xd0\\xce\\xc4'\n" + "assert '\\u4e2d\\u6587'.encode('gb2312') == b'\\xd6\\xd0\\xce\\xc4'\n" "assert '_codecs_cn' not in sys.builtin_module_names\n" "assert '_multibytecodec' in sys.builtin_module_names\n" "print('CPYTHON_TEST_NESTED_IMPORTS: static C API anchors OK')\n"