diff --git a/.nanvix/_docker.py b/.nanvix/_docker.py index 7759258b8cb338d..be58e73ef8506bd 100644 --- a/.nanvix/_docker.py +++ b/.nanvix/_docker.py @@ -22,6 +22,7 @@ import build as build_mod import config +import setup_local as setup_local_mod def _workspace_id(workspace: Path) -> str: @@ -232,27 +233,17 @@ def docker_build( def _generate_setup_local_cmd() -> str: - """Shell command to generate Modules/Setup.local inside the container.""" + """Generate Modules/Setup.local inside the SDK container.""" buildroot = config.DOCKER_BUILDROOT_PATH ws = config.DOCKER_WORKSPACE_PATH - return ( - f"printf '%s\\n' " - f"'# Auto-generated by .nanvix/docker.py -- do not edit manually.' " - f"'' " - f"'# Statically-linked extension modules for Nanvix builds.' " - f"'*static*' " - f"'# Nanvix OS interface module (snapshot, host-mount).' " - f"'_nanvix _nanvixmodule.c' " - f"'# lxml C extension modules (statically linked via pre-built archives).' " - f"'_lxml_etree lxml_etree_builtin.c -L{buildroot}/lib -llxml_etree -lxslt -lexslt -lxml2 -lz' " - f"'_lxml_elementpath lxml_elementpath_builtin.c -L{buildroot}/lib -llxml_elementpath -lxml2 -lz' " - f"'' " - f"'# Phase 0 of the .a -> .so migration: array as a proof-of-concept shared module.' " - f"'# Listed before Setup.stdlib static declaration so makesetup first rule wins.' " - f"'*shared*' " - f"'array arraymodule.c' " - f"> {ws}/Modules/Setup.local" + rendered = setup_local_mod.render_setup_local( + buildroot=buildroot, + header_comment="Auto-generated by .nanvix/_docker.py -- do not edit manually.", + ) + quoted_lines = " ".join( + "'" + line.replace("'", "'\"'\"'") + "'" for line in rendered.splitlines() ) + return f"printf '%s\\n' {quoted_lines} > {ws}/Modules/Setup.local" def _copy_outputs_cmd() -> str: diff --git a/.nanvix/_test.py b/.nanvix/_test.py index f36b8eb20d98197..b0113c4653a4eb4 100644 --- a/.nanvix/_test.py +++ b/.nanvix/_test.py @@ -31,6 +31,94 @@ _DOWNLOADED_RELEASE_MARKER = ".downloaded-release" +# --------------------------------------------------------------------------- +# Shared-extension smoke checks +# --------------------------------------------------------------------------- + +_SO_MODULE_SANITY_CHECKS: tuple[tuple[str, tuple[tuple[str, str], ...]], ...] = ( + ( + "CPYTHON_TEST_DATA_PRIMITIVES", + ( + ("_bisect", "m.bisect_left([1, 3, 5], 4) == 2"), + ("_heapq", "m.heappush([], 1) is None"), + ("_struct", "m.pack('i', 42) == b'\\x2a\\x00\\x00\\x00'"), + ("_random", "hasattr(m, 'Random')"), + ("_opcode", "hasattr(m, 'stack_effect')"), + ("_queue", "hasattr(m, 'SimpleQueue')"), + ("_csv", "hasattr(m, 'reader')"), + ("binascii", "m.hexlify(b'\\xab') == b'ab'"), + ("_json", "hasattr(m, 'encode_basestring_ascii')"), + ("_pickle", "hasattr(m, 'Pickler')"), + ("_zoneinfo", "hasattr(m, 'ZoneInfo')"), + ), + ), + ( + "CPYTHON_TEST_MATH", + ( + ("math", "abs(m.sqrt(4.0) - 2.0) < 1e-9"), + ("cmath", "abs(m.sqrt(complex(-1)) - complex(0, 1)) < 1e-9"), + ("_statistics", "hasattr(m, '_normal_dist_inv_cdf')"), + ("mmap", "hasattr(m, 'mmap')"), + ("_contextvars", "hasattr(m, 'ContextVar')"), + ), + ), + ( + "CPYTHON_TEST_CODECS", + ( + ("unicodedata", "m.lookup('LATIN SMALL LETTER A') == 'a'"), + ("_codecs_cn", "hasattr(m, 'getcodec')"), + ("_codecs_hk", "hasattr(m, 'getcodec')"), + ("_codecs_iso2022", "hasattr(m, 'getcodec')"), + ("_codecs_jp", "hasattr(m, 'getcodec')"), + ("_codecs_kr", "hasattr(m, 'getcodec')"), + ("_codecs_tw", "hasattr(m, 'getcodec')"), + ), + ), + ( + "CPYTHON_TEST_BUNDLED_DEPS", + ( + ("_asyncio", "hasattr(m, 'Future')"), + ("_decimal", "m.Decimal('1.1') + m.Decimal('2.2') == m.Decimal('3.3')"), + ("_elementtree", "hasattr(m, 'XMLParser')"), + ("_md5", "hasattr(m, 'md5')"), + ("_sha1", "hasattr(m, 'sha1')"), + ("_sha2", "hasattr(m, 'sha256')"), + ("_sha3", "hasattr(m, 'sha3_256')"), + ("_blake2", "hasattr(m, 'blake2b')"), + ("select", "hasattr(m, 'select')"), + ("_socket", "hasattr(m, 'socket')"), + ("_posixsubprocess", "hasattr(m, 'fork_exec')"), + ("fcntl", "hasattr(m, 'fcntl')"), + ("termios", "hasattr(m, 'tcgetattr')"), + ), + ), +) + + +def _render_so_sanity_snippets( + checks: tuple[ + tuple[str, tuple[tuple[str, str], ...]], ... + ] = _SO_MODULE_SANITY_CHECKS, +) -> str: + """Render imports that prove each migrated module loads through dlopen.""" + snippets: list[str] = [] + for log_tag, modules in checks: + items = ",\n".join( + f" ({name!r}, lambda m: {check})" for name, check in modules + ) + snippets.append( + f"_so_checks = [\n{items},\n]\n" + "for _name, _check in _so_checks:\n" + " _mod = __import__(_name)\n" + " assert _name not in sys.builtin_module_names, " + "f'{_name} still built-in!'\n" + " assert _check(_mod), f'{_name} sanity check failed'\n" + f" print(f'{log_tag}: " + "{_name} loaded via dlopen from {_mod.__file__}')\n" + ) + return "".join(snippets) + + # --------------------------------------------------------------------------- # Initrd creation helper (standalone mode) # --------------------------------------------------------------------------- @@ -268,6 +356,17 @@ def stage( "assert 'array' not in sys.builtin_module_names, 'array still built-in!'\n" "print(f'CPYTHON_TEST_ARRAY_SO: array loaded via dlopen from {array.__file__}')\n" ) + nested_import_snippet = ( + "import xml.etree.ElementTree as _elementtree_api\n" + "assert _elementtree_api.fromstring('').tag == 'root'\n" + "assert '_elementtree' not in sys.builtin_module_names\n" + "assert 'pyexpat' in sys.builtin_module_names\n" + "import encodings.gb2312\n" + "assert '\\u4e2d\\u6587'.encode('gb2312') == b'\\xd6\\xd0\\xce\\xc4'\n" + "assert '_codecs_cn' not in sys.builtin_module_names\n" + "assert '_multibytecodec' in sys.builtin_module_names\n" + "print('CPYTHON_TEST_NESTED_IMPORTS: static C API anchors OK')\n" + ) # The lxml import is exercised against the in-memory FAT ramfs VFS via # xmlInitParser(). @@ -287,7 +386,11 @@ def stage( (staging / "test_hello.py").write_text( "import sys\n" "print('CPYTHON_TEST_HELLO: Hello from Python', sys.version_info[:2])\n" - "print('CPYTHON_TEST_PLATFORM:', sys.platform)\n" + array_snippet + lxml_snippet + "print('CPYTHON_TEST_PLATFORM:', sys.platform)\n" + + array_snippet + + nested_import_snippet + + _render_so_sanity_snippets() + + lxml_snippet ) # HTTP server smoke-test script must be present in the sysroot before diff --git a/.nanvix/lxml.py b/.nanvix/lxml.py index 9231d408af40f20..cb4180178eb3848 100644 --- a/.nanvix/lxml.py +++ b/.nanvix/lxml.py @@ -8,32 +8,16 @@ from nanvix_zutil import paths import config - -_SETUP_LOCAL_TEMPLATE = """\ -# Auto-generated by .nanvix/lxml.py -- do not edit manually. - -# Statically-linked extension modules for Nanvix builds. -*static* -# Nanvix OS interface module (snapshot, host-mount). -_nanvix _nanvixmodule.c -# lxml C extension modules (statically linked via pre-built archives). -_lxml_etree lxml_etree_builtin.c -L{sysroot}/lib -llxml_etree -lxslt -lexslt -lxml2 -lz -_lxml_elementpath lxml_elementpath_builtin.c -L{sysroot}/lib -llxml_elementpath -lxml2 -lz - -# Phase 0 of the .a -> .so migration: array as a proof-of-concept shared module. -# Listed before Setup.stdlib's static declaration so makesetup's first rule wins. -*shared* -array arraymodule.c -""" +import setup_local as setup_local_mod def generate_setup_local(repo_root: Path, buildroot: Path) -> None: - """Generate Modules/Setup.local with statically-linked module definitions. - - Includes both the _nanvix OS interface module and lxml C extensions. - """ + """Generate Modules/Setup.local for the native-host build path.""" setup_local = repo_root / "Modules" / "Setup.local" - content = _SETUP_LOCAL_TEMPLATE.format(sysroot=buildroot) + content = setup_local_mod.render_setup_local( + buildroot=str(buildroot), + header_comment="Auto-generated by .nanvix/lxml.py -- do not edit manually.", + ) setup_local.write_text(content, encoding="utf-8") print(f"[lxml] Generated {setup_local}") diff --git a/.nanvix/setup_local.py b/.nanvix/setup_local.py new file mode 100644 index 000000000000000..96b7a89c4d70cc9 --- /dev/null +++ b/.nanvix/setup_local.py @@ -0,0 +1,257 @@ +# Copyright(c) The Maintainers of Nanvix. +# Licensed under the MIT License. + +"""Single source of truth for Modules/Setup.local on Nanvix builds. + +The host build path in ``lxml.generate_setup_local()`` and the Docker build +path in ``_docker._generate_setup_local_cmd()`` both render this table. + +Module ordering matters: makesetup applies "first rule wins" semantics when +the same module is declared both here and in ``Modules/Setup.stdlib``. +Declaring a shared duplicate before the upstream static default switches that +module to a runtime-loaded extension. +""" + +from __future__ import annotations + +from enum import Enum +from typing import Iterable, NamedTuple, Sequence + + +class Linkage(Enum): + STATIC = "*static*" + SHARED = "*shared*" + + +class SetupEntry(NamedTuple): + """One Modules/Setup.local entry with optional surrounding comments.""" + + name: str + linkage: Linkage + tokens: Sequence[str] = () + comment: str = "" + section_header: str = "" + + +SETUP_LOCAL_ENTRIES: tuple[SetupEntry, ...] = ( + SetupEntry( + name="_nanvix", + linkage=Linkage.STATIC, + tokens=("_nanvixmodule.c",), + comment="Nanvix OS interface module (snapshot, host-mount).", + ), + SetupEntry( + name="_lxml_etree", + linkage=Linkage.STATIC, + tokens=( + "lxml_etree_builtin.c", + "-L{buildroot}/lib", + "-llxml_etree", + "-lxslt", + "-lexslt", + "-lxml2", + "-lz", + ), + comment="lxml C extensions linked with their pre-built archives.", + ), + SetupEntry( + name="_lxml_elementpath", + linkage=Linkage.STATIC, + tokens=( + "lxml_elementpath_builtin.c", + "-L{buildroot}/lib", + "-llxml_elementpath", + "-lxml2", + "-lz", + ), + ), + SetupEntry( + name="array", + linkage=Linkage.SHARED, + tokens=("arraymodule.c",), + section_header=( + "`array` is the proof-of-concept shared module. It appears before " + "Setup.stdlib so makesetup selects this shared definition." + ), + ), + *( + SetupEntry( + name=name, + linkage=Linkage.SHARED, + tokens=(source,), + section_header=header if index == 0 else "", + ) + for index, (name, source) in enumerate( + ( + ("_bisect", "_bisectmodule.c"), + ("_heapq", "_heapqmodule.c"), + ("_struct", "_struct.c"), + ("_random", "_randommodule.c"), + ("_opcode", "_opcode.c"), + ("_queue", "_queuemodule.c"), + ("_csv", "_csv.c"), + ("binascii", "binascii.c"), + ("_json", "_json.c"), + ("_pickle", "_pickle.c"), + ("_zoneinfo", "_zoneinfo.c"), + ) + ) + for header in ("Data primitives without external dependencies.",) + ), + *( + SetupEntry( + name=name, + linkage=Linkage.SHARED, + tokens=(source,), + section_header=header if index == 0 else "", + ) + for index, (name, source) in enumerate( + ( + ("math", "mathmodule.c"), + ("cmath", "cmathmodule.c"), + ("_statistics", "_statisticsmodule.c"), + ("mmap", "mmapmodule.c"), + ("_contextvars", "_contextvarsmodule.c"), + ) + ) + for header in ("Math and memory modules using the SDK shared runtime.",) + ), + *( + SetupEntry( + name=name, + linkage=Linkage.SHARED, + tokens=(source,), + section_header=header if index == 0 else "", + ) + for index, (name, source) in enumerate( + ( + ("unicodedata", "unicodedata.c"), + ("_codecs_cn", "cjkcodecs/_codecs_cn.c"), + ("_codecs_hk", "cjkcodecs/_codecs_hk.c"), + ("_codecs_iso2022", "cjkcodecs/_codecs_iso2022.c"), + ("_codecs_jp", "cjkcodecs/_codecs_jp.c"), + ("_codecs_kr", "cjkcodecs/_codecs_kr.c"), + ("_codecs_tw", "cjkcodecs/_codecs_tw.c"), + ) + ) + for header in ("Text codecs without external dependencies.",) + ), + # Keep _datetime, pyexpat, and _multibytecodec static: migrated consumers + # import their C APIs while a shared object initializes, which Nanvix cannot + # nest safely yet. + SetupEntry( + name="_asyncio", + linkage=Linkage.SHARED, + tokens=("_asynciomodule.c",), + section_header=( + "Modules with bundled CPython dependencies. Each extension keeps " + "its vendored archive, matching the upstream build." + ), + ), + SetupEntry( + name="_decimal", linkage=Linkage.SHARED, tokens=("_decimal/_decimal.c",) + ), + SetupEntry(name="_elementtree", linkage=Linkage.SHARED, tokens=("_elementtree.c",)), + SetupEntry( + name="_md5", + linkage=Linkage.SHARED, + tokens=( + "md5module.c", + "-I$(srcdir)/Modules/_hacl/include", + "_hacl/Hacl_Hash_MD5.c", + "-D_BSD_SOURCE", + "-D_DEFAULT_SOURCE", + ), + ), + SetupEntry( + name="_sha1", + linkage=Linkage.SHARED, + tokens=( + "sha1module.c", + "-I$(srcdir)/Modules/_hacl/include", + "_hacl/Hacl_Hash_SHA1.c", + "-D_BSD_SOURCE", + "-D_DEFAULT_SOURCE", + ), + ), + SetupEntry( + name="_sha2", + linkage=Linkage.SHARED, + tokens=( + "sha2module.c", + "-I$(srcdir)/Modules/_hacl/include", + "Modules/_hacl/libHacl_Hash_SHA2.a", + ), + ), + SetupEntry( + name="_sha3", + linkage=Linkage.SHARED, + tokens=( + "sha3module.c", + "-I$(srcdir)/Modules/_hacl/include", + "_hacl/Hacl_Hash_SHA3.c", + "-D_BSD_SOURCE", + "-D_DEFAULT_SOURCE", + "$(COMPILER_RT_BUILTINS)", + ), + ), + SetupEntry( + name="_blake2", + linkage=Linkage.SHARED, + tokens=( + "_blake2/blake2module.c", + "_blake2/blake2b_impl.c", + "_blake2/blake2s_impl.c", + ), + ), + SetupEntry(name="select", linkage=Linkage.SHARED, tokens=("selectmodule.c",)), + SetupEntry(name="_socket", linkage=Linkage.SHARED, tokens=("socketmodule.c",)), + SetupEntry( + name="_posixsubprocess", linkage=Linkage.SHARED, tokens=("_posixsubprocess.c",) + ), + SetupEntry(name="fcntl", linkage=Linkage.SHARED, tokens=("fcntlmodule.c",)), + SetupEntry(name="termios", linkage=Linkage.SHARED, tokens=("termios.c", "-lc")), +) + + +def _wrap_comment(text: str, *, width: int = 78) -> list[str]: + """Return wrapped Setup.local comment lines.""" + if not text: + return [] + + import textwrap + + return [f"# {line}" for line in textwrap.wrap(text, width=width - 2)] + + +def render_setup_local( + entries: Iterable[SetupEntry] = SETUP_LOCAL_ENTRIES, + *, + buildroot: str, + header_comment: str, +) -> str: + """Render entries to a complete Modules/Setup.local file.""" + lines = [f"# {header_comment}", ""] + current_linkage: Linkage | None = None + + for entry in entries: + if entry.linkage is not current_linkage: + if current_linkage is not None: + lines.append("") + if entry.section_header: + lines.extend(_wrap_comment(entry.section_header)) + elif current_linkage is None and entry.linkage is Linkage.STATIC: + lines.append("# Statically-linked extension modules for Nanvix builds.") + lines.append(entry.linkage.value) + current_linkage = entry.linkage + elif entry.section_header: + lines.append("") + lines.extend(_wrap_comment(entry.section_header)) + + if entry.comment: + lines.extend(_wrap_comment(entry.comment)) + tokens = " ".join(entry.tokens).replace("{buildroot}", buildroot) + lines.append(f"{entry.name} {tokens}".rstrip()) + + lines.append("") + return "\n".join(lines) diff --git a/Makefile.nanvix b/Makefile.nanvix index f4928c02191c9ca..7ada017b5599845 100644 --- a/Makefile.nanvix +++ b/Makefile.nanvix @@ -41,6 +41,7 @@ TARGET_CXX := $(SDK_BIN)/clang++ --target=$(NANVIX_TARGET_TRIPLE) TARGET_AR := $(SDK_BIN)/llvm-ar TARGET_RANLIB := $(SDK_BIN)/llvm-ranlib TARGET_STRIP := $(SDK_BIN)/llvm-strip +COMPILER_RT_BUILTINS := $(shell $(TARGET_CC) --print-libgcc-file-name) DEPENDENCY_CPPFLAGS := -I$(NANVIX_BUILDROOT)/include DEPENDENCY_LDFLAGS := -L$(NANVIX_BUILDROOT)/lib @@ -61,8 +62,8 @@ CONFIGURE_ENV = \ CFLAGS_NODIST="-fno-semantic-interposition" \ LDFLAGS="$(DEPENDENCY_LDFLAGS) -Wl,--export-dynamic" \ LINKFORSHARED="-Wl,-pie,--export-dynamic,--no-dynamic-linker,-z,notext,-z,norelro,--hash-style=sysv" \ - LDSHARED="$(TARGET_CC) -shared -Wl,--no-as-needed,-z,now,-rpath,/lib,-Bdynamic -L$(NANVIX_SDK_ROOT)/lib -l:libc.so -l:libm.so -Wl,-Bstatic" \ - LDCXXSHARED="$(TARGET_CXX) -shared -Wl,--no-as-needed,-z,now,-rpath,/lib,-Bdynamic -L$(NANVIX_SDK_ROOT)/lib -l:libc.so -l:libm.so -Wl,-Bstatic" \ + LDSHARED="$(TARGET_CC) -shared -nostdlib -Wl,-z,now,--no-dynamic-linker" \ + LDCXXSHARED="$(TARGET_CXX) -shared -nostdlib -Wl,-z,now,--no-dynamic-linker" \ LIBS="$(DEPENDENCY_LIBS)" \ LIBSQLITE3_LIBS="$(DEPENDENCY_LDFLAGS) -lsqlite3" \ LIBSQLITE3_CFLAGS="$(DEPENDENCY_CPPFLAGS)" \ @@ -135,16 +136,15 @@ $(CONFIGURED_MARKER): Makefile.nanvix touch $@ build: $(CONFIGURED_MARKER) - $(MAKE) -j$$(nproc) all + $(MAKE) -j$$(nproc) COMPILER_RT_BUILTINS="$(COMPILER_RT_BUILTINS)" all @if [ -f python ]; then cp -f python python$(EXE); fi @test -f python$(EXE) $(TARGET_STRIP) --strip-all python$(EXE) install: build - $(MAKE) install DESTDIR="$(DESTDIR)" - mkdir -p "$(DESTDIR)/lib" - cp -f "$(NANVIX_SDK_ROOT)/lib/libc.so" "$(DESTDIR)/lib/libc.so" - cp -f "$(NANVIX_SDK_ROOT)/lib/libm.so" "$(DESTDIR)/lib/libm.so" + $(MAKE) COMPILER_RT_BUILTINS="$(COMPILER_RT_BUILTINS)" install DESTDIR="$(DESTDIR)" + find "$(DESTDIR)$(INSTALL_PREFIX)/lib/python3.12/lib-dynload" \ + -type f -name '*.so' -exec $(TARGET_STRIP) --strip-all {} + clean: find . -path './.nanvix' -prune -o -type d -name '__pycache__' \