[nanvix] E: Build stdlib extensions as .so

.nanvix/_docker.py

@@ -22,6 +22,7 @@
     import build as build_mod
 
 import config
+import setup_local as setup_local_mod
 
 
 def _workspace_id(workspace: Path) -> str:
@@ -232,27 +233,17 @@ def docker_build(
 
 
 def _generate_setup_local_cmd() -> str:
-    """Shell command to generate Modules/Setup.local inside the container."""
+    """Generate Modules/Setup.local inside the SDK container."""
     buildroot = config.DOCKER_BUILDROOT_PATH
     ws = config.DOCKER_WORKSPACE_PATH
-    return (
-        f"printf '%s\\n' "
-        f"'# Auto-generated by .nanvix/docker.py -- do not edit manually.' "
-        f"'' "
-        f"'# Statically-linked extension modules for Nanvix builds.' "
-        f"'*static*' "
-        f"'# Nanvix OS interface module (snapshot, host-mount).' "
-        f"'_nanvix _nanvixmodule.c' "
-        f"'# lxml C extension modules (statically linked via pre-built archives).' "
-        f"'_lxml_etree lxml_etree_builtin.c -L{buildroot}/lib -llxml_etree -lxslt -lexslt -lxml2 -lz' "
-        f"'_lxml_elementpath lxml_elementpath_builtin.c -L{buildroot}/lib -llxml_elementpath -lxml2 -lz' "
-        f"'' "
-        f"'# Phase 0 of the .a -> .so migration: array as a proof-of-concept shared module.' "
-        f"'# Listed before Setup.stdlib static declaration so makesetup first rule wins.' "
-        f"'*shared*' "
-        f"'array arraymodule.c' "
-        f"> {ws}/Modules/Setup.local"
+    rendered = setup_local_mod.render_setup_local(
+        buildroot=buildroot,
+        header_comment="Auto-generated by .nanvix/_docker.py -- do not edit manually.",
+    )
+    quoted_lines = " ".join(
+        "'" + line.replace("'", "'\"'\"'") + "'" for line in rendered.splitlines()
     )
+    return f"printf '%s\\n' {quoted_lines} > {ws}/Modules/Setup.local"
 
 
 def _copy_outputs_cmd() -> str:

.nanvix/_test.py

@@ -31,6 +31,94 @@
 
 _DOWNLOADED_RELEASE_MARKER = ".downloaded-release"
 
+# ---------------------------------------------------------------------------
+# Shared-extension smoke checks
+# ---------------------------------------------------------------------------
+
+_SO_MODULE_SANITY_CHECKS: tuple[tuple[str, tuple[tuple[str, str], ...]], ...] = (
+    (
+        "CPYTHON_TEST_DATA_PRIMITIVES",
+        (
+            ("_bisect", "m.bisect_left([1, 3, 5], 4) == 2"),
+            ("_heapq", "m.heappush([], 1) is None"),
+            ("_struct", "m.pack('i', 42) == b'\\x2a\\x00\\x00\\x00'"),
+            ("_random", "hasattr(m, 'Random')"),
+            ("_opcode", "hasattr(m, 'stack_effect')"),
+            ("_queue", "hasattr(m, 'SimpleQueue')"),
+            ("_csv", "hasattr(m, 'reader')"),
+            ("binascii", "m.hexlify(b'\\xab') == b'ab'"),
+            ("_json", "hasattr(m, 'encode_basestring_ascii')"),
+            ("_pickle", "hasattr(m, 'Pickler')"),
+            ("_zoneinfo", "hasattr(m, 'ZoneInfo')"),
+        ),
+    ),
+    (
+        "CPYTHON_TEST_MATH",
+        (
+            ("math", "abs(m.sqrt(4.0) - 2.0) < 1e-9"),
+            ("cmath", "abs(m.sqrt(complex(-1)) - complex(0, 1)) < 1e-9"),
+            ("_statistics", "hasattr(m, '_normal_dist_inv_cdf')"),
+            ("mmap", "hasattr(m, 'mmap')"),
+            ("_contextvars", "hasattr(m, 'ContextVar')"),
+        ),
+    ),
+    (
+        "CPYTHON_TEST_CODECS",
+        (
+            ("unicodedata", "m.lookup('LATIN SMALL LETTER A') == 'a'"),
+            ("_codecs_cn", "hasattr(m, 'getcodec')"),
+            ("_codecs_hk", "hasattr(m, 'getcodec')"),
+            ("_codecs_iso2022", "hasattr(m, 'getcodec')"),
+            ("_codecs_jp", "hasattr(m, 'getcodec')"),
+            ("_codecs_kr", "hasattr(m, 'getcodec')"),
+            ("_codecs_tw", "hasattr(m, 'getcodec')"),
+        ),
+    ),
+    (
+        "CPYTHON_TEST_BUNDLED_DEPS",
+        (
+            ("_asyncio", "hasattr(m, 'Future')"),
+            ("_decimal", "m.Decimal('1.1') + m.Decimal('2.2') == m.Decimal('3.3')"),
+            ("_elementtree", "hasattr(m, 'XMLParser')"),
+            ("_md5", "hasattr(m, 'md5')"),
+            ("_sha1", "hasattr(m, 'sha1')"),
+            ("_sha2", "hasattr(m, 'sha256')"),
+            ("_sha3", "hasattr(m, 'sha3_256')"),
+            ("_blake2", "hasattr(m, 'blake2b')"),
+            ("select", "hasattr(m, 'select')"),
+            ("_socket", "hasattr(m, 'socket')"),
+            ("_posixsubprocess", "hasattr(m, 'fork_exec')"),
+            ("fcntl", "hasattr(m, 'fcntl')"),
+            ("termios", "hasattr(m, 'tcgetattr')"),
+        ),
+    ),
+)
+
+
+def _render_so_sanity_snippets(
+    checks: tuple[
+        tuple[str, tuple[tuple[str, str], ...]], ...
+    ] = _SO_MODULE_SANITY_CHECKS,
+) -> str:
+    """Render imports that prove each migrated module loads through dlopen."""
+    snippets: list[str] = []
+    for log_tag, modules in checks:
+        items = ",\n".join(
+            f"    ({name!r}, lambda m: {check})" for name, check in modules
+        )
+        snippets.append(
+            f"_so_checks = [\n{items},\n]\n"
+            "for _name, _check in _so_checks:\n"
+            "    _mod = __import__(_name)\n"
+            "    assert _name not in sys.builtin_module_names, "
+            "f'{_name} still built-in!'\n"
+            "    assert _check(_mod), f'{_name} sanity check failed'\n"
+            f"    print(f'{log_tag}: "
+            "{_name} loaded via dlopen from {_mod.__file__}')\n"
+        )
+    return "".join(snippets)
+
+
 # ---------------------------------------------------------------------------
 # Initrd creation helper (standalone mode)
 # ---------------------------------------------------------------------------
@@ -268,6 +356,17 @@ def stage(
             "assert 'array' not in sys.builtin_module_names, 'array still built-in!'\n"
             "print(f'CPYTHON_TEST_ARRAY_SO: array loaded via dlopen from {array.__file__}')\n"
         )
+    nested_import_snippet = (
+        "import xml.etree.ElementTree as _elementtree_api\n"
+        "assert _elementtree_api.fromstring('<root/>').tag == 'root'\n"
+        "assert '_elementtree' not in sys.builtin_module_names\n"
+        "assert 'pyexpat' in sys.builtin_module_names\n"
+        "import encodings.gb2312\n"
+        "assert '\\u4e2d\\u6587'.encode('gb2312') == b'\\xd6\\xd0\\xce\\xc4'\n"
+        "assert '_codecs_cn' not in sys.builtin_module_names\n"
+        "assert '_multibytecodec' in sys.builtin_module_names\n"
+        "print('CPYTHON_TEST_NESTED_IMPORTS: static C API anchors OK')\n"
+    )
 
     # The lxml import is exercised against the in-memory FAT ramfs VFS via
     # xmlInitParser().
@@ -287,7 +386,11 @@ def stage(
     (staging / "test_hello.py").write_text(
         "import sys\n"
         "print('CPYTHON_TEST_HELLO: Hello from Python', sys.version_info[:2])\n"
-        "print('CPYTHON_TEST_PLATFORM:', sys.platform)\n" + array_snippet + lxml_snippet
+        "print('CPYTHON_TEST_PLATFORM:', sys.platform)\n"
+        + array_snippet
+        + nested_import_snippet
+        + _render_so_sanity_snippets()
+        + lxml_snippet
     )
 
     # HTTP server smoke-test script must be present in the sysroot before

.nanvix/lxml.py

@@ -8,32 +8,16 @@
 from nanvix_zutil import paths
 
 import config
-
-_SETUP_LOCAL_TEMPLATE = """\
-# Auto-generated by .nanvix/lxml.py -- do not edit manually.
-
-# Statically-linked extension modules for Nanvix builds.
-*static*
-# Nanvix OS interface module (snapshot, host-mount).
-_nanvix _nanvixmodule.c
-# lxml C extension modules (statically linked via pre-built archives).
-_lxml_etree lxml_etree_builtin.c -L{sysroot}/lib -llxml_etree -lxslt -lexslt -lxml2 -lz
-_lxml_elementpath lxml_elementpath_builtin.c -L{sysroot}/lib -llxml_elementpath -lxml2 -lz
-
-# Phase 0 of the .a -> .so migration: array as a proof-of-concept shared module.
-# Listed before Setup.stdlib's static declaration so makesetup's first rule wins.
-*shared*
-array arraymodule.c
-"""
+import setup_local as setup_local_mod
 
 
 def generate_setup_local(repo_root: Path, buildroot: Path) -> None:
-    """Generate Modules/Setup.local with statically-linked module definitions.
-
-    Includes both the _nanvix OS interface module and lxml C extensions.
-    """
+    """Generate Modules/Setup.local for the native-host build path."""
     setup_local = repo_root / "Modules" / "Setup.local"
-    content = _SETUP_LOCAL_TEMPLATE.format(sysroot=buildroot)
+    content = setup_local_mod.render_setup_local(
+        buildroot=str(buildroot),
+        header_comment="Auto-generated by .nanvix/lxml.py -- do not edit manually.",
+    )
     setup_local.write_text(content, encoding="utf-8")
     print(f"[lxml] Generated {setup_local}")