[nanvix] E: Remove lxml built-in shim from cpython

.nanvix/NANVIX.md

@@ -28,9 +28,6 @@ This document describes the port of [CPython](https://www.python.org/) interpret
 - OpenSSL 3.5.0
 - SQLite 3.49.0
 - libffi 3.4.6
-- libxml2 2.12.9
-- libxslt 1.1.42
-- lxml 5.3.0
 - xz 5.2.5
 
 ---
@@ -233,7 +230,7 @@ cd .nanvix/out/test && \
 
 ### Test Suite Status
 
-The `./z test` target runs **160 CPython stdlib test modules** on Nanvix
+The `./z test` target runs **159 CPython stdlib test modules** on Nanvix
 (i686, microvm, 256 MB RAM). Tests are split into batches of 4 modules
 per VM invocation to stay within per-process memory limits.
 
@@ -244,7 +241,7 @@ runner (`python -m test`). A `/tmp` directory is created on the ramfs so
 
 | Metric | Value |
 |--------|-------|
-| **Modules enabled** | 160 |
+| **Modules enabled** | 159 |
 | **Tests passed** | All supported tests |
 | **Tests skipped** | Unsupported Nanvix facilities use explicit skip reasons |
 | **Tests failed** | 0 |

.nanvix/_docker.py

@@ -21,6 +21,7 @@
     import build as build_mod
 
 import config
+import setup_local as setup_local_mod
 
 
 def _workspace_id(workspace: Path) -> str:
@@ -228,21 +229,17 @@ def docker_build(
 
 
 def _generate_setup_local_cmd() -> str:
-    """Shell command to generate Modules/Setup.local inside the container."""
+    """Generate Modules/Setup.local inside the SDK container."""
     buildroot = config.DOCKER_BUILDROOT_PATH
     ws = config.DOCKER_WORKSPACE_PATH
-    return (
-        f"printf '%s\\n' "
-        f"'# Auto-generated by .nanvix/docker.py -- do not edit manually.' "
-        f"'# Statically-linked extension modules for Nanvix builds.' "
-        f"'#' "
-        f"'# Nanvix OS interface module (snapshot, host-mount).' "
-        f"'_nanvix _nanvixmodule.c' "
-        f"'# lxml C extension modules (statically linked via pre-built archives).' "
-        f"'_lxml_etree lxml_etree_builtin.c -L{buildroot}/lib -llxml_etree -lxslt -lexslt -lxml2 -lz' "
-        f"'_lxml_elementpath lxml_elementpath_builtin.c -L{buildroot}/lib -llxml_elementpath -lxml2 -lz' "
-        f"> {ws}/Modules/Setup.local"
+    rendered = setup_local_mod.render_setup_local(
+        buildroot=buildroot,
+        header_comment="Auto-generated by .nanvix/_docker.py -- do not edit manually.",
+    )
+    quoted_lines = " ".join(
+        "'" + line.replace("'", "'\"'\"'") + "'" for line in rendered.splitlines()
     )
+    return f"printf '%s\\n' {quoted_lines} > {ws}/Modules/Setup.local"
 
 
 def _copy_outputs_cmd() -> str:

.nanvix/_test.py

@@ -26,9 +26,96 @@
 
 import build as build_mod
 import config
-import lxml as lxml_mod
 import ramfs as ramfs_mod
 
+# ---------------------------------------------------------------------------
+# Shared-extension smoke checks
+# ---------------------------------------------------------------------------
+
+_SO_MODULE_SANITY_CHECKS: tuple[tuple[str, tuple[tuple[str, str], ...]], ...] = (
+    (
+        "CPYTHON_TEST_DATA_PRIMITIVES",
+        (
+            ("_bisect", "m.bisect_left([1, 3, 5], 4) == 2"),
+            ("_heapq", "m.heappush([], 1) is None"),
+            ("_struct", "m.pack('i', 42) == b'\\x2a\\x00\\x00\\x00'"),
+            ("_random", "hasattr(m, 'Random')"),
+            ("_opcode", "hasattr(m, 'stack_effect')"),
+            ("_queue", "hasattr(m, 'SimpleQueue')"),
+            ("_csv", "hasattr(m, 'reader')"),
+            ("binascii", "m.hexlify(b'\\xab') == b'ab'"),
+            ("_json", "hasattr(m, 'encode_basestring_ascii')"),
+            ("_pickle", "hasattr(m, 'Pickler')"),
+            ("_zoneinfo", "hasattr(m, 'ZoneInfo')"),
+        ),
+    ),
+    (
+        "CPYTHON_TEST_MATH",
+        (
+            ("math", "abs(m.sqrt(4.0) - 2.0) < 1e-9"),
+            ("cmath", "abs(m.sqrt(complex(-1)) - complex(0, 1)) < 1e-9"),
+            ("_statistics", "hasattr(m, '_normal_dist_inv_cdf')"),
+            ("mmap", "hasattr(m, 'mmap')"),
+            ("_contextvars", "hasattr(m, 'ContextVar')"),
+        ),
+    ),
+    (
+        "CPYTHON_TEST_CODECS",
+        (
+            ("unicodedata", "m.lookup('LATIN SMALL LETTER A') == 'a'"),
+            ("_codecs_cn", "hasattr(m, 'getcodec')"),
+            ("_codecs_hk", "hasattr(m, 'getcodec')"),
+            ("_codecs_iso2022", "hasattr(m, 'getcodec')"),
+            ("_codecs_jp", "hasattr(m, 'getcodec')"),
+            ("_codecs_kr", "hasattr(m, 'getcodec')"),
+            ("_codecs_tw", "hasattr(m, 'getcodec')"),
+        ),
+    ),
+    (
+        "CPYTHON_TEST_BUNDLED_DEPS",
+        (
+            ("_asyncio", "hasattr(m, 'Future')"),
+            ("_decimal", "m.Decimal('1.1') + m.Decimal('2.2') == m.Decimal('3.3')"),
+            ("_elementtree", "hasattr(m, 'XMLParser')"),
+            ("_md5", "hasattr(m, 'md5')"),
+            ("_sha1", "hasattr(m, 'sha1')"),
+            ("_sha2", "hasattr(m, 'sha256')"),
+            ("_sha3", "hasattr(m, 'sha3_256')"),
+            ("_blake2", "hasattr(m, 'blake2b')"),
+            ("select", "hasattr(m, 'select')"),
+            ("_socket", "hasattr(m, 'socket')"),
+            ("_posixsubprocess", "hasattr(m, 'fork_exec')"),
+            ("fcntl", "hasattr(m, 'fcntl')"),
+            ("termios", "hasattr(m, 'tcgetattr')"),
+        ),
+    ),
+)
+
+
+def _render_so_sanity_snippets(
+    checks: tuple[
+        tuple[str, tuple[tuple[str, str], ...]], ...
+    ] = _SO_MODULE_SANITY_CHECKS,
+) -> str:
+    """Render imports that prove each migrated module loads through dlopen."""
+    snippets: list[str] = []
+    for log_tag, modules in checks:
+        items = ",\n".join(
+            f"    ({name!r}, lambda m: {check})" for name, check in modules
+        )
+        snippets.append(
+            f"_so_checks = [\n{items},\n]\n"
+            "for _name, _check in _so_checks:\n"
+            "    _mod = __import__(_name)\n"
+            "    assert _name not in sys.builtin_module_names, "
+            "f'{_name} still built-in!'\n"
+            "    assert _check(_mod), f'{_name} sanity check failed'\n"
+            f"    print(f'{log_tag}: "
+            "{_name} loaded via dlopen from {_mod.__file__}')\n"
+        )
+    return "".join(snippets)
+
+
 # ---------------------------------------------------------------------------
 # Initrd creation helper (standalone mode)
 # ---------------------------------------------------------------------------
@@ -237,25 +324,34 @@ def stage(args: build_mod.MakeArgs) -> None:
                 shutil.copy2(scdata_src, scdata_dst)
                 print(f"  Copied {scdata_name} from build dir (make install missed it)")
 
-    # Hello-world test script.  The lxml import is exercised against the
-    # in-memory FAT ramfs VFS via xmlInitParser().
-    lxml_snippet = (
-        "try:\n"
-        "    import lxml.etree\n"
-        "    doc = lxml.etree.fromstring(b'<root><child>lxml OK</child></root>')\n"
-        "    assert doc.tag == 'root'\n"
-        "    assert doc[0].text == 'lxml OK'\n"
-        "    print('CPYTHON_TEST_LXML: lxml.etree import and parse OK')\n"
-        "except ImportError as e:\n"
-        "    print(f'CPYTHON_TEST_LXML_SKIP: {e}')\n"
-        "except Exception as e:\n"
-        "    print(f'CPYTHON_TEST_LXML_FAIL: {e}')\n"
-        "    sys.exit(1)\n"
+    # Hello-world test script.  The array check proves that the first
+    # stdlib module migrated to a shared extension is loaded through dlopen.
+    array_snippet = (
+        "import array\n"
+        "assert 'array' not in sys.builtin_module_names, 'array still built-in!'\n"
+        "_array = array.array('i', [1, 2, 3])\n"
+        "assert _array.tolist() == [1, 2, 3]\n"
+        "print(f'CPYTHON_TEST_ARRAY_SO: array loaded via dlopen from {array.__file__}')\n"
+    )
+    nested_import_snippet = (
+        "import xml.etree.ElementTree as _elementtree_api\n"
+        "assert _elementtree_api.fromstring('<root/>').tag == 'root'\n"
+        "assert '_elementtree' not in sys.builtin_module_names\n"
+        "assert 'pyexpat' in sys.builtin_module_names\n"
+        "import encodings.gb2312\n"
+        "assert '\\u4e2d\\u6587'.encode('gb2312') == b'\\xd6\\xd0\\xce\\xc4'\n"
+        "assert '_codecs_cn' not in sys.builtin_module_names\n"
+        "assert '_multibytecodec' in sys.builtin_module_names\n"
+        "print('CPYTHON_TEST_NESTED_IMPORTS: static C API anchors OK')\n"
     )
+
     (staging / "test_hello.py").write_text(
         "import sys\n"
         "print('CPYTHON_TEST_HELLO: Hello from Python', sys.version_info[:2])\n"
-        "print('CPYTHON_TEST_PLATFORM:', sys.platform)\n" + lxml_snippet
+        "print('CPYTHON_TEST_PLATFORM:', sys.platform)\n"
+        + array_snippet
+        + nested_import_snippet
+        + _render_so_sanity_snippets()
     )
 
     # HTTP server smoke-test script must be present in the sysroot before
@@ -472,26 +568,18 @@ def run_hello(
 
     # Validate output.
     found_hello = False
-    found_lxml = False
     for line in output.splitlines():
         if line.startswith("CPYTHON_TEST_"):
             tag = line.split(":")[0].replace("CPYTHON_TEST_", "")
             print(f"  {tag}: {line.strip()}")
             if tag == "HELLO":
                 found_hello = True
-            elif tag in ("LXML", "LXML_SKIP"):
-                found_lxml = True
 
     if not found_hello:
         print("  FAIL: Hello test did not produce expected output")
         print(output)
         raise RuntimeError("Hello test did not produce expected output")
 
-    if not found_lxml:
-        # lxml staging is best-effort — if the runtime package was not
-        # available (e.g. release asset missing), the test is non-fatal.
-        print("  WARNING: lxml import/parse test did not produce expected output")
-
     print("  PASS")
 
 
@@ -774,8 +862,6 @@ def run_all(
             stage(args)
             stage_ramfs(args)
 
-    lxml_mod.stage_lxml_runtime(staging)
-
     # Hello test.
     run_hello(
         staging,

.nanvix/build.py

@@ -22,7 +22,7 @@
 import _docker as docker_mod
 import _test as test_mod
 import config
-import lxml as lxml_mod
+import setup_local as setup_local_mod
 
 
 @dataclass
@@ -100,7 +100,7 @@ def build(
         buildroot_for_setup = (
             Path(config.DOCKER_BUILDROOT_PATH) if _args.docker else args.buildroot
         )
-        lxml_mod.generate_setup_local(paths.repo_root(), buildroot_for_setup)
+        setup_local_mod.generate_setup_local(paths.repo_root(), buildroot_for_setup)
         _args.run(cwd=paths.repo_root())
         install(dest_dir, args)
 

.nanvix/config.py

@@ -239,8 +239,6 @@ def _manifest_sdk_image() -> str:
     # #323 wave 8 — regex and plistlib
     "test_re",
     "test_plistlib",
-    # #600 — lxml built-in smoke test
-    "test_nanvix_lxml",
     # #526 — _lzma stdlib enablement
     "test_lzma",
     # #327 — network and protocol tests (IPv4 only; IPv6 disabled)
@@ -337,8 +335,8 @@ def _manifest_sdk_image() -> str:
     "lib/pkgconfig",
 ]
 
-# site-packages is no longer trimmed because lxml runtime files may be
-# installed there by downstream packaging.  When the directory is empty
+# site-packages is not trimmed because downstream third-party packages may
+# install runtime files there. When the directory is empty
 # it remains harmlessly on disk (ramfs.trim_sysroot only removes empty
 # bin/).  To force-trim site-packages for minimal images, add the path
 # back into SYSROOT_TRIM_DIRS above.