From 4f4a32865cbb72f3dc0df50534eba89fcefd75b1 Mon Sep 17 00:00:00 2001 From: Ben Copeland Date: Tue, 7 Jul 2026 09:45:30 +0100 Subject: [PATCH 1/2] vm-tests: add ltp test Extract the LTP rootfs referenced by ROOTFS_URL into a chroot and run the requested LTP command files against the booted kernel. Command files come from PULL_LABS_TESTS_JSON (default: smoketest); result logs and a summary CSV are exported through the existing upload paths. Signed-off-by: Ben Copeland --- vm-tests/ltp/README.md | 44 +++++++ vm-tests/ltp/external_requirements.json | 4 + vm-tests/ltp/run.sh | 152 ++++++++++++++++++++++++ 3 files changed, 200 insertions(+) create mode 100644 vm-tests/ltp/README.md create mode 100644 vm-tests/ltp/external_requirements.json create mode 100755 vm-tests/ltp/run.sh diff --git a/vm-tests/ltp/README.md b/vm-tests/ltp/README.md new file mode 100644 index 0000000..6ee3aa8 --- /dev/null +++ b/vm-tests/ltp/README.md @@ -0,0 +1,44 @@ +# ltp + +A vm-test that runs LTP (Linux Test Project) command files from a +KernelCI LTP rootfs inside a chroot on the VM. + +Used by `pull_labs_translate.translate_job` for PULL_LABS jobs whose +`tests[].type` is `ltp` (e.g. the `ltp-smoke-aws-ec2` job in +kernelci-pipeline). The rootfs referenced by `ROOTFS_URL` must carry an +LTP installation in `/opt/ltp`, as the KernelCI Debian `*-ltp` images do. + +## Environment variables + +| Variable | Required | Purpose | +| --- | --- | --- | +| `ROOTFS_URL` | yes | URL to the LTP rootfs tarball (e.g. `full.rootfs.tar.xz`) | +| `PULL_LABS_TESTS_JSON` | no | JSON list of job tests; `parameters` may carry `tst_cmdfiles=` (default: `smoketest`) | +| `KERNEL_URL` | no | URL to the kernel image, only used with `BOOT_HOOK` | +| `MODULES_URL` | with `BOOT_HOOK` | URL to the modules tarball | +| `BOOT_HOOK` | no | Path to an executable that stages the kernel and reboots/kexecs | +| `ARCH` | no | Target architecture string (default: `uname -m`) | +| `KERNELCI_NODE_ID` | no | Job node ID from kernelci-api (recorded in logs) | + +## Output + +- `results_ltp-.log` — LTP result log (one `PASS`/`FAIL`/`CONF` + line per test), uploaded to S3 by `test-vm-client.sh`. +- `results_ltp--output.log` — full LTP console output. +- `results_ltp--failed.log` — failing command lines, if any. +- `benchmark-ltp-.csv` — one `ltp.` bool row per + command file plus `passed`/`failed`/`skipped` counts, in the schema + consumed by `BenchmarkAnalyzer`. + +The script exits non-zero when any command file reports failures, which +marks the VM instance (and hence the job node) as failed. + +## Limitations + +- Without a `BOOT_HOOK`, LTP runs against the currently booted AMI + kernel, not the kernel built by KernelCI (same behaviour as + `url-kernel-boot`). +- The `skipfile`, `workers` and `skip_install` parameters used by the + LAVA/LKFT runner are logged and ignored. +- Per-subtest results are only available in the uploaded logs; the KCIDB + side currently records one row per VM instance. diff --git a/vm-tests/ltp/external_requirements.json b/vm-tests/ltp/external_requirements.json new file mode 100644 index 0000000..a6d8540 --- /dev/null +++ b/vm-tests/ltp/external_requirements.json @@ -0,0 +1,4 @@ +{ + "kernel-rpms/src": false, + "kernel-rpms/binary": false +} diff --git a/vm-tests/ltp/run.sh b/vm-tests/ltp/run.sh new file mode 100755 index 0000000..654c86f --- /dev/null +++ b/vm-tests/ltp/run.sh @@ -0,0 +1,152 @@ +#!/bin/bash + +# SPDX-License-Identifier: Apache-2.0 +# +# Copyright (C) 2026 Linaro Limited +# +# vm-test invoked by pull_labs_poller for PULL_LABS jobs of type "ltp". +# Downloads the LTP rootfs (ROOTFS_URL, a KernelCI Debian image with LTP +# installed in /opt/ltp), extracts it and runs the requested LTP command +# files inside a chroot. Like url-kernel-boot, the kernel under test is +# only booted when a BOOT_HOOK is configured; otherwise LTP runs against +# the currently booted kernel. + +set -euxo pipefail + +: "${ROOTFS_URL:?ROOTFS_URL is required (LTP rootfs, set by pull_labs_translate)}" + +SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +STAGE_DIR="${STAGE_DIR:-/tmp/pullab_ltp}" +CHROOT_DIR="${CHROOT_DIR:-$STAGE_DIR/rootfs}" +mkdir -p "$STAGE_DIR" + +echo "=== ltp ===" +echo "Rootfs URL: $ROOTFS_URL" +echo "Kernel URL: ${KERNEL_URL:-}" +echo "Arch: ${ARCH:-$(uname -m)}" +echo "Node ID: ${KERNELCI_NODE_ID:-unknown}" +echo "Test list: ${PULL_LABS_TESTS_JSON:-${PULL_LABS_TESTS:-}}" + +download() { + local url="$1" + local dest="$2" + echo "Downloading $url -> $dest" + if command -v curl >/dev/null 2>&1; then + curl -fSL --retry 3 --retry-delay 5 -o "$dest" "$url" + elif command -v wget >/dev/null 2>&1; then + wget -q -O "$dest" "$url" + else + echo "ERROR: neither curl nor wget available" >&2 + exit 2 + fi +} + +ROOTFS_TAR="$STAGE_DIR/rootfs.tar" +download "$ROOTFS_URL" "$ROOTFS_TAR" + +# Hook for the deployment-specific boot step, same contract as +# url-kernel-boot: stage the kernel into /boot and kexec/reboot. +if [ -n "${BOOT_HOOK:-}" ] && [ -x "${BOOT_HOOK}" ] && [ -n "${KERNEL_URL:-}" ]; then + KERNEL_BIN="$STAGE_DIR/kernel.bin" + MODULES_TGZ="$STAGE_DIR/modules.tar.gz" + download "$KERNEL_URL" "$KERNEL_BIN" + download "${MODULES_URL:?MODULES_URL is required with BOOT_HOOK}" "$MODULES_TGZ" + echo "Invoking BOOT_HOOK: $BOOT_HOOK" + "$BOOT_HOOK" "$KERNEL_BIN" "$MODULES_TGZ" "" +else + echo "No BOOT_HOOK configured — running LTP against currently booted kernel." +fi + +sudo mkdir -p "$CHROOT_DIR" +sudo tar -xf "$ROOTFS_TAR" -C "$CHROOT_DIR" --numeric-owner + +if ! sudo test -x "$CHROOT_DIR/opt/ltp/runltp"; then + echo "ERROR: /opt/ltp/runltp not found in rootfs — not an LTP rootfs?" >&2 + exit 2 +fi + +cleanup() { + for m in dev/pts dev proc sys; do + sudo umount "$CHROOT_DIR/$m" 2>/dev/null || true + done +} +trap cleanup EXIT + +sudo mount -t proc proc "$CHROOT_DIR/proc" +sudo mount -t sysfs sys "$CHROOT_DIR/sys" +sudo mount --bind /dev "$CHROOT_DIR/dev" +sudo mount --bind /dev/pts "$CHROOT_DIR/dev/pts" +sudo cp -L /etc/resolv.conf "$CHROOT_DIR/etc/resolv.conf" || true + +# Select LTP command files from the job's test list. Parameters that only +# the LAVA/LKFT runner understands (skipfile, workers, skip_install) are +# reported and ignored. +CMDFILES="$(python3 - <<'PYEOF' +import json +import os +import sys + +tests = json.loads(os.environ.get("PULL_LABS_TESTS_JSON") or "[]") +files = [] +for t in tests: + if t.get("type") != "ltp": + continue + params = dict( + kv.split("=", 1) + for kv in (t.get("parameters") or "").split() + if "=" in kv + ) + for name in ("skipfile", "workers", "skip_install"): + if name in params: + print(f"WARNING: ignoring unsupported LTP parameter " + f"{name}={params[name]}", file=sys.stderr) + for f in params.get("tst_cmdfiles", "smoketest").split(","): + if f and f not in files: + files.append(f) +print(" ".join(files) if files else "smoketest") +PYEOF +)" + +RESULT_CSV="${SOURCE_DIR}/benchmark-ltp-$(uname -r).csv" +echo "metric,unit,value,more_is_better,kernel_version,instance_id,instance_type,arch" > "$RESULT_CSV" +CSV_SUFFIX="$(uname -r),${HOSTNAME:-unknown},${INSTANCE_TYPE:-unknown},${ARCH:-$(uname -m)}" + +OVERALL_RC=0 +for cmdfile in $CMDFILES; do + echo "=== Running LTP cmdfile: $cmdfile ===" + rc=0 + sudo chroot "$CHROOT_DIR" /opt/ltp/runltp -f "$cmdfile" -p -q \ + -l "/ltp-$cmdfile.log" -o "/ltp-$cmdfile-output.log" \ + -C "/ltp-$cmdfile-failed.log" || rc=$? + + for suffix in .log -output.log -failed.log; do + if sudo test -f "$CHROOT_DIR/ltp-$cmdfile$suffix"; then + sudo cp "$CHROOT_DIR/ltp-$cmdfile$suffix" \ + "$SOURCE_DIR/results_ltp-$cmdfile$suffix" + sudo chown "$(id -u):$(id -g)" \ + "$SOURCE_DIR/results_ltp-$cmdfile$suffix" + fi + done + + passed=0; failed=0; skipped=0 + if [ -f "$SOURCE_DIR/results_ltp-$cmdfile.log" ]; then + passed=$(grep -c ' PASS ' "$SOURCE_DIR/results_ltp-$cmdfile.log" || true) + failed=$(grep -c ' FAIL ' "$SOURCE_DIR/results_ltp-$cmdfile.log" || true) + skipped=$(grep -c ' CONF ' "$SOURCE_DIR/results_ltp-$cmdfile.log" || true) + fi + echo "LTP $cmdfile: rc=$rc passed=$passed failed=$failed skipped=$skipped" + echo "ltp.$cmdfile,bool,$([ "$rc" -eq 0 ] && echo 1 || echo 0),true,$CSV_SUFFIX" >> "$RESULT_CSV" + echo "ltp.$cmdfile.passed,count,$passed,true,$CSV_SUFFIX" >> "$RESULT_CSV" + echo "ltp.$cmdfile.failed,count,$failed,false,$CSV_SUFFIX" >> "$RESULT_CSV" + echo "ltp.$cmdfile.skipped,count,$skipped,false,$CSV_SUFFIX" >> "$RESULT_CSV" + + if [ "$rc" -ne 0 ]; then + OVERALL_RC=1 + fi +done + +if [ "$OVERALL_RC" -ne 0 ]; then + echo "Test execution completed: FAILURE" + exit 1 +fi +echo "Test execution completed: SUCCESS" From ef314607e1064d734f5316833b9c9082bcaf333d Mon Sep 17 00:00:00 2001 From: Ben Copeland Date: Tue, 7 Jul 2026 09:45:30 +0100 Subject: [PATCH 2/2] translate: route ltp jobs to the ltp vm-test Map PULL_LABS test type ltp to the new ltp vm-test instead of url-kernel-boot, and pass each test's parameters string and timeout_s to the VM as PULL_LABS_TESTS_JSON, which the id:type pairs in PULL_LABS_TESTS cannot carry. Signed-off-by: Ben Copeland --- docs/07-kernelci-kcidb-integration.md | 3 +- .../pull_labs_translate.py | 18 ++++++++- tests/test_pull_labs_translate.py | 37 ++++++++++++++++++- 3 files changed, 54 insertions(+), 4 deletions(-) diff --git a/docs/07-kernelci-kcidb-integration.md b/docs/07-kernelci-kcidb-integration.md index 6c8bd87..7d7af08 100644 --- a/docs/07-kernelci-kcidb-integration.md +++ b/docs/07-kernelci-kcidb-integration.md @@ -126,7 +126,7 @@ It **never** returns `"incomplete"` - that value is reserved for infrastructure | `x86_64` | `c5a.4xlarge` | AL2023 `...al2023-ami-kernel-default-x86_64` | | `arm64`/`aarch64` | `c6g.4xlarge` | AL2023 `...al2023-ami-kernel-default-arm64` | -`DEFAULT_TEST_TYPE_MAP`: `baseline`, `ltp`, `unixbench` all map to `url-kernel-boot`. Unknown types fall back to `url-kernel-boot` via `_resolve_test_dir`, which uses `test_type_map["_default"]` if present, else the literal `"url-kernel-boot"`. +`DEFAULT_TEST_TYPE_MAP`: `ltp` maps to the `ltp` vm-test; `baseline` and `unixbench` map to `url-kernel-boot`. Unknown types fall back to `url-kernel-boot` via `_resolve_test_dir`, which uses `test_type_map["_default"]` if present, else the literal `"url-kernel-boot"`. The `test_params` dict carries: @@ -134,6 +134,7 @@ The `test_params` dict carries: - `ROOTFS_URL` (only if `artifacts.rootfs` or `artifacts.ramdisk` present). - `KERNELCI_NODE_ID` (only if `node_id` was passed). - `PULL_LABS_TESTS` (only if the job has tests) - a comma-joined list of `id:type` pairs. +- `PULL_LABS_TESTS_JSON` (only if the job has tests) - a JSON list of `{id, type, parameters, timeout_s}` objects, carrying the free-form `parameters` string that `PULL_LABS_TESTS` cannot. The job `timeout` defaults to `3600`, is coerced to `int`, and maps to the VM entry's `max_runtime`. Each job becomes exactly one entry in `test_config.vms[*]` (one VM per job). diff --git a/src/kernel_ci_cloud_labs/pull_labs_translate.py b/src/kernel_ci_cloud_labs/pull_labs_translate.py index 2695d21..b25e2b3 100644 --- a/src/kernel_ci_cloud_labs/pull_labs_translate.py +++ b/src/kernel_ci_cloud_labs/pull_labs_translate.py @@ -16,6 +16,7 @@ """ import copy +import json import uuid from typing import Any, Dict, List, Optional @@ -41,7 +42,7 @@ # Extend via test_type_map argument; unknown types fall back to "url-kernel-boot". DEFAULT_TEST_TYPE_MAP: Dict[str, str] = { "baseline": "url-kernel-boot", - "ltp": "url-kernel-boot", + "ltp": "ltp", "unixbench": "url-kernel-boot", } @@ -129,6 +130,21 @@ def translate_job( f"{t.get('id', t.get('type', 'unknown'))}:{t.get('type', 'unknown')}" for t in job_tests ) + # Full test descriptions, including the free-form `parameters` string + # the id:type pairs above cannot carry; PULL_LABS_TESTS is kept for + # compatibility with existing vm-tests. + test_params["PULL_LABS_TESTS_JSON"] = json.dumps( + [ + { + "id": t.get("id", t.get("type", "unknown")), + "type": t.get("type", "unknown"), + "parameters": t.get("parameters", ""), + "timeout_s": t.get("timeout_s"), + } + for t in job_tests + ], + separators=(",", ":"), + ) timeout = jobdef.get("timeout") or 3600 try: diff --git a/tests/test_pull_labs_translate.py b/tests/test_pull_labs_translate.py index 68a33a2..18f3972 100644 --- a/tests/test_pull_labs_translate.py +++ b/tests/test_pull_labs_translate.py @@ -5,6 +5,8 @@ """Unit tests for pull_labs_translate.""" +import json + import pytest from kernel_ci_cloud_labs.pull_labs_translate import ( @@ -73,6 +75,10 @@ def test_unknown_arch_falls_back_to_x86(self): vm = out["test_config"]["vms"][0] assert vm["instance_type"] == DEFAULT_PLATFORM_MAP["x86_64"]["instance_type"] + def test_default_map_routes_ltp_to_own_vm_test(self): + assert DEFAULT_TEST_TYPE_MAP["ltp"] == "ltp" + assert DEFAULT_TEST_TYPE_MAP["baseline"] == "url-kernel-boot" + def test_test_types_map_to_vm_test_dirs(self): jobdef = _job(tests=[ {"id": "ltp-syscalls", "type": "ltp"}, @@ -81,12 +87,39 @@ def test_test_types_map_to_vm_test_dirs(self): ]) out = translate_job(jobdef, BASE_CONFIG, node_id="n1") vm = out["test_config"]["vms"][0] - # All three map (via default fallback) to url-kernel-boot; dedup - assert vm["test"] == ["url-kernel-boot"] + # ltp has its own vm-test; baseline and unknown types fall back to + # url-kernel-boot and are deduplicated + assert vm["test"] == ["ltp", "url-kernel-boot"] assert "PULL_LABS_TESTS" in vm["test_params"] assert "ltp-syscalls:ltp" in vm["test_params"]["PULL_LABS_TESTS"] assert "baseline-x86:baseline" in vm["test_params"]["PULL_LABS_TESTS"] + def test_tests_json_carries_parameters(self): + jobdef = _job(tests=[ + { + "id": "ltp-smoketest", + "type": "ltp", + "timeout_s": 3600, + "parameters": "tst_cmdfiles=smoketest skip_install=true", + }, + {"id": "baseline-x86", "type": "baseline"}, + ]) + out = translate_job(jobdef, BASE_CONFIG, node_id="n1") + tests = json.loads(out["test_config"]["vms"][0]["test_params"]["PULL_LABS_TESTS_JSON"]) + assert tests[0] == { + "id": "ltp-smoketest", + "type": "ltp", + "parameters": "tst_cmdfiles=smoketest skip_install=true", + "timeout_s": 3600, + } + assert tests[1]["id"] == "baseline-x86" + assert tests[1]["parameters"] == "" + assert tests[1]["timeout_s"] is None + + def test_no_tests_json_without_tests(self): + out = translate_job(_job(tests=[]), BASE_CONFIG) + assert "PULL_LABS_TESTS_JSON" not in out["test_config"]["vms"][0]["test_params"] + def test_rootfs_artifact_passed_through(self): jobdef = _job(extra_artifacts={"rootfs": "https://x/rootfs.bin"}) out = translate_job(jobdef, BASE_CONFIG)