From fd2c5089b93c7c3d6f126354dc9f4d91b74c13fc Mon Sep 17 00:00:00 2001 From: Elias Oelschner <62939318+levno-710@users.noreply.github.com> Date: Mon, 15 Jun 2026 20:33:38 +0200 Subject: [PATCH 1/2] Switch to Docker-based test runner with lua5.1, upstream LuaJIT, and Luau --- .dockerignore | 15 ++ .github/workflows/Test.yml | 14 +- .github/workflows/release.yml | 8 +- CONTRIBUTING.md | 23 +++ Dockerfile | 39 ++++ README.md | 8 +- docker-test-runner.lua | 369 ++++++++++++++++++++++++++++++++++ scripts/run-tests.sh | 106 ++++++++++ 8 files changed, 565 insertions(+), 17 deletions(-) create mode 100644 .dockerignore create mode 100644 Dockerfile create mode 100644 docker-test-runner.lua create mode 100755 scripts/run-tests.sh diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..5156ac2f --- /dev/null +++ b/.dockerignore @@ -0,0 +1,15 @@ +node_modules +web/node_modules +web/dist +web/dist-ssr +web/public/docs +web/test-results +web/playwright-report +.vite +.vscode +dist +build +Dockerfile +.dockerignore +.git +.gitignore diff --git a/.github/workflows/Test.yml b/.github/workflows/Test.yml index fc051409..ae23e0cd 100644 --- a/.github/workflows/Test.yml +++ b/.github/workflows/Test.yml @@ -1,5 +1,5 @@ name: Test -on: +on: push: branches: - master @@ -8,14 +8,10 @@ on: - master jobs: - test-linux: + test-docker: runs-on: ubuntu-latest steps: - name: Checkout repo - uses: actions/checkout@master - - name: Install Lua - uses: leafo/gh-actions-lua@master - with: - luaVersion: 5.1 - - name: Run test case - run: lua ./tests.lua --Linux --CI + uses: actions/checkout@v4 + - name: Build and run tests + run: bash scripts/run-tests.sh -b -n 10 --ci diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 5dd7bef6..69ce0eea 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -14,12 +14,8 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - name: Install Lua 5.1 - run: | - sudo apt-get update - sudo apt-get install -y lua5.1 - - name: Run tests - run: lua5.1 ./tests.lua --Linux + - name: Build and run tests + run: bash scripts/run-tests.sh -b -n 10 --ci package: name: Package (${{ matrix.os_name }}) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 3d3ce86c..290ff141 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -10,6 +10,29 @@ Thanks for contributing to Prometheus. - Add or update tests when changing behavior. - Document user-visible changes clearly. +## Running Tests + +Tests run inside a Docker container with lua5.1, upstream LuaJIT (built from source), and Luau: + +```bash +./scripts/run-tests.sh # Run all tests (default: 10 iterations) +./scripts/run-tests.sh -b # Build image (needed first time or after Dockerfile changes) +./scripts/run-tests.sh -n 5 # Run with 5 iterations +./scripts/run-tests.sh -c config.lua # Use a custom config +./scripts/run-tests.sh -v # Verbose output +``` + +### Test File Metadata + +Test files can include metadata comments at the top of the file: + +| Annotation | Effect | +|-----------|--------| +| `-- @skip` | Skip this test entirely | +| `-- @luau-only` | Only run with Luau | +| `-- @runtime lua51 luajit` | Only run with specified runtimes | +| `-- @skip-preset Weak` | Skip a specific preset for this test | + ## Reporting Bugs When opening a bug report, include: diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..7f1c0455 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,39 @@ +FROM ubuntu:24.04 + +RUN apt-get update && apt-get install -y \ + lua5.1 \ + git \ + build-essential \ + cmake \ + ninja-build \ + unzip \ + curl \ + && rm -rf /var/lib/apt/lists/* + +# Build upstream LuaJIT from source (apt version has fatal bugs incompatible with Prometheus) +RUN git clone https://luajit.org/git/luajit.git /opt/luajit-src \ + && cd /opt/luajit-src \ + && make -j$(nproc) \ + && make install \ + && ldconfig \ + && rm -rf /opt/luajit-src + +# Build Luau from source +RUN git clone https://github.com/luau-lang/luau.git /opt/luau-src \ + && cd /opt/luau-src \ + && mkdir build && cd build \ + && cmake .. -GNinja -DCMAKE_BUILD_TYPE=Release \ + && ninja \ + && cp luau /usr/local/bin/luau \ + && cp luau-analyze /usr/local/bin/luau-analyze 2>/dev/null || true \ + && cp luau-compile /usr/local/bin/luau-compile 2>/dev/null || true \ + && cp luau-reduce /usr/local/bin/luau-reduce 2>/dev/null || true \ + && rm -rf /opt/luau-src + +# Verify installations +RUN lua5.1 -v && luajit -v && printf 'print("luau OK")\n' > /tmp/luau_check.lua && luau /tmp/luau_check.lua && rm /tmp/luau_check.lua + +WORKDIR /app +COPY . /app + +ENTRYPOINT ["lua5.1", "docker-test-runner.lua"] diff --git a/README.md b/README.md index 6b897e19..3ecd7a6f 100644 --- a/README.md +++ b/README.md @@ -138,10 +138,14 @@ For more advanced use cases, configuration, and presets, see the [documentation] ## Tests -To run the Prometheus test suite: +The test suite runs inside Docker with lua5.1, upstream LuaJIT (built from source), and Luau: ```bash -lua ./tests.lua [--Linux] +./scripts/run-tests.sh # Run all tests (default: 10 iterations) +./scripts/run-tests.sh -b # Build image and run tests +./scripts/run-tests.sh -n 5 # Run with 5 iterations +./scripts/run-tests.sh -c config.lua # Use a custom config +./scripts/run-tests.sh -v # Verbose output ``` --- diff --git a/docker-test-runner.lua b/docker-test-runner.lua new file mode 100644 index 00000000..460417ae --- /dev/null +++ b/docker-test-runner.lua @@ -0,0 +1,369 @@ +-- docker-test-runner.lua +-- Docker-based test orchestrator for Prometheus +-- Runs each test script against all Lua runtimes and presets, comparing output + +local Prometheus = require("src.prometheus") +local Presets = require("src.presets") + +-- === Argument Parsing === +local args = {} +for _, a in ipairs(arg) do + local key, val = a:match("^%-%-([%w_]+)=(.+)$") + if key then + args[key] = val + elseif a:match("^%-%-([%w_]+)$") then + args[a:match("^%-%-(.+)$")] = true + end +end + +local ITERATIONS = math.max(tonumber(args.iterations) or 10, 1) +local CUSTOM_CONFIG = args.config +local CI_MODE = args.CI == true or args.ci == true +local TEST_DIR = "./tests/" +local PASS_RUNNERS = (args["pass-runners"] ~= nil) +local VERBOSE = args.verbose == true + +-- === Runtimes === +local RUNTIMES = { + { + name = "lua5.1", + cmd = "lua5.1", + }, + { + name = "luajit", + cmd = "luajit", + }, + { + name = "luau", + cmd = "luau", + }, +} + +-- === Helpers === +local function shell_escape(s) + return "'" .. s:gsub("'", "'\\''") .. "'" +end + +local function exec(cmd) + local handle = io.popen(cmd .. " 2>&1") + if not handle then + return nil, "failed to execute: " .. cmd + end + local output = handle:read("*a") + local ok = handle:close() + -- In Lua 5.1, popen:close() returns true for zero exit, nil otherwise + -- In LuaJIT/Lua 5.2+, returns true/nil with additional status info + if ok then + return output, 0 + else + return output, 1 + end +end + +local function trim(s) + return (s:gsub("^%s+", ""):gsub("%s+$", "")) +end + +local function normalize_output(s) + s = s:gsub("\r\n", "\n"):gsub("\r", "\n") + local lines = {} + for line in s:gmatch("[^\n]*") do + table.insert(lines, trim(line)) + end + while #lines > 0 and lines[#lines] == "" do + table.remove(lines) + end + return table.concat(lines, "\n") +end + +local colors = { + red = "\27[31m", + green = "\27[32m", + yellow = "\27[33m", + magenta = "\27[35m", + cyan = "\27[36m", + reset = "\27[0m", + bold = "\27[1m", +} +local function fmt(col, text) + return (colors[col] or "") .. text .. colors.reset +end + +-- === Metadata parsing === +-- Test files can have metadata comments at the top: +-- -- @skip (skip this test entirely) +-- -- @luau-only (only run with luau) +-- -- @runtime lua51 luajit (only run with specified runtimes) +-- -- @skip-preset Weak (skip a specific preset) +local function parse_metadata(code) + local meta = { + runtimes = {}, + skip_presets = {}, + skip = false, + } + for line in code:gmatch("[^\n]*") do + if not line:match("^%s*%-%-%s*@") then + break + end + local key, val = line:match("^%s*%-%-%s*@(%w+)%s+(.+)") + if key then + if key == "runtime" then + for r in val:gmatch("%S+") do + meta.runtimes[r:lower()] = true + end + elseif key == "skip_preset" or key == "skip-preset" then + meta.skip_presets[val] = true + elseif key == "luau_only" or key == "luau-only" then + meta.runtimes["luau"] = true + end + end + if line:match("^%s*%-%-%s*@skip%s*$") then + meta.skip = true + end + if line:match("^%s*%-%-%s*@luau%-only%s*$") then + meta.runtimes["luau"] = true + end + end + return meta +end + +-- === Baseline capture === +local function run_script(runtime, script_path) + local out, status = exec(runtime.cmd .. " " .. shell_escape(script_path)) + if status ~= 0 then + return nil, "exit code " .. tostring(status) .. ": " .. (out or "") + end + return normalize_output(out), nil +end + +-- === List test files === +local function scandir(dir) + local handle = io.popen("ls -1 " .. shell_escape(dir)) + if not handle then + error("Failed to list directory: " .. dir) + end + local files = {} + for name in handle:lines() do + if name:match("%.lua$") then + table.insert(files, name) + end + end + handle:close() + table.sort(files) + return files +end + +-- === Shallow copy === +local function shallowcopy(orig) + if type(orig) ~= "table" then + return orig + end + local copy = {} + for k, v in pairs(orig) do + copy[k] = v + end + return copy +end + +-- === Test a single runfile against a single test file === +-- Returns true if passed, false if failed +local function test_runfile(filename, code, meta, runfile, active_runtimes, baselines) + local runfile_name = runfile._name + + -- Check for skip-preset + if meta.skip_presets[runfile_name] then + print(" " .. fmt("yellow", "[SKIP]") .. " preset " .. runfile_name) + return true + end + + for iter = 1, ITERATIONS do + -- Remove AntiTamper step before testing + local steps = {} + for _, step in ipairs(runfile.Steps) do + if step.Name ~= "AntiTamper" then + table.insert(steps, shallowcopy(step)) + end + end + local cfg = shallowcopy(runfile) + cfg.Steps = steps + + local pipeline = Prometheus.Pipeline:fromConfig(cfg) + pipeline:setNameGenerator("MangledShuffled") + + local ok, obfuscated = pcall(pipeline.apply, pipeline, code) + if not ok then + local err_msg = "obfuscation error: " .. tostring(obfuscated) + print(" " .. fmt("red", "[FAIL] ") .. runfile_name .. " #" .. iter .. " - " .. fmt("red", err_msg)) + if type(obfuscated) == "string" and #obfuscated > 200 then + print(" " .. obfuscated:sub(1, 200) .. "...") + end + return false + end + + -- Write obfuscated code to a temp file + local tmpfile = "/tmp/prometheus_test_obfuscated.lua" + local tfh = io.open(tmpfile, "w") + if not tfh then + print(" " .. fmt("red", "[FAIL] ") .. runfile_name .. " #" .. iter .. " - cannot write temp file") + return false + end + tfh:write(obfuscated) + tfh:close() + + -- Run against each active runtime + for _, rt in ipairs(active_runtimes) do + local out, err = run_script(rt, tmpfile) + if err then + print(" " .. fmt("red", "[FAIL] ") .. runfile_name .. " #" .. iter .. " on " .. rt.name .. fmt("red", " - error")) + print(" " .. fmt("yellow", "[ERR] ") .. err:sub(1, 300)) + print(" " .. fmt("yellow", "[SRC] ") .. obfuscated:gsub("\n", "\\n"):sub(1, 300)) + os.remove(tmpfile) + return false + end + + local expected = baselines[rt.name] + if normalize_output(out) ~= normalize_output(expected) then + print(" " .. fmt("red", "[FAIL] ") .. runfile_name .. " #" .. iter .. " on " .. rt.name .. " - output mismatch") + local max_show = 300 + print(" " .. fmt("yellow", "[EXPECTED] ") .. expected:sub(1, max_show)) + print(" " .. fmt("yellow", "[GOT] ") .. out:sub(1, max_show)) + os.remove(tmpfile) + return false + end + end + + os.remove(tmpfile) + end + + local runtime_list = {} + for _, rt in ipairs(active_runtimes) do + table.insert(runtime_list, rt.name) + end + local n_runtimes = #active_runtimes + print(string.format(" " .. fmt("green", "[PASS]") .. " %s (%d iterations on %s)", runfile_name, ITERATIONS, table.concat(runtime_list, ", "))) + return true +end + +-- === Main === + +-- Determine which presets/runfiles to use +local runfiles = {} +if CUSTOM_CONFIG then + local cfg, err = loadfile(CUSTOM_CONFIG) + if not cfg then + error("Failed to load custom config: " .. tostring(err)) + end + local loaded = cfg() + if type(loaded[1]) == "table" then + for i, c in ipairs(loaded) do + c._name = c.Name or ("custom-" .. i) + table.insert(runfiles, c) + end + else + loaded._name = loaded.Name or "custom" + table.insert(runfiles, loaded) + end +else + for name, preset in pairs(Presets) do + local copy = shallowcopy(preset) + copy._name = name + table.insert(runfiles, copy) + end + table.sort(runfiles, function(a, b) return a._name < b._name end) +end + +-- Suppress Prometheus logger output during tests +Prometheus.Logger.logLevel = Prometheus.Logger.LogLevel.Error + +print(fmt("bold", "=== Prometheus Docker Test Runner ===")) +print(string.format("Test files: %s", TEST_DIR)) +print(string.format("Iterations: %d", ITERATIONS)) +print(string.format("Preset(s): %s", CUSTOM_CONFIG and "custom (" .. CUSTOM_CONFIG .. ")" or "all built-in")) +print(string.format("CI mode: %s", CI_MODE and "yes" or "no")) +print("") + +local test_files = scandir(TEST_DIR) +local total_passed = 0 +local total_failed = 0 + +for _, filename in ipairs(test_files) do + local filepath = TEST_DIR .. filename + local fh = io.open(filepath, "r") + if not fh then + print(fmt("red", "[ERROR] ") .. "cannot open " .. filepath) + total_failed = total_failed + 1 + else + local code = fh:read("*a") + fh:close() + + local meta = parse_metadata(code) + if meta.skip then + print(fmt("yellow", "[SKIP] ") .. filename .. " (marked @skip)") + else + -- Determine active runtimes + local active_runtimes = {} + local has_runtime_filter = next(meta.runtimes) ~= nil + for _, rt in ipairs(RUNTIMES) do + if has_runtime_filter then + if meta.runtimes[rt.name] then + table.insert(active_runtimes, rt) + end + else + table.insert(active_runtimes, rt) + end + end + + if #active_runtimes == 0 then + print(fmt("yellow", "[SKIP] ") .. filename .. " (no matching runtimes)") + else + -- Get baselines + local baselines = {} + local baseline_ok = true + for _, rt in ipairs(active_runtimes) do + local out, err = run_script(rt, filepath) + if err then + print(fmt("red", "[BASELINE FAIL] ") .. filename .. " on " .. rt.name .. ": " .. err) + baseline_ok = false + break + end + if PASS_RUNNERS then + print(string.format(" " .. fmt("green", "[BASELINE] ") .. "%s on %s PASSED", filename, rt.name)) + end + baselines[rt.name] = out + end + + if not baseline_ok then + total_failed = total_failed + 1 + else + print(fmt("magenta", "[TEST] ") .. filename) + + local file_failed = false + for _, runfile in ipairs(runfiles) do + if not test_runfile(filename, code, meta, runfile, active_runtimes, baselines) then + file_failed = true + break + end + end + + if file_failed then + total_failed = total_failed + 1 + else + total_passed = total_passed + 1 + end + end + end + end + end +end + +print("") +local total = total_passed + total_failed +print(string.format("Total: %d passed, %d failed, %d total", total_passed, total_failed, total)) + +if total_failed == 0 then + print(fmt("green", fmt("bold", "=== ALL TESTS PASSED ==="))) + os.exit(0) +else + print(fmt("red", fmt("bold", string.format("=== %d TEST(S) FAILED ===", total_failed)))) + os.exit(1) +end diff --git a/scripts/run-tests.sh b/scripts/run-tests.sh new file mode 100755 index 00000000..da4e1e43 --- /dev/null +++ b/scripts/run-tests.sh @@ -0,0 +1,106 @@ +#!/bin/bash +# scripts/run-tests.sh +# Convenience script to build and run Prometheus tests in Docker +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_DIR="$(dirname "$SCRIPT_DIR")" + +IMAGE_NAME="${IMAGE_NAME:-prometheus-tests}" +N="${N:-10}" +DOCKER_EXTRA_ARGS="" + +usage() { + cat </dev/null; then + echo "=== Building Docker image '$IMAGE_NAME' ===" + docker build $DOCKER_EXTRA_ARGS -t "$IMAGE_NAME" "$PROJECT_DIR" + echo "" +fi + +# Prepare config mount if custom config is specified +CONFIG_MOUNT="" +RUNNER_ARGS="--iterations=$N $CI_FLAG $VERBOSE_FLAG $PASS_RUNNERS_FLAG" +if [[ -n "$CUSTOM_CONFIG" ]]; then + CONFIG_ABS="$(cd "$(dirname "$CUSTOM_CONFIG")" && pwd)/$(basename "$CUSTOM_CONFIG")" + CONFIG_CONTAINER="/tmp/prometheus_custom_config.lua" + CONFIG_MOUNT="-v $CONFIG_ABS:$CONFIG_CONTAINER:ro" + RUNNER_ARGS="$RUNNER_ARGS --config=$CONFIG_CONTAINER" +fi + +echo "=== Running Prometheus Tests ===" +echo "Iterations: $N" +[[ -n "$CUSTOM_CONFIG" ]] && echo "Custom config: $CUSTOM_CONFIG" + +docker run --rm \ + $CONFIG_MOUNT \ + "$IMAGE_NAME" \ + $RUNNER_ARGS From 51baddd4fe92f72d0c4e09dcab96a0a60cbf30b9 Mon Sep 17 00:00:00 2001 From: Elias Oelschner <62939318+levno-710@users.noreply.github.com> Date: Mon, 15 Jun 2026 20:38:45 +0200 Subject: [PATCH 2/2] Remove luajit, use pre-built Luau binary instead of building from source --- CONTRIBUTING.md | 2 +- Dockerfile | 30 +++++++----------------------- README.md | 2 +- docker-test-runner.lua | 4 ---- scripts/run-tests.sh | 4 ++-- 5 files changed, 11 insertions(+), 31 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 290ff141..c294b845 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -12,7 +12,7 @@ Thanks for contributing to Prometheus. ## Running Tests -Tests run inside a Docker container with lua5.1, upstream LuaJIT (built from source), and Luau: +Tests run inside a Docker container with lua5.1 and Luau: ```bash ./scripts/run-tests.sh # Run all tests (default: 10 iterations) diff --git a/Dockerfile b/Dockerfile index 7f1c0455..03b02e7f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,35 +3,19 @@ FROM ubuntu:24.04 RUN apt-get update && apt-get install -y \ lua5.1 \ git \ - build-essential \ - cmake \ - ninja-build \ unzip \ curl \ && rm -rf /var/lib/apt/lists/* -# Build upstream LuaJIT from source (apt version has fatal bugs incompatible with Prometheus) -RUN git clone https://luajit.org/git/luajit.git /opt/luajit-src \ - && cd /opt/luajit-src \ - && make -j$(nproc) \ - && make install \ - && ldconfig \ - && rm -rf /opt/luajit-src - -# Build Luau from source -RUN git clone https://github.com/luau-lang/luau.git /opt/luau-src \ - && cd /opt/luau-src \ - && mkdir build && cd build \ - && cmake .. -GNinja -DCMAKE_BUILD_TYPE=Release \ - && ninja \ - && cp luau /usr/local/bin/luau \ - && cp luau-analyze /usr/local/bin/luau-analyze 2>/dev/null || true \ - && cp luau-compile /usr/local/bin/luau-compile 2>/dev/null || true \ - && cp luau-reduce /usr/local/bin/luau-reduce 2>/dev/null || true \ - && rm -rf /opt/luau-src +# Download pre-built Luau binary +RUN curl -sSLo /tmp/luau-ubuntu.zip \ + https://github.com/luau-lang/luau/releases/latest/download/luau-ubuntu.zip \ + && unzip -j /tmp/luau-ubuntu.zip luau -d /usr/local/bin \ + && chmod +x /usr/local/bin/luau \ + && rm /tmp/luau-ubuntu.zip # Verify installations -RUN lua5.1 -v && luajit -v && printf 'print("luau OK")\n' > /tmp/luau_check.lua && luau /tmp/luau_check.lua && rm /tmp/luau_check.lua +RUN lua5.1 -v && printf 'print("luau OK")\n' > /tmp/luau_check.lua && luau /tmp/luau_check.lua && rm /tmp/luau_check.lua WORKDIR /app COPY . /app diff --git a/README.md b/README.md index 3ecd7a6f..ef778801 100644 --- a/README.md +++ b/README.md @@ -138,7 +138,7 @@ For more advanced use cases, configuration, and presets, see the [documentation] ## Tests -The test suite runs inside Docker with lua5.1, upstream LuaJIT (built from source), and Luau: +The test suite runs inside Docker with lua5.1 and Luau: ```bash ./scripts/run-tests.sh # Run all tests (default: 10 iterations) diff --git a/docker-test-runner.lua b/docker-test-runner.lua index 460417ae..eb7ddaab 100644 --- a/docker-test-runner.lua +++ b/docker-test-runner.lua @@ -29,10 +29,6 @@ local RUNTIMES = { name = "lua5.1", cmd = "lua5.1", }, - { - name = "luajit", - cmd = "luajit", - }, { name = "luau", cmd = "luau", diff --git a/scripts/run-tests.sh b/scripts/run-tests.sh index da4e1e43..d22e95b1 100755 --- a/scripts/run-tests.sh +++ b/scripts/run-tests.sh @@ -14,7 +14,7 @@ usage() { cat <