diff --git a/.github/workflows/update-qpk-pin.yml b/.github/workflows/update-qpk-pin.yml index ac1333f..c0e9e1b 100644 --- a/.github/workflows/update-qpk-pin.yml +++ b/.github/workflows/update-qpk-pin.yml @@ -9,6 +9,7 @@ on: - "constraints.txt" - ".github/workflows/open-downstream-qpk-pin-prs.yml" - ".github/workflows/update-qpk-pin.yml" + - "tests/test_update_qpk_pin_workflow.py" - "docs/**" - "**.md" @@ -31,24 +32,30 @@ jobs: - name: Update QPK_PIN and pin manifests id: update run: | + set -euo pipefail SHA=$(git rev-parse HEAD) echo "$SHA" > QPK_PIN - # Update SHAs from remote repos. Package names use hyphenated form. - for pair in "us-equity-strategies:UsEquityStrategies" \ - "hk-equity-strategies:HkEquityStrategies" \ - "cn-equity-strategies:CnEquityStrategies" \ - "crypto-strategies:CryptoStrategies"; do - pkg="${pair%%:*}" - repo="${pair##*:}" - RSHA=$(git ls-remote "https://github.com/QuantStrategyLab/$repo.git" HEAD | cut -f1) - if [ -n "$RSHA" ]; then - sed -i "s|$pkg @ git+https://github.com/QuantStrategyLab/$repo.git@[a-f0-9]*|$pkg @ git+https://github.com/QuantStrategyLab/$repo.git@$RSHA|" qsl-pins.txt - sed -i "s|$pkg @ git+https://github.com/QuantStrategyLab/$repo.git@[a-f0-9]*|$pkg @ git+https://github.com/QuantStrategyLab/$repo.git@$RSHA|" constraints.txt - fi - done - sed -i "s|quant-platform-kit @ git+https://github.com/QuantStrategyLab/QuantPlatformKit.git@[a-f0-9]*|quant-platform-kit @ git+https://github.com/QuantStrategyLab/QuantPlatformKit.git@$SHA|" qsl-pins.txt - sed -i "s|quant-platform-kit @ git+https://github.com/QuantStrategyLab/QuantPlatformKit.git@[a-f0-9]*|quant-platform-kit @ git+https://github.com/QuantStrategyLab/QuantPlatformKit.git@$SHA|" constraints.txt + QPK_SHA="$SHA" python3 - <<'PY' + import os + import re + from pathlib import Path + + prefix = ( + "quant-platform-kit @ " + "git+https://github.com/QuantStrategyLab/QuantPlatformKit.git@" + ) + pattern = re.compile(f"({re.escape(prefix)})[0-9a-f]{{40}}") + for filename in ("qsl-pins.txt", "constraints.txt"): + path = Path(filename) + updated, count = pattern.subn( + lambda match: f"{match.group(1)}{os.environ['QPK_SHA']}", + path.read_text(encoding="utf-8"), + ) + if count != 1: + raise SystemExit(f"qpk_pin_update_failed:{filename}:matches={count}") + path.write_text(updated, encoding="utf-8") + PY if git diff --quiet; then echo "changed=false" >> "$GITHUB_OUTPUT" @@ -56,59 +63,37 @@ jobs: echo "changed=true" >> "$GITHUB_OUTPUT" fi - - name: Verify downstream compatibility + - name: Verify aggregate dependency closure + id: verify if: steps.update.outputs.changed == 'true' run: | set -euo pipefail - echo "::group::Verifying packages installable with new constraints" - python -m pip install --upgrade pip - # Pre-install runtime deps QPK needs but doesn't declare as dependencies - python -m pip install requests numpy pandas - # Install QPK WITHOUT constraints (avoids self-referencing SHA conflict) - python -m pip install -e . - python -c "import quant_platform_kit; print('QPK import OK')" - python -c "from quant_platform_kit.common.strategies import compute_portfolio_drift; print('strategies OK')" - python -c "from quant_platform_kit.common.platform_runner.loader import load_strategy_definition; print('platform_runner OK')" - python -c "from quant_platform_kit.notifications.telegram import send_telegram_message; print('telegram OK')" - python -c "from quant_platform_kit.common.contracts import SnapshotProfileContract; print('contracts OK')" - - # Verify strategy repo refs are fetchable and package metadata is buildable. - # - # Strategy packages currently carry their own direct QPK pin. A full - # dependency solve with the newly generated top-level QPK constraint - # would conflict until downstream repos update those pins, so keep this - # check focused on the package refs generated above. - failed=0 - pin_file="qsl-pins.txt" - for dep in us-equity-strategies hk-equity-strategies cn-equity-strategies crypto-strategies; do - echo "Checking $dep..." - log_file="$(mktemp)" - if python -m pip install --dry-run --no-deps -c "$pin_file" "$dep" >"${log_file}" 2>&1; then - echo " $dep OK" - else - echo " $dep FAILED" - sed -n '1,160p' "${log_file}" - failed=1 - fi - rm -f "${log_file}" - done - if [ "${failed}" -ne 0 ]; then - echo "One or more downstream dependency checks failed." >&2 + resolver_env="$(mktemp -d)" + resolver_log="$(mktemp)" + cleanup() { + rm -rf -- "$resolver_env" + rm -f -- "$resolver_log" + } + trap cleanup EXIT + + if ! python -m venv "$resolver_env" >"$resolver_log" 2>&1; then + echo "::error title=QPK pin validation::aggregate_dependency_bootstrap_failed" exit 1 fi - echo "::endgroup::" - echo "All compatibility checks passed." - - - name: Restore generated editable-install metadata - if: steps.update.outputs.changed == 'true' - run: | - git restore -- \ - src/quant_platform_kit.egg-info/PKG-INFO \ - src/quant_platform_kit.egg-info/SOURCES.txt + resolver_python="$resolver_env/bin/python" + if ! "$resolver_python" -m pip install -r qsl-pins.txt >"$resolver_log" 2>&1; then + echo "::error title=QPK pin validation::aggregate_dependency_resolution_failed" + exit 1 + fi + if ! "$resolver_python" -m pip check >"$resolver_log" 2>&1; then + echo "::error title=QPK pin validation::aggregate_dependency_check_failed" + exit 1 + fi + echo "aggregate_dependency_resolution_passed" - name: Create PR for pin update id: create_pin_pr - if: steps.update.outputs.changed == 'true' + if: steps.update.outputs.changed == 'true' && steps.verify.outcome == 'success' continue-on-error: true uses: peter-evans/create-pull-request@v7 with: @@ -118,11 +103,10 @@ jobs: body: | Automated update of QPK_PIN and QSL Git SHA pin manifests. - Compatibility checks passed ✅ — all downstream-critical QPK modules verified importable. + Aggregate dependency installation and `pip check` passed. - Updated SHAs: + Updated SHA: - QPK: `${{ github.sha }}` - - Strategy repos: latest HEAD from each 🤖 Generated with [Claude Code](https://claude.com/claude-code) branch: auto/qpk-pin-update @@ -139,7 +123,7 @@ jobs: cat >> "$GITHUB_STEP_SUMMARY" <<'MD' ## QPK pin update PR not created - The generated `QPK_PIN` / `qsl-pins.txt` / `constraints.txt` update was verified, but PR creation failed. + The generated QPK-only pin update passed aggregate dependency verification, but PR creation failed. Ensure `QSL_REPO_SYNC_TOKEN` is configured as a **QuantPlatformKit repository secret** (see `docs/qpk_repo_sync_auth.zh-CN.md`). Org policy blocks `GITHUB_TOKEN` PR creation. diff --git a/tests/test_update_qpk_pin_workflow.py b/tests/test_update_qpk_pin_workflow.py new file mode 100644 index 0000000..7cabe83 --- /dev/null +++ b/tests/test_update_qpk_pin_workflow.py @@ -0,0 +1,243 @@ +from __future__ import annotations + +import os +import stat +import subprocess +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] +WORKFLOW_PATH = ROOT / ".github" / "workflows" / "update-qpk-pin.yml" +OLD_QPK_SHA = "5d4bbd0e7ef9a1434010e8b6a69905d39ee55f1b" +STRATEGY_REFS = { + "us-equity-strategies": ( + "UsEquityStrategies", + "702f9989940187e28102e132887f6216edd4ef66", + ), + "hk-equity-strategies": ( + "HkEquityStrategies", + "1a2155e3a48a212e062f0584f6982f2f2b40d955", + ), + "cn-equity-strategies": ( + "CnEquityStrategies", + "bbff0fcea74231b521c990d5c87d4611ab2d8c53", + ), + "crypto-strategies": ( + "CryptoStrategies", + "2083cc03cf4af075d3518d4d6372be027f3f8eab", + ), +} + + +def _workflow() -> str: + return WORKFLOW_PATH.read_text(encoding="utf-8") + + +def _step_run_block(workflow: str, step_name: str) -> str: + lines = workflow.splitlines() + marker = f" - name: {step_name}" + step_index = lines.index(marker) + run_index = next( + index + for index in range(step_index + 1, len(lines)) + if lines[index] == " run: |" + ) + block: list[str] = [] + for line in lines[run_index + 1 :]: + if line.startswith(" - "): + break + if line and not line.startswith(" "): + break + block.append(line[10:] if line else "") + return "\n".join(block) + "\n" + + +def _verification_run_block(workflow: str) -> str: + desired_name = "Verify aggregate dependency closure" + if f" - name: {desired_name}" in workflow: + return _step_run_block(workflow, desired_name) + return _step_run_block(workflow, "Verify downstream compatibility") + + +def _run_script( + script: str, + *, + cwd: Path, + env: dict[str, str] | None = None, +) -> subprocess.CompletedProcess[str]: + return subprocess.run( + ["bash", "-euo", "pipefail", "-c", script], + cwd=cwd, + env={**os.environ, **(env or {})}, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + + +def _manifest_text(qpk_sha: str) -> str: + lines = [ + "# synthetic aggregate fixture", + ( + "quant-platform-kit @ " + "git+https://github.com/QuantStrategyLab/QuantPlatformKit.git@" + f"{qpk_sha}" + ), + ] + lines.extend( + f"{package} @ git+https://github.com/QuantStrategyLab/{repo}.git@{sha}" + for package, (repo, sha) in STRATEGY_REFS.items() + ) + return "\n".join(lines) + "\n" + + +def _init_pin_fixture(root: Path) -> str: + subprocess.run(["git", "init", "-q"], cwd=root, check=True) + subprocess.run(["git", "config", "user.name", "QPK Test"], cwd=root, check=True) + subprocess.run( + ["git", "config", "user.email", "qpk-test@example.invalid"], + cwd=root, + check=True, + ) + root.joinpath("QPK_PIN").write_text(f"{OLD_QPK_SHA}\n", encoding="utf-8") + manifest = _manifest_text(OLD_QPK_SHA) + root.joinpath("qsl-pins.txt").write_text(manifest, encoding="utf-8") + root.joinpath("constraints.txt").write_text(manifest, encoding="utf-8") + subprocess.run(["git", "add", "."], cwd=root, check=True) + subprocess.run(["git", "commit", "-qm", "fixture"], cwd=root, check=True) + return subprocess.run( + ["git", "rev-parse", "HEAD"], + cwd=root, + check=True, + text=True, + stdout=subprocess.PIPE, + ).stdout.strip() + + +def _install_fake_python(bin_dir: Path) -> None: + bin_dir.mkdir(parents=True) + fake_python = bin_dir / "python" + fake_python.write_text( + """#!/usr/bin/env python3 +import os +import shutil +import sys +from pathlib import Path + +args = sys.argv[1:] +if args[:2] == ["-m", "venv"]: + target = Path(args[2]) / "bin" / "python" + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(Path(__file__), target) + target.chmod(0o755) + raise SystemExit(0) +if args[:3] == ["-m", "pip", "install"]: + if "--no-deps" in args: + raise SystemExit(0) + if "-r" in args and os.environ.get("PIN_RESOLVER_FIXTURE") == "conflict": + sys.stderr.write( + f"credential={os.environ['FIXTURE_SECRET']} " + f"path={os.environ['FIXTURE_PRIVATE_PATH']}\\n" + ) + raise SystemExit(42) + raise SystemExit(0) +if args[:3] == ["-m", "pip", "check"] or args[:1] == ["-c"]: + raise SystemExit(0) +raise SystemExit(f"unexpected fake-python arguments: {args!r}") +""", + encoding="utf-8", + ) + fake_python.chmod(fake_python.stat().st_mode | stat.S_IXUSR) + + +def test_pin_update_changes_only_qpk_refs(tmp_path: Path) -> None: + workflow = _workflow() + update_script = _step_run_block(workflow, "Update QPK_PIN and pin manifests") + + assert "git ls-remote" not in update_script + assert all(repo not in update_script for repo, _sha in STRATEGY_REFS.values()) + + expected_qpk_sha = _init_pin_fixture(tmp_path) + output_path = tmp_path / "github-output" + result = _run_script( + update_script, + cwd=tmp_path, + env={"GITHUB_OUTPUT": str(output_path)}, + ) + + assert result.returncode == 0, result.stderr + assert output_path.read_text(encoding="utf-8") == "changed=true\n" + assert tmp_path.joinpath("QPK_PIN").read_text(encoding="utf-8") == ( + f"{expected_qpk_sha}\n" + ) + expected_manifest = _manifest_text(expected_qpk_sha) + assert tmp_path.joinpath("qsl-pins.txt").read_text(encoding="utf-8") == expected_manifest + assert tmp_path.joinpath("constraints.txt").read_text(encoding="utf-8") == expected_manifest + + +def test_dependency_conflict_fails_closed_before_pr_without_sensitive_output( + tmp_path: Path, +) -> None: + workflow = _workflow() + verify_script = _verification_run_block(workflow) + fake_bin = tmp_path / "bin" + _install_fake_python(fake_bin) + tmp_path.joinpath("qsl-pins.txt").write_text( + _manifest_text(OLD_QPK_SHA), encoding="utf-8" + ) + fixture_secret = "fixture-secret-value" + fixture_private_path = "/private/fixture/path" + + result = _run_script( + verify_script, + cwd=tmp_path, + env={ + "PATH": f"{fake_bin}:{os.environ['PATH']}", + "PIN_RESOLVER_FIXTURE": "conflict", + "FIXTURE_SECRET": fixture_secret, + "FIXTURE_PRIVATE_PATH": fixture_private_path, + }, + ) + output = result.stdout + result.stderr + + assert result.returncode != 0 + assert "aggregate_dependency_resolution_failed" in output + assert fixture_secret not in output + assert fixture_private_path not in output + assert "--no-deps" not in workflow + assert "- name: Verify aggregate dependency closure" in workflow + assert workflow.index("- name: Verify aggregate dependency closure") < workflow.index( + "- name: Create PR for pin update" + ) + + +def test_dependency_success_reaches_only_guarded_pr_step(tmp_path: Path) -> None: + workflow = _workflow() + verify_script = _verification_run_block(workflow) + fake_bin = tmp_path / "bin" + _install_fake_python(fake_bin) + tmp_path.joinpath("qsl-pins.txt").write_text( + _manifest_text(OLD_QPK_SHA), encoding="utf-8" + ) + result = _run_script( + verify_script, + cwd=tmp_path, + env={ + "PATH": f"{fake_bin}:{os.environ['PATH']}", + "PIN_RESOLVER_FIXTURE": "success", + "FIXTURE_SECRET": "unused-fixture-secret", + "FIXTURE_PRIVATE_PATH": "/unused/fixture/path", + }, + ) + + assert result.returncode == 0, result.stderr + assert "aggregate_dependency_resolution_passed" in result.stdout + assert "id: verify" in workflow + assert ( + "if: steps.update.outputs.changed == 'true' && " + "steps.verify.outcome == 'success'" + ) in workflow + assert ' - ".github/workflows/update-qpk-pin.yml"' in workflow + assert ' - "tests/test_update_qpk_pin_workflow.py"' in workflow + assert "workflow_dispatch:" not in workflow