diff --git a/.github/scripts/README.md b/.github/scripts/README.md new file mode 100644 index 00000000..d13e3185 --- /dev/null +++ b/.github/scripts/README.md @@ -0,0 +1,23 @@ +# Source PR CI evidence + +`ci.yml` runs the existing `cargo xtask ci` gate on opened, synchronized, +reopened and ready-for-review PRs, without source/path or draft filters, plus main +pushes and manual dispatch. The existing platform packaging triggers are unchanged. +The read-only hosted Linux job tests the PR merge result and records both head +and checkout identities. No PR-target execution or signing/deployment credentials +are used. The 150-minute cold-build limit is provisional for this new job layout. + +The runner installs Mesa Lavapipe and probes its Vulkan adapter. A failed probe +does not prevent CPU gates from running, but fails the final coverage report. +`RUST_TEST_NOCAPTURE=1` exposes the existing GPU tests' adapter/parity/skip messages. +The reporter requires a successful gate, executed Rust tests and positive GPU +evidence; runtime no-adapter skips fail acceptance. Intentionally ignored platform +cases are counted/reported separately. Complete logs and source identity are saved +as artifacts. Software Vulkan coverage does not qualify physical GPU drivers, +other operating systems or tests deliberately ignored on Linux. + +Run the report regression tests with +`python3 -m unittest discover -s .github/scripts -p 'test_*.py' -v`. +Trigger simulation and fixture tests do not prove GitHub events or runner images +work: after approved publication/landing, inspect actual PR run URLs and results, +including a ready-for-review event, before requiring the new check for merges. diff --git a/.github/scripts/ci_report.py b/.github/scripts/ci_report.py new file mode 100644 index 00000000..2c75faea --- /dev/null +++ b/.github/scripts/ci_report.py @@ -0,0 +1,56 @@ +"""Report the actual quality gate and GPU execution evidence; never treat skips as parity.""" +import json +import os +from pathlib import Path +import re +import subprocess +import sys + +GPU_PROOF = re.compile('effectcraft-gpu tests: adapter ') +GPU_SKIP = re.compile(r"skipp", re.I) +GPU_CONTEXT = re.compile(r"GPU|no\s+(?:GPU\s+)?adapter|no device", re.I) + + +def report(text, adapter_outcome, quality_outcome): + text = re.sub(r"\x1b\[[0-?]*[ -/]*[@-~]", "", text) + totals = re.findall(r"test result: (?:ok|FAILED)\. (\d+) passed; (\d+) failed; (\d+) ignored", text) + counts = [sum(int(row[i]) for row in totals) for i in range(3)] + skips = [line for line in text.splitlines() if "ignored" not in line.lower() and GPU_SKIP.search(line) and GPU_CONTEXT.search(line)] + problems = [] + if adapter_outcome != "success": + problems.append("Vulkan software-adapter probe did not succeed") + if quality_outcome != "success": + problems.append("Full cargo xtask ci did not pass") + if counts[0] == 0: + problems.append("No executed Rust tests were recorded") + if not GPU_PROOF.search(text): + problems.append("No successful GPU execution evidence was recorded") + if skips: + problems.append("GPU tests skipped at runtime; this is not GPU parity acceptance") + lines = ["## Source PR quality gate", "", f"- Adapter probe: {adapter_outcome}", f"- Full gate: {quality_outcome}", f"- Printed Rust totals: {counts[0]} passed, {counts[1]} failed, {counts[2]} ignored", f"- GPU runtime-skip messages: {len(skips)}"] + lines += ["- " + problem for problem in problems] + if skips: + lines += ["", "```text", *skips[:20], "```"] + lines += ["", "Counts come from printed Rust summaries; complete logs and source identity are retained as artifacts.", "Software Vulkan coverage does not qualify physical GPU drivers or other operating systems."] + return (1 if problems else 0), "\n".join(lines) + "\n" + + +def main(): + root = Path(os.environ["RUNNER_TEMP"]) + if len(sys.argv) > 1 and sys.argv[1] == "record": + event = json.loads(Path(os.environ["GITHUB_EVENT_PATH"]).read_text()) + pr = event.get("pull_request", {}) + identity = {"event": os.environ.get("GITHUB_EVENT_NAME"), "action": event.get("action"), "pull_request": event.get("number"), "head_sha": pr.get("head", {}).get("sha"), "github_sha": os.environ.get("GITHUB_SHA"), "checkout_sha": subprocess.check_output(["git", "rev-parse", "HEAD"], text=True).strip()} + (root / "source-identity.json").write_text(json.dumps(identity, indent=2)) + print(json.dumps(identity, indent=2)) + return 0 + log = root / "quality-gate.log" + code, summary = report(log.read_text(errors="replace") if log.exists() else "", os.environ.get("ADAPTER_OUTCOME", "missing"), os.environ.get("QUALITY_OUTCOME", "missing")) + with Path(os.environ["GITHUB_STEP_SUMMARY"]).open("a") as stream: + stream.write(summary) + print(summary) + return code + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.github/scripts/test_ci_report.py b/.github/scripts/test_ci_report.py new file mode 100644 index 00000000..2308462b --- /dev/null +++ b/.github/scripts/test_ci_report.py @@ -0,0 +1,47 @@ +import unittest +from ci_report import report + +POSITIVE = 'effectcraft-gpu tests: adapter llvmpipe (Vulkan)' + "\ntest result: ok. 12 passed; 0 failed; 1 ignored; 0 measured; 0 filtered out\n" + + +class CoverageReportTests(unittest.TestCase): + def test_executed_gpu_tests_and_quality_gate_pass(self): + code, text = report(POSITIVE, "success", "success") + self.assertEqual(code, 0) + self.assertIn("12 passed, 0 failed, 1 ignored", text) + + def test_adapter_probe_failure_is_not_parity(self): + code, text = report(POSITIVE, "failure", "success") + self.assertEqual(code, 1) + self.assertIn("probe did not succeed", text) + + def test_runtime_gpu_skip_cannot_pass_with_other_gpu_evidence(self): + code, text = report(POSITIVE + "\nno GPU adapter; skipping\n", "success", "success") + self.assertEqual(code, 1) + self.assertIn("GPU tests skipped at runtime", text) + + def test_quality_gate_failure_is_reported(self): + code, text = report(POSITIVE, "success", "failure") + self.assertEqual(code, 1) + self.assertIn("Full cargo xtask ci did not pass", text) + + def test_no_selected_tests_or_missing_logs_fail(self): + self.assertEqual(report("", "success", "success")[0], 1) + self.assertEqual(report('effectcraft-gpu tests: adapter llvmpipe (Vulkan)', "success", "success")[0], 1) + + def test_ignored_platform_case_is_reported_not_runtime_skip(self): + code, text = report(POSITIVE + "test windows_case ... ignored, no adapter skips\n", "success", "success") + self.assertEqual(code, 0) + self.assertIn("GPU runtime-skip messages: 0", text) + + def test_expected_no_adapter_error_without_skip_is_not_runtime_skip(self): + self.assertEqual(report(POSITIVE + "expected constructor error: no adapter\n", "success", "success")[0], 0) + + def test_no_gpu_execution_evidence_fails(self): + code, text = report("test result: ok. 12 passed; 0 failed; 0 ignored\n", "success", "success") + self.assertEqual(code, 1) + self.assertIn("No successful GPU execution evidence", text) + + +if __name__ == "__main__": + unittest.main() diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 00000000..f15ecf83 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,102 @@ +# General source coverage; platform packaging retains its own scoped workflows. +name: CI +on: + pull_request: + types: [opened, synchronize, reopened, ready_for_review] + push: + branches: [main] + workflow_dispatch: +permissions: + contents: read +concurrency: + group: source-pr-ci-${{ github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} +env: + CARGO_TERM_COLOR: always + CARGO_BUILD_JOBS: "2" + CARGO_INCREMENTAL: "0" + RUST_BACKTRACE: "1" + RUST_TEST_THREADS: "2" + # Passing GPU tests must expose adapter/skip evidence instead of hiding it in captured output. + RUST_TEST_NOCAPTURE: "1" + WGPU_BACKEND: vulkan + EFFECTCRAFT_REQUIRE_GPU: "1" +jobs: + source-pr-ci: + name: Source PR quality gate (Linux) + runs-on: ubuntu-24.04 + # Provisional cold-build bound for this new full-workspace layout; recalibrate after hosted runs. + timeout-minutes: 150 + defaults: + run: + shell: bash + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: Record event and tested merge/head identity + run: python3 .github/scripts/ci_report.py record + + - name: Prepare the disposable hosted runner + timeout-minutes: 5 + run: | + df -h / + sudo rm -rf /usr/local/lib/android /usr/share/dotnet /opt/ghc /usr/local/.ghcup /opt/hostedtoolcache/CodeQL + mkdir -p "$RUNNER_TEMP/xdg" + chmod 700 "$RUNNER_TEMP/xdg" + echo "XDG_RUNTIME_DIR=$RUNNER_TEMP/xdg" >> "$GITHUB_ENV" + df -h / + - name: Install native test and software Vulkan dependencies + timeout-minutes: 10 + run: | + sudo apt-get update + sudo apt-get install -y --no-install-recommends \ + cmake pkg-config libasound2-dev libxkbcommon-dev libwayland-dev libx11-dev \ + libxcursor-dev libxrandr-dev libxi-dev libgl1-mesa-dev libgtk-3-dev \ + libfontconfig1-dev libfreetype6-dev libvulkan1 mesa-vulkan-drivers vulkan-tools ffmpeg + - uses: dtolnay/rust-toolchain@686976e191b89faba57d3206551f0f330d8cb249 # stable + with: + components: clippy, rustfmt + targets: wasm32-unknown-unknown + - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + with: + key: source-pr-ci-linux + save-if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }} + cache-on-failure: true + - name: Test coverage reporting + run: python3 -m unittest discover -s .github/scripts -p 'test_*.py' -v + - name: Require and identify the Vulkan software adapter + id: adapter + # Run the CPU quality gates even if the adapter probe fails; the final report still fails. + continue-on-error: true + timeout-minutes: 2 + run: | + icd=$(find /usr/share/vulkan/icd.d -maxdepth 1 -name 'lvp_icd*.json' -print -quit) + if [ -z "$icd" ]; then + echo "::error::Lavapipe ICD missing; GPU comparisons cannot be accepted" + exit 1 + fi + export VK_DRIVER_FILES="$icd" VK_ICD_FILENAMES="$icd" + echo "VK_DRIVER_FILES=$icd" >> "$GITHUB_ENV" + echo "VK_ICD_FILENAMES=$icd" >> "$GITHUB_ENV" + vulkaninfo --summary 2>&1 | tee "$RUNNER_TEMP/vulkan-adapter.log" + - name: Full repository quality gate + id: quality + + run: cargo xtask ci 2>&1 | tee "$RUNNER_TEMP/quality-gate.log" + - name: Report executed tests, adapter availability and skips + if: ${{ always() && !cancelled() }} + env: + ADAPTER_OUTCOME: ${{ steps.adapter.outcome }} + QUALITY_OUTCOME: ${{ steps.quality.outcome }} + run: python3 .github/scripts/ci_report.py + - name: Preserve source identity and full CI evidence + if: ${{ always() && !cancelled() }} + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: source-pr-ci-evidence + retention-days: 7 + path: | + ${{ runner.temp }}/source-identity.json + ${{ runner.temp }}/vulkan-adapter.log + ${{ runner.temp }}/quality-gate.log