ci: run the full source quality gate on pull requests

This commit is contained in:
skylartaylor
2026-10-15 04:19:48 +02:00
parent 05955c08b4
commit 381559abb1
4 changed files with 228 additions and 0 deletions
+23
View File
@@ -0,0 +1,23 @@
# Source PR CI evidence
`ci.yml` runs the existing `cargo xtask ci` gate on opened, synchronized,
reopened and ready-for-review PRs, without source/path or draft filters, plus main
pushes and manual dispatch. The existing platform packaging triggers are unchanged.
The read-only hosted Linux job tests the PR merge result and records both head
and checkout identities. No PR-target execution or signing/deployment credentials
are used. The 150-minute cold-build limit is provisional for this new job layout.
The runner installs Mesa Lavapipe and probes its Vulkan adapter. A failed probe
does not prevent CPU gates from running, but fails the final coverage report.
`RUST_TEST_NOCAPTURE=1` exposes the existing GPU tests' adapter/parity/skip messages.
The reporter requires a successful gate, executed Rust tests and positive GPU
evidence; runtime no-adapter skips fail acceptance. Intentionally ignored platform
cases are counted/reported separately. Complete logs and source identity are saved
as artifacts. Software Vulkan coverage does not qualify physical GPU drivers,
other operating systems or tests deliberately ignored on Linux.
Run the report regression tests with
`python3 -m unittest discover -s .github/scripts -p 'test_*.py' -v`.
Trigger simulation and fixture tests do not prove GitHub events or runner images
work: after approved publication/landing, inspect actual PR run URLs and results,
including a ready-for-review event, before requiring the new check for merges.
+56
View File
@@ -0,0 +1,56 @@
"""Report the actual quality gate and GPU execution evidence; never treat skips as parity."""
import json
import os
from pathlib import Path
import re
import subprocess
import sys
GPU_PROOF = re.compile('effectcraft-gpu tests: adapter ')
GPU_SKIP = re.compile(r"skipp", re.I)
GPU_CONTEXT = re.compile(r"GPU|no\s+(?:GPU\s+)?adapter|no device", re.I)
def report(text, adapter_outcome, quality_outcome):
text = re.sub(r"\x1b\[[0-?]*[ -/]*[@-~]", "", text)
totals = re.findall(r"test result: (?:ok|FAILED)\. (\d+) passed; (\d+) failed; (\d+) ignored", text)
counts = [sum(int(row[i]) for row in totals) for i in range(3)]
skips = [line for line in text.splitlines() if "ignored" not in line.lower() and GPU_SKIP.search(line) and GPU_CONTEXT.search(line)]
problems = []
if adapter_outcome != "success":
problems.append("Vulkan software-adapter probe did not succeed")
if quality_outcome != "success":
problems.append("Full cargo xtask ci did not pass")
if counts[0] == 0:
problems.append("No executed Rust tests were recorded")
if not GPU_PROOF.search(text):
problems.append("No successful GPU execution evidence was recorded")
if skips:
problems.append("GPU tests skipped at runtime; this is not GPU parity acceptance")
lines = ["## Source PR quality gate", "", f"- Adapter probe: {adapter_outcome}", f"- Full gate: {quality_outcome}", f"- Printed Rust totals: {counts[0]} passed, {counts[1]} failed, {counts[2]} ignored", f"- GPU runtime-skip messages: {len(skips)}"]
lines += ["- " + problem for problem in problems]
if skips:
lines += ["", "```text", *skips[:20], "```"]
lines += ["", "Counts come from printed Rust summaries; complete logs and source identity are retained as artifacts.", "Software Vulkan coverage does not qualify physical GPU drivers or other operating systems."]
return (1 if problems else 0), "\n".join(lines) + "\n"
def main():
root = Path(os.environ["RUNNER_TEMP"])
if len(sys.argv) > 1 and sys.argv[1] == "record":
event = json.loads(Path(os.environ["GITHUB_EVENT_PATH"]).read_text())
pr = event.get("pull_request", {})
identity = {"event": os.environ.get("GITHUB_EVENT_NAME"), "action": event.get("action"), "pull_request": event.get("number"), "head_sha": pr.get("head", {}).get("sha"), "github_sha": os.environ.get("GITHUB_SHA"), "checkout_sha": subprocess.check_output(["git", "rev-parse", "HEAD"], text=True).strip()}
(root / "source-identity.json").write_text(json.dumps(identity, indent=2))
print(json.dumps(identity, indent=2))
return 0
log = root / "quality-gate.log"
code, summary = report(log.read_text(errors="replace") if log.exists() else "", os.environ.get("ADAPTER_OUTCOME", "missing"), os.environ.get("QUALITY_OUTCOME", "missing"))
with Path(os.environ["GITHUB_STEP_SUMMARY"]).open("a") as stream:
stream.write(summary)
print(summary)
return code
if __name__ == "__main__":
raise SystemExit(main())
+47
View File
@@ -0,0 +1,47 @@
import unittest
from ci_report import report
POSITIVE = 'effectcraft-gpu tests: adapter llvmpipe (Vulkan)' + "\ntest result: ok. 12 passed; 0 failed; 1 ignored; 0 measured; 0 filtered out\n"
class CoverageReportTests(unittest.TestCase):
def test_executed_gpu_tests_and_quality_gate_pass(self):
code, text = report(POSITIVE, "success", "success")
self.assertEqual(code, 0)
self.assertIn("12 passed, 0 failed, 1 ignored", text)
def test_adapter_probe_failure_is_not_parity(self):
code, text = report(POSITIVE, "failure", "success")
self.assertEqual(code, 1)
self.assertIn("probe did not succeed", text)
def test_runtime_gpu_skip_cannot_pass_with_other_gpu_evidence(self):
code, text = report(POSITIVE + "\nno GPU adapter; skipping\n", "success", "success")
self.assertEqual(code, 1)
self.assertIn("GPU tests skipped at runtime", text)
def test_quality_gate_failure_is_reported(self):
code, text = report(POSITIVE, "success", "failure")
self.assertEqual(code, 1)
self.assertIn("Full cargo xtask ci did not pass", text)
def test_no_selected_tests_or_missing_logs_fail(self):
self.assertEqual(report("", "success", "success")[0], 1)
self.assertEqual(report('effectcraft-gpu tests: adapter llvmpipe (Vulkan)', "success", "success")[0], 1)
def test_ignored_platform_case_is_reported_not_runtime_skip(self):
code, text = report(POSITIVE + "test windows_case ... ignored, no adapter skips\n", "success", "success")
self.assertEqual(code, 0)
self.assertIn("GPU runtime-skip messages: 0", text)
def test_expected_no_adapter_error_without_skip_is_not_runtime_skip(self):
self.assertEqual(report(POSITIVE + "expected constructor error: no adapter\n", "success", "success")[0], 0)
def test_no_gpu_execution_evidence_fails(self):
code, text = report("test result: ok. 12 passed; 0 failed; 0 ignored\n", "success", "success")
self.assertEqual(code, 1)
self.assertIn("No successful GPU execution evidence", text)
if __name__ == "__main__":
unittest.main()