mirror of
https://github.com/storytold/effectcraft.git
synced 2026-10-25 18:47:07 -06:00
ci: run the full source quality gate on pull requests
This commit is contained in:
@@ -0,0 +1,23 @@
|
||||
# Source PR CI evidence
|
||||
|
||||
`ci.yml` runs the existing `cargo xtask ci` gate on opened, synchronized,
|
||||
reopened and ready-for-review PRs, without source/path or draft filters, plus main
|
||||
pushes and manual dispatch. The existing platform packaging triggers are unchanged.
|
||||
The read-only hosted Linux job tests the PR merge result and records both head
|
||||
and checkout identities. No PR-target execution or signing/deployment credentials
|
||||
are used. The 150-minute cold-build limit is provisional for this new job layout.
|
||||
|
||||
The runner installs Mesa Lavapipe and probes its Vulkan adapter. A failed probe
|
||||
does not prevent CPU gates from running, but fails the final coverage report.
|
||||
`RUST_TEST_NOCAPTURE=1` exposes the existing GPU tests' adapter/parity/skip messages.
|
||||
The reporter requires a successful gate, executed Rust tests and positive GPU
|
||||
evidence; runtime no-adapter skips fail acceptance. Intentionally ignored platform
|
||||
cases are counted/reported separately. Complete logs and source identity are saved
|
||||
as artifacts. Software Vulkan coverage does not qualify physical GPU drivers,
|
||||
other operating systems or tests deliberately ignored on Linux.
|
||||
|
||||
Run the report regression tests with
|
||||
`python3 -m unittest discover -s .github/scripts -p 'test_*.py' -v`.
|
||||
Trigger simulation and fixture tests do not prove GitHub events or runner images
|
||||
work: after approved publication/landing, inspect actual PR run URLs and results,
|
||||
including a ready-for-review event, before requiring the new check for merges.
|
||||
@@ -0,0 +1,56 @@
|
||||
"""Report the actual quality gate and GPU execution evidence; never treat skips as parity."""
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
GPU_PROOF = re.compile('effectcraft-gpu tests: adapter ')
|
||||
GPU_SKIP = re.compile(r"skipp", re.I)
|
||||
GPU_CONTEXT = re.compile(r"GPU|no\s+(?:GPU\s+)?adapter|no device", re.I)
|
||||
|
||||
|
||||
def report(text, adapter_outcome, quality_outcome):
|
||||
text = re.sub(r"\x1b\[[0-?]*[ -/]*[@-~]", "", text)
|
||||
totals = re.findall(r"test result: (?:ok|FAILED)\. (\d+) passed; (\d+) failed; (\d+) ignored", text)
|
||||
counts = [sum(int(row[i]) for row in totals) for i in range(3)]
|
||||
skips = [line for line in text.splitlines() if "ignored" not in line.lower() and GPU_SKIP.search(line) and GPU_CONTEXT.search(line)]
|
||||
problems = []
|
||||
if adapter_outcome != "success":
|
||||
problems.append("Vulkan software-adapter probe did not succeed")
|
||||
if quality_outcome != "success":
|
||||
problems.append("Full cargo xtask ci did not pass")
|
||||
if counts[0] == 0:
|
||||
problems.append("No executed Rust tests were recorded")
|
||||
if not GPU_PROOF.search(text):
|
||||
problems.append("No successful GPU execution evidence was recorded")
|
||||
if skips:
|
||||
problems.append("GPU tests skipped at runtime; this is not GPU parity acceptance")
|
||||
lines = ["## Source PR quality gate", "", f"- Adapter probe: {adapter_outcome}", f"- Full gate: {quality_outcome}", f"- Printed Rust totals: {counts[0]} passed, {counts[1]} failed, {counts[2]} ignored", f"- GPU runtime-skip messages: {len(skips)}"]
|
||||
lines += ["- " + problem for problem in problems]
|
||||
if skips:
|
||||
lines += ["", "```text", *skips[:20], "```"]
|
||||
lines += ["", "Counts come from printed Rust summaries; complete logs and source identity are retained as artifacts.", "Software Vulkan coverage does not qualify physical GPU drivers or other operating systems."]
|
||||
return (1 if problems else 0), "\n".join(lines) + "\n"
|
||||
|
||||
|
||||
def main():
|
||||
root = Path(os.environ["RUNNER_TEMP"])
|
||||
if len(sys.argv) > 1 and sys.argv[1] == "record":
|
||||
event = json.loads(Path(os.environ["GITHUB_EVENT_PATH"]).read_text())
|
||||
pr = event.get("pull_request", {})
|
||||
identity = {"event": os.environ.get("GITHUB_EVENT_NAME"), "action": event.get("action"), "pull_request": event.get("number"), "head_sha": pr.get("head", {}).get("sha"), "github_sha": os.environ.get("GITHUB_SHA"), "checkout_sha": subprocess.check_output(["git", "rev-parse", "HEAD"], text=True).strip()}
|
||||
(root / "source-identity.json").write_text(json.dumps(identity, indent=2))
|
||||
print(json.dumps(identity, indent=2))
|
||||
return 0
|
||||
log = root / "quality-gate.log"
|
||||
code, summary = report(log.read_text(errors="replace") if log.exists() else "", os.environ.get("ADAPTER_OUTCOME", "missing"), os.environ.get("QUALITY_OUTCOME", "missing"))
|
||||
with Path(os.environ["GITHUB_STEP_SUMMARY"]).open("a") as stream:
|
||||
stream.write(summary)
|
||||
print(summary)
|
||||
return code
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,47 @@
|
||||
import unittest
|
||||
from ci_report import report
|
||||
|
||||
POSITIVE = 'effectcraft-gpu tests: adapter llvmpipe (Vulkan)' + "\ntest result: ok. 12 passed; 0 failed; 1 ignored; 0 measured; 0 filtered out\n"
|
||||
|
||||
|
||||
class CoverageReportTests(unittest.TestCase):
|
||||
def test_executed_gpu_tests_and_quality_gate_pass(self):
|
||||
code, text = report(POSITIVE, "success", "success")
|
||||
self.assertEqual(code, 0)
|
||||
self.assertIn("12 passed, 0 failed, 1 ignored", text)
|
||||
|
||||
def test_adapter_probe_failure_is_not_parity(self):
|
||||
code, text = report(POSITIVE, "failure", "success")
|
||||
self.assertEqual(code, 1)
|
||||
self.assertIn("probe did not succeed", text)
|
||||
|
||||
def test_runtime_gpu_skip_cannot_pass_with_other_gpu_evidence(self):
|
||||
code, text = report(POSITIVE + "\nno GPU adapter; skipping\n", "success", "success")
|
||||
self.assertEqual(code, 1)
|
||||
self.assertIn("GPU tests skipped at runtime", text)
|
||||
|
||||
def test_quality_gate_failure_is_reported(self):
|
||||
code, text = report(POSITIVE, "success", "failure")
|
||||
self.assertEqual(code, 1)
|
||||
self.assertIn("Full cargo xtask ci did not pass", text)
|
||||
|
||||
def test_no_selected_tests_or_missing_logs_fail(self):
|
||||
self.assertEqual(report("", "success", "success")[0], 1)
|
||||
self.assertEqual(report('effectcraft-gpu tests: adapter llvmpipe (Vulkan)', "success", "success")[0], 1)
|
||||
|
||||
def test_ignored_platform_case_is_reported_not_runtime_skip(self):
|
||||
code, text = report(POSITIVE + "test windows_case ... ignored, no adapter skips\n", "success", "success")
|
||||
self.assertEqual(code, 0)
|
||||
self.assertIn("GPU runtime-skip messages: 0", text)
|
||||
|
||||
def test_expected_no_adapter_error_without_skip_is_not_runtime_skip(self):
|
||||
self.assertEqual(report(POSITIVE + "expected constructor error: no adapter\n", "success", "success")[0], 0)
|
||||
|
||||
def test_no_gpu_execution_evidence_fails(self):
|
||||
code, text = report("test result: ok. 12 passed; 0 failed; 0 ignored\n", "success", "success")
|
||||
self.assertEqual(code, 1)
|
||||
self.assertIn("No successful GPU execution evidence", text)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user