mirror of
https://github.com/storytold/effectcraft.git
synced 2026-11-03 06:52:21 -06:00
ci: run the full source quality gate on pull requests
This commit is contained in:
skylartaylor
@@ -0,0 +1,23 @@
|
||||
# Source PR CI evidence
|
||||
|
||||
`ci.yml` runs the existing `cargo xtask ci` gate on opened, synchronized,
|
||||
reopened and ready-for-review PRs, without source/path or draft filters, plus main
|
||||
pushes and manual dispatch. The existing platform packaging triggers are unchanged.
|
||||
The read-only hosted Linux job tests the PR merge result and records both head
|
||||
and checkout identities. No PR-target execution or signing/deployment credentials
|
||||
are used. The 150-minute cold-build limit is provisional for this new job layout.
|
||||
|
||||
The runner installs Mesa Lavapipe and probes its Vulkan adapter. A failed probe
|
||||
does not prevent CPU gates from running, but fails the final coverage report.
|
||||
`RUST_TEST_NOCAPTURE=1` exposes the existing GPU tests' adapter/parity/skip messages.
|
||||
The reporter requires a successful gate, executed Rust tests and positive GPU
|
||||
evidence; runtime no-adapter skips fail acceptance. Intentionally ignored platform
|
||||
cases are counted/reported separately. Complete logs and source identity are saved
|
||||
as artifacts. Software Vulkan coverage does not qualify physical GPU drivers,
|
||||
other operating systems or tests deliberately ignored on Linux.
|
||||
|
||||
Run the report regression tests with
|
||||
`python3 -m unittest discover -s .github/scripts -p 'test_*.py' -v`.
|
||||
Trigger simulation and fixture tests do not prove GitHub events or runner images
|
||||
work: after approved publication/landing, inspect actual PR run URLs and results,
|
||||
including a ready-for-review event, before requiring the new check for merges.
|
||||
@@ -0,0 +1,56 @@
|
||||
"""Report the actual quality gate and GPU execution evidence; never treat skips as parity."""
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
GPU_PROOF = re.compile('effectcraft-gpu tests: adapter ')
|
||||
GPU_SKIP = re.compile(r"skipp", re.I)
|
||||
GPU_CONTEXT = re.compile(r"GPU|no\s+(?:GPU\s+)?adapter|no device", re.I)
|
||||
|
||||
|
||||
def report(text, adapter_outcome, quality_outcome):
|
||||
text = re.sub(r"\x1b\[[0-?]*[ -/]*[@-~]", "", text)
|
||||
totals = re.findall(r"test result: (?:ok|FAILED)\. (\d+) passed; (\d+) failed; (\d+) ignored", text)
|
||||
counts = [sum(int(row[i]) for row in totals) for i in range(3)]
|
||||
skips = [line for line in text.splitlines() if "ignored" not in line.lower() and GPU_SKIP.search(line) and GPU_CONTEXT.search(line)]
|
||||
problems = []
|
||||
if adapter_outcome != "success":
|
||||
problems.append("Vulkan software-adapter probe did not succeed")
|
||||
if quality_outcome != "success":
|
||||
problems.append("Full cargo xtask ci did not pass")
|
||||
if counts[0] == 0:
|
||||
problems.append("No executed Rust tests were recorded")
|
||||
if not GPU_PROOF.search(text):
|
||||
problems.append("No successful GPU execution evidence was recorded")
|
||||
if skips:
|
||||
problems.append("GPU tests skipped at runtime; this is not GPU parity acceptance")
|
||||
lines = ["## Source PR quality gate", "", f"- Adapter probe: {adapter_outcome}", f"- Full gate: {quality_outcome}", f"- Printed Rust totals: {counts[0]} passed, {counts[1]} failed, {counts[2]} ignored", f"- GPU runtime-skip messages: {len(skips)}"]
|
||||
lines += ["- " + problem for problem in problems]
|
||||
if skips:
|
||||
lines += ["", "```text", *skips[:20], "```"]
|
||||
lines += ["", "Counts come from printed Rust summaries; complete logs and source identity are retained as artifacts.", "Software Vulkan coverage does not qualify physical GPU drivers or other operating systems."]
|
||||
return (1 if problems else 0), "\n".join(lines) + "\n"
|
||||
|
||||
|
||||
def main():
|
||||
root = Path(os.environ["RUNNER_TEMP"])
|
||||
if len(sys.argv) > 1 and sys.argv[1] == "record":
|
||||
event = json.loads(Path(os.environ["GITHUB_EVENT_PATH"]).read_text())
|
||||
pr = event.get("pull_request", {})
|
||||
identity = {"event": os.environ.get("GITHUB_EVENT_NAME"), "action": event.get("action"), "pull_request": event.get("number"), "head_sha": pr.get("head", {}).get("sha"), "github_sha": os.environ.get("GITHUB_SHA"), "checkout_sha": subprocess.check_output(["git", "rev-parse", "HEAD"], text=True).strip()}
|
||||
(root / "source-identity.json").write_text(json.dumps(identity, indent=2))
|
||||
print(json.dumps(identity, indent=2))
|
||||
return 0
|
||||
log = root / "quality-gate.log"
|
||||
code, summary = report(log.read_text(errors="replace") if log.exists() else "", os.environ.get("ADAPTER_OUTCOME", "missing"), os.environ.get("QUALITY_OUTCOME", "missing"))
|
||||
with Path(os.environ["GITHUB_STEP_SUMMARY"]).open("a") as stream:
|
||||
stream.write(summary)
|
||||
print(summary)
|
||||
return code
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,47 @@
|
||||
import unittest
|
||||
from ci_report import report
|
||||
|
||||
POSITIVE = 'effectcraft-gpu tests: adapter llvmpipe (Vulkan)' + "\ntest result: ok. 12 passed; 0 failed; 1 ignored; 0 measured; 0 filtered out\n"
|
||||
|
||||
|
||||
class CoverageReportTests(unittest.TestCase):
|
||||
def test_executed_gpu_tests_and_quality_gate_pass(self):
|
||||
code, text = report(POSITIVE, "success", "success")
|
||||
self.assertEqual(code, 0)
|
||||
self.assertIn("12 passed, 0 failed, 1 ignored", text)
|
||||
|
||||
def test_adapter_probe_failure_is_not_parity(self):
|
||||
code, text = report(POSITIVE, "failure", "success")
|
||||
self.assertEqual(code, 1)
|
||||
self.assertIn("probe did not succeed", text)
|
||||
|
||||
def test_runtime_gpu_skip_cannot_pass_with_other_gpu_evidence(self):
|
||||
code, text = report(POSITIVE + "\nno GPU adapter; skipping\n", "success", "success")
|
||||
self.assertEqual(code, 1)
|
||||
self.assertIn("GPU tests skipped at runtime", text)
|
||||
|
||||
def test_quality_gate_failure_is_reported(self):
|
||||
code, text = report(POSITIVE, "success", "failure")
|
||||
self.assertEqual(code, 1)
|
||||
self.assertIn("Full cargo xtask ci did not pass", text)
|
||||
|
||||
def test_no_selected_tests_or_missing_logs_fail(self):
|
||||
self.assertEqual(report("", "success", "success")[0], 1)
|
||||
self.assertEqual(report('effectcraft-gpu tests: adapter llvmpipe (Vulkan)', "success", "success")[0], 1)
|
||||
|
||||
def test_ignored_platform_case_is_reported_not_runtime_skip(self):
|
||||
code, text = report(POSITIVE + "test windows_case ... ignored, no adapter skips\n", "success", "success")
|
||||
self.assertEqual(code, 0)
|
||||
self.assertIn("GPU runtime-skip messages: 0", text)
|
||||
|
||||
def test_expected_no_adapter_error_without_skip_is_not_runtime_skip(self):
|
||||
self.assertEqual(report(POSITIVE + "expected constructor error: no adapter\n", "success", "success")[0], 0)
|
||||
|
||||
def test_no_gpu_execution_evidence_fails(self):
|
||||
code, text = report("test result: ok. 12 passed; 0 failed; 0 ignored\n", "success", "success")
|
||||
self.assertEqual(code, 1)
|
||||
self.assertIn("No successful GPU execution evidence", text)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,102 @@
|
||||
# General source coverage; platform packaging retains its own scoped workflows.
|
||||
name: CI
|
||||
on:
|
||||
pull_request:
|
||||
types: [opened, synchronize, reopened, ready_for_review]
|
||||
push:
|
||||
branches: [main]
|
||||
workflow_dispatch:
|
||||
permissions:
|
||||
contents: read
|
||||
concurrency:
|
||||
group: source-pr-ci-${{ github.ref }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
|
||||
env:
|
||||
CARGO_TERM_COLOR: always
|
||||
CARGO_BUILD_JOBS: "2"
|
||||
CARGO_INCREMENTAL: "0"
|
||||
RUST_BACKTRACE: "1"
|
||||
RUST_TEST_THREADS: "2"
|
||||
# Passing GPU tests must expose adapter/skip evidence instead of hiding it in captured output.
|
||||
RUST_TEST_NOCAPTURE: "1"
|
||||
WGPU_BACKEND: vulkan
|
||||
EFFECTCRAFT_REQUIRE_GPU: "1"
|
||||
jobs:
|
||||
source-pr-ci:
|
||||
name: Source PR quality gate (Linux)
|
||||
runs-on: ubuntu-24.04
|
||||
# Provisional cold-build bound for this new full-workspace layout; recalibrate after hosted runs.
|
||||
timeout-minutes: 150
|
||||
defaults:
|
||||
run:
|
||||
shell: bash
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
persist-credentials: false
|
||||
- name: Record event and tested merge/head identity
|
||||
run: python3 .github/scripts/ci_report.py record
|
||||
|
||||
- name: Prepare the disposable hosted runner
|
||||
timeout-minutes: 5
|
||||
run: |
|
||||
df -h /
|
||||
sudo rm -rf /usr/local/lib/android /usr/share/dotnet /opt/ghc /usr/local/.ghcup /opt/hostedtoolcache/CodeQL
|
||||
mkdir -p "$RUNNER_TEMP/xdg"
|
||||
chmod 700 "$RUNNER_TEMP/xdg"
|
||||
echo "XDG_RUNTIME_DIR=$RUNNER_TEMP/xdg" >> "$GITHUB_ENV"
|
||||
df -h /
|
||||
- name: Install native test and software Vulkan dependencies
|
||||
timeout-minutes: 10
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y --no-install-recommends \
|
||||
cmake pkg-config libasound2-dev libxkbcommon-dev libwayland-dev libx11-dev \
|
||||
libxcursor-dev libxrandr-dev libxi-dev libgl1-mesa-dev libgtk-3-dev \
|
||||
libfontconfig1-dev libfreetype6-dev libvulkan1 mesa-vulkan-drivers vulkan-tools ffmpeg
|
||||
- uses: dtolnay/rust-toolchain@686976e191b89faba57d3206551f0f330d8cb249 # stable
|
||||
with:
|
||||
components: clippy, rustfmt
|
||||
targets: wasm32-unknown-unknown
|
||||
- uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2
|
||||
with:
|
||||
key: source-pr-ci-linux
|
||||
save-if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }}
|
||||
cache-on-failure: true
|
||||
- name: Test coverage reporting
|
||||
run: python3 -m unittest discover -s .github/scripts -p 'test_*.py' -v
|
||||
- name: Require and identify the Vulkan software adapter
|
||||
id: adapter
|
||||
# Run the CPU quality gates even if the adapter probe fails; the final report still fails.
|
||||
continue-on-error: true
|
||||
timeout-minutes: 2
|
||||
run: |
|
||||
icd=$(find /usr/share/vulkan/icd.d -maxdepth 1 -name 'lvp_icd*.json' -print -quit)
|
||||
if [ -z "$icd" ]; then
|
||||
echo "::error::Lavapipe ICD missing; GPU comparisons cannot be accepted"
|
||||
exit 1
|
||||
fi
|
||||
export VK_DRIVER_FILES="$icd" VK_ICD_FILENAMES="$icd"
|
||||
echo "VK_DRIVER_FILES=$icd" >> "$GITHUB_ENV"
|
||||
echo "VK_ICD_FILENAMES=$icd" >> "$GITHUB_ENV"
|
||||
vulkaninfo --summary 2>&1 | tee "$RUNNER_TEMP/vulkan-adapter.log"
|
||||
- name: Full repository quality gate
|
||||
id: quality
|
||||
|
||||
run: cargo xtask ci 2>&1 | tee "$RUNNER_TEMP/quality-gate.log"
|
||||
- name: Report executed tests, adapter availability and skips
|
||||
if: ${{ always() && !cancelled() }}
|
||||
env:
|
||||
ADAPTER_OUTCOME: ${{ steps.adapter.outcome }}
|
||||
QUALITY_OUTCOME: ${{ steps.quality.outcome }}
|
||||
run: python3 .github/scripts/ci_report.py
|
||||
- name: Preserve source identity and full CI evidence
|
||||
if: ${{ always() && !cancelled() }}
|
||||
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
|
||||
with:
|
||||
name: source-pr-ci-evidence
|
||||
retention-days: 7
|
||||
path: |
|
||||
${{ runner.temp }}/source-identity.json
|
||||
${{ runner.temp }}/vulkan-adapter.log
|
||||
${{ runner.temp }}/quality-gate.log
|
||||
Reference in New Issue
Block a user