Phase 6.4.25c: re-score existing v2b-bf16 gate 5 results with corrected veto
New offline script re-scans the code payloads already saved in the gate 5 result JSONs (holdout, full, generalization) using the corrected forbidden- behavior list -- no live Penpot needed, reuses scan_forbidden/score_prompt from 35_gate5_penpot_design.py as the single source of truth. Holdout result changes materially: 3 of the 5 flagged prompts (pd-008, pd-009, pd-035) were flagged only for flex.appendChild. With that pattern removed, forbidden-API rate drops from 8.3% (5/60) to 3.3% (2/60), which is now under the <=5% fallback threshold -- the holdout gate now passes. Full mode and the generalization probe are unaffected: neither run's transcripts ever triggered the flex.appendChild pattern, so their scores are identical before and after. Original result files are kept untouched; corrected results are written alongside as *-corrected.json.
This commit is contained in:
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
@@ -0,0 +1,134 @@
|
||||
"""Re-escanea resultados YA MEDIDOS de la puerta 5 con la lista de comportamiento prohibido
|
||||
CORREGIDA (post hallazgo 54: `board.flex.appendChild` no esta roto, se saco del veto), sin volver
|
||||
a correr el modelo ni tocar Penpot. Los payloads de `code` ya quedaron guardados en los JSON de
|
||||
resultados, asi que el veto se puede recalcular offline con `scan_forbidden` / `score_prompt` del
|
||||
propio scripts/35_gate5_penpot_design.py (fuente de verdad unica, nunca se reimplementa la logica).
|
||||
|
||||
python3 scripts/36_gate5_rescan_veto.py
|
||||
|
||||
Escribe *-corrected.json junto a cada archivo de entrada. No modifica los originales.
|
||||
"""
|
||||
import importlib.util
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parent.parent
|
||||
GATE5_PATH = REPO_ROOT / "scripts" / "35_gate5_penpot_design.py"
|
||||
|
||||
FILES_HOLDOUT = [REPO_ROOT / "data" / "gate5_holdout_v2b-bf16-holdout.json"]
|
||||
FILES_FULL = [
|
||||
REPO_ROOT / "data" / "gate5_results_v2b-bf16-full.json",
|
||||
REPO_ROOT / "data" / "gate5_results_v2b-bf16-generalization.json",
|
||||
]
|
||||
|
||||
|
||||
def load_gate5_module():
|
||||
spec = importlib.util.spec_from_file_location("gate5_penpot_design", GATE5_PATH)
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
def rescan_holdout(gate5, path):
|
||||
data = json.loads(path.read_text(encoding="utf-8"))
|
||||
patrones = {}
|
||||
con_api_prohibida = 0
|
||||
cambios = []
|
||||
for r in data["resultados"]:
|
||||
veto_hits = []
|
||||
for code in r.get("code_payloads", []):
|
||||
veto, _otros = gate5.scan_forbidden(code)
|
||||
veto_hits.extend(veto)
|
||||
antes = bool(r.get("veto_hits"))
|
||||
r["veto_hits"] = veto_hits
|
||||
despues = bool(veto_hits)
|
||||
if antes != despues:
|
||||
cambios.append({"id": r.get("id"), "antes_vetado": antes, "despues_vetado": despues})
|
||||
if despues:
|
||||
con_api_prohibida += 1
|
||||
for hit in veto_hits:
|
||||
patrones[hit["patron"]] = patrones.get(hit["patron"], 0) + 1
|
||||
total = data["total"]
|
||||
tasa = con_api_prohibida / total if total else 0.0
|
||||
data["con_api_prohibida"] = con_api_prohibida
|
||||
data["tasa"] = tasa
|
||||
data["aprueba"] = tasa <= data["umbral"]
|
||||
data["patrones"] = patrones
|
||||
data["_rescan_veto_corregido"] = {
|
||||
"motivo": "hallazgo 54: board.flex.appendChild no esta roto, se sacio del veto de comportamiento prohibido",
|
||||
"cambios_por_prompt": cambios,
|
||||
}
|
||||
return data, cambios
|
||||
|
||||
|
||||
def rescan_full(gate5, path):
|
||||
data = json.loads(path.read_text(encoding="utf-8"))
|
||||
cambios = []
|
||||
scores_antes = []
|
||||
scores_despues = []
|
||||
for p in data["prompts"]:
|
||||
score_antes = p["score"]
|
||||
scores_antes.append(score_antes)
|
||||
veto_hits = []
|
||||
for entry in p.get("code_payloads", []):
|
||||
code = entry["code"] if isinstance(entry, dict) else entry
|
||||
veto, _otros = gate5.scan_forbidden(code)
|
||||
veto_hits.extend(veto)
|
||||
p["veto_hits"] = veto_hits
|
||||
p["metricas_crudas"]["forbiddenBehavior"] = len(veto_hits)
|
||||
raw = p["metricas_crudas"]
|
||||
resultado = gate5.score_prompt(p["id"], raw)
|
||||
p["score"] = resultado["score"]
|
||||
p["pasadas"] = resultado["pasadas"]
|
||||
p["aplicables"] = resultado["aplicables"]
|
||||
p["veto_violado"] = resultado["veto_violado"]
|
||||
p["metricas"] = resultado["metricas"]
|
||||
scores_despues.append(resultado["score"])
|
||||
if resultado["score"] != score_antes:
|
||||
cambios.append({"id": p["id"], "score_antes": score_antes, "score_despues": resultado["score"]})
|
||||
score_medio_antes = sum(scores_antes) / len(scores_antes) if scores_antes else 0.0
|
||||
score_medio_despues = sum(scores_despues) / len(scores_despues) if scores_despues else 0.0
|
||||
veto_limpio_despues = sum(1 for p in data["prompts"] if not p["veto_violado"])
|
||||
prompts_60_despues = sum(1 for p in data["prompts"] if p["score"] >= 60)
|
||||
data["_rescan_veto_corregido"] = {
|
||||
"motivo": "hallazgo 54: board.flex.appendChild no esta roto, se sacio del veto de comportamiento prohibido",
|
||||
"score_medio_antes": round(score_medio_antes, 2),
|
||||
"score_medio_despues": round(score_medio_despues, 2),
|
||||
"veto_limpio_despues": f"{veto_limpio_despues}/{len(data['prompts'])}",
|
||||
"prompts_60_despues": f"{prompts_60_despues}/{len(data['prompts'])}",
|
||||
"cambios_por_prompt": cambios,
|
||||
}
|
||||
return data, cambios
|
||||
|
||||
|
||||
def main():
|
||||
gate5 = load_gate5_module()
|
||||
|
||||
for path in FILES_HOLDOUT:
|
||||
if not path.exists():
|
||||
print(f"[SKIP] {path} no existe")
|
||||
continue
|
||||
data, cambios = rescan_holdout(gate5, path)
|
||||
out = path.with_name(path.stem + "-corrected.json")
|
||||
out.write_text(json.dumps(data, indent=2, ensure_ascii=False), encoding="utf-8")
|
||||
print(f"[HOLDOUT] {path.name}: con_api_prohibida={data['con_api_prohibida']} "
|
||||
f"tasa={data['tasa']:.4f} aprueba={data['aprueba']} patrones={data['patrones']}")
|
||||
print(f" cambios: {cambios}")
|
||||
print(f" -> {out}")
|
||||
|
||||
for path in FILES_FULL:
|
||||
if not path.exists():
|
||||
print(f"[SKIP] {path} no existe")
|
||||
continue
|
||||
data, cambios = rescan_full(gate5, path)
|
||||
out = path.with_name(path.stem + "-corrected.json")
|
||||
out.write_text(json.dumps(data, indent=2, ensure_ascii=False), encoding="utf-8")
|
||||
info = data["_rescan_veto_corregido"]
|
||||
print(f"[FULL] {path.name}: score_medio {info['score_medio_antes']} -> {info['score_medio_despues']} "
|
||||
f"veto_limpio_despues={info['veto_limpio_despues']} prompts_60_despues={info['prompts_60_despues']}")
|
||||
print(f" cambios: {cambios}")
|
||||
print(f" -> {out}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user