diff --git a/scripts/32_gate2_toolcalls.py b/scripts/32_gate2_toolcalls.py index 50585b1..3bc6ca3 100644 --- a/scripts/32_gate2_toolcalls.py +++ b/scripts/32_gate2_toolcalls.py @@ -35,6 +35,10 @@ HOLDOUT_PATH = REPO_ROOT / "data" / "holdout_prompts.jsonl" RESULTS_PATH = REPO_ROOT / "data" / os.environ.get("GATE2_RESULTS_FILENAME", "gate2_results.json") BASE_URL = os.environ.get("VLLM_EVAL_URL", "http://localhost:8001") MODEL_NAME = os.environ.get("VLLM_EVAL_MODEL", "qwen3.6-35b-a3b-mcp-bf16") +# 1024 dejaba cortar la respuesta a mitad de razonamiento en modelos con +# --reasoning-parser activo antes de emitir el tool_call -- ver hallazgo de +# Fase 5 (misma causa que el fix de gate3, max_tokens 512->2048). +GATE2_MAX_TOKENS = int(os.environ.get("GATE2_MAX_TOKENS", "2048")) def load_holdout(): @@ -92,13 +96,13 @@ def validate_tool_call(tool_call, tools): return True, None -def call_vllm(prompt, tools, timeout=120): +def call_vllm(prompt, tools, timeout=240): payload = { "model": MODEL_NAME, "messages": [{"role": "user", "content": prompt}], "tools": to_openai_tools(tools), "tool_choice": "auto", - "max_tokens": 1024, + "max_tokens": GATE2_MAX_TOKENS, "temperature": 0.0, } resp = requests.post(f"{BASE_URL}/v1/chat/completions", json=payload, timeout=timeout) @@ -133,11 +137,24 @@ def main(): continue message = response["choices"][0]["message"] + # Se guarda siempre el texto completo (content + reasoning) para poder auditar + # con criterio humano los casos que fallan o quedan sin tool_call -- antes no se + # guardaba nada de esto, lo que hacia imposible diagnosticar truncamiento. + content = message.get("content") or "" + reasoning = message.get("reasoning") or "" tool_calls = message.get("tool_calls") or [] if not tool_calls: stats[mcp]["no_tool_call"] += 1 stats["__global__"]["no_tool_call"] += 1 - results.append({"mcp": mcp, "prompt": ex["prompt"], "tool_calls": None, "valid": None}) + results.append({ + "mcp": mcp, + "prompt": ex["prompt"], + "tool_calls": None, + "valid": None, + "content": content, + "reasoning": reasoning, + "finish_reason": response["choices"][0].get("finish_reason"), + }) continue all_valid = True @@ -161,6 +178,9 @@ def main(): "tool_calls": [tc["function"]["name"] for tc in tool_calls], "valid": all_valid, "errors": errors, + "content": content, + "reasoning": reasoning, + "finish_reason": response["choices"][0].get("finish_reason"), }) if (i + 1) % 20 == 0: