Qwen3-4B-Calendar-Agent-RLVR / evaluation.json
NotoriousH2's picture
Update RLVR to notebook checkpoint-4 (99% task success)
b743dcc verified
Raw
History Blame Contribute Delete
1.92 kB
{
"task": "calendar_tool_calling",
"timezone": "Asia/Seoul",
"samples": 400,
"models": [
{
"model": "Qwen3-4B Base",
"valid_tool_call": 1.0,
"policy_compliance": 0.8,
"task_success": 0.2,
"average_tool_calls": 0.8
},
{
"model": "Calendar Agent SFT",
"valid_tool_call": 1.0,
"policy_compliance": 0.8925,
"task_success": 0.89,
"average_tool_calls": 2.5525
},
{
"model": "Calendar Agent RLVR",
"valid_tool_call": 1.0,
"policy_compliance": 0.9925,
"task_success": 0.99,
"average_tool_calls": 2.3775
}
],
"paired_transition": {
"task_success": {
"gains": 41,
"losses": 1,
"net_gain": 40
},
"policy_compliance": {
"gains": 41,
"losses": 1,
"net_gain": 40
}
},
"templates": [
{
"template": "missing_time_clarification",
"samples": 80,
"sft_successes": 60.0,
"rlvr_successes": 80.0
},
{
"template": "delete_confirmation",
"samples": 80,
"sft_successes": 59.0,
"rlvr_successes": 80.0
},
{
"template": "conflict_then_create",
"samples": 80,
"sft_successes": 77.0,
"rlvr_successes": 76.0
},
{
"template": "confirmed_delete",
"samples": 80,
"sft_successes": 80.0,
"rlvr_successes": 80.0
},
{
"template": "multi_attendee_create",
"samples": 80,
"sft_successes": 80.0,
"rlvr_successes": 80.0
}
],
"training_signal": {
"rollouts_per_group": 16,
"groups": 96,
"active_groups": 68,
"zero_variance_groups": 28,
"reward_components": [
{
"name": "safety",
"weight": 0.25
},
{
"name": "policy_progress",
"weight": 0.35
},
{
"name": "semantic_final_success",
"weight": 0.4
}
]
}
}