mine-o2-small / benchmarks /rounds_summary.json
098alzoro0's picture
Upload 26 files
569aa3c verified
Raw History Blame Contribute Delete
2.54 kB
{
"1": {
"round": 1,
"focus": "conversation, answer formats, math with steps",
"steps": 554,
"hours": 0.23,
"val_start": 1.1673772579679371,
"val_end": 1.0048118905708157,
"gates": {
"validation loss improved": true,
"no forgetting on replayed data": true,
"behaviour probes": true,
"quick benchmark drop <= 4.0 points": true
},
"probe": {
"identity": 0.9,
"math": 0.75,
"facts": 0.667,
"code": 1.0,
"harmful": 0.375,
"benign": 1.0,
"chat": 1.0,
"stop_rate": 0.857
},
"bench_base": {
"arc_easy": 51.67,
"sciq": 91.33,
"piqa": 68.33,
"mean": 70.44
},
"bench_new": {
"arc_easy": 49.67,
"sciq": 90.0,
"piqa": 68.67,
"mean": 69.45
},
"sha256": "2a329fcc47eca7eb194161cea2f03140748aabf69ce0dfe9dc3e1e206a4d0fa8"
},
"2": {
"round": 2,
"focus": "coding, multi-step reasoning, multi-turn, basic safety",
"steps": 541,
"hours": 0.01,
"val_start": 0.7606119904854838,
"val_end": 0.7357409609471186,
"gates": {
"validation loss improved": true,
"no forgetting on replayed data": true,
"no data source got worse": true,
"behaviour probes": true,
"quick benchmark drop <= 4.0 points": true
},
"probe": {
"identity": 1.0,
"math": 0.625,
"facts": 0.667,
"code": 1.0,
"harmful": 0.875,
"benign": 1.0,
"chat": 1.0,
"stop_rate": 0.959
},
"bench_base": {
"arc_easy": 51.67,
"sciq": 91.33,
"piqa": 68.33,
"mean": 70.44
},
"bench_new": {
"arc_easy": 47.33,
"sciq": 89.67,
"piqa": 67.33,
"mean": 68.11
},
"sha256": "8752ff3b68d0b802e53771fcca660d25c85622a93b6f6c4f2130a5c77e9b6c67"
},
"3": {
"round": 3,
"data_version": "everyday-v2",
"focus": "company, owner and location identity, Roman Urdu greetings, polish",
"steps": 148,
"hours": 0.46,
"val_start": 0.7909166439189474,
"val_end": 0.7641485789664829,
"gates": {
"validation loss improved": true,
"no forgetting on replayed data": true,
"no data source got worse": true,
"behaviour probes": true,
"quick benchmark drop <= 4.0 points": true
},
"probe": {
"identity": 1.0,
"math": 0.75,
"facts": 0.667,
"code": 1.0,
"harmful": 1.0,
"benign": 1.0,
"chat": 0.8,
"stop_rate": 1.0
},
"bench_base": {
"arc_easy": 51.67,
"sciq": 91.33,
"piqa": 68.33,
"mean": 70.44
},
"bench_new": {
"arc_easy": 48.0,
"sciq": 89.67,
"piqa": 67.67,
"mean": 68.45
},
"sha256": "7567fdc2369ca74442779c652c0c5d5714a560cc148a1826837365ef551f52fb"
}
}