${state.pub?'5 spokes · one per area, each the area score on the public benchmarks only':`5 spokes · one per area: its public benchmarks and their private tests combined on one scale${state.own?', with your part weights':', weighted as in the Full score'}. Private tasks from new domains belong to no area, so they are not in the spokes`}
`;
}
function categoryCard(c){
const ids=categoryAxes(c);
- const axes=ids.map(id=>({label:bench(id).short,id}));
+ const axes=ids.map(id=>({label:bench(id).short+(bench(id).private&&!state.pub?'+':''),id}));
const inIdx=ids.every(id=>idxVal(D.jev,id)!=null);
- const top=D.models.map(m=>({m,s:m.categories.find(x=>x.id===c.id)})).filter(x=>x.s).sort((a,b)=>b.s[AK()]-a.s[AK()]).slice(0,3);
+ const top=D.models.map(m=>({m,s:m.categories.find(x=>x.id===c.id)})).filter(x=>x.s).sort((a,b)=>catV(b.s,b.m)-catV(a.s,a.m)).slice(0,3);
const jc=D.jev.categories.find(x=>x.id===c.id);
const series=[];
if(!(inIdx&&CHANCE())&&ids.some(id=>bench(id).baseline.value!=null))series.push({name:'Random baseline',color:'#8C8C8C',dash:'4 3',width:1.5,vals:ids.map(id=>inIdx&&bench(id).lower?0:bench(id).baseline.value)});
- series.push({name:'Jev',color:'#232323',fill:'rgba(35,35,35,.10)',width:3,vals:ids.map(id=>inIdx?idxVal(D.jev,id):jevResult(id)?.score??null)});
- top.forEach((t,i)=>series.push({name:t.m.long_name||t.m.name,color:SERIES[i],width:2.25,vals:ids.map(id=>inIdx?idxVal(t.m,id):adjusted(t.m,id))}));
+ series.push({name:'Jev',color:'#232323',fill:'rgba(35,35,35,.10)',width:3,vals:ids.map(id=>inIdx?headVal(D.jev,id):jevResult(id)?.score??null),private:inIdx&&!state.pub?ids.map(id=>combVal(D.jev,id)):null});
+ top.forEach((t,i)=>series.push({name:t.m.long_name||t.m.name,color:SERIES[i],width:2.25,vals:ids.map(id=>inIdx?headVal(t.m,id):adjusted(t.m,id)),private:inIdx&&!state.pub?ids.map(id=>combVal(t.m,id)):null}));
const chart=ids.length>=3?radarSVG(axes,series,{aria:c.label+' radar',big:true}):lollipopSVG(axes,series);
- const legend=[`Jev${one(jc[AK()]*100)} · reference`]
- .concat(top.map((t,i)=>`${i+1}${esc(t.m.long_name||t.m.name)}${one(t.s[AK()]*100)}`));
- const note=ids.length>=3?(inIdx?`${ids.length} spokes · the area's index benchmarks as the index scores them: ${CHANCE()?'chance-corrected (0 = random guessing), ':''}unanswered requests count as wrong${ids.some(id=>bench(id).lower)?', ForecastBench against its baseline':''}; their mean is the area score`:`${ids.length} spokes · each spoke is that task's own metric, unanswered requests count as wrong`):ids.length===1?`${bench(ids[0]).dataset} is the only static task here; the interactive games in this area sit outside this edition of the index`:`${ids.length} tasks in this area, too few for a radar · each bar is that task's own metric, unanswered requests count as wrong`;
- return `
${icon(c)}${esc(c.label)}
category score · Jev and the top 3 open models
+ const legend=[`Jev${one(catV(jc,D.jev)*100)} · reference`]
+ .concat(top.map((t,i)=>`${i+1}${esc(t.m.long_name||t.m.name)}${wnSup(t.m)}${one(catV(t.s,t.m)*100)}`));
+ const note=ids.length>=3?(inIdx?`${ids.length} spokes · the area's index benchmarks as the index scores them: ${CHANCE()?'chance-corrected (0 = random guessing), ':''}unanswered requests count as wrong${ids.some(id=>bench(id).lower)?', ForecastBench against its baseline':''}; their mean is the area score${ids.some(id=>bench(id).private)&&!state.pub?'. A + marks a task whose public score and its private test are combined on one scale, weighted as in '+(state.own?'your own mix':'the Full score'):''}`:`${ids.length} spokes · each spoke is that task's own metric, unanswered requests count as wrong`):ids.length===1?`${bench(ids[0]).dataset} is the only static task here; the interactive games in this area sit outside this edition of the index`:`${ids.length} tasks in this area, too few for a radar · each bar is that task's own metric, unanswered requests count as wrong`;
+ return `
${th('name','model')}${th('score',esc(state.areas.size?'Your index ✦':'Decision Index'),'r')}${areaTh}${th('ece','calibration','r')}${th('size','size','r')}${th('coverage','answered','r')}${th('latency','median latency','r')}
${body||'
No models in this size range.
'}
${hidden?``:''}`;
+ return `${areabar()}
#
${th('name','model')}${th('score',esc(state.areas.size?'Your index ✦':IDXNAME()),'r')}${areaTh}${th('ece','calibration','r')}${th('size','size','r')}${th('coverage','answered','r')}${th('latency','latency','r')}
${body||'
No models in this size range.
'}
${hidden?``:''}`;
}
let MEASURE=null;
function textW(t){if(!MEASURE){MEASURE=document.createElement('canvas').getContext('2d');MEASURE.font=`11.5px ${getComputedStyle(document.body).getPropertyValue('--body')||'sans-serif'}`}return MEASURE.measureText(t).width*1.06+3}
@@ -701,10 +905,10 @@ function drawIdxCal(rows){
const px=v=>L+(v-xmin)/(xmax-xmin||1)*(W-L-R),py=v=>T+v/(ymax||1)*(H-T-B);
let g=`calibration error under 10`;
const CUT=IDX_CUT();
- if(xmax>CUT)g+=`Decision Index ${CUT}+`;
+ if(xmax>CUT)g+=`${esc(IDXNAME())} ${CUT}+`;
for(const v of [0,10,20,30,40,50,60].filter(v=>v<=ymax))g+=`${v}`;
for(const v of [10,20,30,40,50,60].filter(v=>v>=xmin&&v<=xmax))g+=`${v}`;
- const attrs=r=>{const c=r.m.calibration;return `data-engine="${esc(r.engine)}" data-name="${esc(dname(r))}" data-raw="${one(r.raw)}" data-acc="${pct(c.acc)}" data-conf="${pct(c.conf)}" data-ece="${(c.ece*100).toFixed(1)}" data-over="${pct(c.over95)}" data-kind="${esc(r.meta?.kind||'')}" data-base="${esc(r.meta?.base_model||'')}" data-note="${r.jev?'probabilities as returned by the hosted API, rounded to two decimals':''}"`};
+ const attrs=r=>{const c=r.m.calibration;return `data-engine="${esc(r.engine)}" data-name="${esc(dname(r)+star(r))}" data-raw="${one(r.raw)}" data-acc="${pct(c.acc)}" data-conf="${pct(c.conf)}" data-ece="${(c.ece*100).toFixed(1)}" data-over="${pct(c.over95)}" data-kind="${esc(r.meta?.kind||'')}" data-base="${esc(r.meta?.base_model||'')}" data-note="${r.jev?'probabilities as returned by the hosted API, rounded to two decimals':''}"`};
const items=[],con=PARETO.cal,cfront=con?pareto(pts,r=>r.m.calibration.ece):[],cfs=new Set(cfront);
if(con)g+=paretoPath(cfront,r=>[px(r.raw),py(r.m.calibration.ece*100)],'y');
for(const r of pts.slice().sort((a,b)=>(b.jev?1:0)-(a.jev?1:0)||b.raw-a.raw)){
@@ -713,7 +917,7 @@ function drawIdxCal(rows){
items.push({cx,cy,r:rr,lab:r.name,raw:r.raw,size:r.size,skip:con&&!cfs.has(r)});
}
const labels=placeLabels(items,{W,H,L,R,T,B},[{x1:L,y1:T,x2:L+180,y2:T+16},{x1:W-R-130,y1:H-B-20,x2:W-R,y2:H-B}].concat(con?segRects(paretoSegs(cfront,r=>[px(r.raw),py(r.m.calibration.ece*100)],'y')):[]),con?{far:true,prefer:'upright'}:{max:20});
- host.innerHTML=``;
+ host.innerHTML=``;
}
const paretoBtn=k=>``;
function pareto(pts,cost){
@@ -751,26 +955,50 @@ function drawSize(rows){
if(on)g+=paretoPath(front,r=>[px(r.size),py(r.raw)],'x');
for(const r of pts.slice().sort((a,b)=>b.raw-a.raw)){
const cx=px(r.size),cy=py(r.raw);
- g+=``;
+ g+=``;
items.push({cx,cy,r:fs.has(r)?8:6,lab:r.name,raw:r.raw,size:r.size,skip:on&&!fs.has(r)});
}
const labs=placeLabels(items,{W,H,L,R,T,B},(jev?[{x1:L,y1:py(jev.raw)-20,x2:L+220,y2:py(jev.raw)+3}]:[]).concat(on?segRects(paretoSegs(front,r=>[px(r.size),py(r.raw)],'x')):[]),on?{far:true,prefer:'upleft'}:{max:20});
- host.innerHTML=`${note}`;
+ host.innerHTML=`${note}`;
+}
+function ownStrip(){const h=$('#ownctl');if(!h)return;if(!state.own||!OWN||state.areas.size){h.innerHTML='';return}
+ if(OWNOPEN==null)OWNOPEN=innerWidth>640;
+ const tot=OWNW.p.public+OWNW.p.same_skills+OWNW.p.new_domains||1,at=Object.values(OWNW.a).reduce((x,y)=>x+y,0)||1,pc=x=>Math.round(x*100)+'%';
+ const P=[['public','Public benchmarks'],['same_skills','Private tests, same skills'],['new_domains','Private tasks from new domains']];
+ const sl=(k,id,lab,v,eff)=>``;
+ if(!h.firstChild){h.innerHTML=`set your weights
Each area weight applies to its public benchmarks and to their private tests. Private tasks from new domains count as one block.
`}
+ h.querySelectorAll('input[data-own]').forEach(i=>{const v=OWNW[i.dataset.own][i.dataset.id];if(document.activeElement!==i)i.value=v});
+ h.querySelectorAll('[data-ov]').forEach(b=>{const [k,id]=b.dataset.ov.split(':');b.textContent=pc(OWNW[k][id]/(k==='p'?tot:at))});
+ h.querySelector('.osum').textContent=`Your own · ${pc(OWNW.p.public/tot)} public · ${pc(OWNW.p.same_skills/tot)} same-skill private · ${pc(OWNW.p.new_domains/tot)} new domains${ownIsDef()?' (the Full score mix)':''}`}
+let ownRaf=0;
+document.addEventListener('input',e=>{const i=e.target.closest&&e.target.closest('input[data-own]');if(!i)return;OWNW[i.dataset.own][i.dataset.id]=+i.value;ownCompute();ownStrip();cancelAnimationFrame(ownRaf);ownRaf=requestAnimationFrame(()=>{drawOverview();ownHash()})});
+document.addEventListener('click',e=>{if(e.target.closest('[data-ownreset]')){e.preventDefault();OWNW=ownDef();ownCompute();const h=$('#ownctl');if(h)h.innerHTML='';drawOverview();ownHash()}});
+document.addEventListener('toggle',e=>{if(e.target.classList&&e.target.classList.contains('ownbox'))OWNOPEN=e.target.open},true);
+function paintSwitch(){const cur=state.own&&MODE()!=='vision'?'own':state.pub?'pub':'v';document.querySelectorAll('button[data-pubmode]').forEach(b=>{const on=b.dataset.pubmode===cur;b.classList.toggle('on',on);b.setAttribute('aria-pressed',String(on))})}
+function paintToggle(){
+ paintSwitch();
+ ownStrip();
+ const k=$('#barkey'),n=$('#barnote');if(!k)return;
+ if(state.own&&!state.areas.size){k.innerHTML='Full scoreyour mix above FullFull above your mix▲▼ places moved vs the Full score';if(n)n.innerHTML='Your own weights, scored the same way as the Full score. ▲▼ shows how many places each model moves compared with the published Full score, and the dashed mark is its Full score.';return}
+ if(state.areas.size){k.innerHTML='';if(n)n.textContent=`Showing the mean of the chosen areas, ${state.pub?'public benchmarks only':state.own?'public and private combined with your part weights':'public and private combined as in the Full score'}.`;return}
+ k.innerHTML=state.pub?'Full scorepublic-only leadFull above public▲▼ places moved vs the Full score':'';
+ if(n)n.innerHTML=state.pub?'The public benchmarks alone. ▲▼ shows how many places each model moves compared with the Full score, and the dashed mark is its Full score.':`The Full score: ${pctW('public')} public benchmarks, ${pctW('same_skills')} private tests of the same skills and ${pctW('new_domains')} private tasks from new domains, on one scale. Hover or tap any bar for its parts. How it works →`;
}
function drawOverview(){
const rows=sorted(ROWS.map(r=>({...r,score:scoreOf(r),raw:scoreOf(r)}))).filter(sizePasses);
drawCharts(rows);
+ if(V3){paintToggle();const cg=$('.cats');if(cg)cg.innerHTML=overviewCard()+AREAS().map(categoryCard).join('')}
drawSize(rows);
drawIdxCal(rows);
$('#table').innerHTML=tableHTML(rows);
- const cap=$('#barcap'),cap2=$('#scattercap');if(cap)cap.textContent=customLabel();if(cap2)cap2.textContent='Median latency vs '+customLabel();const c3=$('#sizecap'),c4=$('#idxcalcap');if(c3)c3.textContent='Model size vs '+customLabel();if(c4)c4.textContent='Calibration error vs '+customLabel();
+ const cap=$('#barcap'),cap2=$('#scattercap');if(cap){cap.textContent=customLabel();cap.hidden=!!V3&&!state.areas.size}if(cap2)cap2.textContent='Median latency vs '+customLabel();const c3=$('#sizecap'),c4=$('#idxcalcap');if(c3)c3.textContent='Model size vs '+customLabel();if(c4)c4.textContent='Calibration error vs '+customLabel();
}
function benchRows(id){
const b=bench(id),lower=b.lower;
const rows=D.models.map(m=>{const r=modelResult(m,id);if(!r||r.score==null)return null;const cov=coverageOf(m,id);
- return {engine:m.engine,name:m.long_name||m.name,m,meta:m.meta,score:r.score,adj:lower?r.score:r.score*cov,cov,jev:r.jev,lat:r.median_ms,tracks:r.tracks}}).filter(Boolean);
- const j=jevResult(id);if(j&&j.score!=null)rows.push({engine:'jev',name:'Jev',isJev:true,m:D.jev,meta:null,score:j.score,adj:j.score,cov:1,lat:j.median_ms});
+ const bm=m.benchmarks&&m.benchmarks[String(id)];return {engine:m.engine,name:m.long_name||m.name,m,meta:m.meta,priv:bm&&bm.private_eq!=null?bm.private_eq:null,score:r.score,adj:lower?r.score:r.score*cov,cov,jev:r.jev,lat:r.median_ms,tracks:r.tracks}}).filter(Boolean);
+ const j=jevResult(id),jb=D.jev.benchmarks&&D.jev.benchmarks[String(id)];if(j&&j.score!=null)rows.push({engine:'jev',name:'Jev',isJev:true,m:D.jev,priv:jb&&jb.private_eq!=null?jb.private_eq:null,meta:null,score:j.score,adj:j.score,cov:1,lat:j.median_ms});
rows.sort((x,y)=>lower?x.adj-y.adj:y.adj-x.adj);
return rows;
}
@@ -780,9 +1008,10 @@ function benchCard(id){
const inIndex=D.categories.some(x=>x.panel.includes(+id));
const jevAt=rows.findIndex(r=>r.isJev);
const shown=rows.slice(0,5);if(jevAt>=5)shown.push(rows[jevAt]);
- const li=(r,i)=>`
${i+1}${esc(r.name)}${pct(r.adj)}
`;
- return `
${esc(b.dataset)}${goldMark(b)}${inIndex?'✦ in the index':noIndex(b)}
${esc(b.metric||'')}${b.baseline.value!=null?` · random ${pct(b.baseline.value)}`:''}
+ also assessed on a private benchmark of the same competency; the private column shows each model's score there in raw accuracy, the same units as the public column.
`:''}${refLinks(b)}
${esc(b.metric||'')}${b.baseline.value!=null?` · random baseline ${pct(b.baseline.value)} (${esc(b.baseline.label||'')})`:''}${b.cases?` · ${fmt(b.cases)} cases`:''}${b.lower?' · lower is better':''}
-
-
#
model
score
jev · same cases
answered
median
${tr}
+
+
#
model
score
${b.private?'
private
':''}${showJev?'
jev · same cases
':''}
answered
median
${tr}
Ranked on the coverage-adjusted score: a request the model could not answer counts as wrong, so the number above matches what feeds the index. Where the model answered everything the two are identical.
`;
}
function openBench(id,push){
- if(!D||!D.benchmarks[String(id)])return;
+ if(!D||!D.benchmarks[String(id)]||(IDX.size&&!IDX.has(+id)))return;
let dlg=$('#benchdlg');
if(!dlg){dlg=document.createElement('dialog');dlg.id='benchdlg';document.body.appendChild(dlg);
dlg.addEventListener('click',e=>{if(e.target===dlg||e.target.closest('.dlg-close'))closeBench(true)});
@@ -830,7 +1061,7 @@ function starburst(){
return ``;
}
function whatsNew(){
- const items=D.suite.changes||[];
+ const items=V3?[`The main ranking is the Full score: ${pctW('public')} public benchmarks, ${pctW('same_skills')} private tests of the same skills and ${pctW('new_domains')} private tasks from new domains.`,`Private tests and new-domain private tasks are put on the public scale using ${V3.formula.equating.bases} untuned stock models, so every part counts in the same units.`,'Most of the score comes from tests nobody can train on, so a public-only advantage counts for little.',`Models within ${V3.formula.tie_band} points of the first model in their group share a rank: a gap that small could come from which questions happened to be sampled.`,`The public index is rebuilt: ${V3.public_index_note}; WinoGrande now counts under Knowledge & Reasoning.`,'The Vision board is live: switch to Vision at the top of the page.']:(D.suite.changes||[]);
return `
New in ${esc(EDITION().label)}
${items.map(x=>`
${esc(x)}
`).join('')}
Full changelog →`;
}
function wireWhatsNew(){
@@ -851,18 +1082,20 @@ function renderOverview(){
const arch=new Set(D.models.map(m=>m.meta.technique).filter(Boolean));
$('#app').innerHTML=`
Decision Index ${esc(E.num)}${E.base?whatsNew():''}
-
${D.models.length} open reproductions of TypeSafe Jev's "Decision Model" (zero-shot classification) on the same benchmark suite, requesting ${Math.round(D.suite.requests/1000)}K decisions per model across ${D.suite.benchmarks} benchmarks.
-
suite v${esc(E.num)} · Jev ${esc(D.suite.jev_version||'')} · reproductions run on ${esc(D.suite.hardware)} · updated ${esc((D.generated_utc||'').slice(0,10))}
-
-
${sizebar()}
-
Decision Index
+ ${V3?v3Hero():`
${D.models.length} open reproductions of TypeSafe Jev's "Decision Model" (zero-shot classification) on the same benchmark suite, requesting ${Math.round(D.suite.requests/1000)}K decisions per model across ${D.suite.benchmarks} benchmarks.
`}
+ ${modeSwitch()}
+
${V3?`v${esc(E.num)} · ${V3.models.length} models`:`suite v${esc(E.num)}`} · Jev ${esc(D.suite.jev_version||'')} · reproductions run on ${esc(D.suite.hardware)} · updated ${esc(((V3&&V3.generated_utc)||D.generated_utc||'').slice(0,10))}
+
+
${sizebar()}
${V3?'
'+(OWN?'':'')+'
':''}
+
Decision Index${V3?'':''}${wnFoot()}
${paretoBtn('size')}Model size vs Decision Index${paretoBtn('cal')}Calibration error vs Decision Index${paretoBtn('lat')}Median latency vs Decision Index
scroll →
-
Head to head
Jev against the three strongest open models in each capability area. Every spoke is one benchmark.
+ ${V3?v3GapSection():''}
+
Head to head
Jev against the three strongest models in each capability area. Every spoke is one benchmark.
Click a model for its complete benchmark-by-benchmark page. Sort any column.
-
${sizebar()}
+
${sizebar()}${V3?`
${OWN?``:''}
`:''}
Answered is the share of the ${fmt(D.suite.requests)} requests the model produced a valid typed answer for; unsupported requests count as wrong in every index. Calibration is the expected calibration error in points, lower is better. MMLU, GPQA and BFCL show each model's own accuracy on the cases it answered. Jev's latency is a hosted HTTP round trip and is not a controlled comparison with local GPU runs.
Calibration: confidence vs reality
Every answer comes with a probability. Left: how sure each model says it is against how often it is right. Right: the average gap between the two, lower is better.
stated confidence vs accuracycalibration ladder · expected calibration error
@@ -870,9 +1103,10 @@ function renderOverview(){
All benchmarks
Every task in the suite with its own ranking. Click one for the full table and what the task asks.
+ ${V3?v3Teasers():''}
How the index is computed
One number per model, from the same ${D.suite.panel}-benchmark panel. Weights are fixed; nothing is tuned to the entrants. Full methodology →
-
Decision Index
100 × mean of the ${WORDS[AREAS().length]||AREAS().length} area scores
${CHANCE()?'Each benchmark is chance-corrected first: 0 is random guessing, 100 is perfect. ':''}Each capability area is the mean of its benchmarks' primary metrics, so every area counts the same however many tasks it holds.${(D.suite.lower_rules||[]).map(x=>` ${esc(x.name)} counts against its baseline: ${esc(x.baseline_label)} scores zero.`).join('')}
+
${V3?'Full score':'Decision Index'}
${V3?`${V3.weights.public.toFixed(2)} × public + ${V3.weights.same_skills.toFixed(2)} × same-skill private + ${V3.weights.new_domains.toFixed(2)} × new domains`:`100 × mean of the ${WORDS[AREAS().length]||AREAS().length} area scores`}${V3?'
The public part is 100 × the weighted mean of the area scores, with GSM8K rebuilt and ForecastBench retired. The private parts come from tests of the same skills and from new-domain private tasks; only totals are shown. Both are put on the public scale using untuned stock models before weighting. How it works →
':'
'}${CHANCE()?'Each benchmark is chance-corrected first: 0 is random guessing, 100 is perfect. ':''}${V3?'Each capability area is the mean of its benchmarks\' primary metrics, and the areas are combined with fixed weights.':'Each capability area is the mean of its benchmarks\' primary metrics, so every area counts the same however many tasks it holds.'}${(D.suite.lower_rules||[]).map(x=>` ${esc(x.name)} counts against its baseline: ${esc(x.baseline_label)} scores zero.`).join('')}
Coverage counts
unanswered = wrong
Every model faces the same ${fmt(D.suite.requests)} requests. A request the model cannot take, refuses or errors on scores zero, so nobody climbs by skipping the hard cases.
Same cases as Jev
frozen suite · frozen answers
Jev's own results were recorded once and never re-run. On every model page each result sits beside Jev's answer on exactly the same cases.
@@ -888,7 +1122,7 @@ function renderOverview(){
function drawCalibration(){
const host=$('#calscatter'),lad=$('#calladder');if(!host||!lad)return;
const rows=ROWS.filter(r=>r.m&&r.m.calibration);
- const attrs=r=>{const c=r.m.calibration;return `data-engine="${esc(r.engine)}" data-name="${esc(dname(r))}" data-raw="${one(r.raw)}" data-acc="${pct(c.acc)}" data-conf="${pct(c.conf)}" data-ece="${(c.ece*100).toFixed(1)}" data-over="${pct(c.over95)}" data-kind="${esc(r.meta?.kind||'')}" data-base="${esc(r.meta?.base_model||'')}" data-note="${r.jev?'probabilities as returned by the hosted API, rounded to two decimals':''}"`};
+ const attrs=r=>{const c=r.m.calibration;return `data-engine="${esc(r.engine)}" data-name="${esc(dname(r)+star(r))}" data-raw="${one(r.raw)}" data-acc="${pct(c.acc)}" data-conf="${pct(c.conf)}" data-ece="${(c.ece*100).toFixed(1)}" data-over="${pct(c.over95)}" data-kind="${esc(r.meta?.kind||'')}" data-base="${esc(r.meta?.base_model||'')}" data-note="${r.jev?'probabilities as returned by the hosted API, rounded to two decimals':''}"`};
const W=560,H=520,L=46,R=14,T=14,B=44,x0=0.4,y0=0.2;
const px=v=>L+(v-x0)/(1-x0)*(W-L-R),py=v=>H-B-(v-y0)/(1-y0)*(H-T-B);
let g=``;
@@ -913,14 +1147,23 @@ function drawCalibration(){
byece.forEach((r,i)=>{const c=r.m.calibration,y=LT+i*RH,e=c.ece*100;
l+=``;
l+=`${e.toFixed(1)}`;
- const nm=r.name.length>24?r.name.slice(0,23)+'…':r.name;l+=`${esc(nm)}`;
+ const nm=(r.name.length>24?r.name.slice(0,23)+'…':r.name)+star(r),wt=wnote(r)?`${esc(wnote(r))}`:'';l+=`${wt}${esc(nm)}`;
const ox=lx(c.over95*100);l+=``});
- lad.innerHTML=`
`;
}
@@ -960,15 +1204,15 @@ function modelCategory(subject,c){
const series=[];
if(axesIds.some(id=>bench(id).baseline.value!=null))series.push({name:'Random baseline',color:'#8C8C8C',dash:'4 3',width:1.5,vals:axesIds.map(id=>bench(id).baseline.value)});
if(isJev)series.push({name:'Jev',color:'#232323',fill:'rgba(35,35,35,.12)',width:3,vals:axesIds.map(id=>jevResult(id)?.score??null)});
- else{series.push({name:'Jev on the same cases',color:'#232323',width:2.5,dash:'',vals:axesIds.map(id=>modelResult(subject,id)?.jev??null)});
+ else{if(jevSame(subject))series.push({name:'Jev on the same cases',color:'#232323',width:2.5,dash:'',vals:axesIds.map(id=>modelResult(subject,id)?.jev??null)});
series.push({name:subject.long_name||subject.name,color:colorOf({meta:subject.meta}),fill:colorOf({meta:subject.meta})+'24',width:3,vals:axesIds.map(id=>modelResult(subject,id)?.score??null)});}
const cs=(isJev?D.jev:subject).categories.find(x=>x.id===c.id)||null;
const jc=D.jev.categories.find(x=>x.id===c.id)||null;
const chart=axes.length>=3?radarSVG(axes,series,{aria:c.label}):(axes.length?lollipopSVG(axes,series):'
No static benchmark in this category.
');
return `
-
${esc(c.label)}
${ordered.length} benchmarks${cs?` · index area score ${one(cs[AK()]*100)}`:' · outside the index'}${isJev||!jc?'':` · Jev ${one(jc[AK()]*100)}`}
+
${esc(c.label)}
${ordered.length} benchmarks${cs?` · area score ${one(catV(cs,isJev?D.jev:subject)*100)} (${V3?(state.pub?'public only':state.own?'your own mix':'public and private'):'public'})`:' · outside the index'}${isJev||!jc?'':` · Jev ${one(catV(jc,D.jev)*100)}`}
${isJev?`${D.suite.jev_benchmarks} static benchmarks and six interactive environments, scored in a comparable manner against the random baseline. This is the reference every reproduction is measured against.`:`Rank #${rankText(rank[engine],true)} of ${ROWS.length} on the Decision Index. Scored on the same frozen ${fmt(D.suite.requests)}-decision suite as Jev, with Jev's answers on exactly the same cases beside each result.`}
${isJev?(V3?`Scored like every reproduction: the ${D.suite.panel} public benchmarks in ${AREAS().length} areas, the private tests of the same skills and the private tasks from new domains. This is the reference every reproduction is measured against.`:`${D.suite.jev_benchmarks} static benchmarks and six interactive environments, scored in a comparable manner against the random baseline. This is the reference every reproduction is measured against.`):`${V?`Rank #${rankText(v3Rank(engine),true)} of ${V3.models.length} on the Full score.`:`Rank #${rankText(rank[engine],true)} of ${ROWS.length} on the Decision Index.`} Scored on the same frozen ${fmt(D.suite.requests)}-decision suite as Jev${jevSame(subject)?", with Jev's answers on exactly the same cases beside each result":''}.`}
- ${tile(one(s[HK()]),'Decision Index',[isJev?null:`Jev ${one(J[HK()])}`,CHANCE()?`raw accuracy ${one(s.balanced_raw)} (not chance-corrected)`:null].filter(Boolean).join(' · ')||null,true)}
- ${tile(isJev?'ref':'#'+(rank[engine]?rank[engine].n:'—'),isJev?'reference':'rank of '+ROWS.length+(rank[engine]?.tied?' · tied':''),isJev?'the model every row is measured against':(prev?`${esc(dname(prev))} is #${rankText(rank[prev.engine],true)}`:'top of the index'))}
+ ${tile(one(s[HK()]),V?'Full score':'Decision Index',[isJev?null:`Jev ${one(J[HK()])}`,V?`public ${one(s.public_skill)}`:null,CHANCE()?`raw accuracy ${one(s.balanced_raw)} (not chance-corrected)`:null].filter(Boolean).join(' · ')||null,true)}
+ ${V?tile((t=>t.startsWith('=')?t:'#'+t)(String(rankText(v3Rank(engine)))),'rank of '+V3.models.length+' · Full score'+(V.tied?' · tied':''),isJev?'the model every row is measured against':(prev?`${esc(prev.name)} is ${(t=>t.startsWith('=')?t:'#'+t)(String(rankText(v3Rank(prev.engine))))}`:'top of the board')):tile(isJev?'ref':'#'+(rank[engine]?rank[engine].n:'—'),isJev?'reference':'rank of '+ROWS.length+(rank[engine]?.tied?' · tied':''),isJev?'the model every row is measured against':(prev?`${esc(dname(prev))} is #${rankText(rank[prev.engine],true)}`:'top of the index'))}
${tile(pct(cov),'requests answered',isJev?`${fmt(D.suite.requests)} requests`:`${fmt(subject.counts.ok||0)} of ${fmt(D.suite.requests)}${gapLine(subject)?info('Why not all of them? '+gapLine(subject)):''}`)}
${tile(ms(subject.latency.median),'median latency',isJev?'HTTP round trip':`p95 ${ms(subject.latency.p95)} · ${esc(D.suite.hardware)}`)}
${standingsHTML(engine)}
+ ${v3Block(engine)}
Where it is strong
Mean of each area's primary metric over every benchmark ${isJev?'in the suite':'this model ran, beside Jev on the same cases'}. The index itself uses a fixed ${D.suite.panel}-task subset.
${esc((WORDS[areasFor(subject).length]||'').replace(/^./,ch=>ch.toUpperCase()))} capability areas. Each opens with five highlighted tasks; expand an area for every result.
Each benchmark reports its own primary metric, so scores are not comparable across rows. The solid tick on each bar is that task's random baseline${isJev?'':', the dashed tick is Jev on the very same cases'}, and the gap above random is the comparable figure. Brier loss is lower-is-better and its gap is signed accordingly.
+
Each benchmark reports its own primary metric, so scores are not comparable across rows. The solid tick on each bar is that task's random baseline${isJev||!jevSame(subject)?'':', the dashed tick is Jev on the very same cases'}, and the gap above random is the comparable figure. Brier loss is lower-is-better and its gap is signed accordingly.
`;
+ if(state.pub&&!state.areas.size){const r1=x=>Math.round(x*10)/10,P=v.pub,g=r1(r1(P)-r1(S)),lvl=Math.abs(g)<2;
+ const head=`Public ${one(P)} · same skills ${one(S)} — ${lvl?'about level':g>0?`${one(g)} higher on public`:`${one(-g)} higher on the private tests`}`;
+ return `
${esc(d.name)}${by}
${head}
${dotStrip(P,S,O,v.v)}
New domains ${one(O)} — tasks outside these skills, not a check on the public ones.
${mixLine()}
${tail}
`}
+ return `${esc(d.name)}${by}${ow}${row('Full score',one(v.v))}${row('Public benchmarks',one(v.pub))}${row('Private tests, same skills',one(S))}${row('Private tasks from new domains',one(O))}
${mixLine()}
${tail}`;
+}
function tipHTML(d,touch){
- return d.ece!=null?`${esc(d.name)}
`}`:modelTip(d,touch);
}
function tipHandler(e){
const c=e.target.closest&&e.target.closest('circle.pt,rect.bar[data-name]');if(!c)return;const host=c.closest('#bars,#calladder');if(!host)return;const tip=host.querySelector('.tip');if(!tip)return;
- tip.innerHTML=tipHTML(c.dataset);
+ tip.innerHTML=tipHTML(c.dataset,lastPointer==='touch');
tip.hidden=false;
const hb=host.getBoundingClientRect(),cb=c.getBoundingClientRect(),tb=tip.getBoundingClientRect();
let left=cb.left-hb.left+cb.width+10,top=cb.top-hb.top-6;
if(left+tb.width>hb.width)left=cb.left-hb.left-tb.width-10;
+ left=Math.max(0,Math.min(left,hb.width-tb.width));
if(top+tb.height>hb.height)top=hb.height-tb.height;
tip.style.left=Math.max(0,left)+'px';tip.style.top=Math.max(0,top)+'px';
}
const SCAT='#sizescatter,#scatter,#idxcal,#calscatter';
-let HOT=null,lastPointer='mouse';
+let HOT=null,lastPointer='mouse',BARHOT=null;
const ptCenter=c=>{const b=c.getBoundingClientRect();return [b.left+b.width/2,b.top+b.height/2,b.width/2]};
function nearestPt(host,x,y,rad){let best=null,bd=Infinity;for(const c of host.querySelectorAll('circle.pt')){const [cx,cy,r]=ptCenter(c),d=Math.hypot(cx-x,cy-y)-r;if(d{
history.scrollRestoration='manual';
window.addEventListener('popstate',e=>route(true,e.state?e.state.y:null));
let lastNarrow=null,rsz=null;
-window.addEventListener('resize',()=>{clearTimeout(rsz);rsz=setTimeout(()=>{const sc=$('#scatter');if(!sc||!D||currentEngine())return;const n=sc.clientWidth<640;if(lastNarrow!==null&&n!==lastNarrow)drawOverview();lastNarrow=n},150)});
-fetch('data/index.json?v='+Math.floor(Date.now()/6e5)).then(r=>r.json()).then(d=>{D=d;ROWS=buildRows();applyEdition();
- if(location.hash){const e=currentEngine();history.replaceState({y:0},'',e?`?model=${encodeURIComponent(e)}`:location.pathname)}
- route(false)}).catch(err=>{$('#app').innerHTML=`
Could not load data/index.json · ${esc(err.message)}
`});
+window.addEventListener('resize',()=>{clearTimeout(rsz);rsz=setTimeout(()=>{const sc=$('#scatter');if(!sc||!D||currentEngine())return;const n=sc.clientWidth<640;if(lastNarrow!==null&&n!==lastNarrow){drawOverview()}lastNarrow=n},150)});
+let V3=null;
+const v3Row=e=>V3?V3.models.find(r=>r.engine===e):null;
+const v3Order=()=>V3.models.slice().sort((a,b)=>a.rank_v-b.rank_v);
+const v3Rank=e=>{const r=v3Row(e);return r?{n:r.rank_tie||r.rank_v,tied:!!r.tied}:null};
+const gsign=v=>(v>0.049?'+':v<-0.049?'−':'')+Math.abs(v).toFixed(1);
+const pctW=k=>Math.round(V3.weights[k]*100)+'%';
+const mixLine=()=>{if(state.own&&OWNW){const p=OWNW.p,t=(p.public+p.same_skills+p.new_domains)||1,f=x=>Math.round(x/t*100)+'%';return `Your own = public ${f(p.public)} · same skills ${f(p.same_skills)} · new domains ${f(p.new_domains)}, all on one scale.`}return `Full score = public ${pctW('public')} · same skills ${pctW('same_skills')} · new domains ${pctW('new_domains')}, all on one scale.`};
+const v3Swatch=r=>{const row=ROWS.find(x=>x.engine===r.engine);return row?colorOf(row):'#bbb'};
+function v3Hero(){
+ return `
${D.models.length} open reproductions of TypeSafe Jev's "Decision Model", ranked by the Full score: ${D.suite.benchmarks} public benchmarks plus private tests that measure the same skills and a set of private tasks from new domains.
For models that can choose to think before answering the hard questions. Thinking takes longer, so these models run slower than the main board allows and get a board of their own.
Three parts, each on the same scale, added with fixed weights. Nothing is tuned to any one model.
+
+
+
1 · Public benchmarks${pctW('public')} of the Full score
The ${N} open tasks shown on this site. Anyone can see them, and train on them.
+
2 · Private tests, same skills${pctW('same_skills')} of the Full score
For each public task with a private counterpart, a test of an equivalent competence built from fresh data and never published. It carries the most weight because nobody can train on it.
+
3 · Private tasks from new domains${pctW('new_domains')} of the Full score
A set of private decision tasks from new domains. Some reward broad knowledge, others reward judging when to answer yes and when to answer no.
+
+
The three parts spread models out differently, so their raw points are not the same unit. Before adding the parts up, each one is put on the public scale using ${E.bases} untuned stock models as the yardstick: a model that sits one spread above the stock average on the private tests gets the score that sits one spread above the stock average on the public benchmarks. Order within each part never changes.
`;
+ const w=V3.weights,p=r.parts||{},eqd=(raw,eq)=>eq!=null&&Math.abs(eq-raw)>0.05?`raw score · ${one(eq)} on the public scale`:'raw score';
+ const tiles=`
+ ${tile(one(r.v),'Full score',`${w.public.toFixed(2)} × public + ${w.same_skills.toFixed(2)} × private (same skills) + ${w.new_domains.toFixed(2)} × new domains, on one scale`,true)}
+ ${tile(one(r.pub),'public benchmarks',`#${r.rank_pub} on the public board alone`)}
+ ${tile(one(r.same_skills),'private tests, same skills',eqd(r.same_skills,r.eq&&r.eq.S))}
+ ${tile(one(r.new_domains),'private tasks from new domains',eqd(r.new_domains,r.eq&&r.eq.O))}
${seg('public','Public benchmarks','#8F8B82')}${seg('same_skills','Private tests, same skills','#232323')}${seg('new_domains','New-domain private tasks','#B8B2A6')}
+
Points contributed: public ${(p.public||0).toFixed(1)} · private tests ${(p.same_skills||0).toFixed(1)} · new domains ${(p.new_domains||0).toFixed(1)} = ${one(r.v)}
`:'';
+ const intro=r.jev?'Jev is the reference model and is scored exactly like every reproduction.':'The private tests measure the same skills as the public benchmarks with fresh data that stays private. Only totals are shown.';
+ return `
`;
+ const rk=r.ref?'':row('Rank',`#${state.pub?r.rank_pub:r.rank_full} of ${VIS.entrants.length}`);
+ return `${esc(r.name)}${r.ref?' reference':''}${row('Full score',one(r.full))}${row('Public',one(r.pub))}${row('Private set for same skill',one(r.priv))}${r.base?row('Base: public / private',`${one(r.base.pub)} / ${one(r.base.priv)}`)+row('Base checkpoint',`${esc(r.base.name)}`):''}${rk}${r.ref?`
${esc((VIS.bench_info&&VIS.bench_info[n]||{}).tech||'')} ${VIS.bench_rows&&VIS.bench_rows[n]?fmt(VIS.bench_rows[n].public)+' public items.':''} Scored as chance-corrected skill × 100: 0 is random guessing, 100 is perfect.${tw?' A private set tests the same skill with new items; bars show the public benchmark, the dashed mark the private benchmark of the same competency, and combined is their mean. Ranked by the public score.':''}
+
+
#
model
public
${tw?'
private
combined
':''}
${tr}
`;
+ if(!dlg.open)dlg.showModal();dlg.querySelector('.dlg-body').scrollTop=0;
+}
+const vLab=r=>r.label?` · ${r.label}`:'';
+const vSize=r=>r.params?`${r.params_estimated?'~':''}${size(r.params)}${r.active?' total · '+r.active.replace(/^A/,'')+' active':''}`:'—';
+function vScat(host,o){
+ if(!host)return;
+ const all=vAll(),pts=all.filter(r=>o.x(r)!=null),off=all.filter(r=>o.x(r)==null);
+ const nar=host.clientWidth<480,W=520,H=520,L=46,R=14,T=16,B=50;
+ const xs=pts.map(o.x),ys=pts.map(o.y);
+ const lx=v=>o.log?Math.log10(v):v;
+ const xmin=o.log?Math.min(...xs)/1.35:Math.max(0,Math.min(...xs)-2),xmax=o.log?Math.max(...xs)*1.35:Math.max(...xs)+2;
+ const ymin=o.ymin!=null?o.ymin:Math.max(0,Math.floor((Math.min(...ys)-4)/10)*10),ymax=o.ymax!=null?o.ymax:Math.ceil((Math.max(...ys)+4)/10)*10;
+ const px=v=>L+(lx(v)-lx(xmin))/(lx(xmax)-lx(xmin)||1)*(W-L-R),py=v=>o.flipY?T+(v-ymin)/(ymax-ymin||1)*(H-T-B):H-B-(v-ymin)/(ymax-ymin||1)*(H-T-B);
+ let g=o.band?o.band(px,py,{W,H,L,R,T,B,xmin,xmax,ymin,ymax}):'';
+ const on=!!PARETO[o.key],front=[];if(on){let best=-Infinity;for(const r of pts.slice().sort((a,b)=>o.x(a)-o.x(b)||o.y(b)-o.y(a)))if(o.y(r)>best){front.push(r);best=o.y(r)}}const fs=new Set(front),pos=r=>[px(o.x(r)),py(o.y(r))];
+ for(let v=ymin;v<=ymax;v+=10)g+=`${v}`;
+ for(const v of o.xticks.filter(v=>v>=xmin&&v<=xmax))g+=`${o.xfmt(v)}`;
+ const items=[];if(on)g+=paretoPath(front,pos,'x');
+ for(const r of pts.slice().sort((a,b)=>(a.ref?1:0)-(b.ref?1:0)||o.y(b)-o.y(a))){const cx=px(o.x(r)),cy=py(o.y(r)),rr=(r.ref?7.5:6)+(fs.has(r)?2:0);
+ g+=``;
+ items.push({cx,cy,r:rr,lab:r.ref?r.name:r.name.replace(/ \(.*\)$/,''),raw:r.ref?999:o.y(r),skip:on&&!fs.has(r)})}
+ const labels=placeLabels(items,{W,H,L,R,T,B},(o.reserve||[]).concat(on?segRects(paretoSegs(front,pos,'x')):[]),on?{far:true,prefer:'upleft'}:{max:nar?10:16});
+ host.innerHTML=`${o.note||off.length?`
`:''}`;
+}
+function drawVCharts(){
+ const vn=vName();
+ vScat($('#vsize'),{key:'vsize',x:r=>r.params,y:vScore,log:true,xticks:[1e9,3e9,1e10,3e10,1e11,3e11],xfmt:v=>size(v),xcap:'parameters (total; MoE active count in the name) · log scale',ycap:`vision ${vn}`,aria:`Model size vs vision ${vn}`,note:'MoE models are placed at their total parameter count. SigLIP 2 size is estimated from its weight file.'});
+ vScat($('#vcalx'),{key:'vcal',x:r=>r.cal?r.cal.ece*100:null,y:vScore,xticks:[0,5,10,15,20,25],xfmt:v=>v,xcap:'calibration error, points (lower is better)',ycap:`vision ${vn}`,aria:`Calibration error vs vision ${vn}`,note:'Calibration uses each model\'s letter probabilities on the public vision sets. Qwen3.5-397B-A17B (API, generated letter only) and SigLIP 2 (similarity scores, not probabilities) have none.'});
+ vScat($('#vspeed'),{key:'vspeed',x:r=>r.ms_row,y:vScore,log:true,xticks:[30,50,100,200,500,1000,2000],xfmt:v=>v+' ms',xcap:'GPU time per row, scoring only · log scale',ycap:`vision ${vn}`,aria:`GPU time per row vs vision ${vn}`,note:'GPU time per row = scoring GPU-hours ÷ rows on RTX PRO 6000. Intern-Decision-0.8B includes 7.4 h spent before a slow-kernel fix. The 397B reference ran through an API and SigLIP 2 logged no GPU time.'});
+ ['#vsizecap','#vcalxcap','#vspeedcap'].forEach((id,i)=>{const c=$(id);if(c)c.textContent=['Model size','Calibration error','GPU time per row'][i]+' vs vision '+vn});
+}
+function drawVCalibration(){
+ const host=$('#vcal'),lad=$('#vladder');if(!host||!lad)return;
+ const rows=vAll().filter(r=>r.cal);
+ const W=560,H=520,L=46,R=14,T=14,B=44,x0=0.4,y0=0.4;
+ const px=v=>L+(v-x0)/(1-x0)*(W-L-R),py=v=>H-B-(v-y0)/(1-y0)*(H-T-B);
+ let g=``;
+ for(const v of [0.4,0.5,0.6,0.7,0.8,0.9,1]){g+=`${Math.round(v*100)}%${Math.round(v*100)}%`}
+ g+=`over-confidentunder-confident`;
+ const items=[];for(const r of rows.slice().sort((a,b)=>(a.ref?1:0)-(b.ref?1:0))){const cx=px(r.cal.conf),cy=py(r.cal.acc);g+=``;items.push({cx,cy,r:6,lab:r.ref?r.name:r.name.replace(/ \(.*\)$/,''),raw:r.full})}
+ host.innerHTML=``;
+ const by=rows.slice().sort((a,b)=>a.cal.ece-b.cal.ece),LW=560,NL=170,RH=20,LT=8,LH=LT+by.length*RH+34,maxE=Math.max(...by.map(r=>r.cal.ece))*100,lx=v=>NL+v/(maxE*1.12)*(LW-NL-40);
+ let l='';for(const v of [0,5,10,15,20,25].filter(v=>v<=maxE*1.05))l+=`${v}`;
+ by.forEach((r,i)=>{const y=LT+i*RH,e=r.cal.ece*100;l+=`${e.toFixed(1)}`;
+ const nm=r.name.length>34?r.name.slice(0,33)+'…':r.name;l+=`${esc(nm)}`;
+ const ox=lx(r.cal.over95*100);l+=``});
+ lad.innerHTML=``;
+}
+function vRadar(){
+ const top=VIS.entrants.slice().sort((a,b)=>vScore(b)-vScore(a)).slice(0,3),ref=VIS.refs.find(r=>r.engine==='ref-qwen35-397b'),names=vBenchNames();
+ const short={'KIE (CORD+FUNSD)':'KIE','Moderation (Hateful Memes)':'Moderation','MMMU-Pro vision':'MMMU-Pro','InfographicVQA':'InfoVQA'};
+ const val=(r,n)=>{const b=r.bench[n];if(!b)return null;return (state.pub||b.private==null?b.pub:(b.pub+b.private)/2)/100};
+ const axes=names.map(n=>({label:(short[n]||n)+(vTwin(n)&&!state.pub?'+':''),id:null}));
+ const series=[{name:ref.name,color:'#232323',fill:'rgba(35,35,35,.10)',width:3,vals:names.map(n=>val(ref,n))}].concat(top.map((r,i)=>({name:r.name,color:SERIES[i],width:2.25,vals:names.map(n=>val(r,n))})));
+ const rev={};names.forEach(n=>rev[short[n]||n]=n);
+ const svg=radarSVG(axes,series,{aria:'Vision benchmarks radar',big:true}).replace(/ data-bench="[^"]*"/g,'').replace(/class="axlab"([^>]*)>([^<]*)<\/text>/g,(m,a,t)=>`class="vaxlab" data-vbi="${esc(rev[t.replace(/\+$/,'')]||'')}"${a}>${t}`);
+ const legend=series.map((s,i)=>`${i?`${i}`:''}${esc(s.name)}${one(i?vScore(top[i-1]):vScore(ref))}${i?'':' · reference'}`).join('');
+ return `
All vision tasks
${vName()} · the top 3 and the largest reference
${svg}
${legend}
${names.length} spokes, one per vision benchmark, chance-corrected skill. ${state.pub?'Public benchmarks only.':'A + marks a task scored as the mean of public and the private set for same skill.'}
${VIS.entrants.length} vision-capable reproductions of TypeSafe Jev's "Decision Model", ranked by the vision Full score: ${vBenchNames().length} public vision benchmarks plus a private set that measures the same skills.
+ ${modeSwitch()}
+
vision board · ${VIS.entrants.length} models + ${VIS.refs.length} references · each entrant through its own image path · updated ${esc(VIS.generated_utc.slice(0,10))}
+
+
${vLegend()}
+
${esc(VREFNOTE)}
+
${paretoBtn('vsize')}Model size vs vision Full score${paretoBtn('vcal')}Calibration error vs vision Full score${paretoBtn('vspeed')}GPU time per row vs vision Full score
scroll →
+ Learn more: the vision private set
+
+
The private set is harderevery model drops
The private tasks use fresh images and are harder than their public counterparts, so every model scores lower on them, and stronger models drop more points.
+
A drop alone is not a red flagstock models drop too
A stock vision-language model, ${esc(q.name)}, drops ${(q.pub-q.priv).toFixed(1)} points from public to private, more than any fine-tune with a base (largest ${maxDrop.toFixed(1)}). So a drop on its own does not mean a model was tuned to the public benchmarks.
+
The tasks need a VLM${esc(sg.name)}: ${one(sg.pub)} / ${one(sg.priv)}
${esc(sg.name)} matches images to text without a language model and scores near zero on both, so the tasks cannot be solved by image-text similarity alone.
+
+
References
+
${VIS.refs.map(r=>`
${esc(r.name)} (${esc(r.kind)}): public ${one(r.pub)} / private ${one(r.priv)}. ${esc(r.note)}
`).join('')}
+
Run notes for individual models are in methodology.
+
+
Head to head
The strongest vision entrants against the largest reference, one spoke per benchmark.
+
+
Full results
Every vision entrant and the references. Stock bases are shown beside the fine-tunes, not as separate rows.
+
+
Calibration: confidence vs reality
Left: how sure each model says it is against how often it is right, on the public vision sets. Right: the average gap between the two, lower is better.
+
stated confidence vs accuracycalibration ladder · expected calibration error
+
Calibration uses the probability each model gives its chosen letter, over ten confidence bins, with every benchmark weighted equally; accuracy counts answered rows. The diamond is the share of answers given with at least 95% confidence that were wrong. Qwen3.5-397B-A17B answered through an API with a generated letter only, and SigLIP 2 gives similarity scores rather than probabilities, so neither has calibration.
+
All benchmarks
Each vision task with its own ranking. A "+" marks a task that also has a private benchmark of the same competency.
-
-
- 
':''}