{ "meta": { "id": "_parity", "title": "Parity against the previous Space's published figures", "sources": [ { "repo": "hf-space:Vineethsain/defenseclaw-slm-toolcall", "path": "_build-figures.json", "sha256": "ac634d586f16d12dc325ec6672317870fd689608d410de0ba77dd87764c90da8", "commit": "3c4ba84da3f1884a8e9119cb21e5402d7421258a" }, { "repo": "hf-space:Vineethsain/defenseclaw-slm-toolcall", "path": "index.html", "sha256": "54a9d6cba7ab2b288e0818b8cc57e41e7fc9e63b54b6fc0251979ca63f531110", "commit": "3c4ba84da3f1884a8e9119cb21e5402d7421258a" } ], "extracted_at": "2026-10-16T14:45:21+00:00", "notes": "Every key of the Space's _build-figures.json (its exact-value manifest) rebuilt from the data files in this directory with the Space's own formatting rules, plus the leaderboard's data-x attributes checked against leaderboard.json." }, "data": { "figures_published": 577, "counts": { "matched": 577, "mismatched": 0, "missing": 0 }, "figures": { "acc.beat": { "published": "3", "reproduced": "3", "match": true }, "acc.gain": { "published": "12", "reproduced": "12", "match": true }, "acc.s2.allow": { "published": "0.8857741681949175", "reproduced": "0.8857741681949175", "match": true }, "acc.s3.allow": { "published": "0.9909707468540612", "reproduced": "0.9909707468540612", "match": true }, "arith.bytes": { "published": "22,586", "reproduced": "22,586", "match": true }, "arith.driver": { "published": "benchmarks/scripts/benchmark_run_system_one.py", "reproduced": "benchmarks/scripts/benchmark_run_system_one.py", "match": true }, "arith.importers": { "published": "3", "reproduced": "3", "match": true }, "arith.path": { "published": "benchmarks/system_one/reproduce/07-analysis/rescoring/remine.py", "reproduced": "benchmarks/system_one/reproduce/07-analysis/rescoring/remine.py", "match": true }, "arith.sha": { "published": "97df17a30891446d94a9df923ec343a6c7bd29adea3435b0303e4604440d5a6f", "reproduced": "97df17a30891446d94a9df923ec343a6c7bd29adea3435b0303e4604440d5a6f", "match": true }, "arith.verified": { "published": "2026-09-23", "reproduced": "2026-09-23", "match": true }, "auc.above": { "published": "20", "reproduced": "20", "match": true }, "auc.below": { "published": "2", "reproduced": "2", "match": true }, "auc.coincide": { "published": "22", "reproduced": "22", "match": true }, "auc.inside": { "published": "0", "reproduced": "0", "match": true }, "band.best.mid": { "published": "granite-4.0-micro", "reproduced": "granite-4.0-micro", "match": true }, "band.best.mid.f1": { "published": "0.07775377969762419", "reproduced": "0.07775377969762419", "match": true }, "band.best.mid.params.short": { "published": "3.40B", "reproduced": "3.40B", "match": true }, "band.best.mid.tp": { "published": "18", "reproduced": "18", "match": true }, "band.best.under": { "published": "falcon3-1b-instruct", "reproduced": "falcon3-1b-instruct", "match": true }, "band.hi": { "published": "0.528794503868809", "reproduced": "0.528794503868809", "match": true }, "band.lo": { "published": "0.471205496131191", "reproduced": "0.471205496131191", "match": true }, "band.max.model": { "published": "gemma-3-4b-it", "reproduced": "gemma-3-4b-it", "match": true }, "band.max.params": { "published": "4,300,079,472", "reproduced": "4,300,079,472", "match": true }, "band.max.params.short": { "published": "4.30B", "reproduced": "4.30B", "match": true }, "band.mid": { "published": "8", "reproduced": "8", "match": true }, "band.min.model": { "published": "prompt-guard-2-22m", "reproduced": "prompt-guard-2-22m", "match": true }, "band.min.params": { "published": "70,830,722", "reproduced": "70,830,722", "match": true }, "band.min.params.short": { "published": "70.8M", "reproduced": "70.8M", "match": true }, "band.se": { "published": "0.014691343359335349", "reproduced": "0.014691343359335349", "match": true }, "band.spread": { "published": "61", "reproduced": "61", "match": true }, "band.under": { "published": "14", "reproduced": "14", "match": true }, "band.upper": { "published": "0", "reproduced": "0", "match": true }, "beat.control": { "published": "12", "reproduced": "12", "match": true }, "beat.control.oracle": { "published": "7", "reproduced": "7", "match": true }, "beat.control.shipped": { "published": "12", "reproduced": "12", "match": true }, "best.boot.hi": { "published": "0.14285714285714285", "reproduced": "0.14285714285714285", "match": true }, "best.boot.lo": { "published": "0.06746031746031746", "reproduced": "0.06746031746031746", "match": true }, "boot.n": { "published": "2,000", "reproduced": "2,000", "match": true }, "boot.seed": { "published": "20260924", "reproduced": "20260924", "match": true }, "cal.buckets": { "published": "10", "reproduced": "10", "match": true }, "cal.empty": { "published": "69", "reproduced": "69", "match": true }, "cal.sparse": { "published": "51", "reproduced": "51", "match": true }, "cap.deberta.tp": { "published": "1", "reproduced": "1", "match": true }, "cap.exact": { "published": "0.00384502", "reproduced": "0.00384502", "match": true }, "cap.f1": { "published": "0.10548523206751055", "reproduced": "0.10548523206751055", "match": true }, "cap.f1.accuracy": { "published": "0.8889179984280848", "reproduced": "0.8889179984280848", "match": true }, "cap.f1.band": { "published": "under 3B", "reproduced": "under 3B", "match": true }, "cap.f1.fp": { "published": "13", "reproduced": "13", "match": true }, "cap.f1.missed": { "published": "411", "reproduced": "411", "match": true }, "cap.f1.model": { "published": "falcon3-1b-instruct", "reproduced": "falcon3-1b-instruct", "match": true }, "cap.f1.params": { "published": "1,669,408,768", "reproduced": "1,669,408,768", "match": true }, "cap.f1.params.short": { "published": "1.67B", "reproduced": "1.67B", "match": true }, "cap.f1.precision": { "published": "0.6578947368421053", "reproduced": "0.6578947368421053", "match": true }, "cap.f1.recall": { "published": "0.05733944954128441", "reproduced": "0.05733944954128441", "match": true }, "cap.f1.threshold": { "published": "0.6170100906975968", "reproduced": "0.6170100906975968", "match": true }, "cap.f1.tp": { "published": "25", "reproduced": "25", "match": true }, "cap.maxfp": { "published": "13", "reproduced": "13", "match": true }, "cap.thresholds": { "published": "22", "reproduced": "22", "match": true }, "cap.value": { "published": "0.00384502", "reproduced": "0.00384502", "match": true }, "cb.auc": { "published": "0.6622467295653802", "reproduced": "0.6622467295653802", "match": true }, "cb.controlled": { "published": "0.4876428937684744", "reproduced": "0.4876428937684744", "match": true }, "cb.controlled.where": { "published": "inside", "reproduced": "inside", "match": true }, "cb.distinct": { "published": "53", "reproduced": "53", "match": true }, "cb.f1": { "published": "0.2496318114874816", "reproduced": "0.2496318114874816", "match": true }, "cb.fn": { "published": "97", "reproduced": "97", "match": true }, "cb.fp": { "published": "1,941", "reproduced": "1,941", "match": true }, "cb.fpr": { "published": "0.5740905057675244", "reproduced": "0.5740905057675244", "match": true }, "cb.oracle": { "published": "0.30849478390462", "reproduced": "0.30849478390462", "match": true }, "cb.precision": { "published": "0.14868421052631578", "reproduced": "0.14868421052631578", "match": true }, "cb.raw.where": { "published": "above", "reproduced": "above", "match": true }, "cb.recall": { "published": "0.7775229357798165", "reproduced": "0.7775229357798165", "match": true }, "cb.shipped": { "published": "0.2496318114874816", "reproduced": "0.2496318114874816", "match": true }, "cb.spearman": { "published": "0.597867523607295", "reproduced": "0.597867523607295", "match": true }, "cb.tn": { "published": "1,440", "reproduced": "1,440", "match": true }, "cb.tp": { "published": "339", "reproduced": "339", "match": true }, "cl.auc": { "published": "0.46228994190416495", "reproduced": "0.46228994190416495", "match": true }, "cl.controlled": { "published": "0.38895947698618566", "reproduced": "0.38895947698618566", "match": true }, "cl.controlled.where": { "published": "below", "reproduced": "below", "match": true }, "cl.distinct": { "published": "46", "reproduced": "46", "match": true }, "cl.f1": { "published": "0.18932748538011696", "reproduced": "0.18932748538011696", "match": true }, "cl.fn": { "published": "177", "reproduced": "177", "match": true }, "cl.fp": { "published": "2,041", "reproduced": "2,041", "match": true }, "cl.fpr": { "published": "0.603667553978113", "reproduced": "0.603667553978113", "match": true }, "cl.precision": { "published": "0.1126086956521739", "reproduced": "0.1126086956521739", "match": true }, "cl.raw.where": { "published": "below", "reproduced": "below", "match": true }, "cl.recall": { "published": "0.5940366972477065", "reproduced": "0.5940366972477065", "match": true }, "cl.tn": { "published": "1,340", "reproduced": "1,340", "match": true }, "cl.tp": { "published": "259", "reproduced": "259", "match": true }, "control-modernbert-base.f1": { "published": "0.2496318114874816", "reproduced": "0.2496318114874816", "match": true }, "control-modernbert-base.fn": { "published": "97", "reproduced": "97", "match": true }, "control-modernbert-base.fp": { "published": "1,941", "reproduced": "1,941", "match": true }, "control-modernbert-base.fpr": { "published": "0.5740905057675244", "reproduced": "0.5740905057675244", "match": true }, "control-modernbert-base.precision": { "published": "0.14868421052631578", "reproduced": "0.14868421052631578", "match": true }, "control-modernbert-base.recall": { "published": "0.7775229357798165", "reproduced": "0.7775229357798165", "match": true }, "control-modernbert-base.tn": { "published": "1,440", "reproduced": "1,440", "match": true }, "control-modernbert-base.tp": { "published": "339", "reproduced": "339", "match": true }, "control-modernbert-large.f1": { "published": "0.18932748538011696", "reproduced": "0.18932748538011696", "match": true }, "control-modernbert-large.fn": { "published": "177", "reproduced": "177", "match": true }, "control-modernbert-large.fp": { "published": "2,041", "reproduced": "2,041", "match": true }, "control-modernbert-large.fpr": { "published": "0.603667553978113", "reproduced": "0.603667553978113", "match": true }, "control-modernbert-large.precision": { "published": "0.1126086956521739", "reproduced": "0.1126086956521739", "match": true }, "control-modernbert-large.recall": { "published": "0.5940366972477065", "reproduced": "0.5940366972477065", "match": true }, "control-modernbert-large.tn": { "published": "1,340", "reproduced": "1,340", "match": true }, "control-modernbert-large.tp": { "published": "259", "reproduced": "259", "match": true }, "controls.pearson": { "published": "0.1435084459826658", "reproduced": "0.1435084459826658", "match": true }, "corpus.cases": { "published": "4,277", "reproduced": "4,277", "match": true }, "corpus.gradeA": { "published": "17", "reproduced": "17", "match": true }, "corpus.gradeB": { "published": "419", "reproduced": "419", "match": true }, "corpus.gradeC": { "published": "460", "reproduced": "460", "match": true }, "corpus.negatives": { "published": "3,381", "reproduced": "3,381", "match": true }, "corpus.positives": { "published": "436", "reproduced": "436", "match": true }, "corpus.prev.ratio": { "published": "12.65×", "reproduced": "12.65×", "match": true }, "corpus.prevalence": { "published": "11.42%", "reproduced": "11.42%", "match": true }, "corpus.prevalence.exact": { "published": "0.11422583180508253", "reproduced": "0.11422583180508253", "match": true }, "corpus.rows": { "published": "30,310", "reproduced": "30,310", "match": true }, "corpus.scorable": { "published": "3,817", "reproduced": "3,817", "match": true }, "corpus.sha": { "published": "39f2c1df2369952a0525cc4c5575f4bdb590fb3ca8c1bc6805cf4f376c1adbf7", "reproduced": "39f2c1df2369952a0525cc4c5575f4bdb590fb3ca8c1bc6805cf4f376c1adbf7", "match": true }, "cost.models": { "published": "13", "reproduced": "13", "match": true }, "cost.rss.model": { "published": "phi-4-mini-instruct", "reproduced": "phi-4-mini-instruct", "match": true }, "cost.snapshot.spread": { "published": "30", "reproduced": "30", "match": true }, "ctrl.oracle": { "published": "0.30849478390462", "reproduced": "0.30849478390462", "match": true }, "ctrl.verdict": { "published": "did not hold as stated: control-modernbert-large lands below the chance band at 0.38895947698618566", "reproduced": "did not hold as stated: control-modernbert-large lands below the chance band at 0.38895947698618566", "match": true }, "ctrl.verdict.short": { "published": "not met", "reproduced": "not met", "match": true }, "cue.ctxbytes": { "published": "0.7266059794480217", "reproduced": "0.7266059794480217", "match": true }, "cue.ctxevents": { "published": "0.7445374041120237", "reproduced": "0.7445374041120237", "match": true }, "cue.events": { "published": "0.7772553177633239", "reproduced": "0.7772553177633239", "match": true }, "cue.list": { "published": "event count 0.7772553177633239, prompt tokens, max over events 0.7692264380822134, prompt tokens, summed over events 0.7665974048175314, context events 0.7445374041120237 and context bytes, max over events 0.7266059794480217", "reproduced": "event count 0.7772553177633239, prompt tokens, max over events 0.7692264380822134, prompt tokens, summed over events 0.7665974048175314, context events 0.7445374041120237 and context bytes, max over events 0.7266059794480217", "match": true }, "cue.max": { "published": "0.7772553177633239", "reproduced": "0.7772553177633239", "match": true }, "cue.max.name": { "published": "event count", "reproduced": "event count", "match": true }, "cue.n": { "published": "5", "reproduced": "5", "match": true }, "cue.tokens": { "published": "0.7692264380822134", "reproduced": "0.7692264380822134", "match": true }, "cue.tokens.controlled": { "published": "0.47090471737950174", "reproduced": "0.47090471737950174", "match": true }, "cue.tokens.controlled.where": { "published": "below", "reproduced": "below", "match": true }, "cue.tokens.name": { "published": "prompt tokens, max over events", "reproduced": "prompt tokens, max over events", "match": true }, "curve.aucs": { "published": "28", "reproduced": "28", "match": true }, "curve.models": { "published": "22", "reproduced": "22", "match": true }, "curve.points": { "published": "6,515", "reproduced": "6,515", "match": true }, "curve.points.max": { "published": "711", "reproduced": "711", "match": true }, "curve.thresholds.max": { "published": "3,076", "reproduced": "3,076", "match": true }, "curve.thresholds.min": { "published": "14", "reproduced": "14", "match": true }, "deb.auc": { "published": "0.8390509973434926", "reproduced": "0.8390509973434926", "match": true }, "deb.blockcases": { "published": "3,629", "reproduced": "3,629", "match": true }, "deb.blockrows": { "published": "27,460", "reproduced": "27,460", "match": true }, "deb.blockshare": { "published": "95.1%", "reproduced": "95.1%", "match": true }, "deb.controlled": { "published": "0.8347427912167461", "reproduced": "0.8347427912167461", "match": true }, "deb.distinct": { "published": "864", "reproduced": "864", "match": true }, "deb.f1": { "published": "0.21057810578105782", "reproduced": "0.21057810578105782", "match": true }, "deb.fn": { "published": "8", "reproduced": "8", "match": true }, "deb.fp": { "published": "3,201", "reproduced": "3,201", "match": true }, "deb.fp.cb": { "published": "1,941", "reproduced": "1,941", "match": true }, "deb.fp.cl": { "published": "2,041", "reproduced": "2,041", "match": true }, "deb.fpr": { "published": "0.9467613132209406", "reproduced": "0.9467613132209406", "match": true }, "deb.lc.rank": { "published": "1", "reproduced": "1", "match": true }, "deb.oracle": { "published": "0.48", "reproduced": "0.48", "match": true }, "deb.oracle.threshold": { "published": "0.9987551566979116", "reproduced": "0.9987551566979116", "match": true }, "deb.over.floor": { "published": "0.005546363481504557", "reproduced": "0.005546363481504557", "match": true }, "deb.params": { "published": "184,423,682", "reproduced": "184,423,682", "match": true }, "deb.params.short": { "published": "184M", "reproduced": "184M", "match": true }, "deb.precision": { "published": "0.11793882612289887", "reproduced": "0.11793882612289887", "match": true }, "deb.recall": { "published": "0.981651376146789", "reproduced": "0.981651376146789", "match": true }, "deb.revision": { "published": "90c9989b1a342275dd0d1a95aad283c04e075671", "reproduced": "90c9989b1a342275dd0d1a95aad283c04e075671", "match": true }, "deb.sha": { "published": "927baa5ccc8bbf9b7885381ed50781264f848ab1c666e053166f9aab1721eeb4", "reproduced": "927baa5ccc8bbf9b7885381ed50781264f848ab1c666e053166f9aab1721eeb4", "match": true }, "deb.shipped": { "published": "0.21057810578105782", "reproduced": "0.21057810578105782", "match": true }, "deb.tn": { "published": "180", "reproduced": "180", "match": true }, "deb.tp": { "published": "428", "reproduced": "428", "match": true }, "deberta-v3-prompt-injection-v2.f1": { "published": "0.21057810578105782", "reproduced": "0.21057810578105782", "match": true }, "deberta-v3-prompt-injection-v2.fn": { "published": "8", "reproduced": "8", "match": true }, "deberta-v3-prompt-injection-v2.fp": { "published": "3,201", "reproduced": "3,201", "match": true }, "deberta-v3-prompt-injection-v2.fpr": { "published": "0.9467613132209406", "reproduced": "0.9467613132209406", "match": true }, "deberta-v3-prompt-injection-v2.precision": { "published": "0.11793882612289887", "reproduced": "0.11793882612289887", "match": true }, "deberta-v3-prompt-injection-v2.recall": { "published": "0.981651376146789", "reproduced": "0.981651376146789", "match": true }, "deberta-v3-prompt-injection-v2.tn": { "published": "180", "reproduced": "180", "match": true }, "deberta-v3-prompt-injection-v2.tp": { "published": "428", "reproduced": "428", "match": true }, "est.disagree1": { "published": "B_own_input_tokens_quintiles_unweighted_mean", "reproduced": "B_own_input_tokens_quintiles_unweighted_mean", "match": true }, "est.disagree3": { "published": "5", "reproduced": "5", "match": true }, "est.n": { "published": "4", "reproduced": "4", "match": true }, "est.overweight": { "published": "43.6", "reproduced": "43.6", "match": true }, "est.published": { "published": "pair-weighted pooled", "reproduced": "pair-weighted pooled", "match": true }, "est.q0.pos": { "published": "2", "reproduced": "2", "match": true }, "est.q0.share": { "published": "0.459%", "reproduced": "0.459%", "match": true }, "est.q0.wpool": { "published": "0.505%", "reproduced": "0.505%", "match": true }, "est.q0.wunw": { "published": "20%", "reproduced": "20%", "match": true }, "est.rank1": { "published": "deberta-v3-prompt-injection-v2", "reproduced": "deberta-v3-prompt-injection-v2", "match": true }, "est.rank2": { "published": "shieldgemma-2b", "reproduced": "shieldgemma-2b", "match": true }, "est.rank3": { "published": "granite-guardian-3.2-3b-a800m", "reproduced": "granite-guardian-3.2-3b-a800m", "match": true }, "est.rank3.ge10": { "published": "shieldstral-1.0-3b", "reproduced": "shieldstral-1.0-3b", "match": true }, "est.rank3.ge25": { "published": "granite-guardian-3.2-3b-a800m", "reproduced": "granite-guardian-3.2-3b-a800m", "match": true }, "est.rank3.plurality": { "published": "shieldstral-1.0-3b", "reproduced": "shieldstral-1.0-3b", "match": true }, "est.rank3.pooled": { "published": "granite-guardian-3.2-3b-a800m", "reproduced": "granite-guardian-3.2-3b-a800m", "match": true }, "est.rank3.unw": { "published": "shieldstral-1.0-3b", "reproduced": "shieldstral-1.0-3b", "match": true }, "est.schemes": { "published": "9", "reproduced": "9", "match": true }, "est.seratio": { "published": "2.816880566493238", "reproduced": "2.816880566493238", "match": true }, "est.seworse": { "published": "0", "reproduced": "0", "match": true }, "est.source": { "published": "the ranking artifact's own estimator.authoritative field", "reproduced": "the ranking artifact's own estimator.authoritative field", "match": true }, "est.unwfam": { "published": "6", "reproduced": "6", "match": true }, "falcon3-1b-instruct.f1": { "published": "0.33271719038817005", "reproduced": "0.33271719038817005", "match": true }, "falcon3-1b-instruct.fn": { "published": "256", "reproduced": "256", "match": true }, "falcon3-1b-instruct.fp": { "published": "466", "reproduced": "466", "match": true }, "falcon3-1b-instruct.fpr": { "published": "0.1378290446613428", "reproduced": "0.1378290446613428", "match": true }, "falcon3-1b-instruct.precision": { "published": "0.2786377708978328", "reproduced": "0.2786377708978328", "match": true }, "falcon3-1b-instruct.recall": { "published": "0.41284403669724773", "reproduced": "0.41284403669724773", "match": true }, "falcon3-1b-instruct.tn": { "published": "2,915", "reproduced": "2,915", "match": true }, "falcon3-1b-instruct.tp": { "published": "180", "reproduced": "180", "match": true }, "falcon3-3b-instruct.f1": { "published": "0.12439418416801293", "reproduced": "0.12439418416801293", "match": true }, "falcon3-3b-instruct.fn": { "published": "359", "reproduced": "359", "match": true }, "falcon3-3b-instruct.fp": { "published": "725", "reproduced": "725", "match": true }, "falcon3-3b-instruct.fpr": { "published": "0.21443359952676722", "reproduced": "0.21443359952676722", "match": true }, "falcon3-3b-instruct.precision": { "published": "0.09600997506234414", "reproduced": "0.09600997506234414", "match": true }, "falcon3-3b-instruct.recall": { "published": "0.17660550458715596", "reproduced": "0.17660550458715596", "match": true }, "falcon3-3b-instruct.tn": { "published": "2,656", "reproduced": "2,656", "match": true }, "falcon3-3b-instruct.tp": { "published": "77", "reproduced": "77", "match": true }, "floor.allow.f1": { "published": "0.0", "reproduced": "0.0", "match": true }, "floor.f1": { "published": "0.20503174229955326", "reproduced": "0.20503174229955326", "match": true }, "floor.f1.exact": { "published": "0.20503174229955326", "reproduced": "0.20503174229955326", "match": true }, "floor.f1s": { "published": "0.20503", "reproduced": "0.20503", "match": true }, "floor.fn": { "published": "0", "reproduced": "0", "match": true }, "floor.fp": { "published": "3,381", "reproduced": "3,381", "match": true }, "floor.fpr": { "published": "1.0", "reproduced": "1.0", "match": true }, "floor.precision": { "published": "0.11422583180508253", "reproduced": "0.11422583180508253", "match": true }, "floor.recall": { "published": "1.0", "reproduced": "1.0", "match": true }, "floor.tn": { "published": "0", "reproduced": "0", "match": true }, "floor.tp": { "published": "436", "reproduced": "436", "match": true }, "fpr.wilson.max": { "published": "0.0065677323742142305", "reproduced": "0.0065677323742142305", "match": true }, "fpr.wilson.over": { "published": "14", "reproduced": "14", "match": true }, "gate.artifacts": { "published": "20", "reproduced": "20", "match": true }, "gate.asserts": { "published": "2,251", "reproduced": "2,251", "match": true }, "gate.charts": { "published": "22", "reproduced": "22", "match": true }, "gate.cleared": { "published": "2026-09-23", "reproduced": "2026-09-23", "match": true }, "gemma-3-1b-it.f1": { "published": "0.20748867159551634", "reproduced": "0.20748867159551634", "match": true }, "gemma-3-1b-it.fn": { "published": "1", "reproduced": "1", "match": true }, "gemma-3-1b-it.fp": { "published": "3,322", "reproduced": "3,322", "match": true }, "gemma-3-1b-it.fpr": { "published": "0.9825495415557527", "reproduced": "0.9825495415557527", "match": true }, "gemma-3-1b-it.precision": { "published": "0.11578387010912962", "reproduced": "0.11578387010912962", "match": true }, "gemma-3-1b-it.recall": { "published": "0.9977064220183486", "reproduced": "0.9977064220183486", "match": true }, "gemma-3-1b-it.tn": { "published": "59", "reproduced": "59", "match": true }, "gemma-3-1b-it.tp": { "published": "435", "reproduced": "435", "match": true }, "gm.bytes": { "published": "8,600,277,880", "reproduced": "8,600,277,880", "match": true }, "gm.dead": { "published": "419,816,304", "reproduced": "419,816,304", "match": true }, "gm.gib": { "published": "8.0096", "reproduced": "8.0096", "match": true }, "gm.projector": { "published": "2,950,272", "reproduced": "2,950,272", "match": true }, "gm.share": { "published": "9.76%", "reproduced": "9.76%", "match": true }, "gm.text": { "published": "3,880,263,168", "reproduced": "3,880,263,168", "match": true }, "gm.vision": { "published": "416,866,032", "reproduced": "416,866,032", "match": true }, "granite-4.0-1b.f1": { "published": "0.2584493041749503", "reproduced": "0.2584493041749503", "match": true }, "granite-4.0-1b.fn": { "published": "306", "reproduced": "306", "match": true }, "granite-4.0-1b.fp": { "published": "440", "reproduced": "440", "match": true }, "granite-4.0-1b.fpr": { "published": "0.13013901212658976", "reproduced": "0.13013901212658976", "match": true }, "granite-4.0-1b.precision": { "published": "0.22807017543859648", "reproduced": "0.22807017543859648", "match": true }, "granite-4.0-1b.recall": { "published": "0.2981651376146789", "reproduced": "0.2981651376146789", "match": true }, "granite-4.0-1b.tn": { "published": "2,941", "reproduced": "2,941", "match": true }, "granite-4.0-1b.tp": { "published": "130", "reproduced": "130", "match": true }, "granite-4.0-micro.f1": { "published": "0.2507462686567164", "reproduced": "0.2507462686567164", "match": true }, "granite-4.0-micro.fn": { "published": "352", "reproduced": "352", "match": true }, "granite-4.0-micro.fp": { "published": "150", "reproduced": "150", "match": true }, "granite-4.0-micro.fpr": { "published": "0.044365572315882874", "reproduced": "0.044365572315882874", "match": true }, "granite-4.0-micro.precision": { "published": "0.358974358974359", "reproduced": "0.358974358974359", "match": true }, "granite-4.0-micro.recall": { "published": "0.1926605504587156", "reproduced": "0.1926605504587156", "match": true }, "granite-4.0-micro.tn": { "published": "3,231", "reproduced": "3,231", "match": true }, "granite-4.0-micro.tp": { "published": "84", "reproduced": "84", "match": true }, "granite-guardian-3.1-2b.f1": { "published": "0", "reproduced": "0", "match": true }, "granite-guardian-3.1-2b.fn": { "published": "436", "reproduced": "436", "match": true }, "granite-guardian-3.1-2b.fp": { "published": "0", "reproduced": "0", "match": true }, "granite-guardian-3.1-2b.fpr": { "published": "0", "reproduced": "0", "match": true }, "granite-guardian-3.1-2b.precision": { "published": "n/a", "reproduced": "n/a", "match": true }, "granite-guardian-3.1-2b.recall": { "published": "0", "reproduced": "0", "match": true }, "granite-guardian-3.1-2b.tn": { "published": "3,381", "reproduced": "3,381", "match": true }, "granite-guardian-3.1-2b.tp": { "published": "0", "reproduced": "0", "match": true }, "granite-guardian-3.2-3b-a800m.f1": { "published": "0.36666666666666664", "reproduced": "0.36666666666666664", "match": true }, "granite-guardian-3.2-3b-a800m.fn": { "published": "227", "reproduced": "227", "match": true }, "granite-guardian-3.2-3b-a800m.fp": { "published": "495", "reproduced": "495", "match": true }, "granite-guardian-3.2-3b-a800m.fpr": { "published": "0.14640638864241348", "reproduced": "0.14640638864241348", "match": true }, "granite-guardian-3.2-3b-a800m.precision": { "published": "0.296875", "reproduced": "0.296875", "match": true }, "granite-guardian-3.2-3b-a800m.recall": { "published": "0.4793577981651376", "reproduced": "0.4793577981651376", "match": true }, "granite-guardian-3.2-3b-a800m.tn": { "published": "2,886", "reproduced": "2,886", "match": true }, "granite-guardian-3.2-3b-a800m.tp": { "published": "209", "reproduced": "209", "match": true }, "heldout.floor.f1": { "published": "0.017896910555937968", "reproduced": "0.017896910555937968", "match": true }, "heldout.models": { "published": "6", "reproduced": "6", "match": true }, "heldout.overlap": { "published": "0", "reproduced": "0", "match": true }, "heldout.rows": { "published": "100,001", "reproduced": "100,001", "match": true }, "heldout.sha": { "published": "0ccbc08fb408ffefc89e051c585cfe22433b1ed4c9714f60cc0f7e242969fa03", "reproduced": "0ccbc08fb408ffefc89e051c585cfe22433b1ed4c9714f60cc0f7e242969fa03", "match": true }, "heldout.stage0": { "published": "3", "reproduced": "3", "match": true }, "labels.bytes": { "published": "8,768", "reproduced": "8,768", "match": true }, "labels.fn": { "published": "truth_grade", "reproduced": "truth_grade", "match": true }, "labels.path": { "published": "benchmarks/scripts/benchmark_inventory_system_one_sources.py", "reproduced": "benchmarks/scripts/benchmark_inventory_system_one_sources.py", "match": true }, "labels.sha": { "published": "b1c66462dfd8a6cbb8a3332e5e548774e10e4ba426facdfb43cf5fbacf194995", "reproduced": "b1c66462dfd8a6cbb8a3332e5e548774e10e4ba426facdfb43cf5fbacf194995", "match": true }, "lap.deberta.arm": { "published": "deberta-v3-prompt-injection-v2", "reproduced": "deberta-v3-prompt-injection-v2", "match": true }, "lap.deberta.rpm": { "published": "186.7", "reproduced": "186.7", "match": true }, "lap.encoder.ratio": { "published": "6.1", "reproduced": "6.1", "match": true }, "lap.fastest": { "published": "OLMo-2-1B", "reproduced": "OLMo-2-1B", "match": true }, "lap.fastest.arm": { "published": "olmo-2-1b-instruct", "reproduced": "olmo-2-1b-instruct", "match": true }, "lap.fastest.pass": { "published": "1h39m", "reproduced": "1h39m", "match": true }, "lap.fastest.rpm": { "published": "30.38", "reproduced": "30.38", "match": true }, "lap.headroom": { "published": "4.863", "reproduced": "4.863", "match": true }, "lap.models": { "published": "14", "reproduced": "14", "match": true }, "lap.modernbert.arm": { "published": "control-modernbert-base", "reproduced": "control-modernbert-base", "match": true }, "lap.modernbert.rpm": { "published": "44.6", "reproduced": "44.6", "match": true }, "lap.pass.best": { "published": "1h48m", "reproduced": "1h48m", "match": true }, "lap.published": { "published": "13", "reproduced": "13", "match": true }, "lap.q4max": { "published": "2.323", "reproduced": "2.323", "match": true }, "lap.q4max.key": { "published": "phi-4-mini-instruct", "reproduced": "phi-4-mini-instruct", "match": true }, "lap.q4max.model": { "published": "Phi-4-mini", "reproduced": "Phi-4-mini", "match": true }, "lap.q4min": { "published": "0.871", "reproduced": "0.871", "match": true }, "lap.q4min.key": { "published": "olmo-2-1b-instruct", "reproduced": "olmo-2-1b-instruct", "match": true }, "lap.q4min.model": { "published": "OLMo-2-1B", "reproduced": "OLMo-2-1B", "match": true }, "lap.rpm.best": { "published": "27.67", "reproduced": "27.67", "match": true }, "lap.rpm.shieldstral": { "published": "9.91", "reproduced": "9.91", "match": true }, "lap.rss": { "published": "3.137", "reproduced": "3.137", "match": true }, "lap.rss.key": { "published": "phi-4-mini-instruct", "reproduced": "phi-4-mini-instruct", "match": true }, "lap.rss.mmap": { "published": "5.048", "reproduced": "5.048", "match": true }, "lap.rss.model": { "published": "Phi-4-mini", "reproduced": "Phi-4-mini", "match": true }, "lap.slowest": { "published": "granite-4.0-micro", "reproduced": "granite-4.0-micro", "match": true }, "lap.slowest.arm": { "published": "granite-4.0-micro", "reproduced": "granite-4.0-micro", "match": true }, "lap.slowest.pass": { "published": "5h12m", "reproduced": "5h12m", "match": true }, "lap.slowest.rpm": { "published": "9.61", "reproduced": "9.61", "match": true }, "lg.categories": { "published": "13", "reproduced": "13", "match": true }, "llama-3.2-1b-instruct.f1": { "published": "0.25620094191522763", "reproduced": "0.25620094191522763", "match": true }, "llama-3.2-1b-instruct.fn": { "published": "28", "reproduced": "28", "match": true }, "llama-3.2-1b-instruct.fp": { "published": "2,341", "reproduced": "2,341", "match": true }, "llama-3.2-1b-instruct.fpr": { "published": "0.6923986986098787", "reproduced": "0.6923986986098787", "match": true }, "llama-3.2-1b-instruct.precision": { "published": "0.14841760640232812", "reproduced": "0.14841760640232812", "match": true }, "llama-3.2-1b-instruct.recall": { "published": "0.9357798165137615", "reproduced": "0.9357798165137615", "match": true }, "llama-3.2-1b-instruct.tn": { "published": "1,040", "reproduced": "1,040", "match": true }, "llama-3.2-1b-instruct.tp": { "published": "408", "reproduced": "408", "match": true }, "llama-3.2-3b-instruct.f1": { "published": "0.3548387096774194", "reproduced": "0.3548387096774194", "match": true }, "llama-3.2-3b-instruct.fn": { "published": "194", "reproduced": "194", "match": true }, "llama-3.2-3b-instruct.fp": { "published": "686", "reproduced": "686", "match": true }, "llama-3.2-3b-instruct.fpr": { "published": "0.2028985507246377", "reproduced": "0.2028985507246377", "match": true }, "llama-3.2-3b-instruct.precision": { "published": "0.2607758620689655", "reproduced": "0.2607758620689655", "match": true }, "llama-3.2-3b-instruct.recall": { "published": "0.555045871559633", "reproduced": "0.555045871559633", "match": true }, "llama-3.2-3b-instruct.tn": { "published": "2,695", "reproduced": "2,695", "match": true }, "llama-3.2-3b-instruct.tp": { "published": "242", "reproduced": "242", "match": true }, "llama-guard-3-1b.f1": { "published": "0.008888888888888889", "reproduced": "0.008888888888888889", "match": true }, "llama-guard-3-1b.fn": { "published": "434", "reproduced": "434", "match": true }, "llama-guard-3-1b.fp": { "published": "12", "reproduced": "12", "match": true }, "llama-guard-3-1b.fpr": { "published": "0.00354924578527063", "reproduced": "0.00354924578527063", "match": true }, "llama-guard-3-1b.precision": { "published": "0.14285714285714285", "reproduced": "0.14285714285714285", "match": true }, "llama-guard-3-1b.recall": { "published": "0.0045871559633027525", "reproduced": "0.0045871559633027525", "match": true }, "llama-guard-3-1b.tn": { "published": "3,369", "reproduced": "3,369", "match": true }, "llama-guard-3-1b.tp": { "published": "2", "reproduced": "2", "match": true }, "lock.aggregate": { "published": "4", "reproduced": "4", "match": true }, "lock.download": { "published": "80", "reproduced": "80", "match": true }, "lock.entries": { "published": "92", "reproduced": "92", "match": true }, "lock.frozen": { "published": "2026-09-06", "reproduced": "2026-09-06", "match": true }, "lock.licences": { "published": "13", "reproduced": "13", "match": true }, "lock.path": { "published": "benchmarks/datasets.lock.json", "reproduced": "benchmarks/datasets.lock.json", "match": true }, "lock.sha": { "published": "48a6639433402efcb7069439cb90941d953df89bf4a1a342bdd215ba61298679", "reproduced": "48a6639433402efcb7069439cb90941d953df89bf4a1a342bdd215ba61298679", "match": true }, "lock.vendored": { "published": "8", "reproduced": "8", "match": true }, "mirror.a.aucA": { "published": "0.993005875611353", "reproduced": "0.993005875611353", "match": true }, "mirror.a.aucB": { "published": "0.6159385399181317", "reproduced": "0.6159385399181317", "match": true }, "mirror.a.model": { "published": "shieldstral-1.0-3b", "reproduced": "shieldstral-1.0-3b", "match": true }, "mirror.a.rank": { "published": "8", "reproduced": "8", "match": true }, "mirror.b.aucA": { "published": "0.3635557221789642", "reproduced": "0.3635557221789642", "match": true }, "mirror.b.aucB": { "published": "0.6462239008157376", "reproduced": "0.6462239008157376", "match": true }, "mirror.b.model": { "published": "granite-guardian-3.2-3b-a800m", "reproduced": "granite-guardian-3.2-3b-a800m", "match": true }, "mirror.b.rank": { "published": "3", "reproduced": "3", "match": true }, "models.candidates": { "published": "20", "reproduced": "20", "match": true }, "models.deploy": { "published": "22", "reproduced": "22", "match": true }, "models.noshipped": { "published": "gemma-3-4b-it", "reproduced": "gemma-3-4b-it", "match": true }, "models.over.floor": { "published": "10", "reproduced": "10", "match": true }, "models.scored": { "published": "22", "reproduced": "22", "match": true }, "models.shipped": { "published": "21", "reproduced": "21", "match": true }, "models.shipped.cands": { "published": "19", "reproduced": "19", "match": true }, "models.twoclass": { "published": "7", "reproduced": "7", "match": true }, "models.twoclass.controls": { "published": "2", "reproduced": "2", "match": true }, "models.under.floor": { "published": "9", "reproduced": "9", "match": true }, "models.zero.f1": { "published": "5", "reproduced": "5", "match": true }, "olmo-2-1b-instruct.f1": { "published": "0", "reproduced": "0", "match": true }, "olmo-2-1b-instruct.fn": { "published": "436", "reproduced": "436", "match": true }, "olmo-2-1b-instruct.fp": { "published": "0", "reproduced": "0", "match": true }, "olmo-2-1b-instruct.fpr": { "published": "0", "reproduced": "0", "match": true }, "olmo-2-1b-instruct.precision": { "published": "n/a", "reproduced": "n/a", "match": true }, "olmo-2-1b-instruct.recall": { "published": "0", "reproduced": "0", "match": true }, "olmo-2-1b-instruct.tn": { "published": "3,381", "reproduced": "3,381", "match": true }, "olmo-2-1b-instruct.tp": { "published": "0", "reproduced": "0", "match": true }, "oracle.best.f1": { "published": "0.48", "reproduced": "0.48", "match": true }, "oracle.best.model": { "published": "deberta-v3-prompt-injection-v2", "reproduced": "deberta-v3-prompt-injection-v2", "match": true }, "oracle.clear.floor": { "published": "22", "reproduced": "22", "match": true }, "oracle.min.f1": { "published": "0.20699500356887937", "reproduced": "0.20699500356887937", "match": true }, "oracle.min.model": { "published": "falcon3-3b-instruct", "reproduced": "falcon3-3b-instruct", "match": true }, "other.space": { "published": "https://huggingface.co/spaces/Vineethsain/defenseclaw-system-one", "reproduced": "https://huggingface.co/spaces/Vineethsain/defenseclaw-system-one", "match": true }, "over.all.fp": { "published": "139", "reproduced": "139", "match": true }, "over.all.fpr": { "published": "0.04111209701271813", "reproduced": "0.04111209701271813", "match": true }, "over.all.multiple": { "published": "10.7", "reproduced": "10.7", "match": true }, "over.all.recall": { "published": "0.2018348623853211", "reproduced": "0.2018348623853211", "match": true }, "over.all.tp": { "published": "88", "reproduced": "88", "match": true }, "over.defined": { "published": "221", "reproduced": "221", "match": true }, "over.expected": { "published": "15.7", "reproduced": "15.7", "match": true }, "over.full.spend": { "published": "26", "reproduced": "26", "match": true }, "over.half.a": { "published": "falcon3-1b-instruct", "reproduced": "falcon3-1b-instruct", "match": true }, "over.half.allowance": { "published": "6", "reproduced": "6", "match": true }, "over.half.b": { "published": "granite-4.0-micro", "reproduced": "granite-4.0-micro", "match": true }, "over.half.f1": { "published": "0.14107883817427386", "reproduced": "0.14107883817427386", "match": true }, "over.half.fp": { "published": "12", "reproduced": "12", "match": true }, "over.half.fpr": { "published": "0.00354924578527063", "reproduced": "0.00354924578527063", "match": true }, "over.half.gain": { "published": "9", "reproduced": "9", "match": true }, "over.half.precision": { "published": "0.7391304347826086", "reproduced": "0.7391304347826086", "match": true }, "over.half.recall": { "published": "0.0779816513761468", "reproduced": "0.0779816513761468", "match": true }, "over.half.tp": { "published": "34", "reproduced": "34", "match": true }, "over.inbudget": { "published": "73", "reproduced": "73", "match": true }, "over.mean": { "published": "0.017353723495297863", "reproduced": "0.017353723495297863", "match": true }, "over.median": { "published": "0", "reproduced": "0", "match": true }, "over.pairs": { "published": "231", "reproduced": "231", "match": true }, "over.ratio": { "published": "3.95", "reproduced": "3.95", "match": true }, "over.shared": { "published": "62", "reproduced": "62", "match": true }, "over.single.model": { "published": "falcon3-1b-instruct", "reproduced": "falcon3-1b-instruct", "match": true }, "over.single.recall": { "published": "0.05733944954128441", "reproduced": "0.05733944954128441", "match": true }, "over.single.tp": { "published": "25", "reproduced": "25", "match": true }, "over.zero": { "published": "185", "reproduced": "185", "match": true }, "parity.aucdelta": { "published": "0.0", "reproduced": "0.0", "match": true }, "parity.confmatch": { "published": "2", "reproduced": "2", "match": true }, "parity.maxdelta": { "published": "1.30e-09", "reproduced": "1.30e-09", "match": true }, "parity.n": { "published": "3", "reproduced": "3", "match": true }, "pg2.22": { "published": "70,830,722", "reproduced": "70,830,722", "match": true }, "pg2.86": { "published": "278,810,882", "reproduced": "278,810,882", "match": true }, "pg2.arch": { "published": "DebertaV2ForSequenceClassification", "reproduced": "DebertaV2ForSequenceClassification", "match": true }, "pg2.maxpos": { "published": "512", "reproduced": "512", "match": true }, "phi-4-mini-instruct.f1": { "published": "0.36122331217541837", "reproduced": "0.36122331217541837", "match": true }, "phi-4-mini-instruct.fn": { "published": "123", "reproduced": "123", "match": true }, "phi-4-mini-instruct.fp": { "published": "984", "reproduced": "984", "match": true }, "phi-4-mini-instruct.fpr": { "published": "0.29103815439219166", "reproduced": "0.29103815439219166", "match": true }, "phi-4-mini-instruct.precision": { "published": "0.2413261372397841", "reproduced": "0.2413261372397841", "match": true }, "phi-4-mini-instruct.recall": { "published": "0.7178899082568807", "reproduced": "0.7178899082568807", "match": true }, "phi-4-mini-instruct.tn": { "published": "2,397", "reproduced": "2,397", "match": true }, "phi-4-mini-instruct.tp": { "published": "313", "reproduced": "313", "match": true }, "prompt-guard-2-22m.f1": { "published": "0", "reproduced": "0", "match": true }, "prompt-guard-2-22m.fn": { "published": "436", "reproduced": "436", "match": true }, "prompt-guard-2-22m.fp": { "published": "0", "reproduced": "0", "match": true }, "prompt-guard-2-22m.fpr": { "published": "0", "reproduced": "0", "match": true }, "prompt-guard-2-22m.precision": { "published": "n/a", "reproduced": "n/a", "match": true }, "prompt-guard-2-22m.recall": { "published": "0", "reproduced": "0", "match": true }, "prompt-guard-2-22m.tn": { "published": "3,381", "reproduced": "3,381", "match": true }, "prompt-guard-2-22m.tp": { "published": "0", "reproduced": "0", "match": true }, "prompt-guard-2-86m.f1": { "published": "0", "reproduced": "0", "match": true }, "prompt-guard-2-86m.fn": { "published": "436", "reproduced": "436", "match": true }, "prompt-guard-2-86m.fp": { "published": "3", "reproduced": "3", "match": true }, "prompt-guard-2-86m.fpr": { "published": "0.0008873114463176575", "reproduced": "0.0008873114463176575", "match": true }, "prompt-guard-2-86m.precision": { "published": "0", "reproduced": "0", "match": true }, "prompt-guard-2-86m.recall": { "published": "0", "reproduced": "0", "match": true }, "prompt-guard-2-86m.tn": { "published": "3,378", "reproduced": "3,378", "match": true }, "prompt-guard-2-86m.tp": { "published": "0", "reproduced": "0", "match": true }, "qwen.bytes": { "published": "13,659,595,858", "reproduced": "13,659,595,858", "match": true }, "qwen.gib": { "published": "12.72", "reproduced": "12.72", "match": true }, "qwen.n": { "published": "3", "reproduced": "3", "match": true }, "rand.tp": { "published": "1.7", "reproduced": "1.7", "match": true }, "rank1.cap": { "published": "510", "reproduced": "510", "match": true }, "rank1.lc": { "published": "0.8347427912167461", "reproduced": "0.8347427912167461", "match": true }, "rank1.model": { "published": "deberta-v3-prompt-injection-v2", "reproduced": "deberta-v3-prompt-injection-v2", "match": true }, "rank1.raw": { "published": "0.8390509973434926", "reproduced": "0.8390509973434926", "match": true }, "rank1.shrunk": { "published": "17,202", "reproduced": "17,202", "match": true }, "rank1.stable": { "published": "no", "reproduced": "no", "match": true }, "rank2.cap": { "published": "6,144", "reproduced": "6,144", "match": true }, "rank2.lc": { "published": "0.7544168878566688", "reproduced": "0.7544168878566688", "match": true }, "rank2.model": { "published": "shieldgemma-2b", "reproduced": "shieldgemma-2b", "match": true }, "rank2.stable": { "published": "no", "reproduced": "no", "match": true }, "rank3.cap": { "published": "6,144", "reproduced": "6,144", "match": true }, "rank3.lc": { "published": "0.6556844573807082", "reproduced": "0.6556844573807082", "match": true }, "rank3.model": { "published": "granite-guardian-3.2-3b-a800m", "reproduced": "granite-guardian-3.2-3b-a800m", "match": true }, "res.wilson.hi": { "published": "0.08327404239553826", "reproduced": "0.08327404239553826", "match": true }, "res.wilson.lo": { "published": "0.03913701672802459", "reproduced": "0.03913701672802459", "match": true }, "roster.candidates": { "published": "20", "reproduced": "20", "match": true }, "roster.control": { "published": "2", "reproduced": "2", "match": true }, "roster.encoder": { "published": "3", "reproduced": "3", "match": true }, "roster.gated": { "published": "8", "reproduced": "8", "match": true }, "roster.general": { "published": "12", "reproduced": "12", "match": true }, "roster.groups": { "published": "3", "reproduced": "3", "match": true }, "roster.n": { "published": "22", "reproduced": "22", "match": true }, "roster.ranked": { "published": "22", "reproduced": "22", "match": true }, "roster.safety": { "published": "5", "reproduced": "5", "match": true }, "roster.scored": { "published": "0", "reproduced": "0", "match": true }, "roster.snapshot": { "published": "91,787,383,790", "reproduced": "91,787,383,790", "match": true }, "roster.snapshot.gib": { "published": "85.5", "reproduced": "85.5", "match": true }, "roster.ungated": { "published": "14", "reproduced": "14", "match": true }, "s2.gradeA.share": { "published": "3.90%", "reproduced": "3.90%", "match": true }, "s3.band.hi": { "published": "0.5382331240220734", "reproduced": "0.5382331240220734", "match": true }, "s3.band.lo": { "published": "0.46176687597792665", "reproduced": "0.46176687597792665", "match": true }, "s3.band.se": { "published": "0.01950705437633106", "reproduced": "0.01950705437633106", "match": true }, "s3.cases": { "published": "24,476", "reproduced": "24,476", "match": true }, "s3.cue.bytes": { "published": "0.504185543681342", "reproduced": "0.504185543681342", "match": true }, "s3.cue.events": { "published": "0.34604723008084354", "reproduced": "0.34604723008084354", "match": true }, "s3.gradeA": { "published": "193", "reproduced": "193", "match": true }, "s3.gradeA.share": { "published": "87.33%", "reproduced": "87.33%", "match": true }, "s3.gradeB": { "published": "28", "reproduced": "28", "match": true }, "s3.mdauc": { "published": "0.04158611807750677", "reproduced": "0.04158611807750677", "match": true }, "s3.models": { "published": "6", "reproduced": "6", "match": true }, "s3.negatives": { "published": "24,255", "reproduced": "24,255", "match": true }, "s3.pairs": { "published": "15", "reproduced": "15", "match": true }, "s3.positives": { "published": "221", "reproduced": "221", "match": true }, "s3.prevalence": { "published": "0.90%", "reproduced": "0.90%", "match": true }, "s3.varshare": { "published": "98.68%", "reproduced": "98.68%", "match": true }, "settle.mismatch": { "published": "0", "reproduced": "0", "match": true }, "settle.rows": { "published": "30,310", "reproduced": "30,310", "match": true }, "settle.settled": { "published": "22", "reproduced": "22", "match": true }, "settle.unsettled": { "published": "0", "reproduced": "0", "match": true }, "settle.unsettled.note": { "published": "", "reproduced": "", "match": true }, "settle.unsettled.rowsok": { "published": "0", "reproduced": "0", "match": true }, "sh.dead": { "published": "420,083,712", "reproduced": "420,083,712", "match": true }, "sh.mmproj": { "published": "840,284,704", "reproduced": "840,284,704", "match": true }, "sh.projector": { "published": "16,778,240", "reproduced": "16,778,240", "match": true }, "sh.q4bytes": { "published": "2,146,497,312", "reproduced": "2,146,497,312", "match": true }, "sh.q4gib": { "published": "1.9991", "reproduced": "1.9991", "match": true }, "sh.share": { "published": "10.91%", "reproduced": "10.91%", "match": true }, "sh.tensors": { "published": "236", "reproduced": "236", "match": true }, "sh.text": { "published": "3,429,006,336", "reproduced": "3,429,006,336", "match": true }, "sh.vision": { "published": "403,305,472", "reproduced": "403,305,472", "match": true }, "shieldgemma-2b.f1": { "published": "0.02207505518763797", "reproduced": "0.02207505518763797", "match": true }, "shieldgemma-2b.fn": { "published": "431", "reproduced": "431", "match": true }, "shieldgemma-2b.fp": { "published": "12", "reproduced": "12", "match": true }, "shieldgemma-2b.fpr": { "published": "0.00354924578527063", "reproduced": "0.00354924578527063", "match": true }, "shieldgemma-2b.precision": { "published": "0.29411764705882354", "reproduced": "0.29411764705882354", "match": true }, "shieldgemma-2b.recall": { "published": "0.011467889908256881", "reproduced": "0.011467889908256881", "match": true }, "shieldgemma-2b.tn": { "published": "3,369", "reproduced": "3,369", "match": true }, "shieldgemma-2b.tp": { "published": "5", "reproduced": "5", "match": true }, "shieldstral-1.0-3b.f1": { "published": "0.017937219730941704", "reproduced": "0.017937219730941704", "match": true }, "shieldstral-1.0-3b.fn": { "published": "432", "reproduced": "432", "match": true }, "shieldstral-1.0-3b.fp": { "published": "6", "reproduced": "6", "match": true }, "shieldstral-1.0-3b.fpr": { "published": "0.001774622892635315", "reproduced": "0.001774622892635315", "match": true }, "shieldstral-1.0-3b.precision": { "published": "0.4", "reproduced": "0.4", "match": true }, "shieldstral-1.0-3b.recall": { "published": "0.009174311926605505", "reproduced": "0.009174311926605505", "match": true }, "shieldstral-1.0-3b.tn": { "published": "3,375", "reproduced": "3,375", "match": true }, "shieldstral-1.0-3b.tp": { "published": "4", "reproduced": "4", "match": true }, "smollm2-1.7b-instruct.f1": { "published": "0.20790419161676646", "reproduced": "0.20790419161676646", "match": true }, "smollm2-1.7b-instruct.fn": { "published": "2", "reproduced": "2", "match": true }, "smollm2-1.7b-instruct.fp": { "published": "3,305", "reproduced": "3,305", "match": true }, "smollm2-1.7b-instruct.fpr": { "published": "0.9775214433599527", "reproduced": "0.9775214433599527", "match": true }, "smollm2-1.7b-instruct.precision": { "published": "0.11607381652848354", "reproduced": "0.11607381652848354", "match": true }, "smollm2-1.7b-instruct.recall": { "published": "0.9954128440366973", "reproduced": "0.9954128440366973", "match": true }, "smollm2-1.7b-instruct.tn": { "published": "76", "reproduced": "76", "match": true }, "smollm2-1.7b-instruct.tp": { "published": "434", "reproduced": "434", "match": true }, "smollm3-3b.f1": { "published": "0", "reproduced": "0", "match": true }, "smollm3-3b.fn": { "published": "436", "reproduced": "436", "match": true }, "smollm3-3b.fp": { "published": "0", "reproduced": "0", "match": true }, "smollm3-3b.fpr": { "published": "0", "reproduced": "0", "match": true }, "smollm3-3b.precision": { "published": "n/a", "reproduced": "n/a", "match": true }, "smollm3-3b.recall": { "published": "0", "reproduced": "0", "match": true }, "smollm3-3b.tn": { "published": "3,381", "reproduced": "3,381", "match": true }, "smollm3-3b.tp": { "published": "0", "reproduced": "0", "match": true }, "src.big": { "published": "lihaonan0716/mcphunt-agent-traces", "reproduced": "lihaonan0716/mcphunt-agent-traces", "match": true }, "src.big.pos": { "published": "382", "reproduced": "382", "match": true }, "src.big.share": { "published": "87.61%", "reproduced": "87.61%", "match": true }, "src.big.short": { "published": "mcphunt-agent-traces", "reproduced": "mcphunt-agent-traces", "match": true }, "src.datasets": { "published": "10", "reproduced": "10", "match": true }, "src.mcptox": { "published": "0", "reproduced": "0", "match": true }, "src.restricted": { "published": "0", "reproduced": "0", "match": true }, "src.s3.datasets": { "published": "7", "reproduced": "7", "match": true }, "src.s3.withpos": { "published": "4", "reproduced": "4", "match": true }, "src.scorable.datasets": { "published": "8", "reproduced": "8", "match": true }, "src.withpos": { "published": "5", "reproduced": "5", "match": true }, "top.f1": { "published": "0.36666666666666664", "reproduced": "0.36666666666666664", "match": true }, "top.f1.model": { "published": "granite-guardian-3.2-3b-a800m", "reproduced": "granite-guardian-3.2-3b-a800m", "match": true }, "trunc.allcases": { "published": "58", "reproduced": "58", "match": true }, "trunc.auc": { "published": "0.8017085164830626", "reproduced": "0.8017085164830626", "match": true }, "trunc.aucun": { "published": "0.7771006013326832", "reproduced": "0.7771006013326832", "match": true }, "trunc.cap": { "published": "510", "reproduced": "510", "match": true }, "trunc.cases": { "published": "1,996", "reproduced": "1,996", "match": true }, "trunc.casespct": { "published": "52.29%", "reproduced": "52.29%", "match": true }, "trunc.f1": { "published": "0.4799", "reproduced": "0.4799", "match": true }, "trunc.f1un": { "published": "0.4806", "reproduced": "0.4806", "match": true }, "trunc.flagpct": { "published": "6.88%", "reproduced": "6.88%", "match": true }, "trunc.flagrows": { "published": "2,086", "reproduced": "2,086", "match": true }, "trunc.prevratio": { "published": "5.4", "reproduced": "5.4", "match": true }, "trunc.rows": { "published": "17,202", "reproduced": "17,202", "match": true }, "trunc.rowspct": { "published": "56.75%", "reproduced": "56.75%", "match": true }, "trunc.rowstotal": { "published": "30,310", "reproduced": "30,310", "match": true }, "trunc.scoredrows": { "published": "16,975", "reproduced": "16,975", "match": true }, "trunc.scoredrowspct": { "published": "60.59%", "reproduced": "60.59%", "match": true }, "trunc.scoredrowstotal": { "published": "28,018", "reproduced": "28,018", "match": true }, "under.band": { "published": "8", "reproduced": "8", "match": true }, "wilson.z": { "published": "1.959963984540054", "reproduced": "1.959963984540054", "match": true }, "zfp.best.model": { "published": "granite-4.0-micro", "reproduced": "granite-4.0-micro", "match": true }, "zfp.best.recall": { "published": "0.011467889908256881", "reproduced": "0.011467889908256881", "match": true }, "zfp.best.tp": { "published": "5", "reproduced": "5", "match": true }, "zfp.candidates": { "published": "20", "reproduced": "20", "match": true }, "zfp.nonzero": { "published": "7", "reproduced": "7", "match": true }, "zfp.zero": { "published": "13", "reproduced": "13", "match": true }, "zoom.fpr": { "published": "0.02", "reproduced": "0.02", "match": true }, "zoom.multiple": { "published": "5.20", "reproduced": "5.20", "match": true }, "zoom.recall.max": { "published": "0.23623853211009174", "reproduced": "0.23623853211009174", "match": true } }, "leaderboard_markup": { "rows": 22, "matched": 22, "mismatched": [], "missing": [] }, "read_from_artifact_not_data": { "deb.over.floor": "final-comparisons.json headline_comparisons (not a chart value)", "src.s3.datasets": "cohort-curves.json source_census.s3 (sources.json carries it as census.s3)", "src.s3.withpos": "cohort-curves.json source_census.s3", "src.datasets": "cohort-curves.json source_census.s2 (sources.json census.s2)", "src.scorable.datasets": "cohort-curves.json source_census.s2", "src.withpos": "cohort-curves.json source_census.s2", "est.*": "the stability record read through the extractor's cohort layer", "lap.fastest/slowest labels": "pinned laptop-feasibility.json display labels" }, "extra_keys_not_published": [] } }