CompressionBenchmark / availability.js
jzxiao's picture
Merge latest C++ results and clarify Overall timing coverage
799b74b verified
Raw History Blame
11.3 kB
export const DEFAULT_PROFILE = 'cpp';
const PROFILE_LABELS = {
cpp: 'C++ 路 latest published result per dataset and method',
java: 'Java 路 imported published results',
python: 'Python 路 imported published results',
};
const FINITE = value => typeof value === 'number' && Number.isFinite(value);
const METRIC_AVAILABLE = (row,key) => key==='rank_coverage'?FINITE(row?.[key])&&row[key]>0:FINITE(row?.[key]);
const STRICT_COMPLETE_COLUMNS = new Set(['compression_time','decompression_time','normalized_compression_time','normalized_decompression_time','composite_score']);
const sourceRows = row => Array.isArray(row?.source) ? row.source : [];
const timingAdmitted = record => /^timing_admitted_/i.test(String(record?.timing_profile_status ?? ''));
export function profileLabel(profile, fallback = profile) {
return PROFILE_LABELS[profile] || fallback;
}
export function isByteOnlyProfile(profile) {
return false;
}
export function defaultColumnVisible(profile, column) {
return Boolean(column.defaultVisible);
}
export function timingAvailability(row) {
const source = sourceRows(row),evidence=source.length?source:[row];
if (evidence.some(record => !timingAdmitted(record))) {
return {
code: 'timing_not_admitted',
short: 'Timing not admitted',
detail: 'Timing exists only as diagnostic or unqualified evidence for at least one selected dataset and is excluded from timing and composite rankings.',
};
}
if (!FINITE(row?.compression_time) || !FINITE(row?.decompression_time)) {
return {
code: 'incomplete_timing',
short: 'Incomplete timing',
detail: 'At least one selected dataset lacks a complete admitted compression/decompression timing pair.',
};
}
return {code: 'available', short: 'Available', detail: 'All selected timing inputs are admitted.'};
}
export function metricAvailability(row, key) {
if (FINITE(row?.[key])) return {code: 'available', short: '', detail: ''};
if (['compression_time_timed_subset', 'decompression_time_timed_subset'].includes(key)) {
const status=row?.timing_subset_status;
if(status==='no_admitted_timing')return {code:status,short:'No admitted timing',detail:'None of the current winning rate records has admitted timing.'};
if(status==='mixed_timing_contracts')return {code:status,short:'Mixed timing contracts',detail:'Admitted timing comes from different source profiles, hosts, or timing contracts and is not averaged.'};
if(status==='unbound_timing_source')return {code:status,short:'Timing source unbound',detail:'Source profile, host, and timing contract are not all bound, so subset timing is withheld.'};
if(status==='mixed_timing_aggregation')return {code:status,short:'Mixed timing basis',detail:'Column-level and dataset-level timing aggregation cannot be mixed.'};
return {code:'incomplete_timing',short:'Incomplete timing',detail:'A valid timed subset is unavailable.'};
}
if (['compression_time', 'decompression_time'].includes(key)) return timingAvailability(row);
if (['normalized_compression_time', 'normalized_decompression_time', 'composite_score'].includes(key)) {
const timing = timingAvailability(row);
if (timing.code !== 'available') return timing;
if (Number(row?.workload_signature_count) > 1) return {
code: 'support_mismatch',
short: 'Support mismatch',
detail: `Composite metrics are withheld because this view contains ${Number(row.workload_signature_count).toLocaleString('en-US')} distinct dataset/support signatures.`,
};
return {code: 'not_eligible', short: 'Not eligible', detail: 'No eligible normalization cohort is available for this row.'};
}
if (key === 'average_rank') {
const timing = timingAvailability(row);
if (timing.code !== 'available') return timing;
return {code: 'support_mismatch', short: 'Support mismatch', detail: 'No matched dataset/support cohort provides a rank for this row.'};
}
if (['overall_rate', 'compressed_bytes', 'original_bytes'].includes(key)) return {
code: 'incomplete_bytes', short: 'Incomplete bytes', detail: 'Complete compressed/original byte totals are unavailable for the selected records.',
};
return {code: 'unavailable', short: 'Unavailable', detail: 'This metric is unavailable for the selected records.'};
}
export function summarizeAvailability(rows, requestedColumns) {
const total = rows.length;
const counts = Object.fromEntries(requestedColumns.map(column => [column.key, rows.filter(row => METRIC_AVAILABLE(row,column.key)).length]));
const hidden = requestedColumns.filter(column => (counts[column.key] || 0) === 0);
const timingNotAdmitted = rows.filter(row => timingAvailability(row).code === 'timing_not_admitted').length;
const incompleteTiming = rows.filter(row => timingAvailability(row).code === 'incomplete_timing').length;
const supportMismatch = rows.filter(row =>
!FINITE(row?.composite_score) && timingAvailability(row).code === 'available' && Number(row?.workload_signature_count) > 1
).length;
const noComparableCohort = rows.filter(row =>
!FINITE(row?.composite_score) && timingAvailability(row).code === 'available' && Number(row?.workload_signature_count) <= 1 && Number(row?.normalization_reference_rows) < 2
).length;
const parts = [
`${counts.compression_rate || 0}/${total} rates`,
`${counts.compression_time || 0}/${total} admitted timings`,
`${counts.average_rank || 0}/${total} matched-support ranks`,
`${counts.composite_score || 0}/${total} composite scores`,
];
const reasons = [];
if (timingNotAdmitted) reasons.push(`${timingNotAdmitted} timing not admitted`);
if (incompleteTiming) reasons.push(`${incompleteTiming} incomplete timing`);
if (supportMismatch) {
const signatures = Math.max(0, ...rows.map(row => Number(row?.workload_signature_count) || 0));
reasons.push(`${supportMismatch} composite scores withheld across ${signatures} dataset/support signatures`);
}
if(noComparableCohort)reasons.push(`${noComparableCohort} lack a comparable normalization cohort`);
return {total, counts, hidden, timingNotAdmitted, incompleteTiming, supportMismatch, noComparableCohort, parts, reasons};
}
export function visibleColumns(columns, rows, profile, showUnavailable, touched = new Set()) {
const requested = columns.filter(column => touched.has(column.key) ? column.visible : defaultColumnVisible(profile, column));
if (showUnavailable) return columns.filter(column => touched.has(column.key) ? column.visible : Boolean(column.defaultVisible));
return requested.filter(column => STRICT_COMPLETE_COLUMNS.has(column.key)?rows.length>0&&rows.every(row=>METRIC_AVAILABLE(row,column.key)):rows.some(row => METRIC_AVAILABLE(row,column.key)));
}
export function profileScopeText({profile, measuredDatasets, catalogDatasets}) {
const prefix = `${profileLabel(profile)} 路 ${measuredDatasets} measured datasets in this profile / ${catalogDatasets} catalog datasets.`;
if (profile === DEFAULT_PROFILE) return `${prefix} Duplicate C++ dataset/method pairs use the latest receipt-bound authority. Source profile and host remain visible in details and CSV. Strict timing and scores require complete same-contract coverage; partial admitted timing is labeled as a timed subset.`;
return `${prefix} These imported published results retain their original evidence boundaries; unqualified timing is not promoted into rankings.`;
}
export function timingSourceIdentity(record) {
const profile = record?.source_comparison_profile ?? record?.comparison_profile ?? record?.runtime ?? '';
const host = record?.source_execution_host ?? record?.execution_host ?? '';
const contract = record?.timing_contract ?? record?.timing_profile_status ?? '';
return `${profile}\u0000${host}\u0000${contract}`;
}
export function aggregateTimedSubset(source) {
const denominator = new Set(sourceRows({source}).map(record => record.dataset)).size;
const admitted = sourceRows({source}).filter(record =>
timingAdmitted(record) && FINITE(record?.compression_time) && FINITE(record?.decompression_time) &&
Number(record.compression_time) >= 0 && Number(record.decompression_time) >= 0
);
const boundIdentity = record => {
const profile=record?.source_comparison_profile ?? record?.comparison_profile ?? record?.runtime ?? '';
const host=record?.source_execution_host ?? record?.execution_host ?? '';
const contract=record?.timing_contract ?? record?.timing_profile_status ?? '';
return profile && host && contract;
};
const identities = [...new Set(admitted.map(timingSourceIdentity))];
const base = {
timing_subset_coverage: new Set(admitted.map(record => record.dataset)).size,
timing_subset_denominator: denominator,
timing_subset_source_identity: identities.length === 1 ? identities[0] : '',
timing_subset_source_profiles: [...new Set(admitted.map(record => record.source_comparison_profile ?? record.comparison_profile ?? record.runtime ?? ''))].filter(Boolean),
timing_subset_hosts: [...new Set(admitted.map(record => record.source_execution_host ?? record.execution_host ?? ''))].filter(Boolean),
};
if (!admitted.length) return {...base, compression_time_timed_subset:null, decompression_time_timed_subset:null, timing_subset_status:'no_admitted_timing'};
if (!admitted.every(boundIdentity)) return {...base, compression_time_timed_subset:null, decompression_time_timed_subset:null, timing_subset_status:'unbound_timing_source'};
if (identities.length !== 1) return {...base, compression_time_timed_subset:null, decompression_time_timed_subset:null, timing_subset_status:'mixed_timing_contracts'};
const completeColumnTiming = admitted.every(record => record.timing_aggregation === 'column_mean' && Number.isInteger(record.timing_column_count) && record.timing_column_count > 0);
const completeDatasetTiming = admitted.every(record => record.timing_aggregation === 'dataset_aggregate' && record.timing_column_count === 0);
if (!completeColumnTiming && !completeDatasetTiming) return {...base, compression_time_timed_subset:null, decompression_time_timed_subset:null, timing_subset_status:'mixed_timing_aggregation'};
const aggregate = key => completeColumnTiming
? admitted.reduce((sum,record) => sum + Number(record[key]) * record.timing_column_count, 0) / admitted.reduce((sum,record) => sum + record.timing_column_count, 0)
: admitted.reduce((sum,record) => sum + Number(record[key]), 0) / admitted.length;
return {
...base,
compression_time_timed_subset:aggregate('compression_time'),
decompression_time_timed_subset:aggregate('decompression_time'),
timing_subset_status:completeColumnTiming?'column_mean':'dataset_mean',
};
}
export function strictTimingMode(source) {
const rows=sourceRows({source});
if(!rows.length||rows.some(record=>!timingAdmitted(record)||!FINITE(record?.compression_time)||!FINITE(record?.decompression_time)))return 'unavailable';
const subset=aggregateTimedSubset(rows),datasetCount=new Set(rows.map(record=>record.dataset)).size;
if(subset.timing_subset_coverage!==datasetCount)return 'unavailable';
return ['column_mean','dataset_mean'].includes(subset.timing_subset_status)?subset.timing_subset_status:'unavailable';
}