Spaces:
Running
Running
File size: 15,505 Bytes
0f79625 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 | import './research-results.js?v=overview-5';
import {AudioPlayer} from './audio-player.js?v=media-3';
const COPY={style:['Free-form Style Editing','Localized editing with free-form style descriptions, from a selected passage to a whole utterance.'],text:['Text Editing','Localized changes in English and Mandarin, from a word to a longer passage.'],emotion:['Emotion Editing','Localized expression, with the surrounding delivery preserved.'],prosody:['Prosody Editing','Pitch and speaking-rate control over whole utterances and selected regions.'],pause:['Pause Editing','Insert or reduce pauses at specified boundaries.'],multi:['Compositional Editing','Multiple edit types in a single instruction.'],noise:['Noise Preservation & Denoising','Speech editing with background sound preservation, plus denoising.'],multi_speaker:['Multi-speaker Preservation','Bilingual editing without speaker labels in the instruction.']};
const escape=s=>String(s).replaceAll('&','&').replaceAll('<','<').replaceAll('>','>').replaceAll('"','"');
const kind=o=>['ins','del','sub'].includes(o.kind)?'text':o.kind==='emo'?'emotion':o.kind;
const label=o=>{const p=Object.fromEntries(o.params);return ({ins:`Insert “${o.content}”`,del:'Delete',sub:`Replace with “${p.targ}”`,emo:p.desc?`Style · ${p.desc}`:`Emotion · ${p.type}`,pitch:`Pitch · ${Number(p.semitones)>0?'+':''}${p.semitones} semitones`,rate:`Rate · ${p.factor}×`,pause:`${p.act==='ins'?'Insert':'Reduce'} pause`})[o.kind];};
let expanded=null,releaseScrollAnchor=null;
function timingAt(sample,system,char,point=false){
const spans=system==='source'?sample.source_tokens:sample.target_tokens,alignment=sample.audio_info[system]?.alignment;
if(!alignment?.length)return null;
if(point){const left=spans.findLastIndex(s=>s[1]<=char);return left>=0?alignment[left]?.[1]??null:0;}
let i=spans.findIndex(s=>s[1]>char);if(i<0)i=spans.length-1;
return alignment[i]?.[0]??null;
}
class SampleBlock {
constructor(sample,index,{first=false}={}){
this.sample=sample;this.mode='readable';this.expanded=false;this.active='ours';this.playerMap={};
this.root=document.createElement('article');this.root.className='sample-block';this.root.dataset.sample=sample.candidate_id;
this.root.innerHTML=`<header class="sample-top"><span class="sample-number">${first?'FEATURED':String(index+1).padStart(2,'0')}</span><button class="sample-focus sample-expand" type="button" aria-expanded="false"></button><button class="detail-toggle" type="button" aria-expanded="false"><span>Close detail</span><svg viewBox="0 0 24 24" aria-hidden="true"><path d="m8 5 7 7-7 7"/></svg></button></header><div class="source-transcript"><div class="transcript-heading"><span>Source · edit instruction</span><div class="mode-switch" role="group" aria-label="Instruction display"><button type="button" data-mode="readable" aria-pressed="true">Highlighted</button><button type="button" data-mode="xml" aria-pressed="false">XML</button></div></div><div class="transcript-text source-text"></div></div><div class="audio-comparison"><div class="target-transcript"><div class="transcript-heading"><span>Expected Target transcript</span><span class="seek-hint"></span></div><div class="transcript-text target-text"></div><p class="timing-note"></p></div></div><div class="interaction-note">Drag a waveform to seek · click source words to play the original · click target words or edit labels to compare the results</div><div class="seek-status sr-only" role="status" aria-live="polite"></div>`;
this.root.querySelector('.sample-focus').textContent=sample.focus;
this.root.querySelector('.detail-toggle').onclick=()=>this.setExpanded(!this.expanded);
{
this.root.querySelector('.detail-toggle').hidden=true;
this.root.addEventListener('click',e=>{
if(this.expanded||e.target.closest('.source-transcript,.audio-comparison,.detail-toggle,a,input,select'))return;
if(window.getSelection()?.toString())return;
this.setExpanded(true);
});
}
this.root.querySelectorAll('[data-mode]').forEach(b=>b.onclick=()=>{this.mode=b.dataset.mode;this.root.querySelectorAll('[data-mode]').forEach(e=>e.setAttribute('aria-pressed',String(e===b)));this.renderTranscripts();});
const target=this.root.querySelector('.audio-comparison');
const streams=[{system_id:'source',label:'Source',kind:'source',audio:sample.source_audio},...sample.systems];
target.style.setProperty('--streams',String(streams.length));
for(const stream of streams){const key=stream.system_id;const info=sample.audio_info[key];
const operations=key==='source'?sample.operations.map((o,i)=>({index:i,time:timingAt(sample,'source',o.source.start,o.source.start===o.source.end),kind:kind(o),label:label(o)})):[];
const p=new AudioPlayer({path:stream.audio,label:stream.label.replace('Ours · ',''),kind:stream.kind,info,onPlay:()=>{document.querySelectorAll('.current-word').forEach(e=>e.classList.remove('current-word'));this.active=key;},onTime:t=>this.highlight(key,t),onOperation:i=>this.seekOperation(i),operations});
p.root.dataset.system=key;target.append(p.root);this.playerMap[key]=p;
}
this.root.querySelector('.timing-note').textContent=sample.alignment_note||'';
if(sample.alignment_note){this.root.querySelector('.seek-hint').textContent='';this.root.querySelector('.interaction-note').textContent='Drag a waveform to seek · adjust playback speed for closer listening';}
this.renderTranscripts();
}
setExpanded(value){
releaseScrollAnchor?.();
const previous=expanded, page=document.documentElement;
let frame;
const anchoring=page.style.overflowAnchor;
page.style.overflowAnchor='none';
const viewportTop=this.root.getBoundingClientRect().top;
const release=()=>{cancelAnimationFrame(frame);page.style.overflowAnchor=anchoring;window.removeEventListener('wheel',release);window.removeEventListener('touchmove',release);window.removeEventListener('keydown',release);if(releaseScrollAnchor===release)releaseScrollAnchor=null;};
window.addEventListener('wheel',release,{passive:true});
window.addEventListener('touchmove',release,{passive:true});
window.addEventListener('keydown',release);
releaseScrollAnchor=release;
this.updateExpanded(value);
const start=performance.now(), duration=matchMedia('(prefers-reduced-motion: reduce)').matches?0:300;
// Align the new card while accounting for the previous card's collapse.
const follow=()=>{
const progress=duration?Math.min(1,(performance.now()-start)/duration):1;
const target=value?viewportTop+(24-viewportTop)*(1-(1-progress)**3):viewportTop;
window.scrollBy({top:this.root.getBoundingClientRect().top-target,behavior:'instant'});
if(progress<1||[this,previous].some(card=>card?.animation?.playState==='running'))frame=requestAnimationFrame(follow);
else frame=requestAnimationFrame(()=>{frame=requestAnimationFrame(release);});
};
follow();
}
updateExpanded(value){
const oldHeight=this.root.getBoundingClientRect().height;
this.animation?.cancel();
if(value&&expanded&&expanded!==this)expanded.updateExpanded(false);
this.expanded=value;if(value)expanded=this;else if(expanded===this)expanded=null;
this.root.classList.toggle('expanded',value);const b=this.root.querySelector('.detail-toggle');b.setAttribute('aria-expanded',String(value));b.querySelector('span').textContent='Close detail';
b.hidden=!value;const summary=this.root.querySelector('.sample-expand');summary.setAttribute('aria-expanded',String(value));if(!value&&document.activeElement===b)summary.focus({preventScroll:true});
const newHeight=this.root.getBoundingClientRect().height;
if(!matchMedia('(prefers-reduced-motion: reduce)').matches){this.root.style.overflow='clip';this.animation=this.root.animate([{height:`${oldHeight}px`},{height:`${newHeight}px`}],{duration:260,easing:'cubic-bezier(.2,.7,.2,1)'});this.animation.onfinish=()=>this.root.style.overflow='';this.animation.oncancel=()=>this.root.style.overflow='';}
if(!value){this.mode='readable';this.root.querySelectorAll('[data-mode]').forEach(e=>e.setAttribute('aria-pressed',String(e.dataset.mode==='readable')));this.renderTranscripts();}
}
words(text,start,end,side){
const tokens=side==='source'?this.sample.source_tokens:this.sample.target_tokens;let result='',cursor=start;
for(let i=0;i<tokens.length;i++){const[a,b]=tokens[i];if(a<start||b>end)continue;result+=escape(text.slice(cursor,a));const enabled=side==='source'?this.sample.audio_info.source.alignment[i]:this.sample.audio_info.ours.alignment[i];result+=`<${enabled?'button type="button"':'span'} class="transcript-word" data-side="${side}" data-word="${i}" ${enabled?`title="Play from “${escape(text.slice(a,b))}”"`:''}>${escape(text.slice(a,b))}</${enabled?'button':'span'}>`;cursor=b;}
return result+escape(text.slice(cursor,end));
}
renderTranscripts(){
const s=this.sample;let out='',cursor=0;
s.operations.forEach((o,i)=>{
out+=this.words(s.source_text,cursor,o.source.start,'source');const type=kind(o);const attrs=o.params.map(([k,v])=>` ${k}="${v}"`).join('');const open=`<${o.kind}${attrs}${o.kind==='pause'?'/':''}>`;const inner=o.kind==='ins'?escape(o.content):this.words(s.source_text,o.source.start,o.source.end,'source');
if(this.mode==='xml')out+=`<span class="instruction-operation op-${type}"><span role="button" tabindex="0" class="xml-tag" data-op="${i}" title="Play results from this edit">${escape(open)}</span>${inner}${o.kind!=='pause'?`<span role="button" tabindex="0" class="xml-tag" data-op="${i}">${escape(`</${o.kind}>`)}</span>`:''}</span>`;
else out+=`<span class="instruction-operation op-${type}">${o.kind==='del'?'<s>':''}${o.kind==='ins'?'':inner}${o.kind==='del'?'</s>':''}<span role="button" tabindex="0" class="edit-label" data-wrap="${label(o).length>28?'flow':'keep'}" data-op="${i}" title="Play results from one second before this edit">${escape(label(o))}</span></span>`;
cursor=o.source.end;
});out+=this.words(s.source_text,cursor,s.source_text.length,'source');
if(s.presentation_action)out+=` <button type="button" class="presentation-action" title="Play the enhanced audio">${escape(s.presentation_action)}</button>`;
const src=this.root.querySelector('.source-text');src.innerHTML=out;src.classList.toggle('xml-mode',this.mode==='xml');
const action=src.querySelector('.presentation-action');if(action)action.onclick=()=>this.seekTargets(()=>0,0);
const tgt=this.root.querySelector('.target-text');let target='',pos=0;
for(const o of s.operations){target+=this.words(s.target_text,pos,o.target.start,'target');target+=`<span class="target-operation op-${kind(o)}">${this.words(s.target_text,o.target.start,o.target.end,'target')}</span>`;pos=o.target.end;}
target+=this.words(s.target_text,pos,s.target_text.length,'target');tgt.innerHTML=target;
this.root.querySelectorAll('button[data-word]').forEach(b=>b.onclick=()=>this.seekWord(b.dataset.side,Number(b.dataset.word)));
this.root.querySelectorAll('[data-op]').forEach(b=>{if(!s.audio_info.ours.alignment.length){b.setAttribute('aria-disabled','true');b.tabIndex=-1;b.title='Word timing is unavailable for this example';}else {b.onclick=()=>this.seekOperation(Number(b.dataset.op));b.onkeydown=e=>{if(e.key==='Enter'||e.key===' '){e.preventDefault();b.click();}};}});
}
seekWord(side,index){
if(side==='source'){const span=this.sample.audio_info.source.alignment[index];if(span){this.playerMap.source.seek(span[0]-.25);this.playerMap.source.play();}}
else this.seekTargets(key=>this.sample.audio_info[key].alignment[index]?.[0],.25);
}
seekOperation(index){const o=this.sample.operations[index];this.seekTargets(key=>timingAt(this.sample,key,o.target.start,o.target.start===o.target.end),1);this.root.querySelectorAll('[data-op]').forEach(b=>b.classList.toggle('selected',Number(b.dataset.op)===index));}
seekTargets(getTime,lead){
const missing=[];for(const [key,p] of Object.entries(this.playerMap)){if(key==='source')continue;const t=getTime(key);if(t==null){missing.push(p.label);continue;}p.seek(Math.max(0,t-lead));}
const t=getTime('ours');if(t!=null)this.playerMap.ours.play();
this.root.querySelector('.seek-status').textContent=missing.length?`Timing unavailable for ${missing.join(', ')}.`:'All edited tracks positioned; playing dots.tts.edit.';
}
highlight(key,t){if(!this.expanded||this.active!==key)return;const side=key==='source'?'source':'target',align=this.sample.audio_info[key].alignment;this.root.querySelectorAll(`[data-side="${side}"]`).forEach(b=>{const span=align[Number(b.dataset.word)];b.classList.toggle('current-word',!!span&&t>=span[0]&&t<span[1]);});}
}
const video=document.querySelector('.promotion-video video');
const videoStart=document.querySelector('.video-start');
videoStart?.addEventListener('click',()=>{video.play().catch(()=>{videoStart.hidden=false;});});
video?.addEventListener('play',()=>{videoStart.hidden=true;});
video?.addEventListener('ended',()=>{videoStart.hidden=false;});
async function main(){
const r=await fetch('data/samples.json');if(!r.ok)throw Error('Examples could not be loaded');const data=await r.json();
const container=document.querySelector('#sample-sections');container.replaceChildren();
for(const section of data.sections){const block=document.createElement('section');block.className='card benchmark-section';block.id=section.benchmark;const[title,desc]=COPY[section.benchmark]||[section.title,''];block.innerHTML=`<div class="card-heading benchmark-heading"><h2>${escape(title)}</h2><p>${escape(desc)}</p></div><div class="sample-list"></div>`;container.append(block);section.samples.forEach((sample,i)=>block.querySelector('.sample-list').append(new SampleBlock(sample,i).root));}
document.querySelector('#first-glance-content').replaceChildren(new SampleBlock(data.first_glance,0,{first:true}).root);
const result=await(await fetch('data/benchmark.json')).json();
const metrics=[['Operation<br>success (%) ↑',2,true],['All<br>success (%) ↑',2,true],['Speaker<br>similarity ↑',3,true],['WDTW-Dur<br>(%) ↓',2,false],['WDTW-F0<br>(st) ↓',2,false]];
const best=metrics.map(([, ,up],i)=>(up?Math.max:Math.min)(...result.results.map(row=>row[i+1])));
document.querySelector('#benchmark-results').innerHTML=`<div class="table-scroll" role="region" aria-label="Reference results table" tabindex="0"><table class="results-table"><caption>Mean of one-take and sequential results</caption><thead><tr><th scope="col" rowspan="2">System</th><th scope="colgroup" colspan="2">Instruction following</th><th scope="colgroup" colspan="3">Local preservation</th></tr><tr>${metrics.map(([label])=>`<th scope="col">${label}</th>`).join('')}</tr></thead><tbody>${result.results.map(([name,...values])=>`<tr${name==='dots.tts.edit'?' class="ours"':''}><th scope="row">${escape(name)}${name==='AuK'?'<sup><a href="#auk-note" aria-label="AuK note 1">1</a></sup>':''}</th>${values.map((x,i)=>`<td>${x===best[i]?`<strong>${x.toFixed(metrics[i][1])}</strong>`:x.toFixed(metrics[i][1])}</td>`).join('')}</tr>`).join('')}</tbody></table></div><p class="result-definitions">Operation success measures individual edits; all success requires every requested edit to succeed. WDTW measures duration and pitch changes in regions to preserve. Bold marks the best result.</p>`;
window.addEventListener('pagehide',()=>{for(const media of document.querySelectorAll('audio,video'))media.pause();});
}
main().catch(e=>{document.querySelector('#sample-sections').textContent=e.message;console.error(e);});
|