"""Synthetic stream interpretation using the unchanged endpoint client's measure().""" import json import math import time from types import SimpleNamespace import llm_client SCENARIOS=('answer','thinking','final_answer','no_answer','zero_duration','interrupted','api_error') DEFAULT=dict(scenario='thinking',arrival_delay_ms=200,thinking_ms=600,fragment_interval_ms=400) BOUNDS=dict(arrival_delay_ms=(0,5000),thinking_ms=(0,5000),fragment_interval_ms=(0,2000)) def validate(p): if not isinstance(p,dict) or set(p)!=set(DEFAULT) or p['scenario'] not in SCENARIOS: raise ValueError('Unknown or missing parameters') for key,(lo,hi) in BOUNDS.items(): v=p[key] if type(v) not in (int,float) or not math.isfinite(v) or not lo<=v<=hi or v!=round(v): raise ValueError('Invalid control: '+key) def fixture(p): validate(p) d,t,g=(p[k]/1000 for k in ('arrival_delay_ms','thinking_ms','fragment_interval_ms')) scenario=p['scenario'];events=[] def add(at,**event):events.append(dict(at_s=at,event=event)) add(d,response='') if scenario=='thinking':add(d,thinking='synthetic thinking') first=d+(t if scenario=='thinking' else 0) if scenario not in ('no_answer','final_answer'): for i in range(3):add(first+i*g,response=f'synthetic fragment {i+1}') end=first+3*g if scenario=='api_error':add(end,error='synthetic API error') elif scenario!='interrupted': final=dict(done=True,eval_count=20,eval_duration=0 if scenario=='zero_duration' else 2_000_000_000) if scenario=='final_answer':final['response']='synthetic final answer' add(end,**final) return events,end+.05 class Response: def __init__(self,events,clock,completion):self.events,self.clock,self.completion=events,clock,completion def __enter__(self):return self def __exit__(self,*args):self.clock[0]=self.completion def __iter__(self): for item in self.events: self.clock[0]=item['at_s'] yield b'\n' if item['event'] is None else (json.dumps(item['event'])+'\n').encode() def interpret(events,completion_s): """Replace only the client's I/O and clock; restore module globals after each run.""" clock=[0] old_time,old_urllib=llm_client.time,llm_client.urllib llm_client.time=SimpleNamespace(monotonic=lambda:clock[0]) llm_client.urllib=SimpleNamespace(request=SimpleNamespace(Request=lambda *a,**k:None,urlopen=lambda *a,**k:Response(events,clock,completion_s))) try: return dict(status='ok',**llm_client.measure('synthetic://fixture',{})) except RuntimeError as exc: return dict(status='error',error=str(exc)) finally: llm_client.time,llm_client.urllib=old_time,old_urllib def run(name,p): if name!='llm':raise ValueError('Unknown experiment') start=time.perf_counter();events,completion=fixture(p);result=interpret(events,completion) metrics=[] for key,unit in [('ttfc_s','s'),('first_thinking_s','s'),('client_total_s','s'),('generation_tokens_s','token/s')]: value=result.get(key);m=dict(id=key,value=value,unit=unit) if value is None:m['reason']='Stream failed' if result['status']=='error' else 'No content or positive generation duration' metrics.append(m) lanes={'thinking_events':[],'answer_events':[],'done_events':[],'error_events':[],'empty_events':[]} for e in events: v=e['event'];at=e['at_s'] for field,lane in [('thinking','thinking_events'),('response','answer_events'),('done','done_events'),('error','error_events')]: if v.get(field):lanes[lane].append(at) if not any(v.get(k) for k in ('thinking','response','done','error')):lanes['empty_events'].append(at) series=[dict(id=k,x=v,y=[i]*len(v),x_unit='s',y_unit='lane') for i,(k,v) in enumerate(lanes.items())] return dict(metrics=metrics,series=series,stdout='',provenance=dict(data_type='synthetic',compute_ms=(time.perf_counter()-start)*1000, original_result=result,events=events,synthetic_completion_s=completion,post_stream_overhead_s=.05, limitations='Synthetic virtual clock and server counters; no inference, model, hardware, memory or token-latency measurement.')) if __name__=='__main__': import argparse parser=argparse.ArgumentParser(description=__doc__);parser.add_argument('--parameters',default=json.dumps(DEFAULT)) print(json.dumps(run('llm',json.loads(parser.parse_args().parameters)),allow_nan=False))