Streaming, time to first token and throughput

# A streamed answer: a pause for prefill, then tokens one by one.
answer = "You get 25 days of annual leave plus public holidays .".split()
prefill_s, per_token_s = 0.4, 0.02                   # example timings
 
ttft = prefill_s + per_token_s                        # the first token arrives here
total = prefill_s + len(answer) * per_token_s         # the last token arrives here
print(f"TTFT {ttft:.2f}s, total {total:.2f}s for {len(answer)} tokens")
print("without streaming the user stares at nothing for the whole total time")
 
# Throughput vs latency: bigger batches serve more users per second, but each one waits longer.
def step_time_ms(batch):                              # a decode step gets slower as the batch grows
    return 20 + 0.6 * batch
for batch in (1, 8, 32, 128):
    per_user_tps = 1000 / step_time_ms(batch)        # tokens/s each user sees
    total_tps = per_user_tps * batch                   # tokens/s the GPU produces
    print(f"batch {batch:3}: {per_user_tps:5.1f} tok/s per user, {total_tps:6.0f} tok/s total")

TTFT 0.42s, total 0.62s for 11 tokens
without streaming the user stares at nothing for the whole total time
batch   1:  48.5 tok/s per user,     49 tok/s total
batch   8:  40.3 tok/s per user,    323 tok/s total
batch  32:  25.5 tok/s per user,    816 tok/s total
batch 128:  10.3 tok/s per user,   1322 tok/s total

import anthropic
 
client = anthropic.Anthropic()
with client.messages.stream(model="claude-haiku-5-5", max_tokens=500,
                            messages=[{"role": "user", "content": "Explain TTFT in one paragraph."}]) as stream:
    for text in stream.text_stream:          # text arrives in small pieces
        print(text, end="", flush=True)     # forward each piece to the client (e.g. via SSE)
    final = stream.get_final_message()       # full message + usage at the end