#!/usr/bin/env python3 """bench.py - time streamed chat completions on any OpenAI-compatible gateway. Run it from YOUR network, with YOUR key, against any base URL that speaks the OpenAI chat completions API. It makes N streamed calls with one fixed short prompt and reports, per call and in summary: connect_ms start -> TCP connect and TLS handshake done (a new connection per call) first_byte_ms start -> first byte of the response body first_token_ms start -> first stream event that carries text (answer text, reasoning text or a tool call) total_ms start -> end of the stream output_tokens usage.completion_tokens, as the gateway reports it tokens_per_second output_tokens / (total - first_token), in seconds "start" is one instant, just before the connection is opened, so every time includes your own network path to the gateway and the handshake. Percentiles are nearest-rank (the value at rank ceil(q * n) of the sorted samples), the same rule https://1xai.ir/status uses. What it does not do: it contacts nothing but the base URLs you give it (no telemetry, no update check, no proxy from the environment), it never prints or stores your key, and it never sends one gateway's key to another gateway. It COSTS MONEY: every call is a normal paid API call on your account. The script says how many it will make and asks before it starts, unless --yes. Python 3.8 or newer, standard library only. export BENCH_API_KEY=... # the key is read from the environment python3 bench.py --base-url https://1xai.ir/v1 --model gpt-5-mini # two gateways, called alternately, each with its own key export BENCH_COMPARE_API_KEY=... python3 bench.py --base-url https://1xai.ir/v1 --model gpt-5-mini \\ --compare https://other.example/v1 --compare-model gpt-5-mini Licence: MIT. Source and method: https://1xai.ir/benchmark """ import argparse import datetime import hashlib import http.client import json import math import os import platform import socket import ssl import sys import time from urllib.parse import urlsplit VERSION = "1.0.0" # One fixed prompt, so two runs (or two gateways) do the same work: a short # input and an answer of roughly two hundred tokens that any chat model can # produce without tools. DEFAULT_PROMPT = "Write the numbers from 1 to 60 in English words, separated by commas. Output only the list." DEFAULT_MAX_TOKENS = 256 DEFAULT_N = 20 USER_AGENT = "llm-gateway-bench/" + VERSION KEY_ENV = "BENCH_API_KEY" COMPARE_KEY_ENV = "BENCH_COMPARE_API_KEY" # ----------------------------------------------------------------- statistics def rank(q, n): """Nearest-rank position (1..n) of the q-th quantile among n values.""" if n <= 0: return 0 r = int(math.ceil(q * n - 1e-9)) return max(1, min(n, r)) def percentile(values, q): """Nearest-rank q-th quantile of values, or None when there are none.""" vals = sorted(v for v in values if v is not None) if not vals: return None return vals[rank(q, len(vals)) - 1] def spread(values, digits=1): """p50, p95, min, max and count of a list of numbers (None values left out).""" vals = [v for v in values if v is not None] if not vals: return {"n": 0, "p50": None, "p95": None, "min": None, "max": None} return { "n": len(vals), "p50": round(percentile(vals, 0.50), digits), "p95": round(percentile(vals, 0.95), digits), "min": round(min(vals), digits), "max": round(max(vals), digits), } def summarize(calls): """Summary of one target's calls. Times are over successful calls only.""" ok = [c for c in calls if c.get("ok")] failed = [c for c in calls if not c.get("ok")] reasons = {} for c in failed: reason = c.get("error") or "unknown" reasons[reason] = reasons.get(reason, 0) + 1 rates = [c.get("tokens_per_second") for c in ok] rate_vals = [r for r in rates if r is not None] tokens = sum(c["output_tokens"] for c in ok if c.get("tokens_per_second") is not None) gen_ms = sum(c["total_ms"] - c["first_token_ms"] for c in ok if c.get("tokens_per_second") is not None) tps = { "n": len(rate_vals), # The median call, the slow tail (5 of 100 calls were slower than # p05), and all output tokens over all generation time. "p50": round(percentile(rate_vals, 0.50), 1) if rate_vals else None, "p05": round(percentile(rate_vals, 0.05), 1) if rate_vals else None, "weighted": round(tokens * 1000.0 / gen_ms, 1) if gen_ms > 0 else None, } return { "calls": len(calls), "ok": len(ok), "failed": len(failed), "failure_reasons": reasons, "usage_reported": sum(1 for c in ok if c.get("output_tokens") is not None), "connect_ms": spread([c.get("connect_ms") for c in ok]), "first_byte_ms": spread([c.get("first_byte_ms") for c in ok]), "first_token_ms": spread([c.get("first_token_ms") for c in ok]), "total_ms": spread([c.get("total_ms") for c in ok]), "tokens_per_second": tps, } # ------------------------------------------------------------------ one call def delta_has_text(event): """True when a chat.completion.chunk carries generated content: answer text, reasoning text or a tool call. A role-only first chunk does not.""" for choice in event.get("choices") or []: delta = choice.get("delta") or {} for field in ("content", "reasoning_content", "reasoning", "refusal"): if delta.get(field): return True if delta.get("tool_calls") or delta.get("function_call"): return True return False def redact(text, secrets): """text with every secret replaced, so an echoed key is never shown.""" for s in secrets: if s: text = text.replace(s, "[key removed]") return text class Target(object): """One gateway: where to call, which model, and the key (never printed).""" def __init__(self, label, base_url, model, key): self.label = label self.base_url = base_url.rstrip("/") self.model = model self._key = key u = urlsplit(self.base_url) if u.scheme not in ("http", "https") or not u.hostname: raise ValueError("base URL must start with http:// or https:// : %r" % base_url) if u.username or u.password or u.query or u.fragment: raise ValueError("base URL must not hold credentials, a query or a fragment") self.scheme = u.scheme self.host = u.hostname self.port = u.port or (443 if u.scheme == "https" else 80) self.path = (u.path.rstrip("/") or "") + "/chat/completions" def public(self): return {"label": self.label, "base_url": self.base_url, "model": self.model} def __repr__(self): # a key must not leak through a traceback or a debug print return "Target(%r, %r, %r)" % (self.label, self.base_url, self.model) def build_body(target, opts): body = { "model": target.model, "messages": [{"role": "user", "content": opts["prompt"]}], "stream": True, opts["max_tokens_param"]: opts["max_tokens"], } if opts.get("usage", True): body["stream_options"] = {"include_usage": True} if opts.get("temperature") is not None: body["temperature"] = opts["temperature"] return json.dumps(body).encode("utf-8") def run_one(target, opts, clock=time.perf_counter): """Make one streamed call and time it. Never raises: a failure is a result.""" out = { "ok": False, "status": None, "error": None, "connect_ms": None, "first_byte_ms": None, "first_token_ms": None, "total_ms": None, "output_tokens": None, "tokens_per_second": None, } body = build_body(target, opts) conn = None t0 = clock() def ms(): return round((clock() - t0) * 1000.0, 1) try: if target.scheme == "https": conn = http.client.HTTPSConnection(target.host, target.port, timeout=opts["timeout"], context=ssl.create_default_context()) else: conn = http.client.HTTPConnection(target.host, target.port, timeout=opts["timeout"]) conn.connect() out["connect_ms"] = ms() conn.request("POST", target.path, body=body, headers={ "Authorization": "Bearer " + target._key, "Content-Type": "application/json", "Accept": "text/event-stream", "User-Agent": USER_AGENT, }) resp = conn.getresponse() out["status"] = resp.status if resp.status != 200: detail = resp.read(600).decode("utf-8", "replace") out["error"] = "http_%d" % resp.status out["detail"] = redact(" ".join(detail.split()), [target._key])[:300] return out saw_done = False while True: line = resp.readline() if not line: break if out["first_byte_ms"] is None: out["first_byte_ms"] = ms() line = line.strip() if not line.startswith(b"data:"): continue data = line[5:].strip() if data == b"[DONE]": saw_done = True break try: event = json.loads(data.decode("utf-8", "replace")) except ValueError: continue if not isinstance(event, dict): continue if isinstance(event.get("error"), dict): out["error"] = "stream_error" out["detail"] = redact(str(event["error"].get("message", "")), [target._key])[:300] return out if out["first_token_ms"] is None and delta_has_text(event): out["first_token_ms"] = ms() usage = event.get("usage") if isinstance(usage, dict) and isinstance(usage.get("completion_tokens"), int): out["output_tokens"] = usage["completion_tokens"] out["total_ms"] = ms() if out["first_token_ms"] is None: out["error"] = "no_content" return out if not saw_done and out["output_tokens"] is None: # The stream ended with neither [DONE] nor a usage block: cut short. out["error"] = "stream_cut" return out out["ok"] = True gen_ms = out["total_ms"] - out["first_token_ms"] if out["output_tokens"] and gen_ms > 0: out["tokens_per_second"] = round(out["output_tokens"] * 1000.0 / gen_ms, 1) return out except socket.timeout: out["error"] = "timeout" except ssl.SSLError as e: out["error"] = "tls_error" out["detail"] = redact(str(e), [target._key])[:300] except (OSError, http.client.HTTPException) as e: out["error"] = "connection_error" out["detail"] = redact("%s: %s" % (type(e).__name__, e), [target._key])[:300] finally: if conn is not None: try: conn.close() except Exception: pass if out["total_ms"] is None: out["total_ms"] = ms() return out # ------------------------------------------------------------------- the run def schedule(n_targets, n): """The order of calls: round by round, every target once per round, and the order within a round reversed every other round, so with two gateways neither is always the one called first (A B, B A, A B, ...).""" order = [] for i in range(n): round_ = list(range(n_targets)) if i % 2 == 1: round_.reverse() order.extend((t, i) for t in round_) return order def run(targets, opts, progress=None, sleep=time.sleep): results = [[] for _ in targets] plan = schedule(len(targets), opts["n"]) for step, (t, i) in enumerate(plan): if step > 0 and opts["pause"] > 0: sleep(opts["pause"]) call = run_one(targets[t], opts) call["i"] = i + 1 results[t].append(call) if progress: progress(targets[t], call) return results def cost_notice(targets, opts): total = opts["n"] * len(targets) lines = ["This will make %d PAID streamed chat completion call%s:" % (total, "" if total == 1 else "s")] for t in targets: lines.append(" %d x model %s at %s" % (opts["n"], t.model, t.base_url)) lines.append("Each call sends a prompt of about %d tokens and may generate up to %d output tokens." % (max(1, len(opts["prompt"]) // 4), opts["max_tokens"])) lines.append("Each gateway bills them to the account that owns the key you gave for it.") lines.append("Worst case per gateway: %d input and %d output tokens in total, at that model's price." % (opts["n"] * max(1, len(opts["prompt"]) // 4), opts["n"] * opts["max_tokens"])) return "\n".join(lines) def build_report(targets, results, opts, started_at): return { "tool": "llm-gateway-bench", "version": VERSION, "started_at": started_at, "parameters": { "n": opts["n"], "max_tokens": opts["max_tokens"], "max_tokens_param": opts["max_tokens_param"], "temperature": opts.get("temperature"), "usage_requested": bool(opts.get("usage", True)), "prompt": opts["prompt"], "prompt_sha256": hashlib.sha256(opts["prompt"].encode("utf-8")).hexdigest(), "timeout_s": opts["timeout"], "pause_s": opts["pause"], "order": "alternating" if len(targets) > 1 else "sequential", "connection": "a new connection per call", "percentile_rule": "nearest rank", "python": platform.python_version(), "platform": platform.system(), }, "targets": [ dict(t.public(), summary=summarize(calls), calls=calls) for t, calls in zip(targets, results) ], } def fmt_ms(v): if v is None: return "n/a" if v < 1000: return "%d ms" % round(v) return "%.2f s" % (v / 1000.0) def fmt_rate(v): return "n/a" if v is None else "%.1f" % v def render_markdown(report): p = report["parameters"] lines = [ "| gateway | model | ok / calls | first byte p50 | first byte p95 | first token p50 | first token p95 " "| total p50 | total p95 | tokens/s p50 | tokens/s weighted |", "|---|---|---|---|---|---|---|---|---|---|---|", ] for t in report["targets"]: s = t["summary"] lines.append("| %s | %s | %d / %d | %s | %s | %s | %s | %s | %s | %s | %s |" % ( t["base_url"], t["model"], s["ok"], s["calls"], fmt_ms(s["first_byte_ms"]["p50"]), fmt_ms(s["first_byte_ms"]["p95"]), fmt_ms(s["first_token_ms"]["p50"]), fmt_ms(s["first_token_ms"]["p95"]), fmt_ms(s["total_ms"]["p50"]), fmt_ms(s["total_ms"]["p95"]), fmt_rate(s["tokens_per_second"]["p50"]), fmt_rate(s["tokens_per_second"]["weighted"]), )) lines.append("") lines.append("Run %s with llm-gateway-bench %s: %d streamed call%s per gateway, max %d output tokens, " "%s, %s. Times start before the connection is opened and include the caller's network. " "Percentiles are nearest-rank over successful calls." % (report["started_at"], report["version"], p["n"], "" if p["n"] == 1 else "s", p["max_tokens"], p["connection"], p["order"] + " order")) for t in report["targets"]: s = t["summary"] if s["failed"]: reasons = ", ".join("%s x%d" % (k, v) for k, v in sorted(s["failure_reasons"].items())) lines.append("Failures at %s: %d of %d (%s)." % (t["base_url"], s["failed"], s["calls"], reasons)) if s["ok"] and s["usage_reported"] < s["ok"]: lines.append("%s reported token usage on %d of %d successful calls; tokens/s covers only those." % (t["base_url"], s["usage_reported"], s["ok"])) lines.append("Prompt (sha256 %s): %s" % (p["prompt_sha256"][:12], p["prompt"])) return "\n".join(lines) + "\n" # ----------------------------------------------------------------------- CLI def parse_args(argv): ap = argparse.ArgumentParser( prog="bench.py", description="Time streamed chat completions on any OpenAI-compatible gateway, from your own network.", epilog="The key is read from the environment (%s; for --compare, %s). It is never printed. " "Every call is a paid API call." % (KEY_ENV, COMPARE_KEY_ENV)) ap.add_argument("--base-url", required=True, help="e.g. https://1xai.ir/v1 (the script adds /chat/completions)") ap.add_argument("--model", required=True, help="model id as that gateway names it") ap.add_argument("--n", type=int, default=DEFAULT_N, help="calls per gateway (default %d)" % DEFAULT_N) ap.add_argument("--max-tokens", type=int, default=DEFAULT_MAX_TOKENS, help="output token limit per call (default %d)" % DEFAULT_MAX_TOKENS) ap.add_argument("--max-tokens-param", choices=["max_tokens", "max_completion_tokens"], default="max_tokens", help="name of the limit field; some newer models accept only max_completion_tokens") ap.add_argument("--temperature", type=float, default=None, help="not sent unless given (some models refuse it)") ap.add_argument("--prompt", default=DEFAULT_PROMPT, help="the one prompt every call sends") ap.add_argument("--no-usage", action="store_true", help="do not send stream_options.include_usage (for a gateway that rejects it; no tokens/s then)") ap.add_argument("--timeout", type=float, default=120.0, help="seconds per call, connect and each read (default 120)") ap.add_argument("--pause", type=float, default=0.5, help="seconds between calls (default 0.5)") ap.add_argument("--key-env", default=KEY_ENV, help="environment variable holding the key (default %s)" % KEY_ENV) ap.add_argument("--compare", metavar="BASE_URL", help="a second gateway, called alternately with the first") ap.add_argument("--compare-model", help="model id on the second gateway (default: the same as --model)") ap.add_argument("--compare-key-env", default=COMPARE_KEY_ENV, help="environment variable holding the second gateway's key (default %s)" % COMPARE_KEY_ENV) ap.add_argument("--json", metavar="FILE", help="write the full result as JSON to FILE ('-' for stdout)") ap.add_argument("--markdown", metavar="FILE", help="write the summary table as Markdown to FILE") ap.add_argument("--yes", action="store_true", help="do not ask before making the paid calls") ap.add_argument("--version", action="version", version=VERSION) args = ap.parse_args(argv) if args.n < 1 or args.n > 1000: ap.error("--n must be between 1 and 1000") if args.max_tokens < 1 or args.max_tokens > 8192: ap.error("--max-tokens must be between 1 and 8192") if args.timeout <= 0 or args.pause < 0: ap.error("--timeout must be positive and --pause must not be negative") return args def main(argv=None, env=None, stdin=None, stdout=None, stderr=None): env = os.environ if env is None else env stdin = sys.stdin if stdin is None else stdin stdout = sys.stdout if stdout is None else stdout stderr = sys.stderr if stderr is None else stderr args = parse_args(sys.argv[1:] if argv is None else argv) def say(text=""): stderr.write(text + "\n") stderr.flush() key = (env.get(args.key_env) or "").strip() if not key: say("No key: set the environment variable %s to your API key for %s." % (args.key_env, args.base_url)) return 2 try: targets = [Target("A", args.base_url, args.model, key)] if args.compare: key2 = (env.get(args.compare_key_env) or "").strip() if not key2: # Never fall back to the first key: it belongs to another gateway. say("No key for the second gateway: set %s. The first gateway's key is never sent to it." % args.compare_key_env) return 2 targets.append(Target("B", args.compare, args.compare_model or args.model, key2)) except ValueError as e: say(str(e)) return 2 opts = { "n": args.n, "max_tokens": args.max_tokens, "max_tokens_param": args.max_tokens_param, "temperature": args.temperature, "prompt": args.prompt, "usage": not args.no_usage, "timeout": args.timeout, "pause": args.pause, } say(cost_notice(targets, opts)) if not args.yes: if not stdin or not stdin.isatty(): say("Not started: this is not a terminal, so nobody can confirm. Pass --yes to make the paid calls.") return 2 stderr.write("Continue? [y/N] ") stderr.flush() if stdin.readline().strip().lower() not in ("y", "yes"): say("Not started. No call was made.") return 1 def progress(target, call): if call["ok"]: say(" %s #%d first byte %s, first token %s, total %s, %s tokens/s" % (target.label, call["i"], fmt_ms(call["first_byte_ms"]), fmt_ms(call["first_token_ms"]), fmt_ms(call["total_ms"]), fmt_rate(call["tokens_per_second"]))) else: say(" %s #%d FAILED: %s %s" % (target.label, call["i"], call["error"], call.get("detail", ""))) started_at = datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") results = run(targets, opts, progress=progress) report = build_report(targets, results, opts, started_at) markdown = render_markdown(report) if args.json == "-": stdout.write(json.dumps(report, indent=2) + "\n") else: if args.json: with open(args.json, "w", encoding="utf-8") as f: json.dump(report, f, indent=2) f.write("\n") stdout.write(markdown) if args.markdown: with open(args.markdown, "w", encoding="utf-8") as f: f.write(markdown) stdout.flush() # 0 when every call succeeded, 3 when some failed: a script can tell. return 0 if all(c["ok"] for calls in results for c in calls) else 3 if __name__ == "__main__": try: sys.exit(main()) except KeyboardInterrupt: sys.stderr.write("\nInterrupted. Calls already made were billed.\n") sys.exit(130)