Measure, size, and check inference for your AI agent.

# Check a probe report against your targets, get percentiles, and fetch the skill with your settings. No key; nothing is stored.# Percentiles from raw samples (ms), dropping 2 warm-up samplescurl -s "https://inference.md/api/v1/latency_stats?samples=212,240,198,305,221,250,233,990,215,260&drop_first=2&target=400" | jq .summary# Pass or fail per target, from the probe's reportjq '{measured: .measured, targets: {ttft_p50_ms: 800, ttft_p95_ms: 2000, itl_p95_ms: 100, min_stream_tps: 20, max_error_pct: 1}}' inference-report.json \  | curl -s https://inference.md/api/v1/slo_check -H 'content-type: application/json' -d @- \  | jq '{summary, checks: [.checks[] | {target, result, margin}], notes}'# The skill with your targets written in (an example settings link)curl -s "https://inference.md/inference.md?preset=chat&ttft_p95_ms=1500&test_max_requests=50" -D - -o SKILL.md | grep -i x-settings

// The local probe: measures TTFT, inter-token latency, tokens/s and errors on an endpoint you own or may test. Runs on your machine; sends nothing to inference.md.//// inference-probe.mjs (MIT, from https://inference.md). Read it before you run it.// Provided as is, without warranty, at your own risk: results are estimates from// one run at one load and may be wrong. Terms: https://inference.md/terms.// Node 18 or later, no dependencies.//// It sends streaming requests in the common chat-completions JSON format// (POST <base>/chat/completions with { model, messages, max_tokens, stream: true })// and reads the server-sent events that come back ("data: {...}" lines, ending// with "data: [DONE]"). Then it writes a JSON report and compares it with your// targets. Only test endpoints you own or have written permission to test,// and stay inside their terms and rate limits. Requests cost what your// provider or hardware costs; you are responsible for every request sent.////   export INFERENCE_BASE_URL=http://localhost:8000/v1   # your endpoint//   export INFERENCE_MODEL=my-model                      # the model id it expects//   export INFERENCE_API_KEY=...                         # optional; read from the environment only//   node inference-probe.mjs --requests 20 --concurrency 2 --targets targets.json//   (it asks before sending; add --price-in, --price-out and --max-spend to cap the cost)//// Options (defaults in brackets)://   --requests N         measured requests [20]; asks before more than 100//   --concurrency C      requests at a time [2]; asks before more than 8//   --warmup W           warm-up requests, left out of the results [2]//   --max-tokens T       max output tokens per request [128]//   --prompt TEXT        the prompt [a short fixed prompt]//   --timeout S          seconds per request before it counts as an error [60]//   --targets FILE       JSON with targets, same keys as inference.md settings//                        (ttft_p50_ms, ttft_p95_ms, itl_p95_ms, min_stream_tps, min_total_tps, max_error_pct)//   --price-in P         your price per million input tokens (for the spend check)//   --price-out P        your price per million output tokens (for the spend check)//   --max-spend S        refuse to start if the worst-case cost is above S//   --usage              ask the server for token usage in the stream (not every server supports it)//   --out FILE           report path [inference-report.json]//   --permitted          you confirm you own the endpoint or have written permission to test it//   --yes                skip the size and spend prompts (the --permitted rule still applies)import { writeFileSync, readFileSync } from "node:fs";import { createInterface } from "node:readline/promises";import { stdin, stdout, argv, env, exit } from "node:process";const VERSION = "1.0";function parseArgs(list) {  const o = {};  for (let i = 0; i < list.length; i++) {    const a = list[i];    if (!a.startsWith("--")) continue;    const k = a.slice(2);    if (k === "usage" || k === "yes" || k === "permitted" || k === "help") o[k] = true;    else o[k] = list[++i];  }  return o;}function intOpt(o, k, d, min, max) {  if (o[k] === undefined) return d;  const n = Number(o[k]);  if (!Number.isInteger(n) || n < min || n > max) die(`--${k} must be a whole number from ${min} to ${max}.`);  return n;}function numOpt(o, k, min = 0) {  if (o[k] === undefined) return undefined;  const n = Number(o[k]);  if (!Number.isFinite(n) || n < min) die(`--${k} must be a number of ${min} or more.`);  return n;}function die(msg) {  console.error(`inference-probe: ${msg}`);  exit(1);}async function ask(question) {  if (!stdin.isTTY) return false;  const rl = createInterface({ input: stdin, output: stdout });  const a = await rl.question(`${question} (yes/no) `);  rl.close();  return /^y(es)?$/i.test(a.trim());}// Percentile by linear interpolation between closest ranks (same method as the latency_stats tool).function percentile(xs, q) {  if (!xs.length) return null;  const s = [...xs].sort((a, b) => a - b);  const h = (s.length - 1) * q;  const lo = Math.floor(h);  const hi = Math.min(lo + 1, s.length - 1);  return s[lo] + (h - lo) * (s[hi] - s[lo]);}const r = (n, d = 2) => (n === null || n === undefined ? null : Math.round(n * 10 ** d) / 10 ** d);async function oneRequest(cfg) {  const ctrl = new AbortController();  const timer = setTimeout(() => ctrl.abort(), cfg.timeoutS * 1000);  const t0 = performance.now();  const times = [];  let usageTokens = null;  let finished = false;  try {    const body = { model: cfg.model, messages: [{ role: "user", content: cfg.prompt }], max_tokens: cfg.maxTokens, stream: true };    if (cfg.usage) body.stream_options = { include_usage: true };    const res = await fetch(cfg.url, {      method: "POST",      headers: { "content-type": "application/json", accept: "text/event-stream", ...(cfg.key ? { authorization: `Bearer ${cfg.key}` } : {}) },      body: JSON.stringify(body),      signal: ctrl.signal,    });    if (!res.ok || !res.body) {      await res.body?.cancel().catch(() => {});      return { ok: false, error: `status ${res.status}`, total_ms: performance.now() - t0 };    }    const reader = res.body.getReader();    const dec = new TextDecoder();    let buf = "";    for (;;) {      const { value, done } = await reader.read();      if (done) break;      buf += dec.decode(value, { stream: true });      let nl;      while ((nl = buf.indexOf("\n")) >= 0) {        const line = buf.slice(0, nl).trim();        buf = buf.slice(nl + 1);        if (!line.startsWith("data:")) continue;        const data = line.slice(5).trim();        if (data === "[DONE]") {          finished = true;          continue;        }        let j;        try {          j = JSON.parse(data);        } catch {          continue;        }        const c = j?.choices?.[0];        const text = c?.delta?.content ?? c?.text ?? "";        if (typeof text === "string" && text.length > 0) times.push(performance.now());        if (c?.finish_reason) finished = true;        if (typeof j?.usage?.completion_tokens === "number") usageTokens = j.usage.completion_tokens;      }    }    const end = performance.now();    if (!times.length) return { ok: false, error: "no tokens", total_ms: end - t0 };    if (!finished) return { ok: false, error: "stream ended early", total_ms: end - t0 };    const tokens = usageTokens ?? times.length;    const gaps = times.slice(1).map((t, i) => t - times[i]);    const span = (times[times.length - 1] - times[0]) / 1000;    return {      ok: true,      ttft_ms: times[0] - t0,      total_ms: end - t0,      tokens,      token_source: usageTokens !== null ? "usage" : "chunks",      gaps,      stream_tps: span > 0 && tokens > 1 ? (tokens - 1) / span : null,    };  } catch (e) {    return { ok: false, error: e?.name === "AbortError" ? "timeout" : String(e?.message ?? e).slice(0, 120), total_ms: performance.now() - t0 };  } finally {    clearTimeout(timer);  }}async function runPool(n, conc, cfg) {  const results = [];  let next = 0;  const worker = async () => {    while (next < n) {      next++;      results.push(await oneRequest(cfg));    }  };  await Promise.all(Array.from({ length: Math.min(conc, n) }, worker));  return results;}const TARGETS = [  ["ttft_p50_ms", "ttft_p50_ms", "max"],  ["ttft_p95_ms", "ttft_p95_ms", "max"],  ["itl_p95_ms", "itl_p95_ms", "max"],  ["min_stream_tps", "stream_tps_p10", "min"],  ["min_total_tps", "total_tps", "min"],  ["max_error_pct", "error_pct", "max"],];function compare(measured, targets) {  const checks = [];  for (const [t, m, dir] of TARGETS) {    const target = Number(targets[t]);    if (targets[t] === undefined || !Number.isFinite(target)) continue;    const value = measured[m];    if (value === null || value === undefined) {      checks.push({ target: t, target_value: target, measured_value: null, result: "not measured" });      continue;    }    const margin = dir === "max" ? target - value : value - target;    checks.push({ target: t, target_value: target, measured_value: r(value), result: margin >= 0 ? "pass" : "fail", margin: r(margin) });  }  return checks;}async function main() {  const o = parseArgs(argv.slice(2));  if (o.help) {    console.log(readFileSync(new URL(import.meta.url), "utf8").split("\n").filter((l) => l.startsWith("//")).join("\n"));    return;  }  const base = env.INFERENCE_BASE_URL;  const model = env.INFERENCE_MODEL;  if (!base || !model) die("set INFERENCE_BASE_URL and INFERENCE_MODEL first (see the top of this file).");  let url;  try {    url = new URL(`${base.replace(/\/+$/, "")}/chat/completions`);  } catch {    die("INFERENCE_BASE_URL is not a valid URL.");  }  const requests = intOpt(o, "requests", 20, 1, 100_000);  const concurrency = intOpt(o, "concurrency", 2, 1, 1000);  const warmup = intOpt(o, "warmup", 2, 0, 1000);  const maxTokens = intOpt(o, "max-tokens", 128, 1, 1_000_000);  const timeoutS = intOpt(o, "timeout", 60, 1, 86_400);  const prompt = typeof o.prompt === "string" && o.prompt ? o.prompt : "Write three short sentences about measuring time.";  const out = typeof o.out === "string" ? o.out : "inference-report.json";  let targets = {};  if (o.targets) {    try {      targets = JSON.parse(readFileSync(o.targets, "utf8"));    } catch {      die(`can't read targets from ${o.targets}.`);    }    if (!targets || typeof targets !== "object" || Array.isArray(targets)) die(`${o.targets} must hold a JSON object of targets.`);  }  console.log(`inference-probe ${VERSION}: ${warmup} warm-up + ${requests} measured requests, concurrency ${concurrency}, to ${url.origin}${url.pathname}`);  console.log("Test only endpoints you own or have written permission to test, and stay inside their terms and rate limits. You are responsible for every request sent.");  if (!o.permitted && !(await ask("Do you own this endpoint, or have written permission to test it, within its terms and rate limits?"))) {    die("stopped. Run only against endpoints you own or may test (pass --permitted to confirm).");  }  const local = /^(localhost|127\.\d+\.\d+\.\d+|\[::1\])$/.test(url.hostname);  if (env.INFERENCE_API_KEY && url.protocol === "http:" && !local) console.log("Warning: the API key would be sent over plain http. Use https for any endpoint that isn't on this machine.");  if (requests > 100 && !o.yes && !(await ask(`${requests} requests is more than 100. Continue?`))) die("stopped.");  if (concurrency > 8 && !o.yes && !(await ask(`Concurrency ${concurrency} is more than 8. Continue?`))) die("stopped.");  // Worst-case spend from your own prices: every request uses the full max_tokens. Input tokens are estimated at 4 characters per token.  const priceIn = numOpt(o, "price-in");  const priceOut = numOpt(o, "price-out");  const maxSpend = numOpt(o, "max-spend");  const total = requests + warmup;  const estInputTokens = Math.ceil(prompt.length / 4) + 8;  const worst = priceIn !== undefined || priceOut !== undefined ? (total * (estInputTokens * (priceIn ?? 0) + maxTokens * (priceOut ?? 0))) / 1e6 : null;  if (worst !== null) console.log(`Worst-case cost estimate: ${r(worst, 4)} (your prices, ${total} requests, ${maxTokens} output tokens each).`);  if (maxSpend !== undefined) {    if (worst === null) die("--max-spend needs --price-in and/or --price-out to estimate the cost.");    if (worst > maxSpend) die(`worst-case cost ${r(worst, 4)} is above --max-spend ${maxSpend}. Lower --requests or --max-tokens.`);  } else if (!o.yes && !(await ask(`No spend cap set (--price-in, --price-out and --max-spend). Send up to ${total} requests of up to ${maxTokens} output tokens anyway?`))) {    die("stopped. Pass --price-in, --price-out and --max-spend to cap the cost.");  }  const cfg = { url: url.toString(), model, key: env.INFERENCE_API_KEY, prompt, maxTokens, timeoutS, usage: Boolean(o.usage) };  if (warmup) await runPool(warmup, Math.min(concurrency, warmup), cfg);  const t0 = performance.now();  const results = await runPool(requests, concurrency, cfg);  const wallS = (performance.now() - t0) / 1000;  const ok = results.filter((x) => x.ok);  const errors = results.length - ok.length;  const ttft = ok.map((x) => x.ttft_ms);  const gaps = ok.flatMap((x) => x.gaps);  const streams = ok.map((x) => x.stream_tps).filter((x) => x !== null);  const tokens = ok.reduce((s, x) => s + x.tokens, 0);  const measured = {    requests: results.length,    errors,    error_pct: r((errors / results.length) * 100, 3),    concurrency,    ttft_p50_ms: r(percentile(ttft, 0.5)),    ttft_p95_ms: r(percentile(ttft, 0.95)),    ttft_p99_ms: r(percentile(ttft, 0.99)),    itl_p50_ms: r(percentile(gaps, 0.5)),    itl_p95_ms: r(percentile(gaps, 0.95)),    stream_tps_p10: r(percentile(streams, 0.1)),    stream_tps_p50: r(percentile(streams, 0.5)),    total_tps: r(tokens / wallS),    output_tokens: tokens,    wall_s: r(wallS, 3),  };  const sources = new Set(ok.map((x) => x.token_source));  const notes = [];  if (sources.has("chunks")) notes.push("Token counts are estimates: one streamed chunk was counted as one token. Use --usage if the server reports usage.");  if (results.length < 100) notes.push(`p95 from ${results.length} requests is rough; p99 even more so.`);  const errorKinds = {};  for (const x of results) if (!x.ok) errorKinds[x.error] = (errorKinds[x.error] ?? 0) + 1;  const report = {    tool: "inference-probe",    version: VERSION,    started_at: new Date(Date.now() - wallS * 1000).toISOString(),    endpoint: `${url.origin}${url.pathname}`,    setup: { requests, warmup, concurrency, max_tokens: maxTokens, timeout_s: timeoutS, prompt_chars: prompt.length },    measured,    errors_by_kind: errorKinds,    checks: compare(measured, targets),    notes,    samples: { ttft_ms: ttft.map((x) => r(x)), stream_tps: streams.map((x) => r(x)) },    disclaimer: "A measurement at one time and load, not a guarantee. Figures are estimates. See https://inference.md/terms.",  };  writeFileSync(out, JSON.stringify(report, null, 2));  console.log(`\n${ok.length}/${results.length} ok, ${errors} errors (${measured.error_pct}%), ${measured.total_tps} tokens/s total over ${measured.wall_s} s`);  console.log(`TTFT p50 ${measured.ttft_p50_ms} ms, p95 ${measured.ttft_p95_ms} ms · ITL p95 ${measured.itl_p95_ms} ms · per stream p10 ${measured.stream_tps_p10} tokens/s`);  for (const c of report.checks) console.log(`  ${c.result.padEnd(12)} ${c.target}: measured ${c.measured_value}, target ${c.target_value}${c.margin !== undefined ? `, margin ${c.margin}` : ""}`);  for (const n of notes) console.log(`  note: ${n}`);  console.log(`Report: ${out}`);  if (report.checks.some((c) => c.result === "fail")) exit(2);}main().catch((e) => die(String(e?.message ?? e)));

// Size replicas and cost per million tokens from your own measurements (Node 18+, run with any TypeScript runner).const API = "https://inference.md/api/v1";async function call(tool: string, args: Record<string, unknown>) {  const res = await fetch(`${API}/${tool}`, { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify(args) });  const body = await res.json();  if (!res.ok) throw new Error(`${tool}: ${body.error} ${body.hint ?? ""}`);  return body;}// Your numbers: traffic, request shape, and what one replica did in your own test.const plan = await call("capacity_plan", {  arrival_rate: 120, rate_unit: "minute", peak_factor: 2,  avg_input_tokens: 1500, avg_output_tokens: 300,  ttft_ms: 600, stream_tps: 30,  per_replica_output_tps: 900, per_replica_concurrency: 32,  target_utilization_pct: 70, spare_replicas: 1,});console.log(plan.summary);// Your all-in hourly cost per replica, and a per-token price you want to compare with.const cost = await call("cost_per_mtok", { hourly_cost: 4, throughput_tps: 900, utilization_pct: 60, compare_price_per_mtok: 2 });console.log(cost.summary);console.log(plan.notes.join("\n"));