diff --git a/benchmarks/README.md b/benchmarks/README.md new file mode 100644 index 0000000..5ad2c83 --- /dev/null +++ b/benchmarks/README.md @@ -0,0 +1,56 @@ +# Benchmarks + +What this library sustains, and against what. Run them before setting or changing a scale goal. + +```bash +docker compose run --rm node node benchmarks/run.ts # this library, both ends +COUNT=100000 docker compose run --rm node node benchmarks/run.ts # longer, steadier +``` + +`run.ts` spawns `smsc-sink.ts` — a server that answers every `submit_sm` `ESME_ROK` and stores +nothing — and drives it with `submit-load.ts` at four window sizes. Point `submit-load.ts` at any +SMSC to compare: + +```bash +docker compose -f compose.yaml -f interop-tests/compose.jasmin.yaml run --rm node \ + node benchmarks/submit-load.ts --host=jasmin --port=2775 --username=esme1 --password=esme1pw \ + --count=20000 --concurrency=50 +``` + +**A short run measures the JIT, not the library.** At 2,000 messages per point the same build +reported 16k/s where 100,000 messages reported 40k/s. Give each point several seconds. + +## Results, 2026-09-20 + +Single host, 8 cores, Node 24.18.0 in the project container, loopback. Client and SMSC are separate +processes competing for the same CPUs, so these are a floor for split hosts. + +`maxOutstanding` is the send window, and it is the setting that matters most: + +| Window | msgs/s | with `dlr: true` | +| --- | --- | --- | +| 1 | 7,385 | 7,289 | +| 10 (default) | 25,358 | 24,282 | +| 50 | 37,125 | 37,502 | +| 200 | 40,046 | 39,730 | + +Requesting delivery receipts costs nothing measurable at any window. A window of 1 — one request in +flight at a time — costs 5x, which is the round trip rather than the codec. + +Against real peers, same driver, window 50, 20,000 messages: + +| SMSC | msgs/s | failed | +| --- | --- | --- | +| this library's sink | 37,125 | 0 | +| Jasmin 0.10 | 2,207 | 0 | +| SMPPSim 3.0.0 | — | 19,000 of 20,000 | + +Jasmin routes and persists where the sink does neither, so the gap is not an efficiency ratio +between two comparable things — what it establishes is that this library is not the bottleneck +against a production SMSC, by more than an order of magnitude. SMPPSim's store fills at roughly a +thousand messages and it then refuses the rest, so it cannot be loaded; that is a property of the +simulator, not a result. + +`smppload`, the one purpose-built SMPP load generator among the peers, is blocked by a bind defect +of its own ([findings/07-load.md](../interop-tests/findings/07-load.md)), so no third-party load +tool drives these numbers. diff --git a/benchmarks/run.ts b/benchmarks/run.ts new file mode 100644 index 0000000..18df972 --- /dev/null +++ b/benchmarks/run.ts @@ -0,0 +1,63 @@ +import { spawn } from 'node:child_process'; +import { once } from 'node:events'; +import { createInterface } from 'node:readline'; + +/** + * Node rather than Python, unlike the repo's other standalone scripts: it spawns the two processes + * it measures, and they must run on the same runtime this library is measured under. + */ +const concurrencies = [1, 10, 50, 200]; +const count = Number(process.env.COUNT ?? 5000); + +function node(script: string, args: string[] = []) { + return spawn(process.execPath, [`${import.meta.dirname}/${script}`, ...args], { + stdio: ['ignore', 'pipe', 'inherit'], + }); +} + +async function firstLine(stream: NodeJS.ReadableStream): Promise { + for await (const line of createInterface({ input: stream })) { + return line; + } + + return ''; +} + +const sink = node('smsc-sink.ts'); +const listening: unknown = JSON.parse(await firstLine(sink.stdout)); + +if (typeof listening !== 'object' || listening === null || !('port' in listening)) { + throw new Error('the sink did not report a port'); +} + +const port = String(listening.port); +const rows: Record[] = []; + +for (const dlr of [false, true]) { + for (const concurrency of concurrencies) { + const load = node('submit-load.ts', [ + `--port=${port}`, + `--count=${String(count)}`, + `--concurrency=${String(concurrency)}`, + ...(dlr ? ['--dlr'] : []), + ]); + const reported: unknown = JSON.parse(await firstLine(load.stdout)); + + await once(load, 'exit'); + + if (typeof reported === 'object' && reported !== null) rows.push({ ...reported }); + } +} + +sink.kill('SIGTERM'); +await once(sink, 'exit'); + +process.stdout.write(`\n${'dlr'.padEnd(6)}${'window'.padEnd(9)}${'msgs/s'.padEnd(10)}seconds\n`); + +for (const row of rows) { + const dlr = String(row.dlr).padEnd(6); + const window = String(row.concurrency).padEnd(9); + const rate = String(row.perSecond).padEnd(10); + + process.stdout.write(`${dlr}${window}${rate}${String(row.seconds)}\n`); +} diff --git a/benchmarks/smsc-sink.ts b/benchmarks/smsc-sink.ts new file mode 100644 index 0000000..324c11e --- /dev/null +++ b/benchmarks/smsc-sink.ts @@ -0,0 +1,33 @@ +import { server } from '../src/server.ts'; + +/** + * Answers every submit_sm ESME_ROK and does nothing else, so a measurement against it reads this + * library's own ceiling rather than an SMSC's storage. Prints the bound port on stdout, then waits. + */ +const port = Number(process.env.PORT ?? 0); +const { err, server: smpp } = await server({ port }); + +if (err) { + process.stderr.write(`sink failed to listen: ${err.message}\n`); + process.exit(1); +} + +let answered = 0; + +smpp.on('session', session => { + session.on('sms', async sms => { + answered++; + await sms.sendResp(); + }); +}); + +smpp.on('serverError', reason => { + process.stderr.write(`sink serverError: ${reason.message}\n`); +}); + +process.stdout.write(`${JSON.stringify({ port: smpp.port })}\n`); + +process.on('SIGTERM', () => { + process.stderr.write(`sink answered ${String(answered)}\n`); + void smpp.close({ signal: AbortSignal.abort() }).then(() => process.exit(0)); +}); diff --git a/benchmarks/submit-load.ts b/benchmarks/submit-load.ts new file mode 100644 index 0000000..95de428 --- /dev/null +++ b/benchmarks/submit-load.ts @@ -0,0 +1,72 @@ +import { client } from '../src/client.ts'; + +/** + * Pushes `count` single-segment messages and reports what the wire carried per second. Keeps + * `concurrency` sends in flight so the send window, not the caller, is what bounds the rate. + */ +function arg(name: string, fallback: string): string { + const found = process.argv.find(one => one.startsWith(`--${name}=`)); + + return found === undefined ? fallback : found.slice(name.length + 3); +} + +const host = arg('host', '127.0.0.1'); +const port = Number(arg('port', '2775')); +const count = Number(arg('count', '2000')); +const concurrency = Number(arg('concurrency', '20')); +const message = arg('message', 'benchmark'); +const dlr = process.argv.includes('--dlr'); +const username = arg('username', 'user'); +const password = arg('password', 'pass'); + +const { err, session } = await client({ + host, + maxOutstanding: concurrency, + password, + port, + responseTimeout: 60_000, + username, +}); + +if (err) { + process.stdout.write(`${JSON.stringify({ error: err.message })}\n`); + process.exit(1); +} + +// Narrowing from the guard above does not reach into worker(), which runs after it. +const bound = session; + +let issued = 0; +let failed = 0; +let unanswered = 0; + +async function worker(): Promise { + while (issued < count) { + issued++; + + const sent = await bound.sendSms({ dlr, from: 'BENCH', message, to: '46709771337' }); + + if (sent.err) failed++; + + unanswered += sent.unanswered; + } +} + +const started = process.hrtime.bigint(); + +await Promise.all(Array.from({ length: concurrency }, () => worker())); + +const seconds = Number(process.hrtime.bigint() - started) / 1e9; + +process.stdout.write(`${JSON.stringify({ + concurrency, + count, + dlr, + failed, + perSecond: Math.round(count / seconds), + seconds: Number(seconds.toFixed(3)), + unanswered, +})}\n`); + +await bound.close({ signal: AbortSignal.abort() }); +process.exit(0); diff --git a/tsconfig.json b/tsconfig.json index 8075b0a..3e7323a 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -24,5 +24,5 @@ "types": ["node"], "verbatimModuleSyntax": true }, - "include": ["eslint.config.ts", "interop-tests", "src", "test"] + "include": ["benchmarks", "eslint.config.ts", "interop-tests", "src", "test"] }