Measure what this library sustains, and against Jasmin and SMPPSim

benchmarks/README.md

@@ -0,0 +1,56 @@
+# Benchmarks
+
+What this library sustains, and against what. Run them before setting or changing a scale goal.
+
+```bash
+docker compose run --rm node node benchmarks/run.ts              # this library, both ends
+COUNT=100000 docker compose run --rm node node benchmarks/run.ts # longer, steadier
+```
+
+`run.ts` spawns `smsc-sink.ts` — a server that answers every `submit_sm` `ESME_ROK` and stores
+nothing — and drives it with `submit-load.ts` at four window sizes. Point `submit-load.ts` at any
+SMSC to compare:
+
+```bash
+docker compose -f compose.yaml -f interop-tests/compose.jasmin.yaml run --rm node \
+	node benchmarks/submit-load.ts --host=jasmin --port=2775 --username=esme1 --password=esme1pw \
+	--count=20000 --concurrency=50
+```
+
+**A short run measures the JIT, not the library.** At 2,000 messages per point the same build
+reported 16k/s where 100,000 messages reported 40k/s. Give each point several seconds.
+
+## Results, 2026-09-20
+
+Single host, 8 cores, Node 24.18.0 in the project container, loopback. Client and SMSC are separate
+processes competing for the same CPUs, so these are a floor for split hosts.
+
+`maxOutstanding` is the send window, and it is the setting that matters most:
+
+| Window | msgs/s | with `dlr: true` |
+| --- | --- | --- |
+| 1 | 7,385 | 7,289 |
+| 10 (default) | 25,358 | 24,282 |
+| 50 | 37,125 | 37,502 |
+| 200 | 40,046 | 39,730 |
+
+Requesting delivery receipts costs nothing measurable at any window. A window of 1 — one request in
+flight at a time — costs 5x, which is the round trip rather than the codec.
+
+Against real peers, same driver, window 50, 20,000 messages:
+
+| SMSC | msgs/s | failed |
+| --- | --- | --- |
+| this library's sink | 37,125 | 0 |
+| Jasmin 0.10 | 2,207 | 0 |
+| SMPPSim 3.0.0 | — | 19,000 of 20,000 |
+
+Jasmin routes and persists where the sink does neither, so the gap is not an efficiency ratio
+between two comparable things — what it establishes is that this library is not the bottleneck
+against a production SMSC, by more than an order of magnitude. SMPPSim's store fills at roughly a
+thousand messages and it then refuses the rest, so it cannot be loaded; that is a property of the
+simulator, not a result.
+
+`smppload`, the one purpose-built SMPP load generator among the peers, is blocked by a bind defect
+of its own ([findings/07-load.md](../interop-tests/findings/07-load.md)), so no third-party load
+tool drives these numbers.

benchmarks/run.ts

@@ -0,0 +1,63 @@
+import { spawn } from 'node:child_process';
+import { once } from 'node:events';
+import { createInterface } from 'node:readline';
+
+/**
+ * Node rather than Python, unlike the repo's other standalone scripts: it spawns the two processes
+ * it measures, and they must run on the same runtime this library is measured under.
+ */
+const concurrencies = [1, 10, 50, 200];
+const count = Number(process.env.COUNT ?? 5000);
+
+function node(script: string, args: string[] = []) {
+	return spawn(process.execPath, [`${import.meta.dirname}/${script}`, ...args], {
+		stdio: ['ignore', 'pipe', 'inherit'],
+	});
+}
+
+async function firstLine(stream: NodeJS.ReadableStream): Promise<string> {
+	for await (const line of createInterface({ input: stream })) {
+		return line;
+	}
+
+	return '';
+}
+
+const sink = node('smsc-sink.ts');
+const listening: unknown = JSON.parse(await firstLine(sink.stdout));
+
+if (typeof listening !== 'object' || listening === null || !('port' in listening)) {
+	throw new Error('the sink did not report a port');
+}
+
+const port = String(listening.port);
+const rows: Record<string, unknown>[] = [];
+
+for (const dlr of [false, true]) {
+	for (const concurrency of concurrencies) {
+		const load = node('submit-load.ts', [
+			`--port=${port}`,
+			`--count=${String(count)}`,
+			`--concurrency=${String(concurrency)}`,
+			...(dlr ? ['--dlr'] : []),
+		]);
+		const reported: unknown = JSON.parse(await firstLine(load.stdout));
+
+		await once(load, 'exit');
+
+		if (typeof reported === 'object' && reported !== null) rows.push({ ...reported });
+	}
+}
+
+sink.kill('SIGTERM');
+await once(sink, 'exit');
+
+process.stdout.write(`\n${'dlr'.padEnd(6)}${'window'.padEnd(9)}${'msgs/s'.padEnd(10)}seconds\n`);
+
+for (const row of rows) {
+	const dlr = String(row.dlr).padEnd(6);
+	const window = String(row.concurrency).padEnd(9);
+	const rate = String(row.perSecond).padEnd(10);
+
+	process.stdout.write(`${dlr}${window}${rate}${String(row.seconds)}\n`);
+}

benchmarks/smsc-sink.ts

@@ -0,0 +1,33 @@
+import { server } from '../src/server.ts';
+
+/**
+ * Answers every submit_sm ESME_ROK and does nothing else, so a measurement against it reads this
+ * library's own ceiling rather than an SMSC's storage. Prints the bound port on stdout, then waits.
+ */
+const port = Number(process.env.PORT ?? 0);
+const { err, server: smpp } = await server({ port });
+
+if (err) {
+	process.stderr.write(`sink failed to listen: ${err.message}\n`);
+	process.exit(1);
+}
+
+let answered = 0;
+
+smpp.on('session', session => {
+	session.on('sms', async sms => {
+		answered++;
+		await sms.sendResp();
+	});
+});
+
+smpp.on('serverError', reason => {
+	process.stderr.write(`sink serverError: ${reason.message}\n`);
+});
+
+process.stdout.write(`${JSON.stringify({ port: smpp.port })}\n`);
+
+process.on('SIGTERM', () => {
+	process.stderr.write(`sink answered ${String(answered)}\n`);
+	void smpp.close({ signal: AbortSignal.abort() }).then(() => process.exit(0));
+});

benchmarks/submit-load.ts

@@ -0,0 +1,72 @@
+import { client } from '../src/client.ts';
+
+/**
+ * Pushes `count` single-segment messages and reports what the wire carried per second. Keeps
+ * `concurrency` sends in flight so the send window, not the caller, is what bounds the rate.
+ */
+function arg(name: string, fallback: string): string {
+	const found = process.argv.find(one => one.startsWith(`--${name}=`));
+
+	return found === undefined ? fallback : found.slice(name.length + 3);
+}
+
+const host = arg('host', '127.0.0.1');
+const port = Number(arg('port', '2775'));
+const count = Number(arg('count', '2000'));
+const concurrency = Number(arg('concurrency', '20'));
+const message = arg('message', 'benchmark');
+const dlr = process.argv.includes('--dlr');
+const username = arg('username', 'user');
+const password = arg('password', 'pass');
+
+const { err, session } = await client({
+	host,
+	maxOutstanding: concurrency,
+	password,
+	port,
+	responseTimeout: 60_000,
+	username,
+});
+
+if (err) {
+	process.stdout.write(`${JSON.stringify({ error: err.message })}\n`);
+	process.exit(1);
+}
+
+// Narrowing from the guard above does not reach into worker(), which runs after it.
+const bound = session;
+
+let issued = 0;
+let failed = 0;
+let unanswered = 0;
+
+async function worker(): Promise<void> {
+	while (issued < count) {
+		issued++;
+
+		const sent = await bound.sendSms({ dlr, from: 'BENCH', message, to: '46709771337' });
+
+		if (sent.err) failed++;
+
+		unanswered += sent.unanswered;
+	}
+}
+
+const started = process.hrtime.bigint();
+
+await Promise.all(Array.from({ length: concurrency }, () => worker()));
+
+const seconds = Number(process.hrtime.bigint() - started) / 1e9;
+
+process.stdout.write(`${JSON.stringify({
+	concurrency,
+	count,
+	dlr,
+	failed,
+	perSecond: Math.round(count / seconds),
+	seconds: Number(seconds.toFixed(3)),
+	unanswered,
+})}\n`);
+
+await bound.close({ signal: AbortSignal.abort() });
+process.exit(0);

tsconfig.json

@@ -24,5 +24,5 @@
 		"types": ["node"],
 		"verbatimModuleSyntax": true
 	},
-	"include": ["eslint.config.ts", "interop-tests", "src", "test"]
+	"include": ["benchmarks", "eslint.config.ts", "interop-tests", "src", "test"]
 }