Add message helpers: segmentation, bit counting and SMPP time

2026-08-25 16:23:23 +02:00
parent 79c9df79c6
commit db5dc8cd15
5 changed files with 354 additions and 4 deletions
@@ -87,7 +87,7 @@ implementation (see todo.md).
 | Defect | 0.4.0 behaviour |
 | --- | --- |
 | LATIN1 never decodes | `decodeMsg` loops `consts.ENCODING` without breaking, so `data_coding` 0x03 lands on the alias `ISO_8859_1`, which has no decoder, and silently falls back to ASCII |
-| Oversized segments | `splitMsg` emits 152 GSM chars + 6-byte UDH = 158 octets, over the 140-octet limit; UCS2 gets 66 chars where 67 fit |
+| Short segments | `splitMsg` accumulates a full segment then pushes `msgPart.slice(0, -1)`, so every segment is one character short: 152 GSM characters instead of 153, 66 UCS2 instead of 67. Long messages are split into more segments than they need, and each extra segment is billed |
 | DLR month off by one | `smppDate()` uses `getMonth()` (0-based) without `+1`, so January renders as `00` |
 | Non-standard DLR status | Receipts emit `stat:UNDELIVERABLE`; the spec's field is 7 characters (`UNDELIV`) |
 | Flash destroys UCS2 | `flash: true` overwrites `data_coding` with 0x10, discarding the UCS2 alphabet, which needs 0x18 |
@@ -103,6 +103,17 @@ implementation (see todo.md).
 | Unchecked reads | Wire reads index straight into the buffer, so a short or malformed PDU throws out of the codec. Reads are bounds-checked and return results now |
 | Unrangechecked writes | Integer params are handed to `writeUInt8`/`writeUInt16BE` unvalidated, so an out-of-range value throws from inside Node |

+## GSM 7-bit is sent unpacked
+
+Over SMPP the ESME puts one GSM character per octet in `short_message` and the SMSC packs it into
+septets. The 140-octet limit applies to that packed result, not to what goes on the wire here, which
+is why a concatenated segment is 153 characters plus a 6-octet UDH — 159 octets in `short_message`,
+and entirely correct. Do not "fix" this to 134; that number is the packed payload size and would
+truncate every long message by a fifth.
+
+UCS2 is not packed, so there the two coincide: 67 characters = 134 octets, plus the 6-octet UDH is
+exactly 140.
+
 ## Conventions

 - Hard tabs. Alphabetical ordering for keys, imports and lists unless order is logic-significant.
@@ -180,7 +180,9 @@ commands the codec knows, not just the four the session handles natively.
 0.4.0 had a number of protocol defects. Fixing them changes the bytes it puts on the wire, so if you
 have worked around any of these, remove the workaround:

- Multipart segments were 158 octets, over the 140-octet limit. They are now correctly sized.
+- Every multipart segment was one character short (152 GSM characters instead of 153, 66 UCS2
+  instead of 67), so long messages were split into more segments than necessary — and each extra
+  segment costs a message.
 - LATIN1 (`data_coding` 0x03) was silently decoded as ASCII, corrupting the message.
 - Delivery receipt dates were a month off, and the status field read `UNDELIVERABLE` where the spec
  defines the 7-character `UNDELIV`.
@@ -0,0 +1,184 @@
+import type { Result } from './result.ts';
+import type { EncodingName } from './defs/encodings.ts';
+import { consts } from './defs/constants.ts';
+import { detect, encodingByDataCoding, encodings } from './defs/encodings.ts';
+
+/** A single SMS carries 1120 bits, whatever the alphabet. */
+const singleMessageBits = 1120;
+
+/** Budget per concatenated segment: septets for GSM, octets for UCS2. */
+const segmentUnits = { ASCII: 153, UCS2: 67 * 2 } as const;
+
+export type SplitOptions = {
+	encoding?: EncodingName;
+	reference: number;
+};
+
+export function encodeMessage(
+	message: string,
+	encoding?: EncodingName,
+): { buffer: Buffer; encoding: EncodingName } {
+	const resolved = encoding ?? detect(message);
+
+	return { buffer: encodings[resolved].encode(message), encoding: resolved };
+}
+
+export function decodeMessage(
+	buffer: Buffer,
+	dataCoding: number,
+	esmClass = 0,
+): { message: string; udh: Buffer | undefined } {
+	const encoding = encodingByDataCoding(dataCoding);
+
+	if ((esmClass & consts.ESM_CLASS.UDH_INDICATOR) !== consts.ESM_CLASS.UDH_INDICATOR) {
+		return { message: encodings[encoding].decode(buffer), udh: undefined };
+	}
+
+	const udhLength = (buffer[0] ?? 0) + 1;
+
+	return {
+		message: encodings[encoding].decode(buffer.subarray(udhLength)),
+		udh: buffer.subarray(0, udhLength),
+	};
+}
+
+export function bitCount(message: string, encoding?: EncodingName): number {
+	const resolved = encoding ?? detect(message);
+	const encoded = encodings[resolved].encode(message);
+
+	// GSM characters are packed seven bits to a septet; everything else stays octet-aligned.
+	return resolved === 'ASCII' || resolved === 'FLASH' ? encoded.length * 7 : encoded.length * 8;
+}
+
+/**
+ * Splits a message into concatenation segments, each prefixed with a UDH. A message that fits in a
+ * single SMS is returned as one segment with no UDH.
+ *
+ * Splitting walks code points, so an escaped GSM character never straddles a segment boundary and a
+ * surrogate pair is never cut in half.
+ */
+export function splitMessage(message: string, options: SplitOptions): Buffer[] {
+	const encoding = options.encoding ?? detect(message);
+
+	if (bitCount(message, encoding) <= singleMessageBits) {
+		return [encodings[encoding].encode(message)];
+	}
+
+	const budget = encoding === 'UCS2' ? segmentUnits.UCS2 : segmentUnits.ASCII;
+	const parts: string[] = [];
+	let current = '';
+	let used = 0;
+
+	for (const char of message) {
+		const cost = encodings[encoding].encode(char).length;
+
+		if (used + cost > budget) {
+			parts.push(current);
+			current = '';
+			used = 0;
+		}
+
+		current += char;
+		used += cost;
+	}
+
+	if (current !== '') parts.push(current);
+
+	return parts.map((part, index) => Buffer.concat([
+		Buffer.from([0x05, 0x00, 0x03, options.reference & 0xFF, parts.length, index + 1]),
+		encodings[encoding].encode(part),
+	]));
+}
+
+function pad(value: number, length: number): string {
+	return value.toString().padStart(length, '0');
+}
+
+/**
+ * The YYMMDDhhmm stamp used inside delivery receipt text. UTC, so a receipt reads the same wherever
+ * the process runs.
+ */
+export function smppDate(date: Date): string {
+	return pad(date.getUTCFullYear() % 100, 2)
+		+ pad(date.getUTCMonth() + 1, 2)
+		+ pad(date.getUTCDate(), 2)
+		+ pad(date.getUTCHours(), 2)
+		+ pad(date.getUTCMinutes(), 2);
+}
+
+const absoluteTime = /^(\d\d)(\d\d)(\d\d)(\d\d)(\d\d)(\d\d)(\d)(\d\d)([+-])$/;
+const relativeTime = /^(\d\d)(\d\d)(\d\d)(\d\d)(\d\d)(\d\d)000R$/;
+
+/** The SMPP absolute and relative time format, as used by validity_period and friends. */
+export const smppTime = {
+	/**
+	 * A Date becomes an absolute UTC time; a number is a relative period in seconds. Relative
+	 * periods are expressed in days and below, so anything past 99 days is clamped to that.
+	 */
+	encode(value: Date | number | string): string {
+		if (typeof value === 'string') return value;
+
+		if (typeof value === 'number') {
+			const total = Math.max(0, Math.floor(value));
+			const days = Math.min(99, Math.floor(total / 86400));
+			const capped = days === 99 ? 99 * 86400 + 86399 : total;
+
+			return '0000'
+				+ pad(Math.floor(capped / 86400), 2)
+				+ pad(Math.floor(capped / 3600) % 24, 2)
+				+ pad(Math.floor(capped / 60) % 60, 2)
+				+ pad(capped % 60, 2)
+				+ '000R';
+		}
+
+		return pad(value.getUTCFullYear() % 100, 2)
+			+ pad(value.getUTCMonth() + 1, 2)
+			+ pad(value.getUTCDate(), 2)
+			+ pad(value.getUTCHours(), 2)
+			+ pad(value.getUTCMinutes(), 2)
+			+ pad(value.getUTCSeconds(), 2)
+			+ pad(Math.floor(value.getUTCMilliseconds() / 100), 1)
+			+ '00+';
+	},
+
+	decode(value: string): Result<{ date: Date }> {
+		const relative = relativeTime.exec(value);
+
+		if (relative) {
+			const [, years, months, days, hours, minutes, seconds] = relative;
+			const date = new Date();
+
+			date.setUTCFullYear(date.getUTCFullYear() + Number(years));
+			date.setUTCMonth(date.getUTCMonth() + Number(months));
+			date.setUTCDate(date.getUTCDate() + Number(days));
+			date.setUTCHours(date.getUTCHours() + Number(hours));
+			date.setUTCMinutes(date.getUTCMinutes() + Number(minutes));
+			date.setUTCSeconds(date.getUTCSeconds() + Number(seconds));
+
+			return { date };
+		}
+
+		const absolute = absoluteTime.exec(value);
+
+		if (!absolute) {
+			return { err: new Error(`Not an SMPP time: ${JSON.stringify(value)}`) };
+		}
+
+		const [, years, months, days, hours, minutes, seconds, tenths, quarters, sign] = absolute;
+		const century = Math.floor(new Date().getUTCFullYear() / 100) * 100;
+		const millis = Date.UTC(
+			century + Number(years),
+			Number(months) - 1,
+			Number(days),
+			Number(hours),
+			Number(minutes),
+			Number(seconds),
+			Number(tenths) * 100,
+		);
+
+		// The offset says how far the stamp's local time runs ahead of UTC, so undo it.
+		const offset = Number(quarters) * 15 * 60_000;
+
+		return { date: new Date(sign === '+' ? millis - offset : millis + offset) };
+	},
+};
@@ -0,0 +1,152 @@
+import assert from 'node:assert/strict';
+import test, { describe } from 'node:test';
+import {
+	bitCount,
+	decodeMessage,
+	encodeMessage,
+	smppDate,
+	smppTime,
+	splitMessage,
+} from '../src/message.ts';
+
+describe('bitCount()', () => {
+	test('counts GSM characters as seven bits each', () => {
+		assert.equal(bitCount('hello'), 35);
+		assert.equal(bitCount(''), 0);
+		assert.equal(bitCount('a'.repeat(160)), 1120);
+	});
+
+	test('counts UCS2 characters as sixteen bits each', () => {
+		assert.equal(bitCount('تست'), 48);
+		assert.equal(bitCount('a'.repeat(70), 'UCS2'), 1120);
+	});
+
+	test('counts an escaped GSM character as two', () => {
+		assert.equal(bitCount('€'), 14);
+	});
+});
+
+describe('splitMessage()', () => {
+	test('returns a single unwrapped segment when the message fits', () => {
+		const segments = splitMessage('Hello world', { reference: 1 });
+
+		assert.equal(segments.length, 1);
+		assert.deepEqual(segments[0], encodeMessage('Hello world').buffer);
+	});
+
+	test('fits exactly 160 GSM characters into one segment', () => {
+		assert.equal(splitMessage('a'.repeat(160), { reference: 1 }).length, 1);
+		assert.equal(splitMessage('a'.repeat(161), { reference: 1 }).length, 2);
+	});
+
+	test('fits exactly 70 UCS2 characters into one segment', () => {
+		assert.equal(splitMessage('ت'.repeat(70), { reference: 1 }).length, 1);
+		assert.equal(splitMessage('ت'.repeat(71), { reference: 1 }).length, 2);
+	});
+
+	// 0.4.0 pushed msgPart.slice(0, -1), so every segment was one character short and long
+	// messages were split into more segments — and therefore more billed messages — than needed.
+	test('carries 153 GSM characters per segment', () => {
+		const segments = splitMessage('a'.repeat(306), { reference: 7 });
+
+		assert.equal(segments.length, 2);
+
+		for (const segment of segments) {
+			assert.equal(segment.length, 6 + 153);
+		}
+	});
+
+	test('carries 67 UCS2 characters per segment', () => {
+		const segments = splitMessage('ت'.repeat(134), { reference: 7 });
+
+		assert.equal(segments.length, 2);
+
+		for (const segment of segments) {
+			assert.equal(segment.length, 6 + 134);
+		}
+	});
+
+	test('prefixes each segment with a concatenation UDH', () => {
+		const segments = splitMessage('a'.repeat(306), { reference: 0x2A });
+
+		assert.deepEqual(segments[0]?.subarray(0, 6), Buffer.from([0x05, 0x00, 0x03, 0x2A, 2, 1]));
+		assert.deepEqual(segments[1]?.subarray(0, 6), Buffer.from([0x05, 0x00, 0x03, 0x2A, 2, 2]));
+	});
+
+	test('splits on characters, never inside an escape sequence', () => {
+		const segments = splitMessage('€'.repeat(100), { reference: 1 });
+		const rejoined = segments
+			.map(segment => decodeMessage(segment.subarray(6), 0x00).message)
+			.join('');
+
+		assert.equal(rejoined, '€'.repeat(100));
+	});
+});
+
+describe('encodeMessage() and decodeMessage()', () => {
+	test('picks GSM for GSM-safe text and UCS2 otherwise', () => {
+		assert.equal(encodeMessage('Hello').encoding, 'ASCII');
+		assert.equal(encodeMessage('تست').encoding, 'UCS2');
+	});
+
+	test('honours a forced encoding', () => {
+		assert.equal(encodeMessage('Hello', 'UCS2').encoding, 'UCS2');
+	});
+
+	// 0.4.0 resolved data_coding 0x03 to the alias ISO_8859_1, which has no decoder, so every
+	// Latin-1 message was silently decoded as ASCII.
+	test('decodes Latin-1 rather than falling back to ASCII', () => {
+		assert.equal(decodeMessage(Buffer.from([0xE1, 0xE7, 0xDA]), 0x03).message, 'áçÚ');
+	});
+
+	test('round-trips a UCS2 message ending in a zero low byte', () => {
+		const { buffer } = encodeMessage('hej 一');
+
+		assert.equal(decodeMessage(buffer, 0x08).message, 'hej 一');
+	});
+
+	test('strips a UDH when the esm_class says one is present', () => {
+		const withUdh = Buffer.concat([
+			Buffer.from([0x05, 0x00, 0x03, 0x01, 0x02, 0x01]),
+			encodeMessage('part one').buffer,
+		]);
+
+		assert.equal(decodeMessage(withUdh, 0x00, 0x40).message, 'part one');
+	});
+});
+
+describe('smppDate()', () => {
+	// 0.4.0 used getMonth() without adding one, so January rendered as 00 and every delivery
+	// receipt carried a date a month in the past.
+	test('renders the month one-based', () => {
+		assert.equal(smppDate(new Date(Date.UTC(2026, 0, 9, 5, 4))), '2601090504');
+		assert.equal(smppDate(new Date(Date.UTC(2026, 11, 31, 23, 59))), '2612312359');
+	});
+});
+
+describe('smppTime', () => {
+	test('encodes an absolute time', () => {
+		assert.equal(smppTime.encode(new Date(Date.UTC(2026, 7, 25, 14, 30, 0))), '260825143000000+');
+	});
+
+	test('encodes a relative time given in seconds', () => {
+		assert.equal(smppTime.encode(3600), '000000010000000R');
+	});
+
+	test('passes an already-formatted string through', () => {
+		assert.equal(smppTime.encode('260825143000000+'), '260825143000000+');
+	});
+
+	test('decodes an absolute time back to the same instant', () => {
+		const when = new Date(Date.UTC(2026, 7, 25, 14, 30, 0));
+		const { err, date } = smppTime.decode(smppTime.encode(when));
+
+		assert.equal(err, undefined);
+		assert.equal(date.toISOString(), when.toISOString());
+	});
+
+	test('reports malformed input rather than returning an invalid date', () => {
+		assert.ok(smppTime.decode('nonsense').err instanceof Error);
+		assert.ok(smppTime.decode('').err instanceof Error);
+	});
+});
@@ -139,8 +139,9 @@ All done, with tests in `test/encodings.test.ts`, `test/types.test.ts` and `test
 - [ ] `bitCount(msg, encoding?)`, `encodeMessage`, `decodeMessage`, `smppDate`, `splitMessage`.
 - [ ] `smppTime.encode(value)` / `smppTime.decode(value)` — absolute and relative SMPP time formats,
      replacing the dormant `filters.time`. Used by `validityPeriod` and `scheduleDeliveryTime`.
- [ ] **Fix:** segments are 134 GSM characters or 67 UCS2 characters, so that segment + 6-byte UDH
-      is exactly 140 octets. 0.4.0 produces 152/66.
+- [ ] **Fix:** segments carry 153 GSM characters or 67 UCS2 characters. 0.4.0 produces 152/66,
+      because it pushes `msgPart.slice(0, -1)` after accumulating a full segment. Read the
+      "GSM 7-bit is sent unpacked" section of AGENTS.md before touching these numbers.
 - [ ] **Fix:** `smppDate` must add 1 to `getMonth()` and zero-pad correctly.
 - [ ] **Fix:** `decodeMessage` must resolve the alphabet through `encodingByDataCoding` (already
      written and tested) rather than scanning the alias table, which is what broke LATIN1.