feat: serialization of promises

.rfcs/001-serialize-async.md

@@ -0,0 +1,32 @@
+## Serializing promises and other async generators
+
+### Finished JSON output
+
+```js
+// eslint-disable-next-line @typescript-eslint/no-unused-vars, no-unused-vars
+const out = [
+	// <first line> is just a `[` that initializes the array pf the response
+	// <second line>
+	{
+		json: {
+			foo: "bar",
+			iterator: ["AsyncIterator", 1, "__tson"],
+			promise: ["Promise", 0, "__tson"],
+		},
+		nonce: "__tson",
+	},
+	// <second line>
+	// <second line of json>
+	[
+		// ------ this could be streamed ------
+		[1, ["chunk", "chunk from iterator"]],
+		[0, ["resolve", "result of promise"]],
+		[1, ["chunk", "another chunk from iterator"]],
+		[1, ["end"]],
+	],
+];
+```
+
+### Serializing
+
+> This is now implemented

CHANGELOG.md

@@ -1,11 +1,8 @@
-
-
 # [0.10.0](https://github.com/KATT/tupleson/compare/0.9.0...0.10.0) (2023-10-02)
 
-
 ### Features
 
-* use UUIDs for nonce ([#19](https://github.com/KATT/tupleson/issues/19)) ([e347640](https://github.com/KATT/tupleson/commit/e347640dd10bf6ecc6b93f99e3118f572da671b3))
+- use UUIDs for nonce ([#19](https://github.com/KATT/tupleson/issues/19)) ([e347640](https://github.com/KATT/tupleson/commit/e347640dd10bf6ecc6b93f99e3118f572da671b3))
 
 # [0.9.0](https://github.com/KATT/tupleson/compare/0.8.0...0.9.0) (2023-10-01)
 
@@ -72,4 +69,4 @@
 ### Features
 
 - initial version ([#1](https://github.com/KATT/tupleson/issues/1)) ([ccce25b](https://github.com/KATT/tupleson/commit/ccce25b6a039cf2e5c1a774c1ab022f0946ca8d5))
-- initialized repo ✨ ([c9e92a4](https://github.com/KATT/tupleson/commit/c9e92a42c97a8bc1ee3a9214f65626425c8598e3))
+- initialized repo ✨ ([c9e92a4](https://github.com/KATT/tupleson/commit/c9e92a42c97a8bc1ee3a9214f65626425c8598e3))

README.md

@@ -104,9 +104,9 @@ type Obj = typeof obj;
 // -> type Obj = { foo: string; set: Set<number>; }
 ```
 
-### Extend with a custom serializer
+## Extend with a custom serializer
 
-#### [Temporal](https://www.npmjs.com/package/@js-temporal/polyfill)
+### [Temporal](https://www.npmjs.com/package/@js-temporal/polyfill)
 
 > See test reference in [`./src/extend/temporal.test.ts`](./src/extend/temporal.test.ts)
 
@@ -134,7 +134,7 @@ const tson = createTson({
 });
 ```
 
-#### [Decimal.js](https://github.com/MikeMcl/decimal.js)
+### [Decimal.js](https://github.com/MikeMcl/decimal.js)
 
 > See test reference in [`./src/extend/decimal.test.ts`](./src/extend/decimal.test.ts)
 
@@ -154,6 +154,8 @@ const tson = createTson({
 });
 ```
 
+---
+
 <!-- ## All contributors ✨
 
 <a href="https://github.com/KATT/tupleson/graphs/contributors">

cspell.json

@@ -14,18 +14,19 @@
 		"commitlint",
 		"contributorsrc",
 		"conventionalcommits",
+		"Iterarable",
+		"KATT",
 		"knip",
 		"lcov",
 		"markdownlintignore",
 		"npmpackagejsonlintrc",
 		"outro",
 		"packagejson",
 		"quickstart",
-		"tsup",
-		"wontfix",
-		"tson",
 		"stringifier",
-		"KATT",
-		"tupleson"
+		"tson",
+		"tsup",
+		"tupleson",
+		"wontfix"
 	]
 }

src/async/asyncTypes.test.ts

@@ -0,0 +1,5 @@
+import { test } from "vitest";
+
+import "./asyncTypes.js";
+
+test.todo("check that it retains type");

src/async/asyncTypes.ts

@@ -0,0 +1,53 @@
+// eslint-disable-next-line eslint-comments/disable-enable-pair
+/* eslint-disable @typescript-eslint/no-explicit-any */
+import { TsonType } from "../types.js";
+import { TsonBranded, TsonTypeTesterCustom } from "../types.js";
+import { serialized } from "../types.js";
+
+export type TsonAsyncStringifierIterator<TValue> = AsyncIterable<string> & {
+	[serialized]: TValue;
+};
+
+export type TsonAsyncStringifier = <TValue>(
+	value: TValue,
+	space?: number,
+) => TsonAsyncStringifierIterator<TValue>;
+export type TsonAsyncIndex = TsonBranded<number, "AsyncRegistered">;
+export interface TsonTransformerSerializeDeserializeAsync<TValue> {
+	async: true;
+	/**
+	 * From JSON-serializable value
+	 */
+	deserialize: (
+		v: TsonAsyncIndex,
+		register: (index: TsonAsyncIndex) => Promise<TValue>,
+	) => TValue;
+
+	/**
+	 * The key to use when serialized
+	 */
+	key: string;
+	/**
+	 * JSON-serializable value
+	 */
+	serialize: (
+		v: TValue,
+		register: (thing: TValue) => TsonAsyncIndex,
+	) => TsonAsyncIndex;
+}
+
+export interface TsonAsyncType<TValue>
+	extends TsonTransformerSerializeDeserializeAsync<TValue>,
+		TsonTypeTesterCustom {}
+export interface TsonAsyncOptions {
+	/**
+	 * The nonce function every time we start serializing a new object
+	 * Should return a unique value every time it's called
+	 * @default `${crypto.randomUUID} if available, otherwise a random string generated by Math.random`
+	 */
+	nonce?: () => number | string;
+	/**
+	 * The list of types to use
+	 */
+	types: (TsonAsyncType<any> | TsonType<any, any> | TsonType<any, never>)[];
+}

src/async/createTsonAsync.ts

@@ -0,0 +1,8 @@
+import { TsonAsyncOptions } from "./asyncTypes.js";
+import { createTsonParseAsync } from "./deserializeAsync.js";
+import { createAsyncTsonStringify } from "./serializeAsync.js";
+
+export const createTsonAsync = (opts: TsonAsyncOptions) => ({
+	parse: createTsonParseAsync(opts),
+	stringify: createAsyncTsonStringify(opts),
+});

src/async/deserializeAsync.ts

@@ -0,0 +1,210 @@
+/* eslint-disable @typescript-eslint/no-explicit-any, eslint-comments/disable-enable-pair */
+/* eslint-disable @typescript-eslint/no-unused-vars */
+/* eslint-disable @typescript-eslint/no-unsafe-argument */
+import { TsonError } from "../errors.js";
+import { isTsonTuple } from "../internals/isTsonTuple.js";
+import { mapOrReturn } from "../internals/mapOrReturn.js";
+import {
+	TsonNonce,
+	TsonSerialized,
+	TsonTransformerSerializeDeserialize,
+} from "../types.js";
+import {
+	TsonAsyncIndex,
+	TsonAsyncOptions,
+	TsonAsyncStringifierIterator,
+	TsonAsyncType,
+} from "./asyncTypes.js";
+import { PROMISE_RESOLVED, TsonAsyncValueTuple } from "./serializeAsync.js";
+
+type WalkFn = (value: unknown) => unknown;
+type WalkerFactory = (nonce: TsonNonce) => WalkFn;
+
+type AnyTsonTransformerSerializeDeserialize =
+	| TsonAsyncType<any>
+	| TsonTransformerSerializeDeserialize<any, any>;
+
+type TsonParseAsync = <TValue>(
+	string: AsyncIterable<string> | TsonAsyncStringifierIterator<TValue>,
+) => Promise<TValue>;
+
+function createDeferred<T>() {
+	type PromiseResolve = (value: T) => void;
+	type PromiseReject = (reason: unknown) => void;
+	const deferred = {} as {
+		promise: Promise<T>;
+		reject: PromiseReject;
+		resolve: PromiseResolve;
+	};
+	deferred.promise = new Promise<T>((resolve, reject) => {
+		deferred.resolve = resolve;
+		deferred.reject = reject;
+	});
+	return deferred;
+}
+
+type Deferred<T> = ReturnType<typeof createDeferred<T>>;
+
+export function createTsonParseAsync(opts: TsonAsyncOptions): TsonParseAsync {
+	const typeByKey: Record<string, AnyTsonTransformerSerializeDeserialize> = {};
+
+	const deferreds = new Map<TsonAsyncIndex, Deferred<unknown>>();
+	for (const handler of opts.types) {
+		if (handler.key) {
+			if (typeByKey[handler.key]) {
+				throw new Error(`Multiple handlers for key ${handler.key} found`);
+			}
+
+			typeByKey[handler.key] =
+				handler as AnyTsonTransformerSerializeDeserialize;
+		}
+	}
+
+	return (async (iterator) => {
+		const instance = iterator[Symbol.asyncIterator]();
+
+		const walker: WalkerFactory = (nonce) => {
+			const walk: WalkFn = (value) => {
+				if (isTsonTuple(value, nonce)) {
+					const [type, serializedValue] = value;
+					// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+					const transformer = typeByKey[type]!;
+					return transformer.deserialize(
+						walk(serializedValue) as any,
+						(idx) => {
+							const deferred = createDeferred<unknown>();
+
+							deferreds.set(idx, deferred);
+
+							return deferred.promise;
+						},
+					);
+				}
+
+				return mapOrReturn(value, walk);
+			};
+
+			return walk;
+		};
+
+		async function getStreamedValues(
+			buffer: string[],
+			done: boolean,
+			walk: WalkFn,
+		) {
+			function readString(str: string) {
+				str = str.trimStart();
+				if (!str) {
+					return;
+				}
+
+				if (str.startsWith(",")) {
+					// ignore leading comma
+					str = str.slice(1);
+				}
+
+				if (!str.startsWith("[")) {
+					return;
+				}
+
+				// console.log("got something that looks like a value", str);
+
+				const [index, status, result] = JSON.parse(str) as TsonAsyncValueTuple;
+
+				// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+				const deferred = deferreds.get(index)!;
+
+				const walkedResult = walk(result);
+				status === PROMISE_RESOLVED
+					? deferred.resolve(walkedResult)
+					: deferred.reject(
+							walkedResult instanceof Error
+								? walkedResult
+								: new TsonError("Promise rejected on server", {
+										cause: walkedResult,
+								  }),
+					  );
+			}
+
+			buffer.forEach(readString);
+
+			if (done) {
+				return;
+			}
+
+			let nextValue = await instance.next();
+
+			while (!nextValue.done) {
+				nextValue.value.split("\n").forEach(readString);
+
+				nextValue = await instance.next();
+			}
+		}
+
+		async function init() {
+			const lines: string[] = [];
+
+			// get the head of the JSON
+
+			// console.log("getting head of JSON");
+			let lastResult: IteratorResult<string>;
+			do {
+				lastResult = await instance.next();
+
+				lines.push(...(lastResult.value as string).split("\n").filter(Boolean));
+
+				// console.log("got line", lines);
+			} while (lines.length < 4);
+
+			const [
+				/**
+				 * First line is just a `[`
+				 */
+				_firstLine,
+				/**
+				 * Second line is the shape of the JSON
+				 */
+				secondLine,
+				/**
+				 * Third line is a `,`
+				 */
+				_thirdLine,
+				/**
+				 * Fourth line is the start of the values array
+				 */
+				_fourthLine,
+				/**
+				 * Buffer is the rest of the iterator that came in the chunks while we were waiting for the first 4 lines
+				 */
+				...buffer
+			] = lines;
+
+			// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+			const secondValueParsed = JSON.parse(secondLine!) as TsonSerialized<any>;
+
+			const walk = walker(secondValueParsed.nonce);
+
+			void getStreamedValues(buffer, !!lastResult.done, walk).catch((cause) => {
+				// Something went wrong while getting the streamed values
+
+				const err = new TsonError(
+					"Stream interrupted: failed to get streamed values",
+					// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
+					{ cause },
+				);
+
+				// cancel all pending promises
+				for (const deferred of deferreds.values()) {
+					deferred.reject(err);
+				}
+			});
+
+			return walk(secondValueParsed.json);
+		}
+
+		const result = await init().catch((cause: unknown) => {
+			throw new TsonError("Failed to initialize TSON stream", { cause });
+		});
+		return result;
+	}) as TsonParseAsync;
+}