diff --git a/.certs/.gitignore b/.certs/.gitignore new file mode 100644 index 0000000..21924bc --- /dev/null +++ b/.certs/.gitignore @@ -0,0 +1,3 @@ +.terraform +relay_* +terraform.tfstate \ No newline at end of file diff --git a/.certs/.terraform.lock.hcl b/.certs/.terraform.lock.hcl new file mode 100644 index 0000000..65fa884 --- /dev/null +++ b/.certs/.terraform.lock.hcl @@ -0,0 +1,61 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/local" { + version = "2.5.2" + hashes = [ + "h1:JlMZD6nYqJ8sSrFfEAH0Vk/SL8WLZRmFaMUF9PJK5wM=", + "zh:136299545178ce281c56f36965bf91c35407c11897f7082b3b983d86cb79b511", + "zh:3b4486858aa9cb8163378722b642c57c529b6c64bfbfc9461d940a84cd66ebea", + "zh:4855ee628ead847741aa4f4fc9bed50cfdbf197f2912775dd9fe7bc43fa077c0", + "zh:4b8cd2583d1edcac4011caafe8afb7a95e8110a607a1d5fb87d921178074a69b", + "zh:52084ddaff8c8cd3f9e7bcb7ce4dc1eab00602912c96da43c29b4762dc376038", + "zh:71562d330d3f92d79b2952ffdda0dad167e952e46200c767dd30c6af8d7c0ed3", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:805f81ade06ff68fa8b908d31892eaed5c180ae031c77ad35f82cb7a74b97cf4", + "zh:8b6b3ebeaaa8e38dd04e56996abe80db9be6f4c1df75ac3cccc77642899bd464", + "zh:ad07750576b99248037b897de71113cc19b1a8d0bc235eb99173cc83d0de3b1b", + "zh:b9f1c3bfadb74068f5c205292badb0661e17ac05eb23bfe8bd809691e4583d0e", + "zh:cc4cbcd67414fefb111c1bf7ab0bc4beb8c0b553d01719ad17de9a047adff4d1", + ] +} + +provider "registry.terraform.io/hashicorp/tls" { + version = "4.0.6" + hashes = [ + "h1:dYSb3V94K5dDMtrBRLPzBpkMTPn+3cXZ/kIJdtFL+2M=", + "zh:10de0d8af02f2e578101688fd334da3849f56ea91b0d9bd5b1f7a243417fdda8", + "zh:37fc01f8b2bc9d5b055dc3e78bfd1beb7c42cfb776a4c81106e19c8911366297", + "zh:4578ca03d1dd0b7f572d96bd03f744be24c726bfd282173d54b100fd221608bb", + "zh:6c475491d1250050765a91a493ef330adc24689e8837a0f07da5a0e1269e11c1", + "zh:81bde94d53cdababa5b376bbc6947668be4c45ab655de7aa2e8e4736dfd52509", + "zh:abdce260840b7b050c4e401d4f75c7a199fafe58a8b213947a258f75ac18b3e8", + "zh:b754cebfc5184873840f16a642a7c9ef78c34dc246a8ae29e056c79939963c7a", + "zh:c928b66086078f9917aef0eec15982f2e337914c5c4dbc31dd4741403db7eb18", + "zh:cded27bee5f24de6f2ee0cfd1df46a7f88e84aaffc2ecbf3ff7094160f193d50", + "zh:d65eb3867e8f69aaf1b8bb53bd637c99c6b649ba3db16ded50fa9a01076d1a27", + "zh:ecb0c8b528c7a619fa71852bb3fb5c151d47576c5aab2bf3af4db52588722eeb", + "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", + ] +} + +provider "registry.terraform.io/vancluever/acme" { + version = "2.26.0" + constraints = "~> 2.0" + hashes = [ + "h1:4Lk5cb2Fg1q1JEQf1jkrShjPC3ayukp4eFcdL4e+y0w=", + "zh:11f554916ee99d8930de6d7bb5a014ec636b53ef9ba35eea84b0d2522c78230f", + "zh:231c31271c25477c95e0a4972857b6d5e9d7c3a300cbc4b0948566d87bc46e04", + "zh:2ae165ca7a994a4c77801a82ebd9f2f6de33a4c8882381bea575b3385cc251d8", + "zh:2cf01e4694d81b24972f5dab8e5f374aa59100082ff6e2435615d9c0f24cc00e", + "zh:3de6f6f9d052dfaa5d5f366d7ca26bdebb42fc74b6e19325e67420c37ff630d3", + "zh:3fd2b4b680b970394e4d0d49c2a8e5365297e79cea418ce87197cc8bb456d8c7", + "zh:46ea249cc01dce23ff6c8f02106e693be3b046059834b60b670c45a8f4093980", + "zh:57cb181c73b6e7397744d885c788d8815ad6a43f07769e98c6327bbc37272896", + "zh:761f2adf3e63559bd279763eb91247cdebf31401d79853755453274f143cbb36", + "zh:c4a9905bf81d38201c080cb91ea85002194c47ca26619644628184a56c394b7d", + "zh:d6e3a757c357239edefb640807778fb69805b9ae5df84a811a2d505c51089367", + "zh:d713856e4a459e1091cbb19ffb830d25cd88953d3e54acd46db0729c77a531d8", + "zh:f7cb8dec263d0ee223737dad3b6fa8071258f41cfa9e0b8cf7f337f9f501fc3b", + ] +} diff --git a/.certs/README.md b/.certs/README.md new file mode 100644 index 0000000..071f07d --- /dev/null +++ b/.certs/README.md @@ -0,0 +1,24 @@ +## Usage +1. Update the terraform.tfvars file with your domain and email. + +2. Run `terraform init` to initialize the Terraform working directory. + +3. Run `terraform plan` to see the planned changes. + +4. Run `terraform apply` to create the resources and obtain the certificate. +Outputs + +The configuration provides two sensitive outputs: +```bash +certificate_pem: The full certificate chain +private_key_pem: The private key for the certificate +``` + +These can be then be used in your `moq-relay` as it requires SSL/TLS certificates. + +## Note +The generated certificate and key files are saved locally and ignored by git: +```git +.terraform +relay_* +``` \ No newline at end of file diff --git a/.certs/input.tf b/.certs/input.tf new file mode 100644 index 0000000..4d54386 --- /dev/null +++ b/.certs/input.tf @@ -0,0 +1,7 @@ +variable "email" { + description = "Your email address, used for LetsEncrypt" +} + +variable "domain" { + description = "domain name" +} \ No newline at end of file diff --git a/.certs/main.tf b/.certs/main.tf new file mode 100644 index 0000000..a491561 --- /dev/null +++ b/.certs/main.tf @@ -0,0 +1,65 @@ +terraform { + required_providers { + acme = { + source = "vancluever/acme" + version = "~> 2.0" + } + } +} + +provider "acme" { + server_url = "https://acme-v02.api.letsencrypt.org/directory" +} + +resource "acme_registration" "reg" { + email_address = "wanjohiryan33@gmail.com" +} + +resource "tls_private_key" "relay" { + algorithm = "ECDSA" + ecdsa_curve = "P256" +} + +resource "acme_registration" "relay" { + account_key_pem = tls_private_key.relay.private_key_pem + email_address = var.email +} + +resource "acme_certificate" "relay" { + account_key_pem = acme_registration.relay.account_key_pem + common_name = "relay.${var.domain}" + subject_alternative_names = ["*.relay.${var.domain}"] + key_type = tls_private_key.relay.ecdsa_curve + + recursive_nameservers = ["8.8.8.8:53"] + + dns_challenge { + provider = "route53" + } +} + +# New resources to save certificate and private key +resource "local_file" "cert_file" { + content = "${acme_certificate.relay.certificate_pem}${acme_certificate.relay.issuer_pem}" + filename = "${path.module}/relay_cert.crt" + file_permission = "0644" + directory_permission = "0755" +} + +resource "local_file" "key_file" { + content = acme_certificate.relay.private_key_pem + filename = "${path.module}/relay_key.key" + file_permission = "0600" + directory_permission = "0755" +} + +# Outputs for certificate and private key +output "certificate_pem" { + value = "${acme_certificate.relay.certificate_pem}${acme_certificate.relay.issuer_pem}" + sensitive = true +} + +output "private_key_pem" { + value = acme_certificate.relay.private_key_pem + sensitive = true +} diff --git a/.certs/terraform.tfvars b/.certs/terraform.tfvars new file mode 100644 index 0000000..c63348f --- /dev/null +++ b/.certs/terraform.tfvars @@ -0,0 +1,2 @@ +domain = "fst.so" +email = "wanjohiryan33@gmail.com" \ No newline at end of file diff --git a/.gitignore b/.gitignore index 28dc834..97223e6 100644 --- a/.gitignore +++ b/.gitignore @@ -41,4 +41,7 @@ yarn-error.log* .sst #Bun merging errors, EVERY time :( -bun.lockb \ No newline at end of file +bun.lockb + +#tests +id_* \ No newline at end of file diff --git a/apps/www/package.json b/apps/www/package.json index b39d364..7e56b84 100644 --- a/apps/www/package.json +++ b/apps/www/package.json @@ -32,7 +32,9 @@ "@builder.io/qwik": "^1.8.0", "@builder.io/qwik-city": "^1.8.0", "@builder.io/qwik-react": "0.5.0", + "@modular-forms/qwik": "^0.27.0", "@nestri/eslint-config": "*", + "@nestri/moq": "*", "@nestri/typescript-config": "*", "@nestri/ui": "*", "@types/eslint": "8.56.10", @@ -48,6 +50,7 @@ "react-dom": "18.2.0", "typescript": "5.4.5", "undici": "*", + "valibot": "^0.42.1", "vite": "5.3.5", "vite-tsconfig-paths": "^4.2.1", "wrangler": "^3.0.0" diff --git a/apps/www/src/routes/(moq)/moq/checker/index.tsx b/apps/www/src/routes/(moq)/moq/checker/index.tsx new file mode 100644 index 0000000..aee03dc --- /dev/null +++ b/apps/www/src/routes/(moq)/moq/checker/index.tsx @@ -0,0 +1,118 @@ +import * as v from "valibot" +import { Broadcast } from "./tester"; +import { cn } from "@nestri/ui/design"; +import { routeLoader$ } from "@builder.io/qwik-city"; +import { component$, $, useSignal } from "@builder.io/qwik"; +import { MotionComponent, transition, TitleSection, Button } from "@nestri/ui/react"; +import { type InitialValues, type SubmitHandler, useForm, valiForm$ } from "@modular-forms/qwik" + +const Schema = v.object({ + url: v.pipe( + v.string(), + v.minLength(10, "Please input a valid url"), + v.url("Please input a valid url"), + ) +}, "Please fill in all the fields correctly.") + +type Form = v.InferInput; + +export const useFormLoader = routeLoader$>(async () => { + return { + url: "" + } +}) + +const generateRandomWord = (length: number) => { + const characters = 'abcdefghijklmnopqrstuvwxyz'; + return Array.from({ length }, () => characters[Math.floor(Math.random() * characters.length)]).join(''); +}; + +export default component$(() => { + const broadcasterOk = useSignal(); + const [state, { Form, Field }] = useForm
({ + loader: useFormLoader(), + validate: valiForm$(Schema) + }); + + const handleSubmit = $>(async (values) => { + const randomNamespace = generateRandomWord(6); + const sub = await Broadcast.init({ url: values.url, fingerprint: undefined, namespace: randomNamespace }) + + setTimeout(() => { + broadcasterOk.value = sub.isSubscribed() + }, 1000); + }); + + return ( + <> + + +
+ + + {(field, props) => { + return ( +
+
+ +
+ {field.error && (

{field.error}

)} +
+ ) + }} + + + {/* */} + + {/* + + + + + + */} + + Check + +
+ + + {typeof broadcasterOk.value !== "undefined" && broadcasterOk.value == true ? ( + + Your relay is doing okay + + ) : typeof broadcasterOk.value !== "undefined" && ( + + Your relay has an issue + + )} +
+ + + ) +}) \ No newline at end of file diff --git a/apps/www/src/routes/(moq)/moq/checker/tester.ts b/apps/www/src/routes/(moq)/moq/checker/tester.ts new file mode 100644 index 0000000..11446ea --- /dev/null +++ b/apps/www/src/routes/(moq)/moq/checker/tester.ts @@ -0,0 +1,208 @@ +import type { Connection, SubscribeRecv } from "@nestri/moq/transport" +import { asError } from "@nestri/moq/common/error" +import { Client } from "@nestri/moq/transport/client" +import * as Catalog from "@nestri/moq/media/catalog" +import { type GroupWriter } from "@nestri/moq/transport/objects" + +export interface BroadcastConfig { + namespace: string + connection: Connection +} +export interface BroadcasterConfig { + url: string + namespace: string + fingerprint?: string // URL to fetch TLS certificate fingerprint +} + +export interface BroadcastConfigTrack { + input: string + bitrate: number +} + +export class Broadcast { + stream: GroupWriter | null + subscriber: SubscribeRecv | null + subscribed: boolean; + + + readonly config: BroadcastConfig + readonly catalog: Catalog.Root + readonly connection: Connection + readonly namespace: string + + #running: Promise + + constructor(config: BroadcastConfig) { + this.subscribed = false + this.namespace = config.namespace + this.connection = config.connection + this.config = config + //Arbitrary values, just to keep TypeScript happy :) + this.catalog = { + version: 1, + streamingFormat: 1, + streamingFormatVersion: "0.2", + supportsDeltaUpdates: false, + commonTrackFields: { + packaging: "loc", + renderGroup: 1, + }, + tracks: [{ + name: "tester", + namespace: "tester", + selectionParams: {} + }], + } + this.stream = null + this.subscriber = null + + this.#running = this.#run() + } + + static async init(config: BroadcasterConfig): Promise { + const client = new Client({ url: config.url, fingerprint: config.fingerprint, role: "publisher" }) + const connection = await client.connect(); + + return new Broadcast({ connection, namespace: config.namespace }) + } + + async #run() { + try { + await this.connection.announce(this.namespace) + this.subscribed = true + } catch (error) { + + this.subscribed = false + } + + for (; ;) { + const subscriber = await this.connection.subscribed() + + if (!subscriber) { + this.subscribed = false + + break + } + + await subscriber.ack() + + this.subscriber = subscriber + + this.subscribed = true + + const bytes = Catalog.encode(this.catalog); + + const stream = await subscriber.group({ group: 0 }); + + await stream.write({ object: 0, payload: bytes }) + + this.stream = stream + } + } + + isSubscribed(): boolean { + return this.subscribed; + } + + // async #serveSubscribe(subscriber: SubscribeRecv) { + // try { + + // // Send a SUBSCRIBE_OK + // await subscriber.ack() + + // console.log("catalog track name:", subscriber.track) + + // const stream = await subscriber.group({ group: 0 }); + + // // const bytes = this.catalog.encode("Hello World") + + // await stream.write({ object: 0, payload: bytes }) + + + + // } catch (e) { + // const err = asError(e) + // await subscriber.close(1n, `failed to process publish: ${err.message}`) + // } finally { + // // TODO we can't close subscribers because there's no support for clean termination + // // await subscriber.close() + // } + // } + + // async mouseUpdatePosition({ x, y }: { x: number, y: number }, stream: GroupWriter) { + + // const mouse_move = { + // input_type: "mouse_move", + // delta_y: y, + // delta_x: x, + // } + + // const bytes = Catalog.encode(this.catalog) + + // await stream.write({ object: 0, payload: bytes }); + // } + + // async mouseUpdateButtons(e: MouseEvent, stream: GroupWriter) { + // const data: { input_type?: "mouse_key_down" | "mouse_key_up"; button: number; } = { button: e.button }; + + // if (e.type === "mousedown") { + // data["input_type"] = "mouse_key_down" + // } else if (e.type === "mouseup") { + // data["input_type"] = "mouse_key_up" + // } + + // const bytes = Catalog.encode(this.catalog) + + // await stream.write({ object: 0, payload: bytes }); + // } + + // async mouseUpdateWheel(e: WheelEvent, stream: GroupWriter) { + // const data: { input_type?: "mouse_wheel_up" | "mouse_wheel_down" } = {} + + // if (e.deltaY < 0.0) { + // data["input_type"] = "mouse_wheel_up" + // } else { + // data["input_type"] = "mouse_wheel_down" + // } + + // const bytes = Catalog.encode(this.catalog) + + // await stream.write({ object: 0, payload: bytes }); + // } + + // async updateKeyUp(e: KeyboardEvent, stream: GroupWriter) { + // const data = { + // input_type: "key_up", + // key_code: e.keyCode + // } + + // const bytes = Catalog.encode(this.catalog) + + // await stream.write({ object: 0, payload: bytes }); + // } + + // async updateKeyDown(e: KeyboardEvent, stream: GroupWriter) { + // const data = { + // input_type: "key_down", + // key_code: e.keyCode + // } + + // const bytes = Catalog.encode(this.catalog) + + // await stream.write({ object: 0, payload: bytes }); + // } + + close() { + // TODO implement publish close + } + + // Returns the error message when the connection is closed + async closed(): Promise { + try { + await this.#running + return new Error("closed") // clean termination + } catch (e) { + return asError(e) + } + } +} \ No newline at end of file diff --git a/apps/www/tsconfig.json b/apps/www/tsconfig.json index 25cda1a..ab18d60 100644 --- a/apps/www/tsconfig.json +++ b/apps/www/tsconfig.json @@ -2,7 +2,7 @@ "extends": "@nestri/typescript-config/base.json", "compilerOptions": { "allowJs": true, - "target": "ES2017", + "target": "ES2022", "module": "ES2022", "lib": [ "es2022", diff --git a/apps/www/vite.config.ts b/apps/www/vite.config.ts index 3dde541..67713ab 100644 --- a/apps/www/vite.config.ts +++ b/apps/www/vite.config.ts @@ -27,6 +27,17 @@ export default defineConfig((): UserConfig => { qwikVite(), tsconfigPaths(), qwikReact(), + //For Moq-js (SharedArrayBuffer) + { + name: "configure-response-headers", + configureServer: (server) => { + server.middlewares.use((_req, res, next) => { + res.setHeader("Cross-Origin-Embedder-Policy", "require-corp"); + res.setHeader("Cross-Origin-Opener-Policy", "same-origin"); + next(); + }); + }, + }, ], // This tells Vite which dependencies to pre-build in dev mode. optimizeDeps: { @@ -51,6 +62,8 @@ export default defineConfig((): UserConfig => { // } // : undefined, server: { + // https: true, + // proxy:{}, headers: { // Don't cache the server response in dev mode "Cache-Control": "public, max-age=0", diff --git a/bun.lockb b/bun.lockb index 50b5133..203fb55 100755 Binary files a/bun.lockb and b/bun.lockb differ diff --git a/infra/RELAY.md b/infra/RELAY.md new file mode 100644 index 0000000..cb6e2c8 --- /dev/null +++ b/infra/RELAY.md @@ -0,0 +1,87 @@ +# How to Deploy Your Own MoQ Relay on a Server + +This guide will walk you through the steps to deploy your own MoQ relay on a server. + +## Prerequisites + +1. **Server Requirements:** + - Ensure port 443 is open for both TCP and UDP (`:443/udp & :443/tcp`). + - The server should have a minimum of **4GB RAM** and **2 vCPUs**. + - Supports ARM or AMD64 architecture. + +2. **Software Requirements:** + - Docker and `docker-compose` must be installed on the server. You can use [this installation script](https://github.com/docker/docker-install) for Docker. + - Git must be installed to clone the necessary repository. + +3. **Certificates:** + - You will need private and public certificates. It is recommended to use certificates from a trusted CA rather than self-signed certificates. + +## Installation Steps + +### Step 1: Clone the Repository + +Clone the `kixelated/moq-rs` repository to your local machine: + +```bash +git clone https://github.com/kixelated/moq-rs moq +``` + +### Step 2: Verify Port Availability + +Check if port 443 is already in use on your server: + +```bash +sudo netstat -tulpn | grep ':443' | grep LISTEN +``` +or +```bash +sudo lsof -i -P -n | grep LISTEN | grep 443 +``` + +If you find any processes using port 443, consider terminating them. + +### Step 3: Configure Ports + +Navigate to the cloned directory and edit the Docker compose file to use port 443: + +```bash +cd moq +vim docker-compose.yml +``` + +Change the ports section from lines 34 to 35 to: + +```yaml +ports: + - "443:443" + - "443:443/udp" +``` + +### Step 4: Prepare Certificates + +Copy your generated certificates into the `moq/dev` directory and rename them: + +```bash +cp cert.pem moq/dev/localhost.crt +cp key.pem moq/dev/localhost.key +``` + +### Step 5: Start Docker Instances + +Ensure you are in the root directory of the `moq` project, then start the Docker containers: + +```bash +docker compose up -d +``` + +### Step 6: Link Domain to Server IP + +Configure your DNS settings to connect your server's IP address to your domain: + +``` +Record Type: A +Subdomain: relay.fst.so +IP Address: xx.xxx.xx.xxx +``` + +Congratulations, your MoQ server is now set up! You can verify its functionality by using the [MoQ Checker](https://nestri.pages.dev/moq/checker). \ No newline at end of file diff --git a/infra/domain.ts b/infra/domain.ts index 2fd8e80..c11f87b 100644 --- a/infra/domain.ts +++ b/infra/domain.ts @@ -1,9 +1,9 @@ -export const domain = - { - production: "fst.so", - dev: "dev.fst.so", - }[$app.stage] || $app.stage + ".dev.fst.so"; +// export const domain = +// { +// production: "fst.so", +// dev: "dev.fst.so", +// }[$app.stage] || $app.stage + ".dev.fst.so"; -export const zone = cloudflare.getZoneOutput({ - name: "fst.so", -}); \ No newline at end of file +// export const zone = cloudflare.getZoneOutput({ +// name: "fst.so", +// }); \ No newline at end of file diff --git a/infra/github.ts b/infra/github.ts deleted file mode 100644 index cf1e0d5..0000000 --- a/infra/github.ts +++ /dev/null @@ -1,38 +0,0 @@ -import { isPermanentStage } from "./stage"; - -if (isPermanentStage) { - const github = new aws.iam.OpenIdConnectProvider("GithubProvider", { - url: "https://token.actions.githubusercontent.com", - clientIdLists: ["sts.amazonaws.com"], - thumbprintLists: [ - "6938fd4d98bab03faadb97b34396831e3780aea1", - "1c58a3a8518e8759bf075b76b750d4f2df264fcd", - ], - }); - - const githubRole = new aws.iam.Role("GithubRole", { - name: [$app.name, $app.stage, "github"].join("-"), - assumeRolePolicy: { - Version: "2012-10-17", - Statement: [ - { - Effect: "Allow", - Principal: { - Federated: github.arn, - }, - Action: "sts:AssumeRoleWithWebIdentity", - Condition: { - StringLike: github.url.apply((url) => ({ - [`${url}:sub`]: "repo:nestriness/nestri:*", - })), - }, - }, - ], - }, - }); - - new aws.iam.RolePolicyAttachment("GithubRolePolicy", { - policyArn: "arn:aws:iam::aws:policy/AdministratorAccess", - role: githubRole.name, - }); -} \ No newline at end of file diff --git a/infra/relay.ts b/infra/relay.ts new file mode 100644 index 0000000..5da8dc5 --- /dev/null +++ b/infra/relay.ts @@ -0,0 +1,22 @@ +import { resolve as pathResolve } from "node:path"; +import { readFileSync as readFile } from "node:fs"; +//Copy your (known) ssh public key to the remote machine +//ssh-copy-id "-p $port" user@host + +const domain = "fst.so" +const ips = ["95.216.29.238"] + +// Get the hosted zone +const zone = aws.route53.getZone({ name: domain }); + +// Create an A record +const record = new aws.route53.Record("Relay DNS Records", { + zoneId: zone.then(zone => zone.zoneId), + type: "A", + name: `relay.${domain}`, + ttl: 300, + records: ips, +}); + +// Export the URL +export const url = $interpolate`https://${record.name}`; \ No newline at end of file diff --git a/package.json b/package.json index eb26f39..2581272 100644 --- a/package.json +++ b/package.json @@ -9,6 +9,7 @@ }, "devDependencies": { "@cloudflare/workers-types": "4.20240821.1", + "@pulumi/pulumi": "^3.134.0", "@types/aws-lambda": "8.10.145", "prettier": "^3.2.5", "turbo": "^2.0.12", diff --git a/packages/eslint-config/qwik.js b/packages/eslint-config/qwik.js index 8aac57a..3ee1485 100644 --- a/packages/eslint-config/qwik.js +++ b/packages/eslint-config/qwik.js @@ -1,42 +1,49 @@ module.exports = { - env: { - browser: true, - es2021: true, - node: true, + env: { + browser: true, + es2021: true, + node: true, + }, + extends: [ + "eslint:recommended", + "plugin:@typescript-eslint/recommended", + "plugin:qwik/recommended", + ], + parser: "@typescript-eslint/parser", + parserOptions: { + tsconfigRootDir: __dirname, + project: ["./tsconfig.json"], + ecmaVersion: 2021, + sourceType: "module", + ecmaFeatures: { + jsx: true, }, - extends: [ - "eslint:recommended", - "plugin:@typescript-eslint/recommended", - "plugin:qwik/recommended", - ], - parser: "@typescript-eslint/parser", - parserOptions: { - tsconfigRootDir: __dirname, - project: ["./tsconfig.json"], - ecmaVersion: 2021, - sourceType: "module", - ecmaFeatures: { - jsx: true, + }, + plugins: ["@typescript-eslint"], + rules: { + "@typescript-eslint/no-explicit-any": "off", + "@typescript-eslint/explicit-module-boundary-types": "off", + "@typescript-eslint/no-inferrable-types": "off", + "@typescript-eslint/no-non-null-assertion": "off", + "@typescript-eslint/no-empty-interface": "off", + "@typescript-eslint/no-namespace": "off", + "@typescript-eslint/no-empty-function": "off", + "@typescript-eslint/no-this-alias": "off", + "@typescript-eslint/ban-types": "off", + "@typescript-eslint/ban-ts-comment": "off", + // Warn when an unused variable doesn't start with an underscore + "@typescript-eslint/no-unused-vars": [ + "warn", + { + argsIgnorePattern: "^_", + varsIgnorePattern: "^_", + caughtErrorsIgnorePattern: "^_", }, - }, - plugins: ["@typescript-eslint"], - rules: { - "@typescript-eslint/no-explicit-any": "off", - "@typescript-eslint/explicit-module-boundary-types": "off", - "@typescript-eslint/no-inferrable-types": "off", - "@typescript-eslint/no-non-null-assertion": "off", - "@typescript-eslint/no-empty-interface": "off", - "@typescript-eslint/no-namespace": "off", - "@typescript-eslint/no-empty-function": "off", - "@typescript-eslint/no-this-alias": "off", - "@typescript-eslint/ban-types": "off", - "@typescript-eslint/ban-ts-comment": "off", - "prefer-spread": "off", - "no-case-declarations": "off", - "no-console": "off", - "@typescript-eslint/no-unused-vars": ["error"], - "@typescript-eslint/consistent-type-imports": "warn", - "@typescript-eslint/no-unnecessary-condition": "warn", - }, - }; - \ No newline at end of file + ], + "prefer-spread": "off", + "no-case-declarations": "off", + "no-console": "off", + "@typescript-eslint/consistent-type-imports": "warn", + "@typescript-eslint/no-unnecessary-condition": "warn", + }, +}; diff --git a/packages/moq/.eslintrc.cjs b/packages/moq/.eslintrc.cjs new file mode 100644 index 0000000..89e59ac --- /dev/null +++ b/packages/moq/.eslintrc.cjs @@ -0,0 +1,56 @@ +/* eslint-env node */ +module.exports = { + extends: [ + "eslint:recommended", + "plugin:@typescript-eslint/recommended", + "plugin:@typescript-eslint/recommended-requiring-type-checking", + "plugin:@typescript-eslint/strict", + "prettier", + ], + parser: "@typescript-eslint/parser", + plugins: ["@typescript-eslint", "prettier"], + root: true, + env: { + browser: true, + es2022: true, + worker: true, + }, + ignorePatterns: ["dist", "node_modules", ".eslintrc.cjs"], + rules: { + // Allow the ! operator because typescript can't always figure out when something is not undefined + "@typescript-eslint/no-non-null-assertion": "off", + + // Allow `any` because Javascript was not designed to be type safe. + "@typescript-eslint/no-explicit-any": "off", + + // Requring a comment in empty function is silly + "@typescript-eslint/no-empty-function": "off", + + // Warn when an unused variable doesn't start with an underscore + "@typescript-eslint/no-unused-vars": [ + "warn", + { + argsIgnorePattern: "^_", + varsIgnorePattern: "^_", + caughtErrorsIgnorePattern: "^_", + }, + ], + + // The no-unsafe-* rules are a pain an introduce a lot of false-positives. + // Typescript will make sure things are properly typed. + "@typescript-eslint/no-unsafe-call": "off", + "@typescript-eslint/no-unsafe-argument": "off", + "@typescript-eslint/no-unsafe-call": "off", + "@typescript-eslint/no-unsafe-member-access": "off", + "@typescript-eslint/no-unsafe-assignment": "off", + "@typescript-eslint/no-unsafe-return": "off", + + // Make formatting errors into warnings + "prettier/prettier": 1, + }, + + parserOptions: { + project: true, + tsconfigRootDir: __dirname, + }, +} diff --git a/packages/moq/.prettierrc.yaml b/packages/moq/.prettierrc.yaml new file mode 100644 index 0000000..708c01a --- /dev/null +++ b/packages/moq/.prettierrc.yaml @@ -0,0 +1,4 @@ +# note: root .editorconfig is used + +# Don't insert semi-colons unless needed +semi: false diff --git a/packages/moq/README.md b/packages/moq/README.md new file mode 100644 index 0000000..4aa2e40 --- /dev/null +++ b/packages/moq/README.md @@ -0,0 +1,20 @@ +# Media over QUIC + +Media over QUIC (MoQ) is a live media delivery protocol utilizing QUIC streams. +See the [Warp draft](https://datatracker.ietf.org/doc/draft-lcurley-warp/). + +This is a Typescript library that supports both contribution (ingest) and distribution (playback). +It requires a server, such as [moq-rs](https://github.com/kixelated/moq-rs). + +## Usage + +``` +npm install @kixelated/moq +``` + +## License + +Licensed under either: + +- Apache License, Version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or http://www.apache.org/licenses/LICENSE-2.0) +- MIT license ([LICENSE-MIT](LICENSE-MIT) or http://opensource.org/licenses/MIT) diff --git a/packages/moq/common/async.ts b/packages/moq/common/async.ts new file mode 100644 index 0000000..a0fa0b4 --- /dev/null +++ b/packages/moq/common/async.ts @@ -0,0 +1,120 @@ +export class Deferred { + promise: Promise + resolve!: (value: T | PromiseLike) => void + reject!: (reason: any) => void + pending = true + + constructor() { + this.promise = new Promise((resolve, reject) => { + this.resolve = (value) => { + this.pending = false + resolve(value) + } + this.reject = (reason) => { + this.pending = false + reject(reason) + } + }) + } +} + +export type WatchNext = [T, Promise> | undefined] + +export class Watch { + #current: WatchNext + #next = new Deferred>() + + constructor(init: T) { + this.#next = new Deferred>() + this.#current = [init, this.#next.promise] + } + + value(): WatchNext { + return this.#current + } + + update(v: T | ((v: T) => T)) { + if (!this.#next.pending) { + throw new Error("already closed") + } + + // If we're given a function, call it with the current value + if (v instanceof Function) { + v = v(this.#current[0]) + } + + const next = new Deferred>() + this.#current = [v, next.promise] + this.#next.resolve(this.#current) + this.#next = next + } + + close() { + this.#current[1] = undefined + this.#next.resolve(this.#current) + } +} + +// Wakes up a multiple consumers. +export class Notify { + #next = new Deferred() + + async wait() { + return this.#next.promise + } + + wake() { + if (!this.#next.pending) { + throw new Error("closed") + } + + this.#next.resolve() + this.#next = new Deferred() + } + + close() { + this.#next.resolve() + } +} + +// Allows queuing N values, like a Channel. +export class Queue { + #stream: TransformStream + #closed = false + + constructor(capacity = 1) { + const queue = new CountQueuingStrategy({ highWaterMark: capacity }) + this.#stream = new TransformStream({}, undefined, queue) + } + + async push(v: T) { + const w = this.#stream.writable.getWriter() + await w.write(v) + w.releaseLock() + } + + async next(): Promise { + const r = this.#stream.readable.getReader() + const { value, done } = await r.read() + r.releaseLock() + + if (done) return + return value + } + + async abort(err: Error) { + if (this.#closed) return + await this.#stream.writable.abort(err) + this.#closed = true + } + + async close() { + if (this.#closed) return + await this.#stream.writable.close() + this.#closed = true + } + + closed() { + return this.#closed + } +} diff --git a/packages/moq/common/download.ts b/packages/moq/common/download.ts new file mode 100644 index 0000000..4a2a69b --- /dev/null +++ b/packages/moq/common/download.ts @@ -0,0 +1,18 @@ +// Utility function to download a Uint8Array for debugging. +export function download(data: Uint8Array, name: string) { + const blob = new Blob([data], { + type: "application/octet-stream", + }) + + const url = window.URL.createObjectURL(blob) + + const a = document.createElement("a") + a.href = url + a.download = name + document.body.appendChild(a) + a.style.display = "none" + a.click() + a.remove() + + setTimeout(() => window.URL.revokeObjectURL(url), 1000) +} diff --git a/packages/moq/common/error.ts b/packages/moq/common/error.ts new file mode 100644 index 0000000..c627a0b --- /dev/null +++ b/packages/moq/common/error.ts @@ -0,0 +1,14 @@ +// I hate javascript +export function asError(e: any): Error { + if (e instanceof Error) { + return e + } else if (typeof e === "string") { + return new Error(e) + } else { + return new Error(String(e)) + } +} + +export function isError(e: any): e is Error { + return e instanceof Error +} diff --git a/packages/moq/common/index.ts b/packages/moq/common/index.ts new file mode 100644 index 0000000..6d519d3 --- /dev/null +++ b/packages/moq/common/index.ts @@ -0,0 +1 @@ +export { asError } from "./error" diff --git a/packages/moq/common/ring.ts b/packages/moq/common/ring.ts new file mode 100644 index 0000000..97a73e6 --- /dev/null +++ b/packages/moq/common/ring.ts @@ -0,0 +1,176 @@ +// Ring buffer with audio samples. + +enum STATE { + READ_POS = 0, // The current read position + WRITE_POS, // The current write position + LENGTH, // Clever way of saving the total number of enums values. +} + +interface FrameCopyToOptions { + frameCount?: number + frameOffset?: number + planeIndex: number +} + +// This is implemented by AudioData in WebCodecs, but we don't import it because it's a DOM type. +interface Frame { + numberOfFrames: number + numberOfChannels: number + copyTo(dst: Float32Array, options: FrameCopyToOptions): void +} + +// No prototype to make this easier to send via postMessage +export class RingShared { + state: SharedArrayBuffer + + channels: SharedArrayBuffer[] + capacity: number + + constructor(channels: number, capacity: number) { + // Store the current state in a separate ring buffer. + this.state = new SharedArrayBuffer(STATE.LENGTH * Int32Array.BYTES_PER_ELEMENT) + + // Create a buffer for each audio channel + this.channels = [] + for (let i = 0; i < channels; i += 1) { + const buffer = new SharedArrayBuffer(capacity * Float32Array.BYTES_PER_ELEMENT) + this.channels.push(buffer) + } + + this.capacity = capacity + } +} + +export class Ring { + state: Int32Array + channels: Float32Array[] + capacity: number + + constructor(shared: RingShared) { + this.state = new Int32Array(shared.state) + + this.channels = [] + for (const channel of shared.channels) { + this.channels.push(new Float32Array(channel)) + } + + this.capacity = shared.capacity + } + + // Write samples for single audio frame, returning the total number written. + write(frame: Frame): number { + const readPos = Atomics.load(this.state, STATE.READ_POS) + const writePos = Atomics.load(this.state, STATE.WRITE_POS) + + const startPos = writePos + let endPos = writePos + frame.numberOfFrames + + if (endPos > readPos + this.capacity) { + endPos = readPos + this.capacity + if (endPos <= startPos) { + // No space to write + return 0 + } + } + + const startIndex = startPos % this.capacity + const endIndex = endPos % this.capacity + + // Loop over each channel + for (let i = 0; i < this.channels.length; i += 1) { + const channel = this.channels[i] + + // If the AudioData doesn't have enough channels, duplicate it. + const planeIndex = Math.min(i, frame.numberOfChannels - 1) + + if (startIndex < endIndex) { + // One continuous range to copy. + const full = channel.subarray(startIndex, endIndex) + + frame.copyTo(full, { + planeIndex, + frameCount: endIndex - startIndex, + }) + } else { + const first = channel.subarray(startIndex) + const second = channel.subarray(0, endIndex) + + frame.copyTo(first, { + planeIndex, + frameCount: first.length, + }) + + // We need this conditional when startIndex == 0 and endIndex == 0 + // When capacity=4410 and frameCount=1024, this was happening 52s into the audio. + if (second.length) { + frame.copyTo(second, { + planeIndex, + frameOffset: first.length, + frameCount: second.length, + }) + } + } + } + + Atomics.store(this.state, STATE.WRITE_POS, endPos) + + return endPos - startPos + } + + read(dst: Float32Array[]): number { + const readPos = Atomics.load(this.state, STATE.READ_POS) + const writePos = Atomics.load(this.state, STATE.WRITE_POS) + + const startPos = readPos + let endPos = startPos + dst[0].length + + if (endPos > writePos) { + endPos = writePos + if (endPos <= startPos) { + // Nothing to read + return 0 + } + } + + const startIndex = startPos % this.capacity + const endIndex = endPos % this.capacity + + // Loop over each channel + for (let i = 0; i < dst.length; i += 1) { + if (i >= this.channels.length) { + // ignore excess channels + } + + const input = this.channels[i] + const output = dst[i] + + if (startIndex < endIndex) { + const full = input.subarray(startIndex, endIndex) + output.set(full) + } else { + const first = input.subarray(startIndex) + const second = input.subarray(0, endIndex) + + output.set(first) + output.set(second, first.length) + } + } + + Atomics.store(this.state, STATE.READ_POS, endPos) + + return endPos - startPos + } + + clear() { + const pos = Atomics.load(this.state, STATE.WRITE_POS) + Atomics.store(this.state, STATE.READ_POS, pos) + } + + size() { + // TODO is this thread safe? + const readPos = Atomics.load(this.state, STATE.READ_POS) + const writePos = Atomics.load(this.state, STATE.WRITE_POS) + + return writePos - readPos + } +} diff --git a/packages/moq/common/settings.ts b/packages/moq/common/settings.ts new file mode 100644 index 0000000..807fc39 --- /dev/null +++ b/packages/moq/common/settings.ts @@ -0,0 +1,33 @@ +// MediaTrackSettings can represent both audio and video, which means a LOT of possibly undefined properties. +// This is a fork of the MediaTrackSettings interface with properties required for audio or vidfeo. +export interface AudioTrackSettings { + deviceId: string + groupId: string + + autoGainControl: boolean + channelCount: number + echoCancellation: boolean + noiseSuppression: boolean + sampleRate: number + sampleSize: number +} + +export interface VideoTrackSettings { + deviceId: string + groupId: string + + aspectRatio: number + facingMode: "user" | "environment" | "left" | "right" + frameRate: number + height: number + resizeMode: "none" | "crop-and-scale" + width: number +} + +export function isAudioTrackSettings(settings: MediaTrackSettings): settings is AudioTrackSettings { + return "sampleRate" in settings +} + +export function isVideoTrackSettings(settings: MediaTrackSettings): settings is VideoTrackSettings { + return "width" in settings +} diff --git a/packages/moq/common/tsconfig.json b/packages/moq/common/tsconfig.json new file mode 100644 index 0000000..784e219 --- /dev/null +++ b/packages/moq/common/tsconfig.json @@ -0,0 +1,4 @@ +{ + "extends": "../tsconfig.json", + "include": ["."] +} diff --git a/packages/moq/contribute/audio.ts b/packages/moq/contribute/audio.ts new file mode 100644 index 0000000..d42fe28 --- /dev/null +++ b/packages/moq/contribute/audio.ts @@ -0,0 +1,75 @@ +const SUPPORTED = [ + // TODO support AAC + // "mp4a" + "Opus", +] + +export class Encoder { + #encoder!: AudioEncoder + #encoderConfig: AudioEncoderConfig + #decoderConfig?: AudioDecoderConfig + + frames: TransformStream + + constructor(config: AudioEncoderConfig) { + this.#encoderConfig = config + + this.frames = new TransformStream({ + start: this.#start.bind(this), + transform: this.#transform.bind(this), + flush: this.#flush.bind(this), + }) + } + + #start(controller: TransformStreamDefaultController) { + this.#encoder = new AudioEncoder({ + output: (frame, metadata) => { + this.#enqueue(controller, frame, metadata) + }, + error: (err) => { + throw err + }, + }) + + this.#encoder.configure(this.#encoderConfig) + } + + #transform(frame: AudioData) { + this.#encoder.encode(frame) + frame.close() + } + + #enqueue( + controller: TransformStreamDefaultController, + frame: EncodedAudioChunk, + metadata?: EncodedAudioChunkMetadata, + ) { + const config = metadata?.decoderConfig + if (config && !this.#decoderConfig) { + const config = metadata.decoderConfig + if (!config) throw new Error("missing decoder config") + + controller.enqueue(config) + this.#decoderConfig = config + } + + controller.enqueue(frame) + } + + #flush() { + this.#encoder.close() + } + + static async isSupported(config: AudioEncoderConfig) { + // Check if we support a specific codec family + const short = config.codec.substring(0, 4) + if (!SUPPORTED.includes(short)) return false + + const res = await AudioEncoder.isConfigSupported(config) + return !!res.supported + } + + get config() { + return this.#encoderConfig + } +} diff --git a/packages/moq/contribute/broadcast.ts b/packages/moq/contribute/broadcast.ts new file mode 100644 index 0000000..2f8224f --- /dev/null +++ b/packages/moq/contribute/broadcast.ts @@ -0,0 +1,241 @@ +import { Connection, SubscribeRecv } from "../transport" +import { asError } from "../common/error" +import { Segment } from "./segment" +import { Track } from "./track" +import * as Catalog from "../media/catalog" + +import { isAudioTrackSettings, isVideoTrackSettings } from "../common/settings" + +export interface BroadcastConfig { + namespace: string + connection: Connection + media: MediaStream + + audio?: AudioEncoderConfig + video?: VideoEncoderConfig +} + +export interface BroadcastConfigTrack { + codec: string + bitrate: number +} + +export class Broadcast { + #tracks = new Map() + + readonly config: BroadcastConfig + readonly catalog: Catalog.Root + readonly connection: Connection + readonly namespace: string + + #running: Promise + + constructor(config: BroadcastConfig) { + this.connection = config.connection + this.config = config + this.namespace = config.namespace + + const tracks: Catalog.Track[] = [] + + for (const media of this.config.media.getTracks()) { + const track = new Track(media, config) + this.#tracks.set(track.name, track) + + const settings = media.getSettings() + + if (isVideoTrackSettings(settings)) { + if (!config.video) { + throw new Error("no video configuration provided") + } + + const video: Catalog.VideoTrack = { + namespace: this.namespace, + name: `${track.name}.m4s`, + initTrack: `${track.name}.mp4`, + selectionParams: { + mimeType: "video/mp4", + codec: config.video.codec, + width: settings.width, + height: settings.height, + framerate: settings.frameRate, + bitrate: config.video.bitrate, + }, + } + + tracks.push(video) + } else if (isAudioTrackSettings(settings)) { + if (!config.audio) { + throw new Error("no audio configuration provided") + } + + const audio: Catalog.AudioTrack = { + namespace: this.namespace, + name: `${track.name}.m4s`, + initTrack: `${track.name}.mp4`, + selectionParams: { + mimeType: "audio/ogg", + codec: config.audio.codec, + samplerate: settings.sampleRate, + //sampleSize: settings.sampleSize, + channelConfig: `${settings.channelCount}`, + bitrate: config.audio.bitrate, + }, + } + + tracks.push(audio) + } else { + throw new Error(`unknown track type: ${media.kind}`) + } + } + + this.catalog = { + version: 1, + streamingFormat: 1, + streamingFormatVersion: "0.2", + supportsDeltaUpdates: false, + commonTrackFields: { + packaging: "cmaf", + renderGroup: 1, + }, + tracks, + } + + this.#running = this.#run() + } + + async #run() { + await this.connection.announce(this.namespace) + + for (;;) { + const subscriber = await this.connection.subscribed() + if (!subscriber) break + + // Run an async task to serve each subscription. + this.#serveSubscribe(subscriber).catch((e) => { + const err = asError(e) + console.warn("failed to serve subscribe", err) + }) + } + } + + async #serveSubscribe(subscriber: SubscribeRecv) { + try { + const [base, ext] = splitExt(subscriber.track) + if (ext === "catalog") { + await this.#serveCatalog(subscriber, base) + } else if (ext === "mp4") { + await this.#serveInit(subscriber, base) + } else if (ext === "m4s") { + await this.#serveTrack(subscriber, base) + } else { + throw new Error(`unknown subscription: ${subscriber.track}`) + } + } catch (e) { + const err = asError(e) + await subscriber.close(1n, `failed to process subscribe: ${err.message}`) + } finally { + // TODO we can't close subscribers because there's no support for clean termination + // await subscriber.close() + } + } + + async #serveCatalog(subscriber: SubscribeRecv, name: string) { + // We only support ".catalog" + if (name !== "") throw new Error(`unknown catalog: ${name}`) + + const bytes = Catalog.encode(this.catalog) + + // Send a SUBSCRIBE_OK + await subscriber.ack() + + const stream = await subscriber.group({ group: 0 }) + await stream.write({ object: 0, payload: bytes }) + await stream.close() + } + + async #serveInit(subscriber: SubscribeRecv, name: string) { + const track = this.#tracks.get(name) + if (!track) throw new Error(`no track with name ${subscriber.track}`) + + // Send a SUBSCRIBE_OK + await subscriber.ack() + + const init = await track.init() + + const stream = await subscriber.group({ group: 0 }) + await stream.write({ object: 0, payload: init }) + await stream.close() + } + + async #serveTrack(subscriber: SubscribeRecv, name: string) { + const track = this.#tracks.get(name) + if (!track) throw new Error(`no track with name ${subscriber.track}`) + + // Send a SUBSCRIBE_OK + await subscriber.ack() + + const segments = track.segments().getReader() + + for (;;) { + const { value: segment, done } = await segments.read() + if (done) break + + // Serve the segment and log any errors that occur. + this.#serveSegment(subscriber, segment).catch((e) => { + const err = asError(e) + console.warn("failed to serve segment", err) + }) + } + } + + async #serveSegment(subscriber: SubscribeRecv, segment: Segment) { + // Create a new stream for each segment. + const stream = await subscriber.group({ + group: segment.id, + priority: 0, // TODO + }) + + let object = 0 + + // Pipe the segment to the stream. + const chunks = segment.chunks().getReader() + for (;;) { + const { value, done } = await chunks.read() + if (done) break + + await stream.write({ + object, + payload: value, + }) + + object += 1 + } + + await stream.close() + } + + // Attach the captured video stream to the given video element. + attach(video: HTMLVideoElement) { + video.srcObject = this.config.media + } + + close() { + // TODO implement publish close + } + + // Returns the error message when the connection is closed + async closed(): Promise { + try { + await this.#running + return new Error("closed") // clean termination + } catch (e) { + return asError(e) + } + } +} + +function splitExt(s: string): [string, string] { + const i = s.lastIndexOf(".") + if (i < 0) throw new Error(`no extension found`) + return [s.substring(0, i), s.substring(i + 1)] +} diff --git a/packages/moq/contribute/chunk.ts b/packages/moq/contribute/chunk.ts new file mode 100644 index 0000000..2fcfe33 --- /dev/null +++ b/packages/moq/contribute/chunk.ts @@ -0,0 +1,7 @@ +// Extends EncodedVideoChunk, allowing a new "init" type +export interface Chunk { + type: "init" | "key" | "delta" + timestamp: number // microseconds + duration: number // microseconds + data: Uint8Array +} diff --git a/packages/moq/contribute/container.ts b/packages/moq/contribute/container.ts new file mode 100644 index 0000000..ab1b4f1 --- /dev/null +++ b/packages/moq/contribute/container.ts @@ -0,0 +1,165 @@ +import * as MP4 from "../media/mp4" +import { Chunk } from "./chunk" + +type DecoderConfig = AudioDecoderConfig | VideoDecoderConfig +type EncodedChunk = EncodedAudioChunk | EncodedVideoChunk + +export class Container { + #mp4: MP4.ISOFile + #frame?: EncodedAudioChunk | EncodedVideoChunk // 1 frame buffer + #track?: number + #segment = 0 + + encode: TransformStream + + constructor() { + this.#mp4 = new MP4.ISOFile() + this.#mp4.init() + + this.encode = new TransformStream({ + transform: (frame, controller) => { + if (isDecoderConfig(frame)) { + return this.#init(frame, controller) + } else { + return this.#enqueue(frame, controller) + } + }, + }) + } + + #init(frame: DecoderConfig, controller: TransformStreamDefaultController) { + if (this.#track) throw new Error("duplicate decoder config") + + let codec = frame.codec.substring(0, 4) + if (codec == "opus") { + codec = "Opus" + } + + const options: MP4.TrackOptions = { + type: codec, + timescale: 1_000_000, + } + + if (isVideoConfig(frame)) { + options.width = frame.codedWidth + options.height = frame.codedHeight + } else { + options.channel_count = frame.numberOfChannels + options.samplerate = frame.sampleRate + } + + if (!frame.description) throw new Error("missing frame description") + const desc = frame.description as ArrayBufferLike + + if (codec === "avc1") { + options.avcDecoderConfigRecord = desc + } else if (codec === "hev1") { + options.hevcDecoderConfigRecord = desc + } else if (codec === "Opus") { + // description is an identification header: https://datatracker.ietf.org/doc/html/rfc7845#section-5.1 + // The first 8 bytes are the magic string "OpusHead", followed by what we actually want. + const dops = new MP4.BoxParser.dOpsBox(undefined) + + // Annoyingly, the header is little endian while MP4 is big endian, so we have to parse. + const data = new MP4.Stream(desc, 8, MP4.Stream.LITTLE_ENDIAN) + dops.parse(data) + + dops.Version = 0 + options.description = dops + options.hdlr = "soun" + } else { + throw new Error(`unsupported codec: ${codec}`) + } + + this.#track = this.#mp4.addTrack(options) + if (!this.#track) throw new Error("failed to initialize MP4 track") + + const buffer = MP4.ISOFile.writeInitializationSegment(this.#mp4.ftyp!, this.#mp4.moov!, 0, 0) + const data = new Uint8Array(buffer) + + controller.enqueue({ + type: "init", + timestamp: 0, + duration: 0, + data, + }) + } + + #enqueue(frame: EncodedChunk, controller: TransformStreamDefaultController) { + // Check if we should create a new segment + if (frame.type == "key") { + this.#segment += 1 + } else if (this.#segment == 0) { + throw new Error("must start with keyframe") + } + + // We need a one frame buffer to compute the duration + if (!this.#frame) { + this.#frame = frame + return + } + + const duration = frame.timestamp - this.#frame.timestamp + + // TODO avoid this extra copy by writing to the mdat directly + // ...which means changing mp4box.js to take an offset instead of ArrayBuffer + const buffer = new Uint8Array(this.#frame.byteLength) + this.#frame.copyTo(buffer) + + if (!this.#track) throw new Error("missing decoder config") + + // Add the sample to the container + this.#mp4.addSample(this.#track, buffer, { + duration, + dts: this.#frame.timestamp, + cts: this.#frame.timestamp, + is_sync: this.#frame.type == "key", + }) + + const stream = new MP4.Stream(undefined, 0, MP4.Stream.BIG_ENDIAN) + + // Moof and mdat atoms are written in pairs. + // TODO remove the moof/mdat from the Box to reclaim memory once everything works + for (;;) { + const moof = this.#mp4.moofs.shift() + const mdat = this.#mp4.mdats.shift() + + if (!moof && !mdat) break + if (!moof) throw new Error("moof missing") + if (!mdat) throw new Error("mdat missing") + + moof.write(stream) + mdat.write(stream) + } + + // TODO avoid this extra copy by writing to the buffer provided in copyTo + const data = new Uint8Array(stream.buffer) + + controller.enqueue({ + type: this.#frame.type, + timestamp: this.#frame.timestamp, + duration: this.#frame.duration ?? 0, + data, + }) + + this.#frame = frame + } + + /* TODO flush the last frame + #flush(controller: TransformStreamDefaultController) { + if (this.#frame) { + // TODO guess the duration + this.#enqueue(this.#frame, 0, controller) + } + } + */ +} + +function isDecoderConfig(frame: DecoderConfig | EncodedChunk): frame is DecoderConfig { + // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition + return (frame as DecoderConfig).codec !== undefined +} + +function isVideoConfig(frame: DecoderConfig): frame is VideoDecoderConfig { + return (frame as VideoDecoderConfig).codedWidth !== undefined +} diff --git a/packages/moq/contribute/index.ts b/packages/moq/contribute/index.ts new file mode 100644 index 0000000..704f676 --- /dev/null +++ b/packages/moq/contribute/index.ts @@ -0,0 +1,5 @@ +export { Broadcast } from "./broadcast" +export type { BroadcastConfig, BroadcastConfigTrack } from "./broadcast" + +export { Encoder as VideoEncoder } from "./video" +export { Encoder as AudioEncoder } from "./audio" diff --git a/packages/moq/contribute/segment.ts b/packages/moq/contribute/segment.ts new file mode 100644 index 0000000..f0aa819 --- /dev/null +++ b/packages/moq/contribute/segment.ts @@ -0,0 +1,45 @@ +import { Chunk } from "./chunk" + +export class Segment { + id: number + + // Take in a stream of chunks + input: WritableStream + + // Output a stream of bytes, which we fork for each new subscriber. + #cache: ReadableStream + + timestamp = 0 + + constructor(id: number) { + this.id = id + + // Set a max size for each segment, dropping the tail if it gets too long. + // We tee the reader, so this limit applies to the FASTEST reader. + const backpressure = new ByteLengthQueuingStrategy({ highWaterMark: 8_000_000 }) + + const transport = new TransformStream( + { + transform: (chunk: Chunk, controller) => { + // Compute the max timestamp of the segment + this.timestamp = Math.max(chunk.timestamp + chunk.duration) + + // Push the chunk to any listeners. + controller.enqueue(chunk.data) + }, + }, + undefined, + backpressure, + ) + + this.input = transport.writable + this.#cache = transport.readable + } + + // Split the output reader into two parts. + chunks(): ReadableStream { + const [tee, cache] = this.#cache.tee() + this.#cache = cache + return tee + } +} diff --git a/packages/moq/contribute/track.ts b/packages/moq/contribute/track.ts new file mode 100644 index 0000000..cec70d8 --- /dev/null +++ b/packages/moq/contribute/track.ts @@ -0,0 +1,170 @@ +import { Segment } from "./segment" +import { Notify } from "../common/async" +import { Chunk } from "./chunk" +import { Container } from "./container" +import { BroadcastConfig } from "./broadcast" + +import * as Audio from "./audio" +import * as Video from "./video" + +export class Track { + name: string + + #init?: Uint8Array + #segments: Segment[] = [] + + #offset = 0 // number of segments removed from the front of the queue + #closed = false + #error?: Error + #notify = new Notify() + + constructor(media: MediaStreamTrack, config: BroadcastConfig) { + this.name = media.kind + + // We need to split based on type because Typescript is hard + if (isAudioTrack(media)) { + if (!config.audio) throw new Error("no audio config") + this.#runAudio(media, config.audio).catch((err) => this.#close(err)) + } else if (isVideoTrack(media)) { + if (!config.video) throw new Error("no video config") + this.#runVideo(media, config.video).catch((err) => this.#close(err)) + } else { + throw new Error(`unknown track type: ${media.kind}`) + } + } + + async #runAudio(track: MediaStreamAudioTrack, config: AudioEncoderConfig) { + const source = new MediaStreamTrackProcessor({ track }) + const encoder = new Audio.Encoder(config) + const container = new Container() + + // Split the container at keyframe boundaries + const segments = new WritableStream({ + write: (chunk) => this.#write(chunk), + close: () => this.#close(), + abort: (e) => this.#close(e), + }) + + return source.readable.pipeThrough(encoder.frames).pipeThrough(container.encode).pipeTo(segments) + } + + async #runVideo(track: MediaStreamVideoTrack, config: VideoEncoderConfig) { + const source = new MediaStreamTrackProcessor({ track }) + const encoder = new Video.Encoder(config) + const container = new Container() + + // Split the container at keyframe boundaries + const segments = new WritableStream({ + write: (chunk) => this.#write(chunk), + close: () => this.#close(), + abort: (e) => this.#close(e), + }) + + return source.readable.pipeThrough(encoder.frames).pipeThrough(container.encode).pipeTo(segments) + } + + async #write(chunk: Chunk) { + if (chunk.type === "init") { + this.#init = chunk.data + this.#notify.wake() + return + } + + let current = this.#segments.at(-1) + if (!current || chunk.type === "key") { + if (current) { + await current.input.close() + } + + const segment = new Segment(this.#offset + this.#segments.length) + this.#segments.push(segment) + + this.#notify.wake() + + current = segment + + // Clear old segments + while (this.#segments.length > 1) { + const first = this.#segments[0] + + // Expire after 10s + if (chunk.timestamp - first.timestamp < 10_000_000) break + this.#segments.shift() + this.#offset += 1 + + await first.input.abort("expired") + } + } + + const writer = current.input.getWriter() + + if ((writer.desiredSize || 0) > 0) { + await writer.write(chunk) + } else { + console.warn("dropping chunk", writer.desiredSize) + } + + writer.releaseLock() + } + + async #close(e?: Error) { + this.#error = e + + const current = this.#segments.at(-1) + if (current) { + await current.input.close() + } + + this.#closed = true + this.#notify.wake() + } + + async init(): Promise { + while (!this.#init) { + if (this.#closed) throw new Error("track closed") + await this.#notify.wait() + } + + return this.#init + } + + // TODO generize this + segments(): ReadableStream { + let pos = this.#offset + + return new ReadableStream({ + pull: async (controller) => { + for (;;) { + let index = pos - this.#offset + if (index < 0) index = 0 + + if (index < this.#segments.length) { + controller.enqueue(this.#segments[index]) + pos += 1 + return // Called again when more data is requested + } + + if (this.#error) { + controller.error(this.#error) + return + } else if (this.#closed) { + controller.close() + return + } + + // Pull again on wakeup + // NOTE: We can't return until we enqueue at least one segment. + await this.#notify.wait() + } + }, + }) + } +} + +function isAudioTrack(track: MediaStreamTrack): track is MediaStreamAudioTrack { + return track.kind === "audio" +} + +function isVideoTrack(track: MediaStreamTrack): track is MediaStreamVideoTrack { + return track.kind === "video" +} diff --git a/packages/moq/contribute/tsconfig.json b/packages/moq/contribute/tsconfig.json new file mode 100644 index 0000000..70899a7 --- /dev/null +++ b/packages/moq/contribute/tsconfig.json @@ -0,0 +1,18 @@ +{ + "extends": "../tsconfig.json", + "include": ["."], + "compilerOptions": { + "types": ["dom-mediacapture-transform", "dom-webcodecs"] + }, + "references": [ + { + "path": "../common" + }, + { + "path": "../transport" + }, + { + "path": "../media" + } + ] +} diff --git a/packages/moq/contribute/video.ts b/packages/moq/contribute/video.ts new file mode 100644 index 0000000..747c765 --- /dev/null +++ b/packages/moq/contribute/video.ts @@ -0,0 +1,111 @@ +const SUPPORTED = [ + "avc1", // H.264 + "hev1", // HEVC (aka h.265) + // "av01", // TDOO support AV1 +] + +export interface EncoderSupported { + codecs: string[] +} + +export class Encoder { + #encoder!: VideoEncoder + #encoderConfig: VideoEncoderConfig + #decoderConfig?: VideoDecoderConfig + + // true if we should insert a keyframe, undefined when the encoder should decide + #keyframeNext: true | undefined = true + + // Count the number of frames without a keyframe. + #keyframeCounter = 0 + + // Converts raw rames to encoded frames. + frames: TransformStream + + constructor(config: VideoEncoderConfig) { + config.bitrateMode ??= "constant" + config.latencyMode ??= "realtime" + + this.#encoderConfig = config + + this.frames = new TransformStream({ + start: this.#start.bind(this), + transform: this.#transform.bind(this), + flush: this.#flush.bind(this), + }) + } + + static async isSupported(config: VideoEncoderConfig) { + // Check if we support a specific codec family + const short = config.codec.substring(0, 4) + if (!SUPPORTED.includes(short)) return false + + // Default to hardware encoding + config.hardwareAcceleration ??= "prefer-hardware" + + // Default to CBR + config.bitrateMode ??= "constant" + + // Default to realtime encoding + config.latencyMode ??= "realtime" + + const res = await VideoEncoder.isConfigSupported(config) + return !!res.supported + } + + #start(controller: TransformStreamDefaultController) { + this.#encoder = new VideoEncoder({ + output: (frame, metadata) => { + this.#enqueue(controller, frame, metadata) + }, + error: (err) => { + throw err + }, + }) + + this.#encoder.configure(this.#encoderConfig) + } + + #transform(frame: VideoFrame) { + const encoder = this.#encoder + + // Set keyFrame to undefined when we're not sure so the encoder can decide. + encoder.encode(frame, { keyFrame: this.#keyframeNext }) + this.#keyframeNext = undefined + + frame.close() + } + + #enqueue( + controller: TransformStreamDefaultController, + frame: EncodedVideoChunk, + metadata?: EncodedVideoChunkMetadata, + ) { + if (!this.#decoderConfig) { + const config = metadata?.decoderConfig + if (!config) throw new Error("missing decoder config") + + controller.enqueue(config) + this.#decoderConfig = config + } + + if (frame.type === "key") { + this.#keyframeCounter = 0 + } else { + this.#keyframeCounter += 1 + if (this.#keyframeCounter + this.#encoder.encodeQueueSize >= 2 * this.#encoderConfig.framerate!) { + this.#keyframeNext = true + } + } + + controller.enqueue(frame) + } + + #flush() { + this.#encoder.close() + } + + get config() { + return this.#encoderConfig + } +} diff --git a/packages/moq/media/catalog/index.ts b/packages/moq/media/catalog/index.ts new file mode 100644 index 0000000..41170a5 --- /dev/null +++ b/packages/moq/media/catalog/index.ts @@ -0,0 +1,218 @@ +import { Connection } from "../../transport" +import { asError } from "../../common/error" + +export interface CommonTrackFields { + namespace?: string + packaging?: string + renderGroup?: number + altGroup?: number +} + +export interface Root { + version: number + streamingFormat: number + streamingFormatVersion: string + supportsDeltaUpdates: boolean + commonTrackFields: CommonTrackFields + tracks: Track[] +} + +export function encode(catalog: Root): Uint8Array { + const encoder = new TextEncoder() + const str = JSON.stringify(catalog) + return encoder.encode(str) +} + +export function decode(raw: Uint8Array): Root { + const decoder = new TextDecoder() + const str = decoder.decode(raw) + + const catalog = JSON.parse(str) + if (!isRoot(catalog)) { + throw new Error("invalid catalog") + } + + // Merge common track fields into each track. + for (const track of catalog.tracks) { + track.altGroup ??= catalog.commonTrackFields.altGroup + track.namespace ??= catalog.commonTrackFields.namespace + track.packaging ??= catalog.commonTrackFields.packaging + track.renderGroup ??= catalog.commonTrackFields.renderGroup + } + + return catalog +} + +export async function fetch(connection: Connection, namespace: string): Promise { + const subscribe = await connection.subscribe(namespace, ".catalog") + try { + const segment = await subscribe.data() + if (!segment) throw new Error("no catalog data") + + const chunk = await segment.read() + if (!chunk) throw new Error("no catalog chunk") + + await segment.close() + await subscribe.close() // we done + + if (chunk.payload instanceof Uint8Array) { + return decode(chunk.payload) + } else { + throw new Error("invalid catalog chunk") + } + } catch (e) { + const err = asError(e) + + // Close the subscription after we're done. + await subscribe.close(1n, err.message) + + throw err + } +} + +export function isRoot(catalog: any): catalog is Root { + if (!isCatalogFieldValid(catalog, "packaging")) return false + if (!isCatalogFieldValid(catalog, "namespace")) return false + if (!Array.isArray(catalog.tracks)) return false + return catalog.tracks.every((track: any) => isTrack(track)) +} + +export interface Track { + namespace?: string + name: string + depends?: any[] + packaging?: string + renderGroup?: number + selectionParams: SelectionParams // technically optional but not really + altGroup?: number + initTrack?: string + initData?: string +} + +export interface Mp4Track extends Track { + initTrack?: string + initData?: string + selectionParams: Mp4SelectionParams +} + +export interface SelectionParams { + codec?: string + mimeType?: string + bitrate?: number + lang?: string +} + +export interface Mp4SelectionParams extends SelectionParams { + mimeType: "video/mp4" +} + +export interface AudioTrack extends Track { + name: string + selectionParams: AudioSelectionParams +} + +export interface AudioSelectionParams extends SelectionParams { + samplerate: number + channelConfig: string +} + +export interface VideoTrack extends Track { + name: string + selectionParams: VideoSelectionParams + temporalId?: number + spatialId?: number +} + +export interface VideoSelectionParams extends SelectionParams { + width: number + height: number + displayWidth?: number + displayHeight?: number + framerate?: number +} + +export function isTrack(track: any): track is Track { + if (typeof track.name !== "string") return false + return true +} + +export function isMp4Track(track: any): track is Mp4Track { + if (!isTrack(track)) return false + if (typeof track.initTrack !== "string" && typeof track.initData !== "string") return false + if (typeof track.selectionParams.mimeType !== "string") return false + return true +} + +export function isVideoTrack(track: any): track is VideoTrack { + if (!isTrack(track)) return false + return isVideoSelectionParams(track.selectionParams) +} + +export function isVideoSelectionParams(params: any): params is VideoSelectionParams { + if (typeof params.width !== "number") return false + if (typeof params.height !== "number") return false + return true +} + +export function isAudioTrack(track: any): track is AudioTrack { + if (!isTrack(track)) return false + return isAudioSelectionParams(track.selectionParams) +} + +export function isAudioSelectionParams(params: any): params is AudioSelectionParams { + if (typeof params.channelConfig !== "string") return false + if (typeof params.samplerate !== "number") return false + return true +} + +function isCatalogFieldValid(catalog: any, field: string): boolean { + //packaging,namespace if common would be listed in commonTrackFields but if fields + //in commonTrackFields are mentiond in Tracks , the fields in Tracks precedes + + function isValidPackaging(packaging: any): boolean { + return packaging === "cmaf" || packaging === "loc" + } + + function isValidNamespace(namespace: any): boolean { + return typeof namespace === "string" + } + + let isValidField: (value: any) => boolean + if (field === "packaging") { + isValidField = isValidPackaging + } else if (field === "namespace") { + isValidField = isValidNamespace + } else { + throw new Error(`Invalid field: ${field}`) + } + + if (catalog.commonTrackFields[field] !== undefined && !isValidField(catalog.commonTrackFields[field])) { + return false + } + + for (const track of catalog.tracks) { + if (track[field] !== undefined && !isValidField(track[field])) { + return false + } + } + + return true +} + +export function isMediaTrack(track: any): track is Track { + if (track.name.toLowerCase().includes("audio") || track.name.toLowerCase().includes("video")) { + return true + } + + if (track.selectionParams && track.selectionParams.codec) { + const codec = track.selectionParams.codec.toLowerCase() + const acceptedCodecs = ["mp4a", "avc1"] + + for (const acceptedCodec of acceptedCodecs) { + if (codec.includes(acceptedCodec)) { + return true + } + } + } + return false +} diff --git a/packages/moq/media/mp4/index.ts b/packages/moq/media/mp4/index.ts new file mode 100644 index 0000000..5301db3 --- /dev/null +++ b/packages/moq/media/mp4/index.ts @@ -0,0 +1,37 @@ +// Rename some stuff so it's on brand. +// We need a separate file so this file can use the rename too. +import * as MP4 from "./rename" +export * from "./rename" + +export * from "./parser" + +export function isAudioTrack(track: MP4.Track): track is MP4.AudioTrack { + // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition + return (track as MP4.AudioTrack).audio !== undefined +} + +export function isVideoTrack(track: MP4.Track): track is MP4.VideoTrack { + // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition + return (track as MP4.VideoTrack).video !== undefined +} + +// TODO contribute to mp4box +MP4.BoxParser.dOpsBox.prototype.write = function (stream: MP4.Stream) { + this.size = this.ChannelMappingFamily === 0 ? 11 : 13 + this.ChannelMapping!.length + this.writeHeader(stream) + + stream.writeUint8(this.Version) + stream.writeUint8(this.OutputChannelCount) + stream.writeUint16(this.PreSkip) + stream.writeUint32(this.InputSampleRate) + stream.writeInt16(this.OutputGain) + stream.writeUint8(this.ChannelMappingFamily) + + if (this.ChannelMappingFamily !== 0) { + stream.writeUint8(this.StreamCount!) + stream.writeUint8(this.CoupledCount!) + for (const mapping of this.ChannelMapping!) { + stream.writeUint8(mapping) + } + } +} diff --git a/packages/moq/media/mp4/parser.ts b/packages/moq/media/mp4/parser.ts new file mode 100644 index 0000000..7cc78d0 --- /dev/null +++ b/packages/moq/media/mp4/parser.ts @@ -0,0 +1,71 @@ +import * as MP4 from "./index" + +export interface Frame { + track: MP4.Track // The track this frame belongs to + sample: MP4.Sample // The actual sample contain the frame data +} + +// Decode a MP4 container into individual samples. +export class Parser { + info!: MP4.Info + + #mp4 = MP4.New() + #offset = 0 + + #samples: Array = [] + + constructor(init: Uint8Array) { + this.#mp4.onError = (err) => { + console.error("MP4 error", err) + } + + this.#mp4.onReady = (info: MP4.Info) => { + this.info = info + + // Extract all of the tracks, because we don't know if it's audio or video. + for (const track of info.tracks) { + this.#mp4.setExtractionOptions(track.id, track, { nbSamples: 1 }) + } + } + + this.#mp4.onSamples = (_track_id: number, track: MP4.Track, samples: MP4.Sample[]) => { + for (const sample of samples) { + this.#samples.push({ track, sample }) + } + } + + this.#mp4.start() + + // For some reason we need to modify the underlying ArrayBuffer with offset + const copy = new Uint8Array(init) + const buffer = copy.buffer as MP4.ArrayBuffer + buffer.fileStart = this.#offset + + this.#mp4.appendBuffer(buffer) + this.#offset += buffer.byteLength + this.#mp4.flush() + + if (!this.info) { + throw new Error("could not parse MP4 info") + } + } + + decode(chunk: Uint8Array): Array { + const copy = new Uint8Array(chunk) + + // For some reason we need to modify the underlying ArrayBuffer with offset + const buffer = copy.buffer as MP4.ArrayBuffer + buffer.fileStart = this.#offset + + // Parse the data + this.#mp4.appendBuffer(buffer) + this.#mp4.flush() + + this.#offset += buffer.byteLength + + const samples = [...this.#samples] + this.#samples.length = 0 + + return samples + } +} diff --git a/packages/moq/media/mp4/rename.ts b/packages/moq/media/mp4/rename.ts new file mode 100644 index 0000000..36f8fc8 --- /dev/null +++ b/packages/moq/media/mp4/rename.ts @@ -0,0 +1,13 @@ +// Rename some stuff so it's on brand. +export { createFile as New, DataStream as Stream, ISOFile, BoxParser, Log } from "mp4box" + +export type { + MP4ArrayBuffer as ArrayBuffer, + MP4Info as Info, + MP4Track as Track, + MP4AudioTrack as AudioTrack, + MP4VideoTrack as VideoTrack, + Sample, + TrackOptions, + SampleOptions, +} from "mp4box" diff --git a/packages/moq/media/tsconfig.json b/packages/moq/media/tsconfig.json new file mode 100644 index 0000000..2e4ee28 --- /dev/null +++ b/packages/moq/media/tsconfig.json @@ -0,0 +1,15 @@ +{ + "extends": "../tsconfig.json", + "include": ["."], + "compilerOptions": { + "types": ["mp4box"] + }, + "references": [ + { + "path": "../transport" + }, + { + "path": "../common" + } + ] +} diff --git a/packages/moq/package.json b/packages/moq/package.json new file mode 100644 index 0000000..30535dd --- /dev/null +++ b/packages/moq/package.json @@ -0,0 +1,29 @@ +{ + "name": "@nestri/moq", + "type": "module", + "version": "0.1.4", + "description": "Media over QUIC library", + "license": "(MIT OR Apache-2.0)", + "repository": "github:kixelated/moq-js", + "scripts": { + "build": "tsc -b && cp ../LICENSE* ./dist && cp ./README.md ./dist && cp ./package.json ./dist", + "lint": "eslint .", + "fmt": "prettier --write ." + }, + "devDependencies": { + "@types/audioworklet": "^0.0.50", + "@types/dom-mediacapture-transform": "^0.1.6", + "@types/dom-webcodecs": "^0.1.8", + "@typescript/lib-dom": "npm:@types/web@^0.0.115", + "@typescript-eslint/eslint-plugin": "^6.4.0", + "@typescript-eslint/parser": "^6.4.0", + "eslint": "^8.47.0", + "eslint-config-prettier": "^9.0.0", + "eslint-plugin-prettier": "^5.0.0", + "prettier": "^3.0.1", + "typescript": "^5.1.6" + }, + "dependencies": { + "mp4box": "^0.5.2" + } +} diff --git a/packages/moq/playback/audio.ts b/packages/moq/playback/audio.ts new file mode 100644 index 0000000..687885a --- /dev/null +++ b/packages/moq/playback/audio.ts @@ -0,0 +1,50 @@ +/// + +import * as Message from "./worker/message" + +// This is a non-standard way of importing worklet/workers. +// Unfortunately, it's the only option because of a Vite bug: https://github.com/vitejs/vite/issues/11823 +import workletURL from "./worklet/index.ts?worker&url" + +// NOTE: This must be on the main thread +export class Audio { + context: AudioContext + worklet: Promise + + constructor(config: Message.ConfigAudio) { + this.context = new AudioContext({ + latencyHint: "interactive", + sampleRate: config.sampleRate, + }) + + this.worklet = this.load(config) + } + + private async load(config: Message.ConfigAudio): Promise { + // Load the worklet source code. + await this.context.audioWorklet.addModule(workletURL) + + const volume = this.context.createGain() + volume.gain.value = 2.0 + + // Create the worklet + const worklet = new AudioWorkletNode(this.context, "renderer") + + worklet.port.addEventListener("message", this.on.bind(this)) + worklet.onprocessorerror = (e: Event) => { + console.error("Audio worklet error:", e) + } + + // Connect the worklet to the volume node and then to the speakers + worklet.connect(volume) + volume.connect(this.context.destination) + + worklet.port.postMessage({ config }) + + return worklet + } + + private on(_event: MessageEvent) { + // TODO + } +} diff --git a/packages/moq/playback/backend.ts b/packages/moq/playback/backend.ts new file mode 100644 index 0000000..2b336ce --- /dev/null +++ b/packages/moq/playback/backend.ts @@ -0,0 +1,114 @@ +/// + +import * as Message from "./worker/message" +import { Audio } from "./audio" + +import MediaWorker from "./worker?worker" +import { RingShared } from "../common/ring" +import { Root, isAudioTrack } from "../media/catalog" +import { GroupHeader } from "../transport/objects" + +export interface PlayerConfig { + canvas: OffscreenCanvas + catalog: Root +} + +// This is a non-standard way of importing worklet/workers. +// Unfortunately, it's the only option because of a Vite bug: https://github.com/vitejs/vite/issues/11823 + +// Responsible for sending messages to the worker and worklet. +export default class Backend { + // General worker + #worker: Worker + + // The audio context, which must be created on the main thread. + #audio?: Audio + + constructor(config: PlayerConfig) { + // TODO does this block the main thread? If so, make this async + // @ts-expect-error: The Vite typing is wrong https://github.com/vitejs/vite/blob/22bd67d70a1390daae19ca33d7de162140d533d6/packages/vite/client.d.ts#L182 + this.#worker = new MediaWorker({ format: "es" }) + this.#worker.addEventListener("message", this.on.bind(this)) + + let sampleRate: number | undefined + let channels: number | undefined + + for (const track of config.catalog.tracks) { + if (isAudioTrack(track)) { + if (sampleRate && track.selectionParams.samplerate !== sampleRate) { + throw new Error(`TODO multiple audio tracks with different sample rates`) + } + + sampleRate = track.selectionParams.samplerate + + // TODO properly handle weird channel configs + channels = Math.max(+track.selectionParams.channelConfig, channels ?? 0) + } + } + + const msg: Message.Config = {} + + // Only configure audio is we have an audio track + if (sampleRate && channels) { + msg.audio = { + channels: channels, + sampleRate: sampleRate, + ring: new RingShared(2, sampleRate / 10), // 100ms + } + + this.#audio = new Audio(msg.audio) + } + + // TODO only send the canvas if we have a video track + msg.video = { + canvas: config.canvas, + } + + this.send({ config: msg }, msg.video.canvas) + } + + async play() { + await this.#audio?.context.resume() + } + + init(init: Init) { + this.send({ init }) + } + + segment(segment: Segment) { + this.send({ segment }, segment.stream) + } + + async close() { + this.#worker.terminate() + await this.#audio?.context.close() + } + + // Enforce we're sending valid types to the worker + private send(msg: Message.ToWorker, ...transfer: Transferable[]) { + //console.log("sent message from main to worker", msg) + this.#worker.postMessage(msg, transfer) + } + + private on(e: MessageEvent) { + const msg = e.data as Message.FromWorker + + // Don't print the verbose timeline message. + if (!msg.timeline) { + //console.log("received message from worker to main", msg) + } + } +} + +export interface Init { + name: string // name of the init track + data: Uint8Array +} + +export interface Segment { + init: string // name of the init track + kind: "audio" | "video" + header: GroupHeader + buffer: Uint8Array + stream: ReadableStream +} diff --git a/packages/moq/playback/index.ts b/packages/moq/playback/index.ts new file mode 100644 index 0000000..706b2f2 --- /dev/null +++ b/packages/moq/playback/index.ts @@ -0,0 +1,190 @@ +import * as Message from "./worker/message" + +import { Connection } from "../transport/connection" +import * as Catalog from "../media/catalog" +import { asError } from "../common/error" + +import Backend from "./backend" + +import { Client } from "../transport/client" +import { GroupReader } from "../transport/objects" + +export type Range = Message.Range +export type Timeline = Message.Timeline + +export interface PlayerConfig { + url: string + namespace: string + fingerprint?: string // URL to fetch TLS certificate fingerprint + canvas: HTMLCanvasElement +} + +// This class must be created on the main thread due to AudioContext. +export class Player { + #backend: Backend + + // A periodically updated timeline + //#timeline = new Watch(undefined) + + #connection: Connection + #catalog: Catalog.Root + + // Running is a promise that resolves when the player is closed. + // #close is called with no error, while #abort is called with an error. + #running: Promise + #close!: () => void + #abort!: (err: Error) => void + + private constructor(connection: Connection, catalog: Catalog.Root, backend: Backend) { + this.#connection = connection + this.#catalog = catalog + this.#backend = backend + + const abort = new Promise((resolve, reject) => { + this.#close = resolve + this.#abort = reject + }) + + // Async work + this.#running = Promise.race([this.#run(), abort]).catch(this.#close) + } + + static async create(config: PlayerConfig): Promise { + const client = new Client({ url: config.url, fingerprint: config.fingerprint, role: "subscriber" }) + const connection = await client.connect() + + const catalog = await Catalog.fetch(connection, config.namespace) + console.log("catalog", catalog) + + const canvas = config.canvas.transferControlToOffscreen() + const backend = new Backend({ canvas, catalog }) + + return new Player(connection, catalog, backend) + } + + async #run() { + const inits = new Set<[string, string]>() + const tracks = new Array() + + for (const track of this.#catalog.tracks) { + if (!track.namespace) throw new Error("track has no namespace") + if (track.initTrack) inits.add([track.namespace, track.initTrack]) + tracks.push(track) + } + + // Call #runInit on each unique init track + // TODO do this in parallel with #runTrack to remove a round trip + await Promise.all(Array.from(inits).map((init) => this.#runInit(...init))) + + // Call #runTrack on each track + await Promise.all(tracks.map((track) => this.#runTrack(track))) + } + + async #runInit(namespace: string, name: string) { + const sub = await this.#connection.subscribe(namespace, name) + try { + const init = await Promise.race([sub.data(), this.#running]) + if (!init) throw new Error("no init data") + + // We don't care what type of reader we get, we just want the payload. + const chunk = await init.read() + if (!chunk) throw new Error("no init chunk") + if (!(chunk.payload instanceof Uint8Array)) throw new Error("invalid init chunk") + + this.#backend.init({ data: chunk.payload, name }) + } finally { + await sub.close() + } + } + + async #runTrack(track: Catalog.Track) { + if (!track.namespace) throw new Error("track has no namespace") + const sub = await this.#connection.subscribe(track.namespace, track.name) + + try { + for (;;) { + const segment = await Promise.race([sub.data(), this.#running]) + if (!segment) break + + if (!(segment instanceof GroupReader)) { + throw new Error(`expected group reader for segment: ${track.name}`) + } + + const kind = Catalog.isVideoTrack(track) ? "video" : Catalog.isAudioTrack(track) ? "audio" : "unknown" + if (kind == "unknown") { + throw new Error(`unknown track kind: ${track.name}`) + } + + if (!track.initTrack) { + throw new Error(`no init track for segment: ${track.name}`) + } + + const [buffer, stream] = segment.stream.release() + + this.#backend.segment({ + init: track.initTrack, + kind, + header: segment.header, + buffer, + stream, + }) + } + } catch (error) { + console.error("Error in #runTrack:", error) + } finally { + await sub.close() + } + } + + getCatalog() { + return this.#catalog + } + + #onMessage(msg: Message.FromWorker) { + if (msg.timeline) { + //this.#timeline.update(msg.timeline) + } + } + + async close(err?: Error) { + if (err) this.#abort(err) + else this.#close() + + if (this.#connection) this.#connection.close() + if (this.#backend) await this.#backend.close() + } + + async closed(): Promise { + try { + await this.#running + } catch (e) { + return asError(e) + } + } + + /* + play() { + this.#backend.play({ minBuffer: 0.5 }) // TODO configurable + } + + seek(timestamp: number) { + this.#backend.seek({ timestamp }) + } + */ + + async play() { + await this.#backend.play() + } + + /* + async *timeline() { + for (;;) { + const [timeline, next] = this.#timeline.value() + if (timeline) yield timeline + if (!next) break + + await next + } + } + */ +} diff --git a/packages/moq/playback/tsconfig.json b/packages/moq/playback/tsconfig.json new file mode 100644 index 0000000..77318e8 --- /dev/null +++ b/packages/moq/playback/tsconfig.json @@ -0,0 +1,22 @@ +{ + "extends": "../tsconfig.json", + "include": ["."], + "exclude": ["./worklet"], + "compilerOptions": { + "types": ["dom-mediacapture-transform", "dom-webcodecs"] + }, + "references": [ + { + "path": "../common" + }, + { + "path": "../transport" + }, + { + "path": "../media" + } + ], + "paths": { + "@/*": ["*"] + } +} diff --git a/packages/moq/playback/worker/audio.ts b/packages/moq/playback/worker/audio.ts new file mode 100644 index 0000000..de57e36 --- /dev/null +++ b/packages/moq/playback/worker/audio.ts @@ -0,0 +1,73 @@ +import * as Message from "./message" +import { Ring } from "../../common/ring" +import { Component, Frame } from "./timeline" +import * as MP4 from "../../media/mp4" + +// This is run in a worker. +export class Renderer { + #ring: Ring + #timeline: Component + + #decoder!: AudioDecoder + #stream: TransformStream + + constructor(config: Message.ConfigAudio, timeline: Component) { + this.#timeline = timeline + this.#ring = new Ring(config.ring) + + this.#stream = new TransformStream({ + start: this.#start.bind(this), + transform: this.#transform.bind(this), + }) + + this.#run().catch(console.error) + } + + #start(controller: TransformStreamDefaultController) { + this.#decoder = new AudioDecoder({ + output: (frame: AudioData) => { + controller.enqueue(frame) + }, + error: console.warn, + }) + } + + #transform(frame: Frame) { + if (this.#decoder.state !== "configured") { + const track = frame.track + if (!MP4.isAudioTrack(track)) throw new Error("expected audio track") + + // We only support OPUS right now which doesn't need a description. + this.#decoder.configure({ + codec: track.codec, + sampleRate: track.audio.sample_rate, + numberOfChannels: track.audio.channel_count, + }) + } + + const chunk = new EncodedAudioChunk({ + type: frame.sample.is_sync ? "key" : "delta", + timestamp: frame.sample.dts / frame.track.timescale, + duration: frame.sample.duration, + data: frame.sample.data, + }) + + this.#decoder.decode(chunk) + } + + async #run() { + const reader = this.#timeline.frames.pipeThrough(this.#stream).getReader() + + for (;;) { + const { value: frame, done } = await reader.read() + if (done) break + + // Write audio samples to the ring buffer, dropping when there's no space. + const written = this.#ring.write(frame) + + if (written < frame.numberOfFrames) { + console.warn(`droppped ${frame.numberOfFrames - written} audio samples`) + } + } + } +} diff --git a/packages/moq/playback/worker/index.ts b/packages/moq/playback/worker/index.ts new file mode 100644 index 0000000..1275b3f --- /dev/null +++ b/packages/moq/playback/worker/index.ts @@ -0,0 +1,119 @@ +import { Timeline } from "./timeline" + +import * as Audio from "./audio" +import * as Video from "./video" + +import * as MP4 from "../../media/mp4" +import * as Message from "./message" +import { asError } from "../../common/error" +import { Deferred } from "../../common/async" +import { GroupReader, Reader } from "../../transport/objects" + +class Worker { + // Timeline receives samples, buffering them and choosing the timestamp to render. + #timeline = new Timeline() + + // A map of init tracks. + #inits = new Map>() + + // Renderer requests samples, rendering video frames and emitting audio frames. + #audio?: Audio.Renderer + #video?: Video.Renderer + + on(e: MessageEvent) { + const msg = e.data as Message.ToWorker + + if (msg.config) { + this.#onConfig(msg.config) + } else if (msg.init) { + // TODO buffer the init segmnet so we don't hold the stream open. + this.#onInit(msg.init) + } else if (msg.segment) { + this.#onSegment(msg.segment).catch(console.warn) + } else { + throw new Error(`unknown message: + ${JSON.stringify(msg)}`) + } + } + + #onConfig(msg: Message.Config) { + if (msg.audio) { + this.#audio = new Audio.Renderer(msg.audio, this.#timeline.audio) + } + + if (msg.video) { + this.#video = new Video.Renderer(msg.video, this.#timeline.video) + } + } + + #onInit(msg: Message.Init) { + let init = this.#inits.get(msg.name) + if (!init) { + init = new Deferred() + this.#inits.set(msg.name, init) + } + + init.resolve(msg.data) + } + + async #onSegment(msg: Message.Segment) { + let init = this.#inits.get(msg.init) + if (!init) { + init = new Deferred() + this.#inits.set(msg.init, init) + } + + // Create a new stream that we will use to decode. + const container = new MP4.Parser(await init.promise) + + const timeline = msg.kind === "audio" ? this.#timeline.audio : this.#timeline.video + const reader = new GroupReader(msg.header, new Reader(msg.buffer, msg.stream)) + + // Create a queue that will contain each MP4 frame. + const queue = new TransformStream({}) + const segment = queue.writable.getWriter() + + // Add the segment to the timeline + const segments = timeline.segments.getWriter() + await segments.write({ + sequence: msg.header.group, + frames: queue.readable, + }) + segments.releaseLock() + + // Read each chunk, decoding the MP4 frames and adding them to the queue. + for (;;) { + const chunk = await reader.read() + if (!chunk) { + break + } + + if (!(chunk.payload instanceof Uint8Array)) { + throw new Error(`invalid payload: ${chunk.payload}`) + } + + const frames = container.decode(chunk.payload) + for (const frame of frames) { + await segment.write(frame) + } + } + + // We done. + await segment.close() + } +} + +// Pass all events to the worker +const worker = new Worker() +self.addEventListener("message", (msg) => { + try { + worker.on(msg) + } catch (e) { + const err = asError(e) + console.warn("worker error:", err) + } +}) + +// Validates this is an expected message +function _send(msg: Message.FromWorker) { + postMessage(msg) +} diff --git a/packages/moq/playback/worker/message.ts b/packages/moq/playback/worker/message.ts new file mode 100644 index 0000000..e6a1918 --- /dev/null +++ b/packages/moq/playback/worker/message.ts @@ -0,0 +1,98 @@ +import { GroupHeader } from "../../transport/objects" +import { RingShared } from "../../common/ring" + +export interface Config { + audio?: ConfigAudio + video?: ConfigVideo +} + +export interface ConfigAudio { + channels: number + sampleRate: number + + ring: RingShared +} + +export interface ConfigVideo { + canvas: OffscreenCanvas +} + +export interface Init { + name: string // name of the init object + data: Uint8Array +} + +export interface Segment { + init: string // name of the init object + kind: "audio" | "video" + header: GroupHeader + buffer: Uint8Array + stream: ReadableStream +} + +/* +export interface Play { + // Start playback once the minimum buffer size has been reached. + minBuffer: number +} + +export interface Seek { + timestamp: number +} +*/ + +// Sent periodically with the current timeline info. +export interface Timeline { + // The current playback position + timestamp?: number + + // Audio specific information + audio: TimelineAudio + + // Video specific information + video: TimelineVideo +} + +export interface TimelineAudio { + buffer: Range[] +} + +export interface TimelineVideo { + buffer: Range[] +} + +export interface Range { + start: number + end: number +} + +// Used to validate that only the correct messages can be sent. + +// Any top level messages that can be sent to the worker. +export interface ToWorker { + // Sent to configure on startup. + config?: Config + + // Sent on each init/data stream + init?: Init + segment?: Segment + + /* + // Sent to control playback + play?: Play + seek?: Seek + */ +} + +// Any top-level messages that can be sent from the worker. +export interface FromWorker { + // Sent back to the main thread regularly to update the UI + timeline?: Timeline +} + +/* +interface ToWorklet { + config?: Audio.Config +} + +*/ diff --git a/packages/moq/playback/worker/timeline.ts b/packages/moq/playback/worker/timeline.ts new file mode 100644 index 0000000..2eaac02 --- /dev/null +++ b/packages/moq/playback/worker/timeline.ts @@ -0,0 +1,118 @@ +import type { Frame } from "../../media/mp4" +export type { Frame } + +export interface Range { + start: number + end: number +} + +export class Timeline { + // Maintain audio and video seprarately + audio: Component + video: Component + + // Construct a timeline + constructor() { + this.audio = new Component() + this.video = new Component() + } +} + +interface Segment { + sequence: number + frames: ReadableStream +} + +export class Component { + #current?: Segment + + frames: ReadableStream + #segments: TransformStream + + constructor() { + this.frames = new ReadableStream({ + pull: this.#pull.bind(this), + cancel: this.#cancel.bind(this), + }) + + // This is a hack to have an async channel with 100 items. + this.#segments = new TransformStream({}, { highWaterMark: 100 }) + } + + get segments() { + return this.#segments.writable + } + + async #pull(controller: ReadableStreamDefaultController) { + for (;;) { + // Get the next segment to render. + const segments = this.#segments.readable.getReader() + + let res + if (this.#current) { + // Get the next frame to render. + const frames = this.#current.frames.getReader() + + // Wait for either the frames or segments to be ready. + // NOTE: This assume that the first promise gets priority. + res = await Promise.race([frames.read(), segments.read()]) + + frames.releaseLock() + } else { + res = await segments.read() + } + + segments.releaseLock() + + const { value, done } = res + + if (done) { + // We assume the current segment has been closed + // TODO support the segments stream closing + this.#current = undefined + continue + } + + if (!isSegment(value)) { + // Return so the reader can decide when to get the next frame. + controller.enqueue(value) + return + } + + // We didn't get any frames, and instead got a new segment. + if (this.#current) { + if (value.sequence < this.#current.sequence) { + // Our segment is older than the current, abandon it. + await value.frames.cancel("skipping segment; too old") + continue + } else { + // Our segment is newer than the current, cancel the old one. + await this.#current.frames.cancel("skipping segment; too slow") + } + } + + this.#current = value + } + } + + async #cancel(reason: any) { + if (this.#current) { + await this.#current.frames.cancel(reason) + } + + const segments = this.#segments.readable.getReader() + for (;;) { + const { value: segment, done } = await segments.read() + if (done) break + + await segment.frames.cancel(reason) + } + } +} + +// Return if a type is a segment or frame +// eslint-disable-next-line @typescript-eslint/no-redundant-type-constituents +function isSegment(value: Segment | Frame): value is Segment { + // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition + return (value as Segment).frames !== undefined +} diff --git a/packages/moq/playback/worker/video.ts b/packages/moq/playback/worker/video.ts new file mode 100644 index 0000000..3c0fd11 --- /dev/null +++ b/packages/moq/playback/worker/video.ts @@ -0,0 +1,84 @@ +import { Frame, Component } from "./timeline" +import * as MP4 from "../../media/mp4" +import * as Message from "./message" + +export class Renderer { + #canvas: OffscreenCanvas + #timeline: Component + + #decoder!: VideoDecoder + #queue: TransformStream + + constructor(config: Message.ConfigVideo, timeline: Component) { + this.#canvas = config.canvas + this.#timeline = timeline + + this.#queue = new TransformStream({ + start: this.#start.bind(this), + transform: this.#transform.bind(this), + }) + + this.#run().catch(console.error) + } + + async #run() { + const reader = this.#timeline.frames.pipeThrough(this.#queue).getReader() + for (;;) { + const { value: frame, done } = await reader.read() + if (done) break + + self.requestAnimationFrame(() => { + this.#canvas.width = frame.displayWidth + this.#canvas.height = frame.displayHeight + + const ctx = this.#canvas.getContext("2d") + if (!ctx) throw new Error("failed to get canvas context") + + ctx.drawImage(frame, 0, 0, frame.displayWidth, frame.displayHeight) // TODO respect aspect ratio + frame.close() + }) + } + } + + #start(controller: TransformStreamDefaultController) { + this.#decoder = new VideoDecoder({ + output: (frame: VideoFrame) => { + controller.enqueue(frame) + }, + error: console.error, + }) + } + + #transform(frame: Frame) { + // Configure the decoder with the first frame + if (this.#decoder.state !== "configured") { + const { sample, track } = frame + + const desc = sample.description + const box = desc.avcC ?? desc.hvcC ?? desc.vpcC ?? desc.av1C + if (!box) throw new Error(`unsupported codec: ${track.codec}`) + + const buffer = new MP4.Stream(undefined, 0, MP4.Stream.BIG_ENDIAN) + box.write(buffer) + const description = new Uint8Array(buffer.buffer, 8) // Remove the box header. + + if (!MP4.isVideoTrack(track)) throw new Error("expected video track") + + this.#decoder.configure({ + codec: track.codec, + codedHeight: track.video.height, + codedWidth: track.video.width, + description, + // optimizeForLatency: true + }) + } + + const chunk = new EncodedVideoChunk({ + type: frame.sample.is_sync ? "key" : "delta", + data: frame.sample.data, + timestamp: frame.sample.dts / frame.track.timescale, + }) + + this.#decoder.decode(chunk) + } +} diff --git a/packages/moq/playback/worklet/index.ts b/packages/moq/playback/worklet/index.ts new file mode 100644 index 0000000..afe485d --- /dev/null +++ b/packages/moq/playback/worklet/index.ts @@ -0,0 +1,58 @@ +// TODO add support for @/ to avoid relative imports +import { Ring } from "../../common/ring" +import * as Message from "./message" + +class Renderer extends AudioWorkletProcessor { + ring?: Ring + base: number + + constructor() { + // The super constructor call is required. + super() + + this.base = 0 + this.port.onmessage = this.onMessage.bind(this) + } + + onMessage(e: MessageEvent) { + const msg = e.data as Message.From + if (msg.config) { + this.onConfig(msg.config) + } + } + + onConfig(config: Message.Config) { + this.ring = new Ring(config.ring) + } + + // Inputs and outputs in groups of 128 samples. + process(inputs: Float32Array[][], outputs: Float32Array[][], _parameters: Record): boolean { + if (!this.ring) { + // Paused + return true + } + + if (inputs.length != 1 && outputs.length != 1) { + throw new Error("only a single track is supported") + } + + if (this.ring.size() == this.ring.capacity) { + // This is a hack to clear any latency in the ring buffer. + // The proper solution is to play back slightly faster? + console.warn("resyncing ring buffer") + this.ring.clear() + return true + } + + const output = outputs[0] + + const size = this.ring.read(output) + if (size < output.length) { + // TODO trigger rebuffering event + } + + return true + } +} + +registerProcessor("renderer", Renderer) diff --git a/packages/moq/playback/worklet/message.ts b/packages/moq/playback/worklet/message.ts new file mode 100644 index 0000000..e55ebdd --- /dev/null +++ b/packages/moq/playback/worklet/message.ts @@ -0,0 +1,12 @@ +import { RingShared } from "../../common/ring" + +export interface From { + config?: Config +} + +export interface Config { + channels: number + sampleRate: number + + ring: RingShared +} diff --git a/packages/moq/playback/worklet/tsconfig.json b/packages/moq/playback/worklet/tsconfig.json new file mode 100644 index 0000000..77c5a71 --- /dev/null +++ b/packages/moq/playback/worklet/tsconfig.json @@ -0,0 +1,14 @@ +{ + "extends": "../../tsconfig.json", + "include": ["."], + "exclude": ["./index"], + "compilerOptions": { + "lib": ["es2022"], + "types": ["audioworklet"] + }, + "references": [ + { + "path": "../../common" + } + ] +} diff --git a/packages/moq/transport/client.ts b/packages/moq/transport/client.ts new file mode 100644 index 0000000..1ae5102 --- /dev/null +++ b/packages/moq/transport/client.ts @@ -0,0 +1,83 @@ +import * as Stream from "./stream" +import * as Setup from "./setup" +import * as Control from "./control" +import { Objects } from "./objects" +import { Connection } from "./connection" + +export interface ClientConfig { + url: string + + // Parameters used to create the MoQ session + role: Setup.Role + + // If set, the server fingerprint will be fetched from this URL. + // This is required to use self-signed certificates with Chrome (May 2023) + fingerprint?: string +} + +export class Client { + #fingerprint: Promise + + readonly config: ClientConfig + + constructor(config: ClientConfig) { + this.config = config + + this.#fingerprint = this.#fetchFingerprint(config.fingerprint).catch((e) => { + console.warn("failed to fetch fingerprint: ", e) + return undefined + }) + } + + async connect(): Promise { + // Helper function to make creating a promise easier + const options: WebTransportOptions = {} + + const fingerprint = await this.#fingerprint + if (fingerprint) options.serverCertificateHashes = [fingerprint] + + const quic = new WebTransport(this.config.url, options) + await quic.ready + + const stream = await quic.createBidirectionalStream() + + const writer = new Stream.Writer(stream.writable) + const reader = new Stream.Reader(new Uint8Array(), stream.readable) + + const setup = new Setup.Stream(reader, writer) + + // Send the setup message. + await setup.send.client({ versions: [Setup.Version.DRAFT_04], role: this.config.role }) + + // Receive the setup message. + // TODO verify the SETUP response. + const server = await setup.recv.server() + + if (server.version != Setup.Version.DRAFT_04) { + throw new Error(`unsupported server version: ${server.version}`) + } + + const control = new Control.Stream(reader, writer) + const objects = new Objects(quic) + + return new Connection(quic, control, objects) + } + + async #fetchFingerprint(url?: string): Promise { + if (!url) return + + // TODO remove this fingerprint when Chrome WebTransport accepts the system CA + const response = await fetch(url) + const hexString = await response.text() + + const hexBytes = new Uint8Array(hexString.length / 2) + for (let i = 0; i < hexBytes.length; i += 1) { + hexBytes[i] = parseInt(hexString.slice(2 * i, 2 * i + 2), 16) + } + + return { + algorithm: "sha-256", + value: hexBytes, + } + } +} diff --git a/packages/moq/transport/connection.ts b/packages/moq/transport/connection.ts new file mode 100644 index 0000000..55e5c77 --- /dev/null +++ b/packages/moq/transport/connection.ts @@ -0,0 +1,95 @@ +import * as Control from "./control" +import { Objects } from "./objects" +import { asError } from "../common/error" + +import { Publisher } from "./publisher" +import { Subscriber } from "./subscriber" + +export class Connection { + // The established WebTransport session. + #quic: WebTransport + + // Use to receive/send control messages. + #control: Control.Stream + + // Use to receive/send objects. + #objects: Objects + + // Module for contributing tracks. + #publisher: Publisher + + // Module for distributing tracks. + #subscriber: Subscriber + + // Async work running in the background + #running: Promise + + constructor(quic: WebTransport, control: Control.Stream, objects: Objects) { + this.#quic = quic + this.#control = control + this.#objects = objects + + this.#publisher = new Publisher(this.#control, this.#objects) + this.#subscriber = new Subscriber(this.#control, this.#objects) + + this.#running = this.#run() + } + + close(code = 0, reason = "") { + this.#quic.close({ closeCode: code, reason }) + } + + async #run(): Promise { + await Promise.all([this.#runControl(), this.#runObjects()]) + } + + announce(namespace: string) { + return this.#publisher.announce(namespace) + } + + announced() { + return this.#subscriber.announced() + } + + subscribe(namespace: string, track: string) { + return this.#subscriber.subscribe(namespace, track) + } + + subscribed() { + return this.#publisher.subscribed() + } + + async #runControl() { + // Receive messages until the connection is closed. + for (;;) { + const msg = await this.#control.recv() + await this.#recv(msg) + } + } + + async #runObjects() { + for (;;) { + const obj = await this.#objects.recv() + if (!obj) break + + await this.#subscriber.recvObject(obj) + } + } + + async #recv(msg: Control.Message) { + if (Control.isPublisher(msg)) { + await this.#subscriber.recv(msg) + } else { + await this.#publisher.recv(msg) + } + } + + async closed(): Promise { + try { + await this.#running + return new Error("closed") + } catch (e) { + return asError(e) + } + } +} diff --git a/packages/moq/transport/control.ts b/packages/moq/transport/control.ts new file mode 100644 index 0000000..e4cd93d --- /dev/null +++ b/packages/moq/transport/control.ts @@ -0,0 +1,550 @@ +import { Reader, Writer } from "./stream" + +export type Message = Subscriber | Publisher + +// Sent by subscriber +export type Subscriber = Subscribe | Unsubscribe | AnnounceOk | AnnounceError + +export function isSubscriber(m: Message): m is Subscriber { + return ( + m.kind == Msg.Subscribe || m.kind == Msg.Unsubscribe || m.kind == Msg.AnnounceOk || m.kind == Msg.AnnounceError + ) +} + +// Sent by publisher +export type Publisher = SubscribeOk | SubscribeError | SubscribeDone | Announce | Unannounce + +export function isPublisher(m: Message): m is Publisher { + return ( + m.kind == Msg.SubscribeOk || + m.kind == Msg.SubscribeError || + m.kind == Msg.SubscribeDone || + m.kind == Msg.Announce || + m.kind == Msg.Unannounce + ) +} + +// I wish we didn't have to split Msg and Id into separate enums. +// However using the string in the message makes it easier to debug. +// We'll take the tiny performance hit until I'm better at Typescript. +export enum Msg { + // NOTE: object and setup are in other modules + Subscribe = "subscribe", + SubscribeOk = "subscribe_ok", + SubscribeError = "subscribe_error", + SubscribeDone = "subscribe_done", + Unsubscribe = "unsubscribe", + Announce = "announce", + AnnounceOk = "announce_ok", + AnnounceError = "announce_error", + Unannounce = "unannounce", + GoAway = "go_away", +} + +enum Id { + // NOTE: object and setup are in other modules + // Object = 0, + // Setup = 1, + + Subscribe = 0x3, + SubscribeOk = 0x4, + SubscribeError = 0x5, + SubscribeDone = 0xb, + Unsubscribe = 0xa, + Announce = 0x6, + AnnounceOk = 0x7, + AnnounceError = 0x8, + Unannounce = 0x9, + GoAway = 0x10, +} + +export interface Subscribe { + kind: Msg.Subscribe + + id: bigint + trackId: bigint + namespace: string + name: string + + location: Location + + params?: Parameters +} + +export type Location = LatestGroup | LatestObject | AbsoluteStart | AbsoluteRange + +export interface LatestGroup { + mode: "latest_group" +} + +export interface LatestObject { + mode: "latest_object" +} + +export interface AbsoluteStart { + mode: "absolute_start" + start_group: number + start_object: number +} + +export interface AbsoluteRange { + mode: "absolute_range" + start_group: number + start_object: number + end_group: number + end_object: number +} + +export type Parameters = Map + +export interface SubscribeOk { + kind: Msg.SubscribeOk + id: bigint + expires: bigint + latest?: [number, number] +} + +export interface SubscribeDone { + kind: Msg.SubscribeDone + id: bigint + code: bigint + reason: string + final?: [number, number] +} + +export interface SubscribeError { + kind: Msg.SubscribeError + id: bigint + code: bigint + reason: string +} + +export interface Unsubscribe { + kind: Msg.Unsubscribe + id: bigint +} + +export interface Announce { + kind: Msg.Announce + namespace: string + params?: Parameters +} + +export interface AnnounceOk { + kind: Msg.AnnounceOk + namespace: string +} + +export interface AnnounceError { + kind: Msg.AnnounceError + namespace: string + code: bigint + reason: string +} + +export interface Unannounce { + kind: Msg.Unannounce + namespace: string +} + +export class Stream { + private decoder: Decoder + private encoder: Encoder + + #mutex = Promise.resolve() + + constructor(r: Reader, w: Writer) { + this.decoder = new Decoder(r) + this.encoder = new Encoder(w) + } + + // Will error if two messages are read at once. + async recv(): Promise { + const msg = await this.decoder.message() + console.log("received message", msg) + return msg + } + + async send(msg: Message) { + const unlock = await this.#lock() + try { + console.log("sending message", msg) + await this.encoder.message(msg) + } finally { + unlock() + } + } + + async #lock() { + // Make a new promise that we can resolve later. + let done: () => void + const p = new Promise((resolve) => { + done = () => resolve() + }) + + // Wait until the previous lock is done, then resolve our our lock. + const lock = this.#mutex.then(() => done) + + // Save our lock as the next lock. + this.#mutex = p + + // Return the lock. + return lock + } +} + +export class Decoder { + r: Reader + + constructor(r: Reader) { + this.r = r + } + + private async msg(): Promise { + const t = await this.r.u53() + switch (t) { + case Id.Subscribe: + return Msg.Subscribe + case Id.SubscribeOk: + return Msg.SubscribeOk + case Id.SubscribeDone: + return Msg.SubscribeDone + case Id.SubscribeError: + return Msg.SubscribeError + case Id.Unsubscribe: + return Msg.Unsubscribe + case Id.Announce: + return Msg.Announce + case Id.AnnounceOk: + return Msg.AnnounceOk + case Id.AnnounceError: + return Msg.AnnounceError + case Id.Unannounce: + return Msg.Unannounce + case Id.GoAway: + return Msg.GoAway + } + + throw new Error(`unknown control message type: ${t}`) + } + + async message(): Promise { + const t = await this.msg() + switch (t) { + case Msg.Subscribe: + return this.subscribe() + case Msg.SubscribeOk: + return this.subscribe_ok() + case Msg.SubscribeError: + return this.subscribe_error() + case Msg.SubscribeDone: + return this.subscribe_done() + case Msg.Unsubscribe: + return this.unsubscribe() + case Msg.Announce: + return this.announce() + case Msg.AnnounceOk: + return this.announce_ok() + case Msg.Unannounce: + return this.unannounce() + case Msg.AnnounceError: + return this.announce_error() + case Msg.GoAway: + throw new Error("TODO: implement go away") + } + } + + private async subscribe(): Promise { + return { + kind: Msg.Subscribe, + id: await this.r.u62(), + trackId: await this.r.u62(), + namespace: await this.r.string(), + name: await this.r.string(), + location: await this.location(), + params: await this.parameters(), + } + } + + private async location(): Promise { + const mode = await this.r.u62() + if (mode == 1n) { + return { + mode: "latest_group", + } + } else if (mode == 2n) { + return { + mode: "latest_object", + } + } else if (mode == 3n) { + return { + mode: "absolute_start", + start_group: await this.r.u53(), + start_object: await this.r.u53(), + } + } else if (mode == 4n) { + return { + mode: "absolute_range", + start_group: await this.r.u53(), + start_object: await this.r.u53(), + end_group: await this.r.u53(), + end_object: await this.r.u53(), + } + } else { + throw new Error(`invalid filter type: ${mode}`) + } + } + + private async parameters(): Promise { + const count = await this.r.u53() + if (count == 0) return undefined + + const params = new Map() + + for (let i = 0; i < count; i++) { + const id = await this.r.u62() + const size = await this.r.u53() + const value = await this.r.read(size) + + if (params.has(id)) { + throw new Error(`duplicate parameter id: ${id}`) + } + + params.set(id, value) + } + + return params + } + + private async subscribe_ok(): Promise { + const id = await this.r.u62() + const expires = await this.r.u62() + + let latest: [number, number] | undefined + + const flag = await this.r.u8() + if (flag === 1) { + latest = [await this.r.u53(), await this.r.u53()] + } else if (flag !== 0) { + throw new Error(`invalid final flag: ${flag}`) + } + + return { + kind: Msg.SubscribeOk, + id, + expires, + latest, + } + } + + private async subscribe_done(): Promise { + const id = await this.r.u62() + const code = await this.r.u62() + const reason = await this.r.string() + + let final: [number, number] | undefined + + const flag = await this.r.u8() + if (flag === 1) { + final = [await this.r.u53(), await this.r.u53()] + } else if (flag !== 0) { + throw new Error(`invalid final flag: ${flag}`) + } + + return { + kind: Msg.SubscribeDone, + id, + code, + reason, + final, + } + } + + private async subscribe_error(): Promise { + return { + kind: Msg.SubscribeError, + id: await this.r.u62(), + code: await this.r.u62(), + reason: await this.r.string(), + } + } + + private async unsubscribe(): Promise { + return { + kind: Msg.Unsubscribe, + id: await this.r.u62(), + } + } + + private async announce(): Promise { + const namespace = await this.r.string() + + return { + kind: Msg.Announce, + namespace, + params: await this.parameters(), + } + } + + private async announce_ok(): Promise { + return { + kind: Msg.AnnounceOk, + namespace: await this.r.string(), + } + } + + private async announce_error(): Promise { + return { + kind: Msg.AnnounceError, + namespace: await this.r.string(), + code: await this.r.u62(), + reason: await this.r.string(), + } + } + + private async unannounce(): Promise { + return { + kind: Msg.Unannounce, + namespace: await this.r.string(), + } + } +} + +export class Encoder { + w: Writer + + constructor(w: Writer) { + this.w = w + } + + async message(m: Message) { + switch (m.kind) { + case Msg.Subscribe: + return this.subscribe(m) + case Msg.SubscribeOk: + return this.subscribe_ok(m) + case Msg.SubscribeError: + return this.subscribe_error(m) + case Msg.SubscribeDone: + return this.subscribe_done(m) + case Msg.Unsubscribe: + return this.unsubscribe(m) + case Msg.Announce: + return this.announce(m) + case Msg.AnnounceOk: + return this.announce_ok(m) + case Msg.AnnounceError: + return this.announce_error(m) + case Msg.Unannounce: + return this.unannounce(m) + } + } + + async subscribe(s: Subscribe) { + await this.w.u53(Id.Subscribe) + await this.w.u62(s.id) + await this.w.u62(s.trackId) + await this.w.string(s.namespace) + await this.w.string(s.name) + await this.location(s.location) + await this.parameters(s.params) + } + + private async location(l: Location) { + switch (l.mode) { + case "latest_group": + await this.w.u62(1n) + break + case "latest_object": + await this.w.u62(2n) + break + case "absolute_start": + await this.w.u62(3n) + await this.w.u53(l.start_group) + await this.w.u53(l.start_object) + break + case "absolute_range": + await this.w.u62(3n) + await this.w.u53(l.start_group) + await this.w.u53(l.start_object) + await this.w.u53(l.end_group) + await this.w.u53(l.end_object) + } + } + + private async parameters(p: Parameters | undefined) { + if (!p) { + await this.w.u8(0) + return + } + + await this.w.u53(p.size) + for (const [id, value] of p) { + await this.w.u62(id) + await this.w.u53(value.length) + await this.w.write(value) + } + } + + async subscribe_ok(s: SubscribeOk) { + await this.w.u53(Id.SubscribeOk) + await this.w.u62(s.id) + await this.w.u62(s.expires) + + if (s.latest !== undefined) { + await this.w.u8(1) + await this.w.u53(s.latest[0]) + await this.w.u53(s.latest[1]) + } else { + await this.w.u8(0) + } + } + + async subscribe_done(s: SubscribeDone) { + await this.w.u53(Id.SubscribeDone) + await this.w.u62(s.id) + await this.w.u62(s.code) + await this.w.string(s.reason) + + if (s.final !== undefined) { + await this.w.u8(1) + await this.w.u53(s.final[0]) + await this.w.u53(s.final[1]) + } else { + await this.w.u8(0) + } + } + + async subscribe_error(s: SubscribeError) { + await this.w.u53(Id.SubscribeError) + await this.w.u62(s.id) + } + + async unsubscribe(s: Unsubscribe) { + await this.w.u53(Id.Unsubscribe) + await this.w.u62(s.id) + } + + async announce(a: Announce) { + await this.w.u53(Id.Announce) + await this.w.string(a.namespace) + await this.w.u53(0) // parameters + } + + async announce_ok(a: AnnounceOk) { + await this.w.u53(Id.AnnounceOk) + await this.w.string(a.namespace) + } + + async announce_error(a: AnnounceError) { + await this.w.u53(Id.AnnounceError) + await this.w.string(a.namespace) + await this.w.u62(a.code) + await this.w.string(a.reason) + } + + async unannounce(a: Unannounce) { + await this.w.u53(Id.Unannounce) + await this.w.string(a.namespace) + } +} diff --git a/packages/moq/transport/index.ts b/packages/moq/transport/index.ts new file mode 100644 index 0000000..ca86a6c --- /dev/null +++ b/packages/moq/transport/index.ts @@ -0,0 +1,7 @@ +export { Client } from "./client" +export type { ClientConfig } from "./client" + +export { Connection } from "./connection" + +export { SubscribeRecv, AnnounceSend } from "./publisher" +export { AnnounceRecv, SubscribeSend } from "./subscriber" diff --git a/packages/moq/transport/objects.ts b/packages/moq/transport/objects.ts new file mode 100644 index 0000000..05c9ed4 --- /dev/null +++ b/packages/moq/transport/objects.ts @@ -0,0 +1,307 @@ +import { Reader, Writer } from "./stream" +export { Reader, Writer } + +export enum StreamType { + Object = 0x0, + Track = 0x50, + Group = 0x51, +} + +export enum Status { + OBJECT_NULL = 1, + GROUP_NULL = 2, + GROUP_END = 3, + TRACK_END = 4, +} + +export interface TrackHeader { + type: StreamType.Track + sub: bigint + track: bigint + priority: number // VarInt with a u32 maximum value +} + +export interface TrackChunk { + group: number // The group sequence, as a number because 2^53 is enough. + object: number + payload: Uint8Array | Status +} + +export interface GroupHeader { + type: StreamType.Group + sub: bigint + track: bigint + group: number // The group sequence, as a number because 2^53 is enough. + priority: number // VarInt with a u32 maximum value +} + +export interface GroupChunk { + object: number + payload: Uint8Array | Status +} + +export interface ObjectHeader { + type: StreamType.Object + sub: bigint + track: bigint + group: number + object: number + priority: number + status: number +} + +export interface ObjectChunk { + payload: Uint8Array +} + +type WriterType = T extends TrackHeader + ? TrackWriter + : T extends GroupHeader + ? GroupWriter + : T extends ObjectHeader + ? ObjectWriter + : never + +export class Objects { + private quic: WebTransport + + constructor(quic: WebTransport) { + this.quic = quic + } + + async send(h: T): Promise> { + const stream = await this.quic.createUnidirectionalStream() + const w = new Writer(stream) + + await w.u53(h.type) + await w.u62(h.sub) + await w.u62(h.track) + + let res: WriterType + + if (h.type == StreamType.Object) { + await w.u53(h.group) + await w.u53(h.object) + await w.u53(h.priority) + await w.u53(h.status) + + res = new ObjectWriter(h, w) as WriterType + } else if (h.type === StreamType.Group) { + await w.u53(h.group) + await w.u53(h.priority) + + res = new GroupWriter(h, w) as WriterType + } else if (h.type === StreamType.Track) { + await w.u53(h.priority) + + res = new TrackWriter(h, w) as WriterType + } else { + throw new Error("unknown header type") + } + + // console.trace("send object", res.header) + + return res + } + + async recv(): Promise { + const streams = this.quic.incomingUnidirectionalStreams.getReader() + + const { value, done } = await streams.read() + streams.releaseLock() + + if (done) return + + const r = new Reader(new Uint8Array(), value) + const type = (await r.u53()) as StreamType + let res: TrackReader | GroupReader | ObjectReader + + if (type == StreamType.Track) { + const h: TrackHeader = { + type, + sub: await r.u62(), + track: await r.u62(), + priority: await r.u53(), + } + + res = new TrackReader(h, r) + } else if (type == StreamType.Group) { + const h: GroupHeader = { + type, + sub: await r.u62(), + track: await r.u62(), + group: await r.u53(), + priority: await r.u53(), + } + res = new GroupReader(h, r) + } else if (type == StreamType.Object) { + const h = { + type, + sub: await r.u62(), + track: await r.u62(), + group: await r.u53(), + object: await r.u53(), + status: await r.u53(), + priority: await r.u53(), + } + + res = new ObjectReader(h, r) + } else { + throw new Error("unknown stream type") + } + + // console.trace("receive object", res.header) + + return res + } +} + +export class TrackWriter { + constructor( + public header: TrackHeader, + public stream: Writer, + ) {} + + async write(c: TrackChunk) { + await this.stream.u53(c.group) + await this.stream.u53(c.object) + + if (c.payload instanceof Uint8Array) { + await this.stream.u53(c.payload.byteLength) + await this.stream.write(c.payload) + } else { + // empty payload with status + await this.stream.u53(0) + await this.stream.u53(c.payload as number) + } + } + + async close() { + await this.stream.close() + } +} + +export class GroupWriter { + constructor( + public header: GroupHeader, + public stream: Writer, + ) {} + + async write(c: GroupChunk) { + await this.stream.u53(c.object) + if (c.payload instanceof Uint8Array) { + await this.stream.u53(c.payload.byteLength) + await this.stream.write(c.payload) + } else { + await this.stream.u53(0) + await this.stream.u53(c.payload as number) + } + } + + async close() { + await this.stream.close() + } +} + +export class ObjectWriter { + constructor( + public header: ObjectHeader, + public stream: Writer, + ) {} + + async write(c: ObjectChunk) { + await this.stream.write(c.payload) + } + + async close() { + await this.stream.close() + } +} + +export class TrackReader { + constructor( + public header: TrackHeader, + public stream: Reader, + ) {} + + async read(): Promise { + if (await this.stream.done()) { + return + } + + const group = await this.stream.u53() + const object = await this.stream.u53() + const size = await this.stream.u53() + + let payload + if (size == 0) { + payload = (await this.stream.u53()) as Status + } else { + payload = await this.stream.read(size) + } + + return { + group, + object, + payload, + } + } + + async close() { + await this.stream.close() + } +} + +export class GroupReader { + constructor( + public header: GroupHeader, + public stream: Reader, + ) {} + + async read(): Promise { + if (await this.stream.done()) { + return + } + + const object = await this.stream.u53() + const size = await this.stream.u53() + + let payload + if (size == 0) { + payload = (await this.stream.u53()) as Status + } else { + payload = await this.stream.read(size) + } + + return { + object, + payload, + } + } + + async close() { + await this.stream.close() + } +} + +export class ObjectReader { + constructor( + public header: ObjectHeader, + public stream: Reader, + ) {} + + // NOTE: Can only be called once. + async read(): Promise { + if (await this.stream.done()) { + return + } + + return { + payload: await this.stream.readAll(), + } + } + + async close() { + await this.stream.close() + } +} diff --git a/packages/moq/transport/publisher.ts b/packages/moq/transport/publisher.ts new file mode 100644 index 0000000..547344d --- /dev/null +++ b/packages/moq/transport/publisher.ts @@ -0,0 +1,230 @@ +import * as Control from "./control" +import { Queue, Watch } from "../common/async" +import { Objects, GroupWriter, ObjectWriter, StreamType, TrackWriter } from "./objects" + +export class Publisher { + // Used to send control messages + #control: Control.Stream + + // Use to send objects. + #objects: Objects + + // Our announced tracks. + #announce = new Map() + + // Their subscribed tracks. + #subscribe = new Map() + #subscribeQueue = new Queue(Number.MAX_SAFE_INTEGER) // Unbounded queue in case there's no receiver + + constructor(control: Control.Stream, objects: Objects) { + this.#control = control + this.#objects = objects + } + + // Announce a track namespace. + async announce(namespace: string): Promise { + if (this.#announce.has(namespace)) { + throw new Error(`already announce: ${namespace}`) + } + + const announce = new AnnounceSend(this.#control, namespace) + this.#announce.set(namespace, announce) + + await this.#control.send({ + kind: Control.Msg.Announce, + namespace, + }) + + return announce + } + + // Receive the next new subscription + async subscribed() { + return await this.#subscribeQueue.next() + } + + async recv(msg: Control.Subscriber) { + if (msg.kind == Control.Msg.Subscribe) { + await this.recvSubscribe(msg) + } else if (msg.kind == Control.Msg.Unsubscribe) { + this.recvUnsubscribe(msg) + } else if (msg.kind == Control.Msg.AnnounceOk) { + this.recvAnnounceOk(msg) + } else if (msg.kind == Control.Msg.AnnounceError) { + this.recvAnnounceError(msg) + } else { + throw new Error(`unknown control message`) // impossible + } + } + + recvAnnounceOk(msg: Control.AnnounceOk) { + const announce = this.#announce.get(msg.namespace) + if (!announce) { + throw new Error(`announce OK for unknown announce: ${msg.namespace}`) + } + + announce.onOk() + } + + recvAnnounceError(msg: Control.AnnounceError) { + const announce = this.#announce.get(msg.namespace) + if (!announce) { + // TODO debug this + console.warn(`announce error for unknown announce: ${msg.namespace}`) + return + } + + announce.onError(msg.code, msg.reason) + } + + async recvSubscribe(msg: Control.Subscribe) { + if (this.#subscribe.has(msg.id)) { + throw new Error(`duplicate subscribe for id: ${msg.id}`) + } + + const subscribe = new SubscribeRecv(this.#control, this.#objects, msg) + this.#subscribe.set(msg.id, subscribe) + await this.#subscribeQueue.push(subscribe) + + await this.#control.send({ kind: Control.Msg.SubscribeOk, id: msg.id, expires: 0n }) + } + + recvUnsubscribe(_msg: Control.Unsubscribe) { + throw new Error("TODO unsubscribe") + } +} + +export class AnnounceSend { + #control: Control.Stream + + readonly namespace: string + + // The current state, updated by control messages. + #state = new Watch<"init" | "ack" | Error>("init") + + constructor(control: Control.Stream, namespace: string) { + this.#control = control + this.namespace = namespace + } + + async ok() { + for (;;) { + const [state, next] = this.#state.value() + if (state === "ack") return + if (state instanceof Error) throw state + if (!next) throw new Error("closed") + + await next + } + } + + async active() { + for (;;) { + const [state, next] = this.#state.value() + if (state instanceof Error) throw state + if (!next) return + + await next + } + } + + async close() { + // TODO implement unsubscribe + // await this.#inner.sendUnsubscribe() + } + + closed() { + const [state, next] = this.#state.value() + return state instanceof Error || next == undefined + } + + onOk() { + if (this.closed()) return + this.#state.update("ack") + } + + onError(code: bigint, reason: string) { + if (this.closed()) return + + const err = new Error(`ANNOUNCE_ERROR (${code})` + reason ? `: ${reason}` : "") + this.#state.update(err) + } +} + +export class SubscribeRecv { + #control: Control.Stream + #objects: Objects + #id: bigint + #trackId: bigint + + readonly namespace: string + readonly track: string + + // The current state of the subscription. + #state: "init" | "ack" | "closed" = "init" + + constructor(control: Control.Stream, objects: Objects, msg: Control.Subscribe) { + this.#control = control // so we can send messages + this.#objects = objects // so we can send objects + this.#id = msg.id + this.#trackId = msg.trackId + this.namespace = msg.namespace + this.track = msg.name + } + + // Acknowledge the subscription as valid. + async ack() { + if (this.#state !== "init") return + this.#state = "ack" + + // Send the control message. + return this.#control.send({ kind: Control.Msg.SubscribeOk, id: this.#id, expires: 0n }) + } + + // Close the subscription with an error. + async close(code = 0n, reason = "") { + if (this.#state === "closed") return + this.#state = "closed" + + return this.#control.send({ + kind: Control.Msg.SubscribeDone, + id: this.#id, + code, + reason, + }) + } + + // Create a writable data stream for the entire track + async serve(props?: { priority: number }): Promise { + return this.#objects.send({ + type: StreamType.Track, + sub: this.#id, + track: this.#trackId, + priority: props?.priority ?? 0, + }) + } + + // Create a writable data stream for a group within the track + async group(props: { group: number; priority?: number }): Promise { + return this.#objects.send({ + type: StreamType.Group, + sub: this.#id, + track: this.#trackId, + group: props.group, + priority: props.priority ?? 0, + }) + } + + // Create a writable data stream for a single object within the track + async object(props: { group: number; object: number; priority?: number }): Promise { + return this.#objects.send({ + type: StreamType.Object, + sub: this.#id, + track: this.#trackId, + group: props.group, + object: props.object, + priority: props.priority ?? 0, + status: 0, + }) + } +} diff --git a/packages/moq/transport/setup.ts b/packages/moq/transport/setup.ts new file mode 100644 index 0000000..ac81392 --- /dev/null +++ b/packages/moq/transport/setup.ts @@ -0,0 +1,163 @@ +import { Reader, Writer } from "./stream" + +export type Message = Client | Server +export type Role = "publisher" | "subscriber" | "both" + +export enum Version { + DRAFT_00 = 0xff000000, + DRAFT_01 = 0xff000001, + DRAFT_02 = 0xff000002, + DRAFT_03 = 0xff000003, + DRAFT_04 = 0xff000004, + KIXEL_00 = 0xbad00, + KIXEL_01 = 0xbad01, +} + +// NOTE: These are forked from moq-transport-00. +// 1. messages lack a sized length +// 2. parameters are not optional and written in order (role + path) +// 3. role indicates local support only, not remote support + +export interface Client { + versions: Version[] + role: Role + params?: Parameters +} + +export interface Server { + version: Version + params?: Parameters +} + +export class Stream { + recv: Decoder + send: Encoder + + constructor(r: Reader, w: Writer) { + this.recv = new Decoder(r) + this.send = new Encoder(w) + } +} + +export type Parameters = Map + +export class Decoder { + r: Reader + + constructor(r: Reader) { + this.r = r + } + + async client(): Promise { + const type = await this.r.u53() + if (type !== 0x40) throw new Error(`client SETUP type must be 0x40, got ${type}`) + + const count = await this.r.u53() + + const versions = [] + for (let i = 0; i < count; i++) { + const version = await this.r.u53() + versions.push(version) + } + + const params = await this.parameters() + const role = this.role(params?.get(0n)) + + return { + versions, + role, + params, + } + } + + async server(): Promise { + const type = await this.r.u53() + if (type !== 0x41) throw new Error(`server SETUP type must be 0x41, got ${type}`) + + const version = await this.r.u53() + const params = await this.parameters() + + return { + version, + params, + } + } + + private async parameters(): Promise { + const count = await this.r.u53() + if (count == 0) return undefined + + const params = new Map() + + for (let i = 0; i < count; i++) { + const id = await this.r.u62() + const size = await this.r.u53() + const value = await this.r.read(size) + + if (params.has(id)) { + throw new Error(`duplicate parameter id: ${id}`) + } + + params.set(id, value) + } + + return params + } + + role(raw: Uint8Array | undefined): Role { + if (!raw) throw new Error("missing role parameter") + if (raw.length != 1) throw new Error("multi-byte varint not supported") + + switch (raw[0]) { + case 1: + return "publisher" + case 2: + return "subscriber" + case 3: + return "both" + default: + throw new Error(`invalid role: ${raw[0]}`) + } + } +} + +export class Encoder { + w: Writer + + constructor(w: Writer) { + this.w = w + } + + async client(c: Client) { + await this.w.u53(0x40) + await this.w.u53(c.versions.length) + for (const v of c.versions) { + await this.w.u53(v) + } + + // I hate it + const params = c.params ?? new Map() + params.set(0n, new Uint8Array([c.role == "publisher" ? 1 : c.role == "subscriber" ? 2 : 3])) + await this.parameters(params) + } + + async server(s: Server) { + await this.w.u53(0x41) + await this.w.u53(s.version) + await this.parameters(s.params) + } + + private async parameters(p: Parameters | undefined) { + if (!p) { + await this.w.u8(0) + return + } + + await this.w.u53(p.size) + for (const [id, value] of p) { + await this.w.u62(id) + await this.w.u53(value.length) + await this.w.write(value) + } + } +} diff --git a/packages/moq/transport/stream.ts b/packages/moq/transport/stream.ts new file mode 100644 index 0000000..e8ad59c --- /dev/null +++ b/packages/moq/transport/stream.ts @@ -0,0 +1,270 @@ +const MAX_U6 = Math.pow(2, 6) - 1 +const MAX_U14 = Math.pow(2, 14) - 1 +const MAX_U30 = Math.pow(2, 30) - 1 +const MAX_U31 = Math.pow(2, 31) - 1 +const MAX_U53 = Number.MAX_SAFE_INTEGER +const MAX_U62: bigint = 2n ** 62n - 1n + +// Reader wraps a stream and provides convience methods for reading pieces from a stream +// Unfortunately we can't use a BYOB reader because it's not supported with WebTransport+WebWorkers yet. +export class Reader { + #buffer: Uint8Array + #stream: ReadableStream + #reader: ReadableStreamDefaultReader + + constructor(buffer: Uint8Array, stream: ReadableStream) { + this.#buffer = buffer + this.#stream = stream + this.#reader = this.#stream.getReader() + } + + // Adds more data to the buffer, returning true if more data was added. + async #fill(): Promise { + const result = await this.#reader.read() + if (result.done) { + return false + } + + const buffer = new Uint8Array(result.value) + + if (this.#buffer.byteLength == 0) { + this.#buffer = buffer + } else { + const temp = new Uint8Array(this.#buffer.byteLength + buffer.byteLength) + temp.set(this.#buffer) + temp.set(buffer, this.#buffer.byteLength) + this.#buffer = temp + } + + return true + } + + // Add more data to the buffer until it's at least size bytes. + async #fillTo(size: number) { + while (this.#buffer.byteLength < size) { + if (!(await this.#fill())) { + throw new Error("unexpected end of stream") + } + } + } + + // Consumes the first size bytes of the buffer. + #slice(size: number): Uint8Array { + const result = new Uint8Array(this.#buffer.buffer, this.#buffer.byteOffset, size) + this.#buffer = new Uint8Array(this.#buffer.buffer, this.#buffer.byteOffset + size) + + return result + } + + async read(size: number): Promise { + if (size == 0) return new Uint8Array() + + await this.#fillTo(size) + return this.#slice(size) + } + + async readAll(): Promise { + // eslint-disable-next-line no-empty + while (await this.#fill()) {} + return this.#slice(this.#buffer.byteLength) + } + + async string(maxLength?: number): Promise { + const length = await this.u53() + if (maxLength !== undefined && length > maxLength) { + throw new Error(`string length ${length} exceeds max length ${maxLength}`) + } + + const buffer = await this.read(length) + return new TextDecoder().decode(buffer) + } + + async u8(): Promise { + await this.#fillTo(1) + return this.#slice(1)[0] + } + + // Returns a Number using 53-bits, the max Javascript can use for integer math + async u53(): Promise { + const v = await this.u62() + if (v > MAX_U53) { + throw new Error("value larger than 53-bits; use v62 instead") + } + + return Number(v) + } + + // NOTE: Returns a bigint instead of a number since it may be larger than 53-bits + async u62(): Promise { + await this.#fillTo(1) + const size = (this.#buffer[0] & 0xc0) >> 6 + + if (size == 0) { + const first = this.#slice(1)[0] + return BigInt(first) & 0x3fn + } else if (size == 1) { + await this.#fillTo(2) + const slice = this.#slice(2) + const view = new DataView(slice.buffer, slice.byteOffset, slice.byteLength) + + return BigInt(view.getInt16(0)) & 0x3fffn + } else if (size == 2) { + await this.#fillTo(4) + const slice = this.#slice(4) + const view = new DataView(slice.buffer, slice.byteOffset, slice.byteLength) + + return BigInt(view.getUint32(0)) & 0x3fffffffn + } else if (size == 3) { + await this.#fillTo(8) + const slice = this.#slice(8) + const view = new DataView(slice.buffer, slice.byteOffset, slice.byteLength) + + return view.getBigUint64(0) & 0x3fffffffffffffffn + } else { + throw new Error("impossible") + } + } + + async done(): Promise { + if (this.#buffer.byteLength > 0) return false + return !(await this.#fill()) + } + + async close() { + this.#reader.releaseLock() + await this.#stream.cancel() + } + + release(): [Uint8Array, ReadableStream] { + this.#reader.releaseLock() + return [this.#buffer, this.#stream] + } +} + +// Writer wraps a stream and writes chunks of data +export class Writer { + #scratch: Uint8Array + #writer: WritableStreamDefaultWriter + #stream: WritableStream + + constructor(stream: WritableStream) { + this.#stream = stream + this.#scratch = new Uint8Array(8) + this.#writer = this.#stream.getWriter() + } + + async u8(v: number) { + await this.write(setUint8(this.#scratch, v)) + } + + async i32(v: number) { + if (Math.abs(v) > MAX_U31) { + throw new Error(`overflow, value larger than 32-bits: ${v}`) + } + + // We don't use a VarInt, so it always takes 4 bytes. + // This could be improved but nothing is standardized yet. + await this.write(setInt32(this.#scratch, v)) + } + + async u53(v: number) { + if (v < 0) { + throw new Error(`underflow, value is negative: ${v}`) + } else if (v > MAX_U53) { + throw new Error(`overflow, value larger than 53-bits: ${v}`) + } + + await this.write(setVint53(this.#scratch, v)) + } + + async u62(v: bigint) { + if (v < 0) { + throw new Error(`underflow, value is negative: ${v}`) + } else if (v >= MAX_U62) { + throw new Error(`overflow, value larger than 62-bits: ${v}`) + } + + await this.write(setVint62(this.#scratch, v)) + } + + async write(v: Uint8Array) { + await this.#writer.write(v) + } + + async string(str: string) { + const data = new TextEncoder().encode(str) + await this.u53(data.byteLength) + await this.write(data) + } + + async close() { + this.#writer.releaseLock() + await this.#stream.close() + } + + release(): WritableStream { + this.#writer.releaseLock() + return this.#stream + } +} + +function setUint8(dst: Uint8Array, v: number): Uint8Array { + dst[0] = v + return dst.slice(0, 1) +} + +function setUint16(dst: Uint8Array, v: number): Uint8Array { + const view = new DataView(dst.buffer, dst.byteOffset, 2) + view.setUint16(0, v) + + return new Uint8Array(view.buffer, view.byteOffset, view.byteLength) +} + +function setInt32(dst: Uint8Array, v: number): Uint8Array { + const view = new DataView(dst.buffer, dst.byteOffset, 4) + view.setInt32(0, v) + + return new Uint8Array(view.buffer, view.byteOffset, view.byteLength) +} + +function setUint32(dst: Uint8Array, v: number): Uint8Array { + const view = new DataView(dst.buffer, dst.byteOffset, 4) + view.setUint32(0, v) + + return new Uint8Array(view.buffer, view.byteOffset, view.byteLength) +} + +function setVint53(dst: Uint8Array, v: number): Uint8Array { + if (v <= MAX_U6) { + return setUint8(dst, v) + } else if (v <= MAX_U14) { + return setUint16(dst, v | 0x4000) + } else if (v <= MAX_U30) { + return setUint32(dst, v | 0x80000000) + } else if (v <= MAX_U53) { + return setUint64(dst, BigInt(v) | 0xc000000000000000n) + } else { + throw new Error(`overflow, value larger than 53-bits: ${v}`) + } +} + +function setVint62(dst: Uint8Array, v: bigint): Uint8Array { + if (v < MAX_U6) { + return setUint8(dst, Number(v)) + } else if (v < MAX_U14) { + return setUint16(dst, Number(v) | 0x4000) + } else if (v <= MAX_U30) { + return setUint32(dst, Number(v) | 0x80000000) + } else if (v <= MAX_U62) { + return setUint64(dst, BigInt(v) | 0xc000000000000000n) + } else { + throw new Error(`overflow, value larger than 62-bits: ${v}`) + } +} + +function setUint64(dst: Uint8Array, v: bigint): Uint8Array { + const view = new DataView(dst.buffer, dst.byteOffset, 8) + view.setBigUint64(0, v) + + return new Uint8Array(view.buffer, view.byteOffset, view.byteLength) +} diff --git a/packages/moq/transport/subscriber.ts b/packages/moq/transport/subscriber.ts new file mode 100644 index 0000000..bd1db71 --- /dev/null +++ b/packages/moq/transport/subscriber.ts @@ -0,0 +1,197 @@ +import * as Control from "./control" +import { Queue, Watch } from "../common/async" +import { Objects } from "./objects" +import type { TrackReader, GroupReader, ObjectReader } from "./objects" + +export class Subscriber { + // Use to send control messages. + #control: Control.Stream + + // Use to send objects. + #objects: Objects + + // Announced broadcasts. + #announce = new Map() + #announceQueue = new Watch([]) + + // Our subscribed tracks. + #subscribe = new Map() + #subscribeNext = 0n + + constructor(control: Control.Stream, objects: Objects) { + this.#control = control + this.#objects = objects + } + + announced(): Watch { + return this.#announceQueue + } + + async recv(msg: Control.Publisher) { + if (msg.kind == Control.Msg.Announce) { + await this.recvAnnounce(msg) + } else if (msg.kind == Control.Msg.Unannounce) { + this.recvUnannounce(msg) + } else if (msg.kind == Control.Msg.SubscribeOk) { + this.recvSubscribeOk(msg) + } else if (msg.kind == Control.Msg.SubscribeError) { + await this.recvSubscribeError(msg) + } else if (msg.kind == Control.Msg.SubscribeDone) { + await this.recvSubscribeDone(msg) + } else { + throw new Error(`unknown control message`) // impossible + } + } + + async recvAnnounce(msg: Control.Announce) { + if (this.#announce.has(msg.namespace)) { + throw new Error(`duplicate announce for namespace: ${msg.namespace}`) + } + + await this.#control.send({ kind: Control.Msg.AnnounceOk, namespace: msg.namespace }) + + const announce = new AnnounceRecv(this.#control, msg.namespace) + this.#announce.set(msg.namespace, announce) + + this.#announceQueue.update((queue) => [...queue, announce]) + } + + recvUnannounce(_msg: Control.Unannounce) { + throw new Error(`TODO Unannounce`) + } + + async subscribe(namespace: string, track: string) { + const id = this.#subscribeNext++ + + const subscribe = new SubscribeSend(this.#control, id, namespace, track) + this.#subscribe.set(id, subscribe) + + await this.#control.send({ + kind: Control.Msg.Subscribe, + id, + trackId: id, + namespace, + name: track, + location: { + mode: "latest_group", + }, + }) + + return subscribe + } + + recvSubscribeOk(msg: Control.SubscribeOk) { + const subscribe = this.#subscribe.get(msg.id) + if (!subscribe) { + throw new Error(`subscribe ok for unknown id: ${msg.id}`) + } + + subscribe.onOk() + } + + async recvSubscribeError(msg: Control.SubscribeError) { + const subscribe = this.#subscribe.get(msg.id) + if (!subscribe) { + throw new Error(`subscribe error for unknown id: ${msg.id}`) + } + + await subscribe.onError(msg.code, msg.reason) + } + + async recvSubscribeDone(msg: Control.SubscribeDone) { + const subscribe = this.#subscribe.get(msg.id) + if (!subscribe) { + throw new Error(`subscribe error for unknown id: ${msg.id}`) + } + + await subscribe.onError(msg.code, msg.reason) + } + + async recvObject(reader: TrackReader | GroupReader | ObjectReader) { + const subscribe = this.#subscribe.get(reader.header.track) + if (!subscribe) { + throw new Error(`data for for unknown track: ${reader.header.track}`) + } + + await subscribe.onData(reader) + } +} + +export class AnnounceRecv { + #control: Control.Stream + + readonly namespace: string + + // The current state of the announce + #state: "init" | "ack" | "closed" = "init" + + constructor(control: Control.Stream, namespace: string) { + this.#control = control // so we can send messages + this.namespace = namespace + } + + // Acknowledge the subscription as valid. + async ok() { + if (this.#state !== "init") return + this.#state = "ack" + + // Send the control message. + return this.#control.send({ kind: Control.Msg.AnnounceOk, namespace: this.namespace }) + } + + async close(code = 0n, reason = "") { + if (this.#state === "closed") return + this.#state = "closed" + + return this.#control.send({ kind: Control.Msg.AnnounceError, namespace: this.namespace, code, reason }) + } +} + +export class SubscribeSend { + #control: Control.Stream + #id: bigint + + readonly namespace: string + readonly track: string + + // A queue of received streams for this subscription. + #data = new Queue() + + constructor(control: Control.Stream, id: bigint, namespace: string, track: string) { + this.#control = control // so we can send messages + this.#id = id + this.namespace = namespace + this.track = track + } + + async close(_code = 0n, _reason = "") { + // TODO implement unsubscribe + // await this.#inner.sendReset(code, reason) + } + + onOk() { + // noop + } + + async onError(code: bigint, reason: string) { + if (code == 0n) { + return await this.#data.close() + } + + if (reason !== "") { + reason = `: ${reason}` + } + + const err = new Error(`SUBSCRIBE_ERROR (${code})${reason}`) + return await this.#data.abort(err) + } + + async onData(reader: TrackReader | GroupReader | ObjectReader) { + if (!this.#data.closed()) await this.#data.push(reader) + } + + // Receive the next a readable data stream + async data() { + return await this.#data.next() + } +} diff --git a/packages/moq/transport/tsconfig.json b/packages/moq/transport/tsconfig.json new file mode 100644 index 0000000..88da906 --- /dev/null +++ b/packages/moq/transport/tsconfig.json @@ -0,0 +1,9 @@ +{ + "extends": "../tsconfig.json", + "include": ["."], + "references": [ + { + "path": "../common" + } + ] +} diff --git a/packages/moq/tsconfig.json b/packages/moq/tsconfig.json new file mode 100644 index 0000000..2ab7124 --- /dev/null +++ b/packages/moq/tsconfig.json @@ -0,0 +1,42 @@ +{ + "files": [], // don't build anything with these settings. + "compilerOptions": { + "target": "es2022", + "module": "es2022", + "moduleResolution": "node", + "rootDir": ".", + "outDir": "./dist", + "declaration": true, + "strict": true, + "composite": true, + "declarationMap": true, + "sourceMap": true, + "isolatedModules": true, + "types": [], // Don't automatically import any @types modules. + "lib": ["es2022", "dom"], + "typeRoots": ["./types", "../node_modules/@types"] + }, + "references": [ + { + "path": "./common" + }, + { + "path": "./playback" + }, + { + "path": "./playback/worklet" + }, + { + "path": "./contribute" + }, + { + "path": "./transport" + }, + { + "path": "./media" + } + ], + "paths": { + "@/*": ["*"] + } +} diff --git a/packages/moq/types/mp4box.d.ts b/packages/moq/types/mp4box.d.ts new file mode 100644 index 0000000..923887a --- /dev/null +++ b/packages/moq/types/mp4box.d.ts @@ -0,0 +1,1848 @@ +// https://github.com/gpac/mp4box.js/issues/233 + +declare module "mp4box" { + export interface MP4MediaTrack { + id: number + created: Date + modified: Date + movie_duration: number + layer: number + alternate_group: number + volume: number + track_width: number + track_height: number + timescale: number + duration: number + bitrate: number + codec: string + language: string + nb_samples: number + } + + export interface MP4VideoData { + width: number + height: number + } + + export interface MP4VideoTrack extends MP4MediaTrack { + video: MP4VideoData + } + + export interface MP4AudioData { + sample_rate: number + channel_count: number + sample_size: number + } + + export interface MP4AudioTrack extends MP4MediaTrack { + audio: MP4AudioData + } + + export type MP4Track = MP4VideoTrack | MP4AudioTrack + + export interface MP4Info { + duration: number + timescale: number + fragment_duration: number + isFragmented: boolean + isProgressive: boolean + hasIOD: boolean + brands: string[] + created: Date + modified: Date + tracks: MP4Track[] + mime: string + audioTracks: MP4AudioTrack[] + videoTracks: MP4VideoTrack[] + } + + export type MP4ArrayBuffer = ArrayBuffer & { fileStart: number } + + export function createFile(): ISOFile + + export interface Sample { + number: number + track_id: number + timescale: number + description_index: number + description: { + avcC?: BoxParser.avcCBox // h.264 + hvcC?: BoxParser.hvcCBox // hevc + vpcC?: BoxParser.vpcCBox // vp9 + av1C?: BoxParser.av1CBox // av1 + } + data: Uint8Array + size: number + alreadyRead?: number + duration: number + cts: number + dts: number + is_sync: boolean + is_leading?: number + depends_on?: number + is_depended_on?: number + has_redundancy?: number + degradation_priority?: number + offset?: number + subsamples?: any + } + + export interface ExtractionOptions { + nbSamples: number + } + + export class DataStream { + // WARNING, the default is little endian, which is not what MP4 uses. + constructor(buffer?: ArrayBuffer, byteOffset?: number, endianness?: boolean) + getPosition(): number + + get byteLength(): number + get buffer(): ArrayBuffer + set buffer(v: ArrayBuffer) + get byteOffset(): number + set byteOffset(v: number) + get dataView(): DataView + set dataView(v: DataView) + + seek(pos: number): void + isEof(): boolean + + mapFloat32Array(length: number, e?: boolean): any + mapFloat64Array(length: number, e?: boolean): any + mapInt16Array(length: number, e?: boolean): any + mapInt32Array(length: number, e?: boolean): any + mapInt8Array(length: number): any + mapUint16Array(length: number, e?: boolean): any + mapUint32Array(length: number, e?: boolean): any + mapUint8Array(length: number): any + + readInt32Array(length: number, endianness?: boolean): Int32Array + readInt16Array(length: number, endianness?: boolean): Int16Array + readInt8Array(length: number): Int8Array + readUint32Array(length: number, endianness?: boolean): Uint32Array + readUint16Array(length: number, endianness?: boolean): Uint16Array + readUint8Array(length: number): Uint8Array + readFloat64Array(length: number, endianness?: boolean): Float64Array + readFloat32Array(length: number, endianness?: boolean): Float32Array + + readInt32(endianness?: boolean): number + readInt16(endianness?: boolean): number + readInt8(): number + readUint32(endianness?: boolean): number + //readUint32Array(length: any, e: any): any + readUint24(): number + readUint16(endianness?: boolean): number + readUint8(): number + //readUint64(): any + readFloat32(endianness?: boolean): number + readFloat64(endianness?: boolean): number + //readCString(length: number): any + //readString(length: number, encoding: any): any + + static endianness: boolean + + memcpy( + dst: ArrayBufferLike, + dstOffset: number, + src: ArrayBufferLike, + srcOffset: number, + byteLength: number, + ): void + + // TODO I got bored porting all functions + + save(filename: string): void + shift(offset: number): void + + writeInt32Array(arr: Int32Array, endianness?: boolean): void + writeInt16Array(arr: Int16Array, endianness?: boolean): void + writeInt8Array(arr: Int8Array): void + writeUint32Array(arr: Uint32Array, endianness?: boolean): void + writeUint16Array(arr: Uint16Array, endianness?: boolean): void + writeUint8Array(arr: Uint8Array): void + writeFloat64Array(arr: Float64Array, endianness?: boolean): void + writeFloat32Array(arr: Float32Array, endianness?: boolean): void + writeInt32(v: number, endianness?: boolean): void + writeInt16(v: number, endianness?: boolean): void + writeInt8(v: number): void + writeUint32(v: number, endianness?: boolean): void + writeUint16(v: number, endianness?: boolean): void + writeUint8(v: number): void + writeFloat32(v: number, endianness?: boolean): void + writeFloat64(v: number, endianness?: boolean): void + writeUCS2String(s: string, endianness?: boolean, length?: number): void + writeString(s: string, encoding?: string, length?: number): void + writeCString(s: string, length?: number): void + writeUint64(v: number): void + writeUint24(v: number): void + adjustUint32(pos: number, v: number): void + + static LITTLE_ENDIAN: boolean + static BIG_ENDIAN: boolean + + // TODO add correct types; these are exported by dts-gen + readCString(length: any): any + readInt64(): any + readString(length: any, encoding: any): any + readUint64(): any + writeStruct(structDefinition: any, struct: any): void + writeType(t: any, v: any, struct: any): any + + static arrayToNative(array: any, arrayIsLittleEndian: any): any + static flipArrayEndianness(array: any): any + static memcpy(dst: any, dstOffset: any, src: any, srcOffset: any, byteLength: any): void + static nativeToEndian(array: any, littleEndian: any): any + } + + export interface TrackOptions { + id?: number + type?: string + width?: number + height?: number + duration?: number + layer?: number + timescale?: number + media_duration?: number + language?: string + hdlr?: string + + // video + avcDecoderConfigRecord?: any + hevcDecoderConfigRecord?: any + + // audio + balance?: number + channel_count?: number + samplesize?: number + samplerate?: number + + //captions + namespace?: string + schema_location?: string + auxiliary_mime_types?: string + + description?: BoxParser.Box + description_boxes?: BoxParser.Box[] + + default_sample_description_index_id?: number + default_sample_duration?: number + default_sample_size?: number + default_sample_flags?: number + } + + export interface FileOptions { + brands?: string[] + timescale?: number + rate?: number + duration?: number + width?: number + } + + export interface SampleOptions { + sample_description_index?: number + duration?: number + cts?: number + dts?: number + is_sync?: boolean + is_leading?: number + depends_on?: number + is_depended_on?: number + has_redundancy?: number + degradation_priority?: number + subsamples?: any + } + + // TODO add the remaining functions + // TODO move to another module + export class ISOFile { + constructor(stream?: DataStream) + + init(options?: FileOptions): ISOFile + addTrack(options?: TrackOptions): number + addSample(track: number, data: Uint8Array, options?: SampleOptions): Sample + + createSingleSampleMoof(sample: Sample): BoxParser.moofBox + + // helpers + getTrackById(id: number): BoxParser.trakBox | undefined + getTrexById(id: number): BoxParser.trexBox | undefined + + // boxes that are added to the root + boxes: BoxParser.Box[] + mdats: BoxParser.mdatBox[] + moofs: BoxParser.moofBox[] + + ftyp?: BoxParser.ftypBox + moov?: BoxParser.moovBox + + static writeInitializationSegment( + ftyp: BoxParser.ftypBox, + moov: BoxParser.moovBox, + total_duration: number, + sample_duration: number, + ): ArrayBuffer + + // TODO add correct types; these are exported by dts-gen + add(name: any): any + addBox(box: any): any + appendBuffer(ab: any, last: any): any + buildSampleLists(): void + buildTrakSampleLists(trak: any): void + checkBuffer(ab: any): any + createFragment(track_id: any, sampleNumber: any, stream_: any): any + equal(b: any): any + flattenItemInfo(): void + flush(): void + getAllocatedSampleDataSize(): any + getBox(type: any): any + getBoxes(type: any, returnEarly: any): any + getBuffer(): any + getCodecs(): any + getInfo(): any + getItem(item_id: any): any + getMetaHandler(): any + getPrimaryItem(): any + getSample(trak: any, sampleNum: any): any + getTrackSample(track_id: any, number: any): any + getTrackSamplesInfo(track_id: any): any + hasIncompleteMdat(): any + hasItem(name: any): any + initializeSegmentation(): any + itemToFragmentedTrackFile(_options: any): any + parse(): void + print(output: any): void + processIncompleteBox(ret: any): any + processIncompleteMdat(): any + processItems(callback: any): void + processSamples(last: any): void + releaseItem(item_id: any): any + releaseSample(trak: any, sampleNum: any): any + releaseUsedSamples(id: any, sampleNum: any): void + resetTables(): void + restoreParsePosition(): any + save(name: any): void + saveParsePosition(): void + seek(time: any, useRap: any): any + seekTrack(time: any, useRap: any, trak: any): any + setExtractionOptions(id: any, user: any, options: any): void + setSegmentOptions(id: any, user: any, options: any): void + start(): void + stop(): void + unsetExtractionOptions(id: any): void + unsetSegmentOptions(id: any): void + updateSampleLists(): void + updateUsedBytes(box: any, ret: any): void + write(outstream: any): void + + static initSampleGroups(trak: any, traf: any, sbgps: any, trak_sgpds: any, traf_sgpds: any): void + static process_sdtp(sdtp: any, sample: any, number: any): void + static setSampleGroupProperties(trak: any, sample: any, sample_number: any, sample_groups_info: any): void + + // TODO Expand public API; it's difficult to tell what should be public + onMoovStart?: () => void + onReady?: (info: MP4Info) => void + onError?: (e: string) => void + onSamples?: (id: number, user: any, samples: Sample[]) => void + + appendBuffer(data: MP4ArrayBuffer): number + start(): void + stop(): void + flush(): void + + setExtractionOptions(id: number, user: any, options: ExtractionOptions): void + } + + export namespace BoxParser { + export class Box { + size?: number + flags?: number // Do these go here? + data?: Uint8Array + + constructor(type?: string, size?: number) + + add(name: string): Box + addBox(box: Box): Box + set(name: string, value: any): void + addEntry(value: string, prop?: string): void + printHeader(output: any): void + write(stream: DataStream): void + writeHeader(stream: DataStream, msg?: string): void + computeSize(): void + + // TODO add types for these + parse(stream: any): void + parseDataAndRewind(stream: any): void + parseLanguage(stream: any): void + print(output: any): void + } + + // TODO finish add types for these classes + export class AudioSampleEntry extends SampleEntry { + constructor(type: any, size?: number) + + getChannelCount(): any + getSampleRate(): any + getSampleSize(): any + isAudio(): any + parse(stream: any): void + write(stream: any): void + } + + export class CoLLBox extends ContainerBox { + constructor(size?: number) + + parse(stream: any): void + } + + export class ContainerBox extends Box { + constructor(type: any, size?: number, uuid?: any) + + parse(stream: any): void + print(output: any): void + write(stream: any): void + } + + export class FullBox extends Box { + constructor(type: any, size?: number, uuid?: any) + + parse(stream: any): void + parseDataAndRewind(stream: any): void + parseFullHeader(stream: any): void + printHeader(output: any): void + writeHeader(stream: any): void + } + + export class HintSampleEntry extends SampleEntry { + constructor(type: any, size?: number) + } + + export class MetadataSampleEntry extends SampleEntry { + constructor(type: any, size?: number) + + isMetadata(): any + } + + export class OpusSampleEntry extends SampleEntry { + constructor(size?: number) + } + + export class SampleEntry extends Box { + constructor(type: any, size?: number, hdr_size?: number, start?: number) + + getChannelCount(): any + getCodec(): any + getHeight(): any + getSampleRate(): any + getSampleSize(): any + getWidth(): any + isAudio(): any + isHint(): any + isMetadata(): any + isSubtitle(): any + isVideo(): any + parse(stream: any): void + parseDataAndRewind(stream: any): void + parseFooter(stream: any): void + parseHeader(stream: any): void + write(stream: any): void + writeFooter(stream: any): void + writeHeader(stream: any): void + } + + export class SampleGroupEntry { + constructor(type: any) + + parse(stream: any): void + write(stream: any): void + } + + export class SingleItemTypeReferenceBox extends ContainerBox { + constructor(type: any, size?: number, hdr_size?: number, start?: number) + + parse(stream: any): void + } + + export class SingleItemTypeReferenceBoxLarge { + constructor(type: any, size?: number, hdr_size?: number, start?: number) + + parse(stream: any): void + } + + export class SmDmBox extends ContainerBox { + constructor(size?: number) + + parse(stream: any): void + } + + export class SubtitleSampleEntry extends SampleEntry { + constructor(type: any, size?: number) + + isSubtitle(): any + } + + export class SystemSampleEntry extends SampleEntry { + constructor(type: any, size?: number) + } + + export class TextSampleEntry extends SampleEntry { + constructor(type: any, size?: number) + } + + export class TrackGroupTypeBox extends FullBox { + constructor(type: any, size?: number) + + parse(stream: any): void + } + + export class TrackReferenceTypeBox extends ContainerBox { + constructor(type: any, size?: number, hdr_size?: number, start?: number) + + parse(stream: any): void + + write(stream: any): void + } + + export class VisualSampleEntry extends SampleEntry { + constructor(type: any, size?: number) + + getHeight(): any + getWidth(): any + isVideo(): any + parse(stream: any): void + write(stream: any): void + } + + export class a1lxBox extends ContainerBox { + constructor(size?: number) + + parse(stream: any): void + } + + export class a1opBox extends ContainerBox { + constructor(size?: number) + + parse(stream: any): void + } + + export class alstSampleGroupEntry extends SampleGroupEntry { + constructor(size?: number) + + parse(stream: any): void + } + + export class auxCBox extends ContainerBox { + constructor(size?: number) + + parse(stream: any): void + } + + export class av01SampleEntry extends SampleEntry { + constructor(size?: number) + + getCodec(): any + } + + export class av1CBox extends ContainerBox { + constructor(size?: number) + + parse(stream: any): void + } + + export class avc1SampleEntry extends SampleEntry { + constructor(size?: number) + + getCodec(): any + } + + export class avc2SampleEntry extends SampleEntry { + constructor(size?: number) + + getCodec(): any + } + + export class avc3SampleEntry extends SampleEntry { + constructor(size?: number) + + getCodec(): any + } + + export class avc4SampleEntry extends SampleEntry { + constructor(size?: number) + + getCodec(): any + } + + export class avcCBox extends ContainerBox { + constructor(size?: number) + + parse(stream: any): void + write(stream: any): void + } + + export class avllSampleGroupEntry extends SampleGroupEntry { + constructor(size?: number) + + parse(stream: any): void + } + + export class avssSampleGroupEntry extends SampleGroupEntry { + constructor(size?: number) + + parse(stream: any): void + } + + export class btrtBox extends ContainerBox { + constructor(size?: number) + + parse(stream: any): void + } + + export class bxmlBox extends FullBox { + constructor(size?: number) + + parse(stream: any): void + } + + export class clapBox extends ContainerBox { + constructor(size?: number) + + parse(stream: any): void + } + + export class clefBox extends ContainerBox { + constructor(size?: number) + + parse(stream: any): void + } + + export class clliBox extends ContainerBox { + constructor(size?: number) + + parse(stream: any): void + } + + export class co64Box extends ContainerBox { + constructor(size?: number) + + parse(stream: any): void + write(stream: any): void + } + + export class colrBox extends ContainerBox { + constructor(size?: number) + + parse(stream: any): void + } + + export class cprtBox extends ContainerBox { + constructor(size?: number) + + parse(stream: any): void + } + + export class cslgBox extends ContainerBox { + constructor(size?: number) + + parse(stream: any): void + write(stream: any): void + } + + export class cttsBox extends ContainerBox { + constructor(size?: number) + + parse(stream: any): void + unpack(samples: any): void + write(stream: any): void + } + + export class dOpsBox extends ContainerBox { + constructor(size?: number) + + parse(stream: DataStream): void + + Version: number + OutputChannelCount: number + PreSkip: number + InputSampleRate: number + OutputGain: number + ChannelMappingFamily: number + + // When channelMappingFamily != 0 + StreamCount?: number + CoupledCount?: number + ChannelMapping?: number[] + } + + export class dac3Box extends ContainerBox { + constructor(size?: number) + + parse(stream: any): void + } + + export class dec3Box extends ContainerBox { + constructor(size?: number) + + parse(stream: any): void + } + + export class dfLaBox extends ContainerBox { + constructor(size?: number) + + parse(stream: any): void + } + + export class dimmBox extends ContainerBox { + constructor(size?: number) + + parse(stream: any): void + } + + export class dinfBox extends ContainerBox { + constructor(size?: number) + } + + export class dmaxBox extends ContainerBox { + constructor(size?: number) + + parse(stream: any): void + } + + export class dmedBox extends ContainerBox { + constructor(size?: number) + + parse(stream: any): void + } + + export class drefBox extends ContainerBox { + constructor(size?: number) + + parse(stream: any): void + write(stream: any): void + } + + export class drepBox extends ContainerBox { + constructor(size?: number) + + parse(stream: any): void + } + + export class dtrtSampleGroupEntry extends SampleGroupEntry { + constructor(size?: number) + + parse(stream: any): void + } + + export class edtsBox extends ContainerBox { + constructor(size?: number) + } + + export class elngBox extends ContainerBox { + constructor(size?: number) + + parse(stream: any): void + write(stream: any): void + } + + export class elstBox extends ContainerBox { + constructor(size?: number) + + parse(stream: any): void + write(stream: any): void + } + + export class emsgBox extends ContainerBox { + constructor(size?: number) + + parse(stream: any): void + write(stream: any): void + } + + export class encaSampleEntry extends SampleEntry { + constructor(size?: number) + } + + export class encmSampleEntry extends SampleEntry { + constructor(size?: number) + } + + export class encsSampleEntry extends SampleEntry { + constructor(size?: number) + } + + export class enctSampleEntry extends SampleEntry { + constructor(size?: number) + } + + export class encuSampleEntry extends SampleEntry { + constructor(size?: number) + } + + export class encvSampleEntry extends SampleEntry { + constructor(size?: number) + } + + export class enofBox extends ContainerBox { + constructor(size?: number) + + parse(stream: any): void + } + + export class esdsBox extends ContainerBox { + constructor(size?: number) + + parse(stream: any): void + } + + export class fielBox extends ContainerBox { + constructor(size?: number) + + parse(stream: any): void + } + + export class freeBox extends Box { + constructor(size?: number) + } + + export class frmaBox extends ContainerBox { + constructor(size?: number) + + parse(stream: any): void + } + + export class ftypBox extends ContainerBox { + constructor(size?: number) + + parse(stream: any): void + write(stream: any): void + } + + export class hdlrBox extends ContainerBox { + constructor(size?: number) + + parse(stream: any): void + write(stream: any): void + } + + export class hev1SampleEntry extends SampleEntry { + constructor(size?: number) + + getCodec(): any + } + + export class hinfBox extends ContainerBox { + constructor(size?: number) + } + + export class hmhdBox extends FullBox { + constructor(size?: number) + + parse(stream: any): void + } + + export class hntiBox extends ContainerBox { + constructor(size?: number) + } + + export class hvc1SampleEntry extends SampleEntry { + constructor(size?: number) + + getCodec(): any + } + + export class hvcCBox extends ContainerBox { + constructor(size?: number) + + parse(stream: any): void + } + + export class idatBox extends Box { + constructor(size?: number) + } + + export class iinfBox extends ContainerBox { + constructor(size?: number) + + parse(stream: any): void + } + + export class ilocBox extends ContainerBox { + constructor(size?: number) + + parse(stream: any): void + } + + export class imirBox extends ContainerBox { + constructor(size?: number) + + parse(stream: any): void + } + + export class infeBox extends ContainerBox { + constructor(size?: number) + + parse(stream: any): void + } + + export class iodsBox extends FullBox { + constructor(size?: number) + + parse(stream: any): void + } + + export class ipcoBox extends ContainerBox { + constructor(size?: number) + } + + export class ipmaBox extends ContainerBox { + constructor(size?: number) + + parse(stream: any): void + } + + export class iproBox extends FullBox { + constructor(size?: number) + + parse(stream: any): void + } + + export class iprpBox extends ContainerBox { + constructor(size?: number) + ipmas: ipmaBox[] + } + + export class irefBox extends ContainerBox { + constructor(size?: number) + + parse(stream: any): void + } + + export class irotBox extends ContainerBox { + constructor(size?: number) + + parse(stream: any): void + } + + export class ispeBox extends ContainerBox { + constructor(size?: number) + + parse(stream: any): void + } + + export class kindBox extends ContainerBox { + constructor(size?: number) + + parse(stream: any): void + + write(stream: any): void + } + + export class levaBox extends ContainerBox { + constructor(size?: number) + + parse(stream: any): void + } + + export class lselBox extends ContainerBox { + constructor(size?: number) + + parse(stream: any): void + } + + export class maxrBox extends ContainerBox { + constructor(size?: number) + + parse(stream: any): void + } + + export class mdatBox extends Box { + constructor(size?: number) + } + + export class mdcvBox extends ContainerBox { + constructor(size?: number) + + parse(stream: any): void + } + + export class mdhdBox extends ContainerBox { + constructor(size?: number) + + parse(stream: any): void + + write(stream: any): void + } + + export class mdiaBox extends ContainerBox { + constructor(size?: number) + } + + export class mecoBox extends Box { + constructor(size?: number) + } + + export class mehdBox extends ContainerBox { + constructor(size?: number) + + parse(stream: any): void + + write(stream: any): void + } + + export class mereBox extends FullBox { + constructor(size?: number) + + parse(stream: any): void + } + + export class metaBox extends ContainerBox { + constructor(size?: number) + + parse(stream: any): void + } + + export class mettSampleEntry extends SampleEntry { + constructor(size?: number) + + parse(stream: any): void + } + + export class metxSampleEntry extends SampleEntry { + constructor(size?: number) + + parse(stream: any): void + } + + export class mfhdBox extends ContainerBox { + constructor(size?: number) + + parse(stream: any): void + + write(stream: any): void + } + + export class mfraBox extends ContainerBox { + constructor(size?: number) + tfras: tfraBox[] + } + + export class mfroBox extends ContainerBox { + constructor(size?: number) + + parse(stream: any): void + } + + export class minfBox extends ContainerBox { + constructor(size?: number) + } + + export class moofBox extends ContainerBox { + constructor(size?: number) + trafs: trafBox[] + } + + export class moovBox extends ContainerBox { + constructor(size?: number) + traks: trakBox[] + psshs: psshBox[] + } + + export class mp4aSampleEntry extends SampleEntry { + constructor(size?: number) + + getCodec(): any + } + + export class msrcTrackGroupTypeBox extends ContainerBox { + constructor(size?: number) + } + + export class mvexBox extends ContainerBox { + constructor(size?: number) + + trexs: trexBox[] + } + + export class mvhdBox extends ContainerBox { + constructor(size?: number) + + parse(stream: any): void + print(output: any): void + write(stream: any): void + } + + export class mvifSampleGroupEntry extends SampleGroupEntry { + constructor(size?: number) + + parse(stream: any): void + } + + export class nmhdBox extends FullBox { + constructor(size?: number) + + parse(stream: any): void + } + + export class npckBox extends ContainerBox { + constructor(size?: number) + + parse(stream: any): void + } + + export class numpBox extends ContainerBox { + constructor(size?: number) + + parse(stream: any): void + } + + export class padbBox extends ContainerBox { + constructor(size?: number) + + parse(stream: any): void + } + + export class paspBox extends ContainerBox { + constructor(size?: number) + + parse(stream: any): void + } + + export class paylBox extends ContainerBox { + constructor(size?: number) + + parse(stream: any): void + } + + export class paytBox extends ContainerBox { + constructor(size?: number) + + parse(stream: any): void + } + + export class pdinBox extends ContainerBox { + constructor(size?: number) + + parse(stream: any): void + } + + export class pitmBox extends ContainerBox { + constructor(size?: number) + + parse(stream: any): void + } + + export class pixiBox extends ContainerBox { + constructor(size?: number) + + parse(stream: any): void + } + + export class pmaxBox extends ContainerBox { + constructor(size?: number) + + parse(stream: any): void + } + + export class prftBox extends ContainerBox { + constructor(size?: number) + + parse(stream: any): void + } + + export class profBox extends ContainerBox { + constructor(size?: number) + + parse(stream: any): void + } + + export class prolSampleGroupEntry extends SampleGroupEntry { + constructor(size?: number) + + parse(stream: any): void + } + + export class psshBox extends ContainerBox { + constructor(size?: number) + + parse(stream: any): void + } + + export class rashSampleGroupEntry extends SampleGroupEntry { + constructor(size?: number) + + parse(stream: any): void + } + + export class rinfBox extends ContainerBox { + constructor(size?: number) + } + + export class rollSampleGroupEntry extends SampleGroupEntry { + constructor(size?: number) + + parse(stream: any): void + } + + export class saioBox extends ContainerBox { + constructor(size?: number) + + parse(stream: any): void + } + + export class saizBox extends ContainerBox { + constructor(size?: number) + + parse(stream: any): void + } + + export class sbgpBox extends ContainerBox { + constructor(size?: number) + + parse(stream: any): void + + write(stream: any): void + } + + export class sbttSampleEntry extends SampleEntry { + constructor(size?: number) + + parse(stream: any): void + } + + export class schiBox extends ContainerBox { + constructor(size?: number) + } + + export class schmBox extends ContainerBox { + constructor(size?: number) + + parse(stream: any): void + } + + export class scifSampleGroupEntry extends SampleGroupEntry { + constructor(size?: number) + + parse(stream: any): void + } + + export class scnmSampleGroupEntry extends SampleGroupEntry { + constructor(size?: number) + + parse(stream: any): void + } + + export class sdtpBox extends ContainerBox { + constructor(size?: number) + + parse(stream: any): void + } + + export class seigSampleGroupEntry extends SampleGroupEntry { + constructor(size?: number) + + parse(stream: any): void + } + + export class sencBox extends ContainerBox { + constructor(size?: number) + + parse(stream: any): void + } + + export class sgpdBox extends ContainerBox { + constructor(size?: number) + + parse(stream: any): void + write(stream: any): void + } + + export class sidxBox extends ContainerBox { + constructor(size?: number) + + parse(stream: any): void + write(stream: any): void + } + + export class sinfBox extends ContainerBox { + constructor(size?: number) + } + + export class skipBox extends Box { + constructor(size?: number) + } + + export class smhdBox extends ContainerBox { + constructor(size?: number) + + parse(stream: any): void + write(stream: any): void + } + + export class ssixBox extends ContainerBox { + constructor(size?: number) + + parse(stream: any): void + } + + export class stblBox extends ContainerBox { + constructor(size?: number) + + sgpds: sgpdBox[] + sbgps: sbgpBox[] + } + + export class stcoBox extends ContainerBox { + constructor(size?: number) + + parse(stream: any): void + unpack(samples: any): void + write(stream: any): void + } + + export class stdpBox extends ContainerBox { + constructor(size?: number) + + parse(stream: any): void + } + + export class sthdBox extends ContainerBox { + constructor(size?: number) + + parse(stream: any): void + } + + export class stppSampleEntry extends SampleEntry { + constructor(size?: number) + + parse(stream: any): void + write(stream: any): void + } + + export class strdBox extends ContainerBox { + constructor(size?: number) + } + + export class striBox extends ContainerBox { + constructor(size?: number) + + parse(stream: any): void + } + + export class strkBox extends Box { + constructor(size?: number) + } + + export class stsaSampleGroupEntry extends SampleGroupEntry { + constructor(size?: number) + + parse(stream: any): void + } + + export class stscBox extends ContainerBox { + constructor(size?: number) + + parse(stream: any): void + unpack(samples: any): void + write(stream: any): void + } + + export class stsdBox extends ContainerBox { + constructor(size?: number) + + parse(stream: any): void + write(stream: any): void + } + + export class stsgBox extends ContainerBox { + constructor(size?: number) + + parse(stream: any): void + } + + export class stshBox extends ContainerBox { + constructor(size?: number) + + parse(stream: any): void + write(stream: any): void + } + + export class stssBox extends ContainerBox { + constructor(size?: number) + + parse(stream: any): void + write(stream: any): void + } + + export class stszBox extends ContainerBox { + constructor(size?: number) + + parse(stream: any): void + unpack(samples: any): void + write(stream: any): void + } + + export class sttsBox extends ContainerBox { + constructor(size?: number) + + parse(stream: any): void + unpack(samples: any): void + write(stream: any): void + } + + export class stviBox extends ContainerBox { + constructor(size?: number) + + parse(stream: any): void + } + + export class stxtSampleEntry extends SampleEntry { + constructor(size?: number) + + getCodec(): any + parse(stream: any): void + } + + export class stypBox extends ContainerBox { + constructor(size?: number) + + parse(stream: any): void + } + + export class stz2Box extends ContainerBox { + constructor(size?: number) + + parse(stream: any): void + } + + export class subsBox extends ContainerBox { + constructor(size?: number) + + parse(stream: any): void + } + + export class syncSampleGroupEntry extends SampleGroupEntry { + constructor(size?: number) + + parse(stream: any): void + } + + export class taptBox extends ContainerBox { + constructor(size?: number) + } + + export class teleSampleGroupEntry extends SampleGroupEntry { + constructor(size?: number) + + parse(stream: any): void + } + + export class tencBox extends ContainerBox { + constructor(size?: number) + + parse(stream: any): void + } + + export class tfdtBox extends ContainerBox { + constructor(size?: number) + + parse(stream: any): void + write(stream: any): void + } + + export class tfhdBox extends ContainerBox { + constructor(size?: number) + + parse(stream: any): void + write(stream: any): void + } + + export class tfraBox extends ContainerBox { + constructor(size?: number) + + parse(stream: any): void + } + + export class tkhdBox extends ContainerBox { + constructor(size?: number) + + parse(stream: any): void + print(output: any): void + write(stream: any): void + } + + export class tmaxBox extends ContainerBox { + constructor(size?: number) + + parse(stream: any): void + } + + export class tminBox extends ContainerBox { + constructor(size?: number) + + parse(stream: any): void + } + + export class totlBox extends ContainerBox { + constructor(size?: number) + + parse(stream: any): void + } + + export class tpayBox extends ContainerBox { + constructor(size?: number) + + parse(stream: any): void + } + + export class tpylBox extends ContainerBox { + constructor(size?: number) + + parse(stream: any): void + } + + export class trafBox extends ContainerBox { + constructor(size?: number) + truns: trunBox[] + sgpd: sgpdBox[] + sbgp: sbgpBox[] + } + + export class trakBox extends ContainerBox { + constructor(size?: number) + } + + export class trefBox extends ContainerBox { + constructor(size?: number) + + parse(stream: any): void + } + + export class trepBox extends ContainerBox { + constructor(size?: number) + + parse(stream: any): void + } + + export class trexBox extends ContainerBox { + constructor(size?: number) + + parse(stream: any): void + write(stream: any): void + } + + export class trgrBox extends ContainerBox { + constructor(size?: number) + } + + export class trpyBox extends ContainerBox { + constructor(size?: number) + + parse(stream: any): void + } + + export class trunBox extends ContainerBox { + constructor(size?: number) + + parse(stream: any): void + write(stream: any): void + + sample_count: number + + sample_duration?: number[] + sample_size?: number[] + sample_flags?: number[] + sample_composition_time_offset?: number[] + + data_offset?: number + data_offset_position?: number + } + + export class tsasSampleGroupEntry extends SampleGroupEntry { + constructor(size?: number) + + parse(stream: any): void + } + + export class tsclSampleGroupEntry extends SampleGroupEntry { + constructor(size?: number) + + parse(stream: any): void + } + + export class tselBox extends ContainerBox { + constructor(size?: number) + + parse(stream: any): void + } + + export class tx3gSampleEntry extends SampleEntry { + constructor(size?: number) + + parse(stream: any): void + } + + export class txtCBox extends ContainerBox { + constructor(size?: number) + + parse(stream: any): void + } + + export class udtaBox extends ContainerBox { + constructor(size?: number) + kinds: kindBox[] + } + + export class viprSampleGroupEntry extends SampleGroupEntry { + constructor(size?: number) + + parse(stream: any): void + } + + export class vmhdBox extends ContainerBox { + constructor(size?: number) + + parse(stream: any): void + write(stream: any): void + } + + export class vp08SampleEntry extends SampleEntry { + constructor(size?: number) + + getCodec(): any + } + + export class vp09SampleEntry extends SampleEntry { + constructor(size?: number) + + getCodec(): any + } + + export class vpcCBox extends ContainerBox { + constructor(size?: number) + + parse(stream: any): void + } + + export class vttCBox extends ContainerBox { + constructor(size?: number) + + parse(stream: any): void + } + + export class vttcBox extends ContainerBox { + constructor(size?: number) + } + + export class vvc1SampleEntry extends SampleEntry { + constructor(size?: number) + + getCodec(): any + } + + export class vvcCBox extends ContainerBox { + constructor(size?: number) + + parse(stream: any): void + } + + export class vvcNSampleEntry extends SampleEntry { + constructor(size?: number) + } + + export class vvi1SampleEntry extends SampleEntry { + constructor(size?: number) + + getCodec(): any + } + + export class vvnCBox extends ContainerBox { + constructor(size?: number) + + parse(stream: any): void + } + + export class vvs1SampleEntry extends SampleEntry { + constructor(size?: number) + } + + export class wvttSampleEntry extends SampleEntry { + constructor(size?: number) + + parse(stream: any): void + } + + export const BASIC_BOXES: string[] + export const CONTAINER_BOXES: string[][] + export const DIFF_BOXES_PROP_NAMES: string[] + export const DIFF_PRIMITIVE_ARRAY_PROP_NAMES: string[] + export const ERR_INVALID_DATA: number + export const ERR_NOT_ENOUGH_DATA: number + export const FULL_BOXES: string[] + export const OK: number + export const SAMPLE_ENTRY_TYPE_AUDIO: string + export const SAMPLE_ENTRY_TYPE_HINT: string + export const SAMPLE_ENTRY_TYPE_METADATA: string + export const SAMPLE_ENTRY_TYPE_SUBTITLE: string + export const SAMPLE_ENTRY_TYPE_SYSTEM: string + export const SAMPLE_ENTRY_TYPE_TEXT: string + export const SAMPLE_ENTRY_TYPE_VISUAL: string + export const TFHD_FLAG_BASE_DATA_OFFSET: number + export const TFHD_FLAG_DEFAULT_BASE_IS_MOOF: number + export const TFHD_FLAG_DUR_EMPTY: number + export const TFHD_FLAG_SAMPLE_DESC: number + export const TFHD_FLAG_SAMPLE_DUR: number + export const TFHD_FLAG_SAMPLE_FLAGS: number + export const TFHD_FLAG_SAMPLE_SIZE: number + export const TKHD_FLAG_ENABLED: number + export const TKHD_FLAG_IN_MOVIE: number + export const TKHD_FLAG_IN_PREVIEW: number + export const TRUN_FLAGS_CTS_OFFSET: number + export const TRUN_FLAGS_DATA_OFFSET: number + export const TRUN_FLAGS_DURATION: number + export const TRUN_FLAGS_FIRST_FLAG: number + export const TRUN_FLAGS_FLAGS: number + export const TRUN_FLAGS_SIZE: number + export const UUIDs: string[] + export const boxCodes: string[] + export const containerBoxCodes: any[] + export const fullBoxCodes: any[] + + export const sampleEntryCodes: { + Audio: string[] + Hint: any[] + Metadata: string[] + Subtitle: string[] + System: string[] + Text: string[] + Visual: string[] + } + + export const sampleGroupEntryCodes: any[] + + export const trackGroupTypes: any[] + + export function addSubBoxArrays(subBoxNames: any): void + export function boxEqual(box_a: any, box_b: any): any + export function boxEqualFields(box_a: any, box_b: any): any + export function createBoxCtor(type: any, parseMethod: any): void + export function createContainerBoxCtor(type: any, parseMethod: any, subBoxNames: any): void + export function createEncryptedSampleEntryCtor(mediaType: any, type: any, parseMethod: any): void + export function createFullBoxCtor(type: any, parseMethod: any): void + export function createMediaSampleEntryCtor(mediaType: any, parseMethod: any, subBoxNames: any): void + export function createSampleEntryCtor(mediaType: any, type: any, parseMethod: any, subBoxNames: any): void + export function createSampleGroupCtor(type: any, parseMethod: any): void + export function createTrackGroupCtor(type: any, parseMethod: any): void + export function createUUIDBox(uuid: any, isFullBox: any, isContainerBox: any, parseMethod: any): void + export function decimalToHex(d: any, padding: any): any + export function initialize(): void + export function parseHex16(stream: any): any + export function parseOneBox(stream: any, headerOnly: any, parentsize?: number): any + export function parseUUID(stream: any): any + + /* ??? + namespace UUIDBoxes { + export class a2394f525a9b4f14a2446c427c648df4 { + constructor(size?: number) + } + + export class a5d40b30e81411ddba2f0800200c9a66 { + constructor(size?: number) + + parse(stream: any): void + } + + export class d08a4f1810f34a82b6c832d8aba183d3 { + constructor(size?: number) + + parse(stream: any): void + } + + export class d4807ef2ca3946958e5426cb9e46a79f { + constructor(size?: number) + + parse(stream: any): void + } + } + */ + } + + // TODO Add types for the remaining classes found via dts-gen + export class MP4BoxStream { + constructor(arrayBuffer: any) + + getEndPosition(): any + getLength(): any + getPosition(): any + isEos(): any + readAnyInt(size?: number, signed?: boolean): any + readCString(): any + readInt16(): any + readInt16Array(length: any): any + readInt32(): any + readInt32Array(length: any): any + readInt64(): any + readInt8(): any + readString(length: any): any + readUint16(): any + readUint16Array(length: any): any + readUint24(): any + readUint32(): any + readUint32Array(length: any): any + readUint64(): any + readUint8(): any + readUint8Array(length: any): any + seek(pos: any): any + } + + export class MultiBufferStream { + constructor(buffer: any) + + addUsedBytes(nbBytes: any): void + cleanBuffers(): void + findEndContiguousBuf(inputindex: any): any + findPosition(fromStart: any, filePosition: any, markAsUsed: any): any + getEndFilePositionAfter(pos: any): any + getEndPosition(): any + getLength(): any + getPosition(): any + initialized(): any + insertBuffer(ab: any): void + logBufferLevel(info: any): void + mergeNextBuffer(): any + reduceBuffer(buffer: any, offset: any, newLength: any): any + seek(filePosition: any, fromStart: any, markAsUsed: any): any + setAllUsedBytes(): void + } + + export class Textin4Parser { + constructor() + + parseConfig(data: any): any + parseSample(sample: any): any + } + + export class XMLSubtitlein4Parser { + constructor() + + parseSample(sample: any): any + } + + export function MPEG4DescriptorParser(): any + + export namespace BoxParser {} + + export namespace Log { + export const LOG_LEVEL_ERROR = 4 + export const LOG_LEVEL_WARNING = 3 + export const LOG_LEVEL_INFO = 2 + export const LOG_LEVEL_DEBUG = 1 + + export function debug(module: any, msg: any): void + export function error(module: any, msg: any): void + export function getDurationString(duration: any, _timescale: any): any + export function info(module: any, msg: any): void + export function log(module: any, msg: any): void + export function printRanges(ranges: any): any + export function setLogLevel(level: any): void + export function warn(module: any, msg: any): void + } +} diff --git a/packages/moq/types/tsconfig.json b/packages/moq/types/tsconfig.json new file mode 100644 index 0000000..784e219 --- /dev/null +++ b/packages/moq/types/tsconfig.json @@ -0,0 +1,4 @@ +{ + "extends": "../tsconfig.json", + "include": ["."] +} diff --git a/packages/typescript-config/base.json b/packages/typescript-config/base.json index 359bdb6..5064c1e 100644 --- a/packages/typescript-config/base.json +++ b/packages/typescript-config/base.json @@ -2,7 +2,7 @@ "$schema": "https://json.schemastore.org/tsconfig", "compilerOptions": { "allowJs": true, - "target": "ES2017", + "target": "ES2022", "module": "ES2022", "lib": ["es2022", "DOM", "WebWorker", "DOM.Iterable"], "jsx": "react-jsx", diff --git a/packages/ui/package.json b/packages/ui/package.json index 8cc01b9..17a3156 100644 --- a/packages/ui/package.json +++ b/packages/ui/package.json @@ -26,6 +26,7 @@ "@builder.io/qwik-react": "0.5.0", "@fontsource/bricolage-grotesque": "^5.0.7", "@fontsource/geist-sans": "^5.0.3", + "@modular-forms/qwik": "0.26.1", "@nestri/core": "*", "@nestri/eslint-config": "*", "@nestri/typescript-config": "*", @@ -48,6 +49,7 @@ "tailwind-merge": "^2.4.0", "tailwind-variants": "^0.2.1", "tailwindcss": "^3.4.9", - "typescript": "^5.3.3" + "typescript": "^5.3.3", + "valibot": "^0.42.1" } } \ No newline at end of file diff --git a/packages/ui/src/router-head.tsx b/packages/ui/src/router-head.tsx index 0363fe0..d21828b 100644 --- a/packages/ui/src/router-head.tsx +++ b/packages/ui/src/router-head.tsx @@ -15,9 +15,9 @@ export const RouterHead = component$(() => { {/* {head.title} */} {loc.url.pathname === "/" ? "Nestri – Your games. Your rules.": - loc.url.pathname.startsWith("/blog/") + loc.url.pathname.startsWith("/moq/") ? - head.title + `MoQ – Nestri` : `${loc.url.pathname.split("/")[1].charAt(0).toUpperCase() + loc.url.pathname.split("/")[1].slice(1)} – Nestri` } diff --git a/sst-env.d.ts b/sst-env.d.ts index be8895f..9ee5cc1 100644 --- a/sst-env.d.ts +++ b/sst-env.d.ts @@ -7,10 +7,6 @@ declare module "sst" { "type": "sst.sst.Secret" "value": string } - "Relay": { - "type": "sst.aws.Service" - "url": string - } } } export {} diff --git a/sst.config.ts b/sst.config.ts index 8f3f882..ba0495d 100644 --- a/sst.config.ts +++ b/sst.config.ts @@ -6,7 +6,11 @@ export default $config({ name: "nestri", removal: input?.stage === "production" ? "retain" : "remove", home: "aws", - providers: { cloudflare: "5.37.1" }, + providers: { + cloudflare: "5.37.1", + docker: "4.5.5", + "@pulumi/command": "1.0.1", + }, }; }, async run() {