feat!: grpc migration

.gitignore

@@ -2,7 +2,7 @@
 .idea
 .vscode
 
-packages/client-payments/src/proto/nilchain.ts # generated protobuf codec
+packages/*/src/gen-proto # generated protobuf bindings
 !packages/client-wasm/dist
 docs
 logs

.prettierignore

@@ -1,2 +1,2 @@
 packages/wasm/dist
-packages/client-payments/src/gen-proto
+packages/*/src/gen-proto

justfile

@@ -30,7 +30,9 @@ watch-and-build:
     just clean
     npx concurrently -c "auto" \
       "npm -w packages/client-core run build:watch" \
+      "npm -w packages/client-payments run build:proto" \
       "npm -w packages/client-payments run build:watch" \
+      "npm -w packages/client-vms run build:proto" \
       "npm -w packages/client-vms run build:watch" \
       "npm -w packages/client-react-hooks run build:watch"
 
@@ -192,6 +194,7 @@ publish-client-vms args="":
     #!/usr/bin/env bash
     set -euxo pipefail
     npm -w packages/client-vms run clean
+    npm -w packages/client-vms run build:proto
     npm -w packages/client-vms run build
     npm -w packages/client-vms publish {{args}}
 # <<< End @nillion/client-vms <<<

packages/client-payments/src/gen-proto/nillion/meta/v1/msg_pay_for_pb.ts

@@ -60,4 +60,3 @@ export type Amount = Message<"nillion.meta.v1.Amount"> & {
  */
 export const AmountSchema: GenMessage<Amount> = /*@__PURE__*/
   messageDesc(file_nillion_meta_v1_msg_pay_for, 1);
-

packages/client-vms/buf.gen.yaml

@@ -0,0 +1,8 @@
+version: v2
+managed:
+  enabled: true
+clean: true
+plugins:
+  - local: protoc-gen-es
+    opt: target=ts
+    out: src/gen-proto

packages/client-vms/buf.yaml

@@ -0,0 +1,9 @@
+version: v2
+modules:
+  - path: proto
+lint:
+  use:
+    - STANDARD
+breaking:
+  use:
+    - FILE

packages/client-vms/package.json

@@ -16,18 +16,24 @@
     "clean": "rm -rf dist dist-test",
     "build": "node esbuild.config.mjs",
     "build:watch": "node esbuild.config.mjs --watch",
+    "build:proto": "npx buf generate",
     "test": "npx jasmine-browser-runner runSpecs --config=jasmine.config.mjs",
     "test:build": "npx webpack --config webpack.config.mjs",
     "test:build:watch": "npx webpack --config webpack.config.mjs --watch",
     "test:serve": "npx jasmine-browser-runner serve --config=jasmine.config.mjs"
   },
   "dependencies": {
+    "@bufbuild/protobuf": "^2.1.0",
+    "@connectrpc/connect": "^2.0.0-beta.1",
+    "@connectrpc/connect-web": "^2.0.0-beta.1",
     "@nillion/client-core": "0.1.0",
     "@nillion/client-payments": "0.1.0",
     "debug": "^4.3.7",
     "zod": "^3.23.8"
   },
   "devDependencies": {
+    "@bufbuild/buf": "^1.44.0",
+    "@bufbuild/protoc-gen-es": "^2.1.0",
     "@types/debug": "^4.1.12",
     "browserslist": "^4.24.0",
     "esbuild-plugin-browserslist": "^0.15.0"

packages/client-vms/proto/nillion/auth/v1/public_key.proto

@@ -0,0 +1,30 @@
+syntax = "proto3";
+
+package nillion.auth.v1.public_key;
+
+// A public key.
+//
+// Signatures associated to this public key must be a 64 bytes long sequence which is the result of the
+// concatenation of the `r` and `s` scalars in the signature, each encoded in big endian.
+message PublicKey {
+  // The type of this public key.
+  PublicKeyType key_type = 1;
+
+  // The contents of this key.
+  //
+  // This is dependent on the `key_type` in this key:
+  //
+  // * For ED25519 it should be the curve point in its 32 byte compressed form.
+  // * For SECP256K1 it should be the curve point in its 33 byte compressed form.
+  bytes contents = 2;
+}
+
+// A public key type.
+enum PublicKeyType {
+  // An ED25519 public key.
+  ED25519 = 0;
+
+  // A SECP256K1 public key.
+  SECP256K1 = 1;
+}
+

packages/client-vms/proto/nillion/auth/v1/token.proto

@@ -0,0 +1,38 @@
+syntax = "proto3";
+
+package nillion.auth.v1.token;
+
+import "google/protobuf/timestamp.proto";
+import "nillion/auth/v1/public_key.proto";
+
+// A signed authentication token.
+message SignedToken {
+  // A `Token` serialized into bytes.
+  bytes serialized_token = 1;
+
+  // The public key for the private key this token is signed with.
+  nillion.auth.v1.public_key.PublicKey public_key = 2;
+
+  // The serialized token signature.
+  //
+  // The signature must be generated using the private key corresponding to the `public_key` field
+  // over `sha256(serialized_token)`.
+  bytes signature = 3;
+}
+
+// A token.
+message Token {
+  // A nonce that adds entropy to this token.
+  //
+  // The nonce must be 32 bytes long and will be considered invalid otherwise.
+  bytes nonce = 1;
+
+  // The identifier of the node this token is meant to be used against.
+  //
+  // This needs to match the target node's identity for the token to be considered valid, and is used to prevent
+  // a node from taking the token and impersonating the user.
+  bytes target_identity = 2;
+
+  // The timestamp at which this token expires.
+  google.protobuf.Timestamp expires_at = 3;
+}

packages/client-vms/proto/nillion/compute/v1/invoke.proto

@@ -0,0 +1,49 @@
+syntax = "proto3";
+
+package nillion.compute.v1.invoke;
+
+import "nillion/payments/v1/receipt.proto";
+
+// A request to invoke a computation.
+message InvokeComputeRequest {
+  // The receipt that proves this operation was paid for.
+  //
+  // The receipt must be for a `InvokeCompute` operation.
+  nillion.payments.v1.receipt.SignedReceipt signed_receipt = 1;  
+
+  // The value ids for previously stored values being used as parameters to this compute operation.
+  repeated bytes value_ids = 2;
+
+  // The values being used as compute-time parameters, encoded in bincode format.
+  bytes bincode_values = 3;
+
+  // The bindings that define which input party in the program is which user in the network.
+  repeated InputPartyBinding input_bindings = 4;
+
+  // The bindings that define which output party in the program is which users in the network.
+  repeated OutputPartyBinding output_bindings = 5;
+}
+
+// A response to a request to invoke a computation.
+message InvokeComputeResponse {
+  // An identifier for the execution of the computation.
+  bytes compute_id = 1;
+}
+
+// The bindings for input parties in this program.
+message InputPartyBinding {
+  // The name of the party as defined in the program.
+  string party_name = 1;
+
+  // The user identity this party is being bound to.
+  string user_id = 2;
+}
+
+// The bindings for output parties in this program.
+message OutputPartyBinding {
+  // The name of the party as defined in the program.
+  string party_name = 1;
+
+  // The user identities this party is being bound to.
+  repeated string user_ids = 2;
+}

packages/client-vms/proto/nillion/compute/v1/retrieve.proto

@@ -0,0 +1,33 @@
+syntax = "proto3";
+
+package nillion.compute.v1.retrieve;
+
+import "google/protobuf/empty.proto";
+
+// A request to retrieve the results of a computation.
+message RetrieveResultsRequest {
+  // The instance of the computation to retrieve results for.
+  bytes compute_id = 1;
+}
+
+// The response to a request to retrieve the results of a computation.
+message RetrieveResultsResponse {   
+  oneof state {
+    // The node is waiting for the computation to end.
+    //
+    // This message may be sent 0+ times in a row until a `result` is sent.
+    google.protobuf.Empty waiting_computation = 1;
+
+    // The computation finished successfully and yielded these results.
+    ComputationResult success = 2;
+
+    // The error message if any.
+    string error = 3;
+  }
+}
+
+// The result of a computation.
+message ComputationResult {
+    // The computation results, encoded in bincode.
+    bytes bincode_values = 1; 
+}

packages/client-vms/proto/nillion/compute/v1/service.proto

@@ -0,0 +1,30 @@
+syntax = "proto3";
+
+package nillion.compute.v1;
+
+import "google/protobuf/empty.proto";
+import "nillion/compute/v1/invoke.proto";
+import "nillion/compute/v1/stream.proto";
+import "nillion/compute/v1/retrieve.proto";
+
+// A service to perform compute operations.
+service Compute {
+  // Invoke a computation.
+  rpc InvokeCompute(invoke.InvokeComputeRequest) returns (invoke.InvokeComputeResponse);
+
+  // Start a stream of message for a specific instance of a computation.
+  //
+  // This is meant to be used internally by nodes in the network.
+  rpc StreamCompute(stream stream.ComputeStreamMessage) returns (google.protobuf.Empty);
+
+  // Retrieve the result of a computation.
+  //
+  // The user invoking this operation must have been defined as an output party in the program
+  // the execution was tied to.
+  //
+  // This call will wait for the program execution to end if it hasn't done so already. Clients must
+  // wait until either a result or failure message is sent, ignoring any `Waiting` messages sent until
+  // that happens.
+  rpc RetrieveResults(retrieve.RetrieveResultsRequest) returns (stream retrieve.RetrieveResultsResponse);
+}
+

packages/client-vms/proto/nillion/compute/v1/stream.proto

@@ -0,0 +1,15 @@
+syntax = "proto3";
+
+package nillion.compute.v1.stream;
+
+// A message for a compute stream.
+message ComputeStreamMessage {
+  // An identifier for the computation instance.
+  //
+  // Only the first ever message on the stream requires having this attribute set. Any subsequent message will
+  // have this field ignored.
+  bytes compute_id = 1; 
+
+  // The VM message in bincode format.
+  bytes bincode_message = 2;
+}

packages/client-vms/proto/nillion/leader_queries/v1/pool_status.proto

@@ -0,0 +1,37 @@
+syntax = "proto3";
+
+package nillion.leader_queries.v1.pool_status;
+
+import "nillion/preprocessing/v1/element.proto";
+import "nillion/payments/v1/receipt.proto";
+
+// A request to get the preprocessing pool status.
+message PoolStatusRequest {
+  // The receipt that proves this operation was paid for.
+  //
+  // The receipt must be for a `PoolStatus` operation.
+  nillion.payments.v1.receipt.SignedReceipt signed_receipt = 1;
+}
+
+// A response for the request to get the status of the preprocessing pool.
+message PoolStatusResponse {
+  // The offset ranges for each preprocessing element.
+  repeated PreprocessingOffsets offsets = 1;
+
+  // Whether preprocessing is active (at least one element type is being generated).
+  bool preprocessing_active = 2;
+}
+
+// The offsets for a preprocessing element.
+//
+// This is closed-open set, e.g. [start, end)
+message PreprocessingOffsets {
+  // The preprocessing element.
+  nillion.preprocessing.v1.element.PreprocessingElement element = 1;
+
+  // The first offset in the range.
+  uint64 start = 2;
+
+  // The one-past-the-end offset in this range.
+  uint64 end = 3;
+}

packages/client-vms/proto/nillion/leader_queries/v1/service.proto

@@ -0,0 +1,13 @@
+syntax = "proto3";
+
+package nillion.leader_queries.v1;
+
+import "nillion/leader_queries/v1/pool_status.proto";
+
+// A service that allows performing queries against the leader of the cluster.
+service LeaderQueries {
+  // Get the status of the preprocessing pool.
+  rpc PoolStatus(pool_status.PoolStatusRequest) returns (pool_status.PoolStatusResponse);
+}
+
+