From 9399564d1e32864c6af2d49c0dcd1f76d54443f2 Mon Sep 17 00:00:00 2001 From: Chris Rybicki Date: Fri, 11 Aug 2023 12:46:50 -0400 Subject: [PATCH] feat(sdk)!: remove cloud.Queue initialMessages (#3780) Removes the `initialMessages` property from `cloud.Queue` to reduce API surface area in the standard library. This capability has only worked on the `sim` target to date, and it's possible to easily achieve the capability (across clouds) using `cloud.OnDeploy`: ```js bring cloud; let queue = new cloud.Queue(); new cloud.OnDeploy(inflight () => { queue.push("hello"); queue.push("world"); }); ``` This style of API would have more merit if it's something we could provided an optimized implementation for, but it doesn't appear there are dedicated Terraform resources available that represent AWS SQS messages or Google PubSub items. `OnDeploy` also offers more flexibility since it allows the user to dynamically calculate the queue's initial messages based on the results of other inflight operations. Closes #281. BREAKING CHANGE: The field `initialMessages` of `cloud.Queue` has been removed. Instead, use the `cloud.OnDeploy` resource to perform any deploy-time initialization logic. ## Checklist - [x] Title matches [Winglang's style guide](https://www.winglang.io/contributing/start-here/pull_requests#how-are-pull-request-titles-formatted) - [x] Description explains motivation and solution - [x] Tests added (always) - [x] Docs updated (only required for features) - [ ] Added `pr/e2e-full` label if this feature requires end-to-end testing *By submitting this pull request, I confirm that my contribution is made under the terms of the [Wing Cloud Contribution License](https://github.com/winglang/wing/blob/main/CONTRIBUTION_LICENSE.md)*. --- .../04-standard-library/01-cloud/queue.md | 14 -- .../incomplete_inflight_namespace.snap | 2 +- .../completions/namespace_middle_dot.snap | 2 +- .../variable_type_annotation_namespace.snap | 2 +- libs/wingsdk/src/cloud/queue.ts | 6 - libs/wingsdk/src/target-awscdk/queue.ts | 6 - libs/wingsdk/src/target-sim/queue.inflight.ts | 8 - libs/wingsdk/src/target-sim/queue.ts | 4 - .../src/target-sim/schema-resources.ts | 2 - libs/wingsdk/src/target-tf-aws/queue.ts | 6 - .../__snapshots__/file-counter.test.ts.snap | 1 - .../__snapshots__/queue.test.ts.snap | 148 ++++++++++++++++-- libs/wingsdk/test/target-sim/queue.test.ts | 43 +++-- 13 files changed, 171 insertions(+), 73 deletions(-) diff --git a/docs/docs/04-standard-library/01-cloud/queue.md b/docs/docs/04-standard-library/01-cloud/queue.md index 1a6d4ce92f6..d56866e37d8 100644 --- a/docs/docs/04-standard-library/01-cloud/queue.md +++ b/docs/docs/04-standard-library/01-cloud/queue.md @@ -239,25 +239,11 @@ let QueueProps = cloud.QueueProps{ ... }; | **Name** | **Type** | **Description** | | --- | --- | --- | -| initialMessages | MutArray<str> | Initialize the queue with a set of messages. | | retentionPeriod | duration | How long a queue retains a message. | | timeout | duration | How long a queue's consumers have to process a message. | --- -##### `initialMessages`Optional - -```wing -initialMessages: MutArray; -``` - -- *Type:* MutArray<str> -- *Default:* [] - -Initialize the queue with a set of messages. - ---- - ##### `retentionPeriod`Optional ```wing diff --git a/libs/wingc/src/lsp/snapshots/completions/incomplete_inflight_namespace.snap b/libs/wingc/src/lsp/snapshots/completions/incomplete_inflight_namespace.snap index daaec034816..75be35689ec 100644 --- a/libs/wingc/src/lsp/snapshots/completions/incomplete_inflight_namespace.snap +++ b/libs/wingc/src/lsp/snapshots/completions/incomplete_inflight_namespace.snap @@ -203,7 +203,7 @@ source: libs/wingc/src/lsp/completions.rs kind: 22 documentation: kind: markdown - value: "```wing\nstruct QueueProps\n```\n---\nOptions for `Queue`.\n### Fields\n- `initialMessages?` — Initialize the queue with a set of messages.\n- `retentionPeriod?` — How long a queue retains a message.\n- `timeout?` — How long a queue's consumers have to process a message." + value: "```wing\nstruct QueueProps\n```\n---\nOptions for `Queue`.\n### Fields\n- `retentionPeriod?` — How long a queue retains a message.\n- `timeout?` — How long a queue's consumers have to process a message." sortText: hh|QueueProps - label: QueueSetConsumerProps kind: 22 diff --git a/libs/wingc/src/lsp/snapshots/completions/namespace_middle_dot.snap b/libs/wingc/src/lsp/snapshots/completions/namespace_middle_dot.snap index daaec034816..75be35689ec 100644 --- a/libs/wingc/src/lsp/snapshots/completions/namespace_middle_dot.snap +++ b/libs/wingc/src/lsp/snapshots/completions/namespace_middle_dot.snap @@ -203,7 +203,7 @@ source: libs/wingc/src/lsp/completions.rs kind: 22 documentation: kind: markdown - value: "```wing\nstruct QueueProps\n```\n---\nOptions for `Queue`.\n### Fields\n- `initialMessages?` — Initialize the queue with a set of messages.\n- `retentionPeriod?` — How long a queue retains a message.\n- `timeout?` — How long a queue's consumers have to process a message." + value: "```wing\nstruct QueueProps\n```\n---\nOptions for `Queue`.\n### Fields\n- `retentionPeriod?` — How long a queue retains a message.\n- `timeout?` — How long a queue's consumers have to process a message." sortText: hh|QueueProps - label: QueueSetConsumerProps kind: 22 diff --git a/libs/wingc/src/lsp/snapshots/completions/variable_type_annotation_namespace.snap b/libs/wingc/src/lsp/snapshots/completions/variable_type_annotation_namespace.snap index daaec034816..75be35689ec 100644 --- a/libs/wingc/src/lsp/snapshots/completions/variable_type_annotation_namespace.snap +++ b/libs/wingc/src/lsp/snapshots/completions/variable_type_annotation_namespace.snap @@ -203,7 +203,7 @@ source: libs/wingc/src/lsp/completions.rs kind: 22 documentation: kind: markdown - value: "```wing\nstruct QueueProps\n```\n---\nOptions for `Queue`.\n### Fields\n- `initialMessages?` — Initialize the queue with a set of messages.\n- `retentionPeriod?` — How long a queue retains a message.\n- `timeout?` — How long a queue's consumers have to process a message." + value: "```wing\nstruct QueueProps\n```\n---\nOptions for `Queue`.\n### Fields\n- `retentionPeriod?` — How long a queue retains a message.\n- `timeout?` — How long a queue's consumers have to process a message." sortText: hh|QueueProps - label: QueueSetConsumerProps kind: 22 diff --git a/libs/wingsdk/src/cloud/queue.ts b/libs/wingsdk/src/cloud/queue.ts index 357e7aeb15a..3adc3975669 100644 --- a/libs/wingsdk/src/cloud/queue.ts +++ b/libs/wingsdk/src/cloud/queue.ts @@ -24,12 +24,6 @@ export interface QueueProps { * @default undefined */ readonly retentionPeriod?: Duration; - - /** - * Initialize the queue with a set of messages. - * @default [] - */ - readonly initialMessages?: string[]; } /** diff --git a/libs/wingsdk/src/target-awscdk/queue.ts b/libs/wingsdk/src/target-awscdk/queue.ts index b2766149b4b..3d5eb69effe 100644 --- a/libs/wingsdk/src/target-awscdk/queue.ts +++ b/libs/wingsdk/src/target-awscdk/queue.ts @@ -29,12 +29,6 @@ export class Queue extends cloud.Queue { ? Duration.seconds(props.retentionPeriod?.seconds) : undefined, }); - - if ((props.initialMessages ?? []).length) { - throw new Error( - "initialMessages not supported yet for AWS target - https://github.com/winglang/wing/issues/281" - ); - } } public setConsumer( diff --git a/libs/wingsdk/src/target-sim/queue.inflight.ts b/libs/wingsdk/src/target-sim/queue.inflight.ts index 126736c103a..fb420910bdd 100644 --- a/libs/wingsdk/src/target-sim/queue.inflight.ts +++ b/libs/wingsdk/src/target-sim/queue.inflight.ts @@ -25,14 +25,6 @@ export class Queue private readonly retentionPeriod: number; constructor(props: QueueSchema["props"], context: ISimulatorContext) { - if (props.initialMessages) { - this.messages.push( - ...props.initialMessages.map( - (message) => new QueueMessage(this.retentionPeriod, message) - ) - ); - } - this.timeout = props.timeout; this.retentionPeriod = props.retentionPeriod; this.intervalId = setInterval(() => this.processMessages(), 100); // every 0.1 seconds diff --git a/libs/wingsdk/src/target-sim/queue.ts b/libs/wingsdk/src/target-sim/queue.ts index 08d2ce945d0..6e01e2cfb72 100644 --- a/libs/wingsdk/src/target-sim/queue.ts +++ b/libs/wingsdk/src/target-sim/queue.ts @@ -19,7 +19,6 @@ import { BaseResourceSchema } from "../testing/simulator"; export class Queue extends cloud.Queue implements ISimulatorResource { private readonly timeout: Duration; private readonly retentionPeriod: Duration; - private readonly initialMessages: string[] = []; constructor(scope: Construct, id: string, props: cloud.QueueProps = {}) { super(scope, id, props); @@ -31,8 +30,6 @@ export class Queue extends cloud.Queue implements ISimulatorResource { "Retention period must be greater than or equal to timeout" ); } - - this.initialMessages.push(...(props.initialMessages ?? [])); } public setConsumer( @@ -104,7 +101,6 @@ export class Queue extends cloud.Queue implements ISimulatorResource { props: { timeout: this.timeout.seconds, retentionPeriod: this.retentionPeriod.seconds, - initialMessages: this.initialMessages, }, attrs: {} as any, }; diff --git a/libs/wingsdk/src/target-sim/schema-resources.ts b/libs/wingsdk/src/target-sim/schema-resources.ts index a73728b097f..c660c860a9e 100644 --- a/libs/wingsdk/src/target-sim/schema-resources.ts +++ b/libs/wingsdk/src/target-sim/schema-resources.ts @@ -80,8 +80,6 @@ export interface QueueSchema extends BaseResourceSchema { readonly timeout: number; /** How long a queue retains a message, in seconds */ readonly retentionPeriod: number; - /** Initial messages to be pushed to the queue. */ - readonly initialMessages: string[]; }; } diff --git a/libs/wingsdk/src/target-tf-aws/queue.ts b/libs/wingsdk/src/target-tf-aws/queue.ts index fa06e6a25e3..2d57275622d 100644 --- a/libs/wingsdk/src/target-tf-aws/queue.ts +++ b/libs/wingsdk/src/target-tf-aws/queue.ts @@ -35,12 +35,6 @@ export class Queue extends cloud.Queue { messageRetentionSeconds: props.retentionPeriod?.seconds, name: ResourceNames.generateName(this, NAME_OPTS), }); - - if ((props.initialMessages ?? []).length) { - throw new Error( - "initialMessages not supported yet for AWS target - https://github.com/winglang/wing/issues/281" - ); - } } public setConsumer( diff --git a/libs/wingsdk/test/target-sim/__snapshots__/file-counter.test.ts.snap b/libs/wingsdk/test/target-sim/__snapshots__/file-counter.test.ts.snap index 3deb47e728a..191b6be620e 100644 --- a/libs/wingsdk/test/target-sim/__snapshots__/file-counter.test.ts.snap +++ b/libs/wingsdk/test/target-sim/__snapshots__/file-counter.test.ts.snap @@ -79,7 +79,6 @@ bucket: (function(env) { "attrs": {}, "path": "root/HelloWorld/Queue", "props": { - "initialMessages": [], "retentionPeriod": 3600, "timeout": 10, }, diff --git a/libs/wingsdk/test/target-sim/__snapshots__/queue.test.ts.snap b/libs/wingsdk/test/target-sim/__snapshots__/queue.test.ts.snap index 5c8a6e29ec3..8e0f433233b 100644 --- a/libs/wingsdk/test/target-sim/__snapshots__/queue.test.ts.snap +++ b/libs/wingsdk/test/target-sim/__snapshots__/queue.test.ts.snap @@ -16,7 +16,6 @@ exports[`create a queue 1`] = ` "attrs": {}, "path": "root/my_queue", "props": { - "initialMessages": [], "retentionPeriod": 3600, "timeout": 10, }, @@ -136,7 +135,6 @@ async handle(message) { "attrs": {}, "path": "root/my_queue", "props": { - "initialMessages": [], "retentionPeriod": 1, "timeout": 2, }, @@ -369,7 +367,6 @@ async handle(message) { "attrs": {}, "path": "root/my_queue", "props": { - "initialMessages": [], "retentionPeriod": 3600, "timeout": 1, }, @@ -606,7 +603,6 @@ async handle(message) { "attrs": {}, "path": "root/my_queue", "props": { - "initialMessages": [], "retentionPeriod": 3600, "timeout": 1, }, @@ -805,7 +801,6 @@ exports[`queue batch size of 2, purge the queue 2`] = ` "attrs": {}, "path": "root/my_queue", "props": { - "initialMessages": [], "retentionPeriod": 3600, "timeout": 10, }, @@ -867,8 +862,20 @@ exports[`queue with one subscriber, batch size of 5 1`] = ` "wingsdk.cloud.Function created.", "wingsdk.cloud.Queue created.", "wingsdk.sim.EventMapping created.", + "wingsdk.cloud.Function created.", + "Push (message=A).", + "Push (message=B).", + "Push (message=C).", + "Push (message=D).", + "Push (message=E).", + "Push (message=F).", + "Invoke (payload=\\"\\").", + "OnDeploy invoked.", + "wingsdk.cloud.OnDeploy created.", "Sending messages (messages=[\\"A\\",\\"B\\",\\"C\\",\\"D\\",\\"E\\"], subscriber=sim-1).", "Sending messages (messages=[\\"F\\"], subscriber=sim-1).", + "wingsdk.cloud.OnDeploy deleted.", + "wingsdk.cloud.Function deleted.", "wingsdk.sim.EventMapping deleted.", "wingsdk.cloud.Queue deleted.", "wingsdk.cloud.Function deleted.", @@ -878,6 +885,33 @@ exports[`queue with one subscriber, batch size of 5 1`] = ` exports[`queue with one subscriber, batch size of 5 2`] = ` { + ".wing/function_c8ab799f.js": "exports.handler = async function(event) { + return await (new ((function(){ +return class Handler { + constructor(clients) { + for (const [name, client] of Object.entries(clients)) { + this[name] = client; + } + } + async handle() { + await this.queue.push(\\"A\\"); + await this.queue.push(\\"B\\"); + await this.queue.push(\\"C\\"); + await this.queue.push(\\"D\\"); + await this.queue.push(\\"E\\"); + await this.queue.push(\\"F\\"); +} +}; +})())({ +queue: (function(env) { + let handle = process.env[env]; + if (!handle) { + throw new Error(\\"Missing environment variable: \\" + env); + } + return $simulator.findInstance(handle); + })(\\"QUEUE_HANDLE_54fcf4cd\\") +})).handle(event); +};", ".wing/my_queue-setconsumer-e645076f_c8ddc1ce.js": "exports.handler = async function(event) { return await (new (require(\\"[REDACTED]/wingsdk/src/target-sim/queue.setconsumer.inflight.js\\")).QueueSetConsumerHandlerClient({ handler: new ((function(){ return class Handler { @@ -922,14 +956,6 @@ async handle(message) { "attrs": {}, "path": "root/my_queue", "props": { - "initialMessages": [ - "A", - "B", - "C", - "D", - "E", - "F", - ], "retentionPeriod": 3600, "timeout": 10, }, @@ -947,6 +973,27 @@ async handle(message) { }, "type": "wingsdk.sim.EventMapping", }, + { + "attrs": {}, + "path": "root/my_queue_messages/Function", + "props": { + "environmentVariables": { + "QUEUE_HANDLE_54fcf4cd": "\${root/my_queue#attrs.handle}", + }, + "sourceCodeFile": ".wing/function_c8ab799f.js", + "sourceCodeLanguage": "javascript", + "timeout": 60000, + }, + "type": "wingsdk.cloud.Function", + }, + { + "attrs": {}, + "path": "root/my_queue_messages", + "props": { + "functionHandle": "\${root/my_queue_messages/Function#attrs.handle}", + }, + "type": "wingsdk.cloud.OnDeploy", + }, ], "sdkVersion": "0.0.0", }, @@ -982,6 +1029,22 @@ async handle(message) { "id": "Handler", "path": "root/Handler", }, + "OnDeployHandler": { + "attributes": { + "wing:resource:connections": [], + }, + "constructInfo": { + "fqn": "constructs.Construct", + "version": "10.1.314", + }, + "display": { + "description": "An inflight resource", + "hidden": true, + "title": "Inflight", + }, + "id": "OnDeployHandler", + "path": "root/OnDeployHandler", + }, "cloud.TestRunner": { "attributes": { "wing:resource:connections": [], @@ -1007,6 +1070,18 @@ async handle(message) { "relationship": "setConsumer()", "resource": "root/my_queue/my_queue-SetConsumer-e645076f", }, + { + "direction": "inbound", + "implicit": false, + "relationship": "push()", + "resource": "root/my_queue_messages/Function", + }, + { + "direction": "inbound", + "implicit": false, + "relationship": "$inflight_init()", + "resource": "root/my_queue_messages/Function", + }, ], }, "children": { @@ -1085,6 +1160,52 @@ async handle(message) { "id": "my_queue", "path": "root/my_queue", }, + "my_queue_messages": { + "attributes": { + "wing:resource:connections": [], + }, + "children": { + "Function": { + "attributes": { + "wing:resource:connections": [ + { + "direction": "outbound", + "implicit": false, + "relationship": "push()", + "resource": "root/my_queue", + }, + { + "direction": "outbound", + "implicit": false, + "relationship": "$inflight_init()", + "resource": "root/my_queue", + }, + ], + }, + "constructInfo": { + "fqn": "constructs.Construct", + "version": "10.1.314", + }, + "display": { + "description": "A cloud function (FaaS)", + "sourceModule": "@winglang/sdk", + "title": "Function", + }, + "id": "Function", + "path": "root/my_queue_messages/Function", + }, + }, + "constructInfo": { + "fqn": "constructs.Construct", + "version": "10.1.314", + }, + "display": { + "description": "Run code during the app's deployment.", + "title": "OnDeploy", + }, + "id": "my_queue_messages", + "path": "root/my_queue_messages", + }, }, "constructInfo": { "fqn": "constructs.Construct", @@ -1161,7 +1282,6 @@ async handle(message) { "attrs": {}, "path": "root/my_queue", "props": { - "initialMessages": [], "retentionPeriod": 3600, "timeout": 10, }, diff --git a/libs/wingsdk/test/target-sim/queue.test.ts b/libs/wingsdk/test/target-sim/queue.test.ts index 9b4ccba10b9..9955332d337 100644 --- a/libs/wingsdk/test/target-sim/queue.test.ts +++ b/libs/wingsdk/test/target-sim/queue.test.ts @@ -28,7 +28,6 @@ test("create a queue", async () => { }, path: "root/my_queue", props: { - initialMessages: [], timeout: 10, retentionPeriod: 3600, }, @@ -96,11 +95,32 @@ test("queue batch size of 2, purge the queue", async () => { test("queue with one subscriber, batch size of 5", async () => { // GIVEN const app = new SimApp(); + + const queue = cloud.Queue._newQueue(app, "my_queue"); const handler = Testing.makeHandler(app, "Handler", INFLIGHT_CODE); - const queue = cloud.Queue._newQueue(app, "my_queue", { - initialMessages: ["A", "B", "C", "D", "E", "F"], - }); queue.setConsumer(handler, { batchSize: 5 }); + + // initialize the queue with some messages + const onDeployHandler = Testing.makeHandler( + app, + "OnDeployHandler", + `async handle() { + await this.queue.push("A"); + await this.queue.push("B"); + await this.queue.push("C"); + await this.queue.push("D"); + await this.queue.push("E"); + await this.queue.push("F"); +}`, + { + queue: { + obj: queue, + ops: [cloud.QueueInflightMethods.PUSH], + }, + } + ); + cloud.OnDeploy._newOnDeploy(app, "my_queue_messages", onDeployHandler); + const s = await app.startSimulator(); // WHEN @@ -276,17 +296,22 @@ test("queue has display title and description properties", async () => { }); }); -test("queue pops messages", async () => { +test("can pop messages from queue", async () => { // GIVEN const app = new SimApp(); const messages = ["A", "B", "C", "D", "E", "F"]; - cloud.Queue._newQueue(app, "my_queue", { - initialMessages: messages, - }); + cloud.Queue._newQueue(app, "my_queue"); // WHEN const s = await app.startSimulator(); const queueClient = s.getResource("/my_queue") as cloud.IQueueClient; + + // initialize the messages + for (const message of messages) { + await queueClient.push(message); + } + + // try popping them const poppedMessages: Array = []; for (let i = 0; i < messages.length; i++) { poppedMessages.push(await queueClient.pop()); @@ -299,7 +324,7 @@ test("queue pops messages", async () => { expect(poppedOnEmptyQueue).toBeUndefined(); }); -test("empty queue pops nothing", async () => { +test("pop from empty queue returns nothing", async () => { // GIVEN const app = new SimApp(); cloud.Queue._newQueue(app, "my_queue");