diff --git a/crates/agent-sql/src/publications.rs b/crates/agent-sql/src/publications.rs index 8f85b73d4a..3769c68494 100644 --- a/crates/agent-sql/src/publications.rs +++ b/crates/agent-sql/src/publications.rs @@ -287,7 +287,7 @@ pub async fn find_tenant_quotas( where live_specs.spec_type = 'capture' or live_specs.spec_type = 'materialization' or - live_specs.spec_type = 'collection' and live_specs.spec->'derivation' is not null + live_specs.spec_type = 'collection' and live_specs.spec->'derive' is not null ))::integer as tasks_used, (count(live_specs.catalog_name) filter ( where live_specs.spec_type = 'collection' diff --git a/crates/agent-sql/tests/publications.rs b/crates/agent-sql/tests/publications.rs index 2b27ab98e8..57ccc59613 100644 --- a/crates/agent-sql/tests/publications.rs +++ b/crates/agent-sql/tests/publications.rs @@ -266,7 +266,7 @@ async fn test_tenant_usage_quotas() { ('1000000000000000', 'usageA/CollectionA', '1', 'collection', 'bbbbbbbbbbbbbbbb', 'bbbbbbbbbbbbbbbb'), ('2000000000000000', 'usageA/CaptureA', '1', 'capture', 'bbbbbbbbbbbbbbbb', 'bbbbbbbbbbbbbbbb'), ('3000000000000000', 'usageA/MaterializationA', '1', 'materialization', 'bbbbbbbbbbbbbbbb', 'bbbbbbbbbbbbbbbb'), - ('5000000000000000', 'usageA/DerivationA', '{"derivation": {}}'::json, 'collection', 'bbbbbbbbbbbbbbbb', 'bbbbbbbbbbbbbbbb'), + ('5000000000000000', 'usageA/DerivationA', '{"derive": {}}'::json, 'collection', 'bbbbbbbbbbbbbbbb', 'bbbbbbbbbbbbbbbb'), ('6000000000000000', 'usageB/CaptureA', '1', 'capture', 'bbbbbbbbbbbbbbbb', 'bbbbbbbbbbbbbbbb'), ('7000000000000000', 'usageB/CaptureB', '1', 'capture', 'bbbbbbbbbbbbbbbb', 'bbbbbbbbbbbbbbbb'), ('8000000000000000', 'usageB/CaptureC', '1', 'capture', 'bbbbbbbbbbbbbbbb', 'bbbbbbbbbbbbbbbb'), diff --git a/crates/agent/src/discovers/specs.rs b/crates/agent/src/discovers/specs.rs index aaeabc3267..6a10c6dea2 100644 --- a/crates/agent/src/discovers/specs.rs +++ b/crates/agent/src/discovers/specs.rs @@ -138,7 +138,6 @@ pub fn merge_collections( projections: Default::default(), journals: Default::default(), derive: None, - derivation: None, }); if collection.read_schema.is_some() { diff --git a/crates/agent/src/publications/specs.rs b/crates/agent/src/publications/specs.rs index a650bbec7f..19d2595b28 100644 --- a/crates/agent/src/publications/specs.rs +++ b/crates/agent/src/publications/specs.rs @@ -1302,7 +1302,7 @@ mod test { ), p5 as ( insert into draft_specs (id, draft_id, catalog_name, spec, spec_type) values - ('1112000000000000', '1120000000000000', 'usageB/DerivationA', '{"schema": {}, "key": ["foo"], "derivation": {"transform":{"key": {"source": {"name": "usageB/CollectionA"}}}}}'::json, 'collection') + ('1112000000000000', '1120000000000000', 'usageB/DerivationA', '{"schema": {}, "key": ["foo"], "derive": {"using": {"sqlite": {}}, "transforms":[{"name": "a-transform", "source": "usageB/CollectionA"}]}}'::json, 'collection') ), p6 as ( insert into publications (id, job_status, user_id, draft_id) values diff --git a/crates/assemble/src/lib.rs b/crates/assemble/src/lib.rs index d2f5e1633e..d34ecc6e34 100644 --- a/crates/assemble/src/lib.rs +++ b/crates/assemble/src/lib.rs @@ -427,7 +427,6 @@ pub fn collection_spec( key, projections: _, journals, - derivation: _, derive: _, }, } = collection; diff --git a/crates/flowctl/src/raw/discover.rs b/crates/flowctl/src/raw/discover.rs index f4d7fa3e9a..a777d8cf9b 100644 --- a/crates/flowctl/src/raw/discover.rs +++ b/crates/flowctl/src/raw/discover.rs @@ -113,7 +113,6 @@ pub async fn do_discover( .collect::>(), ), derive: None, - derivation: None, projections: Default::default(), journals: Default::default(), }, diff --git a/crates/models/src/collections.rs b/crates/models/src/collections.rs index 5f6d704a2b..52eaeb4933 100644 --- a/crates/models/src/collections.rs +++ b/crates/models/src/collections.rs @@ -1,4 +1,4 @@ -use super::{CompositeKey, Derivation, Field, JournalTemplate, JsonPointer, RawValue, Schema}; +use super::{CompositeKey, Derivation, Field, JournalTemplate, JsonPointer, Schema}; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; use serde_json::{from_value, json}; @@ -38,10 +38,6 @@ pub struct CollectionDef { // # Derivation which builds this collection as transformations of other collections. #[serde(default, skip_serializing_if = "Option::is_none")] pub derive: Option, - // TODO(johnny): Remove after cut-over. - #[serde(default, skip_serializing_if = "Option::is_none")] - #[schemars(skip)] - pub derivation: Option, } impl CollectionDef { @@ -54,7 +50,6 @@ impl CollectionDef { projections: BTreeMap::new(), journals: JournalTemplate::default(), derive: None, - derivation: None, } } } diff --git a/crates/sources/src/indirect.rs b/crates/sources/src/indirect.rs index 259a1b4d1e..75553a4f66 100644 --- a/crates/sources/src/indirect.rs +++ b/crates/sources/src/indirect.rs @@ -191,7 +191,6 @@ fn indirect_collection( projections: _, journals: _, derive, - derivation: _, }, } = collection; let base = base_name(collection); diff --git a/crates/sources/src/inline.rs b/crates/sources/src/inline.rs index c747777fa1..ce33afe934 100644 --- a/crates/sources/src/inline.rs +++ b/crates/sources/src/inline.rs @@ -62,7 +62,6 @@ fn inline_collection( projections: _, journals: _, derive, - derivation: _, }, } = collection; diff --git a/crates/validation/src/collection.rs b/crates/validation/src/collection.rs index 100fbec20c..281653ff0a 100644 --- a/crates/validation/src/collection.rs +++ b/crates/validation/src/collection.rs @@ -45,7 +45,6 @@ fn walk_collection( projections, journals: _, derive: _, - derivation: _, }, } = collection; let scope = Scope::new(scope); diff --git a/examples/derive-patterns/join-one-sided.flow.yaml b/examples/derive-patterns/join-one-sided.flow.yaml index 577b90eb23..b4c87679ea 100644 --- a/examples/derive-patterns/join-one-sided.flow.yaml +++ b/examples/derive-patterns/join-one-sided.flow.yaml @@ -45,24 +45,6 @@ collections: INSERT INTO join_state (key, rhs) VALUES ($Key, JSON_ARRAY($String)) ON CONFLICT (key) DO UPDATE SET rhs = JSON_INSERT(COALESCE(rhs, '[]'), '$[#]', $String); - derivation: - register: - schema: schema.yaml#Join - initial: { RHS: [] } - - transform: - publishLHS: - source: { name: patterns/ints } - shuffle: { key: [/Key] } - publish: { lambda: typescript } - - updateRHS: - source: { name: patterns/strings } - shuffle: { key: [/Key] } - update: { lambda: typescript } - typescript: - module: join-one-sided.flow.ts - tests: patterns/test/one-sided-join: - ingest: diff --git a/go/bindings/testdata/inc-reset-publish.flow.yaml b/go/bindings/testdata/inc-reset-publish.flow.yaml deleted file mode 100644 index 0f76d75e5c..0000000000 --- a/go/bindings/testdata/inc-reset-publish.flow.yaml +++ /dev/null @@ -1,38 +0,0 @@ -collections: - source: - schema: inc-reset-publish.schema.yaml#/$defs/source - key: [/key] - - derivation: - schema: inc-reset-publish.schema.yaml#/$defs/derived - key: [/key] - projections: - aa: - location: /reset - partition: true - bb: - location: /key - partition: true - - derivation: - register: - initial: { type: set, value: 1000 } - schema: inc-reset-publish.schema.yaml#/$defs/register - typescript: - module: inc-reset-publish.ts - transform: - increment: - source: { name: source } - update: { lambda: typescript } - - publish: - source: { name: source } - publish: { lambda: typescript } - - reset: - source: { name: source } - update: { lambda: typescript } - publish: { lambda: typescript } - -storageMappings: - "": { stores: [{ provider: S3, bucket: a-bucket }] } diff --git a/go/bindings/testdata/inc-reset-publish.schema.yaml b/go/bindings/testdata/inc-reset-publish.schema.yaml deleted file mode 100644 index ba4dd3f3d2..0000000000 --- a/go/bindings/testdata/inc-reset-publish.schema.yaml +++ /dev/null @@ -1,36 +0,0 @@ -$defs: - source: - type: object - properties: - key: { type: string } - reset: { type: integer } - required: [key, reset] - - register: - type: object - reduce: { strategy: merge } - oneOf: - - properties: - type: { const: set } - value: - type: integer - reduce: { strategy: lastWriteWins } - - properties: - type: { const: add } - value: - type: integer - reduce: { strategy: sum } - - required: [type, value] - - derived: - $ref: "#/$defs/source" - reduce: { strategy: merge } - - properties: - values: - type: array - items: { type: integer } - reduce: { strategy: append } - invalid-property: false - required: [values] diff --git a/go/bindings/testdata/inc-reset-publish.ts b/go/bindings/testdata/inc-reset-publish.ts deleted file mode 100644 index ba6784a828..0000000000 --- a/go/bindings/testdata/inc-reset-publish.ts +++ /dev/null @@ -1,29 +0,0 @@ -import { IDerivation, Document, Register, IncrementSource, PublishSource, ResetSource } from 'flow/derivation'; - -// Implementation for derivation go/bindings/testdata/inc-reset-publish.flow.yaml#/collections/derivation/derivation. -export class Derivation implements IDerivation { - incrementUpdate( - _source: IncrementSource, - ): Register[] { - throw new Error("Not implemented"); - } - publishPublish( - _source: PublishSource, - _register: Register, - _previous: Register, - ): Document[] { - throw new Error("Not implemented"); - } - resetUpdate( - _source: ResetSource, - ): Register[] { - throw new Error("Not implemented"); - } - resetPublish( - _source: ResetSource, - _register: Register, - _previous: Register, - ): Document[] { - throw new Error("Not implemented"); - } -} diff --git a/go/bindings/testdata/int-strings.flow.yaml b/go/bindings/testdata/int-strings.flow.yaml index f7bd4e878a..d990967a9d 100644 --- a/go/bindings/testdata/int-strings.flow.yaml +++ b/go/bindings/testdata/int-strings.flow.yaml @@ -17,13 +17,3 @@ collections: projections: part_a: /s/1 part_b: /i - derivation: - typescript: - module: | - import stuff; - doOtherStuff(); - transform: - appendStrings: - source: - name: int-string - publish: { lambda: typescript } diff --git a/site/docs/concepts/collections.md b/site/docs/concepts/collections.md index fb6b49d838..396e7015f5 100644 --- a/site/docs/concepts/collections.md +++ b/site/docs/concepts/collections.md @@ -104,7 +104,7 @@ collections: # Derivation that builds this collection from others through transformations. # See the "Derivations" concept page to learn more. # Optional, type: object - derivation: + derive: ~ ``` ## Schemas