cleanup: remove deprecated derivation property of collection spec

It's been deprecated a while now, and no specs in production use it any
longer.

crates/agent-sql/src/publications.rs

@@ -287,7 +287,7 @@ pub async fn find_tenant_quotas(
                     where
                         live_specs.spec_type = 'capture' or
                         live_specs.spec_type = 'materialization' or
-                        live_specs.spec_type = 'collection' and live_specs.spec->'derivation' is not null
+                        live_specs.spec_type = 'collection' and live_specs.spec->'derive' is not null
                 ))::integer as tasks_used,
                 (count(live_specs.catalog_name) filter (
                     where live_specs.spec_type = 'collection'

crates/agent-sql/tests/publications.rs

@@ -266,7 +266,7 @@ async fn test_tenant_usage_quotas() {
             ('1000000000000000', 'usageA/CollectionA', '1', 'collection', 'bbbbbbbbbbbbbbbb', 'bbbbbbbbbbbbbbbb'),
             ('2000000000000000', 'usageA/CaptureA', '1', 'capture', 'bbbbbbbbbbbbbbbb', 'bbbbbbbbbbbbbbbb'),
             ('3000000000000000', 'usageA/MaterializationA', '1', 'materialization', 'bbbbbbbbbbbbbbbb', 'bbbbbbbbbbbbbbbb'),
-            ('5000000000000000', 'usageA/DerivationA', '{"derivation": {}}'::json, 'collection', 'bbbbbbbbbbbbbbbb', 'bbbbbbbbbbbbbbbb'),
+            ('5000000000000000', 'usageA/DerivationA', '{"derive": {}}'::json, 'collection', 'bbbbbbbbbbbbbbbb', 'bbbbbbbbbbbbbbbb'),
             ('6000000000000000', 'usageB/CaptureA', '1', 'capture', 'bbbbbbbbbbbbbbbb', 'bbbbbbbbbbbbbbbb'),
             ('7000000000000000', 'usageB/CaptureB', '1', 'capture', 'bbbbbbbbbbbbbbbb', 'bbbbbbbbbbbbbbbb'),
             ('8000000000000000', 'usageB/CaptureC', '1', 'capture', 'bbbbbbbbbbbbbbbb', 'bbbbbbbbbbbbbbbb'),

crates/agent/src/discovers/specs.rs

@@ -138,7 +138,6 @@ pub fn merge_collections(
                     projections: Default::default(),
                     journals: Default::default(),
                     derive: None,
-                    derivation: None,
                 });
 
         if collection.read_schema.is_some() {

crates/agent/src/publications/specs.rs

@@ -1302,7 +1302,7 @@ mod test {
               ),
               p5 as (
                 insert into draft_specs (id, draft_id, catalog_name, spec, spec_type) values
-                ('1112000000000000', '1120000000000000', 'usageB/DerivationA', '{"schema": {}, "key": ["foo"], "derivation": {"transform":{"key": {"source": {"name": "usageB/CollectionA"}}}}}'::json, 'collection')
+                ('1112000000000000', '1120000000000000', 'usageB/DerivationA', '{"schema": {}, "key": ["foo"], "derive": {"using": {"sqlite": {}}, "transforms":[{"name": "a-transform", "source": "usageB/CollectionA"}]}}'::json, 'collection')
               ),
               p6 as (
                 insert into publications (id, job_status, user_id, draft_id) values

crates/assemble/src/lib.rs

@@ -427,7 +427,6 @@ pub fn collection_spec(
                 key,
                 projections: _,
                 journals,
-                derivation: _,
                 derive: _,
             },
     } = collection;

crates/flowctl/src/raw/discover.rs

@@ -113,7 +113,6 @@ pub async fn do_discover(
                             .collect::<Vec<JsonPointer>>(),
                     ),
                     derive: None,
-                    derivation: None,
                     projections: Default::default(),
                     journals: Default::default(),
                 },

crates/models/src/collections.rs

@@ -1,4 +1,4 @@
-use super::{CompositeKey, Derivation, Field, JournalTemplate, JsonPointer, RawValue, Schema};
+use super::{CompositeKey, Derivation, Field, JournalTemplate, JsonPointer, Schema};
 use schemars::JsonSchema;
 use serde::{Deserialize, Serialize};
 use serde_json::{from_value, json};
@@ -38,10 +38,6 @@ pub struct CollectionDef {
     // # Derivation which builds this collection as transformations of other collections.
     #[serde(default, skip_serializing_if = "Option::is_none")]
     pub derive: Option<Derivation>,
-    // TODO(johnny): Remove after cut-over.
-    #[serde(default, skip_serializing_if = "Option::is_none")]
-    #[schemars(skip)]
-    pub derivation: Option<RawValue>,
 }
 
 impl CollectionDef {
@@ -54,7 +50,6 @@ impl CollectionDef {
             projections: BTreeMap::new(),
             journals: JournalTemplate::default(),
             derive: None,
-            derivation: None,
         }
     }
 }

crates/sources/src/indirect.rs

@@ -191,7 +191,6 @@ fn indirect_collection(
                 projections: _,
                 journals: _,
                 derive,
-                derivation: _,
             },
     } = collection;
     let base = base_name(collection);

crates/sources/src/inline.rs

@@ -62,7 +62,6 @@ fn inline_collection(
                 projections: _,
                 journals: _,
                 derive,
-                derivation: _,
             },
     } = collection;
 

crates/validation/src/collection.rs

@@ -45,7 +45,6 @@ fn walk_collection(
                 projections,
                 journals: _,
                 derive: _,
-                derivation: _,
             },
     } = collection;
     let scope = Scope::new(scope);

examples/derive-patterns/join-one-sided.flow.yaml

@@ -45,24 +45,6 @@ collections:
             INSERT INTO join_state (key, rhs) VALUES ($Key, JSON_ARRAY($String))
             ON CONFLICT (key) DO UPDATE SET rhs = JSON_INSERT(COALESCE(rhs, '[]'), '$[#]', $String);
 
-    derivation:
-      register:
-        schema: schema.yaml#Join
-        initial: { RHS: [] }
-
-      transform:
-        publishLHS:
-          source: { name: patterns/ints }
-          shuffle: { key: [/Key] }
-          publish: { lambda: typescript }
-
-        updateRHS:
-          source: { name: patterns/strings }
-          shuffle: { key: [/Key] }
-          update: { lambda: typescript }
-      typescript:
-        module: join-one-sided.flow.ts
-
 tests:
   patterns/test/one-sided-join:
     - ingest:

go/bindings/testdata/int-strings.flow.yaml

@@ -17,13 +17,3 @@ collections:
     projections:
       part_a: /s/1
       part_b: /i
-    derivation:
-      typescript:
-        module: |
-          import stuff;
-          doOtherStuff();
-      transform:
-        appendStrings:
-          source:
-            name: int-string
-          publish: { lambda: typescript }

site/docs/concepts/collections.md

@@ -104,7 +104,7 @@ collections:
     # Derivation that builds this collection from others through transformations.
     # See the "Derivations" concept page to learn more.
     # Optional, type: object
-    derivation:
+    derive: ~
 ```
 
 ## Schemas