feat(clickhouse): Rework tables configuration (#2657)

- Use the new table configuration for events
- Remove all agg tables
- Add flat value in `events_enriched` table
- Remove pre aggregated services

app/models/clickhouse/events_enriched.rb

@@ -10,15 +10,15 @@ class EventsEnriched < BaseRecord
 #
 # Table name: events_enriched
 #
-#  aggregation_type         :string
-#  code                     :string           not null
-#  filters                  :string           not null
-#  grouped_by               :string           not null
-#  properties               :string           not null
-#  timestamp                :datetime         not null
-#  value                    :string
-#  charge_id                :string           not null
-#  external_subscription_id :string           not null
-#  organization_id          :string           not null
-#  transaction_id           :string           not null
+#  code                       :string           not null
+#  decimal_value              :decimal(26, )
+#  enriched_at                :datetime         not null
+#  precise_total_amount_cents :decimal(40, 15)
+#  properties                 :string           not null
+#  sorted_properties          :string           not null
+#  timestamp                  :datetime         not null
+#  value                      :string
+#  external_subscription_id   :string           not null
+#  organization_id            :string           not null
+#  transaction_id             :string           not null
 #

app/services/events/create_batch_service.rb

@@ -82,7 +82,7 @@ def produce_kafka_event(event)
           code: event.code,
           properties: event.properties,
           ingested_at: Time.zone.now.iso8601[...-1],
-          precise_total_amount_cents: event.precise_total_amount_cents
+          precise_total_amount_cents: event.precise_total_amount_cents.present? ? event.precise_total_amount_cents.to_s : "0.0"
         }.to_json
       )
     end

app/services/events/create_service.rb

@@ -51,9 +51,11 @@ def produce_kafka_event(event)
           external_customer_id: event.external_customer_id,
           external_subscription_id: event.external_subscription_id,
           transaction_id: event.transaction_id,
-          timestamp: event.timestamp.iso8601[...-1], # NOTE: Removes trailing 'Z' to allow clickhouse parsing
+          # NOTE: Removes trailing 'Z' to allow clickhouse parsing
+          timestamp: event.timestamp.iso8601[...-1],
           code: event.code,
-          precise_total_amount_cents: event.precise_total_amount_cents,
+          # NOTE: Default value to 0.0 is required for clickhouse parsing
+          precise_total_amount_cents: event.precise_total_amount_cents.present? ? event.precise_total_amount_cents.to_s : "0.0",
           properties: event.properties,
           ingested_at: Time.zone.now.iso8601[...-1]
         }.to_json

db/clickhouse_migrate/20231024084411_create_events_raw.rb

@@ -15,6 +15,7 @@ def change
       t.datetime :timestamp, null: false, precision: 3
       t.string :code, null: false
       t.string :properties, map: true, null: false
+      t.decimal :precise_total_amount_cents, precision: 40, scale: 15
     end
   end
 end

db/clickhouse_migrate/20231026124912_create_events_raw_queue.rb

@@ -19,6 +19,7 @@ def change
       t.string :timestamp, null: false
       t.string :code, null: false
       t.string :properties, null: false
+      t.decimal :precise_total_amount_cents, precision: 40, scale: 15
     end
   end
 end

db/clickhouse_migrate/20231030163703_create_events_raw_mv.rb

@@ -10,7 +10,8 @@ def change
         transaction_id,
         toDateTime64(timestamp, 3) as timestamp,
         code,
-        JSONExtract(properties, 'Map(String, String)') as properties
+        JSONExtract(properties, 'Map(String, String)') as properties,
+        precise_total_amount_cents
       FROM events_raw_queue
     SQL