From 214fe33f198613a43daa5ebede0a9d92ab97209c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9my=20Denquin?= Date: Mon, 7 Oct 2024 18:12:06 +0200 Subject: [PATCH] feat(clickhouse): Rework tables configuration (#2657) - Use the new table configuration for events - Remove all agg tables - Add flat value in `events_enriched` table - Remove pre aggregated services --- app/models/clickhouse/events_enriched.rb | 22 +- app/services/events/create_batch_service.rb | 2 +- app/services/events/create_service.rb | 6 +- .../stores/clickhouse/pre_aggregated/base.rb | 151 ------ .../clickhouse/pre_aggregated/count_query.rb | 29 -- .../clickhouse/pre_aggregated/latest_query.rb | 49 -- .../clickhouse/pre_aggregated/max_query.rb | 29 -- .../clickhouse/pre_aggregated/sum_query.rb | 29 -- .../20231024084411_create_events_raw.rb | 1 + .../20231026124912_create_events_raw_queue.rb | 1 + .../20231030163703_create_events_raw_mv.rb | 3 +- .../20240705080709_create_events_enriched.rb | 21 +- ...0705084952_create_events_enriched_queue.rb | 5 +- ...0240705085501_create_events_enriched_mv.rb | 5 +- .../20240705090006_create_events_sum_agg.rb | 21 - ...20240705090310_create_events_sum_agg_mv.rb | 29 -- .../20240709095907_create_events_count_agg.rb | 21 - ...240709100047_create_events_count_agg_mv.rb | 29 -- .../20240709135506_create_events_max_agg.rb | 22 - ...20240709135535_create_events_max_agg_mv.rb | 29 -- ...dd_precise_total_amount_cents_to_events.rb | 7 - .../pre_aggregated/count_query_spec.rb | 484 ------------------ .../pre_aggregated/latest_query_spec.rb | 323 ------------ .../pre_aggregated/max_query_spec.rb | 484 ------------------ .../pre_aggregated/sum_query_spec.rb | 484 ------------------ 25 files changed, 36 insertions(+), 2250 deletions(-) delete mode 100644 app/services/events/stores/clickhouse/pre_aggregated/base.rb delete mode 100644 app/services/events/stores/clickhouse/pre_aggregated/count_query.rb delete mode 100644 app/services/events/stores/clickhouse/pre_aggregated/latest_query.rb delete mode 100644 app/services/events/stores/clickhouse/pre_aggregated/max_query.rb delete mode 100644 app/services/events/stores/clickhouse/pre_aggregated/sum_query.rb delete mode 100644 db/clickhouse_migrate/20240705090006_create_events_sum_agg.rb delete mode 100644 db/clickhouse_migrate/20240705090310_create_events_sum_agg_mv.rb delete mode 100644 db/clickhouse_migrate/20240709095907_create_events_count_agg.rb delete mode 100644 db/clickhouse_migrate/20240709100047_create_events_count_agg_mv.rb delete mode 100644 db/clickhouse_migrate/20240709135506_create_events_max_agg.rb delete mode 100644 db/clickhouse_migrate/20240709135535_create_events_max_agg_mv.rb delete mode 100644 db/clickhouse_migrate/20240923124501_add_precise_total_amount_cents_to_events.rb delete mode 100644 spec/services/events/stores/clickhouse/pre_aggregated/count_query_spec.rb delete mode 100644 spec/services/events/stores/clickhouse/pre_aggregated/latest_query_spec.rb delete mode 100644 spec/services/events/stores/clickhouse/pre_aggregated/max_query_spec.rb delete mode 100644 spec/services/events/stores/clickhouse/pre_aggregated/sum_query_spec.rb diff --git a/app/models/clickhouse/events_enriched.rb b/app/models/clickhouse/events_enriched.rb index 07c1450b5b9..d9d0939984e 100644 --- a/app/models/clickhouse/events_enriched.rb +++ b/app/models/clickhouse/events_enriched.rb @@ -10,15 +10,15 @@ class EventsEnriched < BaseRecord # # Table name: events_enriched # -# aggregation_type :string -# code :string not null -# filters :string not null -# grouped_by :string not null -# properties :string not null -# timestamp :datetime not null -# value :string -# charge_id :string not null -# external_subscription_id :string not null -# organization_id :string not null -# transaction_id :string not null +# code :string not null +# decimal_value :decimal(26, ) +# enriched_at :datetime not null +# precise_total_amount_cents :decimal(40, 15) +# properties :string not null +# sorted_properties :string not null +# timestamp :datetime not null +# value :string +# external_subscription_id :string not null +# organization_id :string not null +# transaction_id :string not null # diff --git a/app/services/events/create_batch_service.rb b/app/services/events/create_batch_service.rb index fb114333499..0ed0dbd7bb1 100644 --- a/app/services/events/create_batch_service.rb +++ b/app/services/events/create_batch_service.rb @@ -82,7 +82,7 @@ def produce_kafka_event(event) code: event.code, properties: event.properties, ingested_at: Time.zone.now.iso8601[...-1], - precise_total_amount_cents: event.precise_total_amount_cents + precise_total_amount_cents: event.precise_total_amount_cents.present? ? event.precise_total_amount_cents.to_s : "0.0" }.to_json ) end diff --git a/app/services/events/create_service.rb b/app/services/events/create_service.rb index 985b6ba1114..621cf117d77 100644 --- a/app/services/events/create_service.rb +++ b/app/services/events/create_service.rb @@ -51,9 +51,11 @@ def produce_kafka_event(event) external_customer_id: event.external_customer_id, external_subscription_id: event.external_subscription_id, transaction_id: event.transaction_id, - timestamp: event.timestamp.iso8601[...-1], # NOTE: Removes trailing 'Z' to allow clickhouse parsing + # NOTE: Removes trailing 'Z' to allow clickhouse parsing + timestamp: event.timestamp.iso8601[...-1], code: event.code, - precise_total_amount_cents: event.precise_total_amount_cents, + # NOTE: Default value to 0.0 is required for clickhouse parsing + precise_total_amount_cents: event.precise_total_amount_cents.present? ? event.precise_total_amount_cents.to_s : "0.0", properties: event.properties, ingested_at: Time.zone.now.iso8601[...-1] }.to_json diff --git a/app/services/events/stores/clickhouse/pre_aggregated/base.rb b/app/services/events/stores/clickhouse/pre_aggregated/base.rb deleted file mode 100644 index 52e03666019..00000000000 --- a/app/services/events/stores/clickhouse/pre_aggregated/base.rb +++ /dev/null @@ -1,151 +0,0 @@ -# frozen_string_literal: true - -module Events - module Stores - module Clickhouse - module PreAggregated - class Base < BaseService - def initialize(subscription:, boundaries:) - @subscription = subscription - @boundaries = boundaries - - super - end - - def call - result.charges_units = {} - - append_to_result(pre_aggregated_query, initial_result: result.charges_units) if pre_aggregated_model.present? - append_to_result(enriched_events_query, initial_result: result.charges_units) - - result - end - - protected - - attr_reader :subscription, :boundaries - - delegate :organization, to: :subscription - - def aggregation_type - # NOTE: override in subclasses - raise NotImplementedError - end - - def pre_aggregated_model - # NOTE: override in subclasses - raise NotImplementedError - end - - def clickhouse_aggregation - # NOTE: override in subclasses - raise NotImplementedError - end - - def from_datetime - @from_datetime ||= boundaries[:from_datetime] - end - - def to_datetime - @to_datetime ||= boundaries[:to_datetime] - end - - def check_before_boundaries? - from_datetime.beginning_of_hour != from_datetime - end - - def check_after_boundaries? - to_datetime.end_of_hour != to_datetime # TODO: check for miliseconds - end - - def charge_ids - @charge_ids ||= subscription.plan.charges.joins(:billable_metric) - .merge(BillableMetric.where(aggregation_type: aggregation_type)) - .pluck(:id) - end - - def pre_aggregated_query - sql = pre_aggregated_model.where(organization_id: organization.id) - .where(external_subscription_id: subscription.external_id) - .where(charge_id: charge_ids) - .where(timestamp: from_datetime...) - .where(timestamp: ..to_datetime.beginning_of_hour) - .group(:charge_id, :grouped_by, :filters) - .select("#{clickhouse_aggregation}(value) as units, charge_id, grouped_by, filters") - .to_sql - - pre_aggregated_model.connection.select_all(sql) - end - - def enriched_events_query - return [] if !check_before_boundaries? && !check_after_to_boundaries? - - base_scope = ::Clickhouse::EventsEnriched - .where(organization_id: organization.id) - .where(external_subscription_id: subscription.external_id) - .where(charge_id: charge_ids) - .group(:charge_id, :grouped_by, :filters) - .select("#{clickhouse_aggregation}(toDecimal128(value, #{ClickhouseStore::DECIMAL_SCALE})) as units, charge_id, grouped_by, filters") - - sub_scope = ::Clickhouse::EventsEnriched.all - if check_before_boundaries? - sub_scope = sub_scope.where(timestamp: from_datetime...from_datetime.end_of_hour) - end - - if check_after_boundaries? - sub_scope = sub_scope.or(::Clickhouse::EventsEnriched.where(timestamp: to_datetime.beginning_of_hour...to_datetime)) - end - - ::Clickhouse::EventsEnriched.connection.select_all(base_scope.merge(sub_scope).to_sql) - end - - # NOTE: Build a list of units indexed by charge_id, filters and or grouped_by - # The format of the result is similar to the following: - # - # { - # "XXXXX" => { - # filters: { - # '{"key1":"value","key2":"value"}' => {grouped_by: {}, units: 12.0} - # '{"key5":"value","key3":"value"}' => {grouped_by: {'{"group_1":"value1","group2":"value2"}' => {units: 12.0}}}, units: 0.0} - # } - # grouped_by: {'{"group_1":"value1","group2":"value2"}' => {units: 12.0}}} - # units: 12.0 - # } - # #... - # } - def append_to_result(grouped_rows, initial_result: {}) - grouped_rows.each_with_object(initial_result) do |row, result| - charge_id = row['charge_id'] - units = row['units'] - - result[charge_id] ||= {filters: {}, grouped_by: {}, units: 0} - - if row['filters'].present? - result[charge_id][:filters][row['filters'].to_json] ||= {grouped_by: {}, units: 0} - - if row['grouped_by'].present? - result[charge_id][:filters][row['filters'].to_json][:grouped_by][row['grouped_by'].to_json] ||= {units: 0} - assign_units(result[charge_id][:filters][row['filters'].to_json][:grouped_by][row['grouped_by'].to_json], units) - else - assign_units(result[charge_id][:filters][row['filters'].to_json], units) - end - elsif row['grouped_by'].present? - result[charge_id][:grouped_by][row['grouped_by'].to_json] ||= {units: 0} - assign_units(result[charge_id][:grouped_by][row['grouped_by'].to_json], units) - else - assign_units(result[charge_id], units) - end - - result - end - end - - def assign_units(bucket, units) - # NOTE: override in subclasses - raise NotImplementedError - end - end - end - end - end -end diff --git a/app/services/events/stores/clickhouse/pre_aggregated/count_query.rb b/app/services/events/stores/clickhouse/pre_aggregated/count_query.rb deleted file mode 100644 index a602a5463f6..00000000000 --- a/app/services/events/stores/clickhouse/pre_aggregated/count_query.rb +++ /dev/null @@ -1,29 +0,0 @@ -# frozen_string_literal: true - -module Events - module Stores - module Clickhouse - module PreAggregated - class CountQuery < Base - protected - - def aggregation_type - @aggregation_type ||= 'count_agg' - end - - def pre_aggregated_model - @pre_aggregated_model ||= ::Clickhouse::EventsCountAgg - end - - def clickhouse_aggregation - @clickhouse_aggregation ||= 'sum' - end - - def assign_units(bucket, units) - bucket[:units] += units - end - end - end - end - end -end diff --git a/app/services/events/stores/clickhouse/pre_aggregated/latest_query.rb b/app/services/events/stores/clickhouse/pre_aggregated/latest_query.rb deleted file mode 100644 index b437e2fd50a..00000000000 --- a/app/services/events/stores/clickhouse/pre_aggregated/latest_query.rb +++ /dev/null @@ -1,49 +0,0 @@ -# frozen_string_literal: true - -module Events - module Stores - module Clickhouse - module PreAggregated - class LatestQuery < Base - protected - - def aggregation_type - @aggregation_type ||= 'latest_agg' - end - - def pre_aggregated_model - nil - end - - def clickhouse_aggregation - nil - end - - def assign_units(bucket, units) - bucket[:units] = units - end - - def enriched_events_query - query = ::Clickhouse::EventsEnriched - .where(organization_id: organization.id) - .where(external_subscription_id: subscription.external_id) - .where(charge_id: charge_ids) - .where(timestamp: from_datetime...to_datetime) # TODO: check for miliseconds - .select( - [ - 'DISTINCT ON (events_enriched.charge_id, events_enriched.grouped_by, events_enriched.filters) events_enriched.charge_id', - 'events_enriched.grouped_by', - 'events_enriched.filters', - 'events_enriched.timestamp', - "toDecimal128(events_enriched.value, #{ClickhouseStore::DECIMAL_SCALE}) as units" - ].join(', ') - ) - .order('events_enriched.charge_id, events_enriched.grouped_by, events_enriched.filters, events_enriched.timestamp DESC') - - ::Clickhouse::EventsEnriched.connection.select_all(query.to_sql) - end - end - end - end - end -end diff --git a/app/services/events/stores/clickhouse/pre_aggregated/max_query.rb b/app/services/events/stores/clickhouse/pre_aggregated/max_query.rb deleted file mode 100644 index eefe99dd174..00000000000 --- a/app/services/events/stores/clickhouse/pre_aggregated/max_query.rb +++ /dev/null @@ -1,29 +0,0 @@ -# frozen_string_literal: true - -module Events - module Stores - module Clickhouse - module PreAggregated - class MaxQuery < Base - protected - - def aggregation_type - @aggregation_type ||= 'max_agg' - end - - def pre_aggregated_model - @pre_aggregated_model ||= ::Clickhouse::EventsMaxAgg - end - - def clickhouse_aggregation - @clickhouse_aggregation ||= 'max' - end - - def assign_units(bucket, units) - bucket[:units] = units if units > bucket[:units] - end - end - end - end - end -end diff --git a/app/services/events/stores/clickhouse/pre_aggregated/sum_query.rb b/app/services/events/stores/clickhouse/pre_aggregated/sum_query.rb deleted file mode 100644 index 96880b8b5a9..00000000000 --- a/app/services/events/stores/clickhouse/pre_aggregated/sum_query.rb +++ /dev/null @@ -1,29 +0,0 @@ -# frozen_string_literal: true - -module Events - module Stores - module Clickhouse - module PreAggregated - class SumQuery < Base - protected - - def aggregation_type - @aggregation_type ||= 'sum_agg' - end - - def pre_aggregated_model - @pre_aggregated_model ||= ::Clickhouse::EventsSumAgg - end - - def clickhouse_aggregation - @clickhouse_aggregation ||= 'sum' - end - - def assign_units(bucket, units) - bucket[:units] += units - end - end - end - end - end -end diff --git a/db/clickhouse_migrate/20231024084411_create_events_raw.rb b/db/clickhouse_migrate/20231024084411_create_events_raw.rb index e6ceb68636b..1df6ac135bf 100644 --- a/db/clickhouse_migrate/20231024084411_create_events_raw.rb +++ b/db/clickhouse_migrate/20231024084411_create_events_raw.rb @@ -15,6 +15,7 @@ def change t.datetime :timestamp, null: false, precision: 3 t.string :code, null: false t.string :properties, map: true, null: false + t.decimal :precise_total_amount_cents, precision: 40, scale: 15 end end end diff --git a/db/clickhouse_migrate/20231026124912_create_events_raw_queue.rb b/db/clickhouse_migrate/20231026124912_create_events_raw_queue.rb index 046697df8b2..fe84692a5f5 100644 --- a/db/clickhouse_migrate/20231026124912_create_events_raw_queue.rb +++ b/db/clickhouse_migrate/20231026124912_create_events_raw_queue.rb @@ -19,6 +19,7 @@ def change t.string :timestamp, null: false t.string :code, null: false t.string :properties, null: false + t.decimal :precise_total_amount_cents, precision: 40, scale: 15 end end end diff --git a/db/clickhouse_migrate/20231030163703_create_events_raw_mv.rb b/db/clickhouse_migrate/20231030163703_create_events_raw_mv.rb index 00927e8aa99..ccef9126133 100644 --- a/db/clickhouse_migrate/20231030163703_create_events_raw_mv.rb +++ b/db/clickhouse_migrate/20231030163703_create_events_raw_mv.rb @@ -10,7 +10,8 @@ def change transaction_id, toDateTime64(timestamp, 3) as timestamp, code, - JSONExtract(properties, 'Map(String, String)') as properties + JSONExtract(properties, 'Map(String, String)') as properties, + precise_total_amount_cents FROM events_raw_queue SQL diff --git a/db/clickhouse_migrate/20240705080709_create_events_enriched.rb b/db/clickhouse_migrate/20240705080709_create_events_enriched.rb index f46b655218a..319d2b965df 100644 --- a/db/clickhouse_migrate/20240705080709_create_events_enriched.rb +++ b/db/clickhouse_migrate/20240705080709_create_events_enriched.rb @@ -3,12 +3,19 @@ class CreateEventsEnriched < ActiveRecord::Migration[7.1] def change options = <<-SQL - ReplacingMergeTree - ORDER BY ( + ReplacingMergeTree(timestamp) + PRIMARY KEY ( organization_id, + code, external_subscription_id, + toDate(timestamp) + ) + ORDER BY ( + organization_id, code, - charge_id, + external_subscription_id, + toDate(timestamp), + timestamp, transaction_id ) SQL @@ -20,11 +27,11 @@ def change t.datetime :timestamp, null: false, precision: 3 t.string :transaction_id, null: false t.string :properties, map: true, null: false + t.string :sorted_properties, map: true, null: false, default: -> { 'mapSort(properties)' } t.string :value - t.string :charge_id, null: false - t.string :aggregation_type - t.string :filters, map: :array, null: false - t.string :grouped_by, map: true, null: false + t.decimal :decimal_value, precision: 26 + t.datetime :enriched_at, null: false, precision: 3, default: -> { 'now()' } + t.decimal :precise_total_amount_cents, precision: 40, scale: 15 end end end diff --git a/db/clickhouse_migrate/20240705084952_create_events_enriched_queue.rb b/db/clickhouse_migrate/20240705084952_create_events_enriched_queue.rb index fc1cee6017f..fb68e963d2a 100644 --- a/db/clickhouse_migrate/20240705084952_create_events_enriched_queue.rb +++ b/db/clickhouse_migrate/20240705084952_create_events_enriched_queue.rb @@ -19,10 +19,7 @@ def change t.string :transaction_id, null: false t.string :properties, null: false t.string :value - t.string :charge_id, null: false - t.string :aggregation_type, null: false - t.string :filters - t.string :grouped_by + t.decimal :precise_total_amount_cents, precision: 40, scale: 15 end end end diff --git a/db/clickhouse_migrate/20240705085501_create_events_enriched_mv.rb b/db/clickhouse_migrate/20240705085501_create_events_enriched_mv.rb index 327d0756ab0..b0b76e5fd54 100644 --- a/db/clickhouse_migrate/20240705085501_create_events_enriched_mv.rb +++ b/db/clickhouse_migrate/20240705085501_create_events_enriched_mv.rb @@ -11,10 +11,7 @@ def change code, JSONExtract(properties, 'Map(String, String)') AS properties, value, - charge_id, - aggregation_type, - JSONExtract(coalesce(filters, '{}'), 'Map(String, Array(String))') AS filters, - JSONExtract(coalesce(grouped_by, '{}'), 'Map(String, String)') AS grouped_by + precise_total_amount_cents FROM events_enriched_queue SQL diff --git a/db/clickhouse_migrate/20240705090006_create_events_sum_agg.rb b/db/clickhouse_migrate/20240705090006_create_events_sum_agg.rb deleted file mode 100644 index b2f8589a1b6..00000000000 --- a/db/clickhouse_migrate/20240705090006_create_events_sum_agg.rb +++ /dev/null @@ -1,21 +0,0 @@ -# frozen_string_literal: false - -class CreateEventsSumAgg < ActiveRecord::Migration[7.1] - def change - options = <<-SQL - SummingMergeTree - PRIMARY KEY (organization_id, external_subscription_id, code, charge_id, timestamp, filters, grouped_by) - SQL - - create_table :events_sum_agg, id: false, options: do |t| - t.string :organization_id, null: false - t.string :external_subscription_id, null: false - t.string :code, null: false - t.string :charge_id, null: false - t.decimal :value, precision: 26 - t.datetime :timestamp, precision: 3, null: false - t.string :filters, map: :array, null: false - t.string :grouped_by, map: true, null: false - end - end -end diff --git a/db/clickhouse_migrate/20240705090310_create_events_sum_agg_mv.rb b/db/clickhouse_migrate/20240705090310_create_events_sum_agg_mv.rb deleted file mode 100644 index 492ebab6a83..00000000000 --- a/db/clickhouse_migrate/20240705090310_create_events_sum_agg_mv.rb +++ /dev/null @@ -1,29 +0,0 @@ -# frozen_string_literal: false - -class CreateEventsSumAggMv < ActiveRecord::Migration[7.1] - def change - sql = <<-SQL - SELECT - organization_id, - external_subscription_id, - code, - charge_id, - sum(toDecimal128(value, 26)) as value, - toStartOfHour(timestamp) as timestamp, - filters, - grouped_by - FROM events_enriched - WHERE aggregation_type = 'sum_agg' - GROUP BY - organization_id, - external_subscription_id, - code, - charge_id, - toStartOfHour(timestamp) as timestamp, - filters, - grouped_by - SQL - - create_view :events_sum_agg_mv, materialized: true, as: sql, to: 'events_sum_agg' - end -end diff --git a/db/clickhouse_migrate/20240709095907_create_events_count_agg.rb b/db/clickhouse_migrate/20240709095907_create_events_count_agg.rb deleted file mode 100644 index 9c7fa506b3d..00000000000 --- a/db/clickhouse_migrate/20240709095907_create_events_count_agg.rb +++ /dev/null @@ -1,21 +0,0 @@ -# frozen_string_literal: false - -class CreateEventsCountAgg < ActiveRecord::Migration[7.1] - def change - options = <<-SQL - SummingMergeTree - PRIMARY KEY (organization_id, external_subscription_id, code, charge_id, timestamp, filters, grouped_by) - SQL - - create_table :events_count_agg, id: false, options: do |t| - t.string :organization_id, null: false - t.string :external_subscription_id, null: false - t.string :code, null: false - t.string :charge_id, null: false - t.decimal :value, precision: 26 - t.datetime :timestamp, precision: 3, null: false - t.string :filters, map: :array, null: false - t.string :grouped_by, map: true, null: false - end - end -end diff --git a/db/clickhouse_migrate/20240709100047_create_events_count_agg_mv.rb b/db/clickhouse_migrate/20240709100047_create_events_count_agg_mv.rb deleted file mode 100644 index 0d4479a11ab..00000000000 --- a/db/clickhouse_migrate/20240709100047_create_events_count_agg_mv.rb +++ /dev/null @@ -1,29 +0,0 @@ -# frozen_string_literal: false - -class CreateEventsCountAggMv < ActiveRecord::Migration[7.1] - def change - sql = <<-SQL - SELECT - organization_id, - external_subscription_id, - code, - charge_id, - sum(toDecimal128(value, 26)) as value, - toStartOfHour(timestamp) as timestamp, - filters, - grouped_by - FROM events_enriched - WHERE aggregation_type = 'count_agg' - GROUP BY - organization_id, - external_subscription_id, - code, - charge_id, - toStartOfHour(timestamp) as timestamp, - filters, - grouped_by - SQL - - create_view :events_count_agg_mv, materialized: true, as: sql, to: 'events_count_agg' - end -end diff --git a/db/clickhouse_migrate/20240709135506_create_events_max_agg.rb b/db/clickhouse_migrate/20240709135506_create_events_max_agg.rb deleted file mode 100644 index 7a9f448b0f5..00000000000 --- a/db/clickhouse_migrate/20240709135506_create_events_max_agg.rb +++ /dev/null @@ -1,22 +0,0 @@ -# frozen_string_literal: false - -class CreateEventsMaxAgg < ActiveRecord::Migration[7.1] - def change - options = <<-SQL - AggregatingMergeTree() - PRIMARY KEY (organization_id, external_subscription_id, code, charge_id, timestamp, filters, grouped_by) - SQL - - create_table :events_max_agg, id: false, options: do |t| - t.string :organization_id, null: false - t.string :external_subscription_id, null: false - t.string :code, null: false - t.string :charge_id, null: false - t.datetime :timestamp, precision: 3, null: false - t.string :filters, map: :array, null: false - t.string :grouped_by, map: true, null: false - end - - add_column :events_max_agg, :value, 'AggregateFunction(max, Decimal(38, 26))', null: false # rubocop:disable Rails/NotNullColumn - end -end diff --git a/db/clickhouse_migrate/20240709135535_create_events_max_agg_mv.rb b/db/clickhouse_migrate/20240709135535_create_events_max_agg_mv.rb deleted file mode 100644 index 8993283e608..00000000000 --- a/db/clickhouse_migrate/20240709135535_create_events_max_agg_mv.rb +++ /dev/null @@ -1,29 +0,0 @@ -# frozen_string_literal: false - -class CreateEventsMaxAggMv < ActiveRecord::Migration[7.1] - def change - sql = <<-SQL - SELECT - organization_id, - external_subscription_id, - code, - charge_id, - maxState(toDecimal128(coalesce(value, '0'), 26)) as value, - toStartOfHour(timestamp) as timestamp, - filters, - grouped_by - FROM events_enriched - WHERE aggregation_type = 'max_agg' - GROUP BY - organization_id, - external_subscription_id, - code, - charge_id, - toStartOfHour(timestamp) as timestamp, - filters, - grouped_by - SQL - - create_view :events_max_agg_mv, materialized: true, as: sql, to: 'events_max_agg' - end -end diff --git a/db/clickhouse_migrate/20240923124501_add_precise_total_amount_cents_to_events.rb b/db/clickhouse_migrate/20240923124501_add_precise_total_amount_cents_to_events.rb deleted file mode 100644 index cb683bbf7ad..00000000000 --- a/db/clickhouse_migrate/20240923124501_add_precise_total_amount_cents_to_events.rb +++ /dev/null @@ -1,7 +0,0 @@ -# frozen_string_literal: true - -class AddPreciseTotalAmountCentsToEvents < ActiveRecord::Migration[7.1] - def change - add_column :events_raw, :precise_total_amount_cents, :decimal, precision: 40, scale: 15 - end -end diff --git a/spec/services/events/stores/clickhouse/pre_aggregated/count_query_spec.rb b/spec/services/events/stores/clickhouse/pre_aggregated/count_query_spec.rb deleted file mode 100644 index 054bb107349..00000000000 --- a/spec/services/events/stores/clickhouse/pre_aggregated/count_query_spec.rb +++ /dev/null @@ -1,484 +0,0 @@ -# frozen_string_literal: true - -require 'rails_helper' - -RSpec.describe Events::Stores::Clickhouse::PreAggregated::CountQuery, type: :service, clickhouse: true do - subject(:pre_aggregated_query) { described_class.new(subscription:, boundaries:) } - - let(:subscription) { create(:subscription, customer:, plan:) } - - let(:organization) { create(:organization) } - let(:customer) { create(:customer, organization:) } - - let(:billable_metric1) { create(:billable_metric, organization:) } - let(:billable_metric2) { create(:billable_metric, organization:) } - - let(:plan) { create(:plan, organization:) } - let(:charge1) { create(:standard_charge, billable_metric: billable_metric1, plan:) } - let(:charge2) { create(:standard_charge, billable_metric: billable_metric2, plan:) } - - let(:boundaries) { {from_datetime:, to_datetime:} } - let(:from_datetime) { Time.zone.parse('2024-07-01T03:42:00') } - let(:to_datetime) { Time.zone.parse('2024-07-31T22:47:00') } - - let(:pre_aggregated_events1) do - [ - create( - :clickhouse_events_count_agg, - subscription:, - organization:, - billable_metric: billable_metric1, - charge: charge1, - value: 3.0, - timestamp: Time.zone.parse('2024-07-07T00:00:00') - ), - create( - :clickhouse_events_count_agg, - subscription:, - organization:, - billable_metric: billable_metric1, - charge: charge1, - value: 2.0, - timestamp: Time.zone.parse('2024-07-09T00:00:00') - ) - ] - end - - let(:pre_aggregated_events2) do - [ - create( - :clickhouse_events_count_agg, - subscription:, - organization:, - billable_metric: billable_metric2, - charge: charge2, - value: 4.0, - timestamp: Time.zone.parse('2024-07-04T00:00:00') - ), - create( - :clickhouse_events_count_agg, - subscription:, - organization:, - billable_metric: billable_metric2, - charge: charge2, - value: 8.0, - timestamp: Time.zone.parse('2024-07-10T00:00:00') - ), - create( - :clickhouse_events_count_agg, - subscription:, - organization:, - billable_metric: billable_metric2, - charge: charge2, - value: 2.0, - timestamp: Time.zone.parse('2024-07-16T00:00:00') - ) - ] - end - - before do - pre_aggregated_events1 - pre_aggregated_events2 - end - - describe '.call' do - it 'returns the aggregated count units', aggregate_failures: true do - result = pre_aggregated_query.call - - expect(result).to be_success - expect(result.charges_units.keys).to match_array([charge1.id, charge2.id]) - expect(result.charges_units[charge1.id]).to eq({units: 5.0, filters: {}, grouped_by: {}}) - expect(result.charges_units[charge2.id]).to eq({units: 14.0, filters: {}, grouped_by: {}}) - end - - context 'with enriched events before and after the boundaries' do - let(:enriched_events1) do - [ - create( - :clickhouse_events_enriched, - subscription:, - organization:, - charge: charge1, - value: 1.0, - timestamp: Time.zone.parse('2024-07-01T03:45:50') - ), - create( - :clickhouse_events_enriched, - subscription:, - organization:, - charge: charge1, - value: 1.0, - timestamp: Time.zone.parse('2024-07-31T22:12:00') - ) - ] - end - - let(:enriched_events2) do - [ - create( - :clickhouse_events_enriched, - subscription:, - organization:, - charge: charge2, - value: 1.0, - # Ignored since it should be taken into account by the pre-aggregated query - timestamp: Time.zone.parse('2024-07-01T04:10:50') - ), - create( - :clickhouse_events_enriched, - subscription:, - organization:, - charge: charge2, - value: 1.0, - # Ignored since it should be taken into account by the pre-aggregated query - timestamp: Time.zone.parse('2024-07-31T23:10:00') - ) - ] - end - - before do - enriched_events1 - enriched_events2 - end - - it 'returns the aggregated count units', aggregate_failures: true do - result = pre_aggregated_query.call - - expect(result).to be_success - expect(result.charges_units.keys).to match_array([charge1.id, charge2.id]) - expect(result.charges_units[charge1.id]).to eq({units: 7.0, filters: {}, grouped_by: {}}) - expect(result.charges_units[charge2.id]).to eq({units: 14.0, filters: {}, grouped_by: {}}) - end - end - - context 'with grouped_by' do - let(:charge1) do - create( - :charge, - billable_metric: billable_metric1, - plan:, - properties: {amount: 10.to_s, grouped_by: %w[cloud country]} - ) - end - - let(:pre_aggregated_events1) do - [ - create( - :clickhouse_events_count_agg, - subscription:, - organization:, - billable_metric: billable_metric1, - charge: charge1, - value: 3.0, - timestamp: Time.zone.parse('2024-07-07T00:00:00'), - grouped_by: {cloud: 'aws', country: 'us'} - ), - create( - :clickhouse_events_count_agg, - subscription:, - organization:, - billable_metric: billable_metric1, - charge: charge1, - value: 2.0, - timestamp: Time.zone.parse('2024-07-09T00:00:00'), - grouped_by: {cloud: 'aws', country: 'us'} - ), - create( - :clickhouse_events_count_agg, - subscription:, - organization:, - billable_metric: billable_metric1, - charge: charge1, - value: 2.0, - timestamp: Time.zone.parse('2024-07-09T00:00:00') - ), - create( - :clickhouse_events_count_agg, - subscription:, - organization:, - billable_metric: billable_metric1, - charge: charge1, - value: 2.0, - timestamp: Time.zone.parse('2024-07-09T00:00:00'), - grouped_by: {cloud: 'aws', country: 'canada'} - ) - ] - end - - it 'returns the aggregated count units', aggregate_failures: true do - result = pre_aggregated_query.call - - expect(result).to be_success - expect(result.charges_units.keys).to match_array([charge1.id, charge2.id]) - expect(result.charges_units[charge1.id][:units]).to eq(2.0) - expect(result.charges_units[charge1.id][:grouped_by]).to eq({ - "{\"cloud\":\"aws\",\"country\":\"canada\"}" => {units: 2.0}, - "{\"cloud\":\"aws\",\"country\":\"us\"}" => {units: 5.0} - }) - - expect(result.charges_units[charge2.id]).to eq({units: 14.0, filters: {}, grouped_by: {}}) - end - - context 'with enriched events before and after the boundaries' do - let(:enriched_events1) do - [ - create( - :clickhouse_events_enriched, - subscription:, - organization:, - charge: charge1, - value: 1.0, - timestamp: Time.zone.parse('2024-07-01T03:45:50'), - grouped_by: {cloud: 'aws', country: 'canada'} - ), - create( - :clickhouse_events_enriched, - subscription:, - organization:, - charge: charge1, - value: 1.0, - timestamp: Time.zone.parse('2024-07-31T22:12:00') - ) - ] - end - - before do - enriched_events1 - end - - it 'returns the aggregated count units', aggregate_failures: true do - result = pre_aggregated_query.call - - expect(result).to be_success - expect(result.charges_units.keys).to match_array([charge1.id, charge2.id]) - expect(result.charges_units[charge1.id][:units]).to eq(3.0) - expect(result.charges_units[charge1.id][:grouped_by]).to eq({ - "{\"cloud\":\"aws\",\"country\":\"canada\"}" => {units: 3.0}, - "{\"cloud\":\"aws\",\"country\":\"us\"}" => {units: 5.0} - }) - - expect(result.charges_units[charge2.id]).to eq({units: 14.0, filters: {}, grouped_by: {}}) - end - end - end - - context 'with filters' do - let(:billable_metric_filter11) do - create(:billable_metric_filter, billable_metric: billable_metric1, key: 'cloud', values: %w[aws gcp azure]) - end - let(:billable_metric_filter12) do - create(:billable_metric_filter, billable_metric: billable_metric1, key: 'country', values: %w[us canada france]) - end - - let(:charge_filter1) { create(:charge_filter, charge: charge1, properties: {amount: 10.to_s}) } - let(:charge_filter1_value1) do - create( - :charge_filter_value, - charge_filter: charge_filter1, - billable_metric_filter: billable_metric_filter11, - values: %w[aws] - ) - end - let(:charge_filter1_value2) do - create( - :charge_filter_value, - charge_filter: charge_filter1, - billable_metric_filter: billable_metric_filter12, - values: %w[us canada] - ) - end - - let(:charge_filter2) { create(:charge_filter, charge: charge1, properties: {amount: 10.to_s}) } - let(:charge_filter2_value1) do - create( - :charge_filter_value, - charge_filter: charge_filter1, - billable_metric_filter: billable_metric_filter12, - values: %w[france] - ) - end - - let(:pre_aggregated_events1) do - [ - create( - :clickhouse_events_count_agg, - subscription:, - organization:, - billable_metric: billable_metric1, - charge: charge1, - value: 3.0, - timestamp: Time.zone.parse('2024-07-07T00:00:00'), - filters: {cloud: ['aws'], country: %w[us canada]} - ), - create( - :clickhouse_events_count_agg, - subscription:, - organization:, - billable_metric: billable_metric1, - charge: charge1, - value: 2.0, - timestamp: Time.zone.parse('2024-07-09T00:00:00'), - filters: {cloud: ['aws'], country: %w[us canada]} - ), - create( - :clickhouse_events_count_agg, - subscription:, - organization:, - billable_metric: billable_metric1, - charge: charge1, - value: 2.0, - timestamp: Time.zone.parse('2024-07-09T00:00:00') - ), - create( - :clickhouse_events_count_agg, - subscription:, - organization:, - billable_metric: billable_metric1, - charge: charge1, - value: 2.0, - timestamp: Time.zone.parse('2024-07-09T00:00:00'), - filters: {country: %w[france]} - ) - ] - end - - it 'returns the aggregated count units', aggregate_failures: true do - result = pre_aggregated_query.call - - expect(result).to be_success - expect(result.charges_units.keys).to match_array([charge1.id, charge2.id]) - expect(result.charges_units[charge1.id][:units]).to eq(2.0) - expect(result.charges_units[charge1.id][:filters]).to eq({ - "{\"cloud\":[\"aws\"],\"country\":[\"us\",\"canada\"]}" => {units: 5.0, grouped_by: {}}, - "{\"country\":[\"france\"]}" => {units: 2.0, grouped_by: {}} - }) - - expect(result.charges_units[charge2.id]).to eq({units: 14.0, filters: {}, grouped_by: {}}) - end - - context 'with enriched events before and after the boundaries' do - let(:enriched_events1) do - [ - create( - :clickhouse_events_enriched, - subscription:, - organization:, - charge: charge1, - value: 1.0, - timestamp: Time.zone.parse('2024-07-01T03:45:50'), - filters: {cloud: ['aws'], country: %w[us canada]} - ), - create( - :clickhouse_events_enriched, - subscription:, - organization:, - charge: charge1, - value: 1.0, - timestamp: Time.zone.parse('2024-07-31T22:12:00') - ) - ] - end - - before do - enriched_events1 - end - - it 'returns the aggregated count units', aggregate_failures: true do - result = pre_aggregated_query.call - - expect(result).to be_success - expect(result.charges_units.keys).to match_array([charge1.id, charge2.id]) - expect(result.charges_units[charge1.id][:units]).to eq(3.0) - expect(result.charges_units[charge1.id][:filters]).to eq({ - "{\"cloud\":[\"aws\"],\"country\":[\"us\",\"canada\"]}" => {units: 6.0, grouped_by: {}}, - "{\"country\":[\"france\"]}" => {units: 2.0, grouped_by: {}} - }) - - expect(result.charges_units[charge2.id]).to eq({units: 14.0, filters: {}, grouped_by: {}}) - end - end - - context 'with grouped_by and filters' do - let(:charge1) do - create( - :charge, - billable_metric: billable_metric1, - plan:, - properties: {amount: 10.to_s, grouped_by: %w[region]} - ) - end - - let(:pre_aggregated_events1) do - [ - create( - :clickhouse_events_count_agg, - subscription:, - organization:, - billable_metric: billable_metric1, - charge: charge1, - value: 3.0, - timestamp: Time.zone.parse('2024-07-07T00:00:00'), - filters: {cloud: ['aws'], country: %w[us canada]}, - grouped_by: {region: 'us-east-1'} - ), - create( - :clickhouse_events_count_agg, - subscription:, - organization:, - billable_metric: billable_metric1, - charge: charge1, - value: 2.0, - timestamp: Time.zone.parse('2024-07-09T00:00:00'), - filters: {cloud: ['aws'], country: %w[us canada]}, - grouped_by: {region: 'us-east-1'} - ), - create( - :clickhouse_events_count_agg, - subscription:, - organization:, - billable_metric: billable_metric1, - charge: charge1, - value: 2.0, - timestamp: Time.zone.parse('2024-07-09T00:00:00'), - grouped_by: {region: 'us-east-1'} - ), - create( - :clickhouse_events_count_agg, - subscription:, - organization:, - billable_metric: billable_metric1, - charge: charge1, - value: 2.0, - timestamp: Time.zone.parse('2024-07-09T00:00:00'), - filters: {cloud: ['aws'], country: %w[us canada]}, - grouped_by: {region: 'us-east-2'} - ) - ] - end - - it 'returns the aggregated count units', aggregate_failures: true do - result = pre_aggregated_query.call - - expect(result).to be_success - expect(result.charges_units.keys).to match_array([charge1.id, charge2.id]) - expect(result.charges_units[charge1.id][:units]).to eq(0.0) - expect(result.charges_units[charge1.id][:grouped_by]).to eq( - {"{\"region\":\"us-east-1\"}" => {units: 2.0}} - ) - - expect(result.charges_units[charge1.id][:filters]).to eq({ - "{\"cloud\":[\"aws\"],\"country\":[\"us\",\"canada\"]}" => { - units: 0.0, - grouped_by: { - "{\"region\":\"us-east-1\"}" => {units: 5.0}, - "{\"region\":\"us-east-2\"}" => {units: 2.0} - } - } - }) - - expect(result.charges_units[charge2.id]).to eq({units: 14.0, filters: {}, grouped_by: {}}) - end - end - end - end -end diff --git a/spec/services/events/stores/clickhouse/pre_aggregated/latest_query_spec.rb b/spec/services/events/stores/clickhouse/pre_aggregated/latest_query_spec.rb deleted file mode 100644 index 6f681081a24..00000000000 --- a/spec/services/events/stores/clickhouse/pre_aggregated/latest_query_spec.rb +++ /dev/null @@ -1,323 +0,0 @@ -# frozen_string_literal: true - -require 'rails_helper' - -RSpec.describe Events::Stores::Clickhouse::PreAggregated::LatestQuery, type: :service, clickhouse: true do - subject(:pre_aggregated_query) { described_class.new(subscription:, boundaries:) } - - let(:subscription) { create(:subscription, customer:, plan:) } - - let(:organization) { create(:organization) } - let(:customer) { create(:customer, organization:) } - - let(:billable_metric1) { create(:latest_billable_metric, organization:) } - let(:billable_metric2) { create(:latest_billable_metric, organization:) } - - let(:plan) { create(:plan, organization:) } - let(:charge1) { create(:standard_charge, billable_metric: billable_metric1, plan:) } - let(:charge2) { create(:standard_charge, billable_metric: billable_metric2, plan:) } - - let(:boundaries) { {from_datetime:, to_datetime:} } - let(:from_datetime) { Time.zone.parse('2024-07-01T03:42:00') } - let(:to_datetime) { Time.zone.parse('2024-07-31T22:47:00') } - - let(:enriched_events1) do - [ - create( - :clickhouse_events_enriched, - subscription:, - organization:, - charge: charge1, - value: 2.0, - timestamp: Time.zone.parse('2024-07-01T03:45:50') - ), - create( - :clickhouse_events_enriched, - subscription:, - organization:, - charge: charge1, - value: 7.0, - timestamp: Time.zone.parse('2024-07-31T22:12:00') - ) - ] - end - - let(:enriched_events2) do - [ - create( - :clickhouse_events_enriched, - subscription:, - organization:, - charge: charge2, - value: 4.0, - timestamp: Time.zone.parse('2024-07-04T00:00:00') - ), - create( - :clickhouse_events_enriched, - subscription:, - organization:, - charge: charge2, - value: 8.0, - timestamp: Time.zone.parse('2024-07-10T00:00:00') - ), - create( - :clickhouse_events_enriched, - subscription:, - organization:, - charge: charge2, - value: 2.0, - timestamp: Time.zone.parse('2024-07-16T00:00:00') - ) - ] - end - - before do - enriched_events1 - enriched_events2 - end - - describe '.call' do - it 'returns the latest units', aggregate_failures: true do - result = pre_aggregated_query.call - - expect(result).to be_success - expect(result.charges_units.keys).to match_array([charge1.id, charge2.id]) - expect(result.charges_units[charge1.id]).to eq({units: 7.0, filters: {}, grouped_by: {}}) - expect(result.charges_units[charge2.id]).to eq({units: 2.0, filters: {}, grouped_by: {}}) - end - - context 'with grouped_by' do - let(:charge1) do - create( - :charge, - billable_metric: billable_metric1, - plan:, - properties: {amount: 10.to_s, grouped_by: %w[cloud country]} - ) - end - - let(:enriched_events1) do - [ - create( - :clickhouse_events_enriched, - subscription:, - organization:, - charge: charge1, - value: 3.0, - timestamp: Time.zone.parse('2024-07-07T00:00:00'), - grouped_by: {cloud: 'aws', country: 'us'} - ), - create( - :clickhouse_events_enriched, - subscription:, - organization:, - charge: charge1, - value: 2.0, - timestamp: Time.zone.parse('2024-07-09T00:00:00'), - grouped_by: {cloud: 'aws', country: 'us'} - ), - create( - :clickhouse_events_enriched, - subscription:, - organization:, - charge: charge1, - value: 2.0, - timestamp: Time.zone.parse('2024-07-09T00:00:00') - ), - create( - :clickhouse_events_enriched, - subscription:, - organization:, - charge: charge1, - value: 2.0, - timestamp: Time.zone.parse('2024-07-09T00:00:00'), - grouped_by: {cloud: 'aws', country: 'canada'} - ) - ] - end - - it 'returns the latest units', aggregate_failures: true do - result = pre_aggregated_query.call - - expect(result).to be_success - expect(result.charges_units.keys).to match_array([charge1.id, charge2.id]) - expect(result.charges_units[charge1.id][:units]).to eq(2.0) - expect(result.charges_units[charge1.id][:grouped_by]).to eq({ - "{\"cloud\":\"aws\",\"country\":\"canada\"}" => {units: 2.0}, - "{\"cloud\":\"aws\",\"country\":\"us\"}" => {units: 2.0} - }) - - expect(result.charges_units[charge2.id]).to eq({units: 2.0, filters: {}, grouped_by: {}}) - end - end - - context 'with filters' do - let(:billable_metric_filter11) do - create(:billable_metric_filter, billable_metric: billable_metric1, key: 'cloud', values: %w[aws gcp azure]) - end - let(:billable_metric_filter12) do - create(:billable_metric_filter, billable_metric: billable_metric1, key: 'country', values: %w[us canada france]) - end - - let(:charge_filter1) { create(:charge_filter, charge: charge1, properties: {amount: 10.to_s}) } - let(:charge_filter1_value1) do - create( - :charge_filter_value, - charge_filter: charge_filter1, - billable_metric_filter: billable_metric_filter11, - values: %w[aws] - ) - end - let(:charge_filter1_value2) do - create( - :charge_filter_value, - charge_filter: charge_filter1, - billable_metric_filter: billable_metric_filter12, - values: %w[us canada] - ) - end - - let(:charge_filter2) { create(:charge_filter, charge: charge1, properties: {amount: 10.to_s}) } - let(:charge_filter2_value1) do - create( - :charge_filter_value, - charge_filter: charge_filter1, - billable_metric_filter: billable_metric_filter12, - values: %w[france] - ) - end - - let(:enriched_events1) do - [ - create( - :clickhouse_events_enriched, - subscription:, - organization:, - charge: charge1, - value: 3.0, - timestamp: Time.zone.parse('2024-07-07T00:00:00'), - filters: {cloud: ['aws'], country: %w[us canada]} - ), - create( - :clickhouse_events_enriched, - subscription:, - organization:, - charge: charge1, - value: 2.0, - timestamp: Time.zone.parse('2024-07-09T00:00:00'), - filters: {cloud: ['aws'], country: %w[us canada]} - ), - create( - :clickhouse_events_enriched, - subscription:, - organization:, - charge: charge1, - value: 2.0, - timestamp: Time.zone.parse('2024-07-09T00:00:00') - ), - create( - :clickhouse_events_enriched, - subscription:, - organization:, - charge: charge1, - value: 2.0, - timestamp: Time.zone.parse('2024-07-09T00:00:00'), - filters: {country: %w[france]} - ) - ] - end - - it 'returns the latest units', aggregate_failures: true do - result = pre_aggregated_query.call - - expect(result).to be_success - expect(result.charges_units.keys).to match_array([charge1.id, charge2.id]) - expect(result.charges_units[charge1.id][:units]).to eq(2.0) - expect(result.charges_units[charge1.id][:filters]).to eq({ - "{\"cloud\":[\"aws\"],\"country\":[\"us\",\"canada\"]}" => {units: 2.0, grouped_by: {}}, - "{\"country\":[\"france\"]}" => {units: 2.0, grouped_by: {}} - }) - - expect(result.charges_units[charge2.id]).to eq({units: 2.0, filters: {}, grouped_by: {}}) - end - - context 'with grouped_by and filters' do - let(:charge1) do - create( - :charge, - billable_metric: billable_metric1, - plan:, - properties: {amount: 10.to_s, grouped_by: %w[region]} - ) - end - - let(:enriched_events1) do - [ - create( - :clickhouse_events_enriched, - subscription:, - organization:, - charge: charge1, - value: 3.0, - timestamp: Time.zone.parse('2024-07-07T00:00:00'), - filters: {cloud: ['aws'], country: %w[us canada]}, - grouped_by: {region: 'us-east-1'} - ), - create( - :clickhouse_events_enriched, - subscription:, - organization:, - charge: charge1, - value: 2.0, - timestamp: Time.zone.parse('2024-07-09T00:00:00'), - filters: {cloud: ['aws'], country: %w[us canada]}, - grouped_by: {region: 'us-east-1'} - ), - create( - :clickhouse_events_enriched, - subscription:, - organization:, - charge: charge1, - value: 2.0, - timestamp: Time.zone.parse('2024-07-09T00:00:00'), - grouped_by: {region: 'us-east-1'} - ), - create( - :clickhouse_events_enriched, - subscription:, - organization:, - charge: charge1, - value: 2.0, - timestamp: Time.zone.parse('2024-07-09T00:00:00'), - filters: {cloud: ['aws'], country: %w[us canada]}, - grouped_by: {region: 'us-east-2'} - ) - ] - end - - it 'returns the latest units', aggregate_failures: true do - result = pre_aggregated_query.call - - expect(result).to be_success - expect(result.charges_units.keys).to match_array([charge1.id, charge2.id]) - expect(result.charges_units[charge1.id][:units]).to eq(0.0) - expect(result.charges_units[charge1.id][:grouped_by]).to eq( - {"{\"region\":\"us-east-1\"}" => {units: 2.0}} - ) - - expect(result.charges_units[charge1.id][:filters]).to eq({ - "{\"cloud\":[\"aws\"],\"country\":[\"us\",\"canada\"]}" => { - units: 0.0, - grouped_by: { - "{\"region\":\"us-east-1\"}" => {units: 2.0}, - "{\"region\":\"us-east-2\"}" => {units: 2.0} - } - } - }) - - expect(result.charges_units[charge2.id]).to eq({units: 2.0, filters: {}, grouped_by: {}}) - end - end - end - end -end diff --git a/spec/services/events/stores/clickhouse/pre_aggregated/max_query_spec.rb b/spec/services/events/stores/clickhouse/pre_aggregated/max_query_spec.rb deleted file mode 100644 index 3c52d46296d..00000000000 --- a/spec/services/events/stores/clickhouse/pre_aggregated/max_query_spec.rb +++ /dev/null @@ -1,484 +0,0 @@ -# frozen_string_literal: true - -require 'rails_helper' - -RSpec.describe Events::Stores::Clickhouse::PreAggregated::MaxQuery, type: :service, clickhouse: true do - subject(:pre_aggregated_query) { described_class.new(subscription:, boundaries:) } - - let(:subscription) { create(:subscription, customer:, plan:) } - - let(:organization) { create(:organization) } - let(:customer) { create(:customer, organization:) } - - let(:billable_metric1) { create(:max_billable_metric, organization:) } - let(:billable_metric2) { create(:max_billable_metric, organization:) } - - let(:plan) { create(:plan, organization:) } - let(:charge1) { create(:standard_charge, billable_metric: billable_metric1, plan:) } - let(:charge2) { create(:standard_charge, billable_metric: billable_metric2, plan:) } - - let(:boundaries) { {from_datetime:, to_datetime:} } - let(:from_datetime) { Time.zone.parse('2024-07-01T03:42:00') } - let(:to_datetime) { Time.zone.parse('2024-07-31T22:47:00') } - - let(:pre_aggregated_events1) do - [ - create( - :clickhouse_events_max_agg, - subscription:, - organization:, - billable_metric: billable_metric1, - charge: charge1, - value: 3.0, - timestamp: Time.zone.parse('2024-07-07T00:00:00') - ), - create( - :clickhouse_events_max_agg, - subscription:, - organization:, - billable_metric: billable_metric1, - charge: charge1, - value: 2.0, - timestamp: Time.zone.parse('2024-07-09T00:00:00') - ) - ] - end - - let(:pre_aggregated_events2) do - [ - create( - :clickhouse_events_max_agg, - subscription:, - organization:, - billable_metric: billable_metric2, - charge: charge2, - value: 4.0, - timestamp: Time.zone.parse('2024-07-04T00:00:00') - ), - create( - :clickhouse_events_max_agg, - subscription:, - organization:, - billable_metric: billable_metric2, - charge: charge2, - value: 8.0, - timestamp: Time.zone.parse('2024-07-10T00:00:00') - ), - create( - :clickhouse_events_max_agg, - subscription:, - organization:, - billable_metric: billable_metric2, - charge: charge2, - value: 2.0, - timestamp: Time.zone.parse('2024-07-16T00:00:00') - ) - ] - end - - before do - pre_aggregated_events1 - pre_aggregated_events2 - end - - describe '.call' do - it 'returns the aggregated max units', aggregate_failures: true do - result = pre_aggregated_query.call - - expect(result).to be_success - expect(result.charges_units.keys).to match_array([charge1.id, charge2.id]) - expect(result.charges_units[charge1.id]).to eq({units: 3.0, filters: {}, grouped_by: {}}) - expect(result.charges_units[charge2.id]).to eq({units: 8.0, filters: {}, grouped_by: {}}) - end - - context 'with enriched events before and after the boundaries' do - let(:enriched_events1) do - [ - create( - :clickhouse_events_enriched, - subscription:, - organization:, - charge: charge1, - value: 1.0, - timestamp: Time.zone.parse('2024-07-01T03:45:50') - ), - create( - :clickhouse_events_enriched, - subscription:, - organization:, - charge: charge1, - value: 1.0, - timestamp: Time.zone.parse('2024-07-31T22:12:00') - ) - ] - end - - let(:enriched_events2) do - [ - create( - :clickhouse_events_enriched, - subscription:, - organization:, - charge: charge2, - value: 1.0, - # Ignored since it should be taken into account by the pre-aggregated query - timestamp: Time.zone.parse('2024-07-01T04:10:50') - ), - create( - :clickhouse_events_enriched, - subscription:, - organization:, - charge: charge2, - value: 1.0, - # Ignored since it should be taken into account by the pre-aggregated query - timestamp: Time.zone.parse('2024-07-31T23:10:00') - ) - ] - end - - before do - enriched_events1 - enriched_events2 - end - - it 'returns the aggregated count units', aggregate_failures: true do - result = pre_aggregated_query.call - - expect(result).to be_success - expect(result.charges_units.keys).to match_array([charge1.id, charge2.id]) - expect(result.charges_units[charge1.id]).to eq({units: 3.0, filters: {}, grouped_by: {}}) - expect(result.charges_units[charge2.id]).to eq({units: 8.0, filters: {}, grouped_by: {}}) - end - end - - context 'with grouped_by' do - let(:charge1) do - create( - :charge, - billable_metric: billable_metric1, - plan:, - properties: {amount: 10.to_s, grouped_by: %w[cloud country]} - ) - end - - let(:pre_aggregated_events1) do - [ - create( - :clickhouse_events_max_agg, - subscription:, - organization:, - billable_metric: billable_metric1, - charge: charge1, - value: 3.0, - timestamp: Time.zone.parse('2024-07-07T00:00:00'), - grouped_by: {cloud: 'aws', country: 'us'} - ), - create( - :clickhouse_events_max_agg, - subscription:, - organization:, - billable_metric: billable_metric1, - charge: charge1, - value: 2.0, - timestamp: Time.zone.parse('2024-07-09T00:00:00'), - grouped_by: {cloud: 'aws', country: 'us'} - ), - create( - :clickhouse_events_max_agg, - subscription:, - organization:, - billable_metric: billable_metric1, - charge: charge1, - value: 2.0, - timestamp: Time.zone.parse('2024-07-09T00:00:00') - ), - create( - :clickhouse_events_max_agg, - subscription:, - organization:, - billable_metric: billable_metric1, - charge: charge1, - value: 2.0, - timestamp: Time.zone.parse('2024-07-09T00:00:00'), - grouped_by: {cloud: 'aws', country: 'canada'} - ) - ] - end - - it 'returns the aggregated max units', aggregate_failures: true do - result = pre_aggregated_query.call - - expect(result).to be_success - expect(result.charges_units.keys).to match_array([charge1.id, charge2.id]) - expect(result.charges_units[charge1.id][:units]).to eq(2.0) - expect(result.charges_units[charge1.id][:grouped_by]).to eq({ - "{\"cloud\":\"aws\",\"country\":\"canada\"}" => {units: 2.0}, - "{\"cloud\":\"aws\",\"country\":\"us\"}" => {units: 3.0} - }) - - expect(result.charges_units[charge2.id]).to eq({units: 8.0, filters: {}, grouped_by: {}}) - end - - context 'with enriched events before and after the boundaries' do - let(:enriched_events1) do - [ - create( - :clickhouse_events_enriched, - subscription:, - organization:, - charge: charge1, - value: 1.0, - timestamp: Time.zone.parse('2024-07-01T03:45:50'), - grouped_by: {cloud: 'aws', country: 'canada'} - ), - create( - :clickhouse_events_enriched, - subscription:, - organization:, - charge: charge1, - value: 1.0, - timestamp: Time.zone.parse('2024-07-31T22:12:00') - ) - ] - end - - before do - enriched_events1 - end - - it 'returns the aggregated count units', aggregate_failures: true do - result = pre_aggregated_query.call - - expect(result).to be_success - expect(result.charges_units.keys).to match_array([charge1.id, charge2.id]) - expect(result.charges_units[charge1.id][:units]).to eq(2.0) - expect(result.charges_units[charge1.id][:grouped_by]).to eq({ - "{\"cloud\":\"aws\",\"country\":\"canada\"}" => {units: 2.0}, - "{\"cloud\":\"aws\",\"country\":\"us\"}" => {units: 3.0} - }) - - expect(result.charges_units[charge2.id]).to eq({units: 8.0, filters: {}, grouped_by: {}}) - end - end - end - - context 'with filters' do - let(:billable_metric_filter11) do - create(:billable_metric_filter, billable_metric: billable_metric1, key: 'cloud', values: %w[aws gcp azure]) - end - let(:billable_metric_filter12) do - create(:billable_metric_filter, billable_metric: billable_metric1, key: 'country', values: %w[us canada france]) - end - - let(:charge_filter1) { create(:charge_filter, charge: charge1, properties: {amount: 10.to_s}) } - let(:charge_filter1_value1) do - create( - :charge_filter_value, - charge_filter: charge_filter1, - billable_metric_filter: billable_metric_filter11, - values: %w[aws] - ) - end - let(:charge_filter1_value2) do - create( - :charge_filter_value, - charge_filter: charge_filter1, - billable_metric_filter: billable_metric_filter12, - values: %w[us canada] - ) - end - - let(:charge_filter2) { create(:charge_filter, charge: charge1, properties: {amount: 10.to_s}) } - let(:charge_filter2_value1) do - create( - :charge_filter_value, - charge_filter: charge_filter1, - billable_metric_filter: billable_metric_filter12, - values: %w[france] - ) - end - - let(:pre_aggregated_events1) do - [ - create( - :clickhouse_events_max_agg, - subscription:, - organization:, - billable_metric: billable_metric1, - charge: charge1, - value: 3.0, - timestamp: Time.zone.parse('2024-07-07T00:00:00'), - filters: {cloud: ['aws'], country: %w[us canada]} - ), - create( - :clickhouse_events_max_agg, - subscription:, - organization:, - billable_metric: billable_metric1, - charge: charge1, - value: 2.0, - timestamp: Time.zone.parse('2024-07-09T00:00:00'), - filters: {cloud: ['aws'], country: %w[us canada]} - ), - create( - :clickhouse_events_max_agg, - subscription:, - organization:, - billable_metric: billable_metric1, - charge: charge1, - value: 2.0, - timestamp: Time.zone.parse('2024-07-09T00:00:00') - ), - create( - :clickhouse_events_max_agg, - subscription:, - organization:, - billable_metric: billable_metric1, - charge: charge1, - value: 2.0, - timestamp: Time.zone.parse('2024-07-09T00:00:00'), - filters: {country: %w[france]} - ) - ] - end - - it 'returns the aggregated max units', aggregate_failures: true do - result = pre_aggregated_query.call - - expect(result).to be_success - expect(result.charges_units.keys).to match_array([charge1.id, charge2.id]) - expect(result.charges_units[charge1.id][:units]).to eq(2.0) - expect(result.charges_units[charge1.id][:filters]).to eq({ - "{\"cloud\":[\"aws\"],\"country\":[\"us\",\"canada\"]}" => {units: 3.0, grouped_by: {}}, - "{\"country\":[\"france\"]}" => {units: 2.0, grouped_by: {}} - }) - - expect(result.charges_units[charge2.id]).to eq({units: 8.0, filters: {}, grouped_by: {}}) - end - - context 'with enriched events before and after the boundaries' do - let(:enriched_events1) do - [ - create( - :clickhouse_events_enriched, - subscription:, - organization:, - charge: charge1, - value: 1.0, - timestamp: Time.zone.parse('2024-07-01T03:45:50'), - filters: {cloud: ['aws'], country: %w[us canada]} - ), - create( - :clickhouse_events_enriched, - subscription:, - organization:, - charge: charge1, - value: 1.0, - timestamp: Time.zone.parse('2024-07-31T22:12:00') - ) - ] - end - - before do - enriched_events1 - end - - it 'returns the aggregated count units', aggregate_failures: true do - result = pre_aggregated_query.call - - expect(result).to be_success - expect(result.charges_units.keys).to match_array([charge1.id, charge2.id]) - expect(result.charges_units[charge1.id][:units]).to eq(2.0) - expect(result.charges_units[charge1.id][:filters]).to eq({ - "{\"cloud\":[\"aws\"],\"country\":[\"us\",\"canada\"]}" => {units: 3.0, grouped_by: {}}, - "{\"country\":[\"france\"]}" => {units: 2.0, grouped_by: {}} - }) - - expect(result.charges_units[charge2.id]).to eq({units: 8.0, filters: {}, grouped_by: {}}) - end - end - - context 'with grouped_by and filters' do - let(:charge1) do - create( - :charge, - billable_metric: billable_metric1, - plan:, - properties: {amount: 10.to_s, grouped_by: %w[region]} - ) - end - - let(:pre_aggregated_events1) do - [ - create( - :clickhouse_events_max_agg, - subscription:, - organization:, - billable_metric: billable_metric1, - charge: charge1, - value: 3.0, - timestamp: Time.zone.parse('2024-07-07T00:00:00'), - filters: {cloud: ['aws'], country: %w[us canada]}, - grouped_by: {region: 'us-east-1'} - ), - create( - :clickhouse_events_max_agg, - subscription:, - organization:, - billable_metric: billable_metric1, - charge: charge1, - value: 2.0, - timestamp: Time.zone.parse('2024-07-09T00:00:00'), - filters: {cloud: ['aws'], country: %w[us canada]}, - grouped_by: {region: 'us-east-1'} - ), - create( - :clickhouse_events_max_agg, - subscription:, - organization:, - billable_metric: billable_metric1, - charge: charge1, - value: 2.0, - timestamp: Time.zone.parse('2024-07-09T00:00:00'), - grouped_by: {region: 'us-east-1'} - ), - create( - :clickhouse_events_max_agg, - subscription:, - organization:, - billable_metric: billable_metric1, - charge: charge1, - value: 2.0, - timestamp: Time.zone.parse('2024-07-09T00:00:00'), - filters: {cloud: ['aws'], country: %w[us canada]}, - grouped_by: {region: 'us-east-2'} - ) - ] - end - - it 'returns the aggregated max units', aggregate_failures: true do - result = pre_aggregated_query.call - - expect(result).to be_success - expect(result.charges_units.keys).to match_array([charge1.id, charge2.id]) - expect(result.charges_units[charge1.id][:units]).to eq(0.0) - expect(result.charges_units[charge1.id][:grouped_by]).to eq( - {"{\"region\":\"us-east-1\"}" => {units: 2.0}} - ) - - expect(result.charges_units[charge1.id][:filters]).to eq({ - "{\"cloud\":[\"aws\"],\"country\":[\"us\",\"canada\"]}" => { - units: 0.0, - grouped_by: { - "{\"region\":\"us-east-1\"}" => {units: 3.0}, - "{\"region\":\"us-east-2\"}" => {units: 2.0} - } - } - }) - - expect(result.charges_units[charge2.id]).to eq({units: 8.0, filters: {}, grouped_by: {}}) - end - end - end - end -end diff --git a/spec/services/events/stores/clickhouse/pre_aggregated/sum_query_spec.rb b/spec/services/events/stores/clickhouse/pre_aggregated/sum_query_spec.rb deleted file mode 100644 index 67f9692bc60..00000000000 --- a/spec/services/events/stores/clickhouse/pre_aggregated/sum_query_spec.rb +++ /dev/null @@ -1,484 +0,0 @@ -# frozen_string_literal: true - -require 'rails_helper' - -RSpec.describe Events::Stores::Clickhouse::PreAggregated::SumQuery, type: :service, clickhouse: true do - subject(:pre_aggregated_query) { described_class.new(subscription:, boundaries:) } - - let(:subscription) { create(:subscription, customer:, plan:) } - - let(:organization) { create(:organization) } - let(:customer) { create(:customer, organization:) } - - let(:billable_metric1) { create(:sum_billable_metric, organization:) } - let(:billable_metric2) { create(:sum_billable_metric, organization:) } - - let(:plan) { create(:plan, organization:) } - let(:charge1) { create(:standard_charge, billable_metric: billable_metric1, plan:) } - let(:charge2) { create(:standard_charge, billable_metric: billable_metric2, plan:) } - - let(:boundaries) { {from_datetime:, to_datetime:} } - let(:from_datetime) { Time.zone.parse('2024-07-01T03:42:00') } - let(:to_datetime) { Time.zone.parse('2024-07-31T22:47:00') } - - let(:pre_aggregated_events1) do - [ - create( - :clickhouse_events_sum_agg, - subscription:, - organization:, - billable_metric: billable_metric1, - charge: charge1, - value: 3.0, - timestamp: Time.zone.parse('2024-07-07T00:00:00') - ), - create( - :clickhouse_events_sum_agg, - subscription:, - organization:, - billable_metric: billable_metric1, - charge: charge1, - value: 2.0, - timestamp: Time.zone.parse('2024-07-09T00:00:00') - ) - ] - end - - let(:pre_aggregated_events2) do - [ - create( - :clickhouse_events_sum_agg, - subscription:, - organization:, - billable_metric: billable_metric2, - charge: charge2, - value: 4.0, - timestamp: Time.zone.parse('2024-07-04T00:00:00') - ), - create( - :clickhouse_events_sum_agg, - subscription:, - organization:, - billable_metric: billable_metric2, - charge: charge2, - value: 8.0, - timestamp: Time.zone.parse('2024-07-10T00:00:00') - ), - create( - :clickhouse_events_sum_agg, - subscription:, - organization:, - billable_metric: billable_metric2, - charge: charge2, - value: 2.0, - timestamp: Time.zone.parse('2024-07-16T00:00:00') - ) - ] - end - - before do - pre_aggregated_events1 - pre_aggregated_events2 - end - - describe '.call' do - it 'returns the aggregated count units', aggregate_failures: true do - result = pre_aggregated_query.call - - expect(result).to be_success - expect(result.charges_units.keys).to match_array([charge1.id, charge2.id]) - expect(result.charges_units[charge1.id]).to eq({units: 5.0, filters: {}, grouped_by: {}}) - expect(result.charges_units[charge2.id]).to eq({units: 14.0, filters: {}, grouped_by: {}}) - end - - context 'with enriched events before and after the boundaries' do - let(:enriched_events1) do - [ - create( - :clickhouse_events_enriched, - subscription:, - organization:, - charge: charge1, - value: 1.0, - timestamp: Time.zone.parse('2024-07-01T03:45:50') - ), - create( - :clickhouse_events_enriched, - subscription:, - organization:, - charge: charge1, - value: 1.0, - timestamp: Time.zone.parse('2024-07-31T22:12:00') - ) - ] - end - - let(:enriched_events2) do - [ - create( - :clickhouse_events_enriched, - subscription:, - organization:, - charge: charge2, - value: 1.0, - # Ignored since it should be taken into account by the pre-aggregated query - timestamp: Time.zone.parse('2024-07-01T04:10:50') - ), - create( - :clickhouse_events_enriched, - subscription:, - organization:, - charge: charge2, - value: 1.0, - # Ignored since it should be taken into account by the pre-aggregated query - timestamp: Time.zone.parse('2024-07-31T23:10:00') - ) - ] - end - - before do - enriched_events1 - enriched_events2 - end - - it 'returns the aggregated count units', aggregate_failures: true do - result = pre_aggregated_query.call - - expect(result).to be_success - expect(result.charges_units.keys).to match_array([charge1.id, charge2.id]) - expect(result.charges_units[charge1.id]).to eq({units: 7.0, filters: {}, grouped_by: {}}) - expect(result.charges_units[charge2.id]).to eq({units: 14.0, filters: {}, grouped_by: {}}) - end - end - - context 'with grouped_by' do - let(:charge1) do - create( - :charge, - billable_metric: billable_metric1, - plan:, - properties: {amount: 10.to_s, grouped_by: %w[cloud country]} - ) - end - - let(:pre_aggregated_events1) do - [ - create( - :clickhouse_events_sum_agg, - subscription:, - organization:, - billable_metric: billable_metric1, - charge: charge1, - value: 3.0, - timestamp: Time.zone.parse('2024-07-07T00:00:00'), - grouped_by: {cloud: 'aws', country: 'us'} - ), - create( - :clickhouse_events_sum_agg, - subscription:, - organization:, - billable_metric: billable_metric1, - charge: charge1, - value: 2.0, - timestamp: Time.zone.parse('2024-07-09T00:00:00'), - grouped_by: {cloud: 'aws', country: 'us'} - ), - create( - :clickhouse_events_sum_agg, - subscription:, - organization:, - billable_metric: billable_metric1, - charge: charge1, - value: 2.0, - timestamp: Time.zone.parse('2024-07-09T00:00:00') - ), - create( - :clickhouse_events_sum_agg, - subscription:, - organization:, - billable_metric: billable_metric1, - charge: charge1, - value: 2.0, - timestamp: Time.zone.parse('2024-07-09T00:00:00'), - grouped_by: {cloud: 'aws', country: 'canada'} - ) - ] - end - - it 'returns the aggregated count units', aggregate_failures: true do - result = pre_aggregated_query.call - - expect(result).to be_success - expect(result.charges_units.keys).to match_array([charge1.id, charge2.id]) - expect(result.charges_units[charge1.id][:units]).to eq(2.0) - expect(result.charges_units[charge1.id][:grouped_by]).to eq({ - "{\"cloud\":\"aws\",\"country\":\"canada\"}" => {units: 2.0}, - "{\"cloud\":\"aws\",\"country\":\"us\"}" => {units: 5.0} - }) - - expect(result.charges_units[charge2.id]).to eq({units: 14.0, filters: {}, grouped_by: {}}) - end - - context 'with enriched events before and after the boundaries' do - let(:enriched_events1) do - [ - create( - :clickhouse_events_enriched, - subscription:, - organization:, - charge: charge1, - value: 1.0, - timestamp: Time.zone.parse('2024-07-01T03:45:50'), - grouped_by: {cloud: 'aws', country: 'canada'} - ), - create( - :clickhouse_events_enriched, - subscription:, - organization:, - charge: charge1, - value: 1.0, - timestamp: Time.zone.parse('2024-07-31T22:12:00') - ) - ] - end - - before do - enriched_events1 - end - - it 'returns the aggregated count units', aggregate_failures: true do - result = pre_aggregated_query.call - - expect(result).to be_success - expect(result.charges_units.keys).to match_array([charge1.id, charge2.id]) - expect(result.charges_units[charge1.id][:units]).to eq(3.0) - expect(result.charges_units[charge1.id][:grouped_by]).to eq({ - "{\"cloud\":\"aws\",\"country\":\"canada\"}" => {units: 3.0}, - "{\"cloud\":\"aws\",\"country\":\"us\"}" => {units: 5.0} - }) - - expect(result.charges_units[charge2.id]).to eq({units: 14.0, filters: {}, grouped_by: {}}) - end - end - end - - context 'with filters' do - let(:billable_metric_filter11) do - create(:billable_metric_filter, billable_metric: billable_metric1, key: 'cloud', values: %w[aws gcp azure]) - end - let(:billable_metric_filter12) do - create(:billable_metric_filter, billable_metric: billable_metric1, key: 'country', values: %w[us canada france]) - end - - let(:charge_filter1) { create(:charge_filter, charge: charge1, properties: {amount: 10.to_s}) } - let(:charge_filter1_value1) do - create( - :charge_filter_value, - charge_filter: charge_filter1, - billable_metric_filter: billable_metric_filter11, - values: %w[aws] - ) - end - let(:charge_filter1_value2) do - create( - :charge_filter_value, - charge_filter: charge_filter1, - billable_metric_filter: billable_metric_filter12, - values: %w[us canada] - ) - end - - let(:charge_filter2) { create(:charge_filter, charge: charge1, properties: {amount: 10.to_s}) } - let(:charge_filter2_value1) do - create( - :charge_filter_value, - charge_filter: charge_filter1, - billable_metric_filter: billable_metric_filter12, - values: %w[france] - ) - end - - let(:pre_aggregated_events1) do - [ - create( - :clickhouse_events_sum_agg, - subscription:, - organization:, - billable_metric: billable_metric1, - charge: charge1, - value: 3.0, - timestamp: Time.zone.parse('2024-07-07T00:00:00'), - filters: {cloud: ['aws'], country: %w[us canada]} - ), - create( - :clickhouse_events_sum_agg, - subscription:, - organization:, - billable_metric: billable_metric1, - charge: charge1, - value: 2.0, - timestamp: Time.zone.parse('2024-07-09T00:00:00'), - filters: {cloud: ['aws'], country: %w[us canada]} - ), - create( - :clickhouse_events_sum_agg, - subscription:, - organization:, - billable_metric: billable_metric1, - charge: charge1, - value: 2.0, - timestamp: Time.zone.parse('2024-07-09T00:00:00') - ), - create( - :clickhouse_events_sum_agg, - subscription:, - organization:, - billable_metric: billable_metric1, - charge: charge1, - value: 2.0, - timestamp: Time.zone.parse('2024-07-09T00:00:00'), - filters: {country: %w[france]} - ) - ] - end - - it 'returns the aggregated count units', aggregate_failures: true do - result = pre_aggregated_query.call - - expect(result).to be_success - expect(result.charges_units.keys).to match_array([charge1.id, charge2.id]) - expect(result.charges_units[charge1.id][:units]).to eq(2.0) - expect(result.charges_units[charge1.id][:filters]).to eq({ - "{\"cloud\":[\"aws\"],\"country\":[\"us\",\"canada\"]}" => {units: 5.0, grouped_by: {}}, - "{\"country\":[\"france\"]}" => {units: 2.0, grouped_by: {}} - }) - - expect(result.charges_units[charge2.id]).to eq({units: 14.0, filters: {}, grouped_by: {}}) - end - - context 'with enriched events before and after the boundaries' do - let(:enriched_events1) do - [ - create( - :clickhouse_events_enriched, - subscription:, - organization:, - charge: charge1, - value: 1.0, - timestamp: Time.zone.parse('2024-07-01T03:45:50'), - filters: {cloud: ['aws'], country: %w[us canada]} - ), - create( - :clickhouse_events_enriched, - subscription:, - organization:, - charge: charge1, - value: 1.0, - timestamp: Time.zone.parse('2024-07-31T22:12:00') - ) - ] - end - - before do - enriched_events1 - end - - it 'returns the aggregated count units', aggregate_failures: true do - result = pre_aggregated_query.call - - expect(result).to be_success - expect(result.charges_units.keys).to match_array([charge1.id, charge2.id]) - expect(result.charges_units[charge1.id][:units]).to eq(3.0) - expect(result.charges_units[charge1.id][:filters]).to eq({ - "{\"cloud\":[\"aws\"],\"country\":[\"us\",\"canada\"]}" => {units: 6.0, grouped_by: {}}, - "{\"country\":[\"france\"]}" => {units: 2.0, grouped_by: {}} - }) - - expect(result.charges_units[charge2.id]).to eq({units: 14.0, filters: {}, grouped_by: {}}) - end - end - - context 'with grouped_by and filters' do - let(:charge1) do - create( - :charge, - billable_metric: billable_metric1, - plan:, - properties: {amount: 10.to_s, grouped_by: %w[region]} - ) - end - - let(:pre_aggregated_events1) do - [ - create( - :clickhouse_events_sum_agg, - subscription:, - organization:, - billable_metric: billable_metric1, - charge: charge1, - value: 3.0, - timestamp: Time.zone.parse('2024-07-07T00:00:00'), - filters: {cloud: ['aws'], country: %w[us canada]}, - grouped_by: {region: 'us-east-1'} - ), - create( - :clickhouse_events_sum_agg, - subscription:, - organization:, - billable_metric: billable_metric1, - charge: charge1, - value: 2.0, - timestamp: Time.zone.parse('2024-07-09T00:00:00'), - filters: {cloud: ['aws'], country: %w[us canada]}, - grouped_by: {region: 'us-east-1'} - ), - create( - :clickhouse_events_sum_agg, - subscription:, - organization:, - billable_metric: billable_metric1, - charge: charge1, - value: 2.0, - timestamp: Time.zone.parse('2024-07-09T00:00:00'), - grouped_by: {region: 'us-east-1'} - ), - create( - :clickhouse_events_sum_agg, - subscription:, - organization:, - billable_metric: billable_metric1, - charge: charge1, - value: 2.0, - timestamp: Time.zone.parse('2024-07-09T00:00:00'), - filters: {cloud: ['aws'], country: %w[us canada]}, - grouped_by: {region: 'us-east-2'} - ) - ] - end - - it 'returns the aggregated count units', aggregate_failures: true do - result = pre_aggregated_query.call - - expect(result).to be_success - expect(result.charges_units.keys).to match_array([charge1.id, charge2.id]) - expect(result.charges_units[charge1.id][:units]).to eq(0.0) - expect(result.charges_units[charge1.id][:grouped_by]).to eq( - {"{\"region\":\"us-east-1\"}" => {units: 2.0}} - ) - - expect(result.charges_units[charge1.id][:filters]).to eq({ - "{\"cloud\":[\"aws\"],\"country\":[\"us\",\"canada\"]}" => { - units: 0.0, - grouped_by: { - "{\"region\":\"us-east-1\"}" => {units: 5.0}, - "{\"region\":\"us-east-2\"}" => {units: 2.0} - } - } - }) - - expect(result.charges_units[charge2.id]).to eq({units: 14.0, filters: {}, grouped_by: {}}) - end - end - end - end -end