fix (wallet-race-condition): Retry wallet update upon race condition (#…

…2664)

## Context

Recently, feature for updating wallet balance upon successful payment
was introduced.

## Description

If multiple paid credits are issues, there is a chance that after
successful payment processing we will receive all events from payment
provider in the same request.

Multiple events will schedule multiple jobs for increasing wallet
balance and each job is updating the same resource so there is a high
chance of race condition.

This PR introduced optimistic locking approach with 1 additional retry
if multiple processes are updating the same resource.

app/models/wallet.rb

@@ -57,6 +57,7 @@ def currency
 #  invoice_requires_successful_payment :boolean          default(FALSE), not null
 #  last_balance_sync_at                :datetime
 #  last_consumed_credit_at             :datetime
+#  lock_version                        :integer          default(0), not null
 #  name                                :string
 #  ongoing_balance_cents               :bigint           default(0), not null
 #  ongoing_usage_balance_cents         :bigint           default(0), not null

app/services/wallets/balance/increase_service.rb

@@ -3,34 +3,51 @@
 module Wallets
   module Balance
     class IncreaseService < BaseService
+      MAX_RETRIES = 5
+
       def initialize(wallet:, credits_amount:, reset_consumed_credits: false)
         super
 
         @wallet = wallet
         @credits_amount = credits_amount
         @reset_consumed_credits = reset_consumed_credits
+        @retries = 0
       end
 
       def call
-        currency = wallet.balance.currency
-        amount_cents = wallet.rate_amount * credits_amount * currency.subunit_to_unit
-
-        update_params = {
-          balance_cents: wallet.balance_cents + amount_cents,
-          credits_balance: wallet.credits_balance + credits_amount,
-          last_balance_sync_at: Time.current
-        }
-
-        if reset_consumed_credits
-          update_params[:consumed_credits] = [0.0, wallet.consumed_credits - credits_amount].max
-          update_params[:consumed_amount_cents] = [0, wallet.consumed_amount_cents - amount_cents].max
+        ActiveRecord::Base.transaction do
+          currency = wallet.balance.currency
+          amount_cents = wallet.rate_amount * credits_amount * currency.subunit_to_unit
+
+          update_params = {
+            balance_cents: wallet.balance_cents + amount_cents,
+            credits_balance: wallet.credits_balance + credits_amount,
+            last_balance_sync_at: Time.current
+          }
+
+          if reset_consumed_credits
+            update_params[:consumed_credits] = [0.0, wallet.consumed_credits - credits_amount].max
+            update_params[:consumed_amount_cents] = [0, wallet.consumed_amount_cents - amount_cents].max
+          end
+
+          wallet.update!(update_params)
+          Wallets::Balance::RefreshOngoingService.call(wallet:)
         end
 
-        wallet.update!(update_params)
-        Wallets::Balance::RefreshOngoingService.call(wallet:)
-
         result.wallet = wallet
         result
+      rescue ActiveRecord::StaleObjectError
+        @retries += 1
+
+        if @retries <= MAX_RETRIES
+          sleep(0.5)
+
+          wallet.reload
+
+          retry
+        end
+
+        result.service_failure!(code: 'race_condition_error', message: '')
       end
 
       private

db/migrate/20241008080209_add_lock_version_to_wallets.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+class AddLockVersionToWallets < ActiveRecord::Migration[7.1]
+  def change
+    add_column :wallets, :lock_version, :integer, default: 0, null: false
+  end
+end