account.get_utxo pagination (#1436)

* attempt number 2! * wow it finally works * should failed on no address * add more parse test * check constranints * (test) paginator attempt 3 * spandex * fix address test * return list * bring back paginator * get utxo * fix spec OMG.Crypto.address_t() * remove redundancy * add data paginator * wtf it works * all passed * fix eth_event_test * fix another eth event test failed * fix response schemas for account get utxo * fix request body yamal * improve description * add new line * add new line * mix format * format mix * fix test * fix test * fix naming * format mix * fix warning error * add paging test case * add pagination test * fix limit on swagger * format test * fix get_utxo spect * sort alias * add leser validate * remove empty lines * change name optional_lesser * add todo for watcher info api test cabbage for monday ! * new line * mix format * rename optional lesser * add limit for block and transaction count * if con test * add feature for alice call * alice_addr * take only single account * fix account array * add required * okay runnable test... gosh * add alias for apimoedl * fix fixture * test 2 utxo * format * fix patterm atching * yay working * update info api spec * change to pure call * _alice_priv fix warning * add payload * fix pattern matching * dynamic value * remove fixture * fix key as string for map * test single tx * format * more format * fix sometimes failed test * test pagination * format * fix expression * fix pattern match * remove warning * update make file * use ipoller * fix default limit * default paging * remove unused utxo * fix mix warn * Update priv/cabbage/apps/itest/test/features/watcher_info_api.feature Co-Authored-By: Ino Murko <[email protected]> * fix comments and eng * fix scenario * mix format * fix scnerio * format * fix comments * mix format * use to atom * mix format * mix format * add atom * wow it works * alias * mix format * add helper spec * fix lint * update doc * fix type spec for all existing paginator * fix paginator type * t(%__MODULE__{}) for own paginator * fix the last man standing paginator spec * okay i lied not last one * fix paginator * fix missing ) * change type Co-authored-by: Ino Murko <[email protected]> Co-authored-by: Unnawut Leepaisalsuwanna <[email protected]>
omgnetwork · Apr 16, 2020 · 4389bce · 4389bce
1 parent b22ea96
commit 4389bce
Show file tree

Hide file tree

Showing 29 changed files with 438 additions and 82 deletions.
diff --git a/.gitignore b/.gitignore
@@ -69,9 +69,14 @@ data/
 # local test setup
 localchain_contract_addresses.env
 
+
+# the famous mac .DS_Store
+.DS_Store
+
 # IntelliJ files
 .idea/
 *.iml
 
 #vs code
-.tool_versions
+.tool_versions
+
diff --git a/Makefile b/Makefile
@@ -27,9 +27,11 @@ help:
 	@echo "DOCKER DEVELOPMENT"
 	@echo "------------------"
 	@echo ""
-	@echo "  - \`make docker-build-cluster\`: build child_chain, watcher and watcher_info images \c"
+	@echo "  - \`make docker-build-start-cluster\`: build child_chain, watcher and watcher_info images \c"
 	@echo "from your current code base, then start a cluster with these freshly built images."
 	@echo ""
+	@echo " - \`make docker-build\`" build child_chain, watcher and watcher_info images from your current code base
+	@echo ""
 	@echo "  - \`make docker-update-watcher\`, \`make docker-update-watcher_info\` or \c"
 	@echo "\`make docker-update-child_chain\`: replaces containers with your code changes\c"
 	@echo "for rapid development."
@@ -305,6 +307,8 @@ docker-watcher: docker-watcher-prod docker-watcher-build
 docker-watcher_info: docker-watcher_info-prod docker-watcher_info-build
 docker-child_chain: docker-child_chain-prod docker-child_chain-build
 
+docker-build: docker-watcher docker-watcher_info docker-child_chain
+
 docker-push: docker
 	docker push $(CHILD_CHAIN_IMAGE_NAME)
 	docker push $(WATCHER_IMAGE_NAME)
@@ -315,7 +319,8 @@ docker-start-cluster:
 	SNAPSHOT=SNAPSHOT_MIX_EXIT_PERIOD_SECONDS_120 make init_test && \
 	docker-compose build --no-cache && docker-compose up
 
-docker-build-cluster: docker-child_chain docker-watcher docker-watcher_info
+docker-build-start-cluster:
+	$(MAKE) docker-build
 	SNAPSHOT=SNAPSHOT_MIX_EXIT_PERIOD_SECONDS_120 make init_test && \
 	docker-compose build --no-cache && docker-compose up
 

diff --git a/apps/omg_utils/lib/omg_utils/http_rpc/validators/base.ex b/apps/omg_utils/lib/omg_utils/http_rpc/validators/base.ex
@@ -145,6 +145,12 @@ defmodule OMG.Utils.HttpRPC.Validator.Base do
   def greater({val, []}, _b) when not is_integer(val), do: {val, [:integer]}
   def greater({val, []}, bound), do: {val, greater: bound}
 
+  @spec lesser({any(), list()}, integer()) :: {any(), list()}
+  def lesser({_, [_ | _]} = err, _b), do: err
+  def lesser({val, []}, bound) when is_integer(val) and val < bound, do: {val, []}
+  def lesser({val, []}, _b) when not is_integer(val), do: {val, [:integer]}
+  def lesser({val, []}, bound), do: {val, lesser: bound}
+
   @spec list({any(), list()}, function() | nil) :: {any(), list()}
   def list(tuple, mapper \\ nil)
   def list({_, [_ | _]} = err, _), do: err

diff --git a/apps/omg_utils/lib/omg_utils/paginator.ex b/apps/omg_utils/lib/omg_utils/paginator.ex
@@ -19,8 +19,8 @@ defmodule OMG.Utils.Paginator do
   @default_limit 200
   @first_page 1
 
-  @type t() :: %__MODULE__{
-          data: list(),
+  @type t(data_type) :: %__MODULE__{
+          data: list(data_type),
           data_paging: %{limit: pos_integer(), page: pos_integer()}
         }
 
@@ -45,6 +45,6 @@ defmodule OMG.Utils.Paginator do
     %__MODULE__{data: [], data_paging: data_paging}
   end
 
-  @spec set_data(list(), t()) :: t()
+  @spec set_data(list(), t(%__MODULE__{})) :: t(%__MODULE__{})
   def set_data(data, paginator) when is_list(data), do: %__MODULE__{paginator | data: data}
 end
diff --git a/apps/omg_watcher/test/omg_watcher/integration/block_getter_test.exs b/apps/omg_watcher/test/omg_watcher/integration/block_getter_test.exs
@@ -77,7 +77,7 @@ defmodule OMG.Watcher.Integration.BlockGetterTest do
       |> Enum.map(fn utxo -> Map.take(utxo, fields) end)
 
     utxos =
-      alice.addr
+      %{address: alice.addr}
       |> WatcherHelper.get_utxos()
       |> Enum.map(fn utxo -> Map.take(utxo, fields) end)
 

diff --git a/apps/omg_watcher/test/support/watcher_helper.ex b/apps/omg_watcher/test/support/watcher_helper.ex
@@ -125,8 +125,16 @@ defmodule Support.WatcherHelper do
     |> Map.get("amount")
   end
 
-  def get_utxos(address) do
-    success?("/account.get_utxos", %{"address" => Encoding.to_hex(address)})
+  def get_utxos(params) when is_map(params) do
+    hex_string_address = Encoding.to_hex(params.address)
+    success?("/account.get_utxos", %{params | address: hex_string_address})
+  end
+
+  @doc """
+  shortcut helper for get_utxos that inject pagination data for you
+  """
+  def get_utxos(address, page \\ 1, limit \\ 100) do
+    success?("/account.get_utxos", %{"address" => Encoding.to_hex(address), "page" => page, "limit" => limit})
   end
 
   def get_exitable_utxos(address) do

diff --git a/apps/omg_watcher_info/lib/omg_watcher_info/api/account.ex b/apps/omg_watcher_info/lib/omg_watcher_info/api/account.ex
@@ -16,7 +16,7 @@ defmodule OMG.WatcherInfo.API.Account do
   @moduledoc """
   Module provides operations related to plasma accounts.
   """
-
+  alias OMG.Utils.Paginator
   alias OMG.WatcherInfo.DB
 
   @doc """
@@ -30,8 +30,8 @@ defmodule OMG.WatcherInfo.API.Account do
   @doc """
   Gets all utxos belonging to the given address.
   """
-  @spec get_utxos(OMG.Crypto.address_t()) :: list(%DB.TxOutput{})
-  def get_utxos(address) do
-    DB.TxOutput.get_utxos(address)
+  @spec get_utxos(Keyword.t()) :: Paginator.t(%DB.TxOutput{})
+  def get_utxos(params) do
+    DB.TxOutput.get_utxos(params)
   end
 end
diff --git a/apps/omg_watcher_info/lib/omg_watcher_info/api/block.ex b/apps/omg_watcher_info/lib/omg_watcher_info/api/block.ex
@@ -37,7 +37,7 @@ defmodule OMG.WatcherInfo.API.Block do
   Retrieves a list of blocks.
   Length of the list is limited by `limit` and `page` arguments.
   """
-  @spec get_blocks(Keyword.t()) :: Paginator.t()
+  @spec get_blocks(Keyword.t()) :: Paginator.t(%DB.Block{})
   def get_blocks(constraints) do
     paginator = Paginator.from_constraints(constraints, @default_blocks_limit)
 

diff --git a/apps/omg_watcher_info/lib/omg_watcher_info/api/transaction.ex b/apps/omg_watcher_info/lib/omg_watcher_info/api/transaction.ex
@@ -23,7 +23,6 @@ defmodule OMG.WatcherInfo.API.Transaction do
   alias OMG.WatcherInfo.DB
   alias OMG.WatcherInfo.HttpRPC.Client
   alias OMG.WatcherInfo.UtxoSelection
-
   require Utxo
 
   @default_transactions_limit 200
@@ -46,7 +45,7 @@ defmodule OMG.WatcherInfo.API.Transaction do
 
   Length of the list is limited by `limit` argument
   """
-  @spec get_transactions(Keyword.t()) :: Paginator.t()
+  @spec get_transactions(Keyword.t()) :: Paginator.t(%DB.Transaction{})
   def get_transactions(constraints) do
     paginator = Paginator.from_constraints(constraints, @default_transactions_limit)
 

diff --git a/apps/omg_watcher_info/lib/omg_watcher_info/db/block.ex b/apps/omg_watcher_info/lib/omg_watcher_info/db/block.ex
@@ -87,7 +87,7 @@ defmodule OMG.WatcherInfo.DB.Block do
   @doc """
   Returns a list of blocks
   """
-  @spec get_blocks(Paginator.t()) :: Paginator.t()
+  @spec get_blocks(Paginator.t(%DB.Block{})) :: Paginator.t(%DB.Block{})
   def get_blocks(paginator) do
     query_get_last(paginator.data_paging)
     |> DB.Repo.all()

diff --git a/apps/omg_watcher_info/lib/omg_watcher_info/db/transaction.ex b/apps/omg_watcher_info/lib/omg_watcher_info/db/transaction.ex
@@ -66,7 +66,7 @@ defmodule OMG.WatcherInfo.DB.Transaction do
   Returns transactions possibly filtered by constraints
   * constraints - accepts keyword in the form of [schema_field: value]
   """
-  @spec get_by_filters(Keyword.t(), Paginator.t()) :: Paginator.t()
+  @spec get_by_filters(Keyword.t(), Paginator.t(%__MODULE__{})) :: Paginator.t(%__MODULE__{})
   def get_by_filters(constraints, paginator) do
     allowed_constraints = [:address, :blknum, :txindex, :txtypes, :metadata]
 

diff --git a/apps/omg_watcher_info/lib/omg_watcher_info/db/txoutput.ex b/apps/omg_watcher_info/lib/omg_watcher_info/db/txoutput.ex
@@ -21,6 +21,7 @@ defmodule OMG.WatcherInfo.DB.TxOutput do
   use Ecto.Schema
 
   alias OMG.State.Transaction
+  alias OMG.Utils.Paginator
   alias OMG.Utxo
   alias OMG.WatcherInfo.DB
   alias OMG.WatcherInfo.DB.Repo
@@ -29,6 +30,8 @@ defmodule OMG.WatcherInfo.DB.TxOutput do
 
   import Ecto.Query, only: [from: 2, where: 2]
 
+  @default_get_utxos_limit 200
+
   @type balance() :: %{
           currency: binary(),
           amount: non_neg_integer()
@@ -79,24 +82,18 @@ defmodule OMG.WatcherInfo.DB.TxOutput do
     )
   end
 
-  def get_utxos(owner) do
-    query =
-      from(
-        txoutput in __MODULE__,
-        preload: [:ethevents],
-        left_join: ethevent in assoc(txoutput, :ethevents),
-        # select txoutputs by owner that have neither been spent nor have a corresponding ethevents exit events
-        where: txoutput.owner == ^owner and is_nil(txoutput.spending_txhash) and (is_nil(ethevent) or fragment("
- NOT EXISTS (SELECT 1
-             FROM ethevents_txoutputs AS etfrag
-             JOIN ethevents AS efrag ON
-                      etfrag.root_chain_txhash_event=efrag.root_chain_txhash_event
-                      AND efrag.event_type IN (?)
-                      AND etfrag.child_chain_utxohash = ?)", "standard_exit", txoutput.child_chain_utxohash)),
-        order_by: [asc: :blknum, asc: :txindex, asc: :oindex]
-      )
-
-    Repo.all(query)
+  @spec get_utxos(keyword) :: OMG.Utils.Paginator.t(%__MODULE__{})
+  def get_utxos(params) do
+    address = Keyword.fetch!(params, :address)
+    paginator = Paginator.from_constraints(params, @default_get_utxos_limit)
+    %{limit: limit, page: page} = paginator.data_paging
+    offset = (page - 1) * limit
+
+    address
+    |> query_get_utxos()
+    |> from(limit: ^limit, offset: ^offset)
+    |> Repo.all()
+    |> Paginator.set_data(paginator)
   end
 
   @spec get_balance(OMG.Crypto.address_t()) :: list(balance())
@@ -192,9 +189,33 @@ defmodule OMG.WatcherInfo.DB.TxOutput do
   @spec get_sorted_grouped_utxos(OMG.Crypto.address_t()) :: %{OMG.Crypto.address_t() => list(%__MODULE__{})}
   def get_sorted_grouped_utxos(owner) do
     # TODO: use clever DB query to get following out of DB
-    get_utxos(owner)
+    owner
+    |> get_all_utxos()
     |> Enum.group_by(& &1.currency)
     |> Enum.map(fn {k, v} -> {k, Enum.sort_by(v, & &1.amount, &>=/2)} end)
     |> Map.new()
   end
+
+  defp query_get_utxos(address) do
+    from(
+      txoutput in __MODULE__,
+      preload: [:ethevents],
+      left_join: ethevent in assoc(txoutput, :ethevents),
+      # select txoutputs by owner that have neither been spent nor have a corresponding ethevents exit events
+      where: txoutput.owner == ^address and is_nil(txoutput.spending_txhash) and (is_nil(ethevent) or fragment("
+NOT EXISTS (SELECT 1
+           FROM ethevents_txoutputs AS etfrag
+           JOIN ethevents AS efrag ON
+                    etfrag.root_chain_txhash_event=efrag.root_chain_txhash_event
+                    AND efrag.event_type IN (?)
+                    AND etfrag.child_chain_utxohash = ?)", "standard_exit", txoutput.child_chain_utxohash)),
+      order_by: [asc: :blknum, asc: :txindex, asc: :oindex]
+    )
+  end
+
+  @spec get_all_utxos(OMG.Crypto.address_t()) :: list()
+  defp get_all_utxos(address) do
+    query = query_get_utxos(address)
+    Repo.all(query)
+  end
 end
diff --git a/apps/omg_watcher_info/test/omg_watcher_info/db/eth_event_test.exs b/apps/omg_watcher_info/test/omg_watcher_info/db/eth_event_test.exs
@@ -189,9 +189,10 @@ defmodule OMG.WatcherInfo.DB.EthEventTest do
              root_chain_txhash_event: ^expected_root_chain_txhash_event_3
            } = DB.EthEvent.get(expected_root_chain_txhash_event_3)
 
+    %{data: alice_utxos} = DB.TxOutput.get_utxos(address: alice.addr)
+
     assert [^expected_root_chain_txhash_1, ^expected_root_chain_txhash_2, ^expected_root_chain_txhash_3] =
-             DB.TxOutput.get_utxos(alice.addr)
-             |> Enum.map(fn txoutput ->
+             Enum.map(alice_utxos, fn txoutput ->
                [head | _tail] = txoutput.ethevents
                head.root_chain_txhash
              end)
@@ -225,7 +226,8 @@ defmodule OMG.WatcherInfo.DB.EthEventTest do
                }
              ])
 
-    assert length(DB.TxOutput.get_utxos(expected_owner)) == 1
+    %{data: utxos} = DB.TxOutput.get_utxos(address: expected_owner)
+    assert length(utxos) == 1
 
     assert :ok =
              DB.EthEvent.insert_exits!([
@@ -236,7 +238,8 @@ defmodule OMG.WatcherInfo.DB.EthEventTest do
                }
              ])
 
-    assert Enum.empty?(DB.TxOutput.get_utxos(expected_owner))
+    %{data: utxos_after_exit} = DB.TxOutput.get_utxos(address: expected_owner)
+    assert Enum.empty?(utxos_after_exit)
   end
 
   @tag fixtures: [:alice, :initial_blocks]

diff --git a/apps/omg_watcher_rpc/lib/web/controllers/account.ex b/apps/omg_watcher_rpc/lib/web/controllers/account.ex
@@ -21,6 +21,7 @@ defmodule OMG.WatcherRPC.Web.Controller.Account do
 
   alias OMG.Watcher.API, as: SecurityAPI
   alias OMG.WatcherInfo.API, as: InfoAPI
+  alias OMG.WatcherRPC.Web.Validator.AccountConstraints
 
   @doc """
   Gets plasma account balance
@@ -34,8 +35,8 @@ defmodule OMG.WatcherRPC.Web.Controller.Account do
   end
 
   def get_utxos(conn, params) do
-    with {:ok, address} <- expect(params, "address", :address) do
-      address
+    with {:ok, constraints} <- AccountConstraints.parse(params) do
+      constraints
       |> InfoAPI.Account.get_utxos()
       |> api_response(conn, :utxos)
     end
@@ -45,7 +46,7 @@ defmodule OMG.WatcherRPC.Web.Controller.Account do
     with {:ok, address} <- expect(params, "address", :address) do
       address
       |> SecurityAPI.Account.get_exitable_utxos()
-      |> api_response(conn, :utxos)
+      |> api_response(conn, :exitable_utxos)
     end
   end
 end
diff --git a/apps/omg_watcher_rpc/lib/web/validators/account_constraints.ex b/apps/omg_watcher_rpc/lib/web/validators/account_constraints.ex
@@ -0,0 +1,35 @@
+# Copyright 2019-2020 OmiseGO Pte Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+defmodule OMG.WatcherRPC.Web.Validator.AccountConstraints do
+  @moduledoc """
+  Validates `/account.get_utxos` query parameters
+  """
+
+  alias OMG.WatcherRPC.Web.Validator.Helpers
+
+  @doc """
+  Validates possible query constraints, stops on first error.
+  """
+  @spec parse(%{binary() => any()}) :: {:ok, Keyword.t()} | {:error, any()}
+  def parse(params) do
+    constraints = [
+      {"limit", [pos_integer: true, lesser: 1000, optional: true], :limit},
+      {"page", [:pos_integer, :optional], :page},
+      {"address", [:address], :address}
+    ]
+
+    Helpers.validate_constraints(params, constraints)
+  end
+end
diff --git a/apps/omg_watcher_rpc/lib/web/validators/block_constraints.ex b/apps/omg_watcher_rpc/lib/web/validators/block_constraints.ex
@@ -16,25 +16,18 @@ defmodule OMG.WatcherRPC.Web.Validator.BlockConstraints do
   @moduledoc """
   Validates `/block.all` query parameters
   """
-
-  import OMG.Utils.HttpRPC.Validator.Base, only: [expect: 3]
+  alias OMG.WatcherRPC.Web.Validator.Helpers
 
   @doc """
   Validates possible query constraints, stops on first error.
   """
   @spec parse(%{binary() => any()}) :: {:ok, Keyword.t()} | {:error, any()}
   def parse(params) do
     constraints = [
-      {"limit", [:pos_integer, :optional]},
-      {"page", [:pos_integer, :optional]}
+      {"limit", [pos_integer: true, lesser: 1000, optional: true], :limit},
+      {"page", [:pos_integer, :optional], :page}
     ]
 
-    Enum.reduce_while(constraints, {:ok, []}, fn {key, validators}, {:ok, list} ->
-      case expect(params, key, validators) do
-        {:ok, nil} -> {:cont, {:ok, list}}
-        {:ok, value} -> {:cont, {:ok, [{String.to_existing_atom(key), value} | list]}}
-        error -> {:halt, error}
-      end
-    end)
+    Helpers.validate_constraints(params, constraints)
   end
 end