account.get_utxo pagination

.gitignore

@@ -69,6 +69,11 @@ data/
 # local test setup
 localchain_contract_addresses.env
 
+
+# the famous mac .DS_Store
+.DS_Store
+
 # IntelliJ files
 .idea/
 *.iml
+

Makefile

@@ -27,9 +27,11 @@ help:
 	@echo "DOCKER DEVELOPMENT"
 	@echo "------------------"
 	@echo ""
-	@echo "  - \`make docker-build-cluster\`: build child_chain, watcher and watcher_info images \c"
+	@echo "  - \`make docker-build-start-cluster\`: build child_chain, watcher and watcher_info images \c"
 	@echo "from your current code base, then start a cluster with these freshly built images."
 	@echo ""
+	@echo " - \`make docker-build-local\`" build child_chain, watcher and watcher_info images from your current code base
+	@echo ""
 	@echo "  - \`make docker-update-watcher\`, \`make docker-update-watcher_info\` or \c"
 	@echo "\`make docker-update-child_chain\`: replaces containers with your code changes\c"
 	@echo "for rapid development."
@@ -305,6 +307,8 @@ docker-watcher: docker-watcher-prod docker-watcher-build
 docker-watcher_info: docker-watcher_info-prod docker-watcher_info-build
 docker-child_chain: docker-child_chain-prod docker-child_chain-build
 
+docker-build-local: docker-watcher docker-watcher_info docker-child_chain
+
 docker-push: docker
 	docker push $(CHILD_CHAIN_IMAGE_NAME)
 	docker push $(WATCHER_IMAGE_NAME)
@@ -315,7 +319,8 @@ docker-start-cluster:
 	SNAPSHOT=SNAPSHOT_MIX_EXIT_PERIOD_SECONDS_120 make init_test && \
 	docker-compose build --no-cache && docker-compose up
 
-docker-build-cluster: docker-child_chain docker-watcher docker-watcher_info
+docker-build-start-cluster:
+	$(MAKE) docker-build-local
 	SNAPSHOT=SNAPSHOT_MIX_EXIT_PERIOD_SECONDS_120 make init_test && \
 	docker-compose build --no-cache && docker-compose up
 

apps/omg_utils/lib/omg_utils/http_rpc/validators/base.ex

@@ -145,6 +145,12 @@ defmodule OMG.Utils.HttpRPC.Validator.Base do
   def greater({val, []}, _b) when not is_integer(val), do: {val, [:integer]}
   def greater({val, []}, bound), do: {val, greater: bound}
 
+  @spec lesser({any(), list()}, integer()) :: {any(), list()}
+  def lesser({_, [_ | _]} = err, _b), do: err
+  def lesser({val, []}, bound) when is_integer(val) and val < bound, do: {val, []}
+  def lesser({val, []}, _b) when not is_integer(val), do: {val, [:integer]}
+  def lesser({val, []}, bound), do: {val, lesser: bound}
+
   @spec list({any(), list()}, function() | nil) :: {any(), list()}
   def list(tuple, mapper \\ nil)
   def list({_, [_ | _]} = err, _), do: err

apps/omg_watcher/test/omg_watcher/integration/block_getter_test.exs

@@ -77,7 +77,7 @@ defmodule OMG.Watcher.Integration.BlockGetterTest do
       |> Enum.map(fn utxo -> Map.take(utxo, fields) end)
 
     utxos =
-      alice.addr
+      %{address: alice.addr}
       |> WatcherHelper.get_utxos()
       |> Enum.map(fn utxo -> Map.take(utxo, fields) end)
 

apps/omg_watcher/test/support/watcher_helper.ex

@@ -125,8 +125,16 @@ defmodule Support.WatcherHelper do
     |> Map.get("amount")
   end
 
+  def get_utxos(params) when is_map(params) do
+    hex_string_address = Encoding.to_hex(params.address)
+    success?("/account.get_utxos", %{params | address: hex_string_address})
+  end
+
+  @doc """
+  shortcut helper for get_utxos that inject pagination data for you
+  """
   def get_utxos(address) do
-    success?("/account.get_utxos", %{"address" => Encoding.to_hex(address)})
+    success?("/account.get_utxos", %{"address" => Encoding.to_hex(address), "page" => 1, "limit" => 100})
   end
 
   def get_exitable_utxos(address) do

apps/omg_watcher_info/lib/omg_watcher_info/api/account.ex

@@ -16,7 +16,7 @@ defmodule OMG.WatcherInfo.API.Account do
   @moduledoc """
   Module provides operations related to plasma accounts.
   """
-
+  alias OMG.Utils.Paginator
   alias OMG.WatcherInfo.DB
 
   @doc """
@@ -30,8 +30,8 @@ defmodule OMG.WatcherInfo.API.Account do
   @doc """
   Gets all utxos belonging to the given address.
   """
-  @spec get_utxos(OMG.Crypto.address_t()) :: list(%DB.TxOutput{})
-  def get_utxos(address) do
-    DB.TxOutput.get_utxos(address)
+  @spec get_utxos(Keyword.t()) :: Paginator.t()
+  def get_utxos(params) do
+    DB.TxOutput.get_utxos(params)
   end
 end

apps/omg_watcher_info/lib/omg_watcher_info/db/txoutput.ex

@@ -21,6 +21,7 @@ defmodule OMG.WatcherInfo.DB.TxOutput do
   use Ecto.Schema
 
   alias OMG.State.Transaction
+  alias OMG.Utils.Paginator
   alias OMG.Utxo
   alias OMG.WatcherInfo.DB
   alias OMG.WatcherInfo.DB.Repo
@@ -29,6 +30,8 @@ defmodule OMG.WatcherInfo.DB.TxOutput do
 
   import Ecto.Query, only: [from: 2, where: 2]
 
+  @default_get_utxos_limit 200
+
   @type balance() :: %{
           currency: binary(),
           amount: non_neg_integer()
@@ -79,24 +82,18 @@ defmodule OMG.WatcherInfo.DB.TxOutput do
     )
   end
 
-  def get_utxos(owner) do
-    query =
-      from(
-        txoutput in __MODULE__,
-        preload: [:ethevents],
-        left_join: ethevent in assoc(txoutput, :ethevents),
-        # select txoutputs by owner that have neither been spent nor have a corresponding ethevents exit events
-        where: txoutput.owner == ^owner and is_nil(txoutput.spending_txhash) and (is_nil(ethevent) or fragment("
- NOT EXISTS (SELECT 1
-             FROM ethevents_txoutputs AS etfrag
-             JOIN ethevents AS efrag ON
-                      etfrag.root_chain_txhash_event=efrag.root_chain_txhash_event
-                      AND efrag.event_type IN (?)
-                      AND etfrag.child_chain_utxohash = ?)", "standard_exit", txoutput.child_chain_utxohash)),
-        order_by: [asc: :blknum, asc: :txindex, asc: :oindex]
-      )
-
-    Repo.all(query)
+  @spec get_utxos(keyword) :: OMG.Utils.Paginator.t()
+  def get_utxos(params) do
+    address = Keyword.fetch!(params, :address)
+    paginator = Paginator.from_constraints(params, @default_get_utxos_limit)
+    %{limit: limit, page: page} = paginator.data_paging
+    offset = (page - 1) * limit
+
+    address
+    |> query_get_utxos()
+    |> from(limit: ^limit, offset: ^offset)
+    |> Repo.all()
+    |> Paginator.set_data(paginator)
   end
 
   @spec get_balance(OMG.Crypto.address_t()) :: list(balance())
@@ -192,9 +189,33 @@ defmodule OMG.WatcherInfo.DB.TxOutput do
   @spec get_sorted_grouped_utxos(OMG.Crypto.address_t()) :: %{OMG.Crypto.address_t() => list(%__MODULE__{})}
   def get_sorted_grouped_utxos(owner) do
     # TODO: use clever DB query to get following out of DB
-    get_utxos(owner)
+    owner
+    |> get_all_utxos()
     |> Enum.group_by(& &1.currency)
     |> Enum.map(fn {k, v} -> {k, Enum.sort_by(v, & &1.amount, &>=/2)} end)
     |> Map.new()
   end
+
+  defp query_get_utxos(address) do
+    from(
+      txoutput in __MODULE__,
+      preload: [:ethevents],
+      left_join: ethevent in assoc(txoutput, :ethevents),
+      # select txoutputs by owner that have neither been spent nor have a corresponding ethevents exit events
+      where: txoutput.owner == ^address and is_nil(txoutput.spending_txhash) and (is_nil(ethevent) or fragment("
+NOT EXISTS (SELECT 1
+           FROM ethevents_txoutputs AS etfrag
+           JOIN ethevents AS efrag ON
+                    etfrag.root_chain_txhash_event=efrag.root_chain_txhash_event
+                    AND efrag.event_type IN (?)
+                    AND etfrag.child_chain_utxohash = ?)", "standard_exit", txoutput.child_chain_utxohash)),
+      order_by: [asc: :blknum, asc: :txindex, asc: :oindex]
+    )
+  end
+
+  @spec get_all_utxos(OMG.Crypto.address_t()) :: list()
+  defp get_all_utxos(address) do
+    query = query_get_utxos(address)
+    Repo.all(query)
+  end
 end

apps/omg_watcher_info/test/omg_watcher_info/db/eth_event_test.exs

@@ -189,9 +189,10 @@ defmodule OMG.WatcherInfo.DB.EthEventTest do
              root_chain_txhash_event: ^expected_root_chain_txhash_event_3
            } = DB.EthEvent.get(expected_root_chain_txhash_event_3)
 
+    %{data: alice_utxos} = DB.TxOutput.get_utxos(address: alice.addr)
+
     assert [^expected_root_chain_txhash_1, ^expected_root_chain_txhash_2, ^expected_root_chain_txhash_3] =
-             DB.TxOutput.get_utxos(alice.addr)
-             |> Enum.map(fn txoutput ->
+             Enum.map(alice_utxos, fn txoutput ->
                [head | _tail] = txoutput.ethevents
                head.root_chain_txhash
              end)
@@ -225,7 +226,8 @@ defmodule OMG.WatcherInfo.DB.EthEventTest do
                }
              ])
 
-    assert length(DB.TxOutput.get_utxos(expected_owner)) == 1
+    %{data: utxos} = DB.TxOutput.get_utxos(address: expected_owner)
+    assert length(utxos) == 1
 
     assert :ok =
              DB.EthEvent.insert_exits!([
@@ -236,7 +238,8 @@ defmodule OMG.WatcherInfo.DB.EthEventTest do
                }
              ])
 
-    assert Enum.empty?(DB.TxOutput.get_utxos(expected_owner))
+    %{data: utxos_after_exit} = DB.TxOutput.get_utxos(address: expected_owner)
+    assert Enum.empty?(utxos_after_exit)
   end
 
   @tag fixtures: [:alice, :initial_blocks]

apps/omg_watcher_rpc/lib/web/controllers/account.ex

@@ -21,6 +21,7 @@ defmodule OMG.WatcherRPC.Web.Controller.Account do
 
   alias OMG.Watcher.API, as: SecurityAPI
   alias OMG.WatcherInfo.API, as: InfoAPI
+  alias OMG.WatcherRPC.Web.Validator.AccountConstraints
 
   @doc """
   Gets plasma account balance
@@ -34,8 +35,8 @@ defmodule OMG.WatcherRPC.Web.Controller.Account do
   end
 
   def get_utxos(conn, params) do
-    with {:ok, address} <- expect(params, "address", :address) do
-      address
+    with {:ok, constraints} <- AccountConstraints.parse(params) do
+      constraints
       |> InfoAPI.Account.get_utxos()
       |> api_response(conn, :utxos)
     end
@@ -45,7 +46,7 @@ defmodule OMG.WatcherRPC.Web.Controller.Account do
     with {:ok, address} <- expect(params, "address", :address) do
       address
       |> SecurityAPI.Account.get_exitable_utxos()
-      |> api_response(conn, :utxos)
+      |> api_response(conn, :exitable_utxos)
     end
   end
 end

apps/omg_watcher_rpc/lib/web/validators/account_constraints.ex

@@ -0,0 +1,35 @@
+# Copyright 2019-2020 OmiseGO Pte Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+defmodule OMG.WatcherRPC.Web.Validator.AccountConstraints do
+  @moduledoc """
+  Validates `/account.get_utxos` query parameters
+  """
+
+  import OMG.Utils.HttpRPC.Validator.Base, only: [expect: 3]
+
+  @doc """
+  Validates possible query constraints, stops on first error.
+  """
+  @spec parse(%{binary() => any()}) :: {:ok, Keyword.t()} | {:error, any()}
+  def parse(params) do
+    constraints = [
+      {"limit", [pos_integer: true, lesser: 1000, optional: true], :limit},
+      {"page", [:pos_integer, :optional], :page},
+      {"address", [:address], :address}
+    ]
+
+    OMG.WatcherRPC.Web.Validator.Helpers.validate_constraints(params, constraints)
+  end
+end

apps/omg_watcher_rpc/lib/web/validators/block_constraints.ex

@@ -17,24 +17,16 @@ defmodule OMG.WatcherRPC.Web.Validator.BlockConstraints do
   Validates `/block.all` query parameters
   """
 
-  import OMG.Utils.HttpRPC.Validator.Base, only: [expect: 3]
-
   @doc """
   Validates possible query constraints, stops on first error.
   """
   @spec parse(%{binary() => any()}) :: {:ok, Keyword.t()} | {:error, any()}
   def parse(params) do
     constraints = [
-      {"limit", [:pos_integer, :optional]},
-      {"page", [:pos_integer, :optional]}
+      {"limit", [pos_integer: true, lesser: 1000, optional: true], :limit},
+      {"page", [:pos_integer, :optional], :page}
     ]
 
-    Enum.reduce_while(constraints, {:ok, []}, fn {key, validators}, {:ok, list} ->
-      case expect(params, key, validators) do
-        {:ok, nil} -> {:cont, {:ok, list}}
-        {:ok, value} -> {:cont, {:ok, [{String.to_existing_atom(key), value} | list]}}
-        error -> {:halt, error}
-      end
-    end)
+    OMG.WatcherRPC.Web.Validator.Helpers.validate_constraints(params, constraints)
   end
 end

apps/omg_watcher_rpc/lib/web/validators/helpers.ex

@@ -0,0 +1,38 @@
+# Copyright 2019-2020 OmiseGO Pte Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+defmodule OMG.WatcherRPC.Web.Validator.Helpers do
+  @moduledoc """
+  helper for validators
+  """
+  import OMG.Utils.HttpRPC.Validator.Base, only: [expect: 3]
+
+  @doc """
+  Validates possible query constraints, stops on first error.
+  """
+  def validate_constraints(params, constraints) do
+    Enum.reduce_while(constraints, {:ok, []}, fn {key, validators, atom}, {:ok, list} ->
+      case expect(params, key, validators) do
+        {:ok, nil} ->
+          {:cont, {:ok, list}}
+
+        {:ok, value} ->
+          {:cont, {:ok, [{atom, value} | list]}}
+
+        error ->
+          {:halt, error}
+      end
+    end)
+  end
+end

apps/omg_watcher_rpc/lib/web/validators/transaction_constraints.ex

@@ -16,9 +16,7 @@ defmodule OMG.WatcherRPC.Web.Validator.TransactionConstraints do
   @moduledoc """
   Validates `/transaction.all` query parameters
   """
-
-  import OMG.Utils.HttpRPC.Validator.Base
-
+  import OMG.Utils.HttpRPC.Validator.Base, only: [expect: 3]
   @max_tx_types 16
 
   @doc """
@@ -31,22 +29,11 @@ defmodule OMG.WatcherRPC.Web.Validator.TransactionConstraints do
       {"blknum", [:pos_integer, :optional], :blknum},
       {"metadata", [:hash, :optional], :metadata},
       {"txtypes", [list: &to_tx_type/1, max_length: @max_tx_types, optional: true], :txtypes},
-      {"limit", [:pos_integer, :optional], :limit},
+      {"limit", [pos_integer: true, lesser: 1000, optional: true], :limit},
       {"page", [:pos_integer, :optional], :page}
     ]
 
-    Enum.reduce_while(constraints, {:ok, []}, fn {key, validators, atom}, {:ok, list} ->
-      case expect(params, key, validators) do
-        {:ok, nil} ->
-          {:cont, {:ok, list}}
-
-        {:ok, value} ->
-          {:cont, {:ok, [{atom, value} | list]}}
-
-        error ->
-          {:halt, error}
-      end
-    end)
+    OMG.WatcherRPC.Web.Validator.Helpers.validate_constraints(params, constraints)
   end
 
   defp to_tx_type(tx_type_str) do