Merge branch 'ensemble/hydra-producer'

.gitignore

@@ -17,3 +17,4 @@ dist-newstyle
 ## Dist
 /dist
 *.tar.gz
+/cabal.project.local

.gitmodules

@@ -4,3 +4,6 @@
 [submodule "test/ogmios"]
 	path = test/vectors/ogmios
 	url = [email protected]:cardanosolutions/ogmios.git
+[submodule "test/vectors/hydra"]
+	path = test/vectors/hydra
+	url = [email protected]:input-output-hk/hydra.git

CHANGELOG.md

@@ -1,3 +1,31 @@
+### [2.7.0] - 2023-10-13
+
+#### Added
+
+- Support for indexing a Hydra head.
+
+  Use `--hydra-host` and `--hydra-port` to connect `kupo` to a `hydra-node`.
+
+  > **Note**:
+  >
+  > Hydra heads do not actually form a 'chain'; Hydra doesn't have blocks, but
+  > snapshots which are akin to blocks so we treat them as such. The block
+  > header hash we opted for is arbitrary and is a hash of all the transaction
+  > id present in the snapshot.
+  >
+  > It also comes with diminished capabilities since Hydra doesn't have any
+  > protocol to query metadata for example and Kupo does not store them. So
+  > unlike Ogmios or Cardano-node, metadata cannot be retrieved when Hydra is
+  > used as a chain producer.
+
+#### Changed
+
+- N/A
+
+#### Removed
+
+- N/A
+
 ### [2.6.1] - 2023-08-30
 
 #### Added

cabal.project

@@ -23,7 +23,6 @@ test-show-details: direct
 
 package kupo
   tests: true
-  flags: +production
 
 package websockets-json
   tests: false

docs/api/latest.yaml

@@ -224,6 +224,25 @@ info:
 
     Kupo will synchronize data directly from Ogmios! Neat isn't it?
 
+    ## --hydra-host {hostname} / --hydra-port {port-number}
+
+    Kupo can also be used to index the layer two ledger available in a
+    [Hydra](https://github.com/input-output-hk/hydra/#readme) head. For this, we
+    need to use `--hydra-host` and `--hydra-port` to point to a machine running
+    a `hydra-node` of a head you want to index and the port of the websocket
+    API.
+
+    For example:
+
+    ```console
+    $ kupo \
+      --hydra-host 0.0.0.0 \
+      --hydra-port 4001 \
+      --since origin \
+      --match * \
+      --workdir ./db
+    ```
+
     ## --help
 
     In case you're lost, don't forget that a summary of this manual is available by running:

docs/man/README.md

@@ -32,22 +32,32 @@ kupo - Fast, lightweight & configurable chain-index for Cardano.
 **--node-socket**
 :   Path to the Cardano node domain socket file.
 
-    (**NOTE**: Unused when connecting to Ogmios)
+    (**NOTE**: Unused when connecting to Ogmios or Hydra)
 
 **--node-config**
 :   Path to the Cardano node configuration file.
 
-    (**NOTE**: Unused when connecting to Ogmios)
+    (**NOTE**: Unused when connecting to Ogmios or Hydra)
 
 **--ogmios-host**
 :   Ogmios' host address.
 
-    (**NOTE**: Unused when connecting to a Cardano node)
+    (**NOTE**: Unused when connecting to a Cardano node or Hydra)
 
 **--ogmios-port**
 :   Ogmios' port.
 
-    (**NOTE**: Unused when connecting to a Cardano node)
+    (**NOTE**: Unused when connecting to a Cardano node or Hydra)
+
+**--hydra-host**
+:   Hydra-node host address.
+
+    (**NOTE**: Unused when connecting to a Cardano node or Ogmios)
+
+**--hydra-port**
+:   Hydra-node port.
+
+    (**NOTE**: Unused when connecting to a Cardano node or Ogmios)
 
 **--workdir**
 :   Path to a working directory, where the SQLite database files are stored. By convention, the database is called <u>kupo.sqlite</u>.
@@ -256,6 +266,13 @@ kupo - Fast, lightweight & configurable chain-index for Cardano.
   --since 16588737.4e9bbbb67e3ae262133d94c3da5bffce7b1127fc436e7433b87668dba34c354a\
   --match addr1vyc29pvl2uyzqt8nwxrcxnf558ffm27u3d9calxn8tdudjgz4xq9p\
 
+**kupo**\
+  --hydra-host 0.0.0.0\
+  --hydra-port 4001\
+  --in-memory\
+  --since origin\
+  --match *
+
 # SEE ALSO
 Online documentation and API reference: <https://cardanosolutions.github.io/kupo/>
 

modules/websockets-json/package.yaml

@@ -28,5 +28,6 @@ library:
     - bytestring
     - connection
     - exceptions
+    - io-classes
     - websockets
     - wuss

modules/websockets-json/src/Network/WebSockets/Json.hs

@@ -13,7 +13,7 @@ import Prelude
 import Control.Exception
     ( Exception
     )
-import Control.Monad.Catch
+import Control.Monad.Class.MonadThrow
     ( MonadThrow (..)
     )
 import Control.Monad.IO.Class
@@ -54,7 +54,7 @@ receiveJson
 receiveJson ws decoder =  do
     bytes <- liftIO (WS.receiveData ws)
     either
-        (\(path, err) -> throwM $ MalformedOrUnexpectedResponse bytes path err)
+        (\(path, err) -> throwIO $ MalformedOrUnexpectedResponse bytes path err)
         pure
         (Json.eitherDecodeWith Json.jsonEOF (Json.iparse decoder) bytes)
 

src/Kupo/App.hs

@@ -2,6 +2,7 @@
 --  License, v. 2.0. If a copy of the MPL was not distributed with this
 --  file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
+{-# LANGUAGE PatternSynonyms #-}
 {-# LANGUAGE RecordWildCards #-}
 
 module Kupo.App
@@ -77,6 +78,7 @@ import Kupo.Data.Cardano
     , distanceToTip
     , getPoint
     , getPointSlotNo
+    , pattern GenesisPoint
     )
 import Kupo.Data.ChainSync
     ( ForcedRollbackHandler (..)
@@ -100,6 +102,9 @@ import Kupo.Data.Database
 import Kupo.Data.FetchBlock
     ( FetchBlockClient
     )
+import Kupo.Data.PartialBlock
+    ( PartialBlock
+    )
 import Kupo.Data.Pattern
     ( Codecs (..)
     , Match (..)
@@ -111,6 +116,7 @@ import Kupo.Data.Pattern
 
 import qualified Data.Map as Map
 import qualified Data.Set as Set
+import qualified Kupo.App.ChainSync.Hydra as Hydra
 import qualified Kupo.App.ChainSync.Node as Node
 import qualified Kupo.App.ChainSync.Ogmios as Ogmios
 import qualified Kupo.App.FetchBlock.Node as Node
@@ -166,6 +172,38 @@ newProducer tr chainProducer callback = do
 
                 runOgmios checkpoints (return ()) throwIO restart
 
+        Hydra{hydraHost, hydraPort} -> do
+            logWith tr ConfigurationHydra{hydraHost, hydraPort}
+            mailbox <- atomically (newMailbox mailboxCapacity)
+
+            callback forcedRollbackCallback mailbox $ \_tracerChainSync checkpoints statusToggle -> do
+                let runHydra pts beforeMainLoop onIntersectionNotFound continuation = do
+                        res <- race
+                            (Hydra.connect statusToggle hydraHost hydraPort $
+                                -- NOTE: Kupo does not generally index the genesis 'block' / configuration.
+                                -- That's true for the normal case with ogmios or cardano-node; but for
+                                -- Hydra, we might want to.
+                                --
+                                -- This is debatable and should be backed by use-cases, and may also be
+                                -- confusing since normally Kupo starts indexing only the blocks that
+                                -- follows the given checkpoints. But now we make an exception for genesis.
+                                Hydra.runChainSyncClient mailbox beforeMainLoop (filter (/= GenesisPoint) pts)
+                            )
+                            (atomically (takeTMVar forcedRollbackVar))
+                        case res of
+                            Left notFound ->
+                                onIntersectionNotFound notFound
+                            Right (point, handler) ->
+                                continuation point handler
+
+                let restart point handler =
+                        runHydra [point]
+                            (onSuccess handler)
+                            (\e -> onFailure handler >> throwIO e)
+                            restart
+
+                runHydra checkpoints (return ()) throwIO restart
+
         CardanoNode{nodeSocket, nodeConfig} -> do
             logWith tr ConfigurationCardanoNode{nodeSocket,nodeConfig}
             mailbox <- atomically (newMailbox mailboxCapacity)
@@ -209,6 +247,8 @@ withFetchBlockClient chainProducer callback = do
     case chainProducer of
         Ogmios{ogmiosHost, ogmiosPort} ->
             Ogmios.withFetchBlockClient ogmiosHost ogmiosPort callback
+        Hydra{} ->
+            callback @PartialBlock (\_point respond -> respond Nothing)
         CardanoNode{nodeSocket, nodeConfig} -> do
             NetworkParameters
                 { networkMagic