Create watcher in eth_call mode for Claims contract (#5)

Co-authored-by: Dhruv Srivastava <[email protected]>

packages/claims-watcher/.eslintignore

@@ -0,0 +1,2 @@
+# Don't lint build output.
+dist

packages/claims-watcher/.eslintrc.json

@@ -0,0 +1,28 @@
+{
+    "env": {
+        "browser": true,
+        "es2021": true
+    },
+    "extends": [
+        "semistandard",
+        "plugin:@typescript-eslint/recommended"
+    ],
+    "parser": "@typescript-eslint/parser",
+    "parserOptions": {
+        "ecmaVersion": 12,
+        "sourceType": "module"
+    },
+    "plugins": [
+        "@typescript-eslint"
+    ],
+    "rules": {
+        "indent": ["error", 2, { "SwitchCase": 1 }],
+        "@typescript-eslint/no-explicit-any": "off",
+        "@typescript-eslint/explicit-module-boundary-types": [
+            "warn",
+            {
+                "allowArgumentsExplicitlyTypedAsAny": true
+            }
+        ]
+    }
+}

packages/claims-watcher/README.md

@@ -0,0 +1,204 @@
+# claims-watcher
+
+## Setup
+
+* Run the following command to install required packages:
+
+  ```bash
+  yarn
+  ```
+
+* Create a postgres12 database for the watcher:
+
+  ```bash
+  sudo su - postgres
+  createdb claims-watcher
+  ```
+
+* If the watcher is an `active` watcher:
+
+  Create database for the job queue and enable the `pgcrypto` extension on them (https://github.com/timgit/pg-boss/blob/master/docs/usage.md#intro):
+
+  ```
+  createdb claims-watcher-job-queue
+  ```
+
+  ```
+  postgres@tesla:~$ psql -U postgres -h localhost claims-watcher-job-queue
+  Password for user postgres:
+  psql (12.7 (Ubuntu 12.7-1.pgdg18.04+1))
+  SSL connection (protocol: TLSv1.3, cipher: TLS_AES_256_GCM_SHA384, bits: 256, compression: off)
+  Type "help" for help.
+
+  claims-watcher-job-queue=# CREATE EXTENSION pgcrypto;
+  CREATE EXTENSION
+  claims-watcher-job-queue=# exit
+  ```
+
+* In the [config file](./environments/local.toml):
+
+  * Update the database connection settings.
+
+  * Update the `upstream` config and provide the `ipld-eth-server` GQL API endpoint.
+
+  * Update the `server` config with state checkpoint settings.
+
+## Customize
+
+* Indexing on an event:
+
+  * Edit the custom hook function `handleEvent` (triggered on an event) in [hooks.ts](./src/hooks.ts) to perform corresponding indexing using the `Indexer` object.
+
+  * While using the indexer storage methods for indexing, pass `diff` as true if default state is desired to be generated using the state variables being indexed.
+
+* Generating state:
+
+  * Edit the custom hook function `createInitialState` (triggered if the watcher passes the start block, checkpoint: `true`) in [hooks.ts](./src/hooks.ts) to save an initial `State` using the `Indexer` object.
+
+  * Edit the custom hook function `createStateDiff` (triggered on a block) in [hooks.ts](./src/hooks.ts) to save the state in a `diff` `State` using the `Indexer` object. The default state (if exists) is updated.
+
+  * Edit the custom hook function `createStateCheckpoint` (triggered just before default and CLI checkpoint) in [hooks.ts](./src/hooks.ts) to save the state in a `checkpoint` `State` using the `Indexer` object.
+
+### GQL Caching
+
+To enable GQL requests caching:
+
+* Update the `server.gqlCache` config with required settings.
+
+* In the GQL [schema file](./src/schema.gql), use the `cacheControl` directive to apply cache hints at schema level.
+
+  * Eg. Set `inheritMaxAge` to true for non-scalar fields of a type.
+
+* In the GQL [resolvers file](./src/resolvers.ts), uncomment the `setGQLCacheHints()` calls in resolvers for required queries.
+
+## Run
+
+* If the watcher is a `lazy` watcher:
+
+  * Run the server:
+
+    ```bash
+    yarn server
+    ```
+
+    GQL console: http://localhost:3009/graphql
+
+* If the watcher is an `active` watcher:
+
+  * Run the job-runner:
+
+    ```bash
+    yarn job-runner
+    ```
+
+  * Run the server:
+
+    ```bash
+    yarn server
+    ```
+
+    GQL console: http://localhost:3009/graphql
+
+  * To watch a contract:
+
+    ```bash
+    yarn watch:contract --address <contract-address> --kind <contract-kind> --checkpoint <true | false> --starting-block [block-number]
+    ```
+
+    * `address`: Address or identifier of the contract to be watched.
+    * `kind`: Kind of the contract.
+    * `checkpoint`: Turn checkpointing on (`true` | `false`).
+    * `starting-block`: Starting block for the contract (default: `1`).
+
+    Examples:
+
+    Watch a contract with its address and checkpointing on:
+
+    ```bash
+    yarn watch:contract --address 0x1F78641644feB8b64642e833cE4AFE93DD6e7833 --kind ERC20 --checkpoint true
+    ```
+
+    Watch a contract with its identifier and checkpointing on:
+
+    ```bash
+    yarn watch:contract --address MyProtocol --kind protocol --checkpoint true
+    ```
+
+  * To fill a block range:
+
+    ```bash
+    yarn fill --start-block <from-block> --end-block <to-block>
+    ```
+
+    * `start-block`: Block number to start filling from.
+    * `end-block`: Block number till which to fill.
+
+  * To create a checkpoint for a contract:
+
+    ```bash
+    yarn checkpoint create --address <contract-address> --block-hash [block-hash]
+    ```
+
+    * `address`: Address or identifier of the contract for which to create a checkpoint.
+    * `block-hash`: Hash of a block (in the pruned region) at which to create the checkpoint (default: latest canonical block hash).
+
+  * To reset the watcher to a previous block number:
+
+    * Reset watcher:
+
+      ```bash
+      yarn reset watcher --block-number <previous-block-number>
+      ```
+
+    * Reset job-queue:
+
+      ```bash
+      yarn reset job-queue
+      ```
+
+    * Reset state:
+
+      ```bash
+      yarn reset state --block-number <previous-block-number>
+      ```
+
+    * `block-number`: Block number to which to reset the watcher.
+
+  * To export and import the watcher state:
+
+    * In source watcher, export watcher state:
+
+      ```bash
+      yarn export-state --export-file [export-file-path] --block-number [snapshot-block-height]
+      ```
+
+      * `export-file`: Path of file to which to export the watcher data.
+      * `block-number`: Block height at which to take snapshot for export.
+
+    * In target watcher, run job-runner:
+
+      ```bash
+      yarn job-runner
+      ```
+
+    * Import watcher state:
+
+      ```bash
+      yarn import-state --import-file <import-file-path>
+      ```
+
+      * `import-file`: Path of file from which to import the watcher data.
+
+    * Run server:
+
+      ```bash
+      yarn server
+      ```
+
+  * To inspect a CID:
+
+    ```bash
+    yarn inspect-cid --cid <cid>
+    ```
+
+    * `cid`: CID to be inspected.

packages/claims-watcher/environments/local.toml

@@ -0,0 +1,66 @@
+[server]
+  host = "127.0.0.1"
+  port = 3009
+  kind = "lazy"
+
+  # Checkpointing state.
+  checkpointing = true
+
+  # Checkpoint interval in number of blocks.
+  checkpointInterval = 2000
+
+  # Enable state creation
+  # CAUTION: Disable only if state creation is not desired or can be filled subsequently
+  enableState = true
+
+  # Boolean to filter logs by contract.
+  filterLogs = false
+
+  # Max block range for which to return events in eventsInRange GQL query.
+  # Use -1 for skipping check on block range.
+  maxEventsBlockRange = 1000
+
+  # GQL cache settings
+  [server.gqlCache]
+    enabled = true
+
+    # Max in-memory cache size (in bytes) (default 8 MB)
+    # maxCacheSize
+
+    # GQL cache-control max-age settings (in seconds)
+    maxAge = 15
+
+[metrics]
+  host = "127.0.0.1"
+  port = 9000
+  [metrics.gql]
+    port = 9001
+
+[database]
+  type = "postgres"
+  host = "localhost"
+  port = 5432
+  database = "claims-watcher"
+  username = "postgres"
+  password = "postgres"
+  synchronize = true
+  logging = false
+
+[upstream]
+  [upstream.ethServer]
+    gqlApiEndpoint = "http://127.0.0.1:8082/graphql"
+    rpcProviderEndpoint = "http://127.0.0.1:8081"
+
+  [upstream.cache]
+    name = "requests"
+    enabled = false
+    deleteOnStart = false
+
+[jobQueue]
+  dbConnectionString = "postgres://postgres:postgres@localhost/claims-watcher-job-queue"
+  maxCompletionLagInSecs = 300
+  jobDelayInMilliSecs = 100
+  eventsInBatch = 50
+  blockDelayInMilliSecs = 2000
+  prefetchBlocksInMem = true
+  prefetchBlockCount = 10

packages/claims-watcher/package.json

@@ -0,0 +1,73 @@
+{
+  "name": "@cerc-io/claims-watcher",
+  "version": "0.1.0",
+  "description": "claims-watcher",
+  "private": true,
+  "main": "dist/index.js",
+  "scripts": {
+    "lint": "eslint .",
+    "build": "yarn clean && tsc && yarn copy-assets",
+    "clean": "rm -rf ./dist",
+    "copy-assets": "copyfiles -u 1 src/**/*.gql dist/",
+    "server": "DEBUG=vulcanize:*  YARN_CHILD_PROCESS=true node --enable-source-maps dist/server.js",
+    "server:dev": "DEBUG=vulcanize:* YARN_CHILD_PROCESS=true ts-node src/server.ts",
+    "job-runner": "DEBUG=vulcanize:* YARN_CHILD_PROCESS=true node --enable-source-maps dist/job-runner.js",
+    "job-runner:dev": "DEBUG=vulcanize:* YARN_CHILD_PROCESS=true ts-node src/job-runner.ts",
+    "watch:contract": "DEBUG=vulcanize:* ts-node src/cli/watch-contract.ts",
+    "fill": "DEBUG=vulcanize:* ts-node src/fill.ts",
+    "reset": "DEBUG=vulcanize:* ts-node src/cli/reset.ts",
+    "checkpoint": "DEBUG=vulcanize:* node --enable-source-maps dist/cli/checkpoint.js",
+    "checkpoint:dev": "DEBUG=vulcanize:* ts-node src/cli/checkpoint.ts",
+    "export-state": "DEBUG=vulcanize:* node --enable-source-maps dist/cli/export-state.js",
+    "export-state:dev": "DEBUG=vulcanize:* ts-node src/cli/export-state.ts",
+    "import-state": "DEBUG=vulcanize:* node --enable-source-maps dist/cli/import-state.js",
+    "import-state:dev": "DEBUG=vulcanize:* ts-node src/cli/import-state.ts",
+    "inspect-cid": "DEBUG=vulcanize:* ts-node src/cli/inspect-cid.ts",
+    "index-block": "DEBUG=vulcanize:* ts-node src/cli/index-block.ts"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/cerc-io/watcher-ts.git"
+  },
+  "author": "",
+  "license": "AGPL-3.0",
+  "bugs": {
+    "url": "https://github.com/cerc-io/watcher-ts/issues"
+  },
+  "homepage": "https://github.com/cerc-io/watcher-ts#readme",
+  "dependencies": {
+    "@apollo/client": "^3.3.19",
+    "@ethersproject/providers": "^5.4.4",
+    "@cerc-io/cli": "^0.2.34",
+    "@cerc-io/ipld-eth-client": "^0.2.34",
+    "@cerc-io/solidity-mapper": "^0.2.34",
+    "@cerc-io/util": "^0.2.34",
+    "apollo-type-bigint": "^0.1.3",
+    "debug": "^4.3.1",
+    "ethers": "^5.4.4",
+    "graphql": "^15.5.0",
+    "json-bigint": "^1.0.0",
+    "reflect-metadata": "^0.1.13",
+    "typeorm": "^0.2.32",
+    "yargs": "^17.0.1",
+    "decimal.js": "^10.3.1"
+  },
+  "devDependencies": {
+    "@ethersproject/abi": "^5.3.0",
+    "@types/yargs": "^17.0.0",
+    "@types/debug": "^4.1.5",
+    "@types/json-bigint": "^1.0.1",
+    "@typescript-eslint/eslint-plugin": "^5.47.1",
+    "@typescript-eslint/parser": "^5.47.1",
+    "eslint": "^8.35.0",
+    "eslint-config-semistandard": "^15.0.1",
+    "eslint-config-standard": "^16.0.3",
+    "eslint-plugin-import": "^2.27.5",
+    "eslint-plugin-node": "^11.1.0",
+    "eslint-plugin-promise": "^5.1.0",
+    "eslint-plugin-standard": "^5.0.0",
+    "ts-node": "^10.2.1",
+    "typescript": "^5.0.2",
+    "copyfiles": "^2.4.1"
+  }
+}