Merge pull request #74 from threefoldtech/service_contracts

Service contracts

db/migrations/1672322090839-service_contract.js

@@ -0,0 +1,13 @@
+module.exports = class service_contract1672322090839 {
+  name = 'service_contract1672322090839'
+
+  async up(db) {
+    await db.query(`CREATE TABLE "service_contract" ("id" character varying NOT NULL, "service_contract_id" numeric NOT NULL, "service_twin_id" integer NOT NULL, "consumer_twin_id" integer NOT NULL, "base_fee" numeric NOT NULL, "variable_fee" numeric NOT NULL, "metadata" text NOT NULL, "accepted_by_service" boolean NOT NULL, "accepted_by_consumer" boolean NOT NULL, "last_bill" numeric NOT NULL, "state" character varying(14) NOT NULL, CONSTRAINT "PK_ff58318f8230b8053067edd0343" PRIMARY KEY ("id"))`)
+    await db.query(`CREATE TABLE "service_contract_bill" ("id" character varying NOT NULL, "service_contract_id" numeric NOT NULL, "variable_amount" numeric NOT NULL, "window" numeric NOT NULL, "metadata" text, "amount" numeric NOT NULL, CONSTRAINT "PK_1fd26292c0913e974b774342fa7" PRIMARY KEY ("id"))`)
+  }
+
+  async down(db) {
+    await db.query(`DROP TABLE "service_contract"`)
+    await db.query(`DROP TABLE "service_contract_bill"`)
+  }
+}

docs/development.md

@@ -25,9 +25,13 @@ docker-compose up -d
 Indexer services should now be started, you can check if it's syncing properly by streaming the logs for the indexer:
 
 ```
-docker logs indexer-indexer-1 -f
+docker logs indexer_indexer_1 -f
 ```
 
+You should be able to follow tfchain blocks processing:
+
+![image](https://user-images.githubusercontent.com/73958772/209998096-3d5381d9-97ee-438d-824d-d92d997b42aa.png)
+
 ### Run processor
 
 Check `.env` and adjust the websocket endpoint to your local tfchain address.
@@ -38,10 +42,33 @@ yarn db:up
 yarn process
 ```
 
+You should be able to follow tfchain blocks processing:
+
+![image](https://user-images.githubusercontent.com/73958772/210000023-c575d91a-382e-4fdc-85b3-199a135b493f.png)
+
+
+If you make some changes, don't forget to turn down container before tuning it on again.
+
+```
+docker-compose down
+```
+
 ### Run graphql UI
 
+At this step, by running 
+
+```
+docker ps
+```
+
+it should display such list of running containers:
+
+![image](https://user-images.githubusercontent.com/73958772/210003731-6abd405f-f92a-440d-aa9a-40410e580109.png)
+
+Make sure indexer and processor are both listening to tfchain to be able to browse.
+
 ```
 yarn api
 ```
 
-Browse to http://localhost:4000/graphql to see the UI
+Now you can use the UI (http://localhost:4000/graphql) and run some tests.

docs/typeChanges.md

@@ -1,5 +1,149 @@
 # How to handle type changes on chain
 
-## Type change  
+## Type change
 
-# TODO
+If a type change was made or some types were added we can use the following process to add the corresponding changes.
+
+### 1: Run a local chain
+
+See [./development]
+
+Make sure you increment the `specVersion` before you compile and run tfchain.
+
+### 2: Setup new indexer types
+
+Modify `indexer/typesBundle.json` with the new types additions / changes.
+
+Example: in specVersion 104, `solution_provider_id` was added to the `Contract` object, the typesbundle definition for that looks like:
+
+```json
+{
+    "minmax": [
+        104,
+        null
+    ],
+    "types": {
+        "Contract": {
+          "version": " u32",
+          "state": "ContractState",
+          "contractId": "u64",
+          "twinId": "u32",
+          "contractType": "ContractData",
+          "solutionProviderID": "Option<U64>"
+        }
+    }
+},
+```
+
+Notice the [104, null] definition, this means this type if valid from specVersion 104 until some to define perdiod (null if you don't know).
+
+If specVersion 104 would just add a type without modifiying existing types it could look like this:
+
+```json
+{
+    "minmax": [
+        104,
+        null
+    ],
+    "types": {
+        "newType": {
+          "someId": "u64",
+        }
+    }
+},
+```
+
+When the types.json file has been modified, it needs to be updated in the helm chart as well.
+
+```sh
+kubectl create configmap indexer-config --from-file=./typesBundle.json --dry-run=client --output=yaml > chart/templates/indexer-config.yaml
+```
+
+Also increment the indexer chart version `indexer/chart/chart.yaml`
+
+### 3: Run typegen exploration
+
+`cd typegenLocal`
+
+Copy over the typesBundle.json changes to `typegenLocal/typesBundle.json`.
+
+Once the typesBundle is modified, check if changes are required for generating types in `typegen.json`
+
+If you for example need to track a new event you need to edit the events declaration:
+
+```json
+  "events": [
+    "balances.Transfer",
+    "tfgridModule.EntityStored",
+    "tfgridModule.EntityUpdated",
+    "tfgridModule.EntityDeleted",
+    ...
+    "someModule.newEvent"
+    ...
+```
+
+Once you have done this, it's time to generate the types.
+
+```
+rm tfchainVersions.json
+```
+
+First, explore new types from the metadata:
+
+```
+npx squid-substrate-metadata-explorer \
+                --chain ws://localhost:9944 \
+                --out tfchainVersions.json
+```
+
+Then, run the type generation:
+
+```
+npx squid-substrate-typegen typegen.json
+```
+
+Notice it will re-generate following files:
+
+- src/typesLocal/events.js
+- src/typesLocal/$SPECVERSION.js
+
+### 4: Add new types / event to the main events definitions
+
+All events / types should be moved to `src/types` because we cannot generate types based on 1 chain because this code served more than 1 chain.
+
+At the inception of this project we generated `src/types` from devnet chain, but since we resetted devnet and we don't incrementally upgrade every network
+at all times, we stopped generating `src/types` and started modifiying them manually. While this is not really easy to develop against, it's the only solution for us.
+
+So we need to extend the generated src/types with incremental type changes we make over time.
+
+Steps:
+
+- move new $specVersion.ts file to `src/types`
+- copy changes from `src/typesLocal/events.ts` to `src/types/events.ts` (see which events were added / modified and only copy those over)
+
+### 5: Modify graphql schema
+
+Make changes to the graphql schema but prepare the database first:
+
+```
+yarn db:up
+yarn process
+```
+
+The process will likely crash or stop but the db is initialised with the current schema. Now we can edit the graphql schema file: `schema.graphql`.
+
+Once changes are done to the schema, you can create a database migration:
+
+```
+yarn codegen
+yarn build
+yarn db:create-migration
+```
+
+### 6: Create / extend mappers
+
+Make the corresponding changes in `src/processor.ts` and mappers `src/mappings/..` for the type changes that were made
+
+### 7: Build and test
+
+TODO

indexer/chart/Chart.yaml

@@ -1,5 +1,5 @@
 name: tfchainindexer
 description: Helm Chart for the tfchain hydra indexer
-version: 2.5.0
+version: 2.7.0
 apiVersion: v2
-appVersion: "2.6.1"
+appVersion: '2.7.0'

indexer/chart/templates/indexer-config.yaml

@@ -598,6 +598,38 @@ data:
                   "longitude": "Vec<u8>"
                 }
               }
+            },
+            {
+              "minmax": [
+                122,
+                null
+              ],
+              "types": {
+                "ServiceContract": {
+                  "serviceContractId": "u64",
+                  "serviceTwinId:": "u32",
+                  "consumerTwinId": "u32",
+                  "baseFee": "u64",
+                  "variableFee": "u64",
+                  "metadata": "Vec<u8>",
+                  "acceptedByService": "bool",
+                  "acceptedByConsumer": "bool",
+                  "lastBill": "u64",
+                  "state": "ServiceContractState"
+                },
+                "ServiceContractState": {
+                  "_enum": {
+                    "Created": null,
+                    "AgreementReady": null,
+                    "ApprovedByBoth": null
+                  }
+                },
+                "ServiceContractBill": {
+                  "variableAmount": "u64",
+                  "window": "u64",
+                  "metadata": "Vec<u8>"
+                }
+              }
             }
           ]
         }

indexer/typesBundle.json

@@ -595,6 +595,38 @@
               "longitude": "Vec<u8>"
             }
           }
+        },
+        {
+          "minmax": [
+            122,
+            null
+          ],
+          "types": {
+            "ServiceContract": {
+              "serviceContractId": "u64",
+              "serviceTwinId:": "u32",
+              "consumerTwinId": "u32",
+              "baseFee": "u64",
+              "variableFee": "u64",
+              "metadata": "Vec<u8>",
+              "acceptedByService": "bool",
+              "acceptedByConsumer": "bool",
+              "lastBill": "u64",
+              "state": "ServiceContractState"
+            },
+            "ServiceContractState": {
+              "_enum": {
+                "Created": null,
+                "AgreementReady": null,
+                "ApprovedByBoth": null
+              }
+            },
+            "ServiceContractBill": {
+              "variableAmount": "u64",
+              "window": "u64",
+              "metadata": "Vec<u8>"
+            }
+          }
         }
       ]
     }

schema.graphql

@@ -283,3 +283,30 @@ type RefundTransaction @entity {
   target: String!
   txHash: String!
 }
+
+type ServiceContract @entity {
+  serviceContractID: BigInt!
+  serviceTwinID: Int!
+  consumerTwinID: Int!
+  baseFee: BigInt!
+  variableFee: BigInt!
+  metadata: String!
+  acceptedByService: Boolean!
+  acceptedByConsumer: Boolean!
+  lastBill: BigInt!
+  state: ServiceContractState!
+}
+
+enum ServiceContractState {
+  Created
+  AgreementReady
+  ApprovedByBoth
+}
+
+type ServiceContractBill @entity {
+  serviceContractID: BigInt!
+  variableAmount: BigInt!
+  window: BigInt!
+  metadata: String
+  amount: BigInt!
+}