fix: remap miniMetadata lookup ids to placate TypeRegistry (#1417)

* fix: remap lookup ids when building miniMetadatas

* fix: delete cached balances for disabled tokens

* fix: dont fetch ss58 account balances from secp256k1 substrate chains

* fix: rare crash in chaindata-provider

* chore: added changeset

.changeset/thin-dryers-turn.md

@@ -0,0 +1,8 @@
+---
+"@talismn/chaindata-provider": patch
+"@talismn/balances-react": patch
+"extension-core": patch
+"@talismn/scale": patch
+---
+
+fix: remap lookup ids when building miniMetadatas

package.json

@@ -65,7 +65,7 @@
     "@polkadot/types-create": "10.11.2",
     "@polkadot/types-known": "10.11.2",
     "@polkadot/types-support": "10.11.2",
-    "@polkadot/types": "patch:@polkadot/types@npm:10.11.2#.yarn/patches/@polkadot-types-npm-10.11.2-9fef8e41bc.patch",
+    "@polkadot/types": "10.11.2",
     "@polkadot/hw-ledger-transports": "12.6.2",
     "@polkadot/hw-ledger": "12.6.2",
     "@polkadot/keyring": "12.6.2",

packages/balances-react/src/atoms/balances.ts

@@ -181,15 +181,14 @@ const balancesSubscriptionAtomEffect = atomEffect((get) => {
         addressesByTokenByModule[token.type][token.id] = allAddresses.filter((address) => {
           // for each address, fetch balances only from compatible chains
           return isEthereumAddress(address)
-            ? !!token.evmNetwork?.id || chainsById[token.chain?.id ?? ""]?.account === "secp256k1"
-            : !!token.chain?.id
+            ? token.evmNetwork?.id || chainsById[token.chain?.id ?? ""]?.account === "secp256k1"
+            : token.chain?.id && chainsById[token.chain?.id ?? ""]?.account !== "secp256k1"
         })
       })
 
     // Delete invalid cached balances
     const chainIds = new Set(chains.map((chain) => chain.id))
     const evmNetworkIds = new Set(evmNetworks.map((evmNetwork) => evmNetwork.id))
-    const tokenIds = new Set(tokens.map((token) => token.id))
     await deleteBalances((balance) => {
       // delete cached balances for accounts which don't exist anymore
       if (!balance.address || !allAddresses.includes(balance.address)) return true
@@ -200,8 +199,8 @@ const balancesSubscriptionAtomEffect = atomEffect((get) => {
       if (balance.evmNetworkId !== undefined && !evmNetworkIds.has(balance.evmNetworkId))
         return true
 
-      // delete cached balance when token doesn't exist
-      if (!tokenIds.has(balance.tokenId)) return true
+      // delete cached balance when token doesn't exist / is disabled
+      if (!enabledTokenIds.includes(balance.tokenId)) return true
 
       // delete cached balance when module doesn't exist
       if (!balanceModules.find((module) => module.type === balance.source)) return true
@@ -213,11 +212,12 @@ const balancesSubscriptionAtomEffect = atomEffect((get) => {
       const hasChain = balance.chainId && chainIds.has(balance.chainId)
       const hasEvmNetwork = balance.evmNetworkId && evmNetworkIds.has(balance.evmNetworkId)
       const chainUsesSecp256k1Accounts = chain?.account === "secp256k1"
-      if (!isEthereumAddress(balance.address) && !hasChain) {
-        return true
+      if (!isEthereumAddress(balance.address)) {
+        if (!hasChain) return true
+        if (chainUsesSecp256k1Accounts) return true
       }
-      if (isEthereumAddress(balance.address) && !(hasEvmNetwork || chainUsesSecp256k1Accounts)) {
-        return true
+      if (isEthereumAddress(balance.address)) {
+        if (!hasEvmNetwork && !chainUsesSecp256k1Accounts) return true
       }
 
       // keep balance

packages/chaindata-provider/src/net.ts

@@ -32,8 +32,8 @@ export const availableTokenLogoFilenames = async (): Promise<string[]> =>
     await fetch(
       `${githubApi}/repos/${githubChaindataOrg}/${githubChaindataRepo}/contents/${githubChaindataTokensAssetsDir}`
     ).then((response) => response.json())
-  ).flatMap((entry: unknown) => {
+  )?.flatMap?.((entry: unknown) => {
     if (typeof entry !== "object" || entry === null) return []
     if (!("name" in entry) || typeof entry.name !== "string") return []
     return entry.name
-  })
+  }) ?? []

packages/extension-core/src/domains/balances/store.ts

@@ -353,11 +353,12 @@ export class BalanceStore {
         const hasChain = balance.chainId && chainIds.has(balance.chainId)
         const hasEvmNetwork = balance.evmNetworkId && evmNetworkIds.has(balance.evmNetworkId)
         const chainUsesSecp256k1Accounts = chain?.account === "secp256k1"
-        if (!isEthereumAddress(balance.address) && !hasChain) {
-          return true
+        if (!isEthereumAddress(balance.address)) {
+          if (!hasChain) return true
+          if (chainUsesSecp256k1Accounts) return true
         }
-        if (isEthereumAddress(balance.address) && !(hasEvmNetwork || chainUsesSecp256k1Accounts)) {
-          return true
+        if (isEthereumAddress(balance.address)) {
+          if (!hasEvmNetwork && !chainUsesSecp256k1Accounts) return true
         }
 
         // keep balance
@@ -463,10 +464,10 @@ export class BalanceStore {
             addresses[address]?.includes(chainDetails[token.chain.id]?.genesisHash ?? "")
         )
         .filter((address) => {
-          // for each account, fetch balances only from compatible chains
+          // for each address, fetch balances only from compatible chains
           return isEthereumAddress(address)
-            ? !!token.evmNetwork?.id || chainDetails[token.chain?.id ?? ""]?.account === "secp256k1"
-            : !!token.chain?.id
+            ? token.evmNetwork?.id || chainDetails[token.chain?.id ?? ""]?.account === "secp256k1"
+            : token.chain?.id && chainDetails[token.chain?.id ?? ""]?.account !== "secp256k1"
         })
     })
 

packages/scale/src/metadata/util.ts

@@ -81,6 +81,16 @@ export const filterMetadataPalletsAndItems = (
 
   // ditch the types we aren't keeping
   metadata.tys = metadata.tys.filter((type) => keepTypes.has(type.id))
+
+  // update all type ids to be sequential (fill the gaps left by the deleted types)
+  const newTypeIds = new Map<number, number>()
+  metadata.tys.forEach((ty, index) => newTypeIds.set(ty.id, index))
+  const getNewTypeId = (oldTypeId: number): number => {
+    const newTypeId = newTypeIds.get(oldTypeId)
+    if (typeof newTypeId !== "number") log.error(`Failed to find newTypeId for type ${oldTypeId}`)
+    return newTypeId ?? 0
+  }
+  remapTypeIds(metadata, getNewTypeId)
 }
 
 export const addDependentTypes = (
@@ -104,75 +114,114 @@ export const addDependentTypes = (
     keepTypes.add(type.id)
     addedTypes.add(type.id)
 
+    const paramTypes = type.params
+      .map((param) => param.ty)
+      .filter((type): type is number => typeof type === "number")
+    addDependentSubTypes(paramTypes)
+
     switch (type.type) {
+      case "SizedArray":
+        addDependentSubTypes([type.typeParam])
+        break
+
+      case "BitSequence":
+        addDependentSubTypes([type.bitOrderType, type.bitStoreType])
+        break
+
+      case "Compact":
+        addDependentSubTypes([type.typeParam])
+        break
+
       case "Struct":
-        addDependentSubTypes([
-          ...type.params
-            .map((param) => param.ty)
-            .filter((ty): ty is number => typeof ty === "number"),
-          ...type.fields
-            .map((field) => field.ty)
-            .filter((ty): ty is number => typeof ty === "number"),
-        ])
+        addDependentSubTypes(
+          type.fields.map((field) => field.ty).filter((ty): ty is number => typeof ty === "number")
+        )
         break
 
-      case "Union":
-        addDependentSubTypes([
-          ...type.params
-            .map((param) => param.ty)
-            .filter((ty): ty is number => typeof ty === "number"),
-          ...type.members
-            .flatMap((member) => member.fields.map((field) => field.ty))
-            .filter((ty): ty is number => typeof ty === "number"),
-        ])
+      case "Primitive":
         break
 
       case "Sequence":
-        addDependentSubTypes([
-          ...type.params
-            .map((param) => param.ty)
-            .filter((ty): ty is number => typeof ty === "number"),
-          type.typeParam,
-        ])
+        addDependentSubTypes([type.typeParam])
+        break
+
+      case "Tuple":
+        addDependentSubTypes(type.fields.filter((ty): ty is number => typeof ty === "number"))
+        break
+
+      case "Union":
+        addDependentSubTypes(
+          type.members
+            .flatMap((member) => member.fields.map((field) => field.ty))
+            .filter((ty): ty is number => typeof ty === "number")
+        )
         break
 
+      default: {
+        // force compilation error if any types don't have a case
+        const exhaustiveCheck: never = type
+        log.error(`Unhandled TyMV14 type ${exhaustiveCheck}`)
+      }
+    }
+  }
+}
+
+const remapTypeIds = (metadata: MetadataV14, getNewTypeId: (oldTypeId: number) => number) => {
+  remapLookupTypeIds(metadata, getNewTypeId)
+  remapStorageTypeIds(metadata, getNewTypeId)
+}
+
+const remapLookupTypeIds = (metadata: MetadataV14, getNewTypeId: (oldTypeId: number) => number) => {
+  for (const type of metadata.tys) {
+    type.id = getNewTypeId(type.id)
+
+    for (const param of type.params) {
+      if (typeof param.ty !== "number") continue
+      param.ty = getNewTypeId(param.ty)
+    }
+
+    switch (type.type) {
       case "SizedArray":
-        addDependentSubTypes([
-          ...type.params
-            .map((param) => param.ty)
-            .filter((ty): ty is number => typeof ty === "number"),
-          type.typeParam,
-        ])
+        type.typeParam = getNewTypeId(type.typeParam)
         break
 
-      case "Tuple":
-        addDependentSubTypes([
-          ...type.params
-            .map((param) => param.ty)
-            .filter((ty): ty is number => typeof ty === "number"),
-          ...type.fields.filter((ty): ty is number => typeof ty === "number"),
-        ])
+      case "BitSequence":
+        type.bitOrderType = getNewTypeId(type.bitOrderType)
+        type.bitStoreType = getNewTypeId(type.bitStoreType)
+        break
+
+      case "Compact":
+        type.typeParam = getNewTypeId(type.typeParam)
+        break
+
+      case "Struct":
+        for (const field of type.fields) {
+          if (typeof field.ty !== "number") continue
+          field.ty = getNewTypeId(field.ty)
+        }
         break
 
       case "Primitive":
-        addDependentSubTypes([
-          ...type.params
-            .map((param) => param.ty)
-            .filter((ty): ty is number => typeof ty === "number"),
-        ])
         break
 
-      case "Compact":
-        addDependentSubTypes([
-          ...type.params
-            .map((param) => param.ty)
-            .filter((ty): ty is number => typeof ty === "number"),
-          type.typeParam,
-        ])
+      case "Sequence":
+        type.typeParam = getNewTypeId(type.typeParam)
         break
 
-      case "BitSequence":
-        addDependentSubTypes([type.bitOrderType, type.bitStoreType])
+      case "Tuple":
+        type.fields = type.fields.map((ty) => {
+          if (typeof ty !== "number") return ty
+          return getNewTypeId(ty)
+        })
+        break
+
+      case "Union":
+        for (const member of type.members) {
+          for (const field of member.fields) {
+            if (typeof field.ty !== "number") continue
+            field.ty = getNewTypeId(field.ty)
+          }
+        }
         break
 
       default: {
@@ -183,3 +232,17 @@ export const addDependentTypes = (
     }
   }
 }
+const remapStorageTypeIds = (
+  metadata: MetadataV14,
+  getNewTypeId: (oldTypeId: number) => number
+) => {
+  for (const pallet of metadata.pallets) {
+    for (const item of pallet.storage?.entries ?? []) {
+      if (item.type === "Plain") item.value = getNewTypeId(item.value)
+      if (item.type === "Map") {
+        item.key = getNewTypeId(item.key)
+        item.value = getNewTypeId(item.value)
+      }
+    }
+  }
+}

yarn.lock

@@ -6764,22 +6764,6 @@ __metadata:
   languageName: node
   linkType: hard
 
-"@polkadot/types@patch:@polkadot/types@npm:10.11.2#.yarn/patches/@polkadot-types-npm-10.11.2-9fef8e41bc.patch::locator=talisman%40workspace%3A.":
-  version: 10.11.2
-  resolution: "@polkadot/types@patch:@polkadot/types@npm%3A10.11.2#.yarn/patches/@polkadot-types-npm-10.11.2-9fef8e41bc.patch::version=10.11.2&hash=0a65de&locator=talisman%40workspace%3A."
-  dependencies:
-    "@polkadot/keyring": ^12.6.2
-    "@polkadot/types-augment": 10.11.2
-    "@polkadot/types-codec": 10.11.2
-    "@polkadot/types-create": 10.11.2
-    "@polkadot/util": ^12.6.2
-    "@polkadot/util-crypto": ^12.6.2
-    rxjs: ^7.8.1
-    tslib: ^2.6.2
-  checksum: dc2be7978e88c7a08850b46523c7593d657338478300f5c69f60f228ff6925dffc8a874ab28169028853aeb88cd36c7df8704df63856c57ddccc7c24b67ccba1
-  languageName: node
-  linkType: hard
-
 "@polkadot/ui-keyring@npm:3.6.4":
   version: 3.6.4
   resolution: "@polkadot/ui-keyring@npm:3.6.4"