diff --git a/apps/balances-demo/src/main.tsx b/apps/balances-demo/src/main.tsx index 8e8c4da972..74fc49eb7a 100644 --- a/apps/balances-demo/src/main.tsx +++ b/apps/balances-demo/src/main.tsx @@ -3,11 +3,14 @@ import "anylogger-loglevel" import "./index.css" import { BalancesProvider } from "@talismn/balances-react" +import loglevel from "loglevel" import { StrictMode, useState } from "react" import { createRoot } from "react-dom/client" import { App } from "./App" +loglevel.setLevel("info") + const onfinalityApiKey = undefined const Root = () => { @@ -28,16 +31,23 @@ const Root = () => { // // westend // "0xe143f23803ac50e8f6f8e62695d1ce9e4e1d68aa36c1cd2cfd15340213f3423e", // ]} - // enabledTokens={[ - // // DOT (polkadot relay chain) - // "polkadot-substrate-native", - // // USDC (polkadot asset hub) - // "polkadot-asset-hub-substrate-assets-1337-usdc", - // // ETH (ethereum mainnet) - // "1-evm-native", - // // GM (gm chain) - // "gm-substrate-tokens-gm", - // ]} + enabledTokens={[ + "polkadot-substrate-native", + "acala-substrate-native", + "astar-substrate-native", + "composable-substrate-native", + "crust-substrate-native", + "hydradx-substrate-native", + "interlay-substrate-native", + "karura-substrate-native", + "kintsugi-substrate-native", + "mangata-substrate-native", + "moonbeam-substrate-native", + "moonriver-substrate-native", + "polimec-substrate-native", + "westend-testnet-substrate-native", + "rococo-testnet-substrate-native", + ]} > diff --git a/packages/balances-react/src/atoms/balances.ts b/packages/balances-react/src/atoms/balances.ts index 0186314120..87d12ec9be 100644 --- a/packages/balances-react/src/atoms/balances.ts +++ b/packages/balances-react/src/atoms/balances.ts @@ -181,8 +181,8 @@ const balancesSubscriptionAtomEffect = atomEffect((get) => { addressesByTokenByModule[token.type][token.id] = allAddresses.filter((address) => { // for each address, fetch balances only from compatible chains return isEthereumAddress(address) - ? !!token.evmNetwork?.id || chainsById[token.chain?.id ?? ""]?.account === "secp256k1" - : !!token.chain?.id + ? token.evmNetwork?.id || chainsById[token.chain?.id ?? ""]?.account === "secp256k1" + : token.chain?.id && chainsById[token.chain?.id ?? ""]?.account !== "secp256k1" }) }) @@ -212,11 +212,12 @@ const balancesSubscriptionAtomEffect = atomEffect((get) => { const hasChain = balance.chainId && chainIds.has(balance.chainId) const hasEvmNetwork = balance.evmNetworkId && evmNetworkIds.has(balance.evmNetworkId) const chainUsesSecp256k1Accounts = chain?.account === "secp256k1" - if (!isEthereumAddress(balance.address) && !hasChain) { - return true + if (!isEthereumAddress(balance.address)) { + if (!hasChain) return true + if (chainUsesSecp256k1Accounts) return true } - if (isEthereumAddress(balance.address) && !(hasEvmNetwork || chainUsesSecp256k1Accounts)) { - return true + if (isEthereumAddress(balance.address)) { + if (!hasEvmNetwork && !chainUsesSecp256k1Accounts) return true } // keep balance diff --git a/packages/balances-react/src/atoms/chaindata.ts b/packages/balances-react/src/atoms/chaindata.ts index dcabaf1d0b..4eb629d1ec 100644 --- a/packages/balances-react/src/atoms/chaindata.ts +++ b/packages/balances-react/src/atoms/chaindata.ts @@ -25,14 +25,12 @@ export const miniMetadatasAtom = atom(async (get) => (await get(chaindataAtom)). export const chaindataAtom = atomWithObservable((get) => { const enableTestnets = get(enableTestnetsAtom) - const filterTestnets = (items: T[]) => - items.filter(({ id }) => ["polkadot", "polimec"].includes(id)) - const filterMapTestnets = ( - items: Record - ) => - Object.fromEntries( - Object.entries(items).filter(([, { id }]) => ["polkadot", "polimec"].includes(id)) - ) + const filterTestnets = (items: T[]) => + enableTestnets ? items : items.filter(({ isTestnet }) => !isTestnet) + const filterMapTestnets = (items: Record) => + enableTestnets + ? items + : Object.fromEntries(Object.entries(items).filter(([, { isTestnet }]) => !isTestnet)) const filterEnabledTokens = (tokens: Token[]) => tokens.filter((token) => token.isDefault || ("isCustom" in token && token.isCustom)) diff --git a/packages/balances/src/MiniMetadataUpdater.ts b/packages/balances/src/MiniMetadataUpdater.ts index 741cbb1bd4..bfcd98fe4e 100644 --- a/packages/balances/src/MiniMetadataUpdater.ts +++ b/packages/balances/src/MiniMetadataUpdater.ts @@ -233,7 +233,23 @@ export class MiniMetadataUpdater { await balancesDb.miniMetadatas.bulkDelete(unwantedIds) } - const needUpdates = ["polkadot", "astar", "hydradx", "polimec"] + const needUpdates = [ + "polkadot", + "acala", + "astar", + "composable", + "crust", + "hydradx", + "interlay", + "karura", + "kintsugi", + "mangata", + "moonbeam", + "moonriver", + "polimec", + "westend-testnet", + "rococo-testnet", + ] // const needUpdates = Array.from(statusesByChain.entries()) // .filter(([, status]) => status !== "good") // .map(([chainId]) => chainId) diff --git a/packages/balances/src/modules/SubstrateNativeModule.ts b/packages/balances/src/modules/SubstrateNativeModule.ts index 3864a4a562..8af4281a39 100644 --- a/packages/balances/src/modules/SubstrateNativeModule.ts +++ b/packages/balances/src/modules/SubstrateNativeModule.ts @@ -12,6 +12,7 @@ import { githubTokenLogoUrl, } from "@talismn/chaindata-provider" import { + Binary, V15, compactMetadata, getDynamicBuilder, @@ -24,6 +25,7 @@ import * as $ from "@talismn/subshape-fork" import { Deferred, blake2Concat, decodeAnyAddress, isEthereumAddress } from "@talismn/util" import isEqual from "lodash/isEqual" import { combineLatest, map, scan, share, switchAll } from "rxjs" +import { Struct, u128, u32, u8 } from "scale-ts" import { DefaultBalanceModule, NewBalanceModule, NewTransferParamsType } from "../BalanceModule" import log from "../log" @@ -60,22 +62,23 @@ type ModuleType = "substrate-native" // Theory: new chains will be at least on metadata v14, and so we won't need to hardcode their AccountInfo type. // But for chains we want to support which aren't on metadata v14, hardcode them here: // If the chain upgrades to metadata v14, this override will be ignored :) -// -// TODO: Move the AccountInfoFallback configs for each chain into the ChainMeta section of chaindata -const RegularAccountInfoFallback = JSON.stringify({ - nonce: "u32", - consumers: "u32", - providers: "u32", - sufficients: "u32", - data: { free: "u128", reserved: "u128", miscFrozen: "u128", feeFrozen: "u128" }, +const RegularAccountInfoFallback = Struct({ + nonce: u32, + consumers: u32, + providers: u32, + sufficients: u32, + data: Struct({ free: u128, reserved: u128, miscFrozen: u128, feeFrozen: u128 }), }) -const NoSufficientsAccountInfoFallback = JSON.stringify({ - nonce: "u32", - consumers: "u32", - providers: "u32", - data: { free: "u128", reserved: "u128", miscFrozen: "u128", feeFrozen: "u128" }, +const NoSufficientsAccountInfoFallback = Struct({ + nonce: u32, + consumers: u32, + providers: u32, + data: Struct({ free: u128, reserved: u128, miscFrozen: u128, feeFrozen: u128 }), }) -const AccountInfoOverrides: { [key: ChainId]: string } = { +const AccountInfoOverrides: Record< + string, + typeof RegularAccountInfoFallback | typeof NoSufficientsAccountInfoFallback | undefined +> = { // automata is not yet on metadata v14 "automata": RegularAccountInfoFallback, @@ -344,26 +347,23 @@ export const SubNativeModule: NewBalanceModule< // we queue up our work to clean up our subscription when this promise rejects const callerUnsubscribed = unsubDeferred.promise - subscribeNompoolStaking( - chaindataProvider, - chainConnectors.substrate, - // getOrCreateTypeRegistry, - addressesByToken, - callback, - callerUnsubscribed - ) - subscribeCrowdloans( - chaindataProvider, - chainConnectors.substrate, - // getOrCreateTypeRegistry, - addressesByToken, - callback, - callerUnsubscribed - ) + // subscribeNompoolStaking( + // chaindataProvider, + // chainConnectors.substrate, + // addressesByToken, + // callback, + // callerUnsubscribed + // ) + // subscribeCrowdloans( + // chaindataProvider, + // chainConnectors.substrate, + // addressesByToken, + // callback, + // callerUnsubscribed + // ) subscribeBase( chaindataProvider, chainConnectors.substrate, - // getOrCreateTypeRegistry, addressesByToken, callback, callerUnsubscribed @@ -375,11 +375,7 @@ export const SubNativeModule: NewBalanceModule< async fetchBalances(addressesByToken) { assert(chainConnectors.substrate, "This module requires a substrate chain connector") - const queries = await buildQueries( - chaindataProvider, - // getOrCreateTypeRegistry, - addressesByToken - ) + const queries = await buildQueries(chaindataProvider, addressesByToken) const result = await new RpcStateQueryHelper(chainConnectors.substrate, queries).fetch() return new Balances(result ?? []) @@ -461,7 +457,6 @@ export const SubNativeModule: NewBalanceModule< async function buildQueries( chaindataProvider: ChaindataProvider, - // getOrCreateTypeRegistry: GetOrCreateTypeRegistry, addressesByToken: AddressesByToken ): Promise>> { const chains = await chaindataProvider.chainsById() @@ -481,10 +476,10 @@ async function buildQueries( moduleType: "substrate-native", coders: { base: ["System", "Account"], - reservesDecoder: ["Balances", "Reserves"], - holdsDecoder: ["Balances", "Holds"], - locksDecoder: ["Balances", "Locks"], - freezesDecoder: ["Balances", "Freezes"], + reserves: ["Balances", "Reserves"], + holds: ["Balances", "Holds"], + locks: ["Balances", "Locks"], + freezes: ["Balances", "Freezes"], }, }) @@ -512,8 +507,6 @@ async function buildQueries( return [] } - // if chain is metadata >= v15 and has miniMetadata, this will be true - const hasCoders = chainStorageCoders.get(chainId) !== undefined const [chainMeta] = findChainMeta( miniMetadatas, "substrate-native", @@ -553,13 +546,7 @@ async function buildQueries( let freezesQueryLocks: Array> = [] const baseQuery: RpcStateQuery | undefined = (() => { - const storageHelper = new StorageHelper( - typeRegistry, - "system", - "account", - decodeAnyAddress(address) - ) - const storageDecoder = chainStorageDecoders.get(chainId)?.baseDecoder + // For chains which are using metadata < v15 const getFallbackStateKey = () => { const addressBytes = decodeAnyAddress(address) const addressHash = blake2Concat(addressBytes).replace(/^0x/, "") @@ -569,33 +556,28 @@ async function buildQueries( return `0x${moduleStorageHash}${addressHash}` } - /** - * NOTE: For many MetadataV14 chains, it is not valid to encode an ethereum address into this System.Account state call. - * However, because we have always made that state call in the past, existing users will have the result (a balance of `0`) - * cached in their BalancesDB. - * - * So, until we refactor the storage of this module in a way which nukes the existing cached balances, we'll need to continue - * making these invalid state calls to keep those balances from showing as `cached` or `stale`. - * - * Current logic: - * - * stateKey: string = hasMetadataV14 && storageHelper.stateKey ? storageHelper.stateKey : getFallbackStateKey() - * - * Future (ideal) logic: - * - * stateKey: string | undefined = hasMetadataV14 ? storageHelper.stateKey : getFallbackStateKey() - */ - const stateKey = - hasMetadataV14 && storageHelper.stateKey ? storageHelper.stateKey : getFallbackStateKey() + const scaleCoder = chainStorageCoders.get(chainId)?.base + // NOTE: Only use fallback key when `scaleCoder` is not defined + // i.e. when chain doesn't have metadata v15 + const stateKey = scaleCoder + ? (() => { + try { + return scaleCoder?.enc(address) + } catch (error) { + log.warn(`Invalid address in ${chainId} base query ${address}`, error) + return + } + })() + : getFallbackStateKey() + if (!stateKey) return const decodeResult = (change: string | null) => { - // BEGIN: Handle chains which use metadata < v14 - // eslint-disable-next-line @typescript-eslint/no-explicit-any - let oldChainBalance: any = undefined - if (!hasMetadataV14) { - const accountInfoTypeDef = AccountInfoOverrides[chainId] - if (accountInfoTypeDef === undefined) { - // chain metadata version is < 14 and we also don't have an override hardcoded in + // BEGIN: Handle chains which use metadata < v15 + let oldChainBalance = null + if (!scaleCoder) { + const scaleAccountInfo = AccountInfoOverrides[chainId] + if (scaleAccountInfo === undefined) { + // chain metadata version is < 15 and we also don't have an override hardcoded in // the best way to handle this case: log a warning and return an empty balance log.debug( `Token ${tokenId} on chain ${chainId} has no balance type for decoding. Defaulting to a balance of 0 (zero).` @@ -605,7 +587,7 @@ async function buildQueries( try { // eslint-disable-next-line no-var - oldChainBalance = createType(typeRegistry, accountInfoTypeDef, change) + oldChainBalance = change === null ? null : scaleAccountInfo.dec(change) } catch (error) { log.warn( `Failed to create pre-metadataV14 balance type for token ${tokenId} on chain ${chainId}: ${error?.toString()}` @@ -615,40 +597,47 @@ async function buildQueries( } // END: Handle chains which use metadata < v14 - // eslint-disable-next-line @typescript-eslint/no-explicit-any - const decoded: any = - hasMetadataV14 && storageDecoder - ? change === null - ? null - : storageDecoder.decode($.decodeHex(change)) - : oldChainBalance - - const bigIntOrCodecToBigInt = (value: bigint | i128): bigint => - typeof value === "bigint" ? value : value?.toBigInt?.() - - let free = (bigIntOrCodecToBigInt(decoded?.data?.free) ?? 0n).toString() - let reserved = (bigIntOrCodecToBigInt(decoded?.data?.reserved) ?? 0n).toString() - let miscFrozen = ( - (bigIntOrCodecToBigInt(decoded?.data?.miscFrozen) ?? 0n) + - // some chains don't split their `frozen` amount into `feeFrozen` and `miscFrozen`. - // for those chains, we'll use the `frozen` amount as `miscFrozen`. - (bigIntOrCodecToBigInt(decoded?.data?.frozen) ?? 0n) - ).toString() - let feeFrozen = (bigIntOrCodecToBigInt(decoded?.data?.feeFrozen) ?? 0n).toString() - - // we use the evm-native module to fetch native token balances for ethereum addresses on ethereum networks - // but on moonbeam, moonriver and other chains which use ethereum addresses instead of substrate addresses, - // we use both this module and the evm-native module - if (isEthereumAddress(address) && chain.account !== "secp256k1") - free = reserved = miscFrozen = feeFrozen = "0" - - balanceJson.free = free + /** NOTE: This type is only a hint for typescript, the chain can actually return whatever it wants to */ + type DecodedType = { + data?: { + flags?: bigint + free?: bigint + frozen?: bigint + reserved?: bigint + + // deprecated fields (they only show up on old chains) + feeFrozen?: bigint + miscFrozen?: bigint + } + } + const decoded: DecodedType | null = + change === null + ? null + : (() => { + try { + return scaleCoder?.dec(change) + } catch (error) { + log.warn(`Failed to decode balance on chain ${chainId}`, error) + return null + } + })() ?? oldChainBalance + + balanceJson.free = (decoded?.data?.free ?? 0n).toString() + const otherReserve = balanceJson.reserves.find(({ label }) => label === "reserved") - if (otherReserve) otherReserve.amount = reserved - const feesLock = balanceJson.locks.find(({ label }) => label === "fees") - if (feesLock) feesLock.amount = feeFrozen + if (otherReserve) otherReserve.amount = (decoded?.data?.reserved ?? 0n).toString() + const miscLock = balanceJson.locks.find(({ label }) => label === "misc") - if (miscLock) miscLock.amount = miscFrozen + if (miscLock) + miscLock.amount = ( + (decoded?.data?.miscFrozen ?? 0n) + + // new chains don't split their `frozen` amount into `feeFrozen` and `miscFrozen`. + // for these chains, we'll use the `frozen` amount as `miscFrozen`. + (decoded?.data?.frozen ?? 0n) + ).toString() + + const feesLock = balanceJson.locks.find(({ label }) => label === "fees") + if (feesLock) feesLock.amount = (decoded?.data?.feeFrozen ?? 0n).toString() return new Balance(balanceJson) } @@ -657,27 +646,42 @@ async function buildQueries( })() const locksQuery: RpcStateQuery | undefined = (() => { - const storageHelper = new StorageHelper( - typeRegistry, - "balances", - "locks", - decodeAnyAddress(address) - ) - const storageDecoder = chainStorageDecoders.get(chainId)?.locksDecoder - const stateKey = storageHelper.stateKey + const scaleCoder = chainStorageCoders.get(chainId)?.locks + const stateKey = (() => { + try { + return scaleCoder?.enc(address) + } catch (error) { + log.warn(`Invalid address in ${chainId} locks query ${address}`, error) + return + } + })() if (!stateKey) return - const decodeResult = (change: string | null) => { - if (change === null) return new Balance(balanceJson) - // eslint-disable-next-line @typescript-eslint/no-explicit-any - const decoded: any = - storageDecoder && change !== null ? storageDecoder.decode($.decodeHex(change)) : null + const decodeResult = (change: string | null) => { + /** NOTE: This type is only a hint for typescript, the chain can actually return whatever it wants to */ + type DecodedType = Array<{ + id?: Binary + amount?: bigint + }> + + const decoded: DecodedType | null = + change === null + ? null + : (() => { + try { + const decoded = scaleCoder?.dec(change) + if (Array.isArray(decoded)) return decoded + return null + } catch (error) { + log.warn(`Failed to decode lock on chain ${chainId}`, error) + return null + } + })() ?? null locksQueryLocks = - // eslint-disable-next-line @typescript-eslint/no-explicit-any - decoded?.map?.((lock: any) => ({ - label: getLockedType(lock?.id?.toUtf8?.()), - amount: lock?.amount?.toString?.() ?? "0", + decoded?.map?.((lock) => ({ + label: getLockedType(lock?.id?.asText?.()), + amount: (lock?.amount ?? 0n).toString(), })) ?? [] balanceJson.locks = [ @@ -693,25 +697,40 @@ async function buildQueries( })() const freezesQuery: RpcStateQuery | undefined = (() => { - const storageHelper = new StorageHelper( - typeRegistry, - "balances", - "freezes", - decodeAnyAddress(address) - ) - const storageDecoder = chainStorageDecoders.get(chainId)?.freezesDecoder - const stateKey = storageHelper.stateKey + const scaleCoder = chainStorageCoders.get(chainId)?.freezes + const stateKey = (() => { + try { + return scaleCoder?.enc(address) + } catch (error) { + log.warn(`Invalid address in ${chainId} freezes query ${address}`, error) + return + } + })() if (!stateKey) return - const decodeResult = (change: string | null) => { - if (change === null) return new Balance(balanceJson) - // eslint-disable-next-line @typescript-eslint/no-explicit-any - const decoded: any = - storageDecoder && change !== null ? storageDecoder.decode($.decodeHex(change)) : null + const decodeResult = (change: string | null) => { + /** NOTE: This type is only a hint for typescript, the chain can actually return whatever it wants to */ + type DecodedType = Array<{ + id?: { type?: string } + amount?: bigint + }> + + const decoded: DecodedType | null = + change === null + ? null + : (() => { + try { + const decoded = scaleCoder?.dec(change) + if (Array.isArray(decoded)) return decoded + return null + } catch (error) { + log.warn(`Failed to decode freeze on chain ${chainId}`, error) + return null + } + })() ?? null freezesQueryLocks = - // eslint-disable-next-line @typescript-eslint/no-explicit-any - decoded?.map?.((lock: any) => ({ + decoded?.map?.((lock) => ({ label: getLockedType(lock?.id?.type?.toLowerCase?.()), amount: lock?.amount?.toString?.() ?? "0", })) ?? [] @@ -740,7 +759,6 @@ async function buildQueries( async function subscribeNompoolStaking( chaindataProvider: ChaindataProvider, chainConnector: ChainConnector, - // getOrCreateTypeRegistry: GetOrCreateTypeRegistry, addressesByToken: AddressesByToken, callback: SubscriptionCallback, callerUnsubscribed: Promise @@ -847,23 +865,22 @@ async function subscribeNompoolStaking( const stateKey = storageHelper.stateKey if (!stateKey) return [] const decodeResult = (change: string | null) => { - // eslint-disable-next-line @typescript-eslint/no-explicit-any const decoded: any = storageDecoder && change !== null ? storageDecoder.decode($.decodeHex(change)) : null const poolId: string | undefined = decoded?.poolId?.toString?.() const points: string | undefined = decoded?.points?.toString?.() - const unbondingEras: Array<{ era: string; amount: string }> = - // eslint-disable-next-line @typescript-eslint/no-explicit-any - Array.from(decoded?.unbondingEras ?? []).flatMap((entry: any) => { - const [key, value] = Array.from(entry) + const unbondingEras: Array<{ era: string; amount: string }> = Array.from( + decoded?.unbondingEras ?? [] + ).flatMap((entry: any) => { + const [key, value] = Array.from(entry) - const era = key?.toString?.() - const amount = value?.toString?.() - if (typeof era !== "string" || typeof amount !== "string") return [] + const era = key?.toString?.() + const amount = value?.toString?.() + if (typeof era !== "string" || typeof amount !== "string") return [] - return { era, amount } - }) + return { era, amount } + }) return { tokenId, address, poolId, points, unbondingEras } } @@ -893,7 +910,6 @@ async function subscribeNompoolStaking( const stateKey = storageHelper.stateKey if (!stateKey) return [] const decodeResult = (change: string | null) => { - // eslint-disable-next-line @typescript-eslint/no-explicit-any const decoded: any = storageDecoder && change !== null ? storageDecoder.decode($.decodeHex(change)) : null @@ -925,7 +941,6 @@ async function subscribeNompoolStaking( const stateKey = storageHelper.stateKey if (!stateKey) return [] const decodeResult = (change: string | null) => { - // eslint-disable-next-line @typescript-eslint/no-explicit-any const decoded: any = storageDecoder && change !== null ? storageDecoder.decode($.decodeHex(change)) : null @@ -955,7 +970,6 @@ async function subscribeNompoolStaking( const stateKey = storageHelper.stateKey if (!stateKey) return [] const decodeResult = (change: string | null) => { - // eslint-disable-next-line @typescript-eslint/no-explicit-any const decoded: any = storageDecoder && change !== null ? storageDecoder.decode($.decodeHex(change)) : null @@ -1123,7 +1137,6 @@ async function subscribeNompoolStaking( async function subscribeCrowdloans( chaindataProvider: ChaindataProvider, chainConnector: ChainConnector, - // getOrCreateTypeRegistry: GetOrCreateTypeRegistry, addressesByToken: AddressesByToken, callback: SubscriptionCallback, callerUnsubscribed: Promise @@ -1210,11 +1223,9 @@ async function subscribeCrowdloans( const stateKey = storageHelper.stateKey if (!stateKey) return [] const decodeResult = (change: string | null): number[] => { - // eslint-disable-next-line @typescript-eslint/no-explicit-any const decoded: any = storageDecoder && change !== null ? storageDecoder.decode($.decodeHex(change)) : null - // eslint-disable-next-line @typescript-eslint/no-explicit-any const paraIds = decoded ?? [] return paraIds @@ -1238,7 +1249,6 @@ async function subscribeCrowdloans( const stateKey = storageHelper.stateKey if (!stateKey) return [] const decodeResult = (change: string | null) => { - // eslint-disable-next-line @typescript-eslint/no-explicit-any const decoded: any = storageDecoder && change !== null ? storageDecoder.decode($.decodeHex(change)) : null @@ -1438,7 +1448,6 @@ async function subscribeCrowdloans( async function subscribeBase( chaindataProvider: ChaindataProvider, chainConnector: ChainConnector, - // getOrCreateTypeRegistry: GetOrCreateTypeRegistry, addressesByToken: AddressesByToken, callback: SubscriptionCallback, callerUnsubscribed: Promise diff --git a/packages/balances/src/modules/SubstratePsp22Module.ts b/packages/balances/src/modules/SubstratePsp22Module.ts index bd3d115c12..2355e02b4d 100644 --- a/packages/balances/src/modules/SubstratePsp22Module.ts +++ b/packages/balances/src/modules/SubstratePsp22Module.ts @@ -18,7 +18,7 @@ import { DefaultBalanceModule, NewBalanceModule, NewTransferParamsType } from ". import log from "../log" import { AddressesByToken, Amount, Balance, BalanceJson, Balances, NewBalanceType } from "../types" import psp22Abi from "./abis/psp22.json" -import { makeContractCaller } from "./util/makeContractCaller" +import { makeContractCaller } from "./util" type ModuleType = "substrate-psp22" diff --git a/packages/balances/src/modules/util/InferBalanceModuleTypes.ts b/packages/balances/src/modules/util/InferBalanceModuleTypes.ts new file mode 100644 index 0000000000..3c60d35ec8 --- /dev/null +++ b/packages/balances/src/modules/util/InferBalanceModuleTypes.ts @@ -0,0 +1,42 @@ +import { BalanceModule, NewBalanceModule } from "../../BalanceModule" + +// eslint-disable-next-line @typescript-eslint/no-explicit-any +export type AnyBalanceModule = BalanceModule +// eslint-disable-next-line @typescript-eslint/no-explicit-any +export type AnyNewBalanceModule = NewBalanceModule + +/** + * The following `Infer*` collection of generic types can be used when you want to + * extract one of the generic type arguments from an existing BalanceModule. + * + * For example, you might want to write a function which can accept any BalanceModule + * as an input, and then return the specific TokenType for that module: + * function getTokens(module: T): InferTokenType + * + * Or for another example, you might want a function which can take any BalanceModule `type` + * string as input, and then return some data associated with that module with the correct type: + * function getChainMeta(type: InferModuleType): InferChainMeta | undefined + */ +type InferBalanceModuleTypes = T extends NewBalanceModule< + infer TModuleType, + infer TTokenType, + infer TChainMeta, + infer TModuleConfig, + infer TTransferParams +> + ? { + TModuleType: TModuleType + TTokenType: TTokenType + TChainMeta: TChainMeta + TModuleConfig: TModuleConfig + TTransferParams: TTransferParams + } + : never +export type InferModuleType = + InferBalanceModuleTypes["TModuleType"] +export type InferTokenType = InferBalanceModuleTypes["TTokenType"] +export type InferChainMeta = InferBalanceModuleTypes["TChainMeta"] +export type InferModuleConfig = + InferBalanceModuleTypes["TModuleConfig"] +export type InferTransferParams = + InferBalanceModuleTypes["TTransferParams"] diff --git a/packages/balances/src/modules/util/RpcStateQueryHelper.ts b/packages/balances/src/modules/util/RpcStateQueryHelper.ts new file mode 100644 index 0000000000..944332a5e5 --- /dev/null +++ b/packages/balances/src/modules/util/RpcStateQueryHelper.ts @@ -0,0 +1,105 @@ +import { ChainConnector } from "@talismn/chain-connector" +import { ChainId } from "@talismn/chaindata-provider" +import { hasOwnProperty } from "@talismn/util" +import groupBy from "lodash/groupBy" + +import log from "../../log" +import { SubscriptionCallback, UnsubscribeFn } from "../../types" + +/** + * Pass some these into an `RpcStateQueryHelper` in order to easily batch multiple state queries into the one rpc call. + */ +export type RpcStateQuery = { + chainId: string + stateKey: string + decodeResult: (change: string | null) => T +} + +/** + * Used by a variety of balance modules to help batch multiple state queries into the one rpc call. + */ +export class RpcStateQueryHelper { + #chainConnector: ChainConnector + #queries: Array> + + constructor(chainConnector: ChainConnector, queries: Array>) { + this.#chainConnector = chainConnector + this.#queries = queries + } + + async subscribe( + callback: SubscriptionCallback, + timeout: number | false = false, + subscribeMethod = "state_subscribeStorage", + responseMethod = "state_storage", + unsubscribeMethod = "state_unsubscribeStorage" + ): Promise { + const queriesByChain = groupBy(this.#queries, "chainId") + + const subscriptions = Object.entries(queriesByChain).map(([chainId, queries]) => { + const params = [queries.map(({ stateKey }) => stateKey)] + + const unsub = this.#chainConnector.subscribe( + chainId, + subscribeMethod, + responseMethod, + params, + (error, result) => { + error + ? callback(error) + : callback(null, this.#distributeChangesToQueryDecoders.call(this, chainId, result)) + }, + timeout + ) + + return () => unsub.then((unsubscribe) => unsubscribe(unsubscribeMethod)) + }) + + return () => subscriptions.forEach((unsubscribe) => unsubscribe()) + } + + async fetch(method = "state_queryStorageAt"): Promise { + const queriesByChain = groupBy(this.#queries, "chainId") + + const resultsByChain = await Promise.all( + Object.entries(queriesByChain).map(async ([chainId, queries]) => { + const params = [queries.map(({ stateKey }) => stateKey)] + + const result = (await this.#chainConnector.send(chainId, method, params))[0] + return this.#distributeChangesToQueryDecoders.call(this, chainId, result) + }) + ) + + return resultsByChain.flatMap((result) => result) + } + + #distributeChangesToQueryDecoders(chainId: ChainId, result: unknown): T[] { + if (typeof result !== "object" || result === null) return [] + if (!hasOwnProperty(result, "changes") || typeof result.changes !== "object") return [] + if (!Array.isArray(result.changes)) return [] + + return result.changes.flatMap(([reference, change]: [unknown, unknown]): [T] | [] => { + if (typeof reference !== "string") { + log.warn(`Received non-string reference in RPC result: ${reference}`) + return [] + } + + if (typeof change !== "string" && change !== null) { + log.warn(`Received non-string and non-null change in RPC result: ${reference} | ${change}`) + return [] + } + + const query = this.#queries.find( + ({ chainId: cId, stateKey }) => cId === chainId && stateKey === reference + ) + if (!query) { + log.warn( + `Failed to find query:\n${reference} in\n${this.#queries.map(({ stateKey }) => stateKey)}` + ) + return [] + } + + return [query.decodeResult(change)] + }) + } +} diff --git a/packages/balances/src/modules/util/balances.ts b/packages/balances/src/modules/util/balances.ts new file mode 100644 index 0000000000..59b81af977 --- /dev/null +++ b/packages/balances/src/modules/util/balances.ts @@ -0,0 +1,55 @@ +import { + BalanceModule, + DefaultChainMeta, + DefaultModuleConfig, + DefaultTransferParams, + ExtendableChainMeta, + ExtendableModuleConfig, + ExtendableTokenType, + ExtendableTransferParams, +} from "../../BalanceModule" +import { AddressesByToken, Balances, SubscriptionCallback, UnsubscribeFn } from "../../types" + +/** + * Wraps a BalanceModule's fetch/subscribe methods with a single `balances` method. + * This `balances` method will subscribe if a callback parameter is provided, or otherwise fetch. + */ +export async function balances< + TModuleType extends string, + TTokenType extends ExtendableTokenType, + TChainMeta extends ExtendableChainMeta = DefaultChainMeta, + TModuleConfig extends ExtendableModuleConfig = DefaultModuleConfig, + TTransferParams extends ExtendableTransferParams = DefaultTransferParams +>( + balanceModule: BalanceModule, + addressesByToken: AddressesByToken +): Promise +export async function balances< + TModuleType extends string, + TTokenType extends ExtendableTokenType, + TChainMeta extends ExtendableChainMeta = DefaultChainMeta, + TModuleConfig extends ExtendableModuleConfig = DefaultModuleConfig, + TTransferParams extends ExtendableTransferParams = DefaultTransferParams +>( + balanceModule: BalanceModule, + addressesByToken: AddressesByToken, + callback: SubscriptionCallback +): Promise +export async function balances< + TModuleType extends string, + TTokenType extends ExtendableTokenType, + TChainMeta extends ExtendableChainMeta = DefaultChainMeta, + TModuleConfig extends ExtendableModuleConfig = DefaultModuleConfig, + TTransferParams extends ExtendableTransferParams = DefaultTransferParams +>( + balanceModule: BalanceModule, + addressesByToken: AddressesByToken, + callback?: SubscriptionCallback +): Promise { + // subscription request + if (callback !== undefined) + return await balanceModule.subscribeBalances(addressesByToken, callback) + + // one-off request + return await balanceModule.fetchBalances(addressesByToken) +} diff --git a/packages/balances/src/modules/util/buildStorageCoders.ts b/packages/balances/src/modules/util/buildStorageCoders.ts new file mode 100644 index 0000000000..e073369e45 --- /dev/null +++ b/packages/balances/src/modules/util/buildStorageCoders.ts @@ -0,0 +1,68 @@ +import { ChainId, ChainList } from "@talismn/chaindata-provider" +import { getDynamicBuilder, metadata as scaleMetadata } from "@talismn/scale" + +import log from "../../log" +import { MiniMetadata } from "../../types" +import { findChainMeta } from "./findChainMeta" +import { AnyNewBalanceModule, InferModuleType } from "./InferBalanceModuleTypes" + +export const buildStorageCoders = < + TBalanceModule extends AnyNewBalanceModule, + TCoders extends { [key: string]: [string, string] } +>({ + chainIds, + chains, + miniMetadatas, + moduleType, + coders, +}: { + chainIds: ChainId[] + chains: ChainList + miniMetadatas: Map + moduleType: InferModuleType + coders: TCoders +}) => + new Map( + [...chainIds].flatMap((chainId) => { + const chain = chains[chainId] + if (!chain) return [] + + const [, miniMetadata] = findChainMeta(miniMetadatas, moduleType, chain) + if (!miniMetadata) return [] + if (!miniMetadata.data) return [] + if (miniMetadata.version < 15) return [] + + const decoded = scaleMetadata.dec(miniMetadata.data) + const metadata = decoded.metadata.tag === "v15" && decoded.metadata.value + if (!metadata) return [] + + try { + const scaleBuilder = getDynamicBuilder(metadata) + const builtCoders = Object.fromEntries( + Object.entries(coders).flatMap( + ([key, [module, method]]: [keyof TCoders, [string, string]]) => { + try { + return [[key, scaleBuilder.buildStorage(module, method)] as const] + } catch (cause) { + log.warn( + `Failed to build SCALE coder for chain ${chainId} (${module}::${method})`, + cause + ) + return [] + } + } + ) + ) as { + [Property in keyof TCoders]: ReturnType<(typeof scaleBuilder)["buildStorage"]> | undefined + } + + return [[chainId, builtCoders]] + } catch (cause) { + log.error( + `Failed to build SCALE coders for chain ${chainId} (${JSON.stringify(coders)})`, + cause + ) + return [] + } + }) + ) diff --git a/packages/balances/src/modules/util/deriveStatuses.ts b/packages/balances/src/modules/util/deriveStatuses.ts new file mode 100644 index 0000000000..0db157ef0a --- /dev/null +++ b/packages/balances/src/modules/util/deriveStatuses.ts @@ -0,0 +1,30 @@ +import { BalanceJson } from "../../types" + +/** + * Sets all balance statuses from `live-${string}` to either `live` or `cached` + * + * You should make sure that the input collection `balances` is mutable, because the statuses + * will be changed in-place as a performance consideration. + */ +export const deriveStatuses = ( + validSubscriptionIds: Set, + balances: BalanceJson[] +): BalanceJson[] => { + balances.forEach((balance) => { + if (["live", "cache", "stale", "initializing"].includes(balance.status)) return balance + + if (validSubscriptionIds.size < 1) { + balance.status = "cache" + return balance + } + + if (!validSubscriptionIds.has(balance.status.slice("live-".length))) { + balance.status = "cache" + return balance + } + + balance.status = "live" + return balance + }) + return balances +} diff --git a/packages/balances/src/modules/util/detectTransferMethod.ts b/packages/balances/src/modules/util/detectTransferMethod.ts new file mode 100644 index 0000000000..a794f08cb4 --- /dev/null +++ b/packages/balances/src/modules/util/detectTransferMethod.ts @@ -0,0 +1,29 @@ +import { Metadata, TypeRegistry } from "@polkadot/types" + +/** + * + * Detect Balances::transfer -> Balances::transfer_allow_death migration + * https://github.com/paritytech/substrate/pull/12951 + * + * `transfer_allow_death` is the preferred method, + * so if something goes wrong during detection, we should assume the chain has migrated + * @param metadataRpc string containing the hashed RPC metadata for the chain + * @returns + */ +export const detectTransferMethod = (metadataRpc: `0x${string}`) => { + const pjsMetadata = new Metadata(new TypeRegistry(), metadataRpc) + pjsMetadata.registry.setMetadata(pjsMetadata) + const balancesPallet = pjsMetadata.asLatest.pallets.find((pallet) => pallet.name.eq("Balances")) + + const balancesCallsTypeIndex = balancesPallet?.calls.value.type.toNumber() + const balancesCallsType = + balancesCallsTypeIndex !== undefined + ? pjsMetadata.asLatest.lookup.types[balancesCallsTypeIndex] + : undefined + const hasDeprecatedTransferCall = + balancesCallsType?.type.def.asVariant?.variants.find((variant) => + variant.name.eq("transfer") + ) !== undefined + + return hasDeprecatedTransferCall ? "transfer" : "transferAllowDeath" +} diff --git a/packages/balances/src/modules/util/findChainMeta.ts b/packages/balances/src/modules/util/findChainMeta.ts new file mode 100644 index 0000000000..9201854a47 --- /dev/null +++ b/packages/balances/src/modules/util/findChainMeta.ts @@ -0,0 +1,41 @@ +import { Chain } from "@talismn/chaindata-provider" + +import { MiniMetadata, deriveMiniMetadataId } from "../../types" +import { AnyNewBalanceModule, InferChainMeta, InferModuleType } from "./InferBalanceModuleTypes" + +/** + * Given a `moduleType` and a `chain` from a chaindataProvider, this function will find the chainMeta + * associated with the given balanceModule for the given chain. + */ +export const findChainMeta = ( + miniMetadatas: Map, + moduleType: InferModuleType, + chain?: Chain +): [InferChainMeta | undefined, MiniMetadata | undefined] => { + if (!chain) return [undefined, undefined] + if (!chain.specName) return [undefined, undefined] + if (!chain.specVersion) return [undefined, undefined] + + // TODO: This is spaghetti to import this here, it should be injected into each balance module or something. + const metadataId = deriveMiniMetadataId({ + source: moduleType, + chainId: chain.id, + specName: chain.specName, + specVersion: chain.specVersion, + balancesConfig: JSON.stringify( + chain.balancesConfig?.find((config) => config.moduleType === moduleType)?.moduleConfig ?? {} + ), + }) + + // TODO: Fix this (needs to fetch miniMetadata without being async) + const miniMetadata = miniMetadatas.get(metadataId) + const chainMeta: InferChainMeta | undefined = miniMetadata + ? { + miniMetadata: miniMetadata.data, + metadataVersion: miniMetadata.version, + ...JSON.parse(miniMetadata.extra), + } + : undefined + + return [chainMeta, miniMetadata] +} diff --git a/packages/balances/src/modules/util/getUniqueChainIds.ts b/packages/balances/src/modules/util/getUniqueChainIds.ts new file mode 100644 index 0000000000..05e0c8834d --- /dev/null +++ b/packages/balances/src/modules/util/getUniqueChainIds.ts @@ -0,0 +1,14 @@ +import { ChainId, TokenList } from "@talismn/chaindata-provider" + +import { AddressesByToken } from "../../types" + +export const getUniqueChainIds = ( + addressesByToken: AddressesByToken<{ id: string }>, + tokens: TokenList +): ChainId[] => [ + ...new Set( + Object.keys(addressesByToken) + .map((tokenId) => tokens[tokenId]?.chain?.id) + .flatMap((chainId) => (chainId ? [chainId] : [])) + ), +] diff --git a/packages/balances/src/modules/util/index.ts b/packages/balances/src/modules/util/index.ts index 5ffc395327..626db85002 100644 --- a/packages/balances/src/modules/util/index.ts +++ b/packages/balances/src/modules/util/index.ts @@ -1,94 +1,31 @@ -// import { Metadata as _new_Metadata } from "@alectalisman/polkadotjs-types" -import { - StorageKey, - TypeRegistry, - Metadata as _orig_Metadata, - decorateStorage, -} from "@polkadot/types" +import { Metadata, StorageKey, TypeRegistry, decorateStorage } from "@polkadot/types" import type { Registry } from "@polkadot/types-codec/types" -import { ChainConnector } from "@talismn/chain-connector" -import { Chain, ChainId, ChainList, TokenList } from "@talismn/chaindata-provider" -import { Codec, Storage, getDynamicBuilder, metadata as scaleMetadata } from "@talismn/scale" +import { ChainId, ChainList } from "@talismn/chaindata-provider" import { $metadataV14, getShape } from "@talismn/scale" import * as $ from "@talismn/subshape-fork" -import { hasOwnProperty } from "@talismn/util" import camelCase from "lodash/camelCase" -import groupBy from "lodash/groupBy" - -import { - BalanceModule, - DefaultChainMeta, - DefaultModuleConfig, - DefaultTransferParams, - ExtendableChainMeta, - ExtendableModuleConfig, - ExtendableTokenType, - ExtendableTransferParams, - NewBalanceModule, -} from "../../BalanceModule" -import log from "../../log" -import { - AddressesByToken, - BalanceJson, - Balances, - MiniMetadata, - SubscriptionCallback, - UnsubscribeFn, - deriveMiniMetadataId, -} from "../../types" - -// const Metadata = {} as unknown as _orig_Metadata - -const Metadata = _orig_Metadata - -// const Metadata = _new_Metadata as unknown as typeof _orig_Metadata -/** - * Wraps a BalanceModule's fetch/subscribe methods with a single `balances` method. - * This `balances` method will subscribe if a callback parameter is provided, or otherwise fetch. - */ -export async function balances< - TModuleType extends string, - TTokenType extends ExtendableTokenType, - TChainMeta extends ExtendableChainMeta = DefaultChainMeta, - TModuleConfig extends ExtendableModuleConfig = DefaultModuleConfig, - TTransferParams extends ExtendableTransferParams = DefaultTransferParams ->( - balanceModule: BalanceModule, - addressesByToken: AddressesByToken -): Promise -export async function balances< - TModuleType extends string, - TTokenType extends ExtendableTokenType, - TChainMeta extends ExtendableChainMeta = DefaultChainMeta, - TModuleConfig extends ExtendableModuleConfig = DefaultModuleConfig, - TTransferParams extends ExtendableTransferParams = DefaultTransferParams ->( - balanceModule: BalanceModule, - addressesByToken: AddressesByToken, - callback: SubscriptionCallback -): Promise -export async function balances< - TModuleType extends string, - TTokenType extends ExtendableTokenType, - TChainMeta extends ExtendableChainMeta = DefaultChainMeta, - TModuleConfig extends ExtendableModuleConfig = DefaultModuleConfig, - TTransferParams extends ExtendableTransferParams = DefaultTransferParams ->( - balanceModule: BalanceModule, - addressesByToken: AddressesByToken, - callback?: SubscriptionCallback -): Promise { - // subscription request - if (callback !== undefined) - return await balanceModule.subscribeBalances(addressesByToken, callback) - - // one-off request - return await balanceModule.fetchBalances(addressesByToken) -} +import log from "../../log" +import { MiniMetadata } from "../../types" +import { findChainMeta } from "./findChainMeta" +import { AnyNewBalanceModule, InferModuleType } from "./InferBalanceModuleTypes" + +export * from "./InferBalanceModuleTypes" +export * from "./RpcStateQueryHelper" +export * from "./balances" +export * from "./buildStorageCoders" +export * from "./decodeOutput" +export * from "./deriveStatuses" +export * from "./detectTransferMethod" +export * from "./findChainMeta" +export * from "./getUniqueChainIds" +export * from "./makeContractCaller" +export * from "./subscriptionIds" +// TODO: RM export type GetOrCreateTypeRegistry = (chainId: ChainId, metadataRpc?: `0x${string}`) => Registry +// TODO: RM export const createTypeRegistryCache = () => { const typeRegistryCache: Map< ChainId, @@ -117,157 +54,7 @@ export const createTypeRegistryCache = () => { return { getOrCreateTypeRegistry } } -// eslint-disable-next-line @typescript-eslint/no-explicit-any -export type AnyBalanceModule = BalanceModule -// eslint-disable-next-line @typescript-eslint/no-explicit-any -export type AnyNewBalanceModule = NewBalanceModule - -/** - * The following `Infer*` collection of generic types can be used when you want to - * extract one of the generic type arguments from an existing BalanceModule. - * - * For example, you might want to write a function which can accept any BalanceModule - * as an input, and then return the specific TokenType for that module: - * function getTokens(module: T): InferTokenType - * - * Or for another example, you might want a function which can take any BalanceModule `type` - * string as input, and then return some data associated with that module with the correct type: - * function getChainMeta(type: InferModuleType): InferChainMeta | undefined - */ -type InferBalanceModuleTypes = T extends NewBalanceModule< - infer TModuleType, - infer TTokenType, - infer TChainMeta, - infer TModuleConfig, - infer TTransferParams -> - ? { - TModuleType: TModuleType - TTokenType: TTokenType - TChainMeta: TChainMeta - TModuleConfig: TModuleConfig - TTransferParams: TTransferParams - } - : never -export type InferModuleType = - InferBalanceModuleTypes["TModuleType"] -export type InferTokenType = InferBalanceModuleTypes["TTokenType"] -export type InferChainMeta = InferBalanceModuleTypes["TChainMeta"] -export type InferModuleConfig = - InferBalanceModuleTypes["TModuleConfig"] -export type InferTransferParams = - InferBalanceModuleTypes["TTransferParams"] - -/** - * Given a `moduleType` and a `chain` from a chaindataProvider, this function will find the chainMeta - * associated with the given balanceModule for the given chain. - */ -export const findChainMeta = ( - miniMetadatas: Map, - moduleType: InferModuleType, - chain?: Chain -): [InferChainMeta | undefined, MiniMetadata | undefined] => { - if (!chain) return [undefined, undefined] - if (!chain.specName) return [undefined, undefined] - if (!chain.specVersion) return [undefined, undefined] - - // TODO: This is spaghetti to import this here, it should be injected into each balance module or something. - const metadataId = deriveMiniMetadataId({ - source: moduleType, - chainId: chain.id, - specName: chain.specName, - specVersion: chain.specVersion, - balancesConfig: JSON.stringify( - chain.balancesConfig?.find((config) => config.moduleType === moduleType)?.moduleConfig ?? {} - ), - }) - - // TODO: Fix this (needs to fetch miniMetadata without being async) - const miniMetadata = miniMetadatas.get(metadataId) - const chainMeta: InferChainMeta | undefined = miniMetadata - ? { - miniMetadata: miniMetadata.data, - metadataVersion: miniMetadata.version, - ...JSON.parse(miniMetadata.extra), - } - : undefined - - return [chainMeta, miniMetadata] -} - -export const getValidSubscriptionIds = () => { - return new Set( - localStorage.getItem("TalismanBalancesSubscriptionIds")?.split?.(",")?.filter?.(Boolean) ?? [] - ) -} -export const createSubscriptionId = () => { - // delete current id (if exists) - deleteSubscriptionId() - - // create new id - const subscriptionId = Date.now().toString() - sessionStorage.setItem("TalismanBalancesSubscriptionId", subscriptionId) - - // add to list of current ids - const subscriptionIds = getValidSubscriptionIds() - subscriptionIds.add(subscriptionId) - localStorage.setItem( - "TalismanBalancesSubscriptionIds", - [...subscriptionIds] - .filter(Boolean) - .filter((storageId) => - // filter super old IDs (they tend to stick around when the background script is restarted) - // - // test if the difference between `now` and `then` (subscriptionId - storageId) is greater than 1 week in milliseconds (604_800_000) - // if so, `storageId` is definitely super old and we can just prune it from localStorage - parseInt(subscriptionId, 10) - parseInt(storageId, 10) >= 604_800_000 ? false : true - ) - .join(",") - ) - - return subscriptionId -} -export const deleteSubscriptionId = () => { - const subscriptionId = sessionStorage.getItem("TalismanBalancesSubscriptionId") - if (!subscriptionId) return - - const subscriptionIds = getValidSubscriptionIds() - subscriptionIds.delete(subscriptionId) - localStorage.setItem( - "TalismanBalancesSubscriptionIds", - [...subscriptionIds].filter(Boolean).join(",") - ) -} - -/** - * Sets all balance statuses from `live-${string}` to either `live` or `cached` - * - * You should make sure that the input collection `balances` is mutable, because the statuses - * will be changed in-place as a performance consideration. - */ -export const deriveStatuses = ( - validSubscriptionIds: Set, - balances: BalanceJson[] -): BalanceJson[] => { - balances.forEach((balance) => { - if (["live", "cache", "stale", "initializing"].includes(balance.status)) return balance - - if (validSubscriptionIds.size < 1) { - balance.status = "cache" - return balance - } - - if (!validSubscriptionIds.has(balance.status.slice("live-".length))) { - balance.status = "cache" - return balance - } - - balance.status = "live" - return balance - }) - return balances -} - +// TODO: RM /** * Used by a variety of balance modules to help encode and decode substrate state calls. */ @@ -374,105 +161,8 @@ export class StorageHelper { } } -/** - * Pass some these into an `RpcStateQueryHelper` in order to easily batch multiple state queries into the one rpc call. - */ -export type RpcStateQuery = { - chainId: string - stateKey: string - decodeResult: (change: string | null) => T -} - -/** - * Used by a variety of balance modules to help batch multiple state queries into the one rpc call. - */ -export class RpcStateQueryHelper { - #chainConnector: ChainConnector - #queries: Array> - - constructor(chainConnector: ChainConnector, queries: Array>) { - this.#chainConnector = chainConnector - this.#queries = queries - } - - async subscribe( - callback: SubscriptionCallback, - timeout: number | false = false, - subscribeMethod = "state_subscribeStorage", - responseMethod = "state_storage", - unsubscribeMethod = "state_unsubscribeStorage" - ): Promise { - const queriesByChain = groupBy(this.#queries, "chainId") - - const subscriptions = Object.entries(queriesByChain).map(([chainId, queries]) => { - const params = [queries.map(({ stateKey }) => stateKey)] - - const unsub = this.#chainConnector.subscribe( - chainId, - subscribeMethod, - responseMethod, - params, - (error, result) => { - error - ? callback(error) - : callback(null, this.#distributeChangesToQueryDecoders.call(this, chainId, result)) - }, - timeout - ) - - return () => unsub.then((unsubscribe) => unsubscribe(unsubscribeMethod)) - }) - - return () => subscriptions.forEach((unsubscribe) => unsubscribe()) - } - - async fetch(method = "state_queryStorageAt"): Promise { - const queriesByChain = groupBy(this.#queries, "chainId") - - const resultsByChain = await Promise.all( - Object.entries(queriesByChain).map(async ([chainId, queries]) => { - const params = [queries.map(({ stateKey }) => stateKey)] - - const result = (await this.#chainConnector.send(chainId, method, params))[0] - return this.#distributeChangesToQueryDecoders.call(this, chainId, result) - }) - ) - - return resultsByChain.flatMap((result) => result) - } - - #distributeChangesToQueryDecoders(chainId: ChainId, result: unknown): T[] { - if (typeof result !== "object" || result === null) return [] - if (!hasOwnProperty(result, "changes") || typeof result.changes !== "object") return [] - if (!Array.isArray(result.changes)) return [] - - return result.changes.flatMap(([reference, change]: [unknown, unknown]): [T] | [] => { - if (typeof reference !== "string") { - log.warn(`Received non-string reference in RPC result: ${reference}`) - return [] - } - - if (typeof change !== "string" && change !== null) { - log.warn(`Received non-string and non-null change in RPC result: ${reference} | ${change}`) - return [] - } - - const query = this.#queries.find( - ({ chainId: cId, stateKey }) => cId === chainId && stateKey === reference - ) - if (!query) { - log.warn( - `Failed to find query:\n${reference} in\n${this.#queries.map(({ stateKey }) => stateKey)}` - ) - return [] - } - - return [query.decodeResult(change)] - }) - } -} - -export const subshapeStorageDecoder = +// TODO: RM +const subshapeStorageDecoder = // eslint-disable-next-line @typescript-eslint/no-explicit-any (miniMetadata: MiniMetadata, module: string, method: string) => { if (miniMetadata.version !== 14) { @@ -502,99 +192,7 @@ export const subshapeStorageDecoder = return getShape(metadata, typeId) } -export const getUniqueChainIds = ( - addressesByToken: AddressesByToken<{ id: string }>, - tokens: TokenList -): ChainId[] => [ - ...new Set( - Object.keys(addressesByToken) - .map((tokenId) => tokens[tokenId]?.chain?.id) - .flatMap((chainId) => (chainId ? [chainId] : [])) - ), -] -export const buildStorageCoders = < - TBalanceModule extends AnyNewBalanceModule, - TCoders extends { [key: string]: [string, string] } ->({ - chainIds, - chains, - miniMetadatas, - moduleType, - coders, -}: { - chainIds: ChainId[] - chains: ChainList - miniMetadatas: Map - moduleType: InferModuleType - coders: TCoders -}) => - new Map( - [...chainIds].flatMap((chainId) => { - const chain = chains[chainId] - if (!chain) return [] - - const [, miniMetadata] = findChainMeta(miniMetadatas, moduleType, chain) - if (!miniMetadata) return [] - if (!miniMetadata.data) return [] - if (miniMetadata.version < 15) return [] - - const decoded = scaleMetadata.dec(miniMetadata.data) - const metadata = decoded.metadata.tag === "v15" && decoded.metadata.value - if (!metadata) return [] - - const scaleBuilder = getDynamicBuilder(metadata) - const builtCoders = Object.fromEntries( - Object.entries(coders).map( - ([key, [module, method]]: [keyof TCoders, [string, string]]) => - [key, scaleBuilder.buildStorage(module, method)] as const - ) - ) as { - [Property in keyof TCoders]: ReturnType<(typeof scaleBuilder)["buildStorage"]> | undefined - } - - return [[chainId, builtCoders]] - }) - ) - -/** - * - * Detect Balances::transfer -> Balances::transfer_allow_death migration - * https://github.com/paritytech/substrate/pull/12951 - * - * `transfer_allow_death` is the preferred method, - * so if something goes wrong during detection, we should assume the chain has migrated - * @param metadataRpc string containing the hashed RPC metadata for the chain - * @returns - */ -export const detectTransferMethod = (metadataRpc: `0x${string}`) => { - const pjsMetadata = new Metadata(new TypeRegistry(), metadataRpc) - pjsMetadata.registry.setMetadata(pjsMetadata) - const balancesPallet = pjsMetadata.asLatest.pallets.find((pallet) => pallet.name.eq("Balances")) - - const balancesCallsTypeIndex = balancesPallet?.calls.value.type.toNumber() - const balancesCallsType = - balancesCallsTypeIndex !== undefined - ? pjsMetadata.asLatest.lookup.types[balancesCallsTypeIndex] - : undefined - const hasDeprecatedTransferCall = - balancesCallsType?.type.def.asVariant?.variants.find((variant) => - variant.name.eq("transfer") - ) !== undefined - - return hasDeprecatedTransferCall ? "transfer" : "transferAllowDeath" -} - -// -// -// -// // TODO: RM -// -// -// -// -// - export const buildStorageDecoders = < TBalanceModule extends AnyNewBalanceModule, TDecoders extends { [key: string]: [string, string] } diff --git a/packages/balances/src/modules/util/subscriptionIds.ts b/packages/balances/src/modules/util/subscriptionIds.ts new file mode 100644 index 0000000000..455eb90e82 --- /dev/null +++ b/packages/balances/src/modules/util/subscriptionIds.ts @@ -0,0 +1,41 @@ +const STORAGE_KEY = "TalismanBalancesSubscriptionIds" +const SESSION_KEY = "TalismanBalancesSubscriptionId" + +export const getValidSubscriptionIds = () => { + return new Set(localStorage.getItem(STORAGE_KEY)?.split?.(",")?.filter?.(Boolean) ?? []) +} +export const createSubscriptionId = () => { + // delete current id (if exists) + deleteSubscriptionId() + + // create new id + const subscriptionId = Date.now().toString() + sessionStorage.setItem(SESSION_KEY, subscriptionId) + + // add to list of current ids + const subscriptionIds = getValidSubscriptionIds() + subscriptionIds.add(subscriptionId) + localStorage.setItem( + STORAGE_KEY, + [...subscriptionIds] + .filter(Boolean) + .filter((storageId) => + // filter super old IDs (they tend to stick around when the background script is restarted) + // + // test if the difference between `now` and `then` (subscriptionId - storageId) is greater than 1 week in milliseconds (604_800_000) + // if so, `storageId` is definitely super old and we can just prune it from localStorage + parseInt(subscriptionId, 10) - parseInt(storageId, 10) >= 604_800_000 ? false : true + ) + .join(",") + ) + + return subscriptionId +} +export const deleteSubscriptionId = () => { + const subscriptionId = sessionStorage.getItem(SESSION_KEY) + if (!subscriptionId) return + + const subscriptionIds = getValidSubscriptionIds() + subscriptionIds.delete(subscriptionId) + localStorage.setItem(STORAGE_KEY, [...subscriptionIds].filter(Boolean).join(",")) +} diff --git a/packages/balances/src/modules/util/substrate-native.ts b/packages/balances/src/modules/util/substrate-native.ts index 4c31c935b2..39ab9ea25c 100644 --- a/packages/balances/src/modules/util/substrate-native.ts +++ b/packages/balances/src/modules/util/substrate-native.ts @@ -12,6 +12,10 @@ import { UnsubscribeFn, } from "../../types" +// +// TODO: Turn SubstrateNativeModule into a directory and move this into there +// + /** * Converts a subscription function into an Observable * @@ -92,6 +96,7 @@ export const getLockedType = (input?: string): BalanceLockType => { if (input.includes("vesting")) return "vesting" if (input.includes("calamvst")) return "vesting" // vesting on manta network if (input.includes("ormlvest")) return "vesting" // vesting ORML tokens + if (input.includes("pyconvot")) return "democracy" if (input.includes("democrac")) return "democracy" if (input.includes("democracy")) return "democracy" if (input.includes("phrelect")) return "democracy" // specific to council @@ -116,6 +121,8 @@ export const getLockedType = (input?: string): BalanceLockType => { // ignore technical or undocumented lock types if (input.includes("pdexlock")) return "other" if (input.includes("phala/sp")) return "other" + if (input.includes("aca/earn")) return "other" + if (input.includes("stk_stks")) return "other" // eslint-disable-next-line no-console console.warn(`unknown locked type: ${input}`) diff --git a/packages/extension-core/src/domains/balances/store.ts b/packages/extension-core/src/domains/balances/store.ts index a761acf69f..77aa2c8edf 100644 --- a/packages/extension-core/src/domains/balances/store.ts +++ b/packages/extension-core/src/domains/balances/store.ts @@ -353,11 +353,12 @@ export class BalanceStore { const hasChain = balance.chainId && chainIds.has(balance.chainId) const hasEvmNetwork = balance.evmNetworkId && evmNetworkIds.has(balance.evmNetworkId) const chainUsesSecp256k1Accounts = chain?.account === "secp256k1" - if (!isEthereumAddress(balance.address) && !hasChain) { - return true + if (!isEthereumAddress(balance.address)) { + if (!hasChain) return true + if (chainUsesSecp256k1Accounts) return true } - if (isEthereumAddress(balance.address) && !(hasEvmNetwork || chainUsesSecp256k1Accounts)) { - return true + if (isEthereumAddress(balance.address)) { + if (!hasEvmNetwork && !chainUsesSecp256k1Accounts) return true } // keep balance @@ -463,10 +464,10 @@ export class BalanceStore { addresses[address]?.includes(chainDetails[token.chain.id]?.genesisHash ?? "") ) .filter((address) => { - // for each account, fetch balances only from compatible chains + // for each address, fetch balances only from compatible chains return isEthereumAddress(address) - ? !!token.evmNetwork?.id || chainDetails[token.chain?.id ?? ""]?.account === "secp256k1" - : !!token.chain?.id + ? token.evmNetwork?.id || chainDetails[token.chain?.id ?? ""]?.account === "secp256k1" + : token.chain?.id && chainDetails[token.chain?.id ?? ""]?.account !== "secp256k1" }) }) diff --git a/packages/scale/src/index.ts b/packages/scale/src/index.ts index 670fce8344..f12d24ac04 100644 --- a/packages/scale/src/index.ts +++ b/packages/scale/src/index.ts @@ -1,7 +1,5 @@ // TODO: Get the DX of this lib as close as possible to the DX of `const jsonResult = '{"someKey":"someValue"}'; JSON.decode(jsonResult)` -import { suppressPortableRegistryConsoleWarnings } from "./suppressPortableRegistryConsoleWarnings" - export * from "./subshape/capi" export * from "./subshape/metadata" export * from "./subshape/storage" @@ -9,5 +7,3 @@ export * from "./subshape/storage" export * from "./scale-ts/metadata" export * from "./util" - -suppressPortableRegistryConsoleWarnings() diff --git a/packages/scale/src/scale-ts/metadata/index.ts b/packages/scale/src/scale-ts/metadata/index.ts index c9a03ce086..4beaa00ae3 100644 --- a/packages/scale/src/scale-ts/metadata/index.ts +++ b/packages/scale/src/scale-ts/metadata/index.ts @@ -1,7 +1,7 @@ export { getDynamicBuilder } from "@polkadot-api/metadata-builders" -export type { V15, V15Extrinsic, Codec } from "@polkadot-api/substrate-bindings" +export type { V15, V15Extrinsic, Codec, Binary } from "@polkadot-api/substrate-bindings" export { toHex } from "@polkadot-api/utils" export * from "./metadata" export * from "./v14" -export { v15, Storage } from "@polkadot-api/substrate-bindings" +export { v15 } from "@polkadot-api/substrate-bindings" diff --git a/packages/scale/src/suppressPortableRegistryConsoleWarnings.ts b/packages/scale/src/suppressPortableRegistryConsoleWarnings.ts deleted file mode 100644 index 0a5ddaef5f..0000000000 --- a/packages/scale/src/suppressPortableRegistryConsoleWarnings.ts +++ /dev/null @@ -1,26 +0,0 @@ -const ignoreModuleMessages: Record = { - "PORTABLEREGISTRY:": [ - "Unable to determine runtime Event type, cannot inspect frame_system::EventRecord", - "Unable to determine runtime Call type, cannot inspect sp_runtime::generic::unchecked_extrinsic::UncheckedExtrinsic", - ], -} - -export function suppressPortableRegistryConsoleWarnings() { - /* eslint-disable-next-line no-console */ - const originalWarn = console.warn - - /* eslint-disable-next-line no-console */ - console.warn = (...data: Parameters) => { - const [, dataModule, dataMessage] = data - const ignoreMessages = typeof dataModule === "string" && ignoreModuleMessages[dataModule] - if (Array.isArray(ignoreMessages) && ignoreMessages.includes(dataMessage)) return - if (data[0] === "Unable to map Bytes to a lookup index") return - if (data[0] === "Unable to map [u8; 32] to a lookup index") return - if (data[0] === "Unable to map u16 to a lookup index") return - if (data[0] === "Unable to map u32 to a lookup index") return - if (data[0] === "Unable to map u64 to a lookup index") return - if (data[0] === "Unable to map u128 to a lookup index") return - - originalWarn(...data) - } -} diff --git a/packages/scale/src/util/compactMetadata.ts b/packages/scale/src/util/compactMetadata.ts index 3b9e56e533..02b016282f 100644 --- a/packages/scale/src/util/compactMetadata.ts +++ b/packages/scale/src/util/compactMetadata.ts @@ -19,9 +19,7 @@ export type V15StorageItem = NonNullable["items"][0] * types used in the `System.Account` storage query will remain inside of metadata.lookups. */ export const compactMetadata = ( - metadata: V15, - // TODO: Support V15 | V14 - // metadata: V15 | V14, + metadata: V15 | V14, palletsAndItems: Array<{ pallet: string items: string[] @@ -45,6 +43,7 @@ export const compactMetadata = ( // remove pallet fields we don't care about pallet.calls = undefined pallet.constants = [] + // v15 (NOT v14) has docs if ("docs" in pallet) pallet.docs = [] pallet.errors = undefined pallet.events = undefined @@ -70,9 +69,9 @@ export const compactMetadata = ( // each type can be either "Plain" or "Map" // if it's "Plain" we only need to get the value type // if it's a "Map" we want to keep both the key AND the value types + item.type.tag === "plain" && item.type.value, item.type.tag === "map" && item.type.value.key, item.type.tag === "map" && item.type.value.value, - item.type.value, ]) .filter((type): type is number => typeof type === "number") ) @@ -85,21 +84,37 @@ export const compactMetadata = ( // ditch the types we aren't keeping metadata.lookup = metadata.lookup.filter((type) => keepTypes.has(type.id)) + // update all type ids to be sequential (fill the gaps left by the deleted types) + const newTypeIds = new Map() + metadata.lookup.forEach((type, index) => newTypeIds.set(type.id, index)) + const getNewTypeId = (oldTypeId: number): number => { + const newTypeId = newTypeIds.get(oldTypeId) + if (typeof newTypeId !== "number") log.error(`Failed to find newTypeId for type ${oldTypeId}`) + return newTypeId ?? 0 + } + remapTypeIds(metadata, getNewTypeId) + // ditch the remaining data we don't need to keep in a miniMetata - metadata.apis = [] - metadata.extrinsic.address = 0 - metadata.extrinsic.call = 0 - metadata.extrinsic.extra = 0 - metadata.extrinsic.signature = 0 - metadata.extrinsic.signature = 0 + if ("apis" in metadata) { + // metadata is v15 (NOT v14) + metadata.apis = [] + metadata.extrinsic.address = 0 + metadata.extrinsic.call = 0 + metadata.extrinsic.extra = 0 + metadata.extrinsic.signature = 0 + metadata.extrinsic.signature = 0 + } metadata.extrinsic.signedExtensions = [] - metadata.outerEnums.call = 0 - metadata.outerEnums.error = 0 - metadata.outerEnums.event = 0 + if ("outerEnums" in metadata) { + // metadata is v15 (NOT v14) + metadata.outerEnums.call = 0 + metadata.outerEnums.error = 0 + metadata.outerEnums.event = 0 + } } const addDependentTypes = ( - metadataTysMap: Map, + metadataTysMap: Map, keepTypes: Set, types: number[], // Prevent stack overflow when a type references itself @@ -119,14 +134,14 @@ const addDependentTypes = ( keepTypes.add(type.id) addedTypes.add(type.id) + const paramTypes = type.params + .map((param) => param.type) + .filter((type): type is number => typeof type === "number") + addDependentSubTypes(paramTypes) + switch (type.def.tag) { case "array": - addDependentSubTypes([ - ...type.params - .map((param) => param.type) - .filter((ty): ty is number => typeof ty === "number"), - type.def.value.type, - ]) + addDependentSubTypes([type.def.value.type]) break case "bitSequence": @@ -134,60 +149,46 @@ const addDependentTypes = ( break case "compact": - addDependentSubTypes([ - ...type.params - .map((param) => param.type) - .filter((ty): ty is number => typeof ty === "number"), - type.def.value, - ]) + addDependentSubTypes([type.def.value]) break case "composite": - addDependentSubTypes([ - ...type.params - .map((param) => param.type) - .filter((ty): ty is number => typeof ty === "number"), - ...type.def.value + addDependentSubTypes( + type.def.value .map((field) => field.type) - .filter((ty): ty is number => typeof ty === "number"), - ]) + .filter((type): type is number => typeof type === "number") + ) break case "primitive": - addDependentSubTypes([ - ...type.params - .map((param) => param.type) - .filter((ty): ty is number => typeof ty === "number"), - ]) break case "sequence": - addDependentSubTypes([ - ...type.params - .map((param) => param.type) - .filter((ty): ty is number => typeof ty === "number"), - type.def.value, - ]) + addDependentSubTypes([type.def.value]) break case "tuple": - addDependentSubTypes([ - ...type.params - .map((param) => param.type) - .filter((ty): ty is number => typeof ty === "number"), - ...type.def.value.filter((ty): ty is number => typeof ty === "number"), - ]) + addDependentSubTypes( + type.def.value.filter((type): type is number => typeof type === "number") + ) break case "variant": - addDependentSubTypes([ - ...type.params - .map((param) => param.type) - .filter((ty): ty is number => typeof ty === "number"), - ...type.def.value + addDependentSubTypes( + type.def.value .flatMap((member) => member.fields.map((field) => field.type)) - .filter((ty): ty is number => typeof ty === "number"), - ]) + .filter((type): type is number => typeof type === "number") + ) + break + + case "historicMetaCompat": + log.warn( + `"historicMetaCompat" type found in metadata: this type might be missing from the resulting miniMetadata's lookup map` + ) + // addDependentSubTypes([ + // // TODO: Handle `type.def.value`, which is a string, + // // but `addDependentSubTypes` is only looking for an array of type ids (which are integers) + // ]) break default: { @@ -198,3 +199,84 @@ const addDependentTypes = ( } } } + +const remapTypeIds = (metadata: V14 | V15, getNewTypeId: (oldTypeId: number) => number) => { + remapLookupTypeIds(metadata, getNewTypeId) + remapStorageTypeIds(metadata, getNewTypeId) +} + +const remapLookupTypeIds = (metadata: V14 | V15, getNewTypeId: (oldTypeId: number) => number) => { + for (const type of metadata.lookup) { + type.id = getNewTypeId(type.id) + + for (const param of type.params) { + if (typeof param.type !== "number") continue + param.type = getNewTypeId(param.type) + } + + switch (type.def.tag) { + case "array": + type.def.value.type = getNewTypeId(type.def.value.type) + break + + case "bitSequence": + type.def.value.bitOrderType = getNewTypeId(type.def.value.bitOrderType) + type.def.value.bitStoreType = getNewTypeId(type.def.value.bitStoreType) + break + + case "compact": + type.def.value = getNewTypeId(type.def.value) + break + + case "composite": + for (const field of type.def.value) { + if (typeof field.type !== "number") continue + field.type = getNewTypeId(field.type) + } + break + + case "primitive": + break + + case "sequence": + type.def.value = getNewTypeId(type.def.value) + break + + case "tuple": + type.def.value = type.def.value.map((type) => { + if (typeof type !== "number") return type + return getNewTypeId(type) + }) + break + + case "variant": + for (const member of type.def.value) { + for (const field of member.fields) { + if (typeof field.type !== "number") continue + field.type = getNewTypeId(field.type) + } + } + break + + case "historicMetaCompat": + break + + default: { + // force compilation error if any types don't have a case + const exhaustiveCheck: never = type.def + log.error(`Unhandled V15Type type ${exhaustiveCheck}`) + } + } + } +} +const remapStorageTypeIds = (metadata: V14 | V15, getNewTypeId: (oldTypeId: number) => number) => { + for (const pallet of metadata.pallets) { + for (const item of pallet.storage?.items ?? []) { + if (item.type.tag === "plain") item.type.value = getNewTypeId(item.type.value) + if (item.type.tag === "map") { + item.type.value.key = getNewTypeId(item.type.value.key) + item.type.value.value = getNewTypeId(item.type.value.value) + } + } + } +}