Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: Deployment tags to replace indexer-agent prefix in deployment names #921

Open
wants to merge 11 commits into
base: main
Choose a base branch
from
35 changes: 34 additions & 1 deletion packages/indexer-agent/src/agent.ts
Original file line number Diff line number Diff line change
Expand Up @@ -567,6 +567,34 @@ export class Agent {
},
)

const deploymentTags: Eventual<Map<string, string>> = join({
ticker: timer(60_000),
indexingRules,
}).tryMap(
async ({ indexingRules }) => {
logger.trace('Resolving deployment tags')
const deploymentTags: Map<string, string> = new Map()

// Add offchain subgraphs to the deployment list from rules
Object.values(indexingRules)
.flat()
.filter(rule => rule?.identifier !== 'global')
.forEach(rule => {
deploymentTags.set(
new SubgraphDeploymentID(rule.identifier).toString(),
rule.tag,
)
})
return new Map(deploymentTags)
},
{
onError: error =>
logger.warn(`Failed to resolve deployment tags, trying again later`, {
error,
}),
},
)

const activeAllocations: Eventual<NetworkMapped<Allocation[]>> = timer(
120_000,
).tryMap(
Expand Down Expand Up @@ -708,10 +736,12 @@ export class Agent {
switch (this.deploymentManagement) {
case DeploymentManagementMode.AUTO:
try {
const resolvedDeploymentTags = await deploymentTags.value()
await this.reconcileDeployments(
activeDeployments,
targetDeployments,
eligibleAllocations,
resolvedDeploymentTags,
)
} catch (err) {
logger.warn(
Expand Down Expand Up @@ -904,6 +934,7 @@ export class Agent {
activeDeployments: SubgraphDeploymentID[],
targetDeployments: SubgraphDeploymentID[],
eligibleAllocations: Allocation[],
deploymentTags: Map<string, string>,
): Promise<void> {
const logger = this.logger.child({ function: 'reconcileDeployments' })
logger.debug('Reconcile deployments')
Expand Down Expand Up @@ -974,7 +1005,9 @@ export class Agent {
// Index all new deployments worth indexing
await queue.addAll(
deploy.map(deployment => async () => {
const name = `indexer-agent/${deployment.ipfsHash.slice(-10)}`
const name = `${
deploymentTags.get(deployment.toString()) || 'indexer-agent'
}/${deployment.ipfsHash.slice(-10)}`

logger.info(`Index subgraph deployment`, {
name,
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,53 @@
import { Logger } from '@graphprotocol/common-ts'
import { DataTypes, QueryInterface } from 'sequelize'

interface MigrationContext {
queryInterface: QueryInterface
logger: Logger
}

interface Context {
context: MigrationContext
}

export async function up({ context }: Context): Promise<void> {
const { queryInterface, logger } = context

logger.debug(`Checking if indexing rules table exists`)
const tables = await queryInterface.showAllTables()
if (!tables.includes('IndexingRules')) {
logger.info(`Indexing rules table does not exist, migration not necessary`)
return
}

logger.debug(`Checking if 'IndexingRules' table needs to be migrated`)
const table = await queryInterface.describeTable('IndexingRules')
const subgraphTagColumn = table.tag
if (subgraphTagColumn) {
logger.info(`'tag' column already exists, migration not necessary`)
return
}

logger.info(`Add 'tag' column to 'IndexingRules' table`)
await queryInterface.addColumn('IndexingRules', 'tag', {
type: DataTypes.STRING,
primaryKey: false,
allowNull: false,
defaultValue: 'indexer-agent',
})
}

export async function down({ context }: Context): Promise<void> {
const { queryInterface, logger } = context

return await queryInterface.sequelize.transaction({}, async transaction => {
const tables = await queryInterface.showAllTables()

if (tables.includes('IndexingRules')) {
logger.info(`Remove 'tag' column`)
await context.queryInterface.removeColumn('IndexingRules', 'tag', {
transaction,
})
}
})
}
7 changes: 7 additions & 0 deletions packages/indexer-cli/src/rules.ts
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@ const INDEXING_RULE_PARSERS: Record<keyof IndexingRuleAttributes, (x: never) =>
requireSupported: x => parseBoolean(x),
safety: x => parseBoolean(x),
protocolNetwork: x => x,
tag: x => x,
}

const INDEXING_RULE_FORMATTERS: Record<
Expand All @@ -61,6 +62,7 @@ const INDEXING_RULE_FORMATTERS: Record<
requireSupported: x => x,
safety: x => x,
protocolNetwork: resolveChainAlias,
tag: x => x,
}

const INDEXING_RULE_CONVERTERS_FROM_GRAPHQL: Record<
Expand All @@ -85,6 +87,7 @@ const INDEXING_RULE_CONVERTERS_FROM_GRAPHQL: Record<
requireSupported: x => x,
safety: x => x,
protocolNetwork: x => x,
tag: x => x,
}

const INDEXING_RULE_CONVERTERS_TO_GRAPHQL: Record<
Expand All @@ -109,6 +112,7 @@ const INDEXING_RULE_CONVERTERS_TO_GRAPHQL: Record<
requireSupported: x => x,
safety: x => x,
protocolNetwork: x => x,
tag: x => x,
}

/**
Expand Down Expand Up @@ -267,6 +271,7 @@ export const indexingRules = async (
decisionBasis
requireSupported
safety
tag
}
}
`,
Expand Down Expand Up @@ -307,6 +312,7 @@ export const indexingRule = async (
requireSupported
safety
protocolNetwork
tag
}
}
`,
Expand Down Expand Up @@ -350,6 +356,7 @@ export const setIndexingRule = async (
requireSupported
safety
protocolNetwork
tag
}
}
`,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@ const SET_INDEXING_RULE_MUTATION = gql`
requireSupported
safety
protocolNetwork
tag
}
}
`
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -101,6 +101,7 @@ export const createTestManagementClient = async (
requireSupported: true,
safety: true,
protocolNetwork: 'sepolia',
tag: 'indexer-agent',
},
}

Expand Down Expand Up @@ -137,6 +138,7 @@ export const defaults: IndexerManagementDefaults = {
parallelAllocations: 1,
requireSupported: true,
safety: true,
tag: 'indexer-agent',
},
}

Expand Down
2 changes: 2 additions & 0 deletions packages/indexer-common/src/indexer-management/client.ts
Original file line number Diff line number Diff line change
Expand Up @@ -263,6 +263,7 @@ const SCHEMA_SDL = gql`
requireSupported: Boolean!
safety: Boolean!
protocolNetwork: String!
tag: String
}

input IndexingRuleInput {
Expand All @@ -282,6 +283,7 @@ const SCHEMA_SDL = gql`
requireSupported: Boolean
safety: Boolean
protocolNetwork: String!
tag: String
}

input IndexingRuleIdentifier {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ export interface IndexingRuleAttributes {
requireSupported: boolean
safety: boolean
protocolNetwork: string
tag: string
}

// Unambiguously identify a Indexing Rule in the Database.
Expand Down Expand Up @@ -60,6 +61,7 @@ export interface IndexingRuleCreationAttributes
| 'requireSupported'
| 'safety'
| 'protocolNetwork'
| 'tag'
> {}

export class IndexingRule
Expand All @@ -83,6 +85,7 @@ export class IndexingRule
public requireSupported!: boolean
public safety!: boolean
public protocolNetwork!: string
public tag!: string

public createdAt!: Date
public updatedAt!: Date
Expand Down Expand Up @@ -267,6 +270,12 @@ export const defineIndexingRuleModels = (sequelize: Sequelize): IndexingRuleMode
is: caip2IdRegex,
},
},
tag: {
type: DataTypes.STRING,
primaryKey: false,
allowNull: false,
defaultValue: 'indexer-agent',
},
},
{
modelName: 'IndexingRule',
Expand Down
1 change: 1 addition & 0 deletions packages/indexer-common/src/operator.ts
Original file line number Diff line number Diff line change
Expand Up @@ -109,6 +109,7 @@ export class Operator {
decisionBasis
requireSupported
protocolNetwork
tag
}
}
`,
Expand Down
1 change: 1 addition & 0 deletions packages/indexer-common/src/rules.ts
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@ const INDEXING_RULE_READABLE_TO_MODEL_PARSERS: Record<
requireSupported: (x) => parseBoolean(x),
safety: (x) => parseBoolean(x),
protocolNetwork: (x: string) => validateNetworkIdentifier(x),
tag: (x: string) => x,
}

export const parseIndexingRule = (
Expand Down