diff --git a/grid-proxy/Makefile b/grid-proxy/Makefile index b61d06bf..f3da9342 100644 --- a/grid-proxy/Makefile +++ b/grid-proxy/Makefile @@ -34,6 +34,10 @@ db-fill: ## Fill the database with a randomly generated data --reset \ --seed 13 +db-update: + @echo "Updating node uptimes" + @psql postgresql://postgres:postgres@$(PQ_HOST):5432/tfgrid-graphql < ./internal/explorer/db/helpers.sql + db-dump: ## Load a dump of the database (Args: `p= 0 + // have an empty list instead of null in the json response + if node.Features == nil { + node.Features = []string{} + } return node } @@ -163,9 +168,13 @@ func nodeWithNestedCapacityFromDBNode(info db.Node) types.NodeWithNestedCapacity GPUs: info.Gpus, PriceUsd: math.Round(info.PriceUsd*1000) / 1000, FarmFreeIps: info.FarmFreeIps, + Features: info.Features, } node.Status = nodestatus.DecideNodeStatus(node.Power, node.UpdatedAt) node.Dedicated = info.FarmDedicated || info.NodeContractsCount == 0 || info.Renter != 0 || info.ExtraFee > 0 + if node.Features == nil { + node.Features = []string{} + } return node } diff --git a/grid-proxy/internal/explorer/db/helpers.sql b/grid-proxy/internal/explorer/db/helpers.sql new file mode 100644 index 00000000..30d73a91 --- /dev/null +++ b/grid-proxy/internal/explorer/db/helpers.sql @@ -0,0 +1,24 @@ +-- THIS IS JUST FOR DEBUGGING PURPOSES + + +-- by the time, updated_at gets outdated which break some functionalities +-- that depends on node status. this help update the nodes that +-- got updated in the last period to now. +CREATE OR REPLACE FUNCTION update_node_uptimes() +RETURNS void AS $$ +DECLARE + last_updated_at INT; +BEGIN + -- Step 1: Get the latest uptime report's updated_at timestamp + SELECT updated_at + INTO last_updated_at + FROM node + ORDER BY updated_at DESC + LIMIT 1; + + -- Step 2: Update nodes where updated_at is > last_updated_at - 39 minutes + UPDATE node + SET updated_at = CAST(EXTRACT(epoch FROM NOW()) AS INT) + WHERE updated_at > last_updated_at - 2340; +END; +$$ LANGUAGE plpgsql; diff --git a/grid-proxy/internal/explorer/db/indexer_calls.go b/grid-proxy/internal/explorer/db/indexer_calls.go index 682e0780..4d406986 100644 --- a/grid-proxy/internal/explorer/db/indexer_calls.go +++ b/grid-proxy/internal/explorer/db/indexer_calls.go @@ -76,3 +76,11 @@ func (p *PostgresDatabase) UpsertNodeWorkloads(ctx context.Context, workloads [] } return p.gormDB.WithContext(ctx).Table("node_workloads").Clauses(conflictClause).Create(&workloads).Error } + +func (p *PostgresDatabase) UpsertNodeFeatures(ctx context.Context, features []types.NodeFeatures) error { + conflictClause := clause.OnConflict{ + Columns: []clause.Column{{Name: "node_twin_id"}}, + DoUpdates: clause.AssignmentColumns([]string{"features", "updated_at"}), + } + return p.gormDB.WithContext(ctx).Table("node_features").Clauses(conflictClause).Create(&features).Error +} diff --git a/grid-proxy/internal/explorer/db/postgres.go b/grid-proxy/internal/explorer/db/postgres.go index 77a1bcba..8e9e4d28 100644 --- a/grid-proxy/internal/explorer/db/postgres.go +++ b/grid-proxy/internal/explorer/db/postgres.go @@ -2,6 +2,7 @@ package db import ( "context" + "encoding/json" "fmt" "strings" @@ -132,6 +133,9 @@ func (d *PostgresDatabase) GetLastUpsertsTimestamp() (types.IndexersState, error if res := d.gormDB.Table("node_workloads").Select("updated_at").Where("updated_at IS NOT NULL").Order("updated_at DESC").Limit(1).Scan(&report.Workloads.UpdatedAt); res.Error != nil { return report, errors.Wrap(res.Error, "couldn't get workloads last updated_at") } + if res := d.gormDB.Table("node_features").Select("updated_at").Where("updated_at IS NOT NULL").Order("updated_at DESC").Limit(1).Scan(&report.Features.UpdatedAt); res.Error != nil { + return report, errors.Wrap(res.Error, "couldn't get features last updated_at") + } return report, nil } @@ -143,6 +147,7 @@ func (d *PostgresDatabase) Initialize() error { &types.Speed{}, &types.HasIpv6{}, &types.NodesWorkloads{}, + &types.NodeFeatures{}, ); err != nil { return errors.Wrap(err, "failed to migrate indexer tables") } @@ -363,6 +368,7 @@ func (d *PostgresDatabase) nodeTableQuery(ctx context.Context, filter types.Node "resources_cache.gpus", "health_report.healthy", "node_ipv6.has_ipv6", + "node_features.features as features", "resources_cache.bios", "resources_cache.baseboard", "resources_cache.memory", @@ -380,11 +386,10 @@ func (d *PostgresDatabase) nodeTableQuery(ctx context.Context, filter types.Node LEFT JOIN location ON node.location_id = location.id LEFT JOIN health_report ON node.twin_id = health_report.node_twin_id LEFT JOIN node_ipv6 ON node.twin_id = node_ipv6.node_twin_id + LEFT JOIN node_features ON node.twin_id = node_features.node_twin_id `) - if filter.HasGPU != nil || filter.GpuDeviceName != nil || - filter.GpuVendorName != nil || filter.GpuVendorID != nil || - filter.GpuDeviceID != nil || filter.GpuAvailable != nil { + if filter.IsGpuFilterRequested() { q.Joins( `RIGHT JOIN (?) AS gpu ON gpu.node_twin_id = node.twin_id`, nodeGpuSubquery, ) @@ -411,12 +416,9 @@ func (d *PostgresDatabase) farmTableQuery(ctx context.Context, filter types.Farm "LEFT JOIN public_ips_cache ON public_ips_cache.farm_id = farm.farm_id", ) - if filter.NodeAvailableFor != nil || filter.NodeFreeHRU != nil || - filter.NodeCertified != nil || filter.NodeFreeMRU != nil || - filter.NodeFreeSRU != nil || filter.NodeHasGPU != nil || - filter.NodeRentedBy != nil || len(filter.NodeStatus) != 0 || - filter.NodeTotalCRU != nil || filter.Country != nil || - filter.Region != nil || filter.NodeHasIpv6 != nil { + if filter.IsNodeFilterRequested() { + // TODO: would it be a good option to delegate here to the GetNodes? + // how this will affect the performance benchmark? q.Joins(`RIGHT JOIN (?) AS resources_cache on resources_cache.farm_id = farm.farm_id`, nodeQuery). Group(` farm.id, @@ -485,6 +487,17 @@ func (d *PostgresDatabase) GetFarms(ctx context.Context, filter types.FarmFilter Where("COALESCE(has_ipv6, false) = ?", *filter.NodeHasIpv6) } + if len(filter.NodeFeatures) != 0 { + jsonList, err := json.Marshal(filter.NodeFeatures) + if err != nil { + return nil, 0, errors.Wrap(err, "failed to marshal the features filter to json list") + } + + nodeQuery = nodeQuery. + Joins("LEFT JOIN node_features ON node_features.node_twin_id = node.twin_id"). + Where(`COALESCE(node_features.features, '[]') @> ?`, jsonList) + } + q := d.farmTableQuery(ctx, filter, nodeQuery) if filter.NodeAvailableFor != nil { @@ -735,6 +748,17 @@ func (d *PostgresDatabase) GetNodes(ctx context.Context, filter types.NodeFilter if filter.PriceMax != nil { q = q.Where(`calc_discount(resources_cache.price_usd, ?) <= ?`, limit.Balance, *filter.PriceMax) } + if len(filter.Features) != 0 { + // The @> operator checks if all the right elements exist on the left, + // it needs a proper json object on the right hand side. + // check https://www.postgresql.org/docs/9.4/functions-json.html for jsonb operators + jsonList, err := json.Marshal(filter.Features) + if err != nil { + return nil, 0, errors.Wrap(err, "failed to marshal the features filter to json list") + } + + q = q.Where(`COALESCE(node_features.features, '[]') @> ?`, jsonList) + } // Sorting if limit.Randomize { diff --git a/grid-proxy/internal/explorer/db/types.go b/grid-proxy/internal/explorer/db/types.go index 873c1bda..f571bd61 100644 --- a/grid-proxy/internal/explorer/db/types.go +++ b/grid-proxy/internal/explorer/db/types.go @@ -41,6 +41,7 @@ type Database interface { UpsertNetworkSpeed(ctx context.Context, speeds []types.Speed) error UpsertNodeIpv6Report(ctx context.Context, ips []types.HasIpv6) error UpsertNodeWorkloads(ctx context.Context, workloads []types.NodesWorkloads) error + UpsertNodeFeatures(ctx context.Context, features []types.NodeFeatures) error } type ContractBilling types.ContractBilling @@ -111,6 +112,7 @@ type Node struct { DownloadSpeed float64 PriceUsd float64 FarmFreeIps uint + Features []string `gorm:"type:jsonb;serializer:json"` } // NodePower struct is the farmerbot report for node status diff --git a/grid-proxy/internal/explorer/health.go b/grid-proxy/internal/explorer/health.go index efa4bd75..c051b1bc 100644 --- a/grid-proxy/internal/explorer/health.go +++ b/grid-proxy/internal/explorer/health.go @@ -52,7 +52,8 @@ func createReport(db DBClient, peer rmb.Client, idxIntervals map[string]uint) ty isIndexerStale(indexers.Health.UpdatedAt, idxIntervals["health"]) || isIndexerStale(indexers.Ipv6.UpdatedAt, idxIntervals["ipv6"]) || isIndexerStale(indexers.Speed.UpdatedAt, idxIntervals["speed"]) || - isIndexerStale(indexers.Workloads.UpdatedAt, idxIntervals["workloads"]) { + isIndexerStale(indexers.Workloads.UpdatedAt, idxIntervals["workloads"]) || + isIndexerStale(indexers.Features.UpdatedAt, idxIntervals["features"]) { report.TotalStateOk = false } diff --git a/grid-proxy/internal/explorer/server.go b/grid-proxy/internal/explorer/server.go index 85860b0f..46401e32 100644 --- a/grid-proxy/internal/explorer/server.go +++ b/grid-proxy/internal/explorer/server.go @@ -54,6 +54,7 @@ const ( // @Param node_has_gpu query bool false "True for farms who have at least one node with a GPU" // @Param node_has_ipv6 query bool false "True for farms who have at least one node with an ipv6" // @Param node_certified query bool false "True for farms who have at least one certified node" +// @Param node_features query string false "filter farms with list of supported features on its nods" // @Param country query string false "farm country" // @Param region query string false "farm region" // @Success 200 {object} []types.Farm @@ -66,9 +67,14 @@ func (a *App) listFarms(r *http.Request) (interface{}, mw.Response) { if err := parseQueryParams(r, &filter, &limit); err != nil { return nil, mw.BadRequest(err) } + // TODO: move the validation into the parsing function + // the parser should be generic to accept the different filters if err := limit.Valid(types.Farm{}); err != nil { return nil, mw.BadRequest(err) } + if err := filter.Validate(); err != nil { + return nil, mw.BadRequest(err) + } dbFarms, farmsCount, err := a.cl.Farms(r.Context(), filter, limit) if err != nil { @@ -157,6 +163,7 @@ func (a *App) getStats(r *http.Request) (interface{}, mw.Response) { // @Param owned_by query int false "get nodes owned by twin id" // @Param price_min query string false "get nodes with price greater than this" // @Param price_max query string false "get nodes with price smaller than this" +// @Param features query string false "filter nodes with list of supported features" // @Success 200 {object} []types.Node // @Failure 400 {object} string // @Failure 500 {object} string @@ -215,6 +222,9 @@ func (a *App) listNodes(r *http.Request) (interface{}, mw.Response) { if err := limit.Valid(types.Node{}); err != nil { return nil, mw.BadRequest(err) } + if err := filter.Validate(); err != nil { + return nil, mw.BadRequest(err) + } dbNodes, nodesCount, err := a.cl.Nodes(r.Context(), filter, limit) if err != nil { diff --git a/grid-proxy/internal/indexer/README.md b/grid-proxy/internal/indexer/README.md index 9617d6ef..5fc84321 100644 --- a/grid-proxy/internal/indexer/README.md +++ b/grid-proxy/internal/indexer/README.md @@ -66,3 +66,8 @@ Work a struct that implement the interface `Work` which have three methods: - Interval: `1 hour` - Default caller worker number: 10 - Dump table: `node_workloads` +7. Features indexer: + - Function: get the supported features on each node. + - Interval: `1 day` + - Default caller worker number: 10 + - Dump table: `node_features` diff --git a/grid-proxy/internal/indexer/features.go b/grid-proxy/internal/indexer/features.go new file mode 100644 index 00000000..d1ddaf6a --- /dev/null +++ b/grid-proxy/internal/indexer/features.go @@ -0,0 +1,57 @@ +package indexer + +import ( + "context" + "time" + + "github.com/threefoldtech/tfgrid-sdk-go/grid-proxy/internal/explorer/db" + "github.com/threefoldtech/tfgrid-sdk-go/grid-proxy/pkg/types" + "github.com/threefoldtech/tfgrid-sdk-go/rmb-sdk-go/peer" +) + +const ( + featuresCallCmd = "zos.system.node_features_get" +) + +type FeatureWork struct { + findersInterval map[string]time.Duration +} + +func NewFeatureWork(interval uint) *FeatureWork { + return &FeatureWork{ + findersInterval: map[string]time.Duration{ + "up": time.Duration(interval) * time.Minute, + "new": newNodesCheckInterval, + }, + } +} + +func (w *FeatureWork) Finders() map[string]time.Duration { + return w.findersInterval +} + +func (w *FeatureWork) Get(ctx context.Context, rmb *peer.RpcClient, twinId uint32) ([]types.NodeFeatures, error) { + var features []string + err := callNode(ctx, rmb, featuresCallCmd, nil, twinId, &features) + if err != nil { + return []types.NodeFeatures{}, err + } + + res := parseNodeFeatures(twinId, features) + return []types.NodeFeatures{res}, nil + +} + +func (w *FeatureWork) Upsert(ctx context.Context, db db.Database, batch []types.NodeFeatures) error { + return db.UpsertNodeFeatures(ctx, batch) +} + +func parseNodeFeatures(twinId uint32, features []string) types.NodeFeatures { + res := types.NodeFeatures{ + NodeTwinId: twinId, + UpdatedAt: time.Now().Unix(), + Features: features, + } + + return res +} diff --git a/grid-proxy/pkg/types/farms.go b/grid-proxy/pkg/types/farms.go index dd2f53e6..08a5b8e8 100644 --- a/grid-proxy/pkg/types/farms.go +++ b/grid-proxy/pkg/types/farms.go @@ -38,6 +38,21 @@ type FarmFilter struct { NodeHasGPU *bool `schema:"node_has_gpu,omitempty"` NodeHasIpv6 *bool `schema:"node_has_ipv6,omitempty"` NodeCertified *bool `schema:"node_certified,omitempty"` + NodeFeatures []string `schema:"node_features,omitempty"` Country *string `schema:"country,omitempty"` Region *string `schema:"region,omitempty"` } + +func (f FarmFilter) Validate() error { + return validateNodeFeatures(f.NodeFeatures) +} + +func (f FarmFilter) IsNodeFilterRequested() bool { + return f.NodeAvailableFor != nil || f.NodeFreeHRU != nil || + f.NodeCertified != nil || f.NodeFreeMRU != nil || + f.NodeFreeSRU != nil || f.NodeHasGPU != nil || + f.NodeRentedBy != nil || len(f.NodeStatus) != 0 || + f.NodeTotalCRU != nil || f.Country != nil || + f.Region != nil || f.NodeHasIpv6 != nil || + len(f.NodeFeatures) != 0 +} diff --git a/grid-proxy/pkg/types/indexer.go b/grid-proxy/pkg/types/indexer.go index cc30490e..c7eeb27d 100644 --- a/grid-proxy/pkg/types/indexer.go +++ b/grid-proxy/pkg/types/indexer.go @@ -97,3 +97,13 @@ type Memory struct { Manufacturer string `json:"manufacturer"` Type string `json:"type"` } + +type NodeFeatures struct { + NodeTwinId uint32 `json:"node_twin_id,omitempty" gorm:"unique;not null"` + UpdatedAt int64 `json:"updated_at"` + Features []string `json:"features" gorm:"type:jsonb;serializer:json"` +} + +func (NodeFeatures) TableName() string { + return "node_features" +} diff --git a/grid-proxy/pkg/types/nodes.go b/grid-proxy/pkg/types/nodes.go index 2fc9b392..27261719 100644 --- a/grid-proxy/pkg/types/nodes.go +++ b/grid-proxy/pkg/types/nodes.go @@ -52,6 +52,7 @@ type Node struct { GPUs []NodeGPU `json:"gpus"` PriceUsd float64 `json:"price_usd" sort:"price_usd"` FarmFreeIps uint `json:"farm_free_ips"` + Features []string `json:"features"` _ string `sort:"free_cru"` } @@ -96,6 +97,7 @@ type NodeWithNestedCapacity struct { GPUs []NodeGPU `json:"gpus"` PriceUsd float64 `json:"price_usd"` FarmFreeIps uint `json:"farm_free_ips"` + Features []string `json:"features"` } // PublicConfig node public config @@ -161,4 +163,15 @@ type NodeFilter struct { PriceMax *float64 `schema:"price_max,omitempty"` Excluded []uint64 `schema:"excluded,omitempty"` HasIpv6 *bool `schema:"has_ipv6,omitempty"` + Features []string `schema:"features,omitempty"` +} + +func (f NodeFilter) Validate() error { + return validateNodeFeatures(f.Features) +} + +func (f NodeFilter) IsGpuFilterRequested() bool { + return f.HasGPU != nil || f.GpuDeviceName != nil || + f.GpuVendorName != nil || f.GpuVendorID != nil || + f.GpuDeviceID != nil || f.GpuAvailable != nil } diff --git a/grid-proxy/pkg/types/validators.go b/grid-proxy/pkg/types/validators.go new file mode 100644 index 00000000..a0517007 --- /dev/null +++ b/grid-proxy/pkg/types/validators.go @@ -0,0 +1,24 @@ +package types + +import ( + "fmt" + "slices" +) + +var ( + Zos3NodesFeatures = []string{"zmount", "zdb", "volume", "ipv4", "ip", "gateway-name-proxy", + "gateway-fqdn-proxy", "qsfs", "zlogs", "network", "zmachine"} + Zos4NodesFeatures = []string{"zmount", "zdb", "volume", "ipv4", "ip", "gateway-name-proxy", + "gateway-fqdn-proxy", "qsfs", "zlogs", "zmachine-light", "network-light"} + FeaturesSet = []string{"zmount", "zdb", "volume", "ipv4", "ip", "gateway-name-proxy", + "gateway-fqdn-proxy", "qsfs", "zlogs", "network", "zmachine", "zmachine-light", "network-light"} +) + +func validateNodeFeatures(features []string) error { + for _, feat := range features { + if !slices.Contains(FeaturesSet, feat) { + return fmt.Errorf("%s is not a valid node feature", feat) + } + } + return nil +} diff --git a/grid-proxy/pkg/types/version.go b/grid-proxy/pkg/types/version.go index e1636168..32482669 100644 --- a/grid-proxy/pkg/types/version.go +++ b/grid-proxy/pkg/types/version.go @@ -16,6 +16,7 @@ type IndexersState struct { Speed IndexerState `json:"speed"` Ipv6 IndexerState `json:"ipv6"` Workloads IndexerState `json:"workloads"` + Features IndexerState `json:"features"` } // Health represent the healthiness of the server and connections diff --git a/grid-proxy/tests/queries/farm_test.go b/grid-proxy/tests/queries/farm_test.go index 38a46d7e..73d5e983 100644 --- a/grid-proxy/tests/queries/farm_test.go +++ b/grid-proxy/tests/queries/farm_test.go @@ -11,6 +11,7 @@ import ( "github.com/google/go-cmp/cmp" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/threefoldtech/tfgrid-sdk-go/grid-proxy/pkg/types" proxytypes "github.com/threefoldtech/tfgrid-sdk-go/grid-proxy/pkg/types" mock "github.com/threefoldtech/tfgrid-sdk-go/grid-proxy/tests/queries/mock_client" ) @@ -144,6 +145,10 @@ var farmFilterRandomValueGenerator = map[string]func(agg FarmsAggregate) interfa } return &v }, + "NodeFeatures": func(_ FarmsAggregate) interface{} { + randomLen := rand.Intn(5) + return getRandomSliceFrom(types.FeaturesSet, randomLen) + }, } type FarmsAggregate struct { diff --git a/grid-proxy/tests/queries/mock_client/farms.go b/grid-proxy/tests/queries/mock_client/farms.go index 1b7a1350..b6b7e8a6 100644 --- a/grid-proxy/tests/queries/mock_client/farms.go +++ b/grid-proxy/tests/queries/mock_client/farms.go @@ -110,12 +110,7 @@ func (f *Farm) satisfies(filter types.FarmFilter, data *DBData) bool { return false } - if filter.NodeAvailableFor != nil || filter.NodeCertified != nil || - filter.NodeFreeHRU != nil || filter.NodeFreeMRU != nil || - filter.NodeFreeSRU != nil || filter.NodeHasGPU != nil || - filter.NodeRentedBy != nil || len(filter.NodeStatus) != 0 || - filter.Country != nil || filter.Region != nil || - filter.NodeTotalCRU != nil || filter.NodeHasIpv6 != nil { + if filter.IsNodeFilterRequested() { if !f.satisfyFarmNodesFilter(data, filter) { return false } @@ -188,6 +183,10 @@ func (f *Farm) satisfyFarmNodesFilter(data *DBData, filter types.FarmFilter) boo continue } + if len(filter.NodeFeatures) != 0 && !sliceContains(data.NodeFeatures[uint32(node.TwinID)], filter.NodeFeatures) { + continue + } + return true } return false diff --git a/grid-proxy/tests/queries/mock_client/loader.go b/grid-proxy/tests/queries/mock_client/loader.go index 51589083..b980b345 100644 --- a/grid-proxy/tests/queries/mock_client/loader.go +++ b/grid-proxy/tests/queries/mock_client/loader.go @@ -41,6 +41,7 @@ type DBData struct { Locations map[string]Location HealthReports map[uint32]bool NodeIpv6 map[uint32]bool + NodeFeatures map[uint32][]string DMIs map[uint32]types.Dmi Speeds map[uint32]types.Speed PricingPolicies map[uint]PricingPolicy @@ -49,7 +50,7 @@ type DBData struct { DB *sql.DB } -func loadNodes(db *sql.DB, gormDB *gorm.DB, data *DBData) error { +func loadNodes(gormDB *gorm.DB, data *DBData) error { var nodes []Node err := gormDB.Table("node").Scan(&nodes).Error if err != nil { @@ -598,7 +599,22 @@ func loadNodeIpv6(db *sql.DB, data *DBData) error { return nil } -func loadDMIs(db *sql.DB, gormDB *gorm.DB, data *DBData) error { +func loadNodeFeatures(gormDB *gorm.DB, data *DBData) error { + var featuresReports []types.NodeFeatures + + // load using gorm to utilize the json serializer + if err := gormDB.Table("node_features").Scan(&featuresReports).Error; err != nil { + return err + } + + for _, feat := range featuresReports { + data.NodeFeatures[feat.NodeTwinId] = feat.Features + } + + return nil +} + +func loadDMIs(gormDB *gorm.DB, data *DBData) error { var dmis []types.Dmi err := gormDB.Table("dmi").Scan(&dmis).Error if err != nil { @@ -755,11 +771,12 @@ func Load(db *sql.DB, gormDB *gorm.DB) (DBData, error) { DMIs: make(map[uint32]types.Dmi), Speeds: make(map[uint32]types.Speed), NodeIpv6: make(map[uint32]bool), + NodeFeatures: make(map[uint32][]string), PricingPolicies: make(map[uint]PricingPolicy), WorkloadsNumbers: make(map[uint32]uint32), DB: db, } - if err := loadNodes(db, gormDB, &data); err != nil { + if err := loadNodes(gormDB, &data); err != nil { return data, err } if err := loadFarms(db, &data); err != nil { @@ -807,7 +824,10 @@ func Load(db *sql.DB, gormDB *gorm.DB) (DBData, error) { if err := loadNodeIpv6(db, &data); err != nil { return data, err } - if err := loadDMIs(db, gormDB, &data); err != nil { + if err := loadNodeFeatures(gormDB, &data); err != nil { + return data, err + } + if err := loadDMIs(gormDB, &data); err != nil { return data, err } if err := loadSpeeds(db, &data); err != nil { diff --git a/grid-proxy/tests/queries/mock_client/nodes.go b/grid-proxy/tests/queries/mock_client/nodes.go index 1228bc45..93f74bf7 100644 --- a/grid-proxy/tests/queries/mock_client/nodes.go +++ b/grid-proxy/tests/queries/mock_client/nodes.go @@ -188,6 +188,7 @@ func (g *GridProxyMockClient) Nodes(ctx context.Context, filter types.NodeFilter }, PriceUsd: calcDiscount(calcNodePrice(g.data, node), limit.Balance), FarmFreeIps: uint(g.data.FreeIPs[node.FarmID]), + Features: g.data.NodeFeatures[uint32(node.TwinID)], }) } } @@ -285,6 +286,7 @@ func (g *GridProxyMockClient) Node(ctx context.Context, nodeID uint32) (res type }, PriceUsd: calcNodePrice(g.data, node), FarmFreeIps: uint(g.data.FreeIPs[node.FarmID]), + Features: g.data.NodeFeatures[uint32(node.TwinID)], } return } @@ -334,6 +336,10 @@ func (n *Node) satisfies(f types.NodeFilter, data *DBData) bool { return false } + if len(f.Features) != 0 && !sliceContains(data.NodeFeatures[uint32(n.TwinID)], f.Features) { + return false + } + if f.FreeSRU != nil && int64(*f.FreeSRU) > int64(free.SRU) { return false } @@ -468,10 +474,9 @@ func (n *Node) satisfies(f types.NodeFilter, data *DBData) bool { return false } - foundGpuFilter := f.HasGPU != nil || f.GpuDeviceName != nil || f.GpuVendorName != nil || f.GpuVendorID != nil || f.GpuDeviceID != nil || f.GpuAvailable != nil gpus, foundGpuCards := data.GPUs[uint32(n.TwinID)] - if !foundGpuCards && foundGpuFilter { + if !foundGpuCards && f.IsGpuFilterRequested() { return false } @@ -490,7 +495,7 @@ func (n *Node) satisfies(f types.NodeFilter, data *DBData) bool { } } - if !foundSuitableCard && foundGpuFilter { + if !foundSuitableCard && f.IsGpuFilterRequested() { return false } diff --git a/grid-proxy/tests/queries/mock_client/utils.go b/grid-proxy/tests/queries/mock_client/utils.go index 31ce4f66..a6b91d35 100644 --- a/grid-proxy/tests/queries/mock_client/utils.go +++ b/grid-proxy/tests/queries/mock_client/utils.go @@ -1,6 +1,7 @@ package mock import ( + "slices" "strings" "github.com/threefoldtech/tfgrid-sdk-go/grid-proxy/pkg/types" @@ -54,3 +55,13 @@ func getPage[R Result](res []R, limit types.Limit) ([]R, int) { return res, totalCount } + +func sliceContains(set []string, subset []string) bool { + for _, item := range subset { + if !slices.Contains(set, item) { + return false + } + } + + return true +} diff --git a/grid-proxy/tests/queries/node_test.go b/grid-proxy/tests/queries/node_test.go index 7cd34d5a..c8a4487f 100644 --- a/grid-proxy/tests/queries/node_test.go +++ b/grid-proxy/tests/queries/node_test.go @@ -355,6 +355,10 @@ var nodeFilterRandomValueGenerator = map[string]func(agg NodesAggregate) interfa num := rand.Intn(10) return shuffledIds[:num] }, + "Features": func(_ NodesAggregate) interface{} { + randomLen := rand.Intn(5) + return getRandomSliceFrom(types.FeaturesSet, randomLen) + }, } func TestNode(t *testing.T) { diff --git a/grid-proxy/tools/db/crafter/generator.go b/grid-proxy/tools/db/crafter/generator.go index 39312eae..2c51f32c 100644 --- a/grid-proxy/tools/db/crafter/generator.go +++ b/grid-proxy/tools/db/crafter/generator.go @@ -991,3 +991,31 @@ func (c *Crafter) GenerateNodeWorkloads() error { return nil } + +func (c *Crafter) GenerateNodeFeatures() error { + start := c.NodeStart + end := c.NodeStart + c.NodeCount + nodeTwinsStart := c.TwinStart + (c.FarmStart + c.FarmCount) + + var reports []types.NodeFeatures + for i := start; i < end; i++ { + features := types.Zos3NodesFeatures + if flip(.5) { + features = types.Zos4NodesFeatures + } + + report := types.NodeFeatures{ + NodeTwinId: uint32(nodeTwinsStart + i), + UpdatedAt: time.Now().Unix(), + Features: features, + } + reports = append(reports, report) + } + + if err := c.gormDB.Create(reports).Error; err != nil { + return fmt.Errorf("failed to insert node features reports: %w", err) + } + fmt.Println("node features number reports generated") + + return nil +} diff --git a/grid-proxy/tools/db/generate.go b/grid-proxy/tools/db/generate.go index 180183e0..2385b4e2 100644 --- a/grid-proxy/tools/db/generate.go +++ b/grid-proxy/tools/db/generate.go @@ -140,5 +140,9 @@ func generateData(db *sql.DB, gormDB *gorm.DB, seed int) error { if err := generator.GeneratePricingPolicies(); err != nil { return fmt.Errorf("failed to generate PricingPolicies: %w", err) } + + if err := generator.GenerateNodeFeatures(); err != nil { + return fmt.Errorf("failed to generate NodeFeatures: %w", err) + } return nil } diff --git a/grid-proxy/tools/db/schema.sql b/grid-proxy/tools/db/schema.sql index c7db15b0..66c38325 100644 --- a/grid-proxy/tools/db/schema.sql +++ b/grid-proxy/tools/db/schema.sql @@ -1111,3 +1111,16 @@ CREATE TABLE IF NOT EXISTS public.node_workloads ( ALTER TABLE public.node_workloads OWNER TO postgres; +-- +-- Name: node_features; Type: TABLE; Schema: public; Owner: postgres +-- + +CREATE TABLE IF NOT EXISTS public.node_features ( + node_twin_id bigint NOT NULL, + features jsonb, + updated_at bigint +); + +ALTER TABLE public.node_features + OWNER TO postgres; +