Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

feat: curio: storage filter based on miner ID #11801

Merged
merged 4 commits into from
Apr 4, 2024
Merged

feat: curio: storage filter based on miner ID #11801

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
accc09d
add miner fliter
LexLuthr Apr 1, 2024
4647d08
refactor filter decision code
LexLuthr Apr 2, 2024
ad8ee42
refactor MinerFilter func, update comments
LexLuthr Apr 4, 2024
addcb87
apply suggestions
LexLuthr Apr 4, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 5 additions & 5 deletions api/api_storage.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -200,11 +200,11 @@ type StorageMiner interface {
// StorageBestAlloc returns list of paths where sector files of the specified type can be allocated, ordered by preference.
// Paths with more weight and more % of free space are preferred.
// Note: This method doesn't filter paths based on AllowTypes/DenyTypes.
StorageBestAlloc(ctx context.Context, allocate storiface.SectorFileType, ssize abi.SectorSize, pathType storiface.PathType) ([]storiface.StorageInfo, error) //perm:admin
StorageLock(ctx context.Context, sector abi.SectorID, read storiface.SectorFileType, write storiface.SectorFileType) error //perm:admin
StorageTryLock(ctx context.Context, sector abi.SectorID, read storiface.SectorFileType, write storiface.SectorFileType) (bool, error) //perm:admin
StorageList(ctx context.Context) (map[storiface.ID][]storiface.Decl, error) //perm:admin
StorageGetLocks(ctx context.Context) (storiface.SectorLocks, error) //perm:admin
StorageBestAlloc(ctx context.Context, allocate storiface.SectorFileType, ssize abi.SectorSize, pathType storiface.PathType, miner abi.ActorID) ([]storiface.StorageInfo, error) //perm:admin
StorageLock(ctx context.Context, sector abi.SectorID, read storiface.SectorFileType, write storiface.SectorFileType) error //perm:admin
StorageTryLock(ctx context.Context, sector abi.SectorID, read storiface.SectorFileType, write storiface.SectorFileType) (bool, error) //perm:admin
StorageList(ctx context.Context) (map[storiface.ID][]storiface.Decl, error) //perm:admin
StorageGetLocks(ctx context.Context) (storiface.SectorLocks, error) //perm:admin

StorageLocal(ctx context.Context) (map[storiface.ID]string, error) //perm:admin
StorageStat(ctx context.Context, id storiface.ID) (fsutil.FsStat, error) //perm:admin
Expand Down
8 changes: 4 additions & 4 deletions api/proxy_gen.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

91 changes: 90 additions & 1 deletion build/openrpc/miner.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8897,11 +8897,23 @@
],
"DenyTypes": [
"string value"
],
"AllowMiners": [
"string value"
],
"DenyMiners": [
"string value"
]
}
],
"additionalProperties": false,
"properties": {
"AllowMiners": {
"items": {
"type": "string"
},
"type": "array"
},
"AllowTo": {
"items": {
"type": "string"
Expand All @@ -8920,6 +8932,12 @@
"CanStore": {
"type": "boolean"
},
"DenyMiners": {
"items": {
"type": "string"
},
"type": "array"
},
"DenyTypes": {
"items": {
"type": "string"
Expand Down Expand Up @@ -9078,7 +9096,7 @@
},
{
"name": "Filecoin.StorageBestAlloc",
"description": "```go\nfunc (s *StorageMinerStruct) StorageBestAlloc(p0 context.Context, p1 storiface.SectorFileType, p2 abi.SectorSize, p3 storiface.PathType) ([]storiface.StorageInfo, error) {\n\tif s.Internal.StorageBestAlloc == nil {\n\t\treturn *new([]storiface.StorageInfo), ErrNotSupported\n\t}\n\treturn s.Internal.StorageBestAlloc(p0, p1, p2, p3)\n}\n```",
"description": "```go\nfunc (s *StorageMinerStruct) StorageBestAlloc(p0 context.Context, p1 storiface.SectorFileType, p2 abi.SectorSize, p3 storiface.PathType, p4 abi.ActorID) ([]storiface.StorageInfo, error) {\n\tif s.Internal.StorageBestAlloc == nil {\n\t\treturn *new([]storiface.StorageInfo), ErrNotSupported\n\t}\n\treturn s.Internal.StorageBestAlloc(p0, p1, p2, p3, p4)\n}\n```",
"summary": "StorageBestAlloc returns list of paths where sector files of the specified type can be allocated, ordered by preference.\nPaths with more weight and more % of free space are preferred.\nNote: This method doesn't filter paths based on AllowTypes/DenyTypes.\n",
"paramStructure": "by-position",
"params": [
Expand Down Expand Up @@ -9130,6 +9148,23 @@
},
"required": true,
"deprecated": false
},
{
"name": "p4",
"description": "abi.ActorID",
"summary": "",
"schema": {
"title": "number",
"description": "Number is a number",
"examples": [
1000
],
"type": [
"number"
]
},
"required": true,
"deprecated": false
}
],
"result": {
Expand Down Expand Up @@ -9159,6 +9194,12 @@
],
"DenyTypes": [
"string value"
],
"AllowMiners": [
"string value"
],
"DenyMiners": [
"string value"
]
}
]
Expand All @@ -9167,6 +9208,12 @@
{
"additionalProperties": false,
"properties": {
"AllowMiners": {
"items": {
"type": "string"
},
"type": "array"
},
"AllowTo": {
"items": {
"type": "string"
Expand All @@ -9185,6 +9232,12 @@
"CanStore": {
"type": "boolean"
},
"DenyMiners": {
"items": {
"type": "string"
},
"type": "array"
},
"DenyTypes": {
"items": {
"type": "string"
Expand Down Expand Up @@ -9620,6 +9673,12 @@
],
"DenyTypes": [
"string value"
],
"AllowMiners": [
"string value"
],
"DenyMiners": [
"string value"
]
}
]
Expand All @@ -9628,6 +9687,12 @@
{
"additionalProperties": false,
"properties": {
"AllowMiners": {
"items": {
"type": "string"
},
"type": "array"
},
"AllowTypes": {
"items": {
"type": "string"
Expand All @@ -9646,6 +9711,12 @@
"CanStore": {
"type": "boolean"
},
"DenyMiners": {
"items": {
"type": "string"
},
"type": "array"
},
"DenyTypes": {
"items": {
"type": "string"
Expand Down Expand Up @@ -9833,11 +9904,23 @@
],
"DenyTypes": [
"string value"
],
"AllowMiners": [
"string value"
],
"DenyMiners": [
"string value"
]
}
],
"additionalProperties": false,
"properties": {
"AllowMiners": {
"items": {
"type": "string"
},
"type": "array"
},
"AllowTo": {
"items": {
"type": "string"
Expand All @@ -9856,6 +9939,12 @@
"CanStore": {
"type": "boolean"
},
"DenyMiners": {
"items": {
"type": "string"
},
"type": "array"
},
"DenyTypes": {
"items": {
"type": "string"
Expand Down
18 changes: 17 additions & 1 deletion cmd/curio/tasks/tasks.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,8 @@ import (
"github.com/samber/lo"
"golang.org/x/xerrors"

"github.com/filecoin-project/go-address"

"github.com/filecoin-project/lotus/cmd/curio/deps"
curio "github.com/filecoin-project/lotus/curiosrc"
"github.com/filecoin-project/lotus/curiosrc/chainsched"
Expand Down Expand Up @@ -37,6 +39,20 @@ func StartTasks(ctx context.Context, dependencies *deps.Deps) (*harmonytask.Task
si := dependencies.Si
var activeTasks []harmonytask.TaskInterface

// Get all miner address from config
var miners []address.Address
for _, ad := range cfg.Addresses {
ad := ad
for _, m := range ad.MinerAddresses {
m := m
maddr, err := address.NewFromString(m)
if err != nil {
return nil, xerrors.Errorf("failed to parse the miner address: %w", err)
}
miners = append(miners, maddr)
}
}

sender, sendTask := message.NewSender(full, full, db)
activeTasks = append(activeTasks, sendTask)

Expand Down Expand Up @@ -76,7 +92,7 @@ func StartTasks(ctx context.Context, dependencies *deps.Deps) (*harmonytask.Task
{
// Piece handling
if cfg.Subsystems.EnableParkPiece {
parkPieceTask := piece.NewParkPieceTask(db, must.One(slrLazy.Val()), cfg.Subsystems.ParkPieceMaxTasks)
parkPieceTask := piece.NewParkPieceTask(db, must.One(slrLazy.Val()), cfg.Subsystems.ParkPieceMaxTasks, miners)
LexLuthr marked this conversation as resolved.
Show resolved Hide resolved
cleanupPieceTask := piece.NewCleanupPieceTask(db, must.One(slrLazy.Val()), 0)
activeTasks = append(activeTasks, parkPieceTask, cleanupPieceTask)
}
Expand Down
2 changes: 1 addition & 1 deletion curiosrc/ffi/task_storage.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,7 @@ func (sb *SealCalls) Storage(taskToSectorRef func(taskID harmonytask.TaskID) (Se
func (t *TaskStorage) HasCapacity() bool {
ctx := context.Background()

paths, err := t.sc.sectors.sindex.StorageBestAlloc(ctx, t.alloc, t.ssize, t.pathType)
paths, err := t.sc.sectors.sindex.StorageBestAlloc(ctx, t.alloc, t.ssize, t.pathType, abi.ActorID(0))
LexLuthr marked this conversation as resolved.
Show resolved Hide resolved
if err != nil {
log.Errorf("finding best alloc in HasCapacity: %+v", err)
return false
Expand Down
5 changes: 4 additions & 1 deletion curiosrc/piece/task_park_piece.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,8 @@ import (
logging "github.com/ipfs/go-log/v2"
"golang.org/x/xerrors"

"github.com/filecoin-project/go-address"

"github.com/filecoin-project/lotus/curiosrc/ffi"
"github.com/filecoin-project/lotus/curiosrc/seal"
"github.com/filecoin-project/lotus/lib/harmony/harmonydb"
Expand All @@ -32,13 +34,14 @@ type ParkPieceTask struct {
max int
}

func NewParkPieceTask(db *harmonydb.DB, sc *ffi.SealCalls, max int) *ParkPieceTask {
func NewParkPieceTask(db *harmonydb.DB, sc *ffi.SealCalls, max int, miners []address.Address) *ParkPieceTask {
LexLuthr marked this conversation as resolved.
Show resolved Hide resolved
pt := &ParkPieceTask{
db: db,
sc: sc,

max: max,
}

go pt.pollPieceTasks(context.Background())
return pt
}
Expand Down
8 changes: 6 additions & 2 deletions curiosrc/web/hapi/simpleinfo.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -393,6 +393,8 @@ type machineInfo struct {
FSAvailable int64
Reserved int64
Used int64
AllowMiners string
DenyMiners string
LastHeartbeat time.Time
HeartbeatErr *string

Expand Down Expand Up @@ -458,7 +460,7 @@ func (a *app) clusterNodeInfo(ctx context.Context, id int64) (*machineInfo, erro
}

// query storage info
rows2, err := a.db.Query(ctx, "SELECT storage_id, weight, max_storage, can_seal, can_store, groups, allow_to, allow_types, deny_types, capacity, available, fs_available, reserved, used, last_heartbeat, heartbeat_err FROM storage_path WHERE urls LIKE '%' || $1 || '%'", summaries[0].Info.Host)
rows2, err := a.db.Query(ctx, "SELECT storage_id, weight, max_storage, can_seal, can_store, groups, allow_to, allow_types, deny_types, capacity, available, fs_available, reserved, used, allow_miners, deny_miners, last_heartbeat, heartbeat_err FROM storage_path WHERE urls LIKE '%' || $1 || '%'", summaries[0].Info.Host)
if err != nil {
return nil, err
}
Expand All @@ -481,13 +483,15 @@ func (a *app) clusterNodeInfo(ctx context.Context, id int64) (*machineInfo, erro
FSAvailable int64
Reserved int64
Used int64
AllowMiners string
DenyMiners string
LastHeartbeat time.Time
HeartbeatErr *string

UsedPercent float64
ReservedPercent float64
}
if err := rows2.Scan(&s.ID, &s.Weight, &s.MaxStorage, &s.CanSeal, &s.CanStore, &s.Groups, &s.AllowTo, &s.AllowTypes, &s.DenyTypes, &s.Capacity, &s.Available, &s.FSAvailable, &s.Reserved, &s.Used, &s.LastHeartbeat, &s.HeartbeatErr); err != nil {
if err := rows2.Scan(&s.ID, &s.Weight, &s.MaxStorage, &s.CanSeal, &s.CanStore, &s.Groups, &s.AllowTo, &s.AllowTypes, &s.DenyTypes, &s.Capacity, &s.Available, &s.FSAvailable, &s.Reserved, &s.Used, &s.AllowMiners, &s.DenyMiners, &s.LastHeartbeat, &s.HeartbeatErr); err != nil {
return nil, err
}

Expand Down
12 changes: 12 additions & 0 deletions documentation/en/api-v0-methods-curio.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -179,6 +179,12 @@ Response:
],
"DenyTypes": [
"string value"
],
"AllowMiners": [
"string value"
],
"DenyMiners": [
"string value"
]
}
]
Expand Down Expand Up @@ -218,6 +224,12 @@ Response:
],
"DenyTypes": [
"string value"
],
"AllowMiners": [
"string value"
],
"DenyMiners": [
"string value"
]
}
```
Expand Down
Loading
Loading