Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

feat: curio: storage filter based on miner ID #11801

Merged
merged 4 commits into from
Apr 4, 2024
Merged

feat: curio: storage filter based on miner ID #11801

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
accc09d
add miner fliter
LexLuthr Apr 1, 2024
4647d08
refactor filter decision code
LexLuthr Apr 2, 2024
ad8ee42
refactor MinerFilter func, update comments
LexLuthr Apr 4, 2024
addcb87
apply suggestions
LexLuthr Apr 4, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 5 additions & 5 deletions api/api_storage.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -200,11 +200,11 @@ type StorageMiner interface {
// StorageBestAlloc returns list of paths where sector files of the specified type can be allocated, ordered by preference.
// Paths with more weight and more % of free space are preferred.
// Note: This method doesn't filter paths based on AllowTypes/DenyTypes.
StorageBestAlloc(ctx context.Context, allocate storiface.SectorFileType, ssize abi.SectorSize, pathType storiface.PathType) ([]storiface.StorageInfo, error) //perm:admin
StorageLock(ctx context.Context, sector abi.SectorID, read storiface.SectorFileType, write storiface.SectorFileType) error //perm:admin
StorageTryLock(ctx context.Context, sector abi.SectorID, read storiface.SectorFileType, write storiface.SectorFileType) (bool, error) //perm:admin
StorageList(ctx context.Context) (map[storiface.ID][]storiface.Decl, error) //perm:admin
StorageGetLocks(ctx context.Context) (storiface.SectorLocks, error) //perm:admin
StorageBestAlloc(ctx context.Context, allocate storiface.SectorFileType, ssize abi.SectorSize, pathType storiface.PathType, miner abi.ActorID) ([]storiface.StorageInfo, error) //perm:admin
StorageLock(ctx context.Context, sector abi.SectorID, read storiface.SectorFileType, write storiface.SectorFileType) error //perm:admin
StorageTryLock(ctx context.Context, sector abi.SectorID, read storiface.SectorFileType, write storiface.SectorFileType) (bool, error) //perm:admin
StorageList(ctx context.Context) (map[storiface.ID][]storiface.Decl, error) //perm:admin
StorageGetLocks(ctx context.Context) (storiface.SectorLocks, error) //perm:admin

StorageLocal(ctx context.Context) (map[storiface.ID]string, error) //perm:admin
StorageStat(ctx context.Context, id storiface.ID) (fsutil.FsStat, error) //perm:admin
Expand Down
8 changes: 4 additions & 4 deletions api/proxy_gen.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

91 changes: 90 additions & 1 deletion build/openrpc/miner.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8897,11 +8897,23 @@
],
"DenyTypes": [
"string value"
],
"AllowMiners": [
"string value"
],
"DenyMiners": [
"string value"
]
}
],
"additionalProperties": false,
"properties": {
"AllowMiners": {
"items": {
"type": "string"
},
"type": "array"
},
"AllowTo": {
"items": {
"type": "string"
Expand All @@ -8920,6 +8932,12 @@
"CanStore": {
"type": "boolean"
},
"DenyMiners": {
"items": {
"type": "string"
},
"type": "array"
},
"DenyTypes": {
"items": {
"type": "string"
Expand Down Expand Up @@ -9078,7 +9096,7 @@
},
{
"name": "Filecoin.StorageBestAlloc",
"description": "```go\nfunc (s *StorageMinerStruct) StorageBestAlloc(p0 context.Context, p1 storiface.SectorFileType, p2 abi.SectorSize, p3 storiface.PathType) ([]storiface.StorageInfo, error) {\n\tif s.Internal.StorageBestAlloc == nil {\n\t\treturn *new([]storiface.StorageInfo), ErrNotSupported\n\t}\n\treturn s.Internal.StorageBestAlloc(p0, p1, p2, p3)\n}\n```",
"description": "```go\nfunc (s *StorageMinerStruct) StorageBestAlloc(p0 context.Context, p1 storiface.SectorFileType, p2 abi.SectorSize, p3 storiface.PathType, p4 abi.ActorID) ([]storiface.StorageInfo, error) {\n\tif s.Internal.StorageBestAlloc == nil {\n\t\treturn *new([]storiface.StorageInfo), ErrNotSupported\n\t}\n\treturn s.Internal.StorageBestAlloc(p0, p1, p2, p3, p4)\n}\n```",
"summary": "StorageBestAlloc returns list of paths where sector files of the specified type can be allocated, ordered by preference.\nPaths with more weight and more % of free space are preferred.\nNote: This method doesn't filter paths based on AllowTypes/DenyTypes.\n",
"paramStructure": "by-position",
"params": [
Expand Down Expand Up @@ -9130,6 +9148,23 @@
},
"required": true,
"deprecated": false
},
{
"name": "p4",
"description": "abi.ActorID",
"summary": "",
"schema": {
"title": "number",
"description": "Number is a number",
"examples": [
1000
],
"type": [
"number"
]
},
"required": true,
"deprecated": false
}
],
"result": {
Expand Down Expand Up @@ -9159,6 +9194,12 @@
],
"DenyTypes": [
"string value"
],
"AllowMiners": [
"string value"
],
"DenyMiners": [
"string value"
]
}
]
Expand All @@ -9167,6 +9208,12 @@
{
"additionalProperties": false,
"properties": {
"AllowMiners": {
"items": {
"type": "string"
},
"type": "array"
},
"AllowTo": {
"items": {
"type": "string"
Expand All @@ -9185,6 +9232,12 @@
"CanStore": {
"type": "boolean"
},
"DenyMiners": {
"items": {
"type": "string"
},
"type": "array"
},
"DenyTypes": {
"items": {
"type": "string"
Expand Down Expand Up @@ -9620,6 +9673,12 @@
],
"DenyTypes": [
"string value"
],
"AllowMiners": [
"string value"
],
"DenyMiners": [
"string value"
]
}
]
Expand All @@ -9628,6 +9687,12 @@
{
"additionalProperties": false,
"properties": {
"AllowMiners": {
"items": {
"type": "string"
},
"type": "array"
},
"AllowTypes": {
"items": {
"type": "string"
Expand All @@ -9646,6 +9711,12 @@
"CanStore": {
"type": "boolean"
},
"DenyMiners": {
"items": {
"type": "string"
},
"type": "array"
},
"DenyTypes": {
"items": {
"type": "string"
Expand Down Expand Up @@ -9833,11 +9904,23 @@
],
"DenyTypes": [
"string value"
],
"AllowMiners": [
"string value"
],
"DenyMiners": [
"string value"
]
}
],
"additionalProperties": false,
"properties": {
"AllowMiners": {
"items": {
"type": "string"
},
"type": "array"
},
"AllowTo": {
"items": {
"type": "string"
Expand All @@ -9856,6 +9939,12 @@
"CanStore": {
"type": "boolean"
},
"DenyMiners": {
"items": {
"type": "string"
},
"type": "array"
},
"DenyTypes": {
"items": {
"type": "string"
Expand Down
3 changes: 2 additions & 1 deletion curiosrc/ffi/task_storage.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import (

"github.com/filecoin-project/lotus/lib/harmony/harmonytask"
"github.com/filecoin-project/lotus/lib/harmony/resources"
storagePaths "github.com/filecoin-project/lotus/storage/paths"
"github.com/filecoin-project/lotus/storage/sealer/storiface"
)

Expand Down Expand Up @@ -68,7 +69,7 @@ func (sb *SealCalls) Storage(taskToSectorRef func(taskID harmonytask.TaskID) (Se
func (t *TaskStorage) HasCapacity() bool {
ctx := context.Background()

paths, err := t.sc.sectors.sindex.StorageBestAlloc(ctx, t.alloc, t.ssize, t.pathType)
paths, err := t.sc.sectors.sindex.StorageBestAlloc(ctx, t.alloc, t.ssize, t.pathType, storagePaths.NoMinerFilter)
if err != nil {
log.Errorf("finding best alloc in HasCapacity: %+v", err)
return false
Expand Down
1 change: 1 addition & 0 deletions curiosrc/piece/task_park_piece.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ func NewParkPieceTask(db *harmonydb.DB, sc *ffi.SealCalls, max int) *ParkPieceTa

max: max,
}

go pt.pollPieceTasks(context.Background())
return pt
}
Expand Down
8 changes: 6 additions & 2 deletions curiosrc/web/hapi/simpleinfo.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -393,6 +393,8 @@ type machineInfo struct {
FSAvailable int64
Reserved int64
Used int64
AllowMiners string
DenyMiners string
LastHeartbeat time.Time
HeartbeatErr *string

Expand Down Expand Up @@ -458,7 +460,7 @@ func (a *app) clusterNodeInfo(ctx context.Context, id int64) (*machineInfo, erro
}

// query storage info
rows2, err := a.db.Query(ctx, "SELECT storage_id, weight, max_storage, can_seal, can_store, groups, allow_to, allow_types, deny_types, capacity, available, fs_available, reserved, used, last_heartbeat, heartbeat_err FROM storage_path WHERE urls LIKE '%' || $1 || '%'", summaries[0].Info.Host)
rows2, err := a.db.Query(ctx, "SELECT storage_id, weight, max_storage, can_seal, can_store, groups, allow_to, allow_types, deny_types, capacity, available, fs_available, reserved, used, allow_miners, deny_miners, last_heartbeat, heartbeat_err FROM storage_path WHERE urls LIKE '%' || $1 || '%'", summaries[0].Info.Host)
if err != nil {
return nil, err
}
Expand All @@ -481,13 +483,15 @@ func (a *app) clusterNodeInfo(ctx context.Context, id int64) (*machineInfo, erro
FSAvailable int64
Reserved int64
Used int64
AllowMiners string
DenyMiners string
LastHeartbeat time.Time
HeartbeatErr *string

UsedPercent float64
ReservedPercent float64
}
if err := rows2.Scan(&s.ID, &s.Weight, &s.MaxStorage, &s.CanSeal, &s.CanStore, &s.Groups, &s.AllowTo, &s.AllowTypes, &s.DenyTypes, &s.Capacity, &s.Available, &s.FSAvailable, &s.Reserved, &s.Used, &s.LastHeartbeat, &s.HeartbeatErr); err != nil {
if err := rows2.Scan(&s.ID, &s.Weight, &s.MaxStorage, &s.CanSeal, &s.CanStore, &s.Groups, &s.AllowTo, &s.AllowTypes, &s.DenyTypes, &s.Capacity, &s.Available, &s.FSAvailable, &s.Reserved, &s.Used, &s.AllowMiners, &s.DenyMiners, &s.LastHeartbeat, &s.HeartbeatErr); err != nil {
return nil, err
}

Expand Down
12 changes: 12 additions & 0 deletions documentation/en/api-v0-methods-curio.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -179,6 +179,12 @@ Response:
],
"DenyTypes": [
"string value"
],
"AllowMiners": [
"string value"
],
"DenyMiners": [
"string value"
]
}
]
Expand Down Expand Up @@ -218,6 +224,12 @@ Response:
],
"DenyTypes": [
"string value"
],
"AllowMiners": [
"string value"
],
"DenyMiners": [
"string value"
]
}
```
Expand Down
27 changes: 26 additions & 1 deletion documentation/en/api-v0-methods-miner.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3735,6 +3735,12 @@ Inputs:
],
"DenyTypes": [
"string value"
],
"AllowMiners": [
"string value"
],
"DenyMiners": [
"string value"
]
},
{
Expand Down Expand Up @@ -3782,7 +3788,8 @@ Inputs:
[
1,
34359738368,
"sealing"
"sealing",
1000
]
```

Expand All @@ -3809,6 +3816,12 @@ Response:
],
"DenyTypes": [
"string value"
],
"AllowMiners": [
"string value"
],
"DenyMiners": [
"string value"
]
}
]
Expand Down Expand Up @@ -3928,6 +3941,12 @@ Response:
],
"DenyTypes": [
"string value"
],
"AllowMiners": [
"string value"
],
"DenyMiners": [
"string value"
]
}
]
Expand Down Expand Up @@ -4004,6 +4023,12 @@ Response:
],
"DenyTypes": [
"string value"
],
"AllowMiners": [
"string value"
],
"DenyMiners": [
"string value"
]
}
```
Expand Down
2 changes: 2 additions & 0 deletions lib/harmony/harmonydb/sql/20240401-storage-miner-filter.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
ALTER TABLE storage_path ADD COLUMN IF NOT EXISTS allow_miners varchar DEFAULT '';
ALTER TABLE storage_path ADD COLUMN IF NOT EXISTS deny_miners varchar DEFAULT ''; -- comma separated list of miner addresses
Loading