diff --git a/go.mod b/go.mod index 9b6c7159b39..7655e0f331e 100644 --- a/go.mod +++ b/go.mod @@ -19,7 +19,7 @@ require ( github.com/multiversx/mx-chain-es-indexer-go v1.4.18-0.20231228064619-e3b0caf29058 github.com/multiversx/mx-chain-logger-go v1.0.14-0.20231215125130-a3bed6e76040 github.com/multiversx/mx-chain-scenario-go v1.2.2-0.20231129113427-ad3056f45296 - github.com/multiversx/mx-chain-storage-go v1.0.15-0.20240103193554-5ad54212812d + github.com/multiversx/mx-chain-storage-go v1.0.15-0.20240131142608-5c126467749c github.com/multiversx/mx-chain-vm-common-go v1.5.10-0.20231228070003-ae14e1e0adfa github.com/multiversx/mx-chain-vm-go v1.5.23-0.20231228064104-964359cb8dd3 github.com/multiversx/mx-chain-vm-v1_2-go v1.2.65-0.20231228071026-eed2cb19c216 diff --git a/go.sum b/go.sum index aebf8ac5ff3..64e35192dc1 100644 --- a/go.sum +++ b/go.sum @@ -403,6 +403,8 @@ github.com/multiversx/mx-chain-storage-go v1.0.15-0.20231213110622-e222ba96a9f4 github.com/multiversx/mx-chain-storage-go v1.0.15-0.20231213110622-e222ba96a9f4/go.mod h1:ioCT2oHQ+TyHQYpgjxzlUdy7dCdv56+w5HnBg9z96eY= github.com/multiversx/mx-chain-storage-go v1.0.15-0.20240103193554-5ad54212812d h1:mNf2qlDGSNp6yd4rSJBT93vGseuqraj8/jWWXm1ro+k= github.com/multiversx/mx-chain-storage-go v1.0.15-0.20240103193554-5ad54212812d/go.mod h1:ioCT2oHQ+TyHQYpgjxzlUdy7dCdv56+w5HnBg9z96eY= +github.com/multiversx/mx-chain-storage-go v1.0.15-0.20240131142608-5c126467749c h1:Fr0PM4Kh33QqTHyIqzRQqx049zNvmeKKSCxCFfB/JK4= +github.com/multiversx/mx-chain-storage-go v1.0.15-0.20240131142608-5c126467749c/go.mod h1:ioCT2oHQ+TyHQYpgjxzlUdy7dCdv56+w5HnBg9z96eY= github.com/multiversx/mx-chain-vm-common-go v1.5.10-0.20231228070003-ae14e1e0adfa h1:xdDeUC4yOfiUwctkYioYMjjigBZoZo5RZq1e5WoCVRs= github.com/multiversx/mx-chain-vm-common-go v1.5.10-0.20231228070003-ae14e1e0adfa/go.mod h1:7jjGRykSfLeMs6iQdszlE0lGK2xp9/cctiVdeKbQLLM= github.com/multiversx/mx-chain-vm-go v1.5.23-0.20231228064104-964359cb8dd3 h1:qfzeTPI2oSgxnw52KiVWc2fHMem6FZIkX1Azwy64098= diff --git a/storage/constants.go b/storage/constants.go index b78021138c7..9cd37571521 100644 --- a/storage/constants.go +++ b/storage/constants.go @@ -1,14 +1,14 @@ package storage import ( - "github.com/multiversx/mx-chain-storage-go/storageUnit" + "github.com/multiversx/mx-chain-storage-go/common" ) // MaxRetriesToCreateDB represents the maximum number of times to try to create DB if it failed -const MaxRetriesToCreateDB = storageUnit.MaxRetriesToCreateDB +const MaxRetriesToCreateDB = common.MaxRetriesToCreateDB // SleepTimeBetweenCreateDBRetries represents the number of seconds to sleep between DB creates -const SleepTimeBetweenCreateDBRetries = storageUnit.SleepTimeBetweenCreateDBRetries +const SleepTimeBetweenCreateDBRetries = common.SleepTimeBetweenCreateDBRetries // PathShardPlaceholder represents the placeholder for the shard ID in paths const PathShardPlaceholder = "[S]" diff --git a/storage/factory/persisterCreator.go b/storage/factory/persisterCreator.go index 1357fc37ae4..13398c38a5c 100644 --- a/storage/factory/persisterCreator.go +++ b/storage/factory/persisterCreator.go @@ -5,6 +5,7 @@ import ( "github.com/multiversx/mx-chain-go/storage" "github.com/multiversx/mx-chain-go/storage/database" "github.com/multiversx/mx-chain-go/storage/storageunit" + "github.com/multiversx/mx-chain-storage-go/factory" ) const minNumShards = 2 @@ -51,16 +52,16 @@ func (pc *persisterCreator) Create(path string) (storage.Persister, error) { // CreateBasePersister will create base the persister for the provided path func (pc *persisterCreator) CreateBasePersister(path string) (storage.Persister, error) { var dbType = storageunit.DBType(pc.dbType) - switch dbType { - case storageunit.LvlDB: - return database.NewLevelDB(path, pc.batchDelaySeconds, pc.maxBatchSize, pc.maxOpenFiles) - case storageunit.LvlDBSerial: - return database.NewSerialDB(path, pc.batchDelaySeconds, pc.maxBatchSize, pc.maxOpenFiles) - case storageunit.MemoryDB: - return database.NewMemDB(), nil - default: - return nil, storage.ErrNotSupportedDBType + + argsDB := factory.ArgDB{ + DBType: dbType, + Path: path, + BatchDelaySeconds: pc.batchDelaySeconds, + MaxBatchSize: pc.maxBatchSize, + MaxOpenFiles: pc.maxOpenFiles, } + + return storageunit.NewDB(argsDB) } func (pc *persisterCreator) createShardIDProvider() (storage.ShardIDProvider, error) { diff --git a/storage/factory/persisterFactory_test.go b/storage/factory/persisterFactory_test.go index 860331a22bc..7dd1f987510 100644 --- a/storage/factory/persisterFactory_test.go +++ b/storage/factory/persisterFactory_test.go @@ -3,11 +3,14 @@ package factory_test import ( "fmt" "os" + "path" "testing" + "github.com/multiversx/mx-chain-core-go/core/check" "github.com/multiversx/mx-chain-go/storage" "github.com/multiversx/mx-chain-go/storage/factory" "github.com/multiversx/mx-chain-go/storage/storageunit" + "github.com/multiversx/mx-chain-storage-go/common" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -46,6 +49,41 @@ func TestPersisterFactory_Create(t *testing.T) { }) } +func TestPersisterFactory_CreateWithRetries(t *testing.T) { + t.Parallel() + + t.Run("wrong config should error", func(t *testing.T) { + t.Parallel() + + path := "TEST" + dbConfig := createDefaultDBConfig() + dbConfig.Type = "invalid type" + + persisterFactory, err := factory.NewPersisterFactory(dbConfig) + assert.Nil(t, err) + + db, err := persisterFactory.CreateWithRetries(path) + assert.True(t, check.IfNil(db)) + assert.Equal(t, common.ErrNotSupportedDBType, err) + }) + + t.Run("should work", func(t *testing.T) { + t.Parallel() + + path := path.Join(t.TempDir(), "TEST") + dbConfig := createDefaultDBConfig() + dbConfig.FilePath = path + + persisterFactory, err := factory.NewPersisterFactory(dbConfig) + assert.Nil(t, err) + + db, err := persisterFactory.CreateWithRetries(path) + assert.False(t, check.IfNil(db)) + assert.Nil(t, err) + _ = db.Close() + }) +} + func TestPersisterFactory_Create_ConfigSaveToFilePath(t *testing.T) { t.Parallel() diff --git a/storage/storageunit/constants.go b/storage/storageunit/constants.go index 0e128af8123..022715dbcb7 100644 --- a/storage/storageunit/constants.go +++ b/storage/storageunit/constants.go @@ -1,25 +1,27 @@ package storageunit -import "github.com/multiversx/mx-chain-storage-go/storageUnit" +import ( + "github.com/multiversx/mx-chain-storage-go/common" +) const ( // LRUCache defines a cache identifier with least-recently-used eviction mechanism - LRUCache = storageUnit.LRUCache + LRUCache = common.LRUCache // SizeLRUCache defines a cache identifier with least-recently-used eviction mechanism and fixed size in bytes - SizeLRUCache = storageUnit.SizeLRUCache + SizeLRUCache = common.SizeLRUCache ) // DB types that are currently supported const ( // LvlDB represents a levelDB storage identifier - LvlDB = storageUnit.LvlDB + LvlDB = common.LvlDB // LvlDBSerial represents a levelDB storage with serialized operations identifier - LvlDBSerial = storageUnit.LvlDBSerial + LvlDBSerial = common.LvlDBSerial // MemoryDB represents an in memory storage identifier - MemoryDB = storageUnit.MemoryDB + MemoryDB = common.MemoryDB ) // Shard id provider types that are currently supported const ( - BinarySplit = storageUnit.BinarySplit + BinarySplit = common.BinarySplit ) diff --git a/storage/storageunit/storageunit.go b/storage/storageunit/storageunit.go index 2a9e390b725..c1944777920 100644 --- a/storage/storageunit/storageunit.go +++ b/storage/storageunit/storageunit.go @@ -3,6 +3,8 @@ package storageunit import ( "github.com/multiversx/mx-chain-core-go/marshal" "github.com/multiversx/mx-chain-go/storage" + "github.com/multiversx/mx-chain-storage-go/common" + "github.com/multiversx/mx-chain-storage-go/factory" "github.com/multiversx/mx-chain-storage-go/storageCacherAdapter" "github.com/multiversx/mx-chain-storage-go/storageUnit" ) @@ -12,22 +14,25 @@ import ( type Unit = storageUnit.Unit // CacheConfig holds the configurable elements of a cache -type CacheConfig = storageUnit.CacheConfig +type CacheConfig = common.CacheConfig // DBConfig holds the configurable elements of a database -type DBConfig = storageUnit.DBConfig +type DBConfig = common.DBConfig // NilStorer resembles a disabled implementation of the Storer interface type NilStorer = storageUnit.NilStorer // CacheType represents the type of the supported caches -type CacheType = storageUnit.CacheType +type CacheType = common.CacheType // DBType represents the type of the supported databases -type DBType = storageUnit.DBType +type DBType = common.DBType // ShardIDProviderType represents the type of the supported shard id providers -type ShardIDProviderType = storageUnit.ShardIDProviderType +type ShardIDProviderType = common.ShardIDProviderType + +// ArgDB is a structure that is used to create a new storage.Persister implementation +type ArgDB = factory.ArgDB // NewStorageUnit is the constructor for the storage unit, creating a new storage unit // from the given cacher and persister. @@ -37,12 +42,31 @@ func NewStorageUnit(c storage.Cacher, p storage.Persister) (*Unit, error) { // NewCache creates a new cache from a cache config func NewCache(config CacheConfig) (storage.Cacher, error) { - return storageUnit.NewCache(config) + return factory.NewCache(config) +} + +// NewDB creates a new database from database config +func NewDB(args ArgDB) (storage.Persister, error) { + return factory.NewDB(args) } // NewStorageUnitFromConf creates a new storage unit from a storage unit config func NewStorageUnitFromConf(cacheConf CacheConfig, dbConf DBConfig, persisterFactory storage.PersisterFactoryHandler) (*Unit, error) { - return storageUnit.NewStorageUnitFromConf(cacheConf, dbConf, persisterFactory) + if dbConf.MaxBatchSize > int(cacheConf.Capacity) { + return nil, common.ErrCacheSizeIsLowerThanBatchSize + } + + cache, err := NewCache(cacheConf) + if err != nil { + return nil, err + } + + db, err := persisterFactory.CreateWithRetries(dbConf.FilePath) + if err != nil { + return nil, err + } + + return NewStorageUnit(cache, db) } // NewNilStorer will return a nil storer diff --git a/storage/storageunit/storageunit_test.go b/storage/storageunit/storageunit_test.go index 0652f25b33c..da4aea63b33 100644 --- a/storage/storageunit/storageunit_test.go +++ b/storage/storageunit/storageunit_test.go @@ -72,50 +72,6 @@ func TestNewCache(t *testing.T) { }) } -func TestNewDB(t *testing.T) { - t.Parallel() - - t.Run("wrong config should error", func(t *testing.T) { - t.Parallel() - - path := "TEST" - dbConfig := config.DBConfig{ - FilePath: path, - Type: "invalid type", - BatchDelaySeconds: 5, - MaxBatchSize: 10, - MaxOpenFiles: 10, - } - - persisterFactory, err := factory.NewPersisterFactory(dbConfig) - assert.Nil(t, err) - - db, err := persisterFactory.CreateWithRetries(path) - assert.True(t, check.IfNil(db)) - assert.Equal(t, common.ErrNotSupportedDBType, err) - }) - t.Run("should work", func(t *testing.T) { - t.Parallel() - - path := path.Join(t.TempDir(), "TEST") - dbConfig := config.DBConfig{ - FilePath: path, - Type: "LvlDBSerial", - BatchDelaySeconds: 5, - MaxBatchSize: 10, - MaxOpenFiles: 10, - } - - persisterFactory, err := factory.NewPersisterFactory(dbConfig) - assert.Nil(t, err) - - db, err := persisterFactory.CreateWithRetries(path) - assert.False(t, check.IfNil(db)) - assert.Nil(t, err) - _ = db.Close() - }) -} - func TestNewStorageUnitFromConf(t *testing.T) { t.Parallel()