diff --git a/dataRetriever/factory/dataPoolFactory.go b/dataRetriever/factory/dataPoolFactory.go index 0033d14f686..6e1415ddfd8 100644 --- a/dataRetriever/factory/dataPoolFactory.go +++ b/dataRetriever/factory/dataPoolFactory.go @@ -2,7 +2,6 @@ package factory import ( "fmt" - "os" "time" "github.com/multiversx/mx-chain-core-go/core" @@ -179,22 +178,12 @@ func createTrieSyncDB(args ArgsDataPool) (storage.Persister, error) { shardId := core.GetShardIDString(args.ShardCoordinator.SelfId()) path := args.PathManager.PathForStatic(shardId, mainConfig.TrieSyncStorage.DB.FilePath) - dbConfigHandler := factory.NewDBConfigHandler(mainConfig.TrieSyncStorage.DB) - persisterFactory, err := factory.NewPersisterFactory(dbConfigHandler) + persisterFactory, err := factory.NewPersisterFactory(mainConfig.TrieSyncStorage.DB) if err != nil { return nil, err } - if mainConfig.TrieSyncStorage.DB.UseTmpAsFilePath { - filePath, errTempDir := os.MkdirTemp("", "trieSyncStorage") - if errTempDir != nil { - return nil, errTempDir - } - - path = filePath - } - - db, err := storageunit.NewDB(persisterFactory, path) + db, err := persisterFactory.CreateWithRetries(path) if err != nil { return nil, fmt.Errorf("%w while creating the db for the trie nodes", err) } diff --git a/epochStart/metachain/systemSCs_test.go b/epochStart/metachain/systemSCs_test.go index a519e77e7f7..f74f9238db9 100644 --- a/epochStart/metachain/systemSCs_test.go +++ b/epochStart/metachain/systemSCs_test.go @@ -87,12 +87,11 @@ func createPhysicalUnit(t *testing.T) (storage.Storer, string) { MaxOpenFiles: 10, } - dbConfigHandler := storageFactory.NewDBConfigHandler(dbConfig) - persisterFactory, err := storageFactory.NewPersisterFactory(dbConfigHandler) + persisterFactory, err := storageFactory.NewPersisterFactory(dbConfig) assert.Nil(t, err) cache, _ := storageunit.NewCache(cacheConfig) - persist, _ := storageunit.NewDB(persisterFactory, dir) + persist, _ := persisterFactory.CreateWithRetries(dir) unit, _ := storageunit.NewStorageUnit(cache, persist) return unit, dir diff --git a/genesis/process/genesisBlockCreator.go b/genesis/process/genesisBlockCreator.go index 2e9b14d7db3..d3fecd2f2d1 100644 --- a/genesis/process/genesisBlockCreator.go +++ b/genesis/process/genesisBlockCreator.go @@ -131,8 +131,7 @@ func createStorer(storageConfig config.StorageConfig, folder string) (storage.St dbConfig := factory.GetDBFromConfig(storageConfig.DB) dbConfig.FilePath = path.Join(folder, storageConfig.DB.FilePath) - dbConfigHandler := factory.NewDBConfigHandler(storageConfig.DB) - persisterFactory, err := factory.NewPersisterFactory(dbConfigHandler) + persisterFactory, err := factory.NewPersisterFactory(storageConfig.DB) if err != nil { return nil, err } diff --git a/go.mod b/go.mod index 69c8b07ca2d..c1e098d9c7d 100644 --- a/go.mod +++ b/go.mod @@ -20,7 +20,7 @@ require ( github.com/multiversx/mx-chain-es-indexer-go v1.4.19-0.20240129150813-a772c480d33a github.com/multiversx/mx-chain-logger-go v1.0.14-0.20240129144507-d00e967c890c github.com/multiversx/mx-chain-scenario-go v1.3.1-0.20240129145446-ca4fba98f6d1 - github.com/multiversx/mx-chain-storage-go v1.0.15-0.20240129144933-b1c0d642d7f8 + github.com/multiversx/mx-chain-storage-go v1.0.15-0.20240304133242-faaf1d20b087 github.com/multiversx/mx-chain-vm-common-go v1.5.12-0.20240129145149-4fe61574f566 github.com/multiversx/mx-chain-vm-go v1.5.27-0.20240129150501-7c828af05c83 github.com/multiversx/mx-chain-vm-v1_2-go v1.2.66-0.20240129145751-f814f5525edb diff --git a/go.sum b/go.sum index 5835957d880..c8be913281d 100644 --- a/go.sum +++ b/go.sum @@ -397,8 +397,8 @@ github.com/multiversx/mx-chain-logger-go v1.0.14-0.20240129144507-d00e967c890c h github.com/multiversx/mx-chain-logger-go v1.0.14-0.20240129144507-d00e967c890c/go.mod h1:fH/fR/GEBsDjPkBoZDVJMoYo2HhlA7++DP6QfITJ1N8= github.com/multiversx/mx-chain-scenario-go v1.3.1-0.20240129145446-ca4fba98f6d1 h1:hkeHftnhRuJoT5FrfF97gEtb5aY351SWEjZPaTb6D+Y= github.com/multiversx/mx-chain-scenario-go v1.3.1-0.20240129145446-ca4fba98f6d1/go.mod h1:ndk45i9J9McuCJpTcgiaK4ocd0yhnBBCPrlFwO6GRcs= -github.com/multiversx/mx-chain-storage-go v1.0.15-0.20240129144933-b1c0d642d7f8 h1:/EYv/HGX0OKbeNFt667J0yZRtuJiZH0lEK8YtobuH/c= -github.com/multiversx/mx-chain-storage-go v1.0.15-0.20240129144933-b1c0d642d7f8/go.mod h1:zl1A6teNe39T8yhdZlkX3ckm5aLYrMIJJZ6Ord1E71M= +github.com/multiversx/mx-chain-storage-go v1.0.15-0.20240304133242-faaf1d20b087 h1:liZ6PL4Audkpkx4vCBngGzC48VZUpjjZd+p2mgarrt0= +github.com/multiversx/mx-chain-storage-go v1.0.15-0.20240304133242-faaf1d20b087/go.mod h1:zl1A6teNe39T8yhdZlkX3ckm5aLYrMIJJZ6Ord1E71M= github.com/multiversx/mx-chain-vm-common-go v1.5.12-0.20240129145149-4fe61574f566 h1:zImJa/r6B5L2OLWbKTn5io53U11PPGDla12H2OaJ9y0= github.com/multiversx/mx-chain-vm-common-go v1.5.12-0.20240129145149-4fe61574f566/go.mod h1:OUyhCFqZKqUk1uaPsenyPDwO1830SlHNDU7Q7b6CBVI= github.com/multiversx/mx-chain-vm-go v1.5.27-0.20240129150501-7c828af05c83 h1:G/d9aplnwP/9MrLE3gcANEpGfn5e8ZZufijPv2XVUfw= diff --git a/integrationTests/vm/wasm/delegation/testRunner.go b/integrationTests/vm/wasm/delegation/testRunner.go index 343f3dace0f..e7bcb516b45 100644 --- a/integrationTests/vm/wasm/delegation/testRunner.go +++ b/integrationTests/vm/wasm/delegation/testRunner.go @@ -53,8 +53,7 @@ func RunDelegationStressTest( MaxBatchSize: 45000, MaxOpenFiles: 10, } - dbConfigHandler := factory.NewDBConfigHandler(dbConfig) - persisterFactory, err := factory.NewPersisterFactory(dbConfigHandler) + persisterFactory, err := factory.NewPersisterFactory(dbConfig) if err != nil { return nil, err } diff --git a/process/smartContract/hooks/blockChainHook.go b/process/smartContract/hooks/blockChainHook.go index 827d08da435..18d0dac3d7f 100644 --- a/process/smartContract/hooks/blockChainHook.go +++ b/process/smartContract/hooks/blockChainHook.go @@ -826,8 +826,7 @@ func (bh *BlockChainHookImpl) makeCompiledSCStorage() error { dbConfig := factory.GetDBFromConfig(bh.configSCStorage.DB) dbConfig.FilePath = path.Join(bh.workingDir, defaultCompiledSCPath, bh.configSCStorage.DB.FilePath) - dbConfigHandler := factory.NewDBConfigHandler(bh.configSCStorage.DB) - persisterFactory, err := factory.NewPersisterFactory(dbConfigHandler) + persisterFactory, err := factory.NewPersisterFactory(bh.configSCStorage.DB) if err != nil { return err } diff --git a/storage/constants.go b/storage/constants.go index b78021138c7..9cd37571521 100644 --- a/storage/constants.go +++ b/storage/constants.go @@ -1,14 +1,14 @@ package storage import ( - "github.com/multiversx/mx-chain-storage-go/storageUnit" + "github.com/multiversx/mx-chain-storage-go/common" ) // MaxRetriesToCreateDB represents the maximum number of times to try to create DB if it failed -const MaxRetriesToCreateDB = storageUnit.MaxRetriesToCreateDB +const MaxRetriesToCreateDB = common.MaxRetriesToCreateDB // SleepTimeBetweenCreateDBRetries represents the number of seconds to sleep between DB creates -const SleepTimeBetweenCreateDBRetries = storageUnit.SleepTimeBetweenCreateDBRetries +const SleepTimeBetweenCreateDBRetries = common.SleepTimeBetweenCreateDBRetries // PathShardPlaceholder represents the placeholder for the shard ID in paths const PathShardPlaceholder = "[S]" diff --git a/storage/factory/dbConfigHandler.go b/storage/factory/dbConfigHandler.go index 2e5a611f293..468c42a2ee7 100644 --- a/storage/factory/dbConfigHandler.go +++ b/storage/factory/dbConfigHandler.go @@ -11,8 +11,12 @@ import ( ) const ( - dbConfigFileName = "config.toml" - defaultType = "LvlDBSerial" + dbConfigFileName = "config.toml" + defaultType = "LvlDBSerial" + defaultBatchDelaySeconds = 2 + defaultMaxBatchSize = 100 + defaultMaxOpenFiles = 10 + defaultUseTmpAsFilePath = false ) var ( @@ -20,23 +24,13 @@ var ( ) type dbConfigHandler struct { - dbType string - batchDelaySeconds int - maxBatchSize int - maxOpenFiles int - shardIDProviderType string - numShards int32 + conf config.DBConfig } // NewDBConfigHandler will create a new db config handler instance func NewDBConfigHandler(config config.DBConfig) *dbConfigHandler { return &dbConfigHandler{ - dbType: config.Type, - batchDelaySeconds: config.BatchDelaySeconds, - maxBatchSize: config.MaxBatchSize, - maxOpenFiles: config.MaxOpenFiles, - shardIDProviderType: config.ShardIDProviderType, - numShards: config.NumShards, + conf: config, } } @@ -56,9 +50,10 @@ func (dh *dbConfigHandler) GetDBConfig(path string) (*config.DBConfig, error) { if !empty { dbConfig := &config.DBConfig{ Type: defaultType, - BatchDelaySeconds: dh.batchDelaySeconds, - MaxBatchSize: dh.maxBatchSize, - MaxOpenFiles: dh.maxOpenFiles, + BatchDelaySeconds: dh.conf.BatchDelaySeconds, + MaxBatchSize: dh.conf.MaxBatchSize, + MaxOpenFiles: dh.conf.MaxOpenFiles, + UseTmpAsFilePath: dh.conf.UseTmpAsFilePath, } log.Debug("GetDBConfig: loaded default db config", @@ -68,20 +63,11 @@ func (dh *dbConfigHandler) GetDBConfig(path string) (*config.DBConfig, error) { return dbConfig, nil } - dbConfig := &config.DBConfig{ - Type: dh.dbType, - BatchDelaySeconds: dh.batchDelaySeconds, - MaxBatchSize: dh.maxBatchSize, - MaxOpenFiles: dh.maxOpenFiles, - ShardIDProviderType: dh.shardIDProviderType, - NumShards: dh.numShards, - } - log.Debug("GetDBConfig: loaded db config from main config file", - "configuration", fmt.Sprintf("%+v", dbConfig), + "configuration", fmt.Sprintf("%+v", dh.conf), ) - return dbConfig, nil + return &dh.conf, nil } func readCorrectConfigurationFromToml(dbConfig *config.DBConfig, filePath string) error { diff --git a/storage/factory/export_test.go b/storage/factory/export_test.go index 177bc97358c..854f52e9cc3 100644 --- a/storage/factory/export_test.go +++ b/storage/factory/export_test.go @@ -25,3 +25,8 @@ func NewPersisterCreator(config config.DBConfig) *persisterCreator { func (pc *persisterCreator) CreateShardIDProvider() (storage.ShardIDProvider, error) { return pc.createShardIDProvider() } + +// GetTmpFilePath - +func GetTmpFilePath(path string) (string, error) { + return getTmpFilePath(path) +} diff --git a/storage/factory/openStorage.go b/storage/factory/openStorage.go index 80dae5bc39c..0effada6f04 100644 --- a/storage/factory/openStorage.go +++ b/storage/factory/openStorage.go @@ -3,7 +3,6 @@ package factory import ( "fmt" "path/filepath" - "time" "github.com/multiversx/mx-chain-core-go/core/check" "github.com/multiversx/mx-chain-go/config" @@ -56,8 +55,7 @@ func (o *openStorageUnits) GetMostRecentStorageUnit(dbConfig config.DBConfig) (s return nil, err } - dbConfigHandler := NewDBConfigHandler(dbConfig) - persisterFactory, err := NewPersisterFactory(dbConfigHandler) + persisterFactory, err := NewPersisterFactory(dbConfig) if err != nil { return nil, err } @@ -74,7 +72,7 @@ func (o *openStorageUnits) GetMostRecentStorageUnit(dbConfig config.DBConfig) (s persisterPath := o.getPersisterPath(pathWithoutShard, mostRecentShard, dbConfig) - persister, err := createDB(persisterFactory, persisterPath) + persister, err := persisterFactory.CreateWithRetries(persisterPath) if err != nil { return nil, err } @@ -112,13 +110,12 @@ func (o *openStorageUnits) OpenDB(dbConfig config.DBConfig, shardID uint32, epoc parentDir := o.latestStorageDataProvider.GetParentDirectory() pathWithoutShard := o.getPathWithoutShard(parentDir, epoch) persisterPath := o.getPersisterPath(pathWithoutShard, fmt.Sprintf("%d", shardID), dbConfig) - dbConfigHandler := NewDBConfigHandler(dbConfig) - persisterFactory, err := NewPersisterFactory(dbConfigHandler) + persisterFactory, err := NewPersisterFactory(dbConfig) if err != nil { return nil, err } - persister, err := createDB(persisterFactory, persisterPath) + persister, err := persisterFactory.CreateWithRetries(persisterPath) if err != nil { return nil, err } @@ -131,21 +128,6 @@ func (o *openStorageUnits) OpenDB(dbConfig config.DBConfig, shardID uint32, epoc return storageunit.NewStorageUnit(lruCache, persister) } -func createDB(persisterFactory *PersisterFactory, persisterPath string) (storage.Persister, error) { - var persister storage.Persister - var err error - for i := 0; i < storage.MaxRetriesToCreateDB; i++ { - persister, err = persisterFactory.Create(persisterPath) - if err == nil { - return persister, nil - } - log.Warn("Create Persister failed", "path", persisterPath, "error", err) - //TODO: extract this in a parameter and inject it - time.Sleep(storage.SleepTimeBetweenCreateDBRetries) - } - return nil, err -} - func (o *openStorageUnits) getMostUpToDateDirectory( dbConfig config.DBConfig, pathWithoutShard string, diff --git a/storage/factory/persisterCreator.go b/storage/factory/persisterCreator.go index 1357fc37ae4..0d17287815e 100644 --- a/storage/factory/persisterCreator.go +++ b/storage/factory/persisterCreator.go @@ -5,39 +5,29 @@ import ( "github.com/multiversx/mx-chain-go/storage" "github.com/multiversx/mx-chain-go/storage/database" "github.com/multiversx/mx-chain-go/storage/storageunit" + "github.com/multiversx/mx-chain-storage-go/factory" ) const minNumShards = 2 // persisterCreator is the factory which will handle creating new persisters type persisterCreator struct { - dbType string - batchDelaySeconds int - maxBatchSize int - maxOpenFiles int - shardIDProviderType string - numShards int32 + conf config.DBConfig } func newPersisterCreator(config config.DBConfig) *persisterCreator { return &persisterCreator{ - dbType: config.Type, - batchDelaySeconds: config.BatchDelaySeconds, - maxBatchSize: config.MaxBatchSize, - maxOpenFiles: config.MaxOpenFiles, - shardIDProviderType: config.ShardIDProviderType, - numShards: config.NumShards, + conf: config, } } // Create will create the persister for the provided path -// TODO: refactor to use max tries mechanism func (pc *persisterCreator) Create(path string) (storage.Persister, error) { if len(path) == 0 { return nil, storage.ErrInvalidFilePath } - if pc.numShards < minNumShards { + if pc.conf.NumShards < minNumShards { return pc.CreateBasePersister(path) } @@ -50,23 +40,23 @@ func (pc *persisterCreator) Create(path string) (storage.Persister, error) { // CreateBasePersister will create base the persister for the provided path func (pc *persisterCreator) CreateBasePersister(path string) (storage.Persister, error) { - var dbType = storageunit.DBType(pc.dbType) - switch dbType { - case storageunit.LvlDB: - return database.NewLevelDB(path, pc.batchDelaySeconds, pc.maxBatchSize, pc.maxOpenFiles) - case storageunit.LvlDBSerial: - return database.NewSerialDB(path, pc.batchDelaySeconds, pc.maxBatchSize, pc.maxOpenFiles) - case storageunit.MemoryDB: - return database.NewMemDB(), nil - default: - return nil, storage.ErrNotSupportedDBType + var dbType = storageunit.DBType(pc.conf.Type) + + argsDB := factory.ArgDB{ + DBType: dbType, + Path: path, + BatchDelaySeconds: pc.conf.BatchDelaySeconds, + MaxBatchSize: pc.conf.MaxBatchSize, + MaxOpenFiles: pc.conf.MaxOpenFiles, } + + return storageunit.NewDB(argsDB) } func (pc *persisterCreator) createShardIDProvider() (storage.ShardIDProvider, error) { - switch storageunit.ShardIDProviderType(pc.shardIDProviderType) { + switch storageunit.ShardIDProviderType(pc.conf.ShardIDProviderType) { case storageunit.BinarySplit: - return database.NewShardIDProvider(pc.numShards) + return database.NewShardIDProvider(pc.conf.NumShards) default: return nil, storage.ErrNotSupportedShardIDProviderType } diff --git a/storage/factory/persisterCreator_test.go b/storage/factory/persisterCreator_test.go index a0fdef7e1ef..b1a4cc63796 100644 --- a/storage/factory/persisterCreator_test.go +++ b/storage/factory/persisterCreator_test.go @@ -38,6 +38,19 @@ func TestPersisterCreator_Create(t *testing.T) { require.Equal(t, storage.ErrInvalidFilePath, err) }) + t.Run("use tmp as file path", func(t *testing.T) { + t.Parallel() + + conf := createDefaultDBConfig() + conf.UseTmpAsFilePath = true + + pc := factory.NewPersisterCreator(conf) + + p, err := pc.Create("path1") + require.Nil(t, err) + require.NotNil(t, p) + }) + t.Run("should create non sharded persister", func(t *testing.T) { t.Parallel() diff --git a/storage/factory/persisterFactory.go b/storage/factory/persisterFactory.go index a1305ec2184..321ddf59118 100644 --- a/storage/factory/persisterFactory.go +++ b/storage/factory/persisterFactory.go @@ -1,29 +1,51 @@ package factory import ( - "github.com/multiversx/mx-chain-core-go/core/check" + "os" + "path" + "time" + + "github.com/multiversx/mx-chain-go/config" "github.com/multiversx/mx-chain-go/storage" "github.com/multiversx/mx-chain-go/storage/disabled" ) -// PersisterFactory is the factory which will handle creating new databases -type PersisterFactory struct { +// persisterFactory is the factory which will handle creating new databases +type persisterFactory struct { dbConfigHandler storage.DBConfigHandler } -// NewPersisterFactory will return a new instance of a PersisterFactory -func NewPersisterFactory(dbConfigHandler storage.DBConfigHandler) (*PersisterFactory, error) { - if check.IfNil(dbConfigHandler) { - return nil, storage.ErrNilDBConfigHandler - } +// NewPersisterFactory will return a new instance of persister factory +func NewPersisterFactory(config config.DBConfig) (*persisterFactory, error) { + dbConfigHandler := NewDBConfigHandler(config) - return &PersisterFactory{ + return &persisterFactory{ dbConfigHandler: dbConfigHandler, }, nil } +// CreateWithRetries will return a new instance of a DB with a given path +// It will try to create db multiple times +func (pf *persisterFactory) CreateWithRetries(path string) (storage.Persister, error) { + var persister storage.Persister + var err error + + for i := 0; i < storage.MaxRetriesToCreateDB; i++ { + persister, err = pf.Create(path) + if err == nil { + return persister, nil + } + log.Warn("Create Persister failed", "path", path, "error", err) + + // TODO: extract this in a parameter and inject it + time.Sleep(storage.SleepTimeBetweenCreateDBRetries) + } + + return nil, err +} + // Create will return a new instance of a DB with a given path -func (pf *PersisterFactory) Create(path string) (storage.Persister, error) { +func (pf *persisterFactory) Create(path string) (storage.Persister, error) { if len(path) == 0 { return nil, storage.ErrInvalidFilePath } @@ -33,6 +55,15 @@ func (pf *PersisterFactory) Create(path string) (storage.Persister, error) { return nil, err } + if dbConfig.UseTmpAsFilePath { + filePath, err := getTmpFilePath(path) + if err != nil { + return nil, err + } + + path = filePath + } + pc := newPersisterCreator(*dbConfig) persister, err := pc.Create(path) @@ -49,11 +80,16 @@ func (pf *PersisterFactory) Create(path string) (storage.Persister, error) { } // CreateDisabled will return a new disabled persister -func (pf *PersisterFactory) CreateDisabled() storage.Persister { +func (pf *persisterFactory) CreateDisabled() storage.Persister { return disabled.NewErrorDisabledPersister() } +func getTmpFilePath(p string) (string, error) { + _, file := path.Split(p) + return os.MkdirTemp("", file) +} + // IsInterfaceNil returns true if there is no value under the interface -func (pf *PersisterFactory) IsInterfaceNil() bool { +func (pf *persisterFactory) IsInterfaceNil() bool { return pf == nil } diff --git a/storage/factory/persisterFactory_test.go b/storage/factory/persisterFactory_test.go index 208542a665b..cb7e15b1e47 100644 --- a/storage/factory/persisterFactory_test.go +++ b/storage/factory/persisterFactory_test.go @@ -3,11 +3,15 @@ package factory_test import ( "fmt" "os" + "path" + "strings" "testing" + "github.com/multiversx/mx-chain-core-go/core/check" "github.com/multiversx/mx-chain-go/storage" "github.com/multiversx/mx-chain-go/storage/factory" "github.com/multiversx/mx-chain-go/storage/storageunit" + "github.com/multiversx/mx-chain-storage-go/common" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -15,8 +19,7 @@ import ( func TestNewPersisterFactory(t *testing.T) { t.Parallel() - dbConfigHandler := factory.NewDBConfigHandler(createDefaultDBConfig()) - pf, err := factory.NewPersisterFactory(dbConfigHandler) + pf, err := factory.NewPersisterFactory(createDefaultDBConfig()) require.NotNil(t, pf) require.Nil(t, err) } @@ -27,25 +30,84 @@ func TestPersisterFactory_Create(t *testing.T) { t.Run("invalid file path, should fail", func(t *testing.T) { t.Parallel() - dbConfigHandler := factory.NewDBConfigHandler(createDefaultDBConfig()) - pf, _ := factory.NewPersisterFactory(dbConfigHandler) + pf, _ := factory.NewPersisterFactory(createDefaultDBConfig()) p, err := pf.Create("") require.Nil(t, p) require.Equal(t, storage.ErrInvalidFilePath, err) }) + t.Run("with tmp file path, should work", func(t *testing.T) { + t.Parallel() + + conf := createDefaultDBConfig() + conf.UseTmpAsFilePath = true + + pf, _ := factory.NewPersisterFactory(conf) + + dir := t.TempDir() + + p, err := pf.Create(dir) + require.NotNil(t, p) + require.Nil(t, err) + + // config.toml will be created in tmp path, but cannot be easily checked since + // the file path is not created deterministically + + // should not find in the dir created initially. + _, err = os.Stat(dir + "/config.toml") + require.Error(t, err) + }) + t.Run("should work", func(t *testing.T) { t.Parallel() - dbConfigHandler := factory.NewDBConfigHandler(createDefaultDBConfig()) - pf, _ := factory.NewPersisterFactory(dbConfigHandler) + pf, _ := factory.NewPersisterFactory(createDefaultDBConfig()) dir := t.TempDir() p, err := pf.Create(dir) require.NotNil(t, p) require.Nil(t, err) + + // check config.toml file exists + _, err = os.Stat(dir + "/config.toml") + require.Nil(t, err) + }) +} + +func TestPersisterFactory_CreateWithRetries(t *testing.T) { + t.Parallel() + + t.Run("wrong config should error", func(t *testing.T) { + t.Parallel() + + path := "TEST" + dbConfig := createDefaultDBConfig() + dbConfig.Type = "invalid type" + + persisterFactory, err := factory.NewPersisterFactory(dbConfig) + assert.Nil(t, err) + + db, err := persisterFactory.CreateWithRetries(path) + assert.True(t, check.IfNil(db)) + assert.Equal(t, common.ErrNotSupportedDBType, err) + }) + + t.Run("should work", func(t *testing.T) { + t.Parallel() + + path := path.Join(t.TempDir(), "TEST") + dbConfig := createDefaultDBConfig() + dbConfig.FilePath = path + + persisterFactory, err := factory.NewPersisterFactory(dbConfig) + assert.Nil(t, err) + + db, err := persisterFactory.CreateWithRetries(path) + assert.False(t, check.IfNil(db)) + assert.Nil(t, err) + _ = db.Close() }) } @@ -57,8 +119,7 @@ func TestPersisterFactory_Create_ConfigSaveToFilePath(t *testing.T) { dbConfig := createDefaultBasePersisterConfig() dbConfig.Type = string(storageunit.LvlDB) - dbConfigHandler := factory.NewDBConfigHandler(dbConfig) - pf, _ := factory.NewPersisterFactory(dbConfigHandler) + pf, _ := factory.NewPersisterFactory(dbConfig) dir := t.TempDir() path := dir + "storer/" @@ -77,8 +138,7 @@ func TestPersisterFactory_Create_ConfigSaveToFilePath(t *testing.T) { dbConfig := createDefaultBasePersisterConfig() dbConfig.Type = string(storageunit.LvlDBSerial) - dbConfigHandler := factory.NewDBConfigHandler(dbConfig) - pf, _ := factory.NewPersisterFactory(dbConfigHandler) + pf, _ := factory.NewPersisterFactory(dbConfig) dir := t.TempDir() path := dir + "storer/" @@ -97,8 +157,7 @@ func TestPersisterFactory_Create_ConfigSaveToFilePath(t *testing.T) { dbConfig := createDefaultBasePersisterConfig() dbConfig.Type = string(storageunit.MemoryDB) - dbConfigHandler := factory.NewDBConfigHandler(dbConfig) - pf, _ := factory.NewPersisterFactory(dbConfigHandler) + pf, _ := factory.NewPersisterFactory(dbConfig) dir := t.TempDir() path := dir + "storer/" @@ -117,8 +176,7 @@ func TestPersisterFactory_Create_ConfigSaveToFilePath(t *testing.T) { dbConfig := createDefaultBasePersisterConfig() dbConfig.Type = string(storageunit.MemoryDB) - dbConfigHandler := factory.NewDBConfigHandler(dbConfig) - pf, _ := factory.NewPersisterFactory(dbConfigHandler) + pf, _ := factory.NewPersisterFactory(dbConfig) dir := t.TempDir() path := dir + "storer/" @@ -135,8 +193,7 @@ func TestPersisterFactory_Create_ConfigSaveToFilePath(t *testing.T) { func TestPersisterFactory_CreateDisabled(t *testing.T) { t.Parallel() - dbConfigHandler := factory.NewDBConfigHandler(createDefaultDBConfig()) - factoryInstance, err := factory.NewPersisterFactory(dbConfigHandler) + factoryInstance, err := factory.NewPersisterFactory(createDefaultDBConfig()) require.Nil(t, err) persisterInstance := factoryInstance.CreateDisabled() @@ -147,10 +204,28 @@ func TestPersisterFactory_CreateDisabled(t *testing.T) { func TestPersisterFactory_IsInterfaceNil(t *testing.T) { t.Parallel() - var pf *factory.PersisterFactory - require.True(t, pf.IsInterfaceNil()) - - dbConfigHandler := factory.NewDBConfigHandler(createDefaultDBConfig()) - pf, _ = factory.NewPersisterFactory(dbConfigHandler) + pf, _ := factory.NewPersisterFactory(createDefaultDBConfig()) require.False(t, pf.IsInterfaceNil()) } + +func TestGetTmpFilePath(t *testing.T) { + t.Parallel() + + pathSeparator := "/" + + tmpDir := os.TempDir() + tmpBasePath := tmpDir + pathSeparator + + path, err := factory.GetTmpFilePath("aaaa/bbbb/cccc") + require.Nil(t, err) + require.True(t, strings.Contains(path, tmpBasePath+"cccc")) + + path, _ = factory.GetTmpFilePath("aaaa") + require.True(t, strings.Contains(path, tmpBasePath+"aaaa")) + + path, _ = factory.GetTmpFilePath("") + require.True(t, strings.Contains(path, tmpBasePath+"")) + + path, _ = factory.GetTmpFilePath("/") + require.True(t, strings.Contains(path, tmpBasePath+"")) +} diff --git a/storage/factory/storageServiceFactory.go b/storage/factory/storageServiceFactory.go index f316bfec7d7..c153e6b2cc8 100644 --- a/storage/factory/storageServiceFactory.go +++ b/storage/factory/storageServiceFactory.go @@ -27,6 +27,7 @@ var log = logger.GetOrCreate("storage/factory") const ( minimumNumberOfActivePersisters = 1 minimumNumberOfEpochsToKeep = 2 + emptyDBPathSuffix = "" ) // StorageServiceType defines the type of StorageService @@ -134,11 +135,8 @@ func checkArgs(args StorageServiceFactoryArgs) error { return nil } -// TODO: refactor this function, split it into multiple ones -func (psf *StorageServiceFactory) createAndAddBaseStorageUnits( +func (psf *StorageServiceFactory) createAndAddTxStorageUnits( store dataRetriever.StorageService, - customDatabaseRemover storage.CustomDatabaseRemoverHandler, - shardID string, ) error { disabledCustomDatabaseRemover := disabled.NewDisabledCustomDatabaseRemover() @@ -182,6 +180,21 @@ func (psf *StorageServiceFactory) createAndAddBaseStorageUnits( } store.AddStorer(dataRetriever.ReceiptsUnit, receiptsUnit) + return nil +} + +func (psf *StorageServiceFactory) createAndAddBaseStorageUnits( + store dataRetriever.StorageService, + customDatabaseRemover storage.CustomDatabaseRemoverHandler, + shardID string, +) error { + disabledCustomDatabaseRemover := disabled.NewDisabledCustomDatabaseRemover() + + err := psf.createAndAddTxStorageUnits(store) + if err != nil { + return err + } + scheduledSCRsUnitArgs, err := psf.createPruningStorerArgs(psf.generalConfig.ScheduledSCRsStorage, disabledCustomDatabaseRemover) if err != nil { return err @@ -222,22 +235,7 @@ func (psf *StorageServiceFactory) createAndAddBaseStorageUnits( } store.AddStorer(dataRetriever.MetaBlockUnit, metaBlockUnit) - // metaHdrHashNonce is static - metaHdrHashNonceUnitConfig := GetDBFromConfig(psf.generalConfig.MetaHdrNonceHashStorage.DB) - dbPath := psf.pathManager.PathForStatic(shardID, psf.generalConfig.MetaHdrNonceHashStorage.DB.FilePath) - metaHdrHashNonceUnitConfig.FilePath = dbPath - - dbConfigHandlerInstance := NewDBConfigHandler(psf.generalConfig.MetaHdrNonceHashStorage.DB) - metaHdrHashNoncePersisterCreator, err := NewPersisterFactory(dbConfigHandlerInstance) - if err != nil { - return err - } - - metaHdrHashNonceUnit, err := storageunit.NewStorageUnitFromConf( - GetCacherFromConfig(psf.generalConfig.MetaHdrNonceHashStorage.Cache), - metaHdrHashNonceUnitConfig, - metaHdrHashNoncePersisterCreator, - ) + metaHdrHashNonceUnit, err := psf.createStaticStorageUnit(psf.generalConfig.MetaHdrNonceHashStorage, shardID, emptyDBPathSuffix) if err != nil { return fmt.Errorf("%w for MetaHdrNonceHashStorage", err) } @@ -259,22 +257,8 @@ func (psf *StorageServiceFactory) createAndAddBaseStorageUnits( } store.AddStorer(dataRetriever.UserAccountsUnit, userAccountsUnit) - statusMetricsDbConfig := GetDBFromConfig(psf.generalConfig.StatusMetricsStorage.DB) shardId := core.GetShardIDString(psf.shardCoordinator.SelfId()) - dbPath = psf.pathManager.PathForStatic(shardId, psf.generalConfig.StatusMetricsStorage.DB.FilePath) - statusMetricsDbConfig.FilePath = dbPath - - dbConfigHandlerInstance = NewDBConfigHandler(psf.generalConfig.StatusMetricsStorage.DB) - statusMetricsPersisterCreator, err := NewPersisterFactory(dbConfigHandlerInstance) - if err != nil { - return err - } - - statusMetricsStorageUnit, err := storageunit.NewStorageUnitFromConf( - GetCacherFromConfig(psf.generalConfig.StatusMetricsStorage.Cache), - statusMetricsDbConfig, - statusMetricsPersisterCreator, - ) + statusMetricsStorageUnit, err := psf.createStaticStorageUnit(psf.generalConfig.StatusMetricsStorage, shardId, emptyDBPathSuffix) if err != nil { return fmt.Errorf("%w for StatusMetricsStorage", err) } @@ -289,6 +273,27 @@ func (psf *StorageServiceFactory) createAndAddBaseStorageUnits( return nil } +func (psf *StorageServiceFactory) createStaticStorageUnit( + storageConf config.StorageConfig, + shardID string, + dbPathSuffix string, +) (*storageunit.Unit, error) { + storageUnitDBConf := GetDBFromConfig(storageConf.DB) + dbPath := psf.pathManager.PathForStatic(shardID, storageConf.DB.FilePath) + dbPathSuffix + storageUnitDBConf.FilePath = dbPath + + persisterCreator, err := NewPersisterFactory(storageConf.DB) + if err != nil { + return nil, err + } + + return storageunit.NewStorageUnitFromConf( + GetCacherFromConfig(storageConf.Cache), + storageUnitDBConf, + persisterCreator, + ) +} + // CreateForShard will return the storage service which contains all storers needed for a shard func (psf *StorageServiceFactory) CreateForShard() (dataRetriever.StorageService, error) { // TODO: if there will be a differentiation between the creation or opening of a DB, the DBs could be destroyed on a defer @@ -301,23 +306,8 @@ func (psf *StorageServiceFactory) CreateForShard() (dataRetriever.StorageService } shardID := core.GetShardIDString(psf.shardCoordinator.SelfId()) - - // shardHdrHashNonce storer is static - shardHdrHashNonceConfig := GetDBFromConfig(psf.generalConfig.ShardHdrNonceHashStorage.DB) - dbPath := psf.pathManager.PathForStatic(shardID, psf.generalConfig.ShardHdrNonceHashStorage.DB.FilePath) + shardID - shardHdrHashNonceConfig.FilePath = dbPath - - dbConfigHandlerInstance := NewDBConfigHandler(psf.generalConfig.ShardHdrNonceHashStorage.DB) - shardHdrHashNoncePersisterCreator, err := NewPersisterFactory(dbConfigHandlerInstance) - if err != nil { - return nil, err - } - - shardHdrHashNonceUnit, err := storageunit.NewStorageUnitFromConf( - GetCacherFromConfig(psf.generalConfig.ShardHdrNonceHashStorage.Cache), - shardHdrHashNonceConfig, - shardHdrHashNoncePersisterCreator, - ) + dbPathSuffix := shardID + shardHdrHashNonceUnit, err := psf.createStaticStorageUnit(psf.generalConfig.ShardHdrNonceHashStorage, shardID, dbPathSuffix) if err != nil { return nil, fmt.Errorf("%w for ShardHdrNonceHashStorage", err) } @@ -382,24 +372,10 @@ func (psf *StorageServiceFactory) CreateForMeta() (dataRetriever.StorageService, shardHdrHashNonceUnits := make([]*storageunit.Unit, psf.shardCoordinator.NumberOfShards()) for i := uint32(0); i < psf.shardCoordinator.NumberOfShards(); i++ { - shardHdrHashNonceConfig := GetDBFromConfig(psf.generalConfig.ShardHdrNonceHashStorage.DB) shardID = core.GetShardIDString(core.MetachainShardId) - dbPath := psf.pathManager.PathForStatic(shardID, psf.generalConfig.ShardHdrNonceHashStorage.DB.FilePath) + fmt.Sprintf("%d", i) - shardHdrHashNonceConfig.FilePath = dbPath - - dbConfigHandlerInstance := NewDBConfigHandler(psf.generalConfig.ShardHdrNonceHashStorage.DB) - shardHdrHashNoncePersisterCreator, errLoop := NewPersisterFactory(dbConfigHandlerInstance) - if errLoop != nil { - return nil, errLoop - } - - shardHdrHashNonceUnits[i], errLoop = storageunit.NewStorageUnitFromConf( - GetCacherFromConfig(psf.generalConfig.ShardHdrNonceHashStorage.Cache), - shardHdrHashNonceConfig, - shardHdrHashNoncePersisterCreator, - ) - if errLoop != nil { - return nil, fmt.Errorf("%w for ShardHdrNonceHashStorage on shard %d", errLoop, i) + shardHdrHashNonceUnits[i], err = psf.createStaticStorageUnit(psf.generalConfig.ShardHdrNonceHashStorage, shardID, fmt.Sprintf("%d", i)) + if err != nil { + return nil, fmt.Errorf("%w for ShardHdrNonceHashStorage on shard %d", err, i) } } @@ -527,69 +503,21 @@ func (psf *StorageServiceFactory) setUpDbLookupExtensions(chainStorer *dataRetri chainStorer.AddStorer(dataRetriever.MiniblocksMetadataUnit, miniblocksMetadataPruningStorer) - // Create the miniblocksHashByTxHash (STATIC) storer - miniblockHashByTxHashConfig := psf.generalConfig.DbLookupExtensions.MiniblockHashByTxHashStorageConfig - miniblockHashByTxHashDbConfig := GetDBFromConfig(miniblockHashByTxHashConfig.DB) - miniblockHashByTxHashDbConfig.FilePath = psf.pathManager.PathForStatic(shardID, miniblockHashByTxHashConfig.DB.FilePath) - miniblockHashByTxHashCacherConfig := GetCacherFromConfig(miniblockHashByTxHashConfig.Cache) - - dbConfigHandlerInstance := NewDBConfigHandler(miniblockHashByTxHashConfig.DB) - miniblockHashByTxHashPersisterCreator, err := NewPersisterFactory(dbConfigHandlerInstance) - if err != nil { - return err - } - - miniblockHashByTxHashUnit, err := storageunit.NewStorageUnitFromConf( - miniblockHashByTxHashCacherConfig, - miniblockHashByTxHashDbConfig, - miniblockHashByTxHashPersisterCreator, - ) + miniblockHashByTxHashUnit, err := psf.createStaticStorageUnit(psf.generalConfig.DbLookupExtensions.MiniblockHashByTxHashStorageConfig, shardID, emptyDBPathSuffix) if err != nil { return fmt.Errorf("%w for DbLookupExtensions.MiniblockHashByTxHashStorageConfig", err) } chainStorer.AddStorer(dataRetriever.MiniblockHashByTxHashUnit, miniblockHashByTxHashUnit) - // Create the blockHashByRound (STATIC) storer - blockHashByRoundConfig := psf.generalConfig.DbLookupExtensions.RoundHashStorageConfig - blockHashByRoundDBConfig := GetDBFromConfig(blockHashByRoundConfig.DB) - blockHashByRoundDBConfig.FilePath = psf.pathManager.PathForStatic(shardID, blockHashByRoundConfig.DB.FilePath) - blockHashByRoundCacherConfig := GetCacherFromConfig(blockHashByRoundConfig.Cache) - - dbConfigHandlerInstance = NewDBConfigHandler(blockHashByRoundConfig.DB) - blockHashByRoundPersisterCreator, err := NewPersisterFactory(dbConfigHandlerInstance) - if err != nil { - return err - } - - blockHashByRoundUnit, err := storageunit.NewStorageUnitFromConf( - blockHashByRoundCacherConfig, - blockHashByRoundDBConfig, - blockHashByRoundPersisterCreator, - ) + blockHashByRoundUnit, err := psf.createStaticStorageUnit(psf.generalConfig.DbLookupExtensions.RoundHashStorageConfig, shardID, emptyDBPathSuffix) if err != nil { return fmt.Errorf("%w for DbLookupExtensions.RoundHashStorageConfig", err) } chainStorer.AddStorer(dataRetriever.RoundHdrHashDataUnit, blockHashByRoundUnit) - // Create the epochByHash (STATIC) storer - epochByHashConfig := psf.generalConfig.DbLookupExtensions.EpochByHashStorageConfig - epochByHashDbConfig := GetDBFromConfig(epochByHashConfig.DB) - epochByHashDbConfig.FilePath = psf.pathManager.PathForStatic(shardID, epochByHashConfig.DB.FilePath) - epochByHashCacherConfig := GetCacherFromConfig(epochByHashConfig.Cache) - - dbConfigHandlerInstance = NewDBConfigHandler(epochByHashConfig.DB) - epochByHashPersisterCreator, err := NewPersisterFactory(dbConfigHandlerInstance) - if err != nil { - return err - } - - epochByHashUnit, err := storageunit.NewStorageUnitFromConf( - epochByHashCacherConfig, - epochByHashDbConfig, - epochByHashPersisterCreator, - ) + epochByHashUnit, err := psf.createStaticStorageUnit(psf.generalConfig.DbLookupExtensions.EpochByHashStorageConfig, shardID, emptyDBPathSuffix) if err != nil { return fmt.Errorf("%w for DbLookupExtensions.EpochByHashStorageConfig", err) } @@ -600,7 +528,7 @@ func (psf *StorageServiceFactory) setUpDbLookupExtensions(chainStorer *dataRetri } func (psf *StorageServiceFactory) setUpEsdtSuppliesStorer(chainStorer *dataRetriever.ChainStorer, shardIDStr string) error { - esdtSuppliesUnit, err := psf.createEsdtSuppliesUnit(shardIDStr) + esdtSuppliesUnit, err := psf.createStaticStorageUnit(psf.generalConfig.DbLookupExtensions.ESDTSuppliesStorageConfig, shardIDStr, emptyDBPathSuffix) if err != nil { return fmt.Errorf("%w for DbLookupExtensions.ESDTSuppliesStorageConfig", err) } @@ -613,7 +541,7 @@ func (psf *StorageServiceFactory) setUpEsdtSuppliesStorer(chainStorer *dataRetri } time.Sleep(time.Second) // making sure the unit was properly closed and destroyed - esdtSuppliesUnit, err = psf.createEsdtSuppliesUnit(shardIDStr) + esdtSuppliesUnit, err = psf.createStaticStorageUnit(psf.generalConfig.DbLookupExtensions.ESDTSuppliesStorageConfig, shardIDStr, emptyDBPathSuffix) if err != nil { return err } @@ -623,23 +551,6 @@ func (psf *StorageServiceFactory) setUpEsdtSuppliesStorer(chainStorer *dataRetri return nil } -func (psf *StorageServiceFactory) createEsdtSuppliesUnit(shardIDStr string) (storage.Storer, error) { - esdtSuppliesConfig := psf.generalConfig.DbLookupExtensions.ESDTSuppliesStorageConfig - esdtSuppliesDbConfig := GetDBFromConfig(esdtSuppliesConfig.DB) - esdtSuppliesDbConfig.FilePath = psf.pathManager.PathForStatic(shardIDStr, esdtSuppliesConfig.DB.FilePath) - esdtSuppliesCacherConfig := GetCacherFromConfig(esdtSuppliesConfig.Cache) - - dbConfigHandlerInstance := NewDBConfigHandler(esdtSuppliesConfig.DB) - esdtSuppliesPersisterCreator, err := NewPersisterFactory(dbConfigHandlerInstance) - if err != nil { - return nil, err - } - - return storageunit.NewStorageUnitFromConf( - esdtSuppliesCacherConfig, esdtSuppliesDbConfig, - esdtSuppliesPersisterCreator) -} - func (psf *StorageServiceFactory) createPruningStorerArgs( storageConfig config.StorageConfig, customDatabaseRemover storage.CustomDatabaseRemoverHandler, @@ -655,8 +566,7 @@ func (psf *StorageServiceFactory) createPruningStorerArgs( NumOfActivePersisters: numOfActivePersisters, } - dbConfigHandlerInstance := NewDBConfigHandler(storageConfig.DB) - persisterFactory, err := NewPersisterFactory(dbConfigHandlerInstance) + persisterFactory, err := NewPersisterFactory(storageConfig.DB) if err != nil { return pruning.StorerArgs{}, err } @@ -687,22 +597,8 @@ func (psf *StorageServiceFactory) createTrieEpochRootHashStorerIfNeeded() (stora return storageunit.NewNilStorer(), nil } - trieEpochRootHashDbConfig := GetDBFromConfig(psf.generalConfig.TrieEpochRootHashStorage.DB) shardId := core.GetShardIDString(psf.shardCoordinator.SelfId()) - dbPath := psf.pathManager.PathForStatic(shardId, psf.generalConfig.TrieEpochRootHashStorage.DB.FilePath) - trieEpochRootHashDbConfig.FilePath = dbPath - - dbConfigHandlerInstance := NewDBConfigHandler(psf.generalConfig.TrieEpochRootHashStorage.DB) - esdtSuppliesPersisterCreator, err := NewPersisterFactory(dbConfigHandlerInstance) - if err != nil { - return nil, err - } - - trieEpochRootHashStorageUnit, err := storageunit.NewStorageUnitFromConf( - GetCacherFromConfig(psf.generalConfig.TrieEpochRootHashStorage.Cache), - trieEpochRootHashDbConfig, - esdtSuppliesPersisterCreator, - ) + trieEpochRootHashStorageUnit, err := psf.createStaticStorageUnit(psf.generalConfig.TrieEpochRootHashStorage, shardId, emptyDBPathSuffix) if err != nil { return nil, fmt.Errorf("%w for TrieEpochRootHashStorage", err) } @@ -713,21 +609,8 @@ func (psf *StorageServiceFactory) createTrieEpochRootHashStorerIfNeeded() (stora func (psf *StorageServiceFactory) createTriePersister( storageConfig config.StorageConfig, ) (storage.Storer, error) { - trieDBConfig := GetDBFromConfig(storageConfig.DB) shardID := core.GetShardIDString(psf.shardCoordinator.SelfId()) - dbPath := psf.pathManager.PathForStatic(shardID, storageConfig.DB.FilePath) - trieDBConfig.FilePath = dbPath - - dbConfigHandlerInstance := NewDBConfigHandler(storageConfig.DB) - persisterFactory, err := NewPersisterFactory(dbConfigHandlerInstance) - if err != nil { - return nil, err - } - - return storageunit.NewStorageUnitFromConf( - GetCacherFromConfig(storageConfig.Cache), - trieDBConfig, - persisterFactory) + return psf.createStaticStorageUnit(storageConfig, shardID, emptyDBPathSuffix) } func (psf *StorageServiceFactory) createTriePruningPersister(arg pruning.StorerArgs) (storage.Storer, error) { diff --git a/storage/interface.go b/storage/interface.go index 328eb86c4ed..5dd61cfad1d 100644 --- a/storage/interface.go +++ b/storage/interface.go @@ -211,6 +211,7 @@ type ManagedPeersHolder interface { // PersisterFactoryHandler defines the behaviour of a component which is able to create persisters type PersisterFactoryHandler interface { Create(path string) (Persister, error) + CreateWithRetries(path string) (Persister, error) IsInterfaceNil() bool } diff --git a/storage/latestData/latestDataProvider.go b/storage/latestData/latestDataProvider.go index df6ea7e2418..2b894627de3 100644 --- a/storage/latestData/latestDataProvider.go +++ b/storage/latestData/latestDataProvider.go @@ -132,8 +132,7 @@ func (ldp *latestDataProvider) getEpochDirs() ([]string, error) { } func (ldp *latestDataProvider) getLastEpochAndRoundFromStorage(parentDir string, lastEpoch uint32) (storage.LatestDataFromStorage, error) { - dbConfigHandler := factory.NewDBConfigHandler(ldp.generalConfig.BootstrapStorage.DB) - persisterFactory, err := factory.NewPersisterFactory(dbConfigHandler) + persisterFactory, err := factory.NewPersisterFactory(ldp.generalConfig.BootstrapStorage.DB) if err != nil { return storage.LatestDataFromStorage{}, err } diff --git a/storage/pruning/fullHistoryPruningStorer_test.go b/storage/pruning/fullHistoryPruningStorer_test.go index c83fc5fae34..0e0d43877e8 100644 --- a/storage/pruning/fullHistoryPruningStorer_test.go +++ b/storage/pruning/fullHistoryPruningStorer_test.go @@ -294,16 +294,13 @@ func TestFullHistoryPruningStorer_ConcurrentOperations(t *testing.T) { fmt.Println(testDir) args := getDefaultArgs() - dbConfigHandler := factory.NewDBConfigHandler( - config.DBConfig{ - FilePath: filepath.Join(testDir, dbName), - Type: "LvlDBSerial", - MaxBatchSize: 100, - MaxOpenFiles: 10, - BatchDelaySeconds: 2, - }, - ) - persisterFactory, err := factory.NewPersisterFactory(dbConfigHandler) + persisterFactory, err := factory.NewPersisterFactory(config.DBConfig{ + FilePath: filepath.Join(testDir, dbName), + Type: "LvlDBSerial", + MaxBatchSize: 100, + MaxOpenFiles: 10, + BatchDelaySeconds: 2, + }) require.Nil(t, err) args.PersisterFactory = persisterFactory diff --git a/storage/pruning/pruningStorer_test.go b/storage/pruning/pruningStorer_test.go index 29c3765e2d8..248cc53cda2 100644 --- a/storage/pruning/pruningStorer_test.go +++ b/storage/pruning/pruningStorer_test.go @@ -1053,16 +1053,13 @@ func TestPruningStorer_ConcurrentOperations(t *testing.T) { fmt.Println(testDir) args := getDefaultArgs() - dbConfigHandler := factory.NewDBConfigHandler( - config.DBConfig{ - FilePath: filepath.Join(testDir, dbName), - Type: "LvlDBSerial", - MaxBatchSize: 100, - MaxOpenFiles: 10, - BatchDelaySeconds: 2, - }, - ) - persisterFactory, err := factory.NewPersisterFactory(dbConfigHandler) + persisterFactory, err := factory.NewPersisterFactory(config.DBConfig{ + FilePath: filepath.Join(testDir, dbName), + Type: "LvlDBSerial", + MaxBatchSize: 100, + MaxOpenFiles: 10, + BatchDelaySeconds: 2, + }) require.Nil(t, err) args.PersisterFactory = persisterFactory diff --git a/storage/storageunit/constants.go b/storage/storageunit/constants.go index 0e128af8123..022715dbcb7 100644 --- a/storage/storageunit/constants.go +++ b/storage/storageunit/constants.go @@ -1,25 +1,27 @@ package storageunit -import "github.com/multiversx/mx-chain-storage-go/storageUnit" +import ( + "github.com/multiversx/mx-chain-storage-go/common" +) const ( // LRUCache defines a cache identifier with least-recently-used eviction mechanism - LRUCache = storageUnit.LRUCache + LRUCache = common.LRUCache // SizeLRUCache defines a cache identifier with least-recently-used eviction mechanism and fixed size in bytes - SizeLRUCache = storageUnit.SizeLRUCache + SizeLRUCache = common.SizeLRUCache ) // DB types that are currently supported const ( // LvlDB represents a levelDB storage identifier - LvlDB = storageUnit.LvlDB + LvlDB = common.LvlDB // LvlDBSerial represents a levelDB storage with serialized operations identifier - LvlDBSerial = storageUnit.LvlDBSerial + LvlDBSerial = common.LvlDBSerial // MemoryDB represents an in memory storage identifier - MemoryDB = storageUnit.MemoryDB + MemoryDB = common.MemoryDB ) // Shard id provider types that are currently supported const ( - BinarySplit = storageUnit.BinarySplit + BinarySplit = common.BinarySplit ) diff --git a/storage/storageunit/storageunit.go b/storage/storageunit/storageunit.go index 4e1605efaa7..c1944777920 100644 --- a/storage/storageunit/storageunit.go +++ b/storage/storageunit/storageunit.go @@ -3,6 +3,8 @@ package storageunit import ( "github.com/multiversx/mx-chain-core-go/marshal" "github.com/multiversx/mx-chain-go/storage" + "github.com/multiversx/mx-chain-storage-go/common" + "github.com/multiversx/mx-chain-storage-go/factory" "github.com/multiversx/mx-chain-storage-go/storageCacherAdapter" "github.com/multiversx/mx-chain-storage-go/storageUnit" ) @@ -12,25 +14,25 @@ import ( type Unit = storageUnit.Unit // CacheConfig holds the configurable elements of a cache -type CacheConfig = storageUnit.CacheConfig - -// ArgDB is a structure that is used to create a new storage.Persister implementation -type ArgDB = storageUnit.ArgDB +type CacheConfig = common.CacheConfig // DBConfig holds the configurable elements of a database -type DBConfig = storageUnit.DBConfig +type DBConfig = common.DBConfig // NilStorer resembles a disabled implementation of the Storer interface type NilStorer = storageUnit.NilStorer // CacheType represents the type of the supported caches -type CacheType = storageUnit.CacheType +type CacheType = common.CacheType // DBType represents the type of the supported databases -type DBType = storageUnit.DBType +type DBType = common.DBType // ShardIDProviderType represents the type of the supported shard id providers -type ShardIDProviderType = storageUnit.ShardIDProviderType +type ShardIDProviderType = common.ShardIDProviderType + +// ArgDB is a structure that is used to create a new storage.Persister implementation +type ArgDB = factory.ArgDB // NewStorageUnit is the constructor for the storage unit, creating a new storage unit // from the given cacher and persister. @@ -40,17 +42,31 @@ func NewStorageUnit(c storage.Cacher, p storage.Persister) (*Unit, error) { // NewCache creates a new cache from a cache config func NewCache(config CacheConfig) (storage.Cacher, error) { - return storageUnit.NewCache(config) + return factory.NewCache(config) } // NewDB creates a new database from database config -func NewDB(persisterFactory storage.PersisterFactoryHandler, path string) (storage.Persister, error) { - return storageUnit.NewDB(persisterFactory, path) +func NewDB(args ArgDB) (storage.Persister, error) { + return factory.NewDB(args) } // NewStorageUnitFromConf creates a new storage unit from a storage unit config func NewStorageUnitFromConf(cacheConf CacheConfig, dbConf DBConfig, persisterFactory storage.PersisterFactoryHandler) (*Unit, error) { - return storageUnit.NewStorageUnitFromConf(cacheConf, dbConf, persisterFactory) + if dbConf.MaxBatchSize > int(cacheConf.Capacity) { + return nil, common.ErrCacheSizeIsLowerThanBatchSize + } + + cache, err := NewCache(cacheConf) + if err != nil { + return nil, err + } + + db, err := persisterFactory.CreateWithRetries(dbConf.FilePath) + if err != nil { + return nil, err + } + + return NewStorageUnit(cache, db) } // NewNilStorer will return a nil storer diff --git a/storage/storageunit/storageunit_test.go b/storage/storageunit/storageunit_test.go index 34affcb569f..da4aea63b33 100644 --- a/storage/storageunit/storageunit_test.go +++ b/storage/storageunit/storageunit_test.go @@ -72,52 +72,6 @@ func TestNewCache(t *testing.T) { }) } -func TestNewDB(t *testing.T) { - t.Parallel() - - t.Run("wrong config should error", func(t *testing.T) { - t.Parallel() - - path := "TEST" - dbConfig := config.DBConfig{ - FilePath: path, - Type: "invalid type", - BatchDelaySeconds: 5, - MaxBatchSize: 10, - MaxOpenFiles: 10, - } - - dbConfigHandler := factory.NewDBConfigHandler(dbConfig) - persisterFactory, err := factory.NewPersisterFactory(dbConfigHandler) - assert.Nil(t, err) - - db, err := storageunit.NewDB(persisterFactory, path) - assert.True(t, check.IfNil(db)) - assert.Equal(t, common.ErrNotSupportedDBType, err) - }) - t.Run("should work", func(t *testing.T) { - t.Parallel() - - path := path.Join(t.TempDir(), "TEST") - dbConfig := config.DBConfig{ - FilePath: path, - Type: "LvlDBSerial", - BatchDelaySeconds: 5, - MaxBatchSize: 10, - MaxOpenFiles: 10, - } - - dbConfigHandler := factory.NewDBConfigHandler(dbConfig) - persisterFactory, err := factory.NewPersisterFactory(dbConfigHandler) - assert.Nil(t, err) - - db, err := storageunit.NewDB(persisterFactory, path) - assert.False(t, check.IfNil(db)) - assert.Nil(t, err) - _ = db.Close() - }) -} - func TestNewStorageUnitFromConf(t *testing.T) { t.Parallel() @@ -144,8 +98,7 @@ func TestNewStorageUnitFromConf(t *testing.T) { MaxBatchSize: dbConfig.MaxBatchSize, MaxOpenFiles: dbConfig.MaxOpenFiles, } - dbConfigHandler := factory.NewDBConfigHandler(dbConf) - persisterFactory, err := factory.NewPersisterFactory(dbConfigHandler) + persisterFactory, err := factory.NewPersisterFactory(dbConf) assert.Nil(t, err) unit, err := storageunit.NewStorageUnitFromConf(cacheConfig, dbConfig, persisterFactory) @@ -166,8 +119,7 @@ func TestNewStorageUnitFromConf(t *testing.T) { MaxBatchSize: dbConfig.MaxBatchSize, MaxOpenFiles: dbConfig.MaxOpenFiles, } - dbConfigHandler := factory.NewDBConfigHandler(dbConf) - persisterFactory, err := factory.NewPersisterFactory(dbConfigHandler) + persisterFactory, err := factory.NewPersisterFactory(dbConf) assert.Nil(t, err) unit, err := storageunit.NewStorageUnitFromConf(cacheConfig, dbConfig, persisterFactory) diff --git a/testscommon/dataRetriever/poolFactory.go b/testscommon/dataRetriever/poolFactory.go index 77bdeb610a7..a8f4374e800 100644 --- a/testscommon/dataRetriever/poolFactory.go +++ b/testscommon/dataRetriever/poolFactory.go @@ -98,11 +98,10 @@ func CreatePoolsHolder(numShards uint32, selfShard uint32) dataRetriever.PoolsHo MaxOpenFiles: 10, } - dbConfigHandler := storageFactory.NewDBConfigHandler(dbConfig) - persisterFactory, err := storageFactory.NewPersisterFactory(dbConfigHandler) + persisterFactory, err := storageFactory.NewPersisterFactory(dbConfig) panicIfError("Create persister factory", err) - persister, err := storageunit.NewDB(persisterFactory, tempDir) + persister, err := persisterFactory.CreateWithRetries(tempDir) panicIfError("Create trieSync DB", err) tnf := factory.NewTrieNodeFactory() diff --git a/testscommon/integrationtests/factory.go b/testscommon/integrationtests/factory.go index 4d2f9ad02d8..9acfa7c5e10 100644 --- a/testscommon/integrationtests/factory.go +++ b/testscommon/integrationtests/factory.go @@ -62,8 +62,7 @@ func CreateStorer(parentDir string) storage.Storer { MaxBatchSize: 45000, MaxOpenFiles: 10, } - dbConfigHandler := factory.NewDBConfigHandler(dbConfig) - persisterFactory, err := factory.NewPersisterFactory(dbConfigHandler) + persisterFactory, err := factory.NewPersisterFactory(dbConfig) if err != nil { return nil } diff --git a/update/factory/dataTrieFactory.go b/update/factory/dataTrieFactory.go index f9491350693..dcd83da1bd7 100644 --- a/update/factory/dataTrieFactory.go +++ b/update/factory/dataTrieFactory.go @@ -67,8 +67,7 @@ func NewDataTrieFactory(args ArgsNewDataTrieFactory) (*dataTrieFactory, error) { dbConfig := storageFactory.GetDBFromConfig(args.StorageConfig.DB) dbConfig.FilePath = path.Join(args.SyncFolder, args.StorageConfig.DB.FilePath) - dbConfigHandler := factory.NewDBConfigHandler(args.StorageConfig.DB) - persisterFactory, err := factory.NewPersisterFactory(dbConfigHandler) + persisterFactory, err := factory.NewPersisterFactory(args.StorageConfig.DB) if err != nil { return nil, err } diff --git a/update/factory/exportHandlerFactory.go b/update/factory/exportHandlerFactory.go index 8dd429345bb..c13f25f3f5a 100644 --- a/update/factory/exportHandlerFactory.go +++ b/update/factory/exportHandlerFactory.go @@ -608,8 +608,7 @@ func createStorer(storageConfig config.StorageConfig, folder string) (storage.St dbConfig := storageFactory.GetDBFromConfig(storageConfig.DB) dbConfig.FilePath = path.Join(folder, storageConfig.DB.FilePath) - dbConfigHandler := storageFactory.NewDBConfigHandler(storageConfig.DB) - persisterFactory, err := storageFactory.NewPersisterFactory(dbConfigHandler) + persisterFactory, err := storageFactory.NewPersisterFactory(storageConfig.DB) if err != nil { return nil, err }