Merge pull request #6001 from multiversx/feat/refactor-persister-factory

Feat/refactor persister factory

dataRetriever/factory/dataPoolFactory.go

@@ -2,7 +2,6 @@ package factory
 
 import (
 	"fmt"
-	"os"
 	"time"
 
 	"github.com/multiversx/mx-chain-core-go/core"
@@ -179,22 +178,12 @@ func createTrieSyncDB(args ArgsDataPool) (storage.Persister, error) {
 	shardId := core.GetShardIDString(args.ShardCoordinator.SelfId())
 	path := args.PathManager.PathForStatic(shardId, mainConfig.TrieSyncStorage.DB.FilePath)
 
-	dbConfigHandler := factory.NewDBConfigHandler(mainConfig.TrieSyncStorage.DB)
-	persisterFactory, err := factory.NewPersisterFactory(dbConfigHandler)
+	persisterFactory, err := factory.NewPersisterFactory(mainConfig.TrieSyncStorage.DB)
 	if err != nil {
 		return nil, err
 	}
 
-	if mainConfig.TrieSyncStorage.DB.UseTmpAsFilePath {
-		filePath, errTempDir := os.MkdirTemp("", "trieSyncStorage")
-		if errTempDir != nil {
-			return nil, errTempDir
-		}
-
-		path = filePath
-	}
-
-	db, err := storageunit.NewDB(persisterFactory, path)
+	db, err := persisterFactory.CreateWithRetries(path)
 	if err != nil {
 		return nil, fmt.Errorf("%w while creating the db for the trie nodes", err)
 	}

epochStart/metachain/systemSCs_test.go

@@ -87,12 +87,11 @@ func createPhysicalUnit(t *testing.T) (storage.Storer, string) {
 		MaxOpenFiles:      10,
 	}
 
-	dbConfigHandler := storageFactory.NewDBConfigHandler(dbConfig)
-	persisterFactory, err := storageFactory.NewPersisterFactory(dbConfigHandler)
+	persisterFactory, err := storageFactory.NewPersisterFactory(dbConfig)
 	assert.Nil(t, err)
 
 	cache, _ := storageunit.NewCache(cacheConfig)
-	persist, _ := storageunit.NewDB(persisterFactory, dir)
+	persist, _ := persisterFactory.CreateWithRetries(dir)
 	unit, _ := storageunit.NewStorageUnit(cache, persist)
 
 	return unit, dir

genesis/process/genesisBlockCreator.go

@@ -131,8 +131,7 @@ func createStorer(storageConfig config.StorageConfig, folder string) (storage.St
 	dbConfig := factory.GetDBFromConfig(storageConfig.DB)
 	dbConfig.FilePath = path.Join(folder, storageConfig.DB.FilePath)
 
-	dbConfigHandler := factory.NewDBConfigHandler(storageConfig.DB)
-	persisterFactory, err := factory.NewPersisterFactory(dbConfigHandler)
+	persisterFactory, err := factory.NewPersisterFactory(storageConfig.DB)
 	if err != nil {
 		return nil, err
 	}

go.mod

@@ -20,7 +20,7 @@ require (
 	github.com/multiversx/mx-chain-es-indexer-go v1.4.19-0.20240129150813-a772c480d33a
 	github.com/multiversx/mx-chain-logger-go v1.0.14-0.20240129144507-d00e967c890c
 	github.com/multiversx/mx-chain-scenario-go v1.3.1-0.20240129145446-ca4fba98f6d1
-	github.com/multiversx/mx-chain-storage-go v1.0.15-0.20240129144933-b1c0d642d7f8
+	github.com/multiversx/mx-chain-storage-go v1.0.15-0.20240304133242-faaf1d20b087
 	github.com/multiversx/mx-chain-vm-common-go v1.5.12-0.20240129145149-4fe61574f566
 	github.com/multiversx/mx-chain-vm-go v1.5.27-0.20240129150501-7c828af05c83
 	github.com/multiversx/mx-chain-vm-v1_2-go v1.2.66-0.20240129145751-f814f5525edb

go.sum

@@ -397,8 +397,8 @@ github.com/multiversx/mx-chain-logger-go v1.0.14-0.20240129144507-d00e967c890c h
 github.com/multiversx/mx-chain-logger-go v1.0.14-0.20240129144507-d00e967c890c/go.mod h1:fH/fR/GEBsDjPkBoZDVJMoYo2HhlA7++DP6QfITJ1N8=
 github.com/multiversx/mx-chain-scenario-go v1.3.1-0.20240129145446-ca4fba98f6d1 h1:hkeHftnhRuJoT5FrfF97gEtb5aY351SWEjZPaTb6D+Y=
 github.com/multiversx/mx-chain-scenario-go v1.3.1-0.20240129145446-ca4fba98f6d1/go.mod h1:ndk45i9J9McuCJpTcgiaK4ocd0yhnBBCPrlFwO6GRcs=
-github.com/multiversx/mx-chain-storage-go v1.0.15-0.20240129144933-b1c0d642d7f8 h1:/EYv/HGX0OKbeNFt667J0yZRtuJiZH0lEK8YtobuH/c=
-github.com/multiversx/mx-chain-storage-go v1.0.15-0.20240129144933-b1c0d642d7f8/go.mod h1:zl1A6teNe39T8yhdZlkX3ckm5aLYrMIJJZ6Ord1E71M=
+github.com/multiversx/mx-chain-storage-go v1.0.15-0.20240304133242-faaf1d20b087 h1:liZ6PL4Audkpkx4vCBngGzC48VZUpjjZd+p2mgarrt0=
+github.com/multiversx/mx-chain-storage-go v1.0.15-0.20240304133242-faaf1d20b087/go.mod h1:zl1A6teNe39T8yhdZlkX3ckm5aLYrMIJJZ6Ord1E71M=
 github.com/multiversx/mx-chain-vm-common-go v1.5.12-0.20240129145149-4fe61574f566 h1:zImJa/r6B5L2OLWbKTn5io53U11PPGDla12H2OaJ9y0=
 github.com/multiversx/mx-chain-vm-common-go v1.5.12-0.20240129145149-4fe61574f566/go.mod h1:OUyhCFqZKqUk1uaPsenyPDwO1830SlHNDU7Q7b6CBVI=
 github.com/multiversx/mx-chain-vm-go v1.5.27-0.20240129150501-7c828af05c83 h1:G/d9aplnwP/9MrLE3gcANEpGfn5e8ZZufijPv2XVUfw=

integrationTests/vm/wasm/delegation/testRunner.go

@@ -53,8 +53,7 @@ func RunDelegationStressTest(
 		MaxBatchSize:      45000,
 		MaxOpenFiles:      10,
 	}
-	dbConfigHandler := factory.NewDBConfigHandler(dbConfig)
-	persisterFactory, err := factory.NewPersisterFactory(dbConfigHandler)
+	persisterFactory, err := factory.NewPersisterFactory(dbConfig)
 	if err != nil {
 		return nil, err
 	}

process/smartContract/hooks/blockChainHook.go

@@ -826,8 +826,7 @@ func (bh *BlockChainHookImpl) makeCompiledSCStorage() error {
 	dbConfig := factory.GetDBFromConfig(bh.configSCStorage.DB)
 	dbConfig.FilePath = path.Join(bh.workingDir, defaultCompiledSCPath, bh.configSCStorage.DB.FilePath)
 
-	dbConfigHandler := factory.NewDBConfigHandler(bh.configSCStorage.DB)
-	persisterFactory, err := factory.NewPersisterFactory(dbConfigHandler)
+	persisterFactory, err := factory.NewPersisterFactory(bh.configSCStorage.DB)
 	if err != nil {
 		return err
 	}

storage/constants.go

@@ -1,14 +1,14 @@
 package storage
 
 import (
-	"github.com/multiversx/mx-chain-storage-go/storageUnit"
+	"github.com/multiversx/mx-chain-storage-go/common"
 )
 
 // MaxRetriesToCreateDB represents the maximum number of times to try to create DB if it failed
-const MaxRetriesToCreateDB = storageUnit.MaxRetriesToCreateDB
+const MaxRetriesToCreateDB = common.MaxRetriesToCreateDB
 
 // SleepTimeBetweenCreateDBRetries represents the number of seconds to sleep between DB creates
-const SleepTimeBetweenCreateDBRetries = storageUnit.SleepTimeBetweenCreateDBRetries
+const SleepTimeBetweenCreateDBRetries = common.SleepTimeBetweenCreateDBRetries
 
 // PathShardPlaceholder represents the placeholder for the shard ID in paths
 const PathShardPlaceholder = "[S]"

storage/factory/dbConfigHandler.go

@@ -11,32 +11,26 @@ import (
 )
 
 const (
-	dbConfigFileName = "config.toml"
-	defaultType      = "LvlDBSerial"
+	dbConfigFileName         = "config.toml"
+	defaultType              = "LvlDBSerial"
+	defaultBatchDelaySeconds = 2
+	defaultMaxBatchSize      = 100
+	defaultMaxOpenFiles      = 10
+	defaultUseTmpAsFilePath  = false
 )
 
 var (
 	errInvalidConfiguration = errors.New("invalid configuration")
 )
 
 type dbConfigHandler struct {
-	dbType              string
-	batchDelaySeconds   int
-	maxBatchSize        int
-	maxOpenFiles        int
-	shardIDProviderType string
-	numShards           int32
+	conf config.DBConfig
 }
 
 // NewDBConfigHandler will create a new db config handler instance
 func NewDBConfigHandler(config config.DBConfig) *dbConfigHandler {
 	return &dbConfigHandler{
-		dbType:              config.Type,
-		batchDelaySeconds:   config.BatchDelaySeconds,
-		maxBatchSize:        config.MaxBatchSize,
-		maxOpenFiles:        config.MaxOpenFiles,
-		shardIDProviderType: config.ShardIDProviderType,
-		numShards:           config.NumShards,
+		conf: config,
 	}
 }
 
@@ -56,9 +50,10 @@ func (dh *dbConfigHandler) GetDBConfig(path string) (*config.DBConfig, error) {
 	if !empty {
 		dbConfig := &config.DBConfig{
 			Type:              defaultType,
-			BatchDelaySeconds: dh.batchDelaySeconds,
-			MaxBatchSize:      dh.maxBatchSize,
-			MaxOpenFiles:      dh.maxOpenFiles,
+			BatchDelaySeconds: dh.conf.BatchDelaySeconds,
+			MaxBatchSize:      dh.conf.MaxBatchSize,
+			MaxOpenFiles:      dh.conf.MaxOpenFiles,
+			UseTmpAsFilePath:  dh.conf.UseTmpAsFilePath,
 		}
 
 		log.Debug("GetDBConfig: loaded default db config",
@@ -68,20 +63,11 @@ func (dh *dbConfigHandler) GetDBConfig(path string) (*config.DBConfig, error) {
 		return dbConfig, nil
 	}
 
-	dbConfig := &config.DBConfig{
-		Type:                dh.dbType,
-		BatchDelaySeconds:   dh.batchDelaySeconds,
-		MaxBatchSize:        dh.maxBatchSize,
-		MaxOpenFiles:        dh.maxOpenFiles,
-		ShardIDProviderType: dh.shardIDProviderType,
-		NumShards:           dh.numShards,
-	}
-
 	log.Debug("GetDBConfig: loaded db config from main config file",
-		"configuration", fmt.Sprintf("%+v", dbConfig),
+		"configuration", fmt.Sprintf("%+v", dh.conf),
 	)
 
-	return dbConfig, nil
+	return &dh.conf, nil
 }
 
 func readCorrectConfigurationFromToml(dbConfig *config.DBConfig, filePath string) error {

storage/factory/export_test.go

@@ -25,3 +25,8 @@ func NewPersisterCreator(config config.DBConfig) *persisterCreator {
 func (pc *persisterCreator) CreateShardIDProvider() (storage.ShardIDProvider, error) {
 	return pc.createShardIDProvider()
 }
+
+// GetTmpFilePath -
+func GetTmpFilePath(path string) (string, error) {
+	return getTmpFilePath(path)
+}

storage/factory/openStorage.go

@@ -3,7 +3,6 @@ package factory
 import (
 	"fmt"
 	"path/filepath"
-	"time"
 
 	"github.com/multiversx/mx-chain-core-go/core/check"
 	"github.com/multiversx/mx-chain-go/config"
@@ -56,8 +55,7 @@ func (o *openStorageUnits) GetMostRecentStorageUnit(dbConfig config.DBConfig) (s
 		return nil, err
 	}
 
-	dbConfigHandler := NewDBConfigHandler(dbConfig)
-	persisterFactory, err := NewPersisterFactory(dbConfigHandler)
+	persisterFactory, err := NewPersisterFactory(dbConfig)
 	if err != nil {
 		return nil, err
 	}
@@ -74,7 +72,7 @@ func (o *openStorageUnits) GetMostRecentStorageUnit(dbConfig config.DBConfig) (s
 
 	persisterPath := o.getPersisterPath(pathWithoutShard, mostRecentShard, dbConfig)
 
-	persister, err := createDB(persisterFactory, persisterPath)
+	persister, err := persisterFactory.CreateWithRetries(persisterPath)
 	if err != nil {
 		return nil, err
 	}
@@ -112,13 +110,12 @@ func (o *openStorageUnits) OpenDB(dbConfig config.DBConfig, shardID uint32, epoc
 	parentDir := o.latestStorageDataProvider.GetParentDirectory()
 	pathWithoutShard := o.getPathWithoutShard(parentDir, epoch)
 	persisterPath := o.getPersisterPath(pathWithoutShard, fmt.Sprintf("%d", shardID), dbConfig)
-	dbConfigHandler := NewDBConfigHandler(dbConfig)
-	persisterFactory, err := NewPersisterFactory(dbConfigHandler)
+	persisterFactory, err := NewPersisterFactory(dbConfig)
 	if err != nil {
 		return nil, err
 	}
 
-	persister, err := createDB(persisterFactory, persisterPath)
+	persister, err := persisterFactory.CreateWithRetries(persisterPath)
 	if err != nil {
 		return nil, err
 	}
@@ -131,21 +128,6 @@ func (o *openStorageUnits) OpenDB(dbConfig config.DBConfig, shardID uint32, epoc
 	return storageunit.NewStorageUnit(lruCache, persister)
 }
 
-func createDB(persisterFactory *PersisterFactory, persisterPath string) (storage.Persister, error) {
-	var persister storage.Persister
-	var err error
-	for i := 0; i < storage.MaxRetriesToCreateDB; i++ {
-		persister, err = persisterFactory.Create(persisterPath)
-		if err == nil {
-			return persister, nil
-		}
-		log.Warn("Create Persister failed", "path", persisterPath, "error", err)
-		//TODO: extract this in a parameter and inject it
-		time.Sleep(storage.SleepTimeBetweenCreateDBRetries)
-	}
-	return nil, err
-}
-
 func (o *openStorageUnits) getMostUpToDateDirectory(
 	dbConfig config.DBConfig,
 	pathWithoutShard string,

storage/factory/persisterCreator.go

@@ -5,39 +5,29 @@ import (
 	"github.com/multiversx/mx-chain-go/storage"
 	"github.com/multiversx/mx-chain-go/storage/database"
 	"github.com/multiversx/mx-chain-go/storage/storageunit"
+	"github.com/multiversx/mx-chain-storage-go/factory"
 )
 
 const minNumShards = 2
 
 // persisterCreator is the factory which will handle creating new persisters
 type persisterCreator struct {
-	dbType              string
-	batchDelaySeconds   int
-	maxBatchSize        int
-	maxOpenFiles        int
-	shardIDProviderType string
-	numShards           int32
+	conf config.DBConfig
 }
 
 func newPersisterCreator(config config.DBConfig) *persisterCreator {
 	return &persisterCreator{
-		dbType:              config.Type,
-		batchDelaySeconds:   config.BatchDelaySeconds,
-		maxBatchSize:        config.MaxBatchSize,
-		maxOpenFiles:        config.MaxOpenFiles,
-		shardIDProviderType: config.ShardIDProviderType,
-		numShards:           config.NumShards,
+		conf: config,
 	}
 }
 
 // Create will create the persister for the provided path
-// TODO: refactor to use max tries mechanism
 func (pc *persisterCreator) Create(path string) (storage.Persister, error) {
 	if len(path) == 0 {
 		return nil, storage.ErrInvalidFilePath
 	}
 
-	if pc.numShards < minNumShards {
+	if pc.conf.NumShards < minNumShards {
 		return pc.CreateBasePersister(path)
 	}
 
@@ -50,23 +40,23 @@ func (pc *persisterCreator) Create(path string) (storage.Persister, error) {
 
 // CreateBasePersister will create base the persister for the provided path
 func (pc *persisterCreator) CreateBasePersister(path string) (storage.Persister, error) {
-	var dbType = storageunit.DBType(pc.dbType)
-	switch dbType {
-	case storageunit.LvlDB:
-		return database.NewLevelDB(path, pc.batchDelaySeconds, pc.maxBatchSize, pc.maxOpenFiles)
-	case storageunit.LvlDBSerial:
-		return database.NewSerialDB(path, pc.batchDelaySeconds, pc.maxBatchSize, pc.maxOpenFiles)
-	case storageunit.MemoryDB:
-		return database.NewMemDB(), nil
-	default:
-		return nil, storage.ErrNotSupportedDBType
+	var dbType = storageunit.DBType(pc.conf.Type)
+
+	argsDB := factory.ArgDB{
+		DBType:            dbType,
+		Path:              path,
+		BatchDelaySeconds: pc.conf.BatchDelaySeconds,
+		MaxBatchSize:      pc.conf.MaxBatchSize,
+		MaxOpenFiles:      pc.conf.MaxOpenFiles,
 	}
+
+	return storageunit.NewDB(argsDB)
 }
 
 func (pc *persisterCreator) createShardIDProvider() (storage.ShardIDProvider, error) {
-	switch storageunit.ShardIDProviderType(pc.shardIDProviderType) {
+	switch storageunit.ShardIDProviderType(pc.conf.ShardIDProviderType) {
 	case storageunit.BinarySplit:
-		return database.NewShardIDProvider(pc.numShards)
+		return database.NewShardIDProvider(pc.conf.NumShards)
 	default:
 		return nil, storage.ErrNotSupportedShardIDProviderType
 	}

storage/factory/persisterCreator_test.go

@@ -38,6 +38,19 @@ func TestPersisterCreator_Create(t *testing.T) {
 		require.Equal(t, storage.ErrInvalidFilePath, err)
 	})
 
+	t.Run("use tmp as file path", func(t *testing.T) {
+		t.Parallel()
+
+		conf := createDefaultDBConfig()
+		conf.UseTmpAsFilePath = true
+
+		pc := factory.NewPersisterCreator(conf)
+
+		p, err := pc.Create("path1")
+		require.Nil(t, err)
+		require.NotNil(t, p)
+	})
+
 	t.Run("should create non sharded persister", func(t *testing.T) {
 		t.Parallel()