Ottersync: make --all flag to seed .idx, .vi, etc... (#11540)

Co-authored-by: alex.sharov <[email protected]>
erigontech · Aug 15, 2024 · 8a28557 · 8a28557
1 parent c31eb64
commit 8a28557
Show file tree

Hide file tree

Showing 3 changed files with 36 additions and 8 deletions.
diff --git a/erigon-lib/downloader/downloader.go b/erigon-lib/downloader/downloader.go
@@ -2657,7 +2657,14 @@ func SeedableFiles(dirs datadir.Dirs, chainName string, all bool) ([]string, err
 	if err != nil {
 		return nil, err
 	}
-	files = append(append(append(files, l1...), l2...), l3...)
+	var l4 []string
+	if all {
+		l4, err = seedableStateFilesBySubDir(dirs.Snap, "accessor", all)
+		if err != nil {
+			return nil, err
+		}
+	}
+	files = append(append(append(append(files, l1...), l2...), l3...), l4...)
 	return files, nil
 }
 

diff --git a/erigon-lib/downloader/snaptype/files.go b/erigon-lib/downloader/snaptype/files.go
@@ -201,10 +201,18 @@ func SeedableV2Extensions() []string {
 	return []string{".seg"}
 }
 
+func AllV2Extensions() []string {
+	return []string{".seg", ".idx", ".txt"}
+}
+
 func SeedableV3Extensions() []string {
 	return []string{".kv", ".v", ".ef"}
 }
 
+func AllV3Extensions() []string {
+	return []string{".kv", ".v", ".ef", ".kvei", ".vi", ".efi", ".bt"}
+}
+
 func IsSeedableExtension(name string) bool {
 	for _, ext := range append(SeedableV2Extensions(), SeedableV3Extensions()...) {
 		if strings.HasSuffix(name, ext) {

diff --git a/erigon-lib/downloader/util.go b/erigon-lib/downloader/util.go
@@ -76,19 +76,24 @@ type torrentInfo struct {
 }
 
 func seedableSegmentFiles(dir string, chainName string, skipSeedableCheck bool) ([]string, error) {
-	files, err := dir2.ListFiles(dir, snaptype.SeedableV2Extensions()...)
+	extensions := snaptype.SeedableV2Extensions()
+	if skipSeedableCheck {
+		extensions = snaptype.AllV2Extensions()
+	}
+	files, err := dir2.ListFiles(dir, extensions...)
 	if err != nil {
 		return nil, err
 	}
+
 	res := make([]string, 0, len(files))
 	for _, fPath := range files {
 
 		_, name := filepath.Split(fPath)
-		if !snaptype.IsCorrectFileName(name) {
+		if !skipSeedableCheck && !snaptype.IsCorrectFileName(name) {
 			continue
 		}
 		ff, isStateFile, ok := snaptype.ParseFileName(dir, name)
-		if !ok || isStateFile {
+		if !skipSeedableCheck && (!ok || isStateFile) {
 			continue
 		}
 		if !skipSeedableCheck && !snapcfg.Seedable(chainName, ff) {
@@ -99,17 +104,21 @@ func seedableSegmentFiles(dir string, chainName string, skipSeedableCheck bool)
 	return res, nil
 }
 
-func seedableStateFilesBySubDir(dir, subDir string, skipSeedable bool) ([]string, error) {
+func seedableStateFilesBySubDir(dir, subDir string, skipSeedableCheck bool) ([]string, error) {
 	historyDir := filepath.Join(dir, subDir)
 	dir2.MustExist(historyDir)
-	files, err := dir2.ListFiles(historyDir, snaptype.SeedableV3Extensions()...)
+	extensions := snaptype.SeedableV3Extensions()
+	if skipSeedableCheck {
+		extensions = snaptype.AllV3Extensions()
+	}
+	files, err := dir2.ListFiles(historyDir, extensions...)
 	if err != nil {
 		return nil, err
 	}
 	res := make([]string, 0, len(files))
 	for _, fPath := range files {
 		_, name := filepath.Split(fPath)
-		if !skipSeedable && !snaptype.E3Seedable(name) {
+		if !skipSeedableCheck && !snaptype.E3Seedable(name) {
 			continue
 		}
 		res = append(res, filepath.Join(subDir, name))
@@ -263,7 +272,11 @@ func AllTorrentPaths(dirs datadir.Dirs) ([]string, error) {
 	if err != nil {
 		return nil, err
 	}
-	files = append(append(append(files, l1...), l2...), l3...)
+	l4, err := dir2.ListFiles(dirs.SnapAccessors, ".torrent")
+	if err != nil {
+		return nil, err
+	}
+	files = append(append(append(append(files, l1...), l2...), l3...), l4...)
 	return files, nil
 }