Improve loadGpxItems with bulk load. Drop redundant progress call.

osmandapp · Sep 23, 2024 · f01edd8 · f01edd8
1 parent 15e30d1
commit f01edd8
Show file tree

Hide file tree

Showing 3 changed files with 33 additions and 9 deletions.
diff --git a/OsmAnd-shared/src/commonMain/kotlin/net/osmand/shared/gpx/GpxDbHelper.kt b/OsmAnd-shared/src/commonMain/kotlin/net/osmand/shared/gpx/GpxDbHelper.kt
@@ -4,14 +4,19 @@ import co.touchlab.stately.collections.ConcurrentMutableList
 import co.touchlab.stately.collections.ConcurrentMutableMap
 import co.touchlab.stately.concurrency.Synchronizable
 import co.touchlab.stately.concurrency.synchronize
+import kotlinx.coroutines.Dispatchers
+import kotlinx.coroutines.IO
+import kotlinx.coroutines.async
+import kotlinx.coroutines.awaitAll
+import kotlinx.coroutines.coroutineScope
 import kotlinx.coroutines.runBlocking
 import net.osmand.shared.api.SQLiteAPI.SQLiteConnection
 import net.osmand.shared.data.StringIntPair
 import net.osmand.shared.extensions.currentTimeMillis
 import net.osmand.shared.gpx.GpxReader.GpxDbReaderCallback
 import net.osmand.shared.io.KFile
 import net.osmand.shared.util.LoggerFactory
-import kotlin.random.Random
+
 
 object GpxDbHelper : GpxDbReaderCallback {
 	val log = LoggerFactory.getLogger("GpxDbHelper")
@@ -51,19 +56,34 @@ object GpxDbHelper : GpxDbReaderCallback {
 	private suspend fun loadGpxItems() {
 		val start = currentTimeMillis()
 		val items = getItems()
-		val fileExistenceMap = items.associate { it.file to it.file.exists() }
+		val startEx = currentTimeMillis()
+		val fileExistenceMap = getFileExistenceMap(items)
+		log.info("Time to getFileExistenceMap ${currentTimeMillis() - startEx} ms, ${items.size} items")
 
+		val itemsToCache = mutableMapOf<KFile, GpxDataItem>()
+		val itemsToRemove = mutableSetOf<KFile>()
 		items.forEach { item ->
 			val file = item.file
 			if (fileExistenceMap[file] == true) {
-				dataItems[file] = item
+				itemsToCache[file] = item
 			} else {
-				remove(file)
+				itemsToRemove.add(file)
 			}
 		}
+		putToCacheBulk(itemsToCache);
+		removeFromCacheBulk(itemsToRemove);
 		log.info("Time to loadGpxItems ${currentTimeMillis() - start} ms, ${items.size} items")
 	}
 
+	private suspend fun getFileExistenceMap(
+		items: List<GpxDataItem>,
+		batchSize: Int = 100
+	): Map<KFile, Boolean> = coroutineScope {
+		items.chunked(batchSize).map { batch ->
+			async(Dispatchers.IO) { batch.associate { it.file to it.file.exists() } }
+		}.awaitAll().fold(mutableMapOf()) { acc, map -> acc.apply { putAll(map) } }
+	}
+
 	private fun loadGpxDirItems() {
 		val start = currentTimeMillis()
 		val items = getDirItems()
@@ -96,6 +116,14 @@ object GpxDbHelper : GpxDbReaderCallback {
 		}
 	}
 
+	private fun putToCacheBulk(itemsToCache: Map<KFile, GpxDataItem>) {
+		dataItems.putAll(itemsToCache)
+	}
+
+	private fun removeFromCacheBulk(filesToRemove: Set<KFile>) {
+		dataItems.keys.removeAll(filesToRemove)
+	}
+
 	fun rename(currentFile: KFile, newFile: KFile): Boolean {
 		val success = database.rename(currentFile, newFile)
 		if (success) {

diff --git a/OsmAnd-shared/src/commonMain/kotlin/net/osmand/shared/gpx/TrackFolderLoaderTask.kt b/OsmAnd-shared/src/commonMain/kotlin/net/osmand/shared/gpx/TrackFolderLoaderTask.kt
@@ -64,10 +64,6 @@ class TrackFolderLoaderTask(
 			Companion.cachedRootFolder = TrackFolder(folder)
 		}
 
-		if (progress.isNotEmpty()) {
-			publishProgress(*progress.toTypedArray())
-		}
-
 		listener.tracksLoaded(folder)
 		log.info("Finished loading tracks. Took ${currentTimeMillis() - start}ms")
 

diff --git a/OsmAnd-shared/src/commonMain/kotlin/net/osmand/shared/gpx/filters/TrackFolderAnalysis.kt b/OsmAnd-shared/src/commonMain/kotlin/net/osmand/shared/gpx/filters/TrackFolderAnalysis.kt
@@ -31,7 +31,7 @@ class TrackFolderAnalysis(folder: TracksGroup) {
 			items.addAll(folder.getTrackItems())
 		}
 		var totalDistanceSum = 0.0
-		for (trackItem in items.sortedBy { it.name }) {
+		for (trackItem in items) {
 			val dataItem = trackItem.dataItem
 			val analysis = dataItem?.getAnalysis()
 			if (analysis != null) {