rg_storage: scandir now uses a callback system to save memory

Previously on large lists (2000+) it could fail because no contiguous block large enough could be allocated.

I tried a few methods to work around it (make multiple smaller allocations, use a bucket system, use a linked list).

But they all had the issue that they had to be freed correctly + they were slower + they caused fragmentation.

Now it no longer returns an array but takes a callback to run on every file. Which makes it a tad more annoying to use, but oh well.

On the bright side, filenames are no longer limited to 90 characters!

components/retro-go/rg_gui.c

@@ -882,44 +882,48 @@ void rg_gui_alert(const char *title, const char *message)
     rg_gui_dialog(title, message ? options : options + 1, -1);
 }
 
+typedef struct
+{
+    rg_gui_option_t options[21];
+    size_t count;
+    bool (*validator)(const char *path);
+} file_picker_opts_t;
+
+static bool file_picker_cb(const rg_scandir_t *entry, void *arg)
+{
+    file_picker_opts_t *f = arg;
+    if (f->validator && !(f->validator)(entry->path))
+        return false;
+    f->options[f->count].arg = f->count;
+    f->options[f->count].flags = 1;
+    f->options[f->count].label = strdup(entry->name);
+    f->count++;
+    f->options[f->count] = (rg_gui_option_t)RG_DIALOG_CHOICE_LAST;
+    return f->count < 19;
+}
+
 char *rg_gui_file_picker(const char *title, const char *path, bool (*validator)(const char *path))
 {
-    rg_scandir_t *files = rg_storage_scandir(path, validator, false);
+    file_picker_opts_t options = {
+        .options = {RG_DIALOG_CHOICE_LAST},
+        .count = 0,
+        .validator = validator,
+    };
 
-    if (!files || !files[0].is_valid)
+    if (!rg_storage_scandir(path, file_picker_cb, &options, 0) || options.count < 1)
     {
-        free(files);
         rg_gui_alert(title, "Folder is empty.");
         return NULL;
     }
 
-    size_t count = 0;
-    while (files[count].is_valid)
-        count++;
-
-    RG_LOGI("count=%d\n", count);
-
-    // Unfortunately, at this time, any more than that will blow the stack.
-    count = RG_MIN(count, 20);
-
-    rg_gui_option_t *options = calloc(count + 1, sizeof(rg_gui_option_t));
-    for (size_t i = 0; i < count; ++i)
-    {
-        // To do: check extension...
-        options[i].arg = i;
-        options[i].flags = 1;
-        options[i].label = files[i].name;
-    }
-    options[count] = (rg_gui_option_t)RG_DIALOG_CHOICE_LAST;
-
-    int sel = rg_gui_dialog(title, options, 0);
+    int sel = rg_gui_dialog(title, options.options, 0);
     char *filename = NULL;
 
-    if (sel >= 0 && sel < count)
-        filename = strdup(files[sel].name);
+    if (sel >= 0 && sel < options.count)
+        filename = strdup(options.options[sel].label);
 
-    free(options);
-    free(files);
+    for (size_t i = 0; i < options.count; ++i)
+        free((void *)(options.options[i].label));
 
     return filename;
 }

components/retro-go/rg_storage.c

@@ -269,78 +269,58 @@ static int scandir_natural_sort(const void *a, const void *b)
     return 0;
 }
 
-// FIXME: rg_scandir_t should probably be {count, items[]} to avoid walking the array to get the count...
-rg_scandir_t *rg_storage_scandir(const char *path, bool (*validator)(const char *path), uint32_t flags)
+bool rg_storage_scandir(const char *path, rg_scandir_cb_t *callback, void *arg, uint32_t flags)
 {
-    RG_ASSERT(path, "Bad param");
+    RG_ASSERT(path && callback, "Bad param");
 
     DIR *dir = opendir(path);
     if (!dir)
-        return NULL;
-
-    rg_scandir_t *results = calloc(1, sizeof(rg_scandir_t));
-    size_t capacity = 0;
-    size_t count = 0;
-    struct dirent *ent;
-    struct stat statbuf;
+        return false;
 
     size_t path_len = strlen(path) + 1;
-    size_t name_maxlen = sizeof(results[0].name) - 1;
-    char fullpath[path_len + name_maxlen + 1];
-
-    sprintf(fullpath, "%s/", path);
+    struct stat statbuf;
+    char fullpath[RG_PATH_MAX + 1] = {0};
+    rg_scandir_t result = {
+        .path = strcat(strcpy(fullpath, path), "/"),
+        .name = fullpath + path_len,
+    };
 
-    while ((ent = readdir(dir)))
+    for (struct dirent *ent; (ent = readdir(dir));)
     {
         if (ent->d_name[0] == '.') // Ignore all dot files
             continue;
 
-        if (strlen(ent->d_name) > name_maxlen) // Filename is too long
-            continue;
-
-        strcpy(fullpath + path_len, ent->d_name);
-
-        if (validator && !validator(fullpath))
-            continue;
-
-        if (count + 1 >= capacity)
+        if (path_len + strlen(ent->d_name) >= sizeof(fullpath))
         {
-            capacity += 100;
-            void *temp = realloc(results, (capacity + 1) * sizeof(rg_scandir_t));
-            if (!temp)
-            {
-                RG_LOGW("Not enough memory to finish scan!\n");
-                break;
-            }
-            results = temp;
+            RG_LOGE("File path too long '%s/%s'", path, ent->d_name);
+            continue;
         }
 
-        rg_scandir_t *result = &results[count++];
-
-        strcpy(result->name, ent->d_name);
-        result->is_valid = 1;
+        strcpy(result.name, ent->d_name);
     #if defined(DT_REG) && defined(DT_DIR)
-        result->is_file = ent->d_type == DT_REG;
-        result->is_dir = ent->d_type == DT_DIR;
+        result.is_file = ent->d_type == DT_REG;
+        result.is_dir = ent->d_type == DT_DIR;
     #else
+        // We're forced to stat() if the OS doesn't provide type via dirent
         flags |= RG_SCANDIR_STAT;
     #endif
 
         if ((flags & RG_SCANDIR_STAT) && stat(fullpath, &statbuf) == 0)
         {
-            result->is_file = S_ISREG(statbuf.st_mode);
-            result->is_dir = S_ISDIR(statbuf.st_mode);
-            result->size = statbuf.st_size;
-            result->mtime = statbuf.st_mtime;
+            result.is_file = S_ISREG(statbuf.st_mode);
+            result.is_dir = S_ISDIR(statbuf.st_mode);
+            result.size = statbuf.st_size;
+            result.mtime = statbuf.st_mtime;
         }
-    }
-    memset(&results[count], 0, sizeof(rg_scandir_t));
-    closedir(dir);
 
-    if (flags & RG_SCANDIR_SORT)
-    {
-        qsort(results, count, sizeof(rg_scandir_t), scandir_natural_sort);
+        if (!(callback)(&result, arg))
+        {
+            // Stop if the callback returns false
+            break;
+        }
     }
 
-    return results;
+    closedir(dir);
+
+    return true;
 }

components/retro-go/rg_storage.h

@@ -14,16 +14,18 @@
 #define RG_BASE_PATH_SAVES  RG_BASE_PATH "/saves"
 #define RG_BASE_PATH_THEMES RG_BASE_PATH "/themes"
 
-typedef struct __attribute__((packed))
+typedef struct
 {
-    uint8_t is_valid : 1;
-    uint8_t is_file  : 1;
-    uint8_t is_dir   : 1;
-    uint8_t unused   : 5;
-    char name[91];
-    int32_t mtime, size;
+    char *path;
+    char *name;
+    size_t size;
+    time_t mtime;
+    bool is_file;
+    bool is_dir;
 } rg_scandir_t;
 
+typedef bool (rg_scandir_cb_t)(const rg_scandir_t *file, void *arg);
+
 enum
 {
     RG_SCANDIR_STAT = 1, // This will populate file size
@@ -41,4 +43,4 @@ bool rg_storage_read_file(const char *path, void **data_ptr, size_t *data_len);
 bool rg_storage_write_file(const char *path, const void *data_ptr, const size_t data_len);
 bool rg_storage_delete(const char *path);
 bool rg_storage_mkdir(const char *dir);
-rg_scandir_t *rg_storage_scandir(const char *path, bool (*validator)(const char *path), uint32_t flags);
+bool rg_storage_scandir(const char *path, rg_scandir_cb_t *callback, void *arg, uint32_t flags);

launcher/main/applications.c

@@ -27,75 +27,85 @@ static bool crc_cache_dirty = true;
 static retro_app_t *apps[24];
 static int apps_count = 0;
 
-
-static const char *get_file_path(retro_file_t *file)
-{
-    static char buffer[RG_PATH_MAX + 1];
-    RG_ASSERT(file, "Bad param");
-    snprintf(buffer, RG_PATH_MAX, "%s/%s", file->folder, file->name);
-    return buffer;
-}
-
-static void scan_folder(retro_app_t *app, const char* path, void *parent)
+typedef struct
 {
-    RG_ASSERT(app && path, "Bad param");
+    const char *folder;
+    retro_app_t *app;
+} sc_arg_t;
 
-    RG_LOGI("Scanning directory %s\n", path);
+static void scan_folder(retro_app_t *app, const char* path, void *parent);
 
-    const char *folder = const_string(path);
-    rg_scandir_t *files = rg_storage_scandir(path, NULL, false);
+static bool scan_folder_cb(const rg_scandir_t *entry, void *arg)
+{
+    const char *folder = ((sc_arg_t*)arg)->folder;
+    retro_app_t *app = ((sc_arg_t*)arg)->app;
+    const char *ext = rg_extension(entry->name);
+    uint8_t is_valid = false;
+    uint8_t type = 0x00;
     char ext_buf[32];
 
-    for (rg_scandir_t *entry = files; entry && entry->is_valid; ++entry)
+    if (entry->is_file && ext != NULL)
     {
-        const char *ext = rg_extension(entry->name);
-        uint8_t is_valid = false;
-        uint8_t type = 0x00;
-
-        if (entry->is_file && ext != NULL)
-        {
-            snprintf(ext_buf, sizeof(ext_buf), " %s ", ext);
-            is_valid = strstr(app->extensions, rg_strtolower(ext_buf)) != NULL;
-            type = 0x00;
-        }
-        else if (entry->is_dir)
-        {
-            is_valid = true;
-            type = 0xFF;
-        }
+        snprintf(ext_buf, sizeof(ext_buf), " %s ", ext);
+        is_valid = strstr(app->extensions, rg_strtolower(ext_buf)) != NULL;
+        type = 0x00;
+    }
+    else if (entry->is_dir)
+    {
+        is_valid = true;
+        type = 0xFF;
+    }
 
-        if (!is_valid)
-            continue;
+    if (!is_valid)
+        return true;
 
-        if (app->files_count + 1 > app->files_capacity)
+    if (app->files_count + 1 > app->files_capacity)
+    {
+        size_t new_capacity = app->files_capacity * 1.5;
+        retro_file_t *new_buf = realloc(app->files, new_capacity * sizeof(retro_file_t));
+        if (!new_buf)
         {
-            size_t new_capacity = app->files_capacity * 1.5;
-            retro_file_t *new_buf = realloc(app->files, new_capacity * sizeof(retro_file_t));
-            if (!new_buf)
-            {
-                RG_LOGW("Ran out of memory, file scanning stopped at %d entries ...\n", app->files_count);
-                break;
-            }
-            app->files = new_buf;
-            app->files_capacity = new_capacity;
+            RG_LOGW("Ran out of memory, file scanning stopped at %d entries ...\n", app->files_count);
+            return false;
         }
+        app->files = new_buf;
+        app->files_capacity = new_capacity;
+    }
 
-        app->files[app->files_count++] = (retro_file_t) {
-            .name = strdup(entry->name),
-            .folder = folder,
-            .app = (void*)app,
-            .type = type,
-            .is_valid = true,
-        };
+    app->files[app->files_count++] = (retro_file_t) {
+        .name = strdup(entry->name),
+        .folder = folder,
+        .app = (void*)app,
+        .type = type,
+        .is_valid = true,
+    };
 
-        if (type == 0xFF)
-        {
-            retro_file_t *file = &app->files[app->files_count-1];
-            scan_folder(app, get_file_path(file), file);
-        }
+    if (type == 0xFF)
+    {
+        retro_file_t *file = &app->files[app->files_count-1];
+        scan_folder(app, entry->path, file);
     }
+    return true;
+}
+
+static void scan_folder(retro_app_t *app, const char* path, void *parent)
+{
+    RG_ASSERT(app && path, "Bad param");
+
+    RG_LOGI("Scanning directory %s\n", path);
 
-    free(files);
+    sc_arg_t data = {
+        .folder = const_string(path),
+        .app = app,
+    };
+    rg_storage_scandir(path, scan_folder_cb, &data, 0);
+}
+
+static bool scan_folder_cb2(const rg_scandir_t *entry, void *arg)
+{
+    retro_app_t *app = arg;
+    app->use_crc_covers = entry->name[1] == 0 && isalnum(entry->name[0]);
+    return app->use_crc_covers == false;
 }
 
 static void application_init(retro_app_t *app)
@@ -107,23 +117,23 @@ static void application_init(retro_app_t *app)
 
     // This checks if we have crc cover folders, the idea is to skip the crc later on if we don't!
     // It adds very little delay but it could become an issue if someone has thousands of named files...
-    rg_scandir_t *files = rg_storage_scandir(app->paths.covers, NULL, false);
-    if (!files)
+    if (rg_storage_scandir(app->paths.covers, scan_folder_cb2, app, 0) <= 0)
         rg_storage_mkdir(app->paths.covers);
-    else
-    {
-        for (rg_scandir_t *entry = files; entry->is_valid && !app->use_crc_covers; ++entry)
-            app->use_crc_covers = entry->name[1] == 0 && isalnum(entry->name[0]);
-        free(files);
-    }
-
     rg_storage_mkdir(app->paths.saves);
     rg_storage_mkdir(app->paths.roms);
     scan_folder(app, app->paths.roms, 0);
 
     app->initialized = true;
 }
 
+static const char *get_file_path(retro_file_t *file)
+{
+    static char buffer[RG_PATH_MAX + 1];
+    RG_ASSERT(file, "Bad param");
+    snprintf(buffer, RG_PATH_MAX, "%s/%s", file->folder, file->name);
+    return buffer;
+}
+
 static void application_start(retro_file_t *file, int load_state)
 {
     RG_ASSERT(file, "Unable to find file...");