Skip to content

Commit

Permalink
Experimental ZIP support (#132)
Browse files Browse the repository at this point in the history 
This adds zip support to GB, GBC, NES, SNES, SMS, GG, COL, PCE, LYNX

The first file in the ZIP is loaded.

Still to fix:
- Be smarter about picking a file in the zip
- Reduce unzip memory usage to allow loading larger ROMs
- Remove the extension from cover art so that both game.zip and game.nes map to game.png, instead of game.nes.png and game.zip.png
- Maybe use miniz' ZIP implementation instead of my own
  • Loading branch information
@ducalex
ducalex committed Jul 16, 2024
1 parent 2800e49 commit c7316f3
Show file tree
Hide file tree
Showing 9 changed files with 139 additions and 30 deletions.
51 changes: 47 additions & 4 deletions components/retro-go/rg_storage.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -413,6 +413,49 @@ bool rg_storage_scandir(const char *path, rg_scandir_cb_t *callback, void *arg,
return true;
}

bool rg_storage_read_file(const char *path, void **data_out, size_t *data_len)
{
RG_ASSERT(data_out && data_len, "Bad param");
CHECK_PATH(path);

FILE *fp = fopen(path, "rb");
if (!fp)
{
RG_LOGE("Fopen failed");
return false;
}

size_t data_align = 0x4000;
size_t file_size;
void *file_data;

fseek(fp, 0, SEEK_END);
file_size = ftell(fp);
fseek(fp, 0, SEEK_SET);

file_data = malloc(((file_size & ~data_align) + data_align));
if (!file_data)
{
RG_LOGE("Memory allocation failed");
fclose(fp);
return false;
}

if (!fread(file_data, file_size, 1, fp))
{
RG_LOGE("File read failed");
free(file_data);
fclose(fp);
return false;
}

fclose(fp);

*data_out = file_data;
*data_len = file_size;
return true;
}

#if RG_HAVE_MINIZ
/**
* This is a minimal UNZIP implementation that utilizes only the miniz primitives found in ESP32's ROM.
Expand Down Expand Up @@ -443,17 +486,17 @@ bool rg_storage_unzip_file(const char *zip_path, const char *filter, void **data
RG_ASSERT(data_out && data_len, "Bad param");
CHECK_PATH(zip_path);

zip_header_t header = {0};
size_t data_align = 0x2000;
int header_pos = 0;

FILE *fp = fopen(zip_path, "rb");
if (!fp)
{
RG_LOGE("Fopen failed");
return false;
}

zip_header_t header = {0};
size_t data_align = 0x4000;
int header_pos = 0;

// Very inefficient, we should read a block at a time and search it for a header. But I'm lazy.
// Thankfully the header is usually found on the very first read :)
for (header_pos = 0; !feof(fp); ++header_pos)
Expand Down
22 changes: 13 additions & 9 deletions gwenesis/main/main.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -233,15 +233,19 @@ void app_main(void)

RG_LOGI("Genesis start\n");

FILE *fp = fopen(app->romPath, "rb");
if (!fp)
RG_PANIC("Rom load failed");
fseek(fp, 0, SEEK_END);
size_t rom_size = ftell(fp);
void *rom_data = malloc((rom_size & ~0xFFFF) + 0x10000);
fseek(fp, 0, SEEK_SET);
fread(rom_data, 1, rom_size, fp);
fclose(fp);
size_t rom_size;
void *rom_data;

if (rg_extension_match(app->romPath, "zip"))
{
if (!rg_storage_unzip_file(app->romPath, NULL, &rom_data, &rom_size))
RG_PANIC("ROM file unzipping failed!");
}
else
{
if (!rg_storage_read_file(app->romPath, &rom_data, &rom_size))
RG_PANIC("ROM load failed!");
}

RG_LOGI("load_cartridge(%p, %d)\n", rom_data, rom_size);
load_cartridge(rom_data, rom_size);
Expand Down
20 changes: 10 additions & 10 deletions launcher/main/applications.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -673,18 +673,18 @@ static void application(const char *desc, const char *name, const char *exts, co

void applications_init(void)
{
application("Nintendo Entertainment System", "nes", "nes fc fds nsf", "retro-core", 16);
application("Super Nintendo", "snes", "smc sfc", "retro-core", 0);
application("Nintendo Gameboy", "gb", "gb gbc", "retro-core", 0);
application("Nintendo Gameboy Color", "gbc", "gbc gb", "retro-core", 0);
application("Nintendo Entertainment System", "nes", "nes fc fds nsf zip", "retro-core", 16);
application("Super Nintendo", "snes", "smc sfc zip", "retro-core", 0);
application("Nintendo Gameboy", "gb", "gb gbc zip", "retro-core", 0);
application("Nintendo Gameboy Color", "gbc", "gbc gb zip", "retro-core", 0);
application("Nintendo Game & Watch", "gw", "gw", "retro-core", 0);
// application("Sega SG-1000", "sg1", "sms sg sg1", "retro-core", 0);
application("Sega Master System", "sms", "sms sg", "retro-core", 0);
application("Sega Game Gear", "gg", "gg", "retro-core", 0);
application("Sega Mega Drive", "md", "md gen bin", "gwenesis", 0);
application("Coleco ColecoVision", "col", "col rom", "retro-core", 0);
application("NEC PC Engine", "pce", "pce", "retro-core", 0);
application("Atari Lynx", "lnx", "lnx", "retro-core", 64);
application("Sega Master System", "sms", "sms sg zip", "retro-core", 0);
application("Sega Game Gear", "gg", "gg zip", "retro-core", 0);
application("Sega Mega Drive", "md", "md gen bin zip", "gwenesis", 0);
application("Coleco ColecoVision", "col", "col rom zip", "retro-core", 0);
application("NEC PC Engine", "pce", "pce zip", "retro-core", 0);
application("Atari Lynx", "lnx", "lnx zip", "retro-core", 64);
// application("Atari 2600", "a26", "a26", "stella-go", 0);
// application("Neo Geo Pocket Color", "ngp", "ngp ngc", "ngpocket-go", 0);
application("DOOM", "doom", "wad", "prboom-go", 0);

This comment has been minimized.

Sign in to view

Copy link
@tomvanbraeckel

tomvanbraeckel Jul 21, 2024

Contributor

May I ask, is there a reason not to do Doom? Just to know if I should add it or not.

In any case, I did some tests and was happy to see it compresses very well!

Brings a full Doom to the size of a Squashware Doom, a Squashware Doom to the size of a Squashware 1 Level Doom, and a Squashware 1 Level doom to the size of... well 328KB :-D

-rw-r--r--  1 user user 4.1M Jun 15 08:53 Doom1_4MB.wad
-rw-r--r--  1 user user 1.7M Jul 21 19:43 Doom1_4MB.wad.zip
-rw-r--r--  1 user user 1.8M Mar 23 18:51 squashware_doom1_1.7MB.wad
-rw-r--r--  1 user user 808K Jul 21 19:47 squashware_doom1_1.7MB.wad.zip
-rw-r--r--  1 user user 667K Mar 23 18:51 squashware_doom1_666KB.wad
-rw-r--r--  1 user user 328K Jul 21 19:48 squashware_doom1_666KB.wad.zip

I imagine the RAM usage might be a bit high. On the other hand, we have 8MB of PSRAM on the Fri3D Camp 2024 Badge, which I assume is enough, but I have no idea.

This comment has been minimized.

Sign in to view

Copy link
@ducalex

ducalex Jul 21, 2024

Author Owner

Realistically we'd need to use a compression library that provides stdio wrappers because we need the possibility to seek and we can't preload the entire WAD in memory. zlib's gzopen can do it but I doubt it's very efficient because deflate isn't block-based so I'm guessing it has to redecompress the entire stream from the start every time it seeks? We'd need to use another algorithm.

That being said many years ago I did add memory-based WAD support to our DOOM port so it would be possible to unzip an entire WAD to memory and use it that way but only the shareware would fit in 8MB PSRAM (and it won't work at all on ESP32 4MB, so I'm not going to merge the sample code below).

You could try the following to add support to DOOM:

// Add this in `D_Addfile`
  if (rg_extension_match(file, "zip"))
  {
    wadfile_info_t wad = {0};
    if (rg_storage_unzip_file(file, NULL, &wad.data, &wad.size))
    {
      wad.name = strdup(file);
      // Some of the code expects to see .wad in the name
      strcpy(wad.name + strlen(wad.name) - 3, "wad");
      wadfiles[numwadfiles++] = wad;
      return true;
    }
  }

This comment has been minimized.

Sign in to view

Copy link
@tomvanbraeckel

tomvanbraeckel Jul 22, 2024

Contributor

I tested zip support on NES, GB, GBC and it all works! I think I will include it in the release - it's just too good to save so much space on the very limited internal flash!

As for Doom, I added your suggestion (3aae8ee) and added "zip" to launcher/main/applications.c so it allows me to select the zip file but then the "Select IWAD file" dialog shows up (bad sign) and obviously that one doesn't include the .wad.zip file to select.

I didn't have time to figure out what's wrong exactly - also not how prboom-go handles those WAD and IWAD files - but I thought I would report it already :-)

This comment has been minimized.

Sign in to view

Copy link
@tomvanbraeckel

tomvanbraeckel Jul 22, 2024

Contributor

Oh and BTW, romart works if the filename matches, so if I rename the romart file to ".gbc.zip.png" instead of ".gbc.png" then it works. Sure it would be nicer if that wasn't necessary but it's at least a simple workaround for now :-)

This comment has been minimized.

Sign in to view

Copy link
@tomvanbraeckel

tomvanbraeckel Jul 22, 2024

Contributor

Ah, the .zip starts with a P so it thinks it's a PWAD :-)

Fix:

diff --git a/prboom-go/main/main.c b/prboom-go/main/main.c
index dcd843ac..8ce87b1b 100644
--- a/prboom-go/main/main.c
+++ b/prboom-go/main/main.c
@@ -549,7 +549,9 @@ void app_main()
 
     if ((fp = fopen(app->romPath, "rb")))
     {
-        if (fgetc(fp) == 'P')
+        char firstChar = fgetc(fp);
+        char secondChar = fgetc(fp);
+        if (firstChar == 'P' && secondChar == 'W')
             pwad = app->romPath;
         else
             iwad = app->romPath;

It works now!!!

This comment has been minimized.

Sign in to view

Copy link
@ducalex

ducalex Jul 22, 2024

Author Owner

Oh yeah the I/P distinction will be an issue for zip. Your fix won't allow you to load compressed PWADs but it's probably not a big issue for you right now (they're mods, essentially).

This comment has been minimized.

Sign in to view

Copy link
@tomvanbraeckel

tomvanbraeckel Jul 24, 2024 •
edited
Loading

Contributor

Oh, I didn't know about those... any easy fix I could do to fix that, like comparing an additional character or something? Or is there no easy fix?

This comment has been minimized.

Sign in to view

Copy link
@ducalex

ducalex Jul 25, 2024

Author Owner

None that I can think of. because at this point the data is compressed. I'm reconsidering adding ZIP support to DOOM. If I do it I will make sure PWADs work correctly too :)
Expand Down
13 changes: 12 additions & 1 deletion retro-core/main/main_gbc.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -270,8 +270,19 @@ void gbc_main(void)
gnuboy_set_soundbuffer((void *)audioBuffer, sizeof(audioBuffer) / 2);

// Load ROM
if (gnuboy_load_rom_file(app->romPath) < 0)
if (rg_extension_match(app->romPath, "zip"))
{
void *data;
size_t size;
if (!rg_storage_unzip_file(app->romPath, NULL, &data, &size))
RG_PANIC("ROM file unzipping failed!");
if (gnuboy_load_rom(data, size) < 0)
RG_PANIC("ROM Loading failed!");
}
else if (gnuboy_load_rom_file(app->romPath) < 0)
{
RG_PANIC("ROM Loading failed!");
}

// Load BIOS
if (loadBIOSFile)
Expand Down
13 changes: 12 additions & 1 deletion retro-core/main/main_lynx.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -190,7 +190,18 @@ extern "C" void lynx_main(void)
app->tickRate = 60;

// Init emulator
lynx = new CSystem(app->romPath, MIKIE_PIXEL_FORMAT_16BPP_565_BE, app->sampleRate);
if (rg_extension_match(app->romPath, "zip"))
{
void *data;
size_t size;
if (!rg_storage_unzip_file(app->romPath, NULL, &data, &size))
RG_PANIC("ROM file unzipping failed!");
lynx = new CSystem((UBYTE*)data, size, MIKIE_PIXEL_FORMAT_16BPP_565_BE, app->sampleRate);
}
else
{
lynx = new CSystem(app->romPath, MIKIE_PIXEL_FORMAT_16BPP_565_BE, app->sampleRate);
}

if (lynx->mFileType == HANDY_FILETYPE_ILLEGAL)
{
Expand Down
16 changes: 14 additions & 2 deletions retro-core/main/main_nes.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -208,11 +208,23 @@ void nes_main(void)

nes = nes_init(SYS_DETECT, app->sampleRate, true, RG_BASE_PATH_BIOS "/fds_bios.bin");
if (!nes)
{
RG_PANIC("Init failed.");

int ret = -1;

if (rg_extension_match(app->romPath, "zip"))
{
void *data;
size_t size;
if (!rg_storage_unzip_file(app->romPath, NULL, &data, &size))
RG_PANIC("ROM file unzipping failed!");
ret = nes_insertcart(rom_loadmem(data, size));
}
else
{
ret = nes_loadfile(app->romPath);
}

int ret = nes_loadfile(app->romPath);
if (ret == -1)
RG_PANIC("ROM load failed.");
else if (ret == -2)
Expand Down
11 changes: 10 additions & 1 deletion retro-core/main/main_pce.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -219,7 +219,16 @@ void pce_main(void)

InitPCE(app->sampleRate, true);

if (LoadFile(app->romPath) != 0)
if (rg_extension_match(app->romPath, "zip"))
{
void *data;
size_t size;
if (!rg_storage_unzip_file(app->romPath, NULL, &data, &size))
RG_PANIC("ROM file unzipping failed!");
if (LoadCard(data, size) != 0)
RG_PANIC("ROM loading failed");
}
else if (LoadFile(app->romPath) != 0)
{
RG_PANIC("ROM loading failed");
}
Expand Down
11 changes: 10 additions & 1 deletion retro-core/main/main_sms.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -125,7 +125,16 @@ void sms_main(void)
else
option.console = 0;

if (!load_rom_file(app->romPath))
if (rg_extension_match(app->romPath, "zip"))
{
void *data;
size_t size;
if (!rg_storage_unzip_file(app->romPath, NULL, &data, &size))
RG_PANIC("ROM file unzipping failed!");
if (!load_rom(data, RG_MAX(0x4000, size), size))
RG_PANIC("ROM file loading failed!");
}
else if (!load_rom_file(app->romPath))
RG_PANIC("ROM file loading failed!");

bitmap.width = SMS_WIDTH;
Expand Down
12 changes: 11 additions & 1 deletion retro-core/main/main_snes.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -289,7 +289,17 @@ void snes_main(void)
if (!S9xInitGFX())
RG_PANIC("Graphics init failed!");

if (!LoadROM(app->romPath))
const char *filename = app->romPath;

if (rg_extension_match(filename, "zip"))
{
free(Memory.ROM); // Would be nice to reuse it directly...
if (!rg_storage_unzip_file(filename, NULL, (void **)&Memory.ROM, &Memory.ROM_Size))
RG_PANIC("ROM file unzipping failed!");
filename = NULL;
}

if (!LoadROM(filename))
RG_PANIC("ROM loading failed!");

#ifdef USE_BLARGG_APU
Expand Down

0 comments on commit c7316f3

Please sign in to comment.