Finally use memory mapped file for saves

.github/workflows/build.yml

@@ -14,7 +14,12 @@ jobs:
           sudo apt-get install -y vulkan-tools libvulkan1 libvulkan-dev vulkan-validationlayers-dev spirv-tools
           git clone --recursive https://github.com/fmtlib/fmt
           cd fmt
-          cmake -B build
+          cmake -B build -DFMT_TEST=OFF
+          cd build
+          sudo make install
+          git clone --recursive https://github.com/mandreyel/mio
+          cd mio
+          cmake -B build -DBUILD_TESTING=False -DCMAKE_BUILD_TYPE=Release
           cd build
           sudo make install
       - name: Build Kaizen
@@ -46,6 +51,7 @@ jobs:
           vcpkg install sdl2[vulkan]:x64-windows
           vcpkg install fmt:x64-windows
           vcpkg install nlohmann-json:x64-windows
+          vcpkg install mio:x64-windows
       - name: Build Kaizen
         run: |
           cmake -B build -T clangcl -DCMAKE_TOOLCHAIN_FILE=C:/vcpkg/scripts/buildsystems/vcpkg.cmake -DCMAKE_BUILD_TYPE=Release -S src

src/CMakeLists.txt

@@ -6,6 +6,7 @@ set(CMAKE_CXX_STANDARD 17)
 
 find_package(SDL2 REQUIRED)
 find_package(fmt REQUIRED)
+find_package(mio REQUIRED)
 find_package(nlohmann_json REQUIRED)
 
 option(RAPIDJSON_BUILD_DOC "Build rapidjson documentation." OFF)
@@ -76,4 +77,4 @@ file(REMOVE
   ${PROJECT_BINARY_DIR}/resources/shader.vert)
 
 target_link_libraries(kaizen PUBLIC frontend frontend-imgui
-  discord-rpc imgui nfd parallel-rdp backend fmt::fmt nlohmann_json::nlohmann_json core registers interpreter mem mmio rsp SDL2::SDL2main SDL2::SDL2)
+  discord-rpc imgui nfd parallel-rdp backend fmt::fmt mio::mio nlohmann_json::nlohmann_json core registers interpreter mem mmio rsp SDL2::SDL2main SDL2::SDL2)

src/backend/core/Mem.cpp

@@ -22,25 +22,30 @@ Mem::Mem() {
 
 void Mem::Reset() {
   memset(rom.cart, 0, CART_SIZE);
-  if(sram)
+  flash.Reset();
-    memset(sram, 0, SRAM_SIZE);
+  if(sram.is_open()) {
+    std::error_code error;
+    sram.sync(error);
+    sram.unmap();
+  }
   mmio.Reset();
 }
 
 void Mem::LoadSRAM(SaveType saveType, fs::path path) {
   if(saveType == SAVE_SRAM_256k) {
-    if(sram) {
+    std::error_code error;
-      memset(sram, 0, SRAM_SIZE);
-    } else {
-      sram = (u8 *) calloc(SRAM_SIZE, 1);
-    }
     sramPath = path.replace_extension(".sram").string();
+    if(sram.is_mapped()) {
+      sram.sync(error);
+      if(error) { Util::panic("Could not sync {}", sramPath); }
+      sram.unmap();
+    }
+
     FILE *f = fopen(sramPath.c_str(), "rb");
     if (!f) {
       f = fopen(sramPath.c_str(), "wb");
-      fwrite(sram, 1, SRAM_SIZE, f);
+      u8* dummy = (u8*)calloc(SRAM_SIZE, 1);
-      fclose(f);
+      fwrite(dummy, 1, SRAM_SIZE, f);
-      f = fopen(sramPath.c_str(), "rb");
     }
 
     fseek(f, 0, SEEK_END);
@@ -49,9 +54,10 @@ void Mem::LoadSRAM(SaveType saveType, fs::path path) {
     if (actualSize != SRAM_SIZE) {
       Util::panic("Corrupt SRAM!");
     }
-
-    fread(sram, 1, SRAM_SIZE, f);
     fclose(f);
+    sram = mio::make_mmap_sink(
+      sramPath, 0, mio::map_entire_file, error);
+    if (error) { Util::panic("Could not open {}", sramPath); }
   }
 }
 
@@ -170,13 +176,13 @@ u8 Mem::Read8(n64::Registers &regs, u32 paddr) {
             Util::panic("Accessing cartridge backup with save type SAVE_EEPROM");
             break;
           case SAVE_FLASH_1m:
-            if(flash.saveData) {
+            if(flash.flash.is_open()) {
               return flash.Read8(paddr - CART_REGION_START_2_2);
             } else {
               Util::panic("Invalid backup Write8 if save data is not initialized");
             }
           case SAVE_SRAM_256k:
-            if(sram) {
+            if(sram.is_open()) {
               return sram[paddr - CART_REGION_START_2_2];
             }
         }
@@ -343,14 +349,14 @@ void Mem::Write8(Registers& regs, u32 paddr, u32 val) {
             Util::panic("Accessing cartridge backup with save type SAVE_EEPROM");
             break;
           case SAVE_FLASH_1m:
-            if(flash.saveData) {
+            if(flash.flash.is_open()) {
               flash.Write8(paddr - CART_REGION_START_2_2, val);
             } else {
               Util::panic("Invalid backup Write8 if save data is not initialized");
             }
             break;
           case SAVE_SRAM_256k:
-            if(sram) {
+            if(sram.is_open()) {
               sram[paddr - CART_REGION_START_2_2] = val;
             }
             break;
@@ -466,7 +472,7 @@ void Mem::Write32(Registers& regs, u32 paddr, u32 val) {
       case 0x80000000 ... 0xFFFFFFFF: break;
       case CART_REGION_2_2:
         if (saveType == SAVE_FLASH_1m) {
-          if(flash.saveData) {
+          if(flash.flash.is_open()) {
             flash.Write32(paddr - CART_REGION_START_2_2, val);
           } else {
             Util::panic("Invalid write to cartridge backup if save data is not initialized!");

src/backend/core/Mem.hpp

@@ -44,22 +44,16 @@ enum FlashState : u8 {
 
 struct Flash {
   Flash() = default;
-  ~Flash() {
+  ~Flash() = default;
-    FILE* f = fopen(saveDataPath.c_str(), "wb");
+  void Reset();
-    if(f) {
+  void Load(SaveType, std::string);
-      fwrite(saveData, 1, 1_mb, f);
-      fclose(f);
-    }
-  }
-  void Load(SaveType, fs::path);
   FlashState state{};
   u64 status{};
   size_t eraseOffs{};
   size_t writeOffs{};
   u8 writeBuf[128]{};
-  u8* saveData = nullptr;
+  mio::mmap_sink flash;
-  bool saveDataDirty = false;
+  std::string flashPath{};
-  std::string saveDataPath{};
 
   enum FlashCommands : u8 {
     FLASH_COMMAND_EXECUTE = 0xD2,
@@ -116,7 +110,7 @@ struct Flash {
       case Idle: Util::panic("Flash read byte while in state FLASH_STATE_IDLE");
       case Write: Util::panic("Flash read byte while in state FLASH_STATE_WRITE");
       case Read: {
-        u8 value = saveData[index];
+        u8 value = flash[index];
         Util::debug("Flash read byte in state read: index {:08X} = {:02X}", index, value);
         return value;
       }
@@ -136,9 +130,7 @@ struct Flash {
 };
 
 struct Mem {
-  ~Mem() {
+  ~Mem() = default;
-    free(sram);
-  }
   Mem();
   void Reset();
   void LoadSRAM(SaveType, fs::path);
@@ -197,7 +189,7 @@ private:
   friend struct AI;
   friend struct RSP;
   friend struct Core;
-  u8* sram;
+  mio::mmap_sink sram;
   u8 isviewer[ISVIEWER_SIZE]{};
   std::string sramPath{};
 

src/backend/core/mem/Flash.cpp

@@ -1,32 +1,45 @@
 #include <Mem.hpp>
 
 namespace n64 {
-void Flash::Load(SaveType saveType, fs::path path) {
+constexpr auto FLASH_SIZE = 1_mb;
+
+void Flash::Reset() {
+  if (flash.is_mapped()) {
+    std::error_code error;
+    flash.sync(error);
+    if (error) { Util::panic("Could not sync {}", flashPath); }
+    flash.unmap();
+  }
+}
+
+void Flash::Load(SaveType saveType, std::string path) {
   if(saveType == SAVE_FLASH_1m) {
-    if(saveData) {
+    flashPath = fs::path(path).replace_extension(".flash").string();
-      memset(saveData, 0xff, 1_mb);
+    std::error_code error;
-    } else {
+    if (flash.is_mapped()) {
-      saveData = (u8 *) malloc(1_mb);
+      flash.sync(error);
-      memset(saveData, 0xff, 1_mb);
+      if (error) { Util::panic("Could not sync {}", flashPath); }
+      flash.unmap();
     }
-    saveDataPath = path.replace_extension(".flash").string();
+
-    FILE *f = fopen(saveDataPath.c_str(), "rb");
+    FILE *f = fopen(flashPath.c_str(), "rb");
     if (!f) {
-      f = fopen(saveDataPath.c_str(), "wb");
+      f = fopen(flashPath.c_str(), "wb");
-      fwrite(saveData, 1, 1_mb, f);
+      u8* dummy = (u8*)calloc(FLASH_SIZE, 1);
-      fclose(f);
+      fwrite(dummy, 1, FLASH_SIZE, f);
-      f = fopen(saveDataPath.c_str(), "rb");
     }
 
     fseek(f, 0, SEEK_END);
     size_t actualSize = ftell(f);
     fseek(f, 0, SEEK_SET);
-    if (actualSize != 1_mb) {
+    if (actualSize != FLASH_SIZE) {
       Util::panic("Corrupt flash!");
     }
-
-    fread(saveData, 1, 1_mb, f);
     fclose(f);
+
+    flash = mio::make_mmap_sink(
+      flashPath, 0, mio::map_entire_file, error);
+    if (error) { Util::panic("Could not open {}", path); }
   }
 }
 
@@ -37,15 +50,13 @@ void Flash::CommandExecute() {
       break;
     case FlashState::Erase:
       for (int i = 0; i < 128; i++) {
-        saveData[eraseOffs + i] = 0xFF;
+        flash[eraseOffs + i] = 0xFF;
       }
-      saveDataDirty = true;
       break;
     case FlashState::Write:
       for (int i = 0; i < 128; i++) {
-        saveData[writeOffs + i] = writeBuf[i];
+        flash[writeOffs + i] = writeBuf[i];
       }
-      saveDataDirty = true;
       break;
     case FlashState::Read:
       Util::panic("Execute command when flash in read state");

src/backend/core/mmio/PIF.cpp

@@ -9,24 +9,37 @@
 #define MEMPAK_SIZE 32768
 
 namespace n64 {
-PIF::PIF() {
-  mempak = (u8*)calloc(MEMPAK_SIZE, 1);
-}
-
 void PIF::Reset() {
   memset(joybusDevices, 0, sizeof(JoybusDevice) * 6);
   memset(bootrom, 0, PIF_BOOTROM_SIZE);
   memset(ram, 0, PIF_RAM_SIZE);
+  std::error_code error;
+  if(mempak.is_mapped()) {
+    mempak.sync(error);
+    if (error) { Util::panic("Could not sync {}", mempakPath); }
+    mempak.unmap();
+  }
+  if(eeprom.is_mapped()) {
+    eeprom.sync(error);
+    if (error) { Util::panic("Could not sync {}", eepromPath); }
+    eeprom.unmap();
+  }
 }
 
-void PIF::LoadMempak(fs::path path) {
+void PIF::LoadMempak(std::string path) {
-  mempakPath = path.replace_extension(".mempak").string();
+  mempakPath = fs::path(path).replace_extension(".mempak").string();
+  std::error_code error;
+  if (mempak.is_mapped()) {
+    mempak.sync(error);
+    if (error) { Util::panic("Could not sync {}", mempakPath); }
+    mempak.unmap();
+  }
   FILE* f = fopen(mempakPath.c_str(), "rb");
   if (!f) {
     f = fopen(mempakPath.c_str(), "wb");
-    fwrite(mempak, 1, MEMPAK_SIZE, f);
+    u8* dummy = (u8*)calloc(MEMPAK_SIZE, 1);
-    fclose(f);
+    fwrite(dummy, 1, MEMPAK_SIZE, f);
-    f = fopen(mempakPath.c_str(), "rb");
+    free(dummy);
   }
 
   fseek(f, 0, SEEK_END);
@@ -35,9 +48,11 @@ void PIF::LoadMempak(fs::path path) {
   if (actualSize != MEMPAK_SIZE) {
     Util::panic("Corrupt mempak!");
   }
-
-  fread(mempak, 1, MEMPAK_SIZE, f);
   fclose(f);
+
+  mempak = mio::make_mmap_sink(
+    mempakPath, 0, mio::map_entire_file, error);
+  if (error) { Util::panic("Could not open {}", mempakPath); }
 }
 
 FORCE_INLINE size_t getSaveSize(SaveType saveType) {
@@ -57,20 +72,22 @@ FORCE_INLINE size_t getSaveSize(SaveType saveType) {
   }
 }
 
-void PIF::LoadEeprom(SaveType saveType, fs::path path) {
+void PIF::LoadEeprom(SaveType saveType, std::string path) {
   if(saveType == SAVE_EEPROM_16k || saveType == SAVE_EEPROM_4k) {
-    if (eeprom)
+    eepromPath = fs::path(path).replace_extension(".eeprom").string();
-      free(eeprom);
+    std::error_code error;
+    if (eeprom.is_mapped()) {
+      eeprom.sync(error);
+      if (error) { Util::panic("Could not sync {}", eepromPath); }
+      eeprom.unmap();
+    }
 
     eepromSize = getSaveSize(saveType);
-    eeprom = (u8 *) calloc(eepromSize, 1);
-    eepromPath = path.replace_extension(".eeprom").string();
     FILE *f = fopen(eepromPath.c_str(), "rb");
     if (!f) {
       f = fopen(eepromPath.c_str(), "wb");
-      fwrite(eeprom, 1, eepromSize, f);
+      u8* dummy = (u8*)calloc(eepromSize, 1);
-      fclose(f);
+      fwrite(dummy, 1, eepromSize, f);
-      f = fopen(eepromPath.c_str(), "rb");
     }
 
     fseek(f, 0, SEEK_END);
@@ -79,22 +96,11 @@ void PIF::LoadEeprom(SaveType saveType, fs::path path) {
     if (actualSize != eepromSize) {
       Util::panic("Corrupt eeprom!");
     }
+    fclose(f);
 
-    fread(eeprom, 1, eepromSize, f);
+    eeprom = mio::make_mmap_sink(
-    fclose(f);
+      eepromPath, 0, mio::map_entire_file, error);
-  }
+    if (error) { Util::panic("Could not open {}", eepromPath); }
-}
-
-PIF::~PIF() {
-  FILE* f = fopen(mempakPath.c_str(), "wb");
-  if(f) {
-    fwrite(mempak, 1, MEMPAK_SIZE, f);
-    fclose(f);
-  }
-  f = fopen(eepromPath.c_str(), "wb");
-  if(f) {
-    fwrite(eeprom, 1, eepromSize, f);
-    fclose(f);
   }
 }
 
@@ -257,23 +263,19 @@ void PIF::MempakRead(const u8* cmd, u8* res) const {
       break;
     case ACCESSORY_MEMPACK:
       if (offset <= MEMPAK_SIZE - 0x20) {
-        for (int i = 0; i < 32; i++) {
+        std::copy_n(mempak.begin() + offset, 32, res);
-          res[i] = mempak[offset + i];
-        }
       }
       break;
     case ACCESSORY_RUMBLE_PACK:
-      for (int i = 0; i < 32; i++) {
+      memset(res, 0x80, 32);
-        res[i] = 0x80;
-      }
       break;
   }
 
   // CRC byte
-  res[32] = data_crc(&res[0]);
+  res[32] = data_crc(res);
 }
 
-void PIF::MempakWrite(u8* cmd, u8* res) const {
+void PIF::MempakWrite(u8* cmd, u8* res) {
   // First two bytes in the command are the offset
   u16 offset = cmd[3] << 8;
   offset |= cmd[4];
@@ -288,9 +290,7 @@ void PIF::MempakWrite(u8* cmd, u8* res) const {
       break;
     case ACCESSORY_MEMPACK:
       if (offset <= MEMPAK_SIZE - 0x20) {
-        for (int i = 0; i < 32; i++) {
+        std::copy_n(cmd + 5, 32, mempak.begin() + offset);
-          mempak[offset + i] = cmd[5 + i];
-        }
       }
       break;
     case ACCESSORY_RUMBLE_PACK: break;
@@ -307,15 +307,13 @@ void PIF::EepromRead(const u8* cmd, u8* res, const Mem& mem) const {
       Util::panic("Out of range EEPROM read! offset: {:02X}", offset);
     }
 
-    for (int i = 0; i < 8; i++) {
+    std::copy_n(eeprom.begin() + offset * 8, 8, res);
-      res[i] = eeprom[(offset * 8) + i];
-    }
   } else {
     Util::panic("EEPROM read on bad channel {}", channel);
   }
 }
 
-void PIF::EepromWrite(const u8* cmd, u8* res, const Mem& mem) const {
+void PIF::EepromWrite(const u8* cmd, u8* res, const Mem& mem) {
   assert(mem.saveType == SAVE_EEPROM_4k || mem.saveType == SAVE_EEPROM_16k);
   if (channel == 4) {
     u8 offset = cmd[3];
@@ -323,9 +321,7 @@ void PIF::EepromWrite(const u8* cmd, u8* res, const Mem& mem) const {
       Util::panic("Out of range EEPROM write! offset: {:02X}", offset);
     }
 
-    for (int i = 0; i < 8; i++) {
+    std::copy_n(cmd + 4, 8, eeprom.begin() + offset * 8);
-      eeprom[(offset * 8) + i] = cmd[4 + i];
-    }
 
     res[0] = 0; // Error byte, I guess it always succeeds?
   } else {

src/backend/core/mmio/PIF.hpp

@@ -3,6 +3,7 @@
 #include <SDL_gamecontroller.h>
 #include <GameDB.hpp>
 #include <filesystem>
+#include <mio/mmap.hpp>
 
 namespace fs = std::filesystem;
 
@@ -89,11 +90,11 @@ enum CICType {
 };
 
 struct PIF {
-  PIF();
+  PIF() = default;
-  ~PIF();
+  ~PIF() = default;
   void Reset();
-  void LoadMempak(fs::path);
+  void LoadMempak(std::string);
-  void LoadEeprom(SaveType, fs::path);
+  void LoadEeprom(SaveType, std::string);
   void ProcessCommands(Mem&);
   void InitDevices(SaveType);
   void CICChallenge();
@@ -103,14 +104,15 @@ struct PIF {
   bool ReadButtons(u8*) const;
   void ControllerID(u8*) const;
   void MempakRead(const u8*, u8*) const;
-  void MempakWrite(u8*, u8*) const;
+  void MempakWrite(u8*, u8*);
   void EepromRead(const u8*, u8*, const Mem&) const;
-  void EepromWrite(const u8*, u8*, const Mem&) const;
+  void EepromWrite(const u8*, u8*, const Mem&);
 
   bool gamepadConnected = false;
   SDL_GameController* gamepad{};
   JoybusDevice joybusDevices[6]{};
-  u8 bootrom[PIF_BOOTROM_SIZE]{}, ram[PIF_RAM_SIZE]{}, *mempak{}, *eeprom{};
+  u8 bootrom[PIF_BOOTROM_SIZE]{}, ram[PIF_RAM_SIZE]{};
+  mio::mmap_sink mempak, eeprom;
   int channel = 0;
   std::string mempakPath{}, eepromPath{};
   size_t eepromSize{};