Error handling on failed writes

iowrite.cpp

@@ -1557,6 +1557,7 @@ bool writeRestart(
    fname.width(7);
    fname.fill('0');
    fname << fileIndex << "." << currentDate << ".vlsv";
+   P::lastRestart = fname.str();
 
    phiprof::Timer openTimer {"open"};
    //Open the file with vlsvWriter:
@@ -1588,7 +1589,9 @@ bool writeRestart(
       MPI_Info_set(MPIinfo, factor, stripeChar);
    }
 
-   if( vlsvWriter.open( fname.str(), MPI_COMM_WORLD, masterProcessId, MPIinfo ) == false) return false;
+   if (vlsvWriter.open( fname.str(), MPI_COMM_WORLD, masterProcessId, MPIinfo ) == false) {
+      return false;
+   }
 
    if( MPIinfo != MPI_INFO_NULL ) {
       MPI_Info_free(&MPIinfo);

parameters.cpp

@@ -194,6 +194,8 @@ std::array<FsGridTools::Task_t,3> P::overrideReadFsGridDecomposition = {0,0,0};
 std::string tracerString; /*!< Fieldline tracer to use for coupling ionosphere and magnetosphere */
 bool P::computeCurvature;
 
+std::string P::lastRestart {""};
+
 bool P::addParameters() {
    typedef Readparameters RP;
    // the other default parameters we read through the add/get interface

parameters.h

@@ -224,6 +224,8 @@ struct Parameters {
 
    static bool computeCurvature; /*<! Boolean flag, if true the curvature of magnetic field is computed. */
 
+   static std::string lastRestart;  // Last restart file written
+
    /*! \brief Add the global parameters.
     *
     * This function adds all the parameters that are loaded at a global level.

vlasiator.cpp

@@ -590,7 +590,8 @@ int main(int argn,char* args[]) {
             writeGhosts
          ) == false
       ) {
-         cerr << "FAILED TO WRITE GRID AT " << __FILE__ << " " << __LINE__ << endl;
+         // TODO make this std::format when we get C++20
+         abort_mpi(std::string(__FILE__) + ":" + std::to_string(__LINE__) + ": FAILED TO WRITE GRID", 1);
       }
 
       phiprof::stop("Initialization");
@@ -729,7 +730,8 @@ int main(int argn,char* args[]) {
             writeGhosts
          ) == false
       ) {
-         cerr << "FAILED TO WRITE GRID AT " << __FILE__ << " " << __LINE__ << endl;
+         // TODO make this std::format when we get C++20
+         abort_mpi(std::string(__FILE__) + ":" + std::to_string(__LINE__) + ": FAILED TO WRITE GRID", 1);
       }
 
       P::systemWriteDistributionWriteStride.pop_back();
@@ -946,7 +948,8 @@ int main(int argn,char* args[]) {
                writeGhosts
                ) == false
             ) {
-               cerr << "FAILED TO WRITE GRID AT" << __FILE__ << " " << __LINE__ << endl;
+               // TODO make this std::format when we get C++20
+               abort_mpi(std::string(__FILE__) + ":" + std::to_string(__LINE__) + ": FAILED TO WRITE GRID", 1);
             }
             P::systemWrites[i]++;
             // Special case for large timesteps
@@ -1029,6 +1032,12 @@ int main(int argn,char* args[]) {
                   version,
                   config,
                   outputReducer,"restart",(uint)P::t,P::restartStripeFactor) == false ) {
+            // If restart write fails, remove the malformed file and hope someone clears space soon
+            MPI_Barrier(MPI_COMM_WORLD);
+            if(!P::lastRestart.empty()) {
+               std::remove(P::lastRestart.c_str());
+               P::lastRestart.clear();
+            }
             logFile << "(IO): ERROR Failed to write restart!" << endl << writeVerbose;
             cerr << "FAILED TO WRITE RESTART" << endl;
          }