From b4b4d875bc7f52da1caeb55ef02473d942fbc612 Mon Sep 17 00:00:00 2001
From: Donald Buczek <buczek@molgen.mpg.de>
Date: Mon, 1 Jan 2024 15:12:04 +0100
Subject: [PATCH] mxq_reaper: Retry on spool file write errors

If we can't write the spool file for any reason, do not write to stderr,
as this is the users stderr of the mxq job. Just wait and retry until
success.
---
 mxq_reaper.c | 80 +++++++++++++++++++++++++++++++---------------------
 1 file changed, 48 insertions(+), 32 deletions(-)

diff --git a/mxq_reaper.c b/mxq_reaper.c
index 19a878d6..7bca9e19 100644
--- a/mxq_reaper.c
+++ b/mxq_reaper.c
@@ -73,38 +73,54 @@ int main(int argc, char **argv) {
         sleep(wait);
     }
 
-    char *tmpfilename;
-    if (asprintf(&tmpfilename, "%s.tmp", spoolfilename) == -1)
-        die("%m\n");
+    while (1) {
+        // if anything fails, do not write to stderr, as this is the users
+        // stderr, just wait and retry.
+
+        char *tmpfilename;
+        if (asprintf(&tmpfilename, "%s.tmp", spoolfilename) == -1)
+            goto retry_1;
 
-    FILE *out = fopen(tmpfilename,"w");
-    if (out == NULL)
-        die("%s: %m\n", tmpfilename);
-    fprintf(out,"1 %d %d %ld %ld %ld %ld %ld %ld %ld %ld %ld %ld %ld %ld %ld %ld %ld %ld %ld %ld %ld %ld\n",
-        getpid(),
-        user_status,
-        user_time.tv_sec, user_time.tv_usec,
-        user_rusage.ru_utime.tv_sec, user_rusage.ru_utime.tv_usec,
-        user_rusage.ru_stime.tv_sec, user_rusage.ru_stime.tv_usec,
-        user_rusage.ru_maxrss,
-        user_rusage.ru_ixrss,
-        user_rusage.ru_idrss,
-        user_rusage.ru_isrss,
-        user_rusage.ru_minflt,
-        user_rusage.ru_majflt,
-        user_rusage.ru_nswap,
-        user_rusage.ru_inblock,
-        user_rusage.ru_oublock,
-        user_rusage.ru_msgsnd,
-        user_rusage.ru_msgrcv,
-        user_rusage.ru_nsignals,
-        user_rusage.ru_nvcsw,
-        user_rusage.ru_nivcsw
-    );
-    fflush(out);
-    fsync(fileno(out));
-    fclose(out);
-    if (rename(tmpfilename, spoolfilename) == -1)
-        die("%s: %m\n", spoolfilename);
+        FILE *out = fopen(tmpfilename, "w");
+        if (out == NULL)
+            goto retry_2;
+        if (fprintf(out,"1 %d %d %ld %ld %ld %ld %ld %ld %ld %ld %ld %ld %ld %ld %ld %ld %ld %ld %ld %ld %ld %ld\n",
+                getpid(),
+                user_status,
+                user_time.tv_sec, user_time.tv_usec,
+                user_rusage.ru_utime.tv_sec, user_rusage.ru_utime.tv_usec,
+                user_rusage.ru_stime.tv_sec, user_rusage.ru_stime.tv_usec,
+                user_rusage.ru_maxrss,
+                user_rusage.ru_ixrss,
+                user_rusage.ru_idrss,
+                user_rusage.ru_isrss,
+                user_rusage.ru_minflt,
+                user_rusage.ru_majflt,
+                user_rusage.ru_nswap,
+                user_rusage.ru_inblock,
+                user_rusage.ru_oublock,
+                user_rusage.ru_msgsnd,
+                user_rusage.ru_msgrcv,
+                user_rusage.ru_nsignals,
+                user_rusage.ru_nvcsw,
+                user_rusage.ru_nivcsw) < 0)
+            goto retry_3;
+        if (fflush(out) == EOF)
+            goto retry_3;
+        if (fsync(fileno(out)) == -1)
+            goto retry_3;
+        if (fclose(out) == EOF)
+            goto retry_2;
+        if (rename(tmpfilename, spoolfilename) == -1)
+            goto retry_2;
+        break;
+
+retry_3:
+        fclose(out);
+retry_2:
+        free(tmpfilename);
+retry_1:
+        sleep(10);
+    }
     return 0;
 }