From 9509e6eb3b23c1b5bb2c4a5e597c0c67a6b1b8f3 Mon Sep 17 00:00:00 2001
From: Marius Tolzmann <tolzmann@molgen.mpg.de>
Date: Mon, 26 Oct 2015 13:09:52 +0100
Subject: [PATCH 01/18] mxq_job: Minor cleanup

---
 mxq_job.c | 54 ++++++++++++++++++++++--------------------------------
 1 file changed, 22 insertions(+), 32 deletions(-)

diff --git a/mxq_job.c b/mxq_job.c
index 68fc709d..f894f22b 100644
--- a/mxq_job.c
+++ b/mxq_job.c
@@ -23,37 +23,31 @@
                 " job_flags, " \
                 " job_priority, " \
                 " group_id, " \
-                \
                 " job_workdir, " \
                 " job_argc, " \
                 " job_argv, " \
                 " job_stdout, " \
                 " job_stderr, " \
-                \
                 " job_umask, " \
                 " host_submit, " \
                 " host_id, " \
                 " server_id, " \
                 " host_hostname, " \
-                \
                 " host_pid, " \
                 " host_slots, " \
                 " UNIX_TIMESTAMP(date_submit) as date_submit, " \
                 " UNIX_TIMESTAMP(date_start) as date_start, " \
                 " UNIX_TIMESTAMP(date_end) as date_end, " \
-                \
                 " stats_status, " \
                 " stats_utime_sec, " \
                 " stats_utime_usec, " \
                 " stats_stime_sec, " \
                 " stats_stime_usec, " \
-                \
                 " stats_real_sec, " \
                 " stats_real_usec, " \
                 " stats_maxrss, " \
                 " stats_minflt, " \
                 " stats_majflt, " \
-                \
                 " stats_nswap, " \
                 " stats_inblock, " \
                 " stats_oublock, " \
@@ -73,37 +67,31 @@ static int bind_result_job_fields(struct mx_mysql_bind *result, struct mxq_job *
     res += mx_mysql_bind_var(result, idx++, uint64, &(j->job_flags));
     res += mx_mysql_bind_var(result, idx++, uint16, &(j->job_priority));
     res += mx_mysql_bind_var(result, idx++, uint64, &(j->group_id));
-
     res += mx_mysql_bind_var(result, idx++, string, &(j->job_workdir));
     res += mx_mysql_bind_var(result, idx++, uint16, &(j->job_argc));
     res += mx_mysql_bind_var(result, idx++, string, &(j->job_argv_str));
     res += mx_mysql_bind_var(result, idx++, string, &(j->job_stdout));
     res += mx_mysql_bind_var(result, idx++, string, &(j->job_stderr));
-
     res += mx_mysql_bind_var(result, idx++, uint32, &(j->job_umask));
     res += mx_mysql_bind_var(result, idx++, string, &(j->host_submit));
     res += mx_mysql_bind_var(result, idx++, string, &(j->host_id));
     res += mx_mysql_bind_var(result, idx++, string, &(j->server_id));
     res += mx_mysql_bind_var(result, idx++, string, &(j->host_hostname));
-
     res += mx_mysql_bind_var(result, idx++, uint32, &(j->host_pid));
     res += mx_mysql_bind_var(result, idx++, uint32, &(j->host_slots));
     res += mx_mysql_bind_var(result, idx++,  int64, &(j->date_submit));
     res += mx_mysql_bind_var(result, idx++,  int64, &(j->date_start));
     res += mx_mysql_bind_var(result, idx++,  int64, &(j->date_end));
-
     res += mx_mysql_bind_var(result, idx++,  int32, &(j->stats_status));
     res += mx_mysql_bind_var(result, idx++,  int64, &(j->stats_rusage.ru_utime.tv_sec));
     res += mx_mysql_bind_var(result, idx++,  int64, &(j->stats_rusage.ru_utime.tv_usec));
     res += mx_mysql_bind_var(result, idx++,  int64, &(j->stats_rusage.ru_stime.tv_sec));
     res += mx_mysql_bind_var(result, idx++,  int64, &(j->stats_rusage.ru_stime.tv_usec));
-
     res += mx_mysql_bind_var(result, idx++,  int64, &(j->stats_realtime.tv_sec));
     res += mx_mysql_bind_var(result, idx++,  int64, &(j->stats_realtime.tv_usec));
     res += mx_mysql_bind_var(result, idx++,  int64, &(j->stats_rusage.ru_maxrss));
     res += mx_mysql_bind_var(result, idx++,  int64, &(j->stats_rusage.ru_minflt));
     res += mx_mysql_bind_var(result, idx++,  int64, &(j->stats_rusage.ru_majflt));
-
     res += mx_mysql_bind_var(result, idx++,  int64, &(j->stats_rusage.ru_nswap));
     res += mx_mysql_bind_var(result, idx++,  int64, &(j->stats_rusage.ru_inblock));
     res += mx_mysql_bind_var(result, idx++,  int64, &(j->stats_rusage.ru_oublock));
@@ -457,6 +445,7 @@ int mxq_set_job_status_running(struct mx_mysql *mysql, struct mxq_job *job)
 int mxq_set_job_status_exited(struct mx_mysql *mysql, struct mxq_job *job)
 {
     int res;
+    int idx;
     uint16_t newstatus;
     struct mx_mysql_bind param = {0};
 
@@ -509,27 +498,28 @@ int mxq_set_job_status_exited(struct mx_mysql *mysql, struct mxq_job *job)
     res = mx_mysql_bind_init_param(&param, 20);
     assert(res == 0);
 
+    idx = 0;
     res = 0;
-    res += mx_mysql_bind_var(&param, 0, uint16, &(newstatus));
-    res += mx_mysql_bind_var(&param, 1, int32,  &(job->stats_status));
-    res += mx_mysql_bind_var(&param, 2, int64, &(job->stats_rusage.ru_utime.tv_sec));
-    res += mx_mysql_bind_var(&param, 3, int64, &(job->stats_rusage.ru_utime.tv_usec));
-    res += mx_mysql_bind_var(&param, 4, int64, &(job->stats_rusage.ru_stime.tv_sec));
-    res += mx_mysql_bind_var(&param, 5, int64, &(job->stats_rusage.ru_stime.tv_usec));
-    res += mx_mysql_bind_var(&param, 6, int64, &(job->stats_realtime.tv_sec));
-    res += mx_mysql_bind_var(&param, 7, int64, &(job->stats_realtime.tv_usec));
-    res += mx_mysql_bind_var(&param, 8, int64, &(job->stats_rusage.ru_maxrss));
-    res += mx_mysql_bind_var(&param, 9, int64, &(job->stats_rusage.ru_minflt));
-    res += mx_mysql_bind_var(&param, 10, int64, &(job->stats_rusage.ru_majflt));
-    res += mx_mysql_bind_var(&param, 11, int64, &(job->stats_rusage.ru_nswap));
-    res += mx_mysql_bind_var(&param, 12, int64, &(job->stats_rusage.ru_inblock));
-    res += mx_mysql_bind_var(&param, 13, int64, &(job->stats_rusage.ru_oublock));
-    res += mx_mysql_bind_var(&param, 14, int64, &(job->stats_rusage.ru_nvcsw));
-    res += mx_mysql_bind_var(&param, 15, int64, &(job->stats_rusage.ru_nivcsw));
-    res += mx_mysql_bind_var(&param, 16, uint64, &(job->job_id));
-    res += mx_mysql_bind_var(&param, 17, string, &(job->host_hostname));
-    res += mx_mysql_bind_var(&param, 18, string, &(job->server_id));
-    res += mx_mysql_bind_var(&param, 19, uint32, &(job->host_pid));
+    res += mx_mysql_bind_var(&param, idx++, uint16, &(newstatus));
+    res += mx_mysql_bind_var(&param, idx++,  int32, &(job->stats_status));
+    res += mx_mysql_bind_var(&param, idx++,  int64, &(job->stats_rusage.ru_utime.tv_sec));
+    res += mx_mysql_bind_var(&param, idx++,  int64, &(job->stats_rusage.ru_utime.tv_usec));
+    res += mx_mysql_bind_var(&param, idx++,  int64, &(job->stats_rusage.ru_stime.tv_sec));
+    res += mx_mysql_bind_var(&param, idx++,  int64, &(job->stats_rusage.ru_stime.tv_usec));
+    res += mx_mysql_bind_var(&param, idx++,  int64, &(job->stats_realtime.tv_sec));
+    res += mx_mysql_bind_var(&param, idx++,  int64, &(job->stats_realtime.tv_usec));
+    res += mx_mysql_bind_var(&param, idx++,  int64, &(job->stats_rusage.ru_maxrss));
+    res += mx_mysql_bind_var(&param, idx++,  int64, &(job->stats_rusage.ru_minflt));
+    res += mx_mysql_bind_var(&param, idx++,  int64, &(job->stats_rusage.ru_majflt));
+    res += mx_mysql_bind_var(&param, idx++,  int64, &(job->stats_rusage.ru_nswap));
+    res += mx_mysql_bind_var(&param, idx++,  int64, &(job->stats_rusage.ru_inblock));
+    res += mx_mysql_bind_var(&param, idx++,  int64, &(job->stats_rusage.ru_oublock));
+    res += mx_mysql_bind_var(&param, idx++,  int64, &(job->stats_rusage.ru_nvcsw));
+    res += mx_mysql_bind_var(&param, idx++,  int64, &(job->stats_rusage.ru_nivcsw));
+    res += mx_mysql_bind_var(&param, idx++, uint64, &(job->job_id));
+    res += mx_mysql_bind_var(&param, idx++, string, &(job->host_hostname));
+    res += mx_mysql_bind_var(&param, idx++, string, &(job->server_id));
+    res += mx_mysql_bind_var(&param, idx++, uint32, &(job->host_pid));
     assert(res == 0);
 
     res = mx_mysql_do_statement_noresult_retry_on_fail(mysql, query, &param);

From 9b2a5f47c845018c8d513e57fa32bf404f546f60 Mon Sep 17 00:00:00 2001
From: Marius Tolzmann <tolzmann@molgen.mpg.de>
Date: Mon, 26 Oct 2015 13:51:07 +0100
Subject: [PATCH 02/18] mxqd: Store and use max_sumrss

resolves parts of https://github.molgen.mpg.de/mariux64/mxq/issues/16
---
 mxq_group.c                   |  4 +++-
 mxq_group.h                   |  1 +
 mxq_job.c                     |  8 ++++++--
 mxq_job.h                     |  2 ++
 mxqd.c                        | 10 +++++-----
 mxqdump.c                     |  8 ++++++--
 mysql/alter_tables_0.17.0.sql | 11 +++++++++++
 mysql/create_tables.sql       |  4 ++++
 mysql/create_trigger.sql      |  3 +++
 web/pages/mxq/mxq.in          |  3 +++
 10 files changed, 44 insertions(+), 10 deletions(-)
 create mode 100644 mysql/alter_tables_0.17.0.sql

diff --git a/mxq_group.c b/mxq_group.c
index 2887411d..e9c0ac4d 100644
--- a/mxq_group.c
+++ b/mxq_group.c
@@ -12,7 +12,7 @@
 #include "mx_util.h"
 #include "mx_mysql.h"
 
-#define GROUP_FIELDS_CNT 30
+#define GROUP_FIELDS_CNT 31
 #define GROUP_FIELDS \
             " group_id," \
             " group_name," \
@@ -37,6 +37,7 @@
             " group_jobs_unknown," \
             " group_jobs_restarted," \
             " group_slots_running," \
+            " stats_max_sumrss," \
             " stats_max_maxrss," \
             " stats_max_utime_sec," \
             " stats_max_stime_sec," \
@@ -85,6 +86,7 @@ static int bind_result_group_fields(struct mx_mysql_bind *result, struct mxq_gro
 
     res += mx_mysql_bind_var(result, idx++, uint64, &(g->group_slots_running));
 
+    res += mx_mysql_bind_var(result, idx++, uint64, &(g->stats_max_sumrss));
     res += mx_mysql_bind_var(result, idx++, uint64, &(g->stats_max_maxrss));
     res += mx_mysql_bind_var(result, idx++, int64, &(g->stats_max_utime.tv_sec));
     res += mx_mysql_bind_var(result, idx++, int64, &(g->stats_max_stime.tv_sec));
diff --git a/mxq_group.h b/mxq_group.h
index 46a352b2..7286a5a0 100644
--- a/mxq_group.h
+++ b/mxq_group.h
@@ -41,6 +41,7 @@ struct mxq_group {
 
    uint64_t  group_slots_running;
 
+   uint64_t  stats_max_sumrss;
    uint64_t  stats_max_maxrss;
 
    struct timeval stats_max_utime;
diff --git a/mxq_job.c b/mxq_job.c
index f894f22b..ebde474a 100644
--- a/mxq_job.c
+++ b/mxq_job.c
@@ -16,7 +16,7 @@
 #include "mxq_group.h"
 #include "mxq_job.h"
 
-#define JOB_FIELDS_CNT 35
+#define JOB_FIELDS_CNT 36
 #define JOB_FIELDS \
                 " job_id, " \
                 " job_status, " \
@@ -38,6 +38,7 @@
                 " UNIX_TIMESTAMP(date_submit) as date_submit, " \
                 " UNIX_TIMESTAMP(date_start) as date_start, " \
                 " UNIX_TIMESTAMP(date_end) as date_end, " \
+                " stats_max_sumrss, " \
                 " stats_status, " \
                 " stats_utime_sec, " \
                 " stats_utime_usec, " \
@@ -82,6 +83,7 @@ static int bind_result_job_fields(struct mx_mysql_bind *result, struct mxq_job *
     res += mx_mysql_bind_var(result, idx++,  int64, &(j->date_submit));
     res += mx_mysql_bind_var(result, idx++,  int64, &(j->date_start));
     res += mx_mysql_bind_var(result, idx++,  int64, &(j->date_end));
+    res += mx_mysql_bind_var(result, idx++, uint64, &(j->stats_max_sumrss));
     res += mx_mysql_bind_var(result, idx++,  int32, &(j->stats_status));
     res += mx_mysql_bind_var(result, idx++,  int64, &(j->stats_rusage.ru_utime.tv_sec));
     res += mx_mysql_bind_var(result, idx++,  int64, &(j->stats_rusage.ru_utime.tv_usec));
@@ -474,6 +476,7 @@ int mxq_set_job_status_exited(struct mx_mysql *mysql, struct mxq_job *job)
             "UPDATE mxq_job SET"
             " job_status = ?,"
             " date_end = NULL,"
+            " stats_max_sumrss = ?, "
             " stats_status = ?, "
             " stats_utime_sec = ?, "
             " stats_utime_usec = ?, "
@@ -495,12 +498,13 @@ int mxq_set_job_status_exited(struct mx_mysql *mysql, struct mxq_job *job)
             " AND server_id = ?"
             " AND host_pid = ?";
 
-    res = mx_mysql_bind_init_param(&param, 20);
+    res = mx_mysql_bind_init_param(&param, 21);
     assert(res == 0);
 
     idx = 0;
     res = 0;
     res += mx_mysql_bind_var(&param, idx++, uint16, &(newstatus));
+    res += mx_mysql_bind_var(&param, idx++, uint64, &(job->stats_max_sumrss));
     res += mx_mysql_bind_var(&param, idx++,  int32, &(job->stats_status));
     res += mx_mysql_bind_var(&param, idx++,  int64, &(job->stats_rusage.ru_utime.tv_sec));
     res += mx_mysql_bind_var(&param, idx++,  int64, &(job->stats_rusage.ru_utime.tv_usec));
diff --git a/mxq_job.h b/mxq_job.h
index 42b2d81d..f29baf62 100644
--- a/mxq_job.h
+++ b/mxq_job.h
@@ -52,6 +52,8 @@ struct mxq_job {
 
     struct timeval stats_starttime;
 
+    uint64_t   stats_max_sumrss;
+
     int32_t        stats_status;
     struct timeval stats_realtime;
     struct rusage  stats_rusage;
diff --git a/mxqd.c b/mxqd.c
index 43afc3d8..edc3ff48 100644
--- a/mxqd.c
+++ b/mxqd.c
@@ -1638,16 +1638,16 @@ int killall_over_memory(struct mxq_server *server)
                     continue;
                 }
 
-                memory = pinfo->sum_rss * pagesize / 1024 / 1024;
+                memory = pinfo->sum_rss * pagesize / 1024;
 
                 if (job->max_sum_rss < memory)
                     job->max_sum_rss = memory;
 
-                if (memory <= group->group.job_memory)
+                if (memory/1024 <= group->group.job_memory)
                     continue;
 
-                mx_log_info("killall_over_memory(): used(%llu) > requested(%llu): Sending signal=KILL to job=%s(%d):%lu:%lu pgrp=%d",
-                    memory, group->group.job_memory,
+                mx_log_info("killall_over_memory(): used(%lluMiB) > requested(%lluMiB): Sending signal=KILL to job=%s(%d):%lu:%lu pgrp=%d",
+                    memory/1024, group->group.job_memory,
                     group->group.user_name, group->group.user_uid, group->group.group_id, job->job.job_id, pid);
 
                 kill(-pid, SIGKILL);
@@ -1762,7 +1762,7 @@ int catchall(struct mxq_server *server) {
         g = &job->group->group;
 
         timersub(&now, &j->stats_starttime, &j->stats_realtime);
-
+        j->stats_max_sumrss = job->max_sum_rss;
         j->stats_status = status;
         j->stats_rusage = rusage;
 
diff --git a/mxqdump.c b/mxqdump.c
index 2cac9c13..8b0b9665 100644
--- a/mxqdump.c
+++ b/mxqdump.c
@@ -128,6 +128,7 @@ static int print_group(struct mxq_group *g)
         " max_utime=%lu"
         " max_real=%lu"
         " max_memory=%lukiB"
+        " max_rss=%lukiB"
         " wait_sec=%lu"
         " run_sec=%lu"
         " idle_sec=%lu"
@@ -150,10 +151,11 @@ static int print_group(struct mxq_group *g)
         g->job_threads,
         g->job_memory*1024,
         g->job_time*60,
-        (100UL*(uint64_t)g->stats_max_maxrss/1024UL/g->job_memory),
+        (100UL*(uint64_t)g->stats_max_sumrss/1024UL/g->job_memory),
         (100UL*(uint64_t)g->stats_max_real.tv_sec/60UL/g->job_time),
         g->stats_max_utime.tv_sec,
         g->stats_max_real.tv_sec,
+        g->stats_max_sumrss,
         g->stats_max_maxrss,
         g->stats_wait_sec,
         g->stats_run_sec,
@@ -219,6 +221,7 @@ static int print_job(struct mxq_group *g, struct mxq_job *j)
         " runtime_requested=%us"
         " time_load=%lu%%"
         " memory_requested=%lukiB"
+        " max_memory=%lukiB"
         " max_rss=%lukiB"
         " memory_load=%lu%%"
         " threads=%d"
@@ -244,8 +247,9 @@ static int print_job(struct mxq_group *g, struct mxq_job *j)
         g->job_time*60,
         (100UL*(run_sec)/60UL/g->job_time),
         g->job_memory*1024,
+        j->stats_max_sumrss,
         j->stats_rusage.ru_maxrss,
-        (100UL*j->stats_rusage.ru_maxrss/1024UL/g->job_memory),
+        (100UL*j->stats_max_sumrss/1024UL/g->job_memory),
         g->job_threads,
         j->host_slots,
         mxq_job_status_to_name(j->job_status),
diff --git a/mysql/alter_tables_0.17.0.sql b/mysql/alter_tables_0.17.0.sql
new file mode 100644
index 00000000..b9daa722
--- /dev/null
+++ b/mysql/alter_tables_0.17.0.sql
@@ -0,0 +1,11 @@
+ALTER TABLE mxq_group
+    ADD COLUMN
+        stats_max_sumrss  INT8 UNSIGNED NOT NULL DEFAULT 0
+    AFTER
+        group_date_end;
+
+ALTER TABLE mxq_job
+    ADD COLUMN
+        stats_max_sumrss  INT8 UNSIGNED NOT NULL DEFAULT 0
+    AFTER
+        job_id_first;
diff --git a/mysql/create_tables.sql b/mysql/create_tables.sql
index 978c679f..f2566cc6 100644
--- a/mysql/create_tables.sql
+++ b/mysql/create_tables.sql
@@ -34,6 +34,8 @@ CREATE TABLE IF NOT EXISTS mxq_group (
 
    group_date_end    TIMESTAMP NOT NULL DEFAULT 0,
 
+   stats_max_sumrss  INT8 UNSIGNED NOT NULL DEFAULT 0,
+
    stats_max_maxrss     INT8 UNSIGNED NOT NULL DEFAULT 0,
    stats_max_utime_sec  INT8 UNSIGNED NOT NULL DEFAULT 0,
    stats_max_stime_sec  INT8 UNSIGNED NOT NULL DEFAULT 0,
@@ -94,6 +96,8 @@ CREATE TABLE IF NOT EXISTS mxq_job (
    job_id_old     INT8 UNSIGNED   NULL DEFAULT NULL,
    job_id_first   INT8 UNSIGNED   NULL DEFAULT NULL,
 
+   stats_max_sumrss  INT8 UNSIGNED NOT NULL DEFAULT 0,
+
    stats_status   INT4 UNSIGNED NOT NULL DEFAULT 0,
 
    stats_utime_sec  INT8 UNSIGNED NOT NULL DEFAULT 0,
diff --git a/mysql/create_trigger.sql b/mysql/create_trigger.sql
index ccd2af6e..c4269acf 100644
--- a/mysql/create_trigger.sql
+++ b/mysql/create_trigger.sql
@@ -64,6 +64,7 @@ CREATE TRIGGER mxq_update_job BEFORE UPDATE ON mxq_job
                    group_slots_running=group_slots_running-NEW.host_slots,
                    group_jobs_running=group_jobs_running-1,
                    group_jobs_failed=group_jobs_failed+1,
+                   stats_max_sumrss=GREATEST(stats_max_sumrss, NEW.stats_max_sumrss),
                    stats_max_maxrss=GREATEST(stats_max_maxrss, NEW.stats_maxrss),
                    stats_max_utime_sec=GREATEST(stats_max_utime_sec, NEW.stats_utime_sec),
                    stats_max_stime_sec=GREATEST(stats_max_stime_sec, NEW.stats_stime_sec),
@@ -99,6 +100,7 @@ CREATE TRIGGER mxq_update_job BEFORE UPDATE ON mxq_job
                    group_slots_running=group_slots_running-NEW.host_slots,
                    group_jobs_running=group_jobs_running-1,
                    group_jobs_finished=group_jobs_finished+1,
+                   stats_max_sumrss=GREATEST(stats_max_sumrss, NEW.stats_max_sumrss),
                    stats_max_maxrss=GREATEST(stats_max_maxrss, NEW.stats_maxrss),
                    stats_max_utime_sec=GREATEST(stats_max_utime_sec, NEW.stats_utime_sec),
                    stats_max_stime_sec=GREATEST(stats_max_stime_sec, NEW.stats_stime_sec),
@@ -113,6 +115,7 @@ CREATE TRIGGER mxq_update_job BEFORE UPDATE ON mxq_job
                 WHERE group_id=NEW.group_id;
             ELSEIF NEW.job_status NOT IN (399, 755, 989, 990) THEN
                 UPDATE mxq_group SET
+                   stats_max_sumrss=GREATEST(stats_max_sumrss, NEW.stats_max_sumrss),
                    stats_max_maxrss=GREATEST(stats_max_maxrss, NEW.stats_maxrss),
                    stats_max_utime_sec=GREATEST(stats_max_utime_sec, NEW.stats_utime_sec),
                    stats_max_stime_sec=GREATEST(stats_max_stime_sec, NEW.stats_stime_sec),
diff --git a/web/pages/mxq/mxq.in b/web/pages/mxq/mxq.in
index 6a4f8b2b..db24812d 100755
--- a/web/pages/mxq/mxq.in
+++ b/web/pages/mxq/mxq.in
@@ -269,6 +269,7 @@ group_mtime           : $o{group_mtime}
 
 group_date_end        : $o{group_date_end}
 
+stats_max_sumrss                 : $o{stats_max_maxrss} kiB
 stats_max_maxrss                 : $o{stats_max_maxrss}
 stats_max_utime_sec              : $o{stats_max_utime_sec}
 stats_max_stime_sec              : $o{stats_max_stime_sec}
@@ -364,6 +365,8 @@ job_id_new       : $o{job_id_new}
 job_id_old       : $o{job_id_old}
 job_id_first     : $o{job_id_first}
 
+stats_max_sumrss : $o{stats_max_maxrss} kiB
+
 stats_status     : $o{stats_status}
 
 stats_utime_sec  : $o{stats_utime_sec}

From df2c998bd5e751d08364f47146a7b2c79860f955 Mon Sep 17 00:00:00 2001
From: Marius Tolzmann <tolzmann@molgen.mpg.de>
Date: Mon, 26 Oct 2015 14:15:44 +0100
Subject: [PATCH 03/18] mxqd: Send TERM to process when over memory instead of
 KILL to pgrp

---
 mxqd.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/mxqd.c b/mxqd.c
index edc3ff48..bec2797b 100644
--- a/mxqd.c
+++ b/mxqd.c
@@ -1646,11 +1646,11 @@ int killall_over_memory(struct mxq_server *server)
                 if (memory/1024 <= group->group.job_memory)
                     continue;
 
-                mx_log_info("killall_over_memory(): used(%lluMiB) > requested(%lluMiB): Sending signal=KILL to job=%s(%d):%lu:%lu pgrp=%d",
+                mx_log_info("killall_over_memory(): used(%lluMiB) > requested(%lluMiB): Sending signal=TERM to job=%s(%d):%lu:%lu pid=%d",
                     memory/1024, group->group.job_memory,
                     group->group.user_name, group->group.user_uid, group->group.group_id, job->job.job_id, pid);
 
-                kill(-pid, SIGKILL);
+                kill(pid, SIGTERM);
             }
         }
     }

From 9f5688d51bd95ea3f3892819d0f92ef2ff7c3026 Mon Sep 17 00:00:00 2001
From: Marius Tolzmann <tolzmann@molgen.mpg.de>
Date: Mon, 26 Oct 2015 14:22:04 +0100
Subject: [PATCH 04/18] MXQ bump version

---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index b6b0e805..fe158d5a 100644
--- a/Makefile
+++ b/Makefile
@@ -1,5 +1,5 @@
 MXQ_VERSION_MAJOR = 0
-MXQ_VERSION_MINOR = 17
+MXQ_VERSION_MINOR = 18
 MXQ_VERSION_PATCH = 0
 MXQ_VERSION_EXTRA = "beta"
 MXQ_VERSIONDATE = 2015

From 516c0f644596e2fa9486414946787bd169b02b14 Mon Sep 17 00:00:00 2001
From: Marius Tolzmann <tolzmann@molgen.mpg.de>
Date: Mon, 26 Oct 2015 14:27:19 +0100
Subject: [PATCH 05/18] mxqd: Fix root check

---
 mxqd.c | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/mxqd.c b/mxqd.c
index bec2797b..697eff50 100644
--- a/mxqd.c
+++ b/mxqd.c
@@ -433,6 +433,15 @@ int server_init(struct mxq_server *server, int argc, char *argv[])
 
     MX_GETOPT_FINISH(optctl, argc, argv);
 
+    if (!RUNNING_AS_ROOT) {
+#if defined(MXQ_DEVELOPMENT) || defined(RUNASNORMALUSER)
+        mx_log_notice("Running mxqd as non-root user.");
+#else
+        mx_log_err("Running mxqd as non-root user is not supported at the moment.");
+        exit(EX_USAGE);
+#endif
+    }
+
     if (arg_daemonize && arg_nolog) {
         mx_log_err("Error while using conflicting options --daemonize and --no-log at once.");
         exit(EX_USAGE);
@@ -501,15 +510,6 @@ int server_init(struct mxq_server *server, int argc, char *argv[])
         }
     }
 
-    if (!RUNNING_AS_ROOT) {
-#if defined(MXQ_DEVELOPMENT) || defined(RUNASNORMALUSER)
-        mx_log_notice("Running mxqd as non-root user.");
-#else
-        mx_log_err("Running mxqd as non-root user is not supported at the moment.");
-        exit(EX_USAGE);
-#endif
-    }
-
     res = mx_read_first_line_from_file("/proc/sys/kernel/random/boot_id", &str_bootid);
     assert(res == 36);
     assert(str_bootid);

From 498c7a7c71251fb681b3d62734a6877050330a5f Mon Sep 17 00:00:00 2001
From: Marius Tolzmann <tolzmann@molgen.mpg.de>
Date: Mon, 26 Oct 2015 14:28:52 +0100
Subject: [PATCH 06/18] MXQ bump version

---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index fe158d5a..fa1f1ae7 100644
--- a/Makefile
+++ b/Makefile
@@ -1,6 +1,6 @@
 MXQ_VERSION_MAJOR = 0
 MXQ_VERSION_MINOR = 18
-MXQ_VERSION_PATCH = 0
+MXQ_VERSION_PATCH = 1
 MXQ_VERSION_EXTRA = "beta"
 MXQ_VERSIONDATE = 2015
 

From 6f9e118cd7ca5a628cf24d11c335b81d63598f8a Mon Sep 17 00:00:00 2001
From: Donald Buczek <buczek@molgen.mpg.de>
Date: Mon, 26 Oct 2015 14:28:05 +0100
Subject: [PATCH 07/18] mxqd: fix typo in log

---
 mxqd.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/mxqd.c b/mxqd.c
index 697eff50..167ed126 100644
--- a/mxqd.c
+++ b/mxqd.c
@@ -1143,7 +1143,7 @@ unsigned long start_job(struct mxq_group_list *group)
             group->group.user_name, group->group.user_uid, group->group.group_id, mxqjob.job_id);
 
     cpuset_init_job(&mxqjob.host_cpu_set,&server->cpu_set_available,&server->cpu_set_running,group->slots_per_job);
-    cpuset_log(" job assgined cpus: ",&mxqjob.host_cpu_set);
+    cpuset_log(" job assigned cpus: ",&mxqjob.host_cpu_set);
 
     mx_mysql_disconnect(server->mysql);
 

From a092d65b2265c29cd4dd15fbff2fe8a9ad00ce57 Mon Sep 17 00:00:00 2001
From: Donald Buczek <buczek@molgen.mpg.de>
Date: Thu, 22 Oct 2015 18:38:41 +0200
Subject: [PATCH 08/18] mxq_job: fix memory leak

---
 mxq_job.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/mxq_job.c b/mxq_job.c
index ebde474a..d4bbe21f 100644
--- a/mxq_job.c
+++ b/mxq_job.c
@@ -670,6 +670,7 @@ int mxq_load_job_from_group_for_server(struct mx_mysql *mysql, struct mxq_job *m
         }
         if(res == 1) {
             memcpy(mxqjob, &jobs[0], sizeof(*mxqjob));
+            free(jobs);
             break;
         }
 

From c8a578414f517f628d048454a7bea069c420f1e5 Mon Sep 17 00:00:00 2001
From: Donald Buczek <buczek@molgen.mpg.de>
Date: Thu, 22 Oct 2015 13:34:26 +0200
Subject: [PATCH 09/18] web: show threads in group table and runtime in minutes
 for completed jobs

---
 web/pages/mxq/mxq.in | 18 ++++++++++--------
 1 file changed, 10 insertions(+), 8 deletions(-)

diff --git a/web/pages/mxq/mxq.in b/web/pages/mxq/mxq.in
index db24812d..e938bbec 100755
--- a/web/pages/mxq/mxq.in
+++ b/web/pages/mxq/mxq.in
@@ -312,7 +312,7 @@ sub job {
 
 	$dbh or db_init();
 
-	my $sth=$dbh->prepare('SELECT *,timestampdiff(MINUTE,date_start,now()) as t FROM mxq_job WHERE job_id=? LIMIT 1',undef);
+	my $sth=$dbh->prepare('SELECT *,timestampdiff(MINUTE,date_start,now()) as t,timestampdiff(MINUTE,date_start,date_end) as t2 FROM mxq_job WHERE job_id=? LIMIT 1',undef);
 	$sth->execute($job_id);
 	my %o=%{$sth->fetchrow_hashref('NAME_lc')};
 
@@ -326,8 +326,8 @@ sub job {
 	my $job_stdout=escapeHTML($o{job_stdout});
 	my $job_stderr=escapeHTML($o{job_stderr});
 
-
-	my $ago = defined $o{t} ? "($o{t} minutes ago)" : '';
+	my $ago = $job_status_text eq 'RUNNING' && defined $o{t} ? "($o{t} minutes ago)" : '';
+	my $rt = defined $o{t2} ? "($o{t2} minutes runtime)" : '';
 
 	defined $_ or $_='&lt;null&gt;' for values %o;
 
@@ -359,7 +359,7 @@ host_slots       : $o{host_slots}
 
 date_submit      : $o{date_submit}
 date_start       : $o{date_start} $ago
-date_end         : $o{date_end}
+date_end         : $o{date_end} $rt
 
 job_id_new       : $o{job_id_new}
 job_id_old       : $o{job_id_old}
@@ -397,17 +397,17 @@ sub group_table_rows {
 	$out.=$q->Tr($q->th($head));
 
 	while (my $row=$sth->fetchrow_arrayref()) {
-		my ($group_id,$group_name,$user_name,$group_mtime,$group_status,
+		my ($group_id,$group_name,$job_threads,$user_name,$group_mtime,$group_status,
 			$group_jobs,$group_jobs_inq,$group_jobs_running,$group_jobs_finished,$group_jobs_failed,$group_jobs_cancelled,$group_jobs_unknown
 		)=@$row;
 
 		$out.=$q->Tr(
 			$q->td({class=>'number'},$q->a({href=>selfurl("/group/$group_id")},$group_id)),
 			$q->td($group_name),
+			$q->td({class=>'number'},$job_threads),
 			$q->td($user_name),
 			$q->td($group_mtime),
 			$q->td(group_status($group_status)),
-
 			$q->td({class=>'number'},$group_jobs),
 			$q->td({class=>'number'},$group_jobs_inq),
 			$q->td({class=>'number'},$group_jobs_running),
@@ -429,11 +429,11 @@ sub group_table {
 	my $out;
 
 	my @cols=qw(
-		group_id group_name user_name group_mtime group_status
+		group_id group_name job_threads user_name group_mtime group_status
 		group_jobs group_jobs_inq group_jobs_running group_jobs_finished group_jobs_failed group_jobs_cancelled group_jobs_unknown
 	);
 	my @head=qw(
-		group_id group_name user_name group_mtime group_status
+		group_id group_name threads user_name group_mtime group_status
 		jobs inq running finished failed cancelled unknown
 	);
 
@@ -447,6 +447,7 @@ sub group_table {
 	@cols=qw(
         group_id
         group_name
+        job_threads
         user_name
         group_date_end
         group_status
@@ -460,6 +461,7 @@ sub group_table {
 	@head=qw(
         group_id
         group_name
+        threads
         user_name
         date_end
         group_status

From 0e1b70d0bcbb0402208bd4c26015d262427ad847 Mon Sep 17 00:00:00 2001
From: Donald Buczek <buczek@molgen.mpg.de>
Date: Mon, 26 Oct 2015 15:55:26 +0100
Subject: [PATCH 10/18] make: add LOCALSTATEDIR

---
 Makefile | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/Makefile b/Makefile
index fa1f1ae7..deeda390 100644
--- a/Makefile
+++ b/Makefile
@@ -25,6 +25,7 @@ LIBEXECDIR = ${EPREFIX}/libexec
 DATADIR    = ${PREFIX}/share
 MANDIR     = ${DATADIR}/man
 SYSCONFDIR = ${PREFIX}/etc
+LOCALSTATEDIR = ${PREFIX}/var
 
 DESTDIR=
 
@@ -43,6 +44,11 @@ ifneq (, $(filter /usr /usr/local, ${PREFIX}))
     SYSCONFDIR = /etc
 endif
 
+### set localstatedir /var if prefix /usr || /usr/local
+ifneq (, $(filter /usr /usr/local, ${PREFIX}))
+    LOCALSTATEDIR = /var
+endif
+
 ########################################################################
 
 ### strip /mxq from SYSCONFDIR if set
@@ -125,6 +131,7 @@ CFLAGS += -DLIBEXECDIR=\"${LIBEXECDIR}\"
 CFLAGS += -DDATADIR=\"${DATADIR}\"
 CFLAGS += -DMANDIR=\"${MANDIR}\"
 CFLAGS += -DSYSCONFDIR=\"${SYSCONFDIR}\"
+CFLAGS += -DLOCALSTATEDIR=\"${LOCALSTATEDIR}\"
 CFLAGS += $(EXTRA_CFLAGS)
 
 ########################################################################

From 2d2bb58d8c325aed8d0d8b3060810ce6f588715b Mon Sep 17 00:00:00 2001
From: Marius Tolzmann <tolzmann@molgen.mpg.de>
Date: Tue, 27 Oct 2015 10:40:23 +0100
Subject: [PATCH 11/18] MXQ bump version

---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index deeda390..1d1bd5a2 100644
--- a/Makefile
+++ b/Makefile
@@ -1,6 +1,6 @@
 MXQ_VERSION_MAJOR = 0
 MXQ_VERSION_MINOR = 18
-MXQ_VERSION_PATCH = 1
+MXQ_VERSION_PATCH = 2
 MXQ_VERSION_EXTRA = "beta"
 MXQ_VERSIONDATE = 2015
 

From 2200fd6cced71f473c29fc8d69f94b61b14d077c Mon Sep 17 00:00:00 2001
From: Marius Tolzmann <tolzmann@molgen.mpg.de>
Date: Tue, 27 Oct 2015 11:38:07 +0100
Subject: [PATCH 12/18] mxq_job: Only load jobs assigned for the group that
 should be started

this fixes https://github.molgen.mpg.de/mariux64/mxq/issues/7
---
 mxq_job.c | 10 ++++++----
 mxq_job.h |  2 +-
 2 files changed, 7 insertions(+), 5 deletions(-)

diff --git a/mxq_job.c b/mxq_job.c
index d4bbe21f..4b33fda3 100644
--- a/mxq_job.c
+++ b/mxq_job.c
@@ -601,7 +601,7 @@ int mxq_job_set_tmpfilenames(struct mxq_group *g, struct mxq_job *j)
     return 1;
 }
 
-int mxq_load_job_assigned_to_server(struct mx_mysql *mysql, struct mxq_job **mxq_jobs, char *hostname, char *server_id)
+int mxq_load_job_from_group_assigned_to_server(struct mx_mysql *mysql, struct mxq_job **mxq_jobs, uint64_t group_id, char *hostname, char *server_id)
 {
     int res;
     struct mxq_job *jobs = NULL;
@@ -624,14 +624,16 @@ int mxq_load_job_assigned_to_server(struct mx_mysql *mysql, struct mxq_job **mxq
             " WHERE job_status = " status_str(MXQ_JOB_STATUS_ASSIGNED)
             " AND host_hostname = ?"
             " AND server_id  = ?"
+            " AND group_id = ?"
             " LIMIT 1";
 
-    res = mx_mysql_bind_init_param(&param, 2);
+    res = mx_mysql_bind_init_param(&param, 3);
     assert(res == 0);
 
     res = 0;
     res += mx_mysql_bind_var(&param, 0, string, &hostname);
     res += mx_mysql_bind_var(&param, 1, string, &server_id);
+    res += mx_mysql_bind_var(&param, 2, uint64, &group_id);
     assert(res == 0);
 
     res = bind_result_job_fields(&result, &j);
@@ -662,10 +664,10 @@ int mxq_load_job_from_group_for_server(struct mx_mysql *mysql, struct mxq_job *m
     assert(*host_id);
 
     do {
-        res = mxq_load_job_assigned_to_server(mysql, &jobs, hostname, server_id);
+        res = mxq_load_job_from_group_assigned_to_server(mysql, &jobs, group_id, hostname, server_id);
 
         if(res < 0) {
-            mx_log_err("  group_id=%lu :: mxq_load_job_assigned_to_server: %m", group_id);
+            mx_log_err("  group_id=%lu :: mxq_load_job_from_group_assigned_to_server: %m", group_id);
             return 0;
         }
         if(res == 1) {
diff --git a/mxq_job.h b/mxq_job.h
index f29baf62..3bf62bbf 100644
--- a/mxq_job.h
+++ b/mxq_job.h
@@ -105,7 +105,7 @@ int mxq_set_job_status_running(struct mx_mysql *mysql, struct mxq_job *job);
 int mxq_set_job_status_exited(struct mx_mysql *mysql, struct mxq_job *job);
 int mxq_set_job_status_unknown_for_server(struct mx_mysql *mysql, char *hostname, char *server_id);
 int mxq_job_set_tmpfilenames(struct mxq_group *g, struct mxq_job *j);
-int mxq_load_job_assigned_to_server(struct mx_mysql *mysql, struct mxq_job **mxq_jobs, char *hostname, char *server_id);
+int mxq_load_job_from_group_assigned_to_server(struct mx_mysql *mysql, struct mxq_job **mxq_jobs, uint64_t group_id, char *hostname, char *server_id);
 int mxq_load_job_from_group_for_server(struct mx_mysql *mysql, struct mxq_job *mxqjob, uint64_t group_id, char *hostname, char *server_id, char *host_id);
 
 #endif

From 9c82995d8608c29a90d53efe886c36b0356cbbe1 Mon Sep 17 00:00:00 2001
From: Marius Tolzmann <tolzmann@molgen.mpg.de>
Date: Mon, 26 Oct 2015 16:38:42 +0100
Subject: [PATCH 13/18] mxqd: Introduce dynamic log directory based on
 LOCALSTATEDIR

---
 Makefile | 1 +
 mxq.h    | 6 ++++++
 mxqd.c   | 9 ++++++---
 3 files changed, 13 insertions(+), 3 deletions(-)

diff --git a/Makefile b/Makefile
index 1d1bd5a2..21c95c5b 100644
--- a/Makefile
+++ b/Makefile
@@ -255,6 +255,7 @@ install::
 	$(call quiet-installdir,0755,${DESTDIR}${SYSCONFDIR}/mxq)
 	$(call quiet-installdir,0755,${DESTDIR}${MAN1DIR})
 	$(call quiet-installdir,0755,${DESTDIR}${CGIDIR})
+	$(call quiet-installdir,0755,${DESTDIR}${LOCALSTATEDIR}/log)
 
 ########################################################################
 
diff --git a/mxq.h b/mxq.h
index 54b073f9..aac59fe1 100644
--- a/mxq.h
+++ b/mxq.h
@@ -46,6 +46,12 @@
 #endif
 #define MXQ_MYSQL_DEFAULT_GROUP_STR MXQ_MYSQL_DEFAULT_GROUP
 
+#if defined (LOCALSTATEDIR)
+#   define MXQ_LOGDIR LOCALSTATEDIR "/log"
+#else
+#   define MXQ_LOGDIR "/var/log"
+#endif
+
 static void mxq_print_generic_version(void)
 {
     printf(
diff --git a/mxqd.c b/mxqd.c
index 167ed126..97fb7d60 100644
--- a/mxqd.c
+++ b/mxqd.c
@@ -92,6 +92,9 @@ static void print_usage(void)
     "  -M, --mysql-default-file [mysql-file]    default: %s\n"
     "  -S, --mysql-default-group [mysql-group]  default: %s\n"
     "\n"
+    "Directories:\n"
+    "    LOGDIR      " MXQ_LOGDIR "\n"
+    "\n"
     "Environment:\n"
     "  MXQ_MYSQL_DEFAULT_FILE   change default for [mysql-file]\n"
     "  MXQ_MYSQL_DEFAULT_GROUP  change default for [mysql-group]\n"
@@ -499,11 +502,11 @@ int server_init(struct mxq_server *server, int argc, char *argv[])
     setup_stdin("/dev/null");
 
     if (!arg_nolog) {
-        if (access("/var/log",R_OK|W_OK|X_OK)) {
-            mx_log_err("MAIN: cant write to /var/log: %m");
+        if (access(MXQ_LOGDIR, R_OK|W_OK|X_OK)) {
+            mx_log_err("MAIN: cant write to " MXQ_LOGDIR ": %m");
             exit(EX_IOERR);
         }
-        res = setup_cronolog("/usr/sbin/cronolog", "/var/log/mxqd_log", "/var/log/%Y/mxqd_log-%Y-%m");
+        res = setup_cronolog("/usr/sbin/cronolog", MXQ_LOGDIR "/mxqd_log", MXQ_LOGDIR "/%Y/mxqd_log-%Y-%m");
         if (!res) {
             mx_log_err("MAIN: cronolog setup failed. exiting.");
             exit(EX_IOERR);

From d5d0eac0e8a21d234cb57389a69773be8b084099 Mon Sep 17 00:00:00 2001
From: Marius Tolzmann <tolzmann@molgen.mpg.de>
Date: Tue, 27 Oct 2015 11:44:29 +0100
Subject: [PATCH 14/18] mxqd: Allow to be run as non root user

---
 mxqd.c | 7 +------
 1 file changed, 1 insertion(+), 6 deletions(-)

diff --git a/mxqd.c b/mxqd.c
index 97fb7d60..3e66fff8 100644
--- a/mxqd.c
+++ b/mxqd.c
@@ -437,12 +437,7 @@ int server_init(struct mxq_server *server, int argc, char *argv[])
     MX_GETOPT_FINISH(optctl, argc, argv);
 
     if (!RUNNING_AS_ROOT) {
-#if defined(MXQ_DEVELOPMENT) || defined(RUNASNORMALUSER)
-        mx_log_notice("Running mxqd as non-root user.");
-#else
-        mx_log_err("Running mxqd as non-root user is not supported at the moment.");
-        exit(EX_USAGE);
-#endif
+        mx_log_warning("Running mxqd as non-root user.");
     }
 
     if (arg_daemonize && arg_nolog) {

From 5f694044036ad91d1a92f3284d9226383f6c7078 Mon Sep 17 00:00:00 2001
From: Marius Tolzmann <tolzmann@molgen.mpg.de>
Date: Tue, 27 Oct 2015 12:08:14 +0100
Subject: [PATCH 15/18] mxqd: Reorder server initialisation for clearer error
 logging

---
 mxqd.c | 27 +++++++++++++++------------
 1 file changed, 15 insertions(+), 12 deletions(-)

diff --git a/mxqd.c b/mxqd.c
index 3e66fff8..3d91557a 100644
--- a/mxqd.c
+++ b/mxqd.c
@@ -436,10 +436,6 @@ int server_init(struct mxq_server *server, int argc, char *argv[])
 
     MX_GETOPT_FINISH(optctl, argc, argv);
 
-    if (!RUNNING_AS_ROOT) {
-        mx_log_warning("Running mxqd as non-root user.");
-    }
-
     if (arg_daemonize && arg_nolog) {
         mx_log_err("Error while using conflicting options --daemonize and --no-log at once.");
         exit(EX_USAGE);
@@ -447,13 +443,6 @@ int server_init(struct mxq_server *server, int argc, char *argv[])
 
     memset(server, 0, sizeof(*server));
 
-    res = mx_mysql_initialize(&(server->mysql));
-    assert(res == 0);
-
-    mx_mysql_option_set_default_file(server->mysql,  arg_mysql_default_file);
-    mx_mysql_option_set_default_group(server->mysql, arg_mysql_default_group);
-    mx_mysql_option_set_reconnect(server->mysql, 1);
-
     server->hostname = arg_hostname;
     server->server_id = arg_server_id;
     server->initial_path = arg_initial_path;
@@ -498,16 +487,30 @@ int server_init(struct mxq_server *server, int argc, char *argv[])
 
     if (!arg_nolog) {
         if (access(MXQ_LOGDIR, R_OK|W_OK|X_OK)) {
-            mx_log_err("MAIN: cant write to " MXQ_LOGDIR ": %m");
+            if (!RUNNING_AS_ROOT)
+                mx_log_warning("Running mxqd as non-root user.");
+            mx_log_err("MAIN: can't write to " MXQ_LOGDIR ": %m");
             exit(EX_IOERR);
         }
         res = setup_cronolog("/usr/sbin/cronolog", MXQ_LOGDIR "/mxqd_log", MXQ_LOGDIR "/%Y/mxqd_log-%Y-%m");
         if (!res) {
+            if (!RUNNING_AS_ROOT)
+                mx_log_warning("Running mxqd as non-root user.");
             mx_log_err("MAIN: cronolog setup failed. exiting.");
             exit(EX_IOERR);
         }
     }
 
+    if (!RUNNING_AS_ROOT)
+        mx_log_warning("Running mxqd as non-root user.");
+
+    res = mx_mysql_initialize(&(server->mysql));
+    assert(res == 0);
+
+    mx_mysql_option_set_default_file(server->mysql,  arg_mysql_default_file);
+    mx_mysql_option_set_default_group(server->mysql, arg_mysql_default_group);
+    mx_mysql_option_set_reconnect(server->mysql, 1);
+
     res = mx_read_first_line_from_file("/proc/sys/kernel/random/boot_id", &str_bootid);
     assert(res == 36);
     assert(str_bootid);

From 5f9fbac5f3ef005d4f1f8713da495e723180d20c Mon Sep 17 00:00:00 2001
From: Marius Tolzmann <tolzmann@molgen.mpg.de>
Date: Tue, 27 Oct 2015 14:20:09 +0100
Subject: [PATCH 16/18] mx_util: Add mx_strconcat()

---
 mx_util.c      | 41 +++++++++++++++++++++++++++++++++++++++++
 mx_util.h      |  3 +++
 test_mx_util.c | 22 ++++++++++++++++++++++
 3 files changed, 66 insertions(+)

diff --git a/mx_util.c b/mx_util.c
index f783ed38..e2b488ce 100644
--- a/mx_util.c
+++ b/mx_util.c
@@ -856,6 +856,47 @@ int mx_strscan_ll(char **str, long long int *to)
     return res;
 }
 
+char *_mx_strconcat_do(char *first, ...)
+{
+    va_list ap;
+    char *join = NULL;
+    char *str;
+    char *ptr;
+    size_t len;
+
+    if (!first) {
+        join = strdup("");
+        return join;
+    }
+
+    len = strlen(first);
+
+    va_start(ap, first);
+        do {
+            str = va_arg(ap, char *);
+            if (!str)
+                break;
+            len += strlen(str);
+        } while(1);
+    va_end(ap);
+
+    join = malloc(len+1);
+    if (!join)
+        return NULL;
+
+    ptr = stpcpy(join, first);
+    va_start(ap, first);
+        do {
+            str = va_arg(ap, char *);
+            if (!str)
+                break;
+            ptr = stpcpy(ptr, str);
+        } while(1);
+    va_end(ap);
+
+    return join;
+}
+
 int mx_sleep(unsigned int seconds)
 {
     if (seconds)
diff --git a/mx_util.h b/mx_util.h
index 508476c6..92614fe5 100644
--- a/mx_util.h
+++ b/mx_util.h
@@ -136,6 +136,9 @@ int mx_read_first_line_from_file(char *fname, char **line);
 int mx_strscan_ull(char **str, unsigned long long int *to);
 int mx_strscan_ll(char **str, long long int *to);
 
+char *_mx_strconcat_do(char *first, ...);
+#define mx_strconcat(s, ...) _mx_strconcat_do((s), ##__VA_ARGS__, NULL)
+
 int mx_sleep(unsigned int seconds);
 int mx_sleep_nofail(unsigned int seconds);
 
diff --git a/test_mx_util.c b/test_mx_util.c
index 06cfe728..2f6be8f7 100644
--- a/test_mx_util.c
+++ b/test_mx_util.c
@@ -384,6 +384,27 @@ static void test_mx_strvec() {
     mx_strvec_free(strvec);
 }
 
+static void test_mx_strcat() {
+    char *str;
+    char *str2;
+
+    assert(str = mx_strconcat(NULL));
+    assert(mx_streq(str, ""));
+    mx_free_null(str);
+
+    assert(str = mx_strconcat("abc", "123"));
+    assert(mx_streq(str, "abc123"));
+
+    assert(str2 = mx_strconcat(str, "def", str, "456"));
+    assert(mx_streq(str2, "abc123defabc123456"));
+    mx_free_null(str2);
+
+    assert(str2 = mx_strconcat(str));
+    assert(mx_streq(str, str2));
+    mx_free_null(str);
+    mx_free_null(str2);
+}
+
 static void test_mx_cpuset(void)
 {
     cpu_set_t cpuset;
@@ -421,6 +442,7 @@ int main(int argc, char *argv[])
     test_mx_read_first_line_from_file();
     test_mx_strscan();
     test_mx_strvec();
+    test_mx_strcat();
     test_mx_cpuset();
     return 0;
 }

From 949b506ef056ff16cd7992c95c35f3643ff8cfd4 Mon Sep 17 00:00:00 2001
From: Marius Tolzmann <tolzmann@molgen.mpg.de>
Date: Tue, 27 Oct 2015 14:27:41 +0100
Subject: [PATCH 17/18] mxqd: Add --log-directory[=DIRECTORY]

resolves https://github.molgen.mpg.de/mariux64/mxq/issues/32
---
 mxqd.c | 33 ++++++++++++++++++++++++++++-----
 1 file changed, 28 insertions(+), 5 deletions(-)

diff --git a/mxqd.c b/mxqd.c
index 3d91557a..15fa51df 100644
--- a/mxqd.c
+++ b/mxqd.c
@@ -78,7 +78,9 @@ static void print_usage(void)
 #else
     "      --no-log                      default: write a logfile\n"
 #endif
+    "      --log-directory <logdir>      default: " MXQ_LOGDIR "\n"
     "      --debug                       default: info log level\n"
+    "\n"
     "      --recover-only  (recover from crash and exit)\n"
     "\n"
     "      --initial-path <path>         default: %s\n"
@@ -138,11 +140,26 @@ static void cpuset_clear_running(cpu_set_t *running,cpu_set_t *job) {
 }
 
 /**********************************************************************/
-int setup_cronolog(char *cronolog, char *link, char *format)
+int setup_cronolog(char *cronolog, char *logdir, char *rellink, char *relformat)
 {
     int res;
     int pipe_fd[2];
     int pid;
+    _mx_cleanup_free_ char *link = NULL;
+    _mx_cleanup_free_ char *format = NULL;
+
+    if (logdir) {
+        link   = mx_strconcat(logdir, "/", rellink);
+        format = mx_strconcat(logdir, "/", relformat);
+    } else {
+        link   = strdup(rellink);
+        format = strdup(relformat);
+    }
+
+    if (!link || !format) {
+        mx_log_err("can't allocate filenames: (%m)");
+        return 0;
+    }
 
     res = pipe(pipe_fd);
     if (res == -1) {
@@ -279,6 +296,7 @@ int server_init(struct mxq_server *server, int argc, char *argv[])
     char *arg_mysql_default_group;
     char *arg_mysql_default_file;
     char *arg_pidfile = NULL;
+    char *arg_logdir = NULL;
     char *arg_initial_path;
     char *arg_initial_tmpdir;
     char arg_daemonize = 0;
@@ -299,7 +317,8 @@ int server_init(struct mxq_server *server, int argc, char *argv[])
                 MX_OPTION_NO_ARG("version",            'V'),
                 MX_OPTION_NO_ARG("daemonize",            1),
                 MX_OPTION_NO_ARG("no-log",               3),
-                MX_OPTION_NO_ARG("log",                  4),
+                MX_OPTION_OPTIONAL_ARG("log",            4),
+                MX_OPTION_REQUIRED_ARG("log-directory",  4),
                 MX_OPTION_NO_ARG("debug",                5),
                 MX_OPTION_NO_ARG("recover-only",         9),
                 MX_OPTION_REQUIRED_ARG("pid-file",       2),
@@ -357,6 +376,7 @@ int server_init(struct mxq_server *server, int argc, char *argv[])
 
             case 4:
                 arg_nolog = 0;
+                arg_logdir = optctl.optarg;
                 break;
 
             case 5:
@@ -486,13 +506,16 @@ int server_init(struct mxq_server *server, int argc, char *argv[])
     setup_stdin("/dev/null");
 
     if (!arg_nolog) {
-        if (access(MXQ_LOGDIR, R_OK|W_OK|X_OK)) {
+        if (!arg_logdir)
+            arg_logdir = MXQ_LOGDIR;
+
+        if (access(arg_logdir, R_OK|W_OK|X_OK)) {
             if (!RUNNING_AS_ROOT)
                 mx_log_warning("Running mxqd as non-root user.");
-            mx_log_err("MAIN: can't write to " MXQ_LOGDIR ": %m");
+            mx_log_err("MAIN: can't write to '%s': %m", arg_logdir);
             exit(EX_IOERR);
         }
-        res = setup_cronolog("/usr/sbin/cronolog", MXQ_LOGDIR "/mxqd_log", MXQ_LOGDIR "/%Y/mxqd_log-%Y-%m");
+        res = setup_cronolog("/usr/sbin/cronolog", arg_logdir, "mxqd_log", "%Y/mxqd_log-%Y-%m");
         if (!res) {
             if (!RUNNING_AS_ROOT)
                 mx_log_warning("Running mxqd as non-root user.");

From 24fe8a3f5457a43cbba6d6bddcc40ae8f938fed7 Mon Sep 17 00:00:00 2001
From: Marius Tolzmann <tolzmann@molgen.mpg.de>
Date: Tue, 27 Oct 2015 15:55:29 +0100
Subject: [PATCH 18/18] web: Fix typo when (not) displaying max_sumrss

---
 web/pages/mxq/mxq.in | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/web/pages/mxq/mxq.in b/web/pages/mxq/mxq.in
index e938bbec..422c14b4 100755
--- a/web/pages/mxq/mxq.in
+++ b/web/pages/mxq/mxq.in
@@ -269,7 +269,7 @@ group_mtime           : $o{group_mtime}
 
 group_date_end        : $o{group_date_end}
 
-stats_max_sumrss                 : $o{stats_max_maxrss} kiB
+stats_max_sumrss                 : $o{stats_max_sumrss} kiB
 stats_max_maxrss                 : $o{stats_max_maxrss}
 stats_max_utime_sec              : $o{stats_max_utime_sec}
 stats_max_stime_sec              : $o{stats_max_stime_sec}
@@ -365,7 +365,7 @@ job_id_new       : $o{job_id_new}
 job_id_old       : $o{job_id_old}
 job_id_first     : $o{job_id_first}
 
-stats_max_sumrss : $o{stats_max_maxrss} kiB
+stats_max_sumrss : $o{stats_max_sumrss} kiB
 
 stats_status     : $o{stats_status}