Merge branch 'master' into issues/issue7

* master:
  MXQ bump version
  make: add LOCALSTATEDIR
  web: show threads in group table and runtime in minutes for completed jobs
  mxq_job: fix memory leak
  mxqd: fix typo in log
  MXQ bump version
  mxqd: Fix root check
  MXQ bump version
  mxqd: Send TERM to process when over memory instead of KILL to pgrp
  mxqd: Store and use max_sumrss
  mxq_job: Minor cleanup
  Makefile: Build and run tests by default
  mx_log: Fix typo "ALERT"
  mxq_log: Use mx_streq()
  README: update install section
  mx_log: Add mx_logva*() to use va lists when logging
  mx_log: Fix return value of mx_log_printf()
  test_mx_util: Fix renamed function
  README: Update mysql section
  mysql: Split create_tables into create_tables, drop_tables and create_trigger
  mysql: Drop tables and triggers only if they exist
  mxqd: Kill jobs over memory at most every 60 seconds
  mx_proc: Use pgrp in addition to ppid to group processes
  mxqps: Add new tool to list mxqd processes
  mx_proc: Add process tree related functions
  mxqd: limit killall_over_time to run every minute if jobs are running
  mxqd: Change default for --no-log in development build
  Makefile: remove definition of RUNASNORMALUSER in devel mode
  README: Add development instructions
  MXQ: update credits
  mxqd: Introduce new macro RUNNING_AS_ROOT
  Makefile: Add new target devel and remove nonroot
  mx_proc: Move process related function from mx_util to mx_proc
  mxqd: set MXQ_HOSTID in main process

.gitignore

@@ -2,6 +2,8 @@ mx_getopt.o
 mx_flock.o
 mx_log.o
 mx_util.o
+mx_proc.o
+mxqps.o
 mxq_group.o
 mxqadmin.o
 mxqdump.o
@@ -21,6 +23,7 @@ mxqadmin
 mxqdump
 mxqkill
 mxqd
+mxqps
 test_mx_util
 test_mx_log
 test_mx_mysql

Makefile

@@ -1,8 +1,8 @@
 MXQ_VERSION_MAJOR = 0
-MXQ_VERSION_MINOR = 17
-MXQ_VERSION_PATCH = 0
+MXQ_VERSION_MINOR = 18
+MXQ_VERSION_PATCH = 2
 MXQ_VERSION_EXTRA = "beta"
-MXQ_VERSIONDATE = 2013-2015
+MXQ_VERSIONDATE = 2015
 
 MXQ_VERSION_GIT := $(shell git describe --long 2>/dev/null)
 
@@ -25,6 +25,7 @@ LIBEXECDIR = ${EPREFIX}/libexec
 DATADIR    = ${PREFIX}/share
 MANDIR     = ${DATADIR}/man
 SYSCONFDIR = ${PREFIX}/etc
+LOCALSTATEDIR = ${PREFIX}/var
 
 DESTDIR=
 
@@ -43,6 +44,11 @@ ifneq (, $(filter /usr /usr/local, ${PREFIX}))
     SYSCONFDIR = /etc
 endif
 
+### set localstatedir /var if prefix /usr || /usr/local
+ifneq (, $(filter /usr /usr/local, ${PREFIX}))
+    LOCALSTATEDIR = /var
+endif
+
 ########################################################################
 
 ### strip /mxq from SYSCONFDIR if set
@@ -60,13 +66,17 @@ CGIDIR = ${LIBEXECDIR}/mxq/cgi
 ########################################################################
 
 MXQ_MYSQL_DEFAULT_FILE  = ${SYSCONFDIR}/mxq/mysql.cnf
-MXQ_MYSQL_DEFAULT_GROUP = mxqclient
+MXQ_MYSQL_DEFAULT_GROUP_CLIENT = mxqclient
+MXQ_MYSQL_DEFAULT_GROUP_SERVER = mxqd
+MXQ_MYSQL_DEFAULT_GROUP_DEVELOPMENT = mxqdevel
 
 MXQ_INITIAL_PATH   = /sbin:/bin:/usr/sbin:/usr/bin:/usr/local/sbin:/usr/local/bin
 MXQ_INITIAL_TMPDIR = /tmp
 
 CFLAGS_MXQ_MYSQL_DEFAULT_FILE  = -DMXQ_MYSQL_DEFAULT_FILE=\"$(MXQ_MYSQL_DEFAULT_FILE)\"
-CFLAGS_MXQ_MYSQL_DEFAULT_GROUP = -DMXQ_MYSQL_DEFAULT_GROUP=\"$(MXQ_MYSQL_DEFAULT_GROUP)\"
+CFLAGS_MXQ_MYSQL_DEFAULT_GROUP = -DMXQ_MYSQL_DEFAULT_GROUP_CLIENT=\"$(MXQ_MYSQL_DEFAULT_GROUP_CLIENT)\"
+CFLAGS_MXQ_MYSQL_DEFAULT_GROUP += -DMXQ_MYSQL_DEFAULT_GROUP_SERVER=\"$(MXQ_MYSQL_DEFAULT_GROUP_SERVER)\"
+CFLAGS_MXQ_MYSQL_DEFAULT_GROUP += -DMXQ_MYSQL_DEFAULT_GROUP_DEVELOPMENT=\"$(MXQ_MYSQL_DEFAULT_GROUP_DEVELOPMENT)\"
 CFLAGS_MXQ_INITIAL_PATH        = -DMXQ_INITIAL_PATH=\"$(MXQ_INITIAL_PATH)\"
 CFLAGS_MXQ_INITIAL_TMPDIR      = -DMXQ_INITIAL_TMPDIR=\"$(MXQ_INITIAL_TMPDIR)\"
 
@@ -121,6 +131,7 @@ CFLAGS += -DLIBEXECDIR=\"${LIBEXECDIR}\"
 CFLAGS += -DDATADIR=\"${DATADIR}\"
 CFLAGS += -DMANDIR=\"${MANDIR}\"
 CFLAGS += -DSYSCONFDIR=\"${SYSCONFDIR}\"
+CFLAGS += -DLOCALSTATEDIR=\"${LOCALSTATEDIR}\"
 CFLAGS += $(EXTRA_CFLAGS)
 
 ########################################################################
@@ -182,11 +193,11 @@ manpages/%: manpages/%.xml
 .PHONY: all
 .PHONY: build
 
-all: build
+all: build test
 
-.PHONY: nonroot
-nonroot: CFLAGS += -DRUNASNORMALUSER
-nonroot: all
+.PHONY: devel
+devel: CFLAGS += -DMXQ_DEVELOPMENT
+devel: all
 
 ########################################################################
 
@@ -255,6 +266,10 @@ mx_log.h += mx_log.h
 
 mx_util.h += mx_util.h
 
+### mx_proc.h ----------------------------------------------------------
+
+mx_proc.h += mx_proc.h
+
 ### mx_flock.h ---------------------------------------------------------
 
 mx_flock.h += mx_flock.h
@@ -305,6 +320,12 @@ mx_util.o: $(mx_log.h)
 
 clean: CLEAN += mx_util.o
 
+### mx_proc ------------------------------------------------------------
+
+mx_proc.o: $(mx_proc.h)
+
+clean: CLEAN += mx_proc.o
+
 ### mx_flock.o ---------------------------------------------------------
 
 mx_flock.o: $(mx_flock.h)
@@ -386,6 +407,7 @@ clean: CLEAN += mxq_job.o
 mxqd.o: $(mx_getopt.h)
 mxqd.o: $(mx_flock.h)
 mxqd.o: $(mx_util.h)
+mxqd.o: $(mx_proc.h)
 mxqd.o: $(mx_log.h)
 mxqd.o: $(mxqd.h)
 mxqd.o: $(mxq_group.h)
@@ -418,6 +440,7 @@ clean: CLEAN += mxqsub.o
 
 mxqd: mx_flock.o
 mxqd: mx_util.o
+mxqd: mx_proc.o
 mxqd: mx_log.o
 mxqd: mxq_log.o
 mxqd: mx_getopt.o
@@ -496,6 +519,21 @@ clean: CLEAN += mxqkill
 install:: mxqkill
 	$(call quiet-installforuser,$(SUID_MODE),$(UID_CLIENT),$(GID_CLIENT),mxqkill,${DESTDIR}${BINDIR}/mxqkill)
 
+### mxqps -------------------------------------------------------------
+
+mxqps.o: $(mx_proc.h)
+mxqps.o: $(mx_util.h)
+
+clean: CLEAN += mxqps.o
+
+mxqps: mx_log.o
+mxqps: mx_util.o
+mxqps: mx_proc.o
+
+build: mxqps
+
+clean: CLEAN += mxqps
+
 ########################################################################
 
 fix: FIX += mxqdctl-hostconfig.sh
@@ -535,6 +573,7 @@ test_mx_util.o: $(mx_util.h)
 clean: CLEAN += test_mx_util.o
 
 test_mx_util: mx_util.o
+test_mx_util: mx_proc.o
 test_mx_util: mx_log.o
 clean: CLEAN += test_mx_util
 

README.md

@@ -1,5 +1,6 @@
-# mxq
-MXQ - mariux64 job scheduling system
+# MXQ - mariux64 job scheduling system
+- by Marius Tolzmann <marius.tolzmann@molgen.mpg.de> 2013-2015
+- and Donald Buczek <buczek@molgen.mpg.de> 2015-2015
 
 ## Sources
 ### Main git repository
@@ -13,11 +14,56 @@ https://github.com/mariux/mxq
 ## Installation
 ### Install using `GNU make`
 ```
-make 
-make install [DESTDIR=...]
+make install
+```
+
+or to specify a prefix:
+```
+make PREFIX=... [DESTDIR=...] install
 ```
 ### Install using `bee`
 ```
-bee init $(bee download git://github.molgen.mpg.de/mariux64/mxq.git) -e
+bee init $(bee download git://github.molgen.mpg.de/mariux64/mxq.git) --execute
 bee update mxq
 ```
+
+or to specify a prefix
+```
+bee init $(bee download git://github.molgen.mpg.de/mariux64/mxq.git) --prefix=... --execute
+bee update mxq
+```
+
+## Initial setup
+Definitions of the tables for the MySQL database can be found in
+[mysql/create_tables.sql](https://github.molgen.mpg.de/mariux64/mxq/blob/master/mysql/create_tables.sql)
+and
+[mysql/create_trigger.sql](https://github.molgen.mpg.de/mariux64/mxq/blob/master/mysql/create_trigger.sql)
+Be sure to create those once and check the same 
+[directory for alter_tables*.sql`](https://github.molgen.mpg.de/mariux64/mxq/blob/master/mysql/)
+files when upgrading. 
+```
+mysql [options] [database] <mysql/create_tables.sql
+mysql [options] [database] <mysql/create_trigger.sql
+```
+
+## Development builds
+The `devel` target in the Makefile will enable all devolopment features
+by defining `MXQ_DEVELOPMENT` when compiling C sources.
+
+```
+make clean
+make devel PREFIX=/path/to/test
+make install PREFIX=/path/to/test
+```
+
+### Differences to production builds
+Some new features and improvements are enabled in development builds.
+Those features may not be tested for every situation yet and may result
+in database corruption and/or failing jobs.
+
+#### changed `mxqd` default options
+In devolopment builds `--no-log` is default (enable loggin with `--log`)
+
+#### Development database access
+Devolopment builds default to use `[mxqdevel]` groups from mysql config files 
+for servers and clients (instead of the default `[mxqclient]` and `[mxqd]` groups)

mx_log.c

@@ -87,7 +87,7 @@ int mx_log_printf(const char *fmt, ...)
     res  = fflush(stderr);
     mx_free_null(msg);
 
-    if (len2 != len)
+    if (len2 != len+1)
         return -(errno=EIO);
 
     if (!res)
@@ -109,7 +109,7 @@ static int log_log(int level, int loglevel, char *file, unsigned long line, cons
             prefix = "EMERGENCY: ";
             break;
         case MX_LOG_ALERT:
-            prefix = "AKERT: ";
+            prefix = "ALERT: ";
             break;
         case MX_LOG_CRIT:
             prefix = "CRITCAL ERROR: ";
@@ -137,12 +137,11 @@ static int log_log(int level, int loglevel, char *file, unsigned long line, cons
     return mx_log_printf("%s%s", prefix, msg);
 }
 
-int mx_log_do(int level, char *file, unsigned long line, const char *func, const char *fmt, ...)
+int mx_logva_do(int level, char *file, unsigned long line, const char *func, const char *fmt, va_list ap)
 {
     int loglevel;
     int len;
     char *msg = NULL;
-    va_list ap;
     int res;
     int preserved_errno = errno;
 
@@ -153,9 +152,7 @@ int mx_log_do(int level, char *file, unsigned long line, const char *func, const
         return 0;
     }
 
-    va_start(ap, fmt);
     len = vasprintf(&msg, fmt, ap);
-    va_end(ap);
 
     if (len == -1) {
         errno = preserved_errno;
@@ -173,6 +170,18 @@ int mx_log_do(int level, char *file, unsigned long line, const char *func, const
     return res;
 }
 
+int mx_log_do(int level, char *file, unsigned long line, const char *func, const char *fmt, ...)
+{
+    va_list ap;
+    int res;
+
+    va_start(ap, fmt);
+    res = mx_logva_do(level, file, line, func, fmt, ap);
+    va_end(ap);
+
+    return res;
+}
+
 int mx_log_finish(void)
 {
     if (mx_log_log)

mx_log.h

@@ -22,6 +22,7 @@
 #define MX_LOG_DEBUG   MX_LOG_SYSLOG_TO_MXLOG(LOG_DEBUG)   /* debug-level messages */
 
 #define mx_log(level, fmt, ...)  mx_log_do((level), __FILE__, __LINE__, __func__, (fmt), ##__VA_ARGS__)
+#define mx_logva(level, fmt, ap) mx_logva_do((level), __FILE__, __LINE__, __func__, (fmt), (ap))
 
 #define mx_log_fatal(fmt, ...)   mx_log(MX_LOG_EMERG,   (fmt), ##__VA_ARGS__)
 #define mx_log_emerg(fmt, ...)   mx_log(MX_LOG_EMERG,   (fmt), ##__VA_ARGS__)
@@ -33,6 +34,16 @@
 #define mx_log_info(fmt, ...)    mx_log(MX_LOG_INFO,    (fmt), ##__VA_ARGS__)
 #define mx_log_debug(fmt, ...)   mx_log(MX_LOG_DEBUG,   (fmt), ##__VA_ARGS__)
 
+#define mx_logva_fatal(fmt, ap)   mx_log(MX_LOG_EMERG,   (fmt), (ap))
+#define mx_logva_emerg(fmt, ap)   mx_log(MX_LOG_EMERG,   (fmt), (ap))
+#define mx_logva_alert(fmt, ap)   mx_log(MX_LOG_ALERT,   (fmt), (ap))
+#define mx_logva_crit(fmt, ap)    mx_log(MX_LOG_CRIT,    (fmt), (ap))
+#define mx_logva_err(fmt, ap)     mx_log(MX_LOG_ERR,     (fmt), (ap))
+#define mx_logva_warning(fmt, ap) mx_log(MX_LOG_WARNING, (fmt), (ap))
+#define mx_logva_notice(fmt, ap)  mx_log(MX_LOG_NOTICE,  (fmt), (ap))
+#define mx_logva_info(fmt, ap)    mx_log(MX_LOG_INFO,    (fmt), (ap))
+#define mx_logva_debug(fmt, ap)   mx_log(MX_LOG_DEBUG,   (fmt), (ap))
+
 int mx_log_log(int level, int loglevel, char *file, unsigned long line, const char *func, const char *msg) __attribute__ ((weak));
 int mx_log_print(char *msg, size_t len) __attribute__ ((weak));