Merge branch 'master' into issues/issue16

* master:
  mysql: Split create_tables into create_tables, drop_tables and create_trigger
  mysql: Drop tables and triggers only if they exist
  mxqd: limit killall_over_time to run every minute if jobs are running
  mxqd: Change default for --no-log in development build
  Makefile: remove definition of RUNASNORMALUSER in devel mode
  README: Add development instructions
  MXQ: update credits
  mxqd: Introduce new macro RUNNING_AS_ROOT
  Makefile: Add new target devel and remove nonroot
  Makefile: Add new target nonroot to build non-root mxqd
  Makefile: Define prefix variables in CFLAGS
  mxqd: Use full memory by default for every job when run as non-root
  mxqd: Prefix argument variables with arg_
  mxqd: When run as non-root only load groups for current user

Makefile

@@ -2,7 +2,7 @@ MXQ_VERSION_MAJOR = 0
 MXQ_VERSION_MINOR = 17
 MXQ_VERSION_PATCH = 0
 MXQ_VERSION_EXTRA = "beta"
-MXQ_VERSIONDATE = 2013-2015
+MXQ_VERSIONDATE = 2015
 
 MXQ_VERSION_GIT := $(shell git describe --long 2>/dev/null)
 
@@ -60,13 +60,17 @@ CGIDIR = ${LIBEXECDIR}/mxq/cgi
 ########################################################################
 
 MXQ_MYSQL_DEFAULT_FILE  = ${SYSCONFDIR}/mxq/mysql.cnf
-MXQ_MYSQL_DEFAULT_GROUP = mxqclient
+MXQ_MYSQL_DEFAULT_GROUP_CLIENT = mxqclient
+MXQ_MYSQL_DEFAULT_GROUP_SERVER = mxqd
+MXQ_MYSQL_DEFAULT_GROUP_DEVELOPMENT = mxqdevel
 
 MXQ_INITIAL_PATH   = /sbin:/bin:/usr/sbin:/usr/bin:/usr/local/sbin:/usr/local/bin
 MXQ_INITIAL_TMPDIR = /tmp
 
 CFLAGS_MXQ_MYSQL_DEFAULT_FILE  = -DMXQ_MYSQL_DEFAULT_FILE=\"$(MXQ_MYSQL_DEFAULT_FILE)\"
-CFLAGS_MXQ_MYSQL_DEFAULT_GROUP = -DMXQ_MYSQL_DEFAULT_GROUP=\"$(MXQ_MYSQL_DEFAULT_GROUP)\"
+CFLAGS_MXQ_MYSQL_DEFAULT_GROUP = -DMXQ_MYSQL_DEFAULT_GROUP_CLIENT=\"$(MXQ_MYSQL_DEFAULT_GROUP_CLIENT)\"
+CFLAGS_MXQ_MYSQL_DEFAULT_GROUP += -DMXQ_MYSQL_DEFAULT_GROUP_SERVER=\"$(MXQ_MYSQL_DEFAULT_GROUP_SERVER)\"
+CFLAGS_MXQ_MYSQL_DEFAULT_GROUP += -DMXQ_MYSQL_DEFAULT_GROUP_DEVELOPMENT=\"$(MXQ_MYSQL_DEFAULT_GROUP_DEVELOPMENT)\"
 CFLAGS_MXQ_INITIAL_PATH        = -DMXQ_INITIAL_PATH=\"$(MXQ_INITIAL_PATH)\"
 CFLAGS_MXQ_INITIAL_TMPDIR      = -DMXQ_INITIAL_TMPDIR=\"$(MXQ_INITIAL_TMPDIR)\"
 
@@ -112,6 +116,15 @@ CFLAGS += -DMXQ_VERSION=\"${MXQ_VERSION}\"
 CFLAGS += -DMXQ_VERSIONFULL=\"${MXQ_VERSIONFULL}\"
 CFLAGS += -DMXQ_VERSIONDATE=\"${MXQ_VERSIONDATE}\"
 CFLAGS += -DMXQ_VERSIONEXTRA=\"${MXQ_VERSIONEXTRA}\"
+CFLAGS += -DPREFIX=\"${PREFIX}\"
+CFLAGS += -DEPREFIX=\"${EPREFIX}\"
+CFLAGS += -DSBINDIR=\"${SBINDIR}\"
+CFLAGS += -DBINDIR=\"${BINDIR}\"
+CFLAGS += -DLIBDIR=\"${LIBDIR}\"
+CFLAGS += -DLIBEXECDIR=\"${LIBEXECDIR}\"
+CFLAGS += -DDATADIR=\"${DATADIR}\"
+CFLAGS += -DMANDIR=\"${MANDIR}\"
+CFLAGS += -DSYSCONFDIR=\"${SYSCONFDIR}\"
 CFLAGS += $(EXTRA_CFLAGS)
 
 ########################################################################
@@ -175,6 +188,10 @@ manpages/%: manpages/%.xml
 
 all: build
 
+.PHONY: devel
+devel: CFLAGS += -DMXQ_DEVELOPMENT
+devel: all
+
 ########################################################################
 
 .PHONY: test

README.md

@@ -1,5 +1,6 @@
-# mxq
-MXQ - mariux64 job scheduling system
+# MXQ - mariux64 job scheduling system
+- by Marius Tolzmann <marius.tolzmann@molgen.mpg.de> 2013-2015
+- and Donald Buczek <buczek@molgen.mpg.de> 2015-2015
 
 ## Sources
 ### Main git repository
@@ -13,11 +14,49 @@ https://github.com/mariux/mxq
 ## Installation
 ### Install using `GNU make`
 ```
-make 
-make install [DESTDIR=...]
+make
+make install
+```
+```
+make PREFIX=...
+make PREFIX=... [DESTDIR=...] install
 ```
 ### Install using `bee`
 ```
-bee init $(bee download git://github.molgen.mpg.de/mariux64/mxq.git) -e
+bee init $(bee download git://github.molgen.mpg.de/mariux64/mxq.git) --execute
 bee update mxq
 ```
+```
+bee init $(bee download git://github.molgen.mpg.de/mariux64/mxq.git) --prefix=... --execute
+bee update mxq
+```
+
+## Initial setup
+Definitions of the tables and triggers for the MySQL database can be found in
+[mysql/create_tables.sql](https://github.molgen.mpg.de/mariux64/mxq/blob/master/mysql/create_tables.sql)
+Be sure to create those once and check the same 
+[directory for alter_tables*.sql`](https://github.molgen.mpg.de/mariux64/mxq/blob/master/mysql/)
+files when upgrading. 
+
+
+## Development builds
+The `devel` target in the Makefile will enable all devolopment features
+by defining `MXQ_DEVELOPMENT` when compiling C sources.
+
+```
+make clean
+make devel PREFIX=/path/to/test
+make install PREFIX=/path/to/test
+```
+
+### Differences to production builds
+Some new features and improvements are enabled in development builds.
+Those features may not be tested for every situation yet and may result
+in database corruption and/or failing jobs.
+
+#### changed `mxqd` default options
+In devolopment builds `--no-log` is default (enable loggin with `--log`)
+
+#### Development database access
+Devolopment builds default to use `[mxqdevel]` groups from mysql config files 
+for servers and clients (instead of the default `[mxqclient]` and `[mxqd]` groups)

mxq.h

@@ -17,7 +17,7 @@
 #endif
 
 #ifndef MXQ_VERSIONDATE
-#   define MXQ_VERSIONDATE "2015"
+#   define MXQ_VERSIONDATE "today"
 #endif
 
 #ifndef MXQ_MYSQL_DEFAULT_FILE
@@ -27,16 +27,34 @@
 #   define MXQ_MYSQL_DEFAULT_FILE_STR MXQ_MYSQL_DEFAULT_FILE
 #endif
 
-#ifndef MXQ_MYSQL_DEFAULT_GROUP
-#   define MXQ_MYSQL_DEFAULT_GROUP     program_invocation_short_name
+#ifdef MXQ_DEVELOPMENT
+#   undef MXQ_MYSQL_DEFAULT_GROUP
+#   define MXQ_MYSQL_DEFAULT_GROUP MXQ_MYSQL_DEFAULT_GROUP_DEVELOPMENT
+#else
+#   ifdef MXQ_TYPE_SERVER
+#      ifdef MXQ_MYSQL_DEFAULT_GROUP_SERVER
+#          define MXQ_MYSQL_DEFAULT_GROUP MXQ_MYSQL_DEFAULT_GROUP_SERVER
+#      endif
+#   else
+#      ifdef MXQ_MYSQL_DEFAULT_GROUP_CLIENT
+#           define MXQ_MYSQL_DEFAULT_GROUP MXQ_MYSQL_DEFAULT_GROUP_CLIENT
+#      endif
+#   endif
+#   ifndef MXQ_MYSQL_DEFAULT_GROUP
+#       define MXQ_MYSQL_DEFAULT_GROUP program_invocation_short_name
+#   endif
 #endif
 #define MXQ_MYSQL_DEFAULT_GROUP_STR MXQ_MYSQL_DEFAULT_GROUP
 
 static void mxq_print_generic_version(void)
 {
     printf(
     "%s - " MXQ_VERSIONFULL "\n"
-    "  by Marius Tolzmann <tolzmann@molgen.mpg.de> " MXQ_VERSIONDATE "\n"
+#ifdef MXQ_DEVELOPMENT
+    "DEVELOPMENT VERSION: Do not use in production environments.\n"
+#endif
+    "  by Marius Tolzmann <marius.tolzmann@molgen.mpg.de> 2013-" MXQ_VERSIONDATE "\n"
+    "     and Donald Buczek <buczek@molgen.mpg.de> 2015-" MXQ_VERSIONDATE "\n"
     "  Max Planck Institute for Molecular Genetics - Berlin Dahlem\n",
         program_invocation_short_name
     );

mxq_group.c

@@ -292,39 +292,6 @@ int mxq_load_active_groups_for_user(struct mx_mysql *mysql, struct mxq_group **m
     return res;
 }
 
-int mxq_load_active_groups(struct mx_mysql *mysql, struct mxq_group **mxq_groups)
-{
-    int res;
-    struct mxq_group *groups = NULL;
-    struct mxq_group g = {0};
-    struct mx_mysql_bind result = {0};
-
-    assert(mysql);
-    assert(mxq_groups);
-
-    *mxq_groups = NULL;
-
-    char *query =
-            "SELECT"
-                GROUP_FIELDS
-            " FROM mxq_group"
-            " WHERE (group_jobs_inq > 0 OR group_jobs_running > 0)"
-            " ORDER BY user_name, group_mtime"
-            " LIMIT 1000";
-
-    res = bind_result_group_fields(&result, &g);
-    assert(res == 0);
-
-    res = mx_mysql_do_statement_retry_on_fail(mysql, query, NULL, &result, &g, (void **)&groups, sizeof(*groups));
-    if (res < 0) {
-        mx_log_err("mx_mysql_do_statement_retry_on_fail(): %m");
-        return res;
-    }
-
-    *mxq_groups = groups;
-    return res;
-}
-
 int mxq_load_running_groups(struct mx_mysql *mysql, struct mxq_group **mxq_groups)
 {
     int res;

mxq_group.h

@@ -68,7 +68,6 @@ inline uint64_t mxq_group_jobs_inq(struct mxq_group *g);
 
 int mxq_load_group(struct mx_mysql *mysql, struct mxq_group **mxq_groups, uint64_t group_id);
 int mxq_load_all_groups(struct mx_mysql *mysql, struct mxq_group **mxq_groups);
-int mxq_load_active_groups(struct mx_mysql *mysql, struct mxq_group **mxq_groups);
 int mxq_load_all_groups_for_user(struct mx_mysql *mysql, struct mxq_group **mxq_groups, uint64_t user_uid);
 int mxq_load_active_groups_for_user(struct mx_mysql *mysql, struct mxq_group **mxq_groups, uint64_t user_uid);
 int mxq_load_running_groups(struct mx_mysql *mysql, struct mxq_group **mxq_groups);