Merge remote-tracking branch 'mariux/issues/issue30'

implements parts of #30

* mariux/issues/issue30:
  mxqd: free structures to remove leftover memory in reaper process
  mxqd: be a bit more verbose when starting processes to log pids
  mxq_job: Minor cleanup
  mx_flock: export mx_flock_free() to free without releasing lock
  mxqd: Fix memory leak for host_cpu_set_str
  mxqd: Fix kill signals: send kill to pgrp instead of reaper pid
  mxqd: Cleanup reaper_process()
  mxqd: Cleanup user_process()
  mxqd: Cleanup init_child_process()
  mxqd: Cleanup job_has_finished() and job_lost()
  mxqd: Remove fspool_unlink()
  test_mxqd_control: Init server structure
  mxqd: Fix fspool_process_file()
  mxqd: Fix and rename server_reload_running() to load_running_jobs()
  mxqd: Rename load_groups() to load_running_groups()
  mxqd_control: Refactor and export server structure management
  mxqd: Cleanup start_job()
  mxqd: Cleanup server_close()
  mxqd: Cleanup server_dump()
  mxqd: Rename server_find_user() to server_find_user_by_uid()
  mxqd: Rename lost_scan_one()
  mxqd: Cleanup server_reload_running()
  mxqd: Cleanup catchall()
  mxqd: Cleanup load_groups()
  mxqd: Rename server_find_group() to server_get_group_list_by_group_id()
  mxqd: Rename server_find_job() to server_get_job_list_by_job_id()
  mxqd: Rename server_remove_job() to job_list_remove_self()
  mxqd: Rename server_find_job_by_pid() to server_get_job_list_by_pid()
  mxqd: Rename server_remove_job_by_pid() to server_remove_job_list_by_pid()
  mxqd: Rename killallcancelled() to killall_cancelled()
  mxqd: Cleanup killall()
  mxqd: Cleanup killall_over_time()
  mxqd: Cleanup start_users()
  mxqd: Cleanup start_user()
  mxqd: Rename remove_orphaned_groups() to remove_orphaned_group_lists()
  mxqd: Rename group_list_find_group() to _group_list_find_by_group()
  mxqd: Rename group_add_job() to group_list_add_job()
  mxqd: Rename user_list_find_uid() to _user_list_find_by_uid()
  mxqd: Rename server_update_groupdata() to server_update_group()
  mxqd: Rename server_add_user() to _server_add_group()
  mxqd: Rename user_update_groupdata() to _user_list_update_group()
  mxqd: Rename user_add_group() to _user_list_add_group()
  mxqd: Rename group_init() to _group_list_init()
  mxqd: reaper: ignore signals from mxqd
  mxqd: set cpu_set_running in group_add_job
  database: store and retrieve cpuset of job
  mxq_job: add a string version of host_cpu_set
  mxq_job: refactor (add do_jobs_statement)
  mxqd: do not finish jobs from signals when we have reaper output
  mxqd: killall_over_memory: Send SIGKILL after sending SIGTERM
  mxqd: better loglevels for killall_over_time
  mxqd: remove unused member
  mxqd: let reaper call setsid instread of user process
  mxqd: add job_is_lost
  mxqd: do not kill children in catchall
  mxq_job: add mxq_set_job_status_unknown
  mxqd: add SIGQUIT processing : do not kill or wait for children
  mxqd: let recover_from_previous_crash rebuild state for previous jobs
  mxqd: refactor (add reset_signals)
  mxq_job: add mxq_load_jobs_running_on_server
  mxqd: stop recover_from_previous_crash from deleting running jobs
  mxqd: add reaper
  mxqd: killall_over_memory: rename/cleanup variables
  mxqd: add help functions for fspool (finished job spool directory)
  mxqd: create MXQ_FINISHED_JOBSDIR on startup
  make: add FINISHED_JOBSDIR
  mx_util: add mx_mkdir_p
  mxqd: refactor (add job_has_finished)
  mxqd: refactor (add user_process)

.gitignore

@@ -14,8 +14,11 @@ mxqkill.o
 test_mx_util.o
 test_mx_log.o
 test_mx_mysql.o
+test_mxqd_control.o
 mxq_log.o
 mx_mysql.o
+mxqd_control.o
+
 
 mxqsub
 /mxqsub.1
@@ -26,7 +29,8 @@ mxqd
 mxqps
 test_mx_util
 test_mx_log
-test_mx_mysql
+test_mx_mysq
+test_mxqd_control
 
 /web/pages/mxq/mxq
 web/lighttpd.conf

Makefile

@@ -73,12 +73,15 @@ MXQ_MYSQL_DEFAULT_GROUP_DEVELOPMENT = mxqdevel
 MXQ_INITIAL_PATH   = /sbin:/bin:/usr/sbin:/usr/bin:/usr/local/sbin:/usr/local/bin
 MXQ_INITIAL_TMPDIR = /tmp
 
+MXQ_FINISHED_JOBSDIR = ${LOCALSTATEDIR}/spool/mxqd
+
 CFLAGS_MXQ_MYSQL_DEFAULT_FILE  = -DMXQ_MYSQL_DEFAULT_FILE=\"$(MXQ_MYSQL_DEFAULT_FILE)\"
 CFLAGS_MXQ_MYSQL_DEFAULT_GROUP = -DMXQ_MYSQL_DEFAULT_GROUP_CLIENT=\"$(MXQ_MYSQL_DEFAULT_GROUP_CLIENT)\"
 CFLAGS_MXQ_MYSQL_DEFAULT_GROUP += -DMXQ_MYSQL_DEFAULT_GROUP_SERVER=\"$(MXQ_MYSQL_DEFAULT_GROUP_SERVER)\"
 CFLAGS_MXQ_MYSQL_DEFAULT_GROUP += -DMXQ_MYSQL_DEFAULT_GROUP_DEVELOPMENT=\"$(MXQ_MYSQL_DEFAULT_GROUP_DEVELOPMENT)\"
 CFLAGS_MXQ_INITIAL_PATH        = -DMXQ_INITIAL_PATH=\"$(MXQ_INITIAL_PATH)\"
 CFLAGS_MXQ_INITIAL_TMPDIR      = -DMXQ_INITIAL_TMPDIR=\"$(MXQ_INITIAL_TMPDIR)\"
+CFLAGS_MXQ_FINISHED_JOBSDIR    = -DMXQ_FINISHED_JOBSDIR=\"${MXQ_FINISHED_JOBSDIR}\"
 
 MYSQL_CONFIG = mysql_config
 
@@ -297,6 +300,13 @@ mxq_job.h += mxq_group.h
 
 mxqd.h += mxqd.h
 
+### mxqd_conrol.h ------------------------------------------------------
+
+mxqd_control.h += mxqd_control.h
+mxqd_control.h += mxq_group.h
+mxqd_control.h += mxq_job.h
+mxqd_control.h += mxqd.h
+
 ### mx_getopt.h --------------------------------------------------------
 
 mx_getopt.h += mx_getopt.h
@@ -403,6 +413,12 @@ mxq_job.o: CFLAGS += $(CFLAGS_MYSQL)
 
 clean: CLEAN += mxq_job.o
 
+### mxqd_control.o -----------------------------------------------------
+
+mxqd_control.o: $(mxqd_control.h)
+
+clean: CLEAN += mxqd_control.o
+
 ### mxqd.o -------------------------------------------------------------
 
 mxqd.o: $(mx_getopt.h)
@@ -417,6 +433,7 @@ mxqd.o: $(mx_mysql.h)
 mxqd.o: CFLAGS += $(CFLAGS_MYSQL)
 mxqd.o: CFLAGS += $(CFLAGS_MXQ_INITIAL_PATH)
 mxqd.o: CFLAGS += $(CFLAGS_MXQ_INITIAL_TMPDIR)
+mxqd.o: CFLAGS += $(CFLAGS_MXQ_FINISHED_JOBSDIR)
 mxqd.o: CFLAGS += -Wno-unused-but-set-variable
 
 clean: CLEAN += mxqd.o
@@ -448,6 +465,7 @@ mxqd: mx_getopt.o
 mxqd: mxq_group.o
 mxqd: mxq_job.o
 mxqd: mx_mysql.o
+mxqd: mxqd_control.o
 mxqd: LDLIBS += $(LDLIBS_MYSQL)
 
 build: mxqd
@@ -596,3 +614,18 @@ test_mx_mysql: mx_log.o
 test_mx_mysql: mx_util.o
 test_mx_mysql: LDLIBS += $(LDLIBS_MYSQL)
 clean: CLEAN += test_mx_mysql
+
+
+test_mxqd_control.o: $(mxqd_control.h)
+clean: CLEAN += test_mxqd_control.o
+
+test_mxqd_control: mxqd_control.o
+test_mxqd_control: mx_log.o
+test_mxqd_control: mx_util.o
+test_mxqd_control: mx_mysql.o
+test_mxqd_control: mxq_group.o
+test_mxqd_control: LDLIBS += $(LDLIBS_MYSQL)
+
+clean: CLEAN += test_mxqd_control
+
+test: test_mxqd_control

mx_flock.c

@@ -58,11 +58,7 @@ static inline void _flock_free(struct mx_flock *lock)
     if (!lock)
         return;
 
-    if (lock->fname)
-        mx_free_null(lock->fname);
-
-    if (lock->fd >= 0)
-        _flock_close(lock);
+    mx_free_null(lock->fname);
 
     free(lock);
 }
@@ -112,6 +108,7 @@ struct mx_flock *mx_flock(int operation, char *fmt, ...)
             if (errno == EWOULDBLOCK)
                 return lock;
             mx_log_err("flock(): %m");
+            _flock_close(lock);
             _flock_free(lock);
             return NULL;
         }
@@ -155,3 +152,8 @@ int mx_funlock(struct mx_flock *lock)
 
     return res;
 }
+
+void mx_flock_free(struct mx_flock *lock)
+{
+    _flock_free(lock);
+}

mx_flock.h

@@ -10,5 +10,6 @@ struct mx_flock {
 
 struct mx_flock *mx_flock(int operation, char *fmt, ...);
 int mx_funlock(struct mx_flock *lock);
+void mx_flock_free(struct mx_flock *lock);
 
 #endif

mx_util.c

@@ -11,8 +11,8 @@
 
 #include <ctype.h>
 
-//#include <sys/types.h>
-//#include <sys/stat.h>
+#include <sys/types.h>
+#include <sys/stat.h>
 #include <fcntl.h>
 
 #include "mx_log.h"
@@ -1272,3 +1272,30 @@ char *mx_cpuset_to_str(cpu_set_t* cpuset_ptr)
     mx_strvec_free(strvec);
     return out;
 }
+
+int mx_mkdir_p(char *path, mode_t mode)
+{
+    struct stat st;
+    int err;
+    char *d;
+    _mx_cleanup_free_ char *copy = NULL;
+
+    if (stat(path, &st) == 0)
+        return 0;
+
+    copy=mx_strdup_forever(path);
+    d=copy;
+
+    while (*++d == '/');
+
+    while ((d = strchr(d, '/'))) {
+        *d = '\0';
+        err = stat(copy, &st) && mkdir(copy, mode);
+        *d++ = '/';
+        if (err)
+           return -errno;
+        while (*d == '/')
+            ++d;
+    }
+    return (stat(copy, &st) && mkdir(copy, mode)) ? -errno : 0;
+}

mx_util.h

@@ -162,4 +162,7 @@ char*  mx_strvec_join(char *sep,char **strvec);
 char* mx_cpuset_to_str(cpu_set_t* cpuset_ptr);
 int   mx_str_to_cpuset(cpu_set_t* cpuset_ptr,char *str);
 
+int mx_mkdir_p(char *path, mode_t mode);
+
+
 #endif