From 2200fd6cced71f473c29fc8d69f94b61b14d077c Mon Sep 17 00:00:00 2001 From: Marius Tolzmann Date: Tue, 27 Oct 2015 11:38:07 +0100 Subject: [PATCH 1/6] mxq_job: Only load jobs assigned for the group that should be started this fixes https://github.molgen.mpg.de/mariux64/mxq/issues/7 --- mxq_job.c | 10 ++++++---- mxq_job.h | 2 +- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/mxq_job.c b/mxq_job.c index d4bbe21f..4b33fda3 100644 --- a/mxq_job.c +++ b/mxq_job.c @@ -601,7 +601,7 @@ int mxq_job_set_tmpfilenames(struct mxq_group *g, struct mxq_job *j) return 1; } -int mxq_load_job_assigned_to_server(struct mx_mysql *mysql, struct mxq_job **mxq_jobs, char *hostname, char *server_id) +int mxq_load_job_from_group_assigned_to_server(struct mx_mysql *mysql, struct mxq_job **mxq_jobs, uint64_t group_id, char *hostname, char *server_id) { int res; struct mxq_job *jobs = NULL; @@ -624,14 +624,16 @@ int mxq_load_job_assigned_to_server(struct mx_mysql *mysql, struct mxq_job **mxq " WHERE job_status = " status_str(MXQ_JOB_STATUS_ASSIGNED) " AND host_hostname = ?" " AND server_id = ?" + " AND group_id = ?" " LIMIT 1"; - res = mx_mysql_bind_init_param(¶m, 2); + res = mx_mysql_bind_init_param(¶m, 3); assert(res == 0); res = 0; res += mx_mysql_bind_var(¶m, 0, string, &hostname); res += mx_mysql_bind_var(¶m, 1, string, &server_id); + res += mx_mysql_bind_var(¶m, 2, uint64, &group_id); assert(res == 0); res = bind_result_job_fields(&result, &j); @@ -662,10 +664,10 @@ int mxq_load_job_from_group_for_server(struct mx_mysql *mysql, struct mxq_job *m assert(*host_id); do { - res = mxq_load_job_assigned_to_server(mysql, &jobs, hostname, server_id); + res = mxq_load_job_from_group_assigned_to_server(mysql, &jobs, group_id, hostname, server_id); if(res < 0) { - mx_log_err(" group_id=%lu :: mxq_load_job_assigned_to_server: %m", group_id); + mx_log_err(" group_id=%lu :: mxq_load_job_from_group_assigned_to_server: %m", group_id); return 0; } if(res == 1) { diff --git a/mxq_job.h b/mxq_job.h index f29baf62..3bf62bbf 100644 --- a/mxq_job.h +++ b/mxq_job.h @@ -105,7 +105,7 @@ int mxq_set_job_status_running(struct mx_mysql *mysql, struct mxq_job *job); int mxq_set_job_status_exited(struct mx_mysql *mysql, struct mxq_job *job); int mxq_set_job_status_unknown_for_server(struct mx_mysql *mysql, char *hostname, char *server_id); int mxq_job_set_tmpfilenames(struct mxq_group *g, struct mxq_job *j); -int mxq_load_job_assigned_to_server(struct mx_mysql *mysql, struct mxq_job **mxq_jobs, char *hostname, char *server_id); +int mxq_load_job_from_group_assigned_to_server(struct mx_mysql *mysql, struct mxq_job **mxq_jobs, uint64_t group_id, char *hostname, char *server_id); int mxq_load_job_from_group_for_server(struct mx_mysql *mysql, struct mxq_job *mxqjob, uint64_t group_id, char *hostname, char *server_id, char *host_id); #endif From 9c82995d8608c29a90d53efe886c36b0356cbbe1 Mon Sep 17 00:00:00 2001 From: Marius Tolzmann Date: Mon, 26 Oct 2015 16:38:42 +0100 Subject: [PATCH 2/6] mxqd: Introduce dynamic log directory based on LOCALSTATEDIR --- Makefile | 1 + mxq.h | 6 ++++++ mxqd.c | 9 ++++++--- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 1d1bd5a2..21c95c5b 100644 --- a/Makefile +++ b/Makefile @@ -255,6 +255,7 @@ install:: $(call quiet-installdir,0755,${DESTDIR}${SYSCONFDIR}/mxq) $(call quiet-installdir,0755,${DESTDIR}${MAN1DIR}) $(call quiet-installdir,0755,${DESTDIR}${CGIDIR}) + $(call quiet-installdir,0755,${DESTDIR}${LOCALSTATEDIR}/log) ######################################################################## diff --git a/mxq.h b/mxq.h index 54b073f9..aac59fe1 100644 --- a/mxq.h +++ b/mxq.h @@ -46,6 +46,12 @@ #endif #define MXQ_MYSQL_DEFAULT_GROUP_STR MXQ_MYSQL_DEFAULT_GROUP +#if defined (LOCALSTATEDIR) +# define MXQ_LOGDIR LOCALSTATEDIR "/log" +#else +# define MXQ_LOGDIR "/var/log" +#endif + static void mxq_print_generic_version(void) { printf( diff --git a/mxqd.c b/mxqd.c index 167ed126..97fb7d60 100644 --- a/mxqd.c +++ b/mxqd.c @@ -92,6 +92,9 @@ static void print_usage(void) " -M, --mysql-default-file [mysql-file] default: %s\n" " -S, --mysql-default-group [mysql-group] default: %s\n" "\n" + "Directories:\n" + " LOGDIR " MXQ_LOGDIR "\n" + "\n" "Environment:\n" " MXQ_MYSQL_DEFAULT_FILE change default for [mysql-file]\n" " MXQ_MYSQL_DEFAULT_GROUP change default for [mysql-group]\n" @@ -499,11 +502,11 @@ int server_init(struct mxq_server *server, int argc, char *argv[]) setup_stdin("/dev/null"); if (!arg_nolog) { - if (access("/var/log",R_OK|W_OK|X_OK)) { - mx_log_err("MAIN: cant write to /var/log: %m"); + if (access(MXQ_LOGDIR, R_OK|W_OK|X_OK)) { + mx_log_err("MAIN: cant write to " MXQ_LOGDIR ": %m"); exit(EX_IOERR); } - res = setup_cronolog("/usr/sbin/cronolog", "/var/log/mxqd_log", "/var/log/%Y/mxqd_log-%Y-%m"); + res = setup_cronolog("/usr/sbin/cronolog", MXQ_LOGDIR "/mxqd_log", MXQ_LOGDIR "/%Y/mxqd_log-%Y-%m"); if (!res) { mx_log_err("MAIN: cronolog setup failed. exiting."); exit(EX_IOERR); From d5d0eac0e8a21d234cb57389a69773be8b084099 Mon Sep 17 00:00:00 2001 From: Marius Tolzmann Date: Tue, 27 Oct 2015 11:44:29 +0100 Subject: [PATCH 3/6] mxqd: Allow to be run as non root user --- mxqd.c | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/mxqd.c b/mxqd.c index 97fb7d60..3e66fff8 100644 --- a/mxqd.c +++ b/mxqd.c @@ -437,12 +437,7 @@ int server_init(struct mxq_server *server, int argc, char *argv[]) MX_GETOPT_FINISH(optctl, argc, argv); if (!RUNNING_AS_ROOT) { -#if defined(MXQ_DEVELOPMENT) || defined(RUNASNORMALUSER) - mx_log_notice("Running mxqd as non-root user."); -#else - mx_log_err("Running mxqd as non-root user is not supported at the moment."); - exit(EX_USAGE); -#endif + mx_log_warning("Running mxqd as non-root user."); } if (arg_daemonize && arg_nolog) { From 5f694044036ad91d1a92f3284d9226383f6c7078 Mon Sep 17 00:00:00 2001 From: Marius Tolzmann Date: Tue, 27 Oct 2015 12:08:14 +0100 Subject: [PATCH 4/6] mxqd: Reorder server initialisation for clearer error logging --- mxqd.c | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/mxqd.c b/mxqd.c index 3e66fff8..3d91557a 100644 --- a/mxqd.c +++ b/mxqd.c @@ -436,10 +436,6 @@ int server_init(struct mxq_server *server, int argc, char *argv[]) MX_GETOPT_FINISH(optctl, argc, argv); - if (!RUNNING_AS_ROOT) { - mx_log_warning("Running mxqd as non-root user."); - } - if (arg_daemonize && arg_nolog) { mx_log_err("Error while using conflicting options --daemonize and --no-log at once."); exit(EX_USAGE); @@ -447,13 +443,6 @@ int server_init(struct mxq_server *server, int argc, char *argv[]) memset(server, 0, sizeof(*server)); - res = mx_mysql_initialize(&(server->mysql)); - assert(res == 0); - - mx_mysql_option_set_default_file(server->mysql, arg_mysql_default_file); - mx_mysql_option_set_default_group(server->mysql, arg_mysql_default_group); - mx_mysql_option_set_reconnect(server->mysql, 1); - server->hostname = arg_hostname; server->server_id = arg_server_id; server->initial_path = arg_initial_path; @@ -498,16 +487,30 @@ int server_init(struct mxq_server *server, int argc, char *argv[]) if (!arg_nolog) { if (access(MXQ_LOGDIR, R_OK|W_OK|X_OK)) { - mx_log_err("MAIN: cant write to " MXQ_LOGDIR ": %m"); + if (!RUNNING_AS_ROOT) + mx_log_warning("Running mxqd as non-root user."); + mx_log_err("MAIN: can't write to " MXQ_LOGDIR ": %m"); exit(EX_IOERR); } res = setup_cronolog("/usr/sbin/cronolog", MXQ_LOGDIR "/mxqd_log", MXQ_LOGDIR "/%Y/mxqd_log-%Y-%m"); if (!res) { + if (!RUNNING_AS_ROOT) + mx_log_warning("Running mxqd as non-root user."); mx_log_err("MAIN: cronolog setup failed. exiting."); exit(EX_IOERR); } } + if (!RUNNING_AS_ROOT) + mx_log_warning("Running mxqd as non-root user."); + + res = mx_mysql_initialize(&(server->mysql)); + assert(res == 0); + + mx_mysql_option_set_default_file(server->mysql, arg_mysql_default_file); + mx_mysql_option_set_default_group(server->mysql, arg_mysql_default_group); + mx_mysql_option_set_reconnect(server->mysql, 1); + res = mx_read_first_line_from_file("/proc/sys/kernel/random/boot_id", &str_bootid); assert(res == 36); assert(str_bootid); From 5f9fbac5f3ef005d4f1f8713da495e723180d20c Mon Sep 17 00:00:00 2001 From: Marius Tolzmann Date: Tue, 27 Oct 2015 14:20:09 +0100 Subject: [PATCH 5/6] mx_util: Add mx_strconcat() --- mx_util.c | 41 +++++++++++++++++++++++++++++++++++++++++ mx_util.h | 3 +++ test_mx_util.c | 22 ++++++++++++++++++++++ 3 files changed, 66 insertions(+) diff --git a/mx_util.c b/mx_util.c index f783ed38..e2b488ce 100644 --- a/mx_util.c +++ b/mx_util.c @@ -856,6 +856,47 @@ int mx_strscan_ll(char **str, long long int *to) return res; } +char *_mx_strconcat_do(char *first, ...) +{ + va_list ap; + char *join = NULL; + char *str; + char *ptr; + size_t len; + + if (!first) { + join = strdup(""); + return join; + } + + len = strlen(first); + + va_start(ap, first); + do { + str = va_arg(ap, char *); + if (!str) + break; + len += strlen(str); + } while(1); + va_end(ap); + + join = malloc(len+1); + if (!join) + return NULL; + + ptr = stpcpy(join, first); + va_start(ap, first); + do { + str = va_arg(ap, char *); + if (!str) + break; + ptr = stpcpy(ptr, str); + } while(1); + va_end(ap); + + return join; +} + int mx_sleep(unsigned int seconds) { if (seconds) diff --git a/mx_util.h b/mx_util.h index 508476c6..92614fe5 100644 --- a/mx_util.h +++ b/mx_util.h @@ -136,6 +136,9 @@ int mx_read_first_line_from_file(char *fname, char **line); int mx_strscan_ull(char **str, unsigned long long int *to); int mx_strscan_ll(char **str, long long int *to); +char *_mx_strconcat_do(char *first, ...); +#define mx_strconcat(s, ...) _mx_strconcat_do((s), ##__VA_ARGS__, NULL) + int mx_sleep(unsigned int seconds); int mx_sleep_nofail(unsigned int seconds); diff --git a/test_mx_util.c b/test_mx_util.c index 06cfe728..2f6be8f7 100644 --- a/test_mx_util.c +++ b/test_mx_util.c @@ -384,6 +384,27 @@ static void test_mx_strvec() { mx_strvec_free(strvec); } +static void test_mx_strcat() { + char *str; + char *str2; + + assert(str = mx_strconcat(NULL)); + assert(mx_streq(str, "")); + mx_free_null(str); + + assert(str = mx_strconcat("abc", "123")); + assert(mx_streq(str, "abc123")); + + assert(str2 = mx_strconcat(str, "def", str, "456")); + assert(mx_streq(str2, "abc123defabc123456")); + mx_free_null(str2); + + assert(str2 = mx_strconcat(str)); + assert(mx_streq(str, str2)); + mx_free_null(str); + mx_free_null(str2); +} + static void test_mx_cpuset(void) { cpu_set_t cpuset; @@ -421,6 +442,7 @@ int main(int argc, char *argv[]) test_mx_read_first_line_from_file(); test_mx_strscan(); test_mx_strvec(); + test_mx_strcat(); test_mx_cpuset(); return 0; } From 949b506ef056ff16cd7992c95c35f3643ff8cfd4 Mon Sep 17 00:00:00 2001 From: Marius Tolzmann Date: Tue, 27 Oct 2015 14:27:41 +0100 Subject: [PATCH 6/6] mxqd: Add --log-directory[=DIRECTORY] resolves https://github.molgen.mpg.de/mariux64/mxq/issues/32 --- mxqd.c | 33 ++++++++++++++++++++++++++++----- 1 file changed, 28 insertions(+), 5 deletions(-) diff --git a/mxqd.c b/mxqd.c index 3d91557a..15fa51df 100644 --- a/mxqd.c +++ b/mxqd.c @@ -78,7 +78,9 @@ static void print_usage(void) #else " --no-log default: write a logfile\n" #endif + " --log-directory default: " MXQ_LOGDIR "\n" " --debug default: info log level\n" + "\n" " --recover-only (recover from crash and exit)\n" "\n" " --initial-path default: %s\n" @@ -138,11 +140,26 @@ static void cpuset_clear_running(cpu_set_t *running,cpu_set_t *job) { } /**********************************************************************/ -int setup_cronolog(char *cronolog, char *link, char *format) +int setup_cronolog(char *cronolog, char *logdir, char *rellink, char *relformat) { int res; int pipe_fd[2]; int pid; + _mx_cleanup_free_ char *link = NULL; + _mx_cleanup_free_ char *format = NULL; + + if (logdir) { + link = mx_strconcat(logdir, "/", rellink); + format = mx_strconcat(logdir, "/", relformat); + } else { + link = strdup(rellink); + format = strdup(relformat); + } + + if (!link || !format) { + mx_log_err("can't allocate filenames: (%m)"); + return 0; + } res = pipe(pipe_fd); if (res == -1) { @@ -279,6 +296,7 @@ int server_init(struct mxq_server *server, int argc, char *argv[]) char *arg_mysql_default_group; char *arg_mysql_default_file; char *arg_pidfile = NULL; + char *arg_logdir = NULL; char *arg_initial_path; char *arg_initial_tmpdir; char arg_daemonize = 0; @@ -299,7 +317,8 @@ int server_init(struct mxq_server *server, int argc, char *argv[]) MX_OPTION_NO_ARG("version", 'V'), MX_OPTION_NO_ARG("daemonize", 1), MX_OPTION_NO_ARG("no-log", 3), - MX_OPTION_NO_ARG("log", 4), + MX_OPTION_OPTIONAL_ARG("log", 4), + MX_OPTION_REQUIRED_ARG("log-directory", 4), MX_OPTION_NO_ARG("debug", 5), MX_OPTION_NO_ARG("recover-only", 9), MX_OPTION_REQUIRED_ARG("pid-file", 2), @@ -357,6 +376,7 @@ int server_init(struct mxq_server *server, int argc, char *argv[]) case 4: arg_nolog = 0; + arg_logdir = optctl.optarg; break; case 5: @@ -486,13 +506,16 @@ int server_init(struct mxq_server *server, int argc, char *argv[]) setup_stdin("/dev/null"); if (!arg_nolog) { - if (access(MXQ_LOGDIR, R_OK|W_OK|X_OK)) { + if (!arg_logdir) + arg_logdir = MXQ_LOGDIR; + + if (access(arg_logdir, R_OK|W_OK|X_OK)) { if (!RUNNING_AS_ROOT) mx_log_warning("Running mxqd as non-root user."); - mx_log_err("MAIN: can't write to " MXQ_LOGDIR ": %m"); + mx_log_err("MAIN: can't write to '%s': %m", arg_logdir); exit(EX_IOERR); } - res = setup_cronolog("/usr/sbin/cronolog", MXQ_LOGDIR "/mxqd_log", MXQ_LOGDIR "/%Y/mxqd_log-%Y-%m"); + res = setup_cronolog("/usr/sbin/cronolog", arg_logdir, "mxqd_log", "%Y/mxqd_log-%Y-%m"); if (!res) { if (!RUNNING_AS_ROOT) mx_log_warning("Running mxqd as non-root user.");