From e51a0230e154d65226ca3ff144eaaafcc3074dae Mon Sep 17 00:00:00 2001 From: Marius Tolzmann Date: Fri, 15 May 2015 13:33:44 +0200 Subject: [PATCH 1/9] mx_util: Add mx_calloc*() --- mx_util.c | 17 +++++++++++++++++ mx_util.h | 8 ++++++++ 2 files changed, 25 insertions(+) diff --git a/mx_util.c b/mx_util.c index 14741ba..0829ae9 100644 --- a/mx_util.c +++ b/mx_util.c @@ -607,3 +607,20 @@ int mx_sleep_nofail(unsigned int seconds) mx_sleep(seconds); return 1; } + +void *mx_calloc_forever_sec(size_t nmemb, size_t size, unsigned int time) +{ + void *ptr; + + while (1) { + ptr = calloc(nmemb, size); + if (ptr) + break; + + mx_log_debug("calloc() failed: %m - retrying (forever) in %d second(s).", time); + if (time) + mx_sleep(time); + } + + return ptr; +} diff --git a/mx_util.h b/mx_util.h index 63af526..9fac460 100644 --- a/mx_util.h +++ b/mx_util.h @@ -96,4 +96,12 @@ int mx_open_newfile(char *fname); int mx_sleep(unsigned int seconds); int mx_sleep_nofail(unsigned int seconds); +#ifndef MX_CALLOC_FAIL_WAIT_DEFAULT +# define MX_CALLOC_FAIL_WAIT_DEFAULT 1 +#endif + +#define mx_calloc_forever(n, s) mx_calloc_forever_sec((n), (s), MX_CALLOC_FAIL_WAIT_DEFAULT) + +void *mx_calloc_forever_sec(size_t nmemb, size_t size, unsigned int time); + #endif From c4692e793a8969bfd2d981bcb14a1ae6c37b07f5 Mon Sep 17 00:00:00 2001 From: Marius Tolzmann Date: Wed, 20 May 2015 15:24:56 +0200 Subject: [PATCH 2/9] mxqsub: Remove unused -D option (Fixes memory leak) --- mxqsub.c | 8 -------- 1 file changed, 8 deletions(-) diff --git a/mxqsub.c b/mxqsub.c index 956f053..026326d 100644 --- a/mxqsub.c +++ b/mxqsub.c @@ -480,9 +480,6 @@ int main(int argc, char *argv[]) arg_stdout = "/dev/null"; arg_stderr = "stdout"; arg_umask = getumask(); - arg_env = strvec_new(); - - assert(arg_env); arg_mysql_default_group = getenv("MXQ_MYSQL_DEFAULT_GROUP"); if (!arg_mysql_default_group) @@ -592,11 +589,6 @@ int main(int argc, char *argv[]) arg_stderr = optctl.optarg; break; - case 'D': { - char *arg_env_str; - assert(strvec_push_str(&arg_env, optctl.optarg)); - break; - } case 'u': if (mx_strtou32(optctl.optarg, &arg_umask) < 0) { mx_log_crit("--umask '%s': %m", optctl.optarg); From 532df8dd2f2d83cbd704706565fee401692e1fca Mon Sep 17 00:00:00 2001 From: Marius Tolzmann Date: Wed, 20 May 2015 16:23:07 +0200 Subject: [PATCH 3/9] mxqsub: User mx_mysql functions and remove mxq_mysql support --- Makefile | 2 + mxqsub.c | 438 +++++++++++++++++++++++++++---------------------------- 2 files changed, 219 insertions(+), 221 deletions(-) diff --git a/Makefile b/Makefile index dccd3b3..eeca2e7 100644 --- a/Makefile +++ b/Makefile @@ -343,6 +343,7 @@ mxqsub.o: $(mxq_util.h) mxqsub.o: $(mx_util.h) mxqsub.o: $(mx_log.h) mxqsub.o: $(mxq_mysql.h) +mxqsub.o: $(mx_mysql.h) mxqsub.o: CFLAGS += $(CFLAGS_MYSQL) mxqsub.o: CFLAGS += $(CFLAGS_MXQ_MYSQL_DEFAULT_FILE) @@ -377,6 +378,7 @@ mxqsub: mxq_mysql.o mxqsub: mxq_util.o mxqsub: mx_util.o mxqsub: mx_log.o +mxqsub: mx_mysql.o mxqsub: LDLIBS += $(LDLIBS_MYSQL) build: mxqsub diff --git a/mxqsub.c b/mxqsub.c index f7f31c4..0e7ac4d 100644 --- a/mxqsub.c +++ b/mxqsub.c @@ -21,10 +21,11 @@ #include -#include "mxq_mysql.h" #include "mxq_util.h" + #include "mx_util.h" #include "mx_getopt.h" +#include "mx_mysql.h" #define MXQ_TASK_JOB_FORCE_APPEND (1<<0) @@ -114,58 +115,14 @@ static void print_usage(void) ); } -void mxq_mysql_row_to_group(struct mxq_job *j, MYSQL_ROW row) -{ - int r; - struct mxq_group *g; - - g = j->group_ptr; - - r = 0; - - mx_strtou64(row[r++], &g->group_id); - - mx_strtou8(row[r++], &g->group_status); - mx_strtou16(row[r++], &g->group_priority); - - mx_strtou64(row[r++], &g->group_jobs); - mx_strtou64(row[r++], &g->group_jobs_running); - mx_strtou64(row[r++], &g->group_jobs_finished); - mx_strtou64(row[r++], &g->group_jobs_failed); - mx_strtou64(row[r++], &g->group_jobs_cancelled); - mx_strtou64(row[r++], &g->group_jobs_unknown); - - mx_strtou64(row[r++], &g->group_slots_running); - r++; /* mtime */ - mx_strtou32(row[r++], &g->stats_max_maxrss); - r++; /* utime */ - r++; /* stime */ - r++; /* real */ - - j->group_id = g->group_id; -} - - -static int mxq_mysql_load_group(MYSQL *mysql, struct mxq_job *j) +static int load_group_id(struct mx_mysql *mysql, struct mxq_group *g) { - MYSQL_RES *mres; - MYSQL_ROW row; - - unsigned int num_rows; - unsigned int num_fields; - - _mx_cleanup_free_ char *q_group_name = NULL; - _mx_cleanup_free_ char *q_user_name = NULL; - _mx_cleanup_free_ char *q_user_group = NULL; - _mx_cleanup_free_ char *q_job_command = NULL; - - struct mxq_group *g; - - assert(j); - assert(j->group_ptr); - - g = j->group_ptr; + struct mx_mysql_stmt *stmt = NULL; + unsigned long long num_rows = 0; + int res; + assert(mysql); + assert(g); assert(g->group_id == 0); assert(g->group_name); assert(*g->group_name); @@ -175,95 +132,74 @@ static int mxq_mysql_load_group(MYSQL *mysql, struct mxq_job *j) assert(g->job_command); assert(*g->job_command); assert(g->job_threads); assert(g->job_memory); assert(g->job_time); - q_group_name = mxq_mysql_escape_string(mysql, g->group_name); - q_user_name = mxq_mysql_escape_string(mysql, g->user_name); - q_user_group = mxq_mysql_escape_string(mysql, g->user_group); - q_job_command = mxq_mysql_escape_string(mysql, g->job_command); - - if (!q_group_name || !q_user_name || !q_user_group || !q_job_command) - return 0; - mres = mxq_mysql_query_with_result(mysql, "SELECT " - "group_id," - "group_status," - "group_priority," - - "group_jobs," - "group_jobs_running," - "group_jobs_finished," - "group_jobs_failed," - "group_jobs_cancelled," - "group_jobs_unknown," - - "group_slots_running," - - "group_mtime," - - "stats_max_maxrss," - "stats_max_utime_sec," - "stats_max_stime_sec," - "stats_max_real_sec " - - "FROM mxq_group " - "WHERE group_name = '%s' " - "AND user_uid = %" PRIu32 " " - "AND user_name = '%s' " - "AND user_gid = %" PRIu32 " " - "AND user_group = '%s' " - "AND job_command = '%s' " - "AND job_threads = %" PRIu16 " " - "AND job_memory = %" PRIu64 " " - "AND job_time = %" PRIu32 " " - "AND group_status = 0 " - "ORDER BY group_id " - "LIMIT 1", - q_group_name, g->user_uid, q_user_name, g->user_gid, q_user_group, - q_job_command, g->job_threads, g->job_memory, g->job_time); - - if (!mres) { - mx_log_err("mxq_mysql_select_next_job: Failed to query database: Error: %s", mysql_error(mysql)); - sleep(10); - return 0; + res = mx_mysql_statement_init(mysql, &stmt); + if (res < 0) + return res; + + res = mx_mysql_statement_prepare(stmt, + "SELECT" + " group_id" + " FROM mxq_group " + " WHERE group_name = ?" + " AND user_uid = ?" + " AND user_name = ?" + " AND user_gid = ?" + " AND user_group = ?" + " AND job_command = ?" + " AND job_threads = ?" + " AND job_memory = ?" + " AND job_time = ?" + " AND group_priority = ?" + " AND group_status = 0" + " ORDER BY group_id " + " LIMIT 1"); + if (res < 0) { + mx_log_err("mx_mysql_statement_prepare(): %m"); + return res; } - num_rows = mysql_num_rows(mres); + assert(mx_mysql_statement_field_count(stmt) == 1); + assert(mx_mysql_statement_param_count(stmt) == 10); + + mx_mysql_statement_param_bind(stmt, 0, string, &(g->group_name)); + mx_mysql_statement_param_bind(stmt, 1, uint32, &(g->user_uid)); + mx_mysql_statement_param_bind(stmt, 2, string, &(g->user_name)); + mx_mysql_statement_param_bind(stmt, 3, uint32, &(g->user_gid)); + mx_mysql_statement_param_bind(stmt, 4, string, &(g->user_group)); + mx_mysql_statement_param_bind(stmt, 5, string, &(g->job_command)); + mx_mysql_statement_param_bind(stmt, 6, uint16, &(g->job_threads)); + mx_mysql_statement_param_bind(stmt, 7, uint64, &(g->job_memory)); + mx_mysql_statement_param_bind(stmt, 8, uint32, &(g->job_time)); + mx_mysql_statement_param_bind(stmt, 9, uint16, &(g->group_priority)); + + res = mx_mysql_statement_execute(stmt, &num_rows); + if (res < 0) { + mx_log_err("mx_mysql_statement_execute(): %m"); + return res; + } assert(num_rows <= 1); - if (num_rows == 1) { - num_fields = mysql_num_fields(mres); - assert(num_fields == 15); + if (num_rows) { + mx_mysql_statement_result_bind(stmt, 0, uint64, &(g->group_id)); - row = mysql_fetch_row(mres); - if (!row) { - mx_log_err("mxq_mysql_select_next_job: Failed to fetch row: Error: %s", mysql_error(mysql)); - mysql_free_result(mres); - return 0; + res = mx_mysql_statement_fetch(stmt); + if (res < 0) { + mx_log_err("mx_mysql_statement_fetch(): %m"); + return res; } - mxq_mysql_row_to_group(j, row); } - mysql_free_result(mres); + res = mx_mysql_statement_close(&stmt); - return num_rows; + return (int)num_rows; } - -static int mxq_mysql_add_group(MYSQL *mysql, struct mxq_job *j) +static int add_group(struct mx_mysql *mysql, struct mxq_group *g) { - _mx_cleanup_free_ char *q_group_name = NULL; - _mx_cleanup_free_ char *q_user = NULL; - _mx_cleanup_free_ char *q_group = NULL; - _mx_cleanup_free_ char *q_command = NULL; - - int len; - int res; - int i; - - struct mxq_group *g; - - assert(j); - assert(j->group_ptr); - - g = j->group_ptr; + struct mx_mysql_stmt *stmt = NULL; + unsigned long long num_rows = 0; + unsigned long long insert_id = 0; + int res; assert(g->group_name); assert(*g->group_name); assert(g->group_priority); @@ -272,124 +208,172 @@ static int mxq_mysql_add_group(MYSQL *mysql, struct mxq_job *j) assert(g->job_command); assert(*g->job_command); assert(g->job_threads); assert(g->job_memory); assert(g->job_time); - q_group_name = mxq_mysql_escape_string(mysql, g->group_name); - q_user = mxq_mysql_escape_string(mysql, g->user_name); - q_group = mxq_mysql_escape_string(mysql, g->user_group); - q_command = mxq_mysql_escape_string(mysql, g->job_command); - - if (!q_group_name || !q_user || !q_group || !q_command) - return 0; - - res = mxq_mysql_query(mysql, - "INSERT INTO mxq_group SET " - "group_name = '%s'," - "group_priority = %" PRIu16 "," - - "user_uid = %" PRIu32 "," - "user_name = '%s'," - "user_gid = %" PRIu32 "," - "user_group = '%s'," - - "job_command = '%s'," - - "job_threads = %" PRIu16 "," - "job_memory = %" PRIu64 " ," - "job_time = %" PRIu32 " ", - q_group_name, g->group_priority, - g->user_uid, q_user, g->user_gid, q_group, - q_command, - g->job_threads, g->job_memory, g->job_time); - if (res) { - mx_log_err("Failed to query database: Error: %s", mysql_error(mysql)); - return 0; + res = mx_mysql_statement_init(mysql, &stmt); + if (res < 0) + return res; + + res = mx_mysql_statement_prepare(stmt, + "INSERT INTO mxq_group SET" + " group_name = ?," + + " user_uid = ?," + " user_name = ?," + " user_gid = ?," + " user_group = ?," + + " job_command = ?," + + " job_threads = ?," + " job_memory = ?," + " job_time = ?," + " group_priority = ?"); + if (res < 0) { + mx_log_err("mx_mysql_statement_prepare(): %m"); + return res; } - g->group_id = mysql_insert_id(mysql); - j->group_id = g->group_id; + assert(mx_mysql_statement_field_count(stmt) == 0); + assert(mx_mysql_statement_param_count(stmt) == 10); + + mx_mysql_statement_param_bind(stmt, 0, string, &(g->group_name)); + mx_mysql_statement_param_bind(stmt, 1, uint32, &(g->user_uid)); + mx_mysql_statement_param_bind(stmt, 2, string, &(g->user_name)); + mx_mysql_statement_param_bind(stmt, 3, uint32, &(g->user_gid)); + mx_mysql_statement_param_bind(stmt, 4, string, &(g->user_group)); + mx_mysql_statement_param_bind(stmt, 5, string, &(g->job_command)); + mx_mysql_statement_param_bind(stmt, 6, uint16, &(g->job_threads)); + mx_mysql_statement_param_bind(stmt, 7, uint64, &(g->job_memory)); + mx_mysql_statement_param_bind(stmt, 8, uint32, &(g->job_time)); + mx_mysql_statement_param_bind(stmt, 9, uint16, &(g->group_priority)); + + res = mx_mysql_statement_execute(stmt, &num_rows); + if (res < 0) { + mx_log_err("mx_mysql_statement_execute(): %m"); + return res; + } - return 1; + assert(num_rows == 1); + mx_mysql_statement_insert_id(stmt, &insert_id); + assert(insert_id > 0); + + g->group_id = insert_id; + + res = mx_mysql_statement_close(&stmt); + + return (int)num_rows; } -static int mxq_mysql_add_job(MYSQL *mysql, struct mxq_job *j) +static int add_job(struct mx_mysql *mysql, struct mxq_job *j) { - _mx_cleanup_free_ char *q_workdir = NULL; - _mx_cleanup_free_ char *q_argv = NULL; - _mx_cleanup_free_ char *q_stdout = NULL; - _mx_cleanup_free_ char *q_stderr = NULL; - _mx_cleanup_free_ char *q_submit_host = NULL; - - int len; - int res; - int i; + struct mx_mysql_stmt *stmt = NULL; + unsigned long long num_rows = 0; + unsigned long long insert_id = 0; + int res; assert(j); assert(j->job_priority); assert(j->group_id); assert(j->job_workdir); assert(*j->job_workdir); assert(j->job_argc); assert(j->job_argv); assert(*j->job_argv); + assert(j->job_argv_str); assert(*j->job_argv_str); assert(j->job_stdout); assert(*j->job_stdout); assert(j->job_stderr); assert(*j->job_stderr); assert(j->job_umask); assert(j->host_submit); assert(*j->host_submit); - q_workdir = mxq_mysql_escape_str(mysql, j->job_workdir); - q_argv = mxq_mysql_escape_strvec(mysql, j->job_argv); - q_stdout = mxq_mysql_escape_str(mysql, j->job_stdout); - q_stderr = mxq_mysql_escape_str(mysql, j->job_stderr); - q_submit_host = mxq_mysql_escape_str(mysql, j->host_submit); - - if (!q_workdir || !q_argv - || !q_stdout || !q_stderr || !q_submit_host) - return 0; + res = mx_mysql_statement_init(mysql, &stmt); + if (res < 0) + return res; - res = mxq_mysql_query(mysql, - "INSERT INTO mxq_job SET " - "job_priority = %" PRIu16 "," + res = mx_mysql_statement_prepare(stmt, + "INSERT INTO mxq_job SET" + " job_priority = ?," - "group_id = '%" PRIu64 "'," + " group_id = ?," - "job_workdir = '%s'," - "job_argc = %" PRIu16 " ," - "job_argv = '%s'," + " job_workdir = ?," + " job_argc = ? ," + " job_argv = ?," - "job_stdout = '%s'," - "job_stderr = '%s'," + " job_stdout = ?," + " job_stderr = ?," - "job_umask = %" PRIu32 "," + " job_umask = ?," - "host_submit = '%s'", + " host_submit = ?"); + if (res < 0) { + mx_log_err("mx_mysql_statement_prepare(): %m"); + return res; + } - j->job_priority, - j->group_id, - q_workdir, j->job_argc, q_argv, - q_stdout, q_stderr, - j->job_umask, q_submit_host); - if (res) { - mx_log_err("Failed to query database: Error: %s", mysql_error(mysql)); - return 0; + assert(mx_mysql_statement_field_count(stmt) == 0); + assert(mx_mysql_statement_param_count(stmt) == 9); + + mx_mysql_statement_param_bind(stmt, 0, uint16, &(j->job_priority)); + mx_mysql_statement_param_bind(stmt, 1, uint64, &(j->group_id)); + mx_mysql_statement_param_bind(stmt, 2, string, &(j->job_workdir)); + mx_mysql_statement_param_bind(stmt, 3, uint16, &(j->job_argc)); + mx_mysql_statement_param_bind(stmt, 4, string, &(j->job_argv_str)); + mx_mysql_statement_param_bind(stmt, 5, string, &(j->job_stdout)); + mx_mysql_statement_param_bind(stmt, 6, string, &(j->job_stderr)); + mx_mysql_statement_param_bind(stmt, 7, uint32, &(j->job_umask)); + mx_mysql_statement_param_bind(stmt, 8, string, &(j->host_submit)); + + res = mx_mysql_statement_execute(stmt, &num_rows); + if (res < 0) { + mx_log_err("mx_mysql_statement_execute(): %m"); + return res; } - j->job_id = mysql_insert_id(mysql); - return 1; + assert(num_rows == 1); + mx_mysql_statement_insert_id(stmt, &insert_id); + assert(insert_id > 0); + + j->job_id = insert_id; + + res = mx_mysql_statement_close(&stmt); + + return (int)num_rows; } -static int mxq_submit_task(struct mxq_mysql *mmysql, struct mxq_job *j, int flags) +static int mxq_submit_task(struct mx_mysql *mysql, struct mxq_job *j, int flags) { - MYSQL *mysql; + int res; + struct mxq_group *g; - mysql = mxq_mysql_connect(mmysql); + g = j->group_ptr; + res = load_group_id(mysql, g); + if (res < 0) + return res; - while(!mxq_mysql_load_group(mysql, j)) { - mxq_mysql_add_group(mysql, j); - j->group_ptr->group_id = 0; + if (res == 0) { + res = add_group(mysql, g); + if (res < 0) + return res; + + if (res == 0) { + mx_log_err("Failed to add new group."); + return -(errno=EIO); + } } - mxq_mysql_add_job(mysql, j); - mxq_mysql_close(mysql); + assert(g->group_id); - return 1; -} + j->group_id = g->group_id; + + res = add_job(mysql, j); + if (res < 0) + return res; + if (res == 0) { + mx_log_err("Failed to add job group."); + return -(errno=EIO); + } + + assert(j->job_id); + + return res; +} int main(int argc, char *argv[]) { @@ -409,7 +393,6 @@ int main(int argc, char *argv[]) char *arg_stdout; char *arg_stderr; mode_t arg_umask; - char **arg_env; char *arg_mysql_default_file; char *arg_mysql_default_group; @@ -422,7 +405,8 @@ int main(int argc, char *argv[]) struct mxq_job job; struct mxq_group group; - struct mxq_mysql mmysql; + + struct mx_mysql *mysql = NULL; struct passwd *passwd; struct group *grp; @@ -636,6 +620,9 @@ int main(int argc, char *argv[]) arg_stderr = arg_stderr_absolute; } + arg_args = strvec_to_str(argv); + assert(arg_args); + /******************************************************************/ memset(&job, 0, sizeof(job)); @@ -658,8 +645,9 @@ int main(int argc, char *argv[]) job.job_stderr = arg_stderr; job.job_umask = arg_umask; - job.job_argc = argc; - job.job_argv = argv; + job.job_argc = argc; + job.job_argv = argv; + job.job_argv_str = arg_args; /******************************************************************/ @@ -687,14 +675,23 @@ int main(int argc, char *argv[]) /******************************************************************/ - mmysql.default_file = arg_mysql_default_file; - mmysql.default_group = arg_mysql_default_group; + res = mx_mysql_init(&mysql); + assert(res == 0); - /******************************************************************/ + mx_mysql_option_set_default_file(mysql, arg_mysql_default_file); + mx_mysql_option_set_default_group(mysql, arg_mysql_default_group); - mxq_submit_task(&mmysql, &job, flags); + res = mx_mysql_connect(&mysql); + assert(res == 0); - /******************************************************************/ + res = mxq_submit_task(mysql, &job, flags); + + mx_mysql_finish(&mysql); + + if (res < 0) { + mx_log_err("mxq_submit_task() failed: %m"); + return 1; + } printf("mxq_group_id=%" PRIu64 " \n", group.group_id); printf("mxq_group_name=%s\n", group.group_name); @@ -702,4 +699,3 @@ int main(int argc, char *argv[]) return 0; } - From d01b8e5f163e818437cded3c52d8378bcad6488d Mon Sep 17 00:00:00 2001 From: Marius Tolzmann Date: Wed, 20 May 2015 16:32:09 +0200 Subject: [PATCH 4/9] mx_util: Remove unused variables --- mx_util.c | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/mx_util.c b/mx_util.c index 0829ae9..5befe06 100644 --- a/mx_util.c +++ b/mx_util.c @@ -205,7 +205,6 @@ inline int mx_strtoll(char *str, signed long long int *to) int mx_strtoui(char *str, unsigned int *to) { unsigned long int ul; - char *end; int res; assert(str); @@ -226,7 +225,6 @@ int mx_strtoui(char *str, unsigned int *to) int mx_strtou8(char *str, uint8_t *to) { unsigned long int ul; - char *end; int res; assert(str); @@ -247,7 +245,6 @@ int mx_strtou8(char *str, uint8_t *to) int mx_strtou16(char *str, uint16_t *to) { unsigned long int ul; - char *end; int res; assert(str); @@ -268,7 +265,6 @@ int mx_strtou16(char *str, uint16_t *to) int mx_strtou32(char *str, uint32_t *to) { unsigned long int ul; - char *end; int res; assert(str); @@ -289,7 +285,6 @@ int mx_strtou32(char *str, uint32_t *to) int mx_strtou64(char *str, uint64_t *to) { unsigned long long int ull; - char *end; int res; assert(str); @@ -312,7 +307,6 @@ int mx_strtou64(char *str, uint64_t *to) int mx_strtoi(char *str, signed int *to) { signed long int l; - char *end; int res; assert(str); @@ -333,7 +327,6 @@ int mx_strtoi(char *str, signed int *to) int mx_strtoi8(char *str, int8_t *to) { signed long int l; - char *end; int res; assert(str); @@ -354,7 +347,6 @@ int mx_strtoi8(char *str, int8_t *to) int mx_strtoi16(char *str, int16_t *to) { signed long int l; - char *end; int res; assert(str); @@ -375,7 +367,6 @@ int mx_strtoi16(char *str, int16_t *to) int mx_strtoi32(char *str, int32_t *to) { signed long int l; - char *end; int res; assert(str); @@ -396,7 +387,6 @@ int mx_strtoi32(char *str, int32_t *to) int mx_strtoi64(char *str, int64_t *to) { signed long long int ll; - char *end; int res; assert(str); From 48c63c03e57fe4a19205bcd6b5e8ac71c822e054 Mon Sep 17 00:00:00 2001 From: Marius Tolzmann Date: Thu, 21 May 2015 10:54:10 +0200 Subject: [PATCH 5/9] mxqsub: Improve user input verification --- mxqsub.c | 48 +++++++++++++++++++++++++++++++++++------------- 1 file changed, 35 insertions(+), 13 deletions(-) diff --git a/mxqsub.c b/mxqsub.c index 0e7ac4d..29362e6 100644 --- a/mxqsub.c +++ b/mxqsub.c @@ -410,6 +410,7 @@ int main(int argc, char *argv[]) struct passwd *passwd; struct group *grp; + char *p; int opt; struct mx_getopt_ctl optctl; @@ -504,19 +505,28 @@ int main(int argc, char *argv[]) break; case 1: - mx_log_warning("option --group_id is deprecated. please use --group-name instead."); + case 3: + if (opt == 3) + mx_log_warning("option --group-id is deprecated (usage will change in next version). Using --group-name instead."); + else + mx_log_warning("option --group_id is deprecated. Using --group-name instead."); case 'N': + if (!(*optctl.optarg)) { + mx_log_crit("--group-name '%s': String is empty.", optctl.optarg); + exit(EX_CONFIG); + } arg_group_name = optctl.optarg; break; case 'a': - if (*optctl.optarg) { - char *p; - arg_program_name = optctl.optarg; - p = strchr(arg_program_name, ' '); - if (p) - *p = 0; + p = strchr(optctl.optarg, ' '); + if (p) + *p = 0; + if (!(*optctl.optarg)) { + mx_log_crit("--command-alias '%s': String is empty.", optctl.optarg); + exit(EX_CONFIG); } + arg_program_name = optctl.optarg; break; case 2: @@ -528,11 +538,6 @@ int main(int argc, char *argv[]) } break; - case 3: - mx_log_warning("option --group-id is deprecated (usage will change in next version). please use --group-name instead."); - arg_group_name = optctl.optarg; - break; - case 'j': if (mx_strtou16(optctl.optarg, &arg_threads) < 0) { mx_log_crit("--threads '%s': %m", optctl.optarg); @@ -557,6 +562,10 @@ int main(int argc, char *argv[]) break; case 'w': + if (!(*optctl.optarg)) { + mx_log_crit("--workdir '%s': String is empty.", optctl.optarg); + exit(EX_CONFIG); + } if (optctl.optarg[0] != '/') { mx_log_crit("--workdir '%s': workdir is a relativ path. please use absolute path.", optctl.optarg); @@ -566,10 +575,18 @@ int main(int argc, char *argv[]) break; case 'o': + if (!(*optctl.optarg)) { + mx_log_crit("--stdout '%s': String is empty.", optctl.optarg); + exit(EX_CONFIG); + } arg_stdout = optctl.optarg; break; case 'e': + if (!(*optctl.optarg)) { + mx_log_crit("--stderr '%s': String is empty.", optctl.optarg); + exit(EX_CONFIG); + } arg_stderr = optctl.optarg; break; @@ -602,6 +619,11 @@ int main(int argc, char *argv[]) if (!arg_program_name) arg_program_name = argv[0]; + if (!(*arg_program_name)) { + mx_log_crit(" is empty. Please check usage with '%s --help'.", program_invocation_short_name); + exit(EX_CONFIG); + } + /******************************************************************/ if (*arg_stdout != '/') { @@ -681,7 +703,7 @@ int main(int argc, char *argv[]) mx_mysql_option_set_default_file(mysql, arg_mysql_default_file); mx_mysql_option_set_default_group(mysql, arg_mysql_default_group); - res = mx_mysql_connect(&mysql); + res = mx_mysql_connect_forever(&mysql); assert(res == 0); res = mxq_submit_task(mysql, &job, flags); From 97b381b4e366996947b4cc52cf57d4b108ce2297 Mon Sep 17 00:00:00 2001 From: Marius Tolzmann Date: Thu, 21 May 2015 11:07:43 +0200 Subject: [PATCH 6/9] mxqsub: Handle unrecognized options better this prevent 'mxqsub --help.' to be send to the cluster.. ;) --- mxqsub.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mxqsub.c b/mxqsub.c index 29362e6..2163529 100644 --- a/mxqsub.c +++ b/mxqsub.c @@ -477,7 +477,7 @@ int main(int argc, char *argv[]) /******************************************************************/ mx_getopt_init(&optctl, argc-1, &argv[1], opts); - optctl.flags = MX_FLAG_STOPONUNKNOWN|MX_FLAG_STOPONNOOPT; + optctl.flags = MX_FLAG_STOPONNOOPT; while ((opt=mx_getopt(&optctl, &i)) != MX_GETOPT_END) { if (opt == MX_GETOPT_ERROR) { From ed03295f7de7d464c85eba7e07a0c154ccbcd84b Mon Sep 17 00:00:00 2001 From: Marius Tolzmann Date: Thu, 21 May 2015 12:58:43 +0200 Subject: [PATCH 7/9] mxqsub: Add --verbose option --- mxqsub.c | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/mxqsub.c b/mxqsub.c index 2163529..8b33eb0 100644 --- a/mxqsub.c +++ b/mxqsub.c @@ -99,9 +99,10 @@ static void print_usage(void) "\n" "Other options:\n" "\n" - " --debug set debug log level (default: warning log level)\n" - " --version print version and exit\n" - " --help print this help and exit ;)\n" + " -v | --verbose be more verbose\n" + " --debug set debug log level (default: warning log level)\n" + " --version print version and exit\n" + " --help print this help and exit ;)\n" "\n" "Change how to connect to the mysql server:\n" "\n" @@ -395,6 +396,7 @@ int main(int argc, char *argv[]) mode_t arg_umask; char *arg_mysql_default_file; char *arg_mysql_default_group; + char arg_debug; _mx_cleanup_free_ char *current_workdir = NULL; _mx_cleanup_free_ char *arg_stdout_absolute = NULL; @@ -424,6 +426,7 @@ int main(int argc, char *argv[]) MX_OPTION_REQUIRED_ARG("time", 4), MX_OPTION_NO_ARG("debug", 5), + MX_OPTION_NO_ARG("verbose", 'v'), MX_OPTION_REQUIRED_ARG("group-name", 'N'), MX_OPTION_REQUIRED_ARG("group-priority", 'P'), @@ -465,6 +468,7 @@ int main(int argc, char *argv[]) arg_stdout = "/dev/null"; arg_stderr = "stdout"; arg_umask = getumask(); + arg_debug = 0; arg_mysql_default_group = getenv("MXQ_MYSQL_DEFAULT_GROUP"); if (!arg_mysql_default_group) @@ -494,9 +498,15 @@ int main(int argc, char *argv[]) exit(EX_USAGE); case 5: + arg_debug = 1; mx_log_level_set(MX_LOG_DEBUG); break; + case 'v': + if (!arg_debug) + mx_log_level_set(MX_LOG_INFO); + break; + case 'p': if (mx_strtou16(optctl.optarg, &arg_priority) < 0) { mx_log_crit("--priority '%s': %m", optctl.optarg); @@ -711,7 +721,7 @@ int main(int argc, char *argv[]) mx_mysql_finish(&mysql); if (res < 0) { - mx_log_err("mxq_submit_task() failed: %m"); + mx_log_err("Job submission failed: %m"); return 1; } From 874fd1e5e978d00c865558225f99628efe9e7a2a Mon Sep 17 00:00:00 2001 From: Marius Tolzmann Date: Thu, 21 May 2015 13:09:18 +0200 Subject: [PATCH 8/9] mx_util: Assertion loglevel increased to emergency --- mx_util.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mx_util.h b/mx_util.h index 9fac460..64f100c 100644 --- a/mx_util.h +++ b/mx_util.h @@ -18,7 +18,7 @@ do {\ if (!(test)) {\ errno=(eno);\ - mx_log_debug("Assertion '" #test "' failed. Returning -(errno=" #eno ") [%d]: %m", -errno);\ + mx_log_emerg("Assertion '" #test "' failed. Returning -(errno=" #eno ") [%d]: %m", -errno);\ return -errno;\ }\ } while (0) @@ -27,7 +27,7 @@ do {\ if (!(test)) {\ errno=(eno);\ - mx_log_debug("Assertion '" #test "' failed. Setting errno=" #eno " [%d] and returning NULL: %m", errno);\ + mx_log_emerg("Assertion '" #test "' failed. Setting errno=" #eno " [%d] and returning NULL: %m", errno);\ return NULL;\ }\ } while (0) From 37b7eb88f3141ce69b500a80b3f5f19715e73f89 Mon Sep 17 00:00:00 2001 From: Marius Tolzmann Date: Thu, 21 May 2015 13:21:57 +0200 Subject: [PATCH 9/9] mx_util: Fix missing dependencies in Makefile --- Makefile | 4 ++++ mx_util.c | 1 + 2 files changed, 5 insertions(+) diff --git a/Makefile b/Makefile index 15a7de3..f209195 100644 --- a/Makefile +++ b/Makefile @@ -186,6 +186,7 @@ mxq_mysql.h += $(mxq_util.h) ### mxq_util.h --------------------------------------------------------- mxq_util.h += mxq_util.h +mxq_util.h += $(mx_log.h) mxq_util.h += $(mxq.h) ### mxq_group.h -------------------------------------------------------- @@ -221,6 +222,8 @@ clean: CLEAN += mx_log.o ### mx_util.o ---------------------------------------------------------- +mx_util.o: $(mx_log.h) + clean: CLEAN += mx_util.o ### mx_flock.o ------------------------------------------------------- @@ -427,6 +430,7 @@ test_mx_util.o: $(mx_util.h) clean: CLEAN += test_mx_util.o test_mx_util: mx_util.o +test_mx_util: mx_log.o clean: CLEAN += test_mx_util test: test_mx_util diff --git a/mx_util.c b/mx_util.c index 5befe06..da61176 100644 --- a/mx_util.c +++ b/mx_util.c @@ -13,6 +13,7 @@ //#include #include +#include "mx_log.h" #include "mx_util.h" static inline int _mx_strbeginswith(char *str, const char *start, char **endptr, short ignore_case)