From 41e4282272d3f1645a6e66099e6013afdcaa702a Mon Sep 17 00:00:00 2001 From: Donald Buczek Date: Sun, 18 Oct 2015 20:00:34 +0200 Subject: [PATCH 01/67] mxqd: refactor (add user_process) --- mxqd.c | 81 +++++++++++++++++++++++++++++++--------------------------- 1 file changed, 43 insertions(+), 38 deletions(-) diff --git a/mxqd.c b/mxqd.c index 15fa51df..7eb5f8d1 100644 --- a/mxqd.c +++ b/mxqd.c @@ -1142,6 +1142,47 @@ int mxq_redirect_input(char *stdin_fname) return 1; } +int user_process(struct mxq_group_list *group,struct mxq_job *mxqjob) +{ + int res; + char **argv; + + res = init_child_process(group, mxqjob); + if (!res) + return(-1); + + mxq_job_set_tmpfilenames(&group->group, mxqjob); + + res = mxq_redirect_input("/dev/null"); + if (res < 0) { + mx_log_err(" job=%s(%d):%lu:%lu mxq_redirect_input() failed (%d): %m", + group->group.user_name, group->group.user_uid, group->group.group_id, mxqjob->job_id, + res); + return(res); + } + + res = mxq_redirect_output(mxqjob->tmp_stdout, mxqjob->tmp_stderr); + if (res < 0) { + mx_log_err(" job=%s(%d):%lu:%lu mxq_redirect_output() failed (%d): %m", + group->group.user_name, group->group.user_uid, group->group.group_id, mxqjob->job_id, + res); + return(res); + } + + argv = mx_strvec_from_str(mxqjob->job_argv_str); + if (!argv) { + mx_log_err("job=%s(%d):%lu:%lu Can't recaculate commandline. str_to_strvev(%s) failed: %m", + group->group.user_name, group->group.user_uid, group->group.group_id, mxqjob->job_id, + mxqjob->job_argv_str); + return(-errno); + } + + res=execvp(argv[0], argv); + mx_log_err("job=%s(%d):%lu:%lu execvp(\"%s\", ...): %m", + group->group.user_name, group->group.user_uid, group->group.group_id, mxqjob->job_id, + argv[0]); + return(res); +} unsigned long start_job(struct mxq_group_list *group) { @@ -1150,7 +1191,6 @@ unsigned long start_job(struct mxq_group_list *group) struct mxq_job_list *job; pid_t pid; int res; - char **argv; assert(group); assert(group->user); @@ -1182,43 +1222,8 @@ unsigned long start_job(struct mxq_group_list *group) group->group.user_name, group->group.user_uid, group->group.group_id, mxqjob.job_id, mxqjob.host_pid, getpgrp()); - res = init_child_process(group, &mxqjob); - if (!res) - _exit(EX__MAX + 1); - - mxq_job_set_tmpfilenames(&group->group, &mxqjob); - - - res = mxq_redirect_input("/dev/null"); - if (res < 0) { - mx_log_err(" job=%s(%d):%lu:%lu mxq_redirect_input() failed (%d): %m", - group->group.user_name, group->group.user_uid, group->group.group_id, mxqjob.job_id, - res); - _exit(EX__MAX + 1); - } - - res = mxq_redirect_output(mxqjob.tmp_stdout, mxqjob.tmp_stderr); - if (res < 0) { - mx_log_err(" job=%s(%d):%lu:%lu mxq_redirect_output() failed (%d): %m", - group->group.user_name, group->group.user_uid, group->group.group_id, mxqjob.job_id, - res); - _exit(EX__MAX + 1); - } - - - argv = mx_strvec_from_str(mxqjob.job_argv_str); - if (!argv) { - mx_log_err("job=%s(%d):%lu:%lu Can't recaculate commandline. str_to_strvev(%s) failed: %m", - group->group.user_name, group->group.user_uid, group->group.group_id, mxqjob.job_id, - mxqjob.job_argv_str); - _exit(EX__MAX + 1); - } - - execvp(argv[0], argv); - mx_log_err("job=%s(%d):%lu:%lu execvp(\"%s\", ...): %m", - group->group.user_name, group->group.user_uid, group->group.group_id, mxqjob.job_id, - argv[0]); - _exit(EX__MAX + 1); + res=user_process(group,&mxqjob); + exit(res<0 ? 1 : 0); } gettimeofday(&mxqjob.stats_starttime, NULL); From 155adc614523ed2cb4a83bf0bd6d70c47b93c905 Mon Sep 17 00:00:00 2001 From: Donald Buczek Date: Fri, 23 Oct 2015 10:26:57 +0200 Subject: [PATCH 02/67] mxqd: refactor (add job_has_finished) --- mxqd.c | 72 ++++++++++++++++++++++++++++++++++------------------------ 1 file changed, 42 insertions(+), 30 deletions(-) diff --git a/mxqd.c b/mxqd.c index 7eb5f8d1..9dc59fd3 100644 --- a/mxqd.c +++ b/mxqd.c @@ -1720,6 +1720,47 @@ int killallcancelled(struct mxq_server *server, int sig, unsigned int pgrp) return 0; } +static int job_has_finished (struct mxq_server *server,struct mxq_group *g,struct mxq_job_list *job) +{ + int cnt=0; + int res; + struct mxq_job *j=&job->job; + + mxq_set_job_status_exited(server->mysql, j); + + if (j->job_status == MXQ_JOB_STATUS_FINISHED) { + g->group_jobs_finished++; + } else if(j->job_status == MXQ_JOB_STATUS_FAILED) { + g->group_jobs_failed++; + } else if(j->job_status == MXQ_JOB_STATUS_KILLED) { + g->group_jobs_failed++; + } + + mxq_job_set_tmpfilenames(g, j); + + if (!mx_streq(j->job_stdout, "/dev/null")) { + res = rename(j->tmp_stdout, j->job_stdout); + if (res == -1) { + mx_log_err(" job=%s(%d):%lu:%lu host_pid=%d :: rename(stdout) failed: %m", + g->user_name, g->user_uid, g->group_id, j->job_id, j->host_pid); + } + } + + if (!mx_streq(j->job_stderr, "/dev/null") && !mx_streq(j->job_stderr, j->job_stdout)) { + res = rename(j->tmp_stderr, j->job_stderr); + if (res == -1) { + mx_log_err(" job=%s(%d):%lu:%lu host_pid=%d :: rename(stderr) failed: %m", + g->user_name, g->user_uid, g->group_id, j->job_id, j->host_pid); + } + } + + cnt += job->group->slots_per_job; + cpuset_clear_running(&server->cpu_set_running,&j->host_cpu_set); + mxq_job_free_content(j); + free(job); + return cnt; +} + int catchall(struct mxq_server *server) { struct rusage rusage; @@ -1798,38 +1839,9 @@ int catchall(struct mxq_server *server) { mx_log_info(" job=%s(%d):%lu:%lu host_pid=%d stats_status=%d :: child process returned.", g->user_name, g->user_uid, g->group_id, j->job_id, pid, status); - mxq_set_job_status_exited(server->mysql, j); - - if (j->job_status == MXQ_JOB_STATUS_FINISHED) { - g->group_jobs_finished++; - } else if(j->job_status == MXQ_JOB_STATUS_FAILED) { - g->group_jobs_failed++; - } else if(j->job_status == MXQ_JOB_STATUS_KILLED) { - g->group_jobs_failed++; - } - mxq_job_set_tmpfilenames(g, j); + cnt+=job_has_finished(server,g,job); - if (!mx_streq(j->job_stdout, "/dev/null")) { - res = rename(j->tmp_stdout, j->job_stdout); - if (res == -1) { - mx_log_err(" job=%s(%d):%lu:%lu host_pid=%d :: rename(stdout) failed: %m", - g->user_name, g->user_uid, g->group_id, j->job_id, pid); - } - } - - if (!mx_streq(j->job_stderr, "/dev/null") && !mx_streq(j->job_stderr, j->job_stdout)) { - res = rename(j->tmp_stderr, j->job_stderr); - if (res == -1) { - mx_log_err(" job=%s(%d):%lu:%lu host_pid=%d :: rename(stderr) failed: %m", - g->user_name, g->user_uid, g->group_id, j->job_id, pid); - } - } - - cnt += job->group->slots_per_job; - cpuset_clear_running(&server->cpu_set_running,&j->host_cpu_set); - mxq_job_free_content(j); - free(job); } return cnt; From 4df9cfdd2a69526eab308a1384f8a43601f1b34d Mon Sep 17 00:00:00 2001 From: Donald Buczek Date: Thu, 22 Oct 2015 18:03:54 +0200 Subject: [PATCH 03/67] mx_util: add mx_mkdir_p --- mx_util.c | 31 +++++++++++++++++++++++++++++-- mx_util.h | 3 +++ 2 files changed, 32 insertions(+), 2 deletions(-) diff --git a/mx_util.c b/mx_util.c index f2a74db2..a78dc6b0 100644 --- a/mx_util.c +++ b/mx_util.c @@ -11,8 +11,8 @@ #include -//#include -//#include +#include +#include #include #include "mx_log.h" @@ -1272,3 +1272,30 @@ char *mx_cpuset_to_str(cpu_set_t* cpuset_ptr) mx_strvec_free(strvec); return out; } + +int mx_mkdir_p(char *path, mode_t mode) +{ + struct stat st; + int err; + char *d; + _mx_cleanup_free_ char *copy = NULL; + + if (stat(path, &st) == 0) + return 0; + + copy=mx_strdup_forever(path); + d=copy; + + while (*++d == '/'); + + while ((d = strchr(d, '/'))) { + *d = '\0'; + err = stat(copy, &st) && mkdir(copy, mode); + *d++ = '/'; + if (err) + return -errno; + while (*d == '/') + ++d; + } + return (stat(copy, &st) && mkdir(copy, mode)) ? -errno : 0; +} diff --git a/mx_util.h b/mx_util.h index 92614fe5..c67ded7b 100644 --- a/mx_util.h +++ b/mx_util.h @@ -162,4 +162,7 @@ char* mx_strvec_join(char *sep,char **strvec); char* mx_cpuset_to_str(cpu_set_t* cpuset_ptr); int mx_str_to_cpuset(cpu_set_t* cpuset_ptr,char *str); +int mx_mkdir_p(char *path, mode_t mode); + + #endif From bc0c4d7b73eee511342b6a12f2de78db26066d89 Mon Sep 17 00:00:00 2001 From: Donald Buczek Date: Mon, 26 Oct 2015 15:55:51 +0100 Subject: [PATCH 04/67] make: add FINISHED_JOBSDIR --- Makefile | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Makefile b/Makefile index 21c95c5b..19998d41 100644 --- a/Makefile +++ b/Makefile @@ -73,12 +73,15 @@ MXQ_MYSQL_DEFAULT_GROUP_DEVELOPMENT = mxqdevel MXQ_INITIAL_PATH = /sbin:/bin:/usr/sbin:/usr/bin:/usr/local/sbin:/usr/local/bin MXQ_INITIAL_TMPDIR = /tmp +MXQ_FINISHED_JOBSDIR = ${LOCALSTATEDIR}/spool/mxqd + CFLAGS_MXQ_MYSQL_DEFAULT_FILE = -DMXQ_MYSQL_DEFAULT_FILE=\"$(MXQ_MYSQL_DEFAULT_FILE)\" CFLAGS_MXQ_MYSQL_DEFAULT_GROUP = -DMXQ_MYSQL_DEFAULT_GROUP_CLIENT=\"$(MXQ_MYSQL_DEFAULT_GROUP_CLIENT)\" CFLAGS_MXQ_MYSQL_DEFAULT_GROUP += -DMXQ_MYSQL_DEFAULT_GROUP_SERVER=\"$(MXQ_MYSQL_DEFAULT_GROUP_SERVER)\" CFLAGS_MXQ_MYSQL_DEFAULT_GROUP += -DMXQ_MYSQL_DEFAULT_GROUP_DEVELOPMENT=\"$(MXQ_MYSQL_DEFAULT_GROUP_DEVELOPMENT)\" CFLAGS_MXQ_INITIAL_PATH = -DMXQ_INITIAL_PATH=\"$(MXQ_INITIAL_PATH)\" CFLAGS_MXQ_INITIAL_TMPDIR = -DMXQ_INITIAL_TMPDIR=\"$(MXQ_INITIAL_TMPDIR)\" +CFLAGS_MXQ_FINISHED_JOBSDIR = -DMXQ_FINISHED_JOBSDIR=\"${MXQ_FINISHED_JOBSDIR}\" MYSQL_CONFIG = mysql_config @@ -417,6 +420,7 @@ mxqd.o: $(mx_mysql.h) mxqd.o: CFLAGS += $(CFLAGS_MYSQL) mxqd.o: CFLAGS += $(CFLAGS_MXQ_INITIAL_PATH) mxqd.o: CFLAGS += $(CFLAGS_MXQ_INITIAL_TMPDIR) +mxqd.o: CFLAGS += $(CFLAGS_MXQ_FINISHED_JOBSDIR) mxqd.o: CFLAGS += -Wno-unused-but-set-variable clean: CLEAN += mxqd.o From d14cf237713214641dee0d0b180d19a817954528 Mon Sep 17 00:00:00 2001 From: Donald Buczek Date: Thu, 22 Oct 2015 18:18:04 +0200 Subject: [PATCH 05/67] mxqd: create MXQ_FINISHED_JOBSDIR on startup --- mxqd.c | 8 ++++++++ mxqd.h | 1 + 2 files changed, 9 insertions(+) diff --git a/mxqd.c b/mxqd.c index 9dc59fd3..7d011e26 100644 --- a/mxqd.c +++ b/mxqd.c @@ -479,6 +479,13 @@ int server_init(struct mxq_server *server, int argc, char *argv[]) exit(2); } + mx_asprintf_forever(&server->finished_jobsdir,"%s/%s",MXQ_FINISHED_JOBSDIR,server->server_id); + res=mx_mkdir_p(server->finished_jobsdir,0700); + if (res<0) { + mx_log_err("MAIN: mkdir %s failed: %m. Exiting.",MXQ_FINISHED_JOBSDIR); + exit(EX_IOERR); + } + if (arg_daemonize) { res = daemon(0, 1); if (res == -1) { @@ -1518,6 +1525,7 @@ void server_close(struct mxq_server *server) mx_free_null(server->boot_id); mx_free_null(server->host_id); + mx_free_null(server->finished_jobsdir); } int killall(struct mxq_server *server, int sig, unsigned int pgrp) diff --git a/mxqd.h b/mxqd.h index 4bc1cd0a..1259ddd6 100644 --- a/mxqd.h +++ b/mxqd.h @@ -81,6 +81,7 @@ struct mxq_server { char *hostname; char *server_id; char *pidfilename; + char *finished_jobsdir; struct mx_flock *flock; char *initial_path; From 7a4ec2193506f030ba93fedcd3d4cc8591ce0626 Mon Sep 17 00:00:00 2001 From: Donald Buczek Date: Fri, 23 Oct 2015 15:11:52 +0200 Subject: [PATCH 06/67] mxqd: add help functions for fspool (finished job spool directory) --- mxqd.c | 136 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 135 insertions(+), 1 deletion(-) diff --git a/mxqd.c b/mxqd.c index 7d011e26..95d35127 100644 --- a/mxqd.c +++ b/mxqd.c @@ -10,8 +10,9 @@ #include #include #include - +#include #include +#include #include @@ -1769,6 +1770,139 @@ static int job_has_finished (struct mxq_server *server,struct mxq_group *g,struc return cnt; } +static char *fspool_get_filename (struct mxq_server *server,long unsigned int job_id) +{ + char *fspool_filename; + mx_asprintf_forever(&fspool_filename,"%s/%lu.stat",server->finished_jobsdir,job_id); + return fspool_filename; +} + +void fspool_unlink(struct mxq_server *server,int job_id) { + char *fspool_filename=fspool_get_filename(server,job_id); + unlink(fspool_filename); + free(fspool_filename); +} + +static int fspool_process_file(struct mxq_server *server,char *filename,int job_id) { + FILE *in; + int res; + + pid_t pid; + int status; + struct rusage rusage; + struct timeval realtime; + + struct mxq_job_list *job; + struct mxq_job *j; + struct mxq_group *g; + + in=fopen(filename,"r"); + if (!in) { + return -errno; + } + res=fscanf(in,"1 %d %d %ld %ld %ld %ld %ld %ld %ld %ld %ld %ld %ld %ld %ld %ld %ld %ld %ld %ld %ld %ld", + &pid, + &status, + &realtime.tv_sec,&realtime.tv_usec, + &rusage.ru_utime.tv_sec,&rusage.ru_utime.tv_usec, + &rusage.ru_stime.tv_sec,&rusage.ru_stime.tv_usec, + &rusage.ru_maxrss, + &rusage.ru_ixrss, + &rusage.ru_idrss, + &rusage.ru_isrss, + &rusage.ru_minflt, + &rusage.ru_majflt, + &rusage.ru_nswap, + &rusage.ru_inblock, + &rusage.ru_oublock, + &rusage.ru_msgsnd, + &rusage.ru_msgrcv, + &rusage.ru_nsignals, + &rusage.ru_nvcsw, + &rusage.ru_nivcsw); + fclose(in); + if (res!=22) { + mx_log_err("%s : parse error (res=%d)",filename,res); + if (!errno) + errno=EINVAL; + return -errno; + } + + mx_log_info("job finished (via fspool) : job %d pid %d status %d",job_id,pid,status); + + job = server_remove_job_by_pid(server, pid); + if (!job) { + mx_log_warning("fspool_process_file: %s : job unknown on server",filename); + return(-1); + } + j = &job->job; + assert(job->group); + assert(j->job_id=job_id); + g = &job->group->group; + + j->stats_realtime=realtime; + j->stats_status=status; + j->stats_rusage=rusage; + + job_has_finished(server,g,job); + fspool_unlink(server,job_id); + return(0); +} + +static int fspool_is_valid_name_parse(const char *name,int *job_id) { + const char *c=name; + if (!*c) + return 0; + if (!isdigit(*c++)) + return 0; + while(isdigit(*c)) { + c++; + } + if (strcmp(c,".stat")) { + return 0; + } + if (job_id) { + *job_id=atol(name); + } + return 1; +} + +static int fspool_is_valid_name(const struct dirent *d) +{ + return fspool_is_valid_name_parse(d->d_name,NULL); +} + +static int fspool_scan(struct mxq_server *server) { + int cnt=0; + int entries; + struct dirent **namelist; + int i; + int res; + + + entries=scandir(server->finished_jobsdir,&namelist,&fspool_is_valid_name,&alphasort); + if (entries<0) { + mx_log_err("scandir %s: %m",server->finished_jobsdir); + return cnt; + } + + for (i=0;ifinished_jobsdir,namelist[i]->d_name); + fspool_is_valid_name_parse(namelist[i]->d_name,&job_id); + res=fspool_process_file(server,filename,job_id); + if (res==0) { + cnt++; + } + free(namelist[i]); + free(filename); + } + + free(namelist); + return cnt; +} + int catchall(struct mxq_server *server) { struct rusage rusage; From 214242aaf8e90dd0dd1435b126697785c277e783 Mon Sep 17 00:00:00 2001 From: Donald Buczek Date: Fri, 23 Oct 2015 09:31:42 +0200 Subject: [PATCH 07/67] mxqd: add reaper --- mxqd.c | 101 +++++++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 99 insertions(+), 2 deletions(-) diff --git a/mxqd.c b/mxqd.c index 95d35127..f5f4565b 100644 --- a/mxqd.c +++ b/mxqd.c @@ -1192,6 +1192,98 @@ int user_process(struct mxq_group_list *group,struct mxq_job *mxqjob) return(res); } +int reaper_process(struct mxq_server *server,struct mxq_group_list *group,struct mxq_job *job) { + pid_t pid; + struct rusage rusage; + int status; + pid_t waited_pid; + int waited_status; + struct timeval now; + struct timeval realtime; + _mx_cleanup_free_ char *finished_job_filename=NULL; + _mx_cleanup_free_ char *finished_job_tmpfilename=NULL; + FILE *out; + int res; + + res=prctl(PR_SET_CHILD_SUBREAPER, 1); + if (res<0) { + mx_log_err("set subreaper: %m"); + return res; + } + + pid = fork(); + if (pid < 0) { + mx_log_err("fork: %m"); + return(pid); + } else if (pid == 0) { + res=user_process(group,job); + _exit(EX__MAX+1); + } + gettimeofday(&job->stats_starttime, NULL); + + while (1) { + waited_pid=wait(&waited_status); + if (waited_pid<0) { + if (errno==ECHILD) { + break; + } else { + mx_log_warning("reaper: wait: %m"); + sleep(1); + } + } + if (waited_pid==pid) { + status=waited_status; + } + } + gettimeofday(&now, NULL); + timersub(&now, &job->stats_starttime, &realtime); + res=getrusage(RUSAGE_CHILDREN,&rusage); + if (res<0) { + mx_log_err("reaper: getrusage: %m"); + return(res); + } + + mx_asprintf_forever(&finished_job_filename,"%s/%lu.stat",server->finished_jobsdir,job->job_id); + mx_asprintf_forever(&finished_job_tmpfilename,"%s.tmp",finished_job_filename); + + out=fopen(finished_job_tmpfilename,"w"); + if (!out) { + mx_log_fatal("%s: %m",finished_job_tmpfilename); + return (-errno); + } + + fprintf(out,"1 %d %d %ld %ld %ld %ld %ld %ld %ld %ld %ld %ld %ld %ld %ld %ld %ld %ld %ld %ld %ld %ld\n", + getpid(), + status, + realtime.tv_sec,realtime.tv_usec, + rusage.ru_utime.tv_sec,rusage.ru_utime.tv_usec, + rusage.ru_stime.tv_sec,rusage.ru_stime.tv_usec, + rusage.ru_maxrss, + rusage.ru_ixrss, + rusage.ru_idrss, + rusage.ru_isrss, + rusage.ru_minflt, + rusage.ru_majflt, + rusage.ru_nswap, + rusage.ru_inblock, + rusage.ru_oublock, + rusage.ru_msgsnd, + rusage.ru_msgrcv, + rusage.ru_nsignals, + rusage.ru_nvcsw, + rusage.ru_nivcsw + ); + fflush(out); + fsync(fileno(out)); + fclose(out); + res=rename(finished_job_tmpfilename,finished_job_filename); + if (res<0) { + mx_log_fatal("rename %s: %m",finished_job_tmpfilename); + return(res); + } + return(0); +} + unsigned long start_job(struct mxq_group_list *group) { struct mxq_server *server; @@ -1230,8 +1322,8 @@ unsigned long start_job(struct mxq_group_list *group) group->group.user_name, group->group.user_uid, group->group.group_id, mxqjob.job_id, mxqjob.host_pid, getpgrp()); - res=user_process(group,&mxqjob); - exit(res<0 ? 1 : 0); + res=reaper_process(server,group,&mxqjob); + _exit(res<0 ? EX__MAX+1 : 0); } gettimeofday(&mxqjob.stats_starttime, NULL); @@ -1981,6 +2073,7 @@ int catchall(struct mxq_server *server) { mx_log_info(" job=%s(%d):%lu:%lu host_pid=%d stats_status=%d :: child process returned.", g->user_name, g->user_uid, g->group_id, j->job_id, pid, status); + fspool_unlink(server,j->job_id); cnt+=job_has_finished(server,g,job); @@ -2126,6 +2219,8 @@ int main(int argc, char *argv[]) while (!global_sigint_cnt && !global_sigterm_cnt && !fail) { slots_returned = catchall(&server); + slots_returned += fspool_scan(&server); + if (slots_returned) mx_log_info("slots_returned=%lu :: Main Loop freed %lu slots.", slots_returned, slots_returned); @@ -2178,6 +2273,8 @@ int main(int argc, char *argv[]) while (server.jobs_running) { slots_returned = catchall(&server); + slots_returned += fspool_scan(&server); + if (slots_returned) { mx_log_info("jobs_running=%lu slots_returned=%lu global_sigint_cnt=%d global_sigterm_cnt=%d :", server.jobs_running, slots_returned, global_sigint_cnt, global_sigterm_cnt); From 8d76e231eefc5f99c464cf8d6dc4e27a4cb879f0 Mon Sep 17 00:00:00 2001 From: Donald Buczek Date: Mon, 26 Oct 2015 16:36:41 +0100 Subject: [PATCH 08/67] mxqd: stop recover_from_previous_crash from deleting running jobs --- mxqd.c | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/mxqd.c b/mxqd.c index f5f4565b..31cd8455 100644 --- a/mxqd.c +++ b/mxqd.c @@ -2112,7 +2112,6 @@ int load_groups(struct mxq_server *server) { int recover_from_previous_crash(struct mxq_server *server) { int res1; - int res2; assert(server); assert(server->mysql); @@ -2128,16 +2127,7 @@ int recover_from_previous_crash(struct mxq_server *server) mx_log_info("hostname=%s server_id=%s :: recovered from previous crash: unassigned %d jobs.", server->hostname, server->server_id, res1); - res2 = mxq_set_job_status_unknown_for_server(server->mysql, server->hostname, server->server_id); - if (res2 < 0) { - mx_log_info("mxq_unassign_jobs_of_server() failed: %m"); - return res2; - } - if (res2 > 0) - mx_log_info("hostname=%s server_id=%s :: recovered from previous crash: set job_status='unknown' for %d jobs.", - server->hostname, server->server_id, res2); - - return res1+res2; + return res1; } /**********************************************************************/ From 7ec150a2edb1fd13f3275d57c9480daf01c37887 Mon Sep 17 00:00:00 2001 From: Donald Buczek Date: Wed, 28 Oct 2015 08:48:24 +0100 Subject: [PATCH 09/67] mxq_job: add mxq_load_jobs_running_on_server --- mxq_job.c | 35 +++++++++++++++++++++++++++++++++++ mxq_job.h | 1 + 2 files changed, 36 insertions(+) diff --git a/mxq_job.c b/mxq_job.c index 4b33fda3..0c6f68c1 100644 --- a/mxq_job.c +++ b/mxq_job.c @@ -704,3 +704,38 @@ int mxq_load_job_from_group_for_server(struct mx_mysql *mysql, struct mxq_job *m return 1; } + +int mxq_load_jobs_running_on_server(struct mx_mysql *mysql, struct mxq_job **mxq_jobs, char *hostname, char *server_id) +{ + int res; + *mxq_jobs=NULL; + struct mxq_job j = {0}; + + struct mx_mysql_bind param = {0}; + struct mx_mysql_bind result = {0}; + + char *query = + "SELECT" + JOB_FIELDS + " FROM mxq_job" + " WHERE job_status = " status_str(MXQ_JOB_STATUS_RUNNING) + " AND host_hostname=?" + " AND server_id=?"; + res = mx_mysql_bind_init_param(¶m, 2); + assert(res == 0); + + res=0; + res += mx_mysql_bind_var(¶m, 0, string, &hostname); + res += mx_mysql_bind_var(¶m, 1, string, &server_id); + assert(res == 0); + + res = bind_result_job_fields(&result, &j); + assert(res == 0); + + res = mx_mysql_do_statement(mysql, query, ¶m, &result, &j, (void **)mxq_jobs, sizeof(**mxq_jobs)); + if (res < 0) { + mx_log_err("mx_mysql_do_statement(): %m"); + return res; + } + return res; +} diff --git a/mxq_job.h b/mxq_job.h index 3bf62bbf..df039105 100644 --- a/mxq_job.h +++ b/mxq_job.h @@ -107,5 +107,6 @@ int mxq_set_job_status_unknown_for_server(struct mx_mysql *mysql, char *hostname int mxq_job_set_tmpfilenames(struct mxq_group *g, struct mxq_job *j); int mxq_load_job_from_group_assigned_to_server(struct mx_mysql *mysql, struct mxq_job **mxq_jobs, uint64_t group_id, char *hostname, char *server_id); int mxq_load_job_from_group_for_server(struct mx_mysql *mysql, struct mxq_job *mxqjob, uint64_t group_id, char *hostname, char *server_id, char *host_id); +int mxq_load_jobs_running_on_server(struct mx_mysql *mysql, struct mxq_job **mxq_jobs, char *hostname, char *server_id); #endif From 2c72cac970536137af487753af71bc03ce423c06 Mon Sep 17 00:00:00 2001 From: Donald Buczek Date: Wed, 28 Oct 2015 08:59:32 +0100 Subject: [PATCH 10/67] mxqd: refactor (add reset_signals) --- mxqd.c | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/mxqd.c b/mxqd.c index 31cd8455..d4845460 100644 --- a/mxqd.c +++ b/mxqd.c @@ -887,6 +887,19 @@ static struct mxq_group_list *server_update_groupdata(struct mxq_server *server, return user_update_groupdata(user, group); } +static void reset_signals() +{ + signal(SIGINT, SIG_DFL); + signal(SIGTERM, SIG_DFL); + signal(SIGQUIT, SIG_DFL); + signal(SIGHUP, SIG_DFL); + signal(SIGTSTP, SIG_DFL); + signal(SIGTTIN, SIG_DFL); + signal(SIGTTOU, SIG_DFL); + signal(SIGCHLD, SIG_DFL); + signal(SIGPIPE, SIG_DFL); +} + static int init_child_process(struct mxq_group_list *group, struct mxq_job *j) { struct mxq_group *g; @@ -905,18 +918,7 @@ static int init_child_process(struct mxq_group_list *group, struct mxq_job *j) s = group->user->server; g = &group->group; - /** restore signal handler **/ - signal(SIGINT, SIG_DFL); - signal(SIGTERM, SIG_DFL); - signal(SIGQUIT, SIG_DFL); - signal(SIGHUP, SIG_DFL); - signal(SIGTSTP, SIG_DFL); - signal(SIGTTIN, SIG_DFL); - signal(SIGTTOU, SIG_DFL); - signal(SIGCHLD, SIG_DFL); - - /* reset SIGPIPE which seems to be ignored by mysqlclientlib (?) */ - signal(SIGPIPE, SIG_DFL); + reset_signals(); /** set sessionid and pgrp leader **/ pid = setsid(); @@ -1205,6 +1207,8 @@ int reaper_process(struct mxq_server *server,struct mxq_group_list *group,struc FILE *out; int res; + reset_signals(); + res=prctl(PR_SET_CHILD_SUBREAPER, 1); if (res<0) { mx_log_err("set subreaper: %m"); From 82e899220a9e88aab20be807e718ac81f2bf133c Mon Sep 17 00:00:00 2001 From: Donald Buczek Date: Wed, 28 Oct 2015 23:19:43 +0100 Subject: [PATCH 11/67] mxqd: let recover_from_previous_crash rebuild state for previous jobs --- mxqd.c | 272 +++++++++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 227 insertions(+), 45 deletions(-) diff --git a/mxqd.c b/mxqd.c index d4845460..861b39f7 100644 --- a/mxqd.c +++ b/mxqd.c @@ -652,55 +652,103 @@ void group_init(struct mxq_group_list *group) group->memory_max = memory_max; } -/**********************************************************************/ +static struct mxq_user_list *server_find_user(struct mxq_server *server,uint32_t uid) +{ + struct mxq_user_list *user_list; -struct mxq_job_list *server_remove_job_by_pid(struct mxq_server *server, pid_t pid) + for (user_list=server->users;user_list;user_list=user_list->next) + if (user_list->groups && user_list->groups[0].group.user_uid==uid) { + return user_list; + } + return NULL; +} + +static struct mxq_group_list *server_find_group(struct mxq_server *server,uint64_t group_id) { - struct mxq_user_list *user; - struct mxq_group_list *group; - struct mxq_job_list *job, *prev; + struct mxq_user_list *user_list; + struct mxq_group_list *group_list; - for (user=server->users; user; user=user->next) { - for (group=user->groups; group; group=group->next) { - for (job=group->jobs, prev=NULL; job; prev=job,job=job->next) { - if (job->job.host_pid == pid) { - if (prev) { - prev->next = job->next; - } else { - assert(group->jobs); - assert(group->jobs == job); + for (user_list=server->users;user_list;user_list=user_list->next) + for (group_list=user_list->groups;group_list;group_list=group_list->next) + if (group_list->group.group_id==group_id) + return group_list; + return NULL; +} - group->jobs = job->next; - } +static struct mxq_job_list *server_find_job(struct mxq_server *server,uint64_t job_id) +{ + struct mxq_user_list *user_list; + struct mxq_group_list *group_list; + struct mxq_job_list *job_list; + + for (user_list=server->users;user_list;user_list=user_list->next) + for (group_list=user_list->groups;group_list;group_list=group_list->next) + for (job_list=group_list->jobs;job_list;job_list=job_list->next) + if (job_list->job.job_id==job_id) + return job_list; + return NULL; +} - group->job_cnt--; - user->job_cnt--; - server->job_cnt--; +static struct mxq_job_list *server_find_job_by_pid(struct mxq_server *server,pid_t pid) +{ + struct mxq_user_list *user_list; + struct mxq_group_list *group_list; + struct mxq_job_list *job_list; + + for (user_list=server->users;user_list;user_list=user_list->next) + for (group_list=user_list->groups;group_list;group_list=group_list->next) + for (job_list=group_list->jobs;job_list;job_list=job_list->next) + if (job_list->job.host_pid==pid) + return job_list; + return NULL; +} - group->slots_running -= job->job.host_slots; - user->slots_running -= job->job.host_slots; - server->slots_running -= job->job.host_slots; - group->threads_running -= group->group.job_threads; - user->threads_running -= group->group.job_threads; - server->threads_running -= group->group.job_threads; +void server_remove_job(struct mxq_job_list *job) { + struct mxq_group_list *group=job->group; + struct mxq_user_list *user=group->user; + struct mxq_server *server=user->server; - group->group.group_jobs_running--; + struct mxq_job_list **prev; - group->jobs_running--; - user->jobs_running--; - server->jobs_running--; + for (prev=&group->jobs;*prev;prev=&(*prev)->next) { + if (*prev==job) { + *prev=job->next; + group->job_cnt--; + user->job_cnt--; + server->job_cnt--; - group->memory_used -= group->group.job_memory; - user->memory_used -= group->group.job_memory; - server->memory_used -= group->group.job_memory; + group->slots_running -= job->job.host_slots; + user->slots_running -= job->job.host_slots; + server->slots_running -= job->job.host_slots; - return job; - } - } + group->threads_running -= group->group.job_threads; + user->threads_running -= group->group.job_threads; + server->threads_running -= group->group.job_threads; + + group->group.group_jobs_running--; + + group->jobs_running--; + user->jobs_running--; + server->jobs_running--; + + group->memory_used -= group->group.job_memory; + user->memory_used -= group->group.job_memory; + server->memory_used -= group->group.job_memory; + break; } } - return NULL; +} + +struct mxq_job_list *server_remove_job_by_pid(struct mxq_server *server, pid_t pid) +{ + struct mxq_job_list *job; + + job=server_find_job_by_pid(server,pid); + if (job) { + server_remove_job(job); + } + return job; } /**********************************************************************/ @@ -1999,6 +2047,118 @@ static int fspool_scan(struct mxq_server *server) { return cnt; } +static int file_exists(char *name) { + int res; + struct stat stat_buf; + + res=stat(name,&stat_buf); + if (res<0) { + if (errno==ENOENT) { + return 0; + } else { + mx_log_warning("%s: %m",name); + return 1; + } + } else { + return 1; + } +} + +static int fspool_file_exists(struct mxq_server *server,uint64_t job_id) { + _mx_cleanup_free_ char *fspool_filename=NULL; + fspool_filename=fspool_get_filename(server,job_id); + return file_exists(fspool_filename); +} + +static int lost_scan_one(struct mxq_server *server) +{ + struct mxq_user_list *user_list; + struct mxq_group_list *group_list; + struct mxq_job_list *job_list; + int res; + + for (user_list=server->users;user_list;user_list=user_list->next) + for (group_list=user_list->groups;group_list;group_list=group_list->next) + for (job_list=group_list->jobs;job_list;job_list=job_list->next) { + res=kill(job_list->job.host_pid,0); + if (res<0) { + if (errno==ESRCH) { + if (!fspool_file_exists(server,job_list->job.job_id)) { + mx_log_warning("pid %u: process is gone. cancel job %d",job_list->job.host_pid,job_list->job.job_id); + server_remove_job_by_pid(server, job_list->job.host_pid); + job_list->job.job_status=MXQ_JOB_STATUS_UNKNOWN; + job_has_finished(server,&group_list->group,job_list); + return 1; + } + } else { + return -errno; + } + } + } + return 0; +} + +static int lost_scan(struct mxq_server *server) +{ + int res; + int count=0; + do { + res=lost_scan_one(server); + if (res<0) + return res; + count+=res; + } while (res>0); + return count; +} + + +static int server_reload_running(struct mxq_server *server) +{ + int job_cnt; + struct mxq_job *jobs; + int j; + + struct mxq_job_list *mxq_job_list; + struct mxq_group_list *mxq_group_list; + struct mxq_user_list *mxq_user_list; + + int group_cnt; + + job_cnt=mxq_load_jobs_running_on_server(server->mysql,&jobs,server->hostname,server->server_id); + if (job_cnt<0) + return job_cnt; + for (j=0;jstats_starttime.tv_sec=job->date_start; + + mxq_job_list=server_find_job(server,job->job_id); + if (!mxq_job_list) { + mxq_group_list=server_find_group(server,job->group_id); + if (!mxq_group_list) { + struct mxq_group *groups=NULL; + struct mxq_group *group; + group_cnt=mxq_load_group(server->mysql,&groups,job->group_id); + if (group_cnt!=1) + continue; + group=&groups[0]; + mxq_user_list=server_find_user(server,group->user_uid); + if (!mxq_user_list) { + mxq_group_list=server_add_user(server,group); + } else { + mxq_group_list=user_add_group(mxq_user_list,group); + } + free(groups); + } + mxq_job_list=mxq_group_list->jobs; + } + group_add_job(mxq_group_list,job); + } + + free(jobs); + return job_cnt; +} + int catchall(struct mxq_server *server) { struct rusage rusage; @@ -2115,23 +2275,47 @@ int load_groups(struct mxq_server *server) { int recover_from_previous_crash(struct mxq_server *server) { - int res1; + int res; assert(server); assert(server->mysql); assert(server->hostname); assert(server->server_id); - res1 = mxq_unassign_jobs_of_server(server->mysql, server->hostname, server->server_id); - if (res1 < 0) { + res = mxq_unassign_jobs_of_server(server->mysql, server->hostname, server->server_id); + if (res < 0) { mx_log_info("mxq_unassign_jobs_of_server() failed: %m"); - return res1; + return res; } - if (res1 > 0) + if (res > 0) mx_log_info("hostname=%s server_id=%s :: recovered from previous crash: unassigned %d jobs.", - server->hostname, server->server_id, res1); + server->hostname, server->server_id, res); - return res1; + res=server_reload_running(server); + if (res<0) { + mx_log_err("recover: server_reload_running: %m"); + return res; + } + if (res>0) + mx_log_info("recover: reload %d running jobs from database", res); + + res=fspool_scan(server); + if (res<0) { + mx_log_err("recover: server_fspool_scan: %m"); + return res; + } + if (res>0) + mx_log_info("recover: processed %d finished jobs from fspool",res); + + res=lost_scan(server); + if (res<0) { + mx_log_err("recover: lost_scan: %m"); + return(res); + } + if (res>0) + mx_log_warning("recover: %d jobs vanished from the system",res); + + return 0; } /**********************************************************************/ @@ -2205,8 +2389,6 @@ int main(int argc, char *argv[]) mx_log_warning("recover_from_previous_crash() failed. Aborting execution."); fail = 1; } - if (res > 0) - mx_log_warning("total %d jobs recovered from previous crash.", res); if (server.recoveronly) fail = 1; From a07647dfe17369cc454f5f1e7f51df9e245ea4a8 Mon Sep 17 00:00:00 2001 From: Donald Buczek Date: Wed, 28 Oct 2015 15:47:49 +0100 Subject: [PATCH 12/67] mxqd: add SIGQUIT processing : do not kill or wait for children --- mxqd.c | 49 ++++++++++++++++++++++++++++--------------------- 1 file changed, 28 insertions(+), 21 deletions(-) diff --git a/mxqd.c b/mxqd.c index 861b39f7..37001471 100644 --- a/mxqd.c +++ b/mxqd.c @@ -53,6 +53,7 @@ volatile sig_atomic_t global_sigint_cnt=0; volatile sig_atomic_t global_sigterm_cnt=0; +volatile sig_atomic_t global_sigquit_cnt=0; int mxq_redirect_output(char *stdout_fname, char *stderr_fname); @@ -2332,6 +2333,11 @@ static void sig_handler(int sig) global_sigterm_cnt++; return; } + + if (sig == SIGQUIT) { + global_sigquit_cnt++; + return; + } } int main(int argc, char *argv[]) @@ -2377,7 +2383,7 @@ int main(int argc, char *argv[]) signal(SIGINT, sig_handler); signal(SIGTERM, sig_handler); - signal(SIGQUIT, SIG_IGN); + signal(SIGQUIT, sig_handler); signal(SIGHUP, SIG_IGN); signal(SIGTSTP, SIG_IGN); signal(SIGTTIN, SIG_IGN); @@ -2393,7 +2399,7 @@ int main(int argc, char *argv[]) if (server.recoveronly) fail = 1; - while (!global_sigint_cnt && !global_sigterm_cnt && !fail) { + while (!global_sigint_cnt && !global_sigterm_cnt && !global_sigquit_cnt && !fail) { slots_returned = catchall(&server); slots_returned += fspool_scan(&server); @@ -2445,28 +2451,29 @@ int main(int argc, char *argv[]) } /*** clean up ***/ - mx_log_info("global_sigint_cnt=%d global_sigterm_cnt=%d : Exiting.", global_sigint_cnt, global_sigterm_cnt); + mx_log_info("global_sigint_cnt=%d global_sigterm_cnt=%d global_sigquit_cnt=%d: Exiting.", global_sigint_cnt, global_sigterm_cnt,global_sigquit_cnt); - while (server.jobs_running) { - slots_returned = catchall(&server); - slots_returned += fspool_scan(&server); + if (global_sigterm_cnt||global_sigint_cnt) { + while (server.jobs_running) { + slots_returned = catchall(&server); + slots_returned += fspool_scan(&server); - if (slots_returned) { - mx_log_info("jobs_running=%lu slots_returned=%lu global_sigint_cnt=%d global_sigterm_cnt=%d :", - server.jobs_running, slots_returned, global_sigint_cnt, global_sigterm_cnt); - continue; + if (slots_returned) { + mx_log_info("jobs_running=%lu slots_returned=%lu global_sigint_cnt=%d global_sigterm_cnt=%d :", + server.jobs_running, slots_returned, global_sigint_cnt, global_sigterm_cnt); + continue; + } + if (global_sigint_cnt) + killall(&server, SIGTERM, 1); + + killallcancelled(&server, SIGTERM, 0); + killallcancelled(&server, SIGINT, 0); + killall_over_time(&server); + killall_over_memory(&server); + mx_log_info("jobs_running=%lu global_sigint_cnt=%d global_sigterm_cnt=%d : Exiting. Wating for jobs to finish. Sleeping for a while.", + server.jobs_running, global_sigint_cnt, global_sigterm_cnt); + sleep(1); } - if (global_sigint_cnt) - killall(&server, SIGTERM, 0); - - killallcancelled(&server, SIGTERM, 0); - killallcancelled(&server, SIGINT, 0); - killall_over_time(&server); - killall_over_memory(&server); - - mx_log_info("jobs_running=%lu global_sigint_cnt=%d global_sigterm_cnt=%d : Exiting. Wating for jobs to finish. Sleeping for a while.", - server.jobs_running, global_sigint_cnt, global_sigterm_cnt); - sleep(1); } mx_mysql_finish(&(server.mysql)); From 272f6fc6716ea833cfe88ed9f5c3208e2a4afeb4 Mon Sep 17 00:00:00 2001 From: Donald Buczek Date: Wed, 28 Oct 2015 12:03:44 +0100 Subject: [PATCH 13/67] mxq_job: add mxq_set_job_status_unknown --- mxq_job.c | 23 +++++++++++++++++++++++ mxq_job.h | 1 + 2 files changed, 24 insertions(+) diff --git a/mxq_job.c b/mxq_job.c index 0c6f68c1..9a2f2f80 100644 --- a/mxq_job.c +++ b/mxq_job.c @@ -573,6 +573,29 @@ int mxq_set_job_status_unknown_for_server(struct mx_mysql *mysql, char *hostname return res; } +int mxq_set_job_status_unknown(struct mx_mysql *mysql, struct mxq_job *job) +{ + int res; + struct mx_mysql_bind param = {0}; + + char *query = + "UPDATE mxq_job SET" + " job_status = " status_str(MXQ_JOB_STATUS_UNKNOWN) + " WHERE job_id = ?"; + + res = mx_mysql_bind_init_param(¶m, 1); + res += mx_mysql_bind_var(¶m, 0, uint64, &job->job_id); + assert(res == 0); + + res = mx_mysql_do_statement_noresult_retry_on_fail(mysql, query, ¶m); + if (res < 0) { + mx_log_err("mx_mysql_do_statement(): %m"); + return res; + } + + return res; +} + int mxq_job_set_tmpfilenames(struct mxq_group *g, struct mxq_job *j) { if (!mx_streq(j->job_stdout, "/dev/null")) { diff --git a/mxq_job.h b/mxq_job.h index df039105..52259ca9 100644 --- a/mxq_job.h +++ b/mxq_job.h @@ -103,6 +103,7 @@ int mxq_unassign_jobs_of_server(struct mx_mysql *mysql, char *hostname, char *se int mxq_set_job_status_loaded_on_server(struct mx_mysql *mysql, struct mxq_job *job); int mxq_set_job_status_running(struct mx_mysql *mysql, struct mxq_job *job); int mxq_set_job_status_exited(struct mx_mysql *mysql, struct mxq_job *job); +int mxq_set_job_status_unknown(struct mx_mysql *mysql, struct mxq_job *job); int mxq_set_job_status_unknown_for_server(struct mx_mysql *mysql, char *hostname, char *server_id); int mxq_job_set_tmpfilenames(struct mxq_group *g, struct mxq_job *j); int mxq_load_job_from_group_assigned_to_server(struct mx_mysql *mysql, struct mxq_job **mxq_jobs, uint64_t group_id, char *hostname, char *server_id); From c6bd46fbd25f0e0611b1185b7ecc7efbb776ec5a Mon Sep 17 00:00:00 2001 From: Donald Buczek Date: Wed, 28 Oct 2015 12:50:46 +0100 Subject: [PATCH 14/67] mxqd: do not kill children in catchall --- mxqd.c | 6 ------ 1 file changed, 6 deletions(-) diff --git a/mxqd.c b/mxqd.c index 37001471..9c20b0ea 100644 --- a/mxqd.c +++ b/mxqd.c @@ -2202,12 +2202,6 @@ int catchall(struct mxq_server *server) { mx_log_err("FIX ME BUG!!! pid=%d errno=%d (%m)", pid, errno); continue; } - /* valid job returned.. */ - - /* kill possible leftovers with SIGKILL */ - res = kill(-siginfo.si_pid, SIGKILL); - if (res == -1) - mx_log_err("kill process group pgrp=%d failed: %m", -siginfo.si_pid); /* reap child and save new state */ pid = wait4(siginfo.si_pid, &status, WNOHANG, &rusage); From 274bfd33211d9c064acf7a91606ac2095825c0d1 Mon Sep 17 00:00:00 2001 From: Donald Buczek Date: Wed, 28 Oct 2015 12:53:15 +0100 Subject: [PATCH 15/67] mxqd: add job_is_lost --- mxqd.c | 36 +++++++++++++++++++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/mxqd.c b/mxqd.c index 9c20b0ea..501973b1 100644 --- a/mxqd.c +++ b/mxqd.c @@ -1915,6 +1915,40 @@ static int job_has_finished (struct mxq_server *server,struct mxq_group *g,struc return cnt; } +static int job_is_lost (struct mxq_server *server,struct mxq_group *g,struct mxq_job_list *job) +{ + int cnt=0; + int res; + struct mxq_job *j=&job->job; + + mxq_set_job_status_unknown(server->mysql, j); + g->group_jobs_unknown++; + + mxq_job_set_tmpfilenames(g, j); + + if (!mx_streq(j->job_stdout, "/dev/null")) { + res = rename(j->tmp_stdout, j->job_stdout); + if (res == -1) { + mx_log_err(" job=%s(%d):%lu:%lu host_pid=%d :: rename(stdout) failed: %m", + g->user_name, g->user_uid, g->group_id, j->job_id, j->host_pid); + } + } + + if (!mx_streq(j->job_stderr, "/dev/null") && !mx_streq(j->job_stderr, j->job_stdout)) { + res = rename(j->tmp_stderr, j->job_stderr); + if (res == -1) { + mx_log_err(" job=%s(%d):%lu:%lu host_pid=%d :: rename(stderr) failed: %m", + g->user_name, g->user_uid, g->group_id, j->job_id, j->host_pid); + } + } + + cnt += job->group->slots_per_job; + cpuset_clear_running(&server->cpu_set_running,&j->host_cpu_set); + mxq_job_free_content(j); + free(job); + return cnt; +} + static char *fspool_get_filename (struct mxq_server *server,long unsigned int job_id) { char *fspool_filename; @@ -2088,7 +2122,7 @@ static int lost_scan_one(struct mxq_server *server) mx_log_warning("pid %u: process is gone. cancel job %d",job_list->job.host_pid,job_list->job.job_id); server_remove_job_by_pid(server, job_list->job.host_pid); job_list->job.job_status=MXQ_JOB_STATUS_UNKNOWN; - job_has_finished(server,&group_list->group,job_list); + job_is_lost(server,&group_list->group,job_list); return 1; } } else { From e3db7214657e8cba68205ab001e63d9f21335eb2 Mon Sep 17 00:00:00 2001 From: Donald Buczek Date: Wed, 28 Oct 2015 13:52:39 +0100 Subject: [PATCH 16/67] mxqd: let reaper call setsid instread of user process --- mxqd.c | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/mxqd.c b/mxqd.c index 501973b1..ca03aa70 100644 --- a/mxqd.c +++ b/mxqd.c @@ -954,7 +954,6 @@ static int init_child_process(struct mxq_group_list *group, struct mxq_job *j) struct mxq_group *g; struct mxq_server *s; struct passwd *passwd; - pid_t pid; int res; int fh; struct rlimit rlim; @@ -969,13 +968,6 @@ static int init_child_process(struct mxq_group_list *group, struct mxq_job *j) reset_signals(); - /** set sessionid and pgrp leader **/ - pid = setsid(); - if (pid == -1) { - mx_log_err("job=%s(%d):%lu:%lu setsid(): %m", - g->user_name, g->user_uid, g->group_id, j->job_id); - } - passwd = getpwuid(g->user_uid); if (!passwd) { mx_log_err("job=%s(%d):%lu:%lu getpwuid(): %m", @@ -1258,6 +1250,11 @@ int reaper_process(struct mxq_server *server,struct mxq_group_list *group,struc reset_signals(); + res = setsid(); + if (res<0) { + mx_log_warning("reaper_process setsid: %m"); + } + res=prctl(PR_SET_CHILD_SUBREAPER, 1); if (res<0) { mx_log_err("set subreaper: %m"); From 4a28bf6fde2fa39c32326dd9d51e896fb6bd203e Mon Sep 17 00:00:00 2001 From: Donald Buczek Date: Wed, 28 Oct 2015 14:35:53 +0100 Subject: [PATCH 17/67] mxqd: remove unused member --- mxqd.h | 2 -- 1 file changed, 2 deletions(-) diff --git a/mxqd.h b/mxqd.h index 1259ddd6..7e56c610 100644 --- a/mxqd.h +++ b/mxqd.h @@ -11,8 +11,6 @@ struct mxq_job_list { struct mxq_job job; unsigned long long int max_sum_rss; - - pid_t pid; }; struct mxq_group_list { From fe360f47b780d2f54ed7cd08e11416c1969d332c Mon Sep 17 00:00:00 2001 From: Donald Buczek Date: Wed, 28 Oct 2015 15:02:59 +0100 Subject: [PATCH 18/67] mxqd: better loglevels for killall_over_time --- mxqd.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mxqd.c b/mxqd.c index ca03aa70..6e489256 100644 --- a/mxqd.c +++ b/mxqd.c @@ -1716,7 +1716,7 @@ int killall_over_time(struct mxq_server *server) /* limit killing to every >= 60 seconds */ mx_within_rate_limit_or_return(60, 1); - mx_log_info("killall_over_time: Sending signals to all jobs running longer than requested."); + mx_log_debug("killall_over_time: Sending signals to all jobs running longer than requested."); gettimeofday(&now, NULL); @@ -1731,13 +1731,13 @@ int killall_over_time(struct mxq_server *server) pid = job->job.host_pid; if (delta.tv_sec <= group->group.job_time*61) { - mx_log_debug("killall_over_time(): Sending signal=XCPU to job=%s(%d):%lu:%lu pid=%d", + mx_log_info("killall_over_time(): Sending signal=XCPU to job=%s(%d):%lu:%lu pid=%d", group->group.user_name, group->group.user_uid, group->group.group_id, job->job.job_id, pid); kill(pid, SIGXCPU); continue; } - mx_log_debug("killall_over_time(): Sending signal=XCPU to job=%s(%d):%lu:%lu pgrp=%d", + mx_log_info("killall_over_time(): Sending signal=XCPU to job=%s(%d):%lu:%lu pgrp=%d", group->group.user_name, group->group.user_uid, group->group.group_id, job->job.job_id, pid); kill(-pid, SIGXCPU); From 29d49c751524108b1094e6fe3d355ade9fbaf0fc Mon Sep 17 00:00:00 2001 From: Donald Buczek Date: Wed, 28 Oct 2015 23:26:16 +0100 Subject: [PATCH 19/67] mxqd: do not finish jobs from signals when we have reaper output --- mxqd.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/mxqd.c b/mxqd.c index 6e489256..31e70622 100644 --- a/mxqd.c +++ b/mxqd.c @@ -2223,7 +2223,7 @@ int catchall(struct mxq_server *server) { assert(siginfo.si_pid > 1); - job = server_remove_job_by_pid(server, siginfo.si_pid); + job = server_find_job_by_pid(server,siginfo.si_pid); if (!job) { mx_log_warning("unknown pid returned.. si_pid=%d si_uid=%d si_code=%d si_status=%d getpgid(si_pid)=%d getsid(si_pid)=%d", siginfo.si_pid, siginfo.si_uid, siginfo.si_code, siginfo.si_status, @@ -2233,6 +2233,12 @@ int catchall(struct mxq_server *server) { mx_log_err("FIX ME BUG!!! pid=%d errno=%d (%m)", pid, errno); continue; } + if (fspool_file_exists(server,job->job.job_id)) { + waitpid(siginfo.si_pid, &status, WNOHANG); + continue; + } + mx_log_err("reaper died. status=%d. Cleaning up job from catchall.",status); + server_remove_job(job); /* reap child and save new state */ pid = wait4(siginfo.si_pid, &status, WNOHANG, &rusage); From 61ddd6f4ca0096a28f70b7bf3a40a6f248a5e6d2 Mon Sep 17 00:00:00 2001 From: Donald Buczek Date: Thu, 29 Oct 2015 16:43:24 +0100 Subject: [PATCH 20/67] mxq_job: refactor (add do_jobs_statement) --- mxq_job.c | 64 ++++++++++++++++++++++++------------------------------- 1 file changed, 28 insertions(+), 36 deletions(-) diff --git a/mxq_job.c b/mxq_job.c index 9a2f2f80..3e8a3e85 100644 --- a/mxq_job.c +++ b/mxq_job.c @@ -12,6 +12,7 @@ #include "mx_util.h" #include "mx_log.h" +#include "mx_util.h" #include "mxq_group.h" #include "mxq_job.h" @@ -164,13 +165,28 @@ void mxq_job_free_content(struct mxq_job *j) j->job_argv = NULL; } +static int do_jobs_statement(struct mx_mysql *mysql, char *query, struct mx_mysql_bind *param, struct mxq_job **jobs) +{ + int res; + struct mxq_job j = {0}; + struct mx_mysql_bind result = {0}; + + res = bind_result_job_fields(&result, &j); + assert(res == 0); + + res = mx_mysql_do_statement(mysql, query, param, &result, &j, (void **)jobs, sizeof(**jobs)); + if (res < 0) { + mx_log_err("mx_mysql_do_statement(): %m"); + return res; + } + return res; +} + int mxq_load_job(struct mx_mysql *mysql, struct mxq_job **mxq_jobs, uint64_t job_id) { int res; struct mxq_job *jobs = NULL; - struct mxq_job j = {0}; struct mx_mysql_bind param = {0}; - struct mx_mysql_bind result = {0}; assert(mysql); assert(mxq_jobs); @@ -189,12 +205,8 @@ int mxq_load_job(struct mx_mysql *mysql, struct mxq_job **mxq_jobs, uint64_t job res = mx_mysql_bind_var(¶m, 0, uint64, &job_id); assert(res == 0); - res = bind_result_job_fields(&result, &j); - assert(res == 0); - - res = mx_mysql_do_statement(mysql, query, ¶m, &result, &j, (void **)&jobs, sizeof(*jobs)); + res=do_jobs_statement(mysql, query, ¶m, &jobs); if (res < 0) { - mx_log_err("mx_mysql_do_statement(): %m"); return res; } @@ -202,13 +214,13 @@ int mxq_load_job(struct mx_mysql *mysql, struct mxq_job **mxq_jobs, uint64_t job return res; } + + int mxq_load_jobs_in_group(struct mx_mysql *mysql, struct mxq_job **mxq_jobs, struct mxq_group *grp) { int res; struct mxq_job *jobs = NULL; - struct mxq_job j = {0}; struct mx_mysql_bind param = {0}; - struct mx_mysql_bind result = {0}; assert(mysql); assert(mxq_jobs); @@ -227,12 +239,8 @@ int mxq_load_jobs_in_group(struct mx_mysql *mysql, struct mxq_job **mxq_jobs, st res = mx_mysql_bind_var(¶m, 0, uint64, &(grp->group_id)); assert(res == 0); - res = bind_result_job_fields(&result, &j); - assert(res == 0); - - res = mx_mysql_do_statement(mysql, query, ¶m, &result, &j, (void **)&jobs, sizeof(*jobs)); + res=do_jobs_statement(mysql, query, ¶m, &jobs); if (res < 0) { - mx_log_err("mx_mysql_do_statement(): %m"); return res; } @@ -244,9 +252,7 @@ int mxq_load_jobs_in_group_with_status(struct mx_mysql *mysql, struct mxq_job ** { int res; struct mxq_job *jobs = NULL; - struct mxq_job j = {0}; struct mx_mysql_bind param = {0}; - struct mx_mysql_bind result = {0}; assert(mysql); assert(mxq_jobs); @@ -268,12 +274,8 @@ int mxq_load_jobs_in_group_with_status(struct mx_mysql *mysql, struct mxq_job ** res += mx_mysql_bind_var(¶m, 1, uint64, &job_status); assert(res == 0); - res = bind_result_job_fields(&result, &j); - assert(res == 0); - - res = mx_mysql_do_statement(mysql, query, ¶m, &result, &j, (void **)&jobs, sizeof(*jobs)); + res=do_jobs_statement(mysql, query, ¶m, &jobs); if (res < 0) { - mx_log_err("mx_mysql_do_statement(): %m"); return res; } @@ -628,9 +630,7 @@ int mxq_load_job_from_group_assigned_to_server(struct mx_mysql *mysql, struct mx { int res; struct mxq_job *jobs = NULL; - struct mxq_job j = {0}; struct mx_mysql_bind param = {0}; - struct mx_mysql_bind result = {0}; assert(mysql); assert(mxq_jobs); @@ -659,12 +659,8 @@ int mxq_load_job_from_group_assigned_to_server(struct mx_mysql *mysql, struct mx res += mx_mysql_bind_var(¶m, 2, uint64, &group_id); assert(res == 0); - res = bind_result_job_fields(&result, &j); - assert(res == 0); - - res = mx_mysql_do_statement(mysql, query, ¶m, &result, &j, (void **)&jobs, sizeof(*jobs)); + res=do_jobs_statement(mysql, query, ¶m, &jobs); if (res < 0) { - mx_log_err("mx_mysql_do_statement(): %m"); return res; } @@ -731,11 +727,9 @@ int mxq_load_job_from_group_for_server(struct mx_mysql *mysql, struct mxq_job *m int mxq_load_jobs_running_on_server(struct mx_mysql *mysql, struct mxq_job **mxq_jobs, char *hostname, char *server_id) { int res; - *mxq_jobs=NULL; - struct mxq_job j = {0}; + struct mxq_job *jobs = NULL; struct mx_mysql_bind param = {0}; - struct mx_mysql_bind result = {0}; char *query = "SELECT" @@ -752,13 +746,11 @@ int mxq_load_jobs_running_on_server(struct mx_mysql *mysql, struct mxq_job **mxq res += mx_mysql_bind_var(¶m, 1, string, &server_id); assert(res == 0); - res = bind_result_job_fields(&result, &j); - assert(res == 0); - - res = mx_mysql_do_statement(mysql, query, ¶m, &result, &j, (void **)mxq_jobs, sizeof(**mxq_jobs)); + res=do_jobs_statement(mysql, query, ¶m, &jobs); if (res < 0) { - mx_log_err("mx_mysql_do_statement(): %m"); return res; } + + *mxq_jobs = jobs; return res; } From 7dee7ef92a8b7c14ebb9f33a546bc64a7b080684 Mon Sep 17 00:00:00 2001 From: Donald Buczek Date: Thu, 29 Oct 2015 16:45:22 +0100 Subject: [PATCH 21/67] mxq_job: add a string version of host_cpu_set --- mxq_job.h | 1 + mxqd.c | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/mxq_job.h b/mxq_job.h index 52259ca9..921a484e 100644 --- a/mxq_job.h +++ b/mxq_job.h @@ -45,6 +45,7 @@ struct mxq_job { uint32_t host_pid; uint32_t host_slots; cpu_set_t host_cpu_set; + char * host_cpu_set_str; int64_t date_submit; int64_t date_start; diff --git a/mxqd.c b/mxqd.c index 31e70622..261bf7f7 100644 --- a/mxqd.c +++ b/mxqd.c @@ -1357,7 +1357,8 @@ unsigned long start_job(struct mxq_group_list *group) group->group.user_name, group->group.user_uid, group->group.group_id, mxqjob.job_id); cpuset_init_job(&mxqjob.host_cpu_set,&server->cpu_set_available,&server->cpu_set_running,group->slots_per_job); - cpuset_log(" job assigned cpus: ",&mxqjob.host_cpu_set); + mxqjob.host_cpu_set_str=mx_cpuset_to_str(&mxqjob.host_cpu_set); + mx_log_info("job assigned cpus: [%s]",mxqjob.host_cpu_set_str); mx_mysql_disconnect(server->mysql); From 35af203b8568a5f1c6fc6932b76366a820bfa90c Mon Sep 17 00:00:00 2001 From: Donald Buczek Date: Thu, 29 Oct 2015 16:47:28 +0100 Subject: [PATCH 22/67] database: store and retrieve cpuset of job --- mxq_job.c | 21 ++++++++++++++------- mysql/alter_tables_0.18.2.sql | 5 +++++ mysql/create_tables.sql | 1 + web/pages/mxq/mxq.in | 1 + 4 files changed, 21 insertions(+), 7 deletions(-) create mode 100644 mysql/alter_tables_0.18.2.sql diff --git a/mxq_job.c b/mxq_job.c index 3e8a3e85..d0e930e8 100644 --- a/mxq_job.c +++ b/mxq_job.c @@ -17,7 +17,7 @@ #include "mxq_group.h" #include "mxq_job.h" -#define JOB_FIELDS_CNT 36 +#define JOB_FIELDS_CNT 37 #define JOB_FIELDS \ " job_id, " \ " job_status, " \ @@ -36,6 +36,7 @@ " host_hostname, " \ " host_pid, " \ " host_slots, " \ + " host_cpu_set, " \ " UNIX_TIMESTAMP(date_submit) as date_submit, " \ " UNIX_TIMESTAMP(date_start) as date_start, " \ " UNIX_TIMESTAMP(date_end) as date_end, " \ @@ -81,6 +82,7 @@ static int bind_result_job_fields(struct mx_mysql_bind *result, struct mxq_job * res += mx_mysql_bind_var(result, idx++, string, &(j->host_hostname)); res += mx_mysql_bind_var(result, idx++, uint32, &(j->host_pid)); res += mx_mysql_bind_var(result, idx++, uint32, &(j->host_slots)); + res += mx_mysql_bind_var(result, idx++, string, &(j->host_cpu_set_str)); res += mx_mysql_bind_var(result, idx++, int64, &(j->date_submit)); res += mx_mysql_bind_var(result, idx++, int64, &(j->date_start)); res += mx_mysql_bind_var(result, idx++, int64, &(j->date_end)); @@ -159,6 +161,7 @@ void mxq_job_free_content(struct mxq_job *j) mx_free_null(j->tmp_stderr); mx_free_null(j->host_submit); mx_free_null(j->host_id); + mx_free_null(j->host_cpu_set_str); mx_free_null(j->server_id); mx_free_null(j->host_hostname); mx_free_null(j->job_argv); @@ -167,7 +170,7 @@ void mxq_job_free_content(struct mxq_job *j) static int do_jobs_statement(struct mx_mysql *mysql, char *query, struct mx_mysql_bind *param, struct mxq_job **jobs) { - int res; + int res,i; struct mxq_job j = {0}; struct mx_mysql_bind result = {0}; @@ -179,6 +182,8 @@ static int do_jobs_statement(struct mx_mysql *mysql, char *query, struct mx_mysq mx_log_err("mx_mysql_do_statement(): %m"); return res; } + for (i=0;ihost_pid)); res += mx_mysql_bind_var(¶m, 1, uint32, &(job->host_slots)); - res += mx_mysql_bind_var(¶m, 2, uint64, &(job->job_id)); - res += mx_mysql_bind_var(¶m, 3, string, &(job->host_hostname)); - res += mx_mysql_bind_var(¶m, 4, string, &(job->server_id)); + res += mx_mysql_bind_var(¶m, 2, string, &(job->host_cpu_set_str)); + res += mx_mysql_bind_var(¶m, 3, uint64, &(job->job_id)); + res += mx_mysql_bind_var(¶m, 4, string, &(job->host_hostname)); + res += mx_mysql_bind_var(¶m, 5, string, &(job->server_id)); assert(res == 0); res = mx_mysql_do_statement_noresult_retry_on_fail(mysql, query, ¶m); diff --git a/mysql/alter_tables_0.18.2.sql b/mysql/alter_tables_0.18.2.sql new file mode 100644 index 00000000..1be5d73d --- /dev/null +++ b/mysql/alter_tables_0.18.2.sql @@ -0,0 +1,5 @@ +ALTER TABLE mxq_job + ADD COLUMN + host_cpu_set VARCHAR(4095) NOT NULL DEFAULT "" + AFTER + host_slots; diff --git a/mysql/create_tables.sql b/mysql/create_tables.sql index f2566cc6..f589f10a 100644 --- a/mysql/create_tables.sql +++ b/mysql/create_tables.sql @@ -87,6 +87,7 @@ CREATE TABLE IF NOT EXISTS mxq_job ( host_hostname VARCHAR(64) NOT NULL DEFAULT "", host_pid INT4 UNSIGNED NOT NULL DEFAULT 0, host_slots INT4 UNSIGNED NOT NULL DEFAULT 0, + host_cpu_set VARCHAR(4095) NOT NULL DEFAULT "", date_submit TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, date_start TIMESTAMP NOT NULL DEFAULT 0, diff --git a/web/pages/mxq/mxq.in b/web/pages/mxq/mxq.in index 422c14b4..0db5c219 100755 --- a/web/pages/mxq/mxq.in +++ b/web/pages/mxq/mxq.in @@ -356,6 +356,7 @@ host_id : $o{host_id} host_hostname : $o{host_hostname} host_pid : $o{host_pid} host_slots : $o{host_slots} +host_cpu_set : $o{host_cpu_set} date_submit : $o{date_submit} date_start : $o{date_start} $ago From 78bec97b84961e737b4e290851c80c06bcb54b57 Mon Sep 17 00:00:00 2001 From: Donald Buczek Date: Fri, 30 Oct 2015 10:27:36 +0100 Subject: [PATCH 23/67] mxqd: set cpu_set_running in group_add_job so it is applied to reloaded jobs as well --- mxqd.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mxqd.c b/mxqd.c index 261bf7f7..276dc687 100644 --- a/mxqd.c +++ b/mxqd.c @@ -126,7 +126,6 @@ static void cpuset_init_job(cpu_set_t *job_cpu_set,cpu_set_t *available,cpu_set_ for (cpu=CPU_SETSIZE-1;slots&&cpu>=0;cpu--) { if (CPU_ISSET(cpu,available) && !CPU_ISSET(cpu,running)) { CPU_SET(cpu,job_cpu_set); - CPU_SET(cpu,running); slots--; } } @@ -828,6 +827,8 @@ struct mxq_job_list *group_add_job(struct mxq_group_list *group, struct mxq_job user->threads_running += mxqgrp->job_threads; server->threads_running += mxqgrp->job_threads; + CPU_OR(&server->cpu_set_running,&server->cpu_set_running,&j->job.host_cpu_set); + mxqgrp->group_jobs_running++; mxqgrp->group_jobs_inq--; From f23144ea74a984e81fed169191d8fbd8fdb830d8 Mon Sep 17 00:00:00 2001 From: Donald Buczek Date: Fri, 30 Oct 2015 11:57:12 +0100 Subject: [PATCH 24/67] mxqd: reaper: ignore signals from mxqd --- mxqd.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/mxqd.c b/mxqd.c index 276dc687..6ebc9c76 100644 --- a/mxqd.c +++ b/mxqd.c @@ -1251,6 +1251,11 @@ int reaper_process(struct mxq_server *server,struct mxq_group_list *group,struc reset_signals(); + signal(SIGINT, SIG_IGN); + signal(SIGTERM, SIG_IGN); + signal(SIGHUP, SIG_IGN); + signal(SIGXCPU, SIG_IGN); + res = setsid(); if (res<0) { mx_log_warning("reaper_process setsid: %m"); From e76d91ed9afad64b6040a4bf36efdcf3677e4886 Mon Sep 17 00:00:00 2001 From: Marius Tolzmann Date: Fri, 30 Oct 2015 16:58:08 +0100 Subject: [PATCH 25/67] mxqd: Rename group_init() to _group_list_init() --- mxqd.c | 99 ++++++++++++++++++++++++++++++++-------------------------- 1 file changed, 55 insertions(+), 44 deletions(-) diff --git a/mxqd.c b/mxqd.c index 6ebc9c76..ea6c01b1 100644 --- a/mxqd.c +++ b/mxqd.c @@ -583,73 +583,84 @@ int server_init(struct mxq_server *server, int argc, char *argv[]) /**********************************************************************/ -void group_init(struct mxq_group_list *group) +static void _group_list_init(struct mxq_group_list *glist) { - struct mxq_server *s; - struct mxq_group *g; + struct mxq_server *server; + struct mxq_group *group; long double memory_threads; long double memory_per_thread; long double memory_max_available; + unsigned long slots_per_job; unsigned long jobs_max; unsigned long slots_max; unsigned long memory_max; - assert(group); - assert(group->user); - assert(group->user->server); + assert(glist); + assert(glist->user); + assert(glist->user->server); - s = group->user->server; - g = &group->group; + server = glist->user->server; + group = &glist->group; - memory_per_thread = (long double)g->job_memory / (long double) g->job_threads; - memory_max_available = (long double)s->memory_total * (long double)s->memory_max_per_slot / memory_per_thread; + memory_per_thread = (long double)group->job_memory / (long double)group->job_threads; + memory_max_available = (long double)server->memory_total * (long double)server->memory_max_per_slot / memory_per_thread; - if (memory_max_available > s->memory_total) - memory_max_available = s->memory_total; + if (memory_max_available > server->memory_total) + memory_max_available = server->memory_total; - slots_per_job = ceill((long double)g->job_memory / s->memory_avg_per_slot); + slots_per_job = ceill((long double)group->job_memory / server->memory_avg_per_slot); - if (slots_per_job < g->job_threads) - slots_per_job = g->job_threads; + if (slots_per_job < group->job_threads) + slots_per_job = group->job_threads; memory_threads = memory_max_available / memory_per_thread; - if (memory_per_thread > s->memory_max_per_slot) { + if (memory_per_thread > server->memory_max_per_slot) { jobs_max = memory_threads + 0.5; - } else if (memory_per_thread > s->memory_avg_per_slot) { + } else if (memory_per_thread > server->memory_avg_per_slot) { jobs_max = memory_threads + 0.5; } else { - jobs_max = s->slots; + jobs_max = server->slots; } - jobs_max /= g->job_threads; + jobs_max /= group->job_threads; /* limit maximum number of jobs on user/group request */ - if (g->job_max_per_node && jobs_max > g->job_max_per_node) - jobs_max = g->job_max_per_node; - - slots_max = jobs_max * slots_per_job; - memory_max = jobs_max * g->job_memory; - - if (group->memory_per_thread != memory_per_thread - || group->memory_max_available != memory_max_available - || group->memory_max_available != memory_max_available - || group->slots_per_job != slots_per_job - || group->jobs_max != jobs_max - || group->slots_max != slots_max - || group->memory_max != memory_max) + if (group->job_max_per_node && jobs_max > group->job_max_per_node) + jobs_max = group->job_max_per_node; + + slots_max = jobs_max * slots_per_job; + memory_max = jobs_max * group->job_memory; + + if (glist->memory_per_thread != memory_per_thread + || glist->memory_max_available != memory_max_available + || glist->memory_max_available != memory_max_available + || glist->slots_per_job != slots_per_job + || glist->jobs_max != jobs_max + || glist->slots_max != slots_max + || glist->memory_max != memory_max) { mx_log_info(" group=%s(%u):%lu jobs_max=%lu slots_max=%lu memory_max=%lu slots_per_job=%lu :: group %sinitialized.", - g->user_name, g->user_uid, g->group_id, jobs_max, slots_max, memory_max, slots_per_job, - group->orphaned?"re":""); - - group->orphaned = 0; - group->memory_per_thread = memory_per_thread; - group->memory_max_available = memory_max_available; - group->slots_per_job = slots_per_job; - group->jobs_max = jobs_max; - group->slots_max = slots_max; - group->memory_max = memory_max; + group->user_name, + group->user_uid, + group->group_id, + jobs_max, + slots_max, + memory_max, + slots_per_job, + glist->orphaned ? "re" : ""); + } + + glist->memory_per_thread = memory_per_thread; + glist->memory_max_available = memory_max_available; + + glist->slots_per_job = slots_per_job; + + glist->jobs_max = jobs_max; + glist->slots_max = slots_max; + glist->memory_max = memory_max; + + glist->orphaned = 0; } static struct mxq_user_list *server_find_user(struct mxq_server *server,uint32_t uid) @@ -868,7 +879,7 @@ struct mxq_group_list *user_add_group(struct mxq_user_list *user, struct mxq_gro assert(user->server); user->server->group_cnt++; - group_init(g); + _group_list_init(g); assert(g); return g; @@ -918,7 +929,7 @@ struct mxq_group_list *user_update_groupdata(struct mxq_user_list *user, struct mxq_group_free_content(&glist->group); memcpy(&glist->group, group, sizeof(*group)); - group_init(glist); + _group_list_init(glist); return glist; } From bbafc693d70235a274e4f37745a0bad587ed790f Mon Sep 17 00:00:00 2001 From: Marius Tolzmann Date: Fri, 30 Oct 2015 17:11:31 +0100 Subject: [PATCH 26/67] mxqd: Rename user_add_group() to _user_list_add_group() --- mxqd.c | 36 +++++++++++++++++------------------- 1 file changed, 17 insertions(+), 19 deletions(-) diff --git a/mxqd.c b/mxqd.c index ea6c01b1..5fd33095 100644 --- a/mxqd.c +++ b/mxqd.c @@ -856,33 +856,31 @@ struct mxq_job_list *group_add_job(struct mxq_group_list *group, struct mxq_job } /**********************************************************************/ -struct mxq_group_list *user_add_group(struct mxq_user_list *user, struct mxq_group *group) +static struct mxq_group_list *_user_list_add_group(struct mxq_user_list *ulist, struct mxq_group *group) { - struct mxq_group_list *g; struct mxq_group_list *glist; + struct mxq_server *server; - assert(user); + assert(ulist); + assert(ulist->server); - g = mx_calloc_forever(1, sizeof(*g)); - assert(g); + server = ulist->server; - glist = user->groups; + glist = mx_calloc_forever(1, sizeof(*glist)); - memcpy(&g->group, group, sizeof(*group)); + memcpy(&glist->group, group, sizeof(*group)); - g->user = user; - g->next = glist; + glist->user = ulist; - user->groups = g; - user->group_cnt++; + glist->next = ulist->groups; + ulist->groups = glist; - assert(user->server); - user->server->group_cnt++; + ulist->group_cnt++; + server->group_cnt++; - _group_list_init(g); + _group_list_init(glist); - assert(g); - return g; + return glist; } /**********************************************************************/ @@ -901,7 +899,7 @@ struct mxq_group_list *server_add_user(struct mxq_server *server, struct mxq_gro user->server = server; - glist = user_add_group(user, group); + glist = _user_list_add_group(user, group); assert(glist); ulist = server->users; @@ -923,7 +921,7 @@ struct mxq_group_list *user_update_groupdata(struct mxq_user_list *user, struct glist = group_list_find_group(user->groups, group); if (!glist) { - return user_add_group(user, group); + return _user_list_add_group(user, group); } mxq_group_free_content(&glist->group); @@ -2196,7 +2194,7 @@ static int server_reload_running(struct mxq_server *server) if (!mxq_user_list) { mxq_group_list=server_add_user(server,group); } else { - mxq_group_list=user_add_group(mxq_user_list,group); + mxq_group_list = _user_list_add_group(mxq_user_list,group); } free(groups); } From e15a41d4ca1ac42735723508a7c61c11ca6a4d0d Mon Sep 17 00:00:00 2001 From: Marius Tolzmann Date: Fri, 30 Oct 2015 17:18:15 +0100 Subject: [PATCH 27/67] mxqd: Rename user_update_groupdata() to _user_list_update_group() --- mxqd.c | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/mxqd.c b/mxqd.c index 5fd33095..41d8e2cb 100644 --- a/mxqd.c +++ b/mxqd.c @@ -915,16 +915,21 @@ struct mxq_group_list *server_add_user(struct mxq_server *server, struct mxq_gro /**********************************************************************/ -struct mxq_group_list *user_update_groupdata(struct mxq_user_list *user, struct mxq_group *group) +static struct mxq_group_list *_user_list_update_group(struct mxq_user_list *ulist, struct mxq_group *group) { struct mxq_group_list *glist; - glist = group_list_find_group(user->groups, group); + assert(ulist); + assert(group); + + glist = group_list_find_group(ulist->groups, group); + if (!glist) { - return _user_list_add_group(user, group); + return _user_list_add_group(ulist, group); } mxq_group_free_content(&glist->group); + memcpy(&glist->group, group, sizeof(*group)); _group_list_init(glist); @@ -943,7 +948,7 @@ static struct mxq_group_list *server_update_groupdata(struct mxq_server *server, return server_add_user(server, group); } - return user_update_groupdata(user, group); + return _user_list_update_group(user, group); } static void reset_signals() From 88d77488ec4782763188107e15b62578c8f2cc79 Mon Sep 17 00:00:00 2001 From: Marius Tolzmann Date: Fri, 30 Oct 2015 17:27:43 +0100 Subject: [PATCH 28/67] mxqd: Rename server_add_user() to _server_add_group() --- mxqd.c | 23 +++++++++-------------- 1 file changed, 9 insertions(+), 14 deletions(-) diff --git a/mxqd.c b/mxqd.c index 41d8e2cb..a0eebc64 100644 --- a/mxqd.c +++ b/mxqd.c @@ -885,31 +885,26 @@ static struct mxq_group_list *_user_list_add_group(struct mxq_user_list *ulist, /**********************************************************************/ -struct mxq_group_list *server_add_user(struct mxq_server *server, struct mxq_group *group) +static struct mxq_group_list *_server_add_group(struct mxq_server *server, struct mxq_group *group) { - struct mxq_user_list *user; struct mxq_user_list *ulist; struct mxq_group_list *glist; assert(server); assert(group); - user = mx_calloc_forever(1, sizeof(*user)); - assert(user); - - user->server = server; + ulist = mx_calloc_forever(1, sizeof(*ulist)); - glist = _user_list_add_group(user, group); - assert(glist); - - ulist = server->users; + ulist->server = server; - user->next = ulist; + ulist->next = server->users; + server->users = ulist; - server->users = user; server->user_cnt++; + glist = _user_list_add_group(ulist, group); assert(glist); + return glist; } @@ -945,7 +940,7 @@ static struct mxq_group_list *server_update_groupdata(struct mxq_server *server, user = user_list_find_uid(server->users, group->user_uid); if (!user) { - return server_add_user(server, group); + return _server_add_group(server, group); } return _user_list_update_group(user, group); @@ -2197,7 +2192,7 @@ static int server_reload_running(struct mxq_server *server) group=&groups[0]; mxq_user_list=server_find_user(server,group->user_uid); if (!mxq_user_list) { - mxq_group_list=server_add_user(server,group); + mxq_group_list = _server_add_group(server,group); } else { mxq_group_list = _user_list_add_group(mxq_user_list,group); } From f1dabdc3b1adb6caf8fdb3873111be61170414b9 Mon Sep 17 00:00:00 2001 From: Marius Tolzmann Date: Fri, 30 Oct 2015 17:33:24 +0100 Subject: [PATCH 29/67] mxqd: Rename server_update_groupdata() to server_update_group() --- mxqd.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/mxqd.c b/mxqd.c index a0eebc64..6626c44c 100644 --- a/mxqd.c +++ b/mxqd.c @@ -934,16 +934,16 @@ static struct mxq_group_list *_user_list_update_group(struct mxq_user_list *ulis /**********************************************************************/ -static struct mxq_group_list *server_update_groupdata(struct mxq_server *server, struct mxq_group *group) +static struct mxq_group_list *server_update_group(struct mxq_server *server, struct mxq_group *group) { - struct mxq_user_list *user; + struct mxq_user_list *ulist; - user = user_list_find_uid(server->users, group->user_uid); - if (!user) { + ulist = user_list_find_uid(server->users, group->user_uid); + if (!ulist) { return _server_add_group(server, group); } - return _user_list_update_group(user, group); + return _user_list_update_group(ulist, group); } static void reset_signals() @@ -2307,7 +2307,7 @@ int load_groups(struct mxq_server *server) { group_cnt = mxq_load_running_groups_for_user(server->mysql, &mxqgroups, getuid()); for (i=0, total=0; i Date: Fri, 30 Oct 2015 17:50:50 +0100 Subject: [PATCH 30/67] mxqd: Rename user_list_find_uid() to _user_list_find_by_uid() --- mxqd.c | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/mxqd.c b/mxqd.c index 6626c44c..1c80f6d0 100644 --- a/mxqd.c +++ b/mxqd.c @@ -764,14 +764,13 @@ struct mxq_job_list *server_remove_job_by_pid(struct mxq_server *server, pid_t p /**********************************************************************/ -struct mxq_user_list *user_list_find_uid(struct mxq_user_list *list, uint32_t uid) +static struct mxq_user_list *_user_list_find_by_uid(struct mxq_user_list *ulist, uint32_t uid) { - struct mxq_user_list *u; + for (; ulist; ulist = ulist->next) { + assert(ulist->groups); - for (u = list; u; u = u->next) { - assert(u->groups); - if (u->groups[0].group.user_uid == uid) { - return u; + if (ulist->groups[0].group.user_uid == uid) { + return ulist; } } return NULL; @@ -938,7 +937,7 @@ static struct mxq_group_list *server_update_group(struct mxq_server *server, str { struct mxq_user_list *ulist; - ulist = user_list_find_uid(server->users, group->user_uid); + ulist = _user_list_find_by_uid(server->users, group->user_uid); if (!ulist) { return _server_add_group(server, group); } From d948c20587974a0721676f11bab7edd189f23216 Mon Sep 17 00:00:00 2001 From: Marius Tolzmann Date: Fri, 30 Oct 2015 18:19:55 +0100 Subject: [PATCH 31/67] mxqd: Rename group_add_job() to group_list_add_job() --- mxqd.c | 75 +++++++++++++++++++++++++++------------------------------- 1 file changed, 35 insertions(+), 40 deletions(-) diff --git a/mxqd.c b/mxqd.c index 1c80f6d0..310178c6 100644 --- a/mxqd.c +++ b/mxqd.c @@ -794,64 +794,59 @@ struct mxq_group_list *group_list_find_group(struct mxq_group_list *list, struct /**********************************************************************/ -struct mxq_job_list *group_add_job(struct mxq_group_list *group, struct mxq_job *job) +struct mxq_job_list *group_list_add_job(struct mxq_group_list *glist, struct mxq_job *job) { - struct mxq_job_list *j; - struct mxq_job_list *jlist; - struct mxq_server *server; - struct mxq_user_list *user; - struct mxq_group *mxqgrp; + struct mxq_job_list *jlist; + struct mxq_user_list *ulist; - assert(group); - assert(group->user); - assert(group->user->server); - assert(job->job_status == MXQ_JOB_STATUS_RUNNING); + struct mxq_group *group; - mxqgrp = &group->group; - user = group->user; - server = user->server; + assert(glist); + assert(glist->user); + assert(glist->user->server); + assert(job->job_status == MXQ_JOB_STATUS_RUNNING); - j = mx_calloc_forever(1, sizeof(*j)); - assert(j); + group = &glist->group; + ulist = glist->user; + server = ulist->server; - jlist = group->jobs; + jlist = mx_calloc_forever(1, sizeof(*jlist)); - memcpy(&j->job, job, sizeof(*job)); + memcpy(&jlist->job, job, sizeof(*job)); - j->group = group; - j->next = jlist; + jlist->group = glist; - group->jobs = j; + jlist->next = glist->jobs; + glist->jobs = jlist; - group->job_cnt++; - user->job_cnt++; + glist->job_cnt++; + ulist->job_cnt++; server->job_cnt++; - group->slots_running += group->slots_per_job; - user->slots_running += group->slots_per_job; - server->slots_running += group->slots_per_job; + glist->slots_running += glist->slots_per_job; + ulist->slots_running += glist->slots_per_job; + server->slots_running += glist->slots_per_job; - group->threads_running += mxqgrp->job_threads; - user->threads_running += mxqgrp->job_threads; - server->threads_running += mxqgrp->job_threads; + glist->threads_running += group->job_threads; + ulist->threads_running += group->job_threads; + server->threads_running += group->job_threads; - CPU_OR(&server->cpu_set_running,&server->cpu_set_running,&j->job.host_cpu_set); + CPU_OR(&server->cpu_set_running, &server->cpu_set_running, &job->host_cpu_set); - mxqgrp->group_jobs_running++; - mxqgrp->group_jobs_inq--; + group->group_jobs_running++; + group->group_jobs_inq--; - group->jobs_running++; - user->jobs_running++; + glist->jobs_running++; + ulist->jobs_running++; server->jobs_running++; - group->memory_used += mxqgrp->job_memory; - user->memory_used += mxqgrp->job_memory; - server->memory_used += mxqgrp->job_memory; + glist->memory_used += group->job_memory; + ulist->memory_used += group->job_memory; + server->memory_used += group->job_memory; - assert(j); - return j; + return jlist; } /**********************************************************************/ @@ -1405,7 +1400,7 @@ unsigned long start_job(struct mxq_group_list *group) mx_log_err("job=%s(%d):%lu:%lu mxq_job_update_status_running(): Job not found.", group->group.user_name, group->group.user_uid, group->group.group_id, mxqjob.job_id); - job = group_add_job(group, &mxqjob); + job = group_list_add_job(group, &mxqjob); assert(job); mx_log_info(" job=%s(%d):%lu:%lu :: added running job to watch queue.", @@ -2199,7 +2194,7 @@ static int server_reload_running(struct mxq_server *server) } mxq_job_list=mxq_group_list->jobs; } - group_add_job(mxq_group_list,job); + group_list_add_job(mxq_group_list,job); } free(jobs); From e11f60e5d2113c42eb8fa0892e235a0b9f6e9004 Mon Sep 17 00:00:00 2001 From: Marius Tolzmann Date: Fri, 30 Oct 2015 18:57:52 +0100 Subject: [PATCH 32/67] mxqd: Rename group_list_find_group() to _group_list_find_by_group() --- mxqd.c | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/mxqd.c b/mxqd.c index 310178c6..234d90b0 100644 --- a/mxqd.c +++ b/mxqd.c @@ -778,15 +778,13 @@ static struct mxq_user_list *_user_list_find_by_uid(struct mxq_user_list *ulist, /**********************************************************************/ -struct mxq_group_list *group_list_find_group(struct mxq_group_list *list, struct mxq_group *group) +struct mxq_group_list *_group_list_find_by_group(struct mxq_group_list *glist, struct mxq_group *group) { - struct mxq_group_list *g; - assert(group); - for (g = list; g; g = g->next) { - if (g->group.group_id == group->group_id) { - return g; + for (; glist; glist = glist->next) { + if (glist->group.group_id == group->group_id) { + return glist; } } return NULL; @@ -911,8 +909,7 @@ static struct mxq_group_list *_user_list_update_group(struct mxq_user_list *ulis assert(ulist); assert(group); - glist = group_list_find_group(ulist->groups, group); - + glist = _group_list_find_by_group(ulist->groups, group); if (!glist) { return _user_list_add_group(ulist, group); } From 6302304fdeb75998da75da20178bb33bca0943b3 Mon Sep 17 00:00:00 2001 From: Marius Tolzmann Date: Fri, 30 Oct 2015 18:58:32 +0100 Subject: [PATCH 33/67] mxqd: Rename remove_orphaned_groups() to remove_orphaned_group_lists() --- mxqd.c | 58 ++++++++++++++++++++++++++++++++++------------------------ 1 file changed, 34 insertions(+), 24 deletions(-) diff --git a/mxqd.c b/mxqd.c index 234d90b0..6fa1d7fc 100644 --- a/mxqd.c +++ b/mxqd.c @@ -1557,58 +1557,68 @@ unsigned long start_users(struct mxq_server *server) /**********************************************************************/ -int remove_orphaned_groups(struct mxq_server *server) +int remove_orphaned_group_lists(struct mxq_server *server) { - struct mxq_user_list *user, *unext, *uprev; - struct mxq_group_list *group, *gnext, *gprev; + struct mxq_user_list *ulist, *unext, *uprev; + struct mxq_group_list *glist, *gnext, *gprev; + + struct mxq_group *group; + int cnt=0; - for (user=server->users, uprev=NULL; user; user=unext) { - unext = user->next; - for (group=user->groups, gprev=NULL; group; group=gnext) { - gnext = group->next; + for (ulist = server->users, uprev = NULL; ulist; ulist = unext) { + unext = ulist->next; - if (group->job_cnt) { - gprev = group; + for (glist = ulist->groups, gprev = NULL; glist; glist = gnext) { + gnext = glist->next; + group = &glist->group; + + if (glist->job_cnt) { + gprev = glist; continue; } - assert(!group->jobs); + assert(!glist->jobs); - if (!group->orphaned && mxq_group_jobs_active(&group->group)) { - group->orphaned = 1; - gprev = group; + if (!glist->orphaned && mxq_group_jobs_active(group)) { + glist->orphaned = 1; + gprev = glist; continue; } if (gprev) { gprev->next = gnext; } else { - assert(group == user->groups); - user->groups = gnext; + assert(glist == ulist->groups); + ulist->groups = gnext; } - mx_log_info("group=%s(%d):%lu : Removing orphaned group.", group->group.user_name, group->group.user_uid, group->group.group_id); + mx_log_info("group=%s(%d):%lu : Removing orphaned group.", + group->user_name, + group->user_uid, + group->group_id); - user->group_cnt--; + ulist->group_cnt--; server->group_cnt--; cnt++; - mxq_group_free_content(&group->group); - mx_free_null(group); + mxq_group_free_content(group); + mx_free_null(glist); } - if(user->groups) { - uprev = user; + + if(ulist->groups) { + uprev = ulist; continue; } if (uprev) { uprev->next = unext; } else { - assert(user == server->users); + assert(ulist == server->users); server->users = unext; } + server->user_cnt--; - mx_free_null(user); + mx_free_null(ulist); mx_log_info("Removed orphaned user. %lu users left.", server->user_cnt); } @@ -2307,7 +2317,7 @@ int load_groups(struct mxq_server *server) { } free(mxqgroups); - remove_orphaned_groups(server); + remove_orphaned_group_lists(server); return total; } From 90a912ca38e52d0cbeb2f5ab3ba5b46595e25ab2 Mon Sep 17 00:00:00 2001 From: Marius Tolzmann Date: Mon, 2 Nov 2015 13:03:42 +0100 Subject: [PATCH 34/67] mxqd: Cleanup start_user() --- mxqd.c | 72 +++++++++++++++++++++++++++++----------------------------- 1 file changed, 36 insertions(+), 36 deletions(-) diff --git a/mxqd.c b/mxqd.c index 6fa1d7fc..3fee5843 100644 --- a/mxqd.c +++ b/mxqd.c @@ -1408,99 +1408,99 @@ unsigned long start_job(struct mxq_group_list *group) /**********************************************************************/ -unsigned long start_user(struct mxq_user_list *user, int job_limit, long slots_to_start) +unsigned long start_user(struct mxq_user_list *ulist, int job_limit, long slots_to_start) { struct mxq_server *server; - struct mxq_group_list *group; + struct mxq_group_list *glist; struct mxq_group_list *gnext = NULL; - struct mxq_group *mxqgrp; + struct mxq_group *group; unsigned int prio; unsigned char started = 0; unsigned long slots_started = 0; int jobs_started = 0; - assert(user); - assert(user->server); - assert(user->groups); + assert(ulist); + assert(ulist->server); + assert(ulist->groups); - server = user->server; - group = user->groups; - mxqgrp = &group->group; + server = ulist->server; + glist = ulist->groups; + group = &glist->group; - prio = mxqgrp->group_priority; + prio = group->group_priority; assert(slots_to_start <= server->slots - server->slots_running); mx_log_debug(" user=%s(%d) slots_to_start=%ld job_limit=%d :: trying to start jobs for user.", - mxqgrp->user_name, mxqgrp->user_uid, slots_to_start, job_limit); + group->user_name, group->user_uid, slots_to_start, job_limit); - for (group=user->groups; group && slots_to_start > 0 && (!job_limit || jobs_started < job_limit); group=gnext) { + for (glist = ulist->groups; glist && slots_to_start > 0 && (!job_limit || jobs_started < job_limit); glist = gnext) { - mxqgrp = &group->group; + group = &glist->group; - assert(group->jobs_running <= mxqgrp->group_jobs); - assert(group->jobs_running <= group->jobs_max); + assert(glist->jobs_running <= group->group_jobs); + assert(glist->jobs_running <= glist->jobs_max); - if (group->jobs_running == mxqgrp->group_jobs) { - gnext = group->next; + if (glist->jobs_running == group->group_jobs) { + gnext = glist->next; if (!gnext && started) { - gnext = group->user->groups; + gnext = ulist->groups; started = 0; } continue; } - if (group->jobs_running == group->jobs_max) { - gnext = group->next; + if (glist->jobs_running == glist->jobs_max) { + gnext = glist->next; if (!gnext && started) { - gnext = group->user->groups; + gnext = ulist->groups; started = 0; } continue; } - if (mxq_group_jobs_inq(mxqgrp) == 0) { - gnext = group->next; + if (mxq_group_jobs_inq(group) == 0) { + gnext = glist->next; if (!gnext && started) { - gnext = group->user->groups; + gnext = ulist->groups; started = 0; } continue; } - if (group->slots_per_job > slots_to_start) { - gnext = group->next; + if (glist->slots_per_job > slots_to_start) { + gnext = glist->next; if (!gnext && started) { - gnext = group->user->groups; + gnext = ulist->groups; started = 0; } continue; } - if (mxqgrp->group_priority < prio) { + if (group->group_priority < prio) { if (started) { - gnext = group->user->groups; + gnext = ulist->groups; started = 0; continue; } - prio = mxqgrp->group_priority; + prio = group->group_priority; } mx_log_info(" group=%s(%d):%lu slots_to_start=%ld slots_per_job=%lu :: trying to start job for group.", - mxqgrp->user_name, mxqgrp->user_uid, mxqgrp->group_id, slots_to_start, group->slots_per_job); + group->user_name, group->user_uid, group->group_id, slots_to_start, glist->slots_per_job); - if (start_job(group)) { + if (start_job(glist)) { - slots_to_start -= group->slots_per_job; + slots_to_start -= glist->slots_per_job; jobs_started++; - slots_started += group->slots_per_job; + slots_started += glist->slots_per_job; started = 1; } - gnext = group->next; + gnext = glist->next; if (!gnext && started) { - gnext = group->user->groups; + gnext = ulist->groups; started = 0; } } From d1ad7cb8bb880723f2db8cb009c6b7d0bb24716b Mon Sep 17 00:00:00 2001 From: Marius Tolzmann Date: Mon, 2 Nov 2015 13:11:18 +0100 Subject: [PATCH 35/67] mxqd: Cleanup start_users() --- mxqd.c | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/mxqd.c b/mxqd.c index 3fee5843..e893ef36 100644 --- a/mxqd.c +++ b/mxqd.c @@ -1511,12 +1511,13 @@ unsigned long start_user(struct mxq_user_list *ulist, int job_limit, long slots_ unsigned long start_users(struct mxq_server *server) { - long slots_to_start; unsigned long slots_started; - int started = 0; unsigned long slots_started_total = 0; + long slots_to_start; + int started = 0; - struct mxq_user_list *user, *unext=NULL; + struct mxq_user_list *ulist; + struct mxq_user_list *unext = NULL; assert(server); @@ -1525,27 +1526,27 @@ unsigned long start_users(struct mxq_server *server) mx_log_debug("=== starting jobs on free_slots=%lu slots for user_cnt=%lu users", server->slots - server->slots_running, server->user_cnt); - for (user=server->users; user; user=user->next) { + for (ulist = server->users; ulist; ulist = ulist->next) { - slots_to_start = server->slots / server->user_cnt - user->slots_running; + slots_to_start = server->slots / server->user_cnt - ulist->slots_running; if (slots_to_start < 0) continue; - if (server->slots - server->slots_running < slots_to_start) - slots_to_start = server->slots - server->slots_running; + if (slots_to_start > (server->slots - server->slots_running)) + slots_to_start = (server->slots - server->slots_running); - slots_started = start_user(user, 0, slots_to_start); + slots_started = start_user(ulist, 0, slots_to_start); slots_started_total += slots_started; } - for (user=server->users; user && server->slots - server->slots_running; user=unext) { + for (ulist = server->users; ulist && server->slots - server->slots_running; ulist = unext) { slots_to_start = server->slots - server->slots_running; - slots_started = start_user(user, 1, slots_to_start); + slots_started = start_user(ulist, 1, slots_to_start); slots_started_total += slots_started; started = (started || slots_started); - unext = user->next; + unext = ulist->next; if (!unext && started) { unext = server->users; started = 0; From 6da2652a6e795ea4b17d2f10f06ebf49c1deb47c Mon Sep 17 00:00:00 2001 From: Marius Tolzmann Date: Mon, 2 Nov 2015 14:04:12 +0100 Subject: [PATCH 36/67] mxqd: Cleanup killall_over_time() --- mxqd.c | 47 +++++++++++++++++++++++++++-------------------- 1 file changed, 27 insertions(+), 20 deletions(-) diff --git a/mxqd.c b/mxqd.c index e893ef36..9cfedf3f 100644 --- a/mxqd.c +++ b/mxqd.c @@ -1717,9 +1717,12 @@ int killall(struct mxq_server *server, int sig, unsigned int pgrp) int killall_over_time(struct mxq_server *server) { - struct mxq_user_list *user; - struct mxq_group_list *group; - struct mxq_job_list *job; + struct mxq_user_list *ulist; + struct mxq_group_list *glist; + struct mxq_job_list *jlist; + + struct mxq_group *group; + struct mxq_job *job; struct timeval now; struct timeval delta; @@ -1738,54 +1741,58 @@ int killall_over_time(struct mxq_server *server) gettimeofday(&now, NULL); - for (user=server->users; user; user=user->next) { - for (group=user->groups; group; group=group->next) { - for (job=group->jobs; job; job=job->next) { - timersub(&now, &job->job.stats_starttime, &delta); + for (ulist = server->users; ulist; ulist = ulist->next) { + for (glist = ulist->groups; glist; glist = glist->next) { + group = &glist->group; + + for (jlist = glist->jobs; jlist; jlist = jlist->next) { + job = &jlist->job; + + timersub(&now, &job->stats_starttime, &delta); - if (delta.tv_sec <= group->group.job_time*60) + if (delta.tv_sec <= group->job_time*60) continue; - pid = job->job.host_pid; + pid = job->host_pid; - if (delta.tv_sec <= group->group.job_time*61) { + if (delta.tv_sec <= group->job_time*61) { mx_log_info("killall_over_time(): Sending signal=XCPU to job=%s(%d):%lu:%lu pid=%d", - group->group.user_name, group->group.user_uid, group->group.group_id, job->job.job_id, pid); + group->user_name, group->user_uid, group->group_id, job->job_id, pid); kill(pid, SIGXCPU); continue; } mx_log_info("killall_over_time(): Sending signal=XCPU to job=%s(%d):%lu:%lu pgrp=%d", - group->group.user_name, group->group.user_uid, group->group.group_id, job->job.job_id, pid); + group->user_name, group->user_uid, group->group_id, job->job_id, pid); kill(-pid, SIGXCPU); - if (delta.tv_sec <= group->group.job_time*63) + if (delta.tv_sec <= group->job_time*63) continue; mx_log_info("killall_over_time(): Sending signal=TERM to job=%s(%d):%lu:%lu pid=%d", - group->group.user_name, group->group.user_uid, group->group.group_id, job->job.job_id, pid); + group->user_name, group->user_uid, group->group_id, job->job_id, pid); kill(pid, SIGTERM); mx_log_info("killall_over_time(): Sending signal=HUP to job=%s(%d):%lu:%lu pgrp=%d", - group->group.user_name, group->group.user_uid, group->group.group_id, job->job.job_id, pid); + group->user_name, group->user_uid, group->group_id, job->job_id, pid); kill(-pid, SIGHUP); - if (delta.tv_sec <= group->group.job_time*64) + if (delta.tv_sec <= group->job_time*64) continue; mx_log_info("killall_over_time(): Sending signal=TERM to job=%s(%d):%lu:%lu pgrp=%d", - group->group.user_name, group->group.user_uid, group->group.group_id, job->job.job_id, pid); + group->user_name, group->user_uid, group->group_id, job->job_id, pid); kill(-pid, SIGTERM); - if (delta.tv_sec <= group->group.job_time*66) + if (delta.tv_sec <= group->job_time*66) continue; mx_log_info("killall_over_time(): Sending signal=KILL to job=%s(%d):%lu:%lu pid=%d", - group->group.user_name, group->group.user_uid, group->group.group_id, job->job.job_id, pid); + group->user_name, group->user_uid, group->group_id, job->job_id, pid); kill(pid, SIGKILL); mx_log_info("killall_over_time(): Sending signal=KILL to job=%s(%d):%lu:%lu pgrp=%d", - group->group.user_name, group->group.user_uid, group->group.group_id, job->job.job_id, pid); + group->user_name, group->user_uid, group->group_id, job->job_id, pid); kill(-pid, SIGKILL); } } From bd347859ab7147009eabe4b8f505bcf842c56141 Mon Sep 17 00:00:00 2001 From: Marius Tolzmann Date: Mon, 2 Nov 2015 14:06:26 +0100 Subject: [PATCH 37/67] mxqd: Cleanup killall() --- mxqd.c | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/mxqd.c b/mxqd.c index 9cfedf3f..0660b58c 100644 --- a/mxqd.c +++ b/mxqd.c @@ -1691,22 +1691,29 @@ void server_close(struct mxq_server *server) int killall(struct mxq_server *server, int sig, unsigned int pgrp) { - struct mxq_user_list *user; - struct mxq_group_list *group; - struct mxq_job_list *job; + struct mxq_user_list *ulist; + struct mxq_group_list *glist; + struct mxq_job_list *jlist; + + struct mxq_group *group; + struct mxq_job *job; + pid_t pid; assert(server); - for (user=server->users; user; user=user->next) { - for (group=user->groups; group; group=group->next) { - for (job=group->jobs; job; job=job->next) { - pid = job->job.host_pid; + for (ulist = server->users; ulist; ulist = ulist->next) { + for (glist = ulist->groups; glist; glist = glist->next) { + group = &glist->group; + + for (jlist = glist->jobs; jlist; jlist = jlist->next) { + job = &jlist->job; + pid = job->host_pid; if (pgrp) pid = -pid; mx_log_info("Sending signal=%d to job=%s(%d):%lu:%lu %s=%d", sig, - group->group.user_name, group->group.user_uid, group->group.group_id, job->job.job_id, + group->user_name, group->user_uid, group->group_id, job->job_id, pgrp?"pgrp":"pid", pid); kill(pid, sig); } From 3a2d802daf6245ec170e70b2a5141a0a389f6457 Mon Sep 17 00:00:00 2001 From: Marius Tolzmann Date: Mon, 2 Nov 2015 14:12:35 +0100 Subject: [PATCH 38/67] mxqd: Rename killallcancelled() to killall_cancelled() and remove sending SIGINT --- mxqd.c | 38 ++++++++++++++++++++++---------------- 1 file changed, 22 insertions(+), 16 deletions(-) diff --git a/mxqd.c b/mxqd.c index 0660b58c..86af9ea8 100644 --- a/mxqd.c +++ b/mxqd.c @@ -1870,31 +1870,39 @@ int killall_over_memory(struct mxq_server *server) return 0; } -int killallcancelled(struct mxq_server *server, int sig, unsigned int pgrp) +int killall_cancelled(struct mxq_server *server, int sig, unsigned int pgrp) { - struct mxq_user_list *user; - struct mxq_group_list *group; - struct mxq_job_list *job; + struct mxq_user_list *ulist; + struct mxq_group_list *glist; + struct mxq_job_list *jlist; + + struct mxq_group *group; + struct mxq_job *job; + pid_t pid; assert(server); - for (user=server->users; user; user=user->next) { - for (group=user->groups; group; group=group->next) { - if (group->group.group_status != MXQ_GROUP_STATUS_CANCELLED) + for (ulist = server->users; ulist; ulist = ulist->next) { + for (glist = ulist->groups; glist; glist = glist->next) { + group = &glist->group; + + if (group->group_status != MXQ_GROUP_STATUS_CANCELLED) continue; - if (group->jobs) + if (glist->jobs) mx_log_debug("Cancelling all running jobs in group=%s(%d):%lu", - group->group.user_name, group->group.user_uid, group->group.group_id); + group->user_name, group->user_uid, group->group_id); - for (job=group->jobs; job; job=job->next) { - pid = job->job.host_pid; + for (jlist = glist->jobs; jlist; jlist = jlist->next) { + job = &jlist->job; + + pid = job->host_pid; if (pgrp) pid = -pid; mx_log_info(" Sending signal=%d to job=%s(%d):%lu:%lu %s=%d", sig, - group->group.user_name, group->group.user_uid, group->group.group_id, job->job.job_id, + group->user_name, group->user_uid, group->group_id, job->job_id, pgrp?"pgrp":"pid", pid); kill(pid, sig); } @@ -2478,8 +2486,7 @@ int main(int argc, char *argv[]) if (group_cnt) mx_log_debug("group_cnt=%d :: %d Groups loaded", group_cnt, group_cnt); - killallcancelled(&server, SIGTERM, 0); - killallcancelled(&server, SIGINT, 0); + killall_cancelled(&server, SIGTERM, 0); killall_over_time(&server); killall_over_memory(&server); @@ -2529,8 +2536,7 @@ int main(int argc, char *argv[]) if (global_sigint_cnt) killall(&server, SIGTERM, 1); - killallcancelled(&server, SIGTERM, 0); - killallcancelled(&server, SIGINT, 0); + killall_cancelled(&server, SIGTERM, 0); killall_over_time(&server); killall_over_memory(&server); mx_log_info("jobs_running=%lu global_sigint_cnt=%d global_sigterm_cnt=%d : Exiting. Wating for jobs to finish. Sleeping for a while.", From 14f772e95ae72a7ebd9a4271a1cc1b9363bf739f Mon Sep 17 00:00:00 2001 From: Marius Tolzmann Date: Mon, 2 Nov 2015 15:26:48 +0100 Subject: [PATCH 39/67] mxqd: Rename server_remove_job_by_pid() to server_remove_job_list_by_pid() --- mxqd.c | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/mxqd.c b/mxqd.c index 86af9ea8..ebe8c68e 100644 --- a/mxqd.c +++ b/mxqd.c @@ -751,15 +751,17 @@ void server_remove_job(struct mxq_job_list *job) { } } -struct mxq_job_list *server_remove_job_by_pid(struct mxq_server *server, pid_t pid) +struct mxq_job_list *server_remove_job_list_by_pid(struct mxq_server *server, pid_t pid) { - struct mxq_job_list *job; + struct mxq_job_list *jlist; - job=server_find_job_by_pid(server,pid); - if (job) { - server_remove_job(job); + assert(server); + + jlist = server_find_job_by_pid(server, pid); + if (jlist) { + server_remove_job(jlist); } - return job; + return jlist; } /**********************************************************************/ @@ -2046,7 +2048,7 @@ static int fspool_process_file(struct mxq_server *server,char *filename,int job_ mx_log_info("job finished (via fspool) : job %d pid %d status %d",job_id,pid,status); - job = server_remove_job_by_pid(server, pid); + job = server_remove_job_list_by_pid(server, pid); if (!job) { mx_log_warning("fspool_process_file: %s : job unknown on server",filename); return(-1); @@ -2157,7 +2159,7 @@ static int lost_scan_one(struct mxq_server *server) if (errno==ESRCH) { if (!fspool_file_exists(server,job_list->job.job_id)) { mx_log_warning("pid %u: process is gone. cancel job %d",job_list->job.host_pid,job_list->job.job_id); - server_remove_job_by_pid(server, job_list->job.host_pid); + server_remove_job_list_by_pid(server, job_list->job.host_pid); job_list->job.job_status=MXQ_JOB_STATUS_UNKNOWN; job_is_lost(server,&group_list->group,job_list); return 1; From 902c6704207ca86a27c30e1d3469a1f0d95f1f8a Mon Sep 17 00:00:00 2001 From: Marius Tolzmann Date: Mon, 2 Nov 2015 15:35:17 +0100 Subject: [PATCH 40/67] mxqd: Rename server_find_job_by_pid() to server_get_job_list_by_pid() --- mxqd.c | 31 +++++++++++++++++++------------ 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/mxqd.c b/mxqd.c index ebe8c68e..3b8aa4bc 100644 --- a/mxqd.c +++ b/mxqd.c @@ -700,21 +700,28 @@ static struct mxq_job_list *server_find_job(struct mxq_server *server,uint64_t return NULL; } -static struct mxq_job_list *server_find_job_by_pid(struct mxq_server *server,pid_t pid) +static struct mxq_job_list *server_get_job_list_by_pid(struct mxq_server *server, pid_t pid) { - struct mxq_user_list *user_list; - struct mxq_group_list *group_list; - struct mxq_job_list *job_list; + struct mxq_user_list *ulist; + struct mxq_group_list *glist; + struct mxq_job_list *jlist; - for (user_list=server->users;user_list;user_list=user_list->next) - for (group_list=user_list->groups;group_list;group_list=group_list->next) - for (job_list=group_list->jobs;job_list;job_list=job_list->next) - if (job_list->job.host_pid==pid) - return job_list; + struct mxq_job *job; + + assert(server); + + for (ulist = server->users; ulist; ulist = ulist->next) { + for (glist = ulist->groups; glist; glist = glist->next) { + for (jlist = glist->jobs; jlist; jlist = jlist->next) { + job = &jlist->job; + if (job->host_pid == pid) + return jlist; + } + } + } return NULL; } - void server_remove_job(struct mxq_job_list *job) { struct mxq_group_list *group=job->group; struct mxq_user_list *user=group->user; @@ -757,7 +764,7 @@ struct mxq_job_list *server_remove_job_list_by_pid(struct mxq_server *server, pi assert(server); - jlist = server_find_job_by_pid(server, pid); + jlist = server_get_job_list_by_pid(server, pid); if (jlist) { server_remove_job(jlist); } @@ -2265,7 +2272,7 @@ int catchall(struct mxq_server *server) { assert(siginfo.si_pid > 1); - job = server_find_job_by_pid(server,siginfo.si_pid); + job = server_get_job_list_by_pid(server,siginfo.si_pid); if (!job) { mx_log_warning("unknown pid returned.. si_pid=%d si_uid=%d si_code=%d si_status=%d getpgid(si_pid)=%d getsid(si_pid)=%d", siginfo.si_pid, siginfo.si_uid, siginfo.si_code, siginfo.si_status, From d13102317d67f8f3642deba2ecccd6d147a3ab10 Mon Sep 17 00:00:00 2001 From: Marius Tolzmann Date: Mon, 2 Nov 2015 16:02:14 +0100 Subject: [PATCH 41/67] mxqd: Rename server_remove_job() to job_list_remove_self() --- mxqd.c | 74 ++++++++++++++++++++++++++++++++++++---------------------- 1 file changed, 46 insertions(+), 28 deletions(-) diff --git a/mxqd.c b/mxqd.c index 3b8aa4bc..88866a07 100644 --- a/mxqd.c +++ b/mxqd.c @@ -722,39 +722,57 @@ static struct mxq_job_list *server_get_job_list_by_pid(struct mxq_server *server return NULL; } -void server_remove_job(struct mxq_job_list *job) { - struct mxq_group_list *group=job->group; - struct mxq_user_list *user=group->user; - struct mxq_server *server=user->server; +void job_list_remove_self(struct mxq_job_list *jlist) +{ + struct mxq_group_list *glist; + struct mxq_user_list *ulist; + struct mxq_server *server; + + struct mxq_job_list **jprevp; + + struct mxq_job *job; + struct mxq_group *group; + + assert(jlist); + assert(jlist->group); + assert(jlist->group->user); + assert(jlist->group->user->server); - struct mxq_job_list **prev; + glist = jlist->group; + ulist = glist->user; + server = ulist->server; - for (prev=&group->jobs;*prev;prev=&(*prev)->next) { - if (*prev==job) { - *prev=job->next; - group->job_cnt--; - user->job_cnt--; - server->job_cnt--; + group = &glist->group; + job = &jlist->job; - group->slots_running -= job->job.host_slots; - user->slots_running -= job->job.host_slots; - server->slots_running -= job->job.host_slots; + for (jprevp = &glist->jobs; *jprevp; jprevp = &(*jprevp)->next) { + if (*jprevp != jlist) + continue; - group->threads_running -= group->group.job_threads; - user->threads_running -= group->group.job_threads; - server->threads_running -= group->group.job_threads; + *jprevp = jlist->next; - group->group.group_jobs_running--; + glist->job_cnt--; + ulist->job_cnt--; + server->job_cnt--; - group->jobs_running--; - user->jobs_running--; - server->jobs_running--; + glist->slots_running -= job->host_slots; + ulist->slots_running -= job->host_slots; + server->slots_running -= job->host_slots; - group->memory_used -= group->group.job_memory; - user->memory_used -= group->group.job_memory; - server->memory_used -= group->group.job_memory; - break; - } + glist->threads_running -= group->job_threads; + ulist->threads_running -= group->job_threads; + server->threads_running -= group->job_threads; + + group->group_jobs_running--; + + glist->jobs_running--; + ulist->jobs_running--; + server->jobs_running--; + + glist->memory_used -= group->job_memory; + ulist->memory_used -= group->job_memory; + server->memory_used -= group->job_memory; + break; } } @@ -766,7 +784,7 @@ struct mxq_job_list *server_remove_job_list_by_pid(struct mxq_server *server, pi jlist = server_get_job_list_by_pid(server, pid); if (jlist) { - server_remove_job(jlist); + job_list_remove_self(jlist); } return jlist; } @@ -2287,7 +2305,7 @@ int catchall(struct mxq_server *server) { continue; } mx_log_err("reaper died. status=%d. Cleaning up job from catchall.",status); - server_remove_job(job); + job_list_remove_self(job); /* reap child and save new state */ pid = wait4(siginfo.si_pid, &status, WNOHANG, &rusage); From b4df08228ea5c3302843f004e3fc3021c1b301e9 Mon Sep 17 00:00:00 2001 From: Marius Tolzmann Date: Mon, 2 Nov 2015 16:13:09 +0100 Subject: [PATCH 42/67] mxqd: Rename server_find_job() to server_get_job_list_by_job_id() --- mxqd.c | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/mxqd.c b/mxqd.c index 88866a07..877a12ed 100644 --- a/mxqd.c +++ b/mxqd.c @@ -686,17 +686,23 @@ static struct mxq_group_list *server_find_group(struct mxq_server *server,uint64 return NULL; } -static struct mxq_job_list *server_find_job(struct mxq_server *server,uint64_t job_id) +static struct mxq_job_list *server_get_job_list_by_job_id(struct mxq_server *server, uint64_t job_id) { - struct mxq_user_list *user_list; - struct mxq_group_list *group_list; - struct mxq_job_list *job_list; + struct mxq_user_list *ulist; + struct mxq_group_list *glist; + struct mxq_job_list *jlist; - for (user_list=server->users;user_list;user_list=user_list->next) - for (group_list=user_list->groups;group_list;group_list=group_list->next) - for (job_list=group_list->jobs;job_list;job_list=job_list->next) - if (job_list->job.job_id==job_id) - return job_list; + struct mxq_job *job; + + for (ulist = server->users; ulist; ulist = ulist->next) { + for (glist = ulist->groups; glist; glist = glist->next) { + for (jlist = glist->jobs; jlist; jlist = jlist->next) { + job = &jlist->job; + if (job->job_id == job_id) + return jlist; + } + } + } return NULL; } @@ -2231,7 +2237,7 @@ static int server_reload_running(struct mxq_server *server) job->stats_starttime.tv_sec=job->date_start; - mxq_job_list=server_find_job(server,job->job_id); + mxq_job_list = server_get_job_list_by_job_id(server, job->job_id); if (!mxq_job_list) { mxq_group_list=server_find_group(server,job->group_id); if (!mxq_group_list) { From a6e7b14f3b9ab7c71004c10469e9a301bddf60b8 Mon Sep 17 00:00:00 2001 From: Marius Tolzmann Date: Mon, 2 Nov 2015 16:19:13 +0100 Subject: [PATCH 43/67] mxqd: Rename server_find_group() to server_get_group_list_by_group_id() --- mxqd.c | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/mxqd.c b/mxqd.c index 877a12ed..4684abab 100644 --- a/mxqd.c +++ b/mxqd.c @@ -674,15 +674,20 @@ static struct mxq_user_list *server_find_user(struct mxq_server *server,uint32_t return NULL; } -static struct mxq_group_list *server_find_group(struct mxq_server *server,uint64_t group_id) +static struct mxq_group_list *server_get_group_list_by_group_id(struct mxq_server *server, uint64_t group_id) { - struct mxq_user_list *user_list; - struct mxq_group_list *group_list; + struct mxq_user_list *ulist; + struct mxq_group_list *glist; - for (user_list=server->users;user_list;user_list=user_list->next) - for (group_list=user_list->groups;group_list;group_list=group_list->next) - if (group_list->group.group_id==group_id) - return group_list; + struct mxq_group *group; + + for (ulist = server->users; ulist; ulist = ulist->next) { + for (glist = ulist->groups; glist; glist = glist->next) { + group = &glist->group; + if (group->group_id == group_id) + return glist; + } + } return NULL; } @@ -2239,7 +2244,7 @@ static int server_reload_running(struct mxq_server *server) mxq_job_list = server_get_job_list_by_job_id(server, job->job_id); if (!mxq_job_list) { - mxq_group_list=server_find_group(server,job->group_id); + mxq_group_list = server_get_group_list_by_group_id(server, job->group_id); if (!mxq_group_list) { struct mxq_group *groups=NULL; struct mxq_group *group; From 19403049e5bb308c9231e02845e5a1cf8556cc15 Mon Sep 17 00:00:00 2001 From: Marius Tolzmann Date: Mon, 2 Nov 2015 16:41:34 +0100 Subject: [PATCH 44/67] mxqd: Cleanup load_groups() --- mxqd.c | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/mxqd.c b/mxqd.c index 4684abab..32f206f5 100644 --- a/mxqd.c +++ b/mxqd.c @@ -2356,27 +2356,35 @@ int catchall(struct mxq_server *server) { return cnt; } -int load_groups(struct mxq_server *server) { - struct mxq_group *mxqgroups = NULL; - struct mxq_group_list *group; - int group_cnt; +int load_groups(struct mxq_server *server) +{ + struct mxq_group_list *glist; + struct mxq_group *grps; + struct mxq_group *group; + + int grp_cnt; int total; int i; + assert(server); + + grps = NULL; + if (RUNNING_AS_ROOT) - group_cnt = mxq_load_running_groups(server->mysql, &mxqgroups); + grp_cnt = mxq_load_running_groups(server->mysql, &grps); else - group_cnt = mxq_load_running_groups_for_user(server->mysql, &mxqgroups, getuid()); + grp_cnt = mxq_load_running_groups_for_user(server->mysql, &grps, getuid()); - for (i=0, total=0; i Date: Mon, 2 Nov 2015 16:54:48 +0100 Subject: [PATCH 45/67] mxqd: Cleanup catchall() --- mxqd.c | 58 ++++++++++++++++++++++++++++++++++++---------------------- 1 file changed, 36 insertions(+), 22 deletions(-) diff --git a/mxqd.c b/mxqd.c index 32f206f5..68247681 100644 --- a/mxqd.c +++ b/mxqd.c @@ -2269,16 +2269,17 @@ static int server_reload_running(struct mxq_server *server) return job_cnt; } -int catchall(struct mxq_server *server) { +int catchall(struct mxq_server *server) +{ + struct mxq_job_list *jlist; + struct mxq_job *job; + struct mxq_group *group; struct rusage rusage; struct timeval now; int status; pid_t pid; int cnt = 0; - struct mxq_job_list *job; - struct mxq_job *j; - struct mxq_group *g; int res; while (1) { @@ -2301,22 +2302,34 @@ int catchall(struct mxq_server *server) { assert(siginfo.si_pid > 1); - job = server_get_job_list_by_pid(server,siginfo.si_pid); - if (!job) { + jlist = server_get_job_list_by_pid(server, siginfo.si_pid); + if (!jlist) { mx_log_warning("unknown pid returned.. si_pid=%d si_uid=%d si_code=%d si_status=%d getpgid(si_pid)=%d getsid(si_pid)=%d", - siginfo.si_pid, siginfo.si_uid, siginfo.si_code, siginfo.si_status, - getpgid(siginfo.si_pid), getsid(siginfo.si_pid)); - pid = waitpid(siginfo.si_pid, &status, WNOHANG); + siginfo.si_pid, + siginfo.si_uid, + siginfo.si_code, + siginfo.si_status, + getpgid(siginfo.si_pid), + getsid(siginfo.si_pid)); + /* collect child, ignore status */ + pid = waitpid(siginfo.si_pid, NULL, WNOHANG); if (pid != siginfo.si_pid) mx_log_err("FIX ME BUG!!! pid=%d errno=%d (%m)", pid, errno); continue; } - if (fspool_file_exists(server,job->job.job_id)) { + + assert(jlist); + assert(jlist->group); + + job = &jlist->job; + group = &jlist->group->group; + + if (fspool_file_exists(server, job->job_id)) { waitpid(siginfo.si_pid, &status, WNOHANG); continue; } mx_log_err("reaper died. status=%d. Cleaning up job from catchall.",status); - job_list_remove_self(job); + job_list_remove_self(jlist); /* reap child and save new state */ pid = wait4(siginfo.si_pid, &status, WNOHANG, &rusage); @@ -2335,22 +2348,23 @@ int catchall(struct mxq_server *server) { gettimeofday(&now, NULL); - j = &job->job; - assert(job->group); - g = &job->group->group; - timersub(&now, &j->stats_starttime, &j->stats_realtime); - j->stats_max_sumrss = job->max_sum_rss; - j->stats_status = status; - j->stats_rusage = rusage; + timersub(&now, &job->stats_starttime, &job->stats_realtime); + job->stats_max_sumrss = jlist->max_sum_rss; + job->stats_status = status; + job->stats_rusage = rusage; mx_log_info(" job=%s(%d):%lu:%lu host_pid=%d stats_status=%d :: child process returned.", - g->user_name, g->user_uid, g->group_id, j->job_id, pid, status); - - fspool_unlink(server,j->job_id); + group->user_name, + group->user_uid, + group->group_id, + job->job_id, + pid, + status); - cnt+=job_has_finished(server,g,job); + fspool_unlink(server, job->job_id); + cnt += job_has_finished(server, group, jlist); } return cnt; From 82a20b4b1ce088b973be64350da5388b5e06440d Mon Sep 17 00:00:00 2001 From: Marius Tolzmann Date: Mon, 2 Nov 2015 17:05:49 +0100 Subject: [PATCH 46/67] mxqd: Cleanup server_reload_running() --- mxqd.c | 64 ++++++++++++++++++++++++++++++---------------------------- 1 file changed, 33 insertions(+), 31 deletions(-) diff --git a/mxqd.c b/mxqd.c index 68247681..a56e912f 100644 --- a/mxqd.c +++ b/mxqd.c @@ -2224,48 +2224,50 @@ static int lost_scan(struct mxq_server *server) static int server_reload_running(struct mxq_server *server) { - int job_cnt; - struct mxq_job *jobs; - int j; + _mx_cleanup_free_ struct mxq_job *jobs = NULL; + + struct mxq_job_list *jlist; + struct mxq_group_list *glist; + struct mxq_user_list *ulist; - struct mxq_job_list *mxq_job_list; - struct mxq_group_list *mxq_group_list; - struct mxq_user_list *mxq_user_list; + struct mxq_group *grps = NULL; + struct mxq_group *group; + struct mxq_job *job; int group_cnt; + int job_cnt; + + int j; - job_cnt=mxq_load_jobs_running_on_server(server->mysql,&jobs,server->hostname,server->server_id); - if (job_cnt<0) + job_cnt = mxq_load_jobs_running_on_server(server->mysql, &jobs, server->hostname, server->server_id); + if (job_cnt < 0) return job_cnt; - for (j=0;jstats_starttime.tv_sec=job->date_start; - - mxq_job_list = server_get_job_list_by_job_id(server, job->job_id); - if (!mxq_job_list) { - mxq_group_list = server_get_group_list_by_group_id(server, job->group_id); - if (!mxq_group_list) { - struct mxq_group *groups=NULL; - struct mxq_group *group; - group_cnt=mxq_load_group(server->mysql,&groups,job->group_id); - if (group_cnt!=1) + + for (j=0; j < job_cnt; j++) { + job = &jobs[j]; + + job->stats_starttime.tv_sec = job->date_start; + + jlist = server_get_job_list_by_job_id(server, job->job_id); + if (!jlist) { + glist = server_get_group_list_by_group_id(server, job->group_id); + if (!glist) { + group_cnt = mxq_load_group(server->mysql, &grps, job->group_id); + if (group_cnt != 1) continue; - group=&groups[0]; - mxq_user_list=server_find_user(server,group->user_uid); - if (!mxq_user_list) { - mxq_group_list = _server_add_group(server,group); + group = &grps[0]; + ulist = server_find_user(server, group->user_uid); + if (!ulist) { + glist = _server_add_group(server, group); } else { - mxq_group_list = _user_list_add_group(mxq_user_list,group); + glist = _user_list_add_group(ulist, group); } - free(groups); + mx_free_null(grps); } - mxq_job_list=mxq_group_list->jobs; + jlist = glist->jobs; } - group_list_add_job(mxq_group_list,job); + group_list_add_job(glist, job); } - - free(jobs); return job_cnt; } From 0ad9f4218f64b899ccd5920959e14265812526c6 Mon Sep 17 00:00:00 2001 From: Marius Tolzmann Date: Mon, 2 Nov 2015 17:13:26 +0100 Subject: [PATCH 47/67] mxqd: Rename lost_scan_one() --- mxqd.c | 50 +++++++++++++++++++++++++++++++------------------- 1 file changed, 31 insertions(+), 19 deletions(-) diff --git a/mxqd.c b/mxqd.c index a56e912f..5d962ebe 100644 --- a/mxqd.c +++ b/mxqd.c @@ -2182,29 +2182,41 @@ static int fspool_file_exists(struct mxq_server *server,uint64_t job_id) { static int lost_scan_one(struct mxq_server *server) { - struct mxq_user_list *user_list; - struct mxq_group_list *group_list; - struct mxq_job_list *job_list; + struct mxq_user_list *ulist; + struct mxq_group_list *glist; + struct mxq_job_list *jlist; + + struct mxq_job *job; + int res; - for (user_list=server->users;user_list;user_list=user_list->next) - for (group_list=user_list->groups;group_list;group_list=group_list->next) - for (job_list=group_list->jobs;job_list;job_list=job_list->next) { - res=kill(job_list->job.host_pid,0); - if (res<0) { - if (errno==ESRCH) { - if (!fspool_file_exists(server,job_list->job.job_id)) { - mx_log_warning("pid %u: process is gone. cancel job %d",job_list->job.host_pid,job_list->job.job_id); - server_remove_job_list_by_pid(server, job_list->job.host_pid); - job_list->job.job_status=MXQ_JOB_STATUS_UNKNOWN; - job_is_lost(server,&group_list->group,job_list); - return 1; - } - } else { - return -errno; - } + for (ulist = server->users; ulist; ulist = ulist->next) { + for (glist = ulist->groups; glist; glist = glist->next) { + for (jlist = glist->jobs; jlist; jlist = jlist->next) { + job = &jlist->job; + res = kill(job->host_pid, 0); + if (res >= 0) + continue; + + /* PID is not */ + + if (errno != ESRCH) + return -errno; + + if (!fspool_file_exists(server, job->job_id)) { + mx_log_warning("pid %u: process is gone. cancel job %d", + jlist->job.host_pid, + jlist->job.job_id); + server_remove_job_list_by_pid(server, job->host_pid); + + job->job_status = MXQ_JOB_STATUS_UNKNOWN; + + job_is_lost(server, &glist->group, jlist); + return 1; } } + } + } return 0; } From e49a90b5776c35b2a544e2d2ac5e00ccf2dfd6b0 Mon Sep 17 00:00:00 2001 From: Marius Tolzmann Date: Mon, 2 Nov 2015 17:23:09 +0100 Subject: [PATCH 48/67] mxqd: Rename server_find_user() to server_find_user_by_uid() --- mxqd.c | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/mxqd.c b/mxqd.c index 5d962ebe..7b96e658 100644 --- a/mxqd.c +++ b/mxqd.c @@ -663,17 +663,6 @@ static void _group_list_init(struct mxq_group_list *glist) glist->orphaned = 0; } -static struct mxq_user_list *server_find_user(struct mxq_server *server,uint32_t uid) -{ - struct mxq_user_list *user_list; - - for (user_list=server->users;user_list;user_list=user_list->next) - if (user_list->groups && user_list->groups[0].group.user_uid==uid) { - return user_list; - } - return NULL; -} - static struct mxq_group_list *server_get_group_list_by_group_id(struct mxq_server *server, uint64_t group_id) { struct mxq_user_list *ulist; @@ -816,6 +805,15 @@ static struct mxq_user_list *_user_list_find_by_uid(struct mxq_user_list *ulist, /**********************************************************************/ +static struct mxq_user_list *server_find_user_by_uid(struct mxq_server *server, uint32_t uid) +{ + assert(server); + + return _user_list_find_by_uid(server->users, uid); +} + +/**********************************************************************/ + struct mxq_group_list *_group_list_find_by_group(struct mxq_group_list *glist, struct mxq_group *group) { assert(group); @@ -2268,7 +2266,7 @@ static int server_reload_running(struct mxq_server *server) if (group_cnt != 1) continue; group = &grps[0]; - ulist = server_find_user(server, group->user_uid); + ulist = server_find_user_by_uid(server, group->user_uid); if (!ulist) { glist = _server_add_group(server, group); } else { From e23bd7292df4c7a0c389f945800d82bd75742744 Mon Sep 17 00:00:00 2001 From: Marius Tolzmann Date: Mon, 2 Nov 2015 17:32:36 +0100 Subject: [PATCH 49/67] mxqd: Cleanup server_dump() --- mxqd.c | 55 ++++++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 40 insertions(+), 15 deletions(-) diff --git a/mxqd.c b/mxqd.c index 7b96e658..90ea6e62 100644 --- a/mxqd.c +++ b/mxqd.c @@ -1664,33 +1664,58 @@ int remove_orphaned_group_lists(struct mxq_server *server) void server_dump(struct mxq_server *server) { - struct mxq_user_list *user; - struct mxq_group_list *group; - struct mxq_job_list *job; + struct mxq_user_list *ulist; + struct mxq_group_list *glist; + struct mxq_job_list *jlist; + + struct mxq_group *group; + struct mxq_job *job; if (!server->user_cnt) return; mx_log_info("====================== SERVER DUMP START ======================"); - for (user=server->users; user; user=user->next) { + for (ulist = server->users; ulist; ulist = ulist->next) { + if (!ulist->groups) { + mx_log_fatal("BUG: missing group in userlist."); + continue; + } + group = &ulist->groups[0].group; mx_log_info(" user=%s(%d) slots_running=%lu", - user->groups->group.user_name, user->groups->group.user_uid, - user->slots_running); - for (group=user->groups; group; group=group->next) { + group->user_name, + group->user_uid, + ulist->slots_running); + + for (glist = ulist->groups; glist; glist = glist->next) { + group = &glist->group; mx_log_info(" group=%s(%d):%lu %s jobs_in_q=%lu", - group->group.user_name, group->group.user_uid, group->group.group_id, - group->group.group_name, mxq_group_jobs_inq(&group->group)); - for (job=group->jobs; job; job=job->next) { + group->user_name, + group->user_uid, + group->group_id, + group->group_name, + mxq_group_jobs_inq(group)); + for (jlist = glist->jobs; jlist; jlist = jlist->next) { + job = &jlist->job; mx_log_info(" job=%s(%d):%lu:%lu %s", - group->group.user_name, group->group.user_uid, group->group.group_id, job->job.job_id, - job->job.job_argv_str); + group->user_name, + group->user_uid, + group->group_id, + job->job_id, + job->job_argv_str); } } } - mx_log_info("memory_used=%lu memory_total=%lu", server->memory_used, server->memory_total); - mx_log_info("slots_running=%lu slots=%lu threads_running=%lu jobs_running=%lu", server->slots_running, server->slots, server->threads_running, server->jobs_running); - cpuset_log("cpu set running",&server->cpu_set_running); + mx_log_info("memory_used=%lu memory_total=%lu", + server->memory_used, + server->memory_total); + mx_log_info("slots_running=%lu slots=%lu threads_running=%lu jobs_running=%lu", + server->slots_running, + server->slots, + server->threads_running, + server->jobs_running); + cpuset_log("cpu set running", + &server->cpu_set_running); mx_log_info("====================== SERVER DUMP END ======================"); } From 0fe2e5239851fb0b71bba9a6320772f86363f056 Mon Sep 17 00:00:00 2001 From: Marius Tolzmann Date: Mon, 2 Nov 2015 17:37:33 +0100 Subject: [PATCH 50/67] mxqd: Cleanup server_close() --- mxqd.c | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/mxqd.c b/mxqd.c index 90ea6e62..1cdc6f39 100644 --- a/mxqd.c +++ b/mxqd.c @@ -1721,23 +1721,23 @@ void server_dump(struct mxq_server *server) void server_close(struct mxq_server *server) { - struct mxq_user_list *user, *unext; - struct mxq_group_list *group, *gnext; - struct mxq_job_list *job, *jnext; - - for (user=server->users; user; user=unext) { - for (group=user->groups; group; group=gnext) { - for (job=group->jobs; job; job=jnext) { - jnext = job->next; - mxq_job_free_content(&job->job); - free(job); + struct mxq_user_list *ulist, *unext; + struct mxq_group_list *glist, *gnext; + struct mxq_job_list *jlist, *jnext; + + for (ulist = server->users; ulist; ulist = unext) { + for (glist = ulist->groups; glist; glist = gnext) { + for (jlist = glist->jobs; jlist; jlist = jnext) { + jnext = jlist->next; + mxq_job_free_content(&jlist->job); + mx_free_null(jlist); } - gnext = group->next; - mxq_group_free_content(&group->group); - free(group); + gnext = glist->next; + mxq_group_free_content(&glist->group); + mx_free_null(glist); } - unext = user->next; - free(user); + unext = ulist->next; + mx_free_null(ulist); } if (server->pidfilename) From 4faf2b6145043bd1b14b6a8ee23b7fc7fca34c8d Mon Sep 17 00:00:00 2001 From: Marius Tolzmann Date: Mon, 2 Nov 2015 17:48:17 +0100 Subject: [PATCH 51/67] mxqd: Cleanup start_job() --- mxqd.c | 63 ++++++++++++++++++++++++++++++++++------------------------ 1 file changed, 37 insertions(+), 26 deletions(-) diff --git a/mxqd.c b/mxqd.c index 1cdc6f39..4664f7d0 100644 --- a/mxqd.c +++ b/mxqd.c @@ -1376,31 +1376,38 @@ int reaper_process(struct mxq_server *server,struct mxq_group_list *group,struc return(0); } -unsigned long start_job(struct mxq_group_list *group) +unsigned long start_job(struct mxq_group_list *glist) { struct mxq_server *server; - struct mxq_job mxqjob; - struct mxq_job_list *job; + struct mxq_job_list *jlist; + + struct mxq_job _mxqjob; + struct mxq_job *job; + + struct mxq_group *group; + pid_t pid; int res; - assert(group); - assert(group->user); - assert(group->user->server); - - server = group->user->server; + assert(glist); + assert(glist->user); + assert(glist->user->server); - res = mxq_load_job_from_group_for_server(server->mysql, &mxqjob, group->group.group_id, server->hostname, server->server_id, server->host_id); + server = glist->user->server; + group = &glist->group; + job = &_mxqjob; + res = mxq_load_job_from_group_for_server(server->mysql, job, group->group_id, server->hostname, server->server_id, server->host_id); if (!res) { return 0; } mx_log_info(" job=%s(%d):%lu:%lu :: new job loaded.", - group->group.user_name, group->group.user_uid, group->group.group_id, mxqjob.job_id); + group->user_name, group->user_uid, group->group_id, job->job_id); + + cpuset_init_job(&job->host_cpu_set, &server->cpu_set_available, &server->cpu_set_running, glist->slots_per_job); + job->host_cpu_set_str = mx_cpuset_to_str(&job->host_cpu_set); - cpuset_init_job(&mxqjob.host_cpu_set,&server->cpu_set_available,&server->cpu_set_running,group->slots_per_job); - mxqjob.host_cpu_set_str=mx_cpuset_to_str(&mxqjob.host_cpu_set); - mx_log_info("job assigned cpus: [%s]",mxqjob.host_cpu_set_str); + mx_log_info("job assigned cpus: [%s]", job->host_cpu_set_str); mx_mysql_disconnect(server->mysql); @@ -1409,35 +1416,39 @@ unsigned long start_job(struct mxq_group_list *group) mx_log_err("fork: %m"); return 0; } else if (pid == 0) { - mxqjob.host_pid = getpid(); + job->host_pid = getpid(); mx_log_info(" job=%s(%d):%lu:%lu host_pid=%d pgrp=%d :: new child process forked.", - group->group.user_name, group->group.user_uid, group->group.group_id, mxqjob.job_id, - mxqjob.host_pid, getpgrp()); + group->user_name, + group->user_uid, + group->group_id, + job->job_id, + job->host_pid, + getpgrp()); - res=reaper_process(server,group,&mxqjob); + res = reaper_process(server, glist, job); _exit(res<0 ? EX__MAX+1 : 0); } - gettimeofday(&mxqjob.stats_starttime, NULL); + gettimeofday(&job->stats_starttime, NULL); mx_mysql_connect_forever(&(server->mysql)); - mxqjob.host_pid = pid; - mxqjob.host_slots = group->slots_per_job; - res = mxq_set_job_status_running(server->mysql, &mxqjob); + job->host_pid = pid; + job->host_slots = glist->slots_per_job; + res = mxq_set_job_status_running(server->mysql, job); if (res < 0) mx_log_err("job=%s(%d):%lu:%lu mxq_job_update_status_running(): %m", - group->group.user_name, group->group.user_uid, group->group.group_id, mxqjob.job_id); + group->user_name, group->user_uid, group->group_id, job->job_id); if (res == 0) mx_log_err("job=%s(%d):%lu:%lu mxq_job_update_status_running(): Job not found.", - group->group.user_name, group->group.user_uid, group->group.group_id, mxqjob.job_id); + group->user_name, group->user_uid, group->group_id, job->job_id); - job = group_list_add_job(group, &mxqjob); - assert(job); + jlist = group_list_add_job(glist, job); + assert(jlist); mx_log_info(" job=%s(%d):%lu:%lu :: added running job to watch queue.", - group->group.user_name, group->group.user_uid, group->group.group_id, mxqjob.job_id); + group->user_name, group->user_uid, group->group_id, job->job_id); return 1; } From 3c419e03a89cf3629cfed5a6e50d5d1907dbf9cc Mon Sep 17 00:00:00 2001 From: Marius Tolzmann Date: Mon, 2 Nov 2015 22:49:15 +0100 Subject: [PATCH 52/67] mxqd_control: Refactor and export server structure management --- .gitignore | 6 +- Makefile | 29 ++++ mxqd.c | 394 +------------------------------------------- mxqd.h | 4 + mxqd_control.c | 385 +++++++++++++++++++++++++++++++++++++++++++ mxqd_control.h | 29 ++++ test_mxqd_control.c | 15 ++ 7 files changed, 469 insertions(+), 393 deletions(-) create mode 100644 mxqd_control.c create mode 100644 mxqd_control.h create mode 100644 test_mxqd_control.c diff --git a/.gitignore b/.gitignore index 4509600b..b7570cea 100644 --- a/.gitignore +++ b/.gitignore @@ -14,8 +14,11 @@ mxqkill.o test_mx_util.o test_mx_log.o test_mx_mysql.o +test_mxqd_control.o mxq_log.o mx_mysql.o +mxqd_control.o + mxqsub /mxqsub.1 @@ -26,7 +29,8 @@ mxqd mxqps test_mx_util test_mx_log -test_mx_mysql +test_mx_mysq +test_mxqd_control /web/pages/mxq/mxq web/lighttpd.conf diff --git a/Makefile b/Makefile index 19998d41..e38d03cb 100644 --- a/Makefile +++ b/Makefile @@ -300,6 +300,13 @@ mxq_job.h += mxq_group.h mxqd.h += mxqd.h +### mxqd_conrol.h ------------------------------------------------------ + +mxqd_control.h += mxqd_control.h +mxqd_control.h += mxq_group.h +mxqd_control.h += mxq_job.h +mxqd_control.h += mxqd.h + ### mx_getopt.h -------------------------------------------------------- mx_getopt.h += mx_getopt.h @@ -406,6 +413,12 @@ mxq_job.o: CFLAGS += $(CFLAGS_MYSQL) clean: CLEAN += mxq_job.o +### mxqd_control.o ----------------------------------------------------- + +mxqd_control.o: $(mxqd_control.h) + +clean: CLEAN += mxqd_control.o + ### mxqd.o ------------------------------------------------------------- mxqd.o: $(mx_getopt.h) @@ -452,6 +465,7 @@ mxqd: mx_getopt.o mxqd: mxq_group.o mxqd: mxq_job.o mxqd: mx_mysql.o +mxqd: mxqd_control.o mxqd: LDLIBS += $(LDLIBS_MYSQL) build: mxqd @@ -600,3 +614,18 @@ test_mx_mysql: mx_log.o test_mx_mysql: mx_util.o test_mx_mysql: LDLIBS += $(LDLIBS_MYSQL) clean: CLEAN += test_mx_mysql + + +test_mxqd_control.o: $(mxqd_control.h) +clean: CLEAN += test_mxqd_control.o + +test_mxqd_control: mxqd_control.o +test_mxqd_control: mx_log.o +test_mxqd_control: mx_util.o +test_mxqd_control: mx_mysql.o +test_mxqd_control: mxq_group.o +test_mxqd_control: LDLIBS += $(LDLIBS_MYSQL) + +clean: CLEAN += test_mxqd_control + +test: test_mxqd_control diff --git a/mxqd.c b/mxqd.c index f2d91896..8bcb8bbf 100644 --- a/mxqd.c +++ b/mxqd.c @@ -41,6 +41,8 @@ #include "mxqd.h" #include "mxq.h" +#include "mxqd_control.h" + #ifndef MXQ_INITIAL_PATH # define MXQ_INITIAL_PATH "/sbin:/bin:/usr/sbin:/usr/bin:/usr/local/sbin:/usr/local/bin" #endif @@ -581,398 +583,6 @@ int server_init(struct mxq_server *server, int argc, char *argv[]) return 1; } -/**********************************************************************/ - -static void _group_list_init(struct mxq_group_list *glist) -{ - struct mxq_server *server; - struct mxq_group *group; - - long double memory_threads; - long double memory_per_thread; - long double memory_max_available; - - unsigned long slots_per_job; - unsigned long jobs_max; - unsigned long slots_max; - unsigned long memory_max; - - assert(glist); - assert(glist->user); - assert(glist->user->server); - - server = glist->user->server; - group = &glist->group; - - memory_per_thread = (long double)group->job_memory / (long double)group->job_threads; - memory_max_available = (long double)server->memory_total * (long double)server->memory_max_per_slot / memory_per_thread; - - if (memory_max_available > server->memory_total) - memory_max_available = server->memory_total; - - slots_per_job = ceill((long double)group->job_memory / server->memory_avg_per_slot); - - if (slots_per_job < group->job_threads) - slots_per_job = group->job_threads; - - memory_threads = memory_max_available / memory_per_thread; - - if (memory_per_thread > server->memory_max_per_slot) { - jobs_max = memory_threads + 0.5; - } else if (memory_per_thread > server->memory_avg_per_slot) { - jobs_max = memory_threads + 0.5; - } else { - jobs_max = server->slots; - } - jobs_max /= group->job_threads; - - /* limit maximum number of jobs on user/group request */ - if (group->job_max_per_node && jobs_max > group->job_max_per_node) - jobs_max = group->job_max_per_node; - - slots_max = jobs_max * slots_per_job; - memory_max = jobs_max * group->job_memory; - - if (glist->memory_per_thread != memory_per_thread - || glist->memory_max_available != memory_max_available - || glist->memory_max_available != memory_max_available - || glist->slots_per_job != slots_per_job - || glist->jobs_max != jobs_max - || glist->slots_max != slots_max - || glist->memory_max != memory_max) { - mx_log_info(" group=%s(%u):%lu jobs_max=%lu slots_max=%lu memory_max=%lu slots_per_job=%lu :: group %sinitialized.", - group->user_name, - group->user_uid, - group->group_id, - jobs_max, - slots_max, - memory_max, - slots_per_job, - glist->orphaned ? "re" : ""); - } - - glist->memory_per_thread = memory_per_thread; - glist->memory_max_available = memory_max_available; - - glist->slots_per_job = slots_per_job; - - glist->jobs_max = jobs_max; - glist->slots_max = slots_max; - glist->memory_max = memory_max; - - glist->orphaned = 0; -} - -static struct mxq_group_list *server_get_group_list_by_group_id(struct mxq_server *server, uint64_t group_id) -{ - struct mxq_user_list *ulist; - struct mxq_group_list *glist; - - struct mxq_group *group; - - for (ulist = server->users; ulist; ulist = ulist->next) { - for (glist = ulist->groups; glist; glist = glist->next) { - group = &glist->group; - if (group->group_id == group_id) - return glist; - } - } - return NULL; -} - -static struct mxq_job_list *server_get_job_list_by_job_id(struct mxq_server *server, uint64_t job_id) -{ - struct mxq_user_list *ulist; - struct mxq_group_list *glist; - struct mxq_job_list *jlist; - - struct mxq_job *job; - - for (ulist = server->users; ulist; ulist = ulist->next) { - for (glist = ulist->groups; glist; glist = glist->next) { - for (jlist = glist->jobs; jlist; jlist = jlist->next) { - job = &jlist->job; - if (job->job_id == job_id) - return jlist; - } - } - } - return NULL; -} - -static struct mxq_job_list *server_get_job_list_by_pid(struct mxq_server *server, pid_t pid) -{ - struct mxq_user_list *ulist; - struct mxq_group_list *glist; - struct mxq_job_list *jlist; - - struct mxq_job *job; - - assert(server); - - for (ulist = server->users; ulist; ulist = ulist->next) { - for (glist = ulist->groups; glist; glist = glist->next) { - for (jlist = glist->jobs; jlist; jlist = jlist->next) { - job = &jlist->job; - if (job->host_pid == pid) - return jlist; - } - } - } - return NULL; -} - -void job_list_remove_self(struct mxq_job_list *jlist) -{ - struct mxq_group_list *glist; - struct mxq_user_list *ulist; - struct mxq_server *server; - - struct mxq_job_list **jprevp; - - struct mxq_job *job; - struct mxq_group *group; - - assert(jlist); - assert(jlist->group); - assert(jlist->group->user); - assert(jlist->group->user->server); - - glist = jlist->group; - ulist = glist->user; - server = ulist->server; - - group = &glist->group; - job = &jlist->job; - - for (jprevp = &glist->jobs; *jprevp; jprevp = &(*jprevp)->next) { - if (*jprevp != jlist) - continue; - - *jprevp = jlist->next; - - glist->job_cnt--; - ulist->job_cnt--; - server->job_cnt--; - - glist->slots_running -= job->host_slots; - ulist->slots_running -= job->host_slots; - server->slots_running -= job->host_slots; - - glist->threads_running -= group->job_threads; - ulist->threads_running -= group->job_threads; - server->threads_running -= group->job_threads; - - group->group_jobs_running--; - - glist->jobs_running--; - ulist->jobs_running--; - server->jobs_running--; - - glist->memory_used -= group->job_memory; - ulist->memory_used -= group->job_memory; - server->memory_used -= group->job_memory; - break; - } -} - -struct mxq_job_list *server_remove_job_list_by_pid(struct mxq_server *server, pid_t pid) -{ - struct mxq_job_list *jlist; - - assert(server); - - jlist = server_get_job_list_by_pid(server, pid); - if (jlist) { - job_list_remove_self(jlist); - } - return jlist; -} - -/**********************************************************************/ - -static struct mxq_user_list *_user_list_find_by_uid(struct mxq_user_list *ulist, uint32_t uid) -{ - for (; ulist; ulist = ulist->next) { - assert(ulist->groups); - - if (ulist->groups[0].group.user_uid == uid) { - return ulist; - } - } - return NULL; -} - -/**********************************************************************/ - -static struct mxq_user_list *server_find_user_by_uid(struct mxq_server *server, uint32_t uid) -{ - assert(server); - - return _user_list_find_by_uid(server->users, uid); -} - -/**********************************************************************/ - -struct mxq_group_list *_group_list_find_by_group(struct mxq_group_list *glist, struct mxq_group *group) -{ - assert(group); - - for (; glist; glist = glist->next) { - if (glist->group.group_id == group->group_id) { - return glist; - } - } - return NULL; -} - -/**********************************************************************/ - -struct mxq_job_list *group_list_add_job(struct mxq_group_list *glist, struct mxq_job *job) -{ - struct mxq_server *server; - - struct mxq_job_list *jlist; - struct mxq_user_list *ulist; - - struct mxq_group *group; - - assert(glist); - assert(glist->user); - assert(glist->user->server); - assert(job->job_status == MXQ_JOB_STATUS_RUNNING); - - group = &glist->group; - ulist = glist->user; - server = ulist->server; - - jlist = mx_calloc_forever(1, sizeof(*jlist)); - - memcpy(&jlist->job, job, sizeof(*job)); - - jlist->group = glist; - - jlist->next = glist->jobs; - glist->jobs = jlist; - - glist->job_cnt++; - ulist->job_cnt++; - server->job_cnt++; - - glist->slots_running += glist->slots_per_job; - ulist->slots_running += glist->slots_per_job; - server->slots_running += glist->slots_per_job; - - glist->threads_running += group->job_threads; - ulist->threads_running += group->job_threads; - server->threads_running += group->job_threads; - - CPU_OR(&server->cpu_set_running, &server->cpu_set_running, &job->host_cpu_set); - - group->group_jobs_running++; - group->group_jobs_inq--; - - glist->jobs_running++; - ulist->jobs_running++; - server->jobs_running++; - - glist->memory_used += group->job_memory; - ulist->memory_used += group->job_memory; - server->memory_used += group->job_memory; - - return jlist; -} -/**********************************************************************/ - -static struct mxq_group_list *_user_list_add_group(struct mxq_user_list *ulist, struct mxq_group *group) -{ - struct mxq_group_list *glist; - struct mxq_server *server; - - assert(ulist); - assert(ulist->server); - - server = ulist->server; - - glist = mx_calloc_forever(1, sizeof(*glist)); - - memcpy(&glist->group, group, sizeof(*group)); - - glist->user = ulist; - - glist->next = ulist->groups; - ulist->groups = glist; - - ulist->group_cnt++; - server->group_cnt++; - - _group_list_init(glist); - - return glist; -} - -/**********************************************************************/ - -static struct mxq_group_list *_server_add_group(struct mxq_server *server, struct mxq_group *group) -{ - struct mxq_user_list *ulist; - struct mxq_group_list *glist; - - assert(server); - assert(group); - - ulist = mx_calloc_forever(1, sizeof(*ulist)); - - ulist->server = server; - - ulist->next = server->users; - server->users = ulist; - - server->user_cnt++; - - glist = _user_list_add_group(ulist, group); - assert(glist); - - return glist; -} - -/**********************************************************************/ - -static struct mxq_group_list *_user_list_update_group(struct mxq_user_list *ulist, struct mxq_group *group) -{ - struct mxq_group_list *glist; - - assert(ulist); - assert(group); - - glist = _group_list_find_by_group(ulist->groups, group); - if (!glist) { - return _user_list_add_group(ulist, group); - } - - mxq_group_free_content(&glist->group); - - memcpy(&glist->group, group, sizeof(*group)); - - _group_list_init(glist); - - return glist; -} - -/**********************************************************************/ - -static struct mxq_group_list *server_update_group(struct mxq_server *server, struct mxq_group *group) -{ - struct mxq_user_list *ulist; - - ulist = _user_list_find_by_uid(server->users, group->user_uid); - if (!ulist) { - return _server_add_group(server, group); - } - - return _user_list_update_group(ulist, group); -} - static void reset_signals() { signal(SIGINT, SIG_DFL); diff --git a/mxqd.h b/mxqd.h index 7e56c610..8b115a99 100644 --- a/mxqd.h +++ b/mxqd.h @@ -2,6 +2,10 @@ #define __MXQ_SERVER_H__ 1 #include "mx_mysql.h" + +#include "mxq_job.h" +#include "mxq_group.h" + #include struct mxq_job_list { diff --git a/mxqd_control.c b/mxqd_control.c new file mode 100644 index 00000000..9055f434 --- /dev/null +++ b/mxqd_control.c @@ -0,0 +1,385 @@ +#define _GNU_SOURCE + +#include +#include +#include + +#include "mxq_job.h" +#include "mxq_group.h" + +#include "mxqd.h" + +static void _group_list_init(struct mxq_group_list *glist) +{ + struct mxq_server *server; + struct mxq_group *group; + + long double memory_threads; + long double memory_per_thread; + long double memory_max_available; + + unsigned long slots_per_job; + unsigned long jobs_max; + unsigned long slots_max; + unsigned long memory_max; + + assert(glist); + assert(glist->user); + assert(glist->user->server); + + server = glist->user->server; + group = &glist->group; + + memory_per_thread = (long double)group->job_memory / (long double)group->job_threads; + memory_max_available = (long double)server->memory_total * (long double)server->memory_max_per_slot / memory_per_thread; + + if (memory_max_available > server->memory_total) + memory_max_available = server->memory_total; + + slots_per_job = ceill((long double)group->job_memory / server->memory_avg_per_slot); + + if (slots_per_job < group->job_threads) + slots_per_job = group->job_threads; + + memory_threads = memory_max_available / memory_per_thread; + + if (memory_per_thread > server->memory_max_per_slot) { + jobs_max = memory_threads + 0.5; + } else if (memory_per_thread > server->memory_avg_per_slot) { + jobs_max = memory_threads + 0.5; + } else { + jobs_max = server->slots; + } + jobs_max /= group->job_threads; + + /* limit maximum number of jobs on user/group request */ + if (group->job_max_per_node && jobs_max > group->job_max_per_node) + jobs_max = group->job_max_per_node; + + slots_max = jobs_max * slots_per_job; + memory_max = jobs_max * group->job_memory; + + if (glist->memory_per_thread != memory_per_thread + || glist->memory_max_available != memory_max_available + || glist->memory_max_available != memory_max_available + || glist->slots_per_job != slots_per_job + || glist->jobs_max != jobs_max + || glist->slots_max != slots_max + || glist->memory_max != memory_max) { + mx_log_info(" group=%s(%u):%lu jobs_max=%lu slots_max=%lu memory_max=%lu slots_per_job=%lu :: group %sinitialized.", + group->user_name, + group->user_uid, + group->group_id, + jobs_max, + slots_max, + memory_max, + slots_per_job, + glist->orphaned ? "re" : ""); + } + + glist->memory_per_thread = memory_per_thread; + glist->memory_max_available = memory_max_available; + + glist->slots_per_job = slots_per_job; + + glist->jobs_max = jobs_max; + glist->slots_max = slots_max; + glist->memory_max = memory_max; + + glist->orphaned = 0; +} + +struct mxq_group_list *server_get_group_list_by_group_id(struct mxq_server *server, uint64_t group_id) +{ + struct mxq_user_list *ulist; + struct mxq_group_list *glist; + + struct mxq_group *group; + + for (ulist = server->users; ulist; ulist = ulist->next) { + for (glist = ulist->groups; glist; glist = glist->next) { + group = &glist->group; + if (group->group_id == group_id) + return glist; + } + } + return NULL; +} + +struct mxq_job_list *server_get_job_list_by_job_id(struct mxq_server *server, uint64_t job_id) +{ + struct mxq_user_list *ulist; + struct mxq_group_list *glist; + struct mxq_job_list *jlist; + + struct mxq_job *job; + + for (ulist = server->users; ulist; ulist = ulist->next) { + for (glist = ulist->groups; glist; glist = glist->next) { + for (jlist = glist->jobs; jlist; jlist = jlist->next) { + job = &jlist->job; + if (job->job_id == job_id) + return jlist; + } + } + } + return NULL; +} + +struct mxq_job_list *server_get_job_list_by_pid(struct mxq_server *server, pid_t pid) +{ + struct mxq_user_list *ulist; + struct mxq_group_list *glist; + struct mxq_job_list *jlist; + + struct mxq_job *job; + + assert(server); + + for (ulist = server->users; ulist; ulist = ulist->next) { + for (glist = ulist->groups; glist; glist = glist->next) { + for (jlist = glist->jobs; jlist; jlist = jlist->next) { + job = &jlist->job; + if (job->host_pid == pid) + return jlist; + } + } + } + return NULL; +} + +void job_list_remove_self(struct mxq_job_list *jlist) +{ + struct mxq_group_list *glist; + struct mxq_user_list *ulist; + struct mxq_server *server; + + struct mxq_job_list **jprevp; + + struct mxq_job *job; + struct mxq_group *group; + + assert(jlist); + assert(jlist->group); + assert(jlist->group->user); + assert(jlist->group->user->server); + + glist = jlist->group; + ulist = glist->user; + server = ulist->server; + + group = &glist->group; + job = &jlist->job; + + for (jprevp = &glist->jobs; *jprevp; jprevp = &(*jprevp)->next) { + if (*jprevp != jlist) + continue; + + *jprevp = jlist->next; + + glist->job_cnt--; + ulist->job_cnt--; + server->job_cnt--; + + glist->slots_running -= job->host_slots; + ulist->slots_running -= job->host_slots; + server->slots_running -= job->host_slots; + + glist->threads_running -= group->job_threads; + ulist->threads_running -= group->job_threads; + server->threads_running -= group->job_threads; + + group->group_jobs_running--; + + glist->jobs_running--; + ulist->jobs_running--; + server->jobs_running--; + + glist->memory_used -= group->job_memory; + ulist->memory_used -= group->job_memory; + server->memory_used -= group->job_memory; + break; + } +} + +struct mxq_job_list *server_remove_job_list_by_pid(struct mxq_server *server, pid_t pid) +{ + struct mxq_job_list *jlist; + + assert(server); + + jlist = server_get_job_list_by_pid(server, pid); + if (jlist) { + job_list_remove_self(jlist); + } + return jlist; +} + +static struct mxq_user_list *_user_list_find_by_uid(struct mxq_user_list *ulist, uint32_t uid) +{ + for (; ulist; ulist = ulist->next) { + assert(ulist->groups); + + if (ulist->groups[0].group.user_uid == uid) { + return ulist; + } + } + return NULL; +} + +struct mxq_user_list *server_find_user_by_uid(struct mxq_server *server, uint32_t uid) +{ + assert(server); + + return _user_list_find_by_uid(server->users, uid); +} + +struct mxq_group_list *_group_list_find_by_group(struct mxq_group_list *glist, struct mxq_group *group) +{ + assert(group); + + for (; glist; glist = glist->next) { + if (glist->group.group_id == group->group_id) { + return glist; + } + } + return NULL; +} + +struct mxq_job_list *group_list_add_job(struct mxq_group_list *glist, struct mxq_job *job) +{ + struct mxq_server *server; + + struct mxq_job_list *jlist; + struct mxq_user_list *ulist; + + struct mxq_group *group; + + assert(glist); + assert(glist->user); + assert(glist->user->server); + assert(job->job_status == MXQ_JOB_STATUS_RUNNING); + + group = &glist->group; + ulist = glist->user; + server = ulist->server; + + jlist = mx_calloc_forever(1, sizeof(*jlist)); + + memcpy(&jlist->job, job, sizeof(*job)); + + jlist->group = glist; + + jlist->next = glist->jobs; + glist->jobs = jlist; + + glist->job_cnt++; + ulist->job_cnt++; + server->job_cnt++; + + glist->slots_running += glist->slots_per_job; + ulist->slots_running += glist->slots_per_job; + server->slots_running += glist->slots_per_job; + + glist->threads_running += group->job_threads; + ulist->threads_running += group->job_threads; + server->threads_running += group->job_threads; + + CPU_OR(&server->cpu_set_running, &server->cpu_set_running, &job->host_cpu_set); + + group->group_jobs_running++; + group->group_jobs_inq--; + + glist->jobs_running++; + ulist->jobs_running++; + server->jobs_running++; + + glist->memory_used += group->job_memory; + ulist->memory_used += group->job_memory; + server->memory_used += group->job_memory; + + return jlist; +} + +struct mxq_group_list *_user_list_add_group(struct mxq_user_list *ulist, struct mxq_group *group) +{ + struct mxq_group_list *glist; + struct mxq_server *server; + + assert(ulist); + assert(ulist->server); + + server = ulist->server; + + glist = mx_calloc_forever(1, sizeof(*glist)); + + memcpy(&glist->group, group, sizeof(*group)); + + glist->user = ulist; + + glist->next = ulist->groups; + ulist->groups = glist; + + ulist->group_cnt++; + server->group_cnt++; + + _group_list_init(glist); + + return glist; +} + +struct mxq_group_list *_server_add_group(struct mxq_server *server, struct mxq_group *group) +{ + struct mxq_user_list *ulist; + struct mxq_group_list *glist; + + assert(server); + assert(group); + + ulist = mx_calloc_forever(1, sizeof(*ulist)); + + ulist->server = server; + + ulist->next = server->users; + server->users = ulist; + + server->user_cnt++; + + glist = _user_list_add_group(ulist, group); + assert(glist); + + return glist; +} + +static struct mxq_group_list *_user_list_update_group(struct mxq_user_list *ulist, struct mxq_group *group) +{ + struct mxq_group_list *glist; + + assert(ulist); + assert(group); + + glist = _group_list_find_by_group(ulist->groups, group); + if (!glist) { + return _user_list_add_group(ulist, group); + } + + mxq_group_free_content(&glist->group); + + memcpy(&glist->group, group, sizeof(*group)); + + _group_list_init(glist); + + return glist; +} + +struct mxq_group_list *server_update_group(struct mxq_server *server, struct mxq_group *group) +{ + struct mxq_user_list *ulist; + + ulist = _user_list_find_by_uid(server->users, group->user_uid); + if (!ulist) { + return _server_add_group(server, group); + } + + return _user_list_update_group(ulist, group); +} diff --git a/mxqd_control.h b/mxqd_control.h new file mode 100644 index 00000000..3a8d69e9 --- /dev/null +++ b/mxqd_control.h @@ -0,0 +1,29 @@ +#ifndef _MXQD_CONTROL_H +#define _MXQD_CONTROL_H + +#include "mxqd.h" +#include "mxq_group.h" +#include "mxq_job.h" + +void job_list_remove_self(struct mxq_job_list *jlist); +struct mxq_job_list *server_remove_job_list_by_pid(struct mxq_server *server, pid_t pid); +struct mxq_user_list *server_find_user_by_uid(struct mxq_server *server, uint32_t uid); +struct mxq_group_list *_group_list_find_by_group(struct mxq_group_list *glist, struct mxq_group *group); +struct mxq_job_list *group_list_add_job(struct mxq_group_list *glist, struct mxq_job *job); +struct mxq_group_list *server_update_group(struct mxq_server *server, struct mxq_group *group); + +struct mxq_group_list *server_get_group_list_by_group_id(struct mxq_server *server, uint64_t group_id); +struct mxq_job_list *server_get_job_list_by_job_id(struct mxq_server *server, uint64_t job_id); +struct mxq_job_list *server_get_job_list_by_pid(struct mxq_server *server, pid_t pid); + +/* +static void _group_list_init(struct mxq_group_list *glist) +static struct mxq_user_list *_user_list_find_by_uid(struct mxq_user_list *ulist, uint32_t uid) + +static struct mxq_group_list *_user_list_update_group(struct mxq_user_list *ulist, struct mxq_group *group) +*/ + +struct mxq_group_list *_server_add_group(struct mxq_server *server, struct mxq_group *group); +struct mxq_group_list *_user_list_add_group(struct mxq_user_list *ulist, struct mxq_group *group); + +#endif diff --git a/test_mxqd_control.c b/test_mxqd_control.c new file mode 100644 index 00000000..8b96e4df --- /dev/null +++ b/test_mxqd_control.c @@ -0,0 +1,15 @@ + +#define _GNU_SOURCE + +#include + +static void test_mxqd_control(void) +{ + assert(1); +} + +int main(int argc, char *argv[]) +{ + test_mxqd_control(); + return 0; +} From dde96c351da506667c7220b8ea01c2761f469c50 Mon Sep 17 00:00:00 2001 From: Marius Tolzmann Date: Tue, 3 Nov 2015 12:33:15 +0100 Subject: [PATCH 53/67] mxqd: Rename load_groups() to load_running_groups() --- mxqd.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mxqd.c b/mxqd.c index 8bcb8bbf..e669d75b 100644 --- a/mxqd.c +++ b/mxqd.c @@ -2049,7 +2049,7 @@ int catchall(struct mxq_server *server) return cnt; } -int load_groups(struct mxq_server *server) +int load_running_groups(struct mxq_server *server) { struct mxq_group_list *glist; struct mxq_group *grps; @@ -2221,7 +2221,7 @@ int main(int argc, char *argv[]) slots_started = 0; } - group_cnt = load_groups(&server); + group_cnt = load_running_groups(&server); if (group_cnt) mx_log_debug("group_cnt=%d :: %d Groups loaded", group_cnt, group_cnt); From 42a6bc8fb6158ca7a0e03fdecf50a38d52334cd6 Mon Sep 17 00:00:00 2001 From: Marius Tolzmann Date: Tue, 3 Nov 2015 12:34:08 +0100 Subject: [PATCH 54/67] mxqd: Fix and rename server_reload_running() to load_running_jobs() --- mxqd.c | 44 ++++++++++++++++++-------------------------- 1 file changed, 18 insertions(+), 26 deletions(-) diff --git a/mxqd.c b/mxqd.c index e669d75b..b86457fe 100644 --- a/mxqd.c +++ b/mxqd.c @@ -1899,19 +1899,15 @@ static int lost_scan(struct mxq_server *server) } -static int server_reload_running(struct mxq_server *server) +static int load_running_jobs(struct mxq_server *server) { _mx_cleanup_free_ struct mxq_job *jobs = NULL; struct mxq_job_list *jlist; struct mxq_group_list *glist; - struct mxq_user_list *ulist; - struct mxq_group *grps = NULL; - struct mxq_group *group; struct mxq_job *job; - int group_cnt; int job_cnt; int j; @@ -1926,24 +1922,17 @@ static int server_reload_running(struct mxq_server *server) job->stats_starttime.tv_sec = job->date_start; jlist = server_get_job_list_by_job_id(server, job->job_id); - if (!jlist) { - glist = server_get_group_list_by_group_id(server, job->group_id); - if (!glist) { - group_cnt = mxq_load_group(server->mysql, &grps, job->group_id); - if (group_cnt != 1) - continue; - group = &grps[0]; - ulist = server_find_user_by_uid(server, group->user_uid); - if (!ulist) { - glist = _server_add_group(server, group); - } else { - glist = _user_list_add_group(ulist, group); - } - mx_free_null(grps); - } - jlist = glist->jobs; + if (jlist) + continue; + + glist = server_get_group_list_by_group_id(server, job->group_id); + if (!glist) { + mx_log_fatal("BUG17: group %lu of job %lu not loaded. skipping job.", + job->group_id, job->job_id); + return -(errno=EUCLEAN); + } else { + group_list_add_job(glist, job); } - group_list_add_job(glist, job); } return job_cnt; } @@ -2102,12 +2091,15 @@ int recover_from_previous_crash(struct mxq_server *server) mx_log_info("hostname=%s server_id=%s :: recovered from previous crash: unassigned %d jobs.", server->hostname, server->server_id, res); - res=server_reload_running(server); - if (res<0) { - mx_log_err("recover: server_reload_running: %m"); + res = load_running_groups(server); + mx_log_info("recover: %d running groups loaded.", res); + + res = load_running_jobs(server); + if (res < 0) { + mx_log_err("recover: load_running_jobs: %m"); return res; } - if (res>0) + if (res > 0) mx_log_info("recover: reload %d running jobs from database", res); res=fspool_scan(server); From 2ac2843bcdc7f816449ecda9990fb0b6e13e52ca Mon Sep 17 00:00:00 2001 From: Marius Tolzmann Date: Tue, 3 Nov 2015 13:00:43 +0100 Subject: [PATCH 55/67] mxqd: Fix fspool_process_file() converted assertiion that job_id(file) == job_id(pid) to error state --- mxqd.c | 57 +++++++++++++++++++++++++++++++++++---------------------- 1 file changed, 35 insertions(+), 22 deletions(-) diff --git a/mxqd.c b/mxqd.c index b86457fe..18b8f9ec 100644 --- a/mxqd.c +++ b/mxqd.c @@ -1711,20 +1711,24 @@ static int fspool_process_file(struct mxq_server *server,char *filename,int job_ struct rusage rusage; struct timeval realtime; - struct mxq_job_list *job; - struct mxq_job *j; - struct mxq_group *g; + struct mxq_job_list *jlist; + struct mxq_job *job; + struct mxq_group *group; in=fopen(filename,"r"); if (!in) { return -errno; } + errno=0; res=fscanf(in,"1 %d %d %ld %ld %ld %ld %ld %ld %ld %ld %ld %ld %ld %ld %ld %ld %ld %ld %ld %ld %ld %ld", &pid, &status, - &realtime.tv_sec,&realtime.tv_usec, - &rusage.ru_utime.tv_sec,&rusage.ru_utime.tv_usec, - &rusage.ru_stime.tv_sec,&rusage.ru_stime.tv_usec, + &realtime.tv_sec, + &realtime.tv_usec, + &rusage.ru_utime.tv_sec, + &rusage.ru_utime.tv_usec, + &rusage.ru_stime.tv_sec, + &rusage.ru_stime.tv_usec, &rusage.ru_maxrss, &rusage.ru_ixrss, &rusage.ru_idrss, @@ -1749,26 +1753,35 @@ static int fspool_process_file(struct mxq_server *server,char *filename,int job_ mx_log_info("job finished (via fspool) : job %d pid %d status %d",job_id,pid,status); - job = server_remove_job_list_by_pid(server, pid); - if (!job) { - mx_log_warning("fspool_process_file: %s : job unknown on server",filename); - return(-1); + jlist = server_remove_job_list_by_pid(server, pid); + if (!jlist) { + mx_log_warning("fspool_process_file: %s : job unknown on server", filename); + return -(errno=ENOENT); } - j = &job->job; - assert(job->group); - assert(j->job_id=job_id); - g = &job->group->group; - j->stats_realtime=realtime; - j->stats_status=status; - j->stats_rusage=rusage; + job = &jlist->job; + if (job->job_id != job_id) { + mx_log_warning("fspool_process_file: %s: job_id(pid)[%ld] != job_id(filename)[%ld]", + filename, + job->job_id, + job_id); + return -(errno=EINVAL); + } + + assert(jlist->group); + + group = &jlist->group->group; + + job->stats_realtime = realtime; + job->stats_status = status; + job->stats_rusage = rusage; - job_has_finished(server,g,job); + job_has_finished(server, group, jlist); fspool_unlink(server,job_id); return(0); } -static int fspool_is_valid_name_parse(const char *name,int *job_id) { +static int fspool_is_valid_name_parse(const char *name, unsigned long long int *job_id) { const char *c=name; if (!*c) return 0; @@ -1781,7 +1794,7 @@ static int fspool_is_valid_name_parse(const char *name,int *job_id) { return 0; } if (job_id) { - *job_id=atol(name); + *job_id = strtoull(name, NULL, 10); } return 1; } @@ -1797,6 +1810,8 @@ static int fspool_scan(struct mxq_server *server) { struct dirent **namelist; int i; int res; + unsigned long long int job_id; + char *filename; entries=scandir(server->finished_jobsdir,&namelist,&fspool_is_valid_name,&alphasort); @@ -1806,8 +1821,6 @@ static int fspool_scan(struct mxq_server *server) { } for (i=0;ifinished_jobsdir,namelist[i]->d_name); fspool_is_valid_name_parse(namelist[i]->d_name,&job_id); res=fspool_process_file(server,filename,job_id); From a28a7c793dd1de347a9e1869e83e7e13d1055d59 Mon Sep 17 00:00:00 2001 From: Marius Tolzmann Date: Tue, 3 Nov 2015 13:01:50 +0100 Subject: [PATCH 56/67] test_mxqd_control: Init server structure --- test_mxqd_control.c | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/test_mxqd_control.c b/test_mxqd_control.c index 8b96e4df..ef115661 100644 --- a/test_mxqd_control.c +++ b/test_mxqd_control.c @@ -3,8 +3,30 @@ #include +#include "mxqd.h" + +#define MEMORY_TOTAL 20480 +#define MEMORY_MAX_PER_SLOT 2048 +#define SLOTS 10 + +void __init_server(struct mxq_server *server) +{ + server->memory_total = MEMORY_TOTAL; + server->memory_max_per_slot = MEMORY_MAX_PER_SLOT; + server->slots = SLOTS; + server->memory_avg_per_slot = MEMORY_TOTAL / SLOTS; +} + + static void test_mxqd_control(void) { + struct mxq_server _server = { 0 }; + struct mxq_server *server; + + server = &_server; + + __init_server(server); + assert(1); } From 1cf685ba0dcc212c81c7d67b6867fd103666a330 Mon Sep 17 00:00:00 2001 From: Marius Tolzmann Date: Tue, 3 Nov 2015 13:56:32 +0100 Subject: [PATCH 57/67] mxqd: Remove fspool_unlink() --- mxqd.c | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/mxqd.c b/mxqd.c index 18b8f9ec..446183ed 100644 --- a/mxqd.c +++ b/mxqd.c @@ -1696,12 +1696,6 @@ static char *fspool_get_filename (struct mxq_server *server,long unsigned int jo return fspool_filename; } -void fspool_unlink(struct mxq_server *server,int job_id) { - char *fspool_filename=fspool_get_filename(server,job_id); - unlink(fspool_filename); - free(fspool_filename); -} - static int fspool_process_file(struct mxq_server *server,char *filename,int job_id) { FILE *in; int res; @@ -1777,7 +1771,7 @@ static int fspool_process_file(struct mxq_server *server,char *filename,int job_ job->stats_rusage = rusage; job_has_finished(server, group, jlist); - fspool_unlink(server,job_id); + unlink(filename); return(0); } @@ -2043,8 +2037,6 @@ int catchall(struct mxq_server *server) pid, status); - fspool_unlink(server, job->job_id); - cnt += job_has_finished(server, group, jlist); } From 81249be48da69cbae810ceb41be2310b4a6a7216 Mon Sep 17 00:00:00 2001 From: Marius Tolzmann Date: Tue, 3 Nov 2015 13:56:53 +0100 Subject: [PATCH 58/67] mxqd: Cleanup job_has_finished() and job_lost() --- mxqd.c | 116 +++++++++++++++++++++++++++++---------------------------- 1 file changed, 60 insertions(+), 56 deletions(-) diff --git a/mxqd.c b/mxqd.c index 446183ed..ffa5a4b2 100644 --- a/mxqd.c +++ b/mxqd.c @@ -1614,78 +1614,82 @@ int killall_cancelled(struct mxq_server *server, int sig, unsigned int pgrp) return 0; } -static int job_has_finished (struct mxq_server *server,struct mxq_group *g,struct mxq_job_list *job) +static void rename_outfiles(struct mxq_group *group, struct mxq_job *job) { - int cnt=0; - int res; - struct mxq_job *j=&job->job; - - mxq_set_job_status_exited(server->mysql, j); - - if (j->job_status == MXQ_JOB_STATUS_FINISHED) { - g->group_jobs_finished++; - } else if(j->job_status == MXQ_JOB_STATUS_FAILED) { - g->group_jobs_failed++; - } else if(j->job_status == MXQ_JOB_STATUS_KILLED) { - g->group_jobs_failed++; - } + int res; - mxq_job_set_tmpfilenames(g, j); + mxq_job_set_tmpfilenames(group, job); - if (!mx_streq(j->job_stdout, "/dev/null")) { - res = rename(j->tmp_stdout, j->job_stdout); - if (res == -1) { - mx_log_err(" job=%s(%d):%lu:%lu host_pid=%d :: rename(stdout) failed: %m", - g->user_name, g->user_uid, g->group_id, j->job_id, j->host_pid); - } + if (!mx_streq(job->job_stdout, "/dev/null")) { + res = rename(job->tmp_stdout, job->job_stdout); + if (res == -1) { + mx_log_err(" job=%s(%d):%lu:%lu host_pid=%d :: rename(stdout) failed: %m", + group->user_name, + group->user_uid, + group->group_id, + job->job_id, + job->host_pid); } + } - if (!mx_streq(j->job_stderr, "/dev/null") && !mx_streq(j->job_stderr, j->job_stdout)) { - res = rename(j->tmp_stderr, j->job_stderr); - if (res == -1) { - mx_log_err(" job=%s(%d):%lu:%lu host_pid=%d :: rename(stderr) failed: %m", - g->user_name, g->user_uid, g->group_id, j->job_id, j->host_pid); - } + if (!mx_streq(job->job_stderr, "/dev/null") && !mx_streq(job->job_stderr, job->job_stdout)) { + res = rename(job->tmp_stderr, job->job_stderr); + if (res == -1) { + mx_log_err(" job=%s(%d):%lu:%lu host_pid=%d :: rename(stderr) failed: %m", + group->user_name, + group->user_uid, + group->group_id, + job->job_id, + job->host_pid); } + } +} + +static int job_has_finished(struct mxq_server *server, struct mxq_group *group, struct mxq_job_list *jlist) +{ + int cnt; + struct mxq_job *job; + + job=&jlist->job; - cnt += job->group->slots_per_job; - cpuset_clear_running(&server->cpu_set_running,&j->host_cpu_set); - mxq_job_free_content(j); - free(job); + mxq_set_job_status_exited(server->mysql, job); + + if (job->job_status == MXQ_JOB_STATUS_FINISHED) { + group->group_jobs_finished++; + } else if(job->job_status == MXQ_JOB_STATUS_FAILED) { + group->group_jobs_failed++; + } else if(job->job_status == MXQ_JOB_STATUS_KILLED) { + group->group_jobs_failed++; + } + + rename_outfiles(group, job); + + cnt = jlist->group->slots_per_job; + cpuset_clear_running(&server->cpu_set_running, &job->host_cpu_set); + mxq_job_free_content(job); + free(jlist); return cnt; } -static int job_is_lost (struct mxq_server *server,struct mxq_group *g,struct mxq_job_list *job) +static int job_is_lost(struct mxq_server *server,struct mxq_group *group, struct mxq_job_list *jlist) { - int cnt=0; - int res; - struct mxq_job *j=&job->job; + int cnt; + struct mxq_job *job; - mxq_set_job_status_unknown(server->mysql, j); - g->group_jobs_unknown++; + assert(jlist->group); + assert(!jlist->next); - mxq_job_set_tmpfilenames(g, j); + job = &jlist->job; - if (!mx_streq(j->job_stdout, "/dev/null")) { - res = rename(j->tmp_stdout, j->job_stdout); - if (res == -1) { - mx_log_err(" job=%s(%d):%lu:%lu host_pid=%d :: rename(stdout) failed: %m", - g->user_name, g->user_uid, g->group_id, j->job_id, j->host_pid); - } - } + mxq_set_job_status_unknown(server->mysql, job); + group->group_jobs_unknown++; - if (!mx_streq(j->job_stderr, "/dev/null") && !mx_streq(j->job_stderr, j->job_stdout)) { - res = rename(j->tmp_stderr, j->job_stderr); - if (res == -1) { - mx_log_err(" job=%s(%d):%lu:%lu host_pid=%d :: rename(stderr) failed: %m", - g->user_name, g->user_uid, g->group_id, j->job_id, j->host_pid); - } - } + rename_outfiles(group, job); - cnt += job->group->slots_per_job; - cpuset_clear_running(&server->cpu_set_running,&j->host_cpu_set); - mxq_job_free_content(j); - free(job); + cnt = jlist->group->slots_per_job; + cpuset_clear_running(&server->cpu_set_running, &job->host_cpu_set); + mxq_job_free_content(job); + free(jlist); return cnt; } From d80144316d81100559d100f006230c8070cd788f Mon Sep 17 00:00:00 2001 From: Marius Tolzmann Date: Tue, 3 Nov 2015 14:37:26 +0100 Subject: [PATCH 59/67] mxqd: Cleanup init_child_process() --- mxqd.c | 110 ++++++++++++++++++++++++++++----------------------------- 1 file changed, 55 insertions(+), 55 deletions(-) diff --git a/mxqd.c b/mxqd.c index ffa5a4b2..0c51538c 100644 --- a/mxqd.c +++ b/mxqd.c @@ -596,36 +596,36 @@ static void reset_signals() signal(SIGPIPE, SIG_DFL); } -static int init_child_process(struct mxq_group_list *group, struct mxq_job *j) +static int init_child_process(struct mxq_group_list *glist, struct mxq_job *job) { - struct mxq_group *g; - struct mxq_server *s; + struct mxq_server *server; + struct mxq_group *group; struct passwd *passwd; int res; int fh; struct rlimit rlim; - assert(j); - assert(group); - assert(group->user); - assert(group->user->server); + assert(job); + assert(glist); + assert(glist->user); + assert(glist->user->server); - s = group->user->server; - g = &group->group; + server = glist->user->server; + group = &glist->group; reset_signals(); - passwd = getpwuid(g->user_uid); + passwd = getpwuid(group->user_uid); if (!passwd) { mx_log_err("job=%s(%d):%lu:%lu getpwuid(): %m", - g->user_name, g->user_uid, g->group_id, j->job_id); + group->user_name, group->user_uid, group->group_id, job->job_id); return 0; } - if (!mx_streq(passwd->pw_name, g->user_name)) { + if (!mx_streq(passwd->pw_name, group->user_name)) { mx_log_err("job=%s(%d):%lu:%lu user_uid=%d does not map to user_name=%s but to pw_name=%s: Invalid user mapping", - g->user_name, g->user_uid, g->group_id, j->job_id, - g->user_uid, g->user_name, passwd->pw_name); + group->user_name, group->user_uid, group->group_id, job->job_id, + group->user_uid, group->user_name, passwd->pw_name); return 0; } @@ -635,56 +635,56 @@ static int init_child_process(struct mxq_group_list *group, struct mxq_job *j) res = clearenv(); if (res != 0) { mx_log_err("job=%s(%d):%lu:%lu clearenv(): %m", - g->user_name, g->user_uid, g->group_id, j->job_id); + group->user_name, group->user_uid, group->group_id, job->job_id); return 0; } - mx_setenv_forever("USER", g->user_name); - mx_setenv_forever("USERNAME", g->user_name); - mx_setenv_forever("LOGNAME", g->user_name); - mx_setenv_forever("PATH", s->initial_path); - mx_setenv_forever("TMPDIR", s->initial_tmpdir); - mx_setenv_forever("PWD", j->job_workdir); + mx_setenv_forever("USER", group->user_name); + mx_setenv_forever("USERNAME", group->user_name); + mx_setenv_forever("LOGNAME", group->user_name); + mx_setenv_forever("PATH", server->initial_path); + mx_setenv_forever("TMPDIR", server->initial_tmpdir); + mx_setenv_forever("PWD", job->job_workdir); mx_setenv_forever("HOME", passwd->pw_dir); mx_setenv_forever("SHELL", passwd->pw_shell); mx_setenv_forever("HOSTNAME", mx_hostname()); - mx_setenvf_forever("JOB_ID", "%lu", j->job_id); - mx_setenvf_forever("MXQ_JOBID", "%lu", j->job_id); - mx_setenvf_forever("MXQ_THREADS", "%d", g->job_threads); - mx_setenvf_forever("MXQ_SLOTS", "%lu", group->slots_per_job); - mx_setenvf_forever("MXQ_MEMORY", "%lu", g->job_memory); - mx_setenvf_forever("MXQ_TIME", "%d", g->job_time); - mx_setenv_forever("MXQ_HOSTID", s->host_id); - mx_setenv_forever("MXQ_HOSTNAME", s->hostname); - mx_setenv_forever("MXQ_SERVERID", s->server_id); + mx_setenvf_forever("JOB_ID", "%lu", job->job_id); + mx_setenvf_forever("MXQ_JOBID", "%lu", job->job_id); + mx_setenvf_forever("MXQ_THREADS", "%d", group->job_threads); + mx_setenvf_forever("MXQ_SLOTS", "%lu", glist->slots_per_job); + mx_setenvf_forever("MXQ_MEMORY", "%lu", group->job_memory); + mx_setenvf_forever("MXQ_TIME", "%d", group->job_time); + mx_setenv_forever("MXQ_HOSTID", server->host_id); + mx_setenv_forever("MXQ_HOSTNAME", server->hostname); + mx_setenv_forever("MXQ_SERVERID", server->server_id); fh = open("/proc/self/loginuid", O_WRONLY|O_TRUNC); if (fh == -1) { mx_log_err("job=%s(%d):%lu:%lu open(%s) failed: %m", - g->user_name, g->user_uid, g->group_id, j->job_id, "/proc/self/loginuid"); + group->user_name, group->user_uid, group->group_id, job->job_id, "/proc/self/loginuid"); return 0; } - dprintf(fh, "%d", g->user_uid); + dprintf(fh, "%d", group->user_uid); close(fh); /* set memory limits */ - rlim.rlim_cur = g->job_memory*1024*1024; - rlim.rlim_max = g->job_memory*1024*1024; + rlim.rlim_cur = group->job_memory*1024*1024; + rlim.rlim_max = group->job_memory*1024*1024; res = setrlimit(RLIMIT_AS, &rlim); if (res == -1) mx_log_err("job=%s(%d):%lu:%lu setrlimit(RLIMIT_AS, ...) failed: %m", - g->user_name, g->user_uid, g->group_id, j->job_id); + group->user_name, group->user_uid, group->group_id, job->job_id); res = setrlimit(RLIMIT_DATA, &rlim); if (res == -1) mx_log_err("job=%s(%d):%lu:%lu setrlimit(RLIMIT_DATA, ...) failed: %m", - g->user_name, g->user_uid, g->group_id, j->job_id); + group->user_name, group->user_uid, group->group_id, job->job_id); res = setrlimit(RLIMIT_RSS, &rlim); if (res == -1) mx_log_err("job=%s(%d):%lu:%lu setrlimit(RLIMIT_RSS, ...) failed: %m", - g->user_name, g->user_uid, g->group_id, j->job_id); + group->user_name, group->user_uid, group->group_id, job->job_id); /* disable core files */ rlim.rlim_cur = 0; @@ -693,57 +693,57 @@ static int init_child_process(struct mxq_group_list *group, struct mxq_job *j) res = setrlimit(RLIMIT_CORE, &rlim); if (res == -1) mx_log_err("job=%s(%d):%lu:%lu setrlimit(RLIMIT_CORE, ...) failed: %m", - g->user_name, g->user_uid, g->group_id, j->job_id); + group->user_name, group->user_uid, group->group_id, job->job_id); /* set single threaded time limits */ - if (g->job_threads == 1) { + if (group->job_threads == 1) { /* set cpu time limits - hardlimit is 105% of softlimit */ - rlim.rlim_cur = g->job_time*60; - rlim.rlim_cur = g->job_time*63; + rlim.rlim_cur = group->job_time*60; + rlim.rlim_cur = group->job_time*63; res = setrlimit(RLIMIT_CPU, &rlim); if (res == -1) mx_log_err("job=%s(%d):%lu:%lu setrlimit(RLIMIT_CPU, ...) failed: %m", - g->user_name, g->user_uid, g->group_id, j->job_id); + group->user_name, group->user_uid, group->group_id, job->job_id); } if(RUNNING_AS_ROOT) { - res = initgroups(passwd->pw_name, g->user_gid); + res = initgroups(passwd->pw_name, group->user_gid); if (res == -1) { mx_log_err("job=%s(%d):%lu:%lu initgroups() failed: %m", - g->user_name, g->user_uid, g->group_id, j->job_id); + group->user_name, group->user_uid, group->group_id, job->job_id); return 0; } - res = setregid(g->user_gid, g->user_gid); + res = setregid(group->user_gid, group->user_gid); if (res == -1) { mx_log_err("job=%s(%d):%lu:%lu setregid(%d, %d) failed: %m", - g->user_name, g->user_uid, g->group_id, j->job_id, - g->user_gid, g->user_gid); + group->user_name, group->user_uid, group->group_id, job->job_id, + group->user_gid, group->user_gid); return 0; } - res = setreuid(g->user_uid, g->user_uid); + res = setreuid(group->user_uid, group->user_uid); if (res == -1) { mx_log_err("job=%s(%d):%lu:%lu setreuid(%d, %d) failed: %m", - g->user_name, g->user_uid, g->group_id, j->job_id, - g->user_uid, g->user_uid); + group->user_name, group->user_uid, group->group_id, job->job_id, + group->user_uid, group->user_uid); return 0; } } - res = chdir(j->job_workdir); + res = chdir(job->job_workdir); if (res == -1) { mx_log_err("job=%s(%d):%lu:%lu chdir(%s) failed: %m", - g->user_name, g->user_uid, g->group_id, j->job_id, - j->job_workdir); + group->user_name, group->user_uid, group->group_id, job->job_id, + job->job_workdir); return 0; } - umask(j->job_umask); + umask(job->job_umask); - res=sched_setaffinity(0,sizeof(j->host_cpu_set),&j->host_cpu_set); + res=sched_setaffinity(0,sizeof(job->host_cpu_set),&job->host_cpu_set); if (res<0) mx_log_warning("sched_setaffinity: $m"); return 1; From 790d4cffd5ab95eb7db0b0325f80c2d5ce665a41 Mon Sep 17 00:00:00 2001 From: Marius Tolzmann Date: Tue, 3 Nov 2015 14:38:16 +0100 Subject: [PATCH 60/67] mxqd: Cleanup user_process() --- mxqd.c | 48 ++++++++++++++++++++++++++++++++---------------- 1 file changed, 32 insertions(+), 16 deletions(-) diff --git a/mxqd.c b/mxqd.c index 0c51538c..5c6d8564 100644 --- a/mxqd.c +++ b/mxqd.c @@ -840,46 +840,62 @@ int mxq_redirect_input(char *stdin_fname) return 1; } -int user_process(struct mxq_group_list *group,struct mxq_job *mxqjob) +int user_process(struct mxq_group_list *glist, struct mxq_job *job) { int res; char **argv; - res = init_child_process(group, mxqjob); + struct mxq_group *group; + + group = &glist->group; + + res = init_child_process(glist, job); if (!res) return(-1); - mxq_job_set_tmpfilenames(&group->group, mxqjob); + mxq_job_set_tmpfilenames(group, job); res = mxq_redirect_input("/dev/null"); if (res < 0) { mx_log_err(" job=%s(%d):%lu:%lu mxq_redirect_input() failed (%d): %m", - group->group.user_name, group->group.user_uid, group->group.group_id, mxqjob->job_id, - res); + group->user_name, + group->user_uid, + group->group_id, + job->job_id, + res); return(res); } - res = mxq_redirect_output(mxqjob->tmp_stdout, mxqjob->tmp_stderr); + res = mxq_redirect_output(job->tmp_stdout, job->tmp_stderr); if (res < 0) { mx_log_err(" job=%s(%d):%lu:%lu mxq_redirect_output() failed (%d): %m", - group->group.user_name, group->group.user_uid, group->group.group_id, mxqjob->job_id, - res); + group->user_name, + group->user_uid, + group->group_id, + job->job_id, + res); return(res); } - argv = mx_strvec_from_str(mxqjob->job_argv_str); + argv = mx_strvec_from_str(job->job_argv_str); if (!argv) { mx_log_err("job=%s(%d):%lu:%lu Can't recaculate commandline. str_to_strvev(%s) failed: %m", - group->group.user_name, group->group.user_uid, group->group.group_id, mxqjob->job_id, - mxqjob->job_argv_str); - return(-errno); + group->user_name, + group->user_uid, + group->group_id, + job->job_id, + job->job_argv_str); + return -errno; } - res=execvp(argv[0], argv); + res = execvp(argv[0], argv); mx_log_err("job=%s(%d):%lu:%lu execvp(\"%s\", ...): %m", - group->group.user_name, group->group.user_uid, group->group.group_id, mxqjob->job_id, - argv[0]); - return(res); + group->user_name, + group->user_uid, + group->group_id, + job->job_id, + argv[0]); + return res; } int reaper_process(struct mxq_server *server,struct mxq_group_list *group,struct mxq_job *job) { From f9be94178b9a79207d4946b72760fa179601e7bf Mon Sep 17 00:00:00 2001 From: Marius Tolzmann Date: Tue, 3 Nov 2015 14:38:33 +0100 Subject: [PATCH 61/67] mxqd: Cleanup reaper_process() --- mxqd.c | 35 ++++++++++++++++++++--------------- 1 file changed, 20 insertions(+), 15 deletions(-) diff --git a/mxqd.c b/mxqd.c index 5c6d8564..3d65a055 100644 --- a/mxqd.c +++ b/mxqd.c @@ -898,7 +898,7 @@ int user_process(struct mxq_group_list *glist, struct mxq_job *job) return res; } -int reaper_process(struct mxq_server *server,struct mxq_group_list *group,struct mxq_job *job) { +int reaper_process(struct mxq_server *server,struct mxq_group_list *glist, struct mxq_job *job) { pid_t pid; struct rusage rusage; int status; @@ -911,6 +911,10 @@ int reaper_process(struct mxq_server *server,struct mxq_group_list *group,struc FILE *out; int res; + struct mxq_group *group; + + group = &glist->group; + reset_signals(); signal(SIGINT, SIG_IGN); @@ -919,12 +923,13 @@ int reaper_process(struct mxq_server *server,struct mxq_group_list *group,struc signal(SIGXCPU, SIG_IGN); res = setsid(); - if (res<0) { - mx_log_warning("reaper_process setsid: %m"); + if (res < 0) { + mx_log_warning("reaper_process setsid: %m"); + return res; } - res=prctl(PR_SET_CHILD_SUBREAPER, 1); - if (res<0) { + res = prctl(PR_SET_CHILD_SUBREAPER, 1); + if (res < 0) { mx_log_err("set subreaper: %m"); return res; } @@ -932,16 +937,16 @@ int reaper_process(struct mxq_server *server,struct mxq_group_list *group,struc pid = fork(); if (pid < 0) { mx_log_err("fork: %m"); - return(pid); + return pid; } else if (pid == 0) { - res=user_process(group,job); + res = user_process(glist, job); _exit(EX__MAX+1); } gettimeofday(&job->stats_starttime, NULL); while (1) { - waited_pid=wait(&waited_status); - if (waited_pid<0) { + waited_pid = wait(&waited_status); + if (waited_pid < 0) { if (errno==ECHILD) { break; } else { @@ -949,20 +954,20 @@ int reaper_process(struct mxq_server *server,struct mxq_group_list *group,struc sleep(1); } } - if (waited_pid==pid) { - status=waited_status; + if (waited_pid == pid) { + status = waited_status; } } gettimeofday(&now, NULL); timersub(&now, &job->stats_starttime, &realtime); - res=getrusage(RUSAGE_CHILDREN,&rusage); - if (res<0) { + res = getrusage(RUSAGE_CHILDREN, &rusage); + if (res < 0) { mx_log_err("reaper: getrusage: %m"); return(res); } - mx_asprintf_forever(&finished_job_filename,"%s/%lu.stat",server->finished_jobsdir,job->job_id); - mx_asprintf_forever(&finished_job_tmpfilename,"%s.tmp",finished_job_filename); + mx_asprintf_forever(&finished_job_filename, "%s/%lu.stat", server->finished_jobsdir, job->job_id); + mx_asprintf_forever(&finished_job_tmpfilename, "%s.tmp", finished_job_filename); out=fopen(finished_job_tmpfilename,"w"); if (!out) { From 2d632f3a4786700346660afb565462d8b465c79c Mon Sep 17 00:00:00 2001 From: Marius Tolzmann Date: Tue, 3 Nov 2015 15:31:08 +0100 Subject: [PATCH 62/67] mxqd: Fix kill signals: send kill to pgrp instead of reaper pid --- mxqd.c | 52 +++++++++++++++++----------------------------------- 1 file changed, 17 insertions(+), 35 deletions(-) diff --git a/mxqd.c b/mxqd.c index 3d65a055..f4a6a1ba 100644 --- a/mxqd.c +++ b/mxqd.c @@ -1465,42 +1465,22 @@ int killall_over_time(struct mxq_server *server) pid = job->host_pid; - if (delta.tv_sec <= group->job_time*61) { - mx_log_info("killall_over_time(): Sending signal=XCPU to job=%s(%d):%lu:%lu pid=%d", - group->user_name, group->user_uid, group->group_id, job->job_id, pid); - kill(pid, SIGXCPU); - continue; - } - mx_log_info("killall_over_time(): Sending signal=XCPU to job=%s(%d):%lu:%lu pgrp=%d", group->user_name, group->user_uid, group->group_id, job->job_id, pid); + kill(-pid, SIGCONT); kill(-pid, SIGXCPU); if (delta.tv_sec <= group->job_time*63) continue; - mx_log_info("killall_over_time(): Sending signal=TERM to job=%s(%d):%lu:%lu pid=%d", - group->user_name, group->user_uid, group->group_id, job->job_id, pid); - kill(pid, SIGTERM); - - mx_log_info("killall_over_time(): Sending signal=HUP to job=%s(%d):%lu:%lu pgrp=%d", - group->user_name, group->user_uid, group->group_id, job->job_id, pid); - kill(-pid, SIGHUP); - - if (delta.tv_sec <= group->job_time*64) - continue; - mx_log_info("killall_over_time(): Sending signal=TERM to job=%s(%d):%lu:%lu pgrp=%d", group->user_name, group->user_uid, group->group_id, job->job_id, pid); + kill(-pid, SIGCONT); kill(-pid, SIGTERM); - if (delta.tv_sec <= group->job_time*66) + if (delta.tv_sec <= group->job_time*66+60*10) continue; - mx_log_info("killall_over_time(): Sending signal=KILL to job=%s(%d):%lu:%lu pid=%d", - group->user_name, group->user_uid, group->group_id, job->job_id, pid); - kill(pid, SIGKILL); - mx_log_info("killall_over_time(): Sending signal=KILL to job=%s(%d):%lu:%lu pgrp=%d", group->user_name, group->user_uid, group->group_id, job->job_id, pid); kill(-pid, SIGKILL); @@ -1576,7 +1556,7 @@ int killall_over_memory(struct mxq_server *server) if (jlist->max_sum_rss/1024 <= group->job_memory) continue; - mx_log_info("killall_over_memory(): used(%lluMiB) > requested(%lluMiB): Sending signal=%d to job=%s(%d):%lu:%lu pid=%d", + mx_log_info("killall_over_memory(): used(%lluMiB) > requested(%lluMiB): Sending signal=%d to job=%s(%d):%lu:%lu pgrp=%d", jlist->max_sum_rss/1024, group->job_memory, signal, @@ -1586,7 +1566,8 @@ int killall_over_memory(struct mxq_server *server) job->job_id, job->host_pid); - kill(job->host_pid, signal); + kill(-job->host_pid, SIGCONT); + kill(-job->host_pid, signal); } } } @@ -1594,7 +1575,7 @@ int killall_over_memory(struct mxq_server *server) return 0; } -int killall_cancelled(struct mxq_server *server, int sig, unsigned int pgrp) +int killall_cancelled(struct mxq_server *server) { struct mxq_user_list *ulist; struct mxq_group_list *glist; @@ -1622,13 +1603,14 @@ int killall_cancelled(struct mxq_server *server, int sig, unsigned int pgrp) job = &jlist->job; pid = job->host_pid; - if (pgrp) - pid = -pid; - mx_log_info(" Sending signal=%d to job=%s(%d):%lu:%lu %s=%d", - sig, - group->user_name, group->user_uid, group->group_id, job->job_id, - pgrp?"pgrp":"pid", pid); - kill(pid, sig); + mx_log_info(" Sending signal=TERM to job=%s(%d):%lu:%lu pgrp=%d", + group->user_name, + group->user_uid, + group->group_id, + job->job_id, + pid); + kill(-pid, SIGCONT); + kill(-pid, SIGTERM); } } } @@ -2247,7 +2229,7 @@ int main(int argc, char *argv[]) if (group_cnt) mx_log_debug("group_cnt=%d :: %d Groups loaded", group_cnt, group_cnt); - killall_cancelled(&server, SIGTERM, 0); + killall_cancelled(&server); killall_over_time(&server); killall_over_memory(&server); @@ -2297,7 +2279,7 @@ int main(int argc, char *argv[]) if (global_sigint_cnt) killall(&server, SIGTERM, 1); - killall_cancelled(&server, SIGTERM, 0); + killall_cancelled(&server); killall_over_time(&server); killall_over_memory(&server); mx_log_info("jobs_running=%lu global_sigint_cnt=%d global_sigterm_cnt=%d : Exiting. Wating for jobs to finish. Sleeping for a while.", From 1291fc52b2a0da0c3ccf426dd85d3ca9b64610d8 Mon Sep 17 00:00:00 2001 From: Marius Tolzmann Date: Tue, 3 Nov 2015 16:43:02 +0100 Subject: [PATCH 63/67] mxqd: Fix memory leak for host_cpu_set_str --- mxqd.c | 1 + 1 file changed, 1 insertion(+) diff --git a/mxqd.c b/mxqd.c index f4a6a1ba..eaa6d52d 100644 --- a/mxqd.c +++ b/mxqd.c @@ -1036,6 +1036,7 @@ unsigned long start_job(struct mxq_group_list *glist) group->user_name, group->user_uid, group->group_id, job->job_id); cpuset_init_job(&job->host_cpu_set, &server->cpu_set_available, &server->cpu_set_running, glist->slots_per_job); + mx_free_null(job->host_cpu_set_str); job->host_cpu_set_str = mx_cpuset_to_str(&job->host_cpu_set); mx_log_info("job assigned cpus: [%s]", job->host_cpu_set_str); From acc1eb6ac92b7742f7662dd70c0536c91fa708ad Mon Sep 17 00:00:00 2001 From: Marius Tolzmann Date: Tue, 3 Nov 2015 16:43:36 +0100 Subject: [PATCH 64/67] mx_flock: export mx_flock_free() to free without releasing lock --- mx_flock.c | 12 +++++++----- mx_flock.h | 1 + 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/mx_flock.c b/mx_flock.c index bdbbaf30..f6939f22 100644 --- a/mx_flock.c +++ b/mx_flock.c @@ -58,11 +58,7 @@ static inline void _flock_free(struct mx_flock *lock) if (!lock) return; - if (lock->fname) - mx_free_null(lock->fname); - - if (lock->fd >= 0) - _flock_close(lock); + mx_free_null(lock->fname); free(lock); } @@ -112,6 +108,7 @@ struct mx_flock *mx_flock(int operation, char *fmt, ...) if (errno == EWOULDBLOCK) return lock; mx_log_err("flock(): %m"); + _flock_close(lock); _flock_free(lock); return NULL; } @@ -155,3 +152,8 @@ int mx_funlock(struct mx_flock *lock) return res; } + +void mx_flock_free(struct mx_flock *lock) +{ + _flock_free(lock); +} diff --git a/mx_flock.h b/mx_flock.h index 22725383..34ab4e8d 100644 --- a/mx_flock.h +++ b/mx_flock.h @@ -10,5 +10,6 @@ struct mx_flock { struct mx_flock *mx_flock(int operation, char *fmt, ...); int mx_funlock(struct mx_flock *lock); +void mx_flock_free(struct mx_flock *lock); #endif From 8a1c40dc8d5d9f22a17f50ecb33b641bae97cf95 Mon Sep 17 00:00:00 2001 From: Marius Tolzmann Date: Tue, 3 Nov 2015 16:45:10 +0100 Subject: [PATCH 65/67] mxq_job: Minor cleanup --- mxq_job.c | 5 ++--- mxqd.c | 4 ++-- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/mxq_job.c b/mxq_job.c index d0e930e8..49d97a46 100644 --- a/mxq_job.c +++ b/mxq_job.c @@ -149,6 +149,7 @@ char *mxq_job_status_to_name(uint64_t status) void mxq_job_free_content(struct mxq_job *j) { mx_free_null(j->job_workdir); + mx_free_null(j->job_argv); mx_free_null(j->job_argv_str); mx_free_null(j->job_stdout); mx_free_null(j->job_stderr); @@ -161,11 +162,9 @@ void mxq_job_free_content(struct mxq_job *j) mx_free_null(j->tmp_stderr); mx_free_null(j->host_submit); mx_free_null(j->host_id); - mx_free_null(j->host_cpu_set_str); mx_free_null(j->server_id); mx_free_null(j->host_hostname); - mx_free_null(j->job_argv); - j->job_argv = NULL; + mx_free_null(j->host_cpu_set_str); } static int do_jobs_statement(struct mx_mysql *mysql, char *query, struct mx_mysql_bind *param, struct mxq_job **jobs) diff --git a/mxqd.c b/mxqd.c index eaa6d52d..1feb59ca 100644 --- a/mxqd.c +++ b/mxqd.c @@ -765,8 +765,8 @@ int mxq_redirect_open(char *fname) } else if (!mx_streq(fname, "/dev/null")) { res = unlink(fname); if (res == -1 && errno != ENOENT) { - mx_log_err("unlink() failed: %m"); - return -2; + mx_log_err("%s: unlink() failed: %m", fname); + return -2; } flags |= O_EXCL; } From 9d57ac316960c6a43bcd906d35eaf6045bb02844 Mon Sep 17 00:00:00 2001 From: Marius Tolzmann Date: Tue, 3 Nov 2015 16:49:18 +0100 Subject: [PATCH 66/67] mxqd: be a bit more verbose when starting processes to log pids --- mxqd.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mxqd.c b/mxqd.c index 1feb59ca..671c3033 100644 --- a/mxqd.c +++ b/mxqd.c @@ -939,6 +939,7 @@ int reaper_process(struct mxq_server *server,struct mxq_group_list *glist, struc mx_log_err("fork: %m"); return pid; } else if (pid == 0) { + mx_log_info("starting user process."); res = user_process(glist, job); _exit(EX__MAX+1); } @@ -1058,6 +1059,8 @@ unsigned long start_job(struct mxq_group_list *glist) job->host_pid, getpgrp()); + mx_log_info("starting reaper process."); + res = reaper_process(server, glist, job); _exit(res<0 ? EX__MAX+1 : 0); } From 7699dae1edcdb7650f1c5bed6e096380c73924da Mon Sep 17 00:00:00 2001 From: Marius Tolzmann Date: Tue, 3 Nov 2015 16:49:29 +0100 Subject: [PATCH 67/67] mxqd: free structures to remove leftover memory in reaper process --- mxqd.c | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/mxqd.c b/mxqd.c index 671c3033..97277ae8 100644 --- a/mxqd.c +++ b/mxqd.c @@ -58,6 +58,7 @@ volatile sig_atomic_t global_sigterm_cnt=0; volatile sig_atomic_t global_sigquit_cnt=0; int mxq_redirect_output(char *stdout_fname, char *stderr_fname); +void server_free(struct mxq_server *server); static void print_usage(void) { @@ -1060,8 +1061,15 @@ unsigned long start_job(struct mxq_group_list *glist) getpgrp()); mx_log_info("starting reaper process."); + mx_mysql_finish(&server->mysql); res = reaper_process(server, glist, job); + + mxq_job_free_content(job); + + mx_log_info("shutting down reaper, bye bye."); + mx_log_finish(); + server_free(server); _exit(res<0 ? EX__MAX+1 : 0); } @@ -1365,7 +1373,7 @@ void server_dump(struct mxq_server *server) mx_log_info("====================== SERVER DUMP END ======================"); } -void server_close(struct mxq_server *server) +void server_free(struct mxq_server *server) { struct mxq_user_list *ulist, *unext; struct mxq_group_list *glist, *gnext; @@ -1386,14 +1394,21 @@ void server_close(struct mxq_server *server) mx_free_null(ulist); } + mx_free_null(server->boot_id); + mx_free_null(server->host_id); + mx_free_null(server->finished_jobsdir); + mx_flock_free(server->flock); +} + +void server_close(struct mxq_server *server) +{ if (server->pidfilename) unlink(server->pidfilename); mx_funlock(server->flock); + server->flock = NULL; - mx_free_null(server->boot_id); - mx_free_null(server->host_id); - mx_free_null(server->finished_jobsdir); + server_free(server); } int killall(struct mxq_server *server, int sig, unsigned int pgrp)