Skip to content

Commit

Permalink
Merge branch 'master' into improvements
Browse files Browse the repository at this point in the history 
* master:
  web: Fix typo when (not) displaying max_sumrss
  mxqd: Add --log-directory[=DIRECTORY]
  mx_util: Add mx_strconcat()
  mxqd: Reorder server initialisation for clearer error logging
  mxqd: Allow to be run as non root user
  mxqd: Introduce dynamic log directory based on LOCALSTATEDIR
  mxq_job: Only load jobs assigned for the group that should be started
  MXQ bump version
  make: add LOCALSTATEDIR
  web: show threads in group table and runtime in minutes for completed jobs
  mxq_job: fix memory leak
  mxqd: fix typo in log
  MXQ bump version
  mxqd: Fix root check
  MXQ bump version
  mxqd: Send TERM to process when over memory instead of KILL to pgrp
  mxqd: Store and use max_sumrss
  mxq_job: Minor cleanup
  • Loading branch information
@mariux
mariux committed Oct 27, 2015
2 parents e345231 + 24fe8a3 commit a9362b2
Show file tree
Hide file tree
Showing 15 changed files with 212 additions and 79 deletions.
12 changes: 10 additions & 2 deletions Makefile
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
MXQ_VERSION_MAJOR = 0
MXQ_VERSION_MINOR = 17
MXQ_VERSION_PATCH = 0
MXQ_VERSION_MINOR = 18
MXQ_VERSION_PATCH = 2
MXQ_VERSION_EXTRA = "beta"
MXQ_VERSIONDATE = 2015

Expand All @@ -25,6 +25,7 @@ LIBEXECDIR = ${EPREFIX}/libexec
DATADIR = ${PREFIX}/share
MANDIR = ${DATADIR}/man
SYSCONFDIR = ${PREFIX}/etc
LOCALSTATEDIR = ${PREFIX}/var

DESTDIR=

Expand All @@ -43,6 +44,11 @@ ifneq (, $(filter /usr /usr/local, ${PREFIX}))
SYSCONFDIR = /etc
endif

### set localstatedir /var if prefix /usr || /usr/local
ifneq (, $(filter /usr /usr/local, ${PREFIX}))
LOCALSTATEDIR = /var
endif

########################################################################

### strip /mxq from SYSCONFDIR if set
Expand Down Expand Up @@ -125,6 +131,7 @@ CFLAGS += -DLIBEXECDIR=\"${LIBEXECDIR}\"
CFLAGS += -DDATADIR=\"${DATADIR}\"
CFLAGS += -DMANDIR=\"${MANDIR}\"
CFLAGS += -DSYSCONFDIR=\"${SYSCONFDIR}\"
CFLAGS += -DLOCALSTATEDIR=\"${LOCALSTATEDIR}\"
CFLAGS += $(EXTRA_CFLAGS)

########################################################################
Expand Down Expand Up @@ -248,6 +255,7 @@ install::
$(call quiet-installdir,0755,${DESTDIR}${SYSCONFDIR}/mxq)
$(call quiet-installdir,0755,${DESTDIR}${MAN1DIR})
$(call quiet-installdir,0755,${DESTDIR}${CGIDIR})
$(call quiet-installdir,0755,${DESTDIR}${LOCALSTATEDIR}/log)

########################################################################

Expand Down
41 changes: 41 additions & 0 deletions mx_util.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -856,6 +856,47 @@ int mx_strscan_ll(char **str, long long int *to)
return res;
}

char *_mx_strconcat_do(char *first, ...)
{
va_list ap;
char *join = NULL;
char *str;
char *ptr;
size_t len;

if (!first) {
join = strdup("");
return join;
}

len = strlen(first);

va_start(ap, first);
do {
str = va_arg(ap, char *);
if (!str)
break;
len += strlen(str);
} while(1);
va_end(ap);

join = malloc(len+1);
if (!join)
return NULL;

ptr = stpcpy(join, first);
va_start(ap, first);
do {
str = va_arg(ap, char *);
if (!str)
break;
ptr = stpcpy(ptr, str);
} while(1);
va_end(ap);

return join;
}

int mx_sleep(unsigned int seconds)
{
if (seconds)
Expand Down
3 changes: 3 additions & 0 deletions mx_util.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -136,6 +136,9 @@ int mx_read_first_line_from_file(char *fname, char **line);
int mx_strscan_ull(char **str, unsigned long long int *to);
int mx_strscan_ll(char **str, long long int *to);

char *_mx_strconcat_do(char *first, ...);
#define mx_strconcat(s, ...) _mx_strconcat_do((s), ##__VA_ARGS__, NULL)

int mx_sleep(unsigned int seconds);
int mx_sleep_nofail(unsigned int seconds);

Expand Down
6 changes: 6 additions & 0 deletions mxq.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,12 @@
#endif
#define MXQ_MYSQL_DEFAULT_GROUP_STR MXQ_MYSQL_DEFAULT_GROUP

#if defined (LOCALSTATEDIR)
# define MXQ_LOGDIR LOCALSTATEDIR "/log"
#else
# define MXQ_LOGDIR "/var/log"
#endif

static void mxq_print_generic_version(void)
{
printf(
Expand Down
4 changes: 3 additions & 1 deletion mxq_group.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
#include "mx_util.h"
#include "mx_mysql.h"

#define GROUP_FIELDS_CNT 30
#define GROUP_FIELDS_CNT 31
#define GROUP_FIELDS \
" group_id," \
" group_name," \
Expand All @@ -37,6 +37,7 @@
" group_jobs_unknown," \
" group_jobs_restarted," \
" group_slots_running," \
" stats_max_sumrss," \
" stats_max_maxrss," \
" stats_max_utime_sec," \
" stats_max_stime_sec," \
Expand Down Expand Up @@ -85,6 +86,7 @@ static int bind_result_group_fields(struct mx_mysql_bind *result, struct mxq_gro

res += mx_mysql_bind_var(result, idx++, uint64, &(g->group_slots_running));

res += mx_mysql_bind_var(result, idx++, uint64, &(g->stats_max_sumrss));
res += mx_mysql_bind_var(result, idx++, uint64, &(g->stats_max_maxrss));
res += mx_mysql_bind_var(result, idx++, int64, &(g->stats_max_utime.tv_sec));
res += mx_mysql_bind_var(result, idx++, int64, &(g->stats_max_stime.tv_sec));
Expand Down
1 change: 1 addition & 0 deletions mxq_group.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@ struct mxq_group {

uint64_t group_slots_running;

uint64_t stats_max_sumrss;
uint64_t stats_max_maxrss;

struct timeval stats_max_utime;
Expand Down
73 changes: 35 additions & 38 deletions mxq_job.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,44 +16,39 @@
#include "mxq_group.h"
#include "mxq_job.h"

#define JOB_FIELDS_CNT 35
#define JOB_FIELDS_CNT 36
#define JOB_FIELDS \
" job_id, " \
" job_status, " \
" job_flags, " \
" job_priority, " \
" group_id, " \
\
" job_workdir, " \
" job_argc, " \
" job_argv, " \
" job_stdout, " \
" job_stderr, " \
\
" job_umask, " \
" host_submit, " \
" host_id, " \
" server_id, " \
" host_hostname, " \
\
" host_pid, " \
" host_slots, " \
" UNIX_TIMESTAMP(date_submit) as date_submit, " \
" UNIX_TIMESTAMP(date_start) as date_start, " \
" UNIX_TIMESTAMP(date_end) as date_end, " \
\
" stats_max_sumrss, " \
" stats_status, " \
" stats_utime_sec, " \
" stats_utime_usec, " \
" stats_stime_sec, " \
" stats_stime_usec, " \
\
" stats_real_sec, " \
" stats_real_usec, " \
" stats_maxrss, " \
" stats_minflt, " \
" stats_majflt, " \
\
" stats_nswap, " \
" stats_inblock, " \
" stats_oublock, " \
Expand All @@ -73,37 +68,32 @@ static int bind_result_job_fields(struct mx_mysql_bind *result, struct mxq_job *
res += mx_mysql_bind_var(result, idx++, uint64, &(j->job_flags));
res += mx_mysql_bind_var(result, idx++, uint16, &(j->job_priority));
res += mx_mysql_bind_var(result, idx++, uint64, &(j->group_id));

res += mx_mysql_bind_var(result, idx++, string, &(j->job_workdir));
res += mx_mysql_bind_var(result, idx++, uint16, &(j->job_argc));
res += mx_mysql_bind_var(result, idx++, string, &(j->job_argv_str));
res += mx_mysql_bind_var(result, idx++, string, &(j->job_stdout));
res += mx_mysql_bind_var(result, idx++, string, &(j->job_stderr));

res += mx_mysql_bind_var(result, idx++, uint32, &(j->job_umask));
res += mx_mysql_bind_var(result, idx++, string, &(j->host_submit));
res += mx_mysql_bind_var(result, idx++, string, &(j->host_id));
res += mx_mysql_bind_var(result, idx++, string, &(j->server_id));
res += mx_mysql_bind_var(result, idx++, string, &(j->host_hostname));

res += mx_mysql_bind_var(result, idx++, uint32, &(j->host_pid));
res += mx_mysql_bind_var(result, idx++, uint32, &(j->host_slots));
res += mx_mysql_bind_var(result, idx++, int64, &(j->date_submit));
res += mx_mysql_bind_var(result, idx++, int64, &(j->date_start));
res += mx_mysql_bind_var(result, idx++, int64, &(j->date_end));

res += mx_mysql_bind_var(result, idx++, uint64, &(j->stats_max_sumrss));
res += mx_mysql_bind_var(result, idx++, int32, &(j->stats_status));
res += mx_mysql_bind_var(result, idx++, int64, &(j->stats_rusage.ru_utime.tv_sec));
res += mx_mysql_bind_var(result, idx++, int64, &(j->stats_rusage.ru_utime.tv_usec));
res += mx_mysql_bind_var(result, idx++, int64, &(j->stats_rusage.ru_stime.tv_sec));
res += mx_mysql_bind_var(result, idx++, int64, &(j->stats_rusage.ru_stime.tv_usec));

res += mx_mysql_bind_var(result, idx++, int64, &(j->stats_realtime.tv_sec));
res += mx_mysql_bind_var(result, idx++, int64, &(j->stats_realtime.tv_usec));
res += mx_mysql_bind_var(result, idx++, int64, &(j->stats_rusage.ru_maxrss));
res += mx_mysql_bind_var(result, idx++, int64, &(j->stats_rusage.ru_minflt));
res += mx_mysql_bind_var(result, idx++, int64, &(j->stats_rusage.ru_majflt));

res += mx_mysql_bind_var(result, idx++, int64, &(j->stats_rusage.ru_nswap));
res += mx_mysql_bind_var(result, idx++, int64, &(j->stats_rusage.ru_inblock));
res += mx_mysql_bind_var(result, idx++, int64, &(j->stats_rusage.ru_oublock));
Expand Down Expand Up @@ -457,6 +447,7 @@ int mxq_set_job_status_running(struct mx_mysql *mysql, struct mxq_job *job)
int mxq_set_job_status_exited(struct mx_mysql *mysql, struct mxq_job *job)
{
int res;
int idx;
uint16_t newstatus;
struct mx_mysql_bind param = {0};

Expand Down Expand Up @@ -485,6 +476,7 @@ int mxq_set_job_status_exited(struct mx_mysql *mysql, struct mxq_job *job)
"UPDATE mxq_job SET"
" job_status = ?,"
" date_end = NULL,"
" stats_max_sumrss = ?, "
" stats_status = ?, "
" stats_utime_sec = ?, "
" stats_utime_usec = ?, "
Expand All @@ -506,30 +498,32 @@ int mxq_set_job_status_exited(struct mx_mysql *mysql, struct mxq_job *job)
" AND server_id = ?"
" AND host_pid = ?";

res = mx_mysql_bind_init_param(&param, 20);
res = mx_mysql_bind_init_param(&param, 21);
assert(res == 0);

idx = 0;
res = 0;
res += mx_mysql_bind_var(&param, 0, uint16, &(newstatus));
res += mx_mysql_bind_var(&param, 1, int32, &(job->stats_status));
res += mx_mysql_bind_var(&param, 2, int64, &(job->stats_rusage.ru_utime.tv_sec));
res += mx_mysql_bind_var(&param, 3, int64, &(job->stats_rusage.ru_utime.tv_usec));
res += mx_mysql_bind_var(&param, 4, int64, &(job->stats_rusage.ru_stime.tv_sec));
res += mx_mysql_bind_var(&param, 5, int64, &(job->stats_rusage.ru_stime.tv_usec));
res += mx_mysql_bind_var(&param, 6, int64, &(job->stats_realtime.tv_sec));
res += mx_mysql_bind_var(&param, 7, int64, &(job->stats_realtime.tv_usec));
res += mx_mysql_bind_var(&param, 8, int64, &(job->stats_rusage.ru_maxrss));
res += mx_mysql_bind_var(&param, 9, int64, &(job->stats_rusage.ru_minflt));
res += mx_mysql_bind_var(&param, 10, int64, &(job->stats_rusage.ru_majflt));
res += mx_mysql_bind_var(&param, 11, int64, &(job->stats_rusage.ru_nswap));
res += mx_mysql_bind_var(&param, 12, int64, &(job->stats_rusage.ru_inblock));
res += mx_mysql_bind_var(&param, 13, int64, &(job->stats_rusage.ru_oublock));
res += mx_mysql_bind_var(&param, 14, int64, &(job->stats_rusage.ru_nvcsw));
res += mx_mysql_bind_var(&param, 15, int64, &(job->stats_rusage.ru_nivcsw));
res += mx_mysql_bind_var(&param, 16, uint64, &(job->job_id));
res += mx_mysql_bind_var(&param, 17, string, &(job->host_hostname));
res += mx_mysql_bind_var(&param, 18, string, &(job->server_id));
res += mx_mysql_bind_var(&param, 19, uint32, &(job->host_pid));
res += mx_mysql_bind_var(&param, idx++, uint16, &(newstatus));
res += mx_mysql_bind_var(&param, idx++, uint64, &(job->stats_max_sumrss));
res += mx_mysql_bind_var(&param, idx++, int32, &(job->stats_status));
res += mx_mysql_bind_var(&param, idx++, int64, &(job->stats_rusage.ru_utime.tv_sec));
res += mx_mysql_bind_var(&param, idx++, int64, &(job->stats_rusage.ru_utime.tv_usec));
res += mx_mysql_bind_var(&param, idx++, int64, &(job->stats_rusage.ru_stime.tv_sec));
res += mx_mysql_bind_var(&param, idx++, int64, &(job->stats_rusage.ru_stime.tv_usec));
res += mx_mysql_bind_var(&param, idx++, int64, &(job->stats_realtime.tv_sec));
res += mx_mysql_bind_var(&param, idx++, int64, &(job->stats_realtime.tv_usec));
res += mx_mysql_bind_var(&param, idx++, int64, &(job->stats_rusage.ru_maxrss));
res += mx_mysql_bind_var(&param, idx++, int64, &(job->stats_rusage.ru_minflt));
res += mx_mysql_bind_var(&param, idx++, int64, &(job->stats_rusage.ru_majflt));
res += mx_mysql_bind_var(&param, idx++, int64, &(job->stats_rusage.ru_nswap));
res += mx_mysql_bind_var(&param, idx++, int64, &(job->stats_rusage.ru_inblock));
res += mx_mysql_bind_var(&param, idx++, int64, &(job->stats_rusage.ru_oublock));
res += mx_mysql_bind_var(&param, idx++, int64, &(job->stats_rusage.ru_nvcsw));
res += mx_mysql_bind_var(&param, idx++, int64, &(job->stats_rusage.ru_nivcsw));
res += mx_mysql_bind_var(&param, idx++, uint64, &(job->job_id));
res += mx_mysql_bind_var(&param, idx++, string, &(job->host_hostname));
res += mx_mysql_bind_var(&param, idx++, string, &(job->server_id));
res += mx_mysql_bind_var(&param, idx++, uint32, &(job->host_pid));
assert(res == 0);

res = mx_mysql_do_statement_noresult_retry_on_fail(mysql, query, &param);
Expand Down Expand Up @@ -607,7 +601,7 @@ int mxq_job_set_tmpfilenames(struct mxq_group *g, struct mxq_job *j)
return 1;
}

int mxq_load_job_assigned_to_server(struct mx_mysql *mysql, struct mxq_job **mxq_jobs, char *hostname, char *server_id)
int mxq_load_job_from_group_assigned_to_server(struct mx_mysql *mysql, struct mxq_job **mxq_jobs, uint64_t group_id, char *hostname, char *server_id)
{
int res;
struct mxq_job *jobs = NULL;
Expand All @@ -630,14 +624,16 @@ int mxq_load_job_assigned_to_server(struct mx_mysql *mysql, struct mxq_job **mxq
" WHERE job_status = " status_str(MXQ_JOB_STATUS_ASSIGNED)
" AND host_hostname = ?"
" AND server_id = ?"
" AND group_id = ?"
" LIMIT 1";

res = mx_mysql_bind_init_param(&param, 2);
res = mx_mysql_bind_init_param(&param, 3);
assert(res == 0);

res = 0;
res += mx_mysql_bind_var(&param, 0, string, &hostname);
res += mx_mysql_bind_var(&param, 1, string, &server_id);
res += mx_mysql_bind_var(&param, 2, uint64, &group_id);
assert(res == 0);

res = bind_result_job_fields(&result, &j);
Expand Down Expand Up @@ -668,14 +664,15 @@ int mxq_load_job_from_group_for_server(struct mx_mysql *mysql, struct mxq_job *m
assert(*host_id);

do {
res = mxq_load_job_assigned_to_server(mysql, &jobs, hostname, server_id);
res = mxq_load_job_from_group_assigned_to_server(mysql, &jobs, group_id, hostname, server_id);

if(res < 0) {
mx_log_err(" group_id=%lu :: mxq_load_job_assigned_to_server: %m", group_id);
mx_log_err(" group_id=%lu :: mxq_load_job_from_group_assigned_to_server: %m", group_id);
return 0;
}
if(res == 1) {
memcpy(mxqjob, &jobs[0], sizeof(*mxqjob));
free(jobs);
break;
}

Expand Down
4 changes: 3 additions & 1 deletion mxq_job.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,8 @@ struct mxq_job {

struct timeval stats_starttime;

uint64_t stats_max_sumrss;

int32_t stats_status;
struct timeval stats_realtime;
struct rusage stats_rusage;
Expand Down Expand Up @@ -103,7 +105,7 @@ int mxq_set_job_status_running(struct mx_mysql *mysql, struct mxq_job *job);
int mxq_set_job_status_exited(struct mx_mysql *mysql, struct mxq_job *job);
int mxq_set_job_status_unknown_for_server(struct mx_mysql *mysql, char *hostname, char *server_id);
int mxq_job_set_tmpfilenames(struct mxq_group *g, struct mxq_job *j);
int mxq_load_job_assigned_to_server(struct mx_mysql *mysql, struct mxq_job **mxq_jobs, char *hostname, char *server_id);
int mxq_load_job_from_group_assigned_to_server(struct mx_mysql *mysql, struct mxq_job **mxq_jobs, uint64_t group_id, char *hostname, char *server_id);
int mxq_load_job_from_group_for_server(struct mx_mysql *mysql, struct mxq_job *mxqjob, uint64_t group_id, char *hostname, char *server_id, char *host_id);

#endif
Loading

0 comments on commit a9362b2

Please sign in to comment.