diff --git a/.gitignore b/.gitignore index e9d5b00d..4599a3ca 100644 --- a/.gitignore +++ b/.gitignore @@ -21,6 +21,9 @@ test_mxqd_control.o mxq_log.o mx_mysql.o mxqd_control.o +keywordset.o +test_keywordset.o +mxqset.o mxqsub @@ -30,10 +33,12 @@ mxqdump mxqkill mxqd mxqps +mxqset test_mx_util test_mx_log test_mx_mysq test_mxqd_control +test_keywordset /web/pages/mxq/mxq web/lighttpd.conf diff --git a/.vimrc b/.vimrc new file mode 100644 index 00000000..cd72c889 --- /dev/null +++ b/.vimrc @@ -0,0 +1,5 @@ +set tabstop=4 +set shiftwidth=4 +set softtabstop=4 +set expandtab +set nosmarttab diff --git a/Makefile b/Makefile index 6e14493d..6aca8321 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ MXQ_VERSION_MAJOR = 0 MXQ_VERSION_MINOR = 26 -MXQ_VERSION_PATCH = 1 +MXQ_VERSION_PATCH = 2 MXQ_VERSION_EXTRA = "beta" MXQ_VERSIONDATE = 2016 @@ -341,6 +341,10 @@ mxqd_control.h += mxqd.h mx_getopt.h += mx_getopt.h +### keywordset.h ------------------------------------------------------- + +keywordset.h += keywordset.h + ######################################################################## ### mx_getopt.o -------------------------------------------------------- @@ -412,6 +416,16 @@ mxqadmin.o: CFLAGS += $(CFLAGS_MYSQL) clean: CLEAN += mxqadmin.o +### mxqset.o ---------------------------------------------------------- + +mxqsset.o: $(mx_mysql.h) +mxqsset.o: $(keywordset.h) +mxqkill.o: $(mxq.h) +mxqkill.o: $(mxq_group.h) +mxqset.o: CFLAGS += $(CFLAGS_MYSQL) + +clean: CLEAN += mxqsset.o + ### mxqkill.o ---------------------------------------------------------- mxqkill.o: $(mx_log.h) @@ -472,6 +486,7 @@ mxqd.o: $(mxq_daemon.h) mxqd.o: $(mxq_group.h) mxqd.o: $(mxq_job.h) mxqd.o: $(mx_mysql.h) +mxqd.o: $(keywordset.h) mxqd.o: CFLAGS += $(CFLAGS_MYSQL) mxqd.o: CFLAGS += $(CFLAGS_MXQ_INITIAL_PATH) mxqd.o: CFLAGS += $(CFLAGS_MXQ_INITIAL_TMPDIR) @@ -490,10 +505,18 @@ mxqsub.o: $(mxq.h) mxqsub.o: $(mxq_group.h) mxqsub.o: $(mxq_job.h) mxqsub.o: $(mx_util.h) +mxqsub.o: $(keywordset.h) mxqsub.o: CFLAGS += $(CFLAGS_MYSQL) clean: CLEAN += mxqsub.o +### keywordset.o ----------------------------------------------------- + +keywordset.o: $(keywordset.h) +keywordset.o: xmalloc.h + +clean: CLEAN += keywordset.o + ######################################################################## ### mxqd --------------------------------------------------------------- @@ -509,6 +532,7 @@ mxqd: mxq_group.o mxqd: mxq_job.o mxqd: mx_mysql.o mxqd: mxqd_control.o +mxqd: keywordset.o mxqd: LDLIBS += $(LDLIBS_MYSQL) build: mxqd @@ -524,6 +548,7 @@ mxqsub: mx_getopt.o mxqsub: mx_util.o mxqsub: mx_log.o mxqsub: mx_mysql.o +mxqsub: keywordset.o mxqsub: LDLIBS += $(LDLIBS_MYSQL) build: mxqsub @@ -566,6 +591,21 @@ clean: CLEAN += mxqadmin install:: mxqadmin $(call quiet-installforuser,$(SUID_MODE),$(UID_CLIENT),$(GID_CLIENT),mxqadmin,${DESTDIR}${BINDIR}/mxqadmin) +### mxqset --------------------------------------------------------------- + +mxqset: mx_mysql.o +mxqset: mx_log.o +mxqset: mx_util.o +mxqset: keywordset.o +mxqset: LDLIBS += $(LDLIBS_MYSQL) + +build: mxqset + +clean: CLEAN += mxqset + +install:: mxqset + $(call quiet-installforuser,$(SUID_MODE),$(UID_CLIENT),$(GID_CLIENT),mxqset,${DESTDIR}${BINDIR}/mxqset) + ### mxqkill ------------------------------------------------------------ mxqkill: mx_log.o @@ -678,3 +718,9 @@ test_mxqd_control: LDLIBS += $(LDLIBS_MYSQL) clean: CLEAN += test_mxqd_control test: test_mxqd_control + +test_keywordset: $(test_keywordset.h) +test_keywordset: test_keywordset.o +test_keywordset: keywordset.o +clean: CLEAN += test_keywordset.o +test: test_keywordset diff --git a/keywordset.c b/keywordset.c new file mode 100644 index 00000000..b95adb5c --- /dev/null +++ b/keywordset.c @@ -0,0 +1,162 @@ +#include "keywordset.h" +#include +#include +#include +#include +#include "xmalloc.h" + +#define KEYWORDSET_INITIAL_SLOTS (4-2) + +struct keywordset { + int nr_slots; + int used; + char **names; +}; + +static int find_name(struct keywordset *kws, char *name, size_t len) { + int i; + int j; + for ( i = 0; i < kws->used ; i++ ) { + j = 0; + while(1) { + if (kws->names[i][j] == 0) + break; + if (kws->names[i][j] != name[j]) + break; + j++; + if (j==len) + return i; + } + } + return -1; +} + +static void expand(struct keywordset *kws) { + int new_slots=(kws->nr_slots+2)*2-2; + kws->names=xrealloc(kws->names,new_slots*sizeof(*kws->names)); + kws->nr_slots=new_slots; +} + +static void add_name(struct keywordset *kws, char *name, size_t len) { + int i=find_name(kws, name, len); + if (i>=0) { + free(kws->names[i]); + kws->names[i] = xstrndup(name, len); + } else { + if (kws->used == kws->nr_slots) + expand(kws); + kws->names[kws->used++] = xstrndup(name, len); + } +} + +static void remove_name(struct keywordset *kws, char *name, size_t len) { + int i=find_name(kws, name, len); + if (i>=0) { + free(kws->names[i]); + memmove(&(kws->names[i]), &(kws->names[i+1]), (kws->used-i-1)*sizeof(*kws->names)); + kws->used--; + } +} + +void keywordset_purge(struct keywordset *kws) { + int i; + for ( i = 0 ; i < kws->used ; i++) + free(kws->names[i]); + kws->used = 0; +} + +enum PHASE { + PHASE_SET, + PHASE_UPDATE, +}; + +static void keywordset_update_phase(struct keywordset *kws, char *input, enum PHASE phase) { + char *c=input; + char *name_start; + char action; + int purged=0; + while (*c) { + while (*c && isspace(*c)) + c++; + + if (*c == '+' || *c == '-') { + action = *c; + c++; + } else { + action = ' '; + } + + if (*c) { + name_start=c++; + while (*c && !isspace(*c)) + c++; + if (phase == PHASE_SET && action==' ') { + if (!purged) { + keywordset_purge(kws); + purged = 1; + } + add_name(kws, name_start, c-name_start); + } else if (phase == PHASE_UPDATE) { + if (action == '+') + add_name(kws, name_start, c-name_start); + else if (action == '-') + remove_name(kws, name_start, c-name_start); + } + } + } +} + +void keywordset_update(struct keywordset *kws, char *input) { + keywordset_update_phase(kws, input, PHASE_SET); + keywordset_update_phase(kws, input, PHASE_UPDATE); +} + +struct keywordset *keywordset_new(char *input) { + struct keywordset *kws = xmalloc(sizeof(*kws)); + kws->nr_slots = KEYWORDSET_INITIAL_SLOTS; + kws->used = 0; + kws->names = xmalloc(KEYWORDSET_INITIAL_SLOTS*sizeof(*kws->names)); + if (input) + keywordset_update(kws, input); + return kws; +} + +static int cmp(const void *a, const void *b) { + return strcmp(*(char **)a, *(char **)b); +} + +char *keywordset_get(struct keywordset *kws) { + char **names=xmalloc(kws->used * sizeof(*names)); + memcpy(names, kws->names, kws->used * sizeof(*names)); + qsort(names, kws->used, sizeof(*names), cmp); + size_t len = 0; + int i; + for (i=0; iused; i++) { + len += strlen(names[i]); + } + size_t outlen = len + (kws->used >= 2 ? kws->used-1 : 0); + char *out=xmalloc(outlen + 1 ); + char *p=out; + for ( i = 0 ; i < kws->used ; i++) { + p=stpcpy(p, names[i]); + *p++ = ' '; + } + out[outlen] = 0; + free(names); + return(out); +} + +int keywordset_ismember(struct keywordset *kws, char *name) { + if (find_name(kws, name, strlen(name)) >= 0) + return 1; + else + return 0; +} + +void keywordset_free(struct keywordset *kws) { + int i; + for ( i = 0 ; i < kws->used ; i++) + free(kws->names[i]); + free(kws->names); + free(kws); +} diff --git a/keywordset.h b/keywordset.h new file mode 100644 index 00000000..7484235b --- /dev/null +++ b/keywordset.h @@ -0,0 +1,11 @@ +#ifndef _KEYWORDSET_H +#define _KEYWORDSET_H + +struct keywordset *keywordset_new(char *input); +void keywordset_update(struct keywordset *kws, char *input); +char *keywordset_get(struct keywordset *kws); +int keywordset_ismember(struct keywordset *kws, char *name); +void keywordset_purge(struct keywordset *kws); +void keywordset_free(struct keywordset *kws); + +#endif diff --git a/mxq.h b/mxq.h index 418331d2..39c460db 100644 --- a/mxq.h +++ b/mxq.h @@ -60,7 +60,7 @@ # define MXQ_MAX_PENDING_JOBS_PER_GROUP 10000 #endif -static void mxq_print_generic_version(void) +__attribute__ ((unused)) static void mxq_print_generic_version(void) { printf( "%s - " MXQ_VERSIONFULL "\n" diff --git a/mxq_group.c b/mxq_group.c index 34f8c7ac..25daa998 100644 --- a/mxq_group.c +++ b/mxq_group.c @@ -12,13 +12,15 @@ #include "mx_util.h" #include "mx_mysql.h" -#define GROUP_FIELDS_CNT 33 +#define GROUP_FIELDS_CNT 35 #define GROUP_FIELDS \ " group_id," \ " group_name," \ " group_status," \ " group_flags," \ " group_priority," \ + " group_blacklist," \ + " group_whitelist," \ " user_uid," \ " user_name," \ " user_gid," \ @@ -62,6 +64,8 @@ static int bind_result_group_fields(struct mx_mysql_bind *result, struct mxq_gro res += mx_mysql_bind_var(result, idx++, uint8, &(g->group_status)); res += mx_mysql_bind_var(result, idx++, uint64, &(g->group_flags)); res += mx_mysql_bind_var(result, idx++, uint16, &(g->group_priority)); + res += mx_mysql_bind_var(result, idx++, string, &(g->group_blacklist)); + res += mx_mysql_bind_var(result, idx++, string, &(g->group_whitelist)); res += mx_mysql_bind_var(result, idx++, uint32, &(g->user_uid)); res += mx_mysql_bind_var(result, idx++, string, &(g->user_name)); @@ -105,6 +109,8 @@ static int bind_result_group_fields(struct mx_mysql_bind *result, struct mxq_gro void mxq_group_free_content(struct mxq_group *g) { + mx_free_null(g->group_whitelist); + mx_free_null(g->group_blacklist); mx_free_null(g->group_name); mx_free_null(g->user_name); mx_free_null(g->user_group); diff --git a/mxq_group.h b/mxq_group.h index 762b04ae..d8fbd733 100644 --- a/mxq_group.h +++ b/mxq_group.h @@ -14,6 +14,8 @@ struct mxq_group { uint8_t group_status; uint64_t group_flags; uint16_t group_priority; + char * group_blacklist; + char * group_whitelist; uint32_t user_uid; char * user_name; diff --git a/mxqd.c b/mxqd.c index aef3ab0d..73d2eaaa 100644 --- a/mxqd.c +++ b/mxqd.c @@ -43,6 +43,7 @@ #include "mxq.h" #include "mxqd_control.h" +#include "keywordset.h" #ifndef MXQ_INITIAL_PATH # define MXQ_INITIAL_PATH "/sbin:/bin:/usr/sbin:/usr/bin:/usr/local/sbin:/usr/local/bin" @@ -546,6 +547,16 @@ int server_init(struct mxq_server *server, int argc, char *argv[]) } server->hostname = arg_hostname; + { + char *dot=index(arg_hostname,'.'); + if (dot) { + server->hostname_short = mx_malloc_forever(dot-arg_hostname+1); + strncpy(server->hostname_short, arg_hostname, dot-arg_hostname); + server->hostname_short[dot-arg_hostname] = 0; + } else + server->hostname_short = mx_strdup_forever(arg_hostname); + } + server->daemon_name = arg_daemon_name; server->initial_path = arg_initial_path; server->initial_tmpdir = arg_initial_tmpdir; @@ -1277,6 +1288,39 @@ unsigned long start_job(struct mxq_group_list *glist) /**********************************************************************/ +static int server_is_qualified(struct mxq_server *server, struct mxq_group *group) { + + int is_qualified; + + if (*group->group_whitelist != 0) { + is_qualified = 0; + struct keywordset *kws = keywordset_new(group->group_whitelist); + if ( keywordset_ismember(kws, server->hostname_short) + || keywordset_ismember(kws, server->hostname) ) + is_qualified = 1; + keywordset_free(kws); + } else { + is_qualified = 1; + } + + if (*group->group_blacklist != 0) { + struct keywordset *kws = keywordset_new(group->group_blacklist); + if ( keywordset_ismember(kws, server->hostname_short) + || keywordset_ismember(kws, server->hostname) ) + is_qualified = 0; + keywordset_free(kws); + } + return (is_qualified); +} + +static int server_is_qualified_cached(struct mxq_server *server, struct mxq_group_list *glist) { + if (!glist->server_is_qualified_evaluated) { + glist->server_is_qualified = server_is_qualified(server, &glist->group); + glist->server_is_qualified_evaluated = 1; + } + return glist->server_is_qualified; +} + unsigned long start_user(struct mxq_user_list *ulist, long slots_to_start) { struct mxq_server *server; @@ -1319,6 +1363,8 @@ unsigned long start_user(struct mxq_user_list *ulist, long slots_to_start) if (df_scratch/1024/1024/1024 < group->job_tmpdir_size + 20) { continue; } + if (!server_is_qualified_cached(server, glist)) + continue; mx_log_info(" group=%s(%d):%lu slots_to_start=%ld slots_per_job=%lu :: trying to start job for group.", group->user_name, group->user_uid, group->group_id, slots_to_start, glist->slots_per_job); @@ -1472,6 +1518,7 @@ void server_free(struct mxq_server *server) mx_free_null(server->finished_jobsdir); mx_flock_free(server->flock); mx_free_null(server->supgid); + mx_free_null(server->hostname_short); mx_log_finish(); } @@ -2308,10 +2355,8 @@ static void server_umount_stale_job_mountdirs(struct mxq_server *server) { int i; entries=scandir(MXQ_JOB_TMPDIR_MNTDIR,&namelist,&job_mountdirs_is_valid_name,&alphasort); - if (entries<0) { - mx_log_err("scandir %s: %m", MXQ_JOB_TMPDIR_MNTDIR); + if (entries<=0) return; - } for (i=0;id_name, &job_id)) { if (server_get_job_list_by_job_id(server, job_id) == NULL) { diff --git a/mxqd.h b/mxqd.h index 5c9ce7e3..87aa2236 100644 --- a/mxqd.h +++ b/mxqd.h @@ -43,6 +43,9 @@ struct mxq_group_list { unsigned long global_threads_running; unsigned long global_slots_running; + int server_is_qualified_evaluated; + int server_is_qualified; + short orphaned; }; @@ -95,6 +98,7 @@ struct mxq_server { unsigned long long int starttime; char *host_id; char *hostname; + char *hostname_short; char *daemon_name; char *pidfilename; char *finished_jobsdir; diff --git a/mxqd_control.c b/mxqd_control.c index 36814e26..bbbd8549 100644 --- a/mxqd_control.c +++ b/mxqd_control.c @@ -103,6 +103,9 @@ static void _group_list_init(struct mxq_group_list *glist) glist->slots_max = slots_max; glist->memory_max = memory_max; + glist->server_is_qualified_evaluated = 0; + glist->server_is_qualified = 0; + glist->orphaned = 0; } diff --git a/mxqdump.c b/mxqdump.c index 8a39e9e2..ace53d47 100644 --- a/mxqdump.c +++ b/mxqdump.c @@ -135,6 +135,8 @@ static int print_group(struct mxq_group *g) " idle_sec=%lu" " job_command=%s" " group_name=%s" + " blacklist='%s'" + " whitelist='%s'" "\n", g->user_name, g->user_uid, @@ -163,7 +165,9 @@ static int print_group(struct mxq_group *g) g->stats_run_sec, g->stats_idle_sec, g->job_command, - g->group_name); + g->group_name, + g->group_blacklist, + g->group_whitelist); } static int print_job(struct mxq_group *g, struct mxq_job *j) diff --git a/mxqset.c b/mxqset.c new file mode 100644 index 00000000..18cf375a --- /dev/null +++ b/mxqset.c @@ -0,0 +1,245 @@ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include "mx_mysql.h" +#include "mxq.h" +#include "mxq_group.h" +#include "keywordset.h" +#include "stdarg.h" + +static __attribute__ ((noreturn)) void die(char *msg, ...) { + va_list ap; + + va_start(ap, msg); + fprintf(stderr, "%s: ",program_invocation_short_name); + vfprintf(stderr, msg, ap); + va_end(ap); + exit(1); +} + +static void verify_group_permission(struct mx_mysql *mysql , long unsigned groupid, uid_t client_uid) { + struct mx_mysql_stmt *stmt = NULL; + unsigned long long num_rows; + unsigned uid; + + stmt = mx_mysql_statement_prepare(mysql,"SELECT user_uid FROM mxq_group WHERE group_id = ? LIMIT 1"); + if (!stmt) + die("%m\n"); + mx_mysql_statement_param_bind(stmt, 0, uint64, &(groupid)); + if (mx_mysql_statement_execute(stmt, &num_rows) < 0) + die("%m\n"); + if (num_rows < 1) + die("no such group %ld\n", groupid); + mx_mysql_statement_result_bind(stmt, 0, uint32, &uid); + if (mx_mysql_statement_fetch(stmt) < 0) + die("%m\n"); + if ( client_uid != 0 && client_uid != uid ) + die("no permission to access this group\n"); + mx_mysql_statement_close(&stmt); +} + +enum OPEN_CLOSED { + OPEN_CLOSED_UNSET = 0, + OPEN_CLOSED_CLOSED, + OPEN_CLOSED_OPEN, +}; + +static void update_closed_flag(struct mx_mysql *mysql, long unsigned groupid, enum OPEN_CLOSED open_closed) { + + struct mx_mysql_stmt *stmt = NULL; + unsigned long long num_rows; + + uint64_t mask = ~MXQ_GROUP_FLAG_CLOSED; + uint64_t value = open_closed == OPEN_CLOSED_CLOSED ? MXQ_GROUP_FLAG_CLOSED : 0; + + stmt = mx_mysql_statement_prepare(mysql, + "UPDATE mxq_group SET group_flags = ( group_flags & ?) | ? WHERE group_id = ?"); + if (!stmt) + die("%m\n"); + mx_mysql_statement_param_bind(stmt, 0, uint64, &(mask)); + mx_mysql_statement_param_bind(stmt, 1, uint64, &(value)); + mx_mysql_statement_param_bind(stmt, 2, uint64, &(groupid)); + if (mx_mysql_statement_execute(stmt, &num_rows) < 0) + die("%m\n"); + if (num_rows < 1) + die("no such group %ld\n", groupid); + mx_mysql_statement_close(&stmt); +} + +static void update_blacklist(struct mx_mysql *mysql, long unsigned groupid, char *update_blacklist) { + + struct mx_mysql_stmt *stmt = NULL; + unsigned long long num_rows; + char *group_blacklist = NULL; + struct keywordset *blacklist; + + if (strcmp(update_blacklist, "") !=0 ) { + stmt = mx_mysql_statement_prepare(mysql, + "SELECT group_blacklist FROM mxq_group WHERE group_id = ? LIMIT 1"); + if (!stmt) + die("%m\n"); + mx_mysql_statement_param_bind(stmt, 0, uint64, &groupid); + if (mx_mysql_statement_execute(stmt, &num_rows) < 0) + die("%m\n"); + if (num_rows < 1) + die("no such group %ld\n", groupid); + mx_mysql_statement_result_bind(stmt, 0, string, &group_blacklist); + if (mx_mysql_statement_fetch(stmt) < 0) + die("%m\n"); + mx_mysql_statement_close(&stmt); + blacklist = keywordset_new(group_blacklist); + free(group_blacklist); + } else { + blacklist = keywordset_new(NULL); + } + keywordset_update(blacklist, update_blacklist); + + group_blacklist = keywordset_get(blacklist); + + stmt = mx_mysql_statement_prepare(mysql, + "UPDATE mxq_group SET group_blacklist = ? WHERE group_id = ?"); + if (!stmt) + die("%m\n"); + mx_mysql_statement_param_bind(stmt, 0, string, &group_blacklist); + mx_mysql_statement_param_bind(stmt, 1, uint64, &groupid); + if (mx_mysql_statement_execute(stmt, &num_rows) < 0) + die("%m\n"); + if (num_rows < 1) + die("no such group %ld\n", groupid); + mx_mysql_statement_close(&stmt); + free(group_blacklist); + keywordset_free(blacklist); +} + +static void update_whitelist(struct mx_mysql *mysql, long unsigned groupid, char *update_whitelist) { + + struct mx_mysql_stmt *stmt = NULL; + unsigned long long num_rows; + char *group_whitelist = NULL; + struct keywordset *whitelist; + + if (strcmp(update_whitelist, "") !=0 ) { + stmt = mx_mysql_statement_prepare(mysql, + "SELECT group_whitelist FROM mxq_group WHERE group_id = ? LIMIT 1"); + if (!stmt) + die("%m\n"); + mx_mysql_statement_param_bind(stmt, 0, uint64, &groupid); + if (mx_mysql_statement_execute(stmt, &num_rows) < 0) + die("%m\n"); + if (num_rows < 1) + die("no such group %ld\n", groupid); + mx_mysql_statement_result_bind(stmt, 0, string, &group_whitelist); + if (mx_mysql_statement_fetch(stmt) < 0) + die("%m\n"); + mx_mysql_statement_close(&stmt); + whitelist = keywordset_new(group_whitelist); + free(group_whitelist); + } else { + whitelist = keywordset_new(NULL); + } + keywordset_update(whitelist, update_whitelist); + + group_whitelist = keywordset_get(whitelist); + + stmt = mx_mysql_statement_prepare(mysql, + "UPDATE mxq_group SET group_whitelist = ? WHERE group_id = ?"); + if (!stmt) + die("%m\n"); + mx_mysql_statement_param_bind(stmt, 0, string, &group_whitelist); + mx_mysql_statement_param_bind(stmt, 1, uint64, &groupid); + if (mx_mysql_statement_execute(stmt, &num_rows) < 0) + die("%m\n"); + if (num_rows < 1) + die("no such group %ld\n", groupid); + mx_mysql_statement_close(&stmt); + free(group_whitelist); + keywordset_free(whitelist); +} + +struct opts { + enum OPEN_CLOSED open_closed; + char *update_blacklist; + char *update_whitelist; +}; + +static error_t parser (int key, char *arg, struct argp_state *state) { + struct opts *opts = state->input; + switch (key) { + case 10: + opts->open_closed = OPEN_CLOSED_CLOSED; + return 0; + case 11: + opts->open_closed = OPEN_CLOSED_OPEN; + return 0; + case 13: + opts->update_blacklist = arg; + return 0; + case 15: + opts->update_whitelist = arg; + return 0; + } + return ARGP_ERR_UNKNOWN; +} + +static const struct argp_option options[] = { + {"closed", 10, NULL, 0, NULL}, + {"open", 11, NULL, 0, NULL}, + {"blacklist", 13, "", 0, NULL}, + {"whitelist", 15, "", 0, NULL}, + {0} +}; + +static const struct argp argp = { options, parser, NULL, NULL }; + +static __attribute__ ((noreturn)) void exit_usage(char *argv0) { + fprintf(stderr, +"usage: %s group GID [group-options]\n" +"\n" +"group options:\n" +"\n" +" --open\n" +" --closed\n" +" --blacklist=STRING\n" +" --whitelist=STRING\n" + ,program_invocation_short_name); + exit(EX_USAGE); +} + +int main(int argc, char **argv) { + + char *argv0=argv[0]; + struct mx_mysql *mysql = NULL; + int groupid; + uid_t uid = getuid(); + + if (argc<3 || strcmp(argv[1],"group") != 0) + exit_usage(argv0); + groupid=atoi(argv[2]); + + int sts; + struct opts opts={0}; + + sts=argp_parse (&argp, argc-3, &argv[3], ARGP_PARSE_ARGV0|ARGP_SILENT, NULL, &opts); + if (sts) + exit_usage(argv0); + + assert(mx_mysql_initialize(&mysql) == 0); + mx_mysql_option_set_default_file(mysql, MXQ_MYSQL_DEFAULT_FILE); + mx_mysql_option_set_default_group(mysql, MXQ_MYSQL_DEFAULT_GROUP); + assert(mx_mysql_connect_forever(&mysql) == 0); + + verify_group_permission(mysql, groupid, uid); + + if ( opts.open_closed != OPEN_CLOSED_UNSET) + update_closed_flag(mysql, groupid, opts.open_closed); + if ( opts.update_blacklist != NULL) + update_blacklist(mysql, groupid, opts.update_blacklist); + if ( opts.update_whitelist != NULL) + update_whitelist(mysql, groupid, opts.update_whitelist); + + mx_mysql_finish(&mysql); +} diff --git a/mxqsub.c b/mxqsub.c index e9900d11..5c66f8ad 100644 --- a/mxqsub.c +++ b/mxqsub.c @@ -32,6 +32,7 @@ #include "mx_util.h" #include "mx_getopt.h" #include "mx_mysql.h" +#include "keywordset.h" #include "mxq.h" @@ -71,7 +72,9 @@ static void print_usage(void) "\n" " -j, --threads=NUMBER set number of threads (default: 1)\n" " -m, --memory=SIZE set amount of memory (default: 2G)\n" - " --tmpdir=SIZE set size of MXQ_JOB_TMPDIR (default: 0)\n" + " --tmpdir=SIZE set size of MXQ_JOB_TMPDIR (default: 0)\n" + " --blacklist=STRING set list of blacklisted servers (default: '')\n" + " --whitelist=STRING set list of whitelisted servers (default: '')\n" "\n" " [SIZE] may be suffixed with a combination of T, G and M\n" " to specify tebibytes, gibibytes and mebibytes.\n" @@ -168,6 +171,8 @@ static int load_group_id(struct mx_mysql *mysql, struct mxq_group *g) " AND job_tmpdir_size = ?" " AND job_max_per_node = ?" " AND group_priority = ?" + " AND group_blacklist = ?" + " AND group_whitelist = ?" " AND group_status = 0" " AND group_flags & ? = 0 " " ORDER BY group_id DESC" @@ -189,7 +194,9 @@ static int load_group_id(struct mx_mysql *mysql, struct mxq_group *g) res += mx_mysql_statement_param_bind(stmt, 9, uint32, &(g->job_tmpdir_size)); res += mx_mysql_statement_param_bind(stmt, 10, uint16, &(g->job_max_per_node)); res += mx_mysql_statement_param_bind(stmt, 11, uint16, &(g->group_priority)); - res += mx_mysql_statement_param_bind(stmt, 12, uint64, &(flags)); + res += mx_mysql_statement_param_bind(stmt, 12, string, &(g->group_blacklist)); + res += mx_mysql_statement_param_bind(stmt, 13, string, &(g->group_whitelist)); + res += mx_mysql_statement_param_bind(stmt, 14, uint64, &(flags)); assert(res == 0); res = mx_mysql_statement_execute(stmt, &num_rows); @@ -253,6 +260,8 @@ static int load_group_id_by_group_id(struct mx_mysql *mysql, struct mxq_group *g " AND job_tmpdir_size = ?" " AND job_max_per_node = ?" " AND group_priority = ?" + " AND group_blacklist = ?" + " AND group_whitelist = ?" " AND group_status = 0" " AND group_id = ?" " AND group_flags & ? = 0 " @@ -275,8 +284,10 @@ static int load_group_id_by_group_id(struct mx_mysql *mysql, struct mxq_group *g res += mx_mysql_statement_param_bind(stmt, 9, uint32, &(g->job_tmpdir_size)); res += mx_mysql_statement_param_bind(stmt, 10, uint16, &(g->job_max_per_node)); res += mx_mysql_statement_param_bind(stmt, 11, uint16, &(g->group_priority)); - res += mx_mysql_statement_param_bind(stmt, 12, uint64, &(g->group_id)); - res += mx_mysql_statement_param_bind(stmt, 13, uint64, &(flags)); + res += mx_mysql_statement_param_bind(stmt, 12, string, &(g->group_blacklist)); + res += mx_mysql_statement_param_bind(stmt, 12, string, &(g->group_whitelist)); + res += mx_mysql_statement_param_bind(stmt, 14, uint64, &(g->group_id)); + res += mx_mysql_statement_param_bind(stmt, 15, uint64, &(flags)); assert(res == 0); res = mx_mysql_statement_execute(stmt, &num_rows); @@ -332,6 +343,8 @@ static int load_group_id_run_or_wait(struct mx_mysql *mysql, struct mxq_group *g " AND job_tmpdir_size = ?" " AND job_max_per_node = ?" " AND group_priority = ?" + " AND group_blacklist = ?" + " AND group_whitelist = ?" " AND group_status = 0" " AND (" "group_jobs_running > 0" @@ -358,7 +371,9 @@ static int load_group_id_run_or_wait(struct mx_mysql *mysql, struct mxq_group *g res += mx_mysql_statement_param_bind(stmt, 9, uint32, &(g->job_tmpdir_size)); res += mx_mysql_statement_param_bind(stmt, 10, uint16, &(g->job_max_per_node)); res += mx_mysql_statement_param_bind(stmt, 11, uint16, &(g->group_priority)); - res += mx_mysql_statement_param_bind(stmt, 12, uint64, &(flags)); + res += mx_mysql_statement_param_bind(stmt, 12, string, &(g->group_blacklist)); + res += mx_mysql_statement_param_bind(stmt, 13, string, &(g->group_whitelist)); + res += mx_mysql_statement_param_bind(stmt, 14, uint64, &(flags)); assert(res == 0); res = mx_mysql_statement_execute(stmt, &num_rows); @@ -415,7 +430,9 @@ static int add_group(struct mx_mysql *mysql, struct mxq_group *g) " job_time = ?," " job_tmpdir_size = ?," " job_max_per_node = ?," - " group_priority = ?"); + " group_priority = ?," + " group_blacklist = ?," + " group_whitelist = ?"); if (!stmt) { mx_log_err("mx_mysql_statement_prepare(): %m"); return -errno; @@ -433,6 +450,8 @@ static int add_group(struct mx_mysql *mysql, struct mxq_group *g) res += mx_mysql_statement_param_bind(stmt, 9, uint32, &(g->job_tmpdir_size)); res += mx_mysql_statement_param_bind(stmt,10, uint16, &(g->job_max_per_node)); res += mx_mysql_statement_param_bind(stmt,11, uint16, &(g->group_priority)); + res += mx_mysql_statement_param_bind(stmt,12, string, &(g->group_blacklist)); + res += mx_mysql_statement_param_bind(stmt,13, string, &(g->group_whitelist)); assert(res == 0); res = mx_mysql_statement_execute(stmt, &num_rows); @@ -639,6 +658,8 @@ int main(int argc, char *argv[]) u_int16_t arg_priority; char *arg_group_name; u_int16_t arg_group_priority; + char *arg_blacklist; + char *arg_whitelist; char *arg_program_name; u_int16_t arg_threads; u_int64_t arg_memory; @@ -659,6 +680,8 @@ int main(int argc, char *argv[]) _mx_cleanup_free_ char *arg_stdout_absolute = NULL; _mx_cleanup_free_ char *arg_stderr_absolute = NULL; _mx_cleanup_free_ char *arg_args = NULL; + _mx_cleanup_free_ char *blacklist = NULL; + _mx_cleanup_free_ char *whitelist = NULL; int flags = 0; @@ -713,6 +736,8 @@ int main(int argc, char *argv[]) MX_OPTION_OPTIONAL_ARG("mysql-default-file", 'M'), MX_OPTION_OPTIONAL_ARG("mysql-default-group", 'S'), MX_OPTION_REQUIRED_ARG("tmpdir", 7), + MX_OPTION_REQUIRED_ARG("blacklist", 8), + MX_OPTION_REQUIRED_ARG("whitelist", 9), MX_OPTION_END }; @@ -738,6 +763,8 @@ int main(int argc, char *argv[]) arg_jobflags = 0; arg_groupid = UINT64_UNSET; arg_tmpdir = 0; + arg_blacklist = NULL; + arg_whitelist = NULL; arg_mysql_default_group = getenv("MXQ_MYSQL_DEFAULT_GROUP"); if (!arg_mysql_default_group) @@ -942,6 +969,14 @@ int main(int argc, char *argv[]) } break; + case 8: + arg_blacklist = optctl.optarg; + break; + + case 9: + arg_whitelist = optctl.optarg; + break; + } } @@ -997,6 +1032,23 @@ int main(int argc, char *argv[]) arg_args = mx_strvec_to_str(argv); assert(arg_args); + if ( arg_blacklist != NULL ) { + struct keywordset *kws = keywordset_new(arg_blacklist); + blacklist = keywordset_get(kws); + keywordset_free(kws); + } else { + blacklist = mx_strdup_forever(""); + } + + if ( arg_whitelist != NULL ) { + struct keywordset *kws = keywordset_new(arg_whitelist); + whitelist = keywordset_get(kws); + keywordset_free(kws); + } else { + whitelist = mx_strdup_forever(""); + } + + /******************************************************************/ memset(&job, 0, sizeof(job)); @@ -1013,6 +1065,8 @@ int main(int argc, char *argv[]) group.job_memory = arg_memory; group.job_time = arg_time; group.job_tmpdir_size = arg_tmpdir; + group.group_blacklist = blacklist; + group.group_whitelist = whitelist; group.job_max_per_node = arg_max_per_node; diff --git a/mysql/create_tables.sql b/mysql/create_tables.sql index c3c0f9d2..3cf4f348 100644 --- a/mysql/create_tables.sql +++ b/mysql/create_tables.sql @@ -4,6 +4,8 @@ CREATE TABLE IF NOT EXISTS mxq_group ( group_status INT1 UNSIGNED NOT NULL DEFAULT 0, group_flags INT8 UNSIGNED NOT NULL DEFAULT 0, group_priority INT2 UNSIGNED NOT NULL DEFAULT 127, + group_blacklist VARCHAR(1000) NOT NULL DEFAULT '', + group_whitelist VARCHAR(1000) NOT NULL DEFAULT '', user_uid INT4 UNSIGNED NOT NULL, user_name VARCHAR(256) NOT NULL, diff --git a/test_keywordset.c b/test_keywordset.c new file mode 100644 index 00000000..df6e8345 --- /dev/null +++ b/test_keywordset.c @@ -0,0 +1,50 @@ +#include +#include +#include "keywordset.h" +#include +#include + +static void test_new(char *init, char *expect) { + struct keywordset *kws = keywordset_new(init); + char *s = keywordset_get(kws); + if (strcmp(s, expect)) + fprintf(stderr, "FAIL: new from '%s' got '%s' expected '%s'\n", init, s, expect); \ + free(s); + keywordset_free(kws); +} + +static void test_update(struct keywordset *kws, char *update, char *expect) { + char *init = keywordset_get(kws); + keywordset_update(kws, update); + char *s = keywordset_get(kws); + if (strcmp(s, expect)) + fprintf(stderr, "FAIL: update '%s' with '%s' got '%s' expected '%s'\n", init, update, s, expect); + free(s); + free(init); +} + +int main() { + + test_new(NULL, ""); + test_new("", ""); + test_new("h7 h8 h9 h1 h2 h3", "h1 h2 h3 h7 h8 h9"); + test_new(" h7\th8 h9 h1 h2 h3 \n", "h1 h2 h3 h7 h8 h9"); + + struct keywordset *kws = keywordset_new("a b c"); + + test_update(kws, "d e f", "d e f"); + test_update(kws, "+g +h +i", "d e f g h i"); + test_update(kws, "-e -h", "d f g i"); + test_update(kws, "-i +x", "d f g x"); + test_update(kws, "-x +m +n -n x y +a", "a m y"); + + test_update(kws, "-x +x", "a m x y"); + test_update(kws, "+z -z", "a m x y"); + + keywordset_purge(kws); + test_update(kws, "", ""); + keywordset_purge(kws); + test_update(kws, "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"); + + keywordset_free(kws); +} diff --git a/web/pages/mxq/mxq.in b/web/pages/mxq/mxq.in index 8a7bca74..47e672fa 100755 --- a/web/pages/mxq/mxq.in +++ b/web/pages/mxq/mxq.in @@ -392,6 +392,8 @@ group_name : $group_name group_status : $group_status_text group_flags : $o->{group_flags} group_priority : $o->{group_priority} +group_blacklist: $o->{group_blacklist} +group_whitelist: $o->{group_whitelist} user_uid : $o->{user_uid} user_name : $o->{user_name} diff --git a/xmalloc.h b/xmalloc.h new file mode 100644 index 00000000..23705bdd --- /dev/null +++ b/xmalloc.h @@ -0,0 +1,33 @@ +#ifndef _XMALLOC_H +#define _XMALLOC_H 1 + +#include +#include + +__attribute__ ((noreturn, unused)) static void out_of_memory() { + fprintf(stderr,"out of memory\n"); + abort(); +} + +__attribute__ ((unused)) static void *xmalloc(size_t size) { + void *ptr = malloc(size); + if (ptr == NULL) + out_of_memory(); + return(ptr); +} + +__attribute__ ((unused)) static void *xrealloc(void *ptr, size_t size) { + void *outptr = realloc(ptr, size); + if (outptr == NULL) + out_of_memory(); + return(outptr); +} + +__attribute__ ((unused)) static char *xstrndup(const char *s, size_t n) { + char *ptr = strndup(s, n); + if (ptr == NULL) + out_of_memory(); + return(ptr); +} + +#endif