diff --git a/Makefile b/Makefile index dd12d47c..97f7cd86 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ MXQ_VERSION_MAJOR = 0 MXQ_VERSION_MINOR = 27 -MXQ_VERSION_PATCH = 0 +MXQ_VERSION_PATCH = 1 MXQ_VERSION_EXTRA = "beta" MXQ_VERSIONDATE = 2021 diff --git a/mxq_daemon.c b/mxq_daemon.c index 724d8502..dea5086b 100644 --- a/mxq_daemon.c +++ b/mxq_daemon.c @@ -13,7 +13,7 @@ #include "mxq_daemon.h" -#define DAEMON_FIELDS_CNT 18 +#define DAEMON_FIELDS_CNT 19 #define DAEMON_FIELDS \ " daemon_id," \ " daemon_name," \ @@ -36,7 +36,8 @@ " UNIX_TIMESTAMP(daemon_start) as daemon_start," \ " UNIX_TIMESTAMP(daemon_stop) as daemon_stop," \ " daemon_flags," \ - " tags" + " tags," \ + " prerequisites" #undef _to_string #undef status_str @@ -79,6 +80,7 @@ static int bind_result_daemon_fields(struct mx_mysql_bind *result, struct mxq_da res += mx_mysql_bind_var(result, idx++, int32, &(daemon->daemon_flags)); res += mx_mysql_bind_var(result, idx++, string, &(daemon->tags)); + res += mx_mysql_bind_var(result, idx++, string, &(daemon->prerequisites)); return res; } @@ -143,7 +145,8 @@ int mxq_daemon_register(struct mx_mysql *mysql, struct mxq_daemon *daemon) " daemon_start = CURRENT_TIMESTAMP()," " daemon_stop = 0," " daemon_flags = ?," - " tags = ?" + " tags = ?," + " prerequisites = ?" ); if (!stmt) { mx_log_err("mx_mysql_statement_prepare(): %m"); @@ -170,6 +173,7 @@ int mxq_daemon_register(struct mx_mysql *mysql, struct mxq_daemon *daemon) res += mx_mysql_statement_param_bind(stmt, idx++, uint64, &(daemon->daemon_memory_limit_slot_hard)); res += mx_mysql_statement_param_bind(stmt, idx++, int32, &(daemon->daemon_flags)); res += mx_mysql_statement_param_bind(stmt, idx++, string, &(daemon->tags)); + res += mx_mysql_statement_param_bind(stmt, idx++, string, &(daemon->prerequisites)); assert(res ==0); diff --git a/mxq_daemon.h b/mxq_daemon.h index c1069f46..65474982 100644 --- a/mxq_daemon.h +++ b/mxq_daemon.h @@ -49,6 +49,7 @@ struct mxq_daemon { int daemon_flags; char *tags; + char *prerequisites; }; void mxq_daemon_free_content(struct mxq_daemon *daemon); diff --git a/mxq_group.c b/mxq_group.c index 8760a448..6e52df29 100644 --- a/mxq_group.c +++ b/mxq_group.c @@ -12,7 +12,7 @@ #include "mx_util.h" #include "mx_mysql.h" -#define GROUP_FIELDS_CNT 36 +#define GROUP_FIELDS_CNT 37 #define GROUP_FIELDS \ " group_id," \ " group_name," \ @@ -22,6 +22,7 @@ " group_blacklist," \ " group_whitelist," \ " prerequisites," \ + " tags," \ " user_uid," \ " user_name," \ " user_gid," \ @@ -68,6 +69,7 @@ static int bind_result_group_fields(struct mx_mysql_bind *result, struct mxq_gro res += mx_mysql_bind_var(result, idx++, string, &(g->group_blacklist)); res += mx_mysql_bind_var(result, idx++, string, &(g->group_whitelist)); res += mx_mysql_bind_var(result, idx++, string, &(g->prerequisites)); + res += mx_mysql_bind_var(result, idx++, string, &(g->tags)); res += mx_mysql_bind_var(result, idx++, uint32, &(g->user_uid)); res += mx_mysql_bind_var(result, idx++, string, &(g->user_name)); @@ -112,6 +114,7 @@ static int bind_result_group_fields(struct mx_mysql_bind *result, struct mxq_gro void mxq_group_free_content(struct mxq_group *g) { mx_free_null(g->prerequisites); + mx_free_null(g->tags); mx_free_null(g->group_whitelist); mx_free_null(g->group_blacklist); mx_free_null(g->group_name); diff --git a/mxq_group.h b/mxq_group.h index 5d4e3c54..6543fd2d 100644 --- a/mxq_group.h +++ b/mxq_group.h @@ -17,6 +17,7 @@ struct mxq_group { char * group_blacklist; char * group_whitelist; char * prerequisites; + char * tags; uint32_t user_uid; char * user_name; diff --git a/mxqd.c b/mxqd.c index 40160bb9..3941912f 100644 --- a/mxqd.c +++ b/mxqd.c @@ -44,6 +44,7 @@ #include "mxqd_control.h" #include "keywordset.h" +#include "parser.tab.h" #ifndef MXQ_INITIAL_PATH # define MXQ_INITIAL_PATH "/sbin:/bin:/usr/sbin:/usr/bin:/usr/local/sbin:/usr/local/bin" @@ -79,8 +80,8 @@ static void print_usage(void) "options:\n" " -j, --slots default: depends on number of cores\n" " -m, --memory default: 2G\n" - " -t, --max-time default: 0 (unlimited)" - "\n" + " -t, --max-time default: 0 (unlimited)\n" + " --prerequisites default: ''\n" " -x, --max-memory-per-slot-soft \n" " root user: default: /\n" " non-root user: default: \n" @@ -424,6 +425,22 @@ static void read_cpufeatures(struct keywordset *kws) { fclose(proc_cpuinfo); } +static int expression_is_valid(char *expr) { + struct keywordset *tags = keywordset_new(NULL); + struct parser_context parser_context = { + .input = expr, + .tags = tags, + .pos = 0, + .result = 0, + }; + int sts = yyparse(&parser_context); + keywordset_free(tags); + if (sts) + return 0; + else + return 1; +} + int server_init(struct mxq_server *server, int argc, char *argv[]) { assert(server); @@ -438,6 +455,7 @@ int server_init(struct mxq_server *server, int argc, char *argv[]) char *arg_logdir = NULL; char *arg_initial_path; char *arg_initial_tmpdir; + char *arg_prerequisites = ""; char arg_daemonize = 0; char arg_nolog = 0; char arg_recoveronly = 0; @@ -478,6 +496,7 @@ int server_init(struct mxq_server *server, int argc, char *argv[]) MX_OPTION_OPTIONAL_ARG("mysql-default-file", 'M'), MX_OPTION_OPTIONAL_ARG("mysql-default-group", 'S'), MX_OPTION_OPTIONAL_ARG("max-time", 't'), + MX_OPTION_OPTIONAL_ARG("prerequisites", 11), MX_OPTION_END }; @@ -631,6 +650,10 @@ int server_init(struct mxq_server *server, int argc, char *argv[]) return -EX_USAGE; } break; + + case 11: + arg_prerequisites = optctl.optarg; + break; } } @@ -662,6 +685,11 @@ int server_init(struct mxq_server *server, int argc, char *argv[]) server->initial_tmpdir = arg_initial_tmpdir; server->recoveronly = arg_recoveronly; + if (*arg_prerequisites != 0 && !expression_is_valid(arg_prerequisites)) { + mx_log_err("syntax error in --prerequisites expression \"%s\"", arg_prerequisites); + return -EX_UNAVAILABLE; + } + server->flock = mx_flock(LOCK_EX, "/dev/shm/mxqd.%s.%s.lck", server->hostname, server->daemon_name); if (!server->flock) { mx_log_err("mx_flock(/dev/shm/mxqd.%s.%s.lck) failed: %m", server->hostname, server->daemon_name); @@ -812,6 +840,7 @@ int server_init(struct mxq_server *server, int argc, char *argv[]) daemon->daemon_maxtime = server->maxtime; daemon->daemon_memory_limit_slot_soft = server->memory_limit_slot_soft; daemon->daemon_memory_limit_slot_hard = server->memory_limit_slot_hard; + daemon->prerequisites = arg_prerequisites; server->tags=keywordset_new("true"); keywordset_add(server->tags, server->hostname); diff --git a/mxqsub.c b/mxqsub.c index 065fd02d..c1c8997e 100644 --- a/mxqsub.c +++ b/mxqsub.c @@ -76,6 +76,7 @@ static void print_usage(void) " --blacklist=STRING set list of blacklisted servers (default: '')\n" " --whitelist=STRING set list of whitelisted servers (default: '')\n" " --prerequisites=STRING set prerequisites (default: '')\n" + " --tags=STRING set tags (default: '')\n" "\n" " [SIZE] may be suffixed with a combination of T, G and M\n" " to specify tebibytes, gibibytes and mebibytes.\n" @@ -175,6 +176,7 @@ static int load_group_id(struct mx_mysql *mysql, struct mxq_group *g) " AND group_blacklist = ?" " AND group_whitelist = ?" " AND prerequisites = ?" + " AND tags = ?" " AND group_status = 0" " AND group_flags & ? = 0 " " ORDER BY group_id DESC" @@ -199,7 +201,8 @@ static int load_group_id(struct mx_mysql *mysql, struct mxq_group *g) res += mx_mysql_statement_param_bind(stmt, 12, string, &(g->group_blacklist)); res += mx_mysql_statement_param_bind(stmt, 13, string, &(g->group_whitelist)); res += mx_mysql_statement_param_bind(stmt, 14, string, &(g->prerequisites)); - res += mx_mysql_statement_param_bind(stmt, 15, uint64, &(flags)); + res += mx_mysql_statement_param_bind(stmt, 15, string, &(g->tags)); + res += mx_mysql_statement_param_bind(stmt, 16, uint64, &(flags)); assert(res == 0); res = mx_mysql_statement_execute(stmt, &num_rows); @@ -266,6 +269,7 @@ static int load_group_id_by_group_id(struct mx_mysql *mysql, struct mxq_group *g " AND group_blacklist = ?" " AND group_whitelist = ?" " AND prerequisites = ?" + " AND tags = ?" " AND group_status = 0" " AND group_id = ?" " AND group_flags & ? = 0 " @@ -291,8 +295,9 @@ static int load_group_id_by_group_id(struct mx_mysql *mysql, struct mxq_group *g res += mx_mysql_statement_param_bind(stmt, 12, string, &(g->group_blacklist)); res += mx_mysql_statement_param_bind(stmt, 13, string, &(g->group_whitelist)); res += mx_mysql_statement_param_bind(stmt, 14, string, &(g->prerequisites)); - res += mx_mysql_statement_param_bind(stmt, 15, uint64, &(g->group_id)); - res += mx_mysql_statement_param_bind(stmt, 16, uint64, &(flags)); + res += mx_mysql_statement_param_bind(stmt, 15, string, &(g->tags)); + res += mx_mysql_statement_param_bind(stmt, 16, uint64, &(g->group_id)); + res += mx_mysql_statement_param_bind(stmt, 17, uint64, &(flags)); assert(res == 0); res = mx_mysql_statement_execute(stmt, &num_rows); @@ -351,6 +356,7 @@ static int load_group_id_run_or_wait(struct mx_mysql *mysql, struct mxq_group *g " AND group_blacklist = ?" " AND group_whitelist = ?" " AND prerequisites = ?" + " AND tags = ?" " AND group_status = 0" " AND (" "group_jobs_running > 0" @@ -380,7 +386,8 @@ static int load_group_id_run_or_wait(struct mx_mysql *mysql, struct mxq_group *g res += mx_mysql_statement_param_bind(stmt, 12, string, &(g->group_blacklist)); res += mx_mysql_statement_param_bind(stmt, 13, string, &(g->group_whitelist)); res += mx_mysql_statement_param_bind(stmt, 14, string, &(g->prerequisites)); - res += mx_mysql_statement_param_bind(stmt, 15, uint64, &(flags)); + res += mx_mysql_statement_param_bind(stmt, 15, string, &(g->tags)); + res += mx_mysql_statement_param_bind(stmt, 16, uint64, &(flags)); assert(res == 0); res = mx_mysql_statement_execute(stmt, &num_rows); @@ -440,7 +447,8 @@ static int add_group(struct mx_mysql *mysql, struct mxq_group *g) " group_priority = ?," " group_blacklist = ?," " group_whitelist = ?," - " prerequisites = ?"); + " prerequisites = ?," + " tags = ?"); if (!stmt) { mx_log_err("mx_mysql_statement_prepare(): %m"); return -errno; @@ -461,6 +469,7 @@ static int add_group(struct mx_mysql *mysql, struct mxq_group *g) res += mx_mysql_statement_param_bind(stmt,12, string, &(g->group_blacklist)); res += mx_mysql_statement_param_bind(stmt,13, string, &(g->group_whitelist)); res += mx_mysql_statement_param_bind(stmt,14, string, &(g->prerequisites)); + res += mx_mysql_statement_param_bind(stmt,15, string, &(g->tags)); assert(res == 0); res = mx_mysql_statement_execute(stmt, &num_rows); @@ -685,6 +694,7 @@ int main(int argc, char *argv[]) char *arg_blacklist; char *arg_whitelist; char *arg_prerequisites; + char *arg_tags; char *arg_program_name; u_int16_t arg_threads; u_int64_t arg_memory; @@ -707,6 +717,7 @@ int main(int argc, char *argv[]) _mx_cleanup_free_ char *arg_args = NULL; _mx_cleanup_free_ char *blacklist = NULL; _mx_cleanup_free_ char *whitelist = NULL; + _mx_cleanup_free_ char *tags = NULL; int flags = 0; @@ -764,6 +775,7 @@ int main(int argc, char *argv[]) MX_OPTION_REQUIRED_ARG("blacklist", 8), MX_OPTION_REQUIRED_ARG("whitelist", 9), MX_OPTION_REQUIRED_ARG("prerequisites", 10), + MX_OPTION_REQUIRED_ARG("tags", 11), MX_OPTION_END }; @@ -792,6 +804,7 @@ int main(int argc, char *argv[]) arg_blacklist = NULL; arg_whitelist = NULL; arg_prerequisites = ""; + arg_tags = NULL; arg_mysql_default_group = getenv("MXQ_MYSQL_DEFAULT_GROUP"); if (!arg_mysql_default_group) @@ -1008,6 +1021,9 @@ int main(int argc, char *argv[]) arg_prerequisites = optctl.optarg; break; + case 11: + arg_tags = optctl.optarg; + break; } } @@ -1082,6 +1098,14 @@ int main(int argc, char *argv[]) whitelist = mx_strdup_forever(""); } + if (arg_tags != NULL) { + struct keywordset *kws = keywordset_new(arg_tags); + tags = keywordset_get(kws); + keywordset_free(kws); + } else { + tags = mx_strdup_forever(""); + } + /******************************************************************/ memset(&job, 0, sizeof(job)); @@ -1101,6 +1125,7 @@ int main(int argc, char *argv[]) group.group_blacklist = blacklist; group.group_whitelist = whitelist; group.prerequisites = arg_prerequisites; + group.tags = tags; group.job_max_per_node = arg_max_per_node; diff --git a/mysql/migrate_010_add_mxq_daemon_tags.sql b/mysql/migrate_010_add_mxq_daemon_tags.sql deleted file mode 100644 index 0057f9df..00000000 --- a/mysql/migrate_010_add_mxq_daemon_tags.sql +++ /dev/null @@ -1,9 +0,0 @@ -ALTER TABLE mxq_daemon MODIFY COLUMN prerequisites VARCHAR(1000) NOT NULL DEFAULT ''; - -ALTER TABLE mxq_daemon - ADD COLUMN - tags VARCHAR(1000) NOT NULL DEFAULT '' - AFTER - prerequisites; - -UPDATE mxq_daemon SET tags=prerequisites; diff --git a/mysql/migrate_010_add_mxq_group_tags.sql b/mysql/migrate_010_add_mxq_group_tags.sql deleted file mode 100644 index 46141af2..00000000 --- a/mysql/migrate_010_add_mxq_group_tags.sql +++ /dev/null @@ -1,6 +0,0 @@ -ALTER TABLE mxq_group MODIFY COLUMN prerequisites VARCHAR(1000) NOT NULL DEFAULT ''; -ALTER TABLE mxq_group - ADD COLUMN - tags VARCHAR(1000) NOT NULL DEFAULT '' - AFTER - prerequisites; diff --git a/mysql/migrate_011_purge_mxq_daemon_prerequisites.sql b/mysql/migrate_011_purge_mxq_daemon_prerequisites.sql new file mode 100644 index 00000000..0d3526a7 --- /dev/null +++ b/mysql/migrate_011_purge_mxq_daemon_prerequisites.sql @@ -0,0 +1 @@ +UPDATE mxq_daemon SET prerequisites=''; diff --git a/web/pages/mxq/mxq.in b/web/pages/mxq/mxq.in index 960a2155..bdfe0b1c 100755 --- a/web/pages/mxq/mxq.in +++ b/web/pages/mxq/mxq.in @@ -395,6 +395,7 @@ group_priority : $o->{group_priority} group_blacklist: $o->{group_blacklist} group_whitelist: $o->{group_whitelist} prerequisites : $o->{prerequisites} +tags : $o->{tags} user_uid : $o->{user_uid} user_name : $o->{user_name} @@ -519,7 +520,7 @@ sub server_detail { daemon_memory_limit_slot_soft daemon_memory_limit_slot_hard daemon_jobs_running daemon_slots_running daemon_threads_running daemon_memory_used - mtime daemon_start daemon_stop daemon_flags tags + mtime daemon_start daemon_stop daemon_flags tags prerequisites ); my $sth=$dbh->prepare('SELECT '.join(',',@cols).' FROM mxq_daemon WHERE daemon_id=? LIMIT 1', undef); $sth->execute($daemon_id); @@ -559,6 +560,7 @@ daemon_stop : $o{daemon_stop} daemon_flags : $o{daemon_flags} tags : $o{tags} +prerequisites : $o{prerequisites} EOF }