Merge pull request #98 from mariux64/add-tags

Add tags
mariux64 · Mar 29, 2021 · 5bff524 · 5bff524
2 parents 78c12d0 + 6cce07f
commit 5bff524
Show file tree

Hide file tree

Showing 11 changed files with 79 additions and 28 deletions.
diff --git a/Makefile b/Makefile
@@ -1,6 +1,6 @@
 MXQ_VERSION_MAJOR = 0
 MXQ_VERSION_MINOR = 27
-MXQ_VERSION_PATCH = 0
+MXQ_VERSION_PATCH = 1
 MXQ_VERSION_EXTRA = "beta"
 MXQ_VERSIONDATE = 2021
 

diff --git a/mxq_daemon.c b/mxq_daemon.c
@@ -13,7 +13,7 @@
 
 #include "mxq_daemon.h"
 
-#define DAEMON_FIELDS_CNT 18
+#define DAEMON_FIELDS_CNT 19
 #define DAEMON_FIELDS \
             " daemon_id," \
             " daemon_name," \
@@ -36,7 +36,8 @@
             " UNIX_TIMESTAMP(daemon_start) as daemon_start," \
             " UNIX_TIMESTAMP(daemon_stop)  as daemon_stop," \
             " daemon_flags," \
-            " tags"
+            " tags," \
+            " prerequisites"
 
 #undef _to_string
 #undef status_str
@@ -79,6 +80,7 @@ static int bind_result_daemon_fields(struct mx_mysql_bind *result, struct mxq_da
 
     res += mx_mysql_bind_var(result, idx++,  int32, &(daemon->daemon_flags));
     res += mx_mysql_bind_var(result, idx++, string, &(daemon->tags));
+    res += mx_mysql_bind_var(result, idx++, string, &(daemon->prerequisites));
 
     return res;
 }
@@ -143,7 +145,8 @@ int mxq_daemon_register(struct mx_mysql *mysql, struct mxq_daemon *daemon)
                 " daemon_start = CURRENT_TIMESTAMP(),"
                 " daemon_stop  = 0,"
                 " daemon_flags = ?,"
-                " tags = ?"
+                " tags = ?,"
+                " prerequisites = ?"
                 );
     if (!stmt) {
         mx_log_err("mx_mysql_statement_prepare(): %m");
@@ -170,6 +173,7 @@ int mxq_daemon_register(struct mx_mysql *mysql, struct mxq_daemon *daemon)
     res += mx_mysql_statement_param_bind(stmt, idx++, uint64, &(daemon->daemon_memory_limit_slot_hard));
     res += mx_mysql_statement_param_bind(stmt, idx++, int32,  &(daemon->daemon_flags));
     res += mx_mysql_statement_param_bind(stmt, idx++, string, &(daemon->tags));
+    res += mx_mysql_statement_param_bind(stmt, idx++, string, &(daemon->prerequisites));
 
     assert(res ==0);
 

diff --git a/mxq_daemon.h b/mxq_daemon.h
@@ -49,6 +49,7 @@ struct mxq_daemon {
 
     int daemon_flags;
     char *tags;
+    char *prerequisites;
 };
 
 void mxq_daemon_free_content(struct mxq_daemon *daemon);

diff --git a/mxq_group.c b/mxq_group.c
@@ -12,7 +12,7 @@
 #include "mx_util.h"
 #include "mx_mysql.h"
 
-#define GROUP_FIELDS_CNT 36
+#define GROUP_FIELDS_CNT 37
 #define GROUP_FIELDS \
             " group_id," \
             " group_name," \
@@ -22,6 +22,7 @@
             " group_blacklist," \
             " group_whitelist," \
             " prerequisites," \
+            " tags," \
             " user_uid," \
             " user_name," \
             " user_gid," \
@@ -68,6 +69,7 @@ static int bind_result_group_fields(struct mx_mysql_bind *result, struct mxq_gro
     res += mx_mysql_bind_var(result, idx++, string, &(g->group_blacklist));
     res += mx_mysql_bind_var(result, idx++, string, &(g->group_whitelist));
     res += mx_mysql_bind_var(result, idx++, string, &(g->prerequisites));
+    res += mx_mysql_bind_var(result, idx++, string, &(g->tags));
 
     res += mx_mysql_bind_var(result, idx++, uint32, &(g->user_uid));
     res += mx_mysql_bind_var(result, idx++, string, &(g->user_name));
@@ -112,6 +114,7 @@ static int bind_result_group_fields(struct mx_mysql_bind *result, struct mxq_gro
 void mxq_group_free_content(struct mxq_group *g)
 {
         mx_free_null(g->prerequisites);
+        mx_free_null(g->tags);
         mx_free_null(g->group_whitelist);
         mx_free_null(g->group_blacklist);
         mx_free_null(g->group_name);

diff --git a/mxq_group.h b/mxq_group.h
@@ -17,6 +17,7 @@ struct mxq_group {
    char *    group_blacklist;
    char *    group_whitelist;
    char *    prerequisites;
+   char *    tags;
 
    uint32_t  user_uid;
    char *    user_name;

diff --git a/mxqd.c b/mxqd.c
@@ -44,6 +44,7 @@
 
 #include "mxqd_control.h"
 #include "keywordset.h"
+#include "parser.tab.h"
 
 #ifndef MXQ_INITIAL_PATH
 #  define MXQ_INITIAL_PATH      "/sbin:/bin:/usr/sbin:/usr/bin:/usr/local/sbin:/usr/local/bin"
@@ -79,8 +80,8 @@ static void print_usage(void)
     "options:\n"
     "  -j, --slots     <slots>           default: depends on number of cores\n"
     "  -m, --memory    <totalmemory>     default: 2G\n"
-    "  -t, --max-time  <minutes>         default: 0 (unlimited)"
-    "\n"
+    "  -t, --max-time  <minutes>         default: 0 (unlimited)\n"
+    "      --prerequisites <expr>        default: ''\n"
     "  -x, --max-memory-per-slot-soft <softlimit>\n"
     "                         root user: default: <totalmemory>/<slots>\n"
     "                     non-root user: default: <totalmemory>\n"
@@ -424,6 +425,22 @@ static void read_cpufeatures(struct keywordset *kws) {
     fclose(proc_cpuinfo);
 }
 
+static int expression_is_valid(char *expr) {
+    struct keywordset *tags = keywordset_new(NULL);
+    struct parser_context parser_context = {
+        .input = expr,
+        .tags = tags,
+        .pos = 0,
+        .result = 0,
+    };
+    int sts = yyparse(&parser_context);
+    keywordset_free(tags);
+    if (sts)
+        return 0;
+    else
+        return 1;
+}
+
 int server_init(struct mxq_server *server, int argc, char *argv[])
 {
     assert(server);
@@ -438,6 +455,7 @@ int server_init(struct mxq_server *server, int argc, char *argv[])
     char *arg_logdir = NULL;
     char *arg_initial_path;
     char *arg_initial_tmpdir;
+    char *arg_prerequisites = "";
     char arg_daemonize = 0;
     char arg_nolog = 0;
     char arg_recoveronly = 0;
@@ -478,6 +496,7 @@ int server_init(struct mxq_server *server, int argc, char *argv[])
                 MX_OPTION_OPTIONAL_ARG("mysql-default-file",  'M'),
                 MX_OPTION_OPTIONAL_ARG("mysql-default-group", 'S'),
                 MX_OPTION_OPTIONAL_ARG("max-time",      't'),
+                MX_OPTION_OPTIONAL_ARG("prerequisites",  11),
                 MX_OPTION_END
     };
 
@@ -631,6 +650,10 @@ int server_init(struct mxq_server *server, int argc, char *argv[])
                     return -EX_USAGE;
                 }
                 break;
+
+            case 11:
+                arg_prerequisites = optctl.optarg;
+                break;
         }
     }
 
@@ -662,6 +685,11 @@ int server_init(struct mxq_server *server, int argc, char *argv[])
     server->initial_tmpdir = arg_initial_tmpdir;
     server->recoveronly = arg_recoveronly;
 
+    if (*arg_prerequisites != 0 && !expression_is_valid(arg_prerequisites)) {
+            mx_log_err("syntax error in --prerequisites expression \"%s\"", arg_prerequisites);
+            return -EX_UNAVAILABLE;
+    }
+
     server->flock = mx_flock(LOCK_EX, "/dev/shm/mxqd.%s.%s.lck", server->hostname, server->daemon_name);
     if (!server->flock) {
         mx_log_err("mx_flock(/dev/shm/mxqd.%s.%s.lck) failed: %m", server->hostname, server->daemon_name);
@@ -812,6 +840,7 @@ int server_init(struct mxq_server *server, int argc, char *argv[])
     daemon->daemon_maxtime = server->maxtime;
     daemon->daemon_memory_limit_slot_soft = server->memory_limit_slot_soft;
     daemon->daemon_memory_limit_slot_hard = server->memory_limit_slot_hard;
+    daemon->prerequisites = arg_prerequisites;
 
     server->tags=keywordset_new("true");
     keywordset_add(server->tags, server->hostname);

diff --git a/mxqsub.c b/mxqsub.c
@@ -76,6 +76,7 @@ static void print_usage(void)
     "  --blacklist=STRING       set list of blacklisted servers (default: '')\n"
     "  --whitelist=STRING       set list of whitelisted servers (default: '')\n"
     "  --prerequisites=STRING   set prerequisites           (default: '')\n"
+    "  --tags=STRING            set tags                    (default: '')\n"
     "\n"
     "        [SIZE] may be suffixed with a combination of T, G and M\n"
     "               to specify tebibytes, gibibytes and mebibytes.\n"
@@ -175,6 +176,7 @@ static int load_group_id(struct mx_mysql *mysql, struct mxq_group *g)
                 " AND group_blacklist = ?"
                 " AND group_whitelist = ?"
                 " AND prerequisites = ?"
+                " AND tags = ?"
                 " AND group_status = 0"
                 " AND group_flags & ? = 0 "
             " ORDER BY group_id DESC"
@@ -199,7 +201,8 @@ static int load_group_id(struct mx_mysql *mysql, struct mxq_group *g)
     res += mx_mysql_statement_param_bind(stmt, 12, string, &(g->group_blacklist));
     res += mx_mysql_statement_param_bind(stmt, 13, string, &(g->group_whitelist));
     res += mx_mysql_statement_param_bind(stmt, 14, string, &(g->prerequisites));
-    res += mx_mysql_statement_param_bind(stmt, 15, uint64, &(flags));
+    res += mx_mysql_statement_param_bind(stmt, 15, string, &(g->tags));
+    res += mx_mysql_statement_param_bind(stmt, 16, uint64, &(flags));
     assert(res == 0);
 
     res = mx_mysql_statement_execute(stmt, &num_rows);
@@ -266,6 +269,7 @@ static int load_group_id_by_group_id(struct mx_mysql *mysql, struct mxq_group *g
                 " AND group_blacklist = ?"
                 " AND group_whitelist = ?"
                 " AND prerequisites = ?"
+                " AND tags = ?"
                 " AND group_status = 0"
                 " AND group_id = ?"
                 " AND group_flags & ? = 0 "
@@ -291,8 +295,9 @@ static int load_group_id_by_group_id(struct mx_mysql *mysql, struct mxq_group *g
     res += mx_mysql_statement_param_bind(stmt, 12, string, &(g->group_blacklist));
     res += mx_mysql_statement_param_bind(stmt, 13, string, &(g->group_whitelist));
     res += mx_mysql_statement_param_bind(stmt, 14, string, &(g->prerequisites));
-    res += mx_mysql_statement_param_bind(stmt, 15, uint64, &(g->group_id));
-    res += mx_mysql_statement_param_bind(stmt, 16, uint64, &(flags));
+    res += mx_mysql_statement_param_bind(stmt, 15, string, &(g->tags));
+    res += mx_mysql_statement_param_bind(stmt, 16, uint64, &(g->group_id));
+    res += mx_mysql_statement_param_bind(stmt, 17, uint64, &(flags));
     assert(res == 0);
 
     res = mx_mysql_statement_execute(stmt, &num_rows);
@@ -351,6 +356,7 @@ static int load_group_id_run_or_wait(struct mx_mysql *mysql, struct mxq_group *g
                 " AND group_blacklist = ?"
                 " AND group_whitelist = ?"
                 " AND prerequisites = ?"
+                " AND tags = ?"
                 " AND group_status = 0"
                 " AND ("
                         "group_jobs_running > 0"
@@ -380,7 +386,8 @@ static int load_group_id_run_or_wait(struct mx_mysql *mysql, struct mxq_group *g
     res += mx_mysql_statement_param_bind(stmt, 12, string, &(g->group_blacklist));
     res += mx_mysql_statement_param_bind(stmt, 13, string, &(g->group_whitelist));
     res += mx_mysql_statement_param_bind(stmt, 14, string, &(g->prerequisites));
-    res += mx_mysql_statement_param_bind(stmt, 15, uint64, &(flags));
+    res += mx_mysql_statement_param_bind(stmt, 15, string, &(g->tags));
+    res += mx_mysql_statement_param_bind(stmt, 16, uint64, &(flags));
     assert(res == 0);
 
     res = mx_mysql_statement_execute(stmt, &num_rows);
@@ -440,7 +447,8 @@ static int add_group(struct mx_mysql *mysql, struct mxq_group *g)
                 " group_priority = ?,"
                 " group_blacklist = ?,"
                 " group_whitelist = ?,"
-                " prerequisites = ?");
+                " prerequisites = ?,"
+                " tags = ?");
     if (!stmt) {
         mx_log_err("mx_mysql_statement_prepare(): %m");
         return -errno;
@@ -461,6 +469,7 @@ static int add_group(struct mx_mysql *mysql, struct mxq_group *g)
     res += mx_mysql_statement_param_bind(stmt,12, string, &(g->group_blacklist));
     res += mx_mysql_statement_param_bind(stmt,13, string, &(g->group_whitelist));
     res += mx_mysql_statement_param_bind(stmt,14, string, &(g->prerequisites));
+    res += mx_mysql_statement_param_bind(stmt,15, string, &(g->tags));
     assert(res == 0);
 
     res = mx_mysql_statement_execute(stmt, &num_rows);
@@ -685,6 +694,7 @@ int main(int argc, char *argv[])
     char      *arg_blacklist;
     char      *arg_whitelist;
     char      *arg_prerequisites;
+    char      *arg_tags;
     char      *arg_program_name;
     u_int16_t  arg_threads;
     u_int64_t  arg_memory;
@@ -707,6 +717,7 @@ int main(int argc, char *argv[])
     _mx_cleanup_free_ char *arg_args = NULL;
     _mx_cleanup_free_ char *blacklist = NULL;
     _mx_cleanup_free_ char *whitelist = NULL;
+    _mx_cleanup_free_ char *tags = NULL;
 
     int flags = 0;
 
@@ -764,6 +775,7 @@ int main(int argc, char *argv[])
                 MX_OPTION_REQUIRED_ARG("blacklist",  8),
                 MX_OPTION_REQUIRED_ARG("whitelist",  9),
                 MX_OPTION_REQUIRED_ARG("prerequisites", 10),
+                MX_OPTION_REQUIRED_ARG("tags", 11),
                 MX_OPTION_END
     };
 
@@ -792,6 +804,7 @@ int main(int argc, char *argv[])
     arg_blacklist      = NULL;
     arg_whitelist      = NULL;
     arg_prerequisites  = "";
+    arg_tags           = NULL;
 
     arg_mysql_default_group = getenv("MXQ_MYSQL_DEFAULT_GROUP");
     if (!arg_mysql_default_group)
@@ -1008,6 +1021,9 @@ int main(int argc, char *argv[])
                 arg_prerequisites = optctl.optarg;
                 break;
 
+            case 11:
+                arg_tags = optctl.optarg;
+                break;
         }
     }
 
@@ -1082,6 +1098,14 @@ int main(int argc, char *argv[])
         whitelist = mx_strdup_forever("");
     }
 
+    if (arg_tags != NULL) {
+        struct keywordset *kws = keywordset_new(arg_tags);
+        tags = keywordset_get(kws);
+        keywordset_free(kws);
+    } else {
+        tags = mx_strdup_forever("");
+    }
+
     /******************************************************************/
 
     memset(&job,   0, sizeof(job));
@@ -1101,6 +1125,7 @@ int main(int argc, char *argv[])
     group.group_blacklist = blacklist;
     group.group_whitelist = whitelist;
     group.prerequisites   = arg_prerequisites;
+    group.tags            = tags;
 
     group.job_max_per_node = arg_max_per_node;
 

diff --git a/mysql/migrate_010_add_mxq_daemon_tags.sql b/mysql/migrate_010_add_mxq_daemon_tags.sql
diff --git a/mysql/migrate_010_add_mxq_group_tags.sql b/mysql/migrate_010_add_mxq_group_tags.sql
diff --git a/mysql/migrate_011_purge_mxq_daemon_prerequisites.sql b/mysql/migrate_011_purge_mxq_daemon_prerequisites.sql
@@ -0,0 +1 @@
+UPDATE mxq_daemon SET prerequisites='';
diff --git a/web/pages/mxq/mxq.in b/web/pages/mxq/mxq.in
@@ -395,6 +395,7 @@ group_priority : $o->{group_priority}
 group_blacklist: $o->{group_blacklist}
 group_whitelist: $o->{group_whitelist}
 prerequisites  : $o->{prerequisites}
+tags           : $o->{tags}
 
 user_uid       : $o->{user_uid}
 user_name      : $o->{user_name}
@@ -519,7 +520,7 @@ sub server_detail {
 		daemon_memory_limit_slot_soft daemon_memory_limit_slot_hard
 		daemon_jobs_running daemon_slots_running
 		daemon_threads_running daemon_memory_used
-		mtime daemon_start daemon_stop daemon_flags tags
+		mtime daemon_start daemon_stop daemon_flags tags prerequisites
 	);
 	my $sth=$dbh->prepare('SELECT '.join(',',@cols).' FROM mxq_daemon WHERE daemon_id=? LIMIT 1', undef);
 	$sth->execute($daemon_id);
@@ -559,6 +560,7 @@ daemon_stop    : $o{daemon_stop}
 daemon_flags   : $o{daemon_flags}
 
 tags           : $o{tags}
+prerequisites  : $o{prerequisites}
 </pre>
 EOF
 }