diff --git a/Makefile b/Makefile index a1a3876..a9f5846 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ MXQ_VERSION_MAJOR = 0 MXQ_VERSION_MINOR = 17 MXQ_VERSION_PATCH = 0 MXQ_VERSION_EXTRA = "beta" -MXQ_VERSIONDATE = 2013-2015 +MXQ_VERSIONDATE = 2015 MXQ_VERSION_GIT := $(shell git describe --long 2>/dev/null) @@ -60,13 +60,17 @@ CGIDIR = ${LIBEXECDIR}/mxq/cgi ######################################################################## MXQ_MYSQL_DEFAULT_FILE = ${SYSCONFDIR}/mxq/mysql.cnf -MXQ_MYSQL_DEFAULT_GROUP = mxqclient +MXQ_MYSQL_DEFAULT_GROUP_CLIENT = mxqclient +MXQ_MYSQL_DEFAULT_GROUP_SERVER = mxqd +MXQ_MYSQL_DEFAULT_GROUP_DEVELOPMENT = mxqdevel MXQ_INITIAL_PATH = /sbin:/bin:/usr/sbin:/usr/bin:/usr/local/sbin:/usr/local/bin MXQ_INITIAL_TMPDIR = /tmp CFLAGS_MXQ_MYSQL_DEFAULT_FILE = -DMXQ_MYSQL_DEFAULT_FILE=\"$(MXQ_MYSQL_DEFAULT_FILE)\" -CFLAGS_MXQ_MYSQL_DEFAULT_GROUP = -DMXQ_MYSQL_DEFAULT_GROUP=\"$(MXQ_MYSQL_DEFAULT_GROUP)\" +CFLAGS_MXQ_MYSQL_DEFAULT_GROUP = -DMXQ_MYSQL_DEFAULT_GROUP_CLIENT=\"$(MXQ_MYSQL_DEFAULT_GROUP_CLIENT)\" +CFLAGS_MXQ_MYSQL_DEFAULT_GROUP += -DMXQ_MYSQL_DEFAULT_GROUP_SERVER=\"$(MXQ_MYSQL_DEFAULT_GROUP_SERVER)\" +CFLAGS_MXQ_MYSQL_DEFAULT_GROUP += -DMXQ_MYSQL_DEFAULT_GROUP_DEVELOPMENT=\"$(MXQ_MYSQL_DEFAULT_GROUP_DEVELOPMENT)\" CFLAGS_MXQ_INITIAL_PATH = -DMXQ_INITIAL_PATH=\"$(MXQ_INITIAL_PATH)\" CFLAGS_MXQ_INITIAL_TMPDIR = -DMXQ_INITIAL_TMPDIR=\"$(MXQ_INITIAL_TMPDIR)\" @@ -112,6 +116,15 @@ CFLAGS += -DMXQ_VERSION=\"${MXQ_VERSION}\" CFLAGS += -DMXQ_VERSIONFULL=\"${MXQ_VERSIONFULL}\" CFLAGS += -DMXQ_VERSIONDATE=\"${MXQ_VERSIONDATE}\" CFLAGS += -DMXQ_VERSIONEXTRA=\"${MXQ_VERSIONEXTRA}\" +CFLAGS += -DPREFIX=\"${PREFIX}\" +CFLAGS += -DEPREFIX=\"${EPREFIX}\" +CFLAGS += -DSBINDIR=\"${SBINDIR}\" +CFLAGS += -DBINDIR=\"${BINDIR}\" +CFLAGS += -DLIBDIR=\"${LIBDIR}\" +CFLAGS += -DLIBEXECDIR=\"${LIBEXECDIR}\" +CFLAGS += -DDATADIR=\"${DATADIR}\" +CFLAGS += -DMANDIR=\"${MANDIR}\" +CFLAGS += -DSYSCONFDIR=\"${SYSCONFDIR}\" CFLAGS += $(EXTRA_CFLAGS) ######################################################################## @@ -175,6 +188,10 @@ manpages/%: manpages/%.xml all: build +.PHONY: devel +devel: CFLAGS += -DMXQ_DEVELOPMENT +devel: all + ######################################################################## .PHONY: test diff --git a/README.md b/README.md index fb37fe7..6dfb260 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,6 @@ -# mxq -MXQ - mariux64 job scheduling system +# MXQ - mariux64 job scheduling system +- by Marius Tolzmann 2013-2015 +- and Donald Buczek 2015-2015 ## Sources ### Main git repository @@ -13,11 +14,49 @@ https://github.com/mariux/mxq ## Installation ### Install using `GNU make` ``` -make -make install [DESTDIR=...] +make +make install +``` +``` +make PREFIX=... +make PREFIX=... [DESTDIR=...] install ``` ### Install using `bee` ``` -bee init $(bee download git://github.molgen.mpg.de/mariux64/mxq.git) -e +bee init $(bee download git://github.molgen.mpg.de/mariux64/mxq.git) --execute bee update mxq ``` +``` +bee init $(bee download git://github.molgen.mpg.de/mariux64/mxq.git) --prefix=... --execute +bee update mxq +``` + +## Initial setup +Definitions of the tables and triggers for the MySQL database can be found in +[mysql/create_tables.sql](https://github.molgen.mpg.de/mariux64/mxq/blob/master/mysql/create_tables.sql) +Be sure to create those once and check the same +[directory for alter_tables*.sql`](https://github.molgen.mpg.de/mariux64/mxq/blob/master/mysql/) +files when upgrading. + + +## Development builds +The `devel` target in the Makefile will enable all devolopment features +by defining `MXQ_DEVELOPMENT` when compiling C sources. + +``` +make clean +make devel PREFIX=/path/to/test +make install PREFIX=/path/to/test +``` + +### Differences to production builds +Some new features and improvements are enabled in development builds. +Those features may not be tested for every situation yet and may result +in database corruption and/or failing jobs. + +#### changed `mxqd` default options +In devolopment builds `--no-log` is default (enable loggin with `--log`) + +#### Development database access +Devolopment builds default to use `[mxqdevel]` groups from mysql config files +for servers and clients (instead of the default `[mxqclient]` and `[mxqd]` groups) diff --git a/mxq.h b/mxq.h index 22c5842..54b073f 100644 --- a/mxq.h +++ b/mxq.h @@ -17,7 +17,7 @@ #endif #ifndef MXQ_VERSIONDATE -# define MXQ_VERSIONDATE "2015" +# define MXQ_VERSIONDATE "today" #endif #ifndef MXQ_MYSQL_DEFAULT_FILE @@ -27,8 +27,22 @@ # define MXQ_MYSQL_DEFAULT_FILE_STR MXQ_MYSQL_DEFAULT_FILE #endif -#ifndef MXQ_MYSQL_DEFAULT_GROUP -# define MXQ_MYSQL_DEFAULT_GROUP program_invocation_short_name +#ifdef MXQ_DEVELOPMENT +# undef MXQ_MYSQL_DEFAULT_GROUP +# define MXQ_MYSQL_DEFAULT_GROUP MXQ_MYSQL_DEFAULT_GROUP_DEVELOPMENT +#else +# ifdef MXQ_TYPE_SERVER +# ifdef MXQ_MYSQL_DEFAULT_GROUP_SERVER +# define MXQ_MYSQL_DEFAULT_GROUP MXQ_MYSQL_DEFAULT_GROUP_SERVER +# endif +# else +# ifdef MXQ_MYSQL_DEFAULT_GROUP_CLIENT +# define MXQ_MYSQL_DEFAULT_GROUP MXQ_MYSQL_DEFAULT_GROUP_CLIENT +# endif +# endif +# ifndef MXQ_MYSQL_DEFAULT_GROUP +# define MXQ_MYSQL_DEFAULT_GROUP program_invocation_short_name +# endif #endif #define MXQ_MYSQL_DEFAULT_GROUP_STR MXQ_MYSQL_DEFAULT_GROUP @@ -36,7 +50,11 @@ static void mxq_print_generic_version(void) { printf( "%s - " MXQ_VERSIONFULL "\n" - " by Marius Tolzmann " MXQ_VERSIONDATE "\n" +#ifdef MXQ_DEVELOPMENT + "DEVELOPMENT VERSION: Do not use in production environments.\n" +#endif + " by Marius Tolzmann 2013-" MXQ_VERSIONDATE "\n" + " and Donald Buczek 2015-" MXQ_VERSIONDATE "\n" " Max Planck Institute for Molecular Genetics - Berlin Dahlem\n", program_invocation_short_name ); diff --git a/mxq_group.c b/mxq_group.c index 65f7757..2887411 100644 --- a/mxq_group.c +++ b/mxq_group.c @@ -292,39 +292,6 @@ int mxq_load_active_groups_for_user(struct mx_mysql *mysql, struct mxq_group **m return res; } -int mxq_load_active_groups(struct mx_mysql *mysql, struct mxq_group **mxq_groups) -{ - int res; - struct mxq_group *groups = NULL; - struct mxq_group g = {0}; - struct mx_mysql_bind result = {0}; - - assert(mysql); - assert(mxq_groups); - - *mxq_groups = NULL; - - char *query = - "SELECT" - GROUP_FIELDS - " FROM mxq_group" - " WHERE (group_jobs_inq > 0 OR group_jobs_running > 0)" - " ORDER BY user_name, group_mtime" - " LIMIT 1000"; - - res = bind_result_group_fields(&result, &g); - assert(res == 0); - - res = mx_mysql_do_statement_retry_on_fail(mysql, query, NULL, &result, &g, (void **)&groups, sizeof(*groups)); - if (res < 0) { - mx_log_err("mx_mysql_do_statement_retry_on_fail(): %m"); - return res; - } - - *mxq_groups = groups; - return res; -} - int mxq_load_running_groups(struct mx_mysql *mysql, struct mxq_group **mxq_groups) { int res; diff --git a/mxq_group.h b/mxq_group.h index e678cb3..46a352b 100644 --- a/mxq_group.h +++ b/mxq_group.h @@ -68,7 +68,6 @@ inline uint64_t mxq_group_jobs_inq(struct mxq_group *g); int mxq_load_group(struct mx_mysql *mysql, struct mxq_group **mxq_groups, uint64_t group_id); int mxq_load_all_groups(struct mx_mysql *mysql, struct mxq_group **mxq_groups); -int mxq_load_active_groups(struct mx_mysql *mysql, struct mxq_group **mxq_groups); int mxq_load_all_groups_for_user(struct mx_mysql *mysql, struct mxq_group **mxq_groups, uint64_t user_uid); int mxq_load_active_groups_for_user(struct mx_mysql *mysql, struct mxq_group **mxq_groups, uint64_t user_uid); int mxq_load_running_groups(struct mx_mysql *mysql, struct mxq_group **mxq_groups); diff --git a/mxqd.c b/mxqd.c index b15d558..43afc3d 100644 --- a/mxqd.c +++ b/mxqd.c @@ -1,6 +1,8 @@ #define _GNU_SOURCE +#define MXQ_TYPE_SERVER + #include #include #include @@ -38,9 +40,6 @@ #include "mxqd.h" #include "mxq.h" -#define MYSQL_DEFAULT_FILE MXQ_MYSQL_DEFAULT_FILE -#define MYSQL_DEFAULT_GROUP "mxqd" - #ifndef MXQ_INITIAL_PATH # define MXQ_INITIAL_PATH "/sbin:/bin:/usr/sbin:/usr/bin:/usr/local/sbin:/usr/local/bin" #endif @@ -49,6 +48,8 @@ # define MXQ_INITIAL_TMPDIR "/tmp" #endif +#define RUNNING_AS_ROOT (getuid() == 0) + volatile sig_atomic_t global_sigint_cnt=0; volatile sig_atomic_t global_sigterm_cnt=0; @@ -72,7 +73,11 @@ static void print_usage(void) "\n" " --pid-file default: create no pid file\n" " --daemonize default: run in foreground\n" +#ifdef MXQ_DEVELOPMENT + " --log default (in development): write no logfile\n" +#else " --no-log default: write a logfile\n" +#endif " --debug default: info log level\n" " --recover-only (recover from crash and exit)\n" "\n" @@ -278,9 +283,9 @@ int server_init(struct mxq_server *server, int argc, char *argv[]) char arg_recoveronly = 0; char *str_bootid; int opt; - unsigned long threads_total = 0; - unsigned long memory_total = 2048; - unsigned long memory_max = 0; + unsigned long arg_threads_total = 0; + unsigned long arg_memory_total = 2048; + unsigned long arg_memory_max = 0; int i; _mx_cleanup_free_ struct mx_proc_pid_stat *pps = NULL; @@ -291,6 +296,7 @@ int server_init(struct mxq_server *server, int argc, char *argv[]) MX_OPTION_NO_ARG("version", 'V'), MX_OPTION_NO_ARG("daemonize", 1), MX_OPTION_NO_ARG("no-log", 3), + MX_OPTION_NO_ARG("log", 4), MX_OPTION_NO_ARG("debug", 5), MX_OPTION_NO_ARG("recover-only", 9), MX_OPTION_REQUIRED_ARG("pid-file", 2), @@ -309,16 +315,20 @@ int server_init(struct mxq_server *server, int argc, char *argv[]) arg_server_id = "main"; arg_hostname = mx_hostname(); +#ifdef MXQ_DEVELOPMENT + arg_nolog = 1; +#endif + arg_initial_path = MXQ_INITIAL_PATH; arg_initial_tmpdir = MXQ_INITIAL_TMPDIR; arg_mysql_default_group = getenv("MXQ_MYSQL_DEFAULT_GROUP"); if (!arg_mysql_default_group) - arg_mysql_default_group = MYSQL_DEFAULT_GROUP; + arg_mysql_default_group = MXQ_MYSQL_DEFAULT_GROUP; arg_mysql_default_file = getenv("MXQ_MYSQL_DEFAULT_FILE"); if (!arg_mysql_default_file) - arg_mysql_default_file = MYSQL_DEFAULT_FILE; + arg_mysql_default_file = MXQ_MYSQL_DEFAULT_FILE; mx_getopt_init(&optctl, argc-1, &argv[1], opts); @@ -342,6 +352,10 @@ int server_init(struct mxq_server *server, int argc, char *argv[]) arg_nolog = 1; break; + case 4: + arg_nolog = 0; + break; + case 5: mx_log_level_set(MX_LOG_DEBUG); break; @@ -363,35 +377,35 @@ int server_init(struct mxq_server *server, int argc, char *argv[]) exit(EX_USAGE); case 'j': - if (mx_strtoul(optctl.optarg, &threads_total) < 0) { + if (mx_strtoul(optctl.optarg, &arg_threads_total) < 0) { mx_log_err("Invalid argument supplied for option --slots '%s': %m", optctl.optarg); exit(1); } break; case 'm': - if (mx_strtoul(optctl.optarg, &memory_total) < 0) { + if (mx_strtoul(optctl.optarg, &arg_memory_total) < 0) { unsigned long long int bytes; if(mx_strtobytes(optctl.optarg, &bytes) < 0) { mx_log_err("Invalid argument supplied for option --memory '%s': %m", optctl.optarg); exit(1); } - memory_total = bytes/1024/1024; + arg_memory_total = bytes/1024/1024; } - if (!memory_total) - memory_total = 2048; + if (!arg_memory_total) + arg_memory_total = 2048; break; case 'x': - if (mx_strtoul(optctl.optarg, &memory_max) < 0) { + if (mx_strtoul(optctl.optarg, &arg_memory_max) < 0) { unsigned long long int bytes; if(mx_strtobytes(optctl.optarg, &bytes) < 0) { mx_log_err("Invalid argument supplied for option --max-memory-per-slot '%s': %m", optctl.optarg); exit(1); } - memory_max = bytes/1024/1024; + arg_memory_max = bytes/1024/1024; } break; @@ -487,8 +501,8 @@ int server_init(struct mxq_server *server, int argc, char *argv[]) } } - if (getuid()) { -#ifdef RUNASNORMALUSER + if (!RUNNING_AS_ROOT) { +#if defined(MXQ_DEVELOPMENT) || defined(RUNASNORMALUSER) mx_log_notice("Running mxqd as non-root user."); #else mx_log_err("Running mxqd as non-root user is not supported at the moment."); @@ -511,14 +525,19 @@ int server_init(struct mxq_server *server, int argc, char *argv[]) mx_asprintf_forever(&server->host_id, "%s-%llx-%x", server->boot_id, server->starttime, getpid()); mx_setenv_forever("MXQ_HOSTID", server->host_id); - server->slots = threads_total; + server->slots = arg_threads_total; res = cpuset_init(server); if (res < 0) { mx_log_err("MAIN: cpuset_init() failed. exiting."); exit(1); } - server->memory_total = memory_total; - server->memory_max_per_slot = memory_max; + server->memory_total = arg_memory_total; + server->memory_max_per_slot = arg_memory_max; + + /* if run as non-root use full memory by default for every job */ + if (!arg_memory_max && !RUNNING_AS_ROOT) + server->memory_max_per_slot = arg_memory_total; + server->memory_avg_per_slot = (long double)server->memory_total / (long double)server->slots; if (server->memory_max_per_slot < server->memory_avg_per_slot) @@ -527,7 +546,6 @@ int server_init(struct mxq_server *server, int argc, char *argv[]) if (server->memory_max_per_slot > server->memory_total) server->memory_max_per_slot = server->memory_total; - return 1; } @@ -967,7 +985,7 @@ static int init_child_process(struct mxq_group_list *group, struct mxq_job *j) g->user_name, g->user_uid, g->group_id, j->job_id); } - if(getuid()==0) { + if(RUNNING_AS_ROOT) { res = initgroups(passwd->pw_name, g->user_gid); if (res == -1) { @@ -1512,8 +1530,11 @@ int killall_over_time(struct mxq_server *server) assert(server); - /* limit killing to every >= 5 minutes */ - mx_within_rate_limit_or_return(5*60, 1); + if (!server->jobs_running) + return 0; + + /* limit killing to every >= 60 seconds */ + mx_within_rate_limit_or_return(60, 1); mx_log_info("killall_over_time: Sending signals to all jobs running longer than requested."); @@ -1786,13 +1807,16 @@ int catchall(struct mxq_server *server) { } int load_groups(struct mxq_server *server) { - struct mxq_group *mxqgroups; + struct mxq_group *mxqgroups = NULL; struct mxq_group_list *group; int group_cnt; int total; int i; - group_cnt = mxq_load_active_groups(server->mysql, &mxqgroups); + if (RUNNING_AS_ROOT) + group_cnt = mxq_load_running_groups(server->mysql, &mxqgroups); + else + group_cnt = mxq_load_running_groups_for_user(server->mysql, &mxqgroups, getuid()); for (i=0, total=0; i " MXQ_VERSIONDATE); + mx_log_info(" by Marius Tolzmann 2013-" MXQ_VERSIONDATE); + mx_log_info(" and Donald Buczek 2015-" MXQ_VERSIONDATE); mx_log_info(" Max Planck Institute for Molecular Genetics - Berlin Dahlem"); +#ifdef MXQ_DEVELOPMENT + mx_log_warning("DEVELOPMENT VERSION: Do not use in production environments."); +#endif mx_log_info("hostname=%s server_id=%s :: MXQ server started.", server.hostname, server.server_id); mx_log_info(" host_id=%s", server.host_id); mx_log_info("slots=%lu memory_total=%lu memory_avg_per_slot=%.0Lf memory_max_per_slot=%ld :: server initialized.", diff --git a/mysql/create_tables.sql b/mysql/create_tables.sql index ce129e4..978c679 100644 --- a/mysql/create_tables.sql +++ b/mysql/create_tables.sql @@ -1,5 +1,3 @@ - -DROP TABLE mxq_group; CREATE TABLE IF NOT EXISTS mxq_group ( group_id INT8 UNSIGNED NOT NULL PRIMARY KEY AUTO_INCREMENT, group_name VARCHAR(511) NOT NULL DEFAULT 'default', @@ -62,7 +60,6 @@ CREATE TABLE IF NOT EXISTS mxq_group ( INDEX(group_name) ); -DROP TABLE mxq_job; CREATE TABLE IF NOT EXISTS mxq_job ( job_id INT8 UNSIGNED NOT NULL PRIMARY KEY AUTO_INCREMENT, job_status INT2 UNSIGNED NOT NULL DEFAULT 0, @@ -123,7 +120,6 @@ CREATE TABLE IF NOT EXISTS mxq_job ( INDEX (server_id(767)) ); -DROP TABLE mxq_server; CREATE TABLE IF NOT EXISTS mxq_server ( host_id INT4 UNSIGNED NOT NULL PRIMARY KEY AUTO_INCREMENT, host_hostname VARCHAR(511) NOT NULL DEFAULT 'localhost', @@ -142,136 +138,3 @@ CREATE TABLE IF NOT EXISTS mxq_server ( server_start TIMESTAMP DEFAULT 0, server_stop TIMESTAMP DEFAULT 0 ); - - - -LOCK TABLES mxq_job WRITE, mxq_group WRITE; -DELIMITER | -DROP TRIGGER mxq_add_group| -CREATE TRIGGER mxq_add_group BEFORE INSERT ON mxq_group - FOR EACH ROW BEGIN - SET NEW.group_mtime = NOW(); - - IF (NEW.group_jobs_running = 0 AND NEW.group_jobs_inq = 0) THEN - SET NEW.group_date_end = NEW.group_mtime; - ELSEIF (NEW.group_jobs_running > 0 OR NEW.group_jobs_inq > 0) THEN - SET NEW.group_date_end = 0; - END IF; - END; -| -DROP TRIGGER mxq_update_group| -CREATE TRIGGER mxq_update_group BEFORE UPDATE ON mxq_group - FOR EACH ROW BEGIN - SET NEW.group_mtime = NOW(); - - IF OLD.group_jobs_inq > 0 AND OLD.group_jobs_running = 0 THEN - SET NEW.stats_wait_sec = OLD.stats_wait_sec + (UNIX_TIMESTAMP(NEW.group_mtime) - UNIX_TIMESTAMP(OLD.group_mtime)); - ELSEIF OLD.group_jobs_running > 0 THEN - SET NEW.stats_run_sec = OLD.stats_run_sec + (UNIX_TIMESTAMP(NEW.group_mtime) - UNIX_TIMESTAMP(OLD.group_mtime)); - END IF; - - IF (NEW.group_jobs_running = 0 AND NEW.group_jobs_inq = 0) AND - (OLD.group_jobs_running > 0 OR OLD.group_jobs_inq > 0) THEN - SET NEW.group_date_end = NEW.group_mtime; - ELSEIF (OLD.group_jobs_running = 0 AND OLD.group_jobs_inq = 0) AND - (NEW.group_jobs_running > 0 OR NEW.group_jobs_inq > 0) THEN - SET NEW.stats_idle_sec = OLD.stats_idle_sec + (UNIX_TIMESTAMP(NEW.group_mtime) - UNIX_TIMESTAMP(OLD.group_date_end)); - SET NEW.group_date_end = 0; - END IF; - END; -| -DROP TRIGGER mxq_add_job| -CREATE TRIGGER mxq_add_job AFTER INSERT ON mxq_job - FOR EACH ROW BEGIN - UPDATE mxq_group SET - group_jobs=group_jobs+1, - group_jobs_inq=group_jobs_inq+1, - group_mtime=NULL - WHERE group_id=NEW.group_id; - END; -| -DROP TRIGGER mxq_update_job| -CREATE TRIGGER mxq_update_job BEFORE UPDATE ON mxq_job - FOR EACH ROW BEGIN - IF NEW.job_status != OLD.job_status THEN - IF NEW.job_status IN (150, 200) AND OLD.job_status IN (0, 100) THEN - UPDATE mxq_group SET - group_jobs_inq=group_jobs_inq-1, - group_jobs_running=group_jobs_running+1, - group_slots_running=group_slots_running+NEW.host_slots, - group_mtime=NULL - WHERE group_id=NEW.group_id; - ELSEIF NEW.job_status = 200 AND OLD.job_status = 150 THEN - UPDATE mxq_group SET - group_slots_running=group_slots_running-OLD.host_slots+NEW.host_slots, - group_mtime=NULL - WHERE group_id=NEW.group_id; - ELSEIF NEW.job_status IN (400, 750) AND OLD.job_status IN (150, 200, 250, 300, 350, 399) THEN - UPDATE mxq_group SET - group_slots_running=group_slots_running-NEW.host_slots, - group_jobs_running=group_jobs_running-1, - group_jobs_failed=group_jobs_failed+1, - stats_max_maxrss=GREATEST(stats_max_maxrss, NEW.stats_maxrss), - stats_max_utime_sec=GREATEST(stats_max_utime_sec, NEW.stats_utime_sec), - stats_max_stime_sec=GREATEST(stats_max_stime_sec, NEW.stats_stime_sec), - stats_max_real_sec=GREATEST(stats_max_real_sec, NEW.stats_real_sec), - stats_total_utime_sec=stats_total_utime_sec+NEW.stats_utime_sec, - stats_total_stime_sec=stats_total_stime_sec+NEW.stats_stime_sec, - stats_total_real_sec=stats_total_real_sec+NEW.stats_real_sec, - group_mtime=NULL - WHERE group_id=NEW.group_id; - ELSEIF NEW.job_status = 990 AND OLD.job_status IN (0, 100, 989) THEN - SET NEW.date_start = NOW(); - SET NEW.date_end = NEW.date_start; - UPDATE mxq_group SET - group_jobs_inq=group_jobs_inq-1, - group_jobs_cancelled=group_jobs_cancelled+1, - group_mtime=NULL - WHERE group_id=NEW.group_id; - ELSEIF NEW.job_status = 999 AND OLD.job_status IN (150, 200, 250, 399) THEN - UPDATE mxq_group SET - group_slots_running=group_slots_running-NEW.host_slots, - group_jobs_running=group_jobs_running-1, - group_jobs_unknown=group_jobs_unknown+1, - group_mtime=NULL - WHERE group_id=NEW.group_id; - ELSEIF NEW.job_status = 999 AND OLD.job_status IN (400, 750, 755) THEN - UPDATE mxq_group SET - group_jobs_failed=group_jobs_failed-1, - group_jobs_unknown=group_jobs_unknown+1, - group_mtime=NULL - WHERE group_id=NEW.group_id; - ELSEIF NEW.job_status = 1000 AND OLD.job_status IN (150, 200, 250, 300, 350, 399) THEN - UPDATE mxq_group SET - group_slots_running=group_slots_running-NEW.host_slots, - group_jobs_running=group_jobs_running-1, - group_jobs_finished=group_jobs_finished+1, - stats_max_maxrss=GREATEST(stats_max_maxrss, NEW.stats_maxrss), - stats_max_utime_sec=GREATEST(stats_max_utime_sec, NEW.stats_utime_sec), - stats_max_stime_sec=GREATEST(stats_max_stime_sec, NEW.stats_stime_sec), - stats_max_real_sec=GREATEST(stats_max_real_sec, NEW.stats_real_sec), - stats_total_utime_sec=stats_total_utime_sec+NEW.stats_utime_sec, - stats_total_stime_sec=stats_total_stime_sec+NEW.stats_stime_sec, - stats_total_real_sec=stats_total_real_sec+NEW.stats_real_sec, - stats_total_utime_sec_finished=stats_total_utime_sec_finished+NEW.stats_utime_sec, - stats_total_stime_sec_finished=stats_total_stime_sec_finished+NEW.stats_stime_sec, - stats_total_real_sec_finished=stats_total_real_sec_finished+NEW.stats_real_sec, - group_mtime=NULL - WHERE group_id=NEW.group_id; - ELSEIF NEW.job_status NOT IN (399, 755, 989, 990) THEN - UPDATE mxq_group SET - stats_max_maxrss=GREATEST(stats_max_maxrss, NEW.stats_maxrss), - stats_max_utime_sec=GREATEST(stats_max_utime_sec, NEW.stats_utime_sec), - stats_max_stime_sec=GREATEST(stats_max_stime_sec, NEW.stats_stime_sec), - stats_max_real_sec=GREATEST(stats_max_real_sec, NEW.stats_real_sec), - stats_total_utime_sec=stats_total_utime_sec+NEW.stats_utime_sec, - stats_total_stime_sec=stats_total_stime_sec+NEW.stats_stime_sec, - stats_total_real_sec=stats_total_real_sec+NEW.stats_real_sec, - group_mtime=NULL - WHERE group_id=NEW.group_id; - END IF; - END IF; - END; -| -DELIMITER ; -UNLOCK TABLES; diff --git a/mysql/create_trigger.sql b/mysql/create_trigger.sql new file mode 100644 index 0000000..ccd2af6 --- /dev/null +++ b/mysql/create_trigger.sql @@ -0,0 +1,130 @@ +LOCK TABLES mxq_job WRITE, mxq_group WRITE; +DELIMITER | +DROP TRIGGER IF EXISTS mxq_add_group| +CREATE TRIGGER mxq_add_group BEFORE INSERT ON mxq_group + FOR EACH ROW BEGIN + SET NEW.group_mtime = NOW(); + + IF (NEW.group_jobs_running = 0 AND NEW.group_jobs_inq = 0) THEN + SET NEW.group_date_end = NEW.group_mtime; + ELSEIF (NEW.group_jobs_running > 0 OR NEW.group_jobs_inq > 0) THEN + SET NEW.group_date_end = 0; + END IF; + END; +| +DROP TRIGGER IF EXISTS mxq_update_group| +CREATE TRIGGER mxq_update_group BEFORE UPDATE ON mxq_group + FOR EACH ROW BEGIN + SET NEW.group_mtime = NOW(); + + IF OLD.group_jobs_inq > 0 AND OLD.group_jobs_running = 0 THEN + SET NEW.stats_wait_sec = OLD.stats_wait_sec + (UNIX_TIMESTAMP(NEW.group_mtime) - UNIX_TIMESTAMP(OLD.group_mtime)); + ELSEIF OLD.group_jobs_running > 0 THEN + SET NEW.stats_run_sec = OLD.stats_run_sec + (UNIX_TIMESTAMP(NEW.group_mtime) - UNIX_TIMESTAMP(OLD.group_mtime)); + END IF; + + IF (NEW.group_jobs_running = 0 AND NEW.group_jobs_inq = 0) AND + (OLD.group_jobs_running > 0 OR OLD.group_jobs_inq > 0) THEN + SET NEW.group_date_end = NEW.group_mtime; + ELSEIF (OLD.group_jobs_running = 0 AND OLD.group_jobs_inq = 0) AND + (NEW.group_jobs_running > 0 OR NEW.group_jobs_inq > 0) THEN + SET NEW.stats_idle_sec = OLD.stats_idle_sec + (UNIX_TIMESTAMP(NEW.group_mtime) - UNIX_TIMESTAMP(OLD.group_date_end)); + SET NEW.group_date_end = 0; + END IF; + END; +| +DROP TRIGGER IF EXISTS mxq_add_job| +CREATE TRIGGER mxq_add_job AFTER INSERT ON mxq_job + FOR EACH ROW BEGIN + UPDATE mxq_group SET + group_jobs=group_jobs+1, + group_jobs_inq=group_jobs_inq+1, + group_mtime=NULL + WHERE group_id=NEW.group_id; + END; +| +DROP TRIGGER IF EXISTS mxq_update_job| +CREATE TRIGGER mxq_update_job BEFORE UPDATE ON mxq_job + FOR EACH ROW BEGIN + IF NEW.job_status != OLD.job_status THEN + IF NEW.job_status IN (150, 200) AND OLD.job_status IN (0, 100) THEN + UPDATE mxq_group SET + group_jobs_inq=group_jobs_inq-1, + group_jobs_running=group_jobs_running+1, + group_slots_running=group_slots_running+NEW.host_slots, + group_mtime=NULL + WHERE group_id=NEW.group_id; + ELSEIF NEW.job_status = 200 AND OLD.job_status = 150 THEN + UPDATE mxq_group SET + group_slots_running=group_slots_running-OLD.host_slots+NEW.host_slots, + group_mtime=NULL + WHERE group_id=NEW.group_id; + ELSEIF NEW.job_status IN (400, 750) AND OLD.job_status IN (150, 200, 250, 300, 350, 399) THEN + UPDATE mxq_group SET + group_slots_running=group_slots_running-NEW.host_slots, + group_jobs_running=group_jobs_running-1, + group_jobs_failed=group_jobs_failed+1, + stats_max_maxrss=GREATEST(stats_max_maxrss, NEW.stats_maxrss), + stats_max_utime_sec=GREATEST(stats_max_utime_sec, NEW.stats_utime_sec), + stats_max_stime_sec=GREATEST(stats_max_stime_sec, NEW.stats_stime_sec), + stats_max_real_sec=GREATEST(stats_max_real_sec, NEW.stats_real_sec), + stats_total_utime_sec=stats_total_utime_sec+NEW.stats_utime_sec, + stats_total_stime_sec=stats_total_stime_sec+NEW.stats_stime_sec, + stats_total_real_sec=stats_total_real_sec+NEW.stats_real_sec, + group_mtime=NULL + WHERE group_id=NEW.group_id; + ELSEIF NEW.job_status = 990 AND OLD.job_status IN (0, 100, 989) THEN + SET NEW.date_start = NOW(); + SET NEW.date_end = NEW.date_start; + UPDATE mxq_group SET + group_jobs_inq=group_jobs_inq-1, + group_jobs_cancelled=group_jobs_cancelled+1, + group_mtime=NULL + WHERE group_id=NEW.group_id; + ELSEIF NEW.job_status = 999 AND OLD.job_status IN (150, 200, 250, 399) THEN + UPDATE mxq_group SET + group_slots_running=group_slots_running-NEW.host_slots, + group_jobs_running=group_jobs_running-1, + group_jobs_unknown=group_jobs_unknown+1, + group_mtime=NULL + WHERE group_id=NEW.group_id; + ELSEIF NEW.job_status = 999 AND OLD.job_status IN (400, 750, 755) THEN + UPDATE mxq_group SET + group_jobs_failed=group_jobs_failed-1, + group_jobs_unknown=group_jobs_unknown+1, + group_mtime=NULL + WHERE group_id=NEW.group_id; + ELSEIF NEW.job_status = 1000 AND OLD.job_status IN (150, 200, 250, 300, 350, 399) THEN + UPDATE mxq_group SET + group_slots_running=group_slots_running-NEW.host_slots, + group_jobs_running=group_jobs_running-1, + group_jobs_finished=group_jobs_finished+1, + stats_max_maxrss=GREATEST(stats_max_maxrss, NEW.stats_maxrss), + stats_max_utime_sec=GREATEST(stats_max_utime_sec, NEW.stats_utime_sec), + stats_max_stime_sec=GREATEST(stats_max_stime_sec, NEW.stats_stime_sec), + stats_max_real_sec=GREATEST(stats_max_real_sec, NEW.stats_real_sec), + stats_total_utime_sec=stats_total_utime_sec+NEW.stats_utime_sec, + stats_total_stime_sec=stats_total_stime_sec+NEW.stats_stime_sec, + stats_total_real_sec=stats_total_real_sec+NEW.stats_real_sec, + stats_total_utime_sec_finished=stats_total_utime_sec_finished+NEW.stats_utime_sec, + stats_total_stime_sec_finished=stats_total_stime_sec_finished+NEW.stats_stime_sec, + stats_total_real_sec_finished=stats_total_real_sec_finished+NEW.stats_real_sec, + group_mtime=NULL + WHERE group_id=NEW.group_id; + ELSEIF NEW.job_status NOT IN (399, 755, 989, 990) THEN + UPDATE mxq_group SET + stats_max_maxrss=GREATEST(stats_max_maxrss, NEW.stats_maxrss), + stats_max_utime_sec=GREATEST(stats_max_utime_sec, NEW.stats_utime_sec), + stats_max_stime_sec=GREATEST(stats_max_stime_sec, NEW.stats_stime_sec), + stats_max_real_sec=GREATEST(stats_max_real_sec, NEW.stats_real_sec), + stats_total_utime_sec=stats_total_utime_sec+NEW.stats_utime_sec, + stats_total_stime_sec=stats_total_stime_sec+NEW.stats_stime_sec, + stats_total_real_sec=stats_total_real_sec+NEW.stats_real_sec, + group_mtime=NULL + WHERE group_id=NEW.group_id; + END IF; + END IF; + END; +| +DELIMITER ; +UNLOCK TABLES; diff --git a/mysql/drop_tables.sql b/mysql/drop_tables.sql new file mode 100644 index 0000000..5ff48f1 --- /dev/null +++ b/mysql/drop_tables.sql @@ -0,0 +1,3 @@ +DROP TABLE IF EXISTS mxq_group; +DROP TABLE IF EXISTS mxq_job; +DROP TABLE IF EXISTS mxq_server;