Skip to content

Commit

Permalink
Merge branch 'improvements'
Browse files Browse the repository at this point in the history 
* improvements:
  mysql: Split create_tables into create_tables, drop_tables and create_trigger
  mysql: Drop tables and triggers only if they exist
  mxqd: limit killall_over_time to run every minute if jobs are running
  mxqd: Change default for --no-log in development build
  Makefile: remove definition of RUNASNORMALUSER in devel mode
  • Loading branch information
@mariux
mariux committed Oct 24, 2015
2 parents bd9deb9 + cce9159 commit 1a1c954
Show file tree
Hide file tree
Showing 5 changed files with 143 additions and 141 deletions.
1 change: 0 additions & 1 deletion Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -189,7 +189,6 @@ manpages/%: manpages/%.xml
all: build

.PHONY: devel
devel: CFLAGS += -DRUNASNORMALUSER
devel: CFLAGS += -DMXQ_DEVELOPMENT
devel: all

Expand Down
13 changes: 10 additions & 3 deletions mxqd.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -313,6 +313,10 @@ int server_init(struct mxq_server *server, int argc, char *argv[])
arg_server_id = "main";
arg_hostname = mx_hostname();

#ifdef MXQ_DEVELOPMENT
arg_nolog = 1;
#endif

arg_initial_path = MXQ_INITIAL_PATH;
arg_initial_tmpdir = MXQ_INITIAL_TMPDIR;

Expand Down Expand Up @@ -1523,8 +1527,11 @@ int killall_over_time(struct mxq_server *server)

assert(server);

/* limit killing to every >= 5 minutes */
mx_within_rate_limit_or_return(5*60, 1);
if (!server->jobs_running)
return 0;

/* limit killing to every >= 60 seconds */
mx_within_rate_limit_or_return(60, 1);

mx_log_info("killall_over_time: Sending signals to all jobs running longer than requested.");

Expand Down Expand Up @@ -1834,7 +1841,7 @@ int main(int argc, char *argv[])
mx_log_info(" and Donald Buczek <buczek@molgen.mpg.de> 2015-" MXQ_VERSIONDATE);
mx_log_info(" Max Planck Institute for Molecular Genetics - Berlin Dahlem");
#ifdef MXQ_DEVELOPMENT
mx_log_warning("DEVELOPMENT VERSION: Do not use in production environments.\n");
mx_log_warning("DEVELOPMENT VERSION: Do not use in production environments.");
#endif
mx_log_info("hostname=%s server_id=%s :: MXQ server started.", server.hostname, server.server_id);
mx_log_info(" host_id=%s", server.host_id);
Expand Down
137 changes: 0 additions & 137 deletions mysql/create_tables.sql
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,3 @@

DROP TABLE mxq_group;
CREATE TABLE IF NOT EXISTS mxq_group (
group_id INT8 UNSIGNED NOT NULL PRIMARY KEY AUTO_INCREMENT,
group_name VARCHAR(511) NOT NULL DEFAULT 'default',
Expand Down Expand Up @@ -62,7 +60,6 @@ CREATE TABLE IF NOT EXISTS mxq_group (
INDEX(group_name)
);

DROP TABLE mxq_job;
CREATE TABLE IF NOT EXISTS mxq_job (
job_id INT8 UNSIGNED NOT NULL PRIMARY KEY AUTO_INCREMENT,
job_status INT2 UNSIGNED NOT NULL DEFAULT 0,
Expand Down Expand Up @@ -123,7 +120,6 @@ CREATE TABLE IF NOT EXISTS mxq_job (
INDEX (server_id(767))
);

DROP TABLE mxq_server;
CREATE TABLE IF NOT EXISTS mxq_server (
host_id INT4 UNSIGNED NOT NULL PRIMARY KEY AUTO_INCREMENT,
host_hostname VARCHAR(511) NOT NULL DEFAULT 'localhost',
Expand All @@ -142,136 +138,3 @@ CREATE TABLE IF NOT EXISTS mxq_server (
server_start TIMESTAMP DEFAULT 0,
server_stop TIMESTAMP DEFAULT 0
);



LOCK TABLES mxq_job WRITE, mxq_group WRITE;
DELIMITER |
DROP TRIGGER mxq_add_group|
CREATE TRIGGER mxq_add_group BEFORE INSERT ON mxq_group
FOR EACH ROW BEGIN
SET NEW.group_mtime = NOW();

IF (NEW.group_jobs_running = 0 AND NEW.group_jobs_inq = 0) THEN
SET NEW.group_date_end = NEW.group_mtime;
ELSEIF (NEW.group_jobs_running > 0 OR NEW.group_jobs_inq > 0) THEN
SET NEW.group_date_end = 0;
END IF;
END;
|
DROP TRIGGER mxq_update_group|
CREATE TRIGGER mxq_update_group BEFORE UPDATE ON mxq_group
FOR EACH ROW BEGIN
SET NEW.group_mtime = NOW();

IF OLD.group_jobs_inq > 0 AND OLD.group_jobs_running = 0 THEN
SET NEW.stats_wait_sec = OLD.stats_wait_sec + (UNIX_TIMESTAMP(NEW.group_mtime) - UNIX_TIMESTAMP(OLD.group_mtime));
ELSEIF OLD.group_jobs_running > 0 THEN
SET NEW.stats_run_sec = OLD.stats_run_sec + (UNIX_TIMESTAMP(NEW.group_mtime) - UNIX_TIMESTAMP(OLD.group_mtime));
END IF;

IF (NEW.group_jobs_running = 0 AND NEW.group_jobs_inq = 0) AND
(OLD.group_jobs_running > 0 OR OLD.group_jobs_inq > 0) THEN
SET NEW.group_date_end = NEW.group_mtime;
ELSEIF (OLD.group_jobs_running = 0 AND OLD.group_jobs_inq = 0) AND
(NEW.group_jobs_running > 0 OR NEW.group_jobs_inq > 0) THEN
SET NEW.stats_idle_sec = OLD.stats_idle_sec + (UNIX_TIMESTAMP(NEW.group_mtime) - UNIX_TIMESTAMP(OLD.group_date_end));
SET NEW.group_date_end = 0;
END IF;
END;
|
DROP TRIGGER mxq_add_job|
CREATE TRIGGER mxq_add_job AFTER INSERT ON mxq_job
FOR EACH ROW BEGIN
UPDATE mxq_group SET
group_jobs=group_jobs+1,
group_jobs_inq=group_jobs_inq+1,
group_mtime=NULL
WHERE group_id=NEW.group_id;
END;
|
DROP TRIGGER mxq_update_job|
CREATE TRIGGER mxq_update_job BEFORE UPDATE ON mxq_job
FOR EACH ROW BEGIN
IF NEW.job_status != OLD.job_status THEN
IF NEW.job_status IN (150, 200) AND OLD.job_status IN (0, 100) THEN
UPDATE mxq_group SET
group_jobs_inq=group_jobs_inq-1,
group_jobs_running=group_jobs_running+1,
group_slots_running=group_slots_running+NEW.host_slots,
group_mtime=NULL
WHERE group_id=NEW.group_id;
ELSEIF NEW.job_status = 200 AND OLD.job_status = 150 THEN
UPDATE mxq_group SET
group_slots_running=group_slots_running-OLD.host_slots+NEW.host_slots,
group_mtime=NULL
WHERE group_id=NEW.group_id;
ELSEIF NEW.job_status IN (400, 750) AND OLD.job_status IN (150, 200, 250, 300, 350, 399) THEN
UPDATE mxq_group SET
group_slots_running=group_slots_running-NEW.host_slots,
group_jobs_running=group_jobs_running-1,
group_jobs_failed=group_jobs_failed+1,
stats_max_maxrss=GREATEST(stats_max_maxrss, NEW.stats_maxrss),
stats_max_utime_sec=GREATEST(stats_max_utime_sec, NEW.stats_utime_sec),
stats_max_stime_sec=GREATEST(stats_max_stime_sec, NEW.stats_stime_sec),
stats_max_real_sec=GREATEST(stats_max_real_sec, NEW.stats_real_sec),
stats_total_utime_sec=stats_total_utime_sec+NEW.stats_utime_sec,
stats_total_stime_sec=stats_total_stime_sec+NEW.stats_stime_sec,
stats_total_real_sec=stats_total_real_sec+NEW.stats_real_sec,
group_mtime=NULL
WHERE group_id=NEW.group_id;
ELSEIF NEW.job_status = 990 AND OLD.job_status IN (0, 100, 989) THEN
SET NEW.date_start = NOW();
SET NEW.date_end = NEW.date_start;
UPDATE mxq_group SET
group_jobs_inq=group_jobs_inq-1,
group_jobs_cancelled=group_jobs_cancelled+1,
group_mtime=NULL
WHERE group_id=NEW.group_id;
ELSEIF NEW.job_status = 999 AND OLD.job_status IN (150, 200, 250, 399) THEN
UPDATE mxq_group SET
group_slots_running=group_slots_running-NEW.host_slots,
group_jobs_running=group_jobs_running-1,
group_jobs_unknown=group_jobs_unknown+1,
group_mtime=NULL
WHERE group_id=NEW.group_id;
ELSEIF NEW.job_status = 999 AND OLD.job_status IN (400, 750, 755) THEN
UPDATE mxq_group SET
group_jobs_failed=group_jobs_failed-1,
group_jobs_unknown=group_jobs_unknown+1,
group_mtime=NULL
WHERE group_id=NEW.group_id;
ELSEIF NEW.job_status = 1000 AND OLD.job_status IN (150, 200, 250, 300, 350, 399) THEN
UPDATE mxq_group SET
group_slots_running=group_slots_running-NEW.host_slots,
group_jobs_running=group_jobs_running-1,
group_jobs_finished=group_jobs_finished+1,
stats_max_maxrss=GREATEST(stats_max_maxrss, NEW.stats_maxrss),
stats_max_utime_sec=GREATEST(stats_max_utime_sec, NEW.stats_utime_sec),
stats_max_stime_sec=GREATEST(stats_max_stime_sec, NEW.stats_stime_sec),
stats_max_real_sec=GREATEST(stats_max_real_sec, NEW.stats_real_sec),
stats_total_utime_sec=stats_total_utime_sec+NEW.stats_utime_sec,
stats_total_stime_sec=stats_total_stime_sec+NEW.stats_stime_sec,
stats_total_real_sec=stats_total_real_sec+NEW.stats_real_sec,
stats_total_utime_sec_finished=stats_total_utime_sec_finished+NEW.stats_utime_sec,
stats_total_stime_sec_finished=stats_total_stime_sec_finished+NEW.stats_stime_sec,
stats_total_real_sec_finished=stats_total_real_sec_finished+NEW.stats_real_sec,
group_mtime=NULL
WHERE group_id=NEW.group_id;
ELSEIF NEW.job_status NOT IN (399, 755, 989, 990) THEN
UPDATE mxq_group SET
stats_max_maxrss=GREATEST(stats_max_maxrss, NEW.stats_maxrss),
stats_max_utime_sec=GREATEST(stats_max_utime_sec, NEW.stats_utime_sec),
stats_max_stime_sec=GREATEST(stats_max_stime_sec, NEW.stats_stime_sec),
stats_max_real_sec=GREATEST(stats_max_real_sec, NEW.stats_real_sec),
stats_total_utime_sec=stats_total_utime_sec+NEW.stats_utime_sec,
stats_total_stime_sec=stats_total_stime_sec+NEW.stats_stime_sec,
stats_total_real_sec=stats_total_real_sec+NEW.stats_real_sec,
group_mtime=NULL
WHERE group_id=NEW.group_id;
END IF;
END IF;
END;
|
DELIMITER ;
UNLOCK TABLES;
130 changes: 130 additions & 0 deletions mysql/create_trigger.sql
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,130 @@
LOCK TABLES mxq_job WRITE, mxq_group WRITE;
DELIMITER |
DROP TRIGGER IF EXISTS mxq_add_group|
CREATE TRIGGER mxq_add_group BEFORE INSERT ON mxq_group
FOR EACH ROW BEGIN
SET NEW.group_mtime = NOW();

IF (NEW.group_jobs_running = 0 AND NEW.group_jobs_inq = 0) THEN
SET NEW.group_date_end = NEW.group_mtime;
ELSEIF (NEW.group_jobs_running > 0 OR NEW.group_jobs_inq > 0) THEN
SET NEW.group_date_end = 0;
END IF;
END;
|
DROP TRIGGER IF EXISTS mxq_update_group|
CREATE TRIGGER mxq_update_group BEFORE UPDATE ON mxq_group
FOR EACH ROW BEGIN
SET NEW.group_mtime = NOW();

IF OLD.group_jobs_inq > 0 AND OLD.group_jobs_running = 0 THEN
SET NEW.stats_wait_sec = OLD.stats_wait_sec + (UNIX_TIMESTAMP(NEW.group_mtime) - UNIX_TIMESTAMP(OLD.group_mtime));
ELSEIF OLD.group_jobs_running > 0 THEN
SET NEW.stats_run_sec = OLD.stats_run_sec + (UNIX_TIMESTAMP(NEW.group_mtime) - UNIX_TIMESTAMP(OLD.group_mtime));
END IF;

IF (NEW.group_jobs_running = 0 AND NEW.group_jobs_inq = 0) AND
(OLD.group_jobs_running > 0 OR OLD.group_jobs_inq > 0) THEN
SET NEW.group_date_end = NEW.group_mtime;
ELSEIF (OLD.group_jobs_running = 0 AND OLD.group_jobs_inq = 0) AND
(NEW.group_jobs_running > 0 OR NEW.group_jobs_inq > 0) THEN
SET NEW.stats_idle_sec = OLD.stats_idle_sec + (UNIX_TIMESTAMP(NEW.group_mtime) - UNIX_TIMESTAMP(OLD.group_date_end));
SET NEW.group_date_end = 0;
END IF;
END;
|
DROP TRIGGER IF EXISTS mxq_add_job|
CREATE TRIGGER mxq_add_job AFTER INSERT ON mxq_job
FOR EACH ROW BEGIN
UPDATE mxq_group SET
group_jobs=group_jobs+1,
group_jobs_inq=group_jobs_inq+1,
group_mtime=NULL
WHERE group_id=NEW.group_id;
END;
|
DROP TRIGGER IF EXISTS mxq_update_job|
CREATE TRIGGER mxq_update_job BEFORE UPDATE ON mxq_job
FOR EACH ROW BEGIN
IF NEW.job_status != OLD.job_status THEN
IF NEW.job_status IN (150, 200) AND OLD.job_status IN (0, 100) THEN
UPDATE mxq_group SET
group_jobs_inq=group_jobs_inq-1,
group_jobs_running=group_jobs_running+1,
group_slots_running=group_slots_running+NEW.host_slots,
group_mtime=NULL
WHERE group_id=NEW.group_id;
ELSEIF NEW.job_status = 200 AND OLD.job_status = 150 THEN
UPDATE mxq_group SET
group_slots_running=group_slots_running-OLD.host_slots+NEW.host_slots,
group_mtime=NULL
WHERE group_id=NEW.group_id;
ELSEIF NEW.job_status IN (400, 750) AND OLD.job_status IN (150, 200, 250, 300, 350, 399) THEN
UPDATE mxq_group SET
group_slots_running=group_slots_running-NEW.host_slots,
group_jobs_running=group_jobs_running-1,
group_jobs_failed=group_jobs_failed+1,
stats_max_maxrss=GREATEST(stats_max_maxrss, NEW.stats_maxrss),
stats_max_utime_sec=GREATEST(stats_max_utime_sec, NEW.stats_utime_sec),
stats_max_stime_sec=GREATEST(stats_max_stime_sec, NEW.stats_stime_sec),
stats_max_real_sec=GREATEST(stats_max_real_sec, NEW.stats_real_sec),
stats_total_utime_sec=stats_total_utime_sec+NEW.stats_utime_sec,
stats_total_stime_sec=stats_total_stime_sec+NEW.stats_stime_sec,
stats_total_real_sec=stats_total_real_sec+NEW.stats_real_sec,
group_mtime=NULL
WHERE group_id=NEW.group_id;
ELSEIF NEW.job_status = 990 AND OLD.job_status IN (0, 100, 989) THEN
SET NEW.date_start = NOW();
SET NEW.date_end = NEW.date_start;
UPDATE mxq_group SET
group_jobs_inq=group_jobs_inq-1,
group_jobs_cancelled=group_jobs_cancelled+1,
group_mtime=NULL
WHERE group_id=NEW.group_id;
ELSEIF NEW.job_status = 999 AND OLD.job_status IN (150, 200, 250, 399) THEN
UPDATE mxq_group SET
group_slots_running=group_slots_running-NEW.host_slots,
group_jobs_running=group_jobs_running-1,
group_jobs_unknown=group_jobs_unknown+1,
group_mtime=NULL
WHERE group_id=NEW.group_id;
ELSEIF NEW.job_status = 999 AND OLD.job_status IN (400, 750, 755) THEN
UPDATE mxq_group SET
group_jobs_failed=group_jobs_failed-1,
group_jobs_unknown=group_jobs_unknown+1,
group_mtime=NULL
WHERE group_id=NEW.group_id;
ELSEIF NEW.job_status = 1000 AND OLD.job_status IN (150, 200, 250, 300, 350, 399) THEN
UPDATE mxq_group SET
group_slots_running=group_slots_running-NEW.host_slots,
group_jobs_running=group_jobs_running-1,
group_jobs_finished=group_jobs_finished+1,
stats_max_maxrss=GREATEST(stats_max_maxrss, NEW.stats_maxrss),
stats_max_utime_sec=GREATEST(stats_max_utime_sec, NEW.stats_utime_sec),
stats_max_stime_sec=GREATEST(stats_max_stime_sec, NEW.stats_stime_sec),
stats_max_real_sec=GREATEST(stats_max_real_sec, NEW.stats_real_sec),
stats_total_utime_sec=stats_total_utime_sec+NEW.stats_utime_sec,
stats_total_stime_sec=stats_total_stime_sec+NEW.stats_stime_sec,
stats_total_real_sec=stats_total_real_sec+NEW.stats_real_sec,
stats_total_utime_sec_finished=stats_total_utime_sec_finished+NEW.stats_utime_sec,
stats_total_stime_sec_finished=stats_total_stime_sec_finished+NEW.stats_stime_sec,
stats_total_real_sec_finished=stats_total_real_sec_finished+NEW.stats_real_sec,
group_mtime=NULL
WHERE group_id=NEW.group_id;
ELSEIF NEW.job_status NOT IN (399, 755, 989, 990) THEN
UPDATE mxq_group SET
stats_max_maxrss=GREATEST(stats_max_maxrss, NEW.stats_maxrss),
stats_max_utime_sec=GREATEST(stats_max_utime_sec, NEW.stats_utime_sec),
stats_max_stime_sec=GREATEST(stats_max_stime_sec, NEW.stats_stime_sec),
stats_max_real_sec=GREATEST(stats_max_real_sec, NEW.stats_real_sec),
stats_total_utime_sec=stats_total_utime_sec+NEW.stats_utime_sec,
stats_total_stime_sec=stats_total_stime_sec+NEW.stats_stime_sec,
stats_total_real_sec=stats_total_real_sec+NEW.stats_real_sec,
group_mtime=NULL
WHERE group_id=NEW.group_id;
END IF;
END IF;
END;
|
DELIMITER ;
UNLOCK TABLES;
3 changes: 3 additions & 0 deletions mysql/drop_tables.sql
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
DROP TABLE IF EXISTS mxq_group;
DROP TABLE IF EXISTS mxq_job;
DROP TABLE IF EXISTS mxq_server;

0 comments on commit 1a1c954

Please sign in to comment.