From 4fec1a739636cf99185e01d43a00ef76ba459eef Mon Sep 17 00:00:00 2001 From: Donald Buczek Date: Mon, 29 Mar 2021 12:50:54 +0200 Subject: [PATCH 1/6] sql: Remove obsolete migrations --- mysql/alter_tables_0.13.0.sql | 5 ---- mysql/alter_tables_0.15.0.sql | 11 ------- mysql/alter_tables_0.17.0.sql | 11 ------- mysql/alter_tables_0.18.2.sql | 5 ---- mysql/alter_tables_0.21.3.sql | 46 ------------------------------ mysql/alter_tables_0.22.2.sql | 5 ---- mysql/alter_tables_0.24.0-post.sql | 2 -- mysql/alter_tables_0.24.0-pre.sql | 4 --- 8 files changed, 89 deletions(-) delete mode 100644 mysql/alter_tables_0.13.0.sql delete mode 100644 mysql/alter_tables_0.15.0.sql delete mode 100644 mysql/alter_tables_0.17.0.sql delete mode 100644 mysql/alter_tables_0.18.2.sql delete mode 100644 mysql/alter_tables_0.21.3.sql delete mode 100644 mysql/alter_tables_0.22.2.sql delete mode 100644 mysql/alter_tables_0.24.0-post.sql delete mode 100644 mysql/alter_tables_0.24.0-pre.sql diff --git a/mysql/alter_tables_0.13.0.sql b/mysql/alter_tables_0.13.0.sql deleted file mode 100644 index b2046be8..00000000 --- a/mysql/alter_tables_0.13.0.sql +++ /dev/null @@ -1,5 +0,0 @@ -ALTER TABLE mxq_group - ADD COLUMN ( - depends_on_group INT8 UNSIGNED NULL DEFAULT NULL, - dependency_of_group INT8 UNSIGNED NULL DEFAULT NULL - ); diff --git a/mysql/alter_tables_0.15.0.sql b/mysql/alter_tables_0.15.0.sql deleted file mode 100644 index 95faa894..00000000 --- a/mysql/alter_tables_0.15.0.sql +++ /dev/null @@ -1,11 +0,0 @@ -ALTER TABLE mxq_group - ADD COLUMN - job_max_per_node INT2 UNSIGNED NOT NULL DEFAULT 0 - AFTER - job_time; - -ALTER TABLE mxq_job - ADD COLUMN - host_id VARCHAR(1023) NOT NULL DEFAULT "" - AFTER - server_id; diff --git a/mysql/alter_tables_0.17.0.sql b/mysql/alter_tables_0.17.0.sql deleted file mode 100644 index b9daa722..00000000 --- a/mysql/alter_tables_0.17.0.sql +++ /dev/null @@ -1,11 +0,0 @@ -ALTER TABLE mxq_group - ADD COLUMN - stats_max_sumrss INT8 UNSIGNED NOT NULL DEFAULT 0 - AFTER - group_date_end; - -ALTER TABLE mxq_job - ADD COLUMN - stats_max_sumrss INT8 UNSIGNED NOT NULL DEFAULT 0 - AFTER - job_id_first; diff --git a/mysql/alter_tables_0.18.2.sql b/mysql/alter_tables_0.18.2.sql deleted file mode 100644 index 1be5d73d..00000000 --- a/mysql/alter_tables_0.18.2.sql +++ /dev/null @@ -1,5 +0,0 @@ -ALTER TABLE mxq_job - ADD COLUMN - host_cpu_set VARCHAR(4095) NOT NULL DEFAULT "" - AFTER - host_slots; diff --git a/mysql/alter_tables_0.21.3.sql b/mysql/alter_tables_0.21.3.sql deleted file mode 100644 index 19365a67..00000000 --- a/mysql/alter_tables_0.21.3.sql +++ /dev/null @@ -1,46 +0,0 @@ - -ALTER TABLE mxq_group - MODIFY COLUMN - group_name VARCHAR(128) NOT NULL DEFAULT 'default', - MODIFY COLUMN - user_name VARCHAR(256) NOT NULL, - MODIFY COLUMN - user_group VARCHAR(256) NOT NULL, - MODIFY COLUMN - job_command VARCHAR(256) NOT NULL; - -ALTER TABLE mxq_job - MODIFY COLUMN - job_workdir VARCHAR(4096) NOT NULL, - MODIFY COLUMN - job_argc INT4 UNSIGNED NOT NULL, - MODIFY COLUMN - job_argv VARCHAR(32768) NOT NULL, - MODIFY COLUMN - job_stdout VARCHAR(4096) NOT NULL DEFAULT '/dev/null', - MODIFY COLUMN - job_stderr VARCHAR(4096) NOT NULL DEFAULT '/dev/null', - MODIFY COLUMN - server_id VARCHAR(64) NOT NULL DEFAULT "", - MODIFY COLUMN - host_id VARCHAR(128) NOT NULL DEFAULT "", - MODIFY COLUMN - host_cpu_set VARCHAR(4096) NOT NULL DEFAULT "", - DROP INDEX - server_id, - ADD INDEX - server_id (server_id(64)); - -ALTER TABLE mxq_job - ADD INDEX - host_id (host_id(128)); - -DROP TABLE IF EXISTS mxq_server; - -ALTER TABLE mxq_job - ADD COLUMN - daemon_id INT4 UNSIGNED NOT NULL DEFAULT 0 - AFTER - host_id, - ADD INDEX - daemon_id (daemon_id); diff --git a/mysql/alter_tables_0.22.2.sql b/mysql/alter_tables_0.22.2.sql deleted file mode 100644 index 309ed9e7..00000000 --- a/mysql/alter_tables_0.22.2.sql +++ /dev/null @@ -1,5 +0,0 @@ -ALTER TABLE mxq_group - ADD COLUMN - group_sum_starttime INT8 UNSIGNED NOT NULL DEFAULT 0 - AFTER group_slots_running; - diff --git a/mysql/alter_tables_0.24.0-post.sql b/mysql/alter_tables_0.24.0-post.sql deleted file mode 100644 index dadfc7d7..00000000 --- a/mysql/alter_tables_0.24.0-post.sql +++ /dev/null @@ -1,2 +0,0 @@ -ALTER TABLE mxq_daemon - DROP COLUMN daemon_time; diff --git a/mysql/alter_tables_0.24.0-pre.sql b/mysql/alter_tables_0.24.0-pre.sql deleted file mode 100644 index a0adb28c..00000000 --- a/mysql/alter_tables_0.24.0-pre.sql +++ /dev/null @@ -1,4 +0,0 @@ -ALTER TABLE mxq_daemon - ADD COLUMN - daemon_maxtime INT8 UNSIGNED NOT NULL DEFAULT 0 - AFTER daemon_memory; From a9bead05e2af47befca24cefb8925967add91145 Mon Sep 17 00:00:00 2001 From: Donald Buczek Date: Mon, 29 Mar 2021 12:51:29 +0200 Subject: [PATCH 2/6] sql: Remove obsolete fixup scripts --- mysql/drop_tables.sql | 3 - mysql/fix_cancelled_job_dates.sql | 8 -- mysql/fix_host_id.sql | 7 -- mysql/fix_stats_runwaitidle_sec.sql | 173 ---------------------------- mysql/fix_sum_starttime.sql | 41 ------- 5 files changed, 232 deletions(-) delete mode 100644 mysql/drop_tables.sql delete mode 100644 mysql/fix_cancelled_job_dates.sql delete mode 100644 mysql/fix_host_id.sql delete mode 100644 mysql/fix_stats_runwaitidle_sec.sql delete mode 100644 mysql/fix_sum_starttime.sql diff --git a/mysql/drop_tables.sql b/mysql/drop_tables.sql deleted file mode 100644 index b70e280f..00000000 --- a/mysql/drop_tables.sql +++ /dev/null @@ -1,3 +0,0 @@ -DROP TABLE IF EXISTS mxq_group; -DROP TABLE IF EXISTS mxq_job; -DROP TABLE IF EXISTS mxq_daemon; diff --git a/mysql/fix_cancelled_job_dates.sql b/mysql/fix_cancelled_job_dates.sql deleted file mode 100644 index 4a935764..00000000 --- a/mysql/fix_cancelled_job_dates.sql +++ /dev/null @@ -1,8 +0,0 @@ -UPDATE mxq_job - JOIN mxq_group - ON mxq_job.group_id = mxq_group.group_id - SET mxq_job.date_start = mxq_group.group_date_end, - mxq_job.date_end = mxq_group.group_date_end - WHERE job_status = 990 - AND (date_start = 0 OR date_end = 0); - diff --git a/mysql/fix_host_id.sql b/mysql/fix_host_id.sql deleted file mode 100644 index c3c6fa35..00000000 --- a/mysql/fix_host_id.sql +++ /dev/null @@ -1,7 +0,0 @@ -UPDATE mxq_job - SET - host_id = CONCAT(host_hostname, '::', server_id) - WHERE - host_id = "" - AND server_id != "" - AND host_hostname != ""; diff --git a/mysql/fix_stats_runwaitidle_sec.sql b/mysql/fix_stats_runwaitidle_sec.sql deleted file mode 100644 index ea1c7ce7..00000000 --- a/mysql/fix_stats_runwaitidle_sec.sql +++ /dev/null @@ -1,173 +0,0 @@ - --- fix stats_wait_sec, stats_run_sec, stats_idle_sec in mxq_group --- for all finished groups (group_date_end > 0) - --- do not touch active jobs - -DROP TEMPORARY TABLE IF EXISTS - mytemp; - -SET - @sinq=0, - @srun=0, - @gid=0, - @dt=0, - @ot=0, - @gidchange=0, - @last="none" - ; - -CREATE TEMPORARY TABLE IF NOT EXISTS - mytemp -AS -( - SELECT - group_id, - MAX(IF(last = "run", dtime, 0)) AS run, - MAX(IF(last = "wait", dtime, 0)) AS wait, - MAX(IF(last = "idle", dtime, 0)) AS idle, - MAX(date_end) AS date_end - FROM - ( - SELECT - group_id, - last, - SUM(dt) AS dtime, - MAX(date_end) AS date_end - FROM - ( - SELECT - @gidchange := @gid != group_id AS gidchange, - @gid := group_id AS group_id, - time, - sdinq, - sdrun, - @sinq := IF(@gidchange = 1, - sdinq, - @sinq + sdinq - ) AS sinq, - @srun := IF(@gidchange = 1, - sdrun, - @srun + sdrun - ) AS srun, - IF(@gidchange, "new", @last) as last, - @last := IF(@srun > 0, - "run", - IF(@sinq > 0, - "wait", - "idle" - ) - ) AS phase, - @dt := IF(@gidchange, - 0, - time - @ot - ) AS dt, - @ot := time, - date_end - FROM - ( - SELECT - group_id, - time, - SUM(dinq) as sdinq, - SUM(drun) as sdrun, - MAX(date_end) AS date_end - FROM - ( - ( - SELECT - mxq_job.group_id AS group_id, - job_status, - unix_timestamp(date_submit) AS time, - 1 AS event, - 1 AS dinq, - 0 AS drun, - date_end - FROM - mxq_job - ) - UNION ALL - ( - SELECT - mxq_job.group_id AS group_id, - job_status, - unix_timestamp(date_start), - 2, - -1, - 1, - date_end - FROM - mxq_job - WHERE - date_start > 0 - ) - UNION ALL - ( - SELECT - mxq_job.group_id AS group_id, - job_status, - unix_timestamp(date_end), - 3, - 0, - -1, - date_end - FROM - mxq_job - WHERE - date_end > 0 - ) - ORDER BY - group_id, - time, - event - ) AS S1 - GROUP BY - group_id, - time - ORDER BY - group_id, - time - ) AS S2 - ) AS S3 - WHERE - last != "new" - GROUP BY - group_id, - last - ORDER BY - group_id, - last - ) AS S4 - GROUP BY - group_id -) -; - -SET - @sinq=0, - @srun=0, - @gid=0, - @dt=0, - @ot=0, - @gidchange=0, - @last="none" - ; - -UPDATE - mxq_group AS g - LEFT JOIN - mytemp AS t - ON - g.group_id = t.group_id - SET - g.stats_wait_sec = wait, - g.stats_run_sec = run, - g.stats_idle_sec = idle - WHERE - t.group_id - AND - g.group_date_end -; - -DROP TEMPORARY TABLE IF EXISTS - mytemp; diff --git a/mysql/fix_sum_starttime.sql b/mysql/fix_sum_starttime.sql deleted file mode 100644 index 2f49099a..00000000 --- a/mysql/fix_sum_starttime.sql +++ /dev/null @@ -1,41 +0,0 @@ - -LOCK TABLES mxq_group WRITE, mxq_job WRITE; - -UPDATE - mxq_group -SET - group_slots_running = 0, - group_sum_starttime = 0; - -UPDATE - mxq_group -JOIN ( - SELECT - group_id, - SUM(S) AS rs, - SUM(SS) AS ss - FROM ( - SELECT - group_id, - host_slots, - COUNT(job_id) AS N, - SUM(host_slots) AS S, - SUM(UNIX_TIMESTAMP(date_start)) AS SS - FROM - mxq_job - WHERE - job_status IN (150, 200) - GROUP BY - group_id, - host_slots - ) AS ANONYM - GROUP BY - group_id - ) AS J - ON - mxq_group.group_id = J.group_id - SET - group_slots_running = rs, - group_sum_starttime = ss; - -UNLOCK TABLES; From cba024c97b5630cf2bcfb8f6685f3df9a12633d2 Mon Sep 17 00:00:00 2001 From: Donald Buczek Date: Mon, 29 Mar 2021 12:53:59 +0200 Subject: [PATCH 3/6] sql: Add column tags to mxq_daemon mxq_daemon.prerequisites should be renamed to tags. Add the new name as a new column. The old column prerequisites will be reused for another purpose. Make both columns into VARCHAR(1000) for consistency, where 1000 is an arbitary big enough number. --- mysql/create_tables.sql | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mysql/create_tables.sql b/mysql/create_tables.sql index 6fd0db8d..0ab0f947 100644 --- a/mysql/create_tables.sql +++ b/mysql/create_tables.sql @@ -167,7 +167,8 @@ CREATE TABLE IF NOT EXISTS mxq_daemon ( daemon_stop TIMESTAMP NOT NULL DEFAULT 0, daemon_flags INT4 NOT NULL DEFAULT 0, - prerequisites VARCHAR(1024) NOT NULL DEFAULT '', + prerequisites VARCHAR(1000) NOT NULL DEFAULT '', + tags VARCHAR(1000) NOT NULL DEFAULT '', INDEX (daemon_name(64)), INDEX (hostname(64)) From 3b01ae14929ec68560725d0ef438f9af9d36449b Mon Sep 17 00:00:00 2001 From: Donald Buczek Date: Mon, 29 Mar 2021 13:17:31 +0200 Subject: [PATCH 4/6] sql: Add migration for mxq_daemon.tags We want to rename prerequisites to tags. Additionally, we want a new column prerequisites with new data. However, we can't go the direct way (CHANGE COLUMN prerequisites tags...), because the running mxqd processes use the old name. Create a new column with the new name and copy the data over. We don't really care to much, if data is lost because a mxq daemon restarts during the migration phase, because the former prerequisites and now tags column is only published into the database. Additionally, fix length of prerequisites from 1024 to 1000. --- mysql/migrate_010_add_mxq_daemon_tags.sql | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 mysql/migrate_010_add_mxq_daemon_tags.sql diff --git a/mysql/migrate_010_add_mxq_daemon_tags.sql b/mysql/migrate_010_add_mxq_daemon_tags.sql new file mode 100644 index 00000000..0057f9df --- /dev/null +++ b/mysql/migrate_010_add_mxq_daemon_tags.sql @@ -0,0 +1,9 @@ +ALTER TABLE mxq_daemon MODIFY COLUMN prerequisites VARCHAR(1000) NOT NULL DEFAULT ''; + +ALTER TABLE mxq_daemon + ADD COLUMN + tags VARCHAR(1000) NOT NULL DEFAULT '' + AFTER + prerequisites; + +UPDATE mxq_daemon SET tags=prerequisites; From b91cb5d62df2f49de326bf2d7317047e718688da Mon Sep 17 00:00:00 2001 From: Donald Buczek Date: Mon, 29 Mar 2021 14:56:50 +0200 Subject: [PATCH 5/6] sql: Add column tags to mxq_group Add new column tags to mxq_group and change length of prerequisites for consistency, where 1000 is an arbitary large number. --- mysql/create_tables.sql | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mysql/create_tables.sql b/mysql/create_tables.sql index 0ab0f947..ce12f9ab 100644 --- a/mysql/create_tables.sql +++ b/mysql/create_tables.sql @@ -6,7 +6,8 @@ CREATE TABLE IF NOT EXISTS mxq_group ( group_priority INT2 UNSIGNED NOT NULL DEFAULT 127, group_blacklist VARCHAR(1000) NOT NULL DEFAULT '', group_whitelist VARCHAR(1000) NOT NULL DEFAULT '', - prerequisites VARCHAR(40) NOT NULL DEFAULT '', + prerequisites VARCHAR(1000) NOT NULL DEFAULT '', + tags VARCHAR(1000) NOT NULL DEFAULT '', user_uid INT4 UNSIGNED NOT NULL, user_name VARCHAR(256) NOT NULL, From b2decaf647e34870e9bf68c87908fbb92d425bbc Mon Sep 17 00:00:00 2001 From: Donald Buczek Date: Mon, 29 Mar 2021 15:03:21 +0200 Subject: [PATCH 6/6] sql: Add migration for mxq_group.tags Add new column tags to mxq_groups. Additionally, fix length of prerequisites from 1024 to 1000. --- mysql/migrate_010_add_mxq_group_tags.sql | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 mysql/migrate_010_add_mxq_group_tags.sql diff --git a/mysql/migrate_010_add_mxq_group_tags.sql b/mysql/migrate_010_add_mxq_group_tags.sql new file mode 100644 index 00000000..46141af2 --- /dev/null +++ b/mysql/migrate_010_add_mxq_group_tags.sql @@ -0,0 +1,6 @@ +ALTER TABLE mxq_group MODIFY COLUMN prerequisites VARCHAR(1000) NOT NULL DEFAULT ''; +ALTER TABLE mxq_group + ADD COLUMN + tags VARCHAR(1000) NOT NULL DEFAULT '' + AFTER + prerequisites;