Merge branch 'group_stats_sec'

* group_stats_sec: mysql: Optimzie update of stats_{wait,run,idle}_sec mysql: Fix SQL formatting mxqdump: Dump {wait,run,idle}_sec in group view mxq_group: Query stats_{wait,run,idle}_sec mxqdump: Reformat code to print jobs and groups
mariux · Aug 18, 2015 · 42e731c · 42e731c
2 parents 9d7d5f7 + 3d7ab31
commit 42e731c
Show file tree

Hide file tree

Showing 4 changed files with 146 additions and 87 deletions.
diff --git a/mxq_group.c b/mxq_group.c
@@ -12,7 +12,7 @@
 #include "mx_util.h"
 #include "mx_mysql.h"
 
-#define GROUP_FIELDS_CNT 26
+#define GROUP_FIELDS_CNT 29
 #define GROUP_FIELDS \
             " group_id," \
             " group_name," \
@@ -39,7 +39,11 @@
             " stats_max_maxrss," \
             " stats_max_utime_sec," \
             " stats_max_stime_sec," \
-            " stats_max_real_sec"
+            " stats_max_real_sec," \
+            " stats_wait_sec," \
+            " stats_run_sec," \
+            " stats_idle_sec"
+
 
 static int bind_result_group_fields(struct mx_mysql_bind *result, struct mxq_group *g)
 {
@@ -83,6 +87,10 @@ static int bind_result_group_fields(struct mx_mysql_bind *result, struct mxq_gro
     res += mx_mysql_bind_var(result, idx++, int64, &(g->stats_max_stime.tv_sec));
     res += mx_mysql_bind_var(result, idx++, int64, &(g->stats_max_real.tv_sec));
 
+    res += mx_mysql_bind_var(result, idx++, uint64, &(g->stats_wait_sec));
+    res += mx_mysql_bind_var(result, idx++, uint64, &(g->stats_run_sec));
+    res += mx_mysql_bind_var(result, idx++, uint64, &(g->stats_idle_sec));
+
     return res;
 }
 

diff --git a/mxq_group.h b/mxq_group.h
@@ -48,6 +48,10 @@ struct mxq_group {
    struct timeval stats_max_utime;
    struct timeval stats_max_stime;
    struct timeval stats_max_real;
+
+   uint64_t  stats_wait_sec;
+   uint64_t  stats_run_sec;
+   uint64_t  stats_idle_sec;
 };
 
 #define MXQ_GROUP_STATUS_OK            0

diff --git a/mxqdump.c b/mxqdump.c
@@ -107,20 +107,57 @@ static void print_usage(void)
 
 static int print_group(struct mxq_group *g)
 {
-    return printf("user=%s uid=%u group_id=%lu pri=%d jobs_total=%lu run_jobs=%lu run_slots=%lu failed=%lu"
-                    " finished=%lu cancelled=%lu unknown=%lu inq=%lu"
-                    " job_threads=%u job_memory=%lukiB job_time=%us"
-                    " memory_load=%lu%% time_load=%lu%%"
-                    " max_utime=%lu max_real=%lu max_memory=%lukiB job_command=%s group_name=%s\n",
-        g->user_name, g->user_uid, g->group_id, g->group_priority, g->group_jobs,
-        g->group_jobs_running, g->group_slots_running, g->group_jobs_failed,
-        g->group_jobs_finished, g->group_jobs_cancelled, g->group_jobs_unknown,
+    return printf("user=%s"
+        " uid=%u"
+        " group_id=%lu"
+        " pri=%d"
+        " jobs_total=%lu"
+        " run_jobs=%lu"
+        " run_slots=%lu"
+        " failed=%lu"
+        " finished=%lu"
+        " cancelled=%lu"
+        " unknown=%lu"
+        " inq=%lu"
+        " job_threads=%u"
+        " job_memory=%lukiB"
+        " job_time=%us"
+        " memory_load=%lu%%"
+        " time_load=%lu%%"
+        " max_utime=%lu"
+        " max_real=%lu"
+        " max_memory=%lukiB"
+        " wait_sec=%lu"
+        " run_sec=%lu"
+        " idle_sec=%lu"
+        " job_command=%s"
+        " group_name=%s"
+        "\n",
+        g->user_name,
+        g->user_uid,
+        g->group_id,
+        g->group_priority,
+        g->group_jobs,
+        g->group_jobs_running,
+        g->group_slots_running,
+        g->group_jobs_failed,
+        g->group_jobs_finished,
+        g->group_jobs_cancelled,
+        g->group_jobs_unknown,
         g->group_jobs_inq,
-        g->job_threads, g->job_memory*1024, g->job_time*60,
+        g->job_threads,
+        g->job_memory*1024,
+        g->job_time*60,
         (100UL*(uint64_t)g->stats_max_maxrss/1024UL/g->job_memory),
         (100UL*(uint64_t)g->stats_max_real.tv_sec/60UL/g->job_time),
-        g->stats_max_utime.tv_sec, g->stats_max_real.tv_sec,
-        g->stats_max_maxrss, g->job_command, g->group_name);
+        g->stats_max_utime.tv_sec,
+        g->stats_max_real.tv_sec,
+        g->stats_max_maxrss,
+        g->stats_wait_sec,
+        g->stats_run_sec,
+        g->stats_idle_sec,
+        g->job_command,
+        g->group_name);
 }
 
 static char *restart_to_string(uint64_t flags)
@@ -169,20 +206,49 @@ static int print_job(struct mxq_group *g, struct mxq_job *j)
         run_sec = (j->date_end - j->date_start);
     }
 
-    return printf("job=%s(%u):%lu:%lu host_pid=%u server=%s::%s group_name=%s waittime=%lus runtime=%lus utime=%lus stime=%lus runtime_requested=%us time_load=%lu%% "
-                    "memory_requested=%lukiB max_rss=%lukiB memory_load=%lu%% threads=%d slots=%u status=%s(%d) stats_status=%u restart=%s workdir=%s command=%s"
-                    "\n",
-        g->user_name, g->user_uid, g->group_id, j->job_id,
+    return printf("job=%s(%u):%lu:%lu"
+        " host_pid=%u"
+        " server=%s::%s"
+        " group_name=%s"
+        " waittime=%lus"
+        " runtime=%lus"
+        " utime=%lus"
+        " stime=%lus"
+        " runtime_requested=%us"
+        " time_load=%lu%%"
+        " memory_requested=%lukiB"
+        " max_rss=%lukiB"
+        " memory_load=%lu%%"
+        " threads=%d"
+        " slots=%u"
+        " status=%s(%d)"
+        " stats_status=%u"
+        " restart=%s"
+        " workdir=%s"
+        " command=%s"
+        "\n",
+        g->user_name,
+        g->user_uid,
+        g->group_id,
+        j->job_id,
         j->host_pid,
-        j->host_hostname, j->server_id,
+        j->host_hostname,
+        j->server_id,
         g->group_name,
-        wait_sec, run_sec,
-        j->stats_rusage.ru_utime.tv_sec,j->stats_rusage.ru_stime.tv_sec,g->job_time*60,
+        wait_sec,
+        run_sec,
+        j->stats_rusage.ru_utime.tv_sec,
+        j->stats_rusage.ru_stime.tv_sec,
+        g->job_time*60,
         (100UL*(run_sec)/60UL/g->job_time),
-        g->job_memory*1024, j->stats_rusage.ru_maxrss,
+        g->job_memory*1024,
+        j->stats_rusage.ru_maxrss,
         (100UL*j->stats_rusage.ru_maxrss/1024UL/g->job_memory),
-        g->job_threads, j->host_slots,
-        mxq_job_status_to_name(j->job_status), j->job_status, j->stats_status,
+        g->job_threads,
+        j->host_slots,
+        mxq_job_status_to_name(j->job_status),
+        j->job_status,
+        j->stats_status,
         restart_to_string(j->job_flags),
         j->job_workdir,
         j->job_argv_str);

diff --git a/mysql/fix_stats_runwaitidle_sec.sql b/mysql/fix_stats_runwaitidle_sec.sql
@@ -4,38 +4,34 @@
 
 -- do not touch active jobs
 
-DROP TEMPORARY TABLE IF EXISTS mytemp;
+DROP TEMPORARY TABLE IF EXISTS
+    mytemp;
 
-SET @sinq=0,@srun=0,@gid=0,@dt=0,@ot=0;
+SET
+    @sinq=0,
+    @srun=0,
+    @gid=0,
+    @dt=0,
+    @ot=0
+    ;
 
-CREATE TEMPORARY TABLE IF NOT EXISTS mytemp AS
+CREATE TEMPORARY TABLE IF NOT EXISTS
+    mytemp
+AS
 (
     SELECT
         group_id,
         MAX(IF(last = "run", dtime, 0))  AS run,
         MAX(IF(last = "wait", dtime, 0)) AS wait,
         MAX(IF(last = "idle", dtime, 0)) AS idle,
-        MAX(date_end) AS date_end,
-        group_date_end,
-        stats_run_sec,
-        stats_wait_sec,
-        stats_idle_sec
+        MAX(date_end) AS date_end
     FROM
         (
             SELECT
-                gidchange,
                 group_id,
-                time,
-                sinq,
-                srun,
-                phase,
                 last,
                 SUM(dt) AS dtime,
-                MAX(date_end) AS date_end,
-                group_date_end,
-                stats_wait_sec,
-                stats_run_sec,
-                stats_idle_sec
+                MAX(date_end) AS date_end
             FROM
                 (
                     SELECT
@@ -65,23 +61,15 @@ CREATE TEMPORARY TABLE IF NOT EXISTS mytemp AS
                                     time - @ot
                                 ) AS dt,
                         @ot   := time,
-                        date_end,
-                        group_date_end,
-                        stats_wait_sec,
-                        stats_run_sec,
-                        stats_idle_sec
+                        date_end
                     FROM
                         (
                             SELECT
                                 group_id,
                                 time,
                                 SUM(dinq) as sdinq,
                                 SUM(drun) as sdrun,
-                                MAX(date_end) AS date_end,
-                                group_date_end,
-                                stats_wait_sec,
-                                stats_run_sec,
-                                stats_idle_sec
+                                MAX(date_end) AS date_end
                             FROM
                                 (
                                     (
@@ -92,15 +80,9 @@ CREATE TEMPORARY TABLE IF NOT EXISTS mytemp AS
                                             1 AS event,
                                             1 AS dinq,
                                             0 AS drun,
-                                            date_end,
-                                            group_date_end,
-                                            stats_wait_sec,
-                                            stats_run_sec,
-                                            stats_idle_sec
+                                            date_end
                                         FROM
-                                            mxq_job, mxq_group
-                                        WHERE
-                                            mxq_job.group_id = mxq_group.group_id
+                                            mxq_job
                                     )
                                     UNION ALL
                                     (
@@ -111,16 +93,11 @@ CREATE TEMPORARY TABLE IF NOT EXISTS mytemp AS
                                             2,
                                             -1,
                                             1,
-                                            date_end,
-                                            group_date_end,
-                                            stats_wait_sec,
-                                            stats_run_sec,
-                                            stats_idle_sec
+                                            date_end
                                         FROM
-                                            mxq_job, mxq_group
+                                            mxq_job
                                         WHERE
-                                            mxq_job.group_id = mxq_group.group_id
-                                            AND date_start > 0
+                                            date_start > 0
                                     )
                                     UNION ALL
                                     (
@@ -131,34 +108,33 @@ CREATE TEMPORARY TABLE IF NOT EXISTS mytemp AS
                                             3,
                                             0,
                                             -1,
-                                            date_end,
-                                            group_date_end,
-                                            stats_wait_sec,
-                                            stats_run_sec,
-                                            stats_idle_sec
+                                            date_end
                                         FROM
-                                            mxq_job, mxq_group
+                                            mxq_job
                                         WHERE
-                                            mxq_job.group_id = mxq_group.group_id
-                                            AND date_end > 0
+                                            date_end > 0
                                     )
                                     ORDER BY
                                         group_id,
                                         time,
                                         event
                                 ) AS S1
                             GROUP BY
-                                group_id, time
+                                group_id,
+                                time
                             ORDER BY
-                                group_id, time
+                                group_id,
+                                time
                         ) AS S2
                 ) AS S3
             WHERE
-                    last != "new"
+                last != "new"
             GROUP BY
-                group_id, last
+                group_id,
+                last
             ORDER BY
-                group_id, last
+                group_id,
+                last
         ) AS S4
     GROUP BY
         group_id
@@ -167,16 +143,21 @@ CREATE TEMPORARY TABLE IF NOT EXISTS mytemp AS
 
 SET @sinq=0,@srun=0,@gid=0,@dt=0,@ot=0;
 
-UPDATE mxq_group AS g
-    LEFT JOIN mytemp AS t
-    ON g.group_id = t.group_id
+UPDATE
+        mxq_group AS g
+    LEFT JOIN
+        mytemp AS t
+    ON
+        g.group_id = t.group_id
     SET
         g.stats_wait_sec = wait,
         g.stats_run_sec = run,
         g.stats_idle_sec = idle
-    WHERE t.group_id
-        AND t.group_date_end
-        AND g.group_date_end
+    WHERE
+        t.group_id
+    AND
+        g.group_date_end
 ;
 
-DROP TEMPORARY TABLE IF EXISTS mytemp;
+DROP TEMPORARY TABLE IF EXISTS
+    mytemp;