From 0d52c135bee4d3f9be31bcff52c87e86ca62b64d Mon Sep 17 00:00:00 2001 From: Donald Buczek Date: Fri, 4 Nov 2016 17:48:18 +0100 Subject: [PATCH 01/42] web: use different mysql default file for r/o access The cgi script currently only needs read-only access to the database. With this change, the cgi script uses another mysql default file which is assuemed to contain read-only credentials and can be provided public readable. This enables the webserver to run without special privileges. --- Makefile | 2 ++ web/pages/mxq/mxq.in | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 7e722fb7..efe1b9e3 100644 --- a/Makefile +++ b/Makefile @@ -66,6 +66,7 @@ CGIDIR = ${LIBEXECDIR}/mxq/cgi ######################################################################## MXQ_MYSQL_DEFAULT_FILE = ${SYSCONFDIR}/mxq/mysql.cnf +MXQ_MYSQL_RO_DEFAULT_FILE = ${SYSCONFDIR}/mxq/mysql_ro.cnf MXQ_MYSQL_DEFAULT_GROUP_CLIENT = mxqclient MXQ_MYSQL_DEFAULT_GROUP_SERVER = mxqd MXQ_MYSQL_DEFAULT_GROUP_DEVELOPMENT = mxqdevel @@ -158,6 +159,7 @@ sed-rules = -e 's,@PREFIX@,${PREFIX},g' \ -e 's,@DATADIR@,${DATADIR},g' \ -e 's,@MXQ_VERSION@,${MXQ_VERSION},g' \ -e 's,@MXQ_MYSQL_DEFAULT_FILE@,${MXQ_MYSQL_DEFAULT_FILE},g' \ + -e 's,@MXQ_MYSQL_RO_DEFAULT_FILE@,${MXQ_MYSQL_RO_DEFAULT_FILE},g' \ -e 's,@CGIDIR@,${CGIDIR},g' \ -e 's,@HTTP_USER@,${HTTP_USER},g' \ -e 's,@HTTP_GROUP@,${HTTP_GROUP},g' \ diff --git a/web/pages/mxq/mxq.in b/web/pages/mxq/mxq.in index d35021f8..84168953 100755 --- a/web/pages/mxq/mxq.in +++ b/web/pages/mxq/mxq.in @@ -75,8 +75,8 @@ sub my_url { sub db_init { - open my $test,"<@MXQ_MYSQL_DEFAULT_FILE@" or die "@MXQ_MYSQL_DEFAULT_FILE@: $!\n"; - $dbh= DBI->connect('DBI:mysql:mysql_read_default_file=@MXQ_MYSQL_DEFAULT_FILE@',undef,undef, { RaiseError => 1, AutoCommit => 1 }); + open my $test,"<@MXQ_MYSQL_RO_DEFAULT_FILE@" or die "@MXQ_MYSQL_RO_DEFAULT_FILE@: $!\n"; + $dbh= DBI->connect('DBI:mysql:mysql_read_default_file=@MXQ_MYSQL_RO_DEFAULT_FILE@',undef,undef, { RaiseError => 1, AutoCommit => 1 }); # mxq_group.h register_group_status( OK => 0, From 7ba8999b3412b0df47150c5b3e1c7a14b6480a1a Mon Sep 17 00:00:00 2001 From: Donald Buczek Date: Sat, 5 Nov 2016 15:53:39 +0100 Subject: [PATCH 02/42] web: removed unused jquery links --- web/pages/mxq/mxq.in | 4 ---- 1 file changed, 4 deletions(-) diff --git a/web/pages/mxq/mxq.in b/web/pages/mxq/mxq.in index 84168953..283b1780 100755 --- a/web/pages/mxq/mxq.in +++ b/web/pages/mxq/mxq.in @@ -33,10 +33,6 @@ EOF sub HEAD { return <<'EOF'.STYLE().<<'EOF'; - - - - -EOF - -EOF + return "\n".STYLE()."\n"; } sub my_url { From 87eaa277ed36bdd0be918da08e7b7d0fc3b2c9b1 Mon Sep 17 00:00:00 2001 From: Donald Buczek Date: Sat, 5 Nov 2016 18:53:19 +0100 Subject: [PATCH 16/42] web: add doctype declaration and html element Make html a little bit cleaner. --- web/pages/mxq/mxq.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/pages/mxq/mxq.in b/web/pages/mxq/mxq.in index bad66c6d..a7267176 100755 --- a/web/pages/mxq/mxq.in +++ b/web/pages/mxq/mxq.in @@ -32,7 +32,7 @@ EOF } sub HEAD { - return "\n".STYLE()."\n"; + return "\n\n".STYLE()."\n"; } sub my_url { From 9a201ede78280ccbce10728ae94f8b816fa7f205 Mon Sep 17 00:00:00 2001 From: Peter Marquardt Date: Sat, 5 Nov 2016 19:05:36 +0100 Subject: [PATCH 17/42] web: right-align pid in server table --- web/pages/mxq/mxq.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/pages/mxq/mxq.in b/web/pages/mxq/mxq.in index a7267176..189cf790 100755 --- a/web/pages/mxq/mxq.in +++ b/web/pages/mxq/mxq.in @@ -719,7 +719,7 @@ sub server() { $q->td($mxq_version), # $q->td($boot_id), # $q->td($pid_starttime), - $q->td($daemon_pid), + $q->td({class=>'number'},$daemon_pid), $q->td({class=>'number'},$daemon_slots), $q->td({class=>'number'},$daemon_memory), $q->td({class=>'number'},$daemon_maxtime), From c4fa2d3b009bf1283bcb136b5acacf28de0954b4 Mon Sep 17 00:00:00 2001 From: Donald Buczek Date: Sat, 5 Nov 2016 19:22:10 +0100 Subject: [PATCH 18/42] web: improve server table shorten header width and use himan readble units. Leave maxtime empty if server has no limit Shorten header, use human readble units --- web/pages/mxq/mxq.in | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/web/pages/mxq/mxq.in b/web/pages/mxq/mxq.in index 189cf790..a42a73ef 100755 --- a/web/pages/mxq/mxq.in +++ b/web/pages/mxq/mxq.in @@ -675,9 +675,9 @@ sub server() { # 'boot_id', # 'pid_starttime', 'pid', - 'slots
available', - 'memory
available', - 'max runtime
(minutes)', + 'slots', + 'memory', + 'time
min', # 'time', 'slotmem_soft
available', # 'slotmem_hard', @@ -721,14 +721,14 @@ sub server() { # $q->td($pid_starttime), $q->td({class=>'number'},$daemon_pid), $q->td({class=>'number'},$daemon_slots), - $q->td({class=>'number'},$daemon_memory), - $q->td({class=>'number'},$daemon_maxtime), - $q->td({class=>'number'},$daemon_memory_limit_slot_soft), + $q->td({class=>'number'},size($daemon_memory*1000000)), + $q->td({class=>'number'},$daemon_maxtime ? $daemon_maxtime : ''), + $q->td({class=>'number'},size($daemon_memory_limit_slot_soft*1000000)), # $q->td({class=>'number'},$daemon_memory_limit_slot_hard), $q->td({class=>'number'},$daemon_jobs_running), $q->td({class=>'number'},$daemon_slots_running), $q->td({class=>'number'},$daemon_threads_running), - $q->td({class=>'number'},$daemon_memory_used), + $q->td({class=>'number'},size($daemon_memory_used*1000000)), # $q->td($mtime), # $q->td($daemon_start), # $q->td($daemon_stop), From 4f8404e122f3cd91b2d77df3820cca3ffda8d106 Mon Sep 17 00:00:00 2001 From: Peter Marquardt Date: Sat, 5 Nov 2016 22:30:17 +0100 Subject: [PATCH 19/42] web: make group details toggleable --- web/pages/mxq/mxq.in | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/web/pages/mxq/mxq.in b/web/pages/mxq/mxq.in index a42a73ef..647e4a15 100755 --- a/web/pages/mxq/mxq.in +++ b/web/pages/mxq/mxq.in @@ -315,6 +315,22 @@ sub group_detail { my $job_command=escapeHTML($o{job_command}); $out.=<<"EOF"; +

Summary

+
+$o{user_name} submitted
+
+        $job_command
+
+in group "$group_name", the group status is $group_status_text.
+
+
+ +

[show group details ...]

+ + + EOF return $out; From 61122b3f6e3ad64c6db71b27b3966967dc0e1d49 Mon Sep 17 00:00:00 2001 From: Peter Marquardt Date: Sat, 5 Nov 2016 22:55:31 +0100 Subject: [PATCH 20/42] web: add date_end and runtime to groups job table --- web/pages/mxq/mxq.in | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/web/pages/mxq/mxq.in b/web/pages/mxq/mxq.in index 647e4a15..e25c08f6 100755 --- a/web/pages/mxq/mxq.in +++ b/web/pages/mxq/mxq.in @@ -264,25 +264,29 @@ sub job_table_of_group { my $out; - my @cols=qw(job_id job_status date_submit date_start host_hostname host_pid); + my @qcols=qw(job_id job_status date_submit date_start date_end host_hostname host_pid); + push @qcols,'timestampdiff(SECOND,date_start,date_end) as runtime'; + my @scols=qw(job_id job_status date_submit date_start date_end host_hostname host_pid runtime); - my $sth=$dbh->prepare('SELECT '.join(',',@cols).' FROM mxq_job,mxq_group WHERE mxq_job.group_id=mxq_group.group_id AND mxq_job.group_id=? ORDER BY job_id DESC'); + my $sth=$dbh->prepare('SELECT '.join(',',@qcols).' FROM mxq_job,mxq_group WHERE mxq_job.group_id=mxq_group.group_id AND mxq_job.group_id=? ORDER BY job_id DESC'); $sth->execute($group_id); $out.=''; - $out.=$q->Tr($q->th(\@cols)); + $out.=$q->Tr($q->th(\@scols)); while (my $row=$sth->fetchrow_arrayref()) { - my ($job_id,$job_status,$date_submit,$date_start,$host_hostname,$host_pid)=@$row; + my ($job_id,$job_status,$date_submit,$date_start,$date_end,$host_hostname,$host_pid,$runtime)=@$row; $out.=$q->Tr( $q->td({class=>'number'},$q->a({href=>selfurl("/job/$job_id")},$job_id)), $q->td(job_status($job_status)), $q->td($date_submit), $q->td($date_start), + $q->td($date_end), $q->td($host_hostname), $q->td({class=>'number'},$host_pid), + $q->td({class=>'number'},days($runtime,2)), ); } $out.='
'; From 57886d15995619367e716f247b1806f09b018072 Mon Sep 17 00:00:00 2001 From: Donald Buczek Date: Sat, 5 Nov 2016 23:16:41 +0100 Subject: [PATCH 21/42] web: remove redundant header We already show that information from the encapsulating context. --- web/pages/mxq/mxq.in | 4 ---- 1 file changed, 4 deletions(-) diff --git a/web/pages/mxq/mxq.in b/web/pages/mxq/mxq.in index e25c08f6..7b1ca9bd 100755 --- a/web/pages/mxq/mxq.in +++ b/web/pages/mxq/mxq.in @@ -311,8 +311,6 @@ sub group_detail { $sth->execute($group_id); my %o=%{$sth->fetchrow_hashref('NAME_lc')}; - $out.=h2("Group Detail group_id $o{group_id}"); - my $group_status_text=group_status($o{'group_status'}); my $group_name=escapeHTML($o{group_name}); @@ -431,8 +429,6 @@ sub job { defined $_ or $_='<null>' for values %o; - $out.=h2("Job Details $o{job_id}"); - $out.=<<"EOF"; 
 job_status       : $job_status_text

From c561ff2e034043191eaeedff4c99edfbb3e360de Mon Sep 17 00:00:00 2001
From: Peter Marquardt 
Date: Sat, 5 Nov 2016 23:26:11 +0100
Subject: [PATCH 22/42] web: allow job list to be filtered by status

---
 web/pages/mxq/mxq.in | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/web/pages/mxq/mxq.in b/web/pages/mxq/mxq.in
index 7b1ca9bd..d7792078 100755
--- a/web/pages/mxq/mxq.in
+++ b/web/pages/mxq/mxq.in
@@ -260,7 +260,7 @@ sub job_table_running {
 
 sub job_table_of_group {
 
-	my ($group_id)=@_;
+	my ($group_id,$job_status_filter)=@_;
 
 	my $out;
 
@@ -268,7 +268,9 @@ sub job_table_of_group {
 	push @qcols,'timestampdiff(SECOND,date_start,date_end) as runtime';
 	my @scols=qw(job_id job_status date_submit date_start date_end host_hostname host_pid runtime);
 
-	my $sth=$dbh->prepare('SELECT '.join(',',@qcols).' FROM mxq_job,mxq_group WHERE mxq_job.group_id=mxq_group.group_id AND mxq_job.group_id=? ORDER BY job_id DESC');
+	my $jobfilter = defined($job_status_filter) ? "AND mxq_job.job_status=$job_status_filter" : '';
+
+	my $sth=$dbh->prepare('SELECT '.join(',',@qcols).' FROM mxq_job,mxq_group WHERE mxq_job.group_id=mxq_group.group_id AND mxq_job.group_id=? '.$jobfilter.' ORDER BY job_id DESC');
 	$sth->execute($group_id);
 
 	$out.='';
@@ -393,12 +395,13 @@ EOF
 }
 
 sub group {
-	my ($group_id)=@_;
+	my ($group_id,$job_status_filter)=@_;
 
 	my $out=h1('MXQ Group '.$group_id);
 
 	$out.=group_detail($group_id);
-	$out.=h2('Jobs of this group').job_table_of_group($group_id);
+	my $what = defined($job_status_filter) ? job_status($job_status_filter) . ' jobs' : 'Jobs';
+	$out.=h2($what . ' of this group ').job_status_filter_buttons($group_id).job_table_of_group($group_id,$job_status_filter);
 
 	return $out;
 }
@@ -808,6 +811,8 @@ if ($path_info eq '') {
 	print header().HEAD().top_menu().active_jobs();
 } elsif ($path_info =~ /\/group\/(\d+)$/) {
 	print header().HEAD().top_menu().group($1);
+} elsif ($path_info =~ /\/group\/(\d+)\/(\d+)$/) {
+	print header().HEAD().top_menu().group($1,$2);
 } elsif ($path_info =~ /\/job\/(\d+)$/) {
 	print header().HEAD().top_menu().job($1);
 } else {

From 25d1fc58dbb25615c9ad090ccdab72fce6785b24 Mon Sep 17 00:00:00 2001
From: Donald Buczek 
Date: Sun, 6 Nov 2016 11:47:25 +0100
Subject: [PATCH 23/42] web: add status selector "ALL"

---
 web/pages/mxq/mxq.in | 1 +
 1 file changed, 1 insertion(+)

diff --git a/web/pages/mxq/mxq.in b/web/pages/mxq/mxq.in
index d7792078..6e1ee12a 100755
--- a/web/pages/mxq/mxq.in
+++ b/web/pages/mxq/mxq.in
@@ -297,6 +297,7 @@ sub job_table_of_group {
 sub job_status_filter_buttons {
 	my $group_id = shift;
 	my @r=();
+	push @r,$q->a({href=>selfurl("/group/$group_id")},'ALL');
 	for my $js ( 0, 200, 1000, 750, 400 ) {
 		push @r,$q->a({href=>selfurl("/group/$group_id/$js")},job_status($js));
 	}

From 524da86b066597357cba4a62b03e5c27b38bba46 Mon Sep 17 00:00:00 2001
From: Donald Buczek 
Date: Sun, 6 Nov 2016 11:47:55 +0100
Subject: [PATCH 24/42] web: put status selector into paragraph

We want more space between the selector and the table
---
 web/pages/mxq/mxq.in | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/web/pages/mxq/mxq.in b/web/pages/mxq/mxq.in
index 6e1ee12a..d54e61af 100755
--- a/web/pages/mxq/mxq.in
+++ b/web/pages/mxq/mxq.in
@@ -301,7 +301,7 @@ sub job_status_filter_buttons {
 	for my $js ( 0, 200, 1000, 750, 400 ) {
 		push @r,$q->a({href=>selfurl("/group/$group_id/$js")},job_status($js));
 	}
-	return join(' | ',@r);
+	return '
'.join(' | ',@r).'
';
 }
 
 sub group_detail {

From 2e3fbe0264316b6024c6133f1169e9f567c326ff Mon Sep 17 00:00:00 2001
From: Peter Marquardt 
Date: Sat, 5 Nov 2016 23:43:32 +0100
Subject: [PATCH 25/42] web: visible separation between batches of submits

---
 web/pages/mxq/mxq.in | 15 ++++++++++++++-
 1 file changed, 14 insertions(+), 1 deletion(-)

diff --git a/web/pages/mxq/mxq.in b/web/pages/mxq/mxq.in
index d54e61af..9ca8f4e1 100755
--- a/web/pages/mxq/mxq.in
+++ b/web/pages/mxq/mxq.in
@@ -27,6 +27,9 @@ table.server, table.server th,table.server td { border: 1px solid black;	border-
 
 td.number { text-align: right; }
 
+tr.bg0 { background: #F0F0F0; }
+tr.bg1 { background: #E0E0E0; }
+
 
 EOF
 }
@@ -266,6 +269,7 @@ sub job_table_of_group {
 
 	my @qcols=qw(job_id job_status date_submit date_start date_end host_hostname host_pid);
 	push @qcols,'timestampdiff(SECOND,date_start,date_end) as runtime';
+	push @qcols,'UNIX_TIMESTAMP(date_submit) as ds';
 	my @scols=qw(job_id job_status date_submit date_start date_end host_hostname host_pid runtime);
 
 	my $jobfilter = defined($job_status_filter) ? "AND mxq_job.job_status=$job_status_filter" : '';
@@ -277,10 +281,19 @@ sub job_table_of_group {
 
 	$out.=$q->Tr($q->th(\@scols));
 
+	my $lds;
+	my $col=1;
+
 	while (my $row=$sth->fetchrow_arrayref()) {
-		my ($job_id,$job_status,$date_submit,$date_start,$date_end,$host_hostname,$host_pid,$runtime)=@$row;
+		my ($job_id,$job_status,$date_submit,$date_start,$date_end,$host_hostname,$host_pid,$runtime,$ds)=@$row;
+		if (defined $lds && abs($lds-$ds)>10) {
+			$col = 1-$col;
+			$out.=$q->Tr($q->td(' '));
+		}
+		$lds=$ds;
 
 		$out.=$q->Tr(
+			{class=>'bg'.$col},
 			$q->td({class=>'number'},$q->a({href=>selfurl("/job/$job_id")},$job_id)),
 			$q->td(job_status($job_status)),
 			$q->td($date_submit),

From ff47463fc4fbdddda1736061161d6e67016048d3 Mon Sep 17 00:00:00 2001
From: Peter Arndt 
Date: Sun, 6 Nov 2016 12:14:26 +0100
Subject: [PATCH 26/42] web: use courier font in tables

---
 web/pages/mxq/mxq.in | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/web/pages/mxq/mxq.in b/web/pages/mxq/mxq.in
index 9ca8f4e1..bf08af77 100755
--- a/web/pages/mxq/mxq.in
+++ b/web/pages/mxq/mxq.in
@@ -21,9 +21,9 @@ body { background:#ddd;  }
 
 div.dumper {border: 2px solid black; border-radius: 5px;  margin: 10px 0; padding: 0 10px;background: #faa; }
 
-table.jobs, table.jobs th,table.jobs td { border: 1px solid black;	border-collapse: collapse;}
-table.groups, table.groups th,table.groups td { border: 1px solid black;	border-collapse: collapse;}
-table.server, table.server th,table.server td { border: 1px solid black;	border-collapse: collapse;}
+table.jobs,   table.jobs   th,table.jobs   td {font-family:courier; border: 1px solid black; border-collapse: collapse;}
+table.groups, table.groups th,table.groups td {font-family:courier; border: 1px solid black; border-collapse: collapse;}
+table.server, table.server th,table.server td {font-family:courier; border: 1px solid black; border-collapse: collapse;}
 
 td.number { text-align: right; }
 

From 87b739c97ea73dee4277689bce33bcf3680f3f3b Mon Sep 17 00:00:00 2001
From: Peter Arndt 
Date: Sun, 6 Nov 2016 12:25:48 +0100
Subject: [PATCH 27/42] web: add function colorize

---
 web/pages/mxq/mxq.in | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/web/pages/mxq/mxq.in b/web/pages/mxq/mxq.in
index bf08af77..ccb9b649 100755
--- a/web/pages/mxq/mxq.in
+++ b/web/pages/mxq/mxq.in
@@ -197,6 +197,12 @@ sub split_cmd {
 	return $in;
 }
 
+sub colorize {
+	my ($color,$html) = @_;
+	return $html if ($html eq "0"); # zeros remain black
+	return "$html";
+}
+
 sub DataDumper {
 	use Data::Dumper;
 	$Data::Dumper::Indent=0;

From 3ac90e6e37fc3828294ea71fa9f69ef97ab30ca9 Mon Sep 17 00:00:00 2001
From: Peter Arndt 
Date: Sun, 6 Nov 2016 12:31:44 +0100
Subject: [PATCH 28/42] web: add color to group table counts

---
 web/pages/mxq/mxq.in | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/web/pages/mxq/mxq.in b/web/pages/mxq/mxq.in
index ccb9b649..4cdbbc82 100755
--- a/web/pages/mxq/mxq.in
+++ b/web/pages/mxq/mxq.in
@@ -571,12 +571,12 @@ sub group_table {
 			$q->td($group_mtime),
 			$q->td(group_status($group_status)),
 			$q->td({class=>'number'},$group_jobs),
-			$q->td({class=>'number'},$group_jobs_inq),
-			$q->td({class=>'number'},$group_jobs_running),
-			$q->td({class=>'number'},$group_jobs_finished),
-			$q->td({class=>'number'},$group_jobs_failed),
-			$q->td({class=>'number'},$group_jobs_cancelled),
-			$q->td({class=>'number'},$group_jobs_unknown)
+			$q->td({class=>'number'},colorize("blue",$group_jobs_inq)),
+			$q->td({class=>'number'},colorize("orange",$group_jobs_running)),
+			$q->td({class=>'number'},colorize("green",$group_jobs_finished)),
+			$q->td({class=>'number'},colorize("red",$group_jobs_failed)),
+			$q->td({class=>'number'},colorize("red",$group_jobs_cancelled)),
+			$q->td({class=>'number'},colorize("red",$group_jobs_unknown))
 		);
 	}
 	$out.='
';
@@ -643,10 +643,10 @@ sub group_table {
 			$q->td($group_date_end),
 			$q->td(group_status($group_status)),
 			$q->td({class=>'number'},$group_jobs),
-			$q->td({class=>'number'},$group_jobs_finished),
-			$q->td({class=>'number'},$group_jobs_failed),
-			$q->td({class=>'number'},$group_jobs_cancelled),
-			$q->td({class=>'number'},$group_jobs_unknown)
+			$q->td({class=>'number'},colorize("green",$group_jobs_finished)),
+			$q->td({class=>'number'},colorize("red",$group_jobs_failed)),
+			$q->td({class=>'number'},colorize("red",$group_jobs_cancelled)),
+			$q->td({class=>'number'},colorize("red",$group_jobs_unknown))
 		);
 	}
 	$out.='';

From ba4d17be97096b19b99e593ccccba38336f17590 Mon Sep 17 00:00:00 2001
From: Peter Arndt 
Date: Sun, 6 Nov 2016 13:14:07 +0100
Subject: [PATCH 29/42] web: shorten text representation of function
 humanSeconds

---
 web/pages/mxq/mxq.in | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/web/pages/mxq/mxq.in b/web/pages/mxq/mxq.in
index 4cdbbc82..f4729db7 100755
--- a/web/pages/mxq/mxq.in
+++ b/web/pages/mxq/mxq.in
@@ -122,10 +122,10 @@ sub daemon_status           { return get_status(\%DAEMON_STATUS_NAME,@_) }
 sub humanSeconds {
 	my ($seconds)=@_;
 
-	$seconds<60    and return sprintf ('%2d s',$seconds);
-	$seconds<3600  and return sprintf ('%2d m %2d s',$seconds/60,$seconds%60);
-	$seconds<86400 and return sprintf ('%2d h %2d m',$seconds/3600,$seconds%3600/60);
-	return sprintf('%2d d %2d h',$seconds/86400,$seconds%86400/3600);
+	$seconds<60    and return sprintf ('%2ds',$seconds);
+	$seconds<3600  and return sprintf ('%2dm %2ds',$seconds/60,$seconds%60);
+	$seconds<86400 and return sprintf ('%2dh %2dm',$seconds/3600,$seconds%3600/60);
+	return sprintf('%2dd %2dh',$seconds/86400,$seconds%86400/3600);
 }
 
 sub size {

From b6ba1bca190ecc9cf636d64f5d6484f802c1ae1c Mon Sep 17 00:00:00 2001
From: Peter Arndt 
Date: Sun, 6 Nov 2016 13:14:57 +0100
Subject: [PATCH 30/42] web: add function colorize_status

---
 web/pages/mxq/mxq.in | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/web/pages/mxq/mxq.in b/web/pages/mxq/mxq.in
index f4729db7..237a5ea8 100755
--- a/web/pages/mxq/mxq.in
+++ b/web/pages/mxq/mxq.in
@@ -203,6 +203,15 @@ sub colorize {
 	return "$html";
 }
 
+sub colorize_status {
+	my ($status)=@_;
+	return colorize("green", $status) if $status =~ m/FINISHED/;
+	return colorize("orange",$status) if $status =~ m/RUNNING/;
+	return colorize("blue",  $status) if $status =~ m/INQ/;
+	return colorize("red",   $status) if $status =~ m/KILL|FAIL/;
+	return $status;
+}
+
 sub DataDumper {
 	use Data::Dumper;
 	$Data::Dumper::Indent=0;

From 26f8c6b2b67f3e7e484369d457c0983607fffdc4 Mon Sep 17 00:00:00 2001
From: Donald Buczek 
Date: Sun, 6 Nov 2016 17:54:03 +0100
Subject: [PATCH 31/42] web: shorten relative times in groups table

---
 web/pages/mxq/mxq.in | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/web/pages/mxq/mxq.in b/web/pages/mxq/mxq.in
index 237a5ea8..042f498d 100755
--- a/web/pages/mxq/mxq.in
+++ b/web/pages/mxq/mxq.in
@@ -646,8 +646,8 @@ sub group_table {
 			$q->td({class=>'number'},$job_threads),
 			$q->td({class=>'number'},size($job_memory*1000**2)),
 			$q->td({class=>'number'},size($stats_max_sumrss*1000)),
-			$q->td({class=>'number'},days($job_time*60)),
-			$q->td({class=>'number'},days($stats_run_sec+$stats_idle_sec,2)),
+			$q->td({class=>'number'},humanSeconds($job_time*60)),
+			$q->td({class=>'number'},humanSeconds($stats_run_sec+$stats_idle_sec,2)),
 			$q->td($q->a({href=>my_url('groups',{user_name=>$user_name})},$user_name)),
 			$q->td($group_date_end),
 			$q->td(group_status($group_status)),

From 59adcaa574730d86a9fa5802a2ae75401a2a52b8 Mon Sep 17 00:00:00 2001
From: Donald Buczek 
Date: Sun, 6 Nov 2016 18:13:06 +0100
Subject: [PATCH 32/42] web: refactor group select statement

To prepare for further changes.
---
 web/pages/mxq/mxq.in | 20 ++++++++++++++++----
 1 file changed, 16 insertions(+), 4 deletions(-)

diff --git a/web/pages/mxq/mxq.in b/web/pages/mxq/mxq.in
index 042f498d..c830c965 100755
--- a/web/pages/mxq/mxq.in
+++ b/web/pages/mxq/mxq.in
@@ -282,14 +282,26 @@ sub job_table_of_group {
 
 	my $out;
 
-	my @qcols=qw(job_id job_status date_submit date_start date_end host_hostname host_pid);
-	push @qcols,'timestampdiff(SECOND,date_start,date_end) as runtime';
-	push @qcols,'UNIX_TIMESTAMP(date_submit) as ds';
 	my @scols=qw(job_id job_status date_submit date_start date_end host_hostname host_pid runtime);
 
 	my $jobfilter = defined($job_status_filter) ? "AND mxq_job.job_status=$job_status_filter" : '';
 
-	my $sth=$dbh->prepare('SELECT '.join(',',@qcols).' FROM mxq_job,mxq_group WHERE mxq_job.group_id=mxq_group.group_id AND mxq_job.group_id=? '.$jobfilter.' ORDER BY job_id DESC');
+	my $sth=$dbh->prepare(<<"__EOF__");
+	SELECT
+		job_id,
+		job_status,
+		date_submit,
+		date_start,
+		date_end,
+		host_hostname,
+		host_pid,
+		timestampdiff(SECOND,date_start,date_end) as runtime,
+		UNIX_TIMESTAMP(date_submit) as ds
+	FROM mxq_job,mxq_group
+	WHERE mxq_job.group_id=mxq_group.group_id AND mxq_job.group_id=?
+	$jobfilter
+	ORDER BY job_id DESC
+__EOF__
 	$sth->execute($group_id);
 
 	$out.='';

From 4243b8f136000d7524f017c90a252661b923e2a1 Mon Sep 17 00:00:00 2001
From: Peter Arndt 
Date: Sun, 6 Nov 2016 18:25:37 +0100
Subject: [PATCH 33/42] web: show queuetime and runtime of jobs in condensed
 form

---
 web/pages/mxq/mxq.in | 30 +++++++++++++++++++++++-------
 1 file changed, 23 insertions(+), 7 deletions(-)

diff --git a/web/pages/mxq/mxq.in b/web/pages/mxq/mxq.in
index c830c965..632e5fe2 100755
--- a/web/pages/mxq/mxq.in
+++ b/web/pages/mxq/mxq.in
@@ -282,7 +282,7 @@ sub job_table_of_group {
 
 	my $out;
 
-	my @scols=qw(job_id job_status date_submit date_start date_end host_hostname host_pid runtime);
+	my @scols=qw(job_id job_status qtime rtime host_hostname host_pid);
 
 	my $jobfilter = defined($job_status_filter) ? "AND mxq_job.job_status=$job_status_filter" : '';
 
@@ -295,7 +295,10 @@ sub job_table_of_group {
 		date_end,
 		host_hostname,
 		host_pid,
-		timestampdiff(SECOND,date_start,date_end) as runtime,
+		timestampdiff(SECOND,date_submit,date_start),
+		timestampdiff(SECOND,date_submit,now()),
+		timestampdiff(SECOND,date_start,date_end),
+		timestampdiff(SECOND,date_start,now()),
 		UNIX_TIMESTAMP(date_submit) as ds
 	FROM mxq_job,mxq_group
 	WHERE mxq_job.group_id=mxq_group.group_id AND mxq_job.group_id=?
@@ -312,7 +315,22 @@ __EOF__
 	my $col=1;
 
 	while (my $row=$sth->fetchrow_arrayref()) {
-		my ($job_id,$job_status,$date_submit,$date_start,$date_end,$host_hostname,$host_pid,$runtime,$ds)=@$row;
+		my ($job_id,$job_status,$date_submit,$date_start,$date_end,$host_hostname,$host_pid,
+			$qtime,$qtime_now,$rtime,$rtime_now,$ds)=@$row;
+
+		if (defined($qtime)) {
+			$qtime = humanSeconds($qtime);
+		} else {
+			$qtime = colorize("gray",humanSeconds($qtime_now));
+		}
+
+		if (defined($rtime)) {
+			$rtime = humanSeconds($rtime);
+		} else {
+			$rtime = colorize("gray",humanSeconds($rtime_now));
+		}
+		my $date_info = "submit: $date_submit\nstart:  $date_start\nend:   $date_end";
+
 		if (defined $lds && abs($lds-$ds)>10) {
 			$col = 1-$col;
 			$out.=$q->Tr($q->td(' '));
@@ -323,12 +341,10 @@ __EOF__
 			{class=>'bg'.$col},
 			$q->td({class=>'number'},$q->a({href=>selfurl("/job/$job_id")},$job_id)),
 			$q->td(job_status($job_status)),
-			$q->td($date_submit),
-			$q->td($date_start),
-			$q->td($date_end),
+			$q->td("$qtime"),
+			$q->td("$rtime"),
 			$q->td($host_hostname),
 			$q->td({class=>'number'},$host_pid),
-			$q->td({class=>'number'},days($runtime,2)),
 		);
 	}
 	$out.='
';

From a81283d264f813c2b8ef4703a8e30d77d33501e6 Mon Sep 17 00:00:00 2001
From: Peter Arndt 
Date: Sun, 6 Nov 2016 18:28:39 +0100
Subject: [PATCH 34/42] web: colorize status of groups job table

---
 web/pages/mxq/mxq.in | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/web/pages/mxq/mxq.in b/web/pages/mxq/mxq.in
index 632e5fe2..da6ea067 100755
--- a/web/pages/mxq/mxq.in
+++ b/web/pages/mxq/mxq.in
@@ -340,7 +340,7 @@ __EOF__
 		$out.=$q->Tr(
 			{class=>'bg'.$col},
 			$q->td({class=>'number'},$q->a({href=>selfurl("/job/$job_id")},$job_id)),
-			$q->td(job_status($job_status)),
+			$q->td(colorize_status(job_status($job_status))),
 			$q->td("$qtime"),
 			$q->td("$rtime"),
 			$q->td($host_hostname),

From 500eb5f225f8375abe53f4d601c012ee73f3906a Mon Sep 17 00:00:00 2001
From: Donald Buczek 
Date: Sun, 6 Nov 2016 18:53:37 +0100
Subject: [PATCH 35/42] web: leave out group_mtime and date_end in groups page

---
 web/pages/mxq/mxq.in | 10 ++--------
 1 file changed, 2 insertions(+), 8 deletions(-)

diff --git a/web/pages/mxq/mxq.in b/web/pages/mxq/mxq.in
index da6ea067..f7ff044a 100755
--- a/web/pages/mxq/mxq.in
+++ b/web/pages/mxq/mxq.in
@@ -561,7 +561,6 @@ sub group_table {
 		job_memory
 		job_time
 		user_name
-		group_mtime
 		group_status
 		group_jobs
 		group_jobs_inq group_jobs_running
@@ -575,7 +574,6 @@ sub group_table {
 		req.
memory
 		req.
runtime
 		user
name
-		group
mtime
 		group
status
 		jobs
 		inq
@@ -594,7 +592,7 @@ sub group_table {
 	while (my $row=$sth->fetchrow_arrayref()) {
 		my ($group_id,$group_name,$job_threads,
 			$job_memory,$job_time,
-			$user_name,$group_mtime,$group_status,
+			$user_name,$group_status,
 			$group_jobs,$group_jobs_inq,$group_jobs_running,$group_jobs_finished,$group_jobs_failed,$group_jobs_cancelled,$group_jobs_unknown
 		)=@$row;
 
@@ -605,7 +603,6 @@ sub group_table {
 			$q->td({class=>'number'},size($job_memory*1000**2)),
 			$q->td({class=>'number'},days($job_time*60)),
 			$q->td($q->a({href=>my_url('groups',{user_name=>$user_name})},$user_name)),
-			$q->td($group_mtime),
 			$q->td(group_status($group_status)),
 			$q->td({class=>'number'},$group_jobs),
 			$q->td({class=>'number'},colorize("blue",$group_jobs_inq)),
@@ -628,7 +625,6 @@ sub group_table {
 		stats_run_sec
 		stats_idle_sec
 		user_name
-		group_date_end
 		group_status
 		group_jobs
 		group_jobs_finished group_jobs_failed group_jobs_cancelled group_jobs_unknown
@@ -643,7 +639,6 @@ sub group_table {
 		req.
runtime
 		used
runtime
 		user
name
-		date_end
 		group
status
 		jobs
 		fin
@@ -664,7 +659,7 @@ sub group_table {
 			$job_time,
 			$stats_run_sec,
 			$stats_idle_sec,
-			$user_name,$group_date_end,$group_status,
+			$user_name,$group_status,
 			$group_jobs,$group_jobs_finished,$group_jobs_failed,$group_jobs_cancelled,$group_jobs_unknown
 		)=@$row;
 
@@ -677,7 +672,6 @@ sub group_table {
 			$q->td({class=>'number'},humanSeconds($job_time*60)),
 			$q->td({class=>'number'},humanSeconds($stats_run_sec+$stats_idle_sec,2)),
 			$q->td($q->a({href=>my_url('groups',{user_name=>$user_name})},$user_name)),
-			$q->td($group_date_end),
 			$q->td(group_status($group_status)),
 			$q->td({class=>'number'},$group_jobs),
 			$q->td({class=>'number'},colorize("green",$group_jobs_finished)),

From 3c52bd91823ca598b8804b85f07adbcb0a08870a Mon Sep 17 00:00:00 2001
From: "root@afk /dev/pts/9 141.14.28.170" 
Date: Mon, 7 Nov 2016 14:21:39 +0100
Subject: [PATCH 36/42] web: use monospace fonts

Instead of serif courier fonts use monospace. Make tables contents more readable.
---
 web/pages/mxq/mxq.in | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/web/pages/mxq/mxq.in b/web/pages/mxq/mxq.in
index f7ff044a..dbc49ca2 100755
--- a/web/pages/mxq/mxq.in
+++ b/web/pages/mxq/mxq.in
@@ -21,9 +21,9 @@ body { background:#ddd;  }
 
 div.dumper {border: 2px solid black; border-radius: 5px;  margin: 10px 0; padding: 0 10px;background: #faa; }
 
-table.jobs,   table.jobs   th,table.jobs   td {font-family:courier; border: 1px solid black; border-collapse: collapse;}
-table.groups, table.groups th,table.groups td {font-family:courier; border: 1px solid black; border-collapse: collapse;}
-table.server, table.server th,table.server td {font-family:courier; border: 1px solid black; border-collapse: collapse;}
+table.jobs,   table.jobs   th,table.jobs   td {font-family:monospace; border: 1px solid black; border-collapse: collapse;}
+table.groups, table.groups th,table.groups td {font-family:monospace; border: 1px solid black; border-collapse: collapse;}
+table.server, table.server th,table.server td {font-family:monospace; border: 1px solid black; border-collapse: collapse;}
 
 td.number { text-align: right; }
 

From 22555af875649ab90716ce2e3e3df07ae1e8cc89 Mon Sep 17 00:00:00 2001
From: Donald Buczek 
Date: Wed, 9 Nov 2016 09:46:30 +0100
Subject: [PATCH 37/42] web: show job detail first on job detail page

---
 web/pages/mxq/mxq.in | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/web/pages/mxq/mxq.in b/web/pages/mxq/mxq.in
index dbc49ca2..baf5ffaa 100755
--- a/web/pages/mxq/mxq.in
+++ b/web/pages/mxq/mxq.in
@@ -474,8 +474,6 @@ sub job {
 	$sth->execute($job_id);
 	my %o=%{$sth->fetchrow_hashref('NAME_lc')};
 
-	$out.=group_detail($o{group_id});
-
 	my $job_status_text=job_status($o{'job_status'});
 	my $job_umask_text=sprintf('%03O',$o{job_umask});
 	my $link_group_id=a({href=>selfurl("/group/$o{group_id}")},$o{group_id});
@@ -542,6 +540,9 @@ stats_nvcsw      : $o{stats_nvcsw}
 stats_nivcsw     : $o{stats_nivcsw}
EOF + + $out.=group_detail($o{group_id}); + return $out; } From 9cc2802057719bb4cfe36ae4dbe46263a0c5121e Mon Sep 17 00:00:00 2001 From: Donald Buczek Date: Wed, 9 Nov 2016 10:14:06 +0100 Subject: [PATCH 38/42] web: refactor group_detail into two functions --- web/pages/mxq/mxq.in | 132 ++++++++++++++++++++++++------------------- 1 file changed, 73 insertions(+), 59 deletions(-) diff --git a/web/pages/mxq/mxq.in b/web/pages/mxq/mxq.in index baf5ffaa..1c218f56 100755 --- a/web/pages/mxq/mxq.in +++ b/web/pages/mxq/mxq.in @@ -360,7 +360,71 @@ sub job_status_filter_buttons { return '

'.join(' | ',@r).'

'; } -sub group_detail { +sub group_details_raw { + my ($o)=@_; + + my $group_name=escapeHTML($o->{'group_name'}); + my $job_command=escapeHTML($o->{'job_command'}); + my $group_status_text=group_status($o->{'group_status'}); + + return <<"__EOF__" +
+group_name     : $group_name
+group_status   : $group_status_text
+group_flags    : $o->{group_flags}
+group_priority : $o->{group_priority}
+
+user_uid       : $o->{user_uid}
+user_name      : $o->{user_name}
+user_gid       : $o->{user_gid}
+user_group     : $o->{user_group}
+
+job_command    : $job_command
+job_threads    : $o->{job_threads}
+job_memory     : $o->{job_memory} MiB
+job_time       : $o->{job_time} minutes
+
+job_max_per_node      : $o->{job_max_per_node}
+
+group_jobs            : $o->{group_jobs}
+group_jobs_inq        : $o->{group_jobs_inq}
+group_jobs_running    : $o->{group_jobs_running}
+group_jobs_finished   : $o->{group_jobs_finished}
+group_jobs_failed     : $o->{group_jobs_failed}
+group_jobs_cancelled  : $o->{group_jobs_cancelled}
+group_jobs_unknown    : $o->{group_jobs_unknown}
+group_jobs_restarted  : $o->{group_jobs_restarted}
+
+group_slots_running:  : $o->{group_slots_running}
+
+group_mtime           : $o->{group_mtime}
+
+group_date_end        : $o->{group_date_end}
+
+stats_max_sumrss                 : $o->{stats_max_sumrss} kiB
+stats_max_maxrss                 : $o->{stats_max_maxrss}
+stats_max_utime_sec              : $o->{stats_max_utime_sec}
+stats_max_stime_sec              : $o->{stats_max_stime_sec}
+stats_max_real_sec               : $o->{stats_max_real_sec}
+
+stats_total_utime_sec            : $o->{stats_total_utime_sec}
+stats_total_stime_sec            : $o->{stats_total_stime_sec}
+stats_total_real_sec             : $o->{stats_total_real_sec}
+stats_total_wait_sec             : $o->{stats_total_wait_sec}
+
+stats_wait_sec                   : $o->{stats_wait_sec}
+stats_run_sec                    : $o->{stats_run_sec}
+stats_idle_sec                   : $o->{stats_idle_sec}
+
+stats_total_utime_sec_finished   : $o->{stats_total_utime_sec_finished}
+stats_total_stime_sec_finished   : $o->{stats_total_stime_sec_finished}
+stats_total_real_sec_finished    : $o->{stats_total_real_sec_finished}
+stats_total_wait_sec_finished    : $o->{stats_total_wait_sec_finished}
+
+__EOF__ +} + +sub group_details { my ($group_id)=@_; $dbh or db_init(); @@ -375,7 +439,7 @@ sub group_detail { my $group_name=escapeHTML($o{group_name}); my $job_command=escapeHTML($o{job_command}); - $out.=<<"EOF"; + $out.=<<"__EOF__";

Summary

 $o{user_name} submitted
@@ -392,63 +456,13 @@ in group "$group_name", the group status is $group_status_text.
 
[hide group details]

 
 
-EOF
+__EOF__
 
-		return $out;
+	return $out;
 }
 
 sub group {
@@ -456,7 +470,7 @@ sub group {
 
 	my $out=h1('MXQ Group '.$group_id);
 
-	$out.=group_detail($group_id);
+	$out.=group_details($group_id);
 	my $what = defined($job_status_filter) ? job_status($job_status_filter) . ' jobs' : 'Jobs';
 	$out.=h2($what . ' of this group ').job_status_filter_buttons($group_id).job_table_of_group($group_id,$job_status_filter);
 
@@ -541,7 +555,7 @@ stats_nivcsw     : $o{stats_nivcsw}
EOF - $out.=group_detail($o{group_id}); + $out.=group_details($o{group_id}); return $out; } From b4b33c4528866b85ec4f1b1339085485ba863362 Mon Sep 17 00:00:00 2001 From: Donald Buczek Date: Wed, 9 Nov 2016 10:18:21 +0100 Subject: [PATCH 39/42] web: add function group_details_raw_by_group_id --- web/pages/mxq/mxq.in | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/web/pages/mxq/mxq.in b/web/pages/mxq/mxq.in index 1c218f56..93625e48 100755 --- a/web/pages/mxq/mxq.in +++ b/web/pages/mxq/mxq.in @@ -424,6 +424,14 @@ stats_total_wait_sec_finished : $o->{stats_total_wait_sec_finished} __EOF__ } +sub group_details_raw_by_group_id { + my ($group_id)=@_; + my $sth=$dbh->prepare('SELECT * FROM mxq_group WHERE group_id=? LIMIT 1',undef); + $sth->execute($group_id); + my $o=$sth->fetchrow_hashref('NAME_lc'); + return group_details_raw($o); +} + sub group_details { my ($group_id)=@_; $dbh or db_init(); From 2e4302df47915d792338c31433a756eaaf6919ac Mon Sep 17 00:00:00 2001 From: Donald Buczek Date: Wed, 9 Nov 2016 10:35:45 +0100 Subject: [PATCH 40/42] web: show expaneded group info after job details --- web/pages/mxq/mxq.in | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/web/pages/mxq/mxq.in b/web/pages/mxq/mxq.in index 93625e48..de0952ed 100755 --- a/web/pages/mxq/mxq.in +++ b/web/pages/mxq/mxq.in @@ -563,8 +563,8 @@ stats_nivcsw : $o{stats_nivcsw} EOF - $out.=group_details($o{group_id}); - + $out .= "

group details

"; + $out.=group_details_raw_by_group_id($o{group_id}); return $out; } From 6ab5d3b37ae5bafd93c07c380ec6029aafbad3e9 Mon Sep 17 00:00:00 2001 From: Donald Buczek Date: Tue, 3 Jan 2017 15:12:33 +0100 Subject: [PATCH 41/42] web: limit tables to 100000 rows Avoid killing the browser --- web/pages/mxq/mxq.in | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/web/pages/mxq/mxq.in b/web/pages/mxq/mxq.in index de0952ed..6067027c 100755 --- a/web/pages/mxq/mxq.in +++ b/web/pages/mxq/mxq.in @@ -13,6 +13,8 @@ our $dbh; our $hostconfig; our $q; +our $LIMIT=100000; # max rows + sub STYLE { return <<'EOF';