Skip to content

Commit

Permalink
mxqd: Store and use max_sumrss
Browse files Browse the repository at this point in the history 
resolves parts of mariux64#16
  • Loading branch information
@mariux
mariux committed Oct 26, 2015
1 parent 9509e6e commit 9b2a5f4
Show file tree
Hide file tree
Showing 10 changed files with 44 additions and 10 deletions.
4 changes: 3 additions & 1 deletion mxq_group.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
#include "mx_util.h"
#include "mx_mysql.h"

#define GROUP_FIELDS_CNT 30
#define GROUP_FIELDS_CNT 31
#define GROUP_FIELDS \
" group_id," \
" group_name," \
Expand All @@ -37,6 +37,7 @@
" group_jobs_unknown," \
" group_jobs_restarted," \
" group_slots_running," \
" stats_max_sumrss," \
" stats_max_maxrss," \
" stats_max_utime_sec," \
" stats_max_stime_sec," \
Expand Down Expand Up @@ -85,6 +86,7 @@ static int bind_result_group_fields(struct mx_mysql_bind *result, struct mxq_gro

res += mx_mysql_bind_var(result, idx++, uint64, &(g->group_slots_running));

res += mx_mysql_bind_var(result, idx++, uint64, &(g->stats_max_sumrss));
res += mx_mysql_bind_var(result, idx++, uint64, &(g->stats_max_maxrss));
res += mx_mysql_bind_var(result, idx++, int64, &(g->stats_max_utime.tv_sec));
res += mx_mysql_bind_var(result, idx++, int64, &(g->stats_max_stime.tv_sec));
Expand Down
1 change: 1 addition & 0 deletions mxq_group.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@ struct mxq_group {

uint64_t group_slots_running;

uint64_t stats_max_sumrss;
uint64_t stats_max_maxrss;

struct timeval stats_max_utime;
Expand Down
8 changes: 6 additions & 2 deletions mxq_job.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
#include "mxq_group.h"
#include "mxq_job.h"

#define JOB_FIELDS_CNT 35
#define JOB_FIELDS_CNT 36
#define JOB_FIELDS \
" job_id, " \
" job_status, " \
Expand All @@ -38,6 +38,7 @@
" UNIX_TIMESTAMP(date_submit) as date_submit, " \
" UNIX_TIMESTAMP(date_start) as date_start, " \
" UNIX_TIMESTAMP(date_end) as date_end, " \
" stats_max_sumrss, " \
" stats_status, " \
" stats_utime_sec, " \
" stats_utime_usec, " \
Expand Down Expand Up @@ -82,6 +83,7 @@ static int bind_result_job_fields(struct mx_mysql_bind *result, struct mxq_job *
res += mx_mysql_bind_var(result, idx++, int64, &(j->date_submit));
res += mx_mysql_bind_var(result, idx++, int64, &(j->date_start));
res += mx_mysql_bind_var(result, idx++, int64, &(j->date_end));
res += mx_mysql_bind_var(result, idx++, uint64, &(j->stats_max_sumrss));
res += mx_mysql_bind_var(result, idx++, int32, &(j->stats_status));
res += mx_mysql_bind_var(result, idx++, int64, &(j->stats_rusage.ru_utime.tv_sec));
res += mx_mysql_bind_var(result, idx++, int64, &(j->stats_rusage.ru_utime.tv_usec));
Expand Down Expand Up @@ -474,6 +476,7 @@ int mxq_set_job_status_exited(struct mx_mysql *mysql, struct mxq_job *job)
"UPDATE mxq_job SET"
" job_status = ?,"
" date_end = NULL,"
" stats_max_sumrss = ?, "
" stats_status = ?, "
" stats_utime_sec = ?, "
" stats_utime_usec = ?, "
Expand All @@ -495,12 +498,13 @@ int mxq_set_job_status_exited(struct mx_mysql *mysql, struct mxq_job *job)
" AND server_id = ?"
" AND host_pid = ?";

res = mx_mysql_bind_init_param(&param, 20);
res = mx_mysql_bind_init_param(&param, 21);
assert(res == 0);

idx = 0;
res = 0;
res += mx_mysql_bind_var(&param, idx++, uint16, &(newstatus));
res += mx_mysql_bind_var(&param, idx++, uint64, &(job->stats_max_sumrss));
res += mx_mysql_bind_var(&param, idx++, int32, &(job->stats_status));
res += mx_mysql_bind_var(&param, idx++, int64, &(job->stats_rusage.ru_utime.tv_sec));
res += mx_mysql_bind_var(&param, idx++, int64, &(job->stats_rusage.ru_utime.tv_usec));
Expand Down
2 changes: 2 additions & 0 deletions mxq_job.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,8 @@ struct mxq_job {

struct timeval stats_starttime;

uint64_t stats_max_sumrss;

int32_t stats_status;
struct timeval stats_realtime;
struct rusage stats_rusage;
Expand Down
10 changes: 5 additions & 5 deletions mxqd.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1638,16 +1638,16 @@ int killall_over_memory(struct mxq_server *server)
continue;
}

memory = pinfo->sum_rss * pagesize / 1024 / 1024;
memory = pinfo->sum_rss * pagesize / 1024;

if (job->max_sum_rss < memory)
job->max_sum_rss = memory;

if (memory <= group->group.job_memory)
if (memory/1024 <= group->group.job_memory)
continue;

mx_log_info("killall_over_memory(): used(%llu) > requested(%llu): Sending signal=KILL to job=%s(%d):%lu:%lu pgrp=%d",
memory, group->group.job_memory,
mx_log_info("killall_over_memory(): used(%lluMiB) > requested(%lluMiB): Sending signal=KILL to job=%s(%d):%lu:%lu pgrp=%d",
memory/1024, group->group.job_memory,
group->group.user_name, group->group.user_uid, group->group.group_id, job->job.job_id, pid);

kill(-pid, SIGKILL);
Expand Down Expand Up @@ -1762,7 +1762,7 @@ int catchall(struct mxq_server *server) {
g = &job->group->group;

timersub(&now, &j->stats_starttime, &j->stats_realtime);

j->stats_max_sumrss = job->max_sum_rss;
j->stats_status = status;
j->stats_rusage = rusage;

Expand Down
8 changes: 6 additions & 2 deletions mxqdump.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -128,6 +128,7 @@ static int print_group(struct mxq_group *g)
" max_utime=%lu"
" max_real=%lu"
" max_memory=%lukiB"
" max_rss=%lukiB"
" wait_sec=%lu"
" run_sec=%lu"
" idle_sec=%lu"
Expand All @@ -150,10 +151,11 @@ static int print_group(struct mxq_group *g)
g->job_threads,
g->job_memory*1024,
g->job_time*60,
(100UL*(uint64_t)g->stats_max_maxrss/1024UL/g->job_memory),
(100UL*(uint64_t)g->stats_max_sumrss/1024UL/g->job_memory),
(100UL*(uint64_t)g->stats_max_real.tv_sec/60UL/g->job_time),
g->stats_max_utime.tv_sec,
g->stats_max_real.tv_sec,
g->stats_max_sumrss,
g->stats_max_maxrss,
g->stats_wait_sec,
g->stats_run_sec,
Expand Down Expand Up @@ -219,6 +221,7 @@ static int print_job(struct mxq_group *g, struct mxq_job *j)
" runtime_requested=%us"
" time_load=%lu%%"
" memory_requested=%lukiB"
" max_memory=%lukiB"
" max_rss=%lukiB"
" memory_load=%lu%%"
" threads=%d"
Expand All @@ -244,8 +247,9 @@ static int print_job(struct mxq_group *g, struct mxq_job *j)
g->job_time*60,
(100UL*(run_sec)/60UL/g->job_time),
g->job_memory*1024,
j->stats_max_sumrss,
j->stats_rusage.ru_maxrss,
(100UL*j->stats_rusage.ru_maxrss/1024UL/g->job_memory),
(100UL*j->stats_max_sumrss/1024UL/g->job_memory),
g->job_threads,
j->host_slots,
mxq_job_status_to_name(j->job_status),
Expand Down
11 changes: 11 additions & 0 deletions mysql/alter_tables_0.17.0.sql
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
ALTER TABLE mxq_group
ADD COLUMN
stats_max_sumrss INT8 UNSIGNED NOT NULL DEFAULT 0
AFTER
group_date_end;

ALTER TABLE mxq_job
ADD COLUMN
stats_max_sumrss INT8 UNSIGNED NOT NULL DEFAULT 0
AFTER
job_id_first;
4 changes: 4 additions & 0 deletions mysql/create_tables.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,8 @@ CREATE TABLE IF NOT EXISTS mxq_group (

group_date_end TIMESTAMP NOT NULL DEFAULT 0,

stats_max_sumrss INT8 UNSIGNED NOT NULL DEFAULT 0,

stats_max_maxrss INT8 UNSIGNED NOT NULL DEFAULT 0,
stats_max_utime_sec INT8 UNSIGNED NOT NULL DEFAULT 0,
stats_max_stime_sec INT8 UNSIGNED NOT NULL DEFAULT 0,
Expand Down Expand Up @@ -94,6 +96,8 @@ CREATE TABLE IF NOT EXISTS mxq_job (
job_id_old INT8 UNSIGNED NULL DEFAULT NULL,
job_id_first INT8 UNSIGNED NULL DEFAULT NULL,

stats_max_sumrss INT8 UNSIGNED NOT NULL DEFAULT 0,

stats_status INT4 UNSIGNED NOT NULL DEFAULT 0,

stats_utime_sec INT8 UNSIGNED NOT NULL DEFAULT 0,
Expand Down
3 changes: 3 additions & 0 deletions mysql/create_trigger.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,7 @@ CREATE TRIGGER mxq_update_job BEFORE UPDATE ON mxq_job
group_slots_running=group_slots_running-NEW.host_slots,
group_jobs_running=group_jobs_running-1,
group_jobs_failed=group_jobs_failed+1,
stats_max_sumrss=GREATEST(stats_max_sumrss, NEW.stats_max_sumrss),
stats_max_maxrss=GREATEST(stats_max_maxrss, NEW.stats_maxrss),
stats_max_utime_sec=GREATEST(stats_max_utime_sec, NEW.stats_utime_sec),
stats_max_stime_sec=GREATEST(stats_max_stime_sec, NEW.stats_stime_sec),
Expand Down Expand Up @@ -99,6 +100,7 @@ CREATE TRIGGER mxq_update_job BEFORE UPDATE ON mxq_job
group_slots_running=group_slots_running-NEW.host_slots,
group_jobs_running=group_jobs_running-1,
group_jobs_finished=group_jobs_finished+1,
stats_max_sumrss=GREATEST(stats_max_sumrss, NEW.stats_max_sumrss),
stats_max_maxrss=GREATEST(stats_max_maxrss, NEW.stats_maxrss),
stats_max_utime_sec=GREATEST(stats_max_utime_sec, NEW.stats_utime_sec),
stats_max_stime_sec=GREATEST(stats_max_stime_sec, NEW.stats_stime_sec),
Expand All @@ -113,6 +115,7 @@ CREATE TRIGGER mxq_update_job BEFORE UPDATE ON mxq_job
WHERE group_id=NEW.group_id;
ELSEIF NEW.job_status NOT IN (399, 755, 989, 990) THEN
UPDATE mxq_group SET
stats_max_sumrss=GREATEST(stats_max_sumrss, NEW.stats_max_sumrss),
stats_max_maxrss=GREATEST(stats_max_maxrss, NEW.stats_maxrss),
stats_max_utime_sec=GREATEST(stats_max_utime_sec, NEW.stats_utime_sec),
stats_max_stime_sec=GREATEST(stats_max_stime_sec, NEW.stats_stime_sec),
Expand Down
3 changes: 3 additions & 0 deletions web/pages/mxq/mxq.in
View file
Original file line number Diff line number Diff line change
Expand Up @@ -269,6 +269,7 @@ group_mtime : $o{group_mtime}
group_date_end : $o{group_date_end}
stats_max_sumrss : $o{stats_max_maxrss} kiB
stats_max_maxrss : $o{stats_max_maxrss}
stats_max_utime_sec : $o{stats_max_utime_sec}
stats_max_stime_sec : $o{stats_max_stime_sec}
Expand Down Expand Up @@ -364,6 +365,8 @@ job_id_new : $o{job_id_new}
job_id_old : $o{job_id_old}
job_id_first : $o{job_id_first}
stats_max_sumrss : $o{stats_max_maxrss} kiB
stats_status : $o{stats_status}
stats_utime_sec : $o{stats_utime_sec}
Expand Down

0 comments on commit 9b2a5f4

Please sign in to comment.