From c7c654412af1ff92ac8373caa201bd7829ccaff5 Mon Sep 17 00:00:00 2001 From: Donald Buczek Date: Sat, 2 Apr 2022 13:51:23 +0200 Subject: [PATCH 01/11] mxqd: remove RLIMIT_AS and RLIMIT_RSS Remove RLIMIT_AS, because we to limit physical memory usage, not virtual memory for file-backed mappings. Remove RLIMIT_RSS, because this limit has no effect since Linux 2.4.30. --- mxqd.c | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/mxqd.c b/mxqd.c index a4c35a61..cbbd8a64 100644 --- a/mxqd.c +++ b/mxqd.c @@ -930,21 +930,11 @@ static int init_child_process(struct mxq_group_list *glist, struct mxq_job *job) rlim.rlim_cur = group->job_memory*1024*1024; rlim.rlim_max = group->job_memory*1024*1024; - res = setrlimit(RLIMIT_AS, &rlim); - if (res == -1) - mx_log_err("job=%s(%d):%lu:%lu setrlimit(RLIMIT_AS, ...) failed: %m", - group->user_name, group->user_uid, group->group_id, job->job_id); - res = setrlimit(RLIMIT_DATA, &rlim); if (res == -1) mx_log_err("job=%s(%d):%lu:%lu setrlimit(RLIMIT_DATA, ...) failed: %m", group->user_name, group->user_uid, group->group_id, job->job_id); - res = setrlimit(RLIMIT_RSS, &rlim); - if (res == -1) - mx_log_err("job=%s(%d):%lu:%lu setrlimit(RLIMIT_RSS, ...) failed: %m", - group->user_name, group->user_uid, group->group_id, job->job_id); - /* disable core files */ rlim.rlim_cur = 0; rlim.rlim_cur = 0; From a165e5f6b62f64647f8cedd769be06648f0a1dbf Mon Sep 17 00:00:00 2001 From: Donald Buczek Date: Sat, 2 Apr 2022 15:35:00 +0200 Subject: [PATCH 02/11] test_parser: Fix Off-by-One --- test_parser.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_parser.c b/test_parser.c index 7ff44a04..1c56638b 100644 --- a/test_parser.c +++ b/test_parser.c @@ -68,7 +68,7 @@ int main() { keywordset_free(tags); - static char text[8001]; + static char text[8002]; text[8001] = 0; memset(text, '(', 8000); test_expression(tags, text, 1, 0); From 2e6a295173e0d0c1b506231afdabcc64cc4be134 Mon Sep 17 00:00:00 2001 From: Donald Buczek Date: Sat, 2 Apr 2022 15:37:09 +0200 Subject: [PATCH 03/11] mx_proc: Remove unused member --- mx_proc.h | 2 -- 1 file changed, 2 deletions(-) diff --git a/mx_proc.h b/mx_proc.h index d4abcb35..7e4cede6 100644 --- a/mx_proc.h +++ b/mx_proc.h @@ -7,8 +7,6 @@ struct mx_proc_info { struct mx_proc_pid_stat *pstat; unsigned long long int sum_rss; - - char **environment; }; struct mx_proc_tree { From 91b1993f7e422471a83fa912513e412c1ba2d903 Mon Sep 17 00:00:00 2001 From: Donald Buczek Date: Sat, 2 Apr 2022 15:56:48 +0200 Subject: [PATCH 04/11] mx_proc: Remove unused code --- mx_proc.c | 17 ----------------- mx_proc.h | 1 - 2 files changed, 18 deletions(-) diff --git a/mx_proc.c b/mx_proc.c index 67ecb2e2..809a6d55 100644 --- a/mx_proc.c +++ b/mx_proc.c @@ -99,23 +99,6 @@ int mx_proc_pid_stat(struct mx_proc_pid_stat **pps, pid_t pid) return 0; } -int mx_proc_pid_task_tid_stat(struct mx_proc_pid_stat **pps, pid_t pid, pid_t tid) -{ - struct mx_proc_pid_stat *pstat; - int res; - - pstat = *pps; - if (!pstat) - pstat = mx_calloc_forever(1, sizeof(*pstat)); - - res = mx_proc_pid_stat_read(pstat, "/proc/%d/task/%d/stat", pid, tid); - if (res < 0) - return res; - - *pps = pstat; - return 0; -} - int mx_proc_pid_stat_read(struct mx_proc_pid_stat *pps, char *fmt, ...) { _mx_cleanup_free_ char *fname = NULL; diff --git a/mx_proc.h b/mx_proc.h index 7e4cede6..6ae184a8 100644 --- a/mx_proc.h +++ b/mx_proc.h @@ -74,7 +74,6 @@ struct mx_proc_pid_stat { int mx_proc_pid_stat_read(struct mx_proc_pid_stat *pps, char *fmt, ...); int mx_proc_pid_stat(struct mx_proc_pid_stat **pps, pid_t pid); -int mx_proc_pid_task_tid_stat(struct mx_proc_pid_stat **pps, pid_t pid, pid_t tid); void mx_proc_pid_stat_free_content(struct mx_proc_pid_stat *pps); From 4a4ed5bf0c45c54d7d3b53812436af116d4b271d Mon Sep 17 00:00:00 2001 From: Donald Buczek Date: Sat, 2 Apr 2022 18:34:48 +0200 Subject: [PATCH 05/11] Fix memory leak on error path --- mx_proc.c | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/mx_proc.c b/mx_proc.c index 809a6d55..c5197f28 100644 --- a/mx_proc.c +++ b/mx_proc.c @@ -89,13 +89,11 @@ int mx_proc_pid_stat(struct mx_proc_pid_stat **pps, pid_t pid) pstat = *pps; if (!pstat) - pstat = mx_calloc_forever(1, sizeof(*pstat)); + *pps = pstat = mx_calloc_forever(1, sizeof(*pstat)); res = mx_proc_pid_stat_read(pstat, "/proc/%d/stat", pid); if (res < 0) return res; - - *pps = pstat; return 0; } @@ -370,9 +368,10 @@ int mx_proc_tree(struct mx_proc_tree **newtree) pps = NULL; res = mx_proc_pid_stat(&pps, pid); - if (res < 0) + if (res < 0) { + free(pps); continue; - + } mx_proc_tree_add(pt, pps); } free(namelist); From b70ace55b47f18d53dc15049a03c204df9503e22 Mon Sep 17 00:00:00 2001 From: Donald Buczek Date: Sat, 2 Apr 2022 17:54:27 +0200 Subject: [PATCH 06/11] mx_proc: Add rss_anon to struct mx_proc_pid_stat --- mx_proc.c | 32 ++++++++++++++++++++++++++++++++ mx_proc.h | 2 ++ 2 files changed, 34 insertions(+) diff --git a/mx_proc.c b/mx_proc.c index c5197f28..7d402ebc 100644 --- a/mx_proc.c +++ b/mx_proc.c @@ -4,10 +4,38 @@ #include #include #include +#include #include "mx_util.h" #include "mx_proc.h" +static long long int get_rss_anon(pid_t pid) { + _mx_cleanup_free_ char *fname; + mx_asprintf_forever(&fname, "/proc/%d/status", pid); + _mx_cleanup_fclose_ FILE *file = fopen(fname, "r"); + if (file == NULL) + return -errno; + _mx_cleanup_free_ char *buf = NULL; + size_t n = 0; + while(1) { + size_t len = getline(&buf, &n, file); + if (len == -1) + break; + if (strncmp(buf, "RssAnon:", 8) == 0) { + unsigned long long int anon_rss_kb = strtoull(buf+8, NULL, 10); + if (anon_rss_kb == ULLONG_MAX) + return -errno; + if (anon_rss_kb > LLONG_MAX/1024) { // anon_rss > 8 EiB + return -ERANGE; + } + return anon_rss_kb*1024; + } + } + if (feof(file)) + return 0; /* kernel thread */ + return -errno; +} + static int _mx_proc_pid_stat_strscan(char *str, struct mx_proc_pid_stat *pps) { size_t res = 0; @@ -94,6 +122,10 @@ int mx_proc_pid_stat(struct mx_proc_pid_stat **pps, pid_t pid) res = mx_proc_pid_stat_read(pstat, "/proc/%d/stat", pid); if (res < 0) return res; + long long int rss_anon = get_rss_anon(pid); + if (rss_anon < 0) + return rss_anon; + pstat->rss_anon = rss_anon; return 0; } diff --git a/mx_proc.h b/mx_proc.h index 6ae184a8..c9006a9f 100644 --- a/mx_proc.h +++ b/mx_proc.h @@ -69,6 +69,8 @@ struct mx_proc_pid_stat { unsigned long long int delayacct_blkio_ticks; /* 42 */ unsigned long long int guest_time; /* 43 */ long long int cguest_time; /* 44 */ + + unsigned long long int rss_anon; /* from /proc/PID/status. may be null (kernel thread). unit: bytes */ }; int mx_proc_pid_stat_read(struct mx_proc_pid_stat *pps, char *fmt, ...); From faf0990a24ceaba62d0f8b53b5844c03ce9cb6ad Mon Sep 17 00:00:00 2001 From: Donald Buczek Date: Sat, 2 Apr 2022 18:50:03 +0200 Subject: [PATCH 07/11] mx_proc: Add sum_rss_anon to struct mx_proc_info --- mx_proc.c | 2 ++ mx_proc.h | 1 + 2 files changed, 3 insertions(+) diff --git a/mx_proc.c b/mx_proc.c index 7d402ebc..00b37a6b 100644 --- a/mx_proc.c +++ b/mx_proc.c @@ -167,6 +167,7 @@ static void mx_proc_tree_update_parent_pinfo(struct mx_proc_tree_node *this, str return; this->pinfo.sum_rss += pinfo->sum_rss; + this->pinfo.sum_rss_anon += pinfo->sum_rss_anon; mx_proc_tree_update_parent_pinfo(this->parent, pinfo); } @@ -269,6 +270,7 @@ static struct mx_proc_tree_node *mx_proc_tree_add(struct mx_proc_tree *pt, struc new->pinfo.pstat = pps; new->pinfo.sum_rss = pps->rss; + new->pinfo.sum_rss_anon = pps->rss_anon; if (!(pt->root)) { pt->root = new; diff --git a/mx_proc.h b/mx_proc.h index c9006a9f..428620df 100644 --- a/mx_proc.h +++ b/mx_proc.h @@ -7,6 +7,7 @@ struct mx_proc_info { struct mx_proc_pid_stat *pstat; unsigned long long int sum_rss; + unsigned long long int sum_rss_anon; }; struct mx_proc_tree { From f593d695f7810d3fcdcb71b42d33379e1779bf3a Mon Sep 17 00:00:00 2001 From: Donald Buczek Date: Sat, 2 Apr 2022 19:00:05 +0200 Subject: [PATCH 08/11] mxqps: show rss_anon instead of rss --- mxqps.c | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/mxqps.c b/mxqps.c index ee21424a..65ee66ab 100644 --- a/mxqps.c +++ b/mxqps.c @@ -18,17 +18,13 @@ int filter(const struct dirent *d) return 1; } -#define MX_PROC_TREE_NODE_IS_KERNEL_THREAD(x) ((x)->pinfo.pstat->ppid == 0 && (x)->pinfo.sum_rss == 0) +#define MX_PROC_TREE_NODE_IS_KERNEL_THREAD(x) ((x)->pinfo.pstat->ppid == 0 && (x)->pinfo.sum_rss_anon == 0) int mx_proc_tree_node_print_debug(struct mx_proc_tree_node *ptn, int lvl) { assert(ptn); struct mx_proc_tree_node *current; - long pagesize; - - pagesize = sysconf(_SC_PAGESIZE); - assert(pagesize); for (current = ptn; current; current=current->next) { if (MX_PROC_TREE_NODE_IS_KERNEL_THREAD(current)) @@ -39,8 +35,8 @@ int mx_proc_tree_node_print_debug(struct mx_proc_tree_node *ptn, int lvl) current->pinfo.pstat->ppid, current->pinfo.pstat->pgrp, current->pinfo.pstat->session, - current->pinfo.pstat->rss*pagesize/1024, - current->pinfo.sum_rss*pagesize/1024, + current->pinfo.pstat->rss_anon/1024, + current->pinfo.sum_rss_anon/1024, current->pinfo.pstat->num_threads); if (lvl>0) From 2e95600c890a52e9106993c241f17425d6508618 Mon Sep 17 00:00:00 2001 From: Donald Buczek Date: Sat, 2 Apr 2022 19:15:41 +0200 Subject: [PATCH 09/11] mxqd: killall_over_memory: use sum_rss_anon --- mxqd.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mxqd.c b/mxqd.c index cbbd8a64..bdb31eaa 100644 --- a/mxqd.c +++ b/mxqd.c @@ -1909,7 +1909,7 @@ static int killall_over_memory(struct ppidcache *ppidcache, struct mxq_server *s continue; } - memory = pinfo->sum_rss * pagesize / 1024; + memory = pinfo->sum_rss_anon / 1024; if (jlist->max_sumrss < memory) jlist->max_sumrss = memory; From 3232f06306d7fd229fafe3ba6e75457ec16199cb Mon Sep 17 00:00:00 2001 From: Donald Buczek Date: Sat, 2 Apr 2022 20:14:51 +0200 Subject: [PATCH 10/11] mx_proc: Remove now unused field sum_rss from struct mx_proc_info --- mx_proc.c | 2 -- mx_proc.h | 1 - 2 files changed, 3 deletions(-) diff --git a/mx_proc.c b/mx_proc.c index 00b37a6b..48e590cc 100644 --- a/mx_proc.c +++ b/mx_proc.c @@ -166,7 +166,6 @@ static void mx_proc_tree_update_parent_pinfo(struct mx_proc_tree_node *this, str if (!this) return; - this->pinfo.sum_rss += pinfo->sum_rss; this->pinfo.sum_rss_anon += pinfo->sum_rss_anon; mx_proc_tree_update_parent_pinfo(this->parent, pinfo); @@ -269,7 +268,6 @@ static struct mx_proc_tree_node *mx_proc_tree_add(struct mx_proc_tree *pt, struc pt->nentries++; new->pinfo.pstat = pps; - new->pinfo.sum_rss = pps->rss; new->pinfo.sum_rss_anon = pps->rss_anon; if (!(pt->root)) { diff --git a/mx_proc.h b/mx_proc.h index 428620df..2a125c66 100644 --- a/mx_proc.h +++ b/mx_proc.h @@ -6,7 +6,6 @@ struct mx_proc_info { struct mx_proc_pid_stat *pstat; - unsigned long long int sum_rss; unsigned long long int sum_rss_anon; }; From d563ab69cc19d25fca4db5ea66ccfb637b1a03ac Mon Sep 17 00:00:00 2001 From: Donald Buczek Date: Mon, 4 Apr 2022 07:57:44 +0200 Subject: [PATCH 11/11] Makefile: Update version to 0.30.5 --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index c151d5c5..d9ec74e5 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ MXQ_VERSION_MAJOR = 0 MXQ_VERSION_MINOR = 30 -MXQ_VERSION_PATCH = 4 +MXQ_VERSION_PATCH = 5 MXQ_VERSION_EXTRA = "beta" MXQ_VERSIONDATE = 2022