Merge pull request #133 from mariux64/0.30.6

0.30.6

.gitignore

@@ -30,6 +30,7 @@ parser.tab.o
 test_parser.o
 test_parser
 ppidcache.o
+mxq_reaper.o
 
 mxqsub
 /mxqsub.1
@@ -44,6 +45,7 @@ test_mx_log
 test_mx_mysq
 test_mxqd_control
 test_keywordset
+mxq_reaper
 
 /web/pages/mxq/mxq
 web/lighttpd.conf

Makefile

@@ -1,6 +1,6 @@
 MXQ_VERSION_MAJOR = 0
 MXQ_VERSION_MINOR = 30
-MXQ_VERSION_PATCH = 5
+MXQ_VERSION_PATCH = 6
 MXQ_VERSION_EXTRA = "beta"
 MXQ_VERSIONDATE = 2022
 
@@ -122,7 +122,7 @@ CFLAGS_MYSQL += -DMX_MYSQL_FAIL_WAIT_DEFAULT=5
 
 CFLAGS += -g
 CFLAGS += -O3
-CFLAGS += -Wall
+CFLAGS += -Wall -Wextra -Wno-override-init
 CFLAGS += -DMXQ_VERSION=\"${MXQ_VERSION}\"
 CFLAGS += -DMXQ_VERSIONFULL=\"${MXQ_VERSIONFULL}\"
 CFLAGS += -DMXQ_VERSIONDATE=\"${MXQ_VERSIONDATE}\"
@@ -289,6 +289,13 @@ install::
 
 ########################################################################
 
+.PHONY: scan-build
+
+scan-build::
+	scan-build $(MAKE) build
+
+########################################################################
+
 ### mx_log.h -----------------------------------------------------------
 
 mx_log.h += mx_log.h
@@ -499,7 +506,6 @@ mxqd.o: CFLAGS += $(CFLAGS_MYSQL)
 mxqd.o: CFLAGS += $(CFLAGS_MXQ_INITIAL_PATH)
 mxqd.o: CFLAGS += $(CFLAGS_MXQ_INITIAL_TMPDIR)
 mxqd.o: CFLAGS += $(CFLAGS_MXQ_FINISHED_JOBSDIR)
-mxqd.o: CFLAGS += -Wno-unused-but-set-variable
 
 clean: CLEAN += mxqd.o
 
@@ -587,7 +593,6 @@ mxqdump: mxq_job.o
 mxqdump: mx_util.o
 mxqdump: mx_getopt.o
 mxqdump: LDLIBS += $(LDLIBS_MYSQL)
-mxqdump: CFLAGS += -Wunused-function
 
 build: mxqdump
 
@@ -656,10 +661,17 @@ build: mxqps
 
 clean: CLEAN += mxqps
 
+### mxqps -------------------------------------------------------------
+
+clean: CLEAN += mxq_reaper.o mxq_reaper
+build: mxq_reaper
+install:: mxq_reaper
+	$(call quiet-install,0755,$^,${DESTDIR}${LIBEXECDIR}/mxq/mxq_reaper)
+
 ### script helper -----------------------------------------------------
 
-install:: helper/create_job_tmpdir
-	$(call quiet-install,0755,$^,${DESTDIR}${LIBEXECDIR}/mxq/create_job_tmpdir)
+install:: helper/tmpdir-setup
+	$(call quiet-install,0755,$^,${DESTDIR}${LIBEXECDIR}/mxq/tmpdir-setup)
 
 install:: helper/gpu-setup
 	$(call quiet-install,0755,$^,${DESTDIR}${LIBEXECDIR}/mxq/gpu-setup)

helper/create_job_tmpdir

@@ -37,6 +37,6 @@ if fallocate -l ${MXQ_SIZE}G $filename; then
     fi
     rm $filename
 else
-    test -e $fileame && rm $filename
+    test -e $filename && rm $filename
 fi
 exit $status

helper/tmpdir-setup

@@ -0,0 +1,76 @@
+#! /usr/bin/bash
+
+usage() {
+    cat <<EOF >&2
+usage:
+    $0 create JOBID SIZE UID  # create SIZE GiB /dev/shm/mxqd/mnt/job/$JOBID
+    $0 cleanup JOBID          # cleanup
+EOF
+    exit 1
+}
+
+tmpdir=/scratch/local2/mxqd/tmp
+mntdir=/dev/shm/mxqd/mnt/job
+
+cmd_create() {
+    (( $# == 3 )) || usage
+    MXQ_JOBID=$1
+    MXQ_SIZE=$2
+    MXQ_UID=$3
+
+    filename=$tmpdir/$MXQ_JOBID.tmp
+    mountpoint=$mntdir/$MXQ_JOBID
+
+    umask 006
+    mkdir -p $tmpdir
+    mkdir -p $mntdir
+
+    status=1;
+
+    if fallocate -l ${MXQ_SIZE}G $filename; then
+        if loopdevice=$(losetup --find --show $filename); then
+            if mkfs.ext4 \
+                    -q \
+                    -m 0 \
+                    -E nodiscard,mmp_update_interval=300,lazy_journal_init=1,root_owner=$MXQ_UID:0 \
+                    -O '64bit,ext_attr,filetype,^has_journal,huge_file,inline_data,^mmp,^quota,sparse_super2' \
+                    $loopdevice \
+                    && mkdir -p $mountpoint && mount -Odata=writeback,barrier=0  $loopdevice $mountpoint; then
+                rmdir $mountpoint/lost+found
+                status=0
+            fi
+            losetup -d $loopdevice
+        fi
+        rm $filename
+    else
+        test -e $filename && rm $filename
+    fi
+    exit $status
+}
+
+cmd_cleanup() {
+    (( $# == 1 )) || usage
+    MXQ_JOBID=$1
+
+    (
+        shopt -s dotglob;
+        rm -rf /dev/shm/mxqd/mnt/job/$MXQ_JOBID/*
+        umount /dev/shm/mxqd/mnt/job/$MXQ_JOBID
+        rmdir /dev/shm/mxqd/mnt/job/$MXQ_JOBID
+    ) &
+}
+
+(( $# > 0 )) || usage
+cmd="$1"
+shift;
+case "$cmd" in
+    create)
+        cmd_create "$@"
+        ;;
+    cleanup)
+        cmd_cleanup "$@"
+        ;;
+    *)
+        usage
+        ;;
+esac

keywordset.c

@@ -15,7 +15,7 @@ struct keywordset {
 
 static int find_name(struct keywordset *kws, char *name, size_t len) {
     int i;
-    int j;
+    size_t j;
     for ( i = 0; i < kws->used ; i++ ) {
         j = 0;
         while(1) {

mx_mysql.c

@@ -699,11 +699,9 @@ static int _mx_mysql_bind_string(struct mx_mysql_bind *b, unsigned int index, ch
 
 static int _mx_mysql_bind_validate(struct mx_mysql_bind *b)
 {
-    int i;
-
     mx_assert_return_minus_errno(b, EINVAL);
 
-    for (i=0; i < b->count; i++) {
+    for (unsigned long i=0; i < b->count; i++) {
         if (!(b->data[i].flags)) {
             return -(errno=EBADSLT);
         }
@@ -947,6 +945,7 @@ static int mx_mysql_statement_init(struct mx_mysql *mysql, struct mx_mysql_stmt
     s = mx_calloc_forever(1, sizeof(*s));
 
     s->mysql       = mysql;
+    s->func = "";
 
     do {
         res = mx__mysql_stmt_init(s);
@@ -1015,7 +1014,6 @@ int mx_mysql_statement_fetch(struct mx_mysql_stmt *stmt)
 {
     struct mx_mysql_bind *r;
     int res;
-    int col;
     char *str;
     int no_error = 1;
 
@@ -1044,7 +1042,7 @@ int mx_mysql_statement_fetch(struct mx_mysql_stmt *stmt)
     }
 
     r = &stmt->result;
-    for (col = 0; col < r->count; col++) {
+    for (unsigned long col = 0; col < r->count; col++) {
         if (r->bind[col].buffer_type == MYSQL_TYPE_STRING) {
             str = mx_calloc_forever(r->data[col].length + 1, sizeof(*str));
 
@@ -1063,7 +1061,7 @@ int mx_mysql_statement_fetch(struct mx_mysql_stmt *stmt)
         if (!(r->data[col].is_error))
             continue;
 
-        mx_log_debug("WARNING: result data returned in column with index %d was truncated. query was:", col);
+        mx_log_debug("WARNING: result data returned in column with index %lu was truncated. query was:", col);
         mx_log_debug("       \\ %s", stmt->statement);
         no_error = 0;
     }
@@ -1165,7 +1163,6 @@ static int _mx_mysql_do_statement(struct mx_mysql *mysql, char *query, struct mx
     struct mx_mysql_stmt *stmt = NULL;
     unsigned long long num_rows = 0;
     int res;
-    int cnt = 0;
     char *tmpdata;
 
     assert(mysql);
@@ -1196,7 +1193,7 @@ static int _mx_mysql_do_statement(struct mx_mysql *mysql, char *query, struct mx
     if (result && result->count && num_rows) {
         tmpdata = mx_calloc_forever(num_rows, size);
 
-        for (cnt = 0; cnt < num_rows; cnt++) {
+        for (unsigned long cnt = 0; cnt < num_rows; cnt++) {
             res = mx_mysql_statement_fetch(stmt);
             if (res < 0) {
                 mx_log_err("mx_mysql_statement_fetch(): %m");

mx_proc.c

@@ -10,15 +10,14 @@
 #include "mx_proc.h"
 
 static long long int get_rss_anon(pid_t pid) {
-    _mx_cleanup_free_ char *fname;
-    mx_asprintf_forever(&fname, "/proc/%d/status", pid);
+    _mx_cleanup_free_ char *fname = mx_asprintf_forever("/proc/%d/status", pid);
     _mx_cleanup_fclose_ FILE *file = fopen(fname, "r");
     if (file == NULL)
         return -errno;
     _mx_cleanup_free_ char *buf = NULL;
     size_t n = 0;
     while(1) {
-        size_t len = getline(&buf, &n, file);
+        ssize_t len = getline(&buf, &n, file);
         if (len == -1)
             break;
         if (strncmp(buf, "RssAnon:", 8) == 0) {
@@ -131,15 +130,14 @@ int mx_proc_pid_stat(struct mx_proc_pid_stat **pps, pid_t pid)
 
 int mx_proc_pid_stat_read(struct mx_proc_pid_stat *pps, char *fmt, ...)
 {
-    _mx_cleanup_free_ char *fname = NULL;
     _mx_cleanup_free_ char *line = NULL;
     va_list ap;
     int res;
 
     assert(pps);
 
     va_start(ap, fmt);
-    mx_vasprintf_forever(&fname, fmt, ap);
+    _mx_cleanup_free_ char *fname = fname = mx_vasprintf_forever(fmt, ap);
     va_end(ap);
 
     res = mx_read_first_line_from_file(fname, &line);