re-introduce nginx setup

build.profile

@@ -46,8 +46,8 @@ BUILD_leptonica=leptonica-1.85.0
 BUILD_leptonica_SRCURL="https://beehive.molgen.mpg.de/1732e999e2fef8721348edc2b7283224/${BUILD_leptonica}.tar.gz"
 
 # SRCURL="https://nginx.org/download/nginx-1.27.3.tar.gz"
-#BUILD_nginx=nginx-1.27.3
-#BUILD_nginx_SRCURL="https://beehive.molgen.mpg.de/89773c781ff0c2cd876b03bbc094c258/${BUILD_nginx}.tar.gz"
+BUILD_nginx=nginx-1.27.3
+BUILD_nginx_SRCURL="https://beehive.molgen.mpg.de/89773c781ff0c2cd876b03bbc094c258/${BUILD_nginx}.tar.gz"
 
 # SRCURL="https://github.com/liberationfonts/liberation-fonts/files/7261482/liberation-fonts-ttf-2.1.5.tar.gz"
 BUILD_libfontttf=liberation-fonts-ttf-2.1.5

buildall.sh

@@ -22,8 +22,9 @@ PAPERLESS_WEBSERVER_WORKERS=5
 PAPERLESS_URL=https://localhost
 PAPERLESS_CSRF_TRUSTED_ORIGINS=http://localhost:\${PAPERLESS_PORT}
 
-# SSL_CERTIFICATE=\${PROJECT}/certs/fullchain.pem
-# SSL_CERTIFICATE_KEY=\${PROJECT}/certs/privkey.pem
+NGINX_SSL_DHPARAM=\${PROJECT}/certs/dhparam.pem
+NGINX_SSL_CERTIFICATE=\${PROJECT}/certs/fullchain.pem
+NGINX_SSL_CERTIFICATE_KEY=\${PROJECT}/certs/privkey.pem
 
 _EOP_
 ) ${BUILD_LOCAL}
@@ -66,33 +67,52 @@ ln -fs "${logfile}" "build.log"
 popd
 
 {
-[ -d "${BUILD_DIR}/${BUILD_sqlite}" ] || ./sqlite.build.sh
-[ -d "${BUILD_DIR}/${BUILD_redis}" ] || ./redis.build.sh
-[ -d "${BUILD_DIR}/${BUILD_python}" ] || ./python.build.sh
-[ -d "${BUILD_DIR}/${BUILD_node}" ] || ./node.build.sh
-[ -d "${BUILD_DIR}/${BUILD_libjpegturbo}" ] || ./libjpegturbo.build.sh
-[ -d "${BUILD_DIR}/${BUILD_libtiff}" ] || ./libtiff.build.sh
-[ -d "${BUILD_DIR}/${BUILD_libwebp}" ] || ./libwebp.build.sh
-[ -d "${BUILD_DIR}/${BUILD_ffmpeg}" ] || ./ffmpeg.build.sh
-
-[ -d "${BUILD_DIR}/${BUILD_jbig2dec}" ] || ./jbig2dec.build.sh
-[ -d "${BUILD_DIR}/${BUILD_ghostscript}" ] || ./ghostscript.build.sh
-
-[ -d "${BUILD_DIR}/${BUILD_imagemagick}" ] || ./imagemagick.build.sh
-[ -d "${BUILD_DIR}/${BUILD_leptonica}" ] || ./leptonica.build.sh
-[ -d "${BUILD_DIR}/${BUILD_jbig2enc}" ] || ./jbig2enc.build.sh
-[ -d "${BUILD_DIR}/${BUILD_tesseract}" ] || ./tesseract.build.sh
-
-[ -d "${BUILD_DIR}/${BUILD_pngquant}" ] || ./pngquant.build.sh
-[ -d "${BUILD_DIR}/${BUILD_unpaper}" ] || ./unpaper.build.sh
-[ -d "${BUILD_DIR}/${BUILD_zbar}" ] || ./zbar.build.sh
-[ -d "${BUILD_DIR}/${BUILD_libfontttf}" ] || ./liberation-fonts-ttf.build.sh
-#[ -d "${BUILD_DIR}/${BUILD_nginx}" ] || ./nginx.build.sh
-[ -d "${BUILD_DIR}/${BUILD_qpdf}" ] || ./qpdf.build.sh
-# ./nginx.conf.build.sh
+_pkgs=(
+    sqlite
+    redis
+    python
+    node
+    libjpegturbo
+    libtiff
+    libwebp
+    ffmpeg
+    jbig2dec
+    ghostscript
+    imagemagick
+    leptonica
+    jbig2enc
+    tesseract
+    pngquant
+    unpaper
+    zbar
+    libfontttf
+    qpdf
+    nginx
+)
+
+build() {
+    local _pkg=$1
+    eval "_tag=${BUILD_DIR}/\$BUILD_${_pkg}/.built"
+    if [[ ! -e "$_tag" ]]; then
+        ./${_pkg}.build.sh 2>&1 | tee "${LOGS}/${_pkg}.log";
+        r=${PIPESTATUS[0]}
+        if [[ $r == 0 ]]; then
+            touch "$_tag"
+        else
+            echo "${_pkg}.build.sh failed exit code $r"
+            exit
+        fi
+    fi
+}
+
+for _pkg in "${_pkgs[@]}"; do
+    build "$_pkg"
+done
+
 ./profile.build.sh
 ./startstop.build.sh
-[ -d "$PROJECT/${BUILD_paperless}" ] || ./paperless-ngx.build.sh
+./nginx.conf.build.sh
 
+./paperless-ngx.build.sh
 
 } 2>&1 | tee "${LOGS}/${logfile}" 2>&1

nginx.conf.build.sh

@@ -21,12 +21,12 @@ http {
     client_max_body_size 100M;
     keepalive_timeout    65;
 
-    upstream gunicorn_socket {
+    upstream granian_socket {
         # fail_timeout=0 means we always retry an upstream even if it failed
         # to return a good HTTP response (in case the Unicorn master nukes a
         # single worker for timing out).
 
-        server unix:${DEVSHM}/gunicorn.sock fail_timeout=0;
+        server unix:${DEVSHM}/granian.sock fail_timeout=0;
     }
 
     server {
@@ -65,7 +65,7 @@ tee -a ${CONF} <<_EOP_
         error_page 400 404 405 502 =200 /chicken.html;
 
         location / {
-            proxy_pass http://gunicorn_socket/;
+            proxy_pass http://granian_socket/;
             proxy_http_version 1.1;
             proxy_set_header Upgrade \$http_upgrade;
             proxy_set_header Connection "upgrade";

profile.build.sh

@@ -19,6 +19,7 @@ PATH=$HOME/bin:${PROJECT}/bin:/bin:/sbin:/usr/bin:/usr/sbin
 . ${PROJECT}/${BUILD_python}/profile
 . ${PROJECT}/${BUILD_node}/profile
 . ${PROJECT}/${BUILD_redis}/profile
+. ${PROJECT}/${BUILD_nginx}/profile
 . ${PROJECT}/${BUILD_ghostscript}/profile
 . ${PROJECT}/${BUILD_imagemagick}/profile
 . ${PROJECT}/${BUILD_unpaper}/profile

startstop.build.sh

@@ -27,6 +27,31 @@ function pwait() {
   fi
 }
 
+#### nginx ####
+
+srv_nginx_start() {
+    nginx -t
+    srv_granian_waitup
+    trap rm_pidfiles EXIT
+    nginx -g 'daemon off;'
+}
+
+srv_nginx_restart() {
+    nginx -s reload
+}
+
+srv_nginx_stop() {
+    nginx -s quit || rm -fv "${DEVSHM}"/nginx.pid
+}
+
+srv_nginx_waitup() {
+    until ( printf "" 2>>/dev/null >>/dev/tcp/${PAPERLESS_BIND_ADDR}/${PAPERLESS_PORT} ); do
+      echo "waiting for nginx to be ready..."
+      sleep 1
+    done
+}
+
+
 #### redis ####
 
 srv_redis_start() {
@@ -46,14 +71,7 @@ srv_granian_start() {
     cd "${PROJECT}/paperless-ngx/src"
     rm -fv "${DEVSHM}/granian.pid"
     _opts=(
-# no unix domain socket support in granian
-# https://github.com/emmett-framework/granian/issues/97
-# https://github.com/paperless-ngx/paperless-ngx/discussions/9941
-#      --host "unix:${DEVSHM}/granian.sock"
-      --host ${PAPERLESS_BIND_ADDR}
-      --port ${PAPERLESS_PORT}
-      ${SSL_CERTIFICATE:+--ssl-certificate ${SSL_CERTIFICATE}}
-      ${SSL_CERTIFICATE_KEY:+--ssl-keyfile ${SSL_CERTIFICATE_KEY}}
+      --uds "${DEVSHM}/granian.sock"
       --workers ${PAPERLESS_WEBSERVER_WORKERS:-1}
       --interface asginl
       --pid-file "${DEVSHM}/granian.pid"
@@ -71,7 +89,7 @@ srv_granian_start() {
 }
 
 srv_granian_waitup() {
-    until ( printf "" 2>>/dev/null >>/dev/tcp/${PAPERLESS_BIND_ADDR}/${PAPERLESS_PORT} ); do
+    until ( [[ -e ${DEVSHM}/granian.sock ]] ); do
       echo "waiting for granian server to be ready..."
       sleep 1
     done
@@ -84,7 +102,7 @@ srv_consumer_start() {
     trap rm_pidfiles EXIT
 
     cd "${PROJECT}/paperless-ngx/src"
-    srv_granian_waitup
+    srv_nginx_waitup
 
     ./manage.py document_consumer
 }
@@ -96,7 +114,7 @@ srv_scheduler_start() {
     trap rm_pidfiles EXIT
 
     cd "${PROJECT}/paperless-ngx/src"
-    srv_granian_waitup
+    srv_nginx_waitup
 
     celery --app paperless beat --loglevel INFO
 }
@@ -108,7 +126,7 @@ srv_worker_start() {
     trap rm_pidfiles EXIT
 
     cd "${PROJECT}/paperless-ngx/src"
-    srv_granian_waitup
+    srv_nginx_waitup
 
     celery --app paperless worker --loglevel WARNING
 }
@@ -198,17 +216,23 @@ srv_generic_stop() {
     echo "killing process group \${pgid}"
     KT=\${SECONDS}
     kill -- -\${pgid}||:
-    echo "waiting for process group \${pgid} to die"
+    printf "waiting for pgid %5d (%s) to die\n"  \${pgid} \${srv}
     pwait --pgroup \${pgid}||:
+    while kill -0 -"\${pgid}" > /dev/null 2>&1; do
+      sleep 1
+    done
+    rm -fv "\${PGIDFILE}"
+    rm -fv "\${PIDFILE}"
     KT=\$((\${SECONDS}-\${KT}))
     [[ \${KT} > 1 ]] && echo "kill took \${KT} seconds"
   else
     if [[ -s "\${PIDFILE}" ]]; then
       local pid
       read -a pid < "\${PIDFILE}"||:
-      kill -- \${pid}
-      echo "waiting for process \${pid} to die"
+      kill -- \${pid} ||:
+      printf "waiting for pid %5d (%s) to die\n"  \${pid} \${srv}
       tail -f --pid=\${pid} /dev/null
+      rm -fv "\${PIDFILE}"
     else
       echo "# no pid file found: \${PGIDFILE}"
     fi
@@ -228,7 +252,7 @@ cmd="\${1:-help}"
 srv="\${2:-all}"
 
 if [[ \${srv} = 'all' ]]; then
-  _srv=(redis granian consumer scheduler worker)
+  _srv=(redis granian nginx consumer scheduler worker)
 
   if [[ \${cmd} = 'start' ]]; then
     srv_migrate
@@ -241,7 +265,7 @@ if [[ \${srv} = 'all' ]]; then
 
   for srv in \${_srv[@]}; do
     # echo "### ./startstop.sh \${cmd} \${srv}"
-    "${PROJECT}"/startstop.sh \${cmd} \${srv} || echo "RET: $?"
+    "${PROJECT}"/startstop.sh \${cmd} \${srv} ||:
   done
   exit
 fi
@@ -264,11 +288,11 @@ if [[ \$(type -t srv_\${srv}_\${cmd}) = 'function' ]]; then
             srv_\${srv}_\${cmd};
         } > ${LOGDIR}/srv-\${srv}.log 2>&1
         echo "\${COPROC_PID}" >> "\${PIDFILE}"
-        ps opgid= "\${COPROC_PID}" > "\${PGIDFILE}"
+        ps -o pgid= -p "\${COPROC_PID}" > "\${PGIDFILE}"
         exit
         ;;
       *)
-        echo "########## default fallback to srv_\${srv}_\${cmd} ###############"
+        echo "########## srv_\${srv}_\${cmd}() ###############"
         srv_\${srv}_\${cmd}
         exit
         ;;