From 340fa40a05bbc15c55f4b0ecb61131a5c8e09eb6 Mon Sep 17 00:00:00 2001 From: Peter Marquardt Date: Wed, 3 Sep 2025 15:59:12 +0200 Subject: [PATCH] re-introduce nginx setup nginx: re-added buildall: cleanup startstop: cleanup --- build.profile | 4 +- buildall.sh | 74 ++++++++++++------- ...-fonts-ttf.build.sh => libfontttf.build.sh | 0 nginx.conf.build.sh | 6 +- profile.build.sh | 1 + startstop.build.sh | 62 +++++++++++----- 6 files changed, 96 insertions(+), 51 deletions(-) rename liberation-fonts-ttf.build.sh => libfontttf.build.sh (100%) diff --git a/build.profile b/build.profile index 3676fd3..6f51ead 100644 --- a/build.profile +++ b/build.profile @@ -46,8 +46,8 @@ BUILD_leptonica=leptonica-1.85.0 BUILD_leptonica_SRCURL="https://beehive.molgen.mpg.de/1732e999e2fef8721348edc2b7283224/${BUILD_leptonica}.tar.gz" # SRCURL="https://nginx.org/download/nginx-1.27.3.tar.gz" -#BUILD_nginx=nginx-1.27.3 -#BUILD_nginx_SRCURL="https://beehive.molgen.mpg.de/89773c781ff0c2cd876b03bbc094c258/${BUILD_nginx}.tar.gz" +BUILD_nginx=nginx-1.27.3 +BUILD_nginx_SRCURL="https://beehive.molgen.mpg.de/89773c781ff0c2cd876b03bbc094c258/${BUILD_nginx}.tar.gz" # SRCURL="https://github.com/liberationfonts/liberation-fonts/files/7261482/liberation-fonts-ttf-2.1.5.tar.gz" BUILD_libfontttf=liberation-fonts-ttf-2.1.5 diff --git a/buildall.sh b/buildall.sh index 9aa44db..385260e 100755 --- a/buildall.sh +++ b/buildall.sh @@ -22,8 +22,9 @@ PAPERLESS_WEBSERVER_WORKERS=5 PAPERLESS_URL=https://localhost PAPERLESS_CSRF_TRUSTED_ORIGINS=http://localhost:\${PAPERLESS_PORT} -# SSL_CERTIFICATE=\${PROJECT}/certs/fullchain.pem -# SSL_CERTIFICATE_KEY=\${PROJECT}/certs/privkey.pem +NGINX_SSL_DHPARAM=\${PROJECT}/certs/dhparam.pem +NGINX_SSL_CERTIFICATE=\${PROJECT}/certs/fullchain.pem +NGINX_SSL_CERTIFICATE_KEY=\${PROJECT}/certs/privkey.pem _EOP_ ) ${BUILD_LOCAL} @@ -66,33 +67,52 @@ ln -fs "${logfile}" "build.log" popd { -[ -d "${BUILD_DIR}/${BUILD_sqlite}" ] || ./sqlite.build.sh -[ -d "${BUILD_DIR}/${BUILD_redis}" ] || ./redis.build.sh -[ -d "${BUILD_DIR}/${BUILD_python}" ] || ./python.build.sh -[ -d "${BUILD_DIR}/${BUILD_node}" ] || ./node.build.sh -[ -d "${BUILD_DIR}/${BUILD_libjpegturbo}" ] || ./libjpegturbo.build.sh -[ -d "${BUILD_DIR}/${BUILD_libtiff}" ] || ./libtiff.build.sh -[ -d "${BUILD_DIR}/${BUILD_libwebp}" ] || ./libwebp.build.sh -[ -d "${BUILD_DIR}/${BUILD_ffmpeg}" ] || ./ffmpeg.build.sh - -[ -d "${BUILD_DIR}/${BUILD_jbig2dec}" ] || ./jbig2dec.build.sh -[ -d "${BUILD_DIR}/${BUILD_ghostscript}" ] || ./ghostscript.build.sh - -[ -d "${BUILD_DIR}/${BUILD_imagemagick}" ] || ./imagemagick.build.sh -[ -d "${BUILD_DIR}/${BUILD_leptonica}" ] || ./leptonica.build.sh -[ -d "${BUILD_DIR}/${BUILD_jbig2enc}" ] || ./jbig2enc.build.sh -[ -d "${BUILD_DIR}/${BUILD_tesseract}" ] || ./tesseract.build.sh - -[ -d "${BUILD_DIR}/${BUILD_pngquant}" ] || ./pngquant.build.sh -[ -d "${BUILD_DIR}/${BUILD_unpaper}" ] || ./unpaper.build.sh -[ -d "${BUILD_DIR}/${BUILD_zbar}" ] || ./zbar.build.sh -[ -d "${BUILD_DIR}/${BUILD_libfontttf}" ] || ./liberation-fonts-ttf.build.sh -#[ -d "${BUILD_DIR}/${BUILD_nginx}" ] || ./nginx.build.sh -[ -d "${BUILD_DIR}/${BUILD_qpdf}" ] || ./qpdf.build.sh -# ./nginx.conf.build.sh +_pkgs=( + sqlite + redis + python + node + libjpegturbo + libtiff + libwebp + ffmpeg + jbig2dec + ghostscript + imagemagick + leptonica + jbig2enc + tesseract + pngquant + unpaper + zbar + libfontttf + qpdf + nginx +) + +build() { + local _pkg=$1 + eval "_tag=${BUILD_DIR}/\$BUILD_${_pkg}/.built" + if [[ ! -e "$_tag" ]]; then + ./${_pkg}.build.sh 2>&1 | tee "${LOGS}/${_pkg}.log"; + r=${PIPESTATUS[0]} + if [[ $r == 0 ]]; then + touch "$_tag" + else + echo "${_pkg}.build.sh failed exit code $r" + exit + fi + fi +} + +for _pkg in "${_pkgs[@]}"; do + build "$_pkg" +done + ./profile.build.sh ./startstop.build.sh -[ -d "$PROJECT/${BUILD_paperless}" ] || ./paperless-ngx.build.sh +./nginx.conf.build.sh +./paperless-ngx.build.sh } 2>&1 | tee "${LOGS}/${logfile}" 2>&1 diff --git a/liberation-fonts-ttf.build.sh b/libfontttf.build.sh similarity index 100% rename from liberation-fonts-ttf.build.sh rename to libfontttf.build.sh diff --git a/nginx.conf.build.sh b/nginx.conf.build.sh index a0ee91e..5263845 100755 --- a/nginx.conf.build.sh +++ b/nginx.conf.build.sh @@ -21,12 +21,12 @@ http { client_max_body_size 100M; keepalive_timeout 65; - upstream gunicorn_socket { + upstream granian_socket { # fail_timeout=0 means we always retry an upstream even if it failed # to return a good HTTP response (in case the Unicorn master nukes a # single worker for timing out). - server unix:${DEVSHM}/gunicorn.sock fail_timeout=0; + server unix:${DEVSHM}/granian.sock fail_timeout=0; } server { @@ -65,7 +65,7 @@ tee -a ${CONF} <<_EOP_ error_page 400 404 405 502 =200 /chicken.html; location / { - proxy_pass http://gunicorn_socket/; + proxy_pass http://granian_socket/; proxy_http_version 1.1; proxy_set_header Upgrade \$http_upgrade; proxy_set_header Connection "upgrade"; diff --git a/profile.build.sh b/profile.build.sh index 263a68e..51d4fd3 100755 --- a/profile.build.sh +++ b/profile.build.sh @@ -19,6 +19,7 @@ PATH=$HOME/bin:${PROJECT}/bin:/bin:/sbin:/usr/bin:/usr/sbin . ${PROJECT}/${BUILD_python}/profile . ${PROJECT}/${BUILD_node}/profile . ${PROJECT}/${BUILD_redis}/profile +. ${PROJECT}/${BUILD_nginx}/profile . ${PROJECT}/${BUILD_ghostscript}/profile . ${PROJECT}/${BUILD_imagemagick}/profile . ${PROJECT}/${BUILD_unpaper}/profile diff --git a/startstop.build.sh b/startstop.build.sh index 72b802e..ac7a39f 100755 --- a/startstop.build.sh +++ b/startstop.build.sh @@ -27,6 +27,31 @@ function pwait() { fi } +#### nginx #### + +srv_nginx_start() { + nginx -t + srv_granian_waitup + trap rm_pidfiles EXIT + nginx -g 'daemon off;' +} + +srv_nginx_restart() { + nginx -s reload +} + +srv_nginx_stop() { + nginx -s quit || rm -fv "${DEVSHM}"/nginx.pid +} + +srv_nginx_waitup() { + until ( printf "" 2>>/dev/null >>/dev/tcp/${PAPERLESS_BIND_ADDR}/${PAPERLESS_PORT} ); do + echo "waiting for nginx to be ready..." + sleep 1 + done +} + + #### redis #### srv_redis_start() { @@ -46,14 +71,7 @@ srv_granian_start() { cd "${PROJECT}/paperless-ngx/src" rm -fv "${DEVSHM}/granian.pid" _opts=( -# no unix domain socket support in granian -# https://github.com/emmett-framework/granian/issues/97 -# https://github.com/paperless-ngx/paperless-ngx/discussions/9941 -# --host "unix:${DEVSHM}/granian.sock" - --host ${PAPERLESS_BIND_ADDR} - --port ${PAPERLESS_PORT} - ${SSL_CERTIFICATE:+--ssl-certificate ${SSL_CERTIFICATE}} - ${SSL_CERTIFICATE_KEY:+--ssl-keyfile ${SSL_CERTIFICATE_KEY}} + --uds "${DEVSHM}/granian.sock" --workers ${PAPERLESS_WEBSERVER_WORKERS:-1} --interface asginl --pid-file "${DEVSHM}/granian.pid" @@ -71,7 +89,7 @@ srv_granian_start() { } srv_granian_waitup() { - until ( printf "" 2>>/dev/null >>/dev/tcp/${PAPERLESS_BIND_ADDR}/${PAPERLESS_PORT} ); do + until ( [[ -e ${DEVSHM}/granian.sock ]] ); do echo "waiting for granian server to be ready..." sleep 1 done @@ -84,7 +102,7 @@ srv_consumer_start() { trap rm_pidfiles EXIT cd "${PROJECT}/paperless-ngx/src" - srv_granian_waitup + srv_nginx_waitup ./manage.py document_consumer } @@ -96,7 +114,7 @@ srv_scheduler_start() { trap rm_pidfiles EXIT cd "${PROJECT}/paperless-ngx/src" - srv_granian_waitup + srv_nginx_waitup celery --app paperless beat --loglevel INFO } @@ -108,7 +126,7 @@ srv_worker_start() { trap rm_pidfiles EXIT cd "${PROJECT}/paperless-ngx/src" - srv_granian_waitup + srv_nginx_waitup celery --app paperless worker --loglevel WARNING } @@ -198,17 +216,23 @@ srv_generic_stop() { echo "killing process group \${pgid}" KT=\${SECONDS} kill -- -\${pgid}||: - echo "waiting for process group \${pgid} to die" + printf "waiting for pgid %5d (%s) to die\n" \${pgid} \${srv} pwait --pgroup \${pgid}||: + while kill -0 -"\${pgid}" > /dev/null 2>&1; do + sleep 1 + done + rm -fv "\${PGIDFILE}" + rm -fv "\${PIDFILE}" KT=\$((\${SECONDS}-\${KT})) [[ \${KT} > 1 ]] && echo "kill took \${KT} seconds" else if [[ -s "\${PIDFILE}" ]]; then local pid read -a pid < "\${PIDFILE}"||: - kill -- \${pid} - echo "waiting for process \${pid} to die" + kill -- \${pid} ||: + printf "waiting for pid %5d (%s) to die\n" \${pid} \${srv} tail -f --pid=\${pid} /dev/null + rm -fv "\${PIDFILE}" else echo "# no pid file found: \${PGIDFILE}" fi @@ -228,7 +252,7 @@ cmd="\${1:-help}" srv="\${2:-all}" if [[ \${srv} = 'all' ]]; then - _srv=(redis granian consumer scheduler worker) + _srv=(redis granian nginx consumer scheduler worker) if [[ \${cmd} = 'start' ]]; then srv_migrate @@ -241,7 +265,7 @@ if [[ \${srv} = 'all' ]]; then for srv in \${_srv[@]}; do # echo "### ./startstop.sh \${cmd} \${srv}" - "${PROJECT}"/startstop.sh \${cmd} \${srv} || echo "RET: $?" + "${PROJECT}"/startstop.sh \${cmd} \${srv} ||: done exit fi @@ -264,11 +288,11 @@ if [[ \$(type -t srv_\${srv}_\${cmd}) = 'function' ]]; then srv_\${srv}_\${cmd}; } > ${LOGDIR}/srv-\${srv}.log 2>&1 echo "\${COPROC_PID}" >> "\${PIDFILE}" - ps opgid= "\${COPROC_PID}" > "\${PGIDFILE}" + ps -o pgid= -p "\${COPROC_PID}" > "\${PGIDFILE}" exit ;; *) - echo "########## default fallback to srv_\${srv}_\${cmd} ###############" + echo "########## srv_\${srv}_\${cmd}() ###############" srv_\${srv}_\${cmd} exit ;;