update to 2.16.3

- paperless: 2.16.3
- nodejs: 22.15.1 to 22.16.0
- sqlite: 3.49.2 to 3.50.1
- tesseract: 5.5.0 to 5.5.1
- python: 3.12.10 to 3.12.11
- libjpegturbo: 3.1.0 to 3.1.1
- tesseract: 5.1.0 to 5.1.1
- startstop: kill granian
- granian: kill timeout 1 sec
- granian: force HTTP/1 due to CSRF
- paperless-ngx: add management commands

README.todo

@@ -9,20 +9,6 @@ dot REAME.dot
  ☐ gs ?
      GS_VERSION=10.03.1
 
- ☐ SSL für granian
-
- ☐ paperless-ngx
-
- ☐ # Set Python environment variables
-  ENV PYTHONDONTWRITEBYTECODE=1 \
-    PYTHONUNBUFFERED=1 \
-    # Ignore warning from Whitenoise about async iterators
-    PYTHONWARNINGS="ignore:::django.http.response:517" \
-    PNGX_CONTAINERIZED=1 \
-    # https://docs.astral.sh/uv/reference/settings/#link-mode
-    UV_LINK_MODE=copy \
-    UV_CACHE_DIR=/cache/uv/
-
  ☐ install management commands
    docker/install_management_commands.sh
 

build.profile

@@ -1,6 +1,6 @@
-BUILD_TAG=2.16.1
+BUILD_TAG=2.16.3
 
-BUILD_paperless=paperless-ngx-2.16.1
+BUILD_paperless=paperless-ngx-2.16.3
 
 BUILD_DIR=build
 BUILD_SRC="${PWD}"
@@ -16,22 +16,24 @@ TMPDIR=${TMPDIR:-/tmp/${USER}/${BUILD_TAG}}
 LOGDIR=${PROJECT}/log
 
 # https://sqlite.org
-# wget "https://sqlite.org/2025/sqlite-autoconf-3490200.tar.gz"
-BUILD_sqlite=sqlite-3.49.2
-BUILD_sqlite_SRCURL="https://beehive.molgen.mpg.de/46ef8fec4c97ec77ab27659ad27b28b0/sqlite-autoconf-3490200.tar.gz"
+# wget "https://sqlite.org/2025/sqlite-autoconf-3500100.tar.gz"
+BUILD_sqlite=sqlite-3.50.1
+BUILD_sqlite_SRCURL="https://beehive.molgen.mpg.de/45508b1aeb0a946574d3385249c5fb58/sqlite-autoconf-3500100.tar.gz"
 
 # https://download.redis.io/releases   # community edition !
 # wget https://download.redis.io/releases/redis-7.4.1.tar.gz
 BUILD_redis=redis-7.4.1
 BUILD_redis_SRCURL="https://beehive.molgen.mpg.de/360809807c15da120affb356e55c6388/${BUILD_redis}.tar.gz"
 
-# wget "https://www.python.org/ftp/python/3.12.10/Python-3.12.10.tgz"
-BUILD_python=Python-3.12.10
-BUILD_python_SRCURL="https://beehive.molgen.mpg.de/35c03f014408e26e2b06d576c19cac54/${BUILD_python}.tgz"
+# wget "https://www.python.org/ftp/python/3.12.11/Python-3.12.11.tgz"
+BUILD_python=Python-3.12.11
+BUILD_python_SRCURL="https://beehive.molgen.mpg.de/45bda920329568dd6650b0ac556d17db/${BUILD_python}.tgz"
 
-# wget "https://nodejs.org/download/release/v22.15.1/node-v22.15.1.tar.gz"
-BUILD_node=node-22.15.1
-BUILD_node_SRCURL="https://beehive.molgen.mpg.de/6ee1e546973a07d9a80767c009399d23/${BUILD_node/-/-v}.tar.gz"
+# wget "https://nodejs.org/download/release/v22.16.0/node-v22.16.0.tar.gz"
+BUILD_node=node-22.16.0
+BUILD_node_SRCURL="https://beehive.molgen.mpg.de/ffc2c87efa22a9dbb794810f0a952118/${BUILD_node/-/-v}.tar.gz"
+
+SRCURL[0]="https://beehive.molgen.mpg.de/ffc2c87efa22a9dbb794810f0a952118/node-v22.16.0.tar.gz"
 
 # SRCURL="https://ffmpeg.org//releases/${BUILD_ffmpeg}.tar.gz"
 BUILD_ffmpeg=ffmpeg-7.1
@@ -57,9 +59,9 @@ BUILD_libfontttf_SRCURL="https://beehive.molgen.mpg.de/31b453e0b77bacde410a34a72
 BUILD_libtiff=tiff-4.7.0
 BUILD_libtiff_SRCURL="https://beehive.molgen.mpg.de/3a0fa4a270a4a192b08913f88d0cfbdd/${BUILD_libtiff}.tar.gz"
 
-# SRCURL="https://github.com/libjpeg-turbo/libjpeg-turbo/releases/download/3.1.0/libjpeg-turbo-3.1.0.tar.gz"
-BUILD_libjpegturbo=libjpegturbo-3.1.0
-BUILD_libjpegturbo_SRCURL="https://beehive.molgen.mpg.de/ed3fb4bb4cf794898f11a6d30c54b479/libjpeg-turbo-3.1.0.tar.gz"
+# SRCURL="https://github.com/libjpeg-turbo/libjpeg-turbo/releases/download/3.1.1/libjpeg-turbo-3.1.1.tar.gz"
+BUILD_libjpegturbo=libjpegturbo-3.1.1
+BUILD_libjpegturbo_SRCURL="https://beehive.molgen.mpg.de/e71063e0b043e38207c8cefda6931cbd/libjpeg-turbo-3.1.1.tar.gz"
 
 # SRCURL="https://storage.googleapis.com/downloads.webmproject.org/releases/webp/libwebp-1.5.0.tar.gz"
 BUILD_libwebp=libwebp-1.5.0
@@ -75,7 +77,7 @@ BUILD_pngquant_SRCURL="https://github.com/kornelski/pngquant.git"
 BUILD_unpaper=unpaper-7.0.0-127-g5bcef8a
 BUILD_unpaper_SRCURL="https://github.com/unpaper/unpaper.git"
 
-BUILD_tesseract=tesseract-5.5.0
+BUILD_tesseract=tesseract-5.5.1
 BUILD_tesseract_SRCURL="https://github.com/tesseract-ocr/tesseract.git"
 
 BUILD_tessdata=tessdata_best-4.1.0

paperless-ngx.build.sh

@@ -2,6 +2,7 @@
 {
 set -x
 set -e
+set -u
 
 . build.profile
 
@@ -46,10 +47,6 @@ else
     popd
 fi
 
-#pushd ${BUILD_PKG}
-#patch -p1 --silent --backup --forward --input=${BUILD_SRC}/paperless-ngx.asgi.patch
-#popd
-
 # we need paperless-ngx to be extracted since it reads paperless.conf.example
 # what a mess 8-(
 pushd ${BUILD_SRC}
@@ -62,6 +59,7 @@ mkdir -vp data
 mkdir -vp consume
 mkdir -vp media
 mkdir -vp log
+mkdir -vp bin
 
 popd
 
@@ -73,27 +71,13 @@ pip install uv
 
 pipenv install
 
-# pipenv requirements > requirements.txt
-# pip install -r requirements.txt
-
 pushd src-ui
 npm update -g pnpm
 npm install -g corepack@latest
 corepack enable
 # https://github.com/nodejs/corepack/issues/550
 COREPACK_ENABLE_DOWNLOAD_PROMPT=0 pnpm install --no-color
 
-#npm update npm -g --no-fund
-#ulimit -n 512
-
-#set +e
-#while true; do
-#  npm ci --no-fund
-#  r=$?
-#  [ $r != 232 ] && [ $r != 146 ] && break
-#done
-#set -e
-
 ./node_modules/.bin/ng build --configuration production
 
 popd
@@ -116,6 +100,32 @@ PAPERLESS_ADMIN_MAIL=${PAPERLESS_ADMIN_MAIL:-admin@localhost.lan}
 
 ./manage.py shell -c "from django.contrib.auth.models import User; User.objects.create_superuser('${PAPERLESS_ADMIN_USER}', '${PAPERLESS_ADMIN_MAIL}', '${PAPERLESS_ADMIN_PASSWORD}')" || true
 
+for command in decrypt_documents \
+        document_archiver \
+        document_exporter \
+        document_importer \
+        mail_fetcher \
+        document_create_classifier \
+        document_index \
+        document_renamer \
+        document_retagger \
+        document_thumbnails \
+        document_sanity_checker \
+        document_fuzzy_match \
+        manage_superuser \
+        convert_mariadb_uuid \
+        prune_audit_logs \
+        createsuperuser;
+do
+tee ${PROJECT}/bin/${command} <<_EOSH_
+#!/bin/bash
+. ${PROJECT}/profile
+cd ${PREFIX}/src
+python3 manage.py ${command} "$@"
+_EOSH_
+chmod u=rwx,g=rwx,o=rx ${PROJECT}/bin/${command}
+done
+
 popd
 
 echo "Installing NLTK data"

profile.build.sh

@@ -10,7 +10,7 @@ tee ${PROJECT}/profile <<_EOP_
 set -a
 BUILD_TAG=${BUILD_TAG}
 
-PATH=$HOME/bin:/bin:/sbin:/usr/bin:/usr/sbin
+PATH=$HOME/bin:${PROJECT}/bin:/bin:/sbin:/usr/bin:/usr/sbin
 
 . ${PROJECT}/${BUILD_zbar}/profile
 . ${PROJECT}/${BUILD_jbig2enc}/profile
@@ -39,4 +39,7 @@ TIME_ZONE=Europe/Berlin
 
 PAPERLESS_BIND_ADDR=${PAPERLESS_BIND_ADDR}
 
+PYTHONDONTWRITEBYTECODE=1
+PYTHONUNBUFFERED=1
+
 _EOP_

startstop.build.sh

@@ -62,6 +62,7 @@ srv_gunicorn_start() {
 
     cd "${PROJECT}/paperless-ngx/src"
 
+
     _opts=(
       --config ../gunicorn.conf.py
       paperless.asgi:application
@@ -72,13 +73,13 @@ srv_gunicorn_start() {
       --worker-tmp-dir "${DEVSHM}"
     )
     trap rm_pidfiles EXIT
+
     gunicorn "\${_opts[@]}"
 }
 
 #### granian ####
 
 srv_granian_start() {
-set -x
     cd "${PROJECT}/paperless-ngx/src"
     rm -fv "${DEVSHM}/granian.pid"
     _opts=(
@@ -96,42 +97,14 @@ set -x
       --ws
       --loop uvloop
       --access-log
+      --workers-kill-timeout 1
+      --http 1
       paperless.asgi:application
     )
     trap rm_pidfiles EXIT
-    granian "\${_opts[@]}"
-}
 
-srv_granian_stop() {
-  if [[ -s "${DEVSHM}/granian.pid" ]]; then
-    local pid
-    # ||: since maybe due to missing \n we get some kind of pipefail ?
-    read -a pid < "${DEVSHM}/granian.pid" ||:
-    kill -- \${pid}
-    echo "waiting for process \${pid} to die"
-    tail -f --pid=\${pid} /dev/null
-  else
-    echo "# no pid file found: ${DEVSHM}/granian.pid"
-  fi
-  if [[ -s "\${PGIDFILE}" ]]; then
-    local pgid
-    read -a pgid < "\${PGIDFILE}" ||:
-    echo "killing process group \${pgid}"
-    kill -- -\${pgid}
-    echo "waiting for process group \${pgid} to die"
-    pwait --echo --pgroup \${pgid}
-  else
-    if [[ -s "\${PIDFILE}" ]]; then
-      local pid
-      read -a pid < "\${PIDFILE}" ||:
-      kill -- \${pid}
-      echo "waiting for process \${pid} to die"
-      tail -f --pid=\${pid} /dev/null
-    else
-      echo "# no pid file found: \${PGIDFILE}"
-    fi
-  fi
-  rm -fv "${DEVSHM}/granian.pid"
+    export PYTHONWARNINGS="ignore:::django.http.response:517"
+    granian "\${_opts[@]}"
 }
 
 #### consumer ####
@@ -245,15 +218,18 @@ srv_generic_okfail() {
 srv_generic_stop() {
   if [[ -s "\${PGIDFILE}" ]]; then
     local pgid
-    read -a pgid < "\${PGIDFILE}"
+    read -a pgid < "\${PGIDFILE}"||:
     echo "killing process group \${pgid}"
-    kill -- -\${pgid}
+    KT=\${SECONDS}
+    kill -- -\${pgid}||:
     echo "waiting for process group \${pgid} to die"
-    pwait --pgroup \${pgid}
+    pwait --pgroup \${pgid}||:
+    KT=\$((\${SECONDS}-\${KT}))
+    [[ \${KT} > 1 ]] && echo "kill took \${KT} seconds"
   else
     if [[ -s "\${PIDFILE}" ]]; then
       local pid
-      read -a pid < "\${PIDFILE}"
+      read -a pid < "\${PIDFILE}"||:
       kill -- \${pid}
       echo "waiting for process \${pid} to die"
       tail -f --pid=\${pid} /dev/null