From 4eb524e24a11756f2485bfacd844c89c004db627 Mon Sep 17 00:00:00 2001 From: Peter Marquardt Date: Wed, 11 Jun 2025 15:32:32 +0200 Subject: [PATCH] update to 2.16.3 - paperless: 2.16.3 - nodejs: 22.15.1 to 22.16.0 - sqlite: 3.49.2 to 3.50.1 - tesseract: 5.5.0 to 5.5.1 - python: 3.12.10 to 3.12.11 - libjpegturbo: 3.1.0 to 3.1.1 - tesseract: 5.1.0 to 5.1.1 - startstop: kill granian - granian: kill timeout 1 sec - granian: force HTTP/1 due to CSRF - paperless-ngx: add management commands --- README.todo | 14 ------------ build.profile | 32 ++++++++++++++------------- paperless-ngx.build.sh | 46 +++++++++++++++++++++++--------------- profile.build.sh | 5 ++++- startstop.build.sh | 50 +++++++++++------------------------------- 5 files changed, 62 insertions(+), 85 deletions(-) diff --git a/README.todo b/README.todo index eb8730e..6b3f919 100644 --- a/README.todo +++ b/README.todo @@ -9,20 +9,6 @@ dot REAME.dot ☐ gs ? GS_VERSION=10.03.1 - ☐ SSL für granian - - ☐ paperless-ngx - - ☐ # Set Python environment variables - ENV PYTHONDONTWRITEBYTECODE=1 \ - PYTHONUNBUFFERED=1 \ - # Ignore warning from Whitenoise about async iterators - PYTHONWARNINGS="ignore:::django.http.response:517" \ - PNGX_CONTAINERIZED=1 \ - # https://docs.astral.sh/uv/reference/settings/#link-mode - UV_LINK_MODE=copy \ - UV_CACHE_DIR=/cache/uv/ - ☐ install management commands docker/install_management_commands.sh diff --git a/build.profile b/build.profile index 09d93f2..3866183 100644 --- a/build.profile +++ b/build.profile @@ -1,6 +1,6 @@ -BUILD_TAG=2.16.1 +BUILD_TAG=2.16.3 -BUILD_paperless=paperless-ngx-2.16.1 +BUILD_paperless=paperless-ngx-2.16.3 BUILD_DIR=build BUILD_SRC="${PWD}" @@ -16,22 +16,24 @@ TMPDIR=${TMPDIR:-/tmp/${USER}/${BUILD_TAG}} LOGDIR=${PROJECT}/log # https://sqlite.org -# wget "https://sqlite.org/2025/sqlite-autoconf-3490200.tar.gz" -BUILD_sqlite=sqlite-3.49.2 -BUILD_sqlite_SRCURL="https://beehive.molgen.mpg.de/46ef8fec4c97ec77ab27659ad27b28b0/sqlite-autoconf-3490200.tar.gz" +# wget "https://sqlite.org/2025/sqlite-autoconf-3500100.tar.gz" +BUILD_sqlite=sqlite-3.50.1 +BUILD_sqlite_SRCURL="https://beehive.molgen.mpg.de/45508b1aeb0a946574d3385249c5fb58/sqlite-autoconf-3500100.tar.gz" # https://download.redis.io/releases # community edition ! # wget https://download.redis.io/releases/redis-7.4.1.tar.gz BUILD_redis=redis-7.4.1 BUILD_redis_SRCURL="https://beehive.molgen.mpg.de/360809807c15da120affb356e55c6388/${BUILD_redis}.tar.gz" -# wget "https://www.python.org/ftp/python/3.12.10/Python-3.12.10.tgz" -BUILD_python=Python-3.12.10 -BUILD_python_SRCURL="https://beehive.molgen.mpg.de/35c03f014408e26e2b06d576c19cac54/${BUILD_python}.tgz" +# wget "https://www.python.org/ftp/python/3.12.11/Python-3.12.11.tgz" +BUILD_python=Python-3.12.11 +BUILD_python_SRCURL="https://beehive.molgen.mpg.de/45bda920329568dd6650b0ac556d17db/${BUILD_python}.tgz" -# wget "https://nodejs.org/download/release/v22.15.1/node-v22.15.1.tar.gz" -BUILD_node=node-22.15.1 -BUILD_node_SRCURL="https://beehive.molgen.mpg.de/6ee1e546973a07d9a80767c009399d23/${BUILD_node/-/-v}.tar.gz" +# wget "https://nodejs.org/download/release/v22.16.0/node-v22.16.0.tar.gz" +BUILD_node=node-22.16.0 +BUILD_node_SRCURL="https://beehive.molgen.mpg.de/ffc2c87efa22a9dbb794810f0a952118/${BUILD_node/-/-v}.tar.gz" + +SRCURL[0]="https://beehive.molgen.mpg.de/ffc2c87efa22a9dbb794810f0a952118/node-v22.16.0.tar.gz" # SRCURL="https://ffmpeg.org//releases/${BUILD_ffmpeg}.tar.gz" BUILD_ffmpeg=ffmpeg-7.1 @@ -57,9 +59,9 @@ BUILD_libfontttf_SRCURL="https://beehive.molgen.mpg.de/31b453e0b77bacde410a34a72 BUILD_libtiff=tiff-4.7.0 BUILD_libtiff_SRCURL="https://beehive.molgen.mpg.de/3a0fa4a270a4a192b08913f88d0cfbdd/${BUILD_libtiff}.tar.gz" -# SRCURL="https://github.com/libjpeg-turbo/libjpeg-turbo/releases/download/3.1.0/libjpeg-turbo-3.1.0.tar.gz" -BUILD_libjpegturbo=libjpegturbo-3.1.0 -BUILD_libjpegturbo_SRCURL="https://beehive.molgen.mpg.de/ed3fb4bb4cf794898f11a6d30c54b479/libjpeg-turbo-3.1.0.tar.gz" +# SRCURL="https://github.com/libjpeg-turbo/libjpeg-turbo/releases/download/3.1.1/libjpeg-turbo-3.1.1.tar.gz" +BUILD_libjpegturbo=libjpegturbo-3.1.1 +BUILD_libjpegturbo_SRCURL="https://beehive.molgen.mpg.de/e71063e0b043e38207c8cefda6931cbd/libjpeg-turbo-3.1.1.tar.gz" # SRCURL="https://storage.googleapis.com/downloads.webmproject.org/releases/webp/libwebp-1.5.0.tar.gz" BUILD_libwebp=libwebp-1.5.0 @@ -75,7 +77,7 @@ BUILD_pngquant_SRCURL="https://github.com/kornelski/pngquant.git" BUILD_unpaper=unpaper-7.0.0-127-g5bcef8a BUILD_unpaper_SRCURL="https://github.com/unpaper/unpaper.git" -BUILD_tesseract=tesseract-5.5.0 +BUILD_tesseract=tesseract-5.5.1 BUILD_tesseract_SRCURL="https://github.com/tesseract-ocr/tesseract.git" BUILD_tessdata=tessdata_best-4.1.0 diff --git a/paperless-ngx.build.sh b/paperless-ngx.build.sh index 803d138..a5ea01e 100755 --- a/paperless-ngx.build.sh +++ b/paperless-ngx.build.sh @@ -2,6 +2,7 @@ { set -x set -e +set -u . build.profile @@ -46,10 +47,6 @@ else popd fi -#pushd ${BUILD_PKG} -#patch -p1 --silent --backup --forward --input=${BUILD_SRC}/paperless-ngx.asgi.patch -#popd - # we need paperless-ngx to be extracted since it reads paperless.conf.example # what a mess 8-( pushd ${BUILD_SRC} @@ -62,6 +59,7 @@ mkdir -vp data mkdir -vp consume mkdir -vp media mkdir -vp log +mkdir -vp bin popd @@ -73,9 +71,6 @@ pip install uv pipenv install -# pipenv requirements > requirements.txt -# pip install -r requirements.txt - pushd src-ui npm update -g pnpm npm install -g corepack@latest @@ -83,17 +78,6 @@ corepack enable # https://github.com/nodejs/corepack/issues/550 COREPACK_ENABLE_DOWNLOAD_PROMPT=0 pnpm install --no-color -#npm update npm -g --no-fund -#ulimit -n 512 - -#set +e -#while true; do -# npm ci --no-fund -# r=$? -# [ $r != 232 ] && [ $r != 146 ] && break -#done -#set -e - ./node_modules/.bin/ng build --configuration production popd @@ -116,6 +100,32 @@ PAPERLESS_ADMIN_MAIL=${PAPERLESS_ADMIN_MAIL:-admin@localhost.lan} ./manage.py shell -c "from django.contrib.auth.models import User; User.objects.create_superuser('${PAPERLESS_ADMIN_USER}', '${PAPERLESS_ADMIN_MAIL}', '${PAPERLESS_ADMIN_PASSWORD}')" || true +for command in decrypt_documents \ + document_archiver \ + document_exporter \ + document_importer \ + mail_fetcher \ + document_create_classifier \ + document_index \ + document_renamer \ + document_retagger \ + document_thumbnails \ + document_sanity_checker \ + document_fuzzy_match \ + manage_superuser \ + convert_mariadb_uuid \ + prune_audit_logs \ + createsuperuser; +do +tee ${PROJECT}/bin/${command} <<_EOSH_ +#!/bin/bash +. ${PROJECT}/profile +cd ${PREFIX}/src +python3 manage.py ${command} "$@" +_EOSH_ +chmod u=rwx,g=rwx,o=rx ${PROJECT}/bin/${command} +done + popd echo "Installing NLTK data" diff --git a/profile.build.sh b/profile.build.sh index d6397a8..a3912d5 100755 --- a/profile.build.sh +++ b/profile.build.sh @@ -10,7 +10,7 @@ tee ${PROJECT}/profile <<_EOP_ set -a BUILD_TAG=${BUILD_TAG} -PATH=$HOME/bin:/bin:/sbin:/usr/bin:/usr/sbin +PATH=$HOME/bin:${PROJECT}/bin:/bin:/sbin:/usr/bin:/usr/sbin . ${PROJECT}/${BUILD_zbar}/profile . ${PROJECT}/${BUILD_jbig2enc}/profile @@ -39,4 +39,7 @@ TIME_ZONE=Europe/Berlin PAPERLESS_BIND_ADDR=${PAPERLESS_BIND_ADDR} +PYTHONDONTWRITEBYTECODE=1 +PYTHONUNBUFFERED=1 + _EOP_ diff --git a/startstop.build.sh b/startstop.build.sh index 28f36a5..dd73dc5 100755 --- a/startstop.build.sh +++ b/startstop.build.sh @@ -62,6 +62,7 @@ srv_gunicorn_start() { cd "${PROJECT}/paperless-ngx/src" + _opts=( --config ../gunicorn.conf.py paperless.asgi:application @@ -72,13 +73,13 @@ srv_gunicorn_start() { --worker-tmp-dir "${DEVSHM}" ) trap rm_pidfiles EXIT + gunicorn "\${_opts[@]}" } #### granian #### srv_granian_start() { -set -x cd "${PROJECT}/paperless-ngx/src" rm -fv "${DEVSHM}/granian.pid" _opts=( @@ -96,42 +97,14 @@ set -x --ws --loop uvloop --access-log + --workers-kill-timeout 1 + --http 1 paperless.asgi:application ) trap rm_pidfiles EXIT - granian "\${_opts[@]}" -} -srv_granian_stop() { - if [[ -s "${DEVSHM}/granian.pid" ]]; then - local pid - # ||: since maybe due to missing \n we get some kind of pipefail ? - read -a pid < "${DEVSHM}/granian.pid" ||: - kill -- \${pid} - echo "waiting for process \${pid} to die" - tail -f --pid=\${pid} /dev/null - else - echo "# no pid file found: ${DEVSHM}/granian.pid" - fi - if [[ -s "\${PGIDFILE}" ]]; then - local pgid - read -a pgid < "\${PGIDFILE}" ||: - echo "killing process group \${pgid}" - kill -- -\${pgid} - echo "waiting for process group \${pgid} to die" - pwait --echo --pgroup \${pgid} - else - if [[ -s "\${PIDFILE}" ]]; then - local pid - read -a pid < "\${PIDFILE}" ||: - kill -- \${pid} - echo "waiting for process \${pid} to die" - tail -f --pid=\${pid} /dev/null - else - echo "# no pid file found: \${PGIDFILE}" - fi - fi - rm -fv "${DEVSHM}/granian.pid" + export PYTHONWARNINGS="ignore:::django.http.response:517" + granian "\${_opts[@]}" } #### consumer #### @@ -245,15 +218,18 @@ srv_generic_okfail() { srv_generic_stop() { if [[ -s "\${PGIDFILE}" ]]; then local pgid - read -a pgid < "\${PGIDFILE}" + read -a pgid < "\${PGIDFILE}"||: echo "killing process group \${pgid}" - kill -- -\${pgid} + KT=\${SECONDS} + kill -- -\${pgid}||: echo "waiting for process group \${pgid} to die" - pwait --pgroup \${pgid} + pwait --pgroup \${pgid}||: + KT=\$((\${SECONDS}-\${KT})) + [[ \${KT} > 1 ]] && echo "kill took \${KT} seconds" else if [[ -s "\${PIDFILE}" ]]; then local pid - read -a pid < "\${PIDFILE}" + read -a pid < "\${PIDFILE}"||: kill -- \${pid} echo "waiting for process \${pid} to die" tail -f --pid=\${pid} /dev/null