Merge pull request #12 from mariux64/update-2.5.3

Update 2.5.3

.gitignore

@@ -1,6 +1,7 @@
 /build.local
 /ffmpeg-*
 /jbig2enc-*
+/jbig2dec-*
 /leptonica-*
 /logs
 /nginx-*

Python.build.sh

@@ -5,34 +5,34 @@ set -e
 
 . build.profile
 
-. ${PREFIX}/${BUILD_sqlite}/profile
+. "${PREFIX}/${BUILD_sqlite}/profile"
 
 
 function B_PYTHON {
 
-[ -d "$PREFIX" ] || { echo "PREFIX unset or not a directory";exit; }
+[[ -d "$PREFIX" ]] || { echo "PREFIX unset or not a directory";exit; }
 
 # apt install libbz2-dev
 # wget "https://www.python.org/ftp/python/3.11.8/Python-3.11.8.tgz"
-SRCURL[0]="https://beehive.molgen.mpg.de/7fb0bfaa2f6aae4aadcdb51abe957825/Python-3.11.8.tgz"
+SRCURL="https://beehive.molgen.mpg.de/7fb0bfaa2f6aae4aadcdb51abe957825/Python-3.11.8.tgz"
 
-PREFIX=${PREFIX}/${BUILD_python}
+PREFIX="${PREFIX}/${BUILD_python}"
 
-mkdir -p $PREFIX
+mkdir -p "${PREFIX}"
 
-cat >$PREFIX/profile <<-EOF
-PATH=$PREFIX/bin:\$PATH
+cat >"${PREFIX}"/profile <<-EOF
+PATH="${PREFIX}"/bin:\$PATH
 EOF
 
-test -e ${BUILD_python}.tar.gz || wget -nv ${SRCURL} -O ${BUILD_python}.tar.gz
-test -d ${BUILD_python} || mkdir -pv ${BUILD_python} && tar -xf ${BUILD_python}.tar.gz --strip-components=1 -C ${BUILD_python}
+test -e "${BUILD_python}.tar.gz" || wget -nv "${SRCURL}" -O "${BUILD_python}.tar.gz"
+test -d "${BUILD_python}" || mkdir -pv "${BUILD_python}" && tar -xf "${BUILD_python}.tar.gz" --strip-components=1 -C "${BUILD_python}"
 
-. $PREFIX/profile
+. "${PREFIX}/profile"
 
-cd ${BUILD_python}
+cd "${BUILD_python}"
 
 local _conf=(
-    --prefix ${PREFIX}
+    --prefix "${PREFIX}"
     --enable-loadable-sqlite-extensions
     --enable-optimizations
 )
@@ -44,18 +44,18 @@ SQLITE_PREFIX=$(pkg-config sqlite3 --variable=prefix)
 LDFLAGS="-Wl,-rpath -Wl,-L${SQLITE_PREFIX}/lib" \
 ./configure "${_conf[@]}"
 
-make -j $NPROC
+make -j "${NPROC}"
 make install
 
-ln -vfs python3 ${PREFIX}/bin/python
-ln -vfs python3-config ${PREFIX}/bin/python-config
-ln -vfs pydoc3 ${PREFIX}/bin/pydoc
-ln -vfs idle3 ${PREFIX}/bin/idle
-ln -vfs pip3 ${PREFIX}/bin/pip
+ln -vfs python3 "${PREFIX}/bin/python"
+ln -vfs python3-config "${PREFIX}/bin/python-config"
+ln -vfs pydoc3 "${PREFIX}/bin/pydoc"
+ln -vfs idle3 "${PREFIX}/bin/idle"
+ln -vfs pip3 "${PREFIX}/bin/pip"
 
-. $PREFIX/profile
+. "${PREFIX}/profile"
 
-pip3 install --prefix=$PREFIX --upgrade pip
+pip3 install --prefix="${PREFIX}" --upgrade pip
 python3 -m pip install --no-cache-dir --upgrade pipenv==2023.10.24
 pip3 install meson
 pip3 install sphinx

README.dot

@@ -0,0 +1,38 @@
+digraph depends {
+
+
+  PY [ label="python" ]
+  SQ [ label="sqlite" ]
+  NO [ label="node" ]
+  FF [ label="ffmpeg" ]
+  PN [ label="pngquant" ]
+  UN [ label="unpaper" ]
+  ZB [ label="zbar" ]
+  PL [ label="paperless" ]
+  LE [ label="leptonica" ]
+  TE [ label="tesseract" ]
+  JE [ label="jbig2enc" ]
+  JD [ label="jbig2dec" ]
+  NG [ label="nginx" ]
+
+  SQ -> PY
+
+  PY -> NO
+  SQ -> NO
+
+  FF -> PN
+
+  FF -> UN
+  PY -> UN
+
+  { LE, PY, SQ } -> TE
+
+  LE -> JE
+
+  { SQ, PY } -> NG
+
+  { PY, SQ, NO, FF, PN, ZB, UN, LE, TE, JD, JE, NG } -> PL
+
+}
+
+

README.todo

@@ -1,7 +1,24 @@
+dependency tree:
+
+dot REAME.dot
+
+bug:
+   ☐ tag löschen "löschen|Yes?"
+
+test:
+   ☐ zbar ohne imagemagick
 
  ☐ conf.build-scripte
    sed '/kmous=/d;/XM=/d;$s/$/XM=,/'
 
+encrypted document:
+[2024-02-19 14:07:32,454] [DEBUG] [paperless.parsing.tesseract] Calling OCRmyPDF with args: {'input_file': PosixPath('/home/wwwutz/paperless/2.5.3/media/documents/originals/0000754.pdf'), 'output_file': PosixPath('/scratch/local/paperless/wwwutz/2.5.3/paperless-huijc19r/archive.pdf'), 'use_threads': True, 'jobs': 32, 'language': 'deu+eng', 'output_type': 'pdfa', 'progress_bar': False, 'color_conversion_strategy': 'RGB', 'force_ocr': True, 'clean_final': True, 'deskew': True, 'rotate_pages': True, 'rotate_pages_threshold': 12.0, 'sidecar': PosixPath('/scratch/local/paperless/wwwutz/2.5.3/paperless-huijc19r/sidecar.txt')}
+[2024-02-19 14:07:32,495] [WARNING] [paperless.parsing.tesseract] This file is encrypted, OCR is impossible. Using any text present in the original file.
+[2024-02-19 14:07:32,496] [DEBUG] [paperless.parsing] Execute: convert -density 300 -scale 500x5000> -alpha remove -strip -auto-orient -define pdf:use-cropbox=true /home/wwwutz/paperless/2.5.3/media/documents/originals/0000754.pdf[0] /scratch/local/paperless/wwwutz/2.5.3/paperless-huijc19r/convert.webp
+[2024-02-19 14:07:32,902] [DEBUG] [paperless.parsing.tesseract] Deleting directory /scratch/local/paperless/wwwutz/2.5.3/paperless-huijc19r
+
+
+
 ＿＿＿＿＿＿＿＿＿＿＿＿＿＿＿＿＿＿＿
 Archive:
  ✔ ocrmypdf @done (24-02-15 17:01)

build.profile

@@ -1,4 +1,4 @@
-BUILD_TAG=2.5.2
+BUILD_TAG=2.5.3
 
 . build.local
 
@@ -22,5 +22,6 @@ BUILD_leptonica=leptonica-1.84.1
 BUILD_tesseract=tesseract-5.3.4
 BUILD_nginx=nginx-1.25.3
 BUILD_jbig2enc=jbig2enc-0.28-17-gea05019
+BUILD_jbig2dec=jbig2dec-0.20
 
-BUILD_paperless=paperless-ngx-2.5.2
+BUILD_paperless=paperless-ngx-2.5.3

build.sudo

@@ -11,6 +11,7 @@ declare -a PKGS
 PKGS=(
     autoconf
     autopoint
+    cargo
     g++
     git 
     graphicsmagick-imagemagick-compat
@@ -25,6 +26,7 @@ PKGS=(
     libtool
     ninja-build
     redis-server
+    yasm
 )
 
 if [ $ID = 'linuxmint' ]; then

buildall.sh

@@ -9,9 +9,9 @@ PROJECT=${HOME}/paperless-server/\${BUILD_TAG}
 PREFIX=${HOME}/paperless-server/\${BUILD_TAG}
 TMPDIR=${TMPDIR:-/tmp}/paperless-server/${USER}/\${BUILD_TAG}
 
-PAPERLESS_SECRET_KEY=$(cat /dev/urandom | head -c 64 | openssl base64 |head -1)
+PAPERLESS_SECRET_KEY=$(head -c 64 < /dev/urandom | openssl base64 |head -1)
 PAPERLESS_ADMIN_USER=admin
-PAPERLESS_ADMIN_PASSWORD=$(cat /dev/urandom | head -c 12 | openssl base64 |head -1)
+PAPERLESS_ADMIN_PASSWORD=$(head -c 12 < /dev/urandom | openssl base64 |head -1)
 PAPERLESS_ADMIN_MAIL=admin@localhost.
 
 PAPERLESS_BIND_ADDR=localhost
@@ -29,48 +29,50 @@ fi
 
 . build.profile
 
-mkdir -vp $PREFIX
-mkdir -vp $PREFIX/conf
-mkdir -vp --mode=700 $TMPDIR
-mkdir -vp --mode=700 ${XDG_CACHE_HOME:=${TMPDIR}.cache}
-mkdir -vp --mode=700 $DEVSHM
+mkdir -vp "${PREFIX}"
+mkdir -vp "${PREFIX}/conf"
+mkdir -vp --mode=700 "${TMPDIR}"
+mkdir -vp --mode=700 "${XDG_CACHE_HOME:=${TMPDIR}.cache}"
+mkdir -vp --mode=700 "${DEVSHM}"
 
 # logging
 
 LOGS=${PWD}/logs
 logfile="/usr/bin/cronolog -S $LOGS/build.log $LOGS/build-%Y-%m-%d.log"
 
-mkdir -vp $LOGS
+mkdir -vp "$LOGS"
 
 cnt=1
-logfile=$LOGS/build-$(date +"%Y-%m-%d").$cnt.log
+logfile=$LOGS/build-"$(date +%Y-%m-%d)".$cnt.log
 set -o noclobber
-while ! { > $logfile ; } &> /dev/null; do
+# shellcheck disable=SC2188
+while ! { > "${logfile}" ; } &> /dev/null; do
     cnt=$((cnt + 1))
-    printf -v logfile "%s.%s.log" $LOGS/build-$(date +"%Y-%m-%d") $cnt
+    printf -v logfile "%s.%s.log" "${LOGS}/build-$(date +%Y-%m-%d)" ${cnt}
 done
 set +o noclobber
 
 set -x
 
-ln -fs $logfile $LOGS/build.log
+ln -fs "${logfile}" "${LOGS}/build.log"
 
 {
 ./profile.build.sh
 ./startstop-paperless.build.sh
-[ -d ${BUILD_sqlite} ] || ./sqlite.build.sh
-[ -d ${BUILD_python} ] || ./Python.build.sh
-[ -d ${BUILD_node} ] || ./node.build.sh
-[ -d ${BUILD_ffmpeg} ] || ./ffmpeg.build.sh
-[ -d ${BUILD_pngquant} ] || ./pngquant.build.sh
-[ -d ${BUILD_unpaper} ] || ./unpaper.build.sh
-[ -d ${BUILD_zbar} ] || ./zbar.build.sh
-[ -d ${BUILD_leptonica} ] || ./leptonica.build.sh
-[ -d ${BUILD_tesseract} ] || ./tesseract.build.sh
-[ -d ${BUILD_nginx} ] || ./nginx.build.sh
-[ -d ${BUILD_jbig2enc} ] || ./jbig2enc.build.sh
+[ -d "${BUILD_sqlite}" ] || ./sqlite.build.sh
+[ -d "${BUILD_python}" ] || ./Python.build.sh
+[ -d "${BUILD_node}" ] || ./node.build.sh
+[ -d "${BUILD_ffmpeg}" ] || ./ffmpeg.build.sh
+[ -d "${BUILD_pngquant}" ] || ./pngquant.build.sh
+[ -d "${BUILD_unpaper}" ] || ./unpaper.build.sh
+[ -d "${BUILD_zbar}" ] || ./zbar.build.sh
+[ -d "${BUILD_leptonica}" ] || ./leptonica.build.sh
+[ -d "${BUILD_jbig2enc}" ] || ./jbig2enc.build.sh
+[ -d "${BUILD_jbig2dec}" ] || ./jbig2dec.build.sh
+[ -d "${BUILD_tesseract}" ] || ./tesseract.build.sh
+[ -d "${BUILD_nginx}" ] || ./nginx.build.sh
 ./nginx.conf.build.sh
-[ -d $PROJECT/${BUILD_paperless} ] || ./paperless-ngx.build.sh
+[ -d "$PROJECT/${BUILD_paperless}" ] || ./paperless-ngx.build.sh
 
 
-} 2>&1 | tee $logfile 2>&1
+} 2>&1 | tee "$logfile" 2>&1

cleanall.sh

@@ -8,6 +8,7 @@ declare -a RMRF
 
 RMRF=(
   $BUILD_ffmpeg
+  $BUILD_jbig2dec
   $BUILD_jbig2enc
   $BUILD_leptonica
   $BUILD_nginx

jbig2dec.build.sh

@@ -0,0 +1,51 @@
+#!/bin/bash
+{
+set -x
+set -e
+
+. build.profile
+
+. "${PREFIX}/${BUILD_python}/profile"
+
+function B_JBIG2DEC {
+
+[ -d "$PREFIX" ] || { echo "PREFIX unset or not a directory";exit; }
+
+PREFIX="${PREFIX}/${BUILD_jbig2dec}"
+
+mkdir -p "${PREFIX}"
+
+BUILD_PKG="${BUILD_jbig2dec}"
+
+cat >"$PREFIX/profile" <<-EOF
+PATH="${PREFIX}/bin":\$PATH
+PKG_CONFIG_PATH=${PROJECT}/${BUILD_PKG}/lib/pkgconfig\${PKG_CONFIG_PATH:+:}\$PKG_CONFIG_PATH
+EOF
+
+
+if [ ! -d "${BUILD_PKG}" ]; then
+    if [ ! -e "${BUILD_PKG}.tar" ]; then
+      git clone https://github.com/ArtifexSoftware/jbig2dec.git  --branch "${BUILD_PKG##*-}" --single-branch "${BUILD_PKG}"
+      tar cf "${BUILD_PKG}.tar" "${BUILD_PKG}"
+    else
+      tar -xf "${BUILD_PKG}.tar"
+    fi
+fi
+
+pushd "${BUILD_PKG}"
+
+local _conf=(
+    --prefix "${PREFIX}"
+    )
+
+./autogen.sh "${_conf[@]}"
+
+make -j "${NPROC}"
+make install
+
+popd
+}
+
+B_JBIG2DEC
+
+}

jbig2enc.build.sh

@@ -10,18 +10,18 @@ set -e
 
 function B_JBIG2ENC {
 
-[ -d "$PREFIX" ] || { echo "PREFIX unset or not a directory";exit; }
+[ -d "${PREFIX}" ] || { echo "PREFIX unset or not a directory";exit; }
 
-PREFIX=${PREFIX}/${BUILD_jbig2enc}
+PREFIX="${PREFIX}/${BUILD_jbig2enc}"
 
-mkdir -p $PREFIX
+mkdir -p "${PREFIX}"
 
-cat >$PREFIX/profile <<-EOF
-PATH=$PREFIX/bin:\$PATH
+cat >"${PREFIX}/profile" <<-EOF
+PATH=${PREFIX}/bin:\$PATH
 EOF
 
 
-if [ ! -d ${BUILD_jbig2enc} ]; then
+if [ ! -d "${BUILD_jbig2enc}" ]; then
     if [ ! -e ${BUILD_jbig2enc}.tar ]; then
       git clone https://github.com/agl/jbig2enc.git ${BUILD_jbig2enc}
       tar cf ${BUILD_jbig2enc}.tar ${BUILD_jbig2enc}
@@ -37,10 +37,12 @@ local _conf=(
     --prefix ${PREFIX}
     )
 
-./autogen.sh
+LDFLAGS=$(PKG_CONFIG_PATH=${PKG_CONFIG_PATH} pkg-config --libs-only-L lept)
+export LDFLAGS
+CPPFLAGS="-I${PROJECT}/${BUILD_leptonica}/include"
+export CPPFLAGS
 
-export LDFLAGS=$(PKG_CONFIG_PATH=${PKG_CONFIG_PATH} pkg-config --libs-only-L lept)
-export CPPFLAGS="-I${PROJECT}/${BUILD_leptonica}/include"
+./autogen.sh
 
 ./configure "${_conf[@]}"
 

paperless-ngx.build.sh

@@ -15,6 +15,7 @@ set -e
 . ${PREFIX}/${BUILD_node}/profile
 . ${PREFIX}/${BUILD_leptonica}/profile
 . ${PREFIX}/${BUILD_jbig2enc}/profile
+. ${PREFIX}/${BUILD_jbig2dec}/profile
 . ${PREFIX}/${BUILD_tesseract}/profile
 
 function B_PAPERLESS {

profile.build.sh

@@ -13,6 +13,7 @@ BUILD_TAG=${BUILD_TAG}
 PATH=$HOME/bin:/bin:/sbin:/usr/bin:/usr/sbin
 
 . ${PROJECT}/${BUILD_zbar}/profile
+. ${PROJECT}/${BUILD_jbig2enc}/profile
 . ${PROJECT}/${BUILD_sqlite}/profile
 . ${PROJECT}/${BUILD_python}/profile
 . ${PROJECT}/${BUILD_node}/profile

shellcheckrc

@@ -0,0 +1,3 @@
+disable=SC2174 # (warning): When used with -p, -m only applies to the deepest directory.
+disable=SC2034 # (warning): BLALA appears unused. Verify use (or export if used externally).
+disable=SC1090 # (warning): ShellCheck can't follow non-constant source. Use a directive to specify location.

zbar.build.sh

@@ -45,6 +45,8 @@ local _conf=(
     --with-gtk=no
     --with-dbus=no
     --with-python=no
+    --with-xshm=no
+    --with-imagemagick=no
 )
 
 ./configure "${_conf[@]}"