diff --git a/.gitignore b/.gitignore index 9c0ad2f..3196ae3 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ /build.local /ffmpeg-* /jbig2enc-* +/jbig2dec-* /leptonica-* /logs /nginx-* diff --git a/Python.build.sh b/Python.build.sh index 96de4c5..3955a37 100755 --- a/Python.build.sh +++ b/Python.build.sh @@ -5,34 +5,34 @@ set -e . build.profile -. ${PREFIX}/${BUILD_sqlite}/profile +. "${PREFIX}/${BUILD_sqlite}/profile" function B_PYTHON { -[ -d "$PREFIX" ] || { echo "PREFIX unset or not a directory";exit; } +[[ -d "$PREFIX" ]] || { echo "PREFIX unset or not a directory";exit; } # apt install libbz2-dev # wget "https://www.python.org/ftp/python/3.11.8/Python-3.11.8.tgz" -SRCURL[0]="https://beehive.molgen.mpg.de/7fb0bfaa2f6aae4aadcdb51abe957825/Python-3.11.8.tgz" +SRCURL="https://beehive.molgen.mpg.de/7fb0bfaa2f6aae4aadcdb51abe957825/Python-3.11.8.tgz" -PREFIX=${PREFIX}/${BUILD_python} +PREFIX="${PREFIX}/${BUILD_python}" -mkdir -p $PREFIX +mkdir -p "${PREFIX}" -cat >$PREFIX/profile <<-EOF -PATH=$PREFIX/bin:\$PATH +cat >"${PREFIX}"/profile <<-EOF +PATH="${PREFIX}"/bin:\$PATH EOF -test -e ${BUILD_python}.tar.gz || wget -nv ${SRCURL} -O ${BUILD_python}.tar.gz -test -d ${BUILD_python} || mkdir -pv ${BUILD_python} && tar -xf ${BUILD_python}.tar.gz --strip-components=1 -C ${BUILD_python} +test -e "${BUILD_python}.tar.gz" || wget -nv "${SRCURL}" -O "${BUILD_python}.tar.gz" +test -d "${BUILD_python}" || mkdir -pv "${BUILD_python}" && tar -xf "${BUILD_python}.tar.gz" --strip-components=1 -C "${BUILD_python}" -. $PREFIX/profile +. "${PREFIX}/profile" -cd ${BUILD_python} +cd "${BUILD_python}" local _conf=( - --prefix ${PREFIX} + --prefix "${PREFIX}" --enable-loadable-sqlite-extensions --enable-optimizations ) @@ -44,18 +44,18 @@ SQLITE_PREFIX=$(pkg-config sqlite3 --variable=prefix) LDFLAGS="-Wl,-rpath -Wl,-L${SQLITE_PREFIX}/lib" \ ./configure "${_conf[@]}" -make -j $NPROC +make -j "${NPROC}" make install -ln -vfs python3 ${PREFIX}/bin/python -ln -vfs python3-config ${PREFIX}/bin/python-config -ln -vfs pydoc3 ${PREFIX}/bin/pydoc -ln -vfs idle3 ${PREFIX}/bin/idle -ln -vfs pip3 ${PREFIX}/bin/pip +ln -vfs python3 "${PREFIX}/bin/python" +ln -vfs python3-config "${PREFIX}/bin/python-config" +ln -vfs pydoc3 "${PREFIX}/bin/pydoc" +ln -vfs idle3 "${PREFIX}/bin/idle" +ln -vfs pip3 "${PREFIX}/bin/pip" -. $PREFIX/profile +. "${PREFIX}/profile" -pip3 install --prefix=$PREFIX --upgrade pip +pip3 install --prefix="${PREFIX}" --upgrade pip python3 -m pip install --no-cache-dir --upgrade pipenv==2023.10.24 pip3 install meson pip3 install sphinx diff --git a/README.dot b/README.dot new file mode 100644 index 0000000..6282a3d --- /dev/null +++ b/README.dot @@ -0,0 +1,38 @@ +digraph depends { + + + PY [ label="python" ] + SQ [ label="sqlite" ] + NO [ label="node" ] + FF [ label="ffmpeg" ] + PN [ label="pngquant" ] + UN [ label="unpaper" ] + ZB [ label="zbar" ] + PL [ label="paperless" ] + LE [ label="leptonica" ] + TE [ label="tesseract" ] + JE [ label="jbig2enc" ] + JD [ label="jbig2dec" ] + NG [ label="nginx" ] + + SQ -> PY + + PY -> NO + SQ -> NO + + FF -> PN + + FF -> UN + PY -> UN + + { LE, PY, SQ } -> TE + + LE -> JE + + { SQ, PY } -> NG + + { PY, SQ, NO, FF, PN, ZB, UN, LE, TE, JD, JE, NG } -> PL + +} + + diff --git a/README.todo b/README.todo index 166467b..e9f0130 100644 --- a/README.todo +++ b/README.todo @@ -1,7 +1,24 @@ +dependency tree: + +dot REAME.dot + +bug: + ☐ tag löschen "löschen|Yes?" + +test: + ☐ zbar ohne imagemagick ☐ conf.build-scripte sed '/kmous=/d;/XM=/d;$s/$/XM=,/' +encrypted document: +[2024-02-19 14:07:32,454] [DEBUG] [paperless.parsing.tesseract] Calling OCRmyPDF with args: {'input_file': PosixPath('/home/wwwutz/paperless/2.5.3/media/documents/originals/0000754.pdf'), 'output_file': PosixPath('/scratch/local/paperless/wwwutz/2.5.3/paperless-huijc19r/archive.pdf'), 'use_threads': True, 'jobs': 32, 'language': 'deu+eng', 'output_type': 'pdfa', 'progress_bar': False, 'color_conversion_strategy': 'RGB', 'force_ocr': True, 'clean_final': True, 'deskew': True, 'rotate_pages': True, 'rotate_pages_threshold': 12.0, 'sidecar': PosixPath('/scratch/local/paperless/wwwutz/2.5.3/paperless-huijc19r/sidecar.txt')} +[2024-02-19 14:07:32,495] [WARNING] [paperless.parsing.tesseract] This file is encrypted, OCR is impossible. Using any text present in the original file. +[2024-02-19 14:07:32,496] [DEBUG] [paperless.parsing] Execute: convert -density 300 -scale 500x5000> -alpha remove -strip -auto-orient -define pdf:use-cropbox=true /home/wwwutz/paperless/2.5.3/media/documents/originals/0000754.pdf[0] /scratch/local/paperless/wwwutz/2.5.3/paperless-huijc19r/convert.webp +[2024-02-19 14:07:32,902] [DEBUG] [paperless.parsing.tesseract] Deleting directory /scratch/local/paperless/wwwutz/2.5.3/paperless-huijc19r + + + ___________________ Archive: ✔ ocrmypdf @done (24-02-15 17:01) diff --git a/build.profile b/build.profile index f36511d..7233b69 100644 --- a/build.profile +++ b/build.profile @@ -1,4 +1,4 @@ -BUILD_TAG=2.5.2 +BUILD_TAG=2.5.3 . build.local @@ -22,5 +22,6 @@ BUILD_leptonica=leptonica-1.84.1 BUILD_tesseract=tesseract-5.3.4 BUILD_nginx=nginx-1.25.3 BUILD_jbig2enc=jbig2enc-0.28-17-gea05019 +BUILD_jbig2dec=jbig2dec-0.20 -BUILD_paperless=paperless-ngx-2.5.2 +BUILD_paperless=paperless-ngx-2.5.3 diff --git a/build.sudo b/build.sudo index 2178b13..e6b64f9 100755 --- a/build.sudo +++ b/build.sudo @@ -11,6 +11,7 @@ declare -a PKGS PKGS=( autoconf autopoint + cargo g++ git graphicsmagick-imagemagick-compat @@ -25,6 +26,7 @@ PKGS=( libtool ninja-build redis-server + yasm ) if [ $ID = 'linuxmint' ]; then diff --git a/buildall.sh b/buildall.sh index 5fe6bba..38f09a7 100755 --- a/buildall.sh +++ b/buildall.sh @@ -9,9 +9,9 @@ PROJECT=${HOME}/paperless-server/\${BUILD_TAG} PREFIX=${HOME}/paperless-server/\${BUILD_TAG} TMPDIR=${TMPDIR:-/tmp}/paperless-server/${USER}/\${BUILD_TAG} -PAPERLESS_SECRET_KEY=$(cat /dev/urandom | head -c 64 | openssl base64 |head -1) +PAPERLESS_SECRET_KEY=$(head -c 64 < /dev/urandom | openssl base64 |head -1) PAPERLESS_ADMIN_USER=admin -PAPERLESS_ADMIN_PASSWORD=$(cat /dev/urandom | head -c 12 | openssl base64 |head -1) +PAPERLESS_ADMIN_PASSWORD=$(head -c 12 < /dev/urandom | openssl base64 |head -1) PAPERLESS_ADMIN_MAIL=admin@localhost. PAPERLESS_BIND_ADDR=localhost @@ -29,48 +29,50 @@ fi . build.profile -mkdir -vp $PREFIX -mkdir -vp $PREFIX/conf -mkdir -vp --mode=700 $TMPDIR -mkdir -vp --mode=700 ${XDG_CACHE_HOME:=${TMPDIR}.cache} -mkdir -vp --mode=700 $DEVSHM +mkdir -vp "${PREFIX}" +mkdir -vp "${PREFIX}/conf" +mkdir -vp --mode=700 "${TMPDIR}" +mkdir -vp --mode=700 "${XDG_CACHE_HOME:=${TMPDIR}.cache}" +mkdir -vp --mode=700 "${DEVSHM}" # logging LOGS=${PWD}/logs logfile="/usr/bin/cronolog -S $LOGS/build.log $LOGS/build-%Y-%m-%d.log" -mkdir -vp $LOGS +mkdir -vp "$LOGS" cnt=1 -logfile=$LOGS/build-$(date +"%Y-%m-%d").$cnt.log +logfile=$LOGS/build-"$(date +%Y-%m-%d)".$cnt.log set -o noclobber -while ! { > $logfile ; } &> /dev/null; do +# shellcheck disable=SC2188 +while ! { > "${logfile}" ; } &> /dev/null; do cnt=$((cnt + 1)) - printf -v logfile "%s.%s.log" $LOGS/build-$(date +"%Y-%m-%d") $cnt + printf -v logfile "%s.%s.log" "${LOGS}/build-$(date +%Y-%m-%d)" ${cnt} done set +o noclobber set -x -ln -fs $logfile $LOGS/build.log +ln -fs "${logfile}" "${LOGS}/build.log" { ./profile.build.sh ./startstop-paperless.build.sh -[ -d ${BUILD_sqlite} ] || ./sqlite.build.sh -[ -d ${BUILD_python} ] || ./Python.build.sh -[ -d ${BUILD_node} ] || ./node.build.sh -[ -d ${BUILD_ffmpeg} ] || ./ffmpeg.build.sh -[ -d ${BUILD_pngquant} ] || ./pngquant.build.sh -[ -d ${BUILD_unpaper} ] || ./unpaper.build.sh -[ -d ${BUILD_zbar} ] || ./zbar.build.sh -[ -d ${BUILD_leptonica} ] || ./leptonica.build.sh -[ -d ${BUILD_tesseract} ] || ./tesseract.build.sh -[ -d ${BUILD_nginx} ] || ./nginx.build.sh -[ -d ${BUILD_jbig2enc} ] || ./jbig2enc.build.sh +[ -d "${BUILD_sqlite}" ] || ./sqlite.build.sh +[ -d "${BUILD_python}" ] || ./Python.build.sh +[ -d "${BUILD_node}" ] || ./node.build.sh +[ -d "${BUILD_ffmpeg}" ] || ./ffmpeg.build.sh +[ -d "${BUILD_pngquant}" ] || ./pngquant.build.sh +[ -d "${BUILD_unpaper}" ] || ./unpaper.build.sh +[ -d "${BUILD_zbar}" ] || ./zbar.build.sh +[ -d "${BUILD_leptonica}" ] || ./leptonica.build.sh +[ -d "${BUILD_jbig2enc}" ] || ./jbig2enc.build.sh +[ -d "${BUILD_jbig2dec}" ] || ./jbig2dec.build.sh +[ -d "${BUILD_tesseract}" ] || ./tesseract.build.sh +[ -d "${BUILD_nginx}" ] || ./nginx.build.sh ./nginx.conf.build.sh -[ -d $PROJECT/${BUILD_paperless} ] || ./paperless-ngx.build.sh +[ -d "$PROJECT/${BUILD_paperless}" ] || ./paperless-ngx.build.sh -} 2>&1 | tee $logfile 2>&1 +} 2>&1 | tee "$logfile" 2>&1 diff --git a/cleanall.sh b/cleanall.sh index afb5879..8a53820 100644 --- a/cleanall.sh +++ b/cleanall.sh @@ -8,6 +8,7 @@ declare -a RMRF RMRF=( $BUILD_ffmpeg + $BUILD_jbig2dec $BUILD_jbig2enc $BUILD_leptonica $BUILD_nginx diff --git a/jbig2dec.build.sh b/jbig2dec.build.sh new file mode 100755 index 0000000..1d94d7c --- /dev/null +++ b/jbig2dec.build.sh @@ -0,0 +1,51 @@ +#!/bin/bash +{ +set -x +set -e + +. build.profile + +. "${PREFIX}/${BUILD_python}/profile" + +function B_JBIG2DEC { + +[ -d "$PREFIX" ] || { echo "PREFIX unset or not a directory";exit; } + +PREFIX="${PREFIX}/${BUILD_jbig2dec}" + +mkdir -p "${PREFIX}" + +BUILD_PKG="${BUILD_jbig2dec}" + +cat >"$PREFIX/profile" <<-EOF +PATH="${PREFIX}/bin":\$PATH +PKG_CONFIG_PATH=${PROJECT}/${BUILD_PKG}/lib/pkgconfig\${PKG_CONFIG_PATH:+:}\$PKG_CONFIG_PATH +EOF + + +if [ ! -d "${BUILD_PKG}" ]; then + if [ ! -e "${BUILD_PKG}.tar" ]; then + git clone https://github.com/ArtifexSoftware/jbig2dec.git --branch "${BUILD_PKG##*-}" --single-branch "${BUILD_PKG}" + tar cf "${BUILD_PKG}.tar" "${BUILD_PKG}" + else + tar -xf "${BUILD_PKG}.tar" + fi +fi + +pushd "${BUILD_PKG}" + +local _conf=( + --prefix "${PREFIX}" + ) + +./autogen.sh "${_conf[@]}" + +make -j "${NPROC}" +make install + +popd +} + +B_JBIG2DEC + +} diff --git a/jbig2enc.build.sh b/jbig2enc.build.sh index 9f2dbbb..af42cb1 100755 --- a/jbig2enc.build.sh +++ b/jbig2enc.build.sh @@ -10,18 +10,18 @@ set -e function B_JBIG2ENC { -[ -d "$PREFIX" ] || { echo "PREFIX unset or not a directory";exit; } +[ -d "${PREFIX}" ] || { echo "PREFIX unset or not a directory";exit; } -PREFIX=${PREFIX}/${BUILD_jbig2enc} +PREFIX="${PREFIX}/${BUILD_jbig2enc}" -mkdir -p $PREFIX +mkdir -p "${PREFIX}" -cat >$PREFIX/profile <<-EOF -PATH=$PREFIX/bin:\$PATH +cat >"${PREFIX}/profile" <<-EOF +PATH=${PREFIX}/bin:\$PATH EOF -if [ ! -d ${BUILD_jbig2enc} ]; then +if [ ! -d "${BUILD_jbig2enc}" ]; then if [ ! -e ${BUILD_jbig2enc}.tar ]; then git clone https://github.com/agl/jbig2enc.git ${BUILD_jbig2enc} tar cf ${BUILD_jbig2enc}.tar ${BUILD_jbig2enc} @@ -37,10 +37,12 @@ local _conf=( --prefix ${PREFIX} ) -./autogen.sh +LDFLAGS=$(PKG_CONFIG_PATH=${PKG_CONFIG_PATH} pkg-config --libs-only-L lept) +export LDFLAGS +CPPFLAGS="-I${PROJECT}/${BUILD_leptonica}/include" +export CPPFLAGS -export LDFLAGS=$(PKG_CONFIG_PATH=${PKG_CONFIG_PATH} pkg-config --libs-only-L lept) -export CPPFLAGS="-I${PROJECT}/${BUILD_leptonica}/include" +./autogen.sh ./configure "${_conf[@]}" diff --git a/paperless-ngx.build.sh b/paperless-ngx.build.sh index e05ffac..59a41e0 100755 --- a/paperless-ngx.build.sh +++ b/paperless-ngx.build.sh @@ -15,6 +15,7 @@ set -e . ${PREFIX}/${BUILD_node}/profile . ${PREFIX}/${BUILD_leptonica}/profile . ${PREFIX}/${BUILD_jbig2enc}/profile +. ${PREFIX}/${BUILD_jbig2dec}/profile . ${PREFIX}/${BUILD_tesseract}/profile function B_PAPERLESS { diff --git a/profile.build.sh b/profile.build.sh index 46b552f..78a8e92 100755 --- a/profile.build.sh +++ b/profile.build.sh @@ -13,6 +13,7 @@ BUILD_TAG=${BUILD_TAG} PATH=$HOME/bin:/bin:/sbin:/usr/bin:/usr/sbin . ${PROJECT}/${BUILD_zbar}/profile +. ${PROJECT}/${BUILD_jbig2enc}/profile . ${PROJECT}/${BUILD_sqlite}/profile . ${PROJECT}/${BUILD_python}/profile . ${PROJECT}/${BUILD_node}/profile diff --git a/shellcheckrc b/shellcheckrc new file mode 100644 index 0000000..89ae02f --- /dev/null +++ b/shellcheckrc @@ -0,0 +1,3 @@ +disable=SC2174 # (warning): When used with -p, -m only applies to the deepest directory. +disable=SC2034 # (warning): BLALA appears unused. Verify use (or export if used externally). +disable=SC1090 # (warning): ShellCheck can't follow non-constant source. Use a directive to specify location. diff --git a/zbar.build.sh b/zbar.build.sh index 4b9ba1c..ee50b36 100755 --- a/zbar.build.sh +++ b/zbar.build.sh @@ -45,6 +45,8 @@ local _conf=( --with-gtk=no --with-dbus=no --with-python=no + --with-xshm=no + --with-imagemagick=no ) ./configure "${_conf[@]}"