diff --git a/.gitignore b/.gitignore index eb96dc9..6a948f5 100644 --- a/.gitignore +++ b/.gitignore @@ -10,6 +10,7 @@ /paperless-ngx-* /pngquant-* /Python-* +/redis-* /sqlite-* /tessconfigs /tessdata diff --git a/Python.build.sh b/Python.build.sh index 3955a37..bc00960 100755 --- a/Python.build.sh +++ b/Python.build.sh @@ -2,34 +2,34 @@ { set -x set -e +set -u . build.profile . "${PREFIX}/${BUILD_sqlite}/profile" - function B_PYTHON { -[[ -d "$PREFIX" ]] || { echo "PREFIX unset or not a directory";exit; } - # apt install libbz2-dev # wget "https://www.python.org/ftp/python/3.11.8/Python-3.11.8.tgz" SRCURL="https://beehive.molgen.mpg.de/7fb0bfaa2f6aae4aadcdb51abe957825/Python-3.11.8.tgz" -PREFIX="${PREFIX}/${BUILD_python}" +BUILD_PKG=${BUILD_python} + +PREFIX="${PREFIX}/${BUILD_PKG}" mkdir -p "${PREFIX}" cat >"${PREFIX}"/profile <<-EOF -PATH="${PREFIX}"/bin:\$PATH +PATH="${PREFIX}"/bin:\${PATH} EOF -test -e "${BUILD_python}.tar.gz" || wget -nv "${SRCURL}" -O "${BUILD_python}.tar.gz" -test -d "${BUILD_python}" || mkdir -pv "${BUILD_python}" && tar -xf "${BUILD_python}.tar.gz" --strip-components=1 -C "${BUILD_python}" +test -e "${BUILD_PKG}.tar.gz" || wget -nv "${SRCURL}" -O "${BUILD_PKG}.tar.gz" +test -d "${BUILD_PKG}" || mkdir -pv "${BUILD_PKG}" && tar -xf "${BUILD_PKG}.tar.gz" --strip-components=1 -C "${BUILD_PKG}" . "${PREFIX}/profile" -cd "${BUILD_python}" +cd "${BUILD_PKG}" local _conf=( --prefix "${PREFIX}" diff --git a/README.todo b/README.todo index 1af1118..a23f457 100644 --- a/README.todo +++ b/README.todo @@ -7,6 +7,15 @@ bug: ☐ https://github.com/paperless-ngx/paperless-ngx/discussions/3090 Calling OCRmyPDF with args: {'input_file': PosixPath('/scratch/local/plprj/2024-02-19/paperless-ngx34bhqotl/c913a207C78B05A-B130-45F8-9EC4-281AD914ABFA}.PDF'), 'output_file': PosixPath('/scratch/local/plprj/2024-02-19/paperless-0ee3414u/archive.pdf'), 'use_threads': True, 'jobs': 32, 'language': 'deu+eng', 'output_type': 'pdfa', 'progress_bar': False, 'color_conversion_strategy': 'RGB', 'force_ocr': True, 'clean_final': True, 'deskew': True, 'rotate_pages': True, 'rotate_pages_threshold': 12.0, 'sidecar': PosixPath('/scratch/local/plprj/2024-02-19/paperless-0ee3414u/sidecar.txt')} + ☐ /node_modules/.bin/ng build --configuration production +Locale data for 'ar-AR' cannot be found. Using locale data for 'ar'. +WARNING [src/locale/messages.ar_AR.xlf]: File target locale ('ar') does not match configured locale ('ar-AR') +Locale data for 'af-ZA' cannot be found. Using locale data for 'af'. +WARNING [src/locale/messages.af_ZA.xlf]: File target locale ('af') does not match configured locale ('af-ZA') +Locale data for 'bg-BG' cannot be found. Using locale data for 'bg'. +WARNING [src/locale/messages.bg_BG.xlf]: File target locale ('bg') does not match configured locale ('bg-BG') + + upgrade: ssl-certs in conf umziehen diff --git a/build.profile b/build.profile index c082c9e..8fe6caa 100644 --- a/build.profile +++ b/build.profile @@ -1,17 +1,17 @@ -BUILD_TAG=2.6.1 +BUILD_TAG=2.6.3 . build.local -PAPERLESS_LOGGING_DIR=${PROJECT}/log - DEVSHM=/dev/shm/${USER}/${BUILD_TAG} PATH=$HOME/bin:/bin:/sbin:/usr/bin:/usr/sbin NPROC=$(nproc) XDG_CACHE_HOME=${PREFIX}/cache TMPDIR=${TMPDIR:-/tmp/${USER}/${BUILD_TAG}} +LOGDIR=${PROJECT}/log BUILD_sqlite=sqlite-3.45.1 +BUILD_redis=redis-7.2.4 BUILD_python=Python-3.11.8 BUILD_node=node-20.11.0 BUILD_ffmpeg=ffmpeg-6.1.1 @@ -25,4 +25,4 @@ BUILD_jbig2enc=jbig2enc-0.28-17-gea05019 BUILD_jbig2dec=jbig2dec-0.20 BUILD_libfontttf=liberation-fonts-ttf-2.1.5 -BUILD_paperless=paperless-ngx-2.6.2 +BUILD_paperless=paperless-ngx-2.6.3 diff --git a/buildall.sh b/buildall.sh index 709199b..78d301c 100755 --- a/buildall.sh +++ b/buildall.sh @@ -58,8 +58,9 @@ ln -fs "${logfile}" "${LOGS}/build.log" { ./profile.build.sh -./startstop-paperless.build.sh +./startstop.build.sh [ -d "${BUILD_sqlite}" ] || ./sqlite.build.sh +[ -d "${BUILD_redis}" ] || ./redis.build.sh [ -d "${BUILD_python}" ] || ./Python.build.sh [ -d "${BUILD_node}" ] || ./node.build.sh [ -d "${BUILD_ffmpeg}" ] || ./ffmpeg.build.sh diff --git a/cleanall.sh b/cleanall.sh index 337a674..880a613 100644 --- a/cleanall.sh +++ b/cleanall.sh @@ -18,6 +18,7 @@ RMRF=( "${BUILD_paperless}" "${BUILD_pngquant}" "${BUILD_python}" + "${BUILD_redis}" "${BUILD_sqlite}" "${BUILD_tesseract}" "${BUILD_unpaper}" diff --git a/ffmpeg.build.sh b/ffmpeg.build.sh index 660857c..a6ab428 100755 --- a/ffmpeg.build.sh +++ b/ffmpeg.build.sh @@ -2,30 +2,31 @@ { set -x set -e +set -u . build.profile function B_FFMPEG { -[ -d "$PREFIX" ] || { echo "PREFIX unset or not a directory";exit; } +BUILD_PKG=${BUILD_ffmpeg} # SRCURL[0]="https://ffmpeg.org//releases/${BUILD_ffmpeg}.tar.gz" SRCURL[0]="https://beehive.molgen.mpg.de/cce359cad7ed0d4f0079f7864080ad36/ffmpeg-6.1.1.tar.gz" -PREFIX=${PREFIX}/${BUILD_ffmpeg} +PREFIX=${PREFIX}/${BUILD_PKG} -mkdir -p $PREFIX +mkdir -p ${PREFIX} -cat >$PREFIX/profile <<-EOF -PATH=$PREFIX/bin:\$PATH -PKG_CONFIG_PATH=${PROJECT}/${BUILD_ffmpeg}/lib/pkgconfig${PKG_CONFIG_PATH:+:}$PKG_CONFIG_PATH +cat >${PREFIX}/profile <<-EOF +PATH=${PREFIX}/bin:\$PATH +PKG_CONFIG_PATH=${PROJECT}/${BUILD_PKG}/lib/pkgconfig\${PKG_CONFIG_PATH:+:}\${PKG_CONFIG_PATH:-} EOF -test -e ${BUILD_ffmpeg}.tar.gz || wget -nv ${SRCURL} -O ${BUILD_ffmpeg}.tar.gz -test -d ${BUILD_ffmpeg} || mkdir -pv ${BUILD_ffmpeg} && tar -xf ${BUILD_ffmpeg}.tar.gz --strip-components=1 -C ${BUILD_ffmpeg} +test -e ${BUILD_PKG}.tar.gz || wget -nv ${SRCURL} -O ${BUILD_PKG}.tar.gz +test -d ${BUILD_PKG} || mkdir -pv ${BUILD_PKG} && tar -xf ${BUILD_PKG}.tar.gz --strip-components=1 -C ${BUILD_PKG} -cd ${BUILD_ffmpeg} +cd ${BUILD_PKG} local _conf=( --prefix=${PREFIX} @@ -38,8 +39,7 @@ local _conf=( ./configure "${_conf[@]}" -make -j $NPROC -make install +make -j $NPROC install } diff --git a/jbig2dec.build.sh b/jbig2dec.build.sh index 1d94d7c..36a33db 100755 --- a/jbig2dec.build.sh +++ b/jbig2dec.build.sh @@ -2,6 +2,7 @@ { set -x set -e +set -u . build.profile @@ -9,17 +10,16 @@ set -e function B_JBIG2DEC { -[ -d "$PREFIX" ] || { echo "PREFIX unset or not a directory";exit; } +BUILD_PKG="${BUILD_jbig2dec}" -PREFIX="${PREFIX}/${BUILD_jbig2dec}" +PREFIX="${PREFIX}/${BUILD_PKG}" mkdir -p "${PREFIX}" -BUILD_PKG="${BUILD_jbig2dec}" cat >"$PREFIX/profile" <<-EOF PATH="${PREFIX}/bin":\$PATH -PKG_CONFIG_PATH=${PROJECT}/${BUILD_PKG}/lib/pkgconfig\${PKG_CONFIG_PATH:+:}\$PKG_CONFIG_PATH +PKG_CONFIG_PATH=${PROJECT}/${BUILD_PKG}/lib/pkgconfig\${PKG_CONFIG_PATH:+:}\${PKG_CONFIG_PATH:-} EOF @@ -40,8 +40,7 @@ local _conf=( ./autogen.sh "${_conf[@]}" -make -j "${NPROC}" -make install +make -j "${NPROC}" install popd } diff --git a/jbig2enc.build.sh b/jbig2enc.build.sh index af42cb1..ccdaf37 100755 --- a/jbig2enc.build.sh +++ b/jbig2enc.build.sh @@ -2,6 +2,7 @@ { set -x set -e +set -u . build.profile @@ -10,9 +11,10 @@ set -e function B_JBIG2ENC { -[ -d "${PREFIX}" ] || { echo "PREFIX unset or not a directory";exit; } +BUILD_PKG=${BUILD_jbig2enc} + +PREFIX="${PREFIX}/${BUILD_PKG}" -PREFIX="${PREFIX}/${BUILD_jbig2enc}" mkdir -p "${PREFIX}" @@ -21,17 +23,16 @@ PATH=${PREFIX}/bin:\$PATH EOF -if [ ! -d "${BUILD_jbig2enc}" ]; then - if [ ! -e ${BUILD_jbig2enc}.tar ]; then - git clone https://github.com/agl/jbig2enc.git ${BUILD_jbig2enc} - tar cf ${BUILD_jbig2enc}.tar ${BUILD_jbig2enc} +if [ ! -d "${BUILD_PKG}" ]; then + if [ ! -e ${BUILD_PKG}.tar ]; then + git clone https://github.com/agl/jbig2enc.git ${BUILD_PKG} + tar cf ${BUILD_PKG}.tar ${BUILD_PKG} else - tar -xf ${BUILD_jbig2enc}.tar + tar -xf ${BUILD_PKG}.tar fi fi -pushd ${BUILD_jbig2enc} - +pushd ${BUILD_PKG} local _conf=( --prefix ${PREFIX} @@ -46,8 +47,7 @@ export CPPFLAGS ./configure "${_conf[@]}" -make -j $NPROC -make install +make -j ${NPROC} install popd } diff --git a/leptonica.build.sh b/leptonica.build.sh index 80fa8c1..f12e707 100755 --- a/leptonica.build.sh +++ b/leptonica.build.sh @@ -2,30 +2,30 @@ { set -x set -e +set -u . build.profile function B_LEPTONICA { -[ -d "$PREFIX" ] || { echo "PREFIX unset or not a directory";exit; } - # SRCURL[0]="https://github.com/DanBloomberg/leptonica/releases/download/1.84.1/leptonica-1.84.1.tar.gz" -SRCURL[0]="https://beehive.molgen.mpg.de/01e6de7af2ba93b8350cdc3f7aeb0fbf/leptonica-1.84.1.tar.gz" +SRCURL="https://beehive.molgen.mpg.de/01e6de7af2ba93b8350cdc3f7aeb0fbf/leptonica-1.84.1.tar.gz" + +BUILD_PKG=${BUILD_leptonica} -PREFIX=${PREFIX}/${BUILD_leptonica} +PREFIX=${PREFIX}/${BUILD_PKG} mkdir -p $PREFIX cat >$PREFIX/profile <<-EOF -PATH=$PREFIX/bin:\$PATH -PKG_CONFIG_PATH=${PROJECT}/${BUILD_leptonica}/lib/pkgconfig${PKG_CONFIG_PATH:+:}$PKG_CONFIG_PATH +PATH=${PREFIX}/bin:\$PATH +PKG_CONFIG_PATH=${PROJECT}/${BUILD_PKG}/lib/pkgconfig\${PKG_CONFIG_PATH:+:}\${PKG_CONFIG_PATH:-} EOF +test -e ${BUILD_PKG}.tar.gz || wget -nv ${SRCURL} -O ${BUILD_PKG}.tar.gz +test -d ${BUILD_PKG} || mkdir -pv ${BUILD_PKG} && tar -xf ${BUILD_PKG}.tar.gz --strip-components=1 -C ${BUILD_PKG} -test -e ${BUILD_leptonica}.tar.gz || wget -nv ${SRCURL} -O ${BUILD_leptonica}.tar.gz -test -d ${BUILD_leptonica} || mkdir -pv ${BUILD_leptonica} && tar -xf ${BUILD_leptonica}.tar.gz --strip-components=1 -C ${BUILD_leptonica} - -cd ${BUILD_leptonica} +cd ${BUILD_PKG} local _conf=( --prefix ${PREFIX} @@ -33,8 +33,7 @@ local _conf=( ./configure "${_conf[@]}" -make -j $NPROC -make install +make -j ${NPROC} install } diff --git a/liberation-fonts-ttf.build.sh b/liberation-fonts-ttf.build.sh index 1286097..657abe0 100755 --- a/liberation-fonts-ttf.build.sh +++ b/liberation-fonts-ttf.build.sh @@ -2,20 +2,20 @@ { set -x set -e +set -u . build.profile function B_LIBFONTTTF { -[ -d "${PREFIX}" ] || { echo "PREFIX unset or not a directory";exit; } - # SRCURL="https://github.com/liberationfonts/liberation-fonts/files/7261482/liberation-fonts-ttf-2.1.5.tar.gz" SRCURL="https://beehive.molgen.mpg.de/31b453e0b77bacde410a34a725b34f8a/liberation-fonts-ttf-2.1.5.tar.gz" -PREFIX="${PREFIX}/${BUILD_libfontttf}" - BUILD_PKG="${BUILD_libfontttf}" +PREFIX="${PREFIX}/${BUILD_PKG}" + + mkdir -p "${PREFIX}" test -e "${BUILD_PKG}.tar.gz" || wget -nv "${SRCURL}" -O "${BUILD_PKG}.tar.gz" diff --git a/nginx.build.sh b/nginx.build.sh index 475243a..b959f4a 100755 --- a/nginx.build.sh +++ b/nginx.build.sh @@ -2,10 +2,10 @@ { set -x set -e +set -u . build.profile -[ -d "$PREFIX" ] || { echo "PREFIX unset or not a directory";exit; } . ${PREFIX}/${BUILD_sqlite}/profile . ${PREFIX}/${BUILD_python}/profile @@ -20,19 +20,17 @@ BUILD_EXT=${SRCURL/${SRCURL%.*.*}/} PREFIX=${PREFIX}/${BUILD_PKG} -mkdir -p $PREFIX +mkdir -p ${PREFIX} -cat >$PREFIX/profile <<-EOF -PATH=$PREFIX/sbin:\$PATH +cat >${PREFIX}/profile <<-EOF +PATH=${PREFIX}/sbin:\${PATH} EOF -. $PREFIX/profile - -# erst mal in conf, dauert stuuuunden... -mkdir -p ${PROJECT}/conf -DHPARAM=${PROJECT}/conf/dhparam.pem -test -e ${DHPARAM} || openssl dhparam -dsaparam -out ${DHPARAM} 4096 +. ${PREFIX}/profile +mkdir -p ${PREFIX}/conf +DHPARAM=${PREFIX}/conf/dhparam.pem +test -e ${DHPARAM} || openssl dhparam -dsaparam -out ${DHPARAM} 2048 test -e ${BUILD_PKG}${BUILD_EXT} || wget -nv ${SRCURL} -O ${BUILD_PKG}${BUILD_EXT} @@ -54,7 +52,8 @@ local _conf=( --without-http_scgi_module --without-http_grpc_module --without-http_memcached_module - --error-log-path=${PROJECT}/log/nginx-error.log + --error-log-path=${LOGDIR}/nginx-error.log + --http-log-path=${LOGDIR}/nginx-access.log --http-client-body-temp-path=${DEVSHM}/client-body-temp --http-proxy-temp-path=${DEVSHM}/proxy-temp --http-fastcgi-temp-path=${DEVSHM}/fastcgi-temp \ @@ -67,15 +66,7 @@ local _conf=( CFLAGS='-O2 -fPIC' \ ./configure "${_conf[@]}" -make -j $NPROC - -make install - -rm -fv ${PREFIX}/conf/nginx.conf -ln -vfs ${PROJECT}/conf/nginx.conf ${PREFIX}/conf/ -ln -vfs ${PROJECT}/conf/fullchain.pem ${PREFIX}/conf/ -ln -vfs ${PROJECT}/conf/privkey.pem ${PREFIX}/conf/ -ln -vfs ${DHPARAM} ${PREFIX}/conf/ +make -j ${NPROC} install popd diff --git a/nginx.conf.build.sh b/nginx.conf.build.sh index a094922..9a8713e 100755 --- a/nginx.conf.build.sh +++ b/nginx.conf.build.sh @@ -1,10 +1,11 @@ #!/bin/bash set -e set -x +set -u . build.profile -CONF=${PROJECT}/conf/nginx.conf +CONF=${PROJECT}/${BUILD_nginx}/conf/nginx.conf tee ${CONF} <<_EOP_ # generated by ${0} in $PWD @@ -30,7 +31,7 @@ http { server { server_name ${PAPERLESS_BIND_ADDR}; - access_log ${PAPERLESS_LOGGING_DIR}/nginx-access.log; + access_log ${LOGDIR}/nginx-access.log; listen ${PAPERLESS_BIND_ADDR}:${PAPERLESS_PORT}; _EOP_ @@ -38,8 +39,8 @@ _EOP_ if [ -v PAPERLESS_HTTPS ]; then tee -a ${CONF} <<_EOP_ listen ${PAPERLESS_BIND_ADDR}:${PAPERLESS_HTTPS} ssl; - ssl_certificate fullchain.pem; - ssl_certificate_key privkey.pem; + ssl_certificate ${NGINX_SSL_CERTIFICATE}; # from build.local + ssl_certificate_key ${NGINX_SSL_CERTIFICATE_KEY}; # from build.local ssl_dhparam dhparam.pem; # src/nginx.build.sh ssl_session_timeout 1d; ssl_session_cache shared:MozSSL:10m; diff --git a/node.build.sh b/node.build.sh index 5d9bc1f..df4953b 100755 --- a/node.build.sh +++ b/node.build.sh @@ -2,6 +2,7 @@ { set -x set -e +set -u . build.profile @@ -13,21 +14,21 @@ function B_NODEJS { [ -d "$PREFIX" ] || { echo "PREFIX unset or not a directory";exit; } # wget "https://nodejs.org/download/release/v20.11.0/node-v20.11.0.tar.gz" -SRCURL[0]="https://beehive.molgen.mpg.de/94db35ddc577e52d15a6e242260beea7/node-v20.11.0.tar.gz" +SRCURL="https://beehive.molgen.mpg.de/94db35ddc577e52d15a6e242260beea7/node-v20.11.0.tar.gz" PREFIX=${PREFIX}/${BUILD_node} -mkdir -p $PREFIX +mkdir -p ${PREFIX} -cat >$PREFIX/profile <<-EOF -PATH=$PREFIX/bin:\$PATH -npm_config_cache=$PREFIX/cache +cat >${PREFIX}/profile <<-EOF +PATH=${PREFIX}/bin:\$PATH +npm_config_cache=${PREFIX}/cache EOF test -e ${BUILD_node}.tar.gz || wget -nv ${SRCURL} -O ${BUILD_node}.tar.gz test -d ${BUILD_node} || mkdir -pv ${BUILD_node} && tar -xf ${BUILD_node}.tar.gz --strip-components=1 -C ${BUILD_node} -. $PREFIX/profile +. ${PREFIX}/profile cd ${BUILD_node} diff --git a/paperless.conf.build.sh b/paperless.conf.build.sh index b58e232..b0e5e5e 100755 --- a/paperless.conf.build.sh +++ b/paperless.conf.build.sh @@ -26,7 +26,7 @@ PAPERLESS_CONSUMER_SUBDIRS_AS_TAGS=true PAPERLESS_CONSUMPTION_DIR=${PROJECT}/consume PAPERLESS_DATA_DIR=${PROJECT}/data -PAPERLESS_LOGGING_DIR=${PROJECT}/log +PAPERLESS_LOGGING_DIR=${LOGDIR} PAPERLESS_MEDIA_ROOT=${PROJECT}/media PAPERLESS_NLTK_DIR=${PROJECT}/data/nltk PAPERLESS_SCRATCH_DIR=${TMPDIR} diff --git a/pngquant.build.sh b/pngquant.build.sh index c0ae32a..b777f34 100755 --- a/pngquant.build.sh +++ b/pngquant.build.sh @@ -2,6 +2,7 @@ { set -x set -e +set -u . build.profile @@ -9,8 +10,6 @@ set -e function B_PNGQUANT { -[ -d "$PREFIX" ] || { echo "PREFIX unset or not a directory";exit; } - BUILD_PKG=${BUILD_pngquant} PREFIX=${PREFIX}/${BUILD_PKG} @@ -41,7 +40,7 @@ fi cat >$PREFIX/profile <<-EOF PATH=$PREFIX/bin:\$PATH -PKG_CONFIG_PATH=${PROJECT}/${BUILD_PKG}/lib/pkgconfig${PKG_CONFIG_PATH:+:}$PKG_CONFIG_PATH +PKG_CONFIG_PATH=${PROJECT}/${BUILD_PKG}/lib/pkgconfig\${PKG_CONFIG_PATH:+:}\${PKG_CONFIG_PATH:-} EOF cd ${BUILD_PKG} diff --git a/profile.build.sh b/profile.build.sh index 78a8e92..b5d3f21 100755 --- a/profile.build.sh +++ b/profile.build.sh @@ -17,6 +17,7 @@ PATH=$HOME/bin:/bin:/sbin:/usr/bin:/usr/sbin . ${PROJECT}/${BUILD_sqlite}/profile . ${PROJECT}/${BUILD_python}/profile . ${PROJECT}/${BUILD_node}/profile +. ${PROJECT}/${BUILD_redis}/profile . ${PROJECT}/${BUILD_nginx}/profile . ${PROJECT}/${BUILD_unpaper}/profile . ${PROJECT}/${BUILD_ffmpeg}/profile diff --git a/redis.build.sh b/redis.build.sh new file mode 100755 index 0000000..26312cc --- /dev/null +++ b/redis.build.sh @@ -0,0 +1,36 @@ +#!/bin/bash +{ +set -x +set -e +set -u + +. build.profile + +function B_REDIS { + +SRCURL="https://download.redis.io/releases/redis-7.2.4.tar.gz" + +BUILD_PKG=${BUILD_redis} + +PREFIX=${PREFIX}/${BUILD_PKG} + +mkdir -p $PREFIX + +cat >$PREFIX/profile <<-EOF +PATH=$PREFIX/bin:\$PATH +# PKG_CONFIG_PATH=${PROJECT}/${BUILD_PKG}/lib/pkgconfig\${PKG_CONFIG_PATH:+:}\$PKG_CONFIG_PATH +EOF + + +test -e ${BUILD_PKG}.tar.gz || wget -nv ${SRCURL} -O ${BUILD_PKG}.tar.gz +test -d ${BUILD_PKG} || mkdir -pv ${BUILD_PKG} && tar -xf ${BUILD_PKG}.tar.gz --strip-components=1 -C ${BUILD_PKG} + +cd ${BUILD_PKG} + +make -j "${NPROC}" install PREFIX="${PREFIX}" + +} + +B_REDIS + +} diff --git a/sqlite.build.sh b/sqlite.build.sh index b2f5f3e..f5de93d 100755 --- a/sqlite.build.sh +++ b/sqlite.build.sh @@ -2,13 +2,12 @@ { set -x set -e +set -u . build.profile function B_SQLITE { -[ -d "$PREFIX" ] || { echo "PREFIX unset or not a directory";exit; } - # SRCURL[0]="https://sqlite.org/2024/sqlite-autoconf-3450100.tar.gz" SRCURL[0]="https://beehive.molgen.mpg.de/743c99f996add46273694df83c9140d4/sqlite-3.45.1.tar.gz" @@ -21,7 +20,7 @@ mkdir -p $PREFIX cat >$PREFIX/profile <<-EOF PATH=$PREFIX/bin:\$PATH -PKG_CONFIG_PATH=${PROJECT}/${BUILD_sqlite}/lib/pkgconfig\${PKG_CONFIG_PATH:+:}\$PKG_CONFIG_PATH +PKG_CONFIG_PATH=${PROJECT}/${BUILD_sqlite}/lib/pkgconfig\${PKG_CONFIG_PATH:+:}\${PKG_CONFIG_PATH:-} EOF diff --git a/startstop-paperless.build.sh b/startstop-paperless.build.sh deleted file mode 100755 index ecc1e42..0000000 --- a/startstop-paperless.build.sh +++ /dev/null @@ -1,166 +0,0 @@ -#!/bin/bash -x - -. build.profile - -TO=${PROJECT}/startstop-paperless.sh - -install -m 755 <( cat <<_EOP_ -#!/bin/bash -set -e - -export XDG_RUNTIME_DIR=/run/user/\$(id -u \$USER) - -. profile - -[ -d "../${BUILD_TAG}" ] || { echo "BUILD_TAG unset when generated or not a parent directory";exit; } - -declare -A SDS - -slurp(){ IFS='\n' read -r -d '' \${1} || true; } - -#################################################### -slurp SDS["paperless-${BUILD_TAG}-consumer.service"] <<'_EOS_' -[Unit] -Description=Paperless consumer - -[Service] -WorkingDirectory=${PROJECT}/paperless-ngx/src -ExecStart=/bin/bash -c '. ../../profile && python3 manage.py document_consumer' - -[Install] -WantedBy=default.target -_EOS_ - -#################################################### -slurp SDS["paperless-${BUILD_TAG}-scheduler.service"] <<'_EOS_' -[Unit] -Description=Paperless Celery Beat - -[Service] -WorkingDirectory=${PROJECT}/paperless-ngx/src -ExecStart=/bin/bash -c '. ../../profile && celery --app paperless beat --loglevel INFO' - -[Install] -WantedBy=default.target -_EOS_ - -#################################################### -slurp SDS["paperless-${BUILD_TAG}-task-queue.service"] <<'_EOS_' -[Unit] -Description=Paperless Celery Workers -# Requires=redis.service - -[Service] - -WorkingDirectory=${PROJECT}/paperless-ngx/src -ExecStart=/bin/bash -c '. ../../profile && celery --app paperless worker --loglevel WARNING' - -[Install] -WantedBy=default.target -_EOS_ - -#################################################### -slurp SDS["paperless-${BUILD_TAG}-webserver.service"] <<'_EOS_' -[Unit] -Description=Paperless webserver -After=network.target -Wants=network.target -#Requires=paperless-webserver.socket - -[Service] -WorkingDirectory=${PROJECT}/paperless-ngx/src -ExecStart=/bin/bash -c '. ../../profile && gunicorn -c ../gunicorn.conf.py paperless.asgi:application --bind=unix:${DEVSHM}/gunicorn.sock' -StandardOutput=append:${PAPERLESS_LOGGING_DIR}/gunicorn-stdout.log -StandardError=append:${PAPERLESS_LOGGING_DIR}/gunicorn-stderr.log - -[Install] -WantedBy=default.target -_EOS_ - -#################################################### -slurp SDS["paperless-${BUILD_TAG}-redis.service"] <<'_EOS_' -[Unit] -Description=Paperless redis-server -After=network.target -Wants=network.target -#Requires=paperless-webserver.socket - -[Service] -WorkingDirectory=${PROJECT}/paperless-ngx -ExecStart=/bin/bash -c '. ../profile && redis-server --port 0 --unixsocket "${DEVSHM}/redis.sock"' -StandardOutput=file:${PAPERLESS_LOGGING_DIR}/redis.log - -[Install] -WantedBy=default.target -_EOS_ - -#################################################### -slurp SDS["paperless-${BUILD_TAG}-nginx.service"] <<'_EOS_' -[Unit] -Description=The nginx chicken server -After=syslog.target network-online.target -Wants=network-online.target - -[Service] -Type=forking -PIDFile=${DEVSHM}/nginx.pid -ExecStartPre=${PROJECT}/${BUILD_nginx}/sbin/nginx -t -ExecStart=${PROJECT}/${BUILD_nginx}/sbin/nginx -ExecReload=${PROJECT}/${BUILD_nginx}/nginx -s reload -ExecStop=/bin/kill -s QUIT ${DEVSHM}/nginx.pid -PrivateTmp=true -WorkingDirectory=${PROJECT}/paperless-ngx -StandardOutput=append:${PAPERLESS_LOGGING_DIR}/nginx-stdout.log -StandardError=append:${PAPERLESS_LOGGING_DIR}/nginx-stderr.log - -[Install] -WantedBy=default.target -_EOS_ - -#################################################### -SVC=(\${!SDS[@]}) - -case "\$1" in - start) - [ -d ${DEVSHM} ] || mkdir -p ${DEVSHM} - for s in "\${SVC[@]}"; do - echo \$s - systemctl --user start \$s - done - ;; - restart) - for s in "\${SVC[@]}"; do - echo \$s - systemctl --user restart \$s - done - ;; - stop) - for s in "\${SVC[@]}"; do - echo \$s - systemctl --user stop \$s - done - ;; - status) - systemctl --user list-unit-files paperless-${BUILD_TAG}-\* - for s in "\${SVC[@]}"; do - systemctl --user --no-pager --full status \$s || true - done - ;; - install) - mkdir -p \$HOME/.config/systemd/user - for s in "\${SVC[@]}"; do - ( - echo "# I'm from \$0" - cat <<< "\${SDS["\$s"]}" - ) > \$HOME/.config/systemd/user/\$s - systemctl --user disable \$s - done - systemctl --user daemon-reload - ;; - *) - echo "usage: \$0 {start|stop|restart|install|status}" >&2 - exit 1; - ;; -esac -_EOP_ -) ${TO} diff --git a/startstop.build.sh b/startstop.build.sh new file mode 100755 index 0000000..d0fb86a --- /dev/null +++ b/startstop.build.sh @@ -0,0 +1,203 @@ +#!/bin/bash +set -e +set -u + +. build.profile + +TO=${PROJECT}/startstop.sh + +install -m 755 <( cat <<_EOP_ +#!/bin/bash +set -e +set -u + +export XDG_RUNTIME_DIR=/run/user/\$(id -u \$USER) + +. profile + +cd ${PROJECT} + +cmd="\${1:-help}" +srv="\${2:-all}" + +if [[ \${srv} = 'all' ]]; then + for srv in redis nginx gunicorn consumer scheduler worker; do + echo "### ./startstop.sh \${cmd} \${srv}" + ./startstop.sh \${cmd} \${srv} || echo "RET: $?" + done + exit +fi + +PIDFILE=${DEVSHM}/srv-\${srv}.pid +PGIDFILE=${DEVSHM}/srv-\${srv}.pgid + +function rm_pidfiles() { + rm -fv "\${PIDFILE}" "\${PGIDFILE}" +} + +#### nginx #### + +srv_nginx_start() { + nginx -t + trap rm_pidfiles EXIT + nginx -g 'daemon off;' +} + +srv_nginx_restart() { + nginx -s reload +} + +srv_nginx_stop() { + nginx -s quit +} + +#### redis #### + +srv_redis_start() { + # redis is special: it would start under any condition + local _opts=( + --port 0 + --unixsocket "${DEVSHM}/redis.sock" + --pidfile "${DEVSHM}/redis.pid" + ) + trap rm_pidfiles EXIT + redis-server "\${_opts[@]}" +} + +#### gunicorn #### + +srv_gunicorn_start() { + + cd "${PROJECT}/paperless-ngx/src" + + _opts=( + --config ../gunicorn.conf.py + paperless.asgi:application + --bind "unix:${DEVSHM}/gunicorn.sock" + --pid "${DEVSHM}/gunicorn.pid" + --error-logfile "${LOGDIR}/gunicorn-error.log" + --access-logfile "${LOGDIR}/gunicorn-access.log" + --worker-tmp-dir "${DEVSHM}" + ) + trap rm_pidfiles EXIT + gunicorn "\${_opts[@]}" +} + +#### consumer #### + +srv_consumer_start() { + + cd "${PROJECT}/paperless-ngx/src" + trap rm_pidfiles EXIT + + ./manage.py document_consumer +} + +#### scheduler #### + +srv_scheduler_start() { + + cd "${PROJECT}/paperless-ngx/src" + trap rm_pidfiles EXIT + + celery --app paperless beat --loglevel INFO +} + +#### worker #### + +srv_worker_start() { + + cd "${PROJECT}/paperless-ngx/src" + trap rm_pidfiles EXIT + + celery --app paperless worker --loglevel WARNING +} + +#### generic #### + +srv_generic_status() { + if [[ -s "\${PGIDFILE}" ]]; then + local pgid + read -a pgid < "\${PGIDFILE}" + local pids + pids=\$(pgrep -g \${pgid}) + if [[ \$? = 0 ]]; then + ps -f --pid \${pids} + else + echo "processs group \${pgid} has no running processes for \${PGIDFILE}" + fi + else + if [[ -s "\${PIDFILE}" ]]; then + local pid + read -a pid < "\${PIDFILE}" + ps -f --pid \${pid} --ppid \${pid} + else + echo "# no pgid or pid file found: \${PGIDFILE} \${PIDFILE}" + fi + fi +} + +srv_generic_stop() { + if [[ -s "\${PGIDFILE}" ]]; then + local pgid + read -a pgid < "\${PGIDFILE}" + echo "killing process group \${pgid}" + kill -- -\${pgid} + echo "waiting for process group \${pgid} to die" + pwait --pgroup \${pgid} + else + if [[ -s "\${PIDFILE}" ]]; then + local pid + read -a pid < "\${PIDFILE}" + kill -- \${pid} + echo "waiting for process \${pid} to die" + tail -f --pid=\${pid} /dev/null + else + echo "# no pid file found: \${PGIDFILE}" + fi + fi +} + +srv_generic_restart() { + srv_generic_stop + srv_generic_start +} + +#### main() #### + +if [[ \$(type -t srv_\${srv}_\${cmd}) = 'function' ]]; then + case "\${cmd}" in + start) + mkdir -p "${DEVSHM}" "${LOGDIR}" "${TMPDIR}" + [[ -s "\${PIDFILE}" ]] && echo "# \${PIDFILE} found, refuse start of \${srv}" && exit 0 + [[ -s "\${PGIDFILE}" ]] && echo "# \${PGIDFILE} found, refuse start of \${srv}" && exit 0 + + exec 99>"\${PIDFILE}" || exit 1 + flock 99 || exit 1 + + set -m + coproc { + srv_\${srv}_\${cmd}; + } > ${LOGDIR}/srv-\${srv}.log 2>&1 + echo "\${COPROC_PID}" >> "\${PIDFILE}" + ps opgid= "\${COPROC_PID}" > "\${PGIDFILE}" + exit + ;; + *) + echo "########## default fallback to srv_\${srv}_\${cmd} ###############" + srv_\${srv}_\${cmd} + exit + ;; + esac +else + # echo "srv_\${srv}_\${cmd}() not declared" + if [[ \$(type -t srv_generic_\${cmd}) = 'function' ]]; then + srv_generic_\${cmd} + exit + fi +fi + +echo "#### oh no... ####" + +_EOP_ +) ${TO} diff --git a/tesseract.build.sh b/tesseract.build.sh index 8b4e6ee..7f7dfb6 100755 --- a/tesseract.build.sh +++ b/tesseract.build.sh @@ -2,6 +2,7 @@ { set -x set -e +set -u . build.profile @@ -9,11 +10,11 @@ set -e . ${PREFIX}/${BUILD_python}/profile . ${PREFIX}/${BUILD_leptonica}/profile -function B_TESSERACT { +BUILD_PKG=${BUILD_tesseract} -[ -d "$PREFIX" ] || { echo "PREFIX unset or not a directory";exit; } +function B_TESSERACT { -PREFIX=${PREFIX}/${BUILD_tesseract} +PREFIX=${PREFIX}/${BUILD_PKG} mkdir -p $PREFIX @@ -22,16 +23,16 @@ PATH=$PREFIX/bin:\$PATH EOF -if [ ! -d ${BUILD_tesseract} ]; then - if [ ! -e ${BUILD_tesseract}.tar ]; then - git clone https://github.com/tesseract-ocr/tesseract.git --branch ${BUILD_tesseract##*-} --single-branch ${BUILD_tesseract} - tar cf ${BUILD_tesseract}.tar ${BUILD_tesseract} +if [ ! -d ${BUILD_PKG} ]; then + if [ ! -e ${BUILD_PKG}.tar ]; then + git clone https://github.com/tesseract-ocr/tesseract.git --branch ${BUILD_PKG##*-} --single-branch ${BUILD_PKG} + tar cf ${BUILD_PKG}.tar ${BUILD_PKG} else - tar -xf ${BUILD_tesseract}.tar + tar -xf ${BUILD_PKG}.tar fi fi -pushd ${BUILD_tesseract} +pushd ${BUILD_PKG} local _conf=( --prefix ${PREFIX} @@ -42,15 +43,14 @@ local _conf=( PKG_CONFIG_PATH=${PKG_CONFIG_PATH} \ ./configure "${_conf[@]}" -make -j $NPROC -make install +make -j $NPROC install popd } function B_TESSDATA { -PREFIX=${PROJECT}/${BUILD_tesseract} +PREFIX=${PROJECT}/${BUILD_PKG} if [ ! -d tessdata/,git ]; then if [ ! -e tessdata.tar ]; then diff --git a/unpaper.build.sh b/unpaper.build.sh index 17cf1eb..73c59d1 100755 --- a/unpaper.build.sh +++ b/unpaper.build.sh @@ -2,6 +2,7 @@ { set -x set -e +set -u . build.profile @@ -10,8 +11,6 @@ set -e function B_UNPAPER { -[ -d "$PREFIX" ] || { echo "PREFIX unset or not a directory";exit; } - BUILD_PKG=${BUILD_unpaper} PREFIX=${PREFIX}/${BUILD_PKG} @@ -42,7 +41,7 @@ fi cat >$PREFIX/profile <<-EOF PATH=$PREFIX/bin:\$PATH -PKG_CONFIG_PATH=${PROJECT}/${BUILD_PKG}/lib/pkgconfig${PKG_CONFIG_PATH:+:}$PKG_CONFIG_PATH +PKG_CONFIG_PATH=${PROJECT}/${BUILD_PKG}/lib/pkgconfig\${PKG_CONFIG_PATH:+:}\${PKG_CONFIG_PATH:-} EOF cd ${BUILD_PKG} diff --git a/zbar.build.sh b/zbar.build.sh index ee50b36..d838be0 100755 --- a/zbar.build.sh +++ b/zbar.build.sh @@ -2,6 +2,7 @@ { set -x set -e +set -u . build.profile @@ -21,8 +22,8 @@ mkdir -p $PREFIX cat >$PREFIX/profile <<-EOF PATH=$PREFIX/bin:\$PATH -PKG_CONFIG_PATH=${PROJECT}/${BUILD_PKG}/lib/pkgconfig\${PKG_CONFIG_PATH:+:}\$PKG_CONFIG_PATH -LD_LIBRARY_PATH=${PROJECT}/${BUILD_PKG}/lib\${LD_LIBRARY_PATH:+:}\$LD_LIBRARY_PATH +PKG_CONFIG_PATH=${PROJECT}/${BUILD_PKG}/lib/pkgconfig\${PKG_CONFIG_PATH:+:}\${PKG_CONFIG_PATH:-} +LD_LIBRARY_PATH=${PROJECT}/${BUILD_PKG}/lib\${LD_LIBRARY_PATH:+:}\${LD_LIBRARY_PATH:-} EOF