From 6c8440435e5bf721d0b59f5e64da6e209815edf9 Mon Sep 17 00:00:00 2001
From: Peter Marquardt <wwwutz@molgen.mpg.de>
Date: Thu, 15 Feb 2024 09:24:35 +0100
Subject: [PATCH] add ffmpeg & pngquant

---
 .gitignore              |  2 ++
 build.profile           |  2 ++
 buildall.sh             |  2 ++
 cleanall.sh             |  2 ++
 ffmpeg.build.sh         | 48 +++++++++++++++++++++++++++++++
 paperless-ngx.build.sh  |  3 ++
 paperless.conf.build.sh |  3 +-
 pngquant.build.sh       | 63 +++++++++++++++++++++++++++++++++++++++++
 profile.build.sh        |  2 ++
 unpaper.build.sh        |  2 ++
 10 files changed, 128 insertions(+), 1 deletion(-)
 create mode 100755 ffmpeg.build.sh
 create mode 100755 pngquant.build.sh

diff --git a/.gitignore b/.gitignore
index 609ffa8..7a08b92 100644
--- a/.gitignore
+++ b/.gitignore
@@ -5,12 +5,14 @@
 /leptonica-*
 /nginx-*
 /node-*
+/pngquant-*
 /sqlite-*
 /tessconfigs
 /tessdata
 /tesseract-*
 /unpaper-*
 /zbar-*
+/ffmpeg-*
 *.tar
 *.tar.gz
 *.tar.xz
diff --git a/build.profile b/build.profile
index 85ea90d..f5f1fee 100644
--- a/build.profile
+++ b/build.profile
@@ -14,6 +14,8 @@ TMPDIR=${TMPDIR:-/tmp/${USER}/${BUILD_TAG}}
 BUILD_sqlite=sqlite-3.45.1
 BUILD_python=Python-3.11.8
 BUILD_node=node-20.11.0
+BUILD_ffmpeg=ffmpeg-6.1.1
+BUILD_pngquant=pngquant-3.0.3-2-g88da53e
 BUILD_unpaper=unpaper-7.0.0-99-gf41e9bb
 BUILD_zbar=zbar-0.23.93
 BUILD_leptonica=leptonica-1.84.1
diff --git a/buildall.sh b/buildall.sh
index 1a19d9d..17369f6 100755
--- a/buildall.sh
+++ b/buildall.sh
@@ -61,6 +61,8 @@ ln -fs $logfile $LOGS/build.log
 [ -d ${BUILD_sqlite} ] || ./sqlite.build.sh
 [ -d ${BUILD_python} ] || ./Python.build.sh
 [ -d ${BUILD_node} ] || ./node.build.sh
+[ -d ${BUILD_ffmpeg} ] || ./ffmpeg.build.sh
+[ -d ${BUILD_pngquant} ] || ./pngquant.build.sh
 [ -d ${BUILD_unpaper} ] || ./unpaper.build.sh
 [ -d ${BUILD_zbar} ] || ./zbar.build.sh
 [ -d ${BUILD_leptonica} ] || ./leptonica.build.sh
diff --git a/cleanall.sh b/cleanall.sh
index 34fdf3e..6ceb1ff 100644
--- a/cleanall.sh
+++ b/cleanall.sh
@@ -10,7 +10,9 @@ RMRF=(
   $BUILD_sqlite
   $BUILD_python
   $BUILD_node
+  $BUILD_ffmpeg
   $BUILD_unpaper
+  $BUILD_pngquant
   $BUILD_zbar
   $BUILD_leptonica
   $BUILD_tesseract
diff --git a/ffmpeg.build.sh b/ffmpeg.build.sh
new file mode 100755
index 0000000..660857c
--- /dev/null
+++ b/ffmpeg.build.sh
@@ -0,0 +1,48 @@
+#!/bin/bash
+{
+set -x
+set -e
+
+. build.profile
+
+function B_FFMPEG {
+
+[ -d "$PREFIX" ] || { echo "PREFIX unset or not a directory";exit; }
+
+# SRCURL[0]="https://ffmpeg.org//releases/${BUILD_ffmpeg}.tar.gz"
+SRCURL[0]="https://beehive.molgen.mpg.de/cce359cad7ed0d4f0079f7864080ad36/ffmpeg-6.1.1.tar.gz"
+
+PREFIX=${PREFIX}/${BUILD_ffmpeg}
+
+mkdir -p $PREFIX
+
+cat >$PREFIX/profile <<-EOF
+PATH=$PREFIX/bin:\$PATH
+PKG_CONFIG_PATH=${PROJECT}/${BUILD_ffmpeg}/lib/pkgconfig${PKG_CONFIG_PATH:+:}$PKG_CONFIG_PATH
+EOF
+
+
+test -e ${BUILD_ffmpeg}.tar.gz || wget -nv ${SRCURL} -O ${BUILD_ffmpeg}.tar.gz
+test -d ${BUILD_ffmpeg} || mkdir -pv ${BUILD_ffmpeg} && tar -xf ${BUILD_ffmpeg}.tar.gz --strip-components=1 -C ${BUILD_ffmpeg}
+
+cd ${BUILD_ffmpeg}
+
+local _conf=(
+    --prefix=${PREFIX}
+    --enable-gpl
+    --enable-version3
+    --enable-nonfree
+    --enable-gray
+    --disable-doc
+)
+
+./configure "${_conf[@]}"
+
+make -j $NPROC
+make install
+
+}
+
+B_FFMPEG
+
+}
diff --git a/paperless-ngx.build.sh b/paperless-ngx.build.sh
index f2bc68b..a1bd5f7 100755
--- a/paperless-ngx.build.sh
+++ b/paperless-ngx.build.sh
@@ -5,6 +5,9 @@ set -e
 
 . build.profile
 
+. ${PREFIX}/${BUILD_pngquant}/profile
+. ${PREFIX}/${BUILD_ffmpeg}/profile
+. ${PREFIX}/${BUILD_unpaper}/profile
 . ${PREFIX}/${BUILD_sqlite}/profile
 . ${PREFIX}/${BUILD_zbar}/profile
 . ${PREFIX}/${BUILD_python}/profile
diff --git a/paperless.conf.build.sh b/paperless.conf.build.sh
index 44566a7..3e8f123 100755
--- a/paperless.conf.build.sh
+++ b/paperless.conf.build.sh
@@ -24,8 +24,9 @@ _sed=(
   -e "/#PAPERLESS_SECRET_KEY=change-me/ a PAPERLESS_SECRET_KEY=${PAPERLESS_SECRET_KEY}"
 #  -e '/#PAPERLESS_AUTO_LOGIN_USERNAME=/ a PAPERLESS_AUTO_LOGIN_USERNAME=paperless'
   -e '/#PAPERLESS_OCR_LANGUAGE=/ a PAPERLESS_OCR_LANGUAGE=deu+eng'
-  -e '/#PAPERLESS_OCR_MODE=skip/ a PAPERLESS_OCR_MODE=redo'
+  -e '/#PAPERLESS_OCR_MODE=skip/ a PAPERLESS_OCR_MODE=force'
   -e "/#PAPERLESS_OCR_LANGUAGE=/ a PAPERLESS_NLTK_DIR=${PROJECT}/data/nltk"
+  -e "/#PAPERLESS_OCR_CLEAN=/ a PAPERLESS_OCR_CLEAN=clean-final"
   -e "/#PAPERLESS_CONSUMPTION_DIR=/ a PAPERLESS_CONSUMPTION_DIR=${PROJECT}/consume"
   -e "/#PAPERLESS_CONSUMPTION_DIR=/ a PAPERLESS_LOGGING_DIR=${PROJECT}/log"
   -e '/#PAPERLESS_CONSUMER_RECURSIVE=/ a PAPERLESS_CONSUMER_RECURSIVE=true'
diff --git a/pngquant.build.sh b/pngquant.build.sh
new file mode 100755
index 0000000..c0ae32a
--- /dev/null
+++ b/pngquant.build.sh
@@ -0,0 +1,63 @@
+#!/bin/bash
+{
+set -x
+set -e
+
+. build.profile
+
+. ${PREFIX}/${BUILD_python}/profile
+
+function B_PNGQUANT {
+
+[ -d "$PREFIX" ] || { echo "PREFIX unset or not a directory";exit; }
+
+BUILD_PKG=${BUILD_pngquant}
+
+PREFIX=${PREFIX}/${BUILD_PKG}
+
+if [ ! -d ${BUILD_pngquant} ]; then
+  if [ ! -e ${BUILD_pngquant}.tar ]; then
+    git clone --recursive https://github.com/kornelski/pngquant.git --single-branch ${BUILD_pngquant}
+    pushd ${BUILD_pngquant}
+    GITVER=$(git describe --tags)
+    popd
+    if [ "${GITVER}" != "${BUILD_pngquant##pngquant-}" ]; then
+      echo "${GITVER} should be ${BUILD_pngquant}. update!"
+      exit 1
+    fi
+    tar cf ${BUILD_pngquant}.tar ${BUILD_pngquant}
+  else
+    tar -xf ${BUILD_pngquant}.tar
+  fi
+fi
+
+if [ ! -d ${PREFIX} ]; then
+  git clone ${BUILD_pngquant} $PREFIX
+else
+  pushd ${PREFIX}
+  git pull origin
+  popd
+fi
+
+cat >$PREFIX/profile <<-EOF
+PATH=$PREFIX/bin:\$PATH
+PKG_CONFIG_PATH=${PROJECT}/${BUILD_PKG}/lib/pkgconfig${PKG_CONFIG_PATH:+:}$PKG_CONFIG_PATH
+EOF
+
+cd ${BUILD_PKG}
+
+local _conf=(
+  --release
+)
+
+[ -d /usr/local/package/bin ] && PATH=${PATH}:/usr/local/package/bin
+
+CARGO_HOME=.cargo cargo build "${_conf[@]}"
+
+install -v -Dm0755 ./target/release/pngquant $PREFIX/bin/pngquant
+
+}
+
+B_PNGQUANT
+
+}
diff --git a/profile.build.sh b/profile.build.sh
index ec65f54..46b552f 100755
--- a/profile.build.sh
+++ b/profile.build.sh
@@ -18,6 +18,8 @@ PATH=$HOME/bin:/bin:/sbin:/usr/bin:/usr/sbin
 . ${PROJECT}/${BUILD_node}/profile
 . ${PROJECT}/${BUILD_nginx}/profile
 . ${PROJECT}/${BUILD_unpaper}/profile
+. ${PROJECT}/${BUILD_ffmpeg}/profile
+. ${PROJECT}/${BUILD_pngquant}/profile
 . ${PROJECT}/${BUILD_leptonica}/profile
 . ${PROJECT}/${BUILD_tesseract}/profile
 
diff --git a/unpaper.build.sh b/unpaper.build.sh
index ec4a1e6..17cf1eb 100755
--- a/unpaper.build.sh
+++ b/unpaper.build.sh
@@ -6,6 +6,7 @@ set -e
 . build.profile
 
 . ${PREFIX}/${BUILD_python}/profile
+. ${PREFIX}/${BUILD_ffmpeg}/profile
 
 function B_UNPAPER {
 
@@ -55,6 +56,7 @@ local _cflags=(
 local _conf=(
     --prefix ${PREFIX}
     --buildtype=debugoptimized
+    --pkg-config-path=${PKG_CONFIG_PATH}
 )
 
 CFLAGS="${_cflags[@]}" meson setup "${_conf[@]}" builddir