mariux64 · thomas · Feb 1, 2021 · Jan 29, 2021
diff --git a/python-3.8.6-0.build.sh b/python-3.8.6-0.build.sh
@@ -0,0 +1,370 @@
+#! /bin/bash
+
+# COOKIE=$(mcookie|cut -c-8); grep -v V_GREP_ME $0 > /dev/shm/runme-$COOKIE.sh ; sleep 0.3; exec bash /dev/shm/runme-$COOKIE.sh
+# TESTING=1
+
+PKG=python
+VERSION=3.8.6
+BUILD=0
+PYTHONVER=${VERSION%.*}
+
+CUDADIR=/pkg/cuda-10.2.89-1
+
+PREFIX=/pkg/$PKG-$VERSION-$BUILD
+# tensorflow might not fit into /dev/shm
+if [ -n "$TESTING" ]; then PREFIX=/scratch/local2/$PKG-$VERSION-$BUILD ; fi
+
+PATH=/bin:/usr/bin:/usr/sbin:/usr/local/bin
+
+set -xe
+umask 022
+
+BUILD_TMPDIR=/scratch/local2/$PKG-$VERSION-$BUILD.build.tmp
+test -d $BUILD_TMPDIR && ( chmod -R u+rwx $BUILD_TMPDIR || true ; rm -rf $BUILD_TMPDIR )
+mkdir -p $BUILD_TMPDIR/home
+export TMPDIR=$BUILD_TMPDIR
+export HOME=$BUILD_TMPDIR/home
+
+exec </dev/null
+
+mkdir -p $PREFIX
+cat >$PREFIX/profile <<-EOF
+	. $CUDADIR/profile
+	PATH=$PREFIX/bin:\$PATH
+	export LD_LIBRARY_PATH=$PREFIX/lib\${LD_LIBRARY_PATH:+:}\$LD_LIBRARY_PATH
+	PKG_CONFIG_PATH=$PREFIX/lib/pkgconfig\${PKG_CONFIG_PATH:+:}\$PKG_CONFIG_PATH
+	export PKG_CONFIG_PATH
+	if [ -d $PREFIX/.compatlibs ]; then export LD_LIBRARY_PATH=$PREFIX/.compatlibs\${LD_LIBRARY_PATH:+:}\$LD_LIBRARY_PATH ; fi
+EOF
+. $PREFIX/profile
+
+NPROC=$(( $(nproc) * 4 / 5 + 1 ))
+export MAKEFLAGS="-j $NPROC"
+
+BUILDDIR=$PREFIX/build
+
+mkdir -p $BUILDDIR
+cd $BUILDDIR
+
+# former url: https://www.python.org/ftp/python/$VERSION/Python-$VERSION.tar.xz
+test -e Python-$VERSION.tar.xz || wget --no-verbose https://beehive.molgen.mpg.de/ea132d6f449766623eee886966c7d41f/Python-3.8.6.tgz
+test -d Python-$VERSION        || tar xvf Python-$VERSION.tgz
+cd Python-$VERSION
+
+LDFLAGS="-Wl,-rpath=$PREFIX/lib" \
+./configure \
+        --prefix=$PREFIX \
+        --enable-shared
+
+make -j $NPROC
+# make test || true # 'test_socket' seems to run for ever
+make install
+# some parts may just call 'python', defuse
+# ( cd $PREFIX/bin; ln -s python3.8 python)
+ln -s python3.8 $PREFIX/bin/python
+
+( # sip: tool to create Python bindings for C and C++ libraries.
+  cd $PREFIX/build
+  SIP_VERSION=4.19.18
+  # former url: https://www.riverbankcomputing.com/static/Downloads/sip/$SIP_VERSION/sip-$SIP_VERSION.tar.gz
+  SIP_SRC=https://beehive.molgen.mpg.de/c06bfa2206ddebbff983c272b40a84b9/sip-4.19.18.tar.gz
+  test -e sip-$SIP_VERSION.tar.gz || wget --no-verbose $SIP_SRC
+  test -d sip-$SIP_VERSION || tar -xf sip-$SIP_VERSION.tar.gz
+
+  cd sip-$SIP_VERSION
+
+  #### ATTN!
+  #### python meanders between /python${PYTHONVER} and /python${PYTHONVER}m
+
+  python3 configure.py \
+      -b $PREFIX/bin \
+      -d $PREFIX/lib/python$PYTHONVER/site-packages \
+      -e $PREFIX/include/python${PYTHONVER} \
+      -v $PREFIX/share/sip \
+      --sip-module=sip
+
+  # well, right here share/sip is spoilage, but maybe some packages will use the decent sip later on ...
+  mkdir -p $PREFIX/share/sip
+  make -j $NPROC
+  make install
+)
+
+
+( # now for the private (sic!) version required by newer PyQt
+  cd $PREFIX/build
+  mkdir -p private_sip
+  cd private_sip
+  SIP_VERSION=4.19.18
+  # former url: https://www.riverbankcomputing.com/static/Downloads/sip/$SIP_VERSION/sip-$SIP_VERSION.tar.gz
+  SIP_SRC=https://beehive.molgen.mpg.de/c06bfa2206ddebbff983c272b40a84b9/sip-4.19.18.tar.gz
+  test -e sip-$SIP_VERSION.tar.gz || wget --no-verbose $SIP_SRC
+  test -d sip-$SIP_VERSION || tar -xf sip-$SIP_VERSION.tar.gz
+
+  cd sip-$SIP_VERSION
+
+  # note $PREFIX/share/sip/PyQt5 is mandatory
+  python3 configure.py \
+      -d $PREFIX/lib/python$PYTHONVER/site-packages \
+      -v $PREFIX/share/sip/PyQt5 \
+      --no-tools \
+      --sip-module=PyQt5.sip
+
+  mkdir -p $PREFIX/share/sip/PyQt5
+  make -j $NPROC
+  # The resulting sip.so in site-packages/PyQt5 differs indeed a bit (grep for PyQt5)
+  make install
+)
+
+(
+  PYQT_VERSION=5.13.0
+  # former url: https://www.riverbankcomputing.com/static/Downloads/PyQt5/$PYQT_VERSION/PyQt5_gpl-$PYQT_VERSION.tar.gz
+  PYQT_SRC=https://beehive.molgen.mpg.de/0848fa62dc99ecf2e96f7cdda727c8a2/PyQt5_gpl-5.13.0.tar.gz
+  cd $BUILDDIR
+  test -e PyQt5_gpl-$PYQT_VERSION.tar.gz || wget --no-verbose $PYQT_SRC
+  test -d PyQt5_gpl-$PYQT_VERSION || tar -xf PyQt5_gpl-$PYQT_VERSION.tar.gz
+
+  cd PyQt5_gpl-$PYQT_VERSION
+
+  #### ATTN!
+  #### python meanders between /python${PYTHONVER} and /python${PYTHONVER}m
+
+  # export QTDIR=/usr/local/qt5
+  python3 configure.py \
+    -b $PREFIX/bin \
+    -d $PREFIX/lib/python$PYTHONVER/site-packages \
+    -n $PREFIX/share/qsci \
+    -q /usr/local/qt5/bin/qmake \
+    -v $PREFIX/share/sip/PyQt5 \
+    --sip-incdir=$PREFIX/include/python${PYTHONVER} \
+    --confirm-license \
+    --no-qml-plugin \
+    --no-designer-plugin \
+    --no-dist-info \
+    --verbose
+
+  make -j $NPROC
+
+  # Did I miss something in the docs ???
+  # w/o the *.so files inplace a clean 'make install' fails with file not found ...
+  # -> FileNotFoundError: [Errno 2] No such file or directory:
+  #         '/dev/shm/python-3.7.4-0/lib/python3.7/site-packages/PyQt5/QtCore.so'
+  mkdir -p $PREFIX/lib/python$PYTHONVER/site-packages/PyQt5
+  for F in `find | grep '\.so$' | grep -v '/lib'` ; do
+    cp -vp $F $PREFIX/lib/python$PYTHONVER/site-packages/PyQt5/
+  done
+  make install
+)
+
+(
+  QSCINTILLA_VERSION=2.11.2
+  # former url: https://www.riverbankcomputing.com/static/Downloads/QScintilla/$QSCINTILLA_VERSION/QScintilla_gpl-$QSCINTILLA_VERSION.tar.gz
+  QSCINTILLA_SRC=https://beehive.molgen.mpg.de/2ec4833e790e44698c50ec703f274bab/QScintilla_gpl-2.11.2.tar.gz
+
+  cd $BUILDDIR
+  test -e QScintilla_gpl-$QSCINTILLA_VERSION.tar.gz || wget --no-verbose $QSCINTILLA_SRC
+  test -d QScintilla_gpl-$QSCINTILLA_VERSION || tar -xf QScintilla_gpl-$QSCINTILLA_VERSION.tar.gz
+  cd QScintilla_gpl-$QSCINTILLA_VERSION
+
+  cd Python
+
+  # apidir follows PyQt, not Qt (default is: QT_INSTALL_DATA/qsci)
+  # creation of PEP 376 .dist-info fails in $PREFIX/share
+  python3 configure.py \
+    --pyqt=PyQt5 \
+    --qmake=/usr/local/qt5/bin/qmake \
+    --apidir=$PREFIX/share/qsci \
+    --no-dist-info
+
+  make -j $NPROC
+
+  # Call it a deja-vu, nerve ...
+  # -> FileNotFoundError: [Errno 2] No such file or directory: '/dev/shm/python-3.7.4-0/lib/python3.7/site-packages/PyQt5/Qsci.so'
+  mkdir -p $PREFIX/lib/python$PYTHONVER/site-packages/PyQt5
+  for F in `find | grep '\.so$' | grep -v '/lib'` ; do
+    cp -vp $F $PREFIX/lib/python$PYTHONVER/site-packages/PyQt5
+  done
+  make install
+)
+
+python3 -m ensurepip
+pip3 install --prefix=$PREFIX -I pip
+
+pip3 install --prefix=$PREFIX ipython[all]
+
+pip3 install --prefix=$PREFIX SciPy NumPy Matplotlib pandas SymPy nose statsmodels
+# clean up some dirt that statsmodels placed also into $PREFIX
+rm -vf $PREFIX/LICENSE.txt
+rm -vf $PREFIX/setup.cfg
+
+pip3 install --prefix=$PREFIX Cython
+
+pip3 install --prefix=$PREFIX pysam
+pip3 install --prefix=$PREFIX seaborn
+pip3 install --prefix=$PREFIX scikit-learn
+#pip3 install --prefix=$PREFIX metaseq        # fails (Latest commit fa875d1 on Jan 10, 2017)
+pip3 install --prefix=$PREFIX Mako
+pip3 install --prefix=$PREFIX meson          # required in mariux
+
+pip3 install --prefix=$PREFIX cutadapt       # Helpdesk 01/24/18 16:19, Sabrina Krakau
+pip3 install --prefix=$PREFIX pygobject      # Helpdesk 03/26/18 14:19, Virginie Stanisla
+pip3 install --prefix=$PREFIX blink1         # Blink1-lib 07/04/18 12:00, Niclas Hofmann
+
+pip3 install --prefix=$PREFIX umi-tools
+pip3 install --prefix=$PREFIX sphinx
+pip3 install --prefix=$PREFIX recommonmark
+pip3 install --prefix=$PREFIX fastcluster
+pip3 install --prefix=$PREFIX pydot
+pip3 install --prefix=$PREFIX pygraphviz
+pip3 install --prefix=$PREFIX deeptools
+pip3 install --prefix=$PREFIX HTSeq
+pip3 install --prefix=$PREFIX keras
+pip3 install --prefix=$PREFIX MotifScan      # Alena 30.8.2018
+pip3 install --prefix=$PREFIX MAmotif
+pip3 install --prefix=$PREFIX hic2cool       # Helpdesk Robert Schoepflin 17.09.2018
+#pip3 install --prefix=$PREFIX hifive         # only python 2.7+
+pip3 install --prefix=$PREFIX snakemake
+pip3 install --prefix=$PREFIX virtualenv     # Donald
+pip3 install --prefix=$PREFIX odfpy          # Donald
+pip3 install --prefix=$PREFIX Dumper         # Thomas
+pip3 install --prefix=$PREFIX igraph
+pip3 install --prefix=$PREFIX louvain        # monocle3 prep for Helene
+pip3 install --prefix=$PREFIX selenium       # 19.7.19 Kreitler  -> https://webxray.org/
+pip3 install --prefix=$PREFIX cooltools      # Maria Valieva 17.7.2019
+pip3 install --prefix=$PREFIX gmpy           # From viper context (Helpdesk Laura Glaser 9.8.2019)
+pip3 install --prefix=$PREFIX torch
+pip3 install --prefix=$PREFIX torchvision
+pip3 install --prefix=$PREFIX plotly
+pip3 install --prefix=$PREFIX pyOpenCL
+pip3 install --prefix=$PREFIX docopt
+pip3 install --prefix=$PREFIX dbus-python
+pip3 install --prefix=$PREFIX distro
+pip3 install --prefix=$PREFIX notify
+pip3 install --prefix=$PREFIX ply
+pip3 install --prefix=$PREFIX threadpoolctl
+
+pip3 install --prefix=$PREFIX pyopengl
+# creates error, so remove it
+rm -rf ${PREFIX}/lib/python$PYTHONVER/site-packages/OpenGL/Tk
+
+# pip3 install --prefix=$PREFIX HiCExplorer  # Would be a nice to have, but it likes to downgrade too much, pick some cherries
+pip3 install --prefix=$PREFIX HiCMatrix
+pip3 install --prefix=$PREFIX pybedtools
+pip3 install --prefix=$PREFIX gffutils
+# pyGenomeTracks, version 3.6, Requires-Dist: matplotlib (<=3.3.2,>=3.1.1), somehow they don't catch it ...
+pip3 install --prefix=$PREFIX --no-deps "pyGenomeTracks" # example produces plots, so give it a chance
+pip3 install --prefix=$PREFIX python-bioformats
+
+pip3 install --prefix=$PREFIX jupyterlab      # request Peter Arndt
+pip3 install --prefix=$PREFIX jupyterlab-git
+pip3 install --prefix=$PREFIX jupyterlab-git
+pip3 install --prefix=$PREFIX igv_jupyterlab  # Matthias Lienhard HD 01-04-2020
+pip3 install --prefix=$PREFIX igv-jupyter
+mkdir -vp $PREFIX/share/jupyter/lab/settings  # try to defeat a silly error in jupyter labextension install
+
+pip3 install --prefix=$PREFIX libxml2-python3 # required by several bee packages after swich to python3
+pip3 install --prefix=$PREFIX scour           # https://github.molgen.mpg.de/mariux64/pkg-scripts/issues/98
+pip3 install --prefix=$PREFIX multiqc
+
+pip3 install --prefix=$PREFIX thonny          # nice little ide, kreitler
+pip3 install --prefix=$PREFIX pybind11
+pip3 install --prefix=$PREFIX svgwrite
+pip3 install --prefix=$PREFIX pyomo
+pip3 install --prefix=$PREFIX jsbsim          # kreitler, hobby
+
+pip3 install --prefix=$PREFIX bioservices
+pip3 install --prefix=$PREFIX pairtools
+pip3 install --prefix=$PREFIX SharedArray
+pip3 install --prefix=$PREFIX pyabc           # dependency for stochkit, HD 20.07.20 Gemma Noviello
+pip3 install --prefix=$PREFIX petab
+pip3 install --prefix=$PREFIX tadtool         # HD 26.01.21 Michael Robson
+
+#boost for pycuda
+(
+  . ${PREFIX}/profile
+  BOOST_VERSION=(1.72.0 1_72_0)
+  # former url:https://dl.bintray.com/boostorg/release/${BOOST_VERSION[0]}/source/boost_${BOOST_VERSION[1]}.tar.gz
+  BOOST_SRC=https://beehive.molgen.mpg.de/e2b0b1eac302880461bcbef097171758/boost_1_72_0.tar.gz
+  PKGCFLAGS="-O2 -fPIC"
+
+  cd $BUILDDIR
+  test -e boost_${BOOST_VERSION[1]}.tar.gz || wget --no-verbose $BOOST_SRC
+  test -d boost-${BOOST_VERSION[0]} || mkdir boost-${BOOST_VERSION[0]} && tar -xf boost_${BOOST_VERSION[1]}.tar.gz -C boost-${BOOST_VERSION[0]} --strip-components=1
+
+  #### ATTN!
+  #### python meanders between /python${PYTHONVER} and /python${PYTHONVER}m
+
+  cd boost-${BOOST_VERSION[0]}
+  sed -e "/using python/ s|;|: ${PREFIX}/include/python${PYTHONVER} ;|" \
+      -i bootstrap.sh
+
+  ./bootstrap.sh \
+    --with-toolset=gcc \
+    --with-icu \
+    --with-python=$PREFIX/bin/python3 \
+    --with-libraries=python,thread || exit 1
+
+  ./b2 \
+    $NUMJOBS \
+    --layout=system \
+    --build-dir=build-python3 \
+    --prefix=$PREFIX \
+    --libdir=$PREFIX/lib \
+    --build-type=minimal \
+    toolset=gcc \
+    variant=release \
+    debug-symbols=off \
+    link=shared \
+    threading=multi \
+    runtime-link=shared \
+    python=$PYTHONVER \
+    cflags="$PKGCFLAGS" \
+    cxxflags="$PKGCFLAGS" \
+    install || exit 1
+)
+
+#pycuda git version to get GL support
+(
+  PYCUDAVERS=2019.1.2
+  PYTHON_LIBNAME=boost_python38
+
+  . ${PREFIX}/profile
+  cd $BUILDDIR
+  # former url: https://github.com/inducer/pycuda/archive/v${PYCUDAVERS}/pycuda-${PYCUDAVERS}.tar.gz
+  test -e pycuda-${PYCUDAVERS}.tar.gz || wget --no-verbose https://beehive.molgen.mpg.de/2fdf39e15a9c01e30e97144063b571dc/pycuda-2019.1.2.tar.gz
+  test -d pycuda-${PYCUDAVERS} || tar -xf pycuda-${PYCUDAVERS}.tar.gz
+  #test -d pycuda || git clone --recursive http://git.tiker.net/trees/pycuda.git
+
+  cd pycuda-${PYCUDAVERS}
+  test -e siteconf.py && rm siteconf.py
+  python3 configure.py \
+    --cuda-enable-gl \
+    --cuda-root=$CUDADIR \
+    --no-use-shipped-boost \
+    --boost-python-libname=$PYTHON_LIBNAME \
+    --boost-inc-dir=$PREFIX/include \
+    --boost-lib-dir=$PREFIX/lib \
+    --cudadrv-lib-dir='${CUDA_ROOT}/lib,${CUDA_ROOT}/lib/stubs' \
+    --cudart-lib-dir='${CUDA_ROOT}/lib,${CUDA_ROOT}/lib/stubs' \
+    --curand-lib-dir='${CUDA_ROOT}/lib,${CUDA_ROOT}/lib/stubs'\
+    --ldflags=-L$PREFIX/lib
+
+  python3 setup.py install --prefix $PREFIX
+)
+
+
+# bamToBigWig - just for the taste, HD 20.1.21 T. Zehnder
+(
+   # https://github.com/PanosFirmpas/bamToBigWig
+   cd $PREFIX/bin
+   wget --no-verbose https://beehive.molgen.mpg.de/bc6815740fbec1586054ad42e0b90a2d/bamToBigWig
+   chmod +x bamToBigWig
+)
+
+# tensorflow is omitted in this build.
+
+# try to load all packages, warnings/errors are not fatal, but give some additional hints.
+echo "# START load test."
+python3 -c 'help("modules")' > /dev/null
+echo "# END load test."
+
+exit