python-3.8.6-0.build.sh

#! /bin/bash

# COOKIE=$(mcookie|cut -c-8); grep -v V_GREP_ME $0 > /dev/shm/runme-$COOKIE.sh ; sleep 0.3; exec bash /dev/shm/runme-$COOKIE.sh
# TESTING=1

PKG=python
VERSION=3.8.6
BUILD=0
PYTHONVER=${VERSION%.*}

CUDADIR=/pkg/cuda-10.2.89-1

PREFIX=/pkg/$PKG-$VERSION-$BUILD
# tensorflow might not fit into /dev/shm
if [ -n "$TESTING" ]; then PREFIX=/scratch/local2/$PKG-$VERSION-$BUILD ; fi

PATH=/bin:/usr/bin:/usr/sbin:/usr/local/bin

set -xe
umask 022

BUILD_TMPDIR=/scratch/local2/$PKG-$VERSION-$BUILD.build.tmp
test -d $BUILD_TMPDIR && ( chmod -R u+rwx $BUILD_TMPDIR || true ; rm -rf $BUILD_TMPDIR )
mkdir -p $BUILD_TMPDIR/home
export TMPDIR=$BUILD_TMPDIR
export HOME=$BUILD_TMPDIR/home

exec </dev/null

mkdir -p $PREFIX
cat >$PREFIX/profile <<-EOF
	. $CUDADIR/profile
	PATH=$PREFIX/bin:\$PATH
	export LD_LIBRARY_PATH=$PREFIX/lib\${LD_LIBRARY_PATH:+:}\$LD_LIBRARY_PATH
	PKG_CONFIG_PATH=$PREFIX/lib/pkgconfig\${PKG_CONFIG_PATH:+:}\$PKG_CONFIG_PATH
	export PKG_CONFIG_PATH
	if [ -d $PREFIX/.compatlibs ]; then export LD_LIBRARY_PATH=$PREFIX/.compatlibs\${LD_LIBRARY_PATH:+:}\$LD_LIBRARY_PATH ; fi
EOF
. $PREFIX/profile

NPROC=$(( $(nproc) * 4 / 5 + 1 ))
export MAKEFLAGS="-j $NPROC"

BUILDDIR=$PREFIX/build

mkdir -p $BUILDDIR
cd $BUILDDIR

# former url: https://www.python.org/ftp/python/$VERSION/Python-$VERSION.tar.xz
test -e Python-$VERSION.tar.xz || wget --no-verbose https://beehive.molgen.mpg.de/ea132d6f449766623eee886966c7d41f/Python-3.8.6.tgz
test -d Python-$VERSION        || tar xvf Python-$VERSION.tgz
cd Python-$VERSION

LDFLAGS="-Wl,-rpath=$PREFIX/lib" \
./configure \
        --prefix=$PREFIX \
        --enable-shared

make -j $NPROC
# make test || true # 'test_socket' seems to run for ever
make install
# some parts may just call 'python', defuse
# ( cd $PREFIX/bin; ln -s python3.8 python)
ln -s python3.8 $PREFIX/bin/python

( # sip: tool to create Python bindings for C and C++ libraries.
  cd $PREFIX/build
  SIP_VERSION=4.19.18
  # former url: https://www.riverbankcomputing.com/static/Downloads/sip/$SIP_VERSION/sip-$SIP_VERSION.tar.gz
  SIP_SRC=https://beehive.molgen.mpg.de/c06bfa2206ddebbff983c272b40a84b9/sip-4.19.18.tar.gz
  test -e sip-$SIP_VERSION.tar.gz || wget --no-verbose $SIP_SRC
  test -d sip-$SIP_VERSION || tar -xf sip-$SIP_VERSION.tar.gz

  cd sip-$SIP_VERSION
  
  #### ATTN!
  #### python meanders between /python${PYTHONVER} and /python${PYTHONVER}m

  python3 configure.py \
      -b $PREFIX/bin \
      -d $PREFIX/lib/python$PYTHONVER/site-packages \
      -e $PREFIX/include/python${PYTHONVER} \
      -v $PREFIX/share/sip \
      --sip-module=sip

  # well, right here share/sip is spoilage, but maybe some packages will use the decent sip later on ...
  mkdir -p $PREFIX/share/sip
  make -j $NPROC
  make install
)


( # now for the private (sic!) version required by newer PyQt
  cd $PREFIX/build
  mkdir -p private_sip
  cd private_sip
  SIP_VERSION=4.19.18
  # former url: https://www.riverbankcomputing.com/static/Downloads/sip/$SIP_VERSION/sip-$SIP_VERSION.tar.gz
  SIP_SRC=https://beehive.molgen.mpg.de/c06bfa2206ddebbff983c272b40a84b9/sip-4.19.18.tar.gz
  test -e sip-$SIP_VERSION.tar.gz || wget --no-verbose $SIP_SRC
  test -d sip-$SIP_VERSION || tar -xf sip-$SIP_VERSION.tar.gz

  cd sip-$SIP_VERSION

  # note $PREFIX/share/sip/PyQt5 is mandatory
  python3 configure.py \
      -d $PREFIX/lib/python$PYTHONVER/site-packages \
      -v $PREFIX/share/sip/PyQt5 \
      --no-tools \
      --sip-module=PyQt5.sip

  mkdir -p $PREFIX/share/sip/PyQt5
  make -j $NPROC
  # The resulting sip.so in site-packages/PyQt5 differs indeed a bit (grep for PyQt5)
  make install
)

(
  PYQT_VERSION=5.13.0
  # former url: https://www.riverbankcomputing.com/static/Downloads/PyQt5/$PYQT_VERSION/PyQt5_gpl-$PYQT_VERSION.tar.gz
  PYQT_SRC=https://beehive.molgen.mpg.de/0848fa62dc99ecf2e96f7cdda727c8a2/PyQt5_gpl-5.13.0.tar.gz
  cd $BUILDDIR
  test -e PyQt5_gpl-$PYQT_VERSION.tar.gz || wget --no-verbose $PYQT_SRC
  test -d PyQt5_gpl-$PYQT_VERSION || tar -xf PyQt5_gpl-$PYQT_VERSION.tar.gz

  cd PyQt5_gpl-$PYQT_VERSION

  #### ATTN!
  #### python meanders between /python${PYTHONVER} and /python${PYTHONVER}m

  # export QTDIR=/usr/local/qt5
  python3 configure.py \
    -b $PREFIX/bin \
    -d $PREFIX/lib/python$PYTHONVER/site-packages \
    -n $PREFIX/share/qsci \
    -q /usr/local/qt5/bin/qmake \
    -v $PREFIX/share/sip/PyQt5 \
    --sip-incdir=$PREFIX/include/python${PYTHONVER} \
    --confirm-license \
    --no-qml-plugin \
    --no-designer-plugin \
    --no-dist-info \
    --verbose

  make -j $NPROC

  # Did I miss something in the docs ???
  # w/o the *.so files inplace a clean 'make install' fails with file not found ...
  # -> FileNotFoundError: [Errno 2] No such file or directory:
  #         '/dev/shm/python-3.7.4-0/lib/python3.7/site-packages/PyQt5/QtCore.so'
  mkdir -p $PREFIX/lib/python$PYTHONVER/site-packages/PyQt5
  for F in `find | grep '\.so$' | grep -v '/lib'` ; do
    cp -vp $F $PREFIX/lib/python$PYTHONVER/site-packages/PyQt5/
  done
  make install
)

(
  QSCINTILLA_VERSION=2.11.2
  # former url: https://www.riverbankcomputing.com/static/Downloads/QScintilla/$QSCINTILLA_VERSION/QScintilla_gpl-$QSCINTILLA_VERSION.tar.gz
  QSCINTILLA_SRC=https://beehive.molgen.mpg.de/2ec4833e790e44698c50ec703f274bab/QScintilla_gpl-2.11.2.tar.gz

  cd $BUILDDIR
  test -e QScintilla_gpl-$QSCINTILLA_VERSION.tar.gz || wget --no-verbose $QSCINTILLA_SRC
  test -d QScintilla_gpl-$QSCINTILLA_VERSION || tar -xf QScintilla_gpl-$QSCINTILLA_VERSION.tar.gz
  cd QScintilla_gpl-$QSCINTILLA_VERSION

  cd Python

  # apidir follows PyQt, not Qt (default is: QT_INSTALL_DATA/qsci)
  # creation of PEP 376 .dist-info fails in $PREFIX/share
  python3 configure.py \
    --pyqt=PyQt5 \
    --qmake=/usr/local/qt5/bin/qmake \
    --apidir=$PREFIX/share/qsci \
    --no-dist-info

  make -j $NPROC

  # Call it a deja-vu, nerve ...
  # -> FileNotFoundError: [Errno 2] No such file or directory: '/dev/shm/python-3.7.4-0/lib/python3.7/site-packages/PyQt5/Qsci.so'
  mkdir -p $PREFIX/lib/python$PYTHONVER/site-packages/PyQt5
  for F in `find | grep '\.so$' | grep -v '/lib'` ; do
    cp -vp $F $PREFIX/lib/python$PYTHONVER/site-packages/PyQt5
  done
  make install
)

python3 -m ensurepip
pip3 install --prefix=$PREFIX -I pip

pip3 install --prefix=$PREFIX ipython[all]

pip3 install --prefix=$PREFIX SciPy NumPy Matplotlib pandas SymPy nose statsmodels
# clean up some dirt that statsmodels placed also into $PREFIX
rm -vf $PREFIX/LICENSE.txt
rm -vf $PREFIX/setup.cfg

pip3 install --prefix=$PREFIX Cython

pip3 install --prefix=$PREFIX pysam
pip3 install --prefix=$PREFIX seaborn
pip3 install --prefix=$PREFIX scikit-learn
#pip3 install --prefix=$PREFIX metaseq        # fails (Latest commit fa875d1 on Jan 10, 2017)
pip3 install --prefix=$PREFIX Mako
pip3 install --prefix=$PREFIX meson          # required in mariux

pip3 install --prefix=$PREFIX cutadapt       # Helpdesk 01/24/18 16:19, Sabrina Krakau
pip3 install --prefix=$PREFIX pygobject      # Helpdesk 03/26/18 14:19, Virginie Stanisla
pip3 install --prefix=$PREFIX blink1         # Blink1-lib 07/04/18 12:00, Niclas Hofmann

pip3 install --prefix=$PREFIX umi-tools
pip3 install --prefix=$PREFIX sphinx
pip3 install --prefix=$PREFIX recommonmark
pip3 install --prefix=$PREFIX fastcluster
pip3 install --prefix=$PREFIX pydot
pip3 install --prefix=$PREFIX pygraphviz
pip3 install --prefix=$PREFIX deeptools
pip3 install --prefix=$PREFIX HTSeq
pip3 install --prefix=$PREFIX keras
pip3 install --prefix=$PREFIX MotifScan      # Alena 30.8.2018
pip3 install --prefix=$PREFIX MAmotif
pip3 install --prefix=$PREFIX hic2cool       # Helpdesk Robert Schoepflin 17.09.2018
#pip3 install --prefix=$PREFIX hifive         # only python 2.7+
pip3 install --prefix=$PREFIX snakemake
pip3 install --prefix=$PREFIX virtualenv     # Donald
pip3 install --prefix=$PREFIX odfpy          # Donald
pip3 install --prefix=$PREFIX Dumper         # Thomas
pip3 install --prefix=$PREFIX igraph
pip3 install --prefix=$PREFIX louvain        # monocle3 prep for Helene
pip3 install --prefix=$PREFIX selenium       # 19.7.19 Kreitler  -> https://webxray.org/
pip3 install --prefix=$PREFIX cooltools      # Maria Valieva 17.7.2019
pip3 install --prefix=$PREFIX gmpy           # From viper context (Helpdesk Laura Glaser 9.8.2019)
pip3 install --prefix=$PREFIX torch
pip3 install --prefix=$PREFIX torchvision
pip3 install --prefix=$PREFIX plotly
pip3 install --prefix=$PREFIX pyOpenCL
pip3 install --prefix=$PREFIX docopt
pip3 install --prefix=$PREFIX dbus-python
pip3 install --prefix=$PREFIX distro
pip3 install --prefix=$PREFIX notify
pip3 install --prefix=$PREFIX ply
pip3 install --prefix=$PREFIX threadpoolctl

pip3 install --prefix=$PREFIX pyopengl
# creates error, so remove it
rm -rf ${PREFIX}/lib/python$PYTHONVER/site-packages/OpenGL/Tk

# pip3 install --prefix=$PREFIX HiCExplorer  # Would be a nice to have, but it likes to downgrade too much, pick some cherries
pip3 install --prefix=$PREFIX HiCMatrix
pip3 install --prefix=$PREFIX pybedtools
pip3 install --prefix=$PREFIX gffutils
# pyGenomeTracks, version 3.6, Requires-Dist: matplotlib (<=3.3.2,>=3.1.1), somehow they don't catch it ...
pip3 install --prefix=$PREFIX --no-deps "pyGenomeTracks" # example produces plots, so give it a chance
pip3 install --prefix=$PREFIX python-bioformats

pip3 install --prefix=$PREFIX jupyterlab      # request Peter Arndt
pip3 install --prefix=$PREFIX jupyterlab-git
pip3 install --prefix=$PREFIX jupyterlab-git
pip3 install --prefix=$PREFIX igv_jupyterlab  # Matthias Lienhard HD 01-04-2020
pip3 install --prefix=$PREFIX igv-jupyter
mkdir -vp $PREFIX/share/jupyter/lab/settings  # try to defeat a silly error in jupyter labextension install

pip3 install --prefix=$PREFIX libxml2-python3 # required by several bee packages after swich to python3
pip3 install --prefix=$PREFIX scour           # https://github.molgen.mpg.de/mariux64/pkg-scripts/issues/98
pip3 install --prefix=$PREFIX multiqc

pip3 install --prefix=$PREFIX thonny          # nice little ide, kreitler
pip3 install --prefix=$PREFIX pybind11
pip3 install --prefix=$PREFIX svgwrite
pip3 install --prefix=$PREFIX pyomo
pip3 install --prefix=$PREFIX jsbsim          # kreitler, hobby

pip3 install --prefix=$PREFIX bioservices
pip3 install --prefix=$PREFIX pairtools
pip3 install --prefix=$PREFIX SharedArray
pip3 install --prefix=$PREFIX pyabc           # dependency for stochkit, HD 20.07.20 Gemma Noviello
pip3 install --prefix=$PREFIX petab
pip3 install --prefix=$PREFIX tadtool         # HD 26.01.21 Michael Robson

#boost for pycuda
(
  . ${PREFIX}/profile
  BOOST_VERSION=(1.72.0 1_72_0)
  # former url:https://dl.bintray.com/boostorg/release/${BOOST_VERSION[0]}/source/boost_${BOOST_VERSION[1]}.tar.gz
  BOOST_SRC=https://beehive.molgen.mpg.de/e2b0b1eac302880461bcbef097171758/boost_1_72_0.tar.gz
  PKGCFLAGS="-O2 -fPIC"

  cd $BUILDDIR
  test -e boost_${BOOST_VERSION[1]}.tar.gz || wget --no-verbose $BOOST_SRC
  test -d boost-${BOOST_VERSION[0]} || mkdir boost-${BOOST_VERSION[0]} && tar -xf boost_${BOOST_VERSION[1]}.tar.gz -C boost-${BOOST_VERSION[0]} --strip-components=1

  #### ATTN!
  #### python meanders between /python${PYTHONVER} and /python${PYTHONVER}m

  cd boost-${BOOST_VERSION[0]}
  sed -e "/using python/ s|;|: ${PREFIX}/include/python${PYTHONVER} ;|" \
      -i bootstrap.sh

  ./bootstrap.sh \
    --with-toolset=gcc \
    --with-icu \
    --with-python=$PREFIX/bin/python3 \
    --with-libraries=python,thread || exit 1

  ./b2 \
    $NUMJOBS \
    --layout=system \
    --build-dir=build-python3 \
    --prefix=$PREFIX \
    --libdir=$PREFIX/lib \
    --build-type=minimal \
    toolset=gcc \
    variant=release \
    debug-symbols=off \
    link=shared \
    threading=multi \
    runtime-link=shared \
    python=$PYTHONVER \
    cflags="$PKGCFLAGS" \
    cxxflags="$PKGCFLAGS" \
    install || exit 1
)

#pycuda git version to get GL support
(
  PYCUDAVERS=2019.1.2
  PYTHON_LIBNAME=boost_python38

  . ${PREFIX}/profile
  cd $BUILDDIR
  # former url: https://github.com/inducer/pycuda/archive/v${PYCUDAVERS}/pycuda-${PYCUDAVERS}.tar.gz
  test -e pycuda-${PYCUDAVERS}.tar.gz || wget --no-verbose https://beehive.molgen.mpg.de/2fdf39e15a9c01e30e97144063b571dc/pycuda-2019.1.2.tar.gz
  test -d pycuda-${PYCUDAVERS} || tar -xf pycuda-${PYCUDAVERS}.tar.gz
  #test -d pycuda || git clone --recursive http://git.tiker.net/trees/pycuda.git

  cd pycuda-${PYCUDAVERS}
  test -e siteconf.py && rm siteconf.py
  python3 configure.py \
    --cuda-enable-gl \
    --cuda-root=$CUDADIR \
    --no-use-shipped-boost \
    --boost-python-libname=$PYTHON_LIBNAME \
    --boost-inc-dir=$PREFIX/include \
    --boost-lib-dir=$PREFIX/lib \
    --cudadrv-lib-dir='${CUDA_ROOT}/lib,${CUDA_ROOT}/lib/stubs' \
    --cudart-lib-dir='${CUDA_ROOT}/lib,${CUDA_ROOT}/lib/stubs' \
    --curand-lib-dir='${CUDA_ROOT}/lib,${CUDA_ROOT}/lib/stubs'\
    --ldflags=-L$PREFIX/lib

  python3 setup.py install --prefix $PREFIX
)


# bamToBigWig - just for the taste, HD 20.1.21 T. Zehnder
(
   # https://github.com/PanosFirmpas/bamToBigWig
   cd $PREFIX/bin
   wget --no-verbose https://beehive.molgen.mpg.de/bc6815740fbec1586054ad42e0b90a2d/bamToBigWig
   chmod +x bamToBigWig
)

# tensorflow is omitted in this build.

# try to load all packages, warnings/errors are not fatal, but give some additional hints.
echo "# START load test."
python3 -c 'help("modules")' > /dev/null
echo "# END load test."

exit