python-3.7.7-0.build.sh

#! /bin/bash

PKG=python
VERSION=3.7.7
BUILD=0
PYTHONVER=${VERSION%.*}

CUDADIR=/pkg/cuda-10.2.89-1

PREFIX=/pkg/$PKG-$VERSION-$BUILD
if [ -n "$TESTING" ]; then PREFIX=/dev/shm/$PKG-$VERSION-$BUILD ; fi

PATH=/bin:/usr/bin:/usr/sbin:/usr/local/bin

set -xe
umask 022

BUILD_TMPDIR=/scratch/local2/$PKG-$VERSION-$BUILD.build.tmp
test -d $BUILD_TMPDIR && ( chmod -R u+rwx $BUILD_TMPDIR || true ; rm -rf $BUILD_TMPDIR )
mkdir -p $BUILD_TMPDIR/home
export TMPDIR=$BUILD_TMPDIR
export HOME=$BUILD_TMPDIR/home

exec </dev/null

mkdir -p $PREFIX
cat >$PREFIX/profile <<-EOF
	. $CUDADIR/profile
	PATH=$PREFIX/bin:\$PATH
	export LD_LIBRARY_PATH=$PREFIX/lib\${LD_LIBRARY_PATH:+:}\$LD_LIBRARY_PATH
	PKG_CONFIG_PATH=$PREFIX/lib/pkgconfig\${PKG_CONFIG_PATH:+:\$PKG_CONFIG_PATH}
	export PKG_CONFIG_PATH
	if [ -d $PREFIX/.compatlibs ]; then export LD_LIBRARY_PATH=$PREFIX/.compatlibs\${LD_LIBRARY_PATH:+:\$LD_LIBRARY_PATH} ; fi
EOF
. $PREFIX/profile

export MAKEFLAGS="-j $(nproc)"

BUILDDIR=$PREFIX/build

mkdir -p $BUILDDIR
cd $BUILDDIR

test -e Python-$VERSION.tar.xz || wget https://www.python.org/ftp/python/$VERSION/Python-$VERSION.tar.xz
test -d Python-$VERSION        || tar xvf Python-$VERSION.tar.xz
cd Python-$VERSION

LDFLAGS="-Wl,-rpath=$PREFIX/lib" \
./configure \
        --prefix=$PREFIX \
        --enable-shared

make -j $(nproc)
# make test || true # 'test_socket' seems to run for ever
make install

( # sip: tool to create Python bindings for C and C++ libraries.
  cd $PREFIX/build
  SIP_VERSION=4.19.18
  SIP_SRC=https://www.riverbankcomputing.com/static/Downloads/sip/$SIP_VERSION/sip-$SIP_VERSION.tar.gz
  test -e sip-$SIP_VERSION.tar.gz || wget $SIP_SRC
  test -d sip-$SIP_VERSION || tar -xf sip-$SIP_VERSION.tar.gz

  cd sip-$SIP_VERSION

  python3 configure.py \
      -b $PREFIX/bin \
      -d $PREFIX/lib/python$PYTHONVER/site-packages \
      -e $PREFIX/include/python${PYTHONVER}m \
      -v $PREFIX/share/sip \
      --sip-module=sip

  # well, right here share/sip is spoilage, but maybe some packages will use the decent sip later on ...
  mkdir -p $PREFIX/share/sip
  make -j $(nproc);
  make install
)


( # now for the private (sic!) version required by newer PyQt
  cd $PREFIX/build
  mkdir -p private_sip
  cd private_sip
  SIP_VERSION=4.19.18
  SIP_SRC=https://www.riverbankcomputing.com/static/Downloads/sip/$SIP_VERSION/sip-$SIP_VERSION.tar.gz
  test -e sip-$SIP_VERSION.tar.gz || wget $SIP_SRC
  test -d sip-$SIP_VERSION || tar -xf sip-$SIP_VERSION.tar.gz

  cd sip-$SIP_VERSION

  # note $PREFIX/share/sip/PyQt5 is mandatory
  python3 configure.py \
      -d $PREFIX/lib/python$PYTHONVER/site-packages \
      -v $PREFIX/share/sip/PyQt5 \
      --no-tools \
      --sip-module=PyQt5.sip

  mkdir -p $PREFIX/share/sip/PyQt5
  make -j $(nproc)
  # The resulting sip.so in site-packages/PyQt5 differs indeed a bit (grep for PyQt5)
  make install
)

(
  PYQT_VERSION=5.13.0
  PYQT_SRC=https://www.riverbankcomputing.com/static/Downloads/PyQt5/$PYQT_VERSION/PyQt5_gpl-$PYQT_VERSION.tar.gz

  cd $BUILDDIR
  test -e PyQt5_gpl-$PYQT_VERSION.tar.gz || wget $PYQT_SRC
  test -d PyQt5_gpl-$PYQT_VERSION || tar -xf PyQt5_gpl-$PYQT_VERSION.tar.gz

  cd PyQt5_gpl-$PYQT_VERSION

  # export QTDIR=/usr/local/qt5
  python3 configure.py \
    -b $PREFIX/bin \
    -d $PREFIX/lib/python$PYTHONVER/site-packages \
    -n $PREFIX/share/qsci \
    -q /usr/local/qt5/bin/qmake \
    -v $PREFIX/share/sip/PyQt5 \
    --sip-incdir=$PREFIX/include/python${PYTHONVER}m \
    --confirm-license \
    --no-qml-plugin \
    --no-designer-plugin \
    --no-dist-info \
    --verbose

  make -j $(nproc)

  # Did I miss something in the docs ???
  # w/o the *.so files inplace a clean 'make install' fails with file not found ...
  # -> FileNotFoundError: [Errno 2] No such file or directory:
  #         '/dev/shm/python-3.7.4-0/lib/python3.7/site-packages/PyQt5/QtCore.so'
  mkdir -p $PREFIX/lib/python$PYTHONVER/site-packages/PyQt5
  for F in `find | grep '\.so$' | grep -v '/lib'` ; do
    cp -vp $F $PREFIX/lib/python$PYTHONVER/site-packages/PyQt5/
  done
  make install
)

(
  QSCINTILLA_VERSION=2.11.2
  QSCINTILLA_SRC=https://www.riverbankcomputing.com/static/Downloads/QScintilla/$QSCINTILLA_VERSION/QScintilla_gpl-$QSCINTILLA_VERSION.tar.gz

  cd $BUILDDIR
  test -e QScintilla_gpl-$QSCINTILLA_VERSION.tar.gz || wget $QSCINTILLA_SRC
  test -d QScintilla_gpl-$QSCINTILLA_VERSION || tar -xf QScintilla_gpl-$QSCINTILLA_VERSION.tar.gz
  cd QScintilla_gpl-$QSCINTILLA_VERSION

  cd Python

  # apidir follows PyQt, not Qt (default is: QT_INSTALL_DATA/qsci)
  # creation of PEP 376 .dist-info fails in $PREFIX/share
  python3 configure.py \
    --pyqt=PyQt5 \
    --qmake=/usr/local/qt5/bin/qmake \
    --apidir=$PREFIX/share/qsci \
    --no-dist-info

  make -j $(nproc)

  # Call it a deja-vu, nerve ...
  # -> FileNotFoundError: [Errno 2] No such file or directory: '/dev/shm/python-3.7.4-0/lib/python3.7/site-packages/PyQt5/Qsci.so'
  mkdir -p $PREFIX/lib/python$PYTHONVER/site-packages/PyQt5
  for F in `find | grep '\.so$' | grep -v '/lib'` ; do
    cp -vp $F $PREFIX/lib/python$PYTHONVER/site-packages/PyQt5
  done
  make install
)

python3 -m ensurepip
pip3 install --prefix=$PREFIX -I pip

pip3 install --prefix=$PREFIX ipython[all]

pip3 install --prefix=$PREFIX SciPy NumPy Matplotlib pandas SymPy nose statsmodels
# clean up some dirt that statsmodels placed also into $PREFIX
rm -vf $PREFIX/LICENSE.txt
rm -vf $PREFIX/setup.cfg

pip3 install --prefix=$PREFIX Cython

pip3 install --prefix=$PREFIX pysam
pip3 install --prefix=$PREFIX seaborn
pip3 install --prefix=$PREFIX scikit-learn
#pip3 install --prefix=$PREFIX metaseq      # fails (Latest commit fa875d1 on Jan 10, 2017)
pip3 install --prefix=$PREFIX Mako
pip3 install --prefix=$PREFIX meson          # required in mariux

pip3 install --prefix=$PREFIX cutadapt       # Helpdesk 01/24/18 16:19, Sabrina Krakau
pip3 install --prefix=$PREFIX pygobject      # Helpdesk 03/26/18 14:19, Virginie Stanisla
pip3 install --prefix=$PREFIX blink1         # Blink1-lib 07/04/18 12:00, Niclas Hofmann

pip3 install --prefix=$PREFIX umi-tools
pip3 install --prefix=$PREFIX sphinx
pip3 install --prefix=$PREFIX recommonmark
pip3 install --prefix=$PREFIX fastcluster
pip3 install --prefix=$PREFIX pydot
pip3 install --prefix=$PREFIX pygraphviz
pip3 install --prefix=$PREFIX deeptools
pip3 install --prefix=$PREFIX HTSeq
pip3 install --prefix=$PREFIX keras
pip3 install --prefix=$PREFIX MotifScan      # Alena 30.8.2018
pip3 install --prefix=$PREFIX MAmotif
pip3 install --prefix=$PREFIX hic2cool       # Helpdesk Robert Schoepflin 17.09.2018
#pip3 install --prefix=$PREFIX hifive       # only python 2.7+
pip3 install --prefix=$PREFIX snakemake
pip3 install --prefix=$PREFIX virtualenv     # Donald
pip3 install --prefix=$PREFIX odfpy          # Donald
pip3 install --prefix=$PREFIX Dumper         # Thomas
pip3 install --prefix=$PREFIX igraph
pip3 install --prefix=$PREFIX louvain        # monocle3 prep for Helene
pip3 install --prefix=$PREFIX selenium       # 19.7.19 Kreitler  -> https://webxray.org/
pip3 install --prefix=$PREFIX cooltools      # Maria Valieva 17.7.2019
pip3 install --prefix=$PREFIX gmpy           # From viper context (Helpdesk Laura Glaser 9.8.2019)
pip3 install --prefix=$PREFIX torch
pip3 install --prefix=$PREFIX torchvision
pip3 install --prefix=$PREFIX plotly
pip3 install --prefix=$PREFIX pyOpenCL
pip3 install --prefix=$PREFIX docopt
pip3 install --prefix=$PREFIX dbus-python
pip3 install --prefix=$PREFIX distro
pip3 install --prefix=$PREFIX notify
pip3 install --prefix=$PREFIX ply
pip3 install --prefix=$PREFIX threadpoolctl

pip3 install --prefix=$PREFIX pyopengl
rm -rf ${PREFIX}/lib/python$PYTHONVER/site-packages/OpenGL/Tk
# creates error, so remove it

# pip3 install --prefix=$PREFIX HiCExplorer  # Would be a nice to have, but it likes to downgrade too much, pick some cherries
pip3 install --prefix=$PREFIX HiCMatrix
pip3 install --prefix=$PREFIX "pyGenomeTracks<3.3" # 3.3 depends on HiMatrix>=12 which is not yet available

pip3 install --prefix=$PREFIX python-bioformats #downgrader matplotlib 3.2.0->3.1.1 docutils 0.16 -> 0.15.2

pip3 install --prefix=$PREFIX jupyterlab      # request Peter Arndt
pip3 install --prefix=$PREFIX jupyterlab-git
pip3 install --prefix=$PREFIX libxml2-python3 # required by several bee packages after swich to python3
pip3 install --prefix=$PREFIX scour           # https://github.molgen.mpg.de/mariux64/pkg-scripts/issues/98
pip3 install --prefix=$PREFIX multiqc

#boost for pycuda
(
	. ${PREFIX}/profile
	BOOST_VERSION=(1.72.0 1_72_0)
	BOOST_SRC=https://dl.bintray.com/boostorg/release/${BOOST_VERSION[0]}/source/boost_${BOOST_VERSION[1]}.tar.gz
	PKGCFLAGS="-O2 -fPIC"

	cd $BUILDDIR
	# 28.03.2020: upstream server currently broken - use cached archive
	#test -e boost_${BOOST_VERSION[1]}.tar.gz || wget $BOOST_SRC
	test -e boost_${BOOST_VERSION[1]}.tar.gz || cp /src/mariux/beeroot/downloads/boost_${BOOST_VERSION[1]}.tar.gz .
	test -d boost-${BOOST_VERSION[0]} || mkdir boost-${BOOST_VERSION[0]} && tar -xf boost_${BOOST_VERSION[1]}.tar.gz -C boost-${BOOST_VERSION[0]} --strip-components=1

	cd boost-${BOOST_VERSION[0]}
	sed -e "/using python/ s|;|: ${PREFIX}/include/python${PYTHONVER}m ;|" \
      -i bootstrap.sh

	./bootstrap.sh \
    --with-toolset=gcc \
    --with-icu \
    --with-python=$PREFIX/bin/python3 \
    --with-libraries=python,thread || exit 1

  ./b2 \
    $NUMJOBS \
    --layout=system \
    --build-dir=build-python3 \
    --prefix=$PREFIX \
    --libdir=$PREFIX/lib \
    --build-type=minimal \
    toolset=gcc \
    variant=release \
    debug-symbols=off \
    link=shared \
    threading=multi \
    runtime-link=shared \
    python=$PYTHONVER \
    cflags="$PKGCFLAGS" \
    cxxflags="$PKGCFLAGS" \
    install || exit 1
)

#pycuda git version to get GL support
(
	PYCUDAVERS=2019.1.2

	. ${PREFIX}/profile
	cd $BUILDDIR
	test -e pycuda-${PYCUDAVERS}.tar.gz || wget https://github.com/inducer/pycuda/archive/v${PYCUDAVERS}/pycuda-${PYCUDAVERS}.tar.gz
	test -d pycuda-${PYCUDAVERS} || tar -xf pycuda-${PYCUDAVERS}.tar.gz
	#test -d pycuda || git clone --recursive http://git.tiker.net/trees/pycuda.git

	cd pycuda-${PYCUDAVERS}
	test -e siteconf.py && rm siteconf.py
	python3 configure.py \
		--cuda-enable-gl \
		--cuda-root=$CUDADIR \
		--no-use-shipped-boost \
		--boost-python-libname=boost_python37 \
		--boost-inc-dir=$PREFIX/include \
		--boost-lib-dir=$PREFIX/lib \
		--cudadrv-lib-dir='${CUDA_ROOT}/lib,${CUDA_ROOT}/lib/stubs' \
		--cudart-lib-dir='${CUDA_ROOT}/lib,${CUDA_ROOT}/lib/stubs' \
		--curand-lib-dir='${CUDA_ROOT}/lib,${CUDA_ROOT}/lib/stubs'\
		--ldflags=-L$PREFIX/lib

	python3 setup.py install --prefix $PREFIX
)

#tensorflow
(
	TFVERSION=2.1.0
	TFSRC=https://github.com/tensorflow/tensorflow/archive/v${TFVERSION}/tensorflow-${TFVERSION}.tar.gz


	. ${PREFIX}/profile
	. /pkg/bazel-0.28.1-0/profile
	cd $BUILDDIR
	test -e tensorflow-${TFVERSION}.tar.gz || wget $TFSRC
	test -d tensorflow-${TFVERSION} || tar -xf tensorflow-${TFVERSION}.tar.gz

	cd tensorflow-${TFVERSION}

	#vars for config
	export PYTHON_BIN_PATH=${PREFIX}/bin/python3
	export USE_DEFAULT_PYTHON_LIB_PATH=1
	export TF_NEED_JEMALLOC=1
	export TF_NEED_KAFKA=0
	export TF_NEED_OPENCL_SYCL=0
	export TF_NEED_AWS=0
	export TF_NEED_GCP=0
	export TF_NEED_HDFS=0
	export TF_NEED_S3=0
	export TF_ENABLE_XLA=1
	export TF_NEED_GDR=0
	export TF_NEED_VERBS=0
	export TF_NEED_OPENCL=0
	export TF_NEED_MPI=0
	export TF_NEED_TENSORRT=1
	export TF_NEED_NGRAPH=0
	export TF_NEED_IGNITE=0
	export TF_NEED_ROCM=0
	export TF_SET_ANDROID_WORKSPACE=0
	export TF_DOWNLOAD_CLANG=0
	export TF_NCCL_VERSION=2.5
	export TF_IGNORE_MAX_BAZEL_VERSION=1
	export NCCL_INSTALL_PATH=${CUDADIR}
	export GCC_HOST_COMPILER_PATH=/usr/bin/gcc
	export TF_CUDA_CLANG=0
	export CLANG_CUDA_COMPILER_PATH=/usr/bin/clang
	export TF_CUDA_PATHS=${CUDADIR},${CUDADIR}/extras/CUPTI,${CUDADIR}/nvvm
	export TF_CUDA_VERSION=$(${CUDADIR}/bin/nvcc --version | sed -n 's/^.*release \(.*\),.*/\1/p')
	export TF_CUDNN_VERSION=$(sed -n 's/^#define CUDNN_MAJOR\s*\(.*\).*/\1/p' ${CUDADIR}/include/cudnn.h)
	export TF_CUDA_COMPUTE_CAPABILITIES=3.5,3.7,5.0,5.2,5.3,6.0,6.1,6.2,7.0,7.2,7.5
	export CC_OPT_FLAGS="-march=nehalem"
	export TF_NEED_CUDA=1

	./configure
	bazel build \
		--cxxopt="-march=nehalem" \
		--copt="-march=nehalem" \
		//tensorflow:libtensorflow.so \
		//tensorflow:libtensorflow_cc.so \
		//tensorflow:install_headers \
		//tensorflow/tools/pip_package:build_pip_package

	test -d tensorflow-pip && rm -r tensorflow-pip
	mkdir tensorflow-pip
	./bazel-bin/tensorflow/tools/pip_package/build_pip_package --src ${BUILDDIR}/tensorflow-pip
	cd ${BUILDDIR}
	#remove symlinks into TMPDIR
	test -e tensorflow-pip.tar && rm tensorflow-pip.tar
	tar -chf tensorflow-pip.tar tensorflow-pip
	test -d tensorflow-pip && rm -r tensorflow-pip

	tar -xf tensorflow-pip.tar
	cd ${BUILDDIR}/tensorflow-pip
	python3 setup.py build
	python3 setup.py install --prefix $PREFIX
)

# try to load all packages, maybe we can see installation errors
python3 -c 'help("modules")' > /dev/null

exit