Merge pull request #109 from mariux64/update-python3-to-3.7.6-donald

Update python3 to 3.7.7 (with tensorflow included) Credits go to @david
mariux64 · Apr 1, 2020 · 999be4f · 999be4f
2 parents e711d4d + a4dcb65
commit 999be4f
Showing 1 changed file with 106 additions and 49 deletions.
diff --git a/python-3.7.6-0.build.sh → python-3.7.7-0.build.sh b/python-3.7.6-0.build.sh → python-3.7.7-0.build.sh
@@ -1,14 +1,11 @@
 #! /bin/bash
 
-COOKIE=$(mcookie); grep -v V_GREP_ME $0 > /dev/shm/runme-$COOKIE.sh ; sleep 1; exec bash /dev/shm/runme-$COOKIE.sh $1
-
-
 PKG=python
-VERSION=3.7.6
+VERSION=3.7.7
 BUILD=0
 PYTHONVER=${VERSION%.*}
 
-CUDADIR=/pkg/cuda-10.1.105-0
+CUDADIR=/pkg/cuda-10.2.89-1
 
 PREFIX=/pkg/$PKG-$VERSION-$BUILD
 if [ -n "$TESTING" ]; then PREFIX=/dev/shm/$PKG-$VERSION-$BUILD ; fi
@@ -18,7 +15,7 @@ PATH=/bin:/usr/bin:/usr/sbin:/usr/local/bin
 set -xe
 umask 022
 
-BUILD_TMPDIR=/dev/shm/$PKG-$VERSION-$BUILD.build.tmp
+BUILD_TMPDIR=/scratch/local2/$PKG-$VERSION-$BUILD.build.tmp
 test -d $BUILD_TMPDIR && ( chmod -R u+rwx $BUILD_TMPDIR || true ; rm -rf $BUILD_TMPDIR )
 mkdir -p $BUILD_TMPDIR/home
 export TMPDIR=$BUILD_TMPDIR
@@ -28,10 +25,9 @@ exec </dev/null
 
 mkdir -p $PREFIX
 cat >$PREFIX/profile <<-EOF
-	\. $CUDADIR/profile
+	. $CUDADIR/profile
 	PATH=$PREFIX/bin:\$PATH
 	export LD_LIBRARY_PATH=$PREFIX/lib\${LD_LIBRARY_PATH:+:}\$LD_LIBRARY_PATH
-	export PYTHONPATH=$PREFIX/lib/python3.7/site-packages\${PYTHONPATH:+:}\$PYTHONPATH
 	PKG_CONFIG_PATH=$PREFIX/lib/pkgconfig\${PKG_CONFIG_PATH:+:\$PKG_CONFIG_PATH}
 	export PKG_CONFIG_PATH
 	if [ -d $PREFIX/.compatlibs ]; then export LD_LIBRARY_PATH=$PREFIX/.compatlibs\${LD_LIBRARY_PATH:+:\$LD_LIBRARY_PATH} ; fi
@@ -206,7 +202,7 @@ pip3 install --prefix=$PREFIX HTSeq
 pip3 install --prefix=$PREFIX keras
 pip3 install --prefix=$PREFIX MotifScan      # Alena 30.8.2018
 pip3 install --prefix=$PREFIX MAmotif
-#pip3 install --prefix=$PREFIX hic2cool       # Helpdesk Robert Schoepflin 17.09.2018 Would be a nice to have, but it likes to downgrade too much, pick some cherries
+pip3 install --prefix=$PREFIX hic2cool       # Helpdesk Robert Schoepflin 17.09.2018
 #pip3 install --prefix=$PREFIX hifive       # only python 2.7+
 pip3 install --prefix=$PREFIX snakemake
 pip3 install --prefix=$PREFIX virtualenv     # Donald
@@ -217,7 +213,6 @@ pip3 install --prefix=$PREFIX louvain        # monocle3 prep for Helene
 pip3 install --prefix=$PREFIX selenium       # 19.7.19 Kreitler  -> https://webxray.org/
 pip3 install --prefix=$PREFIX cooltools      # Maria Valieva 17.7.2019
 pip3 install --prefix=$PREFIX gmpy           # From viper context (Helpdesk Laura Glaser 9.8.2019)
-pip3 install --prefix=$PREFIX tensorflow==2.0.1         # tensorflow-2.1.0 needs spezial libs and nightly version(2.2.0) dont
 pip3 install --prefix=$PREFIX torch
 pip3 install --prefix=$PREFIX torchvision
 pip3 install --prefix=$PREFIX plotly
@@ -231,31 +226,34 @@ pip3 install --prefix=$PREFIX threadpoolctl
 
 pip3 install --prefix=$PREFIX pyopengl
 rm -rf ${PREFIX}/lib/python$PYTHONVER/site-packages/OpenGL/Tk
-# create error, so remove it
+# creates error, so remove it
 
 # pip3 install --prefix=$PREFIX HiCExplorer  # Would be a nice to have, but it likes to downgrade too much, pick some cherries
 pip3 install --prefix=$PREFIX HiCMatrix
-pip3 install --prefix=$PREFIX pyGenomeTracks
+pip3 install --prefix=$PREFIX "pyGenomeTracks<3.3" # 3.3 depends on HiMatrix>=12 which is not yet available
 
-pip3 install --prefix=$PREFIX python-bioformats
+pip3 install --prefix=$PREFIX python-bioformats #downgrader matplotlib 3.2.0->3.1.1 docutils 0.16 -> 0.15.2
 
 pip3 install --prefix=$PREFIX jupyterlab      # request Peter Arndt
 pip3 install --prefix=$PREFIX jupyterlab-git
 pip3 install --prefix=$PREFIX libxml2-python3 # required by several bee packages after swich to python3
 pip3 install --prefix=$PREFIX scour           # https://github.molgen.mpg.de/mariux64/pkg-scripts/issues/98
+pip3 install --prefix=$PREFIX multiqc
 
 #boost for pycuda
 (
 	. ${PREFIX}/profile
-	BOOST_VERSION="1.72.0 1_72_0"
-	BOOST_SRC=https://dl.bintray.com/boostorg/release/$( echo $BOOST_VERSION | cut -d" " -f1 )/source/boost_$( echo $BOOST_VERSION | cut -d" " -f2 ).tar.gz
+	BOOST_VERSION=(1.72.0 1_72_0)
+	BOOST_SRC=https://dl.bintray.com/boostorg/release/${BOOST_VERSION[0]}/source/boost_${BOOST_VERSION[1]}.tar.gz
 	PKGCFLAGS="-O2 -fPIC"
 
 	cd $BUILDDIR
-	test -e boost_$( echo $BOOST_VERSION | cut -d" " -f2 ).tar.gz || wget $BOOST_SRC
-	test -d boost-$( echo $BOOST_VERSION | cut -d" " -f1 ) || mkdir boost-$( echo $BOOST_VERSION | cut -d" " -f1 ) && tar -xf boost_$( echo $BOOST_VERSION | cut -d" " -f2 ).tar.gz -C boost-$( echo $BOOST_VERSION | cut -d" " -f1 ) --strip-components=1
+	# 28.03.2020: upstream server currently broken - use cached archive
+	#test -e boost_${BOOST_VERSION[1]}.tar.gz || wget $BOOST_SRC
+	test -e boost_${BOOST_VERSION[1]}.tar.gz || cp /src/mariux/beeroot/downloads/boost_${BOOST_VERSION[1]}.tar.gz .
+	test -d boost-${BOOST_VERSION[0]} || mkdir boost-${BOOST_VERSION[0]} && tar -xf boost_${BOOST_VERSION[1]}.tar.gz -C boost-${BOOST_VERSION[0]} --strip-components=1
 
-	cd boost-$( echo $BOOST_VERSION | cut -d" " -f1 )
+	cd boost-${BOOST_VERSION[0]}
 	sed -e "/using python/ s|;|: ${PREFIX}/include/python${PYTHONVER}m ;|" \
       -i bootstrap.sh
 
@@ -265,24 +263,6 @@ pip3 install --prefix=$PREFIX scour           # https://github.molgen.mpg.de/mar
     --with-python=$PREFIX/bin/python3 \
     --with-libraries=python,thread || exit 1
 
-	./b2 \
-    $NUMJOBS \
-    --layout=system \
-    --build-dir=build-python3 \
-    --prefix=$PREFIX \
-    --libdir=$PREFIX/lib \
-    --build-type=minimal \
-    toolset=gcc \
-    variant=release \
-    debug-symbols=off \
-    link=shared \
-    threading=multi \
-    runtime-link=shared \
-    python=$PYTHONVER \
-    cflags="$PKGCFLAGS" \
-    cxxflags="$PKGCFLAGS" \
-    stage || exit 1
-
   ./b2 \
     $NUMJOBS \
     --layout=system \
@@ -304,26 +284,103 @@ pip3 install --prefix=$PREFIX scour           # https://github.molgen.mpg.de/mar
 
 #pycuda git version to get GL support
 (
+	PYCUDAVERS=2019.1.2
+
 	. ${PREFIX}/profile
 	cd $BUILDDIR
-	test -d pycuda || git clone --recursive http://git.tiker.net/trees/pycuda.git
+	test -e pycuda-${PYCUDAVERS}.tar.gz || wget https://github.com/inducer/pycuda/archive/v${PYCUDAVERS}/pycuda-${PYCUDAVERS}.tar.gz
+	test -d pycuda-${PYCUDAVERS} || tar -xf pycuda-${PYCUDAVERS}.tar.gz
+	#test -d pycuda || git clone --recursive http://git.tiker.net/trees/pycuda.git
 
-	cd pycuda
-	git clean -dffx
+	cd pycuda-${PYCUDAVERS}
+	test -e siteconf.py && rm siteconf.py
 	python3 configure.py \
-    --cuda-enable-gl \
-    --cuda-root=$CUDADIR \
-    --no-use-shipped-boost \
-    --boost-python-libname=boost_python37 \
-    --boost-inc-dir=$PREFIX/include \
-    --boost-lib-dir=$PREFIX/lib \
-    --cudadrv-lib-dir='${CUDA_ROOT}/lib64,${CUDA_ROOT}/lib64/stubs' \
-    --cudart-lib-dir='${CUDA_ROOT}/lib64,${CUDA_ROOT}/lib64/stubs' \
-    --curand-lib-dir='${CUDA_ROOT}/lib64,${CUDA_ROOT}/lib64/stubs'\
-    --ldflags=-L$PREFIX/lib
+		--cuda-enable-gl \
+		--cuda-root=$CUDADIR \
+		--no-use-shipped-boost \
+		--boost-python-libname=boost_python37 \
+		--boost-inc-dir=$PREFIX/include \
+		--boost-lib-dir=$PREFIX/lib \
+		--cudadrv-lib-dir='${CUDA_ROOT}/lib,${CUDA_ROOT}/lib/stubs' \
+		--cudart-lib-dir='${CUDA_ROOT}/lib,${CUDA_ROOT}/lib/stubs' \
+		--curand-lib-dir='${CUDA_ROOT}/lib,${CUDA_ROOT}/lib/stubs'\
+		--ldflags=-L$PREFIX/lib
+
+	python3 setup.py install --prefix $PREFIX
+)
+
+#tensorflow
+(
+	TFVERSION=2.1.0
+	TFSRC=https://github.com/tensorflow/tensorflow/archive/v${TFVERSION}/tensorflow-${TFVERSION}.tar.gz
+
 
+	. ${PREFIX}/profile
+	. /pkg/bazel-0.28.1-0/profile
+	cd $BUILDDIR
+	test -e tensorflow-${TFVERSION}.tar.gz || wget $TFSRC
+	test -d tensorflow-${TFVERSION} || tar -xf tensorflow-${TFVERSION}.tar.gz
+
+	cd tensorflow-${TFVERSION}
+
+	#vars for config
+	export PYTHON_BIN_PATH=${PREFIX}/bin/python3
+	export USE_DEFAULT_PYTHON_LIB_PATH=1
+	export TF_NEED_JEMALLOC=1
+	export TF_NEED_KAFKA=0
+	export TF_NEED_OPENCL_SYCL=0
+	export TF_NEED_AWS=0
+	export TF_NEED_GCP=0
+	export TF_NEED_HDFS=0
+	export TF_NEED_S3=0
+	export TF_ENABLE_XLA=1
+	export TF_NEED_GDR=0
+	export TF_NEED_VERBS=0
+	export TF_NEED_OPENCL=0
+	export TF_NEED_MPI=0
+	export TF_NEED_TENSORRT=1
+	export TF_NEED_NGRAPH=0
+	export TF_NEED_IGNITE=0
+	export TF_NEED_ROCM=0
+	export TF_SET_ANDROID_WORKSPACE=0
+	export TF_DOWNLOAD_CLANG=0
+	export TF_NCCL_VERSION=2.5
+	export TF_IGNORE_MAX_BAZEL_VERSION=1
+	export NCCL_INSTALL_PATH=${CUDADIR}
+	export GCC_HOST_COMPILER_PATH=/usr/bin/gcc
+	export TF_CUDA_CLANG=0
+	export CLANG_CUDA_COMPILER_PATH=/usr/bin/clang
+	export TF_CUDA_PATHS=${CUDADIR},${CUDADIR}/extras/CUPTI,${CUDADIR}/nvvm
+	export TF_CUDA_VERSION=$(${CUDADIR}/bin/nvcc --version | sed -n 's/^.*release \(.*\),.*/\1/p')
+	export TF_CUDNN_VERSION=$(sed -n 's/^#define CUDNN_MAJOR\s*\(.*\).*/\1/p' ${CUDADIR}/include/cudnn.h)
+	export TF_CUDA_COMPUTE_CAPABILITIES=3.5,3.7,5.0,5.2,5.3,6.0,6.1,6.2,7.0,7.2,7.5
+	export CC_OPT_FLAGS="-march=nehalem"
+	export TF_NEED_CUDA=1
+
+	./configure
+	bazel build \
+		--cxxopt="-march=nehalem" \
+		--copt="-march=nehalem" \
+		//tensorflow:libtensorflow.so \
+		//tensorflow:libtensorflow_cc.so \
+		//tensorflow:install_headers \
+		//tensorflow/tools/pip_package:build_pip_package
+
+	test -d tensorflow-pip && rm -r tensorflow-pip
+	mkdir tensorflow-pip
+	./bazel-bin/tensorflow/tools/pip_package/build_pip_package --src ${BUILDDIR}/tensorflow-pip
+	cd ${BUILDDIR}
+	#remove symlinks into TMPDIR
+	test -e tensorflow-pip.tar && rm tensorflow-pip.tar
+	tar -chf tensorflow-pip.tar tensorflow-pip
+	test -d tensorflow-pip && rm -r tensorflow-pip
+
+	tar -xf tensorflow-pip.tar
+	cd ${BUILDDIR}/tensorflow-pip
+	python3 setup.py build
 	python3 setup.py install --prefix $PREFIX
 )
+
 # try to load all packages, maybe we can see installation errors
 python3 -c 'help("modules")' > /dev/null