diff --git a/python-3.7.6-0.build.sh b/python-3.7.7-0.build.sh old mode 100644 new mode 100755 similarity index 68% rename from python-3.7.6-0.build.sh rename to python-3.7.7-0.build.sh index 186ee89..52b1691 --- a/python-3.7.6-0.build.sh +++ b/python-3.7.7-0.build.sh @@ -1,14 +1,11 @@ #! /bin/bash -COOKIE=$(mcookie); grep -v V_GREP_ME $0 > /dev/shm/runme-$COOKIE.sh ; sleep 1; exec bash /dev/shm/runme-$COOKIE.sh $1 - - PKG=python -VERSION=3.7.6 +VERSION=3.7.7 BUILD=0 PYTHONVER=${VERSION%.*} -CUDADIR=/pkg/cuda-10.1.105-0 +CUDADIR=/pkg/cuda-10.2.89-1 PREFIX=/pkg/$PKG-$VERSION-$BUILD if [ -n "$TESTING" ]; then PREFIX=/dev/shm/$PKG-$VERSION-$BUILD ; fi @@ -18,7 +15,7 @@ PATH=/bin:/usr/bin:/usr/sbin:/usr/local/bin set -xe umask 022 -BUILD_TMPDIR=/dev/shm/$PKG-$VERSION-$BUILD.build.tmp +BUILD_TMPDIR=/scratch/local2/$PKG-$VERSION-$BUILD.build.tmp test -d $BUILD_TMPDIR && ( chmod -R u+rwx $BUILD_TMPDIR || true ; rm -rf $BUILD_TMPDIR ) mkdir -p $BUILD_TMPDIR/home export TMPDIR=$BUILD_TMPDIR @@ -28,10 +25,9 @@ exec $PREFIX/profile <<-EOF - \. $CUDADIR/profile + . $CUDADIR/profile PATH=$PREFIX/bin:\$PATH export LD_LIBRARY_PATH=$PREFIX/lib\${LD_LIBRARY_PATH:+:}\$LD_LIBRARY_PATH - export PYTHONPATH=$PREFIX/lib/python3.7/site-packages\${PYTHONPATH:+:}\$PYTHONPATH PKG_CONFIG_PATH=$PREFIX/lib/pkgconfig\${PKG_CONFIG_PATH:+:\$PKG_CONFIG_PATH} export PKG_CONFIG_PATH if [ -d $PREFIX/.compatlibs ]; then export LD_LIBRARY_PATH=$PREFIX/.compatlibs\${LD_LIBRARY_PATH:+:\$LD_LIBRARY_PATH} ; fi @@ -206,7 +202,7 @@ pip3 install --prefix=$PREFIX HTSeq pip3 install --prefix=$PREFIX keras pip3 install --prefix=$PREFIX MotifScan # Alena 30.8.2018 pip3 install --prefix=$PREFIX MAmotif -#pip3 install --prefix=$PREFIX hic2cool # Helpdesk Robert Schoepflin 17.09.2018 Would be a nice to have, but it likes to downgrade too much, pick some cherries +pip3 install --prefix=$PREFIX hic2cool # Helpdesk Robert Schoepflin 17.09.2018 #pip3 install --prefix=$PREFIX hifive # only python 2.7+ pip3 install --prefix=$PREFIX snakemake pip3 install --prefix=$PREFIX virtualenv # Donald @@ -217,7 +213,6 @@ pip3 install --prefix=$PREFIX louvain # monocle3 prep for Helene pip3 install --prefix=$PREFIX selenium # 19.7.19 Kreitler -> https://webxray.org/ pip3 install --prefix=$PREFIX cooltools # Maria Valieva 17.7.2019 pip3 install --prefix=$PREFIX gmpy # From viper context (Helpdesk Laura Glaser 9.8.2019) -pip3 install --prefix=$PREFIX tensorflow==2.0.1 # tensorflow-2.1.0 needs spezial libs and nightly version(2.2.0) dont pip3 install --prefix=$PREFIX torch pip3 install --prefix=$PREFIX torchvision pip3 install --prefix=$PREFIX plotly @@ -231,31 +226,34 @@ pip3 install --prefix=$PREFIX threadpoolctl pip3 install --prefix=$PREFIX pyopengl rm -rf ${PREFIX}/lib/python$PYTHONVER/site-packages/OpenGL/Tk -# create error, so remove it +# creates error, so remove it # pip3 install --prefix=$PREFIX HiCExplorer # Would be a nice to have, but it likes to downgrade too much, pick some cherries pip3 install --prefix=$PREFIX HiCMatrix -pip3 install --prefix=$PREFIX pyGenomeTracks +pip3 install --prefix=$PREFIX "pyGenomeTracks<3.3" # 3.3 depends on HiMatrix>=12 which is not yet available -pip3 install --prefix=$PREFIX python-bioformats +pip3 install --prefix=$PREFIX python-bioformats #downgrader matplotlib 3.2.0->3.1.1 docutils 0.16 -> 0.15.2 pip3 install --prefix=$PREFIX jupyterlab # request Peter Arndt pip3 install --prefix=$PREFIX jupyterlab-git pip3 install --prefix=$PREFIX libxml2-python3 # required by several bee packages after swich to python3 pip3 install --prefix=$PREFIX scour # https://github.molgen.mpg.de/mariux64/pkg-scripts/issues/98 +pip3 install --prefix=$PREFIX multiqc #boost for pycuda ( . ${PREFIX}/profile - BOOST_VERSION="1.72.0 1_72_0" - BOOST_SRC=https://dl.bintray.com/boostorg/release/$( echo $BOOST_VERSION | cut -d" " -f1 )/source/boost_$( echo $BOOST_VERSION | cut -d" " -f2 ).tar.gz + BOOST_VERSION=(1.72.0 1_72_0) + BOOST_SRC=https://dl.bintray.com/boostorg/release/${BOOST_VERSION[0]}/source/boost_${BOOST_VERSION[1]}.tar.gz PKGCFLAGS="-O2 -fPIC" cd $BUILDDIR - test -e boost_$( echo $BOOST_VERSION | cut -d" " -f2 ).tar.gz || wget $BOOST_SRC - test -d boost-$( echo $BOOST_VERSION | cut -d" " -f1 ) || mkdir boost-$( echo $BOOST_VERSION | cut -d" " -f1 ) && tar -xf boost_$( echo $BOOST_VERSION | cut -d" " -f2 ).tar.gz -C boost-$( echo $BOOST_VERSION | cut -d" " -f1 ) --strip-components=1 + # 28.03.2020: upstream server currently broken - use cached archive + #test -e boost_${BOOST_VERSION[1]}.tar.gz || wget $BOOST_SRC + test -e boost_${BOOST_VERSION[1]}.tar.gz || cp /src/mariux/beeroot/downloads/boost_${BOOST_VERSION[1]}.tar.gz . + test -d boost-${BOOST_VERSION[0]} || mkdir boost-${BOOST_VERSION[0]} && tar -xf boost_${BOOST_VERSION[1]}.tar.gz -C boost-${BOOST_VERSION[0]} --strip-components=1 - cd boost-$( echo $BOOST_VERSION | cut -d" " -f1 ) + cd boost-${BOOST_VERSION[0]} sed -e "/using python/ s|;|: ${PREFIX}/include/python${PYTHONVER}m ;|" \ -i bootstrap.sh @@ -265,24 +263,6 @@ pip3 install --prefix=$PREFIX scour # https://github.molgen.mpg.de/mar --with-python=$PREFIX/bin/python3 \ --with-libraries=python,thread || exit 1 - ./b2 \ - $NUMJOBS \ - --layout=system \ - --build-dir=build-python3 \ - --prefix=$PREFIX \ - --libdir=$PREFIX/lib \ - --build-type=minimal \ - toolset=gcc \ - variant=release \ - debug-symbols=off \ - link=shared \ - threading=multi \ - runtime-link=shared \ - python=$PYTHONVER \ - cflags="$PKGCFLAGS" \ - cxxflags="$PKGCFLAGS" \ - stage || exit 1 - ./b2 \ $NUMJOBS \ --layout=system \ @@ -304,26 +284,103 @@ pip3 install --prefix=$PREFIX scour # https://github.molgen.mpg.de/mar #pycuda git version to get GL support ( + PYCUDAVERS=2019.1.2 + . ${PREFIX}/profile cd $BUILDDIR - test -d pycuda || git clone --recursive http://git.tiker.net/trees/pycuda.git + test -e pycuda-${PYCUDAVERS}.tar.gz || wget https://github.com/inducer/pycuda/archive/v${PYCUDAVERS}/pycuda-${PYCUDAVERS}.tar.gz + test -d pycuda-${PYCUDAVERS} || tar -xf pycuda-${PYCUDAVERS}.tar.gz + #test -d pycuda || git clone --recursive http://git.tiker.net/trees/pycuda.git - cd pycuda - git clean -dffx + cd pycuda-${PYCUDAVERS} + test -e siteconf.py && rm siteconf.py python3 configure.py \ - --cuda-enable-gl \ - --cuda-root=$CUDADIR \ - --no-use-shipped-boost \ - --boost-python-libname=boost_python37 \ - --boost-inc-dir=$PREFIX/include \ - --boost-lib-dir=$PREFIX/lib \ - --cudadrv-lib-dir='${CUDA_ROOT}/lib64,${CUDA_ROOT}/lib64/stubs' \ - --cudart-lib-dir='${CUDA_ROOT}/lib64,${CUDA_ROOT}/lib64/stubs' \ - --curand-lib-dir='${CUDA_ROOT}/lib64,${CUDA_ROOT}/lib64/stubs'\ - --ldflags=-L$PREFIX/lib + --cuda-enable-gl \ + --cuda-root=$CUDADIR \ + --no-use-shipped-boost \ + --boost-python-libname=boost_python37 \ + --boost-inc-dir=$PREFIX/include \ + --boost-lib-dir=$PREFIX/lib \ + --cudadrv-lib-dir='${CUDA_ROOT}/lib,${CUDA_ROOT}/lib/stubs' \ + --cudart-lib-dir='${CUDA_ROOT}/lib,${CUDA_ROOT}/lib/stubs' \ + --curand-lib-dir='${CUDA_ROOT}/lib,${CUDA_ROOT}/lib/stubs'\ + --ldflags=-L$PREFIX/lib + + python3 setup.py install --prefix $PREFIX +) + +#tensorflow +( + TFVERSION=2.1.0 + TFSRC=https://github.com/tensorflow/tensorflow/archive/v${TFVERSION}/tensorflow-${TFVERSION}.tar.gz + + . ${PREFIX}/profile + . /pkg/bazel-0.28.1-0/profile + cd $BUILDDIR + test -e tensorflow-${TFVERSION}.tar.gz || wget $TFSRC + test -d tensorflow-${TFVERSION} || tar -xf tensorflow-${TFVERSION}.tar.gz + + cd tensorflow-${TFVERSION} + + #vars for config + export PYTHON_BIN_PATH=${PREFIX}/bin/python3 + export USE_DEFAULT_PYTHON_LIB_PATH=1 + export TF_NEED_JEMALLOC=1 + export TF_NEED_KAFKA=0 + export TF_NEED_OPENCL_SYCL=0 + export TF_NEED_AWS=0 + export TF_NEED_GCP=0 + export TF_NEED_HDFS=0 + export TF_NEED_S3=0 + export TF_ENABLE_XLA=1 + export TF_NEED_GDR=0 + export TF_NEED_VERBS=0 + export TF_NEED_OPENCL=0 + export TF_NEED_MPI=0 + export TF_NEED_TENSORRT=1 + export TF_NEED_NGRAPH=0 + export TF_NEED_IGNITE=0 + export TF_NEED_ROCM=0 + export TF_SET_ANDROID_WORKSPACE=0 + export TF_DOWNLOAD_CLANG=0 + export TF_NCCL_VERSION=2.5 + export TF_IGNORE_MAX_BAZEL_VERSION=1 + export NCCL_INSTALL_PATH=${CUDADIR} + export GCC_HOST_COMPILER_PATH=/usr/bin/gcc + export TF_CUDA_CLANG=0 + export CLANG_CUDA_COMPILER_PATH=/usr/bin/clang + export TF_CUDA_PATHS=${CUDADIR},${CUDADIR}/extras/CUPTI,${CUDADIR}/nvvm + export TF_CUDA_VERSION=$(${CUDADIR}/bin/nvcc --version | sed -n 's/^.*release \(.*\),.*/\1/p') + export TF_CUDNN_VERSION=$(sed -n 's/^#define CUDNN_MAJOR\s*\(.*\).*/\1/p' ${CUDADIR}/include/cudnn.h) + export TF_CUDA_COMPUTE_CAPABILITIES=3.5,3.7,5.0,5.2,5.3,6.0,6.1,6.2,7.0,7.2,7.5 + export CC_OPT_FLAGS="-march=nehalem" + export TF_NEED_CUDA=1 + + ./configure + bazel build \ + --cxxopt="-march=nehalem" \ + --copt="-march=nehalem" \ + //tensorflow:libtensorflow.so \ + //tensorflow:libtensorflow_cc.so \ + //tensorflow:install_headers \ + //tensorflow/tools/pip_package:build_pip_package + + test -d tensorflow-pip && rm -r tensorflow-pip + mkdir tensorflow-pip + ./bazel-bin/tensorflow/tools/pip_package/build_pip_package --src ${BUILDDIR}/tensorflow-pip + cd ${BUILDDIR} + #remove symlinks into TMPDIR + test -e tensorflow-pip.tar && rm tensorflow-pip.tar + tar -chf tensorflow-pip.tar tensorflow-pip + test -d tensorflow-pip && rm -r tensorflow-pip + + tar -xf tensorflow-pip.tar + cd ${BUILDDIR}/tensorflow-pip + python3 setup.py build python3 setup.py install --prefix $PREFIX ) + # try to load all packages, maybe we can see installation errors python3 -c 'help("modules")' > /dev/null