diff --git a/python-3.7.6-0.build.sh b/python-3.7.6-0.build.sh index 186ee89..45a6d23 100644 --- a/python-3.7.6-0.build.sh +++ b/python-3.7.6-0.build.sh @@ -1,6 +1,6 @@ #! /bin/bash -COOKIE=$(mcookie); grep -v V_GREP_ME $0 > /dev/shm/runme-$COOKIE.sh ; sleep 1; exec bash /dev/shm/runme-$COOKIE.sh $1 +#COOKIE=$(mcookie); grep -v V_GREP_ME $0 > /dev/shm/runme-$COOKIE.sh ; sleep 1; exec bash /dev/shm/runme-$COOKIE.sh $1 PKG=python @@ -8,7 +8,7 @@ VERSION=3.7.6 BUILD=0 PYTHONVER=${VERSION%.*} -CUDADIR=/pkg/cuda-10.1.105-0 +CUDADIR=/pkg/cuda-10.2.89-1 PREFIX=/pkg/$PKG-$VERSION-$BUILD if [ -n "$TESTING" ]; then PREFIX=/dev/shm/$PKG-$VERSION-$BUILD ; fi @@ -18,7 +18,7 @@ PATH=/bin:/usr/bin:/usr/sbin:/usr/local/bin set -xe umask 022 -BUILD_TMPDIR=/dev/shm/$PKG-$VERSION-$BUILD.build.tmp +BUILD_TMPDIR=/scratch/local2/$PKG-$VERSION-$BUILD.build.tmp test -d $BUILD_TMPDIR && ( chmod -R u+rwx $BUILD_TMPDIR || true ; rm -rf $BUILD_TMPDIR ) mkdir -p $BUILD_TMPDIR/home export TMPDIR=$BUILD_TMPDIR @@ -175,7 +175,7 @@ make install python3 -m ensurepip pip3 install --prefix=$PREFIX -I pip -pip3 install --prefix=$PREFIX ipython[all] +pip3 install --upgrade --prefix=$PREFIX ipython[all] pip3 install --prefix=$PREFIX SciPy NumPy Matplotlib pandas SymPy nose statsmodels # clean up some dirt that statsmodels placed also into $PREFIX @@ -217,7 +217,6 @@ pip3 install --prefix=$PREFIX louvain # monocle3 prep for Helene pip3 install --prefix=$PREFIX selenium # 19.7.19 Kreitler -> https://webxray.org/ pip3 install --prefix=$PREFIX cooltools # Maria Valieva 17.7.2019 pip3 install --prefix=$PREFIX gmpy # From viper context (Helpdesk Laura Glaser 9.8.2019) -pip3 install --prefix=$PREFIX tensorflow==2.0.1 # tensorflow-2.1.0 needs spezial libs and nightly version(2.2.0) dont pip3 install --prefix=$PREFIX torch pip3 install --prefix=$PREFIX torchvision pip3 install --prefix=$PREFIX plotly @@ -231,18 +230,19 @@ pip3 install --prefix=$PREFIX threadpoolctl pip3 install --prefix=$PREFIX pyopengl rm -rf ${PREFIX}/lib/python$PYTHONVER/site-packages/OpenGL/Tk -# create error, so remove it +# creates error, so remove it # pip3 install --prefix=$PREFIX HiCExplorer # Would be a nice to have, but it likes to downgrade too much, pick some cherries pip3 install --prefix=$PREFIX HiCMatrix -pip3 install --prefix=$PREFIX pyGenomeTracks +pip3 install --prefix=$PREFIX pyGenomeTracks #downgrader cooler 0.8.7-> 0.8.5 -pip3 install --prefix=$PREFIX python-bioformats +pip3 install --prefix=$PREFIX python-bioformats #downgrader matplotlib 3.2.0->3.1.1 docutils 0.16 -> 0.15.2 pip3 install --prefix=$PREFIX jupyterlab # request Peter Arndt pip3 install --prefix=$PREFIX jupyterlab-git pip3 install --prefix=$PREFIX libxml2-python3 # required by several bee packages after swich to python3 pip3 install --prefix=$PREFIX scour # https://github.molgen.mpg.de/mariux64/pkg-scripts/issues/98 +pip3 install --prefix=$PREFIX multiqc #boost for pycuda ( @@ -265,24 +265,6 @@ pip3 install --prefix=$PREFIX scour # https://github.molgen.mpg.de/mar --with-python=$PREFIX/bin/python3 \ --with-libraries=python,thread || exit 1 - ./b2 \ - $NUMJOBS \ - --layout=system \ - --build-dir=build-python3 \ - --prefix=$PREFIX \ - --libdir=$PREFIX/lib \ - --build-type=minimal \ - toolset=gcc \ - variant=release \ - debug-symbols=off \ - link=shared \ - threading=multi \ - runtime-link=shared \ - python=$PYTHONVER \ - cflags="$PKGCFLAGS" \ - cxxflags="$PKGCFLAGS" \ - stage || exit 1 - ./b2 \ $NUMJOBS \ --layout=system \ @@ -304,25 +286,135 @@ pip3 install --prefix=$PREFIX scour # https://github.molgen.mpg.de/mar #pycuda git version to get GL support ( + PYCUDAVERS=2019.1.2 + . ${PREFIX}/profile cd $BUILDDIR - test -d pycuda || git clone --recursive http://git.tiker.net/trees/pycuda.git + test -e pycuda-${PYCUDAVERS}.tar.gz || wget https://github.com/inducer/pycuda/archive/v${PYCUDAVERS}/pycuda-${PYCUDAVERS}.tar.gz + test -d pycuda-${PYCUDAVERS} || tar -xf pycuda-${PYCUDAVERS}.tar.gz + #test -d pycuda || git clone --recursive http://git.tiker.net/trees/pycuda.git - cd pycuda - git clean -dffx + cd pycuda-${PYCUDAVERS} + test -e siteconf.py && rm siteconf.py python3 configure.py \ - --cuda-enable-gl \ - --cuda-root=$CUDADIR \ - --no-use-shipped-boost \ - --boost-python-libname=boost_python37 \ - --boost-inc-dir=$PREFIX/include \ - --boost-lib-dir=$PREFIX/lib \ - --cudadrv-lib-dir='${CUDA_ROOT}/lib64,${CUDA_ROOT}/lib64/stubs' \ - --cudart-lib-dir='${CUDA_ROOT}/lib64,${CUDA_ROOT}/lib64/stubs' \ - --curand-lib-dir='${CUDA_ROOT}/lib64,${CUDA_ROOT}/lib64/stubs'\ - --ldflags=-L$PREFIX/lib + --cuda-enable-gl \ + --cuda-root=$CUDADIR \ + --no-use-shipped-boost \ + --boost-python-libname=boost_python37 \ + --boost-inc-dir=$PREFIX/include \ + --boost-lib-dir=$PREFIX/lib \ + --cudadrv-lib-dir='${CUDA_ROOT}/lib,${CUDA_ROOT}/lib/stubs' \ + --cudart-lib-dir='${CUDA_ROOT}/lib,${CUDA_ROOT}/lib/stubs' \ + --curand-lib-dir='${CUDA_ROOT}/lib,${CUDA_ROOT}/lib/stubs'\ + --ldflags=-L$PREFIX/lib + + python3 setup.py install --prefix $PREFIX +) + +# hic2cool from source to stop downgrading python packages +( + HICVERS=0.8.0 + + . ${PREFIX}/profile + cd $BUILDDIR + test -e hic2cool-${HICVERS}.tar.gz || wget https://github.com/4dn-dcic/hic2cool/archive/${HICVERS}/hic2cool-${HICVERS}.tar.gz + test -d hic2cool-${HICVERS} || tar -xf hic2cool-${HICVERS}.tar.gz + + cd hic2cool-${HICVERS} + + if $( grep '==' requirements.txt >/dev/null ); then + patch -p1 <<'EOFHIC' +diff --git a/requirements.txt b/requirements.txt +index d7e701b..9d3a01c 100644 +--- a/requirements.txt ++++ b/requirements.txt +@@ -1,5 +1,5 @@ +-h5py==2.8.0 +-pandas==0.24.2 +-numpy==1.16.3 +-scipy==1.2.1 +-cooler==0.8.5 ++h5py ++pandas ++numpy ++scipy ++cooler +EOFHIC +fi + pip3 install -r requirements.txt --prefix ${PREFIX} + python3 setup.py install --prefix ${PREFIX} +) + +#tensorflow +( + TFVERSION=2.1.0 + TFSRC=https://github.com/tensorflow/tensorflow/archive/v${TFVERSION}/tensorflow-${TFVERSION}.tar.gz + + . ${PREFIX}/profile + . /pkg/bazel-0.28.1-0/profile + cd $BUILDDIR + test -e tensorflow-${TFVERSION}.tar.gz || wget $TFSRC + test -d tensorflow-${TFVERSION} || tar -xf tensorflow-${TFVERSION}.tar.gz + + cd tensorflow-${TFVERSION} +#vars for config + export PYTHON_BIN_PATH=${PREFIX}/bin/python3 + export USE_DEFAULT_PYTHON_LIB_PATH=1 + export TF_NEED_JEMALLOC=1 + export TF_NEED_KAFKA=0 + export TF_NEED_OPENCL_SYCL=0 + export TF_NEED_AWS=0 + export TF_NEED_GCP=0 + export TF_NEED_HDFS=0 + export TF_NEED_S3=0 + export TF_ENABLE_XLA=1 + export TF_NEED_GDR=0 + export TF_NEED_VERBS=0 + export TF_NEED_OPENCL=0 + export TF_NEED_MPI=0 + export TF_NEED_TENSORRT=1 + export TF_NEED_NGRAPH=0 + export TF_NEED_IGNITE=0 + export TF_NEED_ROCM=0 + export TF_SET_ANDROID_WORKSPACE=0 + export TF_DOWNLOAD_CLANG=0 + export TF_NCCL_VERSION=2.5 + export TF_IGNORE_MAX_BAZEL_VERSION=1 + export NCCL_INSTALL_PATH=${CUDADIR} + export GCC_HOST_COMPILER_PATH=/usr/bin/gcc + export TF_CUDA_CLANG=0 + export CLANG_CUDA_COMPILER_PATH=/usr/bin/clang + export TF_CUDA_PATHS=${CUDADIR},${CUDADIR}/extras/CUPTI,${CUDADIR}/nvvm + export TF_CUDA_VERSION=$(${CUDADIR}/bin/nvcc --version | sed -n 's/^.*release \(.*\),.*/\1/p') + export TF_CUDNN_VERSION=$(sed -n 's/^#define CUDNN_MAJOR\s*\(.*\).*/\1/p' ${CUDADIR}/include/cudnn.h) + export TF_CUDA_COMPUTE_CAPABILITIES=3.5,3.7,5.0,5.2,5.3,6.0,6.1,6.2,7.0,7.2,7.5 + export CC_OPT_FLAGS="-march=nehalem" + export TF_NEED_CUDA=1 + + ./configure + bazel build \ + --cxxopt="-march=nehalem" \ + --copt="-march=nehalem" \ + //tensorflow:libtensorflow.so \ + //tensorflow:libtensorflow_cc.so \ + //tensorflow:install_headers \ + //tensorflow/tools/pip_package:build_pip_package + + test -d tensorflow-pip && rm -r tensorflow-pip + mkdir tensorflow-pip + ./bazel-bin/tensorflow/tools/pip_package/build_pip_package --src ${BUILDDIR}/tensorflow-pip + cd ${BUILDDIR} + #remove symlinks into TMPDIR + test -e tensorflow-pip.tar && rm tensorflow-pip.tar + tar -chf tensorflow-pip.tar tensorflow-pip + test -d tensorflow-pip && rm -r tensorflow-pip + + tar -xf tensorflow-pip.tar + cd ${BUILDDIR}/tensorflow-pip + python3 setup.py build python3 setup.py install --prefix $PREFIX + ) # try to load all packages, maybe we can see installation errors python3 -c 'help("modules")' > /dev/null