Permalink
Cannot retrieve contributors at this time
Name already in use
A tag already exists with the provided branch name. Many Git commands accept both tag and branch names, so creating this branch may cause unexpected behavior. Are you sure you want to create this branch?
pkg-scripts/python-3.7.7-0.build.sh
Go to fileThis commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
executable file
387 lines (322 sloc)
12.8 KB
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| #! /bin/bash | |
| PKG=python | |
| VERSION=3.7.7 | |
| BUILD=0 | |
| PYTHONVER=${VERSION%.*} | |
| CUDADIR=/pkg/cuda-10.2.89-1 | |
| PREFIX=/pkg/$PKG-$VERSION-$BUILD | |
| if [ -n "$TESTING" ]; then PREFIX=/dev/shm/$PKG-$VERSION-$BUILD ; fi | |
| PATH=/bin:/usr/bin:/usr/sbin:/usr/local/bin | |
| set -xe | |
| umask 022 | |
| BUILD_TMPDIR=/scratch/local2/$PKG-$VERSION-$BUILD.build.tmp | |
| test -d $BUILD_TMPDIR && ( chmod -R u+rwx $BUILD_TMPDIR || true ; rm -rf $BUILD_TMPDIR ) | |
| mkdir -p $BUILD_TMPDIR/home | |
| export TMPDIR=$BUILD_TMPDIR | |
| export HOME=$BUILD_TMPDIR/home | |
| exec </dev/null | |
| mkdir -p $PREFIX | |
| cat >$PREFIX/profile <<-EOF | |
| . $CUDADIR/profile | |
| PATH=$PREFIX/bin:\$PATH | |
| export LD_LIBRARY_PATH=$PREFIX/lib\${LD_LIBRARY_PATH:+:}\$LD_LIBRARY_PATH | |
| PKG_CONFIG_PATH=$PREFIX/lib/pkgconfig\${PKG_CONFIG_PATH:+:\$PKG_CONFIG_PATH} | |
| export PKG_CONFIG_PATH | |
| if [ -d $PREFIX/.compatlibs ]; then export LD_LIBRARY_PATH=$PREFIX/.compatlibs\${LD_LIBRARY_PATH:+:\$LD_LIBRARY_PATH} ; fi | |
| EOF | |
| . $PREFIX/profile | |
| export MAKEFLAGS="-j $(nproc)" | |
| BUILDDIR=$PREFIX/build | |
| mkdir -p $BUILDDIR | |
| cd $BUILDDIR | |
| test -e Python-$VERSION.tar.xz || wget https://www.python.org/ftp/python/$VERSION/Python-$VERSION.tar.xz | |
| test -d Python-$VERSION || tar xvf Python-$VERSION.tar.xz | |
| cd Python-$VERSION | |
| LDFLAGS="-Wl,-rpath=$PREFIX/lib" \ | |
| ./configure \ | |
| --prefix=$PREFIX \ | |
| --enable-shared | |
| make -j $(nproc) | |
| # make test || true # 'test_socket' seems to run for ever | |
| make install | |
| ( # sip: tool to create Python bindings for C and C++ libraries. | |
| cd $PREFIX/build | |
| SIP_VERSION=4.19.18 | |
| SIP_SRC=https://www.riverbankcomputing.com/static/Downloads/sip/$SIP_VERSION/sip-$SIP_VERSION.tar.gz | |
| test -e sip-$SIP_VERSION.tar.gz || wget $SIP_SRC | |
| test -d sip-$SIP_VERSION || tar -xf sip-$SIP_VERSION.tar.gz | |
| cd sip-$SIP_VERSION | |
| python3 configure.py \ | |
| -b $PREFIX/bin \ | |
| -d $PREFIX/lib/python$PYTHONVER/site-packages \ | |
| -e $PREFIX/include/python${PYTHONVER}m \ | |
| -v $PREFIX/share/sip \ | |
| --sip-module=sip | |
| # well, right here share/sip is spoilage, but maybe some packages will use the decent sip later on ... | |
| mkdir -p $PREFIX/share/sip | |
| make -j $(nproc); | |
| make install | |
| ) | |
| ( # now for the private (sic!) version required by newer PyQt | |
| cd $PREFIX/build | |
| mkdir -p private_sip | |
| cd private_sip | |
| SIP_VERSION=4.19.18 | |
| SIP_SRC=https://www.riverbankcomputing.com/static/Downloads/sip/$SIP_VERSION/sip-$SIP_VERSION.tar.gz | |
| test -e sip-$SIP_VERSION.tar.gz || wget $SIP_SRC | |
| test -d sip-$SIP_VERSION || tar -xf sip-$SIP_VERSION.tar.gz | |
| cd sip-$SIP_VERSION | |
| # note $PREFIX/share/sip/PyQt5 is mandatory | |
| python3 configure.py \ | |
| -d $PREFIX/lib/python$PYTHONVER/site-packages \ | |
| -v $PREFIX/share/sip/PyQt5 \ | |
| --no-tools \ | |
| --sip-module=PyQt5.sip | |
| mkdir -p $PREFIX/share/sip/PyQt5 | |
| make -j $(nproc) | |
| # The resulting sip.so in site-packages/PyQt5 differs indeed a bit (grep for PyQt5) | |
| make install | |
| ) | |
| ( | |
| PYQT_VERSION=5.13.0 | |
| PYQT_SRC=https://www.riverbankcomputing.com/static/Downloads/PyQt5/$PYQT_VERSION/PyQt5_gpl-$PYQT_VERSION.tar.gz | |
| cd $BUILDDIR | |
| test -e PyQt5_gpl-$PYQT_VERSION.tar.gz || wget $PYQT_SRC | |
| test -d PyQt5_gpl-$PYQT_VERSION || tar -xf PyQt5_gpl-$PYQT_VERSION.tar.gz | |
| cd PyQt5_gpl-$PYQT_VERSION | |
| # export QTDIR=/usr/local/qt5 | |
| python3 configure.py \ | |
| -b $PREFIX/bin \ | |
| -d $PREFIX/lib/python$PYTHONVER/site-packages \ | |
| -n $PREFIX/share/qsci \ | |
| -q /usr/local/qt5/bin/qmake \ | |
| -v $PREFIX/share/sip/PyQt5 \ | |
| --sip-incdir=$PREFIX/include/python${PYTHONVER}m \ | |
| --confirm-license \ | |
| --no-qml-plugin \ | |
| --no-designer-plugin \ | |
| --no-dist-info \ | |
| --verbose | |
| make -j $(nproc) | |
| # Did I miss something in the docs ??? | |
| # w/o the *.so files inplace a clean 'make install' fails with file not found ... | |
| # -> FileNotFoundError: [Errno 2] No such file or directory: | |
| # '/dev/shm/python-3.7.4-0/lib/python3.7/site-packages/PyQt5/QtCore.so' | |
| mkdir -p $PREFIX/lib/python$PYTHONVER/site-packages/PyQt5 | |
| for F in `find | grep '\.so$' | grep -v '/lib'` ; do | |
| cp -vp $F $PREFIX/lib/python$PYTHONVER/site-packages/PyQt5/ | |
| done | |
| make install | |
| ) | |
| ( | |
| QSCINTILLA_VERSION=2.11.2 | |
| QSCINTILLA_SRC=https://www.riverbankcomputing.com/static/Downloads/QScintilla/$QSCINTILLA_VERSION/QScintilla_gpl-$QSCINTILLA_VERSION.tar.gz | |
| cd $BUILDDIR | |
| test -e QScintilla_gpl-$QSCINTILLA_VERSION.tar.gz || wget $QSCINTILLA_SRC | |
| test -d QScintilla_gpl-$QSCINTILLA_VERSION || tar -xf QScintilla_gpl-$QSCINTILLA_VERSION.tar.gz | |
| cd QScintilla_gpl-$QSCINTILLA_VERSION | |
| cd Python | |
| # apidir follows PyQt, not Qt (default is: QT_INSTALL_DATA/qsci) | |
| # creation of PEP 376 .dist-info fails in $PREFIX/share | |
| python3 configure.py \ | |
| --pyqt=PyQt5 \ | |
| --qmake=/usr/local/qt5/bin/qmake \ | |
| --apidir=$PREFIX/share/qsci \ | |
| --no-dist-info | |
| make -j $(nproc) | |
| # Call it a deja-vu, nerve ... | |
| # -> FileNotFoundError: [Errno 2] No such file or directory: '/dev/shm/python-3.7.4-0/lib/python3.7/site-packages/PyQt5/Qsci.so' | |
| mkdir -p $PREFIX/lib/python$PYTHONVER/site-packages/PyQt5 | |
| for F in `find | grep '\.so$' | grep -v '/lib'` ; do | |
| cp -vp $F $PREFIX/lib/python$PYTHONVER/site-packages/PyQt5 | |
| done | |
| make install | |
| ) | |
| python3 -m ensurepip | |
| pip3 install --prefix=$PREFIX -I pip | |
| pip3 install --prefix=$PREFIX ipython[all] | |
| pip3 install --prefix=$PREFIX SciPy NumPy Matplotlib pandas SymPy nose statsmodels | |
| # clean up some dirt that statsmodels placed also into $PREFIX | |
| rm -vf $PREFIX/LICENSE.txt | |
| rm -vf $PREFIX/setup.cfg | |
| pip3 install --prefix=$PREFIX Cython | |
| pip3 install --prefix=$PREFIX pysam | |
| pip3 install --prefix=$PREFIX seaborn | |
| pip3 install --prefix=$PREFIX scikit-learn | |
| #pip3 install --prefix=$PREFIX metaseq # fails (Latest commit fa875d1 on Jan 10, 2017) | |
| pip3 install --prefix=$PREFIX Mako | |
| pip3 install --prefix=$PREFIX meson # required in mariux | |
| pip3 install --prefix=$PREFIX cutadapt # Helpdesk 01/24/18 16:19, Sabrina Krakau | |
| pip3 install --prefix=$PREFIX pygobject # Helpdesk 03/26/18 14:19, Virginie Stanisla | |
| pip3 install --prefix=$PREFIX blink1 # Blink1-lib 07/04/18 12:00, Niclas Hofmann | |
| pip3 install --prefix=$PREFIX umi-tools | |
| pip3 install --prefix=$PREFIX sphinx | |
| pip3 install --prefix=$PREFIX recommonmark | |
| pip3 install --prefix=$PREFIX fastcluster | |
| pip3 install --prefix=$PREFIX pydot | |
| pip3 install --prefix=$PREFIX pygraphviz | |
| pip3 install --prefix=$PREFIX deeptools | |
| pip3 install --prefix=$PREFIX HTSeq | |
| pip3 install --prefix=$PREFIX keras | |
| pip3 install --prefix=$PREFIX MotifScan # Alena 30.8.2018 | |
| pip3 install --prefix=$PREFIX MAmotif | |
| pip3 install --prefix=$PREFIX hic2cool # Helpdesk Robert Schoepflin 17.09.2018 | |
| #pip3 install --prefix=$PREFIX hifive # only python 2.7+ | |
| pip3 install --prefix=$PREFIX snakemake | |
| pip3 install --prefix=$PREFIX virtualenv # Donald | |
| pip3 install --prefix=$PREFIX odfpy # Donald | |
| pip3 install --prefix=$PREFIX Dumper # Thomas | |
| pip3 install --prefix=$PREFIX igraph | |
| pip3 install --prefix=$PREFIX louvain # monocle3 prep for Helene | |
| pip3 install --prefix=$PREFIX selenium # 19.7.19 Kreitler -> https://webxray.org/ | |
| pip3 install --prefix=$PREFIX cooltools # Maria Valieva 17.7.2019 | |
| pip3 install --prefix=$PREFIX gmpy # From viper context (Helpdesk Laura Glaser 9.8.2019) | |
| pip3 install --prefix=$PREFIX torch | |
| pip3 install --prefix=$PREFIX torchvision | |
| pip3 install --prefix=$PREFIX plotly | |
| pip3 install --prefix=$PREFIX pyOpenCL | |
| pip3 install --prefix=$PREFIX docopt | |
| pip3 install --prefix=$PREFIX dbus-python | |
| pip3 install --prefix=$PREFIX distro | |
| pip3 install --prefix=$PREFIX notify | |
| pip3 install --prefix=$PREFIX ply | |
| pip3 install --prefix=$PREFIX threadpoolctl | |
| pip3 install --prefix=$PREFIX pyopengl | |
| rm -rf ${PREFIX}/lib/python$PYTHONVER/site-packages/OpenGL/Tk | |
| # creates error, so remove it | |
| # pip3 install --prefix=$PREFIX HiCExplorer # Would be a nice to have, but it likes to downgrade too much, pick some cherries | |
| pip3 install --prefix=$PREFIX HiCMatrix | |
| pip3 install --prefix=$PREFIX "pyGenomeTracks<3.3" # 3.3 depends on HiMatrix>=12 which is not yet available | |
| pip3 install --prefix=$PREFIX python-bioformats #downgrader matplotlib 3.2.0->3.1.1 docutils 0.16 -> 0.15.2 | |
| pip3 install --prefix=$PREFIX jupyterlab # request Peter Arndt | |
| pip3 install --prefix=$PREFIX jupyterlab-git | |
| pip3 install --prefix=$PREFIX libxml2-python3 # required by several bee packages after swich to python3 | |
| pip3 install --prefix=$PREFIX scour # https://github.molgen.mpg.de/mariux64/pkg-scripts/issues/98 | |
| pip3 install --prefix=$PREFIX multiqc | |
| #boost for pycuda | |
| ( | |
| . ${PREFIX}/profile | |
| BOOST_VERSION=(1.72.0 1_72_0) | |
| BOOST_SRC=https://dl.bintray.com/boostorg/release/${BOOST_VERSION[0]}/source/boost_${BOOST_VERSION[1]}.tar.gz | |
| PKGCFLAGS="-O2 -fPIC" | |
| cd $BUILDDIR | |
| # 28.03.2020: upstream server currently broken - use cached archive | |
| #test -e boost_${BOOST_VERSION[1]}.tar.gz || wget $BOOST_SRC | |
| test -e boost_${BOOST_VERSION[1]}.tar.gz || cp /src/mariux/beeroot/downloads/boost_${BOOST_VERSION[1]}.tar.gz . | |
| test -d boost-${BOOST_VERSION[0]} || mkdir boost-${BOOST_VERSION[0]} && tar -xf boost_${BOOST_VERSION[1]}.tar.gz -C boost-${BOOST_VERSION[0]} --strip-components=1 | |
| cd boost-${BOOST_VERSION[0]} | |
| sed -e "/using python/ s|;|: ${PREFIX}/include/python${PYTHONVER}m ;|" \ | |
| -i bootstrap.sh | |
| ./bootstrap.sh \ | |
| --with-toolset=gcc \ | |
| --with-icu \ | |
| --with-python=$PREFIX/bin/python3 \ | |
| --with-libraries=python,thread || exit 1 | |
| ./b2 \ | |
| $NUMJOBS \ | |
| --layout=system \ | |
| --build-dir=build-python3 \ | |
| --prefix=$PREFIX \ | |
| --libdir=$PREFIX/lib \ | |
| --build-type=minimal \ | |
| toolset=gcc \ | |
| variant=release \ | |
| debug-symbols=off \ | |
| link=shared \ | |
| threading=multi \ | |
| runtime-link=shared \ | |
| python=$PYTHONVER \ | |
| cflags="$PKGCFLAGS" \ | |
| cxxflags="$PKGCFLAGS" \ | |
| install || exit 1 | |
| ) | |
| #pycuda git version to get GL support | |
| ( | |
| PYCUDAVERS=2019.1.2 | |
| . ${PREFIX}/profile | |
| cd $BUILDDIR | |
| test -e pycuda-${PYCUDAVERS}.tar.gz || wget https://github.com/inducer/pycuda/archive/v${PYCUDAVERS}/pycuda-${PYCUDAVERS}.tar.gz | |
| test -d pycuda-${PYCUDAVERS} || tar -xf pycuda-${PYCUDAVERS}.tar.gz | |
| #test -d pycuda || git clone --recursive http://git.tiker.net/trees/pycuda.git | |
| cd pycuda-${PYCUDAVERS} | |
| test -e siteconf.py && rm siteconf.py | |
| python3 configure.py \ | |
| --cuda-enable-gl \ | |
| --cuda-root=$CUDADIR \ | |
| --no-use-shipped-boost \ | |
| --boost-python-libname=boost_python37 \ | |
| --boost-inc-dir=$PREFIX/include \ | |
| --boost-lib-dir=$PREFIX/lib \ | |
| --cudadrv-lib-dir='${CUDA_ROOT}/lib,${CUDA_ROOT}/lib/stubs' \ | |
| --cudart-lib-dir='${CUDA_ROOT}/lib,${CUDA_ROOT}/lib/stubs' \ | |
| --curand-lib-dir='${CUDA_ROOT}/lib,${CUDA_ROOT}/lib/stubs'\ | |
| --ldflags=-L$PREFIX/lib | |
| python3 setup.py install --prefix $PREFIX | |
| ) | |
| #tensorflow | |
| ( | |
| TFVERSION=2.1.0 | |
| TFSRC=https://github.com/tensorflow/tensorflow/archive/v${TFVERSION}/tensorflow-${TFVERSION}.tar.gz | |
| . ${PREFIX}/profile | |
| . /pkg/bazel-0.28.1-0/profile | |
| cd $BUILDDIR | |
| test -e tensorflow-${TFVERSION}.tar.gz || wget $TFSRC | |
| test -d tensorflow-${TFVERSION} || tar -xf tensorflow-${TFVERSION}.tar.gz | |
| cd tensorflow-${TFVERSION} | |
| #vars for config | |
| export PYTHON_BIN_PATH=${PREFIX}/bin/python3 | |
| export USE_DEFAULT_PYTHON_LIB_PATH=1 | |
| export TF_NEED_JEMALLOC=1 | |
| export TF_NEED_KAFKA=0 | |
| export TF_NEED_OPENCL_SYCL=0 | |
| export TF_NEED_AWS=0 | |
| export TF_NEED_GCP=0 | |
| export TF_NEED_HDFS=0 | |
| export TF_NEED_S3=0 | |
| export TF_ENABLE_XLA=1 | |
| export TF_NEED_GDR=0 | |
| export TF_NEED_VERBS=0 | |
| export TF_NEED_OPENCL=0 | |
| export TF_NEED_MPI=0 | |
| export TF_NEED_TENSORRT=1 | |
| export TF_NEED_NGRAPH=0 | |
| export TF_NEED_IGNITE=0 | |
| export TF_NEED_ROCM=0 | |
| export TF_SET_ANDROID_WORKSPACE=0 | |
| export TF_DOWNLOAD_CLANG=0 | |
| export TF_NCCL_VERSION=2.5 | |
| export TF_IGNORE_MAX_BAZEL_VERSION=1 | |
| export NCCL_INSTALL_PATH=${CUDADIR} | |
| export GCC_HOST_COMPILER_PATH=/usr/bin/gcc | |
| export TF_CUDA_CLANG=0 | |
| export CLANG_CUDA_COMPILER_PATH=/usr/bin/clang | |
| export TF_CUDA_PATHS=${CUDADIR},${CUDADIR}/extras/CUPTI,${CUDADIR}/nvvm | |
| export TF_CUDA_VERSION=$(${CUDADIR}/bin/nvcc --version | sed -n 's/^.*release \(.*\),.*/\1/p') | |
| export TF_CUDNN_VERSION=$(sed -n 's/^#define CUDNN_MAJOR\s*\(.*\).*/\1/p' ${CUDADIR}/include/cudnn.h) | |
| export TF_CUDA_COMPUTE_CAPABILITIES=3.5,3.7,5.0,5.2,5.3,6.0,6.1,6.2,7.0,7.2,7.5 | |
| export CC_OPT_FLAGS="-march=nehalem" | |
| export TF_NEED_CUDA=1 | |
| ./configure | |
| bazel build \ | |
| --cxxopt="-march=nehalem" \ | |
| --copt="-march=nehalem" \ | |
| //tensorflow:libtensorflow.so \ | |
| //tensorflow:libtensorflow_cc.so \ | |
| //tensorflow:install_headers \ | |
| //tensorflow/tools/pip_package:build_pip_package | |
| test -d tensorflow-pip && rm -r tensorflow-pip | |
| mkdir tensorflow-pip | |
| ./bazel-bin/tensorflow/tools/pip_package/build_pip_package --src ${BUILDDIR}/tensorflow-pip | |
| cd ${BUILDDIR} | |
| #remove symlinks into TMPDIR | |
| test -e tensorflow-pip.tar && rm tensorflow-pip.tar | |
| tar -chf tensorflow-pip.tar tensorflow-pip | |
| test -d tensorflow-pip && rm -r tensorflow-pip | |
| tar -xf tensorflow-pip.tar | |
| cd ${BUILDDIR}/tensorflow-pip | |
| python3 setup.py build | |
| python3 setup.py install --prefix $PREFIX | |
| ) | |
| # try to load all packages, maybe we can see installation errors | |
| python3 -c 'help("modules")' > /dev/null | |
| exit |