diff --git a/python-3.8.6-0.build.sh b/python-3.8.6-0.build.sh new file mode 100755 index 0000000..11aa332 --- /dev/null +++ b/python-3.8.6-0.build.sh @@ -0,0 +1,370 @@ +#! /bin/bash + +# COOKIE=$(mcookie|cut -c-8); grep -v V_GREP_ME $0 > /dev/shm/runme-$COOKIE.sh ; sleep 0.3; exec bash /dev/shm/runme-$COOKIE.sh +# TESTING=1 + +PKG=python +VERSION=3.8.6 +BUILD=0 +PYTHONVER=${VERSION%.*} + +CUDADIR=/pkg/cuda-10.2.89-1 + +PREFIX=/pkg/$PKG-$VERSION-$BUILD +# tensorflow might not fit into /dev/shm +if [ -n "$TESTING" ]; then PREFIX=/scratch/local2/$PKG-$VERSION-$BUILD ; fi + +PATH=/bin:/usr/bin:/usr/sbin:/usr/local/bin + +set -xe +umask 022 + +BUILD_TMPDIR=/scratch/local2/$PKG-$VERSION-$BUILD.build.tmp +test -d $BUILD_TMPDIR && ( chmod -R u+rwx $BUILD_TMPDIR || true ; rm -rf $BUILD_TMPDIR ) +mkdir -p $BUILD_TMPDIR/home +export TMPDIR=$BUILD_TMPDIR +export HOME=$BUILD_TMPDIR/home + +exec $PREFIX/profile <<-EOF + . $CUDADIR/profile + PATH=$PREFIX/bin:\$PATH + export LD_LIBRARY_PATH=$PREFIX/lib\${LD_LIBRARY_PATH:+:}\$LD_LIBRARY_PATH + PKG_CONFIG_PATH=$PREFIX/lib/pkgconfig\${PKG_CONFIG_PATH:+:}\$PKG_CONFIG_PATH + export PKG_CONFIG_PATH + if [ -d $PREFIX/.compatlibs ]; then export LD_LIBRARY_PATH=$PREFIX/.compatlibs\${LD_LIBRARY_PATH:+:}\$LD_LIBRARY_PATH ; fi +EOF +. $PREFIX/profile + +NPROC=$(( $(nproc) * 4 / 5 + 1 )) +export MAKEFLAGS="-j $NPROC" + +BUILDDIR=$PREFIX/build + +mkdir -p $BUILDDIR +cd $BUILDDIR + +# former url: https://www.python.org/ftp/python/$VERSION/Python-$VERSION.tar.xz +test -e Python-$VERSION.tar.xz || wget --no-verbose https://beehive.molgen.mpg.de/ea132d6f449766623eee886966c7d41f/Python-3.8.6.tgz +test -d Python-$VERSION || tar xvf Python-$VERSION.tgz +cd Python-$VERSION + +LDFLAGS="-Wl,-rpath=$PREFIX/lib" \ +./configure \ + --prefix=$PREFIX \ + --enable-shared + +make -j $NPROC +# make test || true # 'test_socket' seems to run for ever +make install +# some parts may just call 'python', defuse +# ( cd $PREFIX/bin; ln -s python3.8 python) +ln -s python3.8 $PREFIX/bin/python + +( # sip: tool to create Python bindings for C and C++ libraries. + cd $PREFIX/build + SIP_VERSION=4.19.18 + # former url: https://www.riverbankcomputing.com/static/Downloads/sip/$SIP_VERSION/sip-$SIP_VERSION.tar.gz + SIP_SRC=https://beehive.molgen.mpg.de/c06bfa2206ddebbff983c272b40a84b9/sip-4.19.18.tar.gz + test -e sip-$SIP_VERSION.tar.gz || wget --no-verbose $SIP_SRC + test -d sip-$SIP_VERSION || tar -xf sip-$SIP_VERSION.tar.gz + + cd sip-$SIP_VERSION + + #### ATTN! + #### python meanders between /python${PYTHONVER} and /python${PYTHONVER}m + + python3 configure.py \ + -b $PREFIX/bin \ + -d $PREFIX/lib/python$PYTHONVER/site-packages \ + -e $PREFIX/include/python${PYTHONVER} \ + -v $PREFIX/share/sip \ + --sip-module=sip + + # well, right here share/sip is spoilage, but maybe some packages will use the decent sip later on ... + mkdir -p $PREFIX/share/sip + make -j $NPROC + make install +) + + +( # now for the private (sic!) version required by newer PyQt + cd $PREFIX/build + mkdir -p private_sip + cd private_sip + SIP_VERSION=4.19.18 + # former url: https://www.riverbankcomputing.com/static/Downloads/sip/$SIP_VERSION/sip-$SIP_VERSION.tar.gz + SIP_SRC=https://beehive.molgen.mpg.de/c06bfa2206ddebbff983c272b40a84b9/sip-4.19.18.tar.gz + test -e sip-$SIP_VERSION.tar.gz || wget --no-verbose $SIP_SRC + test -d sip-$SIP_VERSION || tar -xf sip-$SIP_VERSION.tar.gz + + cd sip-$SIP_VERSION + + # note $PREFIX/share/sip/PyQt5 is mandatory + python3 configure.py \ + -d $PREFIX/lib/python$PYTHONVER/site-packages \ + -v $PREFIX/share/sip/PyQt5 \ + --no-tools \ + --sip-module=PyQt5.sip + + mkdir -p $PREFIX/share/sip/PyQt5 + make -j $NPROC + # The resulting sip.so in site-packages/PyQt5 differs indeed a bit (grep for PyQt5) + make install +) + +( + PYQT_VERSION=5.13.0 + # former url: https://www.riverbankcomputing.com/static/Downloads/PyQt5/$PYQT_VERSION/PyQt5_gpl-$PYQT_VERSION.tar.gz + PYQT_SRC=https://beehive.molgen.mpg.de/0848fa62dc99ecf2e96f7cdda727c8a2/PyQt5_gpl-5.13.0.tar.gz + cd $BUILDDIR + test -e PyQt5_gpl-$PYQT_VERSION.tar.gz || wget --no-verbose $PYQT_SRC + test -d PyQt5_gpl-$PYQT_VERSION || tar -xf PyQt5_gpl-$PYQT_VERSION.tar.gz + + cd PyQt5_gpl-$PYQT_VERSION + + #### ATTN! + #### python meanders between /python${PYTHONVER} and /python${PYTHONVER}m + + # export QTDIR=/usr/local/qt5 + python3 configure.py \ + -b $PREFIX/bin \ + -d $PREFIX/lib/python$PYTHONVER/site-packages \ + -n $PREFIX/share/qsci \ + -q /usr/local/qt5/bin/qmake \ + -v $PREFIX/share/sip/PyQt5 \ + --sip-incdir=$PREFIX/include/python${PYTHONVER} \ + --confirm-license \ + --no-qml-plugin \ + --no-designer-plugin \ + --no-dist-info \ + --verbose + + make -j $NPROC + + # Did I miss something in the docs ??? + # w/o the *.so files inplace a clean 'make install' fails with file not found ... + # -> FileNotFoundError: [Errno 2] No such file or directory: + # '/dev/shm/python-3.7.4-0/lib/python3.7/site-packages/PyQt5/QtCore.so' + mkdir -p $PREFIX/lib/python$PYTHONVER/site-packages/PyQt5 + for F in `find | grep '\.so$' | grep -v '/lib'` ; do + cp -vp $F $PREFIX/lib/python$PYTHONVER/site-packages/PyQt5/ + done + make install +) + +( + QSCINTILLA_VERSION=2.11.2 + # former url: https://www.riverbankcomputing.com/static/Downloads/QScintilla/$QSCINTILLA_VERSION/QScintilla_gpl-$QSCINTILLA_VERSION.tar.gz + QSCINTILLA_SRC=https://beehive.molgen.mpg.de/2ec4833e790e44698c50ec703f274bab/QScintilla_gpl-2.11.2.tar.gz + + cd $BUILDDIR + test -e QScintilla_gpl-$QSCINTILLA_VERSION.tar.gz || wget --no-verbose $QSCINTILLA_SRC + test -d QScintilla_gpl-$QSCINTILLA_VERSION || tar -xf QScintilla_gpl-$QSCINTILLA_VERSION.tar.gz + cd QScintilla_gpl-$QSCINTILLA_VERSION + + cd Python + + # apidir follows PyQt, not Qt (default is: QT_INSTALL_DATA/qsci) + # creation of PEP 376 .dist-info fails in $PREFIX/share + python3 configure.py \ + --pyqt=PyQt5 \ + --qmake=/usr/local/qt5/bin/qmake \ + --apidir=$PREFIX/share/qsci \ + --no-dist-info + + make -j $NPROC + + # Call it a deja-vu, nerve ... + # -> FileNotFoundError: [Errno 2] No such file or directory: '/dev/shm/python-3.7.4-0/lib/python3.7/site-packages/PyQt5/Qsci.so' + mkdir -p $PREFIX/lib/python$PYTHONVER/site-packages/PyQt5 + for F in `find | grep '\.so$' | grep -v '/lib'` ; do + cp -vp $F $PREFIX/lib/python$PYTHONVER/site-packages/PyQt5 + done + make install +) + +python3 -m ensurepip +pip3 install --prefix=$PREFIX -I pip + +pip3 install --prefix=$PREFIX ipython[all] + +pip3 install --prefix=$PREFIX SciPy NumPy Matplotlib pandas SymPy nose statsmodels +# clean up some dirt that statsmodels placed also into $PREFIX +rm -vf $PREFIX/LICENSE.txt +rm -vf $PREFIX/setup.cfg + +pip3 install --prefix=$PREFIX Cython + +pip3 install --prefix=$PREFIX pysam +pip3 install --prefix=$PREFIX seaborn +pip3 install --prefix=$PREFIX scikit-learn +#pip3 install --prefix=$PREFIX metaseq # fails (Latest commit fa875d1 on Jan 10, 2017) +pip3 install --prefix=$PREFIX Mako +pip3 install --prefix=$PREFIX meson # required in mariux + +pip3 install --prefix=$PREFIX cutadapt # Helpdesk 01/24/18 16:19, Sabrina Krakau +pip3 install --prefix=$PREFIX pygobject # Helpdesk 03/26/18 14:19, Virginie Stanisla +pip3 install --prefix=$PREFIX blink1 # Blink1-lib 07/04/18 12:00, Niclas Hofmann + +pip3 install --prefix=$PREFIX umi-tools +pip3 install --prefix=$PREFIX sphinx +pip3 install --prefix=$PREFIX recommonmark +pip3 install --prefix=$PREFIX fastcluster +pip3 install --prefix=$PREFIX pydot +pip3 install --prefix=$PREFIX pygraphviz +pip3 install --prefix=$PREFIX deeptools +pip3 install --prefix=$PREFIX HTSeq +pip3 install --prefix=$PREFIX keras +pip3 install --prefix=$PREFIX MotifScan # Alena 30.8.2018 +pip3 install --prefix=$PREFIX MAmotif +pip3 install --prefix=$PREFIX hic2cool # Helpdesk Robert Schoepflin 17.09.2018 +#pip3 install --prefix=$PREFIX hifive # only python 2.7+ +pip3 install --prefix=$PREFIX snakemake +pip3 install --prefix=$PREFIX virtualenv # Donald +pip3 install --prefix=$PREFIX odfpy # Donald +pip3 install --prefix=$PREFIX Dumper # Thomas +pip3 install --prefix=$PREFIX igraph +pip3 install --prefix=$PREFIX louvain # monocle3 prep for Helene +pip3 install --prefix=$PREFIX selenium # 19.7.19 Kreitler -> https://webxray.org/ +pip3 install --prefix=$PREFIX cooltools # Maria Valieva 17.7.2019 +pip3 install --prefix=$PREFIX gmpy # From viper context (Helpdesk Laura Glaser 9.8.2019) +pip3 install --prefix=$PREFIX torch +pip3 install --prefix=$PREFIX torchvision +pip3 install --prefix=$PREFIX plotly +pip3 install --prefix=$PREFIX pyOpenCL +pip3 install --prefix=$PREFIX docopt +pip3 install --prefix=$PREFIX dbus-python +pip3 install --prefix=$PREFIX distro +pip3 install --prefix=$PREFIX notify +pip3 install --prefix=$PREFIX ply +pip3 install --prefix=$PREFIX threadpoolctl + +pip3 install --prefix=$PREFIX pyopengl +# creates error, so remove it +rm -rf ${PREFIX}/lib/python$PYTHONVER/site-packages/OpenGL/Tk + +# pip3 install --prefix=$PREFIX HiCExplorer # Would be a nice to have, but it likes to downgrade too much, pick some cherries +pip3 install --prefix=$PREFIX HiCMatrix +pip3 install --prefix=$PREFIX pybedtools +pip3 install --prefix=$PREFIX gffutils +# pyGenomeTracks, version 3.6, Requires-Dist: matplotlib (<=3.3.2,>=3.1.1), somehow they don't catch it ... +pip3 install --prefix=$PREFIX --no-deps "pyGenomeTracks" # example produces plots, so give it a chance +pip3 install --prefix=$PREFIX python-bioformats + +pip3 install --prefix=$PREFIX jupyterlab # request Peter Arndt +pip3 install --prefix=$PREFIX jupyterlab-git +pip3 install --prefix=$PREFIX jupyterlab-git +pip3 install --prefix=$PREFIX igv_jupyterlab # Matthias Lienhard HD 01-04-2020 +pip3 install --prefix=$PREFIX igv-jupyter +mkdir -vp $PREFIX/share/jupyter/lab/settings # try to defeat a silly error in jupyter labextension install + +pip3 install --prefix=$PREFIX libxml2-python3 # required by several bee packages after swich to python3 +pip3 install --prefix=$PREFIX scour # https://github.molgen.mpg.de/mariux64/pkg-scripts/issues/98 +pip3 install --prefix=$PREFIX multiqc + +pip3 install --prefix=$PREFIX thonny # nice little ide, kreitler +pip3 install --prefix=$PREFIX pybind11 +pip3 install --prefix=$PREFIX svgwrite +pip3 install --prefix=$PREFIX pyomo +pip3 install --prefix=$PREFIX jsbsim # kreitler, hobby + +pip3 install --prefix=$PREFIX bioservices +pip3 install --prefix=$PREFIX pairtools +pip3 install --prefix=$PREFIX SharedArray +pip3 install --prefix=$PREFIX pyabc # dependency for stochkit, HD 20.07.20 Gemma Noviello +pip3 install --prefix=$PREFIX petab +pip3 install --prefix=$PREFIX tadtool # HD 26.01.21 Michael Robson + +#boost for pycuda +( + . ${PREFIX}/profile + BOOST_VERSION=(1.72.0 1_72_0) + # former url:https://dl.bintray.com/boostorg/release/${BOOST_VERSION[0]}/source/boost_${BOOST_VERSION[1]}.tar.gz + BOOST_SRC=https://beehive.molgen.mpg.de/e2b0b1eac302880461bcbef097171758/boost_1_72_0.tar.gz + PKGCFLAGS="-O2 -fPIC" + + cd $BUILDDIR + test -e boost_${BOOST_VERSION[1]}.tar.gz || wget --no-verbose $BOOST_SRC + test -d boost-${BOOST_VERSION[0]} || mkdir boost-${BOOST_VERSION[0]} && tar -xf boost_${BOOST_VERSION[1]}.tar.gz -C boost-${BOOST_VERSION[0]} --strip-components=1 + + #### ATTN! + #### python meanders between /python${PYTHONVER} and /python${PYTHONVER}m + + cd boost-${BOOST_VERSION[0]} + sed -e "/using python/ s|;|: ${PREFIX}/include/python${PYTHONVER} ;|" \ + -i bootstrap.sh + + ./bootstrap.sh \ + --with-toolset=gcc \ + --with-icu \ + --with-python=$PREFIX/bin/python3 \ + --with-libraries=python,thread || exit 1 + + ./b2 \ + $NUMJOBS \ + --layout=system \ + --build-dir=build-python3 \ + --prefix=$PREFIX \ + --libdir=$PREFIX/lib \ + --build-type=minimal \ + toolset=gcc \ + variant=release \ + debug-symbols=off \ + link=shared \ + threading=multi \ + runtime-link=shared \ + python=$PYTHONVER \ + cflags="$PKGCFLAGS" \ + cxxflags="$PKGCFLAGS" \ + install || exit 1 +) + +#pycuda git version to get GL support +( + PYCUDAVERS=2019.1.2 + PYTHON_LIBNAME=boost_python38 + + . ${PREFIX}/profile + cd $BUILDDIR + # former url: https://github.com/inducer/pycuda/archive/v${PYCUDAVERS}/pycuda-${PYCUDAVERS}.tar.gz + test -e pycuda-${PYCUDAVERS}.tar.gz || wget --no-verbose https://beehive.molgen.mpg.de/2fdf39e15a9c01e30e97144063b571dc/pycuda-2019.1.2.tar.gz + test -d pycuda-${PYCUDAVERS} || tar -xf pycuda-${PYCUDAVERS}.tar.gz + #test -d pycuda || git clone --recursive http://git.tiker.net/trees/pycuda.git + + cd pycuda-${PYCUDAVERS} + test -e siteconf.py && rm siteconf.py + python3 configure.py \ + --cuda-enable-gl \ + --cuda-root=$CUDADIR \ + --no-use-shipped-boost \ + --boost-python-libname=$PYTHON_LIBNAME \ + --boost-inc-dir=$PREFIX/include \ + --boost-lib-dir=$PREFIX/lib \ + --cudadrv-lib-dir='${CUDA_ROOT}/lib,${CUDA_ROOT}/lib/stubs' \ + --cudart-lib-dir='${CUDA_ROOT}/lib,${CUDA_ROOT}/lib/stubs' \ + --curand-lib-dir='${CUDA_ROOT}/lib,${CUDA_ROOT}/lib/stubs'\ + --ldflags=-L$PREFIX/lib + + python3 setup.py install --prefix $PREFIX +) + + +# bamToBigWig - just for the taste, HD 20.1.21 T. Zehnder +( + # https://github.com/PanosFirmpas/bamToBigWig + cd $PREFIX/bin + wget --no-verbose https://beehive.molgen.mpg.de/bc6815740fbec1586054ad42e0b90a2d/bamToBigWig + chmod +x bamToBigWig +) + +# tensorflow is omitted in this build. + +# try to load all packages, warnings/errors are not fatal, but give some additional hints. +echo "# START load test." +python3 -c 'help("modules")' > /dev/null +echo "# END load test." + +exit