Permalink
Cannot retrieve contributors at this time
Name already in use
A tag already exists with the provided branch name. Many Git commands accept both tag and branch names, so creating this branch may cause unexpected behavior. Are you sure you want to create this branch?
pkg-scripts/python-3.8.9-1.build.sh
Go to fileThis commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
executable file
598 lines (501 sloc)
22.3 KB
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| #! /bin/bash | |
| # COOKIE=$(mcookie|cut -c-8); grep -v V_GREP_ME $0 > /dev/shm/runme-$COOKIE.sh ; sleep 0.3; exec bash /dev/shm/runme-$COOKIE.sh | |
| # TESTING=1 | |
| PKG=python | |
| VERSION=3.8.9 | |
| BUILD=1 | |
| PYTHONVER=${VERSION%.*} | |
| CUDADIR=/pkg/cuda-11.1.1-0 | |
| PREFIX=/pkg/$PKG-$VERSION-$BUILD | |
| if [ -n "$TESTING" ]; then PREFIX=/scratch/local2/$PKG-$VERSION-$BUILD ; fi | |
| PATH=/bin:/usr/bin:/usr/sbin:/usr/local/bin | |
| set -xe | |
| umask 022 | |
| BUILD_TMPDIR=/scratch/local2/$PKG-$VERSION-$BUILD.build.tmp | |
| if [ -n "$TESTING" ]; then BUILD_TMPDIR=/scratch/local2/$PKG-$VERSION-$BUILD.test.build.tmp ; fi | |
| test -d $BUILD_TMPDIR && ( chmod -R u+rwx $BUILD_TMPDIR || true ; rm -rf $BUILD_TMPDIR ) | |
| mkdir -p $BUILD_TMPDIR/home/.cache/pip | |
| mkdir -p $BUILD_TMPDIR/home/.local | |
| # copy script to an accessible location | |
| cp $0 $BUILD_TMPDIR; ME="$BUILD_TMPDIR/$(basename $0)" | |
| export TMPDIR=$BUILD_TMPDIR | |
| export HOME=$BUILD_TMPDIR/home | |
| exec </dev/null | |
| mkdir -p $PREFIX | |
| cat >$PREFIX/profile <<-EOF | |
| . $CUDADIR/profile | |
| PATH=$PREFIX/bin:\$PATH | |
| export LD_LIBRARY_PATH=$PREFIX/lib\${LD_LIBRARY_PATH:+:}\$LD_LIBRARY_PATH | |
| PKG_CONFIG_PATH=$PREFIX/lib/pkgconfig\${PKG_CONFIG_PATH:+:}\$PKG_CONFIG_PATH | |
| export PKG_CONFIG_PATH | |
| if [ -d $PREFIX/.compatlibs ]; then export LD_LIBRARY_PATH=$PREFIX/.compatlibs\${LD_LIBRARY_PATH:+:}\$LD_LIBRARY_PATH ; fi | |
| EOF | |
| . $PREFIX/profile | |
| NPROC=$(( $(nproc) * 4 / 5 + 1 )) | |
| export MAKEFLAGS="-j $NPROC" | |
| BUILDDIR=$PREFIX/build | |
| mkdir -p $BUILDDIR | |
| cd $BUILDDIR | |
| # orig: https://www.python.org/ftp/python/3.8.9/Python-3.8.9.tgz | |
| BEEHIVE=https://beehive.molgen.mpg.de/41a5eaa15818cee7ea59e578564a2629/Python-3.8.9.tgz | |
| test -e Python-$VERSION.tgz || wget -nv $BEEHIVE | |
| test -d Python-$VERSION || tar -xf Python-$VERSION.tgz | |
| cd Python-$VERSION | |
| LDFLAGS="-Wl,-rpath=$PREFIX/lib" \ | |
| ./configure \ | |
| --prefix=$PREFIX \ | |
| --enable-shared | |
| make -j $NPROC | |
| # make test # || true # 'test_socket' seems to run for ever | |
| make install | |
| # it might come to happen that something down the line may just call 'python', defuse ... | |
| test -e $PREFIX/bin/python || ln -s python3 $PREFIX/bin/python | |
| ( # sip: tool to create Python bindings for C and C++ libraries., v6 is out, but not so popular at the moment | |
| cd $BUILDDIR | |
| SIP_VERSION=4.19.24 | |
| # orig: https://www.riverbankcomputing.com/static/Downloads/sip/$SIP_VERSION/sip-$SIP_VERSION.tar.gz | |
| BEEHIVE=https://beehive.molgen.mpg.de/595e9ad6bb0a4b3a6ea92c163a05d19c/sip-4.19.24.tar.gz | |
| test -e sip-$SIP_VERSION.tar.gz || wget -nv $BEEHIVE | |
| test -d sip-$SIP_VERSION || tar -xf sip-$SIP_VERSION.tar.gz | |
| cd sip-$SIP_VERSION | |
| #### ATTN! | |
| #### python meanders between /python${PYTHONVER} and /python${PYTHONVER}m | |
| python3 configure.py \ | |
| -b $PREFIX/bin \ | |
| -d $PREFIX/lib/python$PYTHONVER/site-packages \ | |
| -e $PREFIX/include/python${PYTHONVER} \ | |
| -v $PREFIX/share/sip \ | |
| --sip-module=sip | |
| # well, right here share/sip is spoilage, but maybe some packages will use the decent sip later on ... | |
| mkdir -p $PREFIX/share/sip | |
| make -j $NPROC | |
| make install | |
| ) | |
| ( # now for the private (sic!) version required by newer PyQt | |
| cd $BUILDDIR | |
| SIP_VERSION=4.19.24 | |
| mkdir -p private_sip | |
| cd private_sip | |
| # orig: https://www.riverbankcomputing.com/static/Downloads/sip/$SIP_VERSION/sip-$SIP_VERSION.tar.gz | |
| BEEHIVE=https://beehive.molgen.mpg.de/595e9ad6bb0a4b3a6ea92c163a05d19c/sip-4.19.24.tar.gz | |
| test -e sip-$SIP_VERSION.tar.gz || wget -nv $BEEHIVE | |
| test -d sip-$SIP_VERSION || tar -xf sip-$SIP_VERSION.tar.gz | |
| cd sip-$SIP_VERSION | |
| # note $PREFIX/share/sip/PyQt5 is mandatory | |
| python3 configure.py \ | |
| -d $PREFIX/lib/python$PYTHONVER/site-packages \ | |
| -v $PREFIX/share/sip/PyQt5 \ | |
| --no-tools \ | |
| --sip-module=PyQt5.sip | |
| mkdir -p $PREFIX/share/sip/PyQt5 | |
| make -j $NPROC | |
| # The resulting sip.so in site-packages/PyQt5 differs indeed a bit (grep for PyQt5) | |
| make install | |
| ) | |
| ( | |
| cd $BUILDDIR | |
| PYQT_VERSION=5.15.2 | |
| # orig: https://files.pythonhosted.org/packages/28/6c/640e3f5c734c296a7193079a86842a789edb7988dca39eab44579088a1d1/PyQt5-5.15.2.tar.gz | |
| BEEHIVE=https://beehive.molgen.mpg.de/b94576e9e013210dc5aba061913e4bd4/PyQt5-5.15.2.tar.gz | |
| test -e PyQt5-$PYQT_VERSION.tar.gz || wget -nv $BEEHIVE | |
| test -d PyQt5-$PYQT_VERSION || tar -xf PyQt5-$PYQT_VERSION.tar.gz | |
| cd PyQt5-$PYQT_VERSION | |
| #### ATTN! | |
| #### python meanders between /python${PYTHONVER} and /python${PYTHONVER}m | |
| # export QTDIR=/usr/local/qt5 | |
| python3 configure.py \ | |
| -b $PREFIX/bin \ | |
| -d $PREFIX/lib/python$PYTHONVER/site-packages \ | |
| -n $PREFIX/share/qsci \ | |
| -q /usr/local/qt5/bin/qmake \ | |
| -v $PREFIX/share/sip/PyQt5 \ | |
| --sip-incdir=$PREFIX/include/python${PYTHONVER} \ | |
| --confirm-license \ | |
| --no-qml-plugin \ | |
| --no-designer-plugin \ | |
| --no-dist-info \ | |
| --verbose | |
| make -j $NPROC | |
| make install | |
| ) | |
| ( | |
| cd $BUILDDIR | |
| QSCINTILLA_VERSION=2.11.6 | |
| # orig: https://www.riverbankcomputing.com/static/Downloads/QScintilla/$QSCINTILLA_VERSION/QScintilla-$QSCINTILLA_VERSION.tar.gz | |
| BEEHIVE=https://beehive.molgen.mpg.de/343cd0c2c8b425518df2e51eb994fbc6/QScintilla-2.11.6.tar.gz | |
| test -e QScintilla-$QSCINTILLA_VERSION.tar.gz || wget -nv $BEEHIVE | |
| test -d QScintilla-$QSCINTILLA_VERSION || tar -xf QScintilla-$QSCINTILLA_VERSION.tar.gz | |
| cd QScintilla-$QSCINTILLA_VERSION | |
| cd Python | |
| # apidir follows PyQt, not Qt (default is: QT_INSTALL_DATA/qsci) | |
| python3 configure.py \ | |
| --pyqt=PyQt5 \ | |
| --qmake=/usr/local/qt5/bin/qmake \ | |
| --apidir=$PREFIX/share/qsci \ | |
| --no-dist-info | |
| make -j $NPROC | |
| make install | |
| ) | |
| python3 -m ensurepip | |
| pip3 install --prefix=$PREFIX -I pip | |
| pip3 install --prefix=$PREFIX ipython[all] | |
| # install the basic ballpark ... | |
| # numpy holds no ref to *_MAX_THREADS, albeit it's an issue, fix via profile? | |
| # still persisting: NUMEXPR_MAX_THREADS issue caused by numexpr.utils (braindead defaults from this dependency) | |
| pip3 install --prefix=$PREFIX SciPy NumPy Matplotlib pandas SymPy nose statsmodels | |
| # clean up some dirt that statsmodels placed also into $PREFIX | |
| rm -vf $PREFIX/LICENSE.txt | |
| rm -vf $PREFIX/setup.cfg | |
| pip3 install --prefix=$PREFIX Cython | |
| pip3 install --prefix=$PREFIX pysam | |
| pip3 install --prefix=$PREFIX seaborn | |
| pip3 install --prefix=$PREFIX scikit-learn | |
| #pip3 install --prefix=$PREFIX metaseq # fails (Latest commit fa875d1 on Jan 10, 2017) | |
| pip3 install --prefix=$PREFIX Mako | |
| pip3 install --prefix=$PREFIX meson # required in mariux | |
| pip3 install --prefix=$PREFIX cutadapt # Helpdesk 01/24/18 16:19, Sabrina Krakau | |
| pip3 install --prefix=$PREFIX pygobject # Helpdesk 03/26/18 14:19, Virginie Stanisla | |
| pip3 install --prefix=$PREFIX blink1 # Blink1-lib 07/04/18 12:00, Niclas Hofmann | |
| pip3 install --prefix=$PREFIX umi-tools | |
| pip3 install --prefix=$PREFIX sphinx | |
| pip3 install --prefix=$PREFIX recommonmark | |
| pip3 install --prefix=$PREFIX fastcluster | |
| pip3 install --prefix=$PREFIX pydot | |
| pip3 install --prefix=$PREFIX pygraphviz | |
| pip3 install --prefix=$PREFIX deeptools | |
| pip3 install --prefix=$PREFIX HTSeq | |
| pip3 install --prefix=$PREFIX keras | |
| pip3 install --prefix=$PREFIX keras_preprocessing # tensorflow ... | |
| pip3 install --prefix=$PREFIX MotifScan # Alena 30.8.2018 | |
| pip3 install --prefix=$PREFIX MAmotif | |
| pip3 install --prefix=$PREFIX hic2cool # Helpdesk Robert Schoepflin 17.09.2018 | |
| #pip3 install --prefix=$PREFIX hifive # only python 2.7+ | |
| pip3 install --prefix=$PREFIX snakemake | |
| pip3 install --prefix=$PREFIX virtualenv # Donald | |
| pip3 install --prefix=$PREFIX odfpy # Donald | |
| pip3 install --prefix=$PREFIX Dumper # Thomas | |
| pip3 install --prefix=$PREFIX igraph | |
| pip3 install --prefix=$PREFIX louvain # monocle3 prep for Helene | |
| pip3 install --prefix=$PREFIX selenium # 19.7.19 Kreitler -> https://webxray.org/ | |
| pip3 install --prefix=$PREFIX cooltools # Maria Valieva 17.7.2019 | |
| pip3 install --prefix=$PREFIX gmpy # From viper context (Helpdesk Laura Glaser 9.8.2019) | |
| pip3 install --prefix=$PREFIX torch | |
| pip3 install --prefix=$PREFIX torchvision | |
| pip3 install --prefix=$PREFIX plotly | |
| pip3 install --prefix=$PREFIX pyOpenCL | |
| pip3 install --prefix=$PREFIX docopt | |
| pip3 install --prefix=$PREFIX dbus-python | |
| pip3 install --prefix=$PREFIX distro | |
| pip3 install --prefix=$PREFIX notify | |
| pip3 install --prefix=$PREFIX ply | |
| pip3 install --prefix=$PREFIX threadpoolctl | |
| pip3 install --prefix=$PREFIX pyopengl | |
| # creates error, so remove it | |
| rm -rf ${PREFIX}/lib/python$PYTHONVER/site-packages/OpenGL/Tk | |
| # pip3 install --prefix=$PREFIX HiCExplorer # Would be a nice to have, but it likes to downgrade too much, pick some cherries | |
| pip3 install --prefix=$PREFIX HiCMatrix | |
| pip3 install --prefix=$PREFIX pybedtools | |
| pip3 install --prefix=$PREFIX gffutils | |
| pip3 install --prefix=$PREFIX --no-deps "pyGenomeTracks" # example produces plots, so give it a chance, otherwise it's just a dumbfucked downgrader | |
| pip3 install --prefix=$PREFIX python-bioformats | |
| pip3 install --prefix=$PREFIX jupyterlab # request Peter Arndt | |
| pip3 install --prefix=$PREFIX jupyterlab-git | |
| pip3 install --prefix=$PREFIX igv_jupyterlab # Matthias Lienhard HD 01-04-2020 | |
| pip3 install --prefix=$PREFIX igv-jupyter | |
| mkdir -vp $PREFIX/share/jupyter/lab/settings # try to defeat a silly error in jupyter labextension install | |
| pip3 install --prefix=$PREFIX libxml2-python3 # required by several bee packages after swich to python3 | |
| pip3 install --prefix=$PREFIX scour # https://github.molgen.mpg.de/mariux64/pkg-scripts/issues/98 | |
| pip3 install --prefix=$PREFIX multiqc | |
| pip3 install --prefix=$PREFIX thonny # nice little ide, kreitler | |
| pip3 install --prefix=$PREFIX pybind11 | |
| pip3 install --prefix=$PREFIX svgwrite | |
| pip3 install --prefix=$PREFIX pyomo | |
| pip3 install --prefix=$PREFIX jsbsim # kreitler, hobby | |
| pip3 install --prefix=$PREFIX bioservices | |
| pip3 install --prefix=$PREFIX pairtools | |
| pip3 install --prefix=$PREFIX SharedArray | |
| pip3 install --prefix=$PREFIX pyabc # dependency for stochkit, HD 20.07.20 Gemma Noviello | |
| pip3 install --prefix=$PREFIX petab | |
| pip3 install --prefix=$PREFIX tadtool # HD 26.01.21 Michael Robson | |
| # pip3 install --prefix=$PREFIX perf # fails, issue from Paul sep-20, no clue... | |
| # pip3 install --prefix=$PREFIX glpk-py # kreitler, fails, wants setuptools_scm, do i want setuptools_scm? | |
| pip3 install --prefix=$PREFIX helen # HD, a dependency from a request by helene :) | |
| pip3 install --prefix=$PREFIX umap # HD, used by Matthias Lienhard | |
| pip3 install --prefix=$PREFIX macs2 # HD, Alessa Ringel, Tobias Zehnder, old python2 version broke after glibc update | |
| pip3 install --prefix=$PREFIX macs3 | |
| # boost for pycuda | |
| ( | |
| . ${PREFIX}/profile | |
| BOOST_VERSION=(1.75.0 1_75_0) | |
| PKGCFLAGS="-O2 -fPIC" | |
| cd $BUILDDIR | |
| # orig: https://dl.bintray.com/boostorg/release/1.75.0/source/boost_1_75_0.tar.gz | |
| BEEHIVE=https://beehive.molgen.mpg.de//38813f6feb40387dfe90160debd71251/boost_1_75_0.tar.gz | |
| test -e boost_${BOOST_VERSION[1]}.tar.gz || wget -nv $BEEHIVE | |
| test -d boost-${BOOST_VERSION[0]} || mkdir boost-${BOOST_VERSION[0]} && tar -xf boost_${BOOST_VERSION[1]}.tar.gz -C boost-${BOOST_VERSION[0]} --strip-components=1 | |
| #### ATTN! | |
| #### python meanders between /python${PYTHONVER} and /python${PYTHONVER}m | |
| cd boost-${BOOST_VERSION[0]} | |
| sed -e "/using python/ s|;|: ${PREFIX}/include/python${PYTHONVER} ;|" \ | |
| -i bootstrap.sh | |
| ./bootstrap.sh \ | |
| --with-toolset=gcc \ | |
| --with-icu \ | |
| --with-python=$PREFIX/bin/python3 \ | |
| --with-libraries=python,thread || exit 1 | |
| ./b2 \ | |
| -j $NPROC \ | |
| --layout=system \ | |
| --build-dir=build-python3 \ | |
| --prefix=$PREFIX \ | |
| --libdir=$PREFIX/lib \ | |
| --build-type=minimal \ | |
| toolset=gcc \ | |
| variant=release \ | |
| debug-symbols=off \ | |
| link=shared \ | |
| threading=multi \ | |
| runtime-link=shared \ | |
| python=$PYTHONVER \ | |
| cflags="$PKGCFLAGS" \ | |
| cxxflags="$PKGCFLAGS" \ | |
| install || exit 1 | |
| ) | |
| # use pycuda git version to get GL support | |
| ( | |
| PYCUDAVERS=2020.1 | |
| PYTHON_LIBNAME=boost_python38 | |
| . ${PREFIX}/profile | |
| cd $BUILDDIR | |
| # orig: https://github.com/inducer/pycuda/archive/v2020.1.tar.gz | |
| BEEHIVE=https://beehive.molgen.mpg.de/580562d7af4d3249c5b42a1df8d428a0/pycuda-2020.1.tar.gz | |
| test -e pycuda-${PYCUDAVERS}.tar.gz || wget -nv $BEEHIVE | |
| test -d pycuda-${PYCUDAVERS} || tar -xf pycuda-${PYCUDAVERS}.tar.gz | |
| cd pycuda-${PYCUDAVERS} | |
| test -e siteconf.py && rm siteconf.py | |
| python3 configure.py \ | |
| --cuda-enable-gl \ | |
| --cuda-root=$CUDADIR \ | |
| --no-use-shipped-boost \ | |
| --boost-python-libname=$PYTHON_LIBNAME \ | |
| --boost-inc-dir=$PREFIX/include \ | |
| --boost-lib-dir=$PREFIX/lib \ | |
| --cudadrv-lib-dir='${CUDA_ROOT}/lib,${CUDA_ROOT}/lib/stubs' \ | |
| --cudart-lib-dir='${CUDA_ROOT}/lib,${CUDA_ROOT}/lib/stubs' \ | |
| --curand-lib-dir='${CUDA_ROOT}/lib,${CUDA_ROOT}/lib/stubs' \ | |
| --ldflags=-L$PREFIX/lib | |
| python3 setup.py install --prefix $PREFIX | |
| ) | |
| ( | |
| # orig:https://github.com/nboley/idr/archive/2.0.3/idr-2.0.3.tar.gz # HD: Tobias Zehnder, 18.3.21 | |
| BEEHIVE=https://beehive.molgen.mpg.de/1bec61b526bb1dcf92a5ab99c27852d0/idr-2.0.3.tar.gz | |
| test -e idr-2.0.3.tar.gz || wget -nv $BEEHIVE | |
| test -d idr-2.0.3 || tar -xf idr-2.0.3.tar.gz | |
| cd idr-2.0.3 | |
| python3 setup.py build | |
| python3 setup.py install --prefix $PREFIX | |
| ) | |
| ( | |
| # orig:https://github.com/stacked-git/stgit/archive/v1.0/stgit-1.0.tar.gz # Paul mariux64/pkg-scripts/issues/152 | |
| BEEHIVE=https://beehive.molgen.mpg.de/b20ab5a197fc3a141f4b00c5859c0a3b/stgit-1.0.tar.gz | |
| test -e stgit-1.0.tar.gz || wget -nv $BEEHIVE | |
| test -d stgit-1.0 || tar -xf stgit-1.0.tar.gz | |
| cd stgit-1.0 | |
| python3 setup.py build | |
| python3 setup.py install --prefix $PREFIX | |
| ) | |
| # finaly tensorflaw | |
| ( | |
| TFVERSION=2.3.2 | |
| . ${PREFIX}/profile | |
| . /pkg/bazel-3.4.1-0/profile | |
| cd $BUILDDIR | |
| # orig: https://github.com/tensorflow/tensorflow/archive/v${TFVERSION}/tensorflow-${TFVERSION}.tar.gz | |
| BEEHIVE=https://beehive.molgen.mpg.de/b444d145beb5c8195c0476111747c8fa/tensorflow-2.3.2.tar.gz | |
| test -e tensorflow-${TFVERSION}.tar.gz || wget -nv $BEEHIVE | |
| test -d tensorflow-${TFVERSION} || tar -xf tensorflow-${TFVERSION}.tar.gz | |
| cd tensorflow-${TFVERSION} | |
| # use patch(es) from below ... | |
| sed -ne '/^#Epatch:f16/ s/^#Epatch:\S* // p' $ME | patch -p1 --verbose | |
| sed -ne '/^#Epatch:rtver/ s/^#Epatch:\S* // p' $ME | patch -p1 --verbose | |
| #vars for config | |
| export PYTHON_BIN_PATH=${PREFIX}/bin/python3 | |
| export USE_DEFAULT_PYTHON_LIB_PATH=1 | |
| export TF_NEED_JEMALLOC=1 | |
| export TF_NEED_KAFKA=0 | |
| export TF_NEED_OPENCL_SYCL=0 | |
| export TF_NEED_AWS=0 | |
| export TF_NEED_GCP=0 | |
| export TF_NEED_HDFS=0 | |
| export TF_NEED_S3=0 | |
| export TF_ENABLE_XLA=1 | |
| export TF_NEED_GDR=0 | |
| export TF_NEED_VERBS=0 | |
| export TF_NEED_OPENCL=0 | |
| export TF_NEED_MPI=0 | |
| export TF_NEED_TENSORRT=1 | |
| export TF_NEED_NGRAPH=0 | |
| export TF_NEED_IGNITE=0 | |
| export TF_NEED_ROCM=0 | |
| export TF_SET_ANDROID_WORKSPACE=0 | |
| export TF_DOWNLOAD_CLANG=0 | |
| export TF_NCCL_VERSION=2.7 | |
| export TF_IGNORE_MAX_BAZEL_VERSION=1 | |
| export NCCL_INSTALL_PATH=${CUDADIR} | |
| export GCC_HOST_COMPILER_PATH=/usr/bin/gcc | |
| export TF_CUDA_CLANG=0 | |
| export CLANG_CUDA_COMPILER_PATH=/usr/bin/clang | |
| export TF_CUDA_PATHS=${CUDADIR},${CUDADIR}/extras/CUPTI,${CUDADIR}/nvvm | |
| export TF_CUDA_VERSION=$(${CUDADIR}/bin/nvcc --version | sed -n 's/^.*release \(.*\),.*/\1/p') | |
| export TF_CUDNN_VERSION=$(sed -n 's/^#define CUDNN_MAJOR\s*\(.*\).*/\1/p' ${CUDADIR}/include/cudnn_version.h) | |
| export TF_CUDA_COMPUTE_CAPABILITIES=3.5,3.7,5.0,5.2,5.3,6.0,6.1,6.2,7.0,7.2,7.5 | |
| export CC_OPT_FLAGS="-march=nehalem" | |
| export TF_NEED_CUDA=1 | |
| ./configure | |
| bazel build \ | |
| --cxxopt="-march=nehalem" \ | |
| --copt="-march=nehalem" \ | |
| --noshow_loading_progress \ | |
| --noshow_progress \ | |
| --color="no" \ | |
| //tensorflow:libtensorflow.so \ | |
| //tensorflow:libtensorflow_cc.so \ | |
| //tensorflow:install_headers \ | |
| //tensorflow/tools/pip_package:build_pip_package | |
| bazel shutdown # otherwise it hogs in cwd | |
| # create a _clean/decent_ python package from so far created debris | |
| test -d tensorflow-pip && rm -r tensorflow-pip | |
| mkdir tensorflow-pip | |
| ./bazel-bin/tensorflow/tools/pip_package/build_pip_package --src ${BUILDDIR}/tensorflow-pip | |
| cd ${BUILDDIR} | |
| # ... cope with the ??? symlink-structure placed in bazels' build-TMPDIR (-h) (tell me why ?) | |
| test -e tensorflow-pip.tar && rm tensorflow-pip.tar | |
| tar -chf tensorflow-pip.tar tensorflow-pip | |
| # and perform a regular install | |
| test -d tensorflow-pip && rm -r tensorflow-pip | |
| tar -xf tensorflow-pip.tar | |
| cd ${BUILDDIR}/tensorflow-pip | |
| # inhibit downgrading of packages installed | |
| sed -ne '/^#Epatch:tfdep/ s/^#Epatch:\S* // p' $ME | patch -p2 --verbose | |
| python3 setup.py build | |
| python3 setup.py install --prefix $PREFIX | |
| ) | |
| # load all packages, see warnings and spot installation errors | |
| # But don't care about r2py, a further dependence into R is not wanted! | |
| echo "# START load test." | |
| python3 -c 'help("modules")' > /dev/null | |
| echo "# END load test." | |
| exit | |
| ### PATCHES | |
| # origin? Ooops, I've lost the ref. Found somewhere in TF-github issue-comments, and this code is actually present in TF 2.4 | |
| # note: numpy 1.19 or so borked its API, possibly a common deprecation mech is better to be ignored ... | |
| #Epatch:f16 --- a/tensorflow/python/lib/core/bfloat16.cc 2021-01-04 21:20:31.000000000 +0100 | |
| #Epatch:f16 +++ b/tensorflow/python/lib/core/bfloat16.cc 2021-02-22 17:52:49.012465507 +0100 | |
| #Epatch:f16 @@ -517,7 +517,7 @@ | |
| #Epatch:f16 } | |
| #Epatch:f16 | |
| #Epatch:f16 template <typename InType, typename OutType, typename Functor> | |
| #Epatch:f16 -void BinaryUFunc(char** args, npy_intp* dimensions, npy_intp* steps, | |
| #Epatch:f16 +void BinaryUFunc(char** args, const npy_intp* dimensions, const npy_intp* steps, | |
| #Epatch:f16 void* data) { | |
| #Epatch:f16 const char* i0 = args[0]; | |
| #Epatch:f16 const char* i1 = args[1]; | |
| #Epatch:f16 @@ -532,11 +532,17 @@ | |
| #Epatch:f16 } | |
| #Epatch:f16 } | |
| #Epatch:f16 | |
| #Epatch:f16 +// Numpy changed const-ness of PyUFuncGenericFunction, provide overload. | |
| #Epatch:f16 template <typename Functor> | |
| #Epatch:f16 void CompareUFunc(char** args, npy_intp* dimensions, npy_intp* steps, | |
| #Epatch:f16 void* data) { | |
| #Epatch:f16 BinaryUFunc<bfloat16, npy_bool, Functor>(args, dimensions, steps, data); | |
| #Epatch:f16 } | |
| #Epatch:f16 +template <typename Functor> | |
| #Epatch:f16 +void CompareUFunc(char** args, const npy_intp* dimensions, | |
| #Epatch:f16 + const npy_intp* steps, void* data) { | |
| #Epatch:f16 + BinaryUFunc<bfloat16, npy_bool, Functor>(args, dimensions, steps, data); | |
| #Epatch:f16 +} | |
| #Epatch:f16 | |
| #Epatch:f16 struct Bfloat16EqFunctor { | |
| #Epatch:f16 npy_bool operator()(bfloat16 a, bfloat16 b) { return a == b; } | |
| # https://github.com/tensorflow/tensorflow/issues/43588 | |
| # conclusio: nvidia disted a 'strange' SOVERSION in libcudart, some distro 'fixed' it ... | |
| # ... based on the assumtion that libcudart must be 11.1 | |
| # tensorflaw buddies took the distro -- well, know your code | |
| # n.b. this is a hack for linux only. | |
| #Epatch:rtver --- a/third_party/gpus/cuda_configure.bzl 2021-01-04 21:20:31.000000000 +0100 | |
| #Epatch:rtver +++ b/third_party/gpus/cuda_configure.bzl 2021-02-23 21:16:21.699194719 +0100 | |
| #Epatch:rtver @@ -533,14 +533,14 @@ | |
| #Epatch:rtver "cudart", | |
| #Epatch:rtver cpu_value, | |
| #Epatch:rtver cuda_config.config["cuda_library_dir"], | |
| #Epatch:rtver - cuda_config.cuda_version, | |
| #Epatch:rtver + "11.0", | |
| #Epatch:rtver static = False, | |
| #Epatch:rtver ), | |
| #Epatch:rtver "cudart_static": _check_cuda_lib_params( | |
| #Epatch:rtver "cudart_static", | |
| #Epatch:rtver cpu_value, | |
| #Epatch:rtver cuda_config.config["cuda_library_dir"], | |
| #Epatch:rtver - cuda_config.cuda_version, | |
| #Epatch:rtver + "11.0", | |
| #Epatch:rtver static = True, | |
| #Epatch:rtver ), | |
| #Epatch:rtver "cublas": _check_cuda_lib_params( | |
| #Epatch:rtver --- a/third_party/gpus/cuda/cuda_config.h.tpl 2021-02-25 16:09:26.393892637 +0100 | |
| #Epatch:rtver +++ b/third_party/gpus/cuda/cuda_config.h.tpl 2021-02-25 16:10:29.387584893 +0100 | |
| #Epatch:rtver @@ -17,6 +17,7 @@ | |
| #Epatch:rtver #define CUDA_CUDA_CONFIG_H_ | |
| #Epatch:rtver | |
| #Epatch:rtver #define TF_CUDA_VERSION "%{cuda_version}" | |
| #Epatch:rtver +#define TF_CUDART_VERSION "11.0" | |
| #Epatch:rtver #define TF_CUBLAS_VERSION "%{cublas_version}" | |
| #Epatch:rtver #define TF_CUSOLVER_VERSION "%{cusolver_version}" | |
| #Epatch:rtver #define TF_CURAND_VERSION "%{curand_version}" | |
| #Epatch:rtver --- a/tensorflow/stream_executor/platform/default/dso_loader.cc 2021-01-04 21:20:31.000000000 +0100 | |
| #Epatch:rtver +++ b/tensorflow/stream_executor/platform/default/dso_loader.cc 2021-02-25 16:28:43.867234896 +0100 | |
| #Epatch:rtver @@ -30,7 +30,8 @@ | |
| #Epatch:rtver namespace internal { | |
| #Epatch:rtver | |
| #Epatch:rtver namespace { | |
| #Epatch:rtver -string GetCudaVersion() { return TF_CUDA_VERSION; } | |
| #Epatch:rtver +string GetCudaVersion() { return TF_CUDA_VERSION; } // just libcupti.so | |
| #Epatch:rtver +string GetCudartVersion() { return TF_CUDART_VERSION; } // value is hardcoded in cuda_config.h.tpl | |
| #Epatch:rtver string GetCudnnVersion() { return TF_CUDNN_VERSION; } | |
| #Epatch:rtver string GetCublasVersion() { return TF_CUBLAS_VERSION; } | |
| #Epatch:rtver string GetCusolverVersion() { return TF_CUSOLVER_VERSION; } | |
| #Epatch:rtver @@ -77,7 +78,7 @@ | |
| #Epatch:rtver } | |
| #Epatch:rtver | |
| #Epatch:rtver port::StatusOr<void*> GetCudaRuntimeDsoHandle() { | |
| #Epatch:rtver - return GetDsoHandle("cudart", GetCudaVersion()); | |
| #Epatch:rtver + return GetDsoHandle("cudart", GetCudartVersion()); | |
| #Epatch:rtver } | |
| #Epatch:rtver | |
| #Epatch:rtver port::StatusOr<void*> GetCublasDsoHandle() { | |
| # Tensorflow-pip | |
| # don't downgrade numpy, since the ABI-issue is fixed | |
| # don't downgrade h5py, take the risk of manual fixing :) | |
| # skip stuff present already | |
| #Epatch:tfdep --- a/tensorflow-pip/setup.py 2021-03-05 18:49:44.000000000 +0100 | |
| #Epatch:tfdep +++ b/tensorflow-pip/setup.py 2021-03-18 12:03:02.536396036 +0100 | |
| #Epatch:tfdep @@ -56,19 +56,12 @@ | |
| #Epatch:tfdep 'astunparse == 1.6.3', | |
| #Epatch:tfdep 'gast == 0.3.3', | |
| #Epatch:tfdep 'google_pasta >= 0.1.8', | |
| #Epatch:tfdep - 'h5py >= 2.10.0, < 2.11.0', | |
| #Epatch:tfdep - 'keras_preprocessing >= 1.1.1, < 1.2', | |
| #Epatch:tfdep - # TODO(mihaimaruseac): numpy 1.19.0 has ABI breakage | |
| #Epatch:tfdep - # https://github.com/numpy/numpy/pull/15355 | |
| #Epatch:tfdep - 'numpy >= 1.16.0, < 1.19.0', | |
| #Epatch:tfdep 'opt_einsum >= 2.3.2', | |
| #Epatch:tfdep 'protobuf >= 3.9.2', | |
| #Epatch:tfdep 'tensorboard >= 2.3.0, < 3', | |
| #Epatch:tfdep 'tensorflow_estimator >= 2.3.0, < 2.4.0', | |
| #Epatch:tfdep 'termcolor >= 1.1.0', | |
| #Epatch:tfdep 'wrapt >= 1.11.1', | |
| #Epatch:tfdep - 'wheel >= 0.26', | |
| #Epatch:tfdep - 'six >= 1.12.0', | |
| #Epatch:tfdep ] | |
| #Epatch:tfdep | |
| #Epatch:tfdep if sys.byteorder == 'little': | |