From a7b6a97bee5136fce8395c230f946c7c727613a3 Mon Sep 17 00:00:00 2001 From: thomas Date: Thu, 20 Oct 2022 19:21:11 +0200 Subject: [PATCH] python: add version 3.10.8 Compared to python-3.10.7-0 only a few things changed. - pytorch/cuda-11.6, thus supporting Ampere-GPUs - numba installation is fixed, GPUs are usable now - less offensive content in the build script Version changes: - meson: 0.63.3 - snakemake: 7.16.1 --- python-3.10.8-0.build.sh | 738 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 738 insertions(+) create mode 100755 python-3.10.8-0.build.sh diff --git a/python-3.10.8-0.build.sh b/python-3.10.8-0.build.sh new file mode 100755 index 0000000..7343426 --- /dev/null +++ b/python-3.10.8-0.build.sh @@ -0,0 +1,738 @@ +#! /bin/bash + +# ------------------------------------------------------------------- Debug + +# COOKIE=$(mcookie|cut -c-8); grep -v V_GREP_ME $0 > /dev/shm/runme-$COOKIE.sh ; sleep 0.3; exec bash /dev/shm/runme-$COOKIE.sh +# TESTING=1 +# SKIPTENSORFLOW=1 + +# ---------------------------------------------------------------- Preamble + +PKG=python +VERSION=3.10.8 +BUILD=0 +PYTHONVER=${VERSION%.*} + +CUDADIR=/pkg/cuda-11.5.2-0 + +PREFIX=/pkg/$PKG-$VERSION-$BUILD +if [ -n "$TESTING" ]; then PREFIX=/scratch/local2/$PKG-$VERSION-$BUILD ; fi + +PATH=/bin:/usr/bin:/usr/sbin:/usr/local/bin + +# set -xe +set -e +umask 022 + +BUILD_TMPDIR=/scratch/local2/$PKG-$VERSION-$BUILD.$USER.build.tmp + +test -d $BUILD_TMPDIR && ( chmod -R u+rwx $BUILD_TMPDIR || true ; rm -rf $BUILD_TMPDIR ) +mkdir -p $BUILD_TMPDIR/home/.cache/pip +mkdir -p $BUILD_TMPDIR/home/.local +# copy script to an accessible location +cp $0 $BUILD_TMPDIR; ME="$BUILD_TMPDIR/$(basename $0)" +export TMPDIR=$BUILD_TMPDIR +export HOME=$BUILD_TMPDIR/home + +exec $PREFIX/profile <<-EOF + . $CUDADIR/profile + PATH=$PREFIX/bin:\$PATH + export LD_LIBRARY_PATH=$PREFIX/lib\${LD_LIBRARY_PATH:+:}\$LD_LIBRARY_PATH + PKG_CONFIG_PATH=$PREFIX/lib/pkgconfig\${PKG_CONFIG_PATH:+:}\$PKG_CONFIG_PATH + export PKG_CONFIG_PATH + if [ -d $PREFIX/.compatlibs ]; then export LD_LIBRARY_PATH=$PREFIX/.compatlibs\${LD_LIBRARY_PATH:+:}\$LD_LIBRARY_PATH ; fi +EOF +. $PREFIX/profile + +NPROC=$(( $(nproc) * 4 / 5 + 1 )) +export MAKEFLAGS="-j $NPROC" + +BUILDDIR=$PREFIX/build + +mkdir -p $BUILDDIR +cd $BUILDDIR + +# -------------------------------------- Use git to track package evolution + +test -d .git && rm -rf .git +git init -q; echo -e '[user]\n name = none\n email = of_your_business...' >> .git/config +echo '*' > .gitignore + +function piplist() { + pip3 list | awk '{ printf("%-36s %s\n", $1,$2) }' | grep -v '\----' > GameOfVersions +} + +function track() { + if [ -d .git ]; then + if git add -f GameOfVersions; then + git commit -q -n -m "Package: '$*'" || /bin/true + fi + fi +} + +function install() { + # no-color is nice, but doesn't work when dependencies are installed, so (mis-)use a pipe + pip3 install --no-color --compile --cache-dir=$HOME/.cache/pip --prefix=$PREFIX "${@+$@}" | cat + piplist + track $* +} + +# ---------------------------------------- Build Python & support libraries + +( + # orig: https://www.python.org/ftp/python/3.10.8/Python-3.10.8.tgz + BEEHIVE=https://beehive.molgen.mpg.de/fbe3fff11893916ad1756b15c8a48834/Python-3.10.8.tgz + test -e Python-$VERSION.tgz || wget -nv $BEEHIVE + test -d Python-$VERSION || tar -xf Python-$VERSION.tgz + cd Python-$VERSION + + # BerkeleyDB, the undead ... + # Newer libgdbm_compat.so versions pull libgdbm.so on their own, our libgdbm_compat.so is older ... + # So, let setup.py create '-lgdbm -lgdbm_compat' instead of a sole '-lgdbm_compat' + # verbatim: ndbm_libs = ['gdbm_compat'] => ndbm_libs = ['gdbm','gdbm_compat'] + sed -i -e "/ndbm_libs = / s/_compat'/','gdbm_compat'/" setup.py + + # leave further hints in the log + sed -i -e '/db_setup_debug = False/ s/False/True/' \ + -e '/dbm_setup_debug = False/ s/False/True/' setup.py + + # These may come handy if the Python build needs exclusive stuff from $PREFIX + # INCLUDEDIR=$PREFIX/include \ + # LIBDIR=$PREFIX/lib \ + LDFLAGS="-Wl,-rpath=$PREFIX/lib" \ + ./configure \ + --prefix=$PREFIX \ + --enable-shared + + make -j $NPROC + make install + + # it might come to happen that something down the line may just call 'python', defuse ... + test -e $PREFIX/bin/python || ln -s python3 $PREFIX/bin/python + + python3 -m ensurepip + pip3 install --prefix=$PREFIX -I pip + ( # fix 'please update' noise from pip + cd $PREFIX/lib/python*/site-packages + sed -ne '/^#Epatch:pip/ s/^#Epatch:\S* // p' $ME | patch -p1 --verbose + ) +) +install setuptools==65.3.0 # see numba below +piplist; track START + + +# build various pythonbindings 'by hand', because pip installs do not deliver +# header files and the like. +# BTW: builds need meson + +install meson + +( + DBUS_PYTHON_VERSION=1.2.18 + + # https://files.pythonhosted.org/packages/b1/5c/ccfc167485806c1936f7d3ba97db6c448d0089c5746ba105b6eb22dba60e/dbus-python-1.2.18.tar.gz + BEEHIVE=https://beehive.molgen.mpg.de/468874f3bf61d1b7deac98bbe2cfd8ac/dbus-python-1.2.18.tar.gz + test -e dbus-python-$DBUS_PYTHON_VERSION.tar.gz || wget -nv $BEEHIVE + test -d dbus-python-$DBUS_PYTHON_VERSION || tar -xf dbus-python-$DBUS_PYTHON_VERSION.tar.gz + + cd dbus-python-$DBUS_PYTHON_VERSION + ./configure --prefix=$PREFIX + + make -j $NPROC + make install + +) + +( + PYCAIRO_VERSION=1.21.0 + + # https://files.pythonhosted.org/packages/92/a4/506564f574fa74c90b98690e8ecc8dbae1629f31fcfe0be69de45d9e1605/pycairo-1.21.0.tar.gz + BEEHIVE=https://beehive.molgen.mpg.de/682238eaa8bd7032e36a20b6f53824d1/pycairo-1.21.0.tar.gz + test -e pycairo-$PYCAIRO_VERSION.tar.gz || wget -nv $BEEHIVE + test -d pycairo-$PYCAIRO_VERSION || tar -xf pycairo-$PYCAIRO_VERSION.tar.gz + + cd pycairo-$PYCAIRO_VERSION + + mkdir -p meson-build + cd meson-build + + meson setup \ + --prefix=$PREFIX \ + --buildtype=release \ + .. + + ninja -j $NPROC + ninja install + +) + +( + PYGOBJECT3_VERSION=3.42.2 + + # https://files.pythonhosted.org/packages/fe/40/9afaeb8d3b453fb8596fcb6c7bc2b64e434868c91eda19955742778eff74/PyGObject-3.42.2.tar.gz + BEEHIVE=https://beehive.molgen.mpg.de/80ad843b2a49ccfcc30c6b756b755c09/PyGObject-3.42.2.tar.gz + test -e PyGObject-$PYGOBJECT3_VERSION.tar.gz || wget -nv $BEEHIVE + test -d PyGObject-$PYGOBJECT3_VERSION || tar -xf PyGObject-$PYGOBJECT3_VERSION.tar.gz + + cd PyGObject-$PYGOBJECT3_VERSION + + mkdir -p meson-build + cd meson-build + meson setup \ + --prefix=$PREFIX \ + --buildtype=release \ + -Dtests=false \ + .. + + ninja -j $NPROC + ninja install + +) +piplist; track various pythonbindings + +# build support to use QT from python -- see www.riverbankcomputing.com + +( # sip: tool to create Python bindings for C and C++ libraries. + SIP_VERSION=6.6.2 + + # https://files.pythonhosted.org/packages/5b/cb/c27c925ae07bd03a2597fa1db17bfc2a4ac57da61aeb90f8ec98ffbb975b/sip-6.6.2.tar.gz + BEEHIVE=https://beehive.molgen.mpg.de/620eb75a8b22fb6af9c6044aa015782d/sip-6.6.2.tar.gz + test -e sip-$SIP_VERSION.tar.gz || wget -nv $BEEHIVE + test -d sip-$SIP_VERSION || tar -xf sip-$SIP_VERSION.tar.gz + + cd sip-$SIP_VERSION + + sed -ne '/^#Epatch:sip1/ s/^#Epatch:\S* // p' $ME | patch -p1 --verbose + sed -ne '/^#Epatch:sip2/ s/^#Epatch:\S* // p' $ME | patch -p1 --verbose + + python3 setup.py build + python3 setup.py install --skip-build --optimize=1 + +) + +# QT bindings need some python modules to build + +install PyQt5-sip +install PyQt-builder + +( + PYQT_VERSION=5.15.7 + + # orig: https://files.pythonhosted.org/packages/e1/57/2023316578646e1adab903caab714708422f83a57f97eb34a5d13510f4e1/PyQt5-5.15.7.tar.gz + BEEHIVE=https://beehive.molgen.mpg.de/ae2c68e38b9b36fdf5f932419353a2b3/PyQt5-5.15.7.tar.gz + test -e PyQt5-$PYQT_VERSION.tar.gz || wget -nv $BEEHIVE + test -d PyQt5-$PYQT_VERSION || tar -xf PyQt5-$PYQT_VERSION.tar.gz + + cd PyQt5-$PYQT_VERSION + + sip-build \ + --confirm-license \ + --no-make \ + --api-dir $PREFIX/share/qsci/api/python \ + --dbus $PREFIX/include/dbus-1.0 \ + --no-designer-plugin \ + --no-qml-plugin \ + --jobs $NPROC \ + + cd build + make -j $NPROC + make install + + # Remove unused py2 version of uic modules + rm -r $PREFIX/lib/python*/site-packages/PyQt5/uic/port_v2 + +) + +( + QSCINTILLA_VERSION=2.13.4.dev2206171421 # no 2.14 in sight ... + + # orig: https://www.riverbankcomputing.com/static/Downloads/QScintilla/${PKGVERSION}/QScintilla_src-${PKGVERSION}.tar.gz + BEEHIVE=https://beehive.molgen.mpg.de/d196ebd441f40474dd7732d4d492de69/QScintilla_src-2.13.4.dev2206171421.tar.gz + test -e QScintilla_src-$QSCINTILLA_VERSION.tar.gz || wget -nv $BEEHIVE + test -d QScintilla_src-$QSCINTILLA_VERSION || tar -xf QScintilla_src-$QSCINTILLA_VERSION.tar.gz + + cd QScintilla_src-$QSCINTILLA_VERSION + cd Python + + cp -vp pyproject-qt5.toml pyproject.toml + sip-build \ + --no-make \ + --qsci-features-dir ../src/features \ + --qsci-include-dir ../src \ + --qsci-library-dir ../src \ + --api-dir $PREFIX/share/qsci/api/python \ + --jobs $NPROC + + cd build + make -j $NPROC + make install + +) +piplist; track qt-support-riverbankcomputing + +# -------------------------------------------------------- Install packages + +# Install the psychotic diva first +install numpy==1.23.4 + +install Cython + +# libxml2-python from pypi isn't up to date, cause lxml seems to be the favored binding +# do this early, not that some dependency tries it on its own. +# (btw. libxslt-python isn't ported to python3) +( + LIBXML2_VERSION=2.9.12 + + # orig: http://xmlsoft.org/sources/libxml2-${PKGVERSION}.tar.gz + BEEHIVE=https://beehive.molgen.mpg.de/f433a39be087a9f0b197eb2307ad9f75/libxml2-2.9.12.tar.gz + test -e libxml2-$LIBXML2_VERSION.tar.gz || wget -nv $BEEHIVE + test -d libxml2-$LIBXML2_VERSION || tar -xf libxml2-$LIBXML2_VERSION.tar.gz libxml2-$LIBXML2_VERSION/python + cd libxml2-$LIBXML2_VERSION/python + python3 setup.py build + python3 setup.py install +) +piplist; track legacy-libxml2 + + +# 'numba' the cuda enabled numpy +( + NUMBA_VERSION=0.56.3 + + # orig: https://files.pythonhosted.org/packages/a3/95/7df08647e773a2417e24de8fea9087606b37074f63830a3f2dbabd97f01b/numba-0.56.3.tar.gz + BEEHIVE=https://beehive.molgen.mpg.de/ae0a008447f2f246523af143b1488e88/numba-0.56.3.tar.gz + test -e numba-$NUMBA_VERSION.tar.gz || wget -nv $BEEHIVE + test -d numba-$NUMBA_VERSION || tar -xf numba-$NUMBA_VERSION.tar.gz + cd numba-$NUMBA_VERSION + + # numba assumes that libcudart.so and friends reside in 'CUDA_HOME/lib64' + sed -ne '/^#Epatch:numba/ s/^#Epatch:\S* // p' $ME | patch -p1 --verbose + install . + # useful to see if numba works + cp -p numba/misc/numba_sysinfo.py $PREFIX/bin + chmod 0755 $PREFIX/bin/numba_sysinfo.py +) +piplist; track numba + + +# 'distributed' doesn't like the 'new' tornado 6.2 -- reason: a deprecation warning makes the CI choke (geht's noch?) +( + DISTRIBUTED_VERSION=2022.9.0 + + # orig: https://files.pythonhosted.org/packages/df/46/e8ef41f8dfd553327b6a12f4ab78d9351b54299596fe5b8b165ba40b1550/distributed-2022.9.0.tar.gz + BEEHIVE=https://beehive.molgen.mpg.de/335d718f1ba331ee8e85e40e564559a1/distributed-2022.9.0.tar.gz + test -e distributed-$DISTRIBUTED_VERSION.tar.gz || wget -nv $BEEHIVE + test -d distributed-$DISTRIBUTED_VERSION || tar -xf distributed-$DISTRIBUTED_VERSION.tar.gz + cd distributed-$DISTRIBUTED_VERSION + sed -i -e '/tornado/ s/.*/tornado/' requirements.txt + sed -i -e '/tornado/ s/.*/tornado/' distributed.egg-info/requires.txt + install . +) +piplist; track distributed and tornado + +PACKAGES=$(sed -e s/#.*$// <<- __PKGLIST__ + protobuf==3.19.4 # tensorflow likes to have an slightly older one [< 3.20], no conflicts with this + Matplotlib==3.6.0 # 3.6.1 is the first bugfix release of the 3.6.x series... Well, it is buggy + ipython[all] + + SciPy + pandas + SymPy + nose + statsmodels + pysam + seaborn + scikit-learn + Mako + cutadapt # Helpdesk 01/24/18 16:19, Sabrina Krakau + pygobject # Helpdesk 03/26/18 14:19, Virginie Stanisla + blink1 # Blink1-lib 07/04/18 12:00, Niclas Hofmann + umi-tools + sphinx + recommonmark + fastcluster + pydot + pygraphviz + deeptools + HTSeq + keras + keras_preprocessing # tensorflow ... + MotifScan # Alena 30.8.2018 + MAmotif + hic2cool # Helpdesk Robert Schoepflin 17.09.2018 + snakemake + virtualenv # Donald + odfpy # Donald + Dumper # Thomas + igraph + louvain # monocle3 prep for Helene + selenium # 19.7.19 Kreitler -> https://webxray.org/ + gmpy # From viper context, Helpdesk Laura Glaser 9.8.2019 + plotly + pyOpenCL + docopt + distro + ply + threadpoolctl + HiCMatrix + pybedtools + gffutils + python-bioformats + jupyterlab + jupyterlab-git # request Peter Arndt + igv_jupyterlab # Matthias Lienhard HD 01-04-2020 + igv-jupyter + scour # https://github.molgen.mpg.de/mariux64/pkg-scripts/issues/98 + multiqc + pybind11 + svgwrite + pyomo + bioservices + pairtools + SharedArray + pyabc # dependency for stochkit, HD 20.07.20 Gemma Noviello + petab + tadtool # HD 26.01.21 Michael Robson + pyperf # Paul sep-20 + umap # HD, used by Matthias Lienhard + MACS3 + biopython # HD 2021-07-23 Magalhaes/Hnisz alphafold related -- cool that we have missed biopython so far, LOL + dm-tree + dm-haiku + flatbuffers + contextlib2 + ml_collections + + # additions/fixups since 3.9 + papermill # turns jupyter notebooks into scripts [sic!] used by P.A. [who else, *grin*] + opencv-python + scikit-image + git-cola + cffi-glpk # uses /usr/lib/libglpk.so.40, see if I can get it linked statically - kreitler + czifile + napari[pyqt5] + napari-czifile2 + CrossMap + cooltools # cause trouble with numba/nupy versions -- Maria Valieva 17.7.2019 + scanpy + + __PKGLIST__ +) + +for PKG in $PACKAGES; do + install $PKG +done + +install pyopengl # Tk binding creates an error, so remove it (Expects Tk Togl installation...) +rm -rf ${PREFIX}/lib/python$PYTHONVER/site-packages/OpenGL/Tk + +# torch bundled with a recent CUDA-Toolkit +install torch torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/cu116 + +# install helen after our torch, otherwise we might catch the default torch from pip ... +install helen # HD, a dependency from a request by helene + +mkdir -vp $PREFIX/share/jupyter/lab/settings # try to defeat a silly error in jupyter labextension install + +# --------------------------------------------------- Install misc packages + +( + BOOST_VERSION=(1.80.0 1_80_0) + PKGCFLAGS="-O2 -fPIC" + + # orig: https://dl.bintray.com/boostorg/release/1.75.0/source/boost_1_75_0.tar.gz + BEEHIVE=https://beehive.molgen.mpg.de//38813f6feb40387dfe90160debd71251/boost_1_75_0.tar.gz + # orig: https://boostorg.jfrog.io/artifactory/main/release/1.80.0/source/boost_1_80_0.tar.gz + BEEHIVE=https://beehive.molgen.mpg.de/077f074743ea7b0cb49c6ed43953ae95/boost_1_80_0.tar.gz + test -e boost_${BOOST_VERSION[1]}.tar.gz || wget -nv $BEEHIVE + test -d boost-${BOOST_VERSION[0]} || mkdir boost-${BOOST_VERSION[0]} && tar -xf boost_${BOOST_VERSION[1]}.tar.gz -C boost-${BOOST_VERSION[0]} --strip-components=1 + + cd boost-${BOOST_VERSION[0]} + sed -e "/using python/ s|;|: ${PREFIX}/include/python${PYTHONVER} ;|" \ + -i bootstrap.sh + + ./bootstrap.sh \ + --with-toolset=gcc \ + --with-icu \ + --with-python=$PREFIX/bin/python3 \ + --with-libraries=python,thread || exit 1 + + ./b2 \ + -j $NPROC \ + --layout=system \ + --build-dir=build-python3 \ + --prefix=$PREFIX \ + --libdir=$PREFIX/lib \ + --build-type=minimal \ + toolset=gcc \ + variant=release \ + debug-symbols=off \ + link=shared \ + threading=multi \ + runtime-link=shared \ + python=$PYTHONVER \ + cflags="$PKGCFLAGS" \ + cxxflags="$PKGCFLAGS" \ + install || exit 1 +) + +( + PYCUDAVERS=2022.1 + PYTHON_LIBNAME=boost_python${PYTHONVER/./} + + # orig: https://github.com/inducer/pycuda/archive/v2020.1.tar.gz + # BEEHIVE=https://beehive.molgen.mpg.de/580562d7af4d3249c5b42a1df8d428a0/pycuda-2020.1.tar.gz + # orig: https://files.pythonhosted.org/packages/2d/1f/48a3a5b2c715345e7af1e09361100bd98c3d72b4025371692ab233f523d3/pycuda-2022.1.tar.gz # maybe those very long URLs compensate for other (shorter) things ... + BEEHIVE="https://beehive.molgen.mpg.de/2fad5a15db4153d825822eba7cb25f25/pycuda-2022.1.tar.gz" + test -e pycuda-${PYCUDAVERS}.tar.gz || wget -nv $BEEHIVE + test -d pycuda-${PYCUDAVERS} || tar -xf pycuda-${PYCUDAVERS}.tar.gz + + cd pycuda-${PYCUDAVERS} + test -e siteconf.py && rm siteconf.py + python3 configure.py \ + --cuda-enable-gl \ + --cuda-root=$CUDADIR \ + --no-use-shipped-boost \ + --boost-python-libname=$PYTHON_LIBNAME \ + --boost-inc-dir=$PREFIX/include \ + --boost-lib-dir=$PREFIX/lib \ + --cudadrv-lib-dir='${CUDA_ROOT}/lib,${CUDA_ROOT}/lib/stubs' \ + --cudart-lib-dir='${CUDA_ROOT}/lib,${CUDA_ROOT}/lib/stubs' \ + --curand-lib-dir='${CUDA_ROOT}/lib,${CUDA_ROOT}/lib/stubs' \ + --ldflags=-L$PREFIX/lib + + python3 setup.py build + python3 setup.py install --prefix $PREFIX +) + +( + # orig: https://github.com/nboley/idr/archive/2.0.3/idr-2.0.3.tar.gz # HD: Tobias Zehnder, 18.3.21 + BEEHIVE=https://beehive.molgen.mpg.de/1bec61b526bb1dcf92a5ab99c27852d0/idr-2.0.3.tar.gz + test -e idr-2.0.3.tar.gz || wget -nv $BEEHIVE + test -d idr-2.0.3 || tar -xf idr-2.0.3.tar.gz + cd idr-2.0.3 + python3 setup.py build + python3 setup.py install --prefix $PREFIX +) + +( + # orig: https://github.com/stacked-git/stgit/archive/v1.0/stgit-1.0.tar.gz # Paul mariux64/pkg-scripts/issues/152 + BEEHIVE=https://beehive.molgen.mpg.de/b20ab5a197fc3a141f4b00c5859c0a3b/stgit-1.0.tar.gz + test -e stgit-1.0.tar.gz || wget -nv $BEEHIVE + test -d stgit-1.0 || tar -xf stgit-1.0.tar.gz + cd stgit-1.0 + python3 setup.py build + python3 setup.py install --prefix $PREFIX +) + +( + # https://github.com/brentp/bwa-meth/archive/refs/tags/v0.2.5.tar.gz + BEEHIVE=https://beehive.molgen.mpg.de/33a4c416760dd67e77f17e07791527e2/bwa-meth-0.2.5.tar.gz + test -e bwa-meth-0.2.5.tar.gz || wget -nv $BEEHIVE + test -d bwa-meth-0.2.5 || tar -xf bwa-meth-0.2.5.tar.gz + cd bwa-meth-0.2.5 + python3 setup.py build + python3 setup.py install --prefix $PREFIX +) +piplist; track misc packages + +# ------------------------------------------------ finally build tensorflaw + +install wheel # to build our tf-wheel (i.e. zip archive) + +( + test -n "$SKIPTENSORFLOW" && exit + + TFVERSION=2.10.0 + NCCLSHORTVERSION=$(pkg-config nccl --modversion | grep -Po '\d+\.\d+') + + . /pkg/bazel-5.1.0-0/profile + + # orig: https://github.com/tensorflow/tensorflow/archive/refs/tags/v2.10.0.tar.gz + BEEHIVE=https://beehive.molgen.mpg.de/7ceb5277d482f10d2df8644deb412061/tensorflow-2.10.0.tar.gz + + test -e tensorflow-${TFVERSION}.tar.gz || wget -nv $BEEHIVE + test -d tensorflow-${TFVERSION} || tar -xf tensorflow-${TFVERSION}.tar.gz + + cd tensorflow-${TFVERSION} + + #vars for config + export PYTHON_BIN_PATH=${PREFIX}/bin/python3 + export USE_DEFAULT_PYTHON_LIB_PATH=1 + export TF_NEED_JEMALLOC=1 + export TF_NEED_KAFKA=0 + export TF_NEED_OPENCL_SYCL=0 + export TF_NEED_AWS=0 + export TF_NEED_GCP=0 + export TF_NEED_HDFS=0 + export TF_NEED_S3=0 + export TF_ENABLE_XLA=1 + export TF_NEED_GDR=0 + export TF_NEED_VERBS=0 + export TF_NEED_OPENCL=0 + export TF_NEED_MPI=0 + export TF_NEED_TENSORRT=0 + export TF_NEED_NGRAPH=0 + export TF_NEED_IGNITE=0 + export TF_NEED_ROCM=0 + export TF_SET_ANDROID_WORKSPACE=0 + export TF_DOWNLOAD_CLANG=0 + export TF_NCCL_VERSION=${NCCLSHORTVERSION} + export TF_IGNORE_MAX_BAZEL_VERSION=1 + export NCCL_INSTALL_PATH=${CUDADIR} + export CC=gcc + export CXX=g++ + export GCC_HOST_COMPILER_PATH=/usr/bin/${CC} + export HOST_C_COMPILER=/usr/bin/${CC} + export HOST_CXX_COMPILER=/usr/bin/${CXX} + export TF_CUDA_CLANG=0 + export CLANG_CUDA_COMPILER_PATH=/usr/bin/clang + export TF_CUDA_PATHS=${CUDADIR},${CUDADIR}/extras/CUPTI,${CUDADIR}/nvvm + export TF_CUDA_VERSION=$(${CUDADIR}/bin/nvcc --version | sed -n 's/^.*release \(.*\),.*/\1/p') # 11.5 + export TF_CUDNN_VERSION=$(sed -n 's/^#define CUDNN_MAJOR\s*\(.*\).*/\1/p' ${CUDADIR}/include/cudnn_version.h) + # hardware older than sm_52 is said to be unsupported, compute_86 denotes an emulation + export TF_CUDA_COMPUTE_CAPABILITIES=sm_52,sm_53,sm_60,sm_61,sm_62,sm_70,sm_72,sm_75,sm_80,sm_86,compute_86 + + # stay compatible with the cluster machines + export CC_OPT_FLAGS="-march=nehalem" + export TF_NEED_CUDA=1 + export BAZEL_ARGS="--config=mkl -c opt" + + ./configure + + bazel build \ + --cxxopt="-march=nehalem" \ + --copt="-march=nehalem" \ + --noshow_loading_progress \ + --noshow_progress \ + --color="no" \ + //tensorflow:libtensorflow.so \ + //tensorflow:libtensorflow_cc.so \ + //tensorflow:install_headers \ + //tensorflow/tools/pip_package:build_pip_package + + bazel shutdown # otherwise it hogs in cwd + + # create a _clean/decent_ python package from so far created debris + test -d tensorflow-pip && rm -r tensorflow-pip + mkdir tensorflow-pip + ./bazel-bin/tensorflow/tools/pip_package/build_pip_package --src ${BUILDDIR}/tensorflow-pip + cd ${BUILDDIR} + # ... cope with the ??? symlink-structure placed in bazels' build-TMPDIR (-h) (tell me why ?) + test -e tensorflow-pip.tar && rm tensorflow-pip.tar + tar -chf tensorflow-pip.tar tensorflow-pip + + # and perform a regular install + test -d tensorflow-pip && rm -r tensorflow-pip + tar -xf tensorflow-pip.tar + cd ${BUILDDIR}/tensorflow-pip + + python3 setup.py build + python3 setup.py install --prefix $PREFIX +) +piplist; track tensorflow + +# tidy up a bit +if [ -d $PREFIX/man/man1 ]; then + mv -v $PREFIX/man/man1/* $PREFIX/share/man/man1 || true + rm -rf $PREFIX/man +fi + +# ------------------------------------------------------------ Sanity check + +# load all packages, see warnings and spot installation errors +# But don't care about r2py, a further dependence into R is not wanted! +echo "# START load test." +python3 -c 'help("modules")' > /dev/null +echo "# END load test (passed)." + +echo "# running pip check" +pip3 check + +# I guess it makes sense that files can be read by all users ... +echo "# check for unreadable files" +cd $PREFIX +find \! -perm -004 -exec chmod -c a+r {} + + +exit + +# -------------------------------------------------------- Included patches + +#Epatch:pip # Operation 'Forever young', disable version-check per default. +#Epatch:pip # The option needs to be kept, because some tools make use of +#Epatch:pip # it and would cause pip to fail. +#Epatch:pip --- a/pip/_internal/cli/cmdoptions.py 2022-08-09 17:29:15.853442948 +0200 +#Epatch:pip +++ b/pip/_internal/cli/cmdoptions.py 2022-08-10 10:31:35.124945154 +0200 +#Epatch:pip @@ -892,7 +892,6 @@ +#Epatch:pip dest="disable_pip_version_check", +#Epatch:pip action="store_true", +#Epatch:pip - default=False, +#Epatch:pip - help="Don't periodically check PyPI to determine whether a new version " +#Epatch:pip - "of pip is available for download. Implied with --no-index.", +#Epatch:pip + default=True, +#Epatch:pip + help="Ignore this option, the version check IS disabled.", +#Epatch:pip ) + +# https://riverbankcomputing.com/hg/sip/raw-rev/1430b279a3c9 -> sip-fix-exceptions.patch +#Epatch:sip1 Fixed the handling of exceptions that sub-class C++ exceptions. +#Epatch:sip1 diff -r 6eff81a15822 -r 1430b279a3c9 sipbuild/generator/parser/rules.py +#Epatch:sip1 --- a/sipbuild/generator/parser/rules.py Fri Jun 17 14:25:16 2022 +0100 +#Epatch:sip1 +++ b/sipbuild/generator/parser/rules.py Sun Jun 26 10:01:41 2022 +0100 +#Epatch:sip1 @@ -2293,6 +2293,7 @@ +#Epatch:sip1 +#Epatch:sip1 if len(p) == 4: +#Epatch:sip1 base = p[2] +#Epatch:sip1 + base.make_absolute() +#Epatch:sip1 +#Epatch:sip1 # See if it is a project-defined exception. +#Epatch:sip1 for xd in p.parser.pm.spec.exceptions: + +# https://riverbankcomputing.com/hg/sip/raw-rev/323d39a2d602 -> sip-fix-template-values.patch +#Epatch:sip2 Fixed the instantiation of template values. +#Epatch:sip2 diff -r 1430b279a3c9 -r 323d39a2d602 sipbuild/generator/parser/instantiations.py +#Epatch:sip2 --- a/sipbuild/generator/parser/instantiations.py Sun Jun 26 10:01:41 2022 +0100 +#Epatch:sip2 +++ b/sipbuild/generator/parser/instantiations.py Sun Jul 17 09:41:16 2022 +0100 +#Epatch:sip2 @@ -434,7 +434,8 @@ +#Epatch:sip2 proto_name = proto_value.value.result.definition +#Epatch:sip2 +#Epatch:sip2 if proto_name.is_simple: +#Epatch:sip2 - i_name = ScopedName.parse(template_string(proto_name, expansions)) +#Epatch:sip2 + i_name = ScopedName.parse( +#Epatch:sip2 + template_string(proto_name.base_name, expansions)) +#Epatch:sip2 i_result = Argument(type=ArgumentType.DEFINED, definition=i_name) +#Epatch:sip2 i_fcall = FunctionCall(result=i_result, +#Epatch:sip2 args=proto_value.value.args) + +#Epatch:numba # Fix lib location, turn numba_sysinfo.py into a common diag tool +#Epatch:numba --- a/numba/cuda/cuda_paths.py 2022-10-14 09:57:25.616827200 +0200 +#Epatch:numba +++ b/numba/cuda/cuda_paths.py 2022-10-20 11:20:15.106019523 +0200 +#Epatch:numba @@ -66,3 +66,3 @@ +#Epatch:numba else: +#Epatch:numba - return 'lib64' +#Epatch:numba + return 'lib' +#Epatch:numba +#Epatch:numba --- a/numba/misc/numba_sysinfo.py 2022-10-14 09:57:25.628827000 +0200 +#Epatch:numba +++ b/numba/misc/numba_sysinfo.py 2022-10-20 11:02:54.056767540 +0200 +#Epatch:numba @@ -1 +1,2 @@ +#Epatch:numba +#! /usr/bin/env python3 +#Epatch:numba import json +#Epatch:numba @@ -367,3 +368,3 @@ +#Epatch:numba with redirect_stdout(output): +#Epatch:numba - cudadrv.libs.test(sys.platform, print_paths=False) +#Epatch:numba + cudadrv.libs.test(sys.platform, print_paths=True) +#Epatch:numba sys_info[_cu_lib_test] = output.getvalue() +#Epatch:numba @@ -626,9 +627,2 @@ +#Epatch:numba ("",), +#Epatch:numba - ("__Conda Information__",), +#Epatch:numba - (DisplayMap({k: v for k, v in info.items() +#Epatch:numba - if k.startswith('Conda')}) or ("Conda not available.",)), +#Epatch:numba - ("",), +#Epatch:numba - ("__Installed Packages__",), +#Epatch:numba - DisplaySeq(info.get(_inst_pkg, ("Couldn't retrieve packages info.",))), +#Epatch:numba - ("",), +#Epatch:numba ("__Error log__" if info.get(_errors, []) + +# -------------------------------------------------------- This is the end. +