From 263a3362cb96350e40150a2fd1cd7f077a1cb312 Mon Sep 17 00:00:00 2001 From: david Date: Fri, 24 Mar 2023 13:58:29 +0100 Subject: [PATCH 1/2] Add Python 3.10.10 copy of 3.10.8 script --- python-3.10.10-0.build.sh | 738 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 738 insertions(+) create mode 100644 python-3.10.10-0.build.sh diff --git a/python-3.10.10-0.build.sh b/python-3.10.10-0.build.sh new file mode 100644 index 0000000..7343426 --- /dev/null +++ b/python-3.10.10-0.build.sh @@ -0,0 +1,738 @@ +#! /bin/bash + +# ------------------------------------------------------------------- Debug + +# COOKIE=$(mcookie|cut -c-8); grep -v V_GREP_ME $0 > /dev/shm/runme-$COOKIE.sh ; sleep 0.3; exec bash /dev/shm/runme-$COOKIE.sh +# TESTING=1 +# SKIPTENSORFLOW=1 + +# ---------------------------------------------------------------- Preamble + +PKG=python +VERSION=3.10.8 +BUILD=0 +PYTHONVER=${VERSION%.*} + +CUDADIR=/pkg/cuda-11.5.2-0 + +PREFIX=/pkg/$PKG-$VERSION-$BUILD +if [ -n "$TESTING" ]; then PREFIX=/scratch/local2/$PKG-$VERSION-$BUILD ; fi + +PATH=/bin:/usr/bin:/usr/sbin:/usr/local/bin + +# set -xe +set -e +umask 022 + +BUILD_TMPDIR=/scratch/local2/$PKG-$VERSION-$BUILD.$USER.build.tmp + +test -d $BUILD_TMPDIR && ( chmod -R u+rwx $BUILD_TMPDIR || true ; rm -rf $BUILD_TMPDIR ) +mkdir -p $BUILD_TMPDIR/home/.cache/pip +mkdir -p $BUILD_TMPDIR/home/.local +# copy script to an accessible location +cp $0 $BUILD_TMPDIR; ME="$BUILD_TMPDIR/$(basename $0)" +export TMPDIR=$BUILD_TMPDIR +export HOME=$BUILD_TMPDIR/home + +exec $PREFIX/profile <<-EOF + . $CUDADIR/profile + PATH=$PREFIX/bin:\$PATH + export LD_LIBRARY_PATH=$PREFIX/lib\${LD_LIBRARY_PATH:+:}\$LD_LIBRARY_PATH + PKG_CONFIG_PATH=$PREFIX/lib/pkgconfig\${PKG_CONFIG_PATH:+:}\$PKG_CONFIG_PATH + export PKG_CONFIG_PATH + if [ -d $PREFIX/.compatlibs ]; then export LD_LIBRARY_PATH=$PREFIX/.compatlibs\${LD_LIBRARY_PATH:+:}\$LD_LIBRARY_PATH ; fi +EOF +. $PREFIX/profile + +NPROC=$(( $(nproc) * 4 / 5 + 1 )) +export MAKEFLAGS="-j $NPROC" + +BUILDDIR=$PREFIX/build + +mkdir -p $BUILDDIR +cd $BUILDDIR + +# -------------------------------------- Use git to track package evolution + +test -d .git && rm -rf .git +git init -q; echo -e '[user]\n name = none\n email = of_your_business...' >> .git/config +echo '*' > .gitignore + +function piplist() { + pip3 list | awk '{ printf("%-36s %s\n", $1,$2) }' | grep -v '\----' > GameOfVersions +} + +function track() { + if [ -d .git ]; then + if git add -f GameOfVersions; then + git commit -q -n -m "Package: '$*'" || /bin/true + fi + fi +} + +function install() { + # no-color is nice, but doesn't work when dependencies are installed, so (mis-)use a pipe + pip3 install --no-color --compile --cache-dir=$HOME/.cache/pip --prefix=$PREFIX "${@+$@}" | cat + piplist + track $* +} + +# ---------------------------------------- Build Python & support libraries + +( + # orig: https://www.python.org/ftp/python/3.10.8/Python-3.10.8.tgz + BEEHIVE=https://beehive.molgen.mpg.de/fbe3fff11893916ad1756b15c8a48834/Python-3.10.8.tgz + test -e Python-$VERSION.tgz || wget -nv $BEEHIVE + test -d Python-$VERSION || tar -xf Python-$VERSION.tgz + cd Python-$VERSION + + # BerkeleyDB, the undead ... + # Newer libgdbm_compat.so versions pull libgdbm.so on their own, our libgdbm_compat.so is older ... + # So, let setup.py create '-lgdbm -lgdbm_compat' instead of a sole '-lgdbm_compat' + # verbatim: ndbm_libs = ['gdbm_compat'] => ndbm_libs = ['gdbm','gdbm_compat'] + sed -i -e "/ndbm_libs = / s/_compat'/','gdbm_compat'/" setup.py + + # leave further hints in the log + sed -i -e '/db_setup_debug = False/ s/False/True/' \ + -e '/dbm_setup_debug = False/ s/False/True/' setup.py + + # These may come handy if the Python build needs exclusive stuff from $PREFIX + # INCLUDEDIR=$PREFIX/include \ + # LIBDIR=$PREFIX/lib \ + LDFLAGS="-Wl,-rpath=$PREFIX/lib" \ + ./configure \ + --prefix=$PREFIX \ + --enable-shared + + make -j $NPROC + make install + + # it might come to happen that something down the line may just call 'python', defuse ... + test -e $PREFIX/bin/python || ln -s python3 $PREFIX/bin/python + + python3 -m ensurepip + pip3 install --prefix=$PREFIX -I pip + ( # fix 'please update' noise from pip + cd $PREFIX/lib/python*/site-packages + sed -ne '/^#Epatch:pip/ s/^#Epatch:\S* // p' $ME | patch -p1 --verbose + ) +) +install setuptools==65.3.0 # see numba below +piplist; track START + + +# build various pythonbindings 'by hand', because pip installs do not deliver +# header files and the like. +# BTW: builds need meson + +install meson + +( + DBUS_PYTHON_VERSION=1.2.18 + + # https://files.pythonhosted.org/packages/b1/5c/ccfc167485806c1936f7d3ba97db6c448d0089c5746ba105b6eb22dba60e/dbus-python-1.2.18.tar.gz + BEEHIVE=https://beehive.molgen.mpg.de/468874f3bf61d1b7deac98bbe2cfd8ac/dbus-python-1.2.18.tar.gz + test -e dbus-python-$DBUS_PYTHON_VERSION.tar.gz || wget -nv $BEEHIVE + test -d dbus-python-$DBUS_PYTHON_VERSION || tar -xf dbus-python-$DBUS_PYTHON_VERSION.tar.gz + + cd dbus-python-$DBUS_PYTHON_VERSION + ./configure --prefix=$PREFIX + + make -j $NPROC + make install + +) + +( + PYCAIRO_VERSION=1.21.0 + + # https://files.pythonhosted.org/packages/92/a4/506564f574fa74c90b98690e8ecc8dbae1629f31fcfe0be69de45d9e1605/pycairo-1.21.0.tar.gz + BEEHIVE=https://beehive.molgen.mpg.de/682238eaa8bd7032e36a20b6f53824d1/pycairo-1.21.0.tar.gz + test -e pycairo-$PYCAIRO_VERSION.tar.gz || wget -nv $BEEHIVE + test -d pycairo-$PYCAIRO_VERSION || tar -xf pycairo-$PYCAIRO_VERSION.tar.gz + + cd pycairo-$PYCAIRO_VERSION + + mkdir -p meson-build + cd meson-build + + meson setup \ + --prefix=$PREFIX \ + --buildtype=release \ + .. + + ninja -j $NPROC + ninja install + +) + +( + PYGOBJECT3_VERSION=3.42.2 + + # https://files.pythonhosted.org/packages/fe/40/9afaeb8d3b453fb8596fcb6c7bc2b64e434868c91eda19955742778eff74/PyGObject-3.42.2.tar.gz + BEEHIVE=https://beehive.molgen.mpg.de/80ad843b2a49ccfcc30c6b756b755c09/PyGObject-3.42.2.tar.gz + test -e PyGObject-$PYGOBJECT3_VERSION.tar.gz || wget -nv $BEEHIVE + test -d PyGObject-$PYGOBJECT3_VERSION || tar -xf PyGObject-$PYGOBJECT3_VERSION.tar.gz + + cd PyGObject-$PYGOBJECT3_VERSION + + mkdir -p meson-build + cd meson-build + meson setup \ + --prefix=$PREFIX \ + --buildtype=release \ + -Dtests=false \ + .. + + ninja -j $NPROC + ninja install + +) +piplist; track various pythonbindings + +# build support to use QT from python -- see www.riverbankcomputing.com + +( # sip: tool to create Python bindings for C and C++ libraries. + SIP_VERSION=6.6.2 + + # https://files.pythonhosted.org/packages/5b/cb/c27c925ae07bd03a2597fa1db17bfc2a4ac57da61aeb90f8ec98ffbb975b/sip-6.6.2.tar.gz + BEEHIVE=https://beehive.molgen.mpg.de/620eb75a8b22fb6af9c6044aa015782d/sip-6.6.2.tar.gz + test -e sip-$SIP_VERSION.tar.gz || wget -nv $BEEHIVE + test -d sip-$SIP_VERSION || tar -xf sip-$SIP_VERSION.tar.gz + + cd sip-$SIP_VERSION + + sed -ne '/^#Epatch:sip1/ s/^#Epatch:\S* // p' $ME | patch -p1 --verbose + sed -ne '/^#Epatch:sip2/ s/^#Epatch:\S* // p' $ME | patch -p1 --verbose + + python3 setup.py build + python3 setup.py install --skip-build --optimize=1 + +) + +# QT bindings need some python modules to build + +install PyQt5-sip +install PyQt-builder + +( + PYQT_VERSION=5.15.7 + + # orig: https://files.pythonhosted.org/packages/e1/57/2023316578646e1adab903caab714708422f83a57f97eb34a5d13510f4e1/PyQt5-5.15.7.tar.gz + BEEHIVE=https://beehive.molgen.mpg.de/ae2c68e38b9b36fdf5f932419353a2b3/PyQt5-5.15.7.tar.gz + test -e PyQt5-$PYQT_VERSION.tar.gz || wget -nv $BEEHIVE + test -d PyQt5-$PYQT_VERSION || tar -xf PyQt5-$PYQT_VERSION.tar.gz + + cd PyQt5-$PYQT_VERSION + + sip-build \ + --confirm-license \ + --no-make \ + --api-dir $PREFIX/share/qsci/api/python \ + --dbus $PREFIX/include/dbus-1.0 \ + --no-designer-plugin \ + --no-qml-plugin \ + --jobs $NPROC \ + + cd build + make -j $NPROC + make install + + # Remove unused py2 version of uic modules + rm -r $PREFIX/lib/python*/site-packages/PyQt5/uic/port_v2 + +) + +( + QSCINTILLA_VERSION=2.13.4.dev2206171421 # no 2.14 in sight ... + + # orig: https://www.riverbankcomputing.com/static/Downloads/QScintilla/${PKGVERSION}/QScintilla_src-${PKGVERSION}.tar.gz + BEEHIVE=https://beehive.molgen.mpg.de/d196ebd441f40474dd7732d4d492de69/QScintilla_src-2.13.4.dev2206171421.tar.gz + test -e QScintilla_src-$QSCINTILLA_VERSION.tar.gz || wget -nv $BEEHIVE + test -d QScintilla_src-$QSCINTILLA_VERSION || tar -xf QScintilla_src-$QSCINTILLA_VERSION.tar.gz + + cd QScintilla_src-$QSCINTILLA_VERSION + cd Python + + cp -vp pyproject-qt5.toml pyproject.toml + sip-build \ + --no-make \ + --qsci-features-dir ../src/features \ + --qsci-include-dir ../src \ + --qsci-library-dir ../src \ + --api-dir $PREFIX/share/qsci/api/python \ + --jobs $NPROC + + cd build + make -j $NPROC + make install + +) +piplist; track qt-support-riverbankcomputing + +# -------------------------------------------------------- Install packages + +# Install the psychotic diva first +install numpy==1.23.4 + +install Cython + +# libxml2-python from pypi isn't up to date, cause lxml seems to be the favored binding +# do this early, not that some dependency tries it on its own. +# (btw. libxslt-python isn't ported to python3) +( + LIBXML2_VERSION=2.9.12 + + # orig: http://xmlsoft.org/sources/libxml2-${PKGVERSION}.tar.gz + BEEHIVE=https://beehive.molgen.mpg.de/f433a39be087a9f0b197eb2307ad9f75/libxml2-2.9.12.tar.gz + test -e libxml2-$LIBXML2_VERSION.tar.gz || wget -nv $BEEHIVE + test -d libxml2-$LIBXML2_VERSION || tar -xf libxml2-$LIBXML2_VERSION.tar.gz libxml2-$LIBXML2_VERSION/python + cd libxml2-$LIBXML2_VERSION/python + python3 setup.py build + python3 setup.py install +) +piplist; track legacy-libxml2 + + +# 'numba' the cuda enabled numpy +( + NUMBA_VERSION=0.56.3 + + # orig: https://files.pythonhosted.org/packages/a3/95/7df08647e773a2417e24de8fea9087606b37074f63830a3f2dbabd97f01b/numba-0.56.3.tar.gz + BEEHIVE=https://beehive.molgen.mpg.de/ae0a008447f2f246523af143b1488e88/numba-0.56.3.tar.gz + test -e numba-$NUMBA_VERSION.tar.gz || wget -nv $BEEHIVE + test -d numba-$NUMBA_VERSION || tar -xf numba-$NUMBA_VERSION.tar.gz + cd numba-$NUMBA_VERSION + + # numba assumes that libcudart.so and friends reside in 'CUDA_HOME/lib64' + sed -ne '/^#Epatch:numba/ s/^#Epatch:\S* // p' $ME | patch -p1 --verbose + install . + # useful to see if numba works + cp -p numba/misc/numba_sysinfo.py $PREFIX/bin + chmod 0755 $PREFIX/bin/numba_sysinfo.py +) +piplist; track numba + + +# 'distributed' doesn't like the 'new' tornado 6.2 -- reason: a deprecation warning makes the CI choke (geht's noch?) +( + DISTRIBUTED_VERSION=2022.9.0 + + # orig: https://files.pythonhosted.org/packages/df/46/e8ef41f8dfd553327b6a12f4ab78d9351b54299596fe5b8b165ba40b1550/distributed-2022.9.0.tar.gz + BEEHIVE=https://beehive.molgen.mpg.de/335d718f1ba331ee8e85e40e564559a1/distributed-2022.9.0.tar.gz + test -e distributed-$DISTRIBUTED_VERSION.tar.gz || wget -nv $BEEHIVE + test -d distributed-$DISTRIBUTED_VERSION || tar -xf distributed-$DISTRIBUTED_VERSION.tar.gz + cd distributed-$DISTRIBUTED_VERSION + sed -i -e '/tornado/ s/.*/tornado/' requirements.txt + sed -i -e '/tornado/ s/.*/tornado/' distributed.egg-info/requires.txt + install . +) +piplist; track distributed and tornado + +PACKAGES=$(sed -e s/#.*$// <<- __PKGLIST__ + protobuf==3.19.4 # tensorflow likes to have an slightly older one [< 3.20], no conflicts with this + Matplotlib==3.6.0 # 3.6.1 is the first bugfix release of the 3.6.x series... Well, it is buggy + ipython[all] + + SciPy + pandas + SymPy + nose + statsmodels + pysam + seaborn + scikit-learn + Mako + cutadapt # Helpdesk 01/24/18 16:19, Sabrina Krakau + pygobject # Helpdesk 03/26/18 14:19, Virginie Stanisla + blink1 # Blink1-lib 07/04/18 12:00, Niclas Hofmann + umi-tools + sphinx + recommonmark + fastcluster + pydot + pygraphviz + deeptools + HTSeq + keras + keras_preprocessing # tensorflow ... + MotifScan # Alena 30.8.2018 + MAmotif + hic2cool # Helpdesk Robert Schoepflin 17.09.2018 + snakemake + virtualenv # Donald + odfpy # Donald + Dumper # Thomas + igraph + louvain # monocle3 prep for Helene + selenium # 19.7.19 Kreitler -> https://webxray.org/ + gmpy # From viper context, Helpdesk Laura Glaser 9.8.2019 + plotly + pyOpenCL + docopt + distro + ply + threadpoolctl + HiCMatrix + pybedtools + gffutils + python-bioformats + jupyterlab + jupyterlab-git # request Peter Arndt + igv_jupyterlab # Matthias Lienhard HD 01-04-2020 + igv-jupyter + scour # https://github.molgen.mpg.de/mariux64/pkg-scripts/issues/98 + multiqc + pybind11 + svgwrite + pyomo + bioservices + pairtools + SharedArray + pyabc # dependency for stochkit, HD 20.07.20 Gemma Noviello + petab + tadtool # HD 26.01.21 Michael Robson + pyperf # Paul sep-20 + umap # HD, used by Matthias Lienhard + MACS3 + biopython # HD 2021-07-23 Magalhaes/Hnisz alphafold related -- cool that we have missed biopython so far, LOL + dm-tree + dm-haiku + flatbuffers + contextlib2 + ml_collections + + # additions/fixups since 3.9 + papermill # turns jupyter notebooks into scripts [sic!] used by P.A. [who else, *grin*] + opencv-python + scikit-image + git-cola + cffi-glpk # uses /usr/lib/libglpk.so.40, see if I can get it linked statically - kreitler + czifile + napari[pyqt5] + napari-czifile2 + CrossMap + cooltools # cause trouble with numba/nupy versions -- Maria Valieva 17.7.2019 + scanpy + + __PKGLIST__ +) + +for PKG in $PACKAGES; do + install $PKG +done + +install pyopengl # Tk binding creates an error, so remove it (Expects Tk Togl installation...) +rm -rf ${PREFIX}/lib/python$PYTHONVER/site-packages/OpenGL/Tk + +# torch bundled with a recent CUDA-Toolkit +install torch torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/cu116 + +# install helen after our torch, otherwise we might catch the default torch from pip ... +install helen # HD, a dependency from a request by helene + +mkdir -vp $PREFIX/share/jupyter/lab/settings # try to defeat a silly error in jupyter labextension install + +# --------------------------------------------------- Install misc packages + +( + BOOST_VERSION=(1.80.0 1_80_0) + PKGCFLAGS="-O2 -fPIC" + + # orig: https://dl.bintray.com/boostorg/release/1.75.0/source/boost_1_75_0.tar.gz + BEEHIVE=https://beehive.molgen.mpg.de//38813f6feb40387dfe90160debd71251/boost_1_75_0.tar.gz + # orig: https://boostorg.jfrog.io/artifactory/main/release/1.80.0/source/boost_1_80_0.tar.gz + BEEHIVE=https://beehive.molgen.mpg.de/077f074743ea7b0cb49c6ed43953ae95/boost_1_80_0.tar.gz + test -e boost_${BOOST_VERSION[1]}.tar.gz || wget -nv $BEEHIVE + test -d boost-${BOOST_VERSION[0]} || mkdir boost-${BOOST_VERSION[0]} && tar -xf boost_${BOOST_VERSION[1]}.tar.gz -C boost-${BOOST_VERSION[0]} --strip-components=1 + + cd boost-${BOOST_VERSION[0]} + sed -e "/using python/ s|;|: ${PREFIX}/include/python${PYTHONVER} ;|" \ + -i bootstrap.sh + + ./bootstrap.sh \ + --with-toolset=gcc \ + --with-icu \ + --with-python=$PREFIX/bin/python3 \ + --with-libraries=python,thread || exit 1 + + ./b2 \ + -j $NPROC \ + --layout=system \ + --build-dir=build-python3 \ + --prefix=$PREFIX \ + --libdir=$PREFIX/lib \ + --build-type=minimal \ + toolset=gcc \ + variant=release \ + debug-symbols=off \ + link=shared \ + threading=multi \ + runtime-link=shared \ + python=$PYTHONVER \ + cflags="$PKGCFLAGS" \ + cxxflags="$PKGCFLAGS" \ + install || exit 1 +) + +( + PYCUDAVERS=2022.1 + PYTHON_LIBNAME=boost_python${PYTHONVER/./} + + # orig: https://github.com/inducer/pycuda/archive/v2020.1.tar.gz + # BEEHIVE=https://beehive.molgen.mpg.de/580562d7af4d3249c5b42a1df8d428a0/pycuda-2020.1.tar.gz + # orig: https://files.pythonhosted.org/packages/2d/1f/48a3a5b2c715345e7af1e09361100bd98c3d72b4025371692ab233f523d3/pycuda-2022.1.tar.gz # maybe those very long URLs compensate for other (shorter) things ... + BEEHIVE="https://beehive.molgen.mpg.de/2fad5a15db4153d825822eba7cb25f25/pycuda-2022.1.tar.gz" + test -e pycuda-${PYCUDAVERS}.tar.gz || wget -nv $BEEHIVE + test -d pycuda-${PYCUDAVERS} || tar -xf pycuda-${PYCUDAVERS}.tar.gz + + cd pycuda-${PYCUDAVERS} + test -e siteconf.py && rm siteconf.py + python3 configure.py \ + --cuda-enable-gl \ + --cuda-root=$CUDADIR \ + --no-use-shipped-boost \ + --boost-python-libname=$PYTHON_LIBNAME \ + --boost-inc-dir=$PREFIX/include \ + --boost-lib-dir=$PREFIX/lib \ + --cudadrv-lib-dir='${CUDA_ROOT}/lib,${CUDA_ROOT}/lib/stubs' \ + --cudart-lib-dir='${CUDA_ROOT}/lib,${CUDA_ROOT}/lib/stubs' \ + --curand-lib-dir='${CUDA_ROOT}/lib,${CUDA_ROOT}/lib/stubs' \ + --ldflags=-L$PREFIX/lib + + python3 setup.py build + python3 setup.py install --prefix $PREFIX +) + +( + # orig: https://github.com/nboley/idr/archive/2.0.3/idr-2.0.3.tar.gz # HD: Tobias Zehnder, 18.3.21 + BEEHIVE=https://beehive.molgen.mpg.de/1bec61b526bb1dcf92a5ab99c27852d0/idr-2.0.3.tar.gz + test -e idr-2.0.3.tar.gz || wget -nv $BEEHIVE + test -d idr-2.0.3 || tar -xf idr-2.0.3.tar.gz + cd idr-2.0.3 + python3 setup.py build + python3 setup.py install --prefix $PREFIX +) + +( + # orig: https://github.com/stacked-git/stgit/archive/v1.0/stgit-1.0.tar.gz # Paul mariux64/pkg-scripts/issues/152 + BEEHIVE=https://beehive.molgen.mpg.de/b20ab5a197fc3a141f4b00c5859c0a3b/stgit-1.0.tar.gz + test -e stgit-1.0.tar.gz || wget -nv $BEEHIVE + test -d stgit-1.0 || tar -xf stgit-1.0.tar.gz + cd stgit-1.0 + python3 setup.py build + python3 setup.py install --prefix $PREFIX +) + +( + # https://github.com/brentp/bwa-meth/archive/refs/tags/v0.2.5.tar.gz + BEEHIVE=https://beehive.molgen.mpg.de/33a4c416760dd67e77f17e07791527e2/bwa-meth-0.2.5.tar.gz + test -e bwa-meth-0.2.5.tar.gz || wget -nv $BEEHIVE + test -d bwa-meth-0.2.5 || tar -xf bwa-meth-0.2.5.tar.gz + cd bwa-meth-0.2.5 + python3 setup.py build + python3 setup.py install --prefix $PREFIX +) +piplist; track misc packages + +# ------------------------------------------------ finally build tensorflaw + +install wheel # to build our tf-wheel (i.e. zip archive) + +( + test -n "$SKIPTENSORFLOW" && exit + + TFVERSION=2.10.0 + NCCLSHORTVERSION=$(pkg-config nccl --modversion | grep -Po '\d+\.\d+') + + . /pkg/bazel-5.1.0-0/profile + + # orig: https://github.com/tensorflow/tensorflow/archive/refs/tags/v2.10.0.tar.gz + BEEHIVE=https://beehive.molgen.mpg.de/7ceb5277d482f10d2df8644deb412061/tensorflow-2.10.0.tar.gz + + test -e tensorflow-${TFVERSION}.tar.gz || wget -nv $BEEHIVE + test -d tensorflow-${TFVERSION} || tar -xf tensorflow-${TFVERSION}.tar.gz + + cd tensorflow-${TFVERSION} + + #vars for config + export PYTHON_BIN_PATH=${PREFIX}/bin/python3 + export USE_DEFAULT_PYTHON_LIB_PATH=1 + export TF_NEED_JEMALLOC=1 + export TF_NEED_KAFKA=0 + export TF_NEED_OPENCL_SYCL=0 + export TF_NEED_AWS=0 + export TF_NEED_GCP=0 + export TF_NEED_HDFS=0 + export TF_NEED_S3=0 + export TF_ENABLE_XLA=1 + export TF_NEED_GDR=0 + export TF_NEED_VERBS=0 + export TF_NEED_OPENCL=0 + export TF_NEED_MPI=0 + export TF_NEED_TENSORRT=0 + export TF_NEED_NGRAPH=0 + export TF_NEED_IGNITE=0 + export TF_NEED_ROCM=0 + export TF_SET_ANDROID_WORKSPACE=0 + export TF_DOWNLOAD_CLANG=0 + export TF_NCCL_VERSION=${NCCLSHORTVERSION} + export TF_IGNORE_MAX_BAZEL_VERSION=1 + export NCCL_INSTALL_PATH=${CUDADIR} + export CC=gcc + export CXX=g++ + export GCC_HOST_COMPILER_PATH=/usr/bin/${CC} + export HOST_C_COMPILER=/usr/bin/${CC} + export HOST_CXX_COMPILER=/usr/bin/${CXX} + export TF_CUDA_CLANG=0 + export CLANG_CUDA_COMPILER_PATH=/usr/bin/clang + export TF_CUDA_PATHS=${CUDADIR},${CUDADIR}/extras/CUPTI,${CUDADIR}/nvvm + export TF_CUDA_VERSION=$(${CUDADIR}/bin/nvcc --version | sed -n 's/^.*release \(.*\),.*/\1/p') # 11.5 + export TF_CUDNN_VERSION=$(sed -n 's/^#define CUDNN_MAJOR\s*\(.*\).*/\1/p' ${CUDADIR}/include/cudnn_version.h) + # hardware older than sm_52 is said to be unsupported, compute_86 denotes an emulation + export TF_CUDA_COMPUTE_CAPABILITIES=sm_52,sm_53,sm_60,sm_61,sm_62,sm_70,sm_72,sm_75,sm_80,sm_86,compute_86 + + # stay compatible with the cluster machines + export CC_OPT_FLAGS="-march=nehalem" + export TF_NEED_CUDA=1 + export BAZEL_ARGS="--config=mkl -c opt" + + ./configure + + bazel build \ + --cxxopt="-march=nehalem" \ + --copt="-march=nehalem" \ + --noshow_loading_progress \ + --noshow_progress \ + --color="no" \ + //tensorflow:libtensorflow.so \ + //tensorflow:libtensorflow_cc.so \ + //tensorflow:install_headers \ + //tensorflow/tools/pip_package:build_pip_package + + bazel shutdown # otherwise it hogs in cwd + + # create a _clean/decent_ python package from so far created debris + test -d tensorflow-pip && rm -r tensorflow-pip + mkdir tensorflow-pip + ./bazel-bin/tensorflow/tools/pip_package/build_pip_package --src ${BUILDDIR}/tensorflow-pip + cd ${BUILDDIR} + # ... cope with the ??? symlink-structure placed in bazels' build-TMPDIR (-h) (tell me why ?) + test -e tensorflow-pip.tar && rm tensorflow-pip.tar + tar -chf tensorflow-pip.tar tensorflow-pip + + # and perform a regular install + test -d tensorflow-pip && rm -r tensorflow-pip + tar -xf tensorflow-pip.tar + cd ${BUILDDIR}/tensorflow-pip + + python3 setup.py build + python3 setup.py install --prefix $PREFIX +) +piplist; track tensorflow + +# tidy up a bit +if [ -d $PREFIX/man/man1 ]; then + mv -v $PREFIX/man/man1/* $PREFIX/share/man/man1 || true + rm -rf $PREFIX/man +fi + +# ------------------------------------------------------------ Sanity check + +# load all packages, see warnings and spot installation errors +# But don't care about r2py, a further dependence into R is not wanted! +echo "# START load test." +python3 -c 'help("modules")' > /dev/null +echo "# END load test (passed)." + +echo "# running pip check" +pip3 check + +# I guess it makes sense that files can be read by all users ... +echo "# check for unreadable files" +cd $PREFIX +find \! -perm -004 -exec chmod -c a+r {} + + +exit + +# -------------------------------------------------------- Included patches + +#Epatch:pip # Operation 'Forever young', disable version-check per default. +#Epatch:pip # The option needs to be kept, because some tools make use of +#Epatch:pip # it and would cause pip to fail. +#Epatch:pip --- a/pip/_internal/cli/cmdoptions.py 2022-08-09 17:29:15.853442948 +0200 +#Epatch:pip +++ b/pip/_internal/cli/cmdoptions.py 2022-08-10 10:31:35.124945154 +0200 +#Epatch:pip @@ -892,7 +892,6 @@ +#Epatch:pip dest="disable_pip_version_check", +#Epatch:pip action="store_true", +#Epatch:pip - default=False, +#Epatch:pip - help="Don't periodically check PyPI to determine whether a new version " +#Epatch:pip - "of pip is available for download. Implied with --no-index.", +#Epatch:pip + default=True, +#Epatch:pip + help="Ignore this option, the version check IS disabled.", +#Epatch:pip ) + +# https://riverbankcomputing.com/hg/sip/raw-rev/1430b279a3c9 -> sip-fix-exceptions.patch +#Epatch:sip1 Fixed the handling of exceptions that sub-class C++ exceptions. +#Epatch:sip1 diff -r 6eff81a15822 -r 1430b279a3c9 sipbuild/generator/parser/rules.py +#Epatch:sip1 --- a/sipbuild/generator/parser/rules.py Fri Jun 17 14:25:16 2022 +0100 +#Epatch:sip1 +++ b/sipbuild/generator/parser/rules.py Sun Jun 26 10:01:41 2022 +0100 +#Epatch:sip1 @@ -2293,6 +2293,7 @@ +#Epatch:sip1 +#Epatch:sip1 if len(p) == 4: +#Epatch:sip1 base = p[2] +#Epatch:sip1 + base.make_absolute() +#Epatch:sip1 +#Epatch:sip1 # See if it is a project-defined exception. +#Epatch:sip1 for xd in p.parser.pm.spec.exceptions: + +# https://riverbankcomputing.com/hg/sip/raw-rev/323d39a2d602 -> sip-fix-template-values.patch +#Epatch:sip2 Fixed the instantiation of template values. +#Epatch:sip2 diff -r 1430b279a3c9 -r 323d39a2d602 sipbuild/generator/parser/instantiations.py +#Epatch:sip2 --- a/sipbuild/generator/parser/instantiations.py Sun Jun 26 10:01:41 2022 +0100 +#Epatch:sip2 +++ b/sipbuild/generator/parser/instantiations.py Sun Jul 17 09:41:16 2022 +0100 +#Epatch:sip2 @@ -434,7 +434,8 @@ +#Epatch:sip2 proto_name = proto_value.value.result.definition +#Epatch:sip2 +#Epatch:sip2 if proto_name.is_simple: +#Epatch:sip2 - i_name = ScopedName.parse(template_string(proto_name, expansions)) +#Epatch:sip2 + i_name = ScopedName.parse( +#Epatch:sip2 + template_string(proto_name.base_name, expansions)) +#Epatch:sip2 i_result = Argument(type=ArgumentType.DEFINED, definition=i_name) +#Epatch:sip2 i_fcall = FunctionCall(result=i_result, +#Epatch:sip2 args=proto_value.value.args) + +#Epatch:numba # Fix lib location, turn numba_sysinfo.py into a common diag tool +#Epatch:numba --- a/numba/cuda/cuda_paths.py 2022-10-14 09:57:25.616827200 +0200 +#Epatch:numba +++ b/numba/cuda/cuda_paths.py 2022-10-20 11:20:15.106019523 +0200 +#Epatch:numba @@ -66,3 +66,3 @@ +#Epatch:numba else: +#Epatch:numba - return 'lib64' +#Epatch:numba + return 'lib' +#Epatch:numba +#Epatch:numba --- a/numba/misc/numba_sysinfo.py 2022-10-14 09:57:25.628827000 +0200 +#Epatch:numba +++ b/numba/misc/numba_sysinfo.py 2022-10-20 11:02:54.056767540 +0200 +#Epatch:numba @@ -1 +1,2 @@ +#Epatch:numba +#! /usr/bin/env python3 +#Epatch:numba import json +#Epatch:numba @@ -367,3 +368,3 @@ +#Epatch:numba with redirect_stdout(output): +#Epatch:numba - cudadrv.libs.test(sys.platform, print_paths=False) +#Epatch:numba + cudadrv.libs.test(sys.platform, print_paths=True) +#Epatch:numba sys_info[_cu_lib_test] = output.getvalue() +#Epatch:numba @@ -626,9 +627,2 @@ +#Epatch:numba ("",), +#Epatch:numba - ("__Conda Information__",), +#Epatch:numba - (DisplayMap({k: v for k, v in info.items() +#Epatch:numba - if k.startswith('Conda')}) or ("Conda not available.",)), +#Epatch:numba - ("",), +#Epatch:numba - ("__Installed Packages__",), +#Epatch:numba - DisplaySeq(info.get(_inst_pkg, ("Couldn't retrieve packages info.",))), +#Epatch:numba - ("",), +#Epatch:numba ("__Error log__" if info.get(_errors, []) + +# -------------------------------------------------------- This is the end. + From e9ccc0f526b1c2ca9e5018b28b79898a093ab262 Mon Sep 17 00:00:00 2001 From: david Date: Fri, 31 Mar 2023 10:10:13 +0200 Subject: [PATCH 2/2] Add python-3.10.10-0 install higher versions for some package add fix versions of packages, because of dependencies add ascciidoc --- python-3.10.10-0.build.sh | 268 +++++++++++++++++++++----------------- 1 file changed, 148 insertions(+), 120 deletions(-) mode change 100644 => 100755 python-3.10.10-0.build.sh diff --git a/python-3.10.10-0.build.sh b/python-3.10.10-0.build.sh old mode 100644 new mode 100755 index 7343426..d66a827 --- a/python-3.10.10-0.build.sh +++ b/python-3.10.10-0.build.sh @@ -8,12 +8,14 @@ # ---------------------------------------------------------------- Preamble +#set -x + PKG=python -VERSION=3.10.8 +VERSION=3.10.10 BUILD=0 PYTHONVER=${VERSION%.*} -CUDADIR=/pkg/cuda-11.5.2-0 +CUDADIR=/pkg/cuda-11.7.1-0 PREFIX=/pkg/$PKG-$VERSION-$BUILD if [ -n "$TESTING" ]; then PREFIX=/scratch/local2/$PKG-$VERSION-$BUILD ; fi @@ -83,10 +85,11 @@ function install() { # ---------------------------------------- Build Python & support libraries ( - # orig: https://www.python.org/ftp/python/3.10.8/Python-3.10.8.tgz - BEEHIVE=https://beehive.molgen.mpg.de/fbe3fff11893916ad1756b15c8a48834/Python-3.10.8.tgz - test -e Python-$VERSION.tgz || wget -nv $BEEHIVE - test -d Python-$VERSION || tar -xf Python-$VERSION.tgz + # orig: https://www.python.org/ftp/python/3.10.10/Python-3.10.10.tar.xz + BEEHIVE=https://beehive.molgen.mpg.de/7bf85df71bbe7f95e5370b983e6ae684/Python-3.10.10.tar.xz + + test -e Python-$VERSION.tar.xz || wget -nv $BEEHIVE + test -d Python-$VERSION || tar -xf Python-$VERSION.tar.xz cd Python-$VERSION # BerkeleyDB, the undead ... @@ -131,10 +134,11 @@ piplist; track START install meson ( - DBUS_PYTHON_VERSION=1.2.18 - - # https://files.pythonhosted.org/packages/b1/5c/ccfc167485806c1936f7d3ba97db6c448d0089c5746ba105b6eb22dba60e/dbus-python-1.2.18.tar.gz - BEEHIVE=https://beehive.molgen.mpg.de/468874f3bf61d1b7deac98bbe2cfd8ac/dbus-python-1.2.18.tar.gz + DBUS_PYTHON_VERSION=1.3.2 + + #PyPi + # https://files.pythonhosted.org/packages/c1/d3/6be85a9c772d6ebba0cc3ab37390dd6620006dcced758667e0217fb13307/dbus-python-1.3.2.tar.gz + BEEHIVE=https://beehive.molgen.mpg.de/33be8a4a766e1c7c9a377b8f934ce21a/dbus-python-1.3.2.tar.gz test -e dbus-python-$DBUS_PYTHON_VERSION.tar.gz || wget -nv $BEEHIVE test -d dbus-python-$DBUS_PYTHON_VERSION || tar -xf dbus-python-$DBUS_PYTHON_VERSION.tar.gz @@ -147,10 +151,11 @@ install meson ) ( - PYCAIRO_VERSION=1.21.0 - - # https://files.pythonhosted.org/packages/92/a4/506564f574fa74c90b98690e8ecc8dbae1629f31fcfe0be69de45d9e1605/pycairo-1.21.0.tar.gz - BEEHIVE=https://beehive.molgen.mpg.de/682238eaa8bd7032e36a20b6f53824d1/pycairo-1.21.0.tar.gz + PYCAIRO_VERSION=1.23.0 + + #PyPi + # https://files.pythonhosted.org/packages/69/ca/9e9fa2e8be0876a9bbf046a1be7ee33e61d4fdfbd1fd25c76c1bdfddf8c4/pycairo-1.23.0.tar.gz + BEEHIVE=https://beehive.molgen.mpg.de/7a3729d21659098e1b9a411b62e88966/pycairo-1.23.0.tar.gz test -e pycairo-$PYCAIRO_VERSION.tar.gz || wget -nv $BEEHIVE test -d pycairo-$PYCAIRO_VERSION || tar -xf pycairo-$PYCAIRO_VERSION.tar.gz @@ -170,10 +175,10 @@ install meson ) ( - PYGOBJECT3_VERSION=3.42.2 - - # https://files.pythonhosted.org/packages/fe/40/9afaeb8d3b453fb8596fcb6c7bc2b64e434868c91eda19955742778eff74/PyGObject-3.42.2.tar.gz - BEEHIVE=https://beehive.molgen.mpg.de/80ad843b2a49ccfcc30c6b756b755c09/PyGObject-3.42.2.tar.gz + PYGOBJECT3_VERSION=3.44.0 + + # https://files.pythonhosted.org/packages/07/73/a034fc1bcd6d402d6f72ff618c093f91ac6921f968cc19806b6a1b1b19c8/PyGObject-3.44.0.tar.gz + BEEHIVE=https://beehive.molgen.mpg.de/35d27b533be6a98c7f17ddb6bd0e87d6/PyGObject-3.44.0.tar.gz test -e PyGObject-$PYGOBJECT3_VERSION.tar.gz || wget -nv $BEEHIVE test -d PyGObject-$PYGOBJECT3_VERSION || tar -xf PyGObject-$PYGOBJECT3_VERSION.tar.gz @@ -196,18 +201,15 @@ piplist; track various pythonbindings # build support to use QT from python -- see www.riverbankcomputing.com ( # sip: tool to create Python bindings for C and C++ libraries. - SIP_VERSION=6.6.2 + SIP_VERSION=6.7.7 # https://files.pythonhosted.org/packages/5b/cb/c27c925ae07bd03a2597fa1db17bfc2a4ac57da61aeb90f8ec98ffbb975b/sip-6.6.2.tar.gz - BEEHIVE=https://beehive.molgen.mpg.de/620eb75a8b22fb6af9c6044aa015782d/sip-6.6.2.tar.gz + BEEHIVE=https://beehive.molgen.mpg.de/a131efd79649d31ee11601b19d1e3784/sip-6.7.7.tar.gz test -e sip-$SIP_VERSION.tar.gz || wget -nv $BEEHIVE test -d sip-$SIP_VERSION || tar -xf sip-$SIP_VERSION.tar.gz cd sip-$SIP_VERSION - sed -ne '/^#Epatch:sip1/ s/^#Epatch:\S* // p' $ME | patch -p1 --verbose - sed -ne '/^#Epatch:sip2/ s/^#Epatch:\S* // p' $ME | patch -p1 --verbose - python3 setup.py build python3 setup.py install --skip-build --optimize=1 @@ -219,10 +221,11 @@ install PyQt5-sip install PyQt-builder ( - PYQT_VERSION=5.15.7 + PYQT_VERSION=5.15.9 - # orig: https://files.pythonhosted.org/packages/e1/57/2023316578646e1adab903caab714708422f83a57f97eb34a5d13510f4e1/PyQt5-5.15.7.tar.gz - BEEHIVE=https://beehive.molgen.mpg.de/ae2c68e38b9b36fdf5f932419353a2b3/PyQt5-5.15.7.tar.gz + #PyPi + # orig: https://files.pythonhosted.org/packages/5c/46/b4b6eae1e24d9432905ef1d4e7c28b6610e28252527cdc38f2a75997d8b5/PyQt5-5.15.9.tar.gz + BEEHIVE=https://beehive.molgen.mpg.de/9d97fc06b7ae75e654e946c49e07ff12/PyQt5-5.15.9.tar.gz test -e PyQt5-$PYQT_VERSION.tar.gz || wget -nv $BEEHIVE test -d PyQt5-$PYQT_VERSION || tar -xf PyQt5-$PYQT_VERSION.tar.gz @@ -247,10 +250,10 @@ install PyQt-builder ) ( - QSCINTILLA_VERSION=2.13.4.dev2206171421 # no 2.14 in sight ... + QSCINTILLA_VERSION=2.13.4 # no 2.14 in sight ... - # orig: https://www.riverbankcomputing.com/static/Downloads/QScintilla/${PKGVERSION}/QScintilla_src-${PKGVERSION}.tar.gz - BEEHIVE=https://beehive.molgen.mpg.de/d196ebd441f40474dd7732d4d492de69/QScintilla_src-2.13.4.dev2206171421.tar.gz + # orig: https://www.riverbankcomputing.com/static/Downloads/QScintilla/2.13.4/QScintilla_src-2.13.4.tar.gz + BEEHIVE=https://beehive.molgen.mpg.de/4f83a4a4ad7da40eae80ad23f9fb18f2/QScintilla_src-2.13.4.tar.gz test -e QScintilla_src-$QSCINTILLA_VERSION.tar.gz || wget -nv $BEEHIVE test -d QScintilla_src-$QSCINTILLA_VERSION || tar -xf QScintilla_src-$QSCINTILLA_VERSION.tar.gz @@ -308,7 +311,8 @@ piplist; track legacy-libxml2 cd numba-$NUMBA_VERSION # numba assumes that libcudart.so and friends reside in 'CUDA_HOME/lib64' - sed -ne '/^#Epatch:numba/ s/^#Epatch:\S* // p' $ME | patch -p1 --verbose + sed -ne '/^#Epatch:numba1/ s/^#Epatch:\S* // p' $ME | patch -p1 --verbose + sed -ne '/^#Epatch:numba2/ s/^#Epatch:\S* // p' $ME | patch -p1 --verbose install . # useful to see if numba works cp -p numba/misc/numba_sysinfo.py $PREFIX/bin @@ -316,25 +320,27 @@ piplist; track legacy-libxml2 ) piplist; track numba - -# 'distributed' doesn't like the 'new' tornado 6.2 -- reason: a deprecation warning makes the CI choke (geht's noch?) +# ------------------------------------------------ build tool fuer Thomas ( - DISTRIBUTED_VERSION=2022.9.0 - - # orig: https://files.pythonhosted.org/packages/df/46/e8ef41f8dfd553327b6a12f4ab78d9351b54299596fe5b8b165ba40b1550/distributed-2022.9.0.tar.gz - BEEHIVE=https://beehive.molgen.mpg.de/335d718f1ba331ee8e85e40e564559a1/distributed-2022.9.0.tar.gz - test -e distributed-$DISTRIBUTED_VERSION.tar.gz || wget -nv $BEEHIVE - test -d distributed-$DISTRIBUTED_VERSION || tar -xf distributed-$DISTRIBUTED_VERSION.tar.gz - cd distributed-$DISTRIBUTED_VERSION - sed -i -e '/tornado/ s/.*/tornado/' requirements.txt - sed -i -e '/tornado/ s/.*/tornado/' distributed.egg-info/requires.txt - install . + GYP_VERSION=20230301_c6d8b9f7 + # orig: https://mirror.netcologne.de/slackware/slackware64-current/source/d/gyp/ + BEEHIVE=https://beehive.molgen.mpg.de/ba0357be23ca55555714261afccc7aab/gyp-20230301_c6d8b9f7.tar.xz + test -e gyp-${GYP_VERSION}.tar.xz || wget -nv $BEEHIVE + test -d gyp-${GYP_VERSION} || tar -xf gyp-${GYP_VERSION}.tar.xz + cd gyp-${GYP_VERSION} + sed -ne '/^#Epatch:gyp1/ s/^#Epatch:\S* // p' $ME | patch -p1 --verbose + sed -ne '/^#Epatch:gyp2/ s/^#Epatch:\S* // p' $ME | patch -p1 --verbose + python3 setup.py build + python3 setup.py install --prefix $PREFIX ) -piplist; track distributed and tornado +piplist; track gyp PACKAGES=$(sed -e s/#.*$// <<- __PKGLIST__ protobuf==3.19.4 # tensorflow likes to have an slightly older one [< 3.20], no conflicts with this - Matplotlib==3.6.0 # 3.6.1 is the first bugfix release of the 3.6.x series... Well, it is buggy + onnx==1.12.0 # newer onnx (dep helen) needs newer protobuf + Matplotlib==3.6.3 # 3.6.3 last bugfix for 4.6.x... Well, it is buggy, hopefullt not :) + keras==2.11.0 # tensorflow 2.11.1 requires keras<2.12,>=2.11.0 + Sphinx==4.5.0 # hic2cool(0.8.3) moecht gern was aelteres ipython[all] SciPy @@ -357,7 +363,6 @@ PACKAGES=$(sed -e s/#.*$// <<- __PKGLIST__ pygraphviz deeptools HTSeq - keras keras_preprocessing # tensorflow ... MotifScan # Alena 30.8.2018 MAmotif @@ -417,6 +422,7 @@ PACKAGES=$(sed -e s/#.*$// <<- __PKGLIST__ CrossMap cooltools # cause trouble with numba/nupy versions -- Maria Valieva 17.7.2019 scanpy + asciidoc __PKGLIST__ ) @@ -429,7 +435,7 @@ install pyopengl # Tk binding creates an error, so remove it (Expects Tk Togl in rm -rf ${PREFIX}/lib/python$PYTHONVER/site-packages/OpenGL/Tk # torch bundled with a recent CUDA-Toolkit -install torch torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/cu116 +install torch torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/cu118 # install helen after our torch, otherwise we might catch the default torch from pip ... install helen # HD, a dependency from a request by helene @@ -508,31 +514,34 @@ mkdir -vp $PREFIX/share/jupyter/lab/settings # try to defeat a silly error in j ) ( + IDR_VERSION=2.0.3 # orig: https://github.com/nboley/idr/archive/2.0.3/idr-2.0.3.tar.gz # HD: Tobias Zehnder, 18.3.21 BEEHIVE=https://beehive.molgen.mpg.de/1bec61b526bb1dcf92a5ab99c27852d0/idr-2.0.3.tar.gz - test -e idr-2.0.3.tar.gz || wget -nv $BEEHIVE - test -d idr-2.0.3 || tar -xf idr-2.0.3.tar.gz - cd idr-2.0.3 + test -e idr-${IDR_VERSION}.tar.gz || wget -nv $BEEHIVE + test -d idr-${IDR_VERSION} || tar -xf idr-${IDR_VERSION}.tar.gz + cd idr-${IDR_VERSION} python3 setup.py build python3 setup.py install --prefix $PREFIX ) ( + STGIT_VERSSION=1.0 # orig: https://github.com/stacked-git/stgit/archive/v1.0/stgit-1.0.tar.gz # Paul mariux64/pkg-scripts/issues/152 - BEEHIVE=https://beehive.molgen.mpg.de/b20ab5a197fc3a141f4b00c5859c0a3b/stgit-1.0.tar.gz - test -e stgit-1.0.tar.gz || wget -nv $BEEHIVE - test -d stgit-1.0 || tar -xf stgit-1.0.tar.gz - cd stgit-1.0 + BEEHIVE=https://beehive.molgen.mpg.de/b20ab5a197fc3a141f4b00c5859c0a3b/stgit-1.0.tar.gz + test -e stgit-${STGIT_VERSSION}.tar.gz || wget -nv $BEEHIVE + test -d stgit-${STGIT_VERSSION} || tar -xf stgit-${STGIT_VERSSION}.tar.gz + cd stgit-${STGIT_VERSSION} python3 setup.py build python3 setup.py install --prefix $PREFIX ) ( + METH_VERSION=0.2.5 # https://github.com/brentp/bwa-meth/archive/refs/tags/v0.2.5.tar.gz BEEHIVE=https://beehive.molgen.mpg.de/33a4c416760dd67e77f17e07791527e2/bwa-meth-0.2.5.tar.gz - test -e bwa-meth-0.2.5.tar.gz || wget -nv $BEEHIVE - test -d bwa-meth-0.2.5 || tar -xf bwa-meth-0.2.5.tar.gz - cd bwa-meth-0.2.5 + test -e bwa-meth-${METH_VERSION}.tar.gz || wget -nv $BEEHIVE + test -d bwa-meth-${METH_VERSION} || tar -xf bwa-meth-${METH_VERSION}.tar.gz + cd bwa-meth-${METH_VERSION} python3 setup.py build python3 setup.py install --prefix $PREFIX ) @@ -545,13 +554,13 @@ install wheel # to build our tf-wheel (i.e. zip archive) ( test -n "$SKIPTENSORFLOW" && exit - TFVERSION=2.10.0 + TFVERSION=2.11.1 NCCLSHORTVERSION=$(pkg-config nccl --modversion | grep -Po '\d+\.\d+') . /pkg/bazel-5.1.0-0/profile - # orig: https://github.com/tensorflow/tensorflow/archive/refs/tags/v2.10.0.tar.gz - BEEHIVE=https://beehive.molgen.mpg.de/7ceb5277d482f10d2df8644deb412061/tensorflow-2.10.0.tar.gz + # orig: https://github.com/tensorflow/tensorflow/archive/refs/tags/v2.11.1.tar.gz + BEEHIVE=https://beehive.molgen.mpg.de/e1b60c6e074f8b4c83f184e2c298774f/tensorflow-2.11.1.tar.gz test -e tensorflow-${TFVERSION}.tar.gz || wget -nv $BEEHIVE test -d tensorflow-${TFVERSION} || tar -xf tensorflow-${TFVERSION}.tar.gz @@ -594,7 +603,7 @@ install wheel # to build our tf-wheel (i.e. zip archive) export TF_CUDNN_VERSION=$(sed -n 's/^#define CUDNN_MAJOR\s*\(.*\).*/\1/p' ${CUDADIR}/include/cudnn_version.h) # hardware older than sm_52 is said to be unsupported, compute_86 denotes an emulation export TF_CUDA_COMPUTE_CAPABILITIES=sm_52,sm_53,sm_60,sm_61,sm_62,sm_70,sm_72,sm_75,sm_80,sm_86,compute_86 - + # stay compatible with the cluster machines export CC_OPT_FLAGS="-march=nehalem" export TF_NEED_CUDA=1 @@ -656,8 +665,11 @@ echo "# check for unreadable files" cd $PREFIX find \! -perm -004 -exec chmod -c a+r {} + -exit +echo "For Deeper checks:" +echo "cd ${BUILDDIR} ; git log --patch " + +exit # -------------------------------------------------------- Included patches #Epatch:pip # Operation 'Forever young', disable version-check per default. @@ -675,64 +687,80 @@ exit #Epatch:pip + help="Ignore this option, the version check IS disabled.", #Epatch:pip ) -# https://riverbankcomputing.com/hg/sip/raw-rev/1430b279a3c9 -> sip-fix-exceptions.patch -#Epatch:sip1 Fixed the handling of exceptions that sub-class C++ exceptions. -#Epatch:sip1 diff -r 6eff81a15822 -r 1430b279a3c9 sipbuild/generator/parser/rules.py -#Epatch:sip1 --- a/sipbuild/generator/parser/rules.py Fri Jun 17 14:25:16 2022 +0100 -#Epatch:sip1 +++ b/sipbuild/generator/parser/rules.py Sun Jun 26 10:01:41 2022 +0100 -#Epatch:sip1 @@ -2293,6 +2293,7 @@ -#Epatch:sip1 -#Epatch:sip1 if len(p) == 4: -#Epatch:sip1 base = p[2] -#Epatch:sip1 + base.make_absolute() -#Epatch:sip1 -#Epatch:sip1 # See if it is a project-defined exception. -#Epatch:sip1 for xd in p.parser.pm.spec.exceptions: - -# https://riverbankcomputing.com/hg/sip/raw-rev/323d39a2d602 -> sip-fix-template-values.patch -#Epatch:sip2 Fixed the instantiation of template values. -#Epatch:sip2 diff -r 1430b279a3c9 -r 323d39a2d602 sipbuild/generator/parser/instantiations.py -#Epatch:sip2 --- a/sipbuild/generator/parser/instantiations.py Sun Jun 26 10:01:41 2022 +0100 -#Epatch:sip2 +++ b/sipbuild/generator/parser/instantiations.py Sun Jul 17 09:41:16 2022 +0100 -#Epatch:sip2 @@ -434,7 +434,8 @@ -#Epatch:sip2 proto_name = proto_value.value.result.definition -#Epatch:sip2 -#Epatch:sip2 if proto_name.is_simple: -#Epatch:sip2 - i_name = ScopedName.parse(template_string(proto_name, expansions)) -#Epatch:sip2 + i_name = ScopedName.parse( -#Epatch:sip2 + template_string(proto_name.base_name, expansions)) -#Epatch:sip2 i_result = Argument(type=ArgumentType.DEFINED, definition=i_name) -#Epatch:sip2 i_fcall = FunctionCall(result=i_result, -#Epatch:sip2 args=proto_value.value.args) - -#Epatch:numba # Fix lib location, turn numba_sysinfo.py into a common diag tool -#Epatch:numba --- a/numba/cuda/cuda_paths.py 2022-10-14 09:57:25.616827200 +0200 -#Epatch:numba +++ b/numba/cuda/cuda_paths.py 2022-10-20 11:20:15.106019523 +0200 -#Epatch:numba @@ -66,3 +66,3 @@ -#Epatch:numba else: -#Epatch:numba - return 'lib64' -#Epatch:numba + return 'lib' -#Epatch:numba -#Epatch:numba --- a/numba/misc/numba_sysinfo.py 2022-10-14 09:57:25.628827000 +0200 -#Epatch:numba +++ b/numba/misc/numba_sysinfo.py 2022-10-20 11:02:54.056767540 +0200 -#Epatch:numba @@ -1 +1,2 @@ -#Epatch:numba +#! /usr/bin/env python3 -#Epatch:numba import json -#Epatch:numba @@ -367,3 +368,3 @@ -#Epatch:numba with redirect_stdout(output): -#Epatch:numba - cudadrv.libs.test(sys.platform, print_paths=False) -#Epatch:numba + cudadrv.libs.test(sys.platform, print_paths=True) -#Epatch:numba sys_info[_cu_lib_test] = output.getvalue() -#Epatch:numba @@ -626,9 +627,2 @@ -#Epatch:numba ("",), -#Epatch:numba - ("__Conda Information__",), -#Epatch:numba - (DisplayMap({k: v for k, v in info.items() -#Epatch:numba - if k.startswith('Conda')}) or ("Conda not available.",)), -#Epatch:numba - ("",), -#Epatch:numba - ("__Installed Packages__",), -#Epatch:numba - DisplaySeq(info.get(_inst_pkg, ("Couldn't retrieve packages info.",))), -#Epatch:numba - ("",), -#Epatch:numba ("__Error log__" if info.get(_errors, []) +#Epatch:numba1 # Fix lib location, turn numba_sysinfo.py into a common diag tool +#Epatch:numba1 --- a/numba/cuda/cuda_paths.py 2022-10-14 09:57:25.616827200 +0200 +#Epatch:numba1 +++ b/numba/cuda/cuda_paths.py 2022-10-20 11:20:15.106019523 +0200 +#Epatch:numba1 @@ -68,3 +68,3 @@ +#Epatch:numba1 else: +#Epatch:numba1 - return 'lib64' +#Epatch:numba1 + return 'lib' + +#Epatch:numba2 --- a/numba/misc/numba_sysinfo.py 2022-10-14 09:57:25.628827000 +0200 +#Epatch:numba2 +++ b/numba/misc/numba_sysinfo.py 2022-10-20 11:02:54.056767540 +0200 +#Epatch:numba2 @@ -1 +1,2 @@ +#Epatch:numba2 +#! /usr/bin/env python3 +#Epatch:numba2 import json +#Epatch:numba2 @@ -367,3 +368,3 @@ +#Epatch:numba2 with redirect_stdout(output): +#Epatch:numba2 - cudadrv.libs.test(sys.platform, print_paths=False) +#Epatch:numba2 + cudadrv.libs.test(sys.platform, print_paths=True) +#Epatch:numba2 sys_info[_cu_lib_test] = output.getvalue() +#Epatch:numba2 @@ -626,9 +627,2 @@ +#Epatch:numba2 ("",), +#Epatch:numba2 - ("__Conda Information__",), +#Epatch:numba2 - (DisplayMap({k: v for k, v in info.items() +#Epatch:numba2 - if k.startswith('Conda')}) or ("Conda not available.",)), +#Epatch:numba2 - ("",), +#Epatch:numba2 - ("__Installed Packages__",), +#Epatch:numba2 - DisplaySeq(info.get(_inst_pkg, ("Couldn't retrieve packages info.",))), +#Epatch:numba2 - ("",), +#Epatch:numba2 ("__Error log__" if info.get(_errors, []) + +#Epatch:gyp1 --- a/pylib/gyp/generator/cmake.py +#Epatch:gyp1 +++ b/pylib/gyp/generator/cmake.py +#Epatch:gyp1 @@ -40,9 +40,9 @@ +#Epatch:gyp1 +#Epatch:gyp1 try: +#Epatch:gyp1 # maketrans moved to str in python3. +#Epatch:gyp1 + _maketrans = str.maketrans +#Epatch:gyp1 +except NameError: +#Epatch:gyp1 _maketrans = string.maketrans +#Epatch:gyp1 -except NameError: +#Epatch:gyp1 - _maketrans = str.maketrans +#Epatch:gyp1 +#Epatch:gyp1 generator_default_variables = { +#Epatch:gyp1 'EXECUTABLE_PREFIX': '', +#Epatch:gyp1 @@ -281,7 +281,7 @@ +#Epatch:gyp1 dirs = set(dir for dir in (os.path.dirname(o) for o in outputs) if dir) +#Epatch:gyp1 +#Epatch:gyp1 if int(action.get('process_outputs_as_sources', False)): +#Epatch:gyp1 - extra_sources.extend(zip(cmake_outputs, outputs)) +#Epatch:gyp1 + extra_sources.extend(list(zip(cmake_outputs, outputs))) +#Epatch:gyp1 +#Epatch:gyp1 # add_custom_command +#Epatch:gyp1 output.write('add_custom_command(OUTPUT ') +#Epatch:gyp1 @@ -987,7 +987,7 @@ +#Epatch:gyp1 +#Epatch:gyp1 # XCode settings +#Epatch:gyp1 xcode_settings = config.get('xcode_settings', {}) +#Epatch:gyp1 - for xcode_setting, xcode_value in xcode_settings.viewitems(): +#Epatch:gyp1 + for xcode_setting, xcode_value in xcode_settings.items(): +#Epatch:gyp1 SetTargetProperty(output, cmake_target_name, +#Epatch:gyp1 "XCODE_ATTRIBUTE_%s" % xcode_setting, xcode_value, +#Epatch:gyp1 '' if isinstance(xcode_value, str) else ' ') + +#Epatch:gyp2 diff --git a/pylib/gyp/input.py b/pylib/gyp/input.py +#Epatch:gyp2 index 4c12891..2bea334 100644 +#Epatch:gyp2 --- a/pylib/gyp/input.py +#Epatch:gyp2 +++ b/pylib/gyp/input.py +#Epatch:gyp2 @@ -1183,7 +1183,7 @@ +#Epatch:gyp2 if variable_name in variables: +#Epatch:gyp2 # If the variable is already set, don't set it. +#Epatch:gyp2 continue +#Epatch:gyp2 - if the_dict_key is 'variables' and variable_name in the_dict: +#Epatch:gyp2 + if the_dict_key == 'variables' and variable_name in the_dict: +#Epatch:gyp2 # If the variable is set without a % in the_dict, and the_dict is a +#Epatch:gyp2 # variables dict (making |variables| a varaibles sub-dict of a +#Epatch:gyp2 # variables dict), use the_dict's definition. # -------------------------------------------------------- This is the end. -