diff --git a/python-3.10.12-0.build.sh b/python-3.10.12-0.build.sh new file mode 100755 index 0000000..3cf8e33 --- /dev/null +++ b/python-3.10.12-0.build.sh @@ -0,0 +1,834 @@ +#!/bin/bash + +# ------------------------------------------------------------------- Debug + + COOKIE=$(mcookie|cut -c-8); grep -v V_GREP_ME $0 > /dev/shm/runme-$COOKIE.sh ; sleep 0.3; exec bash /dev/shm/runme-$COOKIE.sh + #TESTING=1 + #SKIPTENSORFLOW=1 + +# ---------------------------------------------------------------- Preamble + +#set -x + +PKG=python +VERSION=3.10.12 +BUILD=0 +PYTHONVER=${VERSION%.*} + +CUDADIR=/pkg/cuda-11.8.0-0 + +PREFIX=/pkg/$PKG-$VERSION-$BUILD +if [ -n "$TESTING" ]; then PREFIX=/scratch/local2/$PKG-$VERSION-$BUILD ; fi + +PATH=/bin:/usr/bin:/usr/sbin:/usr/local/bin + +# set -xe +set -e +umask 022 + +BUILD_TMPDIR=/scratch/local2/$PKG-$VERSION-$BUILD.$USER.build.tmp + +test -d $BUILD_TMPDIR && ( chmod -R u+rwx $BUILD_TMPDIR || true ; rm -rf $BUILD_TMPDIR ) +mkdir -p $BUILD_TMPDIR/home/.cache/pip +mkdir -p $BUILD_TMPDIR/home/.local +# copy script to an accessible location +cp $0 $BUILD_TMPDIR; ME="$BUILD_TMPDIR/$(basename $0)" +export TMPDIR=$BUILD_TMPDIR +export HOME=$BUILD_TMPDIR/home + +exec $PREFIX/profile <<-EOF + . $CUDADIR/profile + PATH=$PREFIX/bin:\$PATH + export LD_LIBRARY_PATH=$PREFIX/lib\${LD_LIBRARY_PATH:+:}\$LD_LIBRARY_PATH + PKG_CONFIG_PATH=$PREFIX/lib/pkgconfig\${PKG_CONFIG_PATH:+:}\$PKG_CONFIG_PATH + export PKG_CONFIG_PATH + if [ -d $PREFIX/.compatlibs ]; then export LD_LIBRARY_PATH=$PREFIX/.compatlibs\${LD_LIBRARY_PATH:+:}\$LD_LIBRARY_PATH ; fi +EOF +. $PREFIX/profile + +NPROC=$(( $(nproc) * 4 / 5 + 1 )) +export MAKEFLAGS="-j $NPROC" + +BUILDDIR=$PREFIX/build + +mkdir -p $BUILDDIR +cd $BUILDDIR + +# -------------------------------------- Use git to track package evolution + +test -d .git && rm -rf .git +git init -q; echo -e '[user]\n name = none\n email = of_your_business...' >> .git/config +echo '*' > .gitignore + +function piplist() { + pip3 list | awk '{ printf("%-36s %s\n", $1,$2) }' | grep -v '\----' > GameOfVersions +} + +function track() { + if [ -d .git ]; then + if git add -f GameOfVersions; then + git commit -q -n -m "Package: '$*'" || /bin/true + fi + fi +} + +function install() { + # no-color is nice, but doesn't work when dependencies are installed, so (mis-)use a pipe + pip3 install --no-color --compile --cache-dir=$HOME/.cache/pip --prefix=$PREFIX "${@+$@}" | cat + piplist + track $* +} + +# ---------------------------------------- Build Python & support libraries + +( + # orig: https://www.python.org/ftp/python/3.10.10/Python-3.10.10.tar.xz + BEEHIVE="https://beehive.molgen.mpg.de/49b0342476b984e106d308c25d657f12/Python-3.10.12.tar.xz" + + test -e Python-$VERSION.tar.xz || wget -nv $BEEHIVE + test -d Python-$VERSION || tar -xf Python-$VERSION.tar.xz + cd Python-$VERSION + + # BerkeleyDB, the undead ... + # Newer libgdbm_compat.so versions pull libgdbm.so on their own, our libgdbm_compat.so is older ... + # So, let setup.py create '-lgdbm -lgdbm_compat' instead of a sole '-lgdbm_compat' + # verbatim: ndbm_libs = ['gdbm_compat'] => ndbm_libs = ['gdbm','gdbm_compat'] + sed -i -e "/ndbm_libs = / s/_compat'/','gdbm_compat'/" setup.py + + # leave further hints in the log + sed -i -e '/db_setup_debug = False/ s/False/True/' \ + -e '/dbm_setup_debug = False/ s/False/True/' setup.py + + # These may come handy if the Python build needs exclusive stuff from $PREFIX + # INCLUDEDIR=$PREFIX/include \ + # LIBDIR=$PREFIX/lib \ + LDFLAGS="-Wl,-rpath=$PREFIX/lib" \ + ./configure \ + --prefix=$PREFIX \ + --enable-shared + + make -j $NPROC + make install + + # it might come to happen that something down the line may just call 'python', defuse ... + test -e $PREFIX/bin/python || ln -s python3 $PREFIX/bin/python + + python3 -m ensurepip + pip3 install --prefix=$PREFIX -I pip + ( # fix 'please update' noise from pip + cd $PREFIX/lib/python*/site-packages + sed -ne '/^#Epatch:pip/ s/^#Epatch:\S* // p' $ME | patch -p1 --verbose + ) +) +install setuptools==65.3.0 # see numba below +piplist; track START + + +# build various pythonbindings 'by hand', because pip installs do not deliver +# header files and the like. +# BTW: builds need meson + +install meson + +( + DBUS_PYTHON_VERSION=1.3.2 + + #PyPi + # https://files.pythonhosted.org/packages/c1/d3/6be85a9c772d6ebba0cc3ab37390dd6620006dcced758667e0217fb13307/dbus-python-1.3.2.tar.gz + BEEHIVE=https://beehive.molgen.mpg.de/33be8a4a766e1c7c9a377b8f934ce21a/dbus-python-1.3.2.tar.gz + test -e dbus-python-$DBUS_PYTHON_VERSION.tar.gz || wget -nv $BEEHIVE + test -d dbus-python-$DBUS_PYTHON_VERSION || tar -xf dbus-python-$DBUS_PYTHON_VERSION.tar.gz + + cd dbus-python-$DBUS_PYTHON_VERSION + ./configure --prefix=$PREFIX + + make -j $NPROC + make install + +) + +# ------------------------------------------------ build tool fuer Thomas +( + set -x + GYP_VERSION=20230301_c6d8b9f7 + # orig: https://mirror.netcologne.de/slackware/slackware64-current/source/d/gyp/ + BEEHIVE=https://beehive.molgen.mpg.de/ba0357be23ca55555714261afccc7aab/gyp-20230301_c6d8b9f7.tar.xz + test -e gyp-${GYP_VERSION}.tar.xz || wget -nv $BEEHIVE + test -d gyp-${GYP_VERSION} || tar -xf gyp-${GYP_VERSION}.tar.xz + cd gyp-${GYP_VERSION} + sed -ne '/^#Epatch:gyp1/ s/^#Epatch:\S* // p' $ME | patch -p1 --verbose + sed -ne '/^#Epatch:gyp2/ s/^#Epatch:\S* // p' $ME | patch -p1 --verbose + python3 setup.py build + python3 setup.py install --prefix $PREFIX +) +piplist; track gyp + +( + PYCAIRO_VERSION=1.23.0 + + #PyPi + # https://files.pythonhosted.org/packages/69/ca/9e9fa2e8be0876a9bbf046a1be7ee33e61d4fdfbd1fd25c76c1bdfddf8c4/pycairo-1.23.0.tar.gz + BEEHIVE=https://beehive.molgen.mpg.de/7a3729d21659098e1b9a411b62e88966/pycairo-1.23.0.tar.gz + test -e pycairo-$PYCAIRO_VERSION.tar.gz || wget -nv $BEEHIVE + test -d pycairo-$PYCAIRO_VERSION || tar -xf pycairo-$PYCAIRO_VERSION.tar.gz + + cd pycairo-$PYCAIRO_VERSION + + mkdir -p meson-build + cd meson-build + + meson setup \ + --prefix=$PREFIX \ + --buildtype=release \ + -Dpython=$PREFIX/bin/python \ + -Dtests=false \ + .. + + ninja -j $NPROC + ninja install + +) + +( + PYGOBJECT3_VERSION=3.44.1 + + # https://files.pythonhosted.org/packages/07/73/a034fc1bcd6d402d6f72ff618c093f91ac6921f968cc19806b6a1b1b19c8/PyGObject-3.44.0.tar.gz + BEEHIVE=https://beehive.molgen.mpg.de/959259478566b24316edaa18da6e0eaa/PyGObject-3.44.1.tar.gz + test -e PyGObject-$PYGOBJECT3_VERSION.tar.gz || wget -nv $BEEHIVE + test -d PyGObject-$PYGOBJECT3_VERSION || tar -xf PyGObject-$PYGOBJECT3_VERSION.tar.gz + + cd PyGObject-$PYGOBJECT3_VERSION + + mkdir -p meson-build + cd meson-build + meson setup \ + --prefix=$PREFIX \ + --buildtype=release \ + -Dpython=$PREFIX/bin/python \ + -Dtests=false \ + .. + + ninja -j $NPROC + ninja install + +) +piplist; track various pythonbindings + +# build support to use QT from python -- see www.riverbankcomputing.com + +( # sip: tool to create Python bindings for C and C++ libraries. + SIP_VERSION=6.7.7 + + # https://files.pythonhosted.org/packages/5b/cb/c27c925ae07bd03a2597fa1db17bfc2a4ac57da61aeb90f8ec98ffbb975b/sip-6.6.2.tar.gz + BEEHIVE=https://beehive.molgen.mpg.de/a131efd79649d31ee11601b19d1e3784/sip-6.7.7.tar.gz + test -e sip-$SIP_VERSION.tar.gz || wget -nv $BEEHIVE + test -d sip-$SIP_VERSION || tar -xf sip-$SIP_VERSION.tar.gz + + cd sip-$SIP_VERSION + + python3 setup.py build + python3 setup.py install --skip-build --optimize=1 + +) + +# QT bindings need some python modules to build + +install PyQt5-sip +install PyQt-builder + +( + PYQT_VERSION=5.15.9 + + #PyPi + # orig: https://files.pythonhosted.org/packages/5c/46/b4b6eae1e24d9432905ef1d4e7c28b6610e28252527cdc38f2a75997d8b5/PyQt5-5.15.9.tar.gz + BEEHIVE=https://beehive.molgen.mpg.de/9d97fc06b7ae75e654e946c49e07ff12/PyQt5-5.15.9.tar.gz + test -e PyQt5-$PYQT_VERSION.tar.gz || wget -nv $BEEHIVE + test -d PyQt5-$PYQT_VERSION || tar -xf PyQt5-$PYQT_VERSION.tar.gz + + cd PyQt5-$PYQT_VERSION + + sip-build \ + --confirm-license \ + --no-make \ + --api-dir $PREFIX/share/qsci/api/python \ + --dbus $PREFIX/include/dbus-1.0 \ + --no-designer-plugin \ + --no-qml-plugin \ + --jobs $NPROC \ + + cd build + make -j $NPROC + make install + + # Remove unused py2 version of uic modules + rm -r $PREFIX/lib/python*/site-packages/PyQt5/uic/port_v2 + +) + +( + QSCINTILLA_VERSION=2.13.4 # no 2.14 in sight ... + + # orig: https://www.riverbankcomputing.com/static/Downloads/QScintilla/2.13.4/QScintilla_src-2.13.4.tar.gz + BEEHIVE=https://beehive.molgen.mpg.de/4f83a4a4ad7da40eae80ad23f9fb18f2/QScintilla_src-2.13.4.tar.gz + test -e QScintilla_src-$QSCINTILLA_VERSION.tar.gz || wget -nv $BEEHIVE + test -d QScintilla_src-$QSCINTILLA_VERSION || tar -xf QScintilla_src-$QSCINTILLA_VERSION.tar.gz + + cd QScintilla_src-$QSCINTILLA_VERSION + cd Python + + cp -vp pyproject-qt5.toml pyproject.toml + sip-build \ + --no-make \ + --qsci-features-dir ../src/features \ + --qsci-include-dir ../src \ + --qsci-library-dir ../src \ + --api-dir $PREFIX/share/qsci/api/python \ + --jobs $NPROC + + cd build + make -j $NPROC + make install + +) + +( + set -x + PYQTWEBENGINE_VERSION=5.15.6 + + #orpg: https://pypi.org/project/PyQtWebEngine/#files + BEEHIVE="https://beehive.molgen.mpg.de/d1e548617de149ba8142a32c1eb9f994/PyQtWebEngine-5.15.6.tar.gz" + test -e PyQtWebEngine-${PYQTWEBENGINE_VERSION}.tar.gz || wget -nv $BEEHIVE + test -d PyQtWebEngine-${PYQTWEBENGINE_VERSION} || tar -xf PyQtWebEngine-${PYQTWEBENGINE_VERSION}.tar.gz + + cd PyQtWebEngine-${PYQTWEBENGINE_VERSION} + sip-build \ + --no-make \ + --api-dir ${PREFIX}/share/qsci/api/python \ + --jobs $NPROC + + cd build + make -j $NPROC + make install +) +piplist; track qt-support-riverbankcomputing + +# -------------------------------------------------------- Install packages + +# Install the psychotic diva first +install numpy==1.23.4 + +install Cython~=0.29 #last stable release, version 3 ganz frisch unlogische versionierung + +# libxml2-python from pypi isn't up to date, cause lxml seems to be the favored binding +# do this early, not that some dependency tries it on its own. +# (btw. libxslt-python isn't ported to python3) +( + LIBXML2_VERSION=2.9.12 + + # orig: http://xmlsoft.org/sources/libxml2-${PKGVERSION}.tar.gz + BEEHIVE=https://beehive.molgen.mpg.de/f433a39be087a9f0b197eb2307ad9f75/libxml2-2.9.12.tar.gz + test -e libxml2-$LIBXML2_VERSION.tar.gz || wget -nv $BEEHIVE + test -d libxml2-$LIBXML2_VERSION || tar -xf libxml2-$LIBXML2_VERSION.tar.gz libxml2-$LIBXML2_VERSION/python + cd libxml2-$LIBXML2_VERSION/python + python3 setup.py build + python3 setup.py install +) +piplist; track legacy-libxml2 + + +# 'numba' the cuda enabled numpy +( + NUMBA_VERSION=0.56.3 + + # orig: https://files.pythonhosted.org/packages/a3/95/7df08647e773a2417e24de8fea9087606b37074f63830a3f2dbabd97f01b/numba-0.56.3.tar.gz + BEEHIVE=https://beehive.molgen.mpg.de/ae0a008447f2f246523af143b1488e88/numba-0.56.3.tar.gz + test -e numba-$NUMBA_VERSION.tar.gz || wget -nv $BEEHIVE + test -d numba-$NUMBA_VERSION || tar -xf numba-$NUMBA_VERSION.tar.gz + cd numba-$NUMBA_VERSION + + # numba assumes that libcudart.so and friends reside in 'CUDA_HOME/lib64' + sed -ne '/^#Epatch:numba1/ s/^#Epatch:\S* // p' $ME | patch -p1 --verbose + sed -ne '/^#Epatch:numba2/ s/^#Epatch:\S* // p' $ME | patch -p1 --verbose + install . + # useful to see if numba works + cp -p numba/misc/numba_sysinfo.py $PREFIX/bin + chmod 0755 $PREFIX/bin/numba_sysinfo.py +) +piplist; track numba + +PACKAGES=$(sed -e s/#.*$// <<- __PKGLIST__ + protobuf==3.19.4 # tensorflow likes to have an slightly older one [< 3.20], no conflicts with this + onnx==1.12.0 # newer onnx (dep helen) needs newer protobuf + Pillow<10.0.0 # pyimagej 1.4.1 downgrade to latest 9 + keras==2.11.0 # tensorflow 2.11.1 requires keras<2.12,>=2.11.0 + urllib3<2.0.0 # oauth brauch wird von tensorflow gebraucht + Sphinx==4.5.0 # hic2cool(0.8.3) moecht gern was aelteres + jedi<0.19.0 # spyder dwowngrade spyder 5.4.4 + black[d] #onnx want but not install it + flask + ipython[all] + + SciPy + pandas + SymPy + nose + statsmodels + pysam + seaborn + scikit-learn + Mako + pyimagej # Mikroscop + cutadapt # Helpdesk 01/24/18 16:19, Sabrina Krakau + pygobject # Helpdesk 03/26/18 14:19, Virginie Stanisla + blink1 # Blink1-lib 07/04/18 12:00, Niclas Hofmann + hvplot #ont/nextflow pipelines + pyarrow #ont/nextflow pipelines + umi-tools + recommonmark + fastcluster + pydot + pygraphviz + deeptools + HTSeq + keras_preprocessing # tensorflow ... + MotifScan # Alena 30.8.2018 + MAmotif + hic2cool # Helpdesk Robert Schoepflin 17.09.2018 + snakemake + virtualenv # Donald + odfpy # Donald + Dumper # Thomas + igraph + louvain # monocle3 prep for Helene + selenium # 19.7.19 Kreitler -> https://webxray.org/ + gmpy # From viper context, Helpdesk Laura Glaser 9.8.2019 + plotly + pyOpenCL + docopt + distro + ply + threadpoolctl + HiCMatrix + pybedtools + gffutils + python-bioformats + jupyter + jupyterlab + jupyterlab-git # request Peter Arndt + igv_jupyterlab # Matthias Lienhard HD 01-04-2020 + igv-notebook + scour # https://github.molgen.mpg.de/mariux64/pkg-scripts/issues/98 + multiqc + pybind11 + svgwrite + pyomo + bioservices + SharedArray + pyabc # dependency for stochkit, HD 20.07.20 Gemma Noviello + petab + tadtool # HD 26.01.21 Michael Robson + pyperf # Paul sep-20 + umap # HD, used by Matthias Lienhard + MACS3 + biopython # HD 2021-07-23 Magalhaes/Hnisz alphafold related -- cool that we have missed biopython so far, LOL + dm-tree + dm-haiku + flatbuffers + contextlib2 + ml_collections + spyder + spyder-kernels + + + # additions/fixups since 3.9 + papermill # turns jupyter notebooks into scripts [sic!] used by P.A. [who else, *grin*] + opencv-python + scikit-image + git-cola + cffi-glpk # uses /usr/lib/libglpk.so.40, see if I can get it linked statically - kreitler + czifile + napari[pyqt5] + napari-czifile2 + CrossMap + scanpy + asciidoc + + __PKGLIST__ +) + +for PKG in $PACKAGES; do + install $PKG +done + +( + #install from git to get .so + pip3 install git+https://github.com/open2c/pairtools.git@v1.0.2 +) +piplist; track 'pairtools' + +( + TRITON_VERSION=2.0.0 + #triton 2.0.0 hat cmake als install dep, ist in 2.1.0 gefixt + + # orig: https://github.com/openai/triton/archive/refs/tags/v2.0.0.tar.gz + BEEHIVE=https://beehive.molgen.mpg.de/56b46016609039fac839f8297c88454c/triton-2.0.0.tar.gz + test -e triton-${TRITON_VERSION}.tar.gz || wget -nv $BEEHIVE + test -d triton-${TRITON_VERSION} || tar -xf triton-${TRITON_VERSION}.tar.gz + cd triton-${TRITON_VERSION}/python + sed -ne '/^#Epatch:triton/ s/^#Epatch:\S* // p' $ME | patch -p1 --verbose + pip install . --prefix=$PREFIX + + piplist; track triton==${TRITON_VERSION} +) + +# torch bundled with a recent CUDA-Toolkit +install torch torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/cu118 + +install pyopengl # Tk binding creates an error, so remove it (Expects Tk Togl installation...) +rm -rf ${PREFIX}/lib/python$PYTHONVER/site-packages/OpenGL/Tk + +# install helen after our torch, otherwise we might catch the default torch from pip ... +install helen # HD, a dependency from a request by helene + +mkdir -vp $PREFIX/share/jupyter/lab/settings # try to defeat a silly error in jupyter labextension install + +# --------------------------------------------------- Install misc packages + +( + BOOST_VERSION=(1.80.0 1_80_0) + PKGCFLAGS="-O2 -fPIC" + + # orig: https://boostorg.jfrog.io/artifactory/main/release/1.80.0/source/boost_1_80_0.tar.gz + BEEHIVE=https://beehive.molgen.mpg.de/077f074743ea7b0cb49c6ed43953ae95/boost_1_80_0.tar.gz + test -e boost_${BOOST_VERSION[1]}.tar.gz || wget -nv $BEEHIVE + test -d boost-${BOOST_VERSION[0]} || mkdir boost-${BOOST_VERSION[0]} && tar -xf boost_${BOOST_VERSION[1]}.tar.gz -C boost-${BOOST_VERSION[0]} --strip-components=1 + + cd boost-${BOOST_VERSION[0]} + sed -e "/using python/ s|;|: ${PREFIX}/include/python${PYTHONVER} ;|" \ + -i bootstrap.sh + + ./bootstrap.sh \ + --with-toolset=gcc \ + --with-icu \ + --with-python=$PREFIX/bin/python3 \ + --with-libraries=python,thread || exit 1 + + ./b2 \ + -j $NPROC \ + --layout=system \ + --build-dir=build-python3 \ + --prefix=$PREFIX \ + --libdir=$PREFIX/lib \ + --build-type=minimal \ + toolset=gcc \ + variant=release \ + debug-symbols=off \ + link=shared \ + threading=multi \ + runtime-link=shared \ + python=$PYTHONVER \ + cflags="$PKGCFLAGS" \ + cxxflags="$PKGCFLAGS" \ + install || exit 1 +) + +( + PYCUDAVERS=2022.1 + PYTHON_LIBNAME=boost_python${PYTHONVER/./} + + # orig: https://files.pythonhosted.org/packages/2d/1f/48a3a5b2c715345e7af1e09361100bd98c3d72b4025371692ab233f523d3/pycuda-2022.1.tar.gz # maybe those very long URLs compensate for other (shorter) things ... + BEEHIVE="https://beehive.molgen.mpg.de/2fad5a15db4153d825822eba7cb25f25/pycuda-2022.1.tar.gz" + test -e pycuda-${PYCUDAVERS}.tar.gz || wget -nv $BEEHIVE + test -d pycuda-${PYCUDAVERS} || tar -xf pycuda-${PYCUDAVERS}.tar.gz + + cd pycuda-${PYCUDAVERS} + test -e siteconf.py && rm siteconf.py + python3 configure.py \ + --cuda-enable-gl \ + --cuda-root=$CUDADIR \ + --no-use-shipped-boost \ + --boost-python-libname=$PYTHON_LIBNAME \ + --boost-inc-dir=$PREFIX/include \ + --boost-lib-dir=$PREFIX/lib \ + --cudadrv-lib-dir='${CUDA_ROOT}/lib,${CUDA_ROOT}/lib/stubs' \ + --cudart-lib-dir='${CUDA_ROOT}/lib,${CUDA_ROOT}/lib/stubs' \ + --curand-lib-dir='${CUDA_ROOT}/lib,${CUDA_ROOT}/lib/stubs' \ + --ldflags=-L$PREFIX/lib + + python3 setup.py build + python3 setup.py install --prefix $PREFIX +) + +( + IDR_VERSION=2.0.3 + # orig: https://github.com/nboley/idr/archive/2.0.3/idr-2.0.3.tar.gz # HD: Tobias Zehnder, 18.3.21 + BEEHIVE=https://beehive.molgen.mpg.de/1bec61b526bb1dcf92a5ab99c27852d0/idr-2.0.3.tar.gz + test -e idr-${IDR_VERSION}.tar.gz || wget -nv $BEEHIVE + test -d idr-${IDR_VERSION} || tar -xf idr-${IDR_VERSION}.tar.gz + cd idr-${IDR_VERSION} + python3 setup.py build + python3 setup.py install --prefix $PREFIX +) + +( + STGIT_VERSSION=1.0 + # orig: https://github.com/stacked-git/stgit/archive/v1.0/stgit-1.0.tar.gz # Paul mariux64/pkg-scripts/issues/152 + BEEHIVE=https://beehive.molgen.mpg.de/b20ab5a197fc3a141f4b00c5859c0a3b/stgit-1.0.tar.gz + test -e stgit-${STGIT_VERSSION}.tar.gz || wget -nv $BEEHIVE + test -d stgit-${STGIT_VERSSION} || tar -xf stgit-${STGIT_VERSSION}.tar.gz + cd stgit-${STGIT_VERSSION} + python3 setup.py build + python3 setup.py install --prefix $PREFIX +) + +( + METH_VERSION=0.2.5 + # https://github.com/brentp/bwa-meth/archive/refs/tags/v0.2.5.tar.gz + BEEHIVE=https://beehive.molgen.mpg.de/33a4c416760dd67e77f17e07791527e2/bwa-meth-0.2.5.tar.gz + test -e bwa-meth-${METH_VERSION}.tar.gz || wget -nv $BEEHIVE + test -d bwa-meth-${METH_VERSION} || tar -xf bwa-meth-${METH_VERSION}.tar.gz + cd bwa-meth-${METH_VERSION} + python3 setup.py build + python3 setup.py install --prefix $PREFIX +) +piplist; track misc packages + +# ------------------------------------------------ finally build tensorflaw + +install wheel # to build our tf-wheel (i.e. zip archive) + +( + test -n "$SKIPTENSORFLOW" && exit + + TFVERSION=2.11.1 + NCCLSHORTVERSION=$(pkg-config nccl --modversion | grep -Po '\d+\.\d+') + + . /pkg/bazel-5.4.1-0/profile + + # orig: https://github.com/tensorflow/tensorflow/archive/refs/tags/v2.11.1.tar.gz + BEEHIVE=https://beehive.molgen.mpg.de/e1b60c6e074f8b4c83f184e2c298774f/tensorflow-2.11.1.tar.gz + + test -e tensorflow-${TFVERSION}.tar.gz || wget -nv $BEEHIVE + test -d tensorflow-${TFVERSION} || tar -xf tensorflow-${TFVERSION}.tar.gz + + cd tensorflow-${TFVERSION} + + #vars for config + export PYTHON_BIN_PATH=${PREFIX}/bin/python3 + export USE_DEFAULT_PYTHON_LIB_PATH=1 + export TF_NEED_JEMALLOC=1 + export TF_NEED_KAFKA=0 + export TF_NEED_OPENCL_SYCL=0 + export TF_NEED_AWS=0 + export TF_NEED_GCP=0 + export TF_NEED_HDFS=0 + export TF_NEED_S3=0 + export TF_ENABLE_XLA=1 + export TF_NEED_GDR=0 + export TF_NEED_VERBS=0 + export TF_NEED_OPENCL=0 + export TF_NEED_MPI=0 + export TF_NEED_TENSORRT=0 + export TF_NEED_NGRAPH=0 + export TF_NEED_IGNITE=0 + export TF_NEED_ROCM=0 + export TF_SET_ANDROID_WORKSPACE=0 + export TF_DOWNLOAD_CLANG=0 + export TF_NCCL_VERSION=${NCCLSHORTVERSION} + export TF_IGNORE_MAX_BAZEL_VERSION=1 + export NCCL_INSTALL_PATH=${CUDADIR} + export CC=gcc + export CXX=g++ + export GCC_HOST_COMPILER_PATH=/usr/bin/${CC} + export HOST_C_COMPILER=/usr/bin/${CC} + export HOST_CXX_COMPILER=/usr/bin/${CXX} + export TF_CUDA_CLANG=0 + export CLANG_CUDA_COMPILER_PATH=/usr/bin/clang + export TF_CUDA_PATHS=${CUDADIR},${CUDADIR}/extras/CUPTI,${CUDADIR}/nvvm + export TF_CUDA_VERSION=$(${CUDADIR}/bin/nvcc --version | sed -n 's/^.*release \(.*\),.*/\1/p') # 11.5 + export TF_CUDNN_VERSION=$(sed -n 's/^#define CUDNN_MAJOR\s*\(.*\).*/\1/p' ${CUDADIR}/include/cudnn_version.h) + # hardware older than sm_52 is said to be unsupported, compute_86 denotes an emulation + export TF_CUDA_COMPUTE_CAPABILITIES=sm_52,sm_53,sm_60,sm_61,sm_62,sm_70,sm_72,sm_75,sm_80,sm_86,compute_86 + + # stay compatible with the cluster machines + export CC_OPT_FLAGS="-march=nehalem" + export TF_NEED_CUDA=1 + export BAZEL_ARGS="--config=mkl -c opt" + + ./configure + + bazel build \ + --cxxopt="-march=nehalem" \ + --copt="-march=nehalem" \ + --noshow_loading_progress \ + --noshow_progress \ + --color="no" \ + //tensorflow:libtensorflow.so \ + //tensorflow:libtensorflow_cc.so \ + //tensorflow:install_headers \ + //tensorflow/tools/pip_package:build_pip_package + + bazel shutdown # otherwise it hogs in cwd + + # create a _clean/decent_ python package from so far created debris + test -d tensorflow-pip && rm -r tensorflow-pip + mkdir tensorflow-pip + ./bazel-bin/tensorflow/tools/pip_package/build_pip_package --src ${BUILDDIR}/tensorflow-pip + cd ${BUILDDIR} + # ... cope with the ??? symlink-structure placed in bazels' build-TMPDIR (-h) (tell me why ?) + test -e tensorflow-pip.tar && rm tensorflow-pip.tar + tar -chf tensorflow-pip.tar tensorflow-pip + + # and perform a regular install + test -d tensorflow-pip && rm -r tensorflow-pip + tar -xf tensorflow-pip.tar + cd ${BUILDDIR}/tensorflow-pip + + python3 setup.py build + python3 setup.py install --prefix $PREFIX +) +piplist; track tensorflow + +# tidy up a bit +if [ -d $PREFIX/man/man1 ]; then + mv -v $PREFIX/man/man1/* $PREFIX/share/man/man1 || true + rm -rf $PREFIX/man +fi + +# ------------------------------------------------------------ Sanity check + +# load all packages, see warnings and spot installation errors +# But don't care about r2py, a further dependence into R is not wanted! +echo "# START load test." +python3 -c 'help("modules")' > /dev/null +echo "# END load test (passed)." + +echo "# running pip check" +pip3 check + +# I guess it makes sense that files can be read by all users ... +echo "# check for unreadable files" +cd $PREFIX +find \! -perm -004 -exec chmod -c a+r {} + + +echo "For Deeper checks:" +echo "cd ${BUILDDIR} ; git log --patch " + + +exit +# -------------------------------------------------------- Included patches + +#Epatch:pip # Operation 'Forever young', disable version-check per default. +#Epatch:pip # The option needs to be kept, because some tools make use of +#Epatch:pip # it and would cause pip to fail. +#Epatch:pip --- a/pip/_internal/cli/cmdoptions.py 2022-08-09 17:29:15.853442948 +0200 +#Epatch:pip +++ b/pip/_internal/cli/cmdoptions.py 2022-08-10 10:31:35.124945154 +0200 +#Epatch:pip @@ -892,7 +892,6 @@ +#Epatch:pip dest="disable_pip_version_check", +#Epatch:pip action="store_true", +#Epatch:pip - default=False, +#Epatch:pip - help="Don't periodically check PyPI to determine whether a new version " +#Epatch:pip - "of pip is available for download. Implied with --no-index.", +#Epatch:pip + default=True, +#Epatch:pip + help="Ignore this option, the version check IS disabled.", +#Epatch:pip ) + +#Epatch:numba1 # Fix lib location, turn numba_sysinfo.py into a common diag tool +#Epatch:numba1 --- a/numba/cuda/cuda_paths.py 2022-10-14 09:57:25.616827200 +0200 +#Epatch:numba1 +++ b/numba/cuda/cuda_paths.py 2022-10-20 11:20:15.106019523 +0200 +#Epatch:numba1 @@ -68,3 +68,3 @@ +#Epatch:numba1 else: +#Epatch:numba1 - return 'lib64' +#Epatch:numba1 + return 'lib' + +#Epatch:numba2 --- a/numba/misc/numba_sysinfo.py 2022-10-14 09:57:25.628827000 +0200 +#Epatch:numba2 +++ b/numba/misc/numba_sysinfo.py 2022-10-20 11:02:54.056767540 +0200 +#Epatch:numba2 @@ -1 +1,2 @@ +#Epatch:numba2 +#! /usr/bin/env python3 +#Epatch:numba2 import json +#Epatch:numba2 @@ -367,3 +368,3 @@ +#Epatch:numba2 with redirect_stdout(output): +#Epatch:numba2 - cudadrv.libs.test(sys.platform, print_paths=False) +#Epatch:numba2 + cudadrv.libs.test(sys.platform, print_paths=True) +#Epatch:numba2 sys_info[_cu_lib_test] = output.getvalue() +#Epatch:numba2 @@ -626,9 +627,2 @@ +#Epatch:numba2 ("",), +#Epatch:numba2 - ("__Conda Information__",), +#Epatch:numba2 - (DisplayMap({k: v for k, v in info.items() +#Epatch:numba2 - if k.startswith('Conda')}) or ("Conda not available.",)), +#Epatch:numba2 - ("",), +#Epatch:numba2 - ("__Installed Packages__",), +#Epatch:numba2 - DisplaySeq(info.get(_inst_pkg, ("Couldn't retrieve packages info.",))), +#Epatch:numba2 - ("",), +#Epatch:numba2 ("__Error log__" if info.get(_errors, []) + +#Epatch:gyp1 --- a/pylib/gyp/generator/cmake.py +#Epatch:gyp1 +++ b/pylib/gyp/generator/cmake.py +#Epatch:gyp1 @@ -41,7 +41,7 @@ +#Epatch:gyp1 try: +#Epatch:gyp1 # maketrans moved to str in python3. +#Epatch:gyp1 + _maketrans = str.maketrans +#Epatch:gyp1 +except NameError: +#Epatch:gyp1 _maketrans = string.maketrans +#Epatch:gyp1 -except NameError: +#Epatch:gyp1 - _maketrans = str.maketrans +#Epatch:gyp1 generator_default_variables = { +#Epatch:gyp1 'EXECUTABLE_PREFIX': '', +#Epatch:gyp1 @@ -283,2 +283,2 @@ +#Epatch:gyp1 if int(action.get('process_outputs_as_sources', False)): +#Epatch:gyp1 - extra_sources.extend(zip(cmake_outputs, outputs)) +#Epatch:gyp1 + extra_sources.extend(list(zip(cmake_outputs, outputs))) +#Epatch:gyp1 @@ -988,6 +988,6 @@ +#Epatch:gyp1 # XCode settings +#Epatch:gyp1 xcode_settings = config.get('xcode_settings', {}) +#Epatch:gyp1 - for xcode_setting, xcode_value in xcode_settings.viewitems(): +#Epatch:gyp1 + for xcode_setting, xcode_value in xcode_settings.items(): +#Epatch:gyp1 SetTargetProperty(output, cmake_target_name, +#Epatch:gyp1 "XCODE_ATTRIBUTE_%s" % xcode_setting, xcode_value, +#Epatch:gyp1 '' if isinstance(xcode_value, str) else ' ') + +#Epatch:gyp2 diff --git a/pylib/gyp/input.py b/pylib/gyp/input.py +#Epatch:gyp2 index 4c12891..2bea334 100644 +#Epatch:gyp2 --- a/pylib/gyp/input.py +#Epatch:gyp2 +++ b/pylib/gyp/input.py +#Epatch:gyp2 @@ -1183,7 +1183,7 @@ +#Epatch:gyp2 if variable_name in variables: +#Epatch:gyp2 # If the variable is already set, don't set it. +#Epatch:gyp2 continue +#Epatch:gyp2 - if the_dict_key is 'variables' and variable_name in the_dict: +#Epatch:gyp2 + if the_dict_key == 'variables' and variable_name in the_dict: +#Epatch:gyp2 # If the variable is set without a % in the_dict, and the_dict is a +#Epatch:gyp2 # variables dict (making |variables| a varaibles sub-dict of a +#Epatch:gyp2 # variables dict), use the_dict's definition. + +#Epatch:triton diff --git a/setup.py b/setup.py +#Epatch:triton index 2ac3accd2..e7357be58 100644 +#Epatch:triton --- a/setup.py +#Epatch:triton +++ b/setup.py +#Epatch:triton @@ -211,10 +211,7 @@ setup( +#Epatch:triton long_description="", +#Epatch:triton packages=["triton", "triton/_C", "triton/language", "triton/tools", "triton/impl", "triton/ops", "triton/runtime", "triton/ops/blocksparse"], +#Epatch:triton install_requires=[ +#Epatch:triton - "cmake", +#Epatch:triton "filelock", +#Epatch:triton - "torch", +#Epatch:triton - "lit", +#Epatch:triton ], +#Epatch:triton package_data={"triton": ["third_party/**/*"]}, +#Epatch:triton include_package_data=True, +#Epatch:triton @@ -233,6 +231,10 @@ setup( +#Epatch:triton ], +#Epatch:triton test_suite="tests", +#Epatch:triton extras_require={ +#Epatch:triton + "build": [ +#Epatch:triton + "cmake>=3.18", +#Epatch:triton + "lit", +#Epatch:triton + ], +#Epatch:triton "tests": [ +#Epatch:triton "autopep8", +#Epatch:triton "flake8", + +# -------------------------------------------------------- This is the end.