diff --git a/python-3.11.7-0.build.sh b/python-3.11.7-0.build.sh new file mode 100755 index 0000000..f989e50 --- /dev/null +++ b/python-3.11.7-0.build.sh @@ -0,0 +1,863 @@ +#!/bin/bash + + +# ------------------------------------------------------------------- Debug + + COOKIE=$(mcookie|cut -c-8); grep -v V_GREP_ME $0 > /dev/shm/runme-$COOKIE.sh ; sleep 0.3; exec bash /dev/shm/runme-$COOKIE.sh + # TESTING=1 + # SKIPTENSORFLOW=1 + +# ---------------------------------------------------------------- Preamble + +PKG=python +VERSION=3.11.7 +BUILD=0 +PYTHONVER=${VERSION%.*} + +CUDADIR=/pkg/cuda-11.8.0-0 + +PREFIX=/pkg/$PKG-$VERSION-$BUILD +if [ -n "$TESTING" ]; then PREFIX=/scratch/local2/$PKG-$VERSION-$BUILD ; fi + +PATH=/bin:/usr/bin:/usr/sbin:/usr/local/bin + +set -e +umask 022 + +BUILD_TMPDIR=/scratch/local2/$PKG-$VERSION-$BUILD.$USER.build.tmp + +# keep pip's download cache when testing +test -z "$TESTING" && test -d $BUILD_TMPDIR && ( chmod -R u+rwx $BUILD_TMPDIR || true ; rm -rf $BUILD_TMPDIR ) +mkdir -p $BUILD_TMPDIR/home/.cache/pip +mkdir -p $BUILD_TMPDIR/home/.local +# copy script to an accessible location +cp $0 $BUILD_TMPDIR; ME="$BUILD_TMPDIR/$(basename $0)" +export TMPDIR=$BUILD_TMPDIR +export HOME=$BUILD_TMPDIR/home + +exec $PREFIX/profile <<-EOF + . $CUDADIR/profile + PATH=$PREFIX/bin:\$PATH + export LD_LIBRARY_PATH=$PREFIX/lib\${LD_LIBRARY_PATH:+:}\$LD_LIBRARY_PATH + PKG_CONFIG_PATH=$PREFIX/lib/pkgconfig\${PKG_CONFIG_PATH:+:}\$PKG_CONFIG_PATH + export PKG_CONFIG_PATH + if [ -d $PREFIX/.compatlibs ]; then export LD_LIBRARY_PATH=$PREFIX/.compatlibs\${LD_LIBRARY_PATH:+:}\$LD_LIBRARY_PATH ; fi +EOF +. $PREFIX/profile + +NPROC=$(( $(nproc) * 4 / 5 + 1 )) +export MAKEFLAGS="-j $NPROC" + +BUILDDIR=$PREFIX/build + +mkdir -p $BUILDDIR +cd $BUILDDIR + + +# -------------------------------------- Use git to track package evolution + +test -d .git && rm -rf .git +git init -q; echo -e '[user]\n name = none\n email = of_your_business...' >> .git/config +echo '*' > .gitignore + +function piplist() { + pip list | awk '{ printf("%-36s %s\n", $1,$2) }' | grep -v '\----' > GameOfVersions +} + +function track() { + if [ -d .git ]; then + if git add -f GameOfVersions; then + git commit -q -n -m "Package: '$*'" || /bin/true + fi + fi +} + +function install() { + echo "# ::INST:: # '${@+$@}'" + # no-color is nice, but doesn't work when dependencies are installed, so (mis-)use a pipe + pip install --no-color --compile --cache-dir=$HOME/.cache/pip --prefix=$PREFIX "${@+$@}" | cat + piplist + track $* +} + + +# ---------------------------------------- Build Python & support libraries + +( + + # orig: https://www.python.org/ftp/python/3.11.7/Python-3.11.7.tgz + BEEHIVE=https://beehive.molgen.mpg.de/ef61f81ec82c490484219c7f0ec96783/Python-3.11.7.tgz + test -e Python-$VERSION.tar.xz || wget -nv $BEEHIVE + test -d Python-$VERSION || tar -xf Python-$VERSION.tgz + cd Python-$VERSION + + # BerkeleyDB, the undead ... + # Newer libgdbm_compat.so versions pull libgdbm.so on their own, our libgdbm_compat.so is older ... + # So, let setup.py create '-lgdbm -lgdbm_compat' instead of a sole '-lgdbm_compat' + # verbatim: ndbm_libs = ['gdbm_compat'] => ndbm_libs = ['gdbm','gdbm_compat'] + sed -i -e "/ndbm_libs = / s/_compat'/','gdbm_compat'/" setup.py + + # leave further hints in the log + sed -i -e '/db_setup_debug = False/ s/False/True/' \ + -e '/dbm_setup_debug = False/ s/False/True/' setup.py + + # These may come handy if the Python build needs exclusive stuff from $PREFIX + # INCLUDEDIR=$PREFIX/include \ + # LIBDIR=$PREFIX/lib \ + LDFLAGS="-Wl,-rpath=$PREFIX/lib" \ + ./configure \ + --prefix=$PREFIX \ + --enable-shared + + make -j $NPROC + make install + + # it might come to happen that something down the line may just call 'python', defuse ... + test -e $PREFIX/bin/python || ln -s python3 $PREFIX/bin/python + + python -m ensurepip + pip3 install --prefix=$PREFIX -I pip + ( # fix 'please update' noise from pip + cd $PREFIX/lib/python*/site-packages + sed -ne '/^#Epatch:pip/ s/^#Epatch:\S* // p' $ME | patch -p1 --verbose + ) + # libblosc2 installs in $PREFIX/lib64, I hate to do this ... + ln -s lib $PREFIX/lib64 +) +install setuptools +piplist; track START + + +# build various pythonbindings 'by hand', because pip installs do not deliver +# header files and the like. +# BTW: builds need meson + +install meson + +( + DBUS_PYTHON_VERSION=1.3.2 + + # PyPi + # https://files.pythonhosted.org/packages/c1/d3/6be85a9c772d6ebba0cc3ab37390dd6620006dcced758667e0217fb13307/dbus-python-1.3.2.tar.gz + BEEHIVE=https://beehive.molgen.mpg.de/33be8a4a766e1c7c9a377b8f934ce21a/dbus-python-1.3.2.tar.gz + test -e dbus-python-$DBUS_PYTHON_VERSION.tar.gz || wget -nv $BEEHIVE + test -d dbus-python-$DBUS_PYTHON_VERSION || tar -xf dbus-python-$DBUS_PYTHON_VERSION.tar.gz + + cd dbus-python-$DBUS_PYTHON_VERSION + ./configure --prefix=$PREFIX + + make -j $NPROC + make install + +) + +( + PYCAIRO_VERSION=1.23.0 + + # PyPi + # https://files.pythonhosted.org/packages/69/ca/9e9fa2e8be0876a9bbf046a1be7ee33e61d4fdfbd1fd25c76c1bdfddf8c4/pycairo-1.23.0.tar.gz + BEEHIVE=https://beehive.molgen.mpg.de/7a3729d21659098e1b9a411b62e88966/pycairo-1.23.0.tar.gz + test -e pycairo-$PYCAIRO_VERSION.tar.gz || wget -nv $BEEHIVE + test -d pycairo-$PYCAIRO_VERSION || tar -xf pycairo-$PYCAIRO_VERSION.tar.gz + + cd pycairo-$PYCAIRO_VERSION + + mkdir -p meson-build + cd meson-build + + meson setup \ + --prefix=$PREFIX \ + --buildtype=release \ + -Dpython=$PREFIX/bin/python \ + -Dtests=false \ + .. + + ninja -j $NPROC + ninja install + +) + +( + PYGOBJECT3_VERSION=3.44.1 + + # https://files.pythonhosted.org/packages/07/73/a034fc1bcd6d402d6f72ff618c093f91ac6921f968cc19806b6a1b1b19c8/PyGObject-3.44.0.tar.gz + BEEHIVE=https://beehive.molgen.mpg.de/959259478566b24316edaa18da6e0eaa/PyGObject-3.44.1.tar.gz + test -e PyGObject-$PYGOBJECT3_VERSION.tar.gz || wget -nv $BEEHIVE + test -d PyGObject-$PYGOBJECT3_VERSION || tar -xf PyGObject-$PYGOBJECT3_VERSION.tar.gz + + cd PyGObject-$PYGOBJECT3_VERSION + + mkdir -p meson-build + cd meson-build + meson setup \ + --prefix=$PREFIX \ + --buildtype=release \ + -Dpython=$PREFIX/bin/python \ + -Dtests=false \ + .. + + ninja -j $NPROC + ninja install + +) +piplist; track various pythonbindings + +# build support to use QT from python -- see www.riverbankcomputing.com + +( # sip: tool to create Python bindings for C and C++ libraries. + SIP_VERSION=6.7.7 + + # https://files.pythonhosted.org/packages/5b/cb/c27c925ae07bd03a2597fa1db17bfc2a4ac57da61aeb90f8ec98ffbb975b/sip-6.6.2.tar.gz + BEEHIVE=https://beehive.molgen.mpg.de/a131efd79649d31ee11601b19d1e3784/sip-6.7.7.tar.gz + test -e sip-$SIP_VERSION.tar.gz || wget -nv $BEEHIVE + test -d sip-$SIP_VERSION || tar -xf sip-$SIP_VERSION.tar.gz + + cd sip-$SIP_VERSION + + pip install . + +) + +# QT bindings need some python modules to build + +install PyQt5-sip +install PyQt-builder + +( + PYQT_VERSION=5.15.9 + + # orig: https://files.pythonhosted.org/packages/5c/46/b4b6eae1e24d9432905ef1d4e7c28b6610e28252527cdc38f2a75997d8b5/PyQt5-5.15.9.tar.gz + BEEHIVE=https://beehive.molgen.mpg.de/9d97fc06b7ae75e654e946c49e07ff12/PyQt5-5.15.9.tar.gz + test -e PyQt5-$PYQT_VERSION.tar.gz || wget -nv $BEEHIVE + test -d PyQt5-$PYQT_VERSION || tar -xf PyQt5-$PYQT_VERSION.tar.gz + + cd PyQt5-$PYQT_VERSION + + sip-build \ + --confirm-license \ + --no-make \ + --api-dir $PREFIX/share/qsci/api/python \ + --dbus $PREFIX/include/dbus-1.0 \ + --no-designer-plugin \ + --no-qml-plugin \ + --jobs $NPROC + + cd build + make -j $NPROC + make install + + # Remove unused py2 version of uic modules + rm -r $PREFIX/lib/python*/site-packages/PyQt5/uic/port_v2 + +) + +( + QSCINTILLA_VERSION=2.13.4 # no 2.14 in sight ... + + # orig: https://www.riverbankcomputing.com/static/Downloads/QScintilla/2.13.4/QScintilla_src-2.13.4.tar.gz + BEEHIVE=https://beehive.molgen.mpg.de/4f83a4a4ad7da40eae80ad23f9fb18f2/QScintilla_src-2.13.4.tar.gz + test -e QScintilla_src-$QSCINTILLA_VERSION.tar.gz || wget -nv $BEEHIVE + test -d QScintilla_src-$QSCINTILLA_VERSION || tar -xf QScintilla_src-$QSCINTILLA_VERSION.tar.gz + + cd QScintilla_src-$QSCINTILLA_VERSION + cd Python + + cp -vp pyproject-qt5.toml pyproject.toml + sip-build \ + --no-make \ + --qsci-features-dir ../src/features \ + --qsci-include-dir ../src \ + --qsci-library-dir ../src \ + --api-dir $PREFIX/share/qsci/api/python \ + --jobs $NPROC + + cd build + make -j $NPROC + make install + +) + +( + set -x + PYQTWEBENGINE_VERSION=5.15.6 + + #orpg: https://pypi.org/project/PyQtWebEngine/#files + BEEHIVE="https://beehive.molgen.mpg.de/d1e548617de149ba8142a32c1eb9f994/PyQtWebEngine-5.15.6.tar.gz" + test -e PyQtWebEngine-${PYQTWEBENGINE_VERSION}.tar.gz || wget -nv $BEEHIVE + test -d PyQtWebEngine-${PYQTWEBENGINE_VERSION} || tar -xf PyQtWebEngine-${PYQTWEBENGINE_VERSION}.tar.gz + + cd PyQtWebEngine-${PYQTWEBENGINE_VERSION} + sip-build \ + --no-make \ + --api-dir ${PREFIX}/share/qsci/api/python \ + --jobs $NPROC + + cd build + make -j $NPROC + make install +) +piplist; track qt-support-riverbankcomputing + + +# -------------------------------------------------------- Install packages + + +install Cython + + +# libxml2-python from pypi isn't up to date, cause lxml seems to be the favored binding +# do this early, not that some dependency tries it on its own. +# (btw. libxslt-python isn't ported to python3) +( + LIBXML2_VERSION=2.9.12 + + # orig: http://xmlsoft.org/sources/libxml2-${PKGVERSION}.tar.gz + BEEHIVE=https://beehive.molgen.mpg.de/f433a39be087a9f0b197eb2307ad9f75/libxml2-2.9.12.tar.gz + test -e libxml2-$LIBXML2_VERSION.tar.gz || wget -nv $BEEHIVE + test -d libxml2-$LIBXML2_VERSION || tar -xf libxml2-$LIBXML2_VERSION.tar.gz libxml2-$LIBXML2_VERSION/python + cd libxml2-$LIBXML2_VERSION/python + pip install . +) +piplist; track legacy-libxml2 + +# Install the most 'sensitive' packages first +install numpy==1.25.2 +install numba==0.58.1 + +install wheel + +# python-javabridge, problems with python 3.11 and recent cython, thus +# regenerate C bindings from *pxd, *pyx -- so must use github, and wheel +( + # use the fork from cellprofiler, as used in pypi + git clone https://github.com/CellProfiler/python-javabridge + cd python-javabridge + git checkout v4.0.3 + sed -ne '/^#Epatch:javabridge/ s/^#Epatch:\S* // p' $ME | patch -p1 --verbose + pip install . +) + +PACKAGES=$(sed -e s/#.*$// <<- __PKGLIST__ + igraph==0.10.8 # louvain + keras==2.14.0 # tensorflow 2.14.1 + ml-dtypes==0.2.0 # tensorflow, dm-haiku, jax, tensorstore + seaborn==0.12.2 # scanpy + wrapt==1.14.1 # tensorflow vs. snakemake + + pytest==6.2.5 + pytest-asyncio==0.19.0 # to get rid of a deprecation warning in 0.20.3, -> 0.19.0 + + urllib3==2.0.7 + ipython[all] + + Pillow + protobuf + onnx + Sphinx + jedi + black[d] # onnx wants it, but does not install it ... + flask + + SciPy + pandas + SymPy + nose + statsmodels + pysam + seaborn + scikit-learn + Mako + cutadapt # Helpdesk 01/24/18 16:19, S. K. + pygobject # Helpdesk 03/26/18 14:19, V. S. + blink1 # Blink1-lib 07/04/18 12:00, N. H. + hvplot # ont/nextflow pipelines + pyarrow # ont/nextflow pipelines + umi-tools + recommonmark + fastcluster + pydot + pygraphviz + deeptools + HTSeq + keras_preprocessing # tensorflow ... + MotifScan # A.B. 30.8.2018 + MAmotif + hic2cool # Helpdesk R. S. 17.09.2018 + snakemake + virtualenv # Donald + odfpy # Donald + Dumper # Thomas + louvain # monocle3 prep for H. K. + selenium # 19.7.19 T.K. -> https://webxray.org/ + plotly + pyOpenCL + docopt + distro + ply + threadpoolctl + HiCMatrix # 17.1 -> https://github.com/deeptools/HiCMatrix/issues/47 + pybedtools + gffutils + jupyter + jupyterlab + jupyterlab-git # request P. A. + igv_jupyterlab # M. L. HD 01-04-2020 + igv-notebook + scour # https://github.molgen.mpg.de/mariux64/pkg-scripts/issues/98 + multiqc + pybind11 + svgwrite + pyomo + bioservices + SharedArray + pyabc # dependency for stochkit, HD 20.07.20 Gemma Noviello + petab + tadtool # HD 26.01.21 M. R. + pyperf # Paul sep-20 + umap # HD, used by M. L. + MACS3 + biopython # HD 2021-07-23 Magalhaes/Hnisz alphafold related -- cool that we have missed biopython so far, LOL + # prevent dm-haiku from fetching 'brand-new' stuff, thus avoiding further havoc + tensorstore==0.1.45 # tensorflow + orbax-checkpoint==0.4.4 # tensorflow + dm-tree + dm-haiku + flatbuffers + contextlib2 + ml_collections + python-bioformats==4.0.6 # 4.0.7 would pull up too much other packages + + + # additions/fixups since 3.9 + spyder + spyder-kernels + papermill # turns jupyter notebooks into scripts [sic!] used by P.A. [who else, *grin*] + opencv-python + scikit-image + git-cola + cffi-glpk # uses /usr/lib/libglpk.so.40, see if I can get it linked statically - kreitler + czifile + # watch https://pypi.org/project/napari/#history + napari[pyqt5]==0.4.19rc2 # 0.4.18 would turn into a psychotic downgrader via numpydoc ... + napari-czifile2 + CrossMap + scanpy + asciidoc + triton + + # additions/fixups since 3.10 + aicspylibczi + filetype + gmpy2 + + # nf-core / epi2me related + dominate + libsass + sigfig + whatshap + Truvari + + __PKGLIST__ +) + +for PKG in $PACKAGES; do + install $PKG +done + + +# remove unneeded dependency, and save the current pillow, then go for pyimagej, nerve ... +( + LABELING_VERSION=0.1.13 + # https://files.pythonhosted.org/packages/71/b0/c3c3c6931850c05e548c0a8cfadb8e6d8c5bd5ad18dce90ad4c909db5867/labeling-0.1.13.tar.gz + BEEHIVE=https://beehive.molgen.mpg.de/6da91e43f9864e9f4d4acb252eb486eb/labeling-0.1.13.tar.gz + test -e labeling-${LABELING_VERSION}.tar.gz || wget -nv $BEEHIVE + test -d labeling-${LABELING_VERSION} || tar -xf labeling-${LABELING_VERSION}.tar.gz + cd labeling-${LABELING_VERSION} + # not needed (tests work...) + sed -i -e '/pillow/ s/<10//' ./setup.py + sed -i -e '/pillow/ s/<10//' ./labeling.egg-info/requires.txt + pip install . --prefix=$PREFIX + + piplist; track labeling==${LABELING_VERSION} +) + +install pyimagej # Microscopy + + +# torch bundled with a recent CUDA-Toolkit +install torch torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/cu118 + +install pyopengl # Tk binding creates an error, so remove it (Expects Tk Togl installation...) +rm -rf ${PREFIX}/lib/python$PYTHONVER/site-packages/OpenGL/Tk + + +mkdir -vp $PREFIX/share/jupyter/lab/settings # try to defeat a silly error in jupyter labextension install + + +# --------------------------------------------------- Install misc packages + +# these mostly bigger packages require either wheel, other treatments, +# or java. + +# GYP is a Meta-Build system: a build system that generates other build systems. https://gyp.gsrc.io/ +( + set -x + GYP_VERSION=20230301_c6d8b9f7 + # orig: https://mirror.netcologne.de/slackware/slackware64-current/source/d/gyp/ + BEEHIVE=https://beehive.molgen.mpg.de/ba0357be23ca55555714261afccc7aab/gyp-20230301_c6d8b9f7.tar.xz + test -e gyp-${GYP_VERSION}.tar.xz || wget -nv $BEEHIVE + test -d gyp-${GYP_VERSION} || tar -xf gyp-${GYP_VERSION}.tar.xz + cd gyp-${GYP_VERSION} + sed -ne '/^#Epatch:gyp/ s/^#Epatch:\S* // p' $ME | patch -p1 --verbose + pip install . +) +piplist; track gyp + +( + BOOST_VERSION=(1.80.0 1_80_0) + PKGCFLAGS="-O2 -fPIC" + + # orig: https://boostorg.jfrog.io/artifactory/main/release/1.80.0/source/boost_1_80_0.tar.gz + BEEHIVE=https://beehive.molgen.mpg.de/077f074743ea7b0cb49c6ed43953ae95/boost_1_80_0.tar.gz + test -e boost_${BOOST_VERSION[1]}.tar.gz || wget -nv $BEEHIVE + test -d boost-${BOOST_VERSION[0]} || mkdir boost-${BOOST_VERSION[0]} && tar -xf boost_${BOOST_VERSION[1]}.tar.gz -C boost-${BOOST_VERSION[0]} --strip-components=1 + + cd boost-${BOOST_VERSION[0]} + sed -e "/using python/ s|;|: ${PREFIX}/include/python${PYTHONVER} ;|" \ + -i bootstrap.sh + + ./bootstrap.sh \ + --with-toolset=gcc \ + --with-icu \ + --with-python=$PREFIX/bin/python3 \ + --with-libraries=python,thread || exit 1 + + ./b2 \ + -j $NPROC \ + --layout=system \ + --build-dir=build-python3 \ + --prefix=$PREFIX \ + --libdir=$PREFIX/lib \ + --build-type=minimal \ + toolset=gcc \ + variant=release \ + debug-symbols=off \ + link=shared \ + threading=multi \ + runtime-link=shared \ + python=$PYTHONVER \ + cflags="$PKGCFLAGS" \ + cxxflags="$PKGCFLAGS" \ + install || exit 1 +) + +( + PYCUDAVERS=2023.1 + PYTHON_LIBNAME=boost_python${PYTHONVER/./} + + # orig: https://files.pythonhosted.org/packages/dd/b2/e30282f3286ddad05ef44548fa5c306a179ed3baceefa699af078e49ce6e/pycuda-2023.1.tar.gz + BEEHIVE="https://beehive.molgen.mpg.de/3193e66e54242ac634825b14944295ac/pycuda-2023.1.tar.gz" + test -e pycuda-${PYCUDAVERS}.tar.gz || wget -nv $BEEHIVE + test -d pycuda-${PYCUDAVERS} || tar -xf pycuda-${PYCUDAVERS}.tar.gz + + cd pycuda-${PYCUDAVERS} + test -e siteconf.py && rm siteconf.py + python configure.py \ + --cuda-enable-gl \ + --cuda-root=$CUDADIR \ + --no-use-shipped-boost \ + --boost-python-libname=$PYTHON_LIBNAME \ + --boost-inc-dir=$PREFIX/include \ + --boost-lib-dir=$PREFIX/lib \ + --cudadrv-lib-dir='${CUDA_ROOT}/lib,${CUDA_ROOT}/lib/stubs' \ + --cudart-lib-dir='${CUDA_ROOT}/lib,${CUDA_ROOT}/lib/stubs' \ + --curand-lib-dir='${CUDA_ROOT}/lib,${CUDA_ROOT}/lib/stubs' \ + --ldflags=-L$PREFIX/lib + + pip install . --prefix=$PREFIX +) +piplist; track pycuda + +( + # install from git to get .so files in case + pip install git+https://github.com/open2c/pairtools.git@v1.0.2 --prefix=$PREFIX + pip install git+https://github.com/brentp/bwa-meth.git@v0.2.7 --prefix=$PREFIX + + # versions > 1.5 will not work with pip install github + pip install git+https://github.com/stacked-git/stgit.git@v1.5 --prefix=$PREFIX +) +piplist; track pairtools bwa-meth stgit + +( + IDR_VERSION=2.0.3 + # orig: https://github.com/nboley/idr/archive/2.0.3/idr-2.0.3.tar.gz # HD: T. Z., 18.3.21 + BEEHIVE=https://beehive.molgen.mpg.de/1bec61b526bb1dcf92a5ab99c27852d0/idr-2.0.3.tar.gz + test -e idr-${IDR_VERSION}.tar.gz || wget -nv $BEEHIVE + test -d idr-${IDR_VERSION} || tar -xf idr-${IDR_VERSION}.tar.gz + cd idr-${IDR_VERSION} + pip install . --prefix=$PREFIX +) +piplist; track IDR + + +# ------------------------------------------------ finally build tensorflaw + +( + test -n "$SKIPTENSORFLOW" && exit + + TFVERSION=2.14.1 + NCCLSHORTVERSION=$(pkg-config nccl --modversion | grep -Po '\d+\.\d+') + + # need newer jre to handle --add-opens=java.base/java.lang=ALL-UNNAMED, and bazel >= 6.1 + . /pkg/openjdk-21.0.0.35-0/profile + . /pkg/bazel-6.2.1-0/profile + + # orig: https://github.com/tensorflow/tensorflow/archive/refs/tags/v2.14.1.tar.gz + BEEHIVE=https://beehive.molgen.mpg.de/c7ba144a4b1b0a6d5fea5ecd274172d3/tensorflow-2.14.1.tar.gz + + test -e tensorflow-${TFVERSION}.tar.gz || wget -nv $BEEHIVE + test -d tensorflow-${TFVERSION} || tar -xf tensorflow-${TFVERSION}.tar.gz + + cd tensorflow-${TFVERSION} + + #vars for config + export PYTHON_BIN_PATH=${PREFIX}/bin/python3 + export USE_DEFAULT_PYTHON_LIB_PATH=1 + export TF_PYTHON_VERSION=3.11 + export TF_NEED_JEMALLOC=1 + export TF_NEED_KAFKA=0 + export TF_NEED_OPENCL_SYCL=0 + export TF_NEED_AWS=0 + export TF_NEED_GCP=0 + export TF_NEED_HDFS=0 + export TF_NEED_S3=0 + export TF_ENABLE_XLA=1 + export TF_NEED_GDR=0 + export TF_NEED_VERBS=0 + export TF_NEED_OPENCL=0 + export TF_NEED_MPI=0 + export TF_NEED_TENSORRT=0 + export TF_NEED_NGRAPH=0 + export TF_NEED_IGNITE=0 + export TF_NEED_ROCM=0 + export TF_SET_ANDROID_WORKSPACE=0 + export TF_DOWNLOAD_CLANG=0 + export TF_NCCL_VERSION=${NCCLSHORTVERSION} + export TF_IGNORE_MAX_BAZEL_VERSION=1 + export NCCL_INSTALL_PATH=${CUDADIR} + export CC=gcc + export CXX=g++ + export GCC_HOST_COMPILER_PATH=/usr/bin/${CC} + export HOST_C_COMPILER=/usr/bin/${CC} + export HOST_CXX_COMPILER=/usr/bin/${CXX} + export TF_CUDA_CLANG=0 + export CLANG_CUDA_COMPILER_PATH=/usr/bin/clang + export TF_CUDA_PATHS=${CUDADIR},${CUDADIR}/extras/CUPTI,${CUDADIR}/nvvm + export TF_CUDA_VERSION=$(${CUDADIR}/bin/nvcc --version | sed -n 's/^.*release \(.*\),.*/\1/p') # 11.5 + export TF_CUDNN_VERSION=$(sed -n 's/^#define CUDNN_MAJOR\s*\(.*\).*/\1/p' ${CUDADIR}/include/cudnn_version.h) + # hardware older than sm_52 is said to be unsupported, compute_86 denotes an emulation + export TF_CUDA_COMPUTE_CAPABILITIES=sm_52,sm_53,sm_60,sm_61,sm_62,sm_70,sm_72,sm_75,sm_80,sm_86,sm_87,sm_89,sm_90,compute_90 + + # stay compatible with the cluster machines + # https://gcc.gnu.org/onlinedocs/gcc/x86-Options.html + export CC_OPT_FLAGS="-march=broadwell" # or haswell to be safe ? no more nehalem + export TF_NEED_CUDA=1 + + export BAZEL_ARGS="--config=mkl -c opt" + + # avoid: Auto-Configuration Warning: 'TMP' environment variable is not set, using 'C:\Windows\Temp' as default :/ + export TMP=$BUILD_TMPDIR/tf_tmp + mkdir -p $TMP + + ./configure + + bazel build \ + --cxxopt="-march=broadwell" \ + --copt="-march=broadwell" \ + --noshow_loading_progress \ + --noshow_progress \ + --color="no" \ + //tensorflow:libtensorflow.so \ + //tensorflow:libtensorflow_cc.so \ + //tensorflow:install_headers \ + //tensorflow/tools/pip_package:build_pip_package + + bazel shutdown # otherwise it hogs in cwd + + # create a _clean/decent_ python package from so far created debris + test -d tensorflow-pip && rm -r tensorflow-pip + mkdir tensorflow-pip + ./bazel-bin/tensorflow/tools/pip_package/build_pip_package --src ${BUILDDIR}/tensorflow-pip + cd ${BUILDDIR} + # ... cope with the weird symlink-structure placed in bazels' build-TMPDIR (-h) (tell me why ?) + test -e tensorflow-pip.tar && rm tensorflow-pip.tar + tar -chf tensorflow-pip.tar tensorflow-pip + + # and perform a regular install + test -d tensorflow-pip && rm -r tensorflow-pip + tar -xf tensorflow-pip.tar + cd ${BUILDDIR}/tensorflow-pip + + pip install . --prefix=$PREFIX +) +piplist; track tensorflow + +# tidy up a bit +if [ -d $PREFIX/man/man1 ]; then + mv -v $PREFIX/man/man1/* $PREFIX/share/man/man1 || true + rm -rf $PREFIX/man +fi + + +# ------------------------------------------------------------ Sanity check + +# load all packages, see warnings and spot installation errors +# But don't care about r2py, a further dependence into R is not wanted! +echo "# START load test." +python -c 'help("modules")' > /dev/null +echo "# END load test (passed)." + +echo "# running pip check" +pip check + +# I guess it makes sense that files can be read by all users ... +echo "# check for unreadable files" +cd $PREFIX +for D in bin etc include lib share; do + find $D \! -perm -004 -exec chmod -c a+r {} + +done + +echo "For Deeper checks:" +echo "cd ${BUILDDIR} ; git log --patch " + + +exit + + +# -------------------------------------------------------- Included patches + +#Epatch:pip # Operation 'Forever young', disable version-check per default. +#Epatch:pip # The option needs to be kept, because some tools make use of +#Epatch:pip # it and would cause pip to fail. +#Epatch:pip --- a/pip/_internal/cli/cmdoptions.py 2022-08-09 17:29:15.853442948 +0200 +#Epatch:pip +++ b/pip/_internal/cli/cmdoptions.py 2022-08-10 10:31:35.124945154 +0200 +#Epatch:pip @@ -892,7 +892,6 @@ +#Epatch:pip dest="disable_pip_version_check", +#Epatch:pip action="store_true", +#Epatch:pip - default=False, +#Epatch:pip - help="Don't periodically check PyPI to determine whether a new version " +#Epatch:pip - "of pip is available for download. Implied with --no-index.", +#Epatch:pip + default=True, +#Epatch:pip + help="Ignore this option, the version check IS disabled.", +#Epatch:pip ) + + +#Epatch:gyp --- a/pylib/gyp/generator/cmake.py +#Epatch:gyp +++ b/pylib/gyp/generator/cmake.py +#Epatch:gyp @@ -41,7 +41,7 @@ +#Epatch:gyp try: +#Epatch:gyp # maketrans moved to str in python3. +#Epatch:gyp + _maketrans = str.maketrans +#Epatch:gyp +except NameError: +#Epatch:gyp _maketrans = string.maketrans +#Epatch:gyp -except NameError: +#Epatch:gyp - _maketrans = str.maketrans +#Epatch:gyp generator_default_variables = { +#Epatch:gyp 'EXECUTABLE_PREFIX': '', +#Epatch:gyp @@ -283,2 +283,2 @@ +#Epatch:gyp if int(action.get('process_outputs_as_sources', False)): +#Epatch:gyp - extra_sources.extend(zip(cmake_outputs, outputs)) +#Epatch:gyp + extra_sources.extend(list(zip(cmake_outputs, outputs))) +#Epatch:gyp @@ -988,6 +988,6 @@ +#Epatch:gyp # XCode settings +#Epatch:gyp xcode_settings = config.get('xcode_settings', {}) +#Epatch:gyp - for xcode_setting, xcode_value in xcode_settings.viewitems(): +#Epatch:gyp + for xcode_setting, xcode_value in xcode_settings.items(): +#Epatch:gyp SetTargetProperty(output, cmake_target_name, +#Epatch:gyp "XCODE_ATTRIBUTE_%s" % xcode_setting, xcode_value, +#Epatch:gyp '' if isinstance(xcode_value, str) else ' ') + +#Epatch:gyp diff --git a/pylib/gyp/input.py b/pylib/gyp/input.py +#Epatch:gyp index 4c12891..2bea334 100644 +#Epatch:gyp --- a/pylib/gyp/input.py +#Epatch:gyp +++ b/pylib/gyp/input.py +#Epatch:gyp @@ -1183,7 +1183,7 @@ +#Epatch:gyp if variable_name in variables: +#Epatch:gyp # If the variable is already set, don't set it. +#Epatch:gyp continue +#Epatch:gyp - if the_dict_key is 'variables' and variable_name in the_dict: +#Epatch:gyp + if the_dict_key == 'variables' and variable_name in the_dict: +#Epatch:gyp # If the variable is set without a % in the_dict, and the_dict is a +#Epatch:gyp # variables dict (making |variables| a varaibles sub-dict of a +#Epatch:gyp # variables dict), use the_dict's definition. + + +#Epatch:javabridge diff --git a/_javabridge_nomac.pyx b/_javabridge_nomac.pyx +#Epatch:javabridge index 9f28939..c437db0 100644 +#Epatch:javabridge --- a/_javabridge_nomac.pyx +#Epatch:javabridge +++ b/_javabridge_nomac.pyx +#Epatch:javabridge @@ -16,30 +16,30 @@ cdef extern from "jni.h": +#Epatch:javabridge jint (*AttachCurrentThreadAsDaemon)(JavaVM *vm, void *penv, void *args) nogil +#Epatch:javabridge jint JNI_CreateJavaVM(JavaVM **pvm, void **penv, void *args) nogil +#Epatch:javabridge +#Epatch:javabridge -cdef int MacStartVM(JavaVM **pvm, JavaVMInitArgs *pVMArgs, char *class_name) nogil: +#Epatch:javabridge +cdef extern int MacStartVM(JavaVM **pvm, JavaVMInitArgs *pVMArgs, char *class_name) nogil: +#Epatch:javabridge return -1 +#Epatch:javabridge +#Epatch:javabridge -cdef void StopVM(JavaVM *vm) nogil: +#Epatch:javabridge +cdef extern void StopVM(JavaVM *vm) nogil: +#Epatch:javabridge vm[0].DestroyJavaVM(vm) +#Epatch:javabridge +#Epatch:javabridge -cdef void MacRunLoopInit() nogil: +#Epatch:javabridge +cdef extern void MacRunLoopInit() nogil: +#Epatch:javabridge pass +#Epatch:javabridge +#Epatch:javabridge -cdef void MacRunLoopRun() nogil: +#Epatch:javabridge +cdef extern void MacRunLoopRun() nogil: +#Epatch:javabridge pass +#Epatch:javabridge +#Epatch:javabridge -cdef void MacRunLoopStop() nogil: +#Epatch:javabridge +cdef extern void MacRunLoopStop() nogil: +#Epatch:javabridge pass +#Epatch:javabridge +#Epatch:javabridge -cdef void MacRunLoopReset() nogil: +#Epatch:javabridge +cdef extern void MacRunLoopReset() nogil: +#Epatch:javabridge pass +#Epatch:javabridge +#Epatch:javabridge -cdef int MacIsMainThread() nogil: +#Epatch:javabridge +cdef extern int MacIsMainThread() nogil: +#Epatch:javabridge return 0 +#Epatch:javabridge +#Epatch:javabridge -cdef void MacRunLoopRunInMode(double timeout) nogil: +#Epatch:javabridge +cdef extern void MacRunLoopRunInMode(double timeout) nogil: +#Epatch:javabridge pass +#Epatch:javabridge +#Epatch:javabridge -cdef int CreateJavaVM(JavaVM **pvm, void **pEnv, void *args) nogil: +#Epatch:javabridge +cdef extern int CreateJavaVM(JavaVM **pvm, void **pEnv, void *args) nogil: +#Epatch:javabridge return JNI_CreateJavaVM(pvm, pEnv, args) +#Epatch:javabridge + +#Epatch:javabridge diff --git a/_javabridge_mac.pyx b/_javabridge_mac.pyx +#Epatch:javabridge index 2e3c524..daed790 100644 +#Epatch:javabridge --- a/_javabridge_mac.pyx +#Epatch:javabridge +++ b/_javabridge_mac.pyx +#Epatch:javabridge @@ -31,11 +31,11 @@ cdef extern from "mac_javabridge_utils.h": +#Epatch:javabridge int MacIsMainThread() nogil +#Epatch:javabridge void MacRunLoopRunInMode(double) nogil +#Epatch:javabridge +#Epatch:javabridge -cdef void StopVM(JavaVM *vm): +#Epatch:javabridge +cdef extern void StopVM(JavaVM *vm): +#Epatch:javabridge MacStopVM() +#Epatch:javabridge +#Epatch:javabridge # +#Epatch:javabridge # Unused stub in Mac +#Epatch:javabridge # +#Epatch:javabridge -cdef int CreateJavaVM(JavaVM **pvm, void **pEnv, void *args): +#Epatch:javabridge +cdef extern int CreateJavaVM(JavaVM **pvm, void **pEnv, void *args): +#Epatch:javabridge return -1 + +# -------------------------------------------------------- This is the end. +