Permalink
Cannot retrieve contributors at this time
Name already in use
A tag already exists with the provided branch name. Many Git commands accept both tag and branch names, so creating this branch may cause unexpected behavior. Are you sure you want to create this branch?
pkg-scripts/python-3.11.7-0.build.sh
Go to fileThis commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
executable file
863 lines (701 sloc)
27.9 KB
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| #!/bin/bash | |
| # ------------------------------------------------------------------- Debug | |
| COOKIE=$(mcookie|cut -c-8); grep -v V_GREP_ME $0 > /dev/shm/runme-$COOKIE.sh ; sleep 0.3; exec bash /dev/shm/runme-$COOKIE.sh | |
| # TESTING=1 | |
| # SKIPTENSORFLOW=1 | |
| # ---------------------------------------------------------------- Preamble | |
| PKG=python | |
| VERSION=3.11.7 | |
| BUILD=0 | |
| PYTHONVER=${VERSION%.*} | |
| CUDADIR=/pkg/cuda-11.8.0-0 | |
| PREFIX=/pkg/$PKG-$VERSION-$BUILD | |
| if [ -n "$TESTING" ]; then PREFIX=/scratch/local2/$PKG-$VERSION-$BUILD ; fi | |
| PATH=/bin:/usr/bin:/usr/sbin:/usr/local/bin | |
| set -e | |
| umask 022 | |
| BUILD_TMPDIR=/scratch/local2/$PKG-$VERSION-$BUILD.$USER.build.tmp | |
| # keep pip's download cache when testing | |
| test -z "$TESTING" && test -d $BUILD_TMPDIR && ( chmod -R u+rwx $BUILD_TMPDIR || true ; rm -rf $BUILD_TMPDIR ) | |
| mkdir -p $BUILD_TMPDIR/home/.cache/pip | |
| mkdir -p $BUILD_TMPDIR/home/.local | |
| # copy script to an accessible location | |
| cp $0 $BUILD_TMPDIR; ME="$BUILD_TMPDIR/$(basename $0)" | |
| export TMPDIR=$BUILD_TMPDIR | |
| export HOME=$BUILD_TMPDIR/home | |
| exec </dev/null | |
| mkdir -p $PREFIX | |
| cat >$PREFIX/profile <<-EOF | |
| . $CUDADIR/profile | |
| PATH=$PREFIX/bin:\$PATH | |
| export LD_LIBRARY_PATH=$PREFIX/lib\${LD_LIBRARY_PATH:+:}\$LD_LIBRARY_PATH | |
| PKG_CONFIG_PATH=$PREFIX/lib/pkgconfig\${PKG_CONFIG_PATH:+:}\$PKG_CONFIG_PATH | |
| export PKG_CONFIG_PATH | |
| if [ -d $PREFIX/.compatlibs ]; then export LD_LIBRARY_PATH=$PREFIX/.compatlibs\${LD_LIBRARY_PATH:+:}\$LD_LIBRARY_PATH ; fi | |
| EOF | |
| . $PREFIX/profile | |
| NPROC=$(( $(nproc) * 4 / 5 + 1 )) | |
| export MAKEFLAGS="-j $NPROC" | |
| BUILDDIR=$PREFIX/build | |
| mkdir -p $BUILDDIR | |
| cd $BUILDDIR | |
| # -------------------------------------- Use git to track package evolution | |
| test -d .git && rm -rf .git | |
| git init -q; echo -e '[user]\n name = none\n email = of_your_business...' >> .git/config | |
| echo '*' > .gitignore | |
| function piplist() { | |
| pip list | awk '{ printf("%-36s %s\n", $1,$2) }' | grep -v '\----' > GameOfVersions | |
| } | |
| function track() { | |
| if [ -d .git ]; then | |
| if git add -f GameOfVersions; then | |
| git commit -q -n -m "Package: '$*'" || /bin/true | |
| fi | |
| fi | |
| } | |
| function install() { | |
| echo "# ::INST:: # '${@+$@}'" | |
| # no-color is nice, but doesn't work when dependencies are installed, so (mis-)use a pipe | |
| pip install --no-color --compile --cache-dir=$HOME/.cache/pip --prefix=$PREFIX "${@+$@}" | cat | |
| piplist | |
| track $* | |
| } | |
| # ---------------------------------------- Build Python & support libraries | |
| ( | |
| # orig: https://www.python.org/ftp/python/3.11.7/Python-3.11.7.tgz | |
| BEEHIVE=https://beehive.molgen.mpg.de/ef61f81ec82c490484219c7f0ec96783/Python-3.11.7.tgz | |
| test -e Python-$VERSION.tar.xz || wget -nv $BEEHIVE | |
| test -d Python-$VERSION || tar -xf Python-$VERSION.tgz | |
| cd Python-$VERSION | |
| # BerkeleyDB, the undead ... | |
| # Newer libgdbm_compat.so versions pull libgdbm.so on their own, our libgdbm_compat.so is older ... | |
| # So, let setup.py create '-lgdbm -lgdbm_compat' instead of a sole '-lgdbm_compat' | |
| # verbatim: ndbm_libs = ['gdbm_compat'] => ndbm_libs = ['gdbm','gdbm_compat'] | |
| sed -i -e "/ndbm_libs = / s/_compat'/','gdbm_compat'/" setup.py | |
| # leave further hints in the log | |
| sed -i -e '/db_setup_debug = False/ s/False/True/' \ | |
| -e '/dbm_setup_debug = False/ s/False/True/' setup.py | |
| # These may come handy if the Python build needs exclusive stuff from $PREFIX | |
| # INCLUDEDIR=$PREFIX/include \ | |
| # LIBDIR=$PREFIX/lib \ | |
| LDFLAGS="-Wl,-rpath=$PREFIX/lib" \ | |
| ./configure \ | |
| --prefix=$PREFIX \ | |
| --enable-shared | |
| make -j $NPROC | |
| make install | |
| # it might come to happen that something down the line may just call 'python', defuse ... | |
| test -e $PREFIX/bin/python || ln -s python3 $PREFIX/bin/python | |
| python -m ensurepip | |
| pip3 install --prefix=$PREFIX -I pip | |
| ( # fix 'please update' noise from pip | |
| cd $PREFIX/lib/python*/site-packages | |
| sed -ne '/^#Epatch:pip/ s/^#Epatch:\S* // p' $ME | patch -p1 --verbose | |
| ) | |
| # libblosc2 installs in $PREFIX/lib64, I hate to do this ... | |
| ln -s lib $PREFIX/lib64 | |
| ) | |
| install setuptools | |
| piplist; track START | |
| # build various pythonbindings 'by hand', because pip installs do not deliver | |
| # header files and the like. | |
| # BTW: builds need meson | |
| install meson | |
| ( | |
| DBUS_PYTHON_VERSION=1.3.2 | |
| # PyPi | |
| # https://files.pythonhosted.org/packages/c1/d3/6be85a9c772d6ebba0cc3ab37390dd6620006dcced758667e0217fb13307/dbus-python-1.3.2.tar.gz | |
| BEEHIVE=https://beehive.molgen.mpg.de/33be8a4a766e1c7c9a377b8f934ce21a/dbus-python-1.3.2.tar.gz | |
| test -e dbus-python-$DBUS_PYTHON_VERSION.tar.gz || wget -nv $BEEHIVE | |
| test -d dbus-python-$DBUS_PYTHON_VERSION || tar -xf dbus-python-$DBUS_PYTHON_VERSION.tar.gz | |
| cd dbus-python-$DBUS_PYTHON_VERSION | |
| ./configure --prefix=$PREFIX | |
| make -j $NPROC | |
| make install | |
| ) | |
| ( | |
| PYCAIRO_VERSION=1.23.0 | |
| # PyPi | |
| # https://files.pythonhosted.org/packages/69/ca/9e9fa2e8be0876a9bbf046a1be7ee33e61d4fdfbd1fd25c76c1bdfddf8c4/pycairo-1.23.0.tar.gz | |
| BEEHIVE=https://beehive.molgen.mpg.de/7a3729d21659098e1b9a411b62e88966/pycairo-1.23.0.tar.gz | |
| test -e pycairo-$PYCAIRO_VERSION.tar.gz || wget -nv $BEEHIVE | |
| test -d pycairo-$PYCAIRO_VERSION || tar -xf pycairo-$PYCAIRO_VERSION.tar.gz | |
| cd pycairo-$PYCAIRO_VERSION | |
| mkdir -p meson-build | |
| cd meson-build | |
| meson setup \ | |
| --prefix=$PREFIX \ | |
| --buildtype=release \ | |
| -Dpython=$PREFIX/bin/python \ | |
| -Dtests=false \ | |
| .. | |
| ninja -j $NPROC | |
| ninja install | |
| ) | |
| ( | |
| PYGOBJECT3_VERSION=3.44.1 | |
| # https://files.pythonhosted.org/packages/07/73/a034fc1bcd6d402d6f72ff618c093f91ac6921f968cc19806b6a1b1b19c8/PyGObject-3.44.0.tar.gz | |
| BEEHIVE=https://beehive.molgen.mpg.de/959259478566b24316edaa18da6e0eaa/PyGObject-3.44.1.tar.gz | |
| test -e PyGObject-$PYGOBJECT3_VERSION.tar.gz || wget -nv $BEEHIVE | |
| test -d PyGObject-$PYGOBJECT3_VERSION || tar -xf PyGObject-$PYGOBJECT3_VERSION.tar.gz | |
| cd PyGObject-$PYGOBJECT3_VERSION | |
| mkdir -p meson-build | |
| cd meson-build | |
| meson setup \ | |
| --prefix=$PREFIX \ | |
| --buildtype=release \ | |
| -Dpython=$PREFIX/bin/python \ | |
| -Dtests=false \ | |
| .. | |
| ninja -j $NPROC | |
| ninja install | |
| ) | |
| piplist; track various pythonbindings | |
| # build support to use QT from python -- see www.riverbankcomputing.com | |
| ( # sip: tool to create Python bindings for C and C++ libraries. | |
| SIP_VERSION=6.7.7 | |
| # https://files.pythonhosted.org/packages/5b/cb/c27c925ae07bd03a2597fa1db17bfc2a4ac57da61aeb90f8ec98ffbb975b/sip-6.6.2.tar.gz | |
| BEEHIVE=https://beehive.molgen.mpg.de/a131efd79649d31ee11601b19d1e3784/sip-6.7.7.tar.gz | |
| test -e sip-$SIP_VERSION.tar.gz || wget -nv $BEEHIVE | |
| test -d sip-$SIP_VERSION || tar -xf sip-$SIP_VERSION.tar.gz | |
| cd sip-$SIP_VERSION | |
| pip install . | |
| ) | |
| # QT bindings need some python modules to build | |
| install PyQt5-sip | |
| install PyQt-builder | |
| ( | |
| PYQT_VERSION=5.15.9 | |
| # orig: https://files.pythonhosted.org/packages/5c/46/b4b6eae1e24d9432905ef1d4e7c28b6610e28252527cdc38f2a75997d8b5/PyQt5-5.15.9.tar.gz | |
| BEEHIVE=https://beehive.molgen.mpg.de/9d97fc06b7ae75e654e946c49e07ff12/PyQt5-5.15.9.tar.gz | |
| test -e PyQt5-$PYQT_VERSION.tar.gz || wget -nv $BEEHIVE | |
| test -d PyQt5-$PYQT_VERSION || tar -xf PyQt5-$PYQT_VERSION.tar.gz | |
| cd PyQt5-$PYQT_VERSION | |
| sip-build \ | |
| --confirm-license \ | |
| --no-make \ | |
| --api-dir $PREFIX/share/qsci/api/python \ | |
| --dbus $PREFIX/include/dbus-1.0 \ | |
| --no-designer-plugin \ | |
| --no-qml-plugin \ | |
| --jobs $NPROC | |
| cd build | |
| make -j $NPROC | |
| make install | |
| # Remove unused py2 version of uic modules | |
| rm -r $PREFIX/lib/python*/site-packages/PyQt5/uic/port_v2 | |
| ) | |
| ( | |
| QSCINTILLA_VERSION=2.13.4 # no 2.14 in sight ... | |
| # orig: https://www.riverbankcomputing.com/static/Downloads/QScintilla/2.13.4/QScintilla_src-2.13.4.tar.gz | |
| BEEHIVE=https://beehive.molgen.mpg.de/4f83a4a4ad7da40eae80ad23f9fb18f2/QScintilla_src-2.13.4.tar.gz | |
| test -e QScintilla_src-$QSCINTILLA_VERSION.tar.gz || wget -nv $BEEHIVE | |
| test -d QScintilla_src-$QSCINTILLA_VERSION || tar -xf QScintilla_src-$QSCINTILLA_VERSION.tar.gz | |
| cd QScintilla_src-$QSCINTILLA_VERSION | |
| cd Python | |
| cp -vp pyproject-qt5.toml pyproject.toml | |
| sip-build \ | |
| --no-make \ | |
| --qsci-features-dir ../src/features \ | |
| --qsci-include-dir ../src \ | |
| --qsci-library-dir ../src \ | |
| --api-dir $PREFIX/share/qsci/api/python \ | |
| --jobs $NPROC | |
| cd build | |
| make -j $NPROC | |
| make install | |
| ) | |
| ( | |
| set -x | |
| PYQTWEBENGINE_VERSION=5.15.6 | |
| #orpg: https://pypi.org/project/PyQtWebEngine/#files | |
| BEEHIVE="https://beehive.molgen.mpg.de/d1e548617de149ba8142a32c1eb9f994/PyQtWebEngine-5.15.6.tar.gz" | |
| test -e PyQtWebEngine-${PYQTWEBENGINE_VERSION}.tar.gz || wget -nv $BEEHIVE | |
| test -d PyQtWebEngine-${PYQTWEBENGINE_VERSION} || tar -xf PyQtWebEngine-${PYQTWEBENGINE_VERSION}.tar.gz | |
| cd PyQtWebEngine-${PYQTWEBENGINE_VERSION} | |
| sip-build \ | |
| --no-make \ | |
| --api-dir ${PREFIX}/share/qsci/api/python \ | |
| --jobs $NPROC | |
| cd build | |
| make -j $NPROC | |
| make install | |
| ) | |
| piplist; track qt-support-riverbankcomputing | |
| # -------------------------------------------------------- Install packages | |
| install Cython | |
| # libxml2-python from pypi isn't up to date, cause lxml seems to be the favored binding | |
| # do this early, not that some dependency tries it on its own. | |
| # (btw. libxslt-python isn't ported to python3) | |
| ( | |
| LIBXML2_VERSION=2.9.12 | |
| # orig: http://xmlsoft.org/sources/libxml2-${PKGVERSION}.tar.gz | |
| BEEHIVE=https://beehive.molgen.mpg.de/f433a39be087a9f0b197eb2307ad9f75/libxml2-2.9.12.tar.gz | |
| test -e libxml2-$LIBXML2_VERSION.tar.gz || wget -nv $BEEHIVE | |
| test -d libxml2-$LIBXML2_VERSION || tar -xf libxml2-$LIBXML2_VERSION.tar.gz libxml2-$LIBXML2_VERSION/python | |
| cd libxml2-$LIBXML2_VERSION/python | |
| pip install . | |
| ) | |
| piplist; track legacy-libxml2 | |
| # Install the most 'sensitive' packages first | |
| install numpy==1.25.2 | |
| install numba==0.58.1 | |
| install wheel | |
| # python-javabridge, problems with python 3.11 and recent cython, thus | |
| # regenerate C bindings from *pxd, *pyx -- so must use github, and wheel | |
| ( | |
| # use the fork from cellprofiler, as used in pypi | |
| git clone https://github.com/CellProfiler/python-javabridge | |
| cd python-javabridge | |
| git checkout v4.0.3 | |
| sed -ne '/^#Epatch:javabridge/ s/^#Epatch:\S* // p' $ME | patch -p1 --verbose | |
| pip install . | |
| ) | |
| PACKAGES=$(sed -e s/#.*$// <<- __PKGLIST__ | |
| igraph==0.10.8 # louvain | |
| keras==2.14.0 # tensorflow 2.14.1 | |
| ml-dtypes==0.2.0 # tensorflow, dm-haiku, jax, tensorstore | |
| seaborn==0.12.2 # scanpy | |
| wrapt==1.14.1 # tensorflow vs. snakemake | |
| pytest==6.2.5 | |
| pytest-asyncio==0.19.0 # to get rid of a deprecation warning in 0.20.3, -> 0.19.0 | |
| urllib3==2.0.7 | |
| ipython[all] | |
| Pillow | |
| protobuf | |
| onnx | |
| Sphinx | |
| jedi | |
| black[d] # onnx wants it, but does not install it ... | |
| flask | |
| SciPy | |
| pandas | |
| SymPy | |
| nose | |
| statsmodels | |
| pysam | |
| seaborn | |
| scikit-learn | |
| Mako | |
| cutadapt # Helpdesk 01/24/18 16:19, S. K. | |
| pygobject # Helpdesk 03/26/18 14:19, V. S. | |
| blink1 # Blink1-lib 07/04/18 12:00, N. H. | |
| hvplot # ont/nextflow pipelines | |
| pyarrow # ont/nextflow pipelines | |
| umi-tools | |
| recommonmark | |
| fastcluster | |
| pydot | |
| pygraphviz | |
| deeptools | |
| HTSeq | |
| keras_preprocessing # tensorflow ... | |
| MotifScan # A.B. 30.8.2018 | |
| MAmotif | |
| hic2cool # Helpdesk R. S. 17.09.2018 | |
| snakemake | |
| virtualenv # Donald | |
| odfpy # Donald | |
| Dumper # Thomas | |
| louvain # monocle3 prep for H. K. | |
| selenium # 19.7.19 T.K. -> https://webxray.org/ | |
| plotly | |
| pyOpenCL | |
| docopt | |
| distro | |
| ply | |
| threadpoolctl | |
| HiCMatrix # 17.1 -> https://github.com/deeptools/HiCMatrix/issues/47 | |
| pybedtools | |
| gffutils | |
| jupyter | |
| jupyterlab | |
| jupyterlab-git # request P. A. | |
| igv_jupyterlab # M. L. HD 01-04-2020 | |
| igv-notebook | |
| scour # https://github.molgen.mpg.de/mariux64/pkg-scripts/issues/98 | |
| multiqc | |
| pybind11 | |
| svgwrite | |
| pyomo | |
| bioservices | |
| SharedArray | |
| pyabc # dependency for stochkit, HD 20.07.20 Gemma Noviello | |
| petab | |
| tadtool # HD 26.01.21 M. R. | |
| pyperf # Paul sep-20 | |
| umap # HD, used by M. L. | |
| MACS3 | |
| biopython # HD 2021-07-23 Magalhaes/Hnisz alphafold related -- cool that we have missed biopython so far, LOL | |
| # prevent dm-haiku from fetching 'brand-new' stuff, thus avoiding further havoc | |
| tensorstore==0.1.45 # tensorflow | |
| orbax-checkpoint==0.4.4 # tensorflow | |
| dm-tree | |
| dm-haiku | |
| flatbuffers | |
| contextlib2 | |
| ml_collections | |
| python-bioformats==4.0.6 # 4.0.7 would pull up too much other packages | |
| # additions/fixups since 3.9 | |
| spyder | |
| spyder-kernels | |
| papermill # turns jupyter notebooks into scripts [sic!] used by P.A. [who else, *grin*] | |
| opencv-python | |
| scikit-image | |
| git-cola | |
| cffi-glpk # uses /usr/lib/libglpk.so.40, see if I can get it linked statically - kreitler | |
| czifile | |
| # watch https://pypi.org/project/napari/#history | |
| napari[pyqt5]==0.4.19rc2 # 0.4.18 would turn into a psychotic downgrader via numpydoc ... | |
| napari-czifile2 | |
| CrossMap | |
| scanpy | |
| asciidoc | |
| triton | |
| # additions/fixups since 3.10 | |
| aicspylibczi | |
| filetype | |
| gmpy2 | |
| # nf-core / epi2me related | |
| dominate | |
| libsass | |
| sigfig | |
| whatshap | |
| Truvari | |
| __PKGLIST__ | |
| ) | |
| for PKG in $PACKAGES; do | |
| install $PKG | |
| done | |
| # remove unneeded dependency, and save the current pillow, then go for pyimagej, nerve ... | |
| ( | |
| LABELING_VERSION=0.1.13 | |
| # https://files.pythonhosted.org/packages/71/b0/c3c3c6931850c05e548c0a8cfadb8e6d8c5bd5ad18dce90ad4c909db5867/labeling-0.1.13.tar.gz | |
| BEEHIVE=https://beehive.molgen.mpg.de/6da91e43f9864e9f4d4acb252eb486eb/labeling-0.1.13.tar.gz | |
| test -e labeling-${LABELING_VERSION}.tar.gz || wget -nv $BEEHIVE | |
| test -d labeling-${LABELING_VERSION} || tar -xf labeling-${LABELING_VERSION}.tar.gz | |
| cd labeling-${LABELING_VERSION} | |
| # not needed (tests work...) | |
| sed -i -e '/pillow/ s/<10//' ./setup.py | |
| sed -i -e '/pillow/ s/<10//' ./labeling.egg-info/requires.txt | |
| pip install . --prefix=$PREFIX | |
| piplist; track labeling==${LABELING_VERSION} | |
| ) | |
| install pyimagej # Microscopy | |
| # torch bundled with a recent CUDA-Toolkit | |
| install torch torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/cu118 | |
| install pyopengl # Tk binding creates an error, so remove it (Expects Tk Togl installation...) | |
| rm -rf ${PREFIX}/lib/python$PYTHONVER/site-packages/OpenGL/Tk | |
| mkdir -vp $PREFIX/share/jupyter/lab/settings # try to defeat a silly error in jupyter labextension install | |
| # --------------------------------------------------- Install misc packages | |
| # these mostly bigger packages require either wheel, other treatments, | |
| # or java. | |
| # GYP is a Meta-Build system: a build system that generates other build systems. https://gyp.gsrc.io/ | |
| ( | |
| set -x | |
| GYP_VERSION=20230301_c6d8b9f7 | |
| # orig: https://mirror.netcologne.de/slackware/slackware64-current/source/d/gyp/ | |
| BEEHIVE=https://beehive.molgen.mpg.de/ba0357be23ca55555714261afccc7aab/gyp-20230301_c6d8b9f7.tar.xz | |
| test -e gyp-${GYP_VERSION}.tar.xz || wget -nv $BEEHIVE | |
| test -d gyp-${GYP_VERSION} || tar -xf gyp-${GYP_VERSION}.tar.xz | |
| cd gyp-${GYP_VERSION} | |
| sed -ne '/^#Epatch:gyp/ s/^#Epatch:\S* // p' $ME | patch -p1 --verbose | |
| pip install . | |
| ) | |
| piplist; track gyp | |
| ( | |
| BOOST_VERSION=(1.80.0 1_80_0) | |
| PKGCFLAGS="-O2 -fPIC" | |
| # orig: https://boostorg.jfrog.io/artifactory/main/release/1.80.0/source/boost_1_80_0.tar.gz | |
| BEEHIVE=https://beehive.molgen.mpg.de/077f074743ea7b0cb49c6ed43953ae95/boost_1_80_0.tar.gz | |
| test -e boost_${BOOST_VERSION[1]}.tar.gz || wget -nv $BEEHIVE | |
| test -d boost-${BOOST_VERSION[0]} || mkdir boost-${BOOST_VERSION[0]} && tar -xf boost_${BOOST_VERSION[1]}.tar.gz -C boost-${BOOST_VERSION[0]} --strip-components=1 | |
| cd boost-${BOOST_VERSION[0]} | |
| sed -e "/using python/ s|;|: ${PREFIX}/include/python${PYTHONVER} ;|" \ | |
| -i bootstrap.sh | |
| ./bootstrap.sh \ | |
| --with-toolset=gcc \ | |
| --with-icu \ | |
| --with-python=$PREFIX/bin/python3 \ | |
| --with-libraries=python,thread || exit 1 | |
| ./b2 \ | |
| -j $NPROC \ | |
| --layout=system \ | |
| --build-dir=build-python3 \ | |
| --prefix=$PREFIX \ | |
| --libdir=$PREFIX/lib \ | |
| --build-type=minimal \ | |
| toolset=gcc \ | |
| variant=release \ | |
| debug-symbols=off \ | |
| link=shared \ | |
| threading=multi \ | |
| runtime-link=shared \ | |
| python=$PYTHONVER \ | |
| cflags="$PKGCFLAGS" \ | |
| cxxflags="$PKGCFLAGS" \ | |
| install || exit 1 | |
| ) | |
| ( | |
| PYCUDAVERS=2023.1 | |
| PYTHON_LIBNAME=boost_python${PYTHONVER/./} | |
| # orig: https://files.pythonhosted.org/packages/dd/b2/e30282f3286ddad05ef44548fa5c306a179ed3baceefa699af078e49ce6e/pycuda-2023.1.tar.gz | |
| BEEHIVE="https://beehive.molgen.mpg.de/3193e66e54242ac634825b14944295ac/pycuda-2023.1.tar.gz" | |
| test -e pycuda-${PYCUDAVERS}.tar.gz || wget -nv $BEEHIVE | |
| test -d pycuda-${PYCUDAVERS} || tar -xf pycuda-${PYCUDAVERS}.tar.gz | |
| cd pycuda-${PYCUDAVERS} | |
| test -e siteconf.py && rm siteconf.py | |
| python configure.py \ | |
| --cuda-enable-gl \ | |
| --cuda-root=$CUDADIR \ | |
| --no-use-shipped-boost \ | |
| --boost-python-libname=$PYTHON_LIBNAME \ | |
| --boost-inc-dir=$PREFIX/include \ | |
| --boost-lib-dir=$PREFIX/lib \ | |
| --cudadrv-lib-dir='${CUDA_ROOT}/lib,${CUDA_ROOT}/lib/stubs' \ | |
| --cudart-lib-dir='${CUDA_ROOT}/lib,${CUDA_ROOT}/lib/stubs' \ | |
| --curand-lib-dir='${CUDA_ROOT}/lib,${CUDA_ROOT}/lib/stubs' \ | |
| --ldflags=-L$PREFIX/lib | |
| pip install . --prefix=$PREFIX | |
| ) | |
| piplist; track pycuda | |
| ( | |
| # install from git to get .so files in case | |
| pip install git+https://github.com/open2c/pairtools.git@v1.0.2 --prefix=$PREFIX | |
| pip install git+https://github.com/brentp/bwa-meth.git@v0.2.7 --prefix=$PREFIX | |
| # versions > 1.5 will not work with pip install github | |
| pip install git+https://github.com/stacked-git/stgit.git@v1.5 --prefix=$PREFIX | |
| ) | |
| piplist; track pairtools bwa-meth stgit | |
| ( | |
| IDR_VERSION=2.0.3 | |
| # orig: https://github.com/nboley/idr/archive/2.0.3/idr-2.0.3.tar.gz # HD: T. Z., 18.3.21 | |
| BEEHIVE=https://beehive.molgen.mpg.de/1bec61b526bb1dcf92a5ab99c27852d0/idr-2.0.3.tar.gz | |
| test -e idr-${IDR_VERSION}.tar.gz || wget -nv $BEEHIVE | |
| test -d idr-${IDR_VERSION} || tar -xf idr-${IDR_VERSION}.tar.gz | |
| cd idr-${IDR_VERSION} | |
| pip install . --prefix=$PREFIX | |
| ) | |
| piplist; track IDR | |
| # ------------------------------------------------ finally build tensorflaw | |
| ( | |
| test -n "$SKIPTENSORFLOW" && exit | |
| TFVERSION=2.14.1 | |
| NCCLSHORTVERSION=$(pkg-config nccl --modversion | grep -Po '\d+\.\d+') | |
| # need newer jre to handle --add-opens=java.base/java.lang=ALL-UNNAMED, and bazel >= 6.1 | |
| . /pkg/openjdk-21.0.0.35-0/profile | |
| . /pkg/bazel-6.2.1-0/profile | |
| # orig: https://github.com/tensorflow/tensorflow/archive/refs/tags/v2.14.1.tar.gz | |
| BEEHIVE=https://beehive.molgen.mpg.de/c7ba144a4b1b0a6d5fea5ecd274172d3/tensorflow-2.14.1.tar.gz | |
| test -e tensorflow-${TFVERSION}.tar.gz || wget -nv $BEEHIVE | |
| test -d tensorflow-${TFVERSION} || tar -xf tensorflow-${TFVERSION}.tar.gz | |
| cd tensorflow-${TFVERSION} | |
| #vars for config | |
| export PYTHON_BIN_PATH=${PREFIX}/bin/python3 | |
| export USE_DEFAULT_PYTHON_LIB_PATH=1 | |
| export TF_PYTHON_VERSION=3.11 | |
| export TF_NEED_JEMALLOC=1 | |
| export TF_NEED_KAFKA=0 | |
| export TF_NEED_OPENCL_SYCL=0 | |
| export TF_NEED_AWS=0 | |
| export TF_NEED_GCP=0 | |
| export TF_NEED_HDFS=0 | |
| export TF_NEED_S3=0 | |
| export TF_ENABLE_XLA=1 | |
| export TF_NEED_GDR=0 | |
| export TF_NEED_VERBS=0 | |
| export TF_NEED_OPENCL=0 | |
| export TF_NEED_MPI=0 | |
| export TF_NEED_TENSORRT=0 | |
| export TF_NEED_NGRAPH=0 | |
| export TF_NEED_IGNITE=0 | |
| export TF_NEED_ROCM=0 | |
| export TF_SET_ANDROID_WORKSPACE=0 | |
| export TF_DOWNLOAD_CLANG=0 | |
| export TF_NCCL_VERSION=${NCCLSHORTVERSION} | |
| export TF_IGNORE_MAX_BAZEL_VERSION=1 | |
| export NCCL_INSTALL_PATH=${CUDADIR} | |
| export CC=gcc | |
| export CXX=g++ | |
| export GCC_HOST_COMPILER_PATH=/usr/bin/${CC} | |
| export HOST_C_COMPILER=/usr/bin/${CC} | |
| export HOST_CXX_COMPILER=/usr/bin/${CXX} | |
| export TF_CUDA_CLANG=0 | |
| export CLANG_CUDA_COMPILER_PATH=/usr/bin/clang | |
| export TF_CUDA_PATHS=${CUDADIR},${CUDADIR}/extras/CUPTI,${CUDADIR}/nvvm | |
| export TF_CUDA_VERSION=$(${CUDADIR}/bin/nvcc --version | sed -n 's/^.*release \(.*\),.*/\1/p') # 11.5 | |
| export TF_CUDNN_VERSION=$(sed -n 's/^#define CUDNN_MAJOR\s*\(.*\).*/\1/p' ${CUDADIR}/include/cudnn_version.h) | |
| # hardware older than sm_52 is said to be unsupported, compute_86 denotes an emulation | |
| export TF_CUDA_COMPUTE_CAPABILITIES=sm_52,sm_53,sm_60,sm_61,sm_62,sm_70,sm_72,sm_75,sm_80,sm_86,sm_87,sm_89,sm_90,compute_90 | |
| # stay compatible with the cluster machines | |
| # https://gcc.gnu.org/onlinedocs/gcc/x86-Options.html | |
| export CC_OPT_FLAGS="-march=broadwell" # or haswell to be safe ? no more nehalem | |
| export TF_NEED_CUDA=1 | |
| export BAZEL_ARGS="--config=mkl -c opt" | |
| # avoid: Auto-Configuration Warning: 'TMP' environment variable is not set, using 'C:\Windows\Temp' as default :/ | |
| export TMP=$BUILD_TMPDIR/tf_tmp | |
| mkdir -p $TMP | |
| ./configure | |
| bazel build \ | |
| --cxxopt="-march=broadwell" \ | |
| --copt="-march=broadwell" \ | |
| --noshow_loading_progress \ | |
| --noshow_progress \ | |
| --color="no" \ | |
| //tensorflow:libtensorflow.so \ | |
| //tensorflow:libtensorflow_cc.so \ | |
| //tensorflow:install_headers \ | |
| //tensorflow/tools/pip_package:build_pip_package | |
| bazel shutdown # otherwise it hogs in cwd | |
| # create a _clean/decent_ python package from so far created debris | |
| test -d tensorflow-pip && rm -r tensorflow-pip | |
| mkdir tensorflow-pip | |
| ./bazel-bin/tensorflow/tools/pip_package/build_pip_package --src ${BUILDDIR}/tensorflow-pip | |
| cd ${BUILDDIR} | |
| # ... cope with the weird symlink-structure placed in bazels' build-TMPDIR (-h) (tell me why ?) | |
| test -e tensorflow-pip.tar && rm tensorflow-pip.tar | |
| tar -chf tensorflow-pip.tar tensorflow-pip | |
| # and perform a regular install | |
| test -d tensorflow-pip && rm -r tensorflow-pip | |
| tar -xf tensorflow-pip.tar | |
| cd ${BUILDDIR}/tensorflow-pip | |
| pip install . --prefix=$PREFIX | |
| ) | |
| piplist; track tensorflow | |
| # tidy up a bit | |
| if [ -d $PREFIX/man/man1 ]; then | |
| mv -v $PREFIX/man/man1/* $PREFIX/share/man/man1 || true | |
| rm -rf $PREFIX/man | |
| fi | |
| # ------------------------------------------------------------ Sanity check | |
| # load all packages, see warnings and spot installation errors | |
| # But don't care about r2py, a further dependence into R is not wanted! | |
| echo "# START load test." | |
| python -c 'help("modules")' > /dev/null | |
| echo "# END load test (passed)." | |
| echo "# running pip check" | |
| pip check | |
| # I guess it makes sense that files can be read by all users ... | |
| echo "# check for unreadable files" | |
| cd $PREFIX | |
| for D in bin etc include lib share; do | |
| find $D \! -perm -004 -exec chmod -c a+r {} + | |
| done | |
| echo "For Deeper checks:" | |
| echo "cd ${BUILDDIR} ; git log --patch " | |
| exit | |
| # -------------------------------------------------------- Included patches | |
| #Epatch:pip # Operation 'Forever young', disable version-check per default. | |
| #Epatch:pip # The option needs to be kept, because some tools make use of | |
| #Epatch:pip # it and would cause pip to fail. | |
| #Epatch:pip --- a/pip/_internal/cli/cmdoptions.py 2022-08-09 17:29:15.853442948 +0200 | |
| #Epatch:pip +++ b/pip/_internal/cli/cmdoptions.py 2022-08-10 10:31:35.124945154 +0200 | |
| #Epatch:pip @@ -892,7 +892,6 @@ | |
| #Epatch:pip dest="disable_pip_version_check", | |
| #Epatch:pip action="store_true", | |
| #Epatch:pip - default=False, | |
| #Epatch:pip - help="Don't periodically check PyPI to determine whether a new version " | |
| #Epatch:pip - "of pip is available for download. Implied with --no-index.", | |
| #Epatch:pip + default=True, | |
| #Epatch:pip + help="Ignore this option, the version check IS disabled.", | |
| #Epatch:pip ) | |
| #Epatch:gyp --- a/pylib/gyp/generator/cmake.py | |
| #Epatch:gyp +++ b/pylib/gyp/generator/cmake.py | |
| #Epatch:gyp @@ -41,7 +41,7 @@ | |
| #Epatch:gyp try: | |
| #Epatch:gyp # maketrans moved to str in python3. | |
| #Epatch:gyp + _maketrans = str.maketrans | |
| #Epatch:gyp +except NameError: | |
| #Epatch:gyp _maketrans = string.maketrans | |
| #Epatch:gyp -except NameError: | |
| #Epatch:gyp - _maketrans = str.maketrans | |
| #Epatch:gyp generator_default_variables = { | |
| #Epatch:gyp 'EXECUTABLE_PREFIX': '', | |
| #Epatch:gyp @@ -283,2 +283,2 @@ | |
| #Epatch:gyp if int(action.get('process_outputs_as_sources', False)): | |
| #Epatch:gyp - extra_sources.extend(zip(cmake_outputs, outputs)) | |
| #Epatch:gyp + extra_sources.extend(list(zip(cmake_outputs, outputs))) | |
| #Epatch:gyp @@ -988,6 +988,6 @@ | |
| #Epatch:gyp # XCode settings | |
| #Epatch:gyp xcode_settings = config.get('xcode_settings', {}) | |
| #Epatch:gyp - for xcode_setting, xcode_value in xcode_settings.viewitems(): | |
| #Epatch:gyp + for xcode_setting, xcode_value in xcode_settings.items(): | |
| #Epatch:gyp SetTargetProperty(output, cmake_target_name, | |
| #Epatch:gyp "XCODE_ATTRIBUTE_%s" % xcode_setting, xcode_value, | |
| #Epatch:gyp '' if isinstance(xcode_value, str) else ' ') | |
| #Epatch:gyp diff --git a/pylib/gyp/input.py b/pylib/gyp/input.py | |
| #Epatch:gyp index 4c12891..2bea334 100644 | |
| #Epatch:gyp --- a/pylib/gyp/input.py | |
| #Epatch:gyp +++ b/pylib/gyp/input.py | |
| #Epatch:gyp @@ -1183,7 +1183,7 @@ | |
| #Epatch:gyp if variable_name in variables: | |
| #Epatch:gyp # If the variable is already set, don't set it. | |
| #Epatch:gyp continue | |
| #Epatch:gyp - if the_dict_key is 'variables' and variable_name in the_dict: | |
| #Epatch:gyp + if the_dict_key == 'variables' and variable_name in the_dict: | |
| #Epatch:gyp # If the variable is set without a % in the_dict, and the_dict is a | |
| #Epatch:gyp # variables dict (making |variables| a varaibles sub-dict of a | |
| #Epatch:gyp # variables dict), use the_dict's definition. | |
| #Epatch:javabridge diff --git a/_javabridge_nomac.pyx b/_javabridge_nomac.pyx | |
| #Epatch:javabridge index 9f28939..c437db0 100644 | |
| #Epatch:javabridge --- a/_javabridge_nomac.pyx | |
| #Epatch:javabridge +++ b/_javabridge_nomac.pyx | |
| #Epatch:javabridge @@ -16,30 +16,30 @@ cdef extern from "jni.h": | |
| #Epatch:javabridge jint (*AttachCurrentThreadAsDaemon)(JavaVM *vm, void *penv, void *args) nogil | |
| #Epatch:javabridge jint JNI_CreateJavaVM(JavaVM **pvm, void **penv, void *args) nogil | |
| #Epatch:javabridge | |
| #Epatch:javabridge -cdef int MacStartVM(JavaVM **pvm, JavaVMInitArgs *pVMArgs, char *class_name) nogil: | |
| #Epatch:javabridge +cdef extern int MacStartVM(JavaVM **pvm, JavaVMInitArgs *pVMArgs, char *class_name) nogil: | |
| #Epatch:javabridge return -1 | |
| #Epatch:javabridge | |
| #Epatch:javabridge -cdef void StopVM(JavaVM *vm) nogil: | |
| #Epatch:javabridge +cdef extern void StopVM(JavaVM *vm) nogil: | |
| #Epatch:javabridge vm[0].DestroyJavaVM(vm) | |
| #Epatch:javabridge | |
| #Epatch:javabridge -cdef void MacRunLoopInit() nogil: | |
| #Epatch:javabridge +cdef extern void MacRunLoopInit() nogil: | |
| #Epatch:javabridge pass | |
| #Epatch:javabridge | |
| #Epatch:javabridge -cdef void MacRunLoopRun() nogil: | |
| #Epatch:javabridge +cdef extern void MacRunLoopRun() nogil: | |
| #Epatch:javabridge pass | |
| #Epatch:javabridge | |
| #Epatch:javabridge -cdef void MacRunLoopStop() nogil: | |
| #Epatch:javabridge +cdef extern void MacRunLoopStop() nogil: | |
| #Epatch:javabridge pass | |
| #Epatch:javabridge | |
| #Epatch:javabridge -cdef void MacRunLoopReset() nogil: | |
| #Epatch:javabridge +cdef extern void MacRunLoopReset() nogil: | |
| #Epatch:javabridge pass | |
| #Epatch:javabridge | |
| #Epatch:javabridge -cdef int MacIsMainThread() nogil: | |
| #Epatch:javabridge +cdef extern int MacIsMainThread() nogil: | |
| #Epatch:javabridge return 0 | |
| #Epatch:javabridge | |
| #Epatch:javabridge -cdef void MacRunLoopRunInMode(double timeout) nogil: | |
| #Epatch:javabridge +cdef extern void MacRunLoopRunInMode(double timeout) nogil: | |
| #Epatch:javabridge pass | |
| #Epatch:javabridge | |
| #Epatch:javabridge -cdef int CreateJavaVM(JavaVM **pvm, void **pEnv, void *args) nogil: | |
| #Epatch:javabridge +cdef extern int CreateJavaVM(JavaVM **pvm, void **pEnv, void *args) nogil: | |
| #Epatch:javabridge return JNI_CreateJavaVM(pvm, pEnv, args) | |
| #Epatch:javabridge | |
| #Epatch:javabridge diff --git a/_javabridge_mac.pyx b/_javabridge_mac.pyx | |
| #Epatch:javabridge index 2e3c524..daed790 100644 | |
| #Epatch:javabridge --- a/_javabridge_mac.pyx | |
| #Epatch:javabridge +++ b/_javabridge_mac.pyx | |
| #Epatch:javabridge @@ -31,11 +31,11 @@ cdef extern from "mac_javabridge_utils.h": | |
| #Epatch:javabridge int MacIsMainThread() nogil | |
| #Epatch:javabridge void MacRunLoopRunInMode(double) nogil | |
| #Epatch:javabridge | |
| #Epatch:javabridge -cdef void StopVM(JavaVM *vm): | |
| #Epatch:javabridge +cdef extern void StopVM(JavaVM *vm): | |
| #Epatch:javabridge MacStopVM() | |
| #Epatch:javabridge | |
| #Epatch:javabridge # | |
| #Epatch:javabridge # Unused stub in Mac | |
| #Epatch:javabridge # | |
| #Epatch:javabridge -cdef int CreateJavaVM(JavaVM **pvm, void **pEnv, void *args): | |
| #Epatch:javabridge +cdef extern int CreateJavaVM(JavaVM **pvm, void **pEnv, void *args): | |
| #Epatch:javabridge return -1 | |
| # -------------------------------------------------------- This is the end. | |