Skip to content
Permalink
master
Switch branches/tags

Name already in use

A tag already exists with the provided branch name. Many Git commands accept both tag and branch names, so creating this branch may cause unexpected behavior. Are you sure you want to create this branch?
Go to file
 
 
Cannot retrieve contributors at this time
executable file 895 lines (731 sloc) 29 KB
#!/bin/bash
# ------------------------------------------------------------------- Debug
COOKIE=$(mcookie|cut -c-8); grep -v V_GREP_ME $0 > /dev/shm/runme-$COOKIE.sh ; sleep 0.3; exec bash /dev/shm/runme-$COOKIE.sh
# TESTING=1
# SKIPTENSORFLOW=1
# ---------------------------------------------------------------- Preamble
PKG=python
VERSION=3.11.7
BUILD=1
PYTHONVER=${VERSION%.*}
CUDADIR=/pkg/cuda-11.8.0-0
PREFIX=/pkg/$PKG-$VERSION-$BUILD
if [ -n "$TESTING" ]; then PREFIX=/scratch/local2/$PKG-$VERSION-$BUILD ; fi
PATH=/bin:/usr/bin:/usr/sbin:/usr/local/bin
set -e
umask 022
BUILD_TMPDIR=/scratch/local2/$PKG-$VERSION-$BUILD.$USER.build.tmp
# keep pip's download cache when testing
test -z "$TESTING" && test -d $BUILD_TMPDIR && ( chmod -R u+rwx $BUILD_TMPDIR || true ; rm -rf $BUILD_TMPDIR )
mkdir -p $BUILD_TMPDIR/home/.cache/pip
mkdir -p $BUILD_TMPDIR/home/.local
# copy script to an accessible location
cp $0 $BUILD_TMPDIR; ME="$BUILD_TMPDIR/$(basename $0)"
export TMPDIR=$BUILD_TMPDIR
export HOME=$BUILD_TMPDIR/home
exec </dev/null
mkdir -p $PREFIX
cat >$PREFIX/profile <<-EOF
. $CUDADIR/profile
PATH=$PREFIX/bin:\$PATH
export LD_LIBRARY_PATH=$PREFIX/lib\${LD_LIBRARY_PATH:+:}\$LD_LIBRARY_PATH
PKG_CONFIG_PATH=$PREFIX/lib/pkgconfig\${PKG_CONFIG_PATH:+:}\$PKG_CONFIG_PATH
export PKG_CONFIG_PATH
if [ -d $PREFIX/.compatlibs ]; then export LD_LIBRARY_PATH=$PREFIX/.compatlibs\${LD_LIBRARY_PATH:+:}\$LD_LIBRARY_PATH ; fi
EOF
. $PREFIX/profile
NPROC=$(( $(nproc) * 4 / 5 + 1 ))
export MAKEFLAGS="-j $NPROC"
BUILDDIR=$PREFIX/build
mkdir -p $BUILDDIR
cd $BUILDDIR
# -------------------------------------- Use git to track package evolution
test -d .git && rm -rf .git
git init -q; echo -e '[user]\n name = none\n email = of_your_business...' >> .git/config
echo '*' > .gitignore
function piplist() {
pip list | awk '{ printf("%-36s %s\n", $1,$2) }' | grep -v '\----' > GameOfVersions
}
function track() {
if [ -d .git ]; then
if git add -f GameOfVersions; then
git commit -q -n -m "Package: '$*'" || /bin/true
fi
fi
}
function install() {
echo "# ::INST:: # '${@+$@}'"
# no-color is nice, but doesn't work when dependencies are installed, so (mis-)use a pipe
pip install --no-color --compile --cache-dir=$HOME/.cache/pip --prefix=$PREFIX "${@+$@}" | cat
piplist
track $*
}
# ---------------------------------------- Build Python & support libraries
(
# orig: https://www.python.org/ftp/python/3.11.7/Python-3.11.7.tgz
BEEHIVE=https://beehive.molgen.mpg.de/ef61f81ec82c490484219c7f0ec96783/Python-3.11.7.tgz
test -e Python-$VERSION.tar.xz || wget -nv $BEEHIVE
test -d Python-$VERSION || tar -xf Python-$VERSION.tgz
cd Python-$VERSION
# BerkeleyDB, the undead ...
# Newer libgdbm_compat.so versions pull libgdbm.so on their own, our libgdbm_compat.so is older ...
# So, let setup.py create '-lgdbm -lgdbm_compat' instead of a sole '-lgdbm_compat'
# verbatim: ndbm_libs = ['gdbm_compat'] => ndbm_libs = ['gdbm','gdbm_compat']
sed -i -e "/ndbm_libs = / s/_compat'/','gdbm_compat'/" setup.py
# leave further hints in the log
sed -i -e '/db_setup_debug = False/ s/False/True/' \
-e '/dbm_setup_debug = False/ s/False/True/' setup.py
# These may come handy if the Python build needs exclusive stuff from $PREFIX
# INCLUDEDIR=$PREFIX/include \
# LIBDIR=$PREFIX/lib \
LDFLAGS="-Wl,-rpath=$PREFIX/lib" \
./configure \
--prefix=$PREFIX \
--enable-shared
make -j $NPROC
make install
# it might come to happen that something down the line may just call 'python', defuse ...
test -e $PREFIX/bin/python || ln -s python3 $PREFIX/bin/python
python -m ensurepip
pip3 install --prefix=$PREFIX -I pip
( # fix 'please update' noise from pip
cd $PREFIX/lib/python*/site-packages
sed -ne '/^#Epatch:pip/ s/^#Epatch:\S* // p' $ME | patch -p1 --verbose
)
# libblosc2 installs in $PREFIX/lib64, I hate to do this ...
ln -s lib $PREFIX/lib64
)
install setuptools
piplist; track START
# build various pythonbindings 'by hand', because pip installs do not deliver
# header files and the like.
# BTW: builds need meson
install meson
(
DBUS_PYTHON_VERSION=1.3.2
# PyPi
# https://files.pythonhosted.org/packages/c1/d3/6be85a9c772d6ebba0cc3ab37390dd6620006dcced758667e0217fb13307/dbus-python-1.3.2.tar.gz
BEEHIVE=https://beehive.molgen.mpg.de/33be8a4a766e1c7c9a377b8f934ce21a/dbus-python-1.3.2.tar.gz
test -e dbus-python-$DBUS_PYTHON_VERSION.tar.gz || wget -nv $BEEHIVE
test -d dbus-python-$DBUS_PYTHON_VERSION || tar -xf dbus-python-$DBUS_PYTHON_VERSION.tar.gz
cd dbus-python-$DBUS_PYTHON_VERSION
./configure --prefix=$PREFIX
make -j $NPROC
make install
)
(
PYCAIRO_VERSION=1.23.0
# PyPi
# https://files.pythonhosted.org/packages/69/ca/9e9fa2e8be0876a9bbf046a1be7ee33e61d4fdfbd1fd25c76c1bdfddf8c4/pycairo-1.23.0.tar.gz
BEEHIVE=https://beehive.molgen.mpg.de/7a3729d21659098e1b9a411b62e88966/pycairo-1.23.0.tar.gz
test -e pycairo-$PYCAIRO_VERSION.tar.gz || wget -nv $BEEHIVE
test -d pycairo-$PYCAIRO_VERSION || tar -xf pycairo-$PYCAIRO_VERSION.tar.gz
cd pycairo-$PYCAIRO_VERSION
mkdir -p meson-build
cd meson-build
meson setup \
--prefix=$PREFIX \
--buildtype=release \
-Dpython=$PREFIX/bin/python \
-Dtests=false \
..
ninja -j $NPROC
ninja install
)
(
PYGOBJECT3_VERSION=3.44.1
# https://files.pythonhosted.org/packages/07/73/a034fc1bcd6d402d6f72ff618c093f91ac6921f968cc19806b6a1b1b19c8/PyGObject-3.44.0.tar.gz
BEEHIVE=https://beehive.molgen.mpg.de/959259478566b24316edaa18da6e0eaa/PyGObject-3.44.1.tar.gz
test -e PyGObject-$PYGOBJECT3_VERSION.tar.gz || wget -nv $BEEHIVE
test -d PyGObject-$PYGOBJECT3_VERSION || tar -xf PyGObject-$PYGOBJECT3_VERSION.tar.gz
cd PyGObject-$PYGOBJECT3_VERSION
mkdir -p meson-build
cd meson-build
meson setup \
--prefix=$PREFIX \
--buildtype=release \
-Dpython=$PREFIX/bin/python \
-Dtests=false \
..
ninja -j $NPROC
ninja install
)
piplist; track various pythonbindings
# build support to use QT from python -- see www.riverbankcomputing.com
( # sip: tool to create Python bindings for C and C++ libraries.
SIP_VERSION=6.7.7
# https://files.pythonhosted.org/packages/5b/cb/c27c925ae07bd03a2597fa1db17bfc2a4ac57da61aeb90f8ec98ffbb975b/sip-6.6.2.tar.gz
BEEHIVE=https://beehive.molgen.mpg.de/a131efd79649d31ee11601b19d1e3784/sip-6.7.7.tar.gz
test -e sip-$SIP_VERSION.tar.gz || wget -nv $BEEHIVE
test -d sip-$SIP_VERSION || tar -xf sip-$SIP_VERSION.tar.gz
cd sip-$SIP_VERSION
pip install .
)
# QT bindings need some python modules to build
install PyQt5-sip
install PyQt-builder
(
PYQT_VERSION=5.15.9
# orig: https://files.pythonhosted.org/packages/5c/46/b4b6eae1e24d9432905ef1d4e7c28b6610e28252527cdc38f2a75997d8b5/PyQt5-5.15.9.tar.gz
BEEHIVE=https://beehive.molgen.mpg.de/9d97fc06b7ae75e654e946c49e07ff12/PyQt5-5.15.9.tar.gz
test -e PyQt5-$PYQT_VERSION.tar.gz || wget -nv $BEEHIVE
test -d PyQt5-$PYQT_VERSION || tar -xf PyQt5-$PYQT_VERSION.tar.gz
cd PyQt5-$PYQT_VERSION
sip-build \
--confirm-license \
--no-make \
--api-dir $PREFIX/share/qsci/api/python \
--dbus $PREFIX/include/dbus-1.0 \
--no-designer-plugin \
--no-qml-plugin \
--jobs $NPROC
cd build
make -j $NPROC
make install
# Remove unused py2 version of uic modules
rm -r $PREFIX/lib/python*/site-packages/PyQt5/uic/port_v2
)
(
QSCINTILLA_VERSION=2.13.4 # no 2.14 in sight ...
# orig: https://www.riverbankcomputing.com/static/Downloads/QScintilla/2.13.4/QScintilla_src-2.13.4.tar.gz
BEEHIVE=https://beehive.molgen.mpg.de/4f83a4a4ad7da40eae80ad23f9fb18f2/QScintilla_src-2.13.4.tar.gz
test -e QScintilla_src-$QSCINTILLA_VERSION.tar.gz || wget -nv $BEEHIVE
test -d QScintilla_src-$QSCINTILLA_VERSION || tar -xf QScintilla_src-$QSCINTILLA_VERSION.tar.gz
cd QScintilla_src-$QSCINTILLA_VERSION
cd Python
cp -vp pyproject-qt5.toml pyproject.toml
sip-build \
--no-make \
--qsci-features-dir ../src/features \
--qsci-include-dir ../src \
--qsci-library-dir ../src \
--api-dir $PREFIX/share/qsci/api/python \
--jobs $NPROC
cd build
make -j $NPROC
make install
)
(
set -x
PYQTWEBENGINE_VERSION=5.15.6
#orpg: https://pypi.org/project/PyQtWebEngine/#files
BEEHIVE="https://beehive.molgen.mpg.de/d1e548617de149ba8142a32c1eb9f994/PyQtWebEngine-5.15.6.tar.gz"
test -e PyQtWebEngine-${PYQTWEBENGINE_VERSION}.tar.gz || wget -nv $BEEHIVE
test -d PyQtWebEngine-${PYQTWEBENGINE_VERSION} || tar -xf PyQtWebEngine-${PYQTWEBENGINE_VERSION}.tar.gz
cd PyQtWebEngine-${PYQTWEBENGINE_VERSION}
sip-build \
--no-make \
--api-dir ${PREFIX}/share/qsci/api/python \
--jobs $NPROC
cd build
make -j $NPROC
make install
)
piplist; track qt-support-riverbankcomputing
# -------------------------------------------------------- Install packages
install Cython
# libxml2-python from pypi isn't up to date, cause lxml seems to be the favored binding
# do this early, not that some dependency tries it on its own.
# (btw. libxslt-python isn't ported to python3)
(
LIBXML2_VERSION=2.9.12
# orig: http://xmlsoft.org/sources/libxml2-${PKGVERSION}.tar.gz
BEEHIVE=https://beehive.molgen.mpg.de/f433a39be087a9f0b197eb2307ad9f75/libxml2-2.9.12.tar.gz
test -e libxml2-$LIBXML2_VERSION.tar.gz || wget -nv $BEEHIVE
test -d libxml2-$LIBXML2_VERSION || tar -xf libxml2-$LIBXML2_VERSION.tar.gz libxml2-$LIBXML2_VERSION/python
cd libxml2-$LIBXML2_VERSION/python
pip install .
)
piplist; track legacy-libxml2
# Install the most 'sensitive' packages first
install numpy==1.25.2
install numba==0.58.1
install wheel
# python-javabridge, problems with python 3.11 and recent cython, thus
# regenerate C bindings from *pxd, *pyx -- so must use github, and wheel
(
# use the fork from cellprofiler, as used in pypi
git clone https://github.com/CellProfiler/python-javabridge
cd python-javabridge
git checkout v4.0.3
sed -ne '/^#Epatch:javabridge/ s/^#Epatch:\S* // p' $ME | patch -p1 --verbose
pip install .
)
PACKAGES=$(sed -e s/#.*$// <<- __PKGLIST__
igraph==0.10.8 # louvain
keras==2.14.0 # tensorflow 2.14.1
ml-dtypes==0.2.0 # tensorflow, dm-haiku, jax, tensorstore
pytest==6.2.5
pytest-asyncio==0.19.0 # to get rid of a deprecation warning in 0.20.3, -> 0.19.0
urllib3==2.0.7
ipython[all]
wrapt==1.14.1 # tensorflow vs. snakemake, snakemake 8.1.1 insists somehow on 1.16.0
snakemake==7.32.4 # December 2024 brougt snakemake 8++
triton=2.1.0 # pytorch
Pillow
protobuf
onnx
Sphinx
jedi
black[d] # onnx wants it, but does not install it ...
flask
SciPy
pandas
SymPy
nose
statsmodels
pysam
seaborn
scikit-learn
Mako
cutadapt # Helpdesk 01/24/18 16:19, S. K.
pygobject # Helpdesk 03/26/18 14:19, V. S.
blink1 # Blink1-lib 07/04/18 12:00, N. H.
hvplot # ont/nextflow pipelines
pyarrow # ont/nextflow pipelines
umi-tools
recommonmark
fastcluster
pydot
pygraphviz
deeptools
HTSeq
keras_preprocessing # tensorflow ...
MotifScan # A.B. 30.8.2018
MAmotif
hic2cool # Helpdesk R. S. 17.09.2018
virtualenv # Donald
odfpy # Donald
Dumper # Thomas
louvain # monocle3 prep for H. K.
selenium # 19.7.19 T.K. -> https://webxray.org/
plotly
pyOpenCL
docopt
distro
ply
threadpoolctl
HiCMatrix # 17.1 -> https://github.com/deeptools/HiCMatrix/issues/47
pybedtools
gffutils
jupyter
jupyterlab
jupyterlab-git # request P. A.
igv_jupyterlab # M. L. HD 01-04-2020
igv-notebook
scour # https://github.molgen.mpg.de/mariux64/pkg-scripts/issues/98
multiqc
pybind11
svgwrite
pyomo
bioservices
SharedArray
pyabc # dependency for stochkit, HD 20.07.20 G. N.
petab
tadtool # HD 26.01.21 M. R.
pyperf # Paul sep-20
umap # HD, used by M. L.
MACS3
biopython # HD 2021-07-23 Magalhaes/Hnisz alphafold related -- cool that we have missed biopython so far, LOL
# prevent dm-haiku from fetching 'brand-new' stuff, thus avoiding further havoc
tensorstore==0.1.45 # tensorflow
orbax-checkpoint==0.4.4 # tensorflow
dm-tree
dm-haiku
flatbuffers
contextlib2
ml_collections
python-bioformats==4.0.6 # 4.0.7 would pull up too much other packages
# additions/fixups since 3.9
spyder
spyder-kernels
papermill # turns jupyter notebooks into scripts [sic!] used by P.A. [who else, *grin*]
opencv-python
scikit-image
git-cola
cffi-glpk # uses /usr/lib/libglpk.so.40, see if I can get it linked statically - kreitler
czifile
# watch https://pypi.org/project/napari/#history
napari[pyqt5]==0.4.19rc7 # 0.4.18 would turn into a psychotic downgrader via numpydoc ...
napari-czifile2
bx-python
# CrossMap - coredumps by help[modules] because of a '__init__.py' in site_packages, fix is below
leidenalg
scanpy # this is a MESS, you might always have broken/ugly plots ...
asciidoc
# additions/fixups since 3.10
aicspylibczi
filetype
gmpy2
shap
xgboost
UpSetPlot
hypothesis
# nf-core / epi2me related
dominate
libsass
sigfig
whatshap
Truvari
nanopack
__PKGLIST__
)
for PKG in $PACKAGES; do
install $PKG
done
# build CrossMap from git, https://github.com/liguowang/CrossMap/issues/67
(
git clone https://github.com/liguowang/CrossMap
cd CrossMap
git rm src/__init__.py
git commit -a -m 'remove badly placed src/__init__.py'
pip install . --prefix=$PREFIX
piplist; track CrossMap
)
# remove unneeded dependency, and save the current pillow, then go for pyimagej, nerve ...
(
LABELING_VERSION=0.1.13
# https://files.pythonhosted.org/packages/71/b0/c3c3c6931850c05e548c0a8cfadb8e6d8c5bd5ad18dce90ad4c909db5867/labeling-0.1.13.tar.gz
BEEHIVE=https://beehive.molgen.mpg.de/6da91e43f9864e9f4d4acb252eb486eb/labeling-0.1.13.tar.gz
test -e labeling-${LABELING_VERSION}.tar.gz || wget -nv $BEEHIVE
test -d labeling-${LABELING_VERSION} || tar -xf labeling-${LABELING_VERSION}.tar.gz
cd labeling-${LABELING_VERSION}
# not needed (tests work...)
sed -i -e '/pillow/ s/<10//' ./setup.py
sed -i -e '/pillow/ s/<10//' ./labeling.egg-info/requires.txt
pip install . --prefix=$PREFIX
piplist; track labeling==${LABELING_VERSION}
)
install pyimagej # Microscopy
# create a wrapper that sets JAVA_HOME
mv -v $PREFIX/bin/imagej $PREFIX/bin/imagej.dist
cat << EOF > $PREFIX/bin/imagej
#! /bin/bash
export JAVA_HOME=/pkg/openjdk-21.0.0.35-0
exec $PREFIX/bin/imagej.dist "\${@+\$@}"
EOF
chmod +x $PREFIX/bin/imagej
# torch bundled with a recent CUDA-Toolkit
install torch torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/cu118
install pyopengl # Tk binding creates an error, so remove it (Expects Tk Togl installation...)
rm -rf ${PREFIX}/lib/python$PYTHONVER/site-packages/OpenGL/Tk
mkdir -vp $PREFIX/share/jupyter/lab/settings # try to defeat a silly error in jupyter labextension install
# --------------------------------------------------- Install misc packages
# these mostly bigger packages require either wheel, other treatments,
# or java.
# GYP is a Meta-Build system: a build system that generates other build systems. https://gyp.gsrc.io/
(
set -x
GYP_VERSION=20230301_c6d8b9f7
# orig: https://mirror.netcologne.de/slackware/slackware64-current/source/d/gyp/
BEEHIVE=https://beehive.molgen.mpg.de/ba0357be23ca55555714261afccc7aab/gyp-20230301_c6d8b9f7.tar.xz
test -e gyp-${GYP_VERSION}.tar.xz || wget -nv $BEEHIVE
test -d gyp-${GYP_VERSION} || tar -xf gyp-${GYP_VERSION}.tar.xz
cd gyp-${GYP_VERSION}
sed -ne '/^#Epatch:gyp/ s/^#Epatch:\S* // p' $ME | patch -p1 --verbose
pip install .
)
piplist; track gyp
(
BOOST_VERSION=(1.80.0 1_80_0)
PKGCFLAGS="-O2 -fPIC"
# orig: https://boostorg.jfrog.io/artifactory/main/release/1.80.0/source/boost_1_80_0.tar.gz
BEEHIVE=https://beehive.molgen.mpg.de/077f074743ea7b0cb49c6ed43953ae95/boost_1_80_0.tar.gz
test -e boost_${BOOST_VERSION[1]}.tar.gz || wget -nv $BEEHIVE
test -d boost-${BOOST_VERSION[0]} || mkdir boost-${BOOST_VERSION[0]} && tar -xf boost_${BOOST_VERSION[1]}.tar.gz -C boost-${BOOST_VERSION[0]} --strip-components=1
cd boost-${BOOST_VERSION[0]}
sed -e "/using python/ s|;|: ${PREFIX}/include/python${PYTHONVER} ;|" \
-i bootstrap.sh
./bootstrap.sh \
--with-toolset=gcc \
--with-icu \
--with-python=$PREFIX/bin/python3 \
--with-libraries=python,thread || exit 1
./b2 \
-j $NPROC \
--layout=system \
--build-dir=build-python3 \
--prefix=$PREFIX \
--libdir=$PREFIX/lib \
--build-type=minimal \
toolset=gcc \
variant=release \
debug-symbols=off \
link=shared \
threading=multi \
runtime-link=shared \
python=$PYTHONVER \
cflags="$PKGCFLAGS" \
cxxflags="$PKGCFLAGS" \
install || exit 1
)
(
PYCUDAVERS=2023.1
PYTHON_LIBNAME=boost_python${PYTHONVER/./}
# orig: https://files.pythonhosted.org/packages/dd/b2/e30282f3286ddad05ef44548fa5c306a179ed3baceefa699af078e49ce6e/pycuda-2023.1.tar.gz
BEEHIVE="https://beehive.molgen.mpg.de/3193e66e54242ac634825b14944295ac/pycuda-2023.1.tar.gz"
test -e pycuda-${PYCUDAVERS}.tar.gz || wget -nv $BEEHIVE
test -d pycuda-${PYCUDAVERS} || tar -xf pycuda-${PYCUDAVERS}.tar.gz
cd pycuda-${PYCUDAVERS}
test -e siteconf.py && rm siteconf.py
python configure.py \
--cuda-enable-gl \
--cuda-root=$CUDADIR \
--no-use-shipped-boost \
--boost-python-libname=$PYTHON_LIBNAME \
--boost-inc-dir=$PREFIX/include \
--boost-lib-dir=$PREFIX/lib \
--cudadrv-lib-dir='${CUDA_ROOT}/lib,${CUDA_ROOT}/lib/stubs' \
--cudart-lib-dir='${CUDA_ROOT}/lib,${CUDA_ROOT}/lib/stubs' \
--curand-lib-dir='${CUDA_ROOT}/lib,${CUDA_ROOT}/lib/stubs' \
--ldflags=-L$PREFIX/lib
pip install . --prefix=$PREFIX
)
piplist; track pycuda
(
# install from git to get .so files in case
pip install git+https://github.com/open2c/pairtools.git@v1.0.2 --prefix=$PREFIX
pip install git+https://github.com/brentp/bwa-meth.git@v0.2.7 --prefix=$PREFIX
# versions > 1.5 will not work with pip install github
pip install git+https://github.com/stacked-git/stgit.git@v1.5 --prefix=$PREFIX
)
piplist; track pairtools bwa-meth stgit
(
IDR_VERSION=2.0.3
# orig: https://github.com/nboley/idr/archive/2.0.3/idr-2.0.3.tar.gz # HD: T. Z., 18.3.21
BEEHIVE=https://beehive.molgen.mpg.de/1bec61b526bb1dcf92a5ab99c27852d0/idr-2.0.3.tar.gz
test -e idr-${IDR_VERSION}.tar.gz || wget -nv $BEEHIVE
test -d idr-${IDR_VERSION} || tar -xf idr-${IDR_VERSION}.tar.gz
cd idr-${IDR_VERSION}
pip install . --prefix=$PREFIX
)
piplist; track IDR
# ------------------------------------------------ finally build tensorflaw
(
test -n "$SKIPTENSORFLOW" && exit
TFVERSION=2.14.1
NCCLSHORTVERSION=$(pkg-config nccl --modversion | grep -Po '\d+\.\d+')
# Need newer JRE to handle --add-opens=java.base/java.lang=ALL-UNNAMED, and bazel >= 6.1
# Also: "error -- unsupported GNU version! gcc versions later than 11 are not supported! The
# nvcc flag '-allow-unsupported-compiler' can be used to override this version check"
. /pkg/openjdk-21.0.0.35-0/profile
. /pkg/bazel-6.2.1-0/profile
. /pkg/gcc-10.3.0-0/profile
pkggcc=/pkg/gcc-10.3.0-0/bin
# orig: https://github.com/tensorflow/tensorflow/archive/refs/tags/v2.14.1.tar.gz
BEEHIVE=https://beehive.molgen.mpg.de/c7ba144a4b1b0a6d5fea5ecd274172d3/tensorflow-2.14.1.tar.gz
test -e tensorflow-${TFVERSION}.tar.gz || wget -nv $BEEHIVE
test -d tensorflow-${TFVERSION} || tar -xf tensorflow-${TFVERSION}.tar.gz
cd tensorflow-${TFVERSION}
#vars for config
export PYTHON_BIN_PATH=${PREFIX}/bin/python3
export USE_DEFAULT_PYTHON_LIB_PATH=1
export TF_PYTHON_VERSION=3.11
export TF_NEED_JEMALLOC=1
export TF_NEED_KAFKA=0
export TF_NEED_OPENCL_SYCL=0
export TF_NEED_AWS=0
export TF_NEED_GCP=0
export TF_NEED_HDFS=0
export TF_NEED_S3=0
export TF_ENABLE_XLA=1
export TF_NEED_GDR=0
export TF_NEED_VERBS=0
export TF_NEED_OPENCL=0
export TF_NEED_MPI=0
export TF_NEED_TENSORRT=0
export TF_NEED_NGRAPH=0
export TF_NEED_IGNITE=0
export TF_NEED_ROCM=0
export TF_SET_ANDROID_WORKSPACE=0
export TF_DOWNLOAD_CLANG=0
export TF_NCCL_VERSION=${NCCLSHORTVERSION}
export TF_IGNORE_MAX_BAZEL_VERSION=1
export NCCL_INSTALL_PATH=${CUDADIR}
export CC=gcc
export CXX=g++
# export GCC_HOST_COMPILER_PATH=/usr/bin/${CC}
# export HOST_C_COMPILER=/usr/bin/${CC}
# export HOST_CXX_COMPILER=/usr/bin/${CXX}
export GCC_HOST_COMPILER_PATH=$pkggcc/${CC}
export HOST_C_COMPILER=$pkggcc/${CC}
export HOST_CXX_COMPILER=$pkggcc/${CXX}
export TF_CUDA_CLANG=0
export CLANG_CUDA_COMPILER_PATH=/usr/bin/clang
export TF_CUDA_PATHS=${CUDADIR},${CUDADIR}/extras/CUPTI,${CUDADIR}/nvvm
export TF_CUDA_VERSION=$(${CUDADIR}/bin/nvcc --version | sed -n 's/^.*release \(.*\),.*/\1/p') # -> '11.8'
export TF_CUDNN_VERSION=$(sed -n 's/^#define CUDNN_MAJOR\s*\(.*\).*/\1/p' ${CUDADIR}/include/cudnn_version.h)
# hardware older than sm_52 is said to be unsupported, compute_86 denotes an emulation
export TF_CUDA_COMPUTE_CAPABILITIES=sm_52,sm_53,sm_60,sm_61,sm_62,sm_70,sm_72,sm_75,sm_80,sm_86,sm_87,sm_89,sm_90,compute_90
# stay compatible with the cluster machines
# https://gcc.gnu.org/onlinedocs/gcc/x86-Options.html
export CC_OPT_FLAGS="-march=broadwell" # or haswell to be safe ? no more nehalem
export TF_NEED_CUDA=1
export BAZEL_ARGS="--config=mkl -c opt"
# avoid: Auto-Configuration Warning: 'TMP' environment variable is not set, using 'C:\Windows\Temp' as default :/
export TMP=$BUILD_TMPDIR/tf_tmp
mkdir -p $TMP
./configure
bazel build \
--cxxopt="-march=broadwell" \
--copt="-march=broadwell" \
--noshow_loading_progress \
--noshow_progress \
--color="no" \
//tensorflow:libtensorflow.so \
//tensorflow:libtensorflow_cc.so \
//tensorflow:install_headers \
//tensorflow/tools/pip_package:build_pip_package
bazel shutdown # otherwise it hogs in cwd
# create a _clean/decent_ python package from so far created debris
test -d tensorflow-pip && rm -r tensorflow-pip
mkdir tensorflow-pip
./bazel-bin/tensorflow/tools/pip_package/build_pip_package --src ${BUILDDIR}/tensorflow-pip
cd ${BUILDDIR}
# ... cope with the weird symlink-structure placed in bazels' build-TMPDIR (-h) (tell me why ?)
test -e tensorflow-pip.tar && rm tensorflow-pip.tar
tar -chf tensorflow-pip.tar tensorflow-pip
# and perform a regular install
test -d tensorflow-pip && rm -r tensorflow-pip
tar -xf tensorflow-pip.tar
cd ${BUILDDIR}/tensorflow-pip
pip install . --prefix=$PREFIX
)
piplist; track tensorflow
# tidy up a bit
if [ -d $PREFIX/man/man1 ]; then
mv -v $PREFIX/man/man1/* $PREFIX/share/man/man1 || true
rm -rf $PREFIX/man
fi
# ------------------------------------------------------------ Sanity check
# load all packages, see warnings and spot installation errors
echo "# START load test."
export NUMEXPR_MAX_THREADS=1
python -c 'help("modules")'
echo "# END load test (passed)."
echo "# running pip check"
pip check
# I guess it makes sense that files can be read by all users ...
echo "# check for unreadable files"
cd $PREFIX
for D in bin etc include lib share; do
find $D \! -perm -004 -exec chmod -c a+r {} +
done
echo "For Deeper checks:"
echo "cd ${BUILDDIR} ; git log --patch | grep ^[\+\-][^\+\-]"
exit
# -------------------------------------------------------- Included patches
#Epatch:pip # Operation 'Forever young', disable version-check per default.
#Epatch:pip # The option needs to be kept, because some tools make use of
#Epatch:pip # it and would cause pip to fail.
#Epatch:pip --- a/pip/_internal/cli/cmdoptions.py 2022-08-09 17:29:15.853442948 +0200
#Epatch:pip +++ b/pip/_internal/cli/cmdoptions.py 2022-08-10 10:31:35.124945154 +0200
#Epatch:pip @@ -892,7 +892,6 @@
#Epatch:pip dest="disable_pip_version_check",
#Epatch:pip action="store_true",
#Epatch:pip - default=False,
#Epatch:pip - help="Don't periodically check PyPI to determine whether a new version "
#Epatch:pip - "of pip is available for download. Implied with --no-index.",
#Epatch:pip + default=True,
#Epatch:pip + help="Ignore this option, the version check IS disabled.",
#Epatch:pip )
#Epatch:gyp --- a/pylib/gyp/generator/cmake.py
#Epatch:gyp +++ b/pylib/gyp/generator/cmake.py
#Epatch:gyp @@ -41,7 +41,7 @@
#Epatch:gyp try:
#Epatch:gyp # maketrans moved to str in python3.
#Epatch:gyp + _maketrans = str.maketrans
#Epatch:gyp +except NameError:
#Epatch:gyp _maketrans = string.maketrans
#Epatch:gyp -except NameError:
#Epatch:gyp - _maketrans = str.maketrans
#Epatch:gyp generator_default_variables = {
#Epatch:gyp 'EXECUTABLE_PREFIX': '',
#Epatch:gyp @@ -283,2 +283,2 @@
#Epatch:gyp if int(action.get('process_outputs_as_sources', False)):
#Epatch:gyp - extra_sources.extend(zip(cmake_outputs, outputs))
#Epatch:gyp + extra_sources.extend(list(zip(cmake_outputs, outputs)))
#Epatch:gyp @@ -988,6 +988,6 @@
#Epatch:gyp # XCode settings
#Epatch:gyp xcode_settings = config.get('xcode_settings', {})
#Epatch:gyp - for xcode_setting, xcode_value in xcode_settings.viewitems():
#Epatch:gyp + for xcode_setting, xcode_value in xcode_settings.items():
#Epatch:gyp SetTargetProperty(output, cmake_target_name,
#Epatch:gyp "XCODE_ATTRIBUTE_%s" % xcode_setting, xcode_value,
#Epatch:gyp '' if isinstance(xcode_value, str) else ' ')
#Epatch:gyp diff --git a/pylib/gyp/input.py b/pylib/gyp/input.py
#Epatch:gyp index 4c12891..2bea334 100644
#Epatch:gyp --- a/pylib/gyp/input.py
#Epatch:gyp +++ b/pylib/gyp/input.py
#Epatch:gyp @@ -1183,7 +1183,7 @@
#Epatch:gyp if variable_name in variables:
#Epatch:gyp # If the variable is already set, don't set it.
#Epatch:gyp continue
#Epatch:gyp - if the_dict_key is 'variables' and variable_name in the_dict:
#Epatch:gyp + if the_dict_key == 'variables' and variable_name in the_dict:
#Epatch:gyp # If the variable is set without a % in the_dict, and the_dict is a
#Epatch:gyp # variables dict (making |variables| a varaibles sub-dict of a
#Epatch:gyp # variables dict), use the_dict's definition.
#Epatch:javabridge diff --git a/_javabridge_nomac.pyx b/_javabridge_nomac.pyx
#Epatch:javabridge index 9f28939..c437db0 100644
#Epatch:javabridge --- a/_javabridge_nomac.pyx
#Epatch:javabridge +++ b/_javabridge_nomac.pyx
#Epatch:javabridge @@ -16,30 +16,30 @@ cdef extern from "jni.h":
#Epatch:javabridge jint (*AttachCurrentThreadAsDaemon)(JavaVM *vm, void *penv, void *args) nogil
#Epatch:javabridge jint JNI_CreateJavaVM(JavaVM **pvm, void **penv, void *args) nogil
#Epatch:javabridge
#Epatch:javabridge -cdef int MacStartVM(JavaVM **pvm, JavaVMInitArgs *pVMArgs, char *class_name) nogil:
#Epatch:javabridge +cdef extern int MacStartVM(JavaVM **pvm, JavaVMInitArgs *pVMArgs, char *class_name) nogil:
#Epatch:javabridge return -1
#Epatch:javabridge
#Epatch:javabridge -cdef void StopVM(JavaVM *vm) nogil:
#Epatch:javabridge +cdef extern void StopVM(JavaVM *vm) nogil:
#Epatch:javabridge vm[0].DestroyJavaVM(vm)
#Epatch:javabridge
#Epatch:javabridge -cdef void MacRunLoopInit() nogil:
#Epatch:javabridge +cdef extern void MacRunLoopInit() nogil:
#Epatch:javabridge pass
#Epatch:javabridge
#Epatch:javabridge -cdef void MacRunLoopRun() nogil:
#Epatch:javabridge +cdef extern void MacRunLoopRun() nogil:
#Epatch:javabridge pass
#Epatch:javabridge
#Epatch:javabridge -cdef void MacRunLoopStop() nogil:
#Epatch:javabridge +cdef extern void MacRunLoopStop() nogil:
#Epatch:javabridge pass
#Epatch:javabridge
#Epatch:javabridge -cdef void MacRunLoopReset() nogil:
#Epatch:javabridge +cdef extern void MacRunLoopReset() nogil:
#Epatch:javabridge pass
#Epatch:javabridge
#Epatch:javabridge -cdef int MacIsMainThread() nogil:
#Epatch:javabridge +cdef extern int MacIsMainThread() nogil:
#Epatch:javabridge return 0
#Epatch:javabridge
#Epatch:javabridge -cdef void MacRunLoopRunInMode(double timeout) nogil:
#Epatch:javabridge +cdef extern void MacRunLoopRunInMode(double timeout) nogil:
#Epatch:javabridge pass
#Epatch:javabridge
#Epatch:javabridge -cdef int CreateJavaVM(JavaVM **pvm, void **pEnv, void *args) nogil:
#Epatch:javabridge +cdef extern int CreateJavaVM(JavaVM **pvm, void **pEnv, void *args) nogil:
#Epatch:javabridge return JNI_CreateJavaVM(pvm, pEnv, args)
#Epatch:javabridge
#Epatch:javabridge diff --git a/_javabridge_mac.pyx b/_javabridge_mac.pyx
#Epatch:javabridge index 2e3c524..daed790 100644
#Epatch:javabridge --- a/_javabridge_mac.pyx
#Epatch:javabridge +++ b/_javabridge_mac.pyx
#Epatch:javabridge @@ -31,11 +31,11 @@ cdef extern from "mac_javabridge_utils.h":
#Epatch:javabridge int MacIsMainThread() nogil
#Epatch:javabridge void MacRunLoopRunInMode(double) nogil
#Epatch:javabridge
#Epatch:javabridge -cdef void StopVM(JavaVM *vm):
#Epatch:javabridge +cdef extern void StopVM(JavaVM *vm):
#Epatch:javabridge MacStopVM()
#Epatch:javabridge
#Epatch:javabridge #
#Epatch:javabridge # Unused stub in Mac
#Epatch:javabridge #
#Epatch:javabridge -cdef int CreateJavaVM(JavaVM **pvm, void **pEnv, void *args):
#Epatch:javabridge +cdef extern int CreateJavaVM(JavaVM **pvm, void **pEnv, void *args):
#Epatch:javabridge return -1
# -------------------------------------------------------- This is the end.