Skip to content
Permalink
master
Switch branches/tags

Name already in use

A tag already exists with the provided branch name. Many Git commands accept both tag and branch names, so creating this branch may cause unexpected behavior. Are you sure you want to create this branch?

pkg-scripts/python-3.10.8-0.build.sh

Go to file
 
 
Cannot retrieve contributors at this time
executable file 738 lines (604 sloc) 25 KB
Raw Blame
Open in GitHub Desktop
#! /bin/bash
# ------------------------------------------------------------------- Debug
# COOKIE=$(mcookie|cut -c-8); grep -v V_GREP_ME $0 > /dev/shm/runme-$COOKIE.sh ; sleep 0.3; exec bash /dev/shm/runme-$COOKIE.sh
# TESTING=1
# SKIPTENSORFLOW=1
# ---------------------------------------------------------------- Preamble
PKG=python
VERSION=3.10.8
BUILD=0
PYTHONVER=${VERSION%.*}
CUDADIR=/pkg/cuda-11.5.2-0
PREFIX=/pkg/$PKG-$VERSION-$BUILD
if [ -n "$TESTING" ]; then PREFIX=/scratch/local2/$PKG-$VERSION-$BUILD ; fi
PATH=/bin:/usr/bin:/usr/sbin:/usr/local/bin
# set -xe
set -e
umask 022
BUILD_TMPDIR=/scratch/local2/$PKG-$VERSION-$BUILD.$USER.build.tmp
test -d $BUILD_TMPDIR && ( chmod -R u+rwx $BUILD_TMPDIR || true ; rm -rf $BUILD_TMPDIR )
mkdir -p $BUILD_TMPDIR/home/.cache/pip
mkdir -p $BUILD_TMPDIR/home/.local
# copy script to an accessible location
cp $0 $BUILD_TMPDIR; ME="$BUILD_TMPDIR/$(basename $0)"
export TMPDIR=$BUILD_TMPDIR
export HOME=$BUILD_TMPDIR/home
exec </dev/null
mkdir -p $PREFIX
cat >$PREFIX/profile <<-EOF
. $CUDADIR/profile
PATH=$PREFIX/bin:\$PATH
export LD_LIBRARY_PATH=$PREFIX/lib\${LD_LIBRARY_PATH:+:}\$LD_LIBRARY_PATH
PKG_CONFIG_PATH=$PREFIX/lib/pkgconfig\${PKG_CONFIG_PATH:+:}\$PKG_CONFIG_PATH
export PKG_CONFIG_PATH
if [ -d $PREFIX/.compatlibs ]; then export LD_LIBRARY_PATH=$PREFIX/.compatlibs\${LD_LIBRARY_PATH:+:}\$LD_LIBRARY_PATH ; fi
EOF
. $PREFIX/profile
NPROC=$(( $(nproc) * 4 / 5 + 1 ))
export MAKEFLAGS="-j $NPROC"
BUILDDIR=$PREFIX/build
mkdir -p $BUILDDIR
cd $BUILDDIR
# -------------------------------------- Use git to track package evolution
test -d .git && rm -rf .git
git init -q; echo -e '[user]\n name = none\n email = of_your_business...' >> .git/config
echo '*' > .gitignore
function piplist() {
pip3 list | awk '{ printf("%-36s %s\n", $1,$2) }' | grep -v '\----' > GameOfVersions
}
function track() {
if [ -d .git ]; then
if git add -f GameOfVersions; then
git commit -q -n -m "Package: '$*'" || /bin/true
fi
fi
}
function install() {
# no-color is nice, but doesn't work when dependencies are installed, so (mis-)use a pipe
pip3 install --no-color --compile --cache-dir=$HOME/.cache/pip --prefix=$PREFIX "${@+$@}" | cat
piplist
track $*
}
# ---------------------------------------- Build Python & support libraries
(
# orig: https://www.python.org/ftp/python/3.10.8/Python-3.10.8.tgz
BEEHIVE=https://beehive.molgen.mpg.de/fbe3fff11893916ad1756b15c8a48834/Python-3.10.8.tgz
test -e Python-$VERSION.tgz || wget -nv $BEEHIVE
test -d Python-$VERSION || tar -xf Python-$VERSION.tgz
cd Python-$VERSION
# BerkeleyDB, the undead ...
# Newer libgdbm_compat.so versions pull libgdbm.so on their own, our libgdbm_compat.so is older ...
# So, let setup.py create '-lgdbm -lgdbm_compat' instead of a sole '-lgdbm_compat'
# verbatim: ndbm_libs = ['gdbm_compat'] => ndbm_libs = ['gdbm','gdbm_compat']
sed -i -e "/ndbm_libs = / s/_compat'/','gdbm_compat'/" setup.py
# leave further hints in the log
sed -i -e '/db_setup_debug = False/ s/False/True/' \
-e '/dbm_setup_debug = False/ s/False/True/' setup.py
# These may come handy if the Python build needs exclusive stuff from $PREFIX
# INCLUDEDIR=$PREFIX/include \
# LIBDIR=$PREFIX/lib \
LDFLAGS="-Wl,-rpath=$PREFIX/lib" \
./configure \
--prefix=$PREFIX \
--enable-shared
make -j $NPROC
make install
# it might come to happen that something down the line may just call 'python', defuse ...
test -e $PREFIX/bin/python || ln -s python3 $PREFIX/bin/python
python3 -m ensurepip
pip3 install --prefix=$PREFIX -I pip
( # fix 'please update' noise from pip
cd $PREFIX/lib/python*/site-packages
sed -ne '/^#Epatch:pip/ s/^#Epatch:\S* // p' $ME | patch -p1 --verbose
)
)
install setuptools==65.3.0 # see numba below
piplist; track START
# build various pythonbindings 'by hand', because pip installs do not deliver
# header files and the like.
# BTW: builds need meson
install meson
(
DBUS_PYTHON_VERSION=1.2.18
# https://files.pythonhosted.org/packages/b1/5c/ccfc167485806c1936f7d3ba97db6c448d0089c5746ba105b6eb22dba60e/dbus-python-1.2.18.tar.gz
BEEHIVE=https://beehive.molgen.mpg.de/468874f3bf61d1b7deac98bbe2cfd8ac/dbus-python-1.2.18.tar.gz
test -e dbus-python-$DBUS_PYTHON_VERSION.tar.gz || wget -nv $BEEHIVE
test -d dbus-python-$DBUS_PYTHON_VERSION || tar -xf dbus-python-$DBUS_PYTHON_VERSION.tar.gz
cd dbus-python-$DBUS_PYTHON_VERSION
./configure --prefix=$PREFIX
make -j $NPROC
make install
)
(
PYCAIRO_VERSION=1.21.0
# https://files.pythonhosted.org/packages/92/a4/506564f574fa74c90b98690e8ecc8dbae1629f31fcfe0be69de45d9e1605/pycairo-1.21.0.tar.gz
BEEHIVE=https://beehive.molgen.mpg.de/682238eaa8bd7032e36a20b6f53824d1/pycairo-1.21.0.tar.gz
test -e pycairo-$PYCAIRO_VERSION.tar.gz || wget -nv $BEEHIVE
test -d pycairo-$PYCAIRO_VERSION || tar -xf pycairo-$PYCAIRO_VERSION.tar.gz
cd pycairo-$PYCAIRO_VERSION
mkdir -p meson-build
cd meson-build
meson setup \
--prefix=$PREFIX \
--buildtype=release \
..
ninja -j $NPROC
ninja install
)
(
PYGOBJECT3_VERSION=3.42.2
# https://files.pythonhosted.org/packages/fe/40/9afaeb8d3b453fb8596fcb6c7bc2b64e434868c91eda19955742778eff74/PyGObject-3.42.2.tar.gz
BEEHIVE=https://beehive.molgen.mpg.de/80ad843b2a49ccfcc30c6b756b755c09/PyGObject-3.42.2.tar.gz
test -e PyGObject-$PYGOBJECT3_VERSION.tar.gz || wget -nv $BEEHIVE
test -d PyGObject-$PYGOBJECT3_VERSION || tar -xf PyGObject-$PYGOBJECT3_VERSION.tar.gz
cd PyGObject-$PYGOBJECT3_VERSION
mkdir -p meson-build
cd meson-build
meson setup \
--prefix=$PREFIX \
--buildtype=release \
-Dtests=false \
..
ninja -j $NPROC
ninja install
)
piplist; track various pythonbindings
# build support to use QT from python -- see www.riverbankcomputing.com
( # sip: tool to create Python bindings for C and C++ libraries.
SIP_VERSION=6.6.2
# https://files.pythonhosted.org/packages/5b/cb/c27c925ae07bd03a2597fa1db17bfc2a4ac57da61aeb90f8ec98ffbb975b/sip-6.6.2.tar.gz
BEEHIVE=https://beehive.molgen.mpg.de/620eb75a8b22fb6af9c6044aa015782d/sip-6.6.2.tar.gz
test -e sip-$SIP_VERSION.tar.gz || wget -nv $BEEHIVE
test -d sip-$SIP_VERSION || tar -xf sip-$SIP_VERSION.tar.gz
cd sip-$SIP_VERSION
sed -ne '/^#Epatch:sip1/ s/^#Epatch:\S* // p' $ME | patch -p1 --verbose
sed -ne '/^#Epatch:sip2/ s/^#Epatch:\S* // p' $ME | patch -p1 --verbose
python3 setup.py build
python3 setup.py install --skip-build --optimize=1
)
# QT bindings need some python modules to build
install PyQt5-sip
install PyQt-builder
(
PYQT_VERSION=5.15.7
# orig: https://files.pythonhosted.org/packages/e1/57/2023316578646e1adab903caab714708422f83a57f97eb34a5d13510f4e1/PyQt5-5.15.7.tar.gz
BEEHIVE=https://beehive.molgen.mpg.de/ae2c68e38b9b36fdf5f932419353a2b3/PyQt5-5.15.7.tar.gz
test -e PyQt5-$PYQT_VERSION.tar.gz || wget -nv $BEEHIVE
test -d PyQt5-$PYQT_VERSION || tar -xf PyQt5-$PYQT_VERSION.tar.gz
cd PyQt5-$PYQT_VERSION
sip-build \
--confirm-license \
--no-make \
--api-dir $PREFIX/share/qsci/api/python \
--dbus $PREFIX/include/dbus-1.0 \
--no-designer-plugin \
--no-qml-plugin \
--jobs $NPROC \
cd build
make -j $NPROC
make install
# Remove unused py2 version of uic modules
rm -r $PREFIX/lib/python*/site-packages/PyQt5/uic/port_v2
)
(
QSCINTILLA_VERSION=2.13.4.dev2206171421 # no 2.14 in sight ...
# orig: https://www.riverbankcomputing.com/static/Downloads/QScintilla/${PKGVERSION}/QScintilla_src-${PKGVERSION}.tar.gz
BEEHIVE=https://beehive.molgen.mpg.de/d196ebd441f40474dd7732d4d492de69/QScintilla_src-2.13.4.dev2206171421.tar.gz
test -e QScintilla_src-$QSCINTILLA_VERSION.tar.gz || wget -nv $BEEHIVE
test -d QScintilla_src-$QSCINTILLA_VERSION || tar -xf QScintilla_src-$QSCINTILLA_VERSION.tar.gz
cd QScintilla_src-$QSCINTILLA_VERSION
cd Python
cp -vp pyproject-qt5.toml pyproject.toml
sip-build \
--no-make \
--qsci-features-dir ../src/features \
--qsci-include-dir ../src \
--qsci-library-dir ../src \
--api-dir $PREFIX/share/qsci/api/python \
--jobs $NPROC
cd build
make -j $NPROC
make install
)
piplist; track qt-support-riverbankcomputing
# -------------------------------------------------------- Install packages
# Install the psychotic diva first
install numpy==1.23.4
install Cython
# libxml2-python from pypi isn't up to date, cause lxml seems to be the favored binding
# do this early, not that some dependency tries it on its own.
# (btw. libxslt-python isn't ported to python3)
(
LIBXML2_VERSION=2.9.12
# orig: http://xmlsoft.org/sources/libxml2-${PKGVERSION}.tar.gz
BEEHIVE=https://beehive.molgen.mpg.de/f433a39be087a9f0b197eb2307ad9f75/libxml2-2.9.12.tar.gz
test -e libxml2-$LIBXML2_VERSION.tar.gz || wget -nv $BEEHIVE
test -d libxml2-$LIBXML2_VERSION || tar -xf libxml2-$LIBXML2_VERSION.tar.gz libxml2-$LIBXML2_VERSION/python
cd libxml2-$LIBXML2_VERSION/python
python3 setup.py build
python3 setup.py install
)
piplist; track legacy-libxml2
# 'numba' the cuda enabled numpy
(
NUMBA_VERSION=0.56.3
# orig: https://files.pythonhosted.org/packages/a3/95/7df08647e773a2417e24de8fea9087606b37074f63830a3f2dbabd97f01b/numba-0.56.3.tar.gz
BEEHIVE=https://beehive.molgen.mpg.de/ae0a008447f2f246523af143b1488e88/numba-0.56.3.tar.gz
test -e numba-$NUMBA_VERSION.tar.gz || wget -nv $BEEHIVE
test -d numba-$NUMBA_VERSION || tar -xf numba-$NUMBA_VERSION.tar.gz
cd numba-$NUMBA_VERSION
# numba assumes that libcudart.so and friends reside in 'CUDA_HOME/lib64'
sed -ne '/^#Epatch:numba/ s/^#Epatch:\S* // p' $ME | patch -p1 --verbose
install .
# useful to see if numba works
cp -p numba/misc/numba_sysinfo.py $PREFIX/bin
chmod 0755 $PREFIX/bin/numba_sysinfo.py
)
piplist; track numba
# 'distributed' doesn't like the 'new' tornado 6.2 -- reason: a deprecation warning makes the CI choke (geht's noch?)
(
DISTRIBUTED_VERSION=2022.9.0
# orig: https://files.pythonhosted.org/packages/df/46/e8ef41f8dfd553327b6a12f4ab78d9351b54299596fe5b8b165ba40b1550/distributed-2022.9.0.tar.gz
BEEHIVE=https://beehive.molgen.mpg.de/335d718f1ba331ee8e85e40e564559a1/distributed-2022.9.0.tar.gz
test -e distributed-$DISTRIBUTED_VERSION.tar.gz || wget -nv $BEEHIVE
test -d distributed-$DISTRIBUTED_VERSION || tar -xf distributed-$DISTRIBUTED_VERSION.tar.gz
cd distributed-$DISTRIBUTED_VERSION
sed -i -e '/tornado/ s/.*/tornado/' requirements.txt
sed -i -e '/tornado/ s/.*/tornado/' distributed.egg-info/requires.txt
install .
)
piplist; track distributed and tornado
PACKAGES=$(sed -e s/#.*$// <<- __PKGLIST__
protobuf==3.19.4 # tensorflow likes to have an slightly older one [< 3.20], no conflicts with this
Matplotlib==3.6.0 # 3.6.1 is the first bugfix release of the 3.6.x series... Well, it is buggy
ipython[all]
SciPy
pandas
SymPy
nose
statsmodels
pysam
seaborn
scikit-learn
Mako
cutadapt # Helpdesk 01/24/18 16:19, Sabrina Krakau
pygobject # Helpdesk 03/26/18 14:19, Virginie Stanisla
blink1 # Blink1-lib 07/04/18 12:00, Niclas Hofmann
umi-tools
sphinx
recommonmark
fastcluster
pydot
pygraphviz
deeptools
HTSeq
keras
keras_preprocessing # tensorflow ...
MotifScan # Alena 30.8.2018
MAmotif
hic2cool # Helpdesk Robert Schoepflin 17.09.2018
snakemake
virtualenv # Donald
odfpy # Donald
Dumper # Thomas
igraph
louvain # monocle3 prep for Helene
selenium # 19.7.19 Kreitler -> https://webxray.org/
gmpy # From viper context, Helpdesk Laura Glaser 9.8.2019
plotly
pyOpenCL
docopt
distro
ply
threadpoolctl
HiCMatrix
pybedtools
gffutils
python-bioformats
jupyterlab
jupyterlab-git # request Peter Arndt
igv_jupyterlab # Matthias Lienhard HD 01-04-2020
igv-jupyter
scour # https://github.molgen.mpg.de/mariux64/pkg-scripts/issues/98
multiqc
pybind11
svgwrite
pyomo
bioservices
pairtools
SharedArray
pyabc # dependency for stochkit, HD 20.07.20 Gemma Noviello
petab
tadtool # HD 26.01.21 Michael Robson
pyperf # Paul sep-20
umap # HD, used by Matthias Lienhard
MACS3
biopython # HD 2021-07-23 Magalhaes/Hnisz alphafold related -- cool that we have missed biopython so far, LOL
dm-tree
dm-haiku
flatbuffers
contextlib2
ml_collections
# additions/fixups since 3.9
papermill # turns jupyter notebooks into scripts [sic!] used by P.A. [who else, *grin*]
opencv-python
scikit-image
git-cola
cffi-glpk # uses /usr/lib/libglpk.so.40, see if I can get it linked statically - kreitler
czifile
napari[pyqt5]
napari-czifile2
CrossMap
cooltools # cause trouble with numba/nupy versions -- Maria Valieva 17.7.2019
scanpy
__PKGLIST__
)
for PKG in $PACKAGES; do
install $PKG
done
install pyopengl # Tk binding creates an error, so remove it (Expects Tk Togl installation...)
rm -rf ${PREFIX}/lib/python$PYTHONVER/site-packages/OpenGL/Tk
# torch bundled with a recent CUDA-Toolkit
install torch torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/cu116
# install helen after our torch, otherwise we might catch the default torch from pip ...
install helen # HD, a dependency from a request by helene
mkdir -vp $PREFIX/share/jupyter/lab/settings # try to defeat a silly error in jupyter labextension install
# --------------------------------------------------- Install misc packages
(
BOOST_VERSION=(1.80.0 1_80_0)
PKGCFLAGS="-O2 -fPIC"
# orig: https://dl.bintray.com/boostorg/release/1.75.0/source/boost_1_75_0.tar.gz
BEEHIVE=https://beehive.molgen.mpg.de//38813f6feb40387dfe90160debd71251/boost_1_75_0.tar.gz
# orig: https://boostorg.jfrog.io/artifactory/main/release/1.80.0/source/boost_1_80_0.tar.gz
BEEHIVE=https://beehive.molgen.mpg.de/077f074743ea7b0cb49c6ed43953ae95/boost_1_80_0.tar.gz
test -e boost_${BOOST_VERSION[1]}.tar.gz || wget -nv $BEEHIVE
test -d boost-${BOOST_VERSION[0]} || mkdir boost-${BOOST_VERSION[0]} && tar -xf boost_${BOOST_VERSION[1]}.tar.gz -C boost-${BOOST_VERSION[0]} --strip-components=1
cd boost-${BOOST_VERSION[0]}
sed -e "/using python/ s|;|: ${PREFIX}/include/python${PYTHONVER} ;|" \
-i bootstrap.sh
./bootstrap.sh \
--with-toolset=gcc \
--with-icu \
--with-python=$PREFIX/bin/python3 \
--with-libraries=python,thread || exit 1
./b2 \
-j $NPROC \
--layout=system \
--build-dir=build-python3 \
--prefix=$PREFIX \
--libdir=$PREFIX/lib \
--build-type=minimal \
toolset=gcc \
variant=release \
debug-symbols=off \
link=shared \
threading=multi \
runtime-link=shared \
python=$PYTHONVER \
cflags="$PKGCFLAGS" \
cxxflags="$PKGCFLAGS" \
install || exit 1
)
(
PYCUDAVERS=2022.1
PYTHON_LIBNAME=boost_python${PYTHONVER/./}
# orig: https://github.com/inducer/pycuda/archive/v2020.1.tar.gz
# BEEHIVE=https://beehive.molgen.mpg.de/580562d7af4d3249c5b42a1df8d428a0/pycuda-2020.1.tar.gz
# orig: https://files.pythonhosted.org/packages/2d/1f/48a3a5b2c715345e7af1e09361100bd98c3d72b4025371692ab233f523d3/pycuda-2022.1.tar.gz # maybe those very long URLs compensate for other (shorter) things ...
BEEHIVE="https://beehive.molgen.mpg.de/2fad5a15db4153d825822eba7cb25f25/pycuda-2022.1.tar.gz"
test -e pycuda-${PYCUDAVERS}.tar.gz || wget -nv $BEEHIVE
test -d pycuda-${PYCUDAVERS} || tar -xf pycuda-${PYCUDAVERS}.tar.gz
cd pycuda-${PYCUDAVERS}
test -e siteconf.py && rm siteconf.py
python3 configure.py \
--cuda-enable-gl \
--cuda-root=$CUDADIR \
--no-use-shipped-boost \
--boost-python-libname=$PYTHON_LIBNAME \
--boost-inc-dir=$PREFIX/include \
--boost-lib-dir=$PREFIX/lib \
--cudadrv-lib-dir='${CUDA_ROOT}/lib,${CUDA_ROOT}/lib/stubs' \
--cudart-lib-dir='${CUDA_ROOT}/lib,${CUDA_ROOT}/lib/stubs' \
--curand-lib-dir='${CUDA_ROOT}/lib,${CUDA_ROOT}/lib/stubs' \
--ldflags=-L$PREFIX/lib
python3 setup.py build
python3 setup.py install --prefix $PREFIX
)
(
# orig: https://github.com/nboley/idr/archive/2.0.3/idr-2.0.3.tar.gz # HD: Tobias Zehnder, 18.3.21
BEEHIVE=https://beehive.molgen.mpg.de/1bec61b526bb1dcf92a5ab99c27852d0/idr-2.0.3.tar.gz
test -e idr-2.0.3.tar.gz || wget -nv $BEEHIVE
test -d idr-2.0.3 || tar -xf idr-2.0.3.tar.gz
cd idr-2.0.3
python3 setup.py build
python3 setup.py install --prefix $PREFIX
)
(
# orig: https://github.com/stacked-git/stgit/archive/v1.0/stgit-1.0.tar.gz # Paul mariux64/pkg-scripts/issues/152
BEEHIVE=https://beehive.molgen.mpg.de/b20ab5a197fc3a141f4b00c5859c0a3b/stgit-1.0.tar.gz
test -e stgit-1.0.tar.gz || wget -nv $BEEHIVE
test -d stgit-1.0 || tar -xf stgit-1.0.tar.gz
cd stgit-1.0
python3 setup.py build
python3 setup.py install --prefix $PREFIX
)
(
# https://github.com/brentp/bwa-meth/archive/refs/tags/v0.2.5.tar.gz
BEEHIVE=https://beehive.molgen.mpg.de/33a4c416760dd67e77f17e07791527e2/bwa-meth-0.2.5.tar.gz
test -e bwa-meth-0.2.5.tar.gz || wget -nv $BEEHIVE
test -d bwa-meth-0.2.5 || tar -xf bwa-meth-0.2.5.tar.gz
cd bwa-meth-0.2.5
python3 setup.py build
python3 setup.py install --prefix $PREFIX
)
piplist; track misc packages
# ------------------------------------------------ finally build tensorflaw
install wheel # to build our tf-wheel (i.e. zip archive)
(
test -n "$SKIPTENSORFLOW" && exit
TFVERSION=2.10.0
NCCLSHORTVERSION=$(pkg-config nccl --modversion | grep -Po '\d+\.\d+')
. /pkg/bazel-5.1.0-0/profile
# orig: https://github.com/tensorflow/tensorflow/archive/refs/tags/v2.10.0.tar.gz
BEEHIVE=https://beehive.molgen.mpg.de/7ceb5277d482f10d2df8644deb412061/tensorflow-2.10.0.tar.gz
test -e tensorflow-${TFVERSION}.tar.gz || wget -nv $BEEHIVE
test -d tensorflow-${TFVERSION} || tar -xf tensorflow-${TFVERSION}.tar.gz
cd tensorflow-${TFVERSION}
#vars for config
export PYTHON_BIN_PATH=${PREFIX}/bin/python3
export USE_DEFAULT_PYTHON_LIB_PATH=1
export TF_NEED_JEMALLOC=1
export TF_NEED_KAFKA=0
export TF_NEED_OPENCL_SYCL=0
export TF_NEED_AWS=0
export TF_NEED_GCP=0
export TF_NEED_HDFS=0
export TF_NEED_S3=0
export TF_ENABLE_XLA=1
export TF_NEED_GDR=0
export TF_NEED_VERBS=0
export TF_NEED_OPENCL=0
export TF_NEED_MPI=0
export TF_NEED_TENSORRT=0
export TF_NEED_NGRAPH=0
export TF_NEED_IGNITE=0
export TF_NEED_ROCM=0
export TF_SET_ANDROID_WORKSPACE=0
export TF_DOWNLOAD_CLANG=0
export TF_NCCL_VERSION=${NCCLSHORTVERSION}
export TF_IGNORE_MAX_BAZEL_VERSION=1
export NCCL_INSTALL_PATH=${CUDADIR}
export CC=gcc
export CXX=g++
export GCC_HOST_COMPILER_PATH=/usr/bin/${CC}
export HOST_C_COMPILER=/usr/bin/${CC}
export HOST_CXX_COMPILER=/usr/bin/${CXX}
export TF_CUDA_CLANG=0
export CLANG_CUDA_COMPILER_PATH=/usr/bin/clang
export TF_CUDA_PATHS=${CUDADIR},${CUDADIR}/extras/CUPTI,${CUDADIR}/nvvm
export TF_CUDA_VERSION=$(${CUDADIR}/bin/nvcc --version | sed -n 's/^.*release \(.*\),.*/\1/p') # 11.5
export TF_CUDNN_VERSION=$(sed -n 's/^#define CUDNN_MAJOR\s*\(.*\).*/\1/p' ${CUDADIR}/include/cudnn_version.h)
# hardware older than sm_52 is said to be unsupported, compute_86 denotes an emulation
export TF_CUDA_COMPUTE_CAPABILITIES=sm_52,sm_53,sm_60,sm_61,sm_62,sm_70,sm_72,sm_75,sm_80,sm_86,compute_86
# stay compatible with the cluster machines
export CC_OPT_FLAGS="-march=nehalem"
export TF_NEED_CUDA=1
export BAZEL_ARGS="--config=mkl -c opt"
./configure
bazel build \
--cxxopt="-march=nehalem" \
--copt="-march=nehalem" \
--noshow_loading_progress \
--noshow_progress \
--color="no" \
//tensorflow:libtensorflow.so \
//tensorflow:libtensorflow_cc.so \
//tensorflow:install_headers \
//tensorflow/tools/pip_package:build_pip_package
bazel shutdown # otherwise it hogs in cwd
# create a _clean/decent_ python package from so far created debris
test -d tensorflow-pip && rm -r tensorflow-pip
mkdir tensorflow-pip
./bazel-bin/tensorflow/tools/pip_package/build_pip_package --src ${BUILDDIR}/tensorflow-pip
cd ${BUILDDIR}
# ... cope with the ??? symlink-structure placed in bazels' build-TMPDIR (-h) (tell me why ?)
test -e tensorflow-pip.tar && rm tensorflow-pip.tar
tar -chf tensorflow-pip.tar tensorflow-pip
# and perform a regular install
test -d tensorflow-pip && rm -r tensorflow-pip
tar -xf tensorflow-pip.tar
cd ${BUILDDIR}/tensorflow-pip
python3 setup.py build
python3 setup.py install --prefix $PREFIX
)
piplist; track tensorflow
# tidy up a bit
if [ -d $PREFIX/man/man1 ]; then
mv -v $PREFIX/man/man1/* $PREFIX/share/man/man1 || true
rm -rf $PREFIX/man
fi
# ------------------------------------------------------------ Sanity check
# load all packages, see warnings and spot installation errors
# But don't care about r2py, a further dependence into R is not wanted!
echo "# START load test."
python3 -c 'help("modules")' > /dev/null
echo "# END load test (passed)."
echo "# running pip check"
pip3 check
# I guess it makes sense that files can be read by all users ...
echo "# check for unreadable files"
cd $PREFIX
find \! -perm -004 -exec chmod -c a+r {} +
exit
# -------------------------------------------------------- Included patches
#Epatch:pip # Operation 'Forever young', disable version-check per default.
#Epatch:pip # The option needs to be kept, because some tools make use of
#Epatch:pip # it and would cause pip to fail.
#Epatch:pip --- a/pip/_internal/cli/cmdoptions.py 2022-08-09 17:29:15.853442948 +0200
#Epatch:pip +++ b/pip/_internal/cli/cmdoptions.py 2022-08-10 10:31:35.124945154 +0200
#Epatch:pip @@ -892,7 +892,6 @@
#Epatch:pip dest="disable_pip_version_check",
#Epatch:pip action="store_true",
#Epatch:pip - default=False,
#Epatch:pip - help="Don't periodically check PyPI to determine whether a new version "
#Epatch:pip - "of pip is available for download. Implied with --no-index.",
#Epatch:pip + default=True,
#Epatch:pip + help="Ignore this option, the version check IS disabled.",
#Epatch:pip )
# https://riverbankcomputing.com/hg/sip/raw-rev/1430b279a3c9 -> sip-fix-exceptions.patch
#Epatch:sip1 Fixed the handling of exceptions that sub-class C++ exceptions.
#Epatch:sip1 diff -r 6eff81a15822 -r 1430b279a3c9 sipbuild/generator/parser/rules.py
#Epatch:sip1 --- a/sipbuild/generator/parser/rules.py Fri Jun 17 14:25:16 2022 +0100
#Epatch:sip1 +++ b/sipbuild/generator/parser/rules.py Sun Jun 26 10:01:41 2022 +0100
#Epatch:sip1 @@ -2293,6 +2293,7 @@
#Epatch:sip1
#Epatch:sip1 if len(p) == 4:
#Epatch:sip1 base = p[2]
#Epatch:sip1 + base.make_absolute()
#Epatch:sip1
#Epatch:sip1 # See if it is a project-defined exception.
#Epatch:sip1 for xd in p.parser.pm.spec.exceptions:
# https://riverbankcomputing.com/hg/sip/raw-rev/323d39a2d602 -> sip-fix-template-values.patch
#Epatch:sip2 Fixed the instantiation of template values.
#Epatch:sip2 diff -r 1430b279a3c9 -r 323d39a2d602 sipbuild/generator/parser/instantiations.py
#Epatch:sip2 --- a/sipbuild/generator/parser/instantiations.py Sun Jun 26 10:01:41 2022 +0100
#Epatch:sip2 +++ b/sipbuild/generator/parser/instantiations.py Sun Jul 17 09:41:16 2022 +0100
#Epatch:sip2 @@ -434,7 +434,8 @@
#Epatch:sip2 proto_name = proto_value.value.result.definition
#Epatch:sip2
#Epatch:sip2 if proto_name.is_simple:
#Epatch:sip2 - i_name = ScopedName.parse(template_string(proto_name, expansions))
#Epatch:sip2 + i_name = ScopedName.parse(
#Epatch:sip2 + template_string(proto_name.base_name, expansions))
#Epatch:sip2 i_result = Argument(type=ArgumentType.DEFINED, definition=i_name)
#Epatch:sip2 i_fcall = FunctionCall(result=i_result,
#Epatch:sip2 args=proto_value.value.args)
#Epatch:numba # Fix lib location, turn numba_sysinfo.py into a common diag tool
#Epatch:numba --- a/numba/cuda/cuda_paths.py 2022-10-14 09:57:25.616827200 +0200
#Epatch:numba +++ b/numba/cuda/cuda_paths.py 2022-10-20 11:20:15.106019523 +0200
#Epatch:numba @@ -66,3 +66,3 @@
#Epatch:numba else:
#Epatch:numba - return 'lib64'
#Epatch:numba + return 'lib'
#Epatch:numba
#Epatch:numba --- a/numba/misc/numba_sysinfo.py 2022-10-14 09:57:25.628827000 +0200
#Epatch:numba +++ b/numba/misc/numba_sysinfo.py 2022-10-20 11:02:54.056767540 +0200
#Epatch:numba @@ -1 +1,2 @@
#Epatch:numba +#! /usr/bin/env python3
#Epatch:numba import json
#Epatch:numba @@ -367,3 +368,3 @@
#Epatch:numba with redirect_stdout(output):
#Epatch:numba - cudadrv.libs.test(sys.platform, print_paths=False)
#Epatch:numba + cudadrv.libs.test(sys.platform, print_paths=True)
#Epatch:numba sys_info[_cu_lib_test] = output.getvalue()
#Epatch:numba @@ -626,9 +627,2 @@
#Epatch:numba ("",),
#Epatch:numba - ("__Conda Information__",),
#Epatch:numba - (DisplayMap({k: v for k, v in info.items()
#Epatch:numba - if k.startswith('Conda')}) or ("Conda not available.",)),
#Epatch:numba - ("",),
#Epatch:numba - ("__Installed Packages__",),
#Epatch:numba - DisplaySeq(info.get(_inst_pkg, ("Couldn't retrieve packages info.",))),
#Epatch:numba - ("",),
#Epatch:numba ("__Error log__" if info.get(_errors, [])
# -------------------------------------------------------- This is the end.