Skip to content
Permalink
master
Switch branches/tags

Name already in use

A tag already exists with the provided branch name. Many Git commands accept both tag and branch names, so creating this branch may cause unexpected behavior. Are you sure you want to create this branch?

pkg-scripts/python-3.10.10-1.build.sh

Go to file
 
 
Cannot retrieve contributors at this time
executable file 808 lines (670 sloc) 27.4 KB
Raw Blame
Open in GitHub Desktop
#!/bin/bash
# ------------------------------------------------------------------- Debug
COOKIE=$(mcookie|cut -c-8); grep -v V_GREP_ME $0 > /dev/shm/runme-$COOKIE.sh ; sleep 0.3; exec bash /dev/shm/runme-$COOKIE.sh
#TESTING=1
#SKIPTENSORFLOW=1
# ---------------------------------------------------------------- Preamble
#set -x
PKG=python
VERSION=3.10.10
BUILD=1
PYTHONVER=${VERSION%.*}
CUDADIR=/pkg/cuda-11.7.1-0
PREFIX=/pkg/$PKG-$VERSION-$BUILD
if [ -n "$TESTING" ]; then PREFIX=/scratch/local2/$PKG-$VERSION-$BUILD ; fi
PATH=/bin:/usr/bin:/usr/sbin:/usr/local/bin
# set -xe
set -e
umask 022
BUILD_TMPDIR=/scratch/local2/$PKG-$VERSION-$BUILD.$USER.build.tmp
test -d $BUILD_TMPDIR && ( chmod -R u+rwx $BUILD_TMPDIR || true ; rm -rf $BUILD_TMPDIR )
mkdir -p $BUILD_TMPDIR/home/.cache/pip
mkdir -p $BUILD_TMPDIR/home/.local
# copy script to an accessible location
cp $0 $BUILD_TMPDIR; ME="$BUILD_TMPDIR/$(basename $0)"
export TMPDIR=$BUILD_TMPDIR
export HOME=$BUILD_TMPDIR/home
exec </dev/null
mkdir -p $PREFIX
cat >$PREFIX/profile <<-EOF
. $CUDADIR/profile
PATH=$PREFIX/bin:\$PATH
export LD_LIBRARY_PATH=$PREFIX/lib\${LD_LIBRARY_PATH:+:}\$LD_LIBRARY_PATH
PKG_CONFIG_PATH=$PREFIX/lib/pkgconfig\${PKG_CONFIG_PATH:+:}\$PKG_CONFIG_PATH
export PKG_CONFIG_PATH
if [ -d $PREFIX/.compatlibs ]; then export LD_LIBRARY_PATH=$PREFIX/.compatlibs\${LD_LIBRARY_PATH:+:}\$LD_LIBRARY_PATH ; fi
EOF
. $PREFIX/profile
NPROC=$(( $(nproc) * 4 / 5 + 1 ))
export MAKEFLAGS="-j $NPROC"
BUILDDIR=$PREFIX/build
mkdir -p $BUILDDIR
cd $BUILDDIR
# -------------------------------------- Use git to track package evolution
test -d .git && rm -rf .git
git init -q; echo -e '[user]\n name = none\n email = of_your_business...' >> .git/config
echo '*' > .gitignore
function piplist() {
pip3 list | awk '{ printf("%-36s %s\n", $1,$2) }' | grep -v '\----' > GameOfVersions
}
function track() {
if [ -d .git ]; then
if git add -f GameOfVersions; then
git commit -q -n -m "Package: '$*'" || /bin/true
fi
fi
}
function install() {
# no-color is nice, but doesn't work when dependencies are installed, so (mis-)use a pipe
pip3 install --no-color --compile --cache-dir=$HOME/.cache/pip --prefix=$PREFIX "${@+$@}" | cat
piplist
track $*
}
# ---------------------------------------- Build Python & support libraries
(
# orig: https://www.python.org/ftp/python/3.10.10/Python-3.10.10.tar.xz
BEEHIVE=https://beehive.molgen.mpg.de/7bf85df71bbe7f95e5370b983e6ae684/Python-3.10.10.tar.xz
test -e Python-$VERSION.tar.xz || wget -nv $BEEHIVE
test -d Python-$VERSION || tar -xf Python-$VERSION.tar.xz
cd Python-$VERSION
# BerkeleyDB, the undead ...
# Newer libgdbm_compat.so versions pull libgdbm.so on their own, our libgdbm_compat.so is older ...
# So, let setup.py create '-lgdbm -lgdbm_compat' instead of a sole '-lgdbm_compat'
# verbatim: ndbm_libs = ['gdbm_compat'] => ndbm_libs = ['gdbm','gdbm_compat']
sed -i -e "/ndbm_libs = / s/_compat'/','gdbm_compat'/" setup.py
# leave further hints in the log
sed -i -e '/db_setup_debug = False/ s/False/True/' \
-e '/dbm_setup_debug = False/ s/False/True/' setup.py
# These may come handy if the Python build needs exclusive stuff from $PREFIX
# INCLUDEDIR=$PREFIX/include \
# LIBDIR=$PREFIX/lib \
LDFLAGS="-Wl,-rpath=$PREFIX/lib" \
./configure \
--prefix=$PREFIX \
--enable-shared
make -j $NPROC
make install
# it might come to happen that something down the line may just call 'python', defuse ...
test -e $PREFIX/bin/python || ln -s python3 $PREFIX/bin/python
python3 -m ensurepip
pip3 install --prefix=$PREFIX -I pip
( # fix 'please update' noise from pip
cd $PREFIX/lib/python*/site-packages
sed -ne '/^#Epatch:pip/ s/^#Epatch:\S* // p' $ME | patch -p1 --verbose
)
)
install setuptools==65.3.0 # see numba below
piplist; track START
# build various pythonbindings 'by hand', because pip installs do not deliver
# header files and the like.
# BTW: builds need meson
install meson
(
DBUS_PYTHON_VERSION=1.3.2
#PyPi
# https://files.pythonhosted.org/packages/c1/d3/6be85a9c772d6ebba0cc3ab37390dd6620006dcced758667e0217fb13307/dbus-python-1.3.2.tar.gz
BEEHIVE=https://beehive.molgen.mpg.de/33be8a4a766e1c7c9a377b8f934ce21a/dbus-python-1.3.2.tar.gz
test -e dbus-python-$DBUS_PYTHON_VERSION.tar.gz || wget -nv $BEEHIVE
test -d dbus-python-$DBUS_PYTHON_VERSION || tar -xf dbus-python-$DBUS_PYTHON_VERSION.tar.gz
cd dbus-python-$DBUS_PYTHON_VERSION
./configure --prefix=$PREFIX
make -j $NPROC
make install
)
# ------------------------------------------------ build tool fuer Thomas
(
set -x
GYP_VERSION=20230301_c6d8b9f7
# orig: https://mirror.netcologne.de/slackware/slackware64-current/source/d/gyp/
BEEHIVE=https://beehive.molgen.mpg.de/ba0357be23ca55555714261afccc7aab/gyp-20230301_c6d8b9f7.tar.xz
test -e gyp-${GYP_VERSION}.tar.xz || wget -nv $BEEHIVE
test -d gyp-${GYP_VERSION} || tar -xf gyp-${GYP_VERSION}.tar.xz
cd gyp-${GYP_VERSION}
sed -ne '/^#Epatch:gyp1/ s/^#Epatch:\S* // p' $ME | patch -p1 --verbose
sed -ne '/^#Epatch:gyp2/ s/^#Epatch:\S* // p' $ME | patch -p1 --verbose
python3 setup.py build
python3 setup.py install --prefix $PREFIX
)
piplist; track gyp
(
PYCAIRO_VERSION=1.23.0
#PyPi
# https://files.pythonhosted.org/packages/69/ca/9e9fa2e8be0876a9bbf046a1be7ee33e61d4fdfbd1fd25c76c1bdfddf8c4/pycairo-1.23.0.tar.gz
BEEHIVE=https://beehive.molgen.mpg.de/7a3729d21659098e1b9a411b62e88966/pycairo-1.23.0.tar.gz
test -e pycairo-$PYCAIRO_VERSION.tar.gz || wget -nv $BEEHIVE
test -d pycairo-$PYCAIRO_VERSION || tar -xf pycairo-$PYCAIRO_VERSION.tar.gz
cd pycairo-$PYCAIRO_VERSION
mkdir -p meson-build
cd meson-build
meson setup \
--prefix=$PREFIX \
--buildtype=release \
-Dpython=$PREFIX/bin/python \
-Dtests=false \
..
ninja -j $NPROC
ninja install
)
(
PYGOBJECT3_VERSION=3.44.1
# https://files.pythonhosted.org/packages/07/73/a034fc1bcd6d402d6f72ff618c093f91ac6921f968cc19806b6a1b1b19c8/PyGObject-3.44.0.tar.gz
BEEHIVE=https://beehive.molgen.mpg.de/959259478566b24316edaa18da6e0eaa/PyGObject-3.44.1.tar.gz
test -e PyGObject-$PYGOBJECT3_VERSION.tar.gz || wget -nv $BEEHIVE
test -d PyGObject-$PYGOBJECT3_VERSION || tar -xf PyGObject-$PYGOBJECT3_VERSION.tar.gz
cd PyGObject-$PYGOBJECT3_VERSION
mkdir -p meson-build
cd meson-build
meson setup \
--prefix=$PREFIX \
--buildtype=release \
-Dpython=$PREFIX/bin/python \
-Dtests=false \
..
ninja -j $NPROC
ninja install
)
piplist; track various pythonbindings
# build support to use QT from python -- see www.riverbankcomputing.com
( # sip: tool to create Python bindings for C and C++ libraries.
SIP_VERSION=6.7.7
# https://files.pythonhosted.org/packages/5b/cb/c27c925ae07bd03a2597fa1db17bfc2a4ac57da61aeb90f8ec98ffbb975b/sip-6.6.2.tar.gz
BEEHIVE=https://beehive.molgen.mpg.de/a131efd79649d31ee11601b19d1e3784/sip-6.7.7.tar.gz
test -e sip-$SIP_VERSION.tar.gz || wget -nv $BEEHIVE
test -d sip-$SIP_VERSION || tar -xf sip-$SIP_VERSION.tar.gz
cd sip-$SIP_VERSION
python3 setup.py build
python3 setup.py install --skip-build --optimize=1
)
# QT bindings need some python modules to build
install PyQt5-sip
install PyQt-builder
(
PYQT_VERSION=5.15.9
#PyPi
# orig: https://files.pythonhosted.org/packages/5c/46/b4b6eae1e24d9432905ef1d4e7c28b6610e28252527cdc38f2a75997d8b5/PyQt5-5.15.9.tar.gz
BEEHIVE=https://beehive.molgen.mpg.de/9d97fc06b7ae75e654e946c49e07ff12/PyQt5-5.15.9.tar.gz
test -e PyQt5-$PYQT_VERSION.tar.gz || wget -nv $BEEHIVE
test -d PyQt5-$PYQT_VERSION || tar -xf PyQt5-$PYQT_VERSION.tar.gz
cd PyQt5-$PYQT_VERSION
sip-build \
--confirm-license \
--no-make \
--api-dir $PREFIX/share/qsci/api/python \
--dbus $PREFIX/include/dbus-1.0 \
--no-designer-plugin \
--no-qml-plugin \
--jobs $NPROC \
cd build
make -j $NPROC
make install
# Remove unused py2 version of uic modules
rm -r $PREFIX/lib/python*/site-packages/PyQt5/uic/port_v2
)
(
QSCINTILLA_VERSION=2.13.4 # no 2.14 in sight ...
# orig: https://www.riverbankcomputing.com/static/Downloads/QScintilla/2.13.4/QScintilla_src-2.13.4.tar.gz
BEEHIVE=https://beehive.molgen.mpg.de/4f83a4a4ad7da40eae80ad23f9fb18f2/QScintilla_src-2.13.4.tar.gz
test -e QScintilla_src-$QSCINTILLA_VERSION.tar.gz || wget -nv $BEEHIVE
test -d QScintilla_src-$QSCINTILLA_VERSION || tar -xf QScintilla_src-$QSCINTILLA_VERSION.tar.gz
cd QScintilla_src-$QSCINTILLA_VERSION
cd Python
cp -vp pyproject-qt5.toml pyproject.toml
sip-build \
--no-make \
--qsci-features-dir ../src/features \
--qsci-include-dir ../src \
--qsci-library-dir ../src \
--api-dir $PREFIX/share/qsci/api/python \
--jobs $NPROC
cd build
make -j $NPROC
make install
)
piplist; track qt-support-riverbankcomputing
# -------------------------------------------------------- Install packages
# Install the psychotic diva first
install numpy==1.23.4
install Cython
# libxml2-python from pypi isn't up to date, cause lxml seems to be the favored binding
# do this early, not that some dependency tries it on its own.
# (btw. libxslt-python isn't ported to python3)
(
LIBXML2_VERSION=2.9.12
# orig: http://xmlsoft.org/sources/libxml2-${PKGVERSION}.tar.gz
BEEHIVE=https://beehive.molgen.mpg.de/f433a39be087a9f0b197eb2307ad9f75/libxml2-2.9.12.tar.gz
test -e libxml2-$LIBXML2_VERSION.tar.gz || wget -nv $BEEHIVE
test -d libxml2-$LIBXML2_VERSION || tar -xf libxml2-$LIBXML2_VERSION.tar.gz libxml2-$LIBXML2_VERSION/python
cd libxml2-$LIBXML2_VERSION/python
python3 setup.py build
python3 setup.py install
)
piplist; track legacy-libxml2
# 'numba' the cuda enabled numpy
(
NUMBA_VERSION=0.56.3
# orig: https://files.pythonhosted.org/packages/a3/95/7df08647e773a2417e24de8fea9087606b37074f63830a3f2dbabd97f01b/numba-0.56.3.tar.gz
BEEHIVE=https://beehive.molgen.mpg.de/ae0a008447f2f246523af143b1488e88/numba-0.56.3.tar.gz
test -e numba-$NUMBA_VERSION.tar.gz || wget -nv $BEEHIVE
test -d numba-$NUMBA_VERSION || tar -xf numba-$NUMBA_VERSION.tar.gz
cd numba-$NUMBA_VERSION
# numba assumes that libcudart.so and friends reside in 'CUDA_HOME/lib64'
sed -ne '/^#Epatch:numba1/ s/^#Epatch:\S* // p' $ME | patch -p1 --verbose
sed -ne '/^#Epatch:numba2/ s/^#Epatch:\S* // p' $ME | patch -p1 --verbose
install .
# useful to see if numba works
cp -p numba/misc/numba_sysinfo.py $PREFIX/bin
chmod 0755 $PREFIX/bin/numba_sysinfo.py
)
piplist; track numba
PACKAGES=$(sed -e s/#.*$// <<- __PKGLIST__
protobuf==3.19.4 # tensorflow likes to have an slightly older one [< 3.20], no conflicts with this
onnx==1.12.0 # newer onnx (dep helen) needs newer protobuf
Matplotlib==3.6.3 # 3.6.3 last bugfix for 4.6.x... Well, it is buggy, hopefullt not :)
keras==2.11.0 # tensorflow 2.11.1 requires keras<2.12,>=2.11.0
tornado==6.1
jupyter_client==7.3.2
Sphinx==4.5.0 # hic2cool(0.8.3) moecht gern was aelteres
jupyter-server<2.0.0 # You must use Jupyter Server v1 to load JupyterLab as notebook extension. Check if it needed
qtconsole==5.4.1 # pairtools wants a lower one; ipython installs latest & greatest
ipython[all]
SciPy
pandas
SymPy
nose
statsmodels
pysam
seaborn
scikit-learn
Mako
cutadapt # Helpdesk 01/24/18 16:19, Sabrina Krakau
pygobject # Helpdesk 03/26/18 14:19, Virginie Stanisla
blink1 # Blink1-lib 07/04/18 12:00, Niclas Hofmann
umi-tools
recommonmark
fastcluster
pydot
pygraphviz
deeptools
HTSeq
keras_preprocessing # tensorflow ...
MotifScan # Alena 30.8.2018
MAmotif
hic2cool # Helpdesk Robert Schoepflin 17.09.2018
snakemake
virtualenv # Donald
odfpy # Donald
Dumper # Thomas
igraph
louvain # monocle3 prep for Helene
selenium # 19.7.19 Kreitler -> https://webxray.org/
gmpy # From viper context, Helpdesk Laura Glaser 9.8.2019
plotly
pyOpenCL
docopt
distro
ply
threadpoolctl
HiCMatrix
pybedtools
gffutils
python-bioformats
jupyter
jupyterlab
jupyterlab-git # request Peter Arndt
igv_jupyterlab # Matthias Lienhard HD 01-04-2020
igv-jupyter
jupyterlab_latex # jupyter notebook need it
scour # https://github.molgen.mpg.de/mariux64/pkg-scripts/issues/98
multiqc
pybind11
svgwrite
pyomo
bioservices
pairtools
SharedArray
pyabc # dependency for stochkit, HD 20.07.20 Gemma Noviello
petab
tadtool # HD 26.01.21 Michael Robson
pyperf # Paul sep-20
umap # HD, used by Matthias Lienhard
MACS3
biopython # HD 2021-07-23 Magalhaes/Hnisz alphafold related -- cool that we have missed biopython so far, LOL
dm-tree
dm-haiku
flatbuffers
contextlib2
ml_collections
# additions/fixups since 3.9
papermill # turns jupyter notebooks into scripts [sic!] used by P.A. [who else, *grin*]
opencv-python
scikit-image
git-cola
cffi-glpk # uses /usr/lib/libglpk.so.40, see if I can get it linked statically - kreitler
czifile
napari[pyqt5]
napari-czifile2
CrossMap
scanpy
asciidoc
__PKGLIST__
)
for PKG in $PACKAGES; do
install $PKG
done
(
TRITON_VERSION=2.0.0
#triton 2.0.0 hat cmake als install dep, ist in 2.1.0 gefixt
# orig: https://github.com/openai/triton/archive/refs/tags/v2.0.0.tar.gz
BEEHIVE=https://beehive.molgen.mpg.de/56b46016609039fac839f8297c88454c/triton-2.0.0.tar.gz
test -e triton-${TRITON_VERSION}.tar.gz || wget -nv $BEEHIVE
test -d triton-${TRITON_VERSION} || tar -xf triton-${TRITON_VERSION}.tar.gz
cd triton-${TRITON_VERSION}/python
sed -ne '/^#Epatch:triton/ s/^#Epatch:\S* // p' $ME | patch -p1 --verbose
pip install . --prefix=$PREFIX
piplist; track triton==2.0.0
)
# torch bundled with a recent CUDA-Toolkit
install torch torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/cu118
install pyopengl # Tk binding creates an error, so remove it (Expects Tk Togl installation...)
rm -rf ${PREFIX}/lib/python$PYTHONVER/site-packages/OpenGL/Tk
# install helen after our torch, otherwise we might catch the default torch from pip ...
install helen # HD, a dependency from a request by helene
mkdir -vp $PREFIX/share/jupyter/lab/settings # try to defeat a silly error in jupyter labextension install
# --------------------------------------------------- Install misc packages
(
BOOST_VERSION=(1.80.0 1_80_0)
PKGCFLAGS="-O2 -fPIC"
# orig: https://dl.bintray.com/boostorg/release/1.75.0/source/boost_1_75_0.tar.gz
BEEHIVE=https://beehive.molgen.mpg.de//38813f6feb40387dfe90160debd71251/boost_1_75_0.tar.gz
# orig: https://boostorg.jfrog.io/artifactory/main/release/1.80.0/source/boost_1_80_0.tar.gz
BEEHIVE=https://beehive.molgen.mpg.de/077f074743ea7b0cb49c6ed43953ae95/boost_1_80_0.tar.gz
test -e boost_${BOOST_VERSION[1]}.tar.gz || wget -nv $BEEHIVE
test -d boost-${BOOST_VERSION[0]} || mkdir boost-${BOOST_VERSION[0]} && tar -xf boost_${BOOST_VERSION[1]}.tar.gz -C boost-${BOOST_VERSION[0]} --strip-components=1
cd boost-${BOOST_VERSION[0]}
sed -e "/using python/ s|;|: ${PREFIX}/include/python${PYTHONVER} ;|" \
-i bootstrap.sh
./bootstrap.sh \
--with-toolset=gcc \
--with-icu \
--with-python=$PREFIX/bin/python3 \
--with-libraries=python,thread || exit 1
./b2 \
-j $NPROC \
--layout=system \
--build-dir=build-python3 \
--prefix=$PREFIX \
--libdir=$PREFIX/lib \
--build-type=minimal \
toolset=gcc \
variant=release \
debug-symbols=off \
link=shared \
threading=multi \
runtime-link=shared \
python=$PYTHONVER \
cflags="$PKGCFLAGS" \
cxxflags="$PKGCFLAGS" \
install || exit 1
)
(
PYCUDAVERS=2022.1
PYTHON_LIBNAME=boost_python${PYTHONVER/./}
# orig: https://github.com/inducer/pycuda/archive/v2020.1.tar.gz
# BEEHIVE=https://beehive.molgen.mpg.de/580562d7af4d3249c5b42a1df8d428a0/pycuda-2020.1.tar.gz
# orig: https://files.pythonhosted.org/packages/2d/1f/48a3a5b2c715345e7af1e09361100bd98c3d72b4025371692ab233f523d3/pycuda-2022.1.tar.gz # maybe those very long URLs compensate for other (shorter) things ...
BEEHIVE="https://beehive.molgen.mpg.de/2fad5a15db4153d825822eba7cb25f25/pycuda-2022.1.tar.gz"
test -e pycuda-${PYCUDAVERS}.tar.gz || wget -nv $BEEHIVE
test -d pycuda-${PYCUDAVERS} || tar -xf pycuda-${PYCUDAVERS}.tar.gz
cd pycuda-${PYCUDAVERS}
test -e siteconf.py && rm siteconf.py
python3 configure.py \
--cuda-enable-gl \
--cuda-root=$CUDADIR \
--no-use-shipped-boost \
--boost-python-libname=$PYTHON_LIBNAME \
--boost-inc-dir=$PREFIX/include \
--boost-lib-dir=$PREFIX/lib \
--cudadrv-lib-dir='${CUDA_ROOT}/lib,${CUDA_ROOT}/lib/stubs' \
--cudart-lib-dir='${CUDA_ROOT}/lib,${CUDA_ROOT}/lib/stubs' \
--curand-lib-dir='${CUDA_ROOT}/lib,${CUDA_ROOT}/lib/stubs' \
--ldflags=-L$PREFIX/lib
python3 setup.py build
python3 setup.py install --prefix $PREFIX
)
(
IDR_VERSION=2.0.3
# orig: https://github.com/nboley/idr/archive/2.0.3/idr-2.0.3.tar.gz # HD: Tobias Zehnder, 18.3.21
BEEHIVE=https://beehive.molgen.mpg.de/1bec61b526bb1dcf92a5ab99c27852d0/idr-2.0.3.tar.gz
test -e idr-${IDR_VERSION}.tar.gz || wget -nv $BEEHIVE
test -d idr-${IDR_VERSION} || tar -xf idr-${IDR_VERSION}.tar.gz
cd idr-${IDR_VERSION}
python3 setup.py build
python3 setup.py install --prefix $PREFIX
)
(
STGIT_VERSSION=1.0
# orig: https://github.com/stacked-git/stgit/archive/v1.0/stgit-1.0.tar.gz # Paul mariux64/pkg-scripts/issues/152
BEEHIVE=https://beehive.molgen.mpg.de/b20ab5a197fc3a141f4b00c5859c0a3b/stgit-1.0.tar.gz
test -e stgit-${STGIT_VERSSION}.tar.gz || wget -nv $BEEHIVE
test -d stgit-${STGIT_VERSSION} || tar -xf stgit-${STGIT_VERSSION}.tar.gz
cd stgit-${STGIT_VERSSION}
python3 setup.py build
python3 setup.py install --prefix $PREFIX
)
(
METH_VERSION=0.2.5
# https://github.com/brentp/bwa-meth/archive/refs/tags/v0.2.5.tar.gz
BEEHIVE=https://beehive.molgen.mpg.de/33a4c416760dd67e77f17e07791527e2/bwa-meth-0.2.5.tar.gz
test -e bwa-meth-${METH_VERSION}.tar.gz || wget -nv $BEEHIVE
test -d bwa-meth-${METH_VERSION} || tar -xf bwa-meth-${METH_VERSION}.tar.gz
cd bwa-meth-${METH_VERSION}
python3 setup.py build
python3 setup.py install --prefix $PREFIX
)
piplist; track misc packages
# ------------------------------------------------ finally build tensorflaw
install wheel # to build our tf-wheel (i.e. zip archive)
(
test -n "$SKIPTENSORFLOW" && exit
TFVERSION=2.11.1
NCCLSHORTVERSION=$(pkg-config nccl --modversion | grep -Po '\d+\.\d+')
. /pkg/bazel-5.1.0-0/profile
# orig: https://github.com/tensorflow/tensorflow/archive/refs/tags/v2.11.1.tar.gz
BEEHIVE=https://beehive.molgen.mpg.de/e1b60c6e074f8b4c83f184e2c298774f/tensorflow-2.11.1.tar.gz
test -e tensorflow-${TFVERSION}.tar.gz || wget -nv $BEEHIVE
test -d tensorflow-${TFVERSION} || tar -xf tensorflow-${TFVERSION}.tar.gz
cd tensorflow-${TFVERSION}
#vars for config
export PYTHON_BIN_PATH=${PREFIX}/bin/python3
export USE_DEFAULT_PYTHON_LIB_PATH=1
export TF_NEED_JEMALLOC=1
export TF_NEED_KAFKA=0
export TF_NEED_OPENCL_SYCL=0
export TF_NEED_AWS=0
export TF_NEED_GCP=0
export TF_NEED_HDFS=0
export TF_NEED_S3=0
export TF_ENABLE_XLA=1
export TF_NEED_GDR=0
export TF_NEED_VERBS=0
export TF_NEED_OPENCL=0
export TF_NEED_MPI=0
export TF_NEED_TENSORRT=0
export TF_NEED_NGRAPH=0
export TF_NEED_IGNITE=0
export TF_NEED_ROCM=0
export TF_SET_ANDROID_WORKSPACE=0
export TF_DOWNLOAD_CLANG=0
export TF_NCCL_VERSION=${NCCLSHORTVERSION}
export TF_IGNORE_MAX_BAZEL_VERSION=1
export NCCL_INSTALL_PATH=${CUDADIR}
export CC=gcc
export CXX=g++
export GCC_HOST_COMPILER_PATH=/usr/bin/${CC}
export HOST_C_COMPILER=/usr/bin/${CC}
export HOST_CXX_COMPILER=/usr/bin/${CXX}
export TF_CUDA_CLANG=0
export CLANG_CUDA_COMPILER_PATH=/usr/bin/clang
export TF_CUDA_PATHS=${CUDADIR},${CUDADIR}/extras/CUPTI,${CUDADIR}/nvvm
export TF_CUDA_VERSION=$(${CUDADIR}/bin/nvcc --version | sed -n 's/^.*release \(.*\),.*/\1/p') # 11.5
export TF_CUDNN_VERSION=$(sed -n 's/^#define CUDNN_MAJOR\s*\(.*\).*/\1/p' ${CUDADIR}/include/cudnn_version.h)
# hardware older than sm_52 is said to be unsupported, compute_86 denotes an emulation
export TF_CUDA_COMPUTE_CAPABILITIES=sm_52,sm_53,sm_60,sm_61,sm_62,sm_70,sm_72,sm_75,sm_80,sm_86,compute_86
# stay compatible with the cluster machines
export CC_OPT_FLAGS="-march=nehalem"
export TF_NEED_CUDA=1
export BAZEL_ARGS="--config=mkl -c opt"
./configure
bazel build \
--cxxopt="-march=nehalem" \
--copt="-march=nehalem" \
--noshow_loading_progress \
--noshow_progress \
--color="no" \
//tensorflow:libtensorflow.so \
//tensorflow:libtensorflow_cc.so \
//tensorflow:install_headers \
//tensorflow/tools/pip_package:build_pip_package
bazel shutdown # otherwise it hogs in cwd
# create a _clean/decent_ python package from so far created debris
test -d tensorflow-pip && rm -r tensorflow-pip
mkdir tensorflow-pip
./bazel-bin/tensorflow/tools/pip_package/build_pip_package --src ${BUILDDIR}/tensorflow-pip
cd ${BUILDDIR}
# ... cope with the ??? symlink-structure placed in bazels' build-TMPDIR (-h) (tell me why ?)
test -e tensorflow-pip.tar && rm tensorflow-pip.tar
tar -chf tensorflow-pip.tar tensorflow-pip
# and perform a regular install
test -d tensorflow-pip && rm -r tensorflow-pip
tar -xf tensorflow-pip.tar
cd ${BUILDDIR}/tensorflow-pip
python3 setup.py build
python3 setup.py install --prefix $PREFIX
)
piplist; track tensorflow
# tidy up a bit
if [ -d $PREFIX/man/man1 ]; then
mv -v $PREFIX/man/man1/* $PREFIX/share/man/man1 || true
rm -rf $PREFIX/man
fi
# ------------------------------------------------------------ Sanity check
# load all packages, see warnings and spot installation errors
# But don't care about r2py, a further dependence into R is not wanted!
echo "# START load test."
python3 -c 'help("modules")' > /dev/null
echo "# END load test (passed)."
echo "# running pip check"
pip3 check
# I guess it makes sense that files can be read by all users ...
echo "# check for unreadable files"
cd $PREFIX
find \! -perm -004 -exec chmod -c a+r {} +
echo "For Deeper checks:"
echo "cd ${BUILDDIR} ; git log --patch "
exit
# -------------------------------------------------------- Included patches
#Epatch:pip # Operation 'Forever young', disable version-check per default.
#Epatch:pip # The option needs to be kept, because some tools make use of
#Epatch:pip # it and would cause pip to fail.
#Epatch:pip --- a/pip/_internal/cli/cmdoptions.py 2022-08-09 17:29:15.853442948 +0200
#Epatch:pip +++ b/pip/_internal/cli/cmdoptions.py 2022-08-10 10:31:35.124945154 +0200
#Epatch:pip @@ -892,7 +892,6 @@
#Epatch:pip dest="disable_pip_version_check",
#Epatch:pip action="store_true",
#Epatch:pip - default=False,
#Epatch:pip - help="Don't periodically check PyPI to determine whether a new version "
#Epatch:pip - "of pip is available for download. Implied with --no-index.",
#Epatch:pip + default=True,
#Epatch:pip + help="Ignore this option, the version check IS disabled.",
#Epatch:pip )
#Epatch:numba1 # Fix lib location, turn numba_sysinfo.py into a common diag tool
#Epatch:numba1 --- a/numba/cuda/cuda_paths.py 2022-10-14 09:57:25.616827200 +0200
#Epatch:numba1 +++ b/numba/cuda/cuda_paths.py 2022-10-20 11:20:15.106019523 +0200
#Epatch:numba1 @@ -68,3 +68,3 @@
#Epatch:numba1 else:
#Epatch:numba1 - return 'lib64'
#Epatch:numba1 + return 'lib'
#Epatch:numba2 --- a/numba/misc/numba_sysinfo.py 2022-10-14 09:57:25.628827000 +0200
#Epatch:numba2 +++ b/numba/misc/numba_sysinfo.py 2022-10-20 11:02:54.056767540 +0200
#Epatch:numba2 @@ -1 +1,2 @@
#Epatch:numba2 +#! /usr/bin/env python3
#Epatch:numba2 import json
#Epatch:numba2 @@ -367,3 +368,3 @@
#Epatch:numba2 with redirect_stdout(output):
#Epatch:numba2 - cudadrv.libs.test(sys.platform, print_paths=False)
#Epatch:numba2 + cudadrv.libs.test(sys.platform, print_paths=True)
#Epatch:numba2 sys_info[_cu_lib_test] = output.getvalue()
#Epatch:numba2 @@ -626,9 +627,2 @@
#Epatch:numba2 ("",),
#Epatch:numba2 - ("__Conda Information__",),
#Epatch:numba2 - (DisplayMap({k: v for k, v in info.items()
#Epatch:numba2 - if k.startswith('Conda')}) or ("Conda not available.",)),
#Epatch:numba2 - ("",),
#Epatch:numba2 - ("__Installed Packages__",),
#Epatch:numba2 - DisplaySeq(info.get(_inst_pkg, ("Couldn't retrieve packages info.",))),
#Epatch:numba2 - ("",),
#Epatch:numba2 ("__Error log__" if info.get(_errors, [])
#Epatch:gyp1 --- a/pylib/gyp/generator/cmake.py
#Epatch:gyp1 +++ b/pylib/gyp/generator/cmake.py
#Epatch:gyp1 @@ -41,7 +41,7 @@
#Epatch:gyp1 try:
#Epatch:gyp1 # maketrans moved to str in python3.
#Epatch:gyp1 + _maketrans = str.maketrans
#Epatch:gyp1 +except NameError:
#Epatch:gyp1 _maketrans = string.maketrans
#Epatch:gyp1 -except NameError:
#Epatch:gyp1 - _maketrans = str.maketrans
#Epatch:gyp1 generator_default_variables = {
#Epatch:gyp1 'EXECUTABLE_PREFIX': '',
#Epatch:gyp1 @@ -283,2 +283,2 @@
#Epatch:gyp1 if int(action.get('process_outputs_as_sources', False)):
#Epatch:gyp1 - extra_sources.extend(zip(cmake_outputs, outputs))
#Epatch:gyp1 + extra_sources.extend(list(zip(cmake_outputs, outputs)))
#Epatch:gyp1 @@ -988,6 +988,6 @@
#Epatch:gyp1 # XCode settings
#Epatch:gyp1 xcode_settings = config.get('xcode_settings', {})
#Epatch:gyp1 - for xcode_setting, xcode_value in xcode_settings.viewitems():
#Epatch:gyp1 + for xcode_setting, xcode_value in xcode_settings.items():
#Epatch:gyp1 SetTargetProperty(output, cmake_target_name,
#Epatch:gyp1 "XCODE_ATTRIBUTE_%s" % xcode_setting, xcode_value,
#Epatch:gyp1 '' if isinstance(xcode_value, str) else ' ')
#Epatch:gyp2 diff --git a/pylib/gyp/input.py b/pylib/gyp/input.py
#Epatch:gyp2 index 4c12891..2bea334 100644
#Epatch:gyp2 --- a/pylib/gyp/input.py
#Epatch:gyp2 +++ b/pylib/gyp/input.py
#Epatch:gyp2 @@ -1183,7 +1183,7 @@
#Epatch:gyp2 if variable_name in variables:
#Epatch:gyp2 # If the variable is already set, don't set it.
#Epatch:gyp2 continue
#Epatch:gyp2 - if the_dict_key is 'variables' and variable_name in the_dict:
#Epatch:gyp2 + if the_dict_key == 'variables' and variable_name in the_dict:
#Epatch:gyp2 # If the variable is set without a % in the_dict, and the_dict is a
#Epatch:gyp2 # variables dict (making |variables| a varaibles sub-dict of a
#Epatch:gyp2 # variables dict), use the_dict's definition.
#Epatch:triton diff --git a/setup.py b/setup.py
#Epatch:triton index 2ac3accd2..e7357be58 100644
#Epatch:triton --- a/setup.py
#Epatch:triton +++ b/setup.py
#Epatch:triton @@ -211,10 +211,8 @@ setup(
#Epatch:triton long_description="",
#Epatch:triton packages=["triton", "triton/_C", "triton/language", "triton/tools", "triton/impl", "triton/ops", "triton/runtime", "triton/ops/blocksparse"],
#Epatch:triton install_requires=[
#Epatch:triton - "cmake",
#Epatch:triton "filelock",
#Epatch:triton "torch",
#Epatch:triton - "lit",
#Epatch:triton ],
#Epatch:triton package_data={"triton": ["third_party/**/*"]},
#Epatch:triton include_package_data=True,
#Epatch:triton @@ -233,6 +231,10 @@ setup(
#Epatch:triton ],
#Epatch:triton test_suite="tests",
#Epatch:triton extras_require={
#Epatch:triton + "build": [
#Epatch:triton + "cmake>=3.18",
#Epatch:triton + "lit",
#Epatch:triton + ],
#Epatch:triton "tests": [
#Epatch:triton "autopep8",
#Epatch:triton "flake8",
# -------------------------------------------------------- This is the end.