From 8f99a7d68692038b144d94f99876eafa1b2c891c Mon Sep 17 00:00:00 2001 From: thomas Date: Tue, 9 Apr 2019 17:18:34 +0200 Subject: [PATCH 1/8] R-3.5.3: Start w copy of R-3.5.2-0.build.sh --- R-3.5.3-0.build.sh | 161 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 161 insertions(+) create mode 100755 R-3.5.3-0.build.sh diff --git a/R-3.5.3-0.build.sh b/R-3.5.3-0.build.sh new file mode 100755 index 0000000..838d254 --- /dev/null +++ b/R-3.5.3-0.build.sh @@ -0,0 +1,161 @@ +#! /bin/bash + +# https://github.com/MathurinD/STASNet ??? +# http://bioconductor.org/packages/stats/ +# +# ^Rcpp (>= 0.10.4) +# ^BH +# RhpcBLASctl +# ^ Rgraphviz +# ^ pheatmap +# ^ lattice +# ^ lhs +# parallel +# + +# https://www.r-project.org/ + +PKG=R +VERSION=3.5.2 +BUILD=0 + +PREFIX=/pkg/$PKG-$VERSION-$BUILD +if [ -n "$TESTING" ]; then PREFIX=/dev/shm/$PKG-$VERSION-$BUILD ; fi + +# neither home nor /usr/local/package +PATH=/bin:/usr/bin:/usr/sbin:/usr/local/bin +export PATH + +set -xe +umask 022 + +BUILD_TMPDIR=/dev/shm/$PKG-$VERSION-$BUILD.build.tmp +test -d $BUILD_TMPDIR && rm -r $BUILD_TMPDIR +mkdir -p $BUILD_TMPDIR/home +export TMPDIR=$BUILD_TMPDIR +export HOME=$BUILD_TMPDIR/home + +exec $PREFIX/profile <<-EOF + PATH=$PREFIX/bin:\$PATH + if [ -d $PREFIX/.compatlibs ]; then export LD_LIBRARY_PATH=$PREFIX/.compatlibs\${LD_LIBRARY_PATH:+:\$LD_LIBRARY_PATH} ; fi +EOF +. $PREFIX/profile + +export MAKEFLAGS="-j $(nproc)" + +BUILDDIR=$BUILD_TMPDIR/build +mkdir -p $BUILDDIR +cd $BUILDDIR + +test -e R-$VERSION.tar.gz || wget https://cran.r-project.org/src/base/R-3/R-$VERSION.tar.gz +test -d R-$VERSION || tar xvf R-$VERSION.tar.gz +cd R-$VERSION + +sed -i 's/en_GB/en_US/g' tests/reg-tests-3.R tests/reg-tests-3.Rout.save + +./configure \ + --prefix=$PREFIX \ + --with-pic \ + --with-recommended-packages \ + --enable-shared \ + --with-x \ + --enable-R-shlib \ + LDFLAGS=-Wl,-rpath,$PREFIX/lib64/R/lib + +export TZ=CEST # some tests need timezone +make +make check +make install + +cd $BUILDDIR + +R --quiet --vanilla -e "update.packages(repos='https://cran.uni-muenster.de/',ask=FALSE)" + +# Bioconductor base packages +# +test -e biocLite.R || wget https://bioconductor.org/biocLite.R +R --quiet --vanilla -e "source('biocLite.R');biocLite()" + +# CRAN packages +# we install with BiocInstaller (biocLite), because some CRAN packages +# have dependencies into Bioconductor packages +# +PKG=( + AdequacyModel BBmisc BH BatchJobs BayesDA Boruta BradleyTerry2 C50 Cairo CircStats Ckmeans.1d.dp DBI + DEoptimR ExomeDepth FField Formula GGally GeneNet GlobalOptions HMM HardyWeinberg Hmisc KernSmooth + LSD Lahman LearnBayes MASS MatrixEQTL Matrix PerfMeas R.cache R.methodsS3 R.oo R.utils R2HTML R6 + RCircos RColorBrewer RCurl RGtk2 RJSONIO RMySQL ROCR RRF RSQLite RUnit RWeka RWekajars RcppArmadillo + RcppEigen Rcpp SparseM SuppDists TFMPvalue TH.data TSP TeachingDemos VGAM VennDiagram WGCNA XML + abind acepack ade4 akima aod ape ash assertthat base64 base64enc bdsmatrix beanplot beeswarm bgmm + bigmemory.sri bigmemory bigrf bit bitops boot bootstrap bpca brew brglm c060 caTools car caret + checkmate chron circlize class clue cluster coda codetools coin colorspace combinat corpcor + corrgram corrplot crayon curl data.table deldir devtools dichromat digest dismo doMC doParallel + doRNG doSNOW dostats dplyr dtw dynamicTreeCut e1071 earth elasticnet energy entropy evaluate evd + exactRankTests fANCOVA fail fastICA fastcluster fastmatch fdrtool fields filehash flashClust foreach + foreign formatR futile.logger futile.options gRbase gbm gclus gdata genetics getopt ggm ggplot2 git2r + glasso glmnet gmp gnm gplots gridBase gridExtra gridSVG grofit gsubfn gtable gtools h2o haplo.stats + harvestr hash hdi hdrcde hexbin hflights highr hsmm htmltools httpuv httr hwriter igraph inline intervals + ipred iterators itertools jsonlite kernlab knitr ks labeling lambda.r lars latticeExtra lattice lava + lazyeval leaps lhs linprog lint lintr lme4 lmtest locfit longitudinal lpSolve magrittr manipulate mapproj + maps maptools maptree markdown matrixStats mclust mda memoise meta mgcv mice microbenchmark mime minqa + misc3d mitools mixtools mlegp mnormt modeltools multcomp multicool munsell mvtnorm nleqslv nlme nloptr + nnet nor1mix numDeriv optparse pROC pamr party partykit pbapply pbkrtest pcalg penalizedSVM peperr pheatmap + pkgmaker plotmo plotrix pls plus plyr prodlim profileModel proto proxy pwr qtl quadprog quantreg qvcalc + rFerns rJava randomForest randomGLM raster rbenchmark registry relaimpo relimp remotes reshape2 reshape rex + rggobi rgl rjson rlecuyer rmarkdown rmeta rngtools robustbase roxygen2 rpart rstudioapi rversions sandwich + scales scalreg scatterplot3d sda segmented sendmailR seqinr seriation setRNG setwidth sfsmisc shape + shiny sm snow snowfall som sp spam spatial spdep splancs spls statmod stepPlr stringdist stringi stringr + strucchange survey survival survrec svMisc svTools tables testthat tgp tidyr vcd venneuler verification + waveslim whisker xlsx xlsxjars xml2 xtable yaml zipfR zoo +) + +P=$(printf ',"%s"' "${PKG[@]}") # ,"ACME","ALL".... +P=${P#,} # "ACME","ALL",... + +R --quiet --vanilla -e "source('biocLite.R');biocLite(c($P))" + +# Bioconductor packages, and user request even if they are from cran, reason: see above + +PKG=( + ACME ALL AnnotationDbi AnnotationForge BSgenome.Ecoli.NCBI.20080805 BSgenome BeadDataPackR BioNet Biobase + BiocGenerics BiocInstaller BiocParallel Biostrings CNEr Category CellNOptR ChIPpeakAnno DESeq2 DESeq + DNAcopy DirichletMultinomial FDb.InfiniumMethylation.hg19 GEOquery GO.db GOTHiC GOstats GSEABase GSEAlm + GenomeGraphs GenomeInfoDb GenomicAlignments GenomicFeatures GenomicRanges GlobalAncova Gviz HTqPCR + Homo.sapiens IRanges MEDIPS OmicCircos OrganismDbi RBGL ROC Rgraphviz Rhtslib Rsamtools S4Vectors + ShortRead TFBSTools TxDb.Hsapiens.UCSC.hg19.knownGene VariantAnnotation XVector affyPLM affy affyio + annotate arrayQualityMetrics bamsignals beadarray biomaRt biovizBase bumphunter chipseq cummeRbund easyRNASeq + edgeR exomeCopy gcrma genefilter geneplotter genomeIntervals ggbio globaltest graph hgu95av2 hgu95av2cdf + illuminaio impute limma lumi methylumi minfi multtest muscle org.Hs.eg.db pcaMethods preprocessCore qvalue + rhdf5 rtracklayer seqLogo siggenes simpleaffy snpStats topGO vsn yeastCC zlibbioc + DEXSeq + qsea # 13.12.2016 lienhard + devtools # 29.03.2017 pacini + YosefLab/scone # 29.03.2017 pacini + rhondabacher/SCnorm # 29.03.2017 pacini + MAST # 29.03.2017 pacini + scDD # 03.03.2017 pacini + zinbwave # 15.03.2018 stanisla + monocle # 21.06.2018 pacinihdf5r + hdf5r Seurat # 20.08.2018 kretzmer + # digest # 29.08.2018 stanisla (update request, already present see above) + QuasR MethylSeekR # 30.08.2018 alena + rnaseqGene apeglm # 4.09.2018 alena + RNAseq123 Glimma # 4.09.2018 alena +) + +P=$(printf ',"%s"' "${PKG[@]}") # ,"ACME","ALL".... +P=${P#,} # "ACME","ALL",... + +R --quiet --vanilla -e "source('biocLite.R');biocLite(c($P))" + +# BPSC needs kpsewhich from '/usr/local/package/bin/kpsewhich', so extend the path here and now +PATH=/bin:/usr/bin:/usr/sbin:/usr/local/bin:/usr/local/package/bin +export PATH +. $PREFIX/profile +R --quiet --vanilla -e 'library("devtools");install_github("nghiavtr/BPSC")' # 3.03.2017 pacini +R --quiet --vanilla -e 'library("devtools");install_github("jokergoo/ComplexHeatmap");install_github("jokergoo/EnrichedHeatmap")' # 15.02.2019 kretzmer + +exit From b397ed5a3d700a590d51158024651188ed041783 Mon Sep 17 00:00:00 2001 From: thomas Date: Tue, 9 Apr 2019 17:21:01 +0200 Subject: [PATCH 2/8] R-3.5.3: Remove old reminders --- R-3.5.3-0.build.sh | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/R-3.5.3-0.build.sh b/R-3.5.3-0.build.sh index 838d254..9e4f49e 100755 --- a/R-3.5.3-0.build.sh +++ b/R-3.5.3-0.build.sh @@ -1,18 +1,5 @@ #! /bin/bash -# https://github.com/MathurinD/STASNet ??? -# http://bioconductor.org/packages/stats/ -# -# ^Rcpp (>= 0.10.4) -# ^BH -# RhpcBLASctl -# ^ Rgraphviz -# ^ pheatmap -# ^ lattice -# ^ lhs -# parallel -# - # https://www.r-project.org/ PKG=R From 2322ba607af51d77f6b3967ed9f2c68563522ebe Mon Sep 17 00:00:00 2001 From: thomas Date: Tue, 9 Apr 2019 17:22:21 +0200 Subject: [PATCH 3/8] R-3.5.3: Adjust version --- R-3.5.3-0.build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/R-3.5.3-0.build.sh b/R-3.5.3-0.build.sh index 9e4f49e..d170b5d 100755 --- a/R-3.5.3-0.build.sh +++ b/R-3.5.3-0.build.sh @@ -3,7 +3,7 @@ # https://www.r-project.org/ PKG=R -VERSION=3.5.2 +VERSION=3.5.3 BUILD=0 PREFIX=/pkg/$PKG-$VERSION-$BUILD From 9f192daa0c084d0e2cb2731127897832861fcb1a Mon Sep 17 00:00:00 2001 From: thomas Date: Wed, 10 Apr 2019 08:49:58 +0200 Subject: [PATCH 4/8] R-3.5.3: Fulfil prerequisites for 'dynverse/dyno' - need the udunits-2 package - need a 'personal access token' for github see: https://dynverse.org/users/1-installation/ https://www.unidata.ucar.edu/software/udunits/ https://github.com/ --- R-3.5.3-0.build.sh | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/R-3.5.3-0.build.sh b/R-3.5.3-0.build.sh index d170b5d..6c901e2 100755 --- a/R-3.5.3-0.build.sh +++ b/R-3.5.3-0.build.sh @@ -2,6 +2,13 @@ # https://www.r-project.org/ +# export GITHUB_PAT='5***b' 38 secret chars :) +if [ -z "$GITHUB_PAT" ] ; then + echo "# Error: GITHUB_PAT is not set, build of 'dynverse/dyno' is supposed to fail." + echo "# Check the dvteam mails for 'personal access token'" + exit 1 +fi + PKG=R VERSION=3.5.3 BUILD=0 @@ -37,6 +44,18 @@ BUILDDIR=$BUILD_TMPDIR/build mkdir -p $BUILDDIR cd $BUILDDIR +( # for 'dynverse/dyno' + test -e udunits-2.2.26.tar.gz || wget ftp://ftp.unidata.ucar.edu/pub/udunits/udunits-2.2.26.tar.gz + test -d udunits-2.2.26 || tar xvf udunits-2.2.26.tar.gz + cd udunits-2.2.26 + CFLAGS='-O2 -fPIC' \ + ./configure --prefix=$PREFIX --libdir=$PREFIX/lib64 --enable-shared=no + make + make install +) +export UDUNITS2_INCLUDE=$PREFIX/include +export UDUNITS2_LIBS=$PREFIX/lib64 + test -e R-$VERSION.tar.gz || wget https://cran.r-project.org/src/base/R-3/R-$VERSION.tar.gz test -d R-$VERSION || tar xvf R-$VERSION.tar.gz cd R-$VERSION From 0e80cf9dbbfa5fa8cad666eb77d2ee646fedb44b Mon Sep 17 00:00:00 2001 From: thomas Date: Wed, 10 Apr 2019 09:03:38 +0200 Subject: [PATCH 5/8] R-3.5.3: Can't use 'R --vanilla' any longer The implicit --no-environ of the --vanilla option would inhibit the evaluation of the GITHUB_PAT variable Bonus: MAKEFLAGS are now evaluated too, speeding up the package builds by approx. 50% --- R-3.5.3-0.build.sh | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/R-3.5.3-0.build.sh b/R-3.5.3-0.build.sh index 6c901e2..c80e2e4 100755 --- a/R-3.5.3-0.build.sh +++ b/R-3.5.3-0.build.sh @@ -40,6 +40,10 @@ EOF export MAKEFLAGS="-j $(nproc)" +# We need the environment (MAKEFLAGS, GITHUB_PAT), so '--vanilla' is no longer an option +# --vanilla : Combine --no-save, --no-restore, --no-site-file --no-init-file and --no-environ +STRACCIATELLA='--no-save --no-restore --no-site-file --no-init-file' + BUILDDIR=$BUILD_TMPDIR/build mkdir -p $BUILDDIR cd $BUILDDIR @@ -78,12 +82,12 @@ make install cd $BUILDDIR -R --quiet --vanilla -e "update.packages(repos='https://cran.uni-muenster.de/',ask=FALSE)" +R $STRACCIATELLA --quiet -e "update.packages(repos='https://cran.uni-muenster.de/',ask=FALSE)" # Bioconductor base packages # test -e biocLite.R || wget https://bioconductor.org/biocLite.R -R --quiet --vanilla -e "source('biocLite.R');biocLite()" +R $STRACCIATELLA --quiet -e "source('biocLite.R');biocLite()" # CRAN packages # we install with BiocInstaller (biocLite), because some CRAN packages @@ -121,7 +125,7 @@ PKG=( P=$(printf ',"%s"' "${PKG[@]}") # ,"ACME","ALL".... P=${P#,} # "ACME","ALL",... -R --quiet --vanilla -e "source('biocLite.R');biocLite(c($P))" +R $STRACCIATELLA --quiet -e "source('biocLite.R');biocLite(c($P))" # Bioconductor packages, and user request even if they are from cran, reason: see above @@ -155,13 +159,13 @@ PKG=( P=$(printf ',"%s"' "${PKG[@]}") # ,"ACME","ALL".... P=${P#,} # "ACME","ALL",... -R --quiet --vanilla -e "source('biocLite.R');biocLite(c($P))" +R $STRACCIATELLA --quiet -e "source('biocLite.R');biocLite(c($P))" # BPSC needs kpsewhich from '/usr/local/package/bin/kpsewhich', so extend the path here and now PATH=/bin:/usr/bin:/usr/sbin:/usr/local/bin:/usr/local/package/bin export PATH . $PREFIX/profile -R --quiet --vanilla -e 'library("devtools");install_github("nghiavtr/BPSC")' # 3.03.2017 pacini -R --quiet --vanilla -e 'library("devtools");install_github("jokergoo/ComplexHeatmap");install_github("jokergoo/EnrichedHeatmap")' # 15.02.2019 kretzmer +R $STRACCIATELLA --quiet -e 'library("devtools");install_github("nghiavtr/BPSC")' # 3.03.2017 pacini +R $STRACCIATELLA --quiet -e 'library("devtools");install_github("jokergoo/ComplexHeatmap");install_github("jokergoo/EnrichedHeatmap")' # 15.02.2019 kretzmer exit From a69130f4c4ba278cbc855d60f3b6f8b2edbbabd6 Mon Sep 17 00:00:00 2001 From: thomas Date: Wed, 10 Apr 2019 09:14:41 +0200 Subject: [PATCH 6/8] R-3.5.3: Add RhpcBLASctl package Control the number of threads on 'BLAS' (Aka 'GotoBLAS', 'OpenBLAS', 'ACML', 'BLIS' and 'MKL'). And possible to control the number of threads in 'OpenMP'. Get a number of logical cores and physical cores if feasible. Sounded good to me... Only works if an external BLAS is used, not the one that's bundled with R --- R-3.5.3-0.build.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/R-3.5.3-0.build.sh b/R-3.5.3-0.build.sh index c80e2e4..2903599 100755 --- a/R-3.5.3-0.build.sh +++ b/R-3.5.3-0.build.sh @@ -154,6 +154,7 @@ PKG=( QuasR MethylSeekR # 30.08.2018 alena rnaseqGene apeglm # 4.09.2018 alena RNAseq123 Glimma # 4.09.2018 alena + RhpcBLASctl # 8.04.2019 Kreitler ) P=$(printf ',"%s"' "${PKG[@]}") # ,"ACME","ALL".... From f4a88f9e1ec0c44d91c902e8193b5d8ada529e54 Mon Sep 17 00:00:00 2001 From: thomas Date: Wed, 10 Apr 2019 09:54:03 +0200 Subject: [PATCH 7/8] R-3.5.3: Adjust github builds and include 'dynverse/dyno' --- R-3.5.3-0.build.sh | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/R-3.5.3-0.build.sh b/R-3.5.3-0.build.sh index 2903599..9b32c90 100755 --- a/R-3.5.3-0.build.sh +++ b/R-3.5.3-0.build.sh @@ -162,11 +162,17 @@ P=${P#,} # "ACME","ALL",... R $STRACCIATELLA --quiet -e "source('biocLite.R');biocLite(c($P))" +# Finally the things that come from github + +R $STRACCIATELLA --quiet -e 'library("devtools");install_github("jokergoo/ComplexHeatmap");install_github("jokergoo/EnrichedHeatmap")' # 15.02.2019 kretzmer + +# dynverse/dyno likes to have Slingshot, SCORPIUS and Monocle3 from a singularity context, well +# at least rcannood/SCORPIUS builds w/o fuzz. +R $STRACCIATELLA --quiet -e 'library("devtools");install_github("rcannood/SCORPIUS", build_vignettes = TRUE)' +R $STRACCIATELLA --quiet -e 'library("devtools");install_github("dynverse/dyno")' # 8.04.2019 Virginie Stanislas + # BPSC needs kpsewhich from '/usr/local/package/bin/kpsewhich', so extend the path here and now -PATH=/bin:/usr/bin:/usr/sbin:/usr/local/bin:/usr/local/package/bin -export PATH -. $PREFIX/profile +PATH=$PATH:/usr/local/package/bin R $STRACCIATELLA --quiet -e 'library("devtools");install_github("nghiavtr/BPSC")' # 3.03.2017 pacini -R $STRACCIATELLA --quiet -e 'library("devtools");install_github("jokergoo/ComplexHeatmap");install_github("jokergoo/EnrichedHeatmap")' # 15.02.2019 kretzmer exit From 7f859016420e339ddcee9f599ed7970fc4b916fa Mon Sep 17 00:00:00 2001 From: thomas Date: Wed, 10 Apr 2019 10:11:48 +0200 Subject: [PATCH 8/8] R-3.5.3: Use a source file for GITHUB_PAT Environment vars from root are not seen by the build user, so a simple export GITHUB_PAT=... before the build won't work. --- R-3.5.3-0.build.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/R-3.5.3-0.build.sh b/R-3.5.3-0.build.sh index 9b32c90..985d87d 100755 --- a/R-3.5.3-0.build.sh +++ b/R-3.5.3-0.build.sh @@ -3,6 +3,7 @@ # https://www.r-project.org/ # export GITHUB_PAT='5***b' 38 secret chars :) +. ./github_pat.source if [ -z "$GITHUB_PAT" ] ; then echo "# Error: GITHUB_PAT is not set, build of 'dynverse/dyno' is supposed to fail." echo "# Check the dvteam mails for 'personal access token'"