From 61f9e1cdfe68d0456332769ff97f4c7997b47caf Mon Sep 17 00:00:00 2001 From: thomas Date: Mon, 17 Feb 2025 14:42:26 +0100 Subject: [PATCH] fragpipe: add version 22.0 Opposed to an other wellknown analysis tool for mass-spec data, this one appears to work rather well under linux. It is noticeable that this package is beeing developed under windows, so some kludges are needed to get it working in a common read-only directory. --- fragpipe-22.0-0.build.sh | 482 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 482 insertions(+) create mode 100755 fragpipe-22.0-0.build.sh diff --git a/fragpipe-22.0-0.build.sh b/fragpipe-22.0-0.build.sh new file mode 100755 index 0000000..5a6eb92 --- /dev/null +++ b/fragpipe-22.0-0.build.sh @@ -0,0 +1,482 @@ +#! /usr/bin/bash + +# ------------------------------------------------------------------- Debug + +# COOKIE=$(mcookie|cut -c-8); grep -v V_GREP_ME $0 > /dev/shm/runme-$COOKIE.sh ; sleep 0.3; exec bash /dev/shm/runme-$COOKIE.sh +# TESTING=1 + +# ---------------------------------------------------------------- Preamble + +PKG=fragpipe +VERSION=22.0 +BUILD=0 + +# some pipeline parts need python +PYTHONVER=3.11.11 + +set -e +umask 022 + +PATH=/bin:/usr/bin:/usr/sbin:/usr/local/bin:/usr/local/package/bin +PREFIX=/pkg/$PKG-$VERSION-$BUILD +if [ -n "$TESTING" ]; then PREFIX=/scratch/local2/$PKG-$VERSION-$BUILD ; fi + +mkdir -p $PREFIX +cat >$PREFIX/profile <<-EOF + . /pkg/openjdk-21.0.0.35-0/profile + PATH=$PREFIX/bin:\$PATH + # avoid interference with the package/pkg python + if [ -z "\$PYTHONUSERBASE" ]; then export PYTHONUSERBASE=\$HOME/.local/fragpipe-python ; fi + if [ -d $PREFIX/.compatlibs ]; then export LD_LIBRARY_PATH=$PREFIX/.compatlibs\${LD_LIBRARY_PATH:+:}\$LD_LIBRARY_PATH ; fi +EOF + +source $PREFIX/profile + +BUILD_TMPDIR=/scratch/local2/$PKG-$VERSION-$BUILD.$USER.build.tmp +# keep pip's download cache when testing +test -z "$TESTING" && test -d $BUILD_TMPDIR && ( chmod -R u+rwx $BUILD_TMPDIR || true ; rm -rf $BUILD_TMPDIR ) +mkdir -p $BUILD_TMPDIR/home +# embedded patches, copy script to an accessible location +cp $0 $BUILD_TMPDIR; ME="$BUILD_TMPDIR/$(basename $0)" +export TMPDIR=$BUILD_TMPDIR +export HOME=$BUILD_TMPDIR/home + +NPROC=$(( $(nproc) * 4 / 5 + 1 )) + +BUILDDIR=$PREFIX/build + +mkdir -p $BUILDDIR +cd $BUILDDIR + + +# ---------------------------------------------------------------- fragpipe + +# the complete installable archive, but permissions are a mess! +# orig: https://github.com/Nesvilab/FragPipe/releases/download/22.0/FragPipe-22.0.zip +BEEHIVE=https://beehive.molgen.mpg.de/e30d93e6c0ef1e2aadd51e732f343451/FragPipe-22.0.zip + +test -e FragPipe-$VERSION.zip || wget -nv $BEEHIVE +test -d fragpipe || bsdtar -xf FragPipe-$VERSION.zip + +mv fragpipe/bin .. +mv fragpipe/lib .. # 'libjava' would be smart, but fp uses lib in lookup routines :/ +mv fragpipe/tools .. +mv fragpipe/workflows ../workflows_dist + +# reset executable attribute in the obvious cases, nerve ... +# some *.py have a shebang, and some ... (you guessed) +# this is safer then accidentally 'defusing' linux binaries within tools +for ext in bat dll exe ico jar obo txt tsv ifl gdb pdb xml json glyc fasta workflow params; do + find $PREFIX -type f -name "*.$ext" -exec chmod -c 0644 {} \+ +done +find $PREFIX -type d -exec chmod -c 0755 {} \+ + +# set jar mtimes according to content (btw. I didn't know that java was already around in 1980 :) +rm -vf jarredater +sed -ne '/^#Epatch:jarredater/ s/^#Epatch:\S* // p' $ME | patch -p0 --verbose +find $PREFIX -name '*.jar' -exec python jarredater {} \+ + +( + # get rid of the way distracting console warning "No interface org.openide.util.spi.SVGLoader instance found" + cd ../lib + mkdir -p fixup + cd fixup + bsdtar -xf ../org-netbeans-swing-outline-RELEASE200.jar + rm -v ./org/netbeans/swing/etable/columns.svg # the root cause + jar -M -c -f org-netbeans-swing-outline-RELEASE200.jar META-INF org + # alternative ... + # mv META-INF/MANIFEST.MF . + # jar -c -f org-netbeans-swing-outline-RELEASE200.jar -m MANIFEST.MF META-INF org + mv -v org-netbeans-swing-outline-RELEASE200.jar .. + cd .. + rm -r fixup +) + +# the source archive , some bigger libs missing (diann torch_cpu), permissions here are again messed up +# 'FragPipe-22.0.tar.gz' -> '/src/mariux/md5repo/bfaae508143e0a67b28a6393cbb7b2fb/FragPipe-22.0.tar.gz' +# orig: https://github.com/Nesvilab/FragPipe/archive/refs/tags/22.0.tar.gz +BEEHIVE=https://beehive.molgen.mpg.de/bfaae508143e0a67b28a6393cbb7b2fb/FragPipe-22.0.tar.gz + +test -e FragPipe-$VERSION.tar.gz || wget -nv $BEEHIVE +test -d FragPipe-$VERSION || tar -xf FragPipe-$VERSION.tar.gz + +cd FragPipe-$VERSION + +# -------------------------------------------------------------- patch time + +# several GUI patches, some are cosmetic, some are related +# to the fact that the installation directory is read only + +cd ./MSFragger-GUI + +# okay, we have them, so we also like to use them ... +sed -i -e '/^\s\s*public static final int maxProcessors/ s/= 128;/= 256;/' \ + src/com/dmtavt/fragpipe/tabs/TabWorkflow.java + +# ... and please do not call home on every start +sed -i \ + -e "/raw.githubusercontent.com/ s|https.*PATH_BUNDLE|file://$PREFIX/etc/Bundle.properties\"|" \ + -e '/^\s\s*+ ".properties"\s*$/ d' \ + src/com/dmtavt/fragpipe/params/ThisAppProps.java + +sed -ne '/^#Epatch:FragpipeLocations / s/^#Epatch:\S* // p' $ME | patch -p2 --verbose +sed -ne '/^#Epatch:Fragpipe / s/^#Epatch:\S* // p' $ME | patch -p2 --verbose + +# ------------------------------------------------------------- compile now + +# a full build, but it will not help with the file attributes, s.a. :/ +# result: build/github-release/FragPipe-22.0.zip +# #> ./gradlew makeReleaseZipNoJre + +# debug/devel builds for fragpipe-22.0.jar +# #> ./gradlew jar --rerun + +# only the recompiled gui is needed ... +# result: build/libs/fragpipe-22.0.jar +./gradlew --console plain --max-workers $NPROC jar +mv -v build/libs/fragpipe-22.0.jar $PREFIX/lib + +./gradlew --stop # since it is a 'daemon' + +# ---------------------------------------------------------- finish install + +cd $PREFIX + +mkdir -p etc ltools updates workflows + +# the file that is supposed to be downloaded from git-hub +cp -v build/FragPipe-22.0/MSFragger-GUI/src/com/dmtavt/fragpipe/Bundle.properties etc + +# this works w/o further installing logging jars: +# JAVA_OPTS=-Dlogback.configurationFile=/package/fragpipe/etc/logback.xml fragpipe +rm -vf etc/logback.xml +sed -ne '/^#Epatch:logback/ s/^#Epatch:\S* // p' $ME | patch -p0 --verbose + + +# the academic license was acquired by H. Schmidt, some of the tools seem to +# talk with google analytics, just in case you wonder :) +REPO_LICENSED_SOFTWARE=/package/fragpipe/src +cd ltools +cp -p $REPO_LICENSED_SOFTWARE/2024-417_Academic_Research_Use_License_05142024.pdf . +bsdtar -xf $REPO_LICENSED_SOFTWARE/MSFragger-4.1.zip +bsdtar -xf $REPO_LICENSED_SOFTWARE/IonQuant-1.10.27.zip +bsdtar -xf $REPO_LICENSED_SOFTWARE/diaTracer-1.1.5.zip + + +# create a neater launcher, but also keep the funny one bundled + +cat > $PREFIX/bin/mxfragpipe <<- EOF + #! /usr/bin/bash + + set -e + + # populate ~/.config/FragPipe/fragpipe/workflows on first start + + XDG_CONFIG_HOME="\${XDG_CONFIG_HOME:-\$HOME/.config}" + export XDG_CONFIG_HOME + WORKFLOWDIR="\$XDG_CONFIG_HOME/FragPipe/fragpipe/workflows" + if [ ! -d "\$WORKFLOWDIR" ] ; then # assume first run ... + mkdir -vp "\$WORKFLOWDIR" + cp --preserve=timestamps /package/fragpipe/workflows.dist/* "\$WORKFLOWDIR" + echo 'fragpipe-config.tools-folder=$PREFIX/ltools' >> "\$XDG_CONFIG_HOME/FragPipe/fragpipe/fragpipe-ui.cache" + fi + + + # avoid stumbling over our filerunner or doublecommander apps + # grep -l inode/directory /usr/share/applications/*.desktop + + if [ ! -e \$XDG_CONFIG_HOME/FragPipe/no_check_filemanager ]; then + FILEMANAGER=\$(xdg-mime query default inode/directory) + if [[ "\$FILEMANAGER" =~ 'doublecmd' || "\$FILEMANAGER" =~ 'filerunner' ]]; then + echo "# ===========================================================================" + echo "#" + echo "# NOTE: Your default filemanager application is set to \$FILEMANAGER ..." + echo "#" + echo "# to fix this issue and switch to thunar, enter:" + echo "# #> xdg-mime default thunar.desktop inode/directory" + echo "#" + echo "# to inhibit this message enter:" + echo "# #> touch \$XDG_CONFIG_HOME/FragPipe/no_check_filemanager" + echo "#" + echo "# ===========================================================================" + sleep 5 + echo "# launching fragpipe now ..." + sleep 1 + fi + fi + + # -classpath '/foobar/lib/*' only adds *.jar files - ohh, what a surprise :) + exec java \$FRAGPIPE_OPTS \$JAVA_OPTS \\ + -classpath '$PREFIX/lib/*' \\ + com.dmtavt.fragpipe.FragPipeMain \\ + "\${@+\$@}" +EOF + +chmod +x $PREFIX/bin/mxfragpipe + +echo '# Fragpipe install is done ...' + +# ------------------------------------------------------------------ python + +cd $BUILDDIR + +# -------------------------------------- Use git to track package evolution + +test -d .git && rm -rf .git +git init -q; echo -e '[user]\n name = none\n email = of_your_business...' >> .git/config +echo '*' > .gitignore + +function piplist() { + pip list | awk '{ printf("%-36s %s\n", $1,$2) }' | grep -v '\----' > GameOfVersions +} + +function track() { + if [ -d .git ]; then + if git add -f GameOfVersions; then + git commit -q -n -m "Package: '$*'" || /bin/true + fi + fi +} + +function install() { + echo "# ::INST:: # '${@+$@}'" + # no-color is nice, but doesn't work when dependencies are installed, so (mis-)use a pipe + pip install --no-color --compile --cache-dir=$HOME/.cache/pip --prefix=$PREFIX "${@+$@}" | cat + piplist + track $* +} + + +# ---------------------------------------- Build Python & support libraries + +( + # orig: https://www.python.org/ftp/python/3.11.11/Python-3.11.11.tgz + BEEHIVE=https://beehive.molgen.mpg.de/9a5b43fcc06810b8ae924b0a080e6569/Python-3.11.11.tgz + + test -e Python-$PYTHONVER.tgz || wget -nv $BEEHIVE + test -d Python-$PYTHONVER || tar -xf Python-$PYTHONVER.tgz + cd Python-$PYTHONVER + + # BerkeleyDB, the undead ... + # Newer libgdbm_compat.so versions pull libgdbm.so on their own, our libgdbm_compat.so is older ... + # So, let setup.py create '-lgdbm -lgdbm_compat' instead of a sole '-lgdbm_compat' + # verbatim: ndbm_libs = ['gdbm_compat'] => ndbm_libs = ['gdbm','gdbm_compat'] + sed -i -e "/ndbm_libs = / s/_compat'/','gdbm_compat'/" setup.py + + # leave further hints in the log + sed -i -e '/db_setup_debug = False/ s/False/True/' \ + -e '/dbm_setup_debug = False/ s/False/True/' setup.py + + # These may come handy if the Python build needs exclusive stuff from $PREFIX + # INCLUDEDIR=$PREFIX/include \ + # LIBDIR=$PREFIX/lib \ + LDFLAGS="-Wl,-rpath=$PREFIX/lib" \ + ./configure \ + --prefix=$PREFIX \ + --enable-shared + + make -j $NPROC + make install + + # it might come to happen that something down the line may just call 'python', defuse ... + test -e $PREFIX/bin/python || ln -s python3 $PREFIX/bin/python + + python -m ensurepip + pip3 install --prefix=$PREFIX -I pip + ( # fix 'please update' noise from pip + cd $PREFIX/lib/python*/site-packages + sed -ne '/^#Epatch:pip/ s/^#Epatch:\S* // p' $ME | patch -p1 --verbose + ) + # I hate to do this ... + ln -s lib $PREFIX/lib64 +) +install setuptools +piplist; track START + +install Cython +install numpy==1.26.4 # The EasyPQP author nailed numpy to this version (needlessly of course, <2 is the right way ...) +install numba==0.60.0 +install wheel + +PACKAGES=$(sed -e s/#.*$// <<- __PKGLIST__ + pyopenms==3.2.0 + easypqp==0.1.50 + lxml==5.3.0 + __PKGLIST__ +) + +for PKG in $PACKAGES; do + install $PKG +done + +# ------------------------------------------------------------ Sanity check + +echo "# --------- *** SANITY CHECKS *** ---------" + +# I guess it makes sense that files can be read by all users ... +echo "# 1) Check for unreadable files ..." +cd $PREFIX +for D in bin include lib share; do + find $D \! -perm -004 -exec chmod -c a+r {} + +done + +# a cheap one :) +echo "# 2) Running pip check" +pip check + +# load all packages, see warnings and spot installation errors +echo "# 3) Module load test." +export NUMEXPR_MAX_THREADS=1 +python -c 'help("modules")' +echo "# Note: reached end of load test" + +echo "For Deeper checks:" +echo "cd ${BUILDDIR} ; git log --patch | grep ^[\+\-][^\+\-]" + +exit + + +# -------------------------------------------------------- Included patches + +#Epatch:pip # Operation 'Forever young', disable version-check per default. +#Epatch:pip # The option needs to be kept, because some tools make use of +#Epatch:pip # it and would cause pip to fail. +#Epatch:pip --- a/pip/_internal/cli/cmdoptions.py 2022-08-09 17:29:15.853442948 +0200 +#Epatch:pip +++ b/pip/_internal/cli/cmdoptions.py 2022-08-10 10:31:35.124945154 +0200 +#Epatch:pip @@ -892,7 +892,6 @@ +#Epatch:pip dest="disable_pip_version_check", +#Epatch:pip action="store_true", +#Epatch:pip - default=False, +#Epatch:pip - help="Don't periodically check PyPI to determine whether a new version " +#Epatch:pip - "of pip is available for download. Implied with --no-index.", +#Epatch:pip + default=True, +#Epatch:pip + help="Ignore this option, the version check IS disabled.", +#Epatch:pip ) + + +#Epatch:jarredater --- jarredater +#Epatch:jarredater +++ jarredater +#Epatch:jarredater @@ -0,0 +1,13 @@ +#Epatch:jarredater +import os, sys, time, zipfile +#Epatch:jarredater + +#Epatch:jarredater +dt_list = [] +#Epatch:jarredater +if len(sys.argv) > 1: +#Epatch:jarredater + for f in sys.argv[1:]: +#Epatch:jarredater + dt_list.clear() +#Epatch:jarredater + zf = zipfile.ZipFile(f, 'r') +#Epatch:jarredater + for i in zf.infolist(): dt_list.append(i.date_time) +#Epatch:jarredater + zf.close() +#Epatch:jarredater + maxtime = max( dt_list ) +#Epatch:jarredater + print('# touching', f, '->', maxtime) +#Epatch:jarredater + epo = time.mktime( maxtime + (0,0,0) ) +#Epatch:jarredater + os.utime(f, (epo, epo)) + + +#Epatch:logback --- etc/logback.xml +#Epatch:logback +++ etc/logback.xml +#Epatch:logback @@ -0,0 +1,12 @@ +#Epatch:logback + +#Epatch:logback + +#Epatch:logback + +#Epatch:logback + +#Epatch:logback + %d %-5level %-42logger{42} - %msg%n +#Epatch:logback + +#Epatch:logback + +#Epatch:logback + +#Epatch:logback + +#Epatch:logback + +#Epatch:logback + +#Epatch:logback + + + +### ========================================================================================= +### VERY CAREFUL NOW, we are patchin WinDos files, lookout for '' / '^M' in your editor ! +### ========================================================================================= + +#Epatch:FragpipeLocations diff --git a/MSFragger-GUI/src/com/dmtavt/fragpipe/FragpipeLocations.java b/MSFragger-GUI/src/com/dmtavt/fragpipe/FragpipeLocations.java +#Epatch:FragpipeLocations index 6f14590e..58f56d5f 100644 +#Epatch:FragpipeLocations --- a/MSFragger-GUI/src/com/dmtavt/fragpipe/FragpipeLocations.java +#Epatch:FragpipeLocations +++ b/MSFragger-GUI/src/com/dmtavt/fragpipe/FragpipeLocations.java +#Epatch:FragpipeLocations @@ -106,8 +106,9 @@ public class FragpipeLocations { +#Epatch:FragpipeLocations tools = dir.resolve(Paths.get("../tools")); +#Epatch:FragpipeLocations } +#Epatch:FragpipeLocations +#Epatch:FragpipeLocations - Path workflows = dir.resolve("../workflows"); +#Epatch:FragpipeLocations - Path longTermStorage = CacheUtils.getTempDir(); +#Epatch:FragpipeLocations + // Path workflows = dir.resolve("../workflows"); +#Epatch:FragpipeLocations + Path workflows = dir.resolve( CacheUtils.getTempDir().toString() + "/workflows" ); +#Epatch:FragpipeLocations + Path longTermStorage = CacheUtils.getTempDir(); // well this yields to > XDG_CONFIG_HOME + "/FragPipe" < +#Epatch:FragpipeLocations +#Epatch:FragpipeLocations // create locations if they don't yet exist +#Epatch:FragpipeLocations List paths = Arrays.asList(dir, cache, tools, workflows, longTermStorage); + + +#Epatch:Fragpipe diff --git a/MSFragger-GUI/src/com/dmtavt/fragpipe/Fragpipe.java b/MSFragger-GUI/src/com/dmtavt/fragpipe/Fragpipe.java +#Epatch:Fragpipe index 16fd82ec..64696321 100644 +#Epatch:Fragpipe --- a/MSFragger-GUI/src/com/dmtavt/fragpipe/Fragpipe.java +#Epatch:Fragpipe +++ b/MSFragger-GUI/src/com/dmtavt/fragpipe/Fragpipe.java +#Epatch:Fragpipe @@ -289,22 +289,36 @@ public class Fragpipe extends JFrameHeadless { +#Epatch:Fragpipe log.debug("Saving ui cache: collected {} properties from UI. Size after merging with cached object: {}.", +#Epatch:Fragpipe tabsAsProps.size(), cache.propsUiState.size()); +#Epatch:Fragpipe try { +#Epatch:Fragpipe - cache.propsUiState.setPath(FragpipeLocations.get().getPathUiCache(false)); +#Epatch:Fragpipe - cache.propsUiState.save(); +#Epatch:Fragpipe + // when we fail here (lib/../cache), the other one will not be written out :/ +#Epatch:Fragpipe + //cache.propsUiState.setPath(FragpipeLocations.get().getPathUiCache(false)); +#Epatch:Fragpipe + //cache.propsUiState.save(); +#Epatch:Fragpipe cache.propsUiState.setPath(FragpipeLocations.get().getPathUiCache(true)); +#Epatch:Fragpipe cache.propsUiState.save(); +#Epatch:Fragpipe } catch (IOException ex) { +#Epatch:Fragpipe - log.error("Error saving ui cache. It won't affect the results."); +#Epatch:Fragpipe + log.error("Error saving ui cache. It won't affect the results." + " (" + +#Epatch:Fragpipe + FragpipeLocations.get().getPathUiCache(true) + ")" +#Epatch:Fragpipe + ); +#Epatch:Fragpipe } +#Epatch:Fragpipe try { +#Epatch:Fragpipe - cache.propsRuntime.setPath(FragpipeLocations.get().getPathRuntimeCache(false)); +#Epatch:Fragpipe - cache.propsRuntime.save(); +#Epatch:Fragpipe + // when we fail here (lib/../cache), the other one will not be written out :/ +#Epatch:Fragpipe + //cache.propsRuntime.setPath(FragpipeLocations.get().getPathRuntimeCache(false)); +#Epatch:Fragpipe + //cache.propsRuntime.save(); +#Epatch:Fragpipe cache.propsRuntime.setPath(FragpipeLocations.get().getPathRuntimeCache(true)); +#Epatch:Fragpipe cache.propsRuntime.save(); +#Epatch:Fragpipe } catch (IOException ex) { +#Epatch:Fragpipe - log.error("Error saving runtime cache. It won't affect the results."); +#Epatch:Fragpipe + log.error("Error saving runtime cache. It won't affect the results." + " (" + +#Epatch:Fragpipe + FragpipeLocations.get().getPathRuntimeCache(true) + ")" +#Epatch:Fragpipe + ); +#Epatch:Fragpipe } +#Epatch:Fragpipe +#Epatch:Fragpipe + /* +#Epatch:Fragpipe + +#Epatch:Fragpipe + -- skipping this for the moment -- +#Epatch:Fragpipe + +#Epatch:Fragpipe + - copy the existing workflows on first start (done by the starter script) +#Epatch:Fragpipe + - then use the users XDG_CONFIG_HOME/FragPipe/fragpipe/workflows directory +#Epatch:Fragpipe + as the sole source for workflows +#Epatch:Fragpipe + +#Epatch:Fragpipe // saving workflows +#Epatch:Fragpipe Path dirWorkflows = FragpipeLocations.get().getDirWorkflows(); +#Epatch:Fragpipe Path lts = FragpipeLocations.get().getPathLongTermStorage().resolve(dirWorkflows.getFileName()); +#Epatch:Fragpipe @@ -315,6 +329,8 @@ public class Fragpipe extends JFrameHeadless { +#Epatch:Fragpipe log.error("Error saving workflows between sessions."); +#Epatch:Fragpipe throw new IllegalStateException(e); +#Epatch:Fragpipe } +#Epatch:Fragpipe + */ +#Epatch:Fragpipe + +#Epatch:Fragpipe } +#Epatch:Fragpipe +#Epatch:Fragpipe public static FormEntry.Builder fe(JComponent comp, String compName) { +#Epatch:Fragpipe @@ -385,9 +401,11 @@ public class Fragpipe extends JFrameHeadless { +#Epatch:Fragpipe System.exit(1); +#Epatch:Fragpipe } +#Epatch:Fragpipe +#Epatch:Fragpipe + /* why are linux users forced to use 'Nimbus' ? +#Epatch:Fragpipe if (!headless) { +#Epatch:Fragpipe SwingUtils.setLaf(); +#Epatch:Fragpipe } +#Epatch:Fragpipe + */ +#Epatch:Fragpipe +#Epatch:Fragpipe FragpipeLoader fragpipeLoader = new FragpipeLoader(); +#Epatch:Fragpipe Bus.register(fragpipeLoader); +