From a84e7f21f0ea479054c761d500e6a90fde9ee498 Mon Sep 17 00:00:00 2001 From: Donald Buczek Date: Thu, 1 Jul 2021 16:30:34 +0200 Subject: [PATCH 01/16] Add vmcontrol.pl Import existing script vmcontrol.pl to repository and install it into /usr/bin. The script currently lives in ~vmprj/bin and is used by multiple projects. we don't want to have a nfs dependency for this script, so just install it into the system. After the planned move of ~vmprj into /var, ~vmprj/bin wouldn't no longer be a good place for scripts, too. --- install.sh | 1 + vmcontrol/vmcontrol.pl | 164 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 165 insertions(+) create mode 100755 vmcontrol/vmcontrol.pl diff --git a/install.sh b/install.sh index 70c85ba..4668108 100755 --- a/install.sh +++ b/install.sh @@ -222,6 +222,7 @@ install_data mxshadow/mxshadow.cert.pem "$DESTDIR$sysconfdir/m install_data mxshadow/mxshadow.conf "$DESTDIR$sysconfdir/mxshadow.conf" install_data mxshadow/mxshadow.service "$DESTDIR$systemdunitdir/mxshadow.service" install_data misc_etc_files/nsswitch.conf "$DESTDIR$sysconfdir/nsswitch.conf" +install_exec vmcontrol/vmcontrol.pl "$DESTDIR$usr_bindir/vmcontrol.pl" postinstall exit diff --git a/vmcontrol/vmcontrol.pl b/vmcontrol/vmcontrol.pl new file mode 100755 index 0000000..1427d9c --- /dev/null +++ b/vmcontrol/vmcontrol.pl @@ -0,0 +1,164 @@ +#! /usr/bin/prun perl perl +use strict; +use warnings; +use IO::Socket::UNIX; +use JSON; +use Data::Dumper; + +our $DEBUG=0; + +our $s; +our $json; + +sub read_json() { + my $buffer=''; + while (1) { + my $obj=$json->incr_parse($buffer); + $DEBUG and defined $obj and print Data::Dumper->Dump([$obj],['RX']); + defined $obj and return $obj; + my $sts=sysread $s,$buffer,1024; + $sts==0 and die "qmp.socket: hangup\n"; + defined $sts or die "qmp.socket: $!\”"; + } +} +sub write_json { + my ($obj)=@_; + $DEBUG and print Data::Dumper->Dump([$obj],['TX']); + $s->print(encode_json($obj)); +} + +sub exec_cmd { + my ($command,@args)=@_; + write_json({'execute'=>$command,'arguments'=>{@args}}); + while(1) { + my $obj=read_json(); + $obj->{'return'} and return $obj->{'return'}; + } +} + +our $PATTERN=qr/^(.+)\.(\d\d\d)\.disk$/; +our %DISK; # ( 'github_root' => [ 'ide-hd0',21 ] + +sub verify_disks { + my ($obj)=@_; + for my $dev (@$obj) { + my $inserted=$dev->{'inserted'} or next; + my $device=$dev->{'device'}; + $device =~ /^ide\d+-cd\d+$/ and next; + my $image=$inserted->{'image'}; + my $filename=$image->{'filename'}; + #printf "verify %-20s %s\n",$device,$filename; + my ($basename,$nnn)=$filename=~$PATTERN or die "$filename: wrong pattern (expected: NAME.\\d\\d\\d.disk)\n"; + -s "$basename.disk" or die "$basename.disk: symlink is missing\n"; + readlink("$basename.disk") eq $filename or die "symlink $basename.disk not pointing to $filename\n"; + + my %FILES; + opendir my $d,"."; + while (readdir $d) { + my ($cmp_basename,$cmp_nnn)= $_=~$PATTERN or next; + $cmp_basename eq $basename or next; + $FILES{$_}=1; + } + delete $FILES{$filename}; + + + my $depth=0; + for (my $backing_image=$image->{'backing-image'};$backing_image;$backing_image=$backing_image->{'backing-image'}) { + $depth++; + my $b_filename=$backing_image->{'filename'}; + #printf " %-20s %s\n",'',$b_filename; + my ($b_basename,$b_nnn)=$b_filename=~$PATTERN or die "backing file $filename: wrong pattern\n"; + $b_basename eq $basename or die "$b_filename: wrong basename\n"; + $b_nnn == $nnn-$depth or die "$b_filename: number should be ".($nnn-$depth)."\n"; + delete $FILES{$b_filename}; + } + if (%FILES) { + warn "unused files: ".join(' ',keys %FILES)."\n"; + } + $DISK{$basename}=[$device,$nnn]; + } +} + +sub verify_no_block_jobs_running { + my $obj=exec_cmd('query-block-jobs'); + @$obj and die "block jobs running!\n"; +} + +sub wait_block_jobs { + while (1) { + my $obj=exec_cmd('query-block-jobs'); + Data::Dumper->Dump([$obj],['O']); + @$obj or return; + for my $job (@$obj) { + my $device=$job->{'device'}; + my $offset=$job->{'offset'}; + my $len=$job->{'len'}; + printf "%-15s %d/%d (%5.1f%%)\n",$device,$offset,$len,$offset*100/$len; + } + sleep 1; + } +} + +sub snapshot { + my ($basename)=@_; + exists $DISK{$basename} or die "$basename does not exist in VM\n"; + my ($devname,$nnn)=@{$DISK{$basename}}; + my $filename=sprintf '%s.%03d.disk',$basename,$nnn+1; + + print "snapshot $devname -> $filename\n"; + exec_cmd('blockdev-snapshot-sync','device'=>$devname,'snapshot-file'=>$filename); + -s "$basename.disk" and unlink "$basename.disk"; + symlink($filename,"$basename.disk") or die "failed to symlink $basename.disk -> $filename: $!\n"; + $DISK{$basename}=[$devname,$nnn+1]; +} + +sub block_stream { + my ($basename)=@_; + exists $DISK{$basename} or die "$basename does not exist in VM\n"; + my ($devname,$nnn)=@{$DISK{$basename}}; + exec_cmd('block-stream','device'=>$devname); +} + + +$json=new JSON; +$s=new IO::Socket::UNIX(Type=>SOCK_STREAM,Peer=>'./qmp.socket') or die "qmp.socket: $!\n"; +exec_cmd('qmp_capabilities'); + + +sub USAGE { + <<"EOF"; +usage: $0 cmd + snapshot NAME # new instance NAME.xxx.disk + block-stream NAME # start stream up to top layer + ( hint: `fstrim -av` on the vm may save some space ) + wait # wait till all block jobs are finished + powerdown # send ACPI powerdown + noop # do nothing, but verify disks as always +EOF +} + +@ARGV or die USAGE; +my ($cmd,@args)=(@ARGV); + +my $block_info=exec_cmd('query-block'); +verify_disks($block_info); + +if ($cmd eq 'snapshot') { + @args==1 or die USAGE; + verify_no_block_jobs_running(); + snapshot(@args); +} elsif ($cmd eq 'block-stream') { + @args==1 or die USAGE; + verify_no_block_jobs_running(); + block_stream(@args); +} elsif ($cmd eq 'wait') { + @args==0 or die USAGE; + wait_block_jobs(); +} elsif ($cmd eq 'powerdown') { + @args==0 or die USAGE; + exec_cmd('system_powerdown'); +} elsif ($cmd eq 'noop') { + @args==0 or die USAGE; +} else { + die USAGE; +} From 1eb2502998b0a585481ef89f1512f3e1ec7ca231 Mon Sep 17 00:00:00 2001 From: thomas Date: Tue, 6 Jul 2021 19:04:15 +0200 Subject: [PATCH 02/16] usrlocalpkg: add binary rstudio package for debian Use for comparison and bug hunting. Introducing also a more or less smart launcher for the case, when the workstation uses the nvidia driver and the remote doesn't. It is called rstudio-nv. --- usrlocalpkg/admin/config | 3 +++ 1 file changed, 3 insertions(+) diff --git a/usrlocalpkg/admin/config b/usrlocalpkg/admin/config index 441311a..6798a30 100644 --- a/usrlocalpkg/admin/config +++ b/usrlocalpkg/admin/config @@ -1411,6 +1411,9 @@ rstudio-1.4.1103-0 rstudio rsession +rstudio-debpkg-1.4.1717-0 + rstudio-debpkg=rstudio + rstudio-debpkg-nv=rstudio-nv R-devel=R-devel-2017-02-02-1 From 0975c41784bafe4c274c6f30510ce15999216b0b Mon Sep 17 00:00:00 2001 From: Donald Buczek Date: Tue, 27 Jul 2021 13:36:34 +0200 Subject: [PATCH 03/16] usrlocalpkg: Update rust default to 1.52.1 --- usrlocalpkg/admin/config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/usrlocalpkg/admin/config b/usrlocalpkg/admin/config index 6798a30..e27bf8b 100644 --- a/usrlocalpkg/admin/config +++ b/usrlocalpkg/admin/config @@ -1515,7 +1515,7 @@ CellProfiler-2.2.0-0 CellProfiler-3.1.5-0 cellprofiler -rustc-1.50.0-0 +rustc-1.52.1-0 rustc cargo cbindgens From 77b1fb2f53382ccea9a9c1a153830510b581ade9 Mon Sep 17 00:00:00 2001 From: Donald Buczek Date: Thu, 29 Jul 2021 09:25:09 +0200 Subject: [PATCH 04/16] usrlocalpkg: Add alias and more wrapper for rust Add package alias "rust" so that we can easily run tools from the current rust package which don't deserve a wrapper on their own. For example prun rust miri Add wrappers for `rustdoc` and `rustfmt`. These are available via cargo, too, ( `cargo doc`, `cargo fmt` ) but, for example, `cargo help fmt` refers to "Options passed to rustfmt" so it would be useful, if `rustfmt --help` would work. Also, external documentation might suggest users to run these commands directly. --- usrlocalpkg/admin/config | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/usrlocalpkg/admin/config b/usrlocalpkg/admin/config index e27bf8b..6f92311 100644 --- a/usrlocalpkg/admin/config +++ b/usrlocalpkg/admin/config @@ -1515,10 +1515,12 @@ CellProfiler-2.2.0-0 CellProfiler-3.1.5-0 cellprofiler -rustc-1.52.1-0 +rust=rustc-1.52.1-0 rustc cargo cbindgens + rustdoc + rustfmt apache-maven-3.6.0-0 mvn mvnDebug From 9ce80222f0060bc922942d0f2a12a094c17a4a85 Mon Sep 17 00:00:00 2001 From: thomas Date: Thu, 29 Jul 2021 13:25:05 +0200 Subject: [PATCH 05/16] usrlocalpkg: cuda, update shortcuts Well, looks like they aren't used that much. --- usrlocalpkg/admin/config | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/usrlocalpkg/admin/config b/usrlocalpkg/admin/config index 6f92311..3218d8c 100644 --- a/usrlocalpkg/admin/config +++ b/usrlocalpkg/admin/config @@ -1426,13 +1426,18 @@ chromium=chromium-89.0.4389.128-0 cuda-9.0=cuda-9.0.176-0 cuda-9.1=cuda-9.1.85-0 +cuda-9.2=cuda-9.2.148-0 cuda-9=cuda-9.0 # keep default 9.0 because of tensorflow cuda-10.0=cuda-10.0.130-0 cuda-10.1=cuda-10.1.105-0 -cuda-10.2=cuda-10.2.89-0 +cuda-10.2=cuda-10.2.89-1 cuda-10=cuda-10.0 # keep default 10.0 because of tensorflow ... +cuda-11.0=cuda-11.0.3-0 +cuda-11.1=ccuda-11.1.1-0 +cuda-11.3=ccuda-11.3.1-0 + gcc-5=gcc-5.5.0-1 ecj gappletviewer From 8e5202ec21b2e91099b6b1e9bc4a85656003c4af Mon Sep 17 00:00:00 2001 From: thomas Date: Thu, 29 Jul 2021 13:26:17 +0200 Subject: [PATCH 06/16] usrlocalpkg: switch to bazel-4.1.0 --- usrlocalpkg/admin/config | 3 +++ 1 file changed, 3 insertions(+) diff --git a/usrlocalpkg/admin/config b/usrlocalpkg/admin/config index 3218d8c..674b420 100644 --- a/usrlocalpkg/admin/config +++ b/usrlocalpkg/admin/config @@ -1627,6 +1627,9 @@ bazel-0.28.1-0 bazel-0.28=bazel bazel=bazel-3.4.1-0 + bazel-3.4.1=bazel + +bazel=bazel-4.1.0-0 bazel From 0f121b7bf92a28d405fd38b68f94da2738f9b8a2 Mon Sep 17 00:00:00 2001 From: thomas Date: Thu, 29 Jul 2021 13:31:13 +0200 Subject: [PATCH 07/16] usrlocalpkg: fix typos in config OMG (I sould keep my door closed!) --- usrlocalpkg/admin/config | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/usrlocalpkg/admin/config b/usrlocalpkg/admin/config index 674b420..d6d20cc 100644 --- a/usrlocalpkg/admin/config +++ b/usrlocalpkg/admin/config @@ -1435,8 +1435,8 @@ cuda-10.2=cuda-10.2.89-1 cuda-10=cuda-10.0 # keep default 10.0 because of tensorflow ... cuda-11.0=cuda-11.0.3-0 -cuda-11.1=ccuda-11.1.1-0 -cuda-11.3=ccuda-11.3.1-0 +cuda-11.1=cuda-11.1.1-0 +cuda-11.3=cuda-11.3.1-0 gcc-5=gcc-5.5.0-1 ecj @@ -1626,7 +1626,7 @@ voreen bazel-0.28.1-0 bazel-0.28=bazel -bazel=bazel-3.4.1-0 +bazel-3.4.1-0 bazel-3.4.1=bazel bazel=bazel-4.1.0-0 From e9c18017f511666907bb8155dacc2dc201f323a7 Mon Sep 17 00:00:00 2001 From: Paul Menzel Date: Mon, 2 Aug 2021 15:16:22 +0200 Subject: [PATCH 08/16] systemd/enable_overcommit_memory: Use `/sbin/sysctl` for Ubuntu MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ubuntu’s package *procps* 2:3.3.16-5ubuntu3.1 installs `sysctl` only in `/sbin/sysctl`, and therefore the service unit fails to be started. Instead of `/usr/sbin/sysctl`, use `/sbin/sysctl`, which is a symbolic link on MarIuX. $ ls -l /sbin/sysctl # usrlinks-1.0-0.x86_64 lrwxrwxrwx 1 root root 16 May 29 15:02 /sbin/sysctl -> /usr/sbin/sysctl Alternatively, as systemd does not require absolute pathnames anymore, only `sysctl` could be used, but be explicit for now. --- misc_systemd_units/enable_overcommit_memory.service | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/misc_systemd_units/enable_overcommit_memory.service b/misc_systemd_units/enable_overcommit_memory.service index 678c68e..302248f 100644 --- a/misc_systemd_units/enable_overcommit_memory.service +++ b/misc_systemd_units/enable_overcommit_memory.service @@ -8,8 +8,8 @@ ConditionHost=|flughafenberlinbrandenburgwillybrandt.molgen.mpg.de [Service] Type=oneshot RemainAfterExit=yes -ExecStart=/usr/sbin/sysctl vm.overcommit_memory=0 -ExecStop=/usr/sbin/sysctl vm.overcommit_memory=2 +ExecStart=/sbin/sysctl vm.overcommit_memory=0 +ExecStop=/sbin/sysctl vm.overcommit_memory=2 [Install] WantedBy=graphical.target From dac25ac73596058534acc0cb2987375e776d41a6 Mon Sep 17 00:00:00 2001 From: thomas Date: Fri, 6 Aug 2021 17:43:47 +0200 Subject: [PATCH 09/16] usrlocalpkg: Add python3.9 and CellProfiler-4 Just provide python3.9, and check a little bit before making it the default. --- usrlocalpkg/admin/config | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/usrlocalpkg/admin/config b/usrlocalpkg/admin/config index d6d20cc..1b6e94c 100644 --- a/usrlocalpkg/admin/config +++ b/usrlocalpkg/admin/config @@ -1352,6 +1352,14 @@ python-3.8=python-3.8.9-1 python3.8=python-3.8 +python-3.9=python-3.9.6-0 + python3.9 + python3.9-config + pip3.9 + pydoc3.9 + +python3.9=python-3.9 + # b) default python, see PEP 0394 too python-3=python-3.8 @@ -1520,6 +1528,9 @@ CellProfiler-2.2.0-0 CellProfiler-3.1.5-0 cellprofiler +CellProfiler-4.2.1-0 + cellprofiler4=cellprofiler + rust=rustc-1.52.1-0 rustc cargo From 7572632d335efe3b702dbebb686a6929badfebc7 Mon Sep 17 00:00:00 2001 From: thomas Date: Fri, 6 Aug 2021 18:04:39 +0200 Subject: [PATCH 10/16] mxsnoop: Add prototype Use to monitor network traffic on site. --- mxsnoop/mxsnoop.sh | 117 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 117 insertions(+) create mode 100755 mxsnoop/mxsnoop.sh diff --git a/mxsnoop/mxsnoop.sh b/mxsnoop/mxsnoop.sh new file mode 100755 index 0000000..a8d39a7 --- /dev/null +++ b/mxsnoop/mxsnoop.sh @@ -0,0 +1,117 @@ +#!/bin/bash + +# +# +--------------+ (!) +----------------------+ +---**** +# | | | br01 | | +# | THE SUSPECT O]===X===[O SCHNUEFFELSTUECK O]======[O WIS-NET +# | | |INLET OUTLET| | +# +--------------+ +----------------------+ +---**** +# (MX_NETDEV) + +# Basics: https://wiki.archlinux.org/title/Network_bridge + +# GATEWAY=10.0.3.1 +# BROADCAST=10.0.3.255 +# NETMASK=255.255.255.0 +# NIDL=24 + +GATEWAY=141.14.16.128 +BROADCAST=141.14.31.255 +NETMASK=255.255.240.0 +NIDL=20 + +INLET=${INLET:-} +OUTLET=${OUTLET:-} +BRIDGE=br01 + +TRANSPARENT=${TRANSPARENT:-} +STP=${STP:-} + +function die() { echo $1; exit 1; } + +type ip > /dev/null || \ + die 'Error: Tools not available, check for ip command.' + +MXHOSTCONF=${MXHOSTCONF:-/etc/local/mxhost.conf} +source $MXHOSTCONF || die "# Error: No 'mxhost.conf' ?" +test -n "$MX_IPADDR" || die '# Error: MX_IPADDR missing.' + +if [ -z "$OUTLET" ]; then + test -n "$MX_NETDEV" || die '# Error: MX_NETDEV missing?' + test -e /sys/class/net/$MX_NETDEV || die "# Error: $MX_NETDEV missing." + OUTLET=$MX_NETDEV +fi + + +# inlet was present at boot time -> netXY, or plugged later -> Search for ethX +if [ -z "$INLET" ]; then # find the (sole) inlet + for D in net{00..09} eth{0..9}; do + test -e /sys/class/net/$D || continue + test $D = $OUTLET && continue + test -n "$INLET" && \ + die 'Error: Too many network devices (netXY) present, use INLET=dev (OUTLET=dev ?).' + INLET=$D + done +fi + +test -z "$INLET" && die '# Error: no second network device found.' + +echo "# Note: about to setup/destroy $BRIDGE between $INLET (INLET) and $OUTLET (OUTLET), IP: $MX_IPADDR" + +cmd="$1" + +case "$cmd" in + + start) + + if [ -z "$STP" ]; then + ip link add dev $BRIDGE type bridge + else + ip link add dev $BRIDGE type bridge stp_state 1 + fi + + ip address flush dev $INLET + ip address flush dev $OUTLET + + ip link set $INLET master $BRIDGE + ip link set $OUTLET master $BRIDGE + + if [ -z "$TRANSPARENT" ] ; then + ip addr add $MX_IPADDR/$NIDL broadcast $BROADCAST dev $BRIDGE + ip link set up dev $BRIDGE + ip route add default via $GATEWAY dev $BRIDGE + fi + + ip link set dev $INLET up + ip link set dev $OUTLET up + + ;; + + stop) + + ip link set $INLET nomaster + ip link set $OUTLET nomaster + ip link delete $BRIDGE type bridge + + ip addr add $MX_IPADDR/$NIDL broadcast $BROADCAST dev $OUTLET + ip link set dev $OUTLET up + ip route add default via $GATEWAY + + ;; + + restart) + + $0 stop + sleep 1 + $0 start + ;; + + *) + + echo "# usage: [INLET=dev] [OUTLET=dev] $0 [start|stop|restart]" + echo "# options: TRANSPARENT=1 doesn't bind IP on the bridge" + echo "# STP=1 enables Spanning Tree" + ;; + +esac + From e0336eb8139246c42253f40b3d5e43d2020ed262 Mon Sep 17 00:00:00 2001 From: thomas Date: Wed, 11 Aug 2021 17:11:18 +0200 Subject: [PATCH 11/16] prun: Add documentation in markdown format Manual pages can be generated with: pandoc --standalone --to man prun.md -o prun.man.1 --- prun/pman.md | 37 ++++++++++++++++++ prun/prun.md | 102 ++++++++++++++++++++++++++++++++++++++++++++++++++ prun/ptype.md | 42 +++++++++++++++++++++ 3 files changed, 181 insertions(+) create mode 100644 prun/pman.md create mode 100644 prun/prun.md create mode 100644 prun/ptype.md diff --git a/prun/pman.md b/prun/pman.md new file mode 100644 index 0000000..8518d23 --- /dev/null +++ b/prun/pman.md @@ -0,0 +1,37 @@ +% PTYPE(1) | Mariux Tools Documentation + +NAME +==== + +**pman** - view manual pages installed in Mariux packages. + +SYNOPSIS +======== + +| **pman command** + +DESCRIPTION +=========== + +**ptype** uses the PATH set in the package wrapper and tries to find a manual page in the package directory. + +EXAMPLES +======== + +**display manpage found in /pkg/stellarium-0.21.0-0/share/man/man1** +: pman stellarium + + +BUGS +==== + +**pman** is very naive in looking up man pages, so your mileage may vary. + +See GitHub Issues: + +SEE ALSO +======== + +**prun(1)**, **ptype(1)**, **man(1)** + + diff --git a/prun/prun.md b/prun/prun.md new file mode 100644 index 0000000..ba36071 --- /dev/null +++ b/prun/prun.md @@ -0,0 +1,102 @@ +% PRUN(1) Version n/a | Mariux Tools Documentation + +NAME +==== + +**prun** - Run Mariux packages (reads 'Package RUN'). + +SYNOPSIS +======== + +| **prun package command** [COMMANDOPTION]... + +DESCRIPTION +=========== + +Runs applications, scripting languages, compilers, etc., from a dedicated directory. + +The main usage is to run software in a specific version, or to provide a given environment. See **EXAMPLES** below. + +There are some advantages over so called 'virtual environments' and the like. Packages are immutable, once installed they never change, thereby increasing the chances that an old program/script will also run in a few years. The packages also +use a standardized call procedure, independent of personal (un-)settings, thus an existing pipeline may be shared between different users. + +The whole 'magic' behind **prun** is, that it transparently sets the PATH variable and possibly LD_LIBRARY_PATH for the actual call without spoiling the default environment. + +EXAMPLES +======== + + +**Run R-Studio with a brand new R version** + +: prun R-4.1.0-0 rstudio + +**Use an ancient python to run an unmaintained script** + +: prun python-2.7.13-2 /project/cruft/bin/old_script.py + +**Run nvidia nvcc from cuda-11.3 and list supported architectures** + +: prun cuda-11.3 nvcc --list-gpu-code + +**Start a plain shell with gcc-10.2 in the PATH** + +: prun gcc-10.2.0-0 bash --norc + +**Or just show the manual page for gcc-10.2** + +: prun gcc-10.2.0-0 man gcc + +**Create an own alias for a scripting language** + +: alias myperl='prun perl-5.18.2-0 perl' + +**Use prun in a script shebang** + +: `#! /usr/bin/prun perl-5.18.2-0 perl` + +**Alternatively using source** + +: An alternative way to set a distinct environment is to source the profile from /pkg/FooBar-12.3. +But beware, such a source command should **never** make it into the **\~/.bashrc** - in the worst case it may inhibit your login! +(N.b. the alias method mentioned above is safe) + +\ +: source /pkg/gcc-10.2.0-0/profile + + +FILES +===== + +*/package/FOO* or */pkg/BAR* + +: Mountpoint (top directory) for selected package. + +*/package/FOO/profile* or */pkg/BAR/profile* + +: Contains settings for PATH, libraries, and possibly other stuff. + +*/usr/local/package/lib/\** + +: Primary wrappers for individual packages. + +*/usr/local/package/admin/config* + +: Definition of binaries provided and optional shortcuts. + + +BUGS +==== + +See GitHub Issues: + +AUTHORS +======= + +D.Buczek, P.Marquardt, M.Tolzmann + +SEE ALSO +======== + +**ptype(1)**, **pman(1)**, **bash(1)**, **ld.so(8)** + + diff --git a/prun/ptype.md b/prun/ptype.md new file mode 100644 index 0000000..72c7d74 --- /dev/null +++ b/prun/ptype.md @@ -0,0 +1,42 @@ +% PTYPE(1) | Mariux Tools Documentation + +NAME +==== + +**ptype** - small **type** like tool for Mariux packages. + +SYNOPSIS +======== + +| **ptype command** + +DESCRIPTION +=========== + +Similar to **type** or **which**. **ptype** gives information about a package binary. **type** would only tell something about the package wrapper, and not about the package where the real program is. + +``` + + #> type snakemake + snakemake is /usr/local/package/bin/snakemake + + #> ptype snakemake + snakemake is /pkg/python-3.8.9-1/bin/snakemake ... + ... (/usr/local/package/lib/python.profile) + + +``` + +BUGS +==== + +**ptype** only handles true wrapper scripts from /usr/local/package/bin, not the aliases. I.e. **ptype R** works, **ptype R4** not. + +See GitHub Issues: + +SEE ALSO +======== + +**prun(1)**, **pman(1)** + + From d841b64c7b2b7c3c2416406e17ca468063e08f59 Mon Sep 17 00:00:00 2001 From: Donald Buczek Date: Thu, 12 Aug 2021 10:39:06 +0200 Subject: [PATCH 12/16] Add build/install logic for prun manpages --- .gitignore | 1 + Makefile | 16 +++++++++++----- install.sh | 4 ++++ 3 files changed, 16 insertions(+), 5 deletions(-) diff --git a/.gitignore b/.gitignore index 2f683ef..41c83a2 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ setuid/setuid +prun/*.1 diff --git a/Makefile b/Makefile index 9810ebc..481b18e 100644 --- a/Makefile +++ b/Makefile @@ -64,13 +64,18 @@ INSTALL=install -v INSTALL_PROGRAM = $(INSTALL) INSTALL_DATA = $(INSTALL) -m 644 -all: - @echo 'Nothing to be done. Ready for "make install"' +manpages += prun/prun.1 prun/pman.1 prun/ptype.1 -install: +all: $(manpages) + +clean: + @rm $(manpages) + +install: all @prefix="$(prefix)" usr_prefix="$(usr_prefix)" usrlocal_prefix="$(usrlocal_prefix)" \ root_exec_prefix="$(root_exec_prefix)" root_bindir="$(root_bindir)" root_sbindir="$(root_sbindir)" \ usr_exec_prefix="$(usr_exec_prefix)" usr_bindir="$(usr_bindir)" usr_sbindir="$(usr_sbindir)" \ + usr_mandir="$(usr_mandir)" \ usrlocal_exec_prefix="$(usrlocal_exec_prefix)" usrlocal_bindir="$(usrlocal_bindir)" usrlocal_sbindir="$(usrlocal_sbindir)" \ sysconfdir="$(sysconfdir)" systemdunitdir="$(systemdunitdir)" \ udev_rulesdir="$(udev_rulesdir)" udev_helperdir="$(udev_helperdir)" \ @@ -79,6 +84,7 @@ install: DESTDIR="$(DESTDIR)" \ ./install.sh -.PHONY: all install - +.PHONY: all install clean +%.1: %.md + pandoc --standalone --to man $< -o $@ diff --git a/install.sh b/install.sh index 4668108..385614b 100755 --- a/install.sh +++ b/install.sh @@ -21,6 +21,7 @@ fi : ${usr_exec_prefix:=$usr_prefix} : ${usr_bindir:=$usr_exec_prefix/bin} : ${usr_sbindir:=$usr_exec_prefix/sbin} +: ${usr_mandir:=$usr_exec_prefix/man} : ${usrlocal_exec_prefix:=$usrlocal_prefix} : ${usrlocal_bindir:=$usrlocal_exec_prefix/bin} @@ -223,6 +224,9 @@ install_data mxshadow/mxshadow.conf "$DESTDIR$sysconfdir/m install_data mxshadow/mxshadow.service "$DESTDIR$systemdunitdir/mxshadow.service" install_data misc_etc_files/nsswitch.conf "$DESTDIR$sysconfdir/nsswitch.conf" install_exec vmcontrol/vmcontrol.pl "$DESTDIR$usr_bindir/vmcontrol.pl" +install_data prun/prun.1 "$DESTDIR$usr_mandir/man1/prun.1" +install_data prun/ptype.1 "$DESTDIR$usr_mandir/man1/ptype.1" +install_data prun/pman.1 "$DESTDIR$usr_mandir/man1/pman.1" postinstall exit From fad0b5454a6d4788d9133a348e7039197b22316b Mon Sep 17 00:00:00 2001 From: Donald Buczek Date: Thu, 12 Aug 2021 12:11:38 +0200 Subject: [PATCH 13/16] Makefile: Shut up "make clean" Use `-f` flag of `rm` to avoid error messages from `make clean` when the targets files don't exist. --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 481b18e..164c794 100644 --- a/Makefile +++ b/Makefile @@ -69,7 +69,7 @@ manpages += prun/prun.1 prun/pman.1 prun/ptype.1 all: $(manpages) clean: - @rm $(manpages) + @rm -f $(manpages) install: all @prefix="$(prefix)" usr_prefix="$(usr_prefix)" usrlocal_prefix="$(usrlocal_prefix)" \ From d7d563834ade87882828968acde738307ef06f5a Mon Sep 17 00:00:00 2001 From: Donald Buczek Date: Thu, 12 Aug 2021 12:14:04 +0200 Subject: [PATCH 14/16] Makefile: Avoid running "pandoc" as root --- Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile b/Makefile index 164c794..e984744 100644 --- a/Makefile +++ b/Makefile @@ -87,4 +87,5 @@ install: all .PHONY: all install clean %.1: %.md + @if [ "$$UID" = 0 ]; then echo "Please build (\`make\`) as non-root before running \`make install\` as root" >&2;exit 1;fi pandoc --standalone --to man $< -o $@ From 52b24e5857ac9bb7cd14b4db2f7e830843e1a504 Mon Sep 17 00:00:00 2001 From: thomas Date: Mon, 16 Aug 2021 13:49:47 +0200 Subject: [PATCH 15/16] prun man pages: cleanup Remove authors from prun.md, reformat Remove copy-and-paste errors in pman.md --- prun/pman.md | 4 ++-- prun/prun.md | 8 +------- 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/prun/pman.md b/prun/pman.md index 8518d23..ae747a8 100644 --- a/prun/pman.md +++ b/prun/pman.md @@ -1,4 +1,4 @@ -% PTYPE(1) | Mariux Tools Documentation +% PMAN(1) | Mariux Tools Documentation NAME ==== @@ -13,7 +13,7 @@ SYNOPSIS DESCRIPTION =========== -**ptype** uses the PATH set in the package wrapper and tries to find a manual page in the package directory. +**pman** uses the PATH set in the package wrapper and tries to find a manual page in the package directory. EXAMPLES ======== diff --git a/prun/prun.md b/prun/prun.md index ba36071..b9cc124 100644 --- a/prun/prun.md +++ b/prun/prun.md @@ -17,8 +17,7 @@ Runs applications, scripting languages, compilers, etc., from a dedicated direct The main usage is to run software in a specific version, or to provide a given environment. See **EXAMPLES** below. -There are some advantages over so called 'virtual environments' and the like. Packages are immutable, once installed they never change, thereby increasing the chances that an old program/script will also run in a few years. The packages also -use a standardized call procedure, independent of personal (un-)settings, thus an existing pipeline may be shared between different users. +There are some advantages over so called 'virtual environments' and the like. Packages are immutable, once installed they never change, thereby increasing the chances that an old program/script will also run in a few years. The packages also use a standardized call procedure, independent of personal (un-)settings, thus an existing pipeline may be shared between different users. The whole 'magic' behind **prun** is, that it transparently sets the PATH variable and possibly LD_LIBRARY_PATH for the actual call without spoiling the default environment. @@ -89,11 +88,6 @@ BUGS See GitHub Issues: -AUTHORS -======= - -D.Buczek, P.Marquardt, M.Tolzmann - SEE ALSO ======== From 78ff0435334c2b8ca5db1f8863a03c9f2d68bdd4 Mon Sep 17 00:00:00 2001 From: thomas Date: Mon, 16 Aug 2021 14:59:43 +0200 Subject: [PATCH 16/16] usrlocalpkg: switch to python-3.9 Summary: - meson version is now 0.59.0 - snakemake is 6.6.1 - pdbfixer is used by AlphaFold, so it is in the path. --- usrlocalpkg/admin/config | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/usrlocalpkg/admin/config b/usrlocalpkg/admin/config index 1b6e94c..268d4ef 100644 --- a/usrlocalpkg/admin/config +++ b/usrlocalpkg/admin/config @@ -1362,7 +1362,7 @@ python3.9=python-3.9 # b) default python, see PEP 0394 too -python-3=python-3.8 +python-3=python-3.9 python3 python3-config idle3=idle @@ -1373,7 +1373,7 @@ python-3=python-3.8 python3=python-3 # c) manage python based tools (meson, snakemake, ...) -# mind python3-wrapper.be0 +# Check existence! And mind python3-wrapper.be0 too python=python-3 bamCompare @@ -1395,6 +1395,7 @@ python=python-3 multiqc nbshow pairtools + pdbfixer pgt pip pyGenomeTracks