diff --git a/.gitignore b/.gitignore index 2f683ef..41c83a2 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ setuid/setuid +prun/*.1 diff --git a/Makefile b/Makefile index 9810ebc..e984744 100644 --- a/Makefile +++ b/Makefile @@ -64,13 +64,18 @@ INSTALL=install -v INSTALL_PROGRAM = $(INSTALL) INSTALL_DATA = $(INSTALL) -m 644 -all: - @echo 'Nothing to be done. Ready for "make install"' +manpages += prun/prun.1 prun/pman.1 prun/ptype.1 -install: +all: $(manpages) + +clean: + @rm -f $(manpages) + +install: all @prefix="$(prefix)" usr_prefix="$(usr_prefix)" usrlocal_prefix="$(usrlocal_prefix)" \ root_exec_prefix="$(root_exec_prefix)" root_bindir="$(root_bindir)" root_sbindir="$(root_sbindir)" \ usr_exec_prefix="$(usr_exec_prefix)" usr_bindir="$(usr_bindir)" usr_sbindir="$(usr_sbindir)" \ + usr_mandir="$(usr_mandir)" \ usrlocal_exec_prefix="$(usrlocal_exec_prefix)" usrlocal_bindir="$(usrlocal_bindir)" usrlocal_sbindir="$(usrlocal_sbindir)" \ sysconfdir="$(sysconfdir)" systemdunitdir="$(systemdunitdir)" \ udev_rulesdir="$(udev_rulesdir)" udev_helperdir="$(udev_helperdir)" \ @@ -79,6 +84,8 @@ install: DESTDIR="$(DESTDIR)" \ ./install.sh -.PHONY: all install - +.PHONY: all install clean +%.1: %.md + @if [ "$$UID" = 0 ]; then echo "Please build (\`make\`) as non-root before running \`make install\` as root" >&2;exit 1;fi + pandoc --standalone --to man $< -o $@ diff --git a/install.sh b/install.sh index 70c85ba..385614b 100755 --- a/install.sh +++ b/install.sh @@ -21,6 +21,7 @@ fi : ${usr_exec_prefix:=$usr_prefix} : ${usr_bindir:=$usr_exec_prefix/bin} : ${usr_sbindir:=$usr_exec_prefix/sbin} +: ${usr_mandir:=$usr_exec_prefix/man} : ${usrlocal_exec_prefix:=$usrlocal_prefix} : ${usrlocal_bindir:=$usrlocal_exec_prefix/bin} @@ -222,6 +223,10 @@ install_data mxshadow/mxshadow.cert.pem "$DESTDIR$sysconfdir/m install_data mxshadow/mxshadow.conf "$DESTDIR$sysconfdir/mxshadow.conf" install_data mxshadow/mxshadow.service "$DESTDIR$systemdunitdir/mxshadow.service" install_data misc_etc_files/nsswitch.conf "$DESTDIR$sysconfdir/nsswitch.conf" +install_exec vmcontrol/vmcontrol.pl "$DESTDIR$usr_bindir/vmcontrol.pl" +install_data prun/prun.1 "$DESTDIR$usr_mandir/man1/prun.1" +install_data prun/ptype.1 "$DESTDIR$usr_mandir/man1/ptype.1" +install_data prun/pman.1 "$DESTDIR$usr_mandir/man1/pman.1" postinstall exit diff --git a/misc_systemd_units/enable_overcommit_memory.service b/misc_systemd_units/enable_overcommit_memory.service index 678c68e..302248f 100644 --- a/misc_systemd_units/enable_overcommit_memory.service +++ b/misc_systemd_units/enable_overcommit_memory.service @@ -8,8 +8,8 @@ ConditionHost=|flughafenberlinbrandenburgwillybrandt.molgen.mpg.de [Service] Type=oneshot RemainAfterExit=yes -ExecStart=/usr/sbin/sysctl vm.overcommit_memory=0 -ExecStop=/usr/sbin/sysctl vm.overcommit_memory=2 +ExecStart=/sbin/sysctl vm.overcommit_memory=0 +ExecStop=/sbin/sysctl vm.overcommit_memory=2 [Install] WantedBy=graphical.target diff --git a/mxsnoop/mxsnoop.sh b/mxsnoop/mxsnoop.sh new file mode 100755 index 0000000..a8d39a7 --- /dev/null +++ b/mxsnoop/mxsnoop.sh @@ -0,0 +1,117 @@ +#!/bin/bash + +# +# +--------------+ (!) +----------------------+ +---**** +# | | | br01 | | +# | THE SUSPECT O]===X===[O SCHNUEFFELSTUECK O]======[O WIS-NET +# | | |INLET OUTLET| | +# +--------------+ +----------------------+ +---**** +# (MX_NETDEV) + +# Basics: https://wiki.archlinux.org/title/Network_bridge + +# GATEWAY=10.0.3.1 +# BROADCAST=10.0.3.255 +# NETMASK=255.255.255.0 +# NIDL=24 + +GATEWAY=141.14.16.128 +BROADCAST=141.14.31.255 +NETMASK=255.255.240.0 +NIDL=20 + +INLET=${INLET:-} +OUTLET=${OUTLET:-} +BRIDGE=br01 + +TRANSPARENT=${TRANSPARENT:-} +STP=${STP:-} + +function die() { echo $1; exit 1; } + +type ip > /dev/null || \ + die 'Error: Tools not available, check for ip command.' + +MXHOSTCONF=${MXHOSTCONF:-/etc/local/mxhost.conf} +source $MXHOSTCONF || die "# Error: No 'mxhost.conf' ?" +test -n "$MX_IPADDR" || die '# Error: MX_IPADDR missing.' + +if [ -z "$OUTLET" ]; then + test -n "$MX_NETDEV" || die '# Error: MX_NETDEV missing?' + test -e /sys/class/net/$MX_NETDEV || die "# Error: $MX_NETDEV missing." + OUTLET=$MX_NETDEV +fi + + +# inlet was present at boot time -> netXY, or plugged later -> Search for ethX +if [ -z "$INLET" ]; then # find the (sole) inlet + for D in net{00..09} eth{0..9}; do + test -e /sys/class/net/$D || continue + test $D = $OUTLET && continue + test -n "$INLET" && \ + die 'Error: Too many network devices (netXY) present, use INLET=dev (OUTLET=dev ?).' + INLET=$D + done +fi + +test -z "$INLET" && die '# Error: no second network device found.' + +echo "# Note: about to setup/destroy $BRIDGE between $INLET (INLET) and $OUTLET (OUTLET), IP: $MX_IPADDR" + +cmd="$1" + +case "$cmd" in + + start) + + if [ -z "$STP" ]; then + ip link add dev $BRIDGE type bridge + else + ip link add dev $BRIDGE type bridge stp_state 1 + fi + + ip address flush dev $INLET + ip address flush dev $OUTLET + + ip link set $INLET master $BRIDGE + ip link set $OUTLET master $BRIDGE + + if [ -z "$TRANSPARENT" ] ; then + ip addr add $MX_IPADDR/$NIDL broadcast $BROADCAST dev $BRIDGE + ip link set up dev $BRIDGE + ip route add default via $GATEWAY dev $BRIDGE + fi + + ip link set dev $INLET up + ip link set dev $OUTLET up + + ;; + + stop) + + ip link set $INLET nomaster + ip link set $OUTLET nomaster + ip link delete $BRIDGE type bridge + + ip addr add $MX_IPADDR/$NIDL broadcast $BROADCAST dev $OUTLET + ip link set dev $OUTLET up + ip route add default via $GATEWAY + + ;; + + restart) + + $0 stop + sleep 1 + $0 start + ;; + + *) + + echo "# usage: [INLET=dev] [OUTLET=dev] $0 [start|stop|restart]" + echo "# options: TRANSPARENT=1 doesn't bind IP on the bridge" + echo "# STP=1 enables Spanning Tree" + ;; + +esac + diff --git a/prun/pman.md b/prun/pman.md new file mode 100644 index 0000000..ae747a8 --- /dev/null +++ b/prun/pman.md @@ -0,0 +1,37 @@ +% PMAN(1) | Mariux Tools Documentation + +NAME +==== + +**pman** - view manual pages installed in Mariux packages. + +SYNOPSIS +======== + +| **pman command** + +DESCRIPTION +=========== + +**pman** uses the PATH set in the package wrapper and tries to find a manual page in the package directory. + +EXAMPLES +======== + +**display manpage found in /pkg/stellarium-0.21.0-0/share/man/man1** +: pman stellarium + + +BUGS +==== + +**pman** is very naive in looking up man pages, so your mileage may vary. + +See GitHub Issues: + +SEE ALSO +======== + +**prun(1)**, **ptype(1)**, **man(1)** + + diff --git a/prun/prun.md b/prun/prun.md new file mode 100644 index 0000000..b9cc124 --- /dev/null +++ b/prun/prun.md @@ -0,0 +1,96 @@ +% PRUN(1) Version n/a | Mariux Tools Documentation + +NAME +==== + +**prun** - Run Mariux packages (reads 'Package RUN'). + +SYNOPSIS +======== + +| **prun package command** [COMMANDOPTION]... + +DESCRIPTION +=========== + +Runs applications, scripting languages, compilers, etc., from a dedicated directory. + +The main usage is to run software in a specific version, or to provide a given environment. See **EXAMPLES** below. + +There are some advantages over so called 'virtual environments' and the like. Packages are immutable, once installed they never change, thereby increasing the chances that an old program/script will also run in a few years. The packages also use a standardized call procedure, independent of personal (un-)settings, thus an existing pipeline may be shared between different users. + +The whole 'magic' behind **prun** is, that it transparently sets the PATH variable and possibly LD_LIBRARY_PATH for the actual call without spoiling the default environment. + +EXAMPLES +======== + + +**Run R-Studio with a brand new R version** + +: prun R-4.1.0-0 rstudio + +**Use an ancient python to run an unmaintained script** + +: prun python-2.7.13-2 /project/cruft/bin/old_script.py + +**Run nvidia nvcc from cuda-11.3 and list supported architectures** + +: prun cuda-11.3 nvcc --list-gpu-code + +**Start a plain shell with gcc-10.2 in the PATH** + +: prun gcc-10.2.0-0 bash --norc + +**Or just show the manual page for gcc-10.2** + +: prun gcc-10.2.0-0 man gcc + +**Create an own alias for a scripting language** + +: alias myperl='prun perl-5.18.2-0 perl' + +**Use prun in a script shebang** + +: `#! /usr/bin/prun perl-5.18.2-0 perl` + +**Alternatively using source** + +: An alternative way to set a distinct environment is to source the profile from /pkg/FooBar-12.3. +But beware, such a source command should **never** make it into the **\~/.bashrc** - in the worst case it may inhibit your login! +(N.b. the alias method mentioned above is safe) + +\ +: source /pkg/gcc-10.2.0-0/profile + + +FILES +===== + +*/package/FOO* or */pkg/BAR* + +: Mountpoint (top directory) for selected package. + +*/package/FOO/profile* or */pkg/BAR/profile* + +: Contains settings for PATH, libraries, and possibly other stuff. + +*/usr/local/package/lib/\** + +: Primary wrappers for individual packages. + +*/usr/local/package/admin/config* + +: Definition of binaries provided and optional shortcuts. + + +BUGS +==== + +See GitHub Issues: + +SEE ALSO +======== + +**ptype(1)**, **pman(1)**, **bash(1)**, **ld.so(8)** + + diff --git a/prun/ptype.md b/prun/ptype.md new file mode 100644 index 0000000..72c7d74 --- /dev/null +++ b/prun/ptype.md @@ -0,0 +1,42 @@ +% PTYPE(1) | Mariux Tools Documentation + +NAME +==== + +**ptype** - small **type** like tool for Mariux packages. + +SYNOPSIS +======== + +| **ptype command** + +DESCRIPTION +=========== + +Similar to **type** or **which**. **ptype** gives information about a package binary. **type** would only tell something about the package wrapper, and not about the package where the real program is. + +``` + + #> type snakemake + snakemake is /usr/local/package/bin/snakemake + + #> ptype snakemake + snakemake is /pkg/python-3.8.9-1/bin/snakemake ... + ... (/usr/local/package/lib/python.profile) + + +``` + +BUGS +==== + +**ptype** only handles true wrapper scripts from /usr/local/package/bin, not the aliases. I.e. **ptype R** works, **ptype R4** not. + +See GitHub Issues: + +SEE ALSO +======== + +**prun(1)**, **pman(1)** + + diff --git a/usrlocalpkg/admin/config b/usrlocalpkg/admin/config index 441311a..268d4ef 100644 --- a/usrlocalpkg/admin/config +++ b/usrlocalpkg/admin/config @@ -1352,9 +1352,17 @@ python-3.8=python-3.8.9-1 python3.8=python-3.8 +python-3.9=python-3.9.6-0 + python3.9 + python3.9-config + pip3.9 + pydoc3.9 + +python3.9=python-3.9 + # b) default python, see PEP 0394 too -python-3=python-3.8 +python-3=python-3.9 python3 python3-config idle3=idle @@ -1365,7 +1373,7 @@ python-3=python-3.8 python3=python-3 # c) manage python based tools (meson, snakemake, ...) -# mind python3-wrapper.be0 +# Check existence! And mind python3-wrapper.be0 too python=python-3 bamCompare @@ -1387,6 +1395,7 @@ python=python-3 multiqc nbshow pairtools + pdbfixer pgt pip pyGenomeTracks @@ -1411,6 +1420,9 @@ rstudio-1.4.1103-0 rstudio rsession +rstudio-debpkg-1.4.1717-0 + rstudio-debpkg=rstudio + rstudio-debpkg-nv=rstudio-nv R-devel=R-devel-2017-02-02-1 @@ -1423,13 +1435,18 @@ chromium=chromium-89.0.4389.128-0 cuda-9.0=cuda-9.0.176-0 cuda-9.1=cuda-9.1.85-0 +cuda-9.2=cuda-9.2.148-0 cuda-9=cuda-9.0 # keep default 9.0 because of tensorflow cuda-10.0=cuda-10.0.130-0 cuda-10.1=cuda-10.1.105-0 -cuda-10.2=cuda-10.2.89-0 +cuda-10.2=cuda-10.2.89-1 cuda-10=cuda-10.0 # keep default 10.0 because of tensorflow ... +cuda-11.0=cuda-11.0.3-0 +cuda-11.1=cuda-11.1.1-0 +cuda-11.3=cuda-11.3.1-0 + gcc-5=gcc-5.5.0-1 ecj gappletviewer @@ -1512,10 +1529,15 @@ CellProfiler-2.2.0-0 CellProfiler-3.1.5-0 cellprofiler -rustc-1.50.0-0 +CellProfiler-4.2.1-0 + cellprofiler4=cellprofiler + +rust=rustc-1.52.1-0 rustc cargo cbindgens + rustdoc + rustfmt apache-maven-3.6.0-0 mvn mvnDebug @@ -1616,7 +1638,10 @@ voreen bazel-0.28.1-0 bazel-0.28=bazel -bazel=bazel-3.4.1-0 +bazel-3.4.1-0 + bazel-3.4.1=bazel + +bazel=bazel-4.1.0-0 bazel diff --git a/vmcontrol/vmcontrol.pl b/vmcontrol/vmcontrol.pl new file mode 100755 index 0000000..1427d9c --- /dev/null +++ b/vmcontrol/vmcontrol.pl @@ -0,0 +1,164 @@ +#! /usr/bin/prun perl perl +use strict; +use warnings; +use IO::Socket::UNIX; +use JSON; +use Data::Dumper; + +our $DEBUG=0; + +our $s; +our $json; + +sub read_json() { + my $buffer=''; + while (1) { + my $obj=$json->incr_parse($buffer); + $DEBUG and defined $obj and print Data::Dumper->Dump([$obj],['RX']); + defined $obj and return $obj; + my $sts=sysread $s,$buffer,1024; + $sts==0 and die "qmp.socket: hangup\n"; + defined $sts or die "qmp.socket: $!\”"; + } +} +sub write_json { + my ($obj)=@_; + $DEBUG and print Data::Dumper->Dump([$obj],['TX']); + $s->print(encode_json($obj)); +} + +sub exec_cmd { + my ($command,@args)=@_; + write_json({'execute'=>$command,'arguments'=>{@args}}); + while(1) { + my $obj=read_json(); + $obj->{'return'} and return $obj->{'return'}; + } +} + +our $PATTERN=qr/^(.+)\.(\d\d\d)\.disk$/; +our %DISK; # ( 'github_root' => [ 'ide-hd0',21 ] + +sub verify_disks { + my ($obj)=@_; + for my $dev (@$obj) { + my $inserted=$dev->{'inserted'} or next; + my $device=$dev->{'device'}; + $device =~ /^ide\d+-cd\d+$/ and next; + my $image=$inserted->{'image'}; + my $filename=$image->{'filename'}; + #printf "verify %-20s %s\n",$device,$filename; + my ($basename,$nnn)=$filename=~$PATTERN or die "$filename: wrong pattern (expected: NAME.\\d\\d\\d.disk)\n"; + -s "$basename.disk" or die "$basename.disk: symlink is missing\n"; + readlink("$basename.disk") eq $filename or die "symlink $basename.disk not pointing to $filename\n"; + + my %FILES; + opendir my $d,"."; + while (readdir $d) { + my ($cmp_basename,$cmp_nnn)= $_=~$PATTERN or next; + $cmp_basename eq $basename or next; + $FILES{$_}=1; + } + delete $FILES{$filename}; + + + my $depth=0; + for (my $backing_image=$image->{'backing-image'};$backing_image;$backing_image=$backing_image->{'backing-image'}) { + $depth++; + my $b_filename=$backing_image->{'filename'}; + #printf " %-20s %s\n",'',$b_filename; + my ($b_basename,$b_nnn)=$b_filename=~$PATTERN or die "backing file $filename: wrong pattern\n"; + $b_basename eq $basename or die "$b_filename: wrong basename\n"; + $b_nnn == $nnn-$depth or die "$b_filename: number should be ".($nnn-$depth)."\n"; + delete $FILES{$b_filename}; + } + if (%FILES) { + warn "unused files: ".join(' ',keys %FILES)."\n"; + } + $DISK{$basename}=[$device,$nnn]; + } +} + +sub verify_no_block_jobs_running { + my $obj=exec_cmd('query-block-jobs'); + @$obj and die "block jobs running!\n"; +} + +sub wait_block_jobs { + while (1) { + my $obj=exec_cmd('query-block-jobs'); + Data::Dumper->Dump([$obj],['O']); + @$obj or return; + for my $job (@$obj) { + my $device=$job->{'device'}; + my $offset=$job->{'offset'}; + my $len=$job->{'len'}; + printf "%-15s %d/%d (%5.1f%%)\n",$device,$offset,$len,$offset*100/$len; + } + sleep 1; + } +} + +sub snapshot { + my ($basename)=@_; + exists $DISK{$basename} or die "$basename does not exist in VM\n"; + my ($devname,$nnn)=@{$DISK{$basename}}; + my $filename=sprintf '%s.%03d.disk',$basename,$nnn+1; + + print "snapshot $devname -> $filename\n"; + exec_cmd('blockdev-snapshot-sync','device'=>$devname,'snapshot-file'=>$filename); + -s "$basename.disk" and unlink "$basename.disk"; + symlink($filename,"$basename.disk") or die "failed to symlink $basename.disk -> $filename: $!\n"; + $DISK{$basename}=[$devname,$nnn+1]; +} + +sub block_stream { + my ($basename)=@_; + exists $DISK{$basename} or die "$basename does not exist in VM\n"; + my ($devname,$nnn)=@{$DISK{$basename}}; + exec_cmd('block-stream','device'=>$devname); +} + + +$json=new JSON; +$s=new IO::Socket::UNIX(Type=>SOCK_STREAM,Peer=>'./qmp.socket') or die "qmp.socket: $!\n"; +exec_cmd('qmp_capabilities'); + + +sub USAGE { + <<"EOF"; +usage: $0 cmd + snapshot NAME # new instance NAME.xxx.disk + block-stream NAME # start stream up to top layer + ( hint: `fstrim -av` on the vm may save some space ) + wait # wait till all block jobs are finished + powerdown # send ACPI powerdown + noop # do nothing, but verify disks as always +EOF +} + +@ARGV or die USAGE; +my ($cmd,@args)=(@ARGV); + +my $block_info=exec_cmd('query-block'); +verify_disks($block_info); + +if ($cmd eq 'snapshot') { + @args==1 or die USAGE; + verify_no_block_jobs_running(); + snapshot(@args); +} elsif ($cmd eq 'block-stream') { + @args==1 or die USAGE; + verify_no_block_jobs_running(); + block_stream(@args); +} elsif ($cmd eq 'wait') { + @args==0 or die USAGE; + wait_block_jobs(); +} elsif ($cmd eq 'powerdown') { + @args==0 or die USAGE; + exec_cmd('system_powerdown'); +} elsif ($cmd eq 'noop') { + @args==0 or die USAGE; +} else { + die USAGE; +}