From 0708053cdfabf901190b4adb2fa432498c3c41bb Mon Sep 17 00:00:00 2001 From: Donald Buczek Date: Mon, 16 Jan 2023 10:47:48 +0100 Subject: [PATCH] libexec_cron: Import sbin scripts from projects Import scripts, which live in projects with user access and are called from the crontab of root, into repository and update install.sh. The scripts are installed into /usr/libexec/cron/ with a name prefixed with "prj_PROJECT_". * afk.molgen.mpg.de 33 7,9,11,13,15,17,19 * * * /usr/local/bin/cronwrap -m wwwutz /project/it/cron/cleanperm.sh --> prj_it_cleanperm.sh * chimichangas.molgen.mpg.de */5 4-23 * * * /project/FlowCytometryLab/home/sbin/fix-users.pl >/dev/null 2>&1 --> prj_FlowCytometryLab_fix-users.pl * doenertier.molgen.mpg.de 10 23 * * * /usr/local/bin/cronwrap -m buczek /project/ag_schulz_images/.sbin/purge_trash.sh --> prj_ag_schulz_images_purge_trash.sh * doenertier.molgen.mpg.de 10 23 * * * /usr/local/bin/cronwrap -m buczek /project/ag_schulz/.sbin/purge_trash.sh --> prj_ag_schulz_purge_trash.sh * furoncles.molgen.mpg.de 30 21 * * * /usr/local/bin/cronwrap -m buczek /project/meissner_external_fastq/.sbin/autoarchive.pl --> prj_meissner_external_fastq_autoarchive.pl * mercwithamouth.molgen.mpg.de 30 4 * * * /usr/local/bin/cronwrap -m buczek,giesselmann /project/minion/sbin/archiver.pl --> prj_minion_archiver.pl * plage.molgen.mpg.de 10 23 * * * /usr/local/bin/cronwrap -m buczek /project/agsgpa/.sbin/purge_trash.sh --> prj_agsgpa_purge_trash.sh * runnynose.molgen.mpg.de 15 23 * * 6 /usr/local/bin/cronwrap -m buczek,klages -v /project/solexawork/sbin/purge_solexawork_cron --> prj_solexawork_purge_solexawork_cron --> prj_solexawork_purge_solexawork * snugglebites.molgen.mpg.de 10 23 * * * /usr/local/bin/cronwrap -m buczek /project/owlmayerUnpublishedData/.sbin/purge_trash.sh --> prj_owlmayerUnpublishedData_purge_trash.sh * snugglebites.molgen.mpg.de 30 05 * * * /usr/local/bin/cronwrap -m buczek,bressin /project/owlmayerUnpublishedData/.sbin/archiver.pl --> prj_owlmayerUnpublishedData_archiver.pl * snugglebites.molgen.mpg.de 33 7,9,11,13,15,17,19 * * * /usr/local/bin/cronwrap -m wwwutz /project/communication/home/cron/cleanperm.sh --> prj_communication_cleanperm.sh * tinytina.molgen.mpg.de #10 23 * * * /usr/local/bin/cronwrap -m buczek /project/agsgpa/.sbin/purge_trash.sh --> prj_agsgpa_purge_trash.sh Notes: prj_solexawork_purge_solexawork_cron calls another script, which is also imported. prj_solexawork_purge_solexawork_cron has been modified to call the imported script from /usr/libexec/cron/. Scripts in packages and projects that we regard as secure (/project/admin , /project/mx , /package/ldap etc.) are not imported. The crontabs need to be updated to refer to installed scripts instead to those in the projects. The scripts in the projects should be deleted to avoid confusion. --- install.sh | 3 + .../prj_FlowCytometryLab_fix-users.pl | 56 ++++++++++++ .../prj_ag_schulz_images_purge_trash.sh | 13 +++ libexec_cron/prj_ag_schulz_purge_trash.sh | 8 ++ libexec_cron/prj_agsgpa_purge_trash.sh | 8 ++ libexec_cron/prj_communication_cleanperm.sh | 13 +++ libexec_cron/prj_it_cleanperm.sh | 11 +++ ...prj_meissner_external_fastq_autoarchive.pl | 43 +++++++++ libexec_cron/prj_minion_archiver.pl | 77 ++++++++++++++++ .../prj_owlmayerUnpublishedData_archiver.pl | 75 ++++++++++++++++ ...prj_owlmayerUnpublishedData_purge_trash.sh | 8 ++ .../prj_solexawork_purge_solexawork.pl | 89 +++++++++++++++++++ .../prj_solexawork_purge_solexawork_cron | 3 + 13 files changed, 407 insertions(+) create mode 100755 libexec_cron/prj_FlowCytometryLab_fix-users.pl create mode 100755 libexec_cron/prj_ag_schulz_images_purge_trash.sh create mode 100755 libexec_cron/prj_ag_schulz_purge_trash.sh create mode 100755 libexec_cron/prj_agsgpa_purge_trash.sh create mode 100755 libexec_cron/prj_communication_cleanperm.sh create mode 100755 libexec_cron/prj_it_cleanperm.sh create mode 100755 libexec_cron/prj_meissner_external_fastq_autoarchive.pl create mode 100755 libexec_cron/prj_minion_archiver.pl create mode 100755 libexec_cron/prj_owlmayerUnpublishedData_archiver.pl create mode 100755 libexec_cron/prj_owlmayerUnpublishedData_purge_trash.sh create mode 100755 libexec_cron/prj_solexawork_purge_solexawork.pl create mode 100755 libexec_cron/prj_solexawork_purge_solexawork_cron diff --git a/install.sh b/install.sh index 4a9d609..706fcd3 100755 --- a/install.sh +++ b/install.sh @@ -257,6 +257,9 @@ install_data misc_etc_files/modprobe.d/ixgbe_sfp.conf "$DESTDIR$sysconfdir/m install_exec forensics/mxvmem "$DESTDIR$usr_bindir/mxvmem" install_data misc_systemd_units/sshd.service "$DESTDIR$systemdunitdir/sshd.service" install_data misc_systemd_units/pacbio-sshd.service "$DESTDIR$systemdunitdir/pacbio-sshd.service" +for f in libexec_cron/*; do + install_exec "$f" "$DESTDIR$usr_exec_prefix/libexec/cron/$(basename "$f")" +done postinstall exit diff --git a/libexec_cron/prj_FlowCytometryLab_fix-users.pl b/libexec_cron/prj_FlowCytometryLab_fix-users.pl new file mode 100755 index 0000000..a3bd348 --- /dev/null +++ b/libexec_cron/prj_FlowCytometryLab_fix-users.pl @@ -0,0 +1,56 @@ +#! /usr/bin/perl + +use strict; + + +sub scandir { + my ($dirname)=@_; + opendir my $dir,$dirname or die "$dirname: $!\n"; + return sort grep !/^\./,readdir $dir; +} + +my $sts=0; + +my @DIRS=scandir('/project/FlowCytometryLab/'); + +for my $name (sort @DIRS) { + $name eq 'home' and next; + my $d="/project/FlowCytometryLab/$name"; + lstat $d or die "$d: $!\n"; + -d _ or next; + if (my $uid=getpwnam($name)) { + my @f1=lstat _; + unless (chdir($d)) { + warn "$d: $!\n"; + $sts=1; + next; + } + my @f2=lstat '.'; + unless ($f1[0] == $f2[0] && $f1[1]==$f2[1]) { + warn "$d: won't work on replaced directory\n"; + $sts=1; + next; + } + system 'find','.', + '-type','d', + '-not','-user',$uid, + '-ls', + '-exec','chown',$uid,'{}',';', + and $sts=1; + system 'find','.', + '-type','f', + '-links','1', + '-not','-user',$uid, + '-ls', + '-exec','chown',$uid,'{}',';', + and $sts=1; + if ( ($f2[2] & 07777) != 02770) { + system 'chmod','02770','.' and $sts=1; + } + } else { + getpwnam("O$name") and next; + warn "$name: no such user\n"; + $sts=1; + } +} +exit $sts; diff --git a/libexec_cron/prj_ag_schulz_images_purge_trash.sh b/libexec_cron/prj_ag_schulz_images_purge_trash.sh new file mode 100755 index 0000000..c054f83 --- /dev/null +++ b/libexec_cron/prj_ag_schulz_images_purge_trash.sh @@ -0,0 +1,13 @@ +#! /bin/bash +set -x +shopt -s nullglob +cd /project/ag_schulz_images/ +for D in .Trash-[0-9]*;do + find "$D" -not -type d -ctime +3 -exec rm {} \; + find "$D" -type d -exec rmdir {} \; 2>/dev/null || true +done +cd /project/ag_schulz_images_bkp/ +for D in .Trash-[0-9]*;do + find "$D" -not -type d -ctime +3 -exec rm {} \; + find "$D" -type d -exec rmdir {} \; 2>/dev/null || true +done diff --git a/libexec_cron/prj_ag_schulz_purge_trash.sh b/libexec_cron/prj_ag_schulz_purge_trash.sh new file mode 100755 index 0000000..27b0bf0 --- /dev/null +++ b/libexec_cron/prj_ag_schulz_purge_trash.sh @@ -0,0 +1,8 @@ +#! /bin/bash +set -x +shopt -s nullglob +cd /project/ag_schulz/ +for D in .Trash-[0-9]*;do + find "$D" -not -type d -ctime +3 -exec rm {} \; + find "$D" -type d -exec rmdir {} \; 2>/dev/null || true +done diff --git a/libexec_cron/prj_agsgpa_purge_trash.sh b/libexec_cron/prj_agsgpa_purge_trash.sh new file mode 100755 index 0000000..4252eba --- /dev/null +++ b/libexec_cron/prj_agsgpa_purge_trash.sh @@ -0,0 +1,8 @@ +#! /bin/bash +set -x +shopt -s nullglob +cd /project/agsgpa/ +for D in .Trash-[0-9]*;do + find "$D" -not -type d -ctime +3 -exec rm {} \; + find "$D" -type d -exec rmdir {} \; 2>/dev/null || true +done diff --git a/libexec_cron/prj_communication_cleanperm.sh b/libexec_cron/prj_communication_cleanperm.sh new file mode 100755 index 0000000..ed70596 --- /dev/null +++ b/libexec_cron/prj_communication_cleanperm.sh @@ -0,0 +1,13 @@ +#!/bin/bash +P=/project/communication +G=commugrp +[ ! -d ${P}/home ] && exit 1 + +LOG=${P}/home/cron/cleanperm.sh.log +NOTHOME="-not -regex ${P}/home/?.*" +date >>${LOG} +find ${P}/ -type d -not -perm -2771 ${NOTHOME} -exec chmod -v 2771 {} ';' >>$LOG 2>&1 +find ${P}/ -type f -perm -u+x -not -perm -g+x ${NOTHOME} -exec chmod -v ug+x,o-rw {} ';' >>$LOG 2>&1 +find ${P}/ -type f -perm -u+rw -not -perm -g+rw -not -perm /u=x ${NOTHOME} -exec chmod -v ug+rw,o-rw {} ';' >>$LOG 2>&1 +find ${P}/ -type f -perm -u+rwx -not -perm -g+rwx ${NOTHOME} -exec chmod -v ug+rwx,o-rwx {} ';' >>$LOG 2>&1 +find ${P}/ -not -group ${G} -ls -exec chgrp -v ${G} {} ';' >>$LOG 2>&1 diff --git a/libexec_cron/prj_it_cleanperm.sh b/libexec_cron/prj_it_cleanperm.sh new file mode 100755 index 0000000..f9265e2 --- /dev/null +++ b/libexec_cron/prj_it_cleanperm.sh @@ -0,0 +1,11 @@ +#!/bin/bash +[ ! -d /project/it/home ] && exit 1 +LOG=/project/it/cron/cleanperm.sh.log +NOTHOME="-not -regex /project/it/home/?.*" +date >>${LOG} +find /project/it/ -type d -not -perm -2771 ${NOTHOME} -exec chmod -v 2771 {} ';' >>$LOG 2>&1 +find /project/it/ -type f -perm -u+x -not -perm -g+x ${NOTHOME} -exec chmod -v ug+x,o-rw {} ';' >>$LOG 2>&1 +find /project/it/ -type f -perm -u+rw -not -perm -g+rw -not -perm /u=x ${NOTHOME} -exec chmod -v ug+rw,o-rw {} ';' >>$LOG 2>&1 +find /project/it/ -type f -perm -u+rwx -not -perm -g+rwx ${NOTHOME} -exec chmod -v ug+rwx,o-rwx {} ';' >>$LOG 2>&1 +find /project/it/EDV-Bestellungen/ -type f -not -perm -o+r -exec chmod -v o+r {} ';' >>$LOG 2>&1 +find /project/it/ -not -group edv -ls -exec chgrp -v edv {} ';' >>$LOG 2>&1 diff --git a/libexec_cron/prj_meissner_external_fastq_autoarchive.pl b/libexec_cron/prj_meissner_external_fastq_autoarchive.pl new file mode 100755 index 0000000..17f9273 --- /dev/null +++ b/libexec_cron/prj_meissner_external_fastq_autoarchive.pl @@ -0,0 +1,43 @@ +#! /usr/bin/perl +use strict; +use warnings; +use File::Find; + +our $PROJECT_PATH = '/project/meissner_external_fastq/'; +our $PROJECT_RAW_PATH = '/amd/furoncles/M/MG000/project/meissner_external_fastq'; + +chdir $PROJECT_RAW_PATH or die "$PROJECT_RAW_PATH: $!\n"; + +my @f1 = lstat $PROJECT_PATH or die "$PROJECT_PATH: $!\n"; +my @f2 = lstat $PROJECT_RAW_PATH or die "$PROJECT_RAW_PATH: $!\n"; + +$f1[0] == $f2[0] && $f1[1] == $f2[1] or die "paths $PROJECT_PATH doeesn't match $PROJECT_RAW_PATH. Has the project been moved?\”"; + +my $twoweeksago = time - 2*7*24*60*60; +print "twoweeksago:",scalar(localtime($twoweeksago)),"\n"; + +my $done = 0; +for my $folder () { + -d $folder or next; + -e "$folder/$folder.is-archived-on-tape.log" and next; + my $mtime = 0; + find ( + sub { + my @f = lstat $_ or die "$File::Find::name: $!\n"; + $f[9] > $mtime and $mtime = $f[9]; + $f[10] > $mtime and $mtime = $f[10]; + }, + $folder + ); + print "mtime of folder $folder is ",scalar(localtime($mtime)),"\n"; + if ($mtime < $twoweeksago) { + system "/project/admin/tools/archiver.pl", $folder and exit 1; + rename "$folder.is-archived-on-tape.log", "$folder/$folder.is-archived-on-tape.log" + or die "rename $folder.is-archived-on-tape.log to $folder/$folder.is-archived-on-tape.log: $!\n"; + $done++; + if ($done >= 2) { + print "limit of max 2 archives per day reached\n"; + exit; + } + } +} diff --git a/libexec_cron/prj_minion_archiver.pl b/libexec_cron/prj_minion_archiver.pl new file mode 100755 index 0000000..b0fb792 --- /dev/null +++ b/libexec_cron/prj_minion_archiver.pl @@ -0,0 +1,77 @@ +#! /usr/bin/perl +use strict; +use warnings; + +sub lock_or_exit { + open our $LOCKFD,'>>','/run/lock/minion-archive' or die "/run/lock/minion-archive: $!\n"; + unless (flock $LOCKFD,6) { # LOCK_EX+LOCK_NB + $!!=11 and die "$!\n"; + warn "already running\n"; + exit; + } +} + +sub scandir { + my ($dirname)=@_; + opendir my $dir,$dirname or die "$dirname: $!\n"; + return sort grep !/^\./,readdir $dir; +} + +sub find_dh_pattern { + my ($dh,$pattern)=@_; + while(my $fn=readdir $dh) { + return $1 if $fn=~$pattern; + } + return undef; +} + +sub warn0 { + warn @_; + return 0; +} + +sub archive { + my ($id)=@_; + my $minionpath="/project/minion/raw/$id"; + lstat $minionpath or return warn0("$minionpath: $!\n"); + -l _ or return warn0("$minionpath: not a symbolic link\n"); + my $target=readlink($minionpath) or die "$minionpath: $!\n"; + $target=~s#/+$##; + my $miniondatapath="/project/miniondata/raw/$id"; + $target eq $miniondatapath or return warn0("$minionpath: invalid target $target\n"); + my ($dev1,$ino1)=lstat $target or return warn0("$target: $!\n"); + -d _ or return warn0("$target: not a directory\n"); + my $fullpath="/amd/mercwithamouth/M/M8026/project/miniondata/raw/$id"; + my ($dev2,$ino2)=lstat $fullpath or die "$fullpath: $!\n"; + unless ($dev1==$dev2 && $ino1==$ino2) { + die "$fullpath not the same as $target\n"; + } + -e "$fullpath.is-archived-on-tape.log" and die "$fullpath.is-archived-on-tape.log already exists\n"; + warn "archive $fullpath\n"; + system '/project/admin/tools/archiver.pl',$fullpath and exit 1; + system 'mv',"$fullpath.is-archived-on-tape.log","/project/minion/archive/$id.ARCHIVE_COMPLETED" and exit 1; + unlink("/project/minion/archive/$id.ARCHIVE_REQUESTED") or die "/project/minion/archive/$id.ARCHIVE_REQUESTED: $!\n"; + return 1; +} + +lock_or_exit(); +chdir '/project/minion/archive' or die "/project/minion/archive: $!\n"; + +while (1) { + my $progress; + my $errors; + + opendir my $dh,'.'; + while (my $id=find_dh_pattern($dh,qr/^(.+)\.ARCHIVE_REQUESTED$/)) { + redo unless -e "$id.ARCHIVE_REQUESTED"; # Operator changed his mind? + if (archive($id)) { + $progress++; + } else { + $errors++; + } + } + unless ($progress) { + exit($errors ? 1 : 0); + } + sleep 5; +} diff --git a/libexec_cron/prj_owlmayerUnpublishedData_archiver.pl b/libexec_cron/prj_owlmayerUnpublishedData_archiver.pl new file mode 100755 index 0000000..b3f7703 --- /dev/null +++ b/libexec_cron/prj_owlmayerUnpublishedData_archiver.pl @@ -0,0 +1,75 @@ +#! /usr/bin/perl +use strict; +use warnings; + +sub lock_or_exit { + open our $LOCKFD,'>>','/run/lock/mayer-minion-archive' or die "/run/lock/mayer-minion-archive: $!\n"; + unless (flock $LOCKFD,6) { # LOCK_EX+LOCK_NB + $!!=11 and die "$!\n"; + warn "already running\n"; + exit; + } +} + +sub scandir { + my ($dirname)=@_; + opendir my $dir,$dirname or die "$dirname: $!\n"; + return sort grep !/^\./,readdir $dir; +} + +sub find_dh_pattern { + my ($dh,$pattern)=@_; + while(my $fn=readdir $dh) { + return $1 if $fn=~$pattern; + } + return undef; +} + +sub warn0 { + warn @_; + return 0; +} + +sub archive { + my ($id)=@_; + my $miniondatapath="/project/owlmayerMinion/raw/$id"; + lstat $miniondatapath or return warn0("$miniondatapath: $!\n"); + + my ($dev1,$ino1)=lstat $miniondatapath or return warn0("$miniondatapath: $!\n"); + -d _ or return warn0("$miniondatapath: not a directory\n"); + my $fullpath="/amd/snugglebites/M/M8033/project/owlmayerMinion/raw/$id"; + my ($dev2,$ino2)=lstat $fullpath or die "$fullpath: $!\n"; + unless ($dev1==$dev2 && $ino1==$ino2) { + die "$fullpath not the same as $miniondatapath\n"; + } + -e "$fullpath.is-archived-on-tape.log" and die "$fullpath.is-archived-on-tape.log already exists\n"; + -e "$id.is-archived-on-tape.log" and die "$id.is-archived-on-tape.log already exists\n"; + warn "archive $fullpath\n"; + system '/project/admin/tools/archiver.pl',$fullpath and exit 1; + system 'mv',"$fullpath.is-archived-on-tape.log","./" and exit 1; + unlink("$id.ARCHIVE_REQUESTED") or die "$id.ARCHIVE_REQUESTED: $!\n"; + system 'chmod','-R','a-w',$fullpath; + return 1; +} + +lock_or_exit(); +chdir '/project/owlmayerUnpublishedData/archive' or die "/project/owlmayerUnpublishedData/archive: $!\n"; + +while (1) { + my $progress; + my $errors; + + opendir my $dh,'.'; + while (my $id=find_dh_pattern($dh,qr/^(.+)\.ARCHIVE_REQUESTED$/)) { + redo unless -e "$id.ARCHIVE_REQUESTED"; # Operator changed his mind? + if (archive($id)) { + $progress++; + } else { + $errors++; + } + } + unless ($progress) { + exit($errors ? 1 : 0); + } + sleep 5; +} diff --git a/libexec_cron/prj_owlmayerUnpublishedData_purge_trash.sh b/libexec_cron/prj_owlmayerUnpublishedData_purge_trash.sh new file mode 100755 index 0000000..a6533eb --- /dev/null +++ b/libexec_cron/prj_owlmayerUnpublishedData_purge_trash.sh @@ -0,0 +1,8 @@ +#! /bin/bash +set -x +shopt -s nullglob +cd /project/owlmayerUnpublishedData/ +for D in .Trash-[0-9]*;do + find "$D" -not -type d -ctime +3 -exec rm {} \; + find "$D" -type d -exec rmdir {} \; 2>/dev/null || true +done diff --git a/libexec_cron/prj_solexawork_purge_solexawork.pl b/libexec_cron/prj_solexawork_purge_solexawork.pl new file mode 100755 index 0000000..697ddc3 --- /dev/null +++ b/libexec_cron/prj_solexawork_purge_solexawork.pl @@ -0,0 +1,89 @@ +#! /usr/local/system/perl/bin/perl +use strict; +use warnings; +use Time::Local; +use Getopt::Long; + +sub USAGE { + "usage: $0 --test|--purge\n"; +} + +our ($opt_test, $opt_purge); + +GetOptions( + 'test' => \$opt_test, + 'purge' => \$opt_purge, +) or die USAGE; +@ARGV and die USAGE; +$opt_test xor $opt_purge or die USAGE; +@ARGV and die USAGE; + +our %FOLDER; # ( mtime => "/project/seqwork23/pipelined/1234_xxx", ... ) + +for (my $n=1;$n<40;$n++) { + my $base = sprintf "/project/solexawork%02d", $n; + -d $base or next; + + opendir my $dir, $base or die "$base : $!\n"; + for (readdir $dir) { + next if /^\.\.?$/; + next if $_ eq 'pipelined'; + next if $_ eq '.nsr'; + next if $_ eq '.snapshot'; + next if /^pipelined\.(findls|inodes|mtime|usage)/; + warn "ignore junk: $base/$_\n"; + } + $base = "$base/pipelined"; + unless (lstat $base) { + warn "missing: $base\n"; + next; + } + unless (-d _) { + warn "$base: not a directory\n"; + next; + } + opendir $dir, $base or die "$base: $!\n"; + for (readdir $dir) { + next if /^\.\.?$/; + next if /\.is-archived-on-tape\.log(\.1)?$/; + next if /\.cache$/; + next if $_ eq 'FULL'; + next if $_ eq '.PMIRROR_STATUS'; + next if $_ eq 'usage'; + next if /\.runs$/; + next if /\.pl$/; + unless (/^\d+_/) { + warn "ignore junk: $base/$_\n"; + next; + } + my $folder = "$base/$_"; + lstat $folder; + unless (-d _) { + warn "$folder: not a directory\n"; + next; + } + my @f=lstat _; + my $mtime = $f[9]; + while (exists $FOLDER{$mtime}) { + $mtime++; + } + $FOLDER{$mtime} = $folder; + } +} + +my @f=localtime; +$f[5]-=10; +my $tenyearsago=Time::Local::timelocal(@f); + +for my $mtime (sort {$a <=> $b} keys %FOLDER) { + my $folder = $FOLDER{$mtime}; + if ($mtime < $tenyearsago) { + my $t=localtime($mtime); + print "rm -rf $folder # mtime $t\n"; + if ($opt_purge) { + system 'rm', '-rf', $folder and exit 1; + } + } else { + last; + } +} diff --git a/libexec_cron/prj_solexawork_purge_solexawork_cron b/libexec_cron/prj_solexawork_purge_solexawork_cron new file mode 100755 index 0000000..a8fc435 --- /dev/null +++ b/libexec_cron/prj_solexawork_purge_solexawork_cron @@ -0,0 +1,3 @@ +#! /bin/bash +echo "===== $(date) =====" >> /project/solexawork/sbin/purge_solexawork.log +/usr/libexec/cron/prj_solexawork_purge_solexawork.pl --purge 2>&1 | tee -a /project/solexawork/sbin/purge_solexawork.log