diff --git a/bin/pbackup b/bin/pbackup
index f5d9e23..fae95d2 100755
--- a/bin/pbackup
+++ b/bin/pbackup
@@ -53,36 +53,36 @@ usage: $0 command [args]
EOF
}
-our $hostname=`/bin/hostname -s`;$? and exit 1;chomp($hostname);
+our $hostname = `/bin/hostname -s` ; $? and exit 1 ; chomp($hostname);
-our $ROOT="/project/pbackup_$hostname";
+our $ROOT = "/project/pbackup_$hostname";
-$ENV{TMPDIR}="$ROOT/db";
+$ENV{TMPDIR} = "$ROOT/db";
################################### tools self contained
sub first_line_of_file {
our ($filename)=@_;
- my $fh=new IO::File ($filename,'<') or die "$filename: $!\n";
- my $line=$fh->getline();
+ my $fh = new IO::File ($filename, '<') or die "$filename: $!\n";
+ my $line = $fh->getline();
chomp($line);
return $line;
}
sub df {
- my ($path)=@_;
- my $pid=open P,'-|';
+ my ($path) = @_;
+ my $pid = open P, '-|';
defined $pid or die "$0: $!\n";
unless ($pid) {
- exec 'df','-k',$path;
+ exec 'df', '-k', $path;
die "$0: $!\n";
}
my $l;
- $l=;
- $l=
;
+ $l =
;
+ $l =
;
chomp $l;
- my ($device,$blocks,$used,$avail,$perc,$ppath)=split " ",$l;
- 1 while ($l=readline(*P));
+ my ($device, $blocks, $used, $avail, $perc, $ppath) = split " ", $l;
+ 1 while ($l = readline(*P));
close P;
return $avail;
}
@@ -90,34 +90,34 @@ sub df {
our %FS_FREE_CACHE;
sub df_cached { # w/o arguments: clear cache
unless (@_) {
- %FS_FREE_CACHE=();
+ %FS_FREE_CACHE = ();
return;
}
- my ($fs)=@_;
- $FS_FREE_CACHE{$fs}=df($fs) unless exists $FS_FREE_CACHE{$fs};
+ my ($fs) = @_;
+ $FS_FREE_CACHE{$fs} = df($fs) unless exists $FS_FREE_CACHE{$fs};
return $FS_FREE_CACHE{$fs};
}
sub du_bytes {
- my ($dir)=@_;
- my $pid=open my $p,'-|';
+ my ($dir) = @_;
+ my $pid = open my $p,'-|';
defined $pid or die "$!\n";
unless ($pid) {
chdir $dir or die "$dir: $!\n";
- exec 'du','--block-size=1','-s','.';
+ exec 'du', '--block-size=1', '-s', '.';
die "$!\n";
}
- my $out=<$p>;
+ my $out = <$p>;
close $p;
$? and die "du failed\n";
- my ($bytes)=split " ",$out;
+ my ($bytes) = split " ", $out;
return $bytes;
}
sub scan_rsync_log {
- my ($logfile)=@_;
- open L,'<',$logfile or die "$logfile: $!\n";
- seek L,-5000,2; # typical entry is 1000 bytes
+ my ($logfile) = @_;
+ open L, '<', $logfile or die "$logfile: $!\n";
+ seek L, -5000, 2; # typical entry is 1000 bytes
my @f;
while () {
@@ -129,17 +129,17 @@ sub scan_rsync_log {
# Matched data is how much data the receiver got locally when recreating the updated files.
if (/^Number of files: ([\d,]+)/) {
- @f=($1);
+ @f = ($1);
} elsif (/^Number of(?: regular)? files transferred: ([\d,]+)/) {
- $f[1]=$1;
+ $f[1] = $1;
} elsif (/^Total file size: ([\d,]+)/) {
- $f[2]=$1;
+ $f[2] = $1;
} elsif (/^Total transferred file size: ([\d,]+)/) {
- $f[3]=$1;
+ $f[3] = $1;
} elsif (/^Literal data: ([\d,]+)/) {
- $f[4]=$1;
+ $f[4] = $1;
} elsif (/^Matched data: ([\d,]+)/) {
- $f[5]=$1;
+ $f[5] = $1;
}
}
s/,//g for @f;
@@ -163,16 +163,16 @@ sub get_db_filename {
sub db_open {
my $db_filename = get_db_filename();;
- $dbh=DBI->connect("dbi:SQLite:dbname=$db_filename","","",{
- AutoCommit=>1,
- PrintError=>0,
- RaiseError=>1,
- PrintWarn=>1,
+ $dbh = DBI->connect("dbi:SQLite:dbname=$db_filename", "", "", {
+ AutoCommit => 1,
+ PrintError => 0,
+ RaiseError => 1,
+ PrintWarn => 1,
sqlite_use_immediate_transaction => 1,
});
$dbh or die "$DBI::errstr\n";
#print "TMO: ",$dbh->sqlite_busy_timeout(),"\n"; # default 30000 msec
- $dbh->sqlite_busy_timeout(24*60*60*1000);
+ $dbh->sqlite_busy_timeout(24 * 60 * 60 * 1000);
}
##################################
@@ -183,53 +183,48 @@ use constant {
};
sub lck_lock {
- my ($lck_name,$lck_mode)=@_;
- $dbh->do('INSERT INTO lock(lck_name,lck_mode,lck_pid) VALUES(?,?,?)',undef,$lck_name,$lck_mode,$$);
+ my ($lck_name, $lck_mode)=@_;
+ $dbh->do('INSERT INTO lock(lck_name, lck_mode, lck_pid) VALUES (?, ?, ?)', undef, $lck_name, $lck_mode, $$);
}
sub lck_can_lock {
- my ($lck_name,$want_mode)=@_;
- my ($lck_mode)=$dbh->selectrow_array('SELECT MAX(lck_mode) FROM lock WHERE lck_name=?',undef,$lck_name);
+ my ($lck_name, $want_mode) = @_;
+ my ($lck_mode) = $dbh->selectrow_array('SELECT MAX(lck_mode) FROM lock WHERE lck_name = ?', undef, $lck_name);
if (!defined $lck_mode) {
return 1;
- } elsif ($lck_mode==LCK_SH) {
- return $want_mode==LCK_SH ? 1 : 0;
+ } elsif ($lck_mode == LCK_SH) {
+ return $want_mode == LCK_SH ? 1 : 0;
} else {
return 0;
}
}
sub lck_unlock {
- my ($lck_name,$lck_mode)=@_;
- $dbh->do('DELETE FROM lock WHERE lck_name=? AND lck_mode=? AND lck_pid=?',undef,$lck_name,$lck_mode,$$);
+ my ($lck_name, $lck_mode) = @_;
+ $dbh->do('DELETE FROM lock WHERE lck_name = ? AND lck_mode = ? AND lck_pid = ?', undef, $lck_name, $lck_mode, $$);
}
sub lck_get_pid {
- my ($lck_name)=@_;
- my ($lck_pid)=$dbh->selectrow_array('SELECT lck_pid FROM lock WHERE lck_name=?',undef,$lck_name);
+ my ($lck_name) = @_;
+ my ($lck_pid) = $dbh->selectrow_array('SELECT lck_pid FROM lock WHERE lck_name = ?', undef, $lck_name);
return $lck_pid;
}
##################################
-our $DATADIR="$ROOT/data";
-our $DAYS=120;
-#our $DAYS=110;
+our $DATADIR = "$ROOT/data";
+our $DAYS = 120;
sub datadirs {
- opendir D,$DATADIR or die "$DATADIR: $!\n";
- my @r=map "$DATADIR/$_",sort grep !/^\./,readdir D;
+ opendir D, $DATADIR or die "$DATADIR: $!\n";
+ my @r = map "$DATADIR/$_", sort grep !/^\./, readdir D;
close D;
return @r;
}
sub datadirs_for_balance {
- my %datadirs=map {$_=>1} datadirs();
-
-# delete $datadirs{"$DATADIR/C3019"}; # we want this empty
-# delete $datadirs{"$DATADIR/C3028"}; # keep avail vor C3019 data
-
+ my %datadirs = map {$_ => 1} datadirs();
return keys %datadirs;
}
@@ -238,41 +233,41 @@ sub free_datadir {
# datadir to write to: the one with the most free space
# uses cached df
- my %df=map {df_cached($_) => $_} datadirs_for_balance();
+ my %df = map {df_cached($_) => $_} datadirs_for_balance();
%df or die "no data directories available\n";
- return $df{(sort {$b<=>$a} keys %df)[0]};
+ return $df{(sort {$b <=> $a} keys %df)[0]};
}
sub logfile {
- my ($job)=@_;
+ my ($job) = @_;
return "$ROOT/log/$job.log";
}
sub openlog {
- my ($job)=@_;
- my $logfile=logfile($job);
- open LOG,'>>',$logfile or die "$logfile: $!\n";
+ my ($job) = @_;
+ my $logfile = logfile($job);
+ open LOG, '>>', $logfile or die "$logfile: $!\n";
}
sub fstag {
- my ($path)=@_;
- $path=~m"/amd/([^/]+)/(\d+)/(home|project|package|src)/" and return "$1_$2"; # /amd/pille/1/home/abt_lh/bukowiec -> pille_1
- $path=~m"/amd/[^/]+/[XM]/([XM]\d\d\d\d)/(home|project|package|src)/" and return "$1"; # /amd/pupsi/X/X0059/home/abt_owl/stelzl -> X0059
- $path=~m"/amd/[^/]+/C/(C\d\d\d\d)/(confidential)/" and return "$1"; # /amd/taphophobie/C/C3024/confidential/clldata/data -> 'C3024'
+ my ($path) = @_;
+ $path =~ m"/amd/([^/]+)/(\d+)/(home|project|package|src)/" and return "$1_$2"; # /amd/pille/1/home/abt_lh/bukowiec -> pille_1
+ $path =~ m"/amd/[^/]+/[XM]/([XM]\d\d\d\d)/(home|project|package|src)/" and return "$1"; # /amd/pupsi/X/X0059/home/abt_owl/stelzl -> X0059
+ $path =~ m"/amd/[^/]+/C/(C\d\d\d\d)/(confidential)/" and return "$1"; # /amd/taphophobie/C/C3024/confidential/clldata/data -> 'C3024'
return '';
}
sub timetag { # $time -> '201210051233'
- my ($time)=@_;
- my @f=localtime($time);
- return sprintf('%04d%02d%02d%02d%02d',$f[5]+1900,$f[4]+1,$f[3],$f[2],$f[1]);
+ my ($time) = @_;
+ my @f = localtime($time);
+ return sprintf('%04d%02d%02d%02d%02d', $f[5]+1900, $f[4]+1, $f[3], $f[2], $f[1]);
}
sub tag2time {
- my ($tag)=@_;
- my ($year,$mon,$mday,$hour,$min)=$tag=~/^(\d\d\d\d)(\d\d)(\d\d)(\d\d)(\d\d)$/ or return undef;
- return timelocal(0,$min,$hour,$mday,$mon-1,$year-1900);
+ my ($tag) = @_;
+ my ($year, $mon, $mday, $hour, $min) = $tag =~ /^(\d\d\d\d)(\d\d)(\d\d)(\d\d)(\d\d)$/ or return undef;
+ return timelocal(0, $min, $hour, $mday, $mon-1, $year-1900);
}
#################################### unique pid
@@ -280,44 +275,43 @@ sub tag2time {
our $BOOT_ID; # system incarnation - lazy init
sub upid_incarnation {
- my ($pid)=@_;
+ my ($pid) = @_;
my $stat;
- defined $BOOT_ID or $BOOT_ID=first_line_of_file('/proc/sys/kernel/random/boot_id');
+ defined $BOOT_ID or $BOOT_ID = first_line_of_file('/proc/sys/kernel/random/boot_id');
-e "/proc/$pid/stat" or return undef;
eval {
- $stat=first_line_of_file("/proc/$pid/stat");
+ $stat = first_line_of_file("/proc/$pid/stat");
};
if ($@) {
- $@=~ /No such file or directory/ and return undef;
+ $@ =~ /No such file or directory/ and return undef;
die $@;
}
- my $stime=(split (" ",$stat))[21];
- return $BOOT_ID.'.'.$stime;
+ my $stime = (split (" ", $stat))[21];
+ return $BOOT_ID . '.' . $stime;
}
sub upid_register_self {
- $dbh->do('INSERT OR REPLACE INTO upid(upid_pid,upid_incarnation) VALUES (?,?)',undef,$$,upid_incarnation($$));
+ $dbh->do('INSERT OR REPLACE INTO upid(upid_pid, upid_incarnation) VALUES (?, ?)', undef, $$, upid_incarnation($$));
}
sub upid_deregister_self {
- $dbh->do('DELETE FROM upid WHERE upid_pid=?',undef,$$);
+ $dbh->do('DELETE FROM upid WHERE upid_pid = ?', undef, $$);
}
sub upid_purge {
- my $t=$dbh->selectall_arrayref('SELECT upid_pid,upid_incarnation FROM upid');
+ my $t = $dbh->selectall_arrayref('SELECT upid_pid, upid_incarnation FROM upid');
for my $r (@$t) {
- my ($upid_pid,$upid_incarnation)=@$r;
- my $incarnation=upid_incarnation($upid_pid);
+ my ($upid_pid, $upid_incarnation) = @$r;
+ my $incarnation = upid_incarnation($upid_pid);
unless (defined $incarnation && $incarnation eq $upid_incarnation) {
- $dbh->do('DELETE FROM upid WHERE upid_pid=? AND upid_incarnation=?',undef,$upid_pid,$upid_incarnation);
- #print "upid purged zombie : $upid_pid $upid_incarnation\n";
+ $dbh->do('DELETE FROM upid WHERE upid_pid = ? AND upid_incarnation = ?', undef, $upid_pid, $upid_incarnation);
}
}
}
sub upid_doing {
- my ($text)=@_;
- $dbh->do('UPDATE upid SET upid_text=?,upid_since=? WHERE upid_pid=? AND upid_text!=?',undef,$text,time,$$,$text);
+ my ($text) = @_;
+ $dbh->do('UPDATE upid SET upid_text = ?, upid_since = ? WHERE upid_pid = ? AND upid_text != ?', undef, $text, time, $$, $text);
}
sub upid_clear {
@@ -325,12 +319,12 @@ sub upid_clear {
}
sub upid_kill {
- my ($pid)=@_;
- $dbh->do('UPDATE upid SET upid_shutdown_pending=1 WHERE upid_pid=?',undef,$pid);
+ my ($pid) = @_;
+ $dbh->do('UPDATE upid SET upid_shutdown_pending = 1 WHERE upid_pid = ?', undef, $pid);
}
sub upid_shutdown_pending {
- my ($upid_shutdown_pending) = $dbh->selectrow_array('SELECT upid_shutdown_pending FROM upid WHERE upid_pid=?',undef,$$);
+ my ($upid_shutdown_pending) = $dbh->selectrow_array('SELECT upid_shutdown_pending FROM upid WHERE upid_pid = ?', undef, $$);
return $upid_shutdown_pending;
}
@@ -338,16 +332,16 @@ sub upid_shutdown_pending {
our %BACKUP_SERVER; # ( "fileserver-or-jobnam" => {backupserver=>1, ...} , ... )
-sub read_server() {
- %BACKUP_SERVER=();
- open IN,'<',"$ROOT/etc/server.dat" or die "$ROOT/etc/server.dat: $!\n";
+sub read_server {
+ %BACKUP_SERVER = ();
+ open IN, '<', "$ROOT/etc/server.dat" or die "$ROOT/etc/server.dat: $!\n";
while () {
chomp;
s/#.*//;
s/\s+$//;
/\S/ or next;
- my ($fileserver_or_jobname,$backupserverlist)=split;
- my @backupserverlist=split ',',$backupserverlist;
+ my ($fileserver_or_jobname, $backupserverlist) = split;
+ my @backupserverlist = split ',', $backupserverlist;
for my $backupserver (@backupserverlist) {
$backupserver eq 'null'
or $backupserver eq 'void'
@@ -355,7 +349,7 @@ sub read_server() {
or $backupserver eq 'gone'
or $backupserver eq 'done'
or warn "$ROOT/etc/server.dat: line $. backup server $backupserver unknown\n";
- $BACKUP_SERVER{$fileserver_or_jobname}{$backupserver}=1;
+ $BACKUP_SERVER{$fileserver_or_jobname}{$backupserver} = 1;
}
}
close IN;
@@ -370,77 +364,70 @@ sub i_want_this_fileserver {
}
sub add_job {
- my ($job_name,$path)=@_; # 'home_kuss','pille:/amd/pille/1/home/abt_mdl/kuss'
+ my ($job_name, $path) = @_; # 'home_kuss','pille:/amd/pille/1/home/abt_mdl/kuss'
my $server;
- ($server,$path)=$path=~/^([^:]+):(\S+)$/;
+ ($server,$path) = $path =~ /^([^:]+):(\S+)$/;
- i_want_this_fileserver($server,$job_name) or return;
+ i_want_this_fileserver($server, $job_name) or return;
- my ($path_id) = $dbh->selectrow_array('SELECT path_id FROM path WHERE path_server=? AND path_path=?',undef,$server,$path);
+ my ($path_id) = $dbh->selectrow_array('SELECT path_id FROM path WHERE path_server = ? AND path_path = ?', undef, $server, $path);
unless (defined $path_id) {
- $dbh->do('INSERT INTO path(path_server,path_path) VALUES (?,?)',undef,$server,$path);
- ($path_id)=$dbh->selectrow_array('SELECT last_insert_rowid()');
- # print "added path $server:$path\n";
+ $dbh->do('INSERT INTO path(path_server, path_path) VALUES (?, ?)', undef, $server, $path);
+ ($path_id) = $dbh->selectrow_array('SELECT last_insert_rowid()');
}
- my ($job_id,$job_enabled) = $dbh->selectrow_array('SELECT job_id,job_enabled FROM job WHERE job_name=? AND job_path_id=?',undef,$job_name,$path_id);
+ my ($job_id, $job_enabled) = $dbh->selectrow_array('SELECT job_id, job_enabled FROM job WHERE job_name = ? AND job_path_id = ?', undef, $job_name, $path_id);
unless (defined $job_id) {
- $dbh->do('INSERT INTO job(job_name,job_path_id) VALUES (?,?)',undef,$job_name,$path_id);
- ($job_id)=$dbh->selectrow_array('SELECT last_insert_rowid()');
- # print "added job $job_name\n";
+ $dbh->do('INSERT INTO job(job_name, job_path_id) VALUES (?, ?)', undef, $job_name, $path_id);
+ ($job_id) = $dbh->selectrow_array('SELECT last_insert_rowid()');
} else {
- $job_enabled or $dbh->do('UPDATE job SET job_enabled=1 WHERE job_id=?',undef,$job_id);
+ $job_enabled or $dbh->do('UPDATE job SET job_enabled = 1 WHERE job_id = ?', undef, $job_id);
}
- # print "job_id $job_id path_id $path_id\n";
- $dbh->do('INSERT INTO vrfy (job_id) VALUES (?)',undef,$job_id);
+ $dbh->do('INSERT INTO vrfy (job_id) VALUES (?)', undef, $job_id);
}
sub read_amd_map
{
- my ($fn,$subkey)=@_;
+ my ($fn, $subkey) = @_;
my %out;
- open FN,'<',$fn or die "$fn: $!\n";
- my $data=join('',);
+ open FN, '<', $fn or die "$fn: $!\n";
+ my $data = join('', );
close FN;
#$data=~s/\\$\s*//mg; # should work, but triggers " Use of uninitialized value" sometimes
$data=~s/\\\s*//mg;
$data=~s/#.+$//mg;
- for my $line (split /\n/,$data)
- {
+ for my $line (split /\n/,$data) {
- my $debug=0;
+ my $debug = 0;
# $line=~/1000g/ and $debug=1;
$debug and print "L: $line\n";
- my ($key,$value)=$line=~/(\S+)\s+(.+)/ or next;
+ my ($key, $value) = $line =~ /(\S+)\s+(.+)/ or next;
next if $key eq '/defaults';
$debug and print " K: $key\n";
my %path; # make paths unqiue
# ignore quoted strings ( assume all white space unquoted )
- my @location=split /\s/,$value;
- for my $loc (@location)
- {
+ my @location = split /\s/,$value;
+ for my $loc (@location) {
$debug and print " L: $loc\n";
- $loc=~s/^-//; # default values for subsequent locations
- for my $sel_or_opt(split /;/,$loc)
- {
+ $loc =~ s/^-//; # default values for subsequent locations
+ for my $sel_or_opt(split /;/, $loc) {
$debug and print " SO: $sel_or_opt\n";
- if ($sel_or_opt =~ m"fs:=/amd/([^/\$]+)/(.+)")
- {
- my $path=$subkey ? "$1:/amd/$1/$2/$key" : "$1:/amd/$1/$2";
+ if ($sel_or_opt =~ m"fs:=/amd/([^/\$]+)/(.+)") {
+ my $path = $subkey ? "$1:/amd/$1/$2/$key" : "$1:/amd/$1/$2";
$path{$path} and next;
$debug and print " A: $key => $path\n";
- $path{$path}=1;
- $out{$key}||=[];
- push @{$out{$key}},$path;
+ $path{$path} = 1;
+ $out{$key} ||= [];
+ push @{$out{$key}}, $path;
} elsif ($sel_or_opt =~ m"fs:=(/jbod/.*)") {
- my $path=$subkey ? "$1/$key" : $1;
+ my $path = $subkey ? "$1/$key" : $1;
$path{$path} and next;
$debug and print " A: $key => $path\n";
- $path{$path}=1;
- $out{$key}||=[];
+ $path{$path} = 1;
+ $out{$key} ||= [];
push @{$out{$key}},$path;
}
}
@@ -452,58 +439,56 @@ sub read_amd_map
our @PATTERN_EXCLUDES;
sub excluded {
- my ($path)=@_;
+ my ($path) = @_;
for my $re (@PATTERN_EXCLUDES) {
- $path=~/$re/ and return 1;
+ $path =~ /$re/ and return 1;
}
return 0;
}
sub read_amd_jobs {
- my ($map,$subkey,$job_prefix,$path_prefix)=@_;
- my %map=read_amd_map("/etc/amd/$map",$subkey);
+ my ($map, $subkey, $job_prefix, $path_prefix) = @_;
+ my %map = read_amd_map("/etc/amd/$map",$subkey);
for (keys %map) {
excluded("$path_prefix/$_") and next;
- my $loc=$map{$_};
- for (my $i=0;$i<@$loc;$i++) {
+ my $loc = $map{$_};
+ for (my $i = 0 ; $i < @$loc ; $i++) {
add_job("${job_prefix}_$_", $loc->[$i]); # home_buczek
}
}
}
sub add_extra_jobs {
- for my $host (split ' ',`hostconfig --list '!rpi'`) {
- my $path="$host:/";
+ for my $host (split ' ', `hostconfig --list '!rpi'`) {
+ my $path = "$host:/";
excluded($path) and next;
- add_job("sys_$host","$host:/");
+ add_job("sys_$host", "$host:/");
}
- add_job('spec_flughafen_homes','flughafenberlinbrandenburgwillybrandt:/home');
+ add_job('spec_flughafen_homes', 'flughafenberlinbrandenburgwillybrandt:/home');
}
sub read_excludes {
- @PATTERN_EXCLUDES=();
- open IN,'<',"$ROOT/etc/excludes.dat" or die "$ROOT/etc/excludes.dat: $!\n";
+ @PATTERN_EXCLUDES = ();
+ open IN, '<', "$ROOT/etc/excludes.dat" or die "$ROOT/etc/excludes.dat: $!\n";
while () {
chomp;
s/#.*//;
s/\s+$//;
/\S/ or next;
eval {
- push @PATTERN_EXCLUDES,qr/^$_$/;
+ push @PATTERN_EXCLUDES, qr/^$_$/;
};
$@ and warn "excludes line $. : $@";
}
close IN;
}
-###########################################################################
-
sub datadir {
- my ($job_name)=@_;
+ my ($job_name) = @_;
- my ($volume_path) = $dbh->selectrow_array('SELECT volume_path FROM volume,stat,job WHERE stat_volume_id=volume_id AND stat_job_id=job_id AND job_name=? ORDER BY stat_started DESC',undef,$job_name);
- return $volume_path||free_datadir();
+ my ($volume_path) = $dbh->selectrow_array('SELECT volume_path FROM volume, stat, job WHERE stat_volume_id = volume_id AND stat_job_id = job_id AND job_name = ? ORDER BY stat_started DESC', undef, $job_name);
+ return $volume_path || free_datadir();
}
sub do_a_job {
@@ -544,7 +529,7 @@ AND due <= 0
ORDER BY due
');
}
- $sth->execute(time-24*60*60, time-8*60*60);
+ $sth->execute(time - 24 * 60 * 60, time - 8 * 60 * 60);
my ($job_id, $job_name, $job_ok, $job_started, $path_server, $path_path, $due);
my $datadir;
my $volume_id;
@@ -570,7 +555,7 @@ ORDER BY due
if ($free < 2) {
next;
}
- ($volume_id) = $dbh->selectrow_array('SELECT volume_id FROM volume WHERE volume_path=?', undef, $datadir);
+ ($volume_id) = $dbh->selectrow_array('SELECT volume_id FROM volume WHERE volume_path = ?', undef, $datadir);
unless (defined $volume_id) {
$dbh->do('INSERT INTO volume(volume_path) VALUES (?)', undef, $datadir);
($volume_id) = $dbh->selectrow_array('SELECT last_insert_rowid()');
@@ -589,7 +574,7 @@ ORDER BY due
}
$sth->finish;
- $dbh->do('UPDATE job SET job_started=? WHERE job_id=?', undef, $started, $job_id);
+ $dbh->do('UPDATE job SET job_started = ? WHERE job_id = ?', undef, $started, $job_id);
upid_doing(sprintf("BACKUP %s %-20s %s:%s -> %s", $type, $job_name, $path_server, $path_path, $datadir));
$dbh->commit;
@@ -633,10 +618,10 @@ EOF
# dont go over amd...
chdir $jobdir or die "$jobdir: $!\n";
- exec 'rsync',
+ exec 'rsync',
'-aHx', # -a : --archive: -recursive --links --perms --times --group --owner --devices --specials
- # -H :-hard-links
- # -x : -one-file-system
+ # -H :-hard-links
+ # -x : -one-file-system
'--fsync',
'--delete',
'--numeric-ids',
@@ -644,7 +629,7 @@ EOF
# '--progress',
'--stats',
'--filter', ". $ROOT/etc/filter.dat",
- # '--log-file',"$ROOT/test.log",
+ # '--log-file', "$ROOT/test.log",
# '-v',
($linkfrom_stat_rowid ? ('--link-dest',"../../$linkfrom_job_name/$linkfrom_run_name") : ()), # relative to destination dir
'-e', 'ssh -oFallBackToRsh=no -oStrictHostKeyChecking=no -oBatchMode=yes -i /root/.ssh/pbackup_ed25519',
@@ -677,7 +662,7 @@ EOF
close LOG;
$dbh->begin_work;
- $dbh->do('UPDATE job SET job_ok=? WHERE job_id=?', undef, ($ok ? 1 : 0), $job_id);
+ $dbh->do('UPDATE job SET job_ok = ? WHERE job_id = ?', undef, ($ok ? 1 : 0), $job_id);
lck_unlock("SERVER.$path_server", LCK_EX);
lck_unlock("JOB.$job_name", LCK_EX);
defined $linkfrom_stat_rowid and lck_unlock("RUN.$linkfrom_stat_rowid", LCK_EX);
@@ -696,10 +681,10 @@ sub add_nextclouddata {
# add individual ncd_USER jobs for subdirectories
my ($job_id, $server, $path) = $dbh->selectrow_array(<<'EOF') or return;
-SELECT job.job_id, path_server, path_path FROM vrfy,job,path
-WHERE job_name='prj_nextclouddata'
-AND job.job_id=vrfy.job_id
-AND job_path_id=path_id
+SELECT job.job_id, path_server, path_path FROM vrfy, job,path
+WHERE job_name = 'prj_nextclouddata'
+AND job.job_id = vrfy.job_id
+AND job_path_id = path_id
EOF
$dbh->do('DELETE FROM vrfy WHERE job_id=?', undef, $job_id);
# NFS is ugly, but until we come up with something better...
@@ -722,14 +707,13 @@ sub add_jobs {
}
sub amd_scan {
-
- my ($force_now)=@_;
+ my ($force_now) = @_;
$dbh->begin_work;
if (!$force_now) {
- my ($g_last_amd_scan)=$dbh->selectrow_array('SELECT g_last_amd_scan FROM global');
- if ($g_last_amd_scan+60*60 >= time) {
+ my ($g_last_amd_scan) = $dbh->selectrow_array('SELECT g_last_amd_scan FROM global');
+ if ($g_last_amd_scan + 60 * 60 >= time) {
$dbh->commit;
return;
}
@@ -741,9 +725,9 @@ sub amd_scan {
read_server();
add_jobs();
- $dbh->do('UPDATE job SET job_enabled=0 WHERE job_id NOT IN vrfy');
+ $dbh->do('UPDATE job SET job_enabled = 0 WHERE job_id NOT IN vrfy');
$dbh->do('DROP TABLE vrfy');
- $dbh->do('UPDATE global SET g_last_amd_scan=?',undef,time);
+ $dbh->do('UPDATE global SET g_last_amd_scan = ?', undef, time);
$dbh->commit;
}
@@ -753,7 +737,7 @@ sub cmd_parse_jobs {
# override add_job()
no warnings 'redefine';
- *add_job = sub { my ($job_name, $path)=@_; printf("$job_name $path\n") };
+ *add_job = sub { my ($job_name, $path) = @_; printf("$job_name $path\n") };
# use same code as amd_scan
add_jobs();
@@ -794,22 +778,22 @@ sub purge_zombies {
sub migrate_one {
df_cached();
- my $datadir_src="$DATADIR/C3019";
+ my $datadir_src = "$DATADIR/C3019";
- my @dd=sort {df_cached($a) <=> df_cached($b)} datadirs_for_balance();
+ my @dd = sort {df_cached($a) <=> df_cached($b)} datadirs_for_balance();
@dd>=2 or return 0;
- my $datadir_dst=$dd[$#dd];
+ my $datadir_dst = $dd[$#dd];
- if (df_cached($datadir_dst)<5*1024*1024*1024) {
+ if (df_cached($datadir_dst) < 5 * 1024 * 1024 * 1024) {
return 0;
}
- my ($job_name,$weight)=$dbh->selectrow_array(<<'EOF',undef,$datadir_src);
-SELECT job_name,MAX(stat_du_bytes)+sum(stat_bytes_transferred) AS weight
-FROM job,stat,volume
-WHERE stat_job_id=job_id AND stat_volume_id=volume_id
-AND volume_path=?
+ my ($job_name, $weight) = $dbh->selectrow_array(<<'EOF', undef, $datadir_src);
+SELECT job_name, MAX(stat_du_bytes) + sum(stat_bytes_transferred) AS weight
+FROM job,stat, volume
+WHERE stat_job_id = job_id AND stat_volume_id = volume_id
+AND volume_path = ?
GROUP BY job_name
ORDER BY weight DESC
LIMIT 1
@@ -819,89 +803,78 @@ EOF
my ($to) = $datadir_dst =~ m($ROOT/data/([XCM]\d\d\d\d)$);
defined $to or die;
- return move_new($job_name,$to,'MIGRATE');
+ return move_new($job_name, $to, 'MIGRATE');
}
sub balance_one {
df_cached();
- my @dd=sort {df_cached($a) <=> df_cached($b)} datadirs_for_balance();
- #for (@dd) {
- # print "$_ : ",human(df_cached($_)),"\n";
- #}
- @dd>=2 or return 0;
+ my @dd = sort {df_cached($a) <=> df_cached($b)} datadirs_for_balance();
+ @dd >= 2 or return 0;
- my $datadir_minfree=$dd[0];
- my $datadir_maxfree=$dd[$#dd];
- #print "datadir minfree: $datadir_minfree \n";
+ my $datadir_minfree = $dd[0];
+ my $datadir_maxfree = $dd[$#dd];
- unless (df_cached($datadir_minfree)<5*1024*1024*1024) {
- # warn "no need to balance: minimum free space over 5 TB\n";
+ unless (df_cached($datadir_minfree) < 5 * 1024 * 1024 * 1024) {
return 0;
}
- #if (df_cached($datadir_maxfree)<10*1024*1024*1024) {
- if (df_cached($datadir_maxfree)<8*1024*1024*1024) {
- # warn "no dataspace over 10 TB free\n";
- # warn "no dataspace over 8 TB free\n";
+ if (df_cached($datadir_maxfree) < 8 * 1024 * 1024 * 1024) {
return 0;
}
- my ($job_name,$weight)=$dbh->selectrow_array(<<'EOF',undef,$datadir_minfree);
-SELECT job_name,MAX(stat_du_bytes)+sum(stat_bytes_transferred) AS weight
-FROM job,stat,volume
-WHERE stat_job_id=job_id AND stat_volume_id=volume_id
-AND volume_path=?
+ my ($job_name, $weight) = $dbh->selectrow_array(<<'EOF', undef, $datadir_minfree);
+SELECT job_name, MAX(stat_du_bytes) + sum(stat_bytes_transferred) AS weight
+FROM job, stat, volume
+WHERE stat_job_id = job_id AND stat_volume_id = volume_id
+AND volume_path = ?
GROUP BY job_name
ORDER BY weight ASC
LIMIT 1
EOF
defined $job_name or return 0;
- #print "move $job_name from $datadir_minfree to $datadir_maxfree weight ",human($weight),"\n";
-
my ($to) = $datadir_maxfree =~ m($ROOT/data/([XCM]\d\d\d\d)$);
defined $to or die;
- return move_new($job_name,$to,'BALANCE');
+ return move_new($job_name, $to, 'BALANCE');
}
sub run_name {
- my ($job_name,$started,$path)=@_;
- my $timetag=timetag($started);
- my $fstag=fstag($path);
+ my ($job_name, $started, $path) = @_;
+ my $timetag = timetag($started);
+ my $fstag = fstag($path);
return "$job_name:$timetag:$fstag";
}
sub parse_run_name {
- my ($run_name)=@_;
- my ($job_name,$timetag,$fstag) = $run_name=~/^([^:]+):([^:]+):([^:]*)$/ or return undef;
- return ($job_name,$timetag,$fstag);
+ my ($run_name) = @_;
+ my ($job_name, $timetag, $fstag) = $run_name=~/^([^:]+):([^:]+):([^:]*)$/ or return undef;
+ return ($job_name, $timetag, $fstag);
}
sub rsync_copy {
- my ($src_path,$dst_path,$link_path)=@_;
+ my ($src_path, $dst_path, $link_path) = @_;
- my $pid=fork;
+ my $pid = fork;
defined $pid or die "$0: $!\n";
unless ($pid) {
- my @cmd=(
+ my @cmd = (
'rsync',
'-aHx', # -a : --archive: -recursive --links --perms --times --group --owner --devices --specials
- # -H :-hard-links
- # -x : -one-file-system
+ # -H :-hard-links
+ # -x : -one-file-system
'--fsync',
'--delete',
'--numeric-ids',
- ($link_path?('--link-dest',$link_path):()),
- $src_path,$dst_path
+ ($link_path ? ('--link-dest', $link_path) : ()),
+ $src_path, $dst_path
);
- # warn "DEBUG: ",join(' ',@cmd),"\n";
exec @cmd;
die "$0: $!\n";
}
- waitpid($pid,0);
+ waitpid($pid, 0);
$? and exit 1;
}
@@ -925,17 +898,17 @@ EOF
}
sub cmd_disable {
- @ARGV==1 or die USAGE;
- my ($job_name)=@ARGV;
+ @ARGV == 1 or die USAGE;
+ my ($job_name) = @ARGV;
- $dbh->do('UPDATE job SET job_enabled=0 WHERE job_name=?',undef,$job_name);
+ $dbh->do('UPDATE job SET job_enabled = 0 WHERE job_name = ?', undef, $job_name);
}
sub cmd_test {
- my $files =0;
+ my $files = 0;
my $jobs = 0;
-my $COUNT_SQL=<<'EOF';
+my $COUNT_SQL = <<'EOF';
SELECT
stat_job_id AS count_job_id,
count(stat_job_id) as count_count
@@ -943,18 +916,18 @@ FROM stat
GROUP BY stat_job_id
EOF
- my $keep=time-$DAYS*24*60*60;
- my $sth=$dbh->prepare(<<"EOF");
-SELECT job_name,stat_started,stat_files,path_server,path_path
-FROM job,stat,path,($COUNT_SQL)
-WHERE stat_job_id=job_id AND job_path_id=path_id
-AND stat_job_id=count_job_id AND count_count>1
+ my $keep = time - $DAYS * 24 * 60 * 60;
+ my $sth = $dbh->prepare(<<"EOF");
+SELECT job_name, stat_started, stat_files, path_server, path_path
+FROM job,stat, path, ($COUNT_SQL)
+WHERE stat_job_id = job_id AND job_path_id = path_id
+AND stat_job_id = count_job_id AND count_count > 1
AND stat_started < ?
ORDER BY stat_started
EOF
$sth->execute($keep);
- while (my $row=$sth->fetchrow_arrayref) {
- my ($job_name,$stat_started,$stat_files,$path_server,$path_path)=@$row;
+ while (my $row = $sth->fetchrow_arrayref) {
+ my ($job_name, $stat_started, $stat_files, $path_server, $path_path) = @$row;
printf " %-21s %s %10s %s:%s\n",
$job_name,
shortDate($stat_started),
@@ -965,36 +938,35 @@ EOF
$jobs++;
}
print "\n";
- print "(expire time : ".shortDate($keep).")\n";
+ print "(expire time : " . shortDate($keep) . ")\n";
print "To do: $files files $jobs jobs\n";
}
sub move_new {
- my ($job_name,$to, $status_tag) = @_; # 'prj_AGHucho' , [ 'C4123' ] , 'BALANCE'
+ my ($job_name, $to, $status_tag) = @_; # 'prj_AGHucho' , [ 'C4123' ] , 'BALANCE'
- $status_tag||='BALANCE ';
+ $status_tag ||= 'BALANCE ';
my $dst_volume_path;
if ($to) {
- $dst_volume_path="$DATADIR/$to";
+ $dst_volume_path = "$DATADIR/$to";
} else {
df_cached(); # clear df cache
- $dst_volume_path=free_datadir(); # $ROOT/data/C4321
- ($to)=$dst_volume_path=~m"([^/]*)$";
+ $dst_volume_path = free_datadir(); # $ROOT/data/C4321
+ ($to) = $dst_volume_path =~ m"([^/]*)$";
}
-d $dst_volume_path or die "$dst_volume_path: $!\n";
- my ($dst_volume_id) = $dbh->selectrow_array('SELECT volume_id FROM volume WHERE volume_path=?',undef,$dst_volume_path);
+ my ($dst_volume_id) = $dbh->selectrow_array('SELECT volume_id FROM volume WHERE volume_path = ?', undef, $dst_volume_path);
unless (defined $dst_volume_id) {
- $dbh->do('INSERT INTO volume(volume_path) VALUES (?)',undef,$dst_volume_path);
- ($dst_volume_id)=$dbh->selectrow_array('SELECT last_insert_rowid()');
+ $dbh->do('INSERT INTO volume(volume_path) VALUES (?)', undef, $dst_volume_path);
+ ($dst_volume_id) = $dbh->selectrow_array('SELECT last_insert_rowid()');
}
while (1) {
- my $free=df($dst_volume_path)/1024/1024/1024;
- if ($free<2) {
- #die "$dst_volume_path: only $free TB free, 2 required.\n";
+ my $free = df($dst_volume_path) / 1024 / 1024 / 1024;
+ if ($free < 2) {
return 0;
}
@@ -1004,12 +976,12 @@ sub move_new {
# newest run not on target volume
#
- my ($stat_rowid,$src_volume_id,$src_volume_path,$stat_started,$path_path,$job_id) = $dbh->selectrow_array(<<'EOF',undef,$job_name,$dst_volume_id);
-SELECT stat.rowid,volume_id,volume_path,stat_started,path_path,job_id
-FROM stat,volume,job,path
+ my ($stat_rowid, $src_volume_id, $src_volume_path, $stat_started, $path_path, $job_id) = $dbh->selectrow_array(<<'EOF', undef, $job_name, $dst_volume_id);
+SELECT stat.rowid, volume_id, volume_path, stat_started, path_path, job_id
+FROM stat, volume, job, path
WHERE
- job_name=? AND volume_id <> ?
- AND stat_job_id=job_id AND job_path_id=path_id AND stat_volume_id=volume_id
+ job_name = ? AND volume_id <> ?
+ AND stat_job_id = job_id AND job_path_id = path_id AND stat_volume_id = volume_id
ORDER BY stat_started DESC
EOF
@@ -1024,8 +996,8 @@ EOF
redo;
}
- my ($from)=$src_volume_path=~m"([^/]*)$"; # 'C1234'
- my $run_name=run_name($job_name,$stat_started,$path_path);
+ my ($from) = $src_volume_path =~ m"([^/]*)$"; # 'C1234'
+ my $run_name = run_name($job_name, $stat_started, $path_path);
#
# run of same job on target volume, nearest date
@@ -1040,66 +1012,66 @@ EOF
redo;
}
- my ($linkfrom_job_name,$linkfrom_stat_started,$linkfrom_path_path)=$dbh->selectrow_array(<<'EOF',undef,$linkfrom_stat_rowid);
-SELECT job_name,stat_started,path_path
-FROM stat,job,path
-WHERE stat_job_id=job_id
-AND job_path_id=path_id
-AND stat.rowid=?
+ my ($linkfrom_job_name, $linkfrom_stat_started, $linkfrom_path_path) = $dbh->selectrow_array(<<'EOF', undef, $linkfrom_stat_rowid);
+SELECT job_name, stat_started, path_path
+FROM stat, job, path
+WHERE stat_job_id = job_id
+AND job_path_id = path_id
+AND stat.rowid = ?
EOF
- my $linkfrom_run_name=run_name($linkfrom_job_name,$linkfrom_stat_started,$linkfrom_path_path);
+ my $linkfrom_run_name = run_name($linkfrom_job_name, $linkfrom_stat_started, $linkfrom_path_path);
warn scalar(localtime)," $status_tag : $from $run_name -> $to $run_name (link $linkfrom_run_name)\n";
upid_doing("$status_tag $from $run_name -> $to $run_name (link $linkfrom_run_name)");
- lck_lock("RUN.$stat_rowid",LCK_EX);
- lck_lock("RUN.$linkfrom_stat_rowid",LCK_EX);
+ lck_lock("RUN.$stat_rowid", LCK_EX);
+ lck_lock("RUN.$linkfrom_stat_rowid", LCK_EX);
$dbh->commit();
- rsync_copy("$src_volume_path/$job_name/$run_name/","$dst_volume_path/$job_name/$run_name","$dst_volume_path/$linkfrom_job_name/$linkfrom_run_name");
- rename("$src_volume_path/$job_name/$run_name","$src_volume_path/$job_name/$run_name.BEING_DELETED") or
+ rsync_copy("$src_volume_path/$job_name/$run_name/", "$dst_volume_path/$job_name/$run_name", "$dst_volume_path/$linkfrom_job_name/$linkfrom_run_name");
+ rename("$src_volume_path/$job_name/$run_name", "$src_volume_path/$job_name/$run_name.BEING_DELETED") or
die "$src_volume_path/$job_name/$run_name $src_volume_path/$job_name/$run_name.BEING_DELETED : $!\n";
- $dbh->do('UPDATE stat SET stat_volume_id=? WHERE rowid=?',undef,$dst_volume_id,$stat_rowid);
- lck_unlock("RUN.$stat_rowid",LCK_EX);
- lck_unlock("RUN.$linkfrom_stat_rowid",LCK_EX);
+ $dbh->do('UPDATE stat SET stat_volume_id = ? WHERE rowid = ?', undef, $dst_volume_id, $stat_rowid);
+ lck_unlock("RUN.$stat_rowid", LCK_EX);
+ lck_unlock("RUN.$linkfrom_stat_rowid", LCK_EX);
} else {
- unless (lck_can_lock("JOB.$job_name",LCK_EX)) {
+ unless (lck_can_lock("JOB.$job_name", LCK_EX)) {
$dbh->commit;
sleep 10;
redo;
}
- warn scalar(localtime)," $status_tag : $from $run_name -> $to $run_name\n";
+ warn scalar(localtime), " $status_tag : $from $run_name -> $to $run_name\n";
upid_doing("$status_tag $from $run_name -> $to $run_name");
- lck_lock("JOB.$job_name",LCK_EX);
+ lck_lock("JOB.$job_name", LCK_EX);
$dbh->commit();
-d "$dst_volume_path/$job_name" or mkdir "$dst_volume_path/$job_name" or die "$dst_volume_path/$job_name: $!\n";
-e "$dst_volume_path/$job_name/$run_name" and die "$dst_volume_path/$job_name/$run_name: exists\n";
system "cp -a $src_volume_path/$job_name/$run_name $dst_volume_path/$job_name/$run_name" and exit 1;
- rename("$src_volume_path/$job_name/$run_name","$src_volume_path/$job_name/$run_name.BEING_DELETED") or
+ rename("$src_volume_path/$job_name/$run_name", "$src_volume_path/$job_name/$run_name.BEING_DELETED") or
die "$src_volume_path/$job_name/$run_name $src_volume_path/$job_name/$run_name.BEING_DELETED : $!\n";
- $dbh->do('UPDATE stat SET stat_volume_id=? WHERE rowid=?',undef,$dst_volume_id,$stat_rowid);
- lck_unlock("JOB.$job_name",LCK_EX);
+ $dbh->do('UPDATE stat SET stat_volume_id = ? WHERE rowid = ?', undef, $dst_volume_id, $stat_rowid);
+ lck_unlock("JOB.$job_name", LCK_EX);
}
upid_doing("$status_tag rm -rf $src_volume_path/$job_name/$run_name.BEING_DELETED");
- warn scalar(localtime)," $status_tag : rm -rf $src_volume_path/$job_name/$run_name.BEING_DELETED\n";
- system "rm -rf $src_volume_path/$job_name/$run_name.BEING_DELETED" and exit 1;
+ warn scalar(localtime), " $status_tag : rm -rf $src_volume_path/$job_name/$run_name.BEING_DELETED\n";
+ system "rm -rf $src_volume_path/$job_name/$run_name.BEING_DELETED" and exit 1;
unlink("$src_volume_path/$job_name/last"); # will be invalid soon anyway
rmdir "$src_volume_path/$job_name"; # (if empty)
}
- warn scalar(localtime)." $status_tag :(done)\n";
+ warn scalar(localtime) . " $status_tag :(done)\n";
return 1;
}
sub shortDate {
- my ($time)=@_;
- my $string=localtime($time); # Sat Nov 3 09:33:14 2012
- return substr($string,4,12).substr($string,19,5);
+ my ($time) = @_;
+ my $string = localtime($time); # Sat Nov 3 09:33:14 2012
+ return substr($string, 4, 12).substr($string, 19, 5);
}
sub humanSeconds {
- my ($seconds)=@_;
+ my ($seconds) = @_;
- $seconds<60 and return sprintf ('%2d s',$seconds);
- $seconds<3600 and return sprintf ('%2d m %2d s',$seconds/60,$seconds%60);
- $seconds<86400 and return sprintf ('%2d h %2d m',$seconds/3600,$seconds%3600/60);
- return sprintf('%2d d %2d h',$seconds/86400,$seconds%86400/3600);
+ $seconds < 60 and return sprintf ('%2d s', $seconds);
+ $seconds < 3600 and return sprintf ('%2d m %2d s', $seconds / 60, $seconds % 60);
+ $seconds < 86400 and return sprintf ('%2d h %2d m', $seconds / 3600, $seconds % 3600 / 60);
+ return sprintf('%2d d %2d h', $seconds / 86400, $seconds % 86400 / 3600);
#
# now
# 12 s
@@ -1110,48 +1082,48 @@ sub humanSeconds {
}
sub humanDue {
- my ($due)=@_;
- $due==0 and return 'DUE now';
- $due>0 and return 'due in '.humanSeconds($due);
+ my ($due) = @_;
+ $due == 0 and return 'DUE now';
+ $due > 0 and return 'due in ' . humanSeconds($due);
$due < -1359900000 and return 'DUE since ever';
- $due<0 and return 'DUE since '.humanSeconds(-$due);
+ $due < 0 and return 'DUE since ' . humanSeconds(-$due);
}
sub cmd_status {
$dbh->begin_work;
my $sth;
- $sth=$dbh->prepare('SELECT upid_pid,upid_text,upid_since FROM upid');
+ $sth = $dbh->prepare('SELECT upid_pid, upid_text, upid_since FROM upid');
$sth->execute;
print "processes\n";
- while (my $row=$sth->fetchrow_arrayref) {
- my ($pid,$what,$upid_since)=@$row;
- my $since=$upid_since ? humanSeconds(time-$upid_since) : '-';
- printf(" %-9s : %9s : %s\n",$pid,$since,$what);
+ while (my $row = $sth->fetchrow_arrayref) {
+ my ($pid, $what, $upid_since) = @$row;
+ my $since = $upid_since ? humanSeconds(time - $upid_since) : '-';
+ printf(" %-9s : %9s : %s\n", $pid, $since, $what);
}
print "next 30 due\n";
- $sth=$dbh->prepare('
-SELECT job_id,job_name,job_ok,job_started,path_server,path_path,
-CASE WHEN job_ok THEN job_started-? ELSE JOB_STARTED-? END as due
-FROM job,path
-WHERE job_path_id=path_id
+ $sth = $dbh->prepare('
+SELECT job_id, job_name, job_ok, job_started, path_server, path_path,
+CASE WHEN job_ok THEN job_started - ? ELSE JOB_STARTED - ? END as due
+FROM job, path
+WHERE job_path_id = path_id
AND job_enabled
ORDER BY due
limit 30
');
- $sth->execute(time-24*60*60,time-8*60*60);
- while (my $row=$sth->fetchrow_arrayref) {
- my ($job_id,$job_name,$job_ok,$job_started,$path_server,$path_path,$due)=@$row;
+ $sth->execute(time - 24 * 60 * 60, time - 8 * 60 * 60);
+ while (my $row = $sth->fetchrow_arrayref) {
+ my ($job_id, $job_name, $job_ok, $job_started, $path_server, $path_path, $due) = @$row;
- my $lpid=lck_get_pid("JOB.$job_name");
+ my $lpid = lck_get_pid("JOB.$job_name");
- my $state=$lpid?"PID $lpid":($job_ok?'ok':$job_started?'fail':'-');
- my $time=$job_started?shortDate($job_started):'never';
- my $src=$path_server.':'.$path_path;
+ my $state = $lpid ? "PID $lpid" : ($job_ok ? 'ok' : $job_started ? 'fail' : '-');
+ my $time = $job_started ? shortDate($job_started) : 'never';
+ my $src = $path_server . ':' . $path_path;
- printf(" %-19s %-9s %-17s %-25s %s\n",humanDue($due),$state,$time,$job_name,$src);
+ printf(" %-19s %-9s %-17s %-25s %s\n", humanDue($due), $state, $time, $job_name, $src);
}
$dbh->commit;
@@ -1160,54 +1132,54 @@ limit 30
sub cmd_jobs {
$dbh->begin_work;
- my $sth=$dbh->prepare('
-SELECT job_id,job_name,job_ok,job_started,job_enabled,path_server,path_path,
-CASE WHEN job_ok THEN job_started-? ELSE JOB_STARTED-? END as due
-FROM job,path
-WHERE job_path_id=path_id
+ my $sth = $dbh->prepare('
+SELECT job_id, job_name, job_ok, job_started, job_enabled, path_server, path_path,
+CASE WHEN job_ok THEN job_started - ? ELSE JOB_STARTED - ? END as due
+FROM job, path
+WHERE job_path_id = path_id
ORDER BY due
');
- $sth->execute(time-24*60*60,time-8*60*60);
- while (my $row=$sth->fetchrow_arrayref) {
- my ($job_id,$job_name,$job_ok,$job_started,$job_enabled,$path_server,$path_path,$due)=@$row;
+ $sth->execute(time - 24 * 60 * 60, time - 8 * 60 * 60);
+ while (my $row = $sth->fetchrow_arrayref) {
+ my ($job_id, $job_name, $job_ok, $job_started, $job_enabled, $path_server, $path_path, $due) = @$row;
- my $lpid=lck_get_pid("JOB.$job_name");
+ my $lpid = lck_get_pid("JOB.$job_name");
- my $state=$lpid?"PID $lpid":($job_ok?'ok':$job_started?'fail':'-');
- my $time=$job_started?shortDate($job_started):'never';
- my $src=$path_server.':'.$path_path;
+ my $state = $lpid ? "PID $lpid" : ($job_ok ? 'ok' : $job_started ? 'fail':'-');
+ my $time = $job_started ? shortDate($job_started) : 'never';
+ my $src = $path_server . ':' . $path_path;
- printf("%19s %-9s %17s %-28s %s\n",($job_enabled?(humanDue($due)):'disabled'),$state,$time,$job_name,$src);
+ printf("%19s %-9s %17s %-28s %s\n", ($job_enabled ? (humanDue($due)) : 'disabled'), $state, $time, $job_name, $src);
}
$dbh->commit;
}
sub human {
- my ($n)=@_;
- $n<1024 and return sprintf("%5d byte",$n);
- $n/=1024;
- $n<1024 and return sprintf("%6.1f KiB",$n);
- $n/=1024;
- $n<1024 and return sprintf("%6.1f MiB",$n);
- $n/=1024;
- $n<1024 and return sprintf("%6.1f GiB",$n);
- $n/=1024;
- return sprintf("%6.1f TiB",$n);
+ my ($n) = @_;
+ $n < 1024 and return sprintf("%5d byte", $n);
+ $n /= 1024;
+ $n < 1024 and return sprintf("%6.1f KiB", $n);
+ $n /= 1024;
+ $n < 1024 and return sprintf("%6.1f MiB", $n);
+ $n /= 1024;
+ $n < 1024 and return sprintf("%6.1f GiB", $n);
+ $n /= 1024;
+ return sprintf("%6.1f TiB", $n);
}
sub show_stat_table {
- my ($sql,@bindings)=@_;
- my $sth=$dbh->prepare($sql);
+ my ($sql, @bindings) = @_;
+ my $sth = $dbh->prepare($sql);
$sth->execute(@bindings);
- while (my $row=$sth->fetchrow_arrayref) {
- my ($job_name,$stat_started,$stat_du_bytes,$stat_seconds,$stat_bytes_transferred,$path_server,$path_path)=@$row;
- printf " %-21s %s %10s (%10s transferred in %9s) %s:%s\n",$job_name,shortDate($stat_started),human($stat_du_bytes),human($stat_bytes_transferred),humanSeconds($stat_seconds),$path_server,$path_path;
+ while (my $row = $sth->fetchrow_arrayref) {
+ my ($job_name, $stat_started, $stat_du_bytes, $stat_seconds, $stat_bytes_transferred, $path_server, $path_path) = @$row;
+ printf " %-21s %s %10s (%10s transferred in %9s) %s:%s\n", $job_name, shortDate($stat_started), human($stat_du_bytes), human($stat_bytes_transferred), humanSeconds($stat_seconds), $path_server, $path_path;
}
}
-our $COUNT_SQL=<<'EOF';
+our $COUNT_SQL = <<'EOF';
SELECT
stat_job_id AS count_job_id,
count(stat_job_id) as count_count
@@ -1217,131 +1189,131 @@ EOF
sub cmd_stat {
- my $last_stat="SELECT stat_job_id AS last_stat_job_id,MAX(stat_started) AS last_stat_started FROM stat GROUP BY stat_job_id";
- my $last_inc_stat="SELECT stat_job_id AS last_stat_job_id,MAX(stat_started) AS last_stat_started FROM stat WHERE stat_type='I' GROUP BY stat_job_id";
- my $join_last="stat_job_id=last_stat_job_id AND stat_started=last_stat_started";
- my $cols="job_name,stat_started,stat_du_bytes,stat_seconds,stat_bytes_transferred,path_server,path_path";
- my $tabs="job,stat,path";
- my $join="stat_job_id=job_id AND job_path_id=path_id";
- my $active="job_enabled";
+ my $last_stat = "SELECT stat_job_id AS last_stat_job_id, MAX(stat_started) AS last_stat_started FROM stat GROUP BY stat_job_id";
+ my $last_inc_stat = "SELECT stat_job_id AS last_stat_job_id, MAX(stat_started) AS last_stat_started FROM stat WHERE stat_type = 'I' GROUP BY stat_job_id";
+ my $join_last = "stat_job_id=last_stat_job_id AND stat_started = last_stat_started";
+ my $cols = "job_name, stat_started, stat_du_bytes, stat_seconds, stat_bytes_transferred, path_server, path_path";
+ my $tabs = "job, stat, path";
+ my $join = "stat_job_id = job_id AND job_path_id = path_id";
+ my $active = "job_enabled";
$dbh->begin_work;
my $sth;
- if (@ARGV==0) {
+ if (@ARGV == 0) {
print "TOP 10 source size (all), last run\n";
- show_stat_table("SELECT $cols FROM $tabs,($last_stat) WHERE $active AND $join AND $join_last ORDER BY stat_du_bytes DESC LIMIT 10");
+ show_stat_table("SELECT $cols FROM $tabs, ($last_stat) WHERE $active AND $join AND $join_last ORDER BY stat_du_bytes DESC LIMIT 10");
print "TOP 10 elapsed (Incrementals) , last run\n";
- show_stat_table("SELECT $cols FROM $tabs,($last_inc_stat) WHERE $active AND $join AND $join_last ORDER BY stat_seconds DESC LIMIT 10");
+ show_stat_table("SELECT $cols FROM $tabs, ($last_inc_stat) WHERE $active AND $join AND $join_last ORDER BY stat_seconds DESC LIMIT 10");
print "TOP 10 transferred (Incrementals) , last run \n";
- show_stat_table("SELECT $cols FROM $tabs,($last_inc_stat) WHERE $active AND $join AND $join_last ORDER BY stat_bytes_transferred DESC LIMIT 10");
+ show_stat_table("SELECT $cols FROM $tabs, ($last_inc_stat) WHERE $active AND $join AND $join_last ORDER BY stat_bytes_transferred DESC LIMIT 10");
} elsif (@ARGV==1 && $ARGV[0] eq 'expire') {
print "OLDEST 20\n";
- show_stat_table("SELECT $cols FROM $tabs,($COUNT_SQL) WHERE $join AND stat_job_id=count_job_id AND count_count>1 ORDER BY stat_started LIMIT 20");
+ show_stat_table("SELECT $cols FROM $tabs, ($COUNT_SQL) WHERE $join AND stat_job_id = count_job_id AND count_count > 1 ORDER BY stat_started LIMIT 20");
- my $keep=time-$DAYS*24*60*60;
+ my $keep = time - $DAYS * 24 * 60 * 60;
print "\n";
- print "(expire time : ".shortDate($keep).")\n";
+ print "(expire time : " . shortDate($keep) . ")\n";
- } elsif (@ARGV==1 && $ARGV[0] eq 'zombies') {
+ } elsif (@ARGV == 1 && $ARGV[0] eq 'zombies') {
- my $keep=time-$DAYS*24*60*60;
+ my $keep = time - $DAYS * 24 * 60 * 60;
print "zombies\n";
- show_stat_table("SELECT $cols FROM $tabs,($COUNT_SQL) WHERE $join AND stat_job_id=count_job_id AND count_count==1 AND stat_startedprepare(<<'EOF');
-SELECT job_name,path_path,MAX(stat_du_bytes),SUM(stat_bytes_transferred),MAX(stat_du_bytes)+SUM(stat_bytes_transferred) AS weight,job_enabled
-FROM job,stat,path
-WHERE stat_job_id=job_id
-AND job_path_id=path_id
+ $sth = $dbh->prepare(<<'EOF');
+SELECT job_name, path_path, MAX(stat_du_bytes), SUM(stat_bytes_transferred), MAX(stat_du_bytes) + SUM(stat_bytes_transferred) AS weight, job_enabled
+FROM job, stat, path
+WHERE stat_job_id = job_id
+AND job_path_id = path_id
GROUP BY job_name
ORDER BY weight DESC
LIMIT 20
EOF
$sth->execute();
- while (my $row=$sth->fetchrow_arrayref) {
- my ($job_name,$path_path,$max_bytes,$sum_transferred,$weight,$job_enabled)=@$row;
- printf "%-25s %20s %11s %11s %11s %s\n",$job_name,fstag($path_path),human($max_bytes),human($sum_transferred),human($weight),$job_enabled?'':'(disabled)';
+ while (my $row = $sth->fetchrow_arrayref) {
+ my ($job_name, $path_path, $max_bytes, $sum_transferred, $weight, $job_enabled) = @$row;
+ printf "%-25s %20s %11s %11s %11s %s\n", $job_name, fstag($path_path), human($max_bytes), human($sum_transferred), human($weight), $job_enabled ? '' : '(disabled)';
}
print "\n";
print "top 20 weight disabled jobs ( max trans weight ) :\n";
$sth=$dbh->prepare(<<'EOF');
-SELECT job_name,path_path,MAX(stat_du_bytes),SUM(stat_bytes_transferred),MAX(stat_du_bytes)+SUM(stat_bytes_transferred) AS weight,job_enabled
-FROM job,stat,path
-WHERE stat_job_id=job_id
-AND job_path_id=path_id
-AND job_enabled=0
+SELECT job_name, path_path, MAX(stat_du_bytes), SUM(stat_bytes_transferred), MAX(stat_du_bytes) + SUM(stat_bytes_transferred) AS weight, job_enabled
+FROM job, stat, path
+WHERE stat_job_id = job_id
+AND job_path_id = path_id
+AND job_enabled = 0
GROUP BY job_name
ORDER BY weight DESC
LIMIT 20
EOF
$sth->execute();
- while (my $row=$sth->fetchrow_arrayref) {
- my ($job_name,$path_path,$max_bytes,$sum_transferred,$weight,$job_enabled)=@$row;
- printf "%-25s %20s %11s %11s %11s %s\n",$job_name,fstag($path_path),human($max_bytes),human($sum_transferred),human($weight),$job_enabled?'':'(disabled)';
+ while (my $row = $sth->fetchrow_arrayref) {
+ my ($job_name, $path_path, $max_bytes, $sum_transferred, $weight, $job_enabled) = @$row;
+ printf "%-25s %20s %11s %11s %11s %s\n", $job_name, fstag($path_path), human($max_bytes), human($sum_transferred), human($weight), $job_enabled ? '' : '(disabled)';
}
- } elsif (@ARGV==1 && $ARGV[0] eq 'week') {
- $sth=$dbh->prepare(<<"EOF");
-SELECT job_name,path_path,SUM(stat_seconds),SUM(stat_bytes_transferred) AS s,job_enabled
+ } elsif (@ARGV == 1 && $ARGV[0] eq 'week') {
+ $sth = $dbh->prepare(<<"EOF");
+SELECT job_name, path_path, SUM(stat_seconds), SUM(stat_bytes_transferred) AS s, job_enabled
FROM $tabs
-WHERE $join AND stat_started>?
+WHERE $join AND stat_started > ?
GROUP BY job_id
ORDER BY s DESC
LIMIT 20
EOF
- $sth->execute(time-7*24*60*60);
- while (my $row=$sth->fetchrow_arrayref) {
- my ($job_name,$path_path,$sum_stat_seconds,$sum_stat_bytes_transferred,$job_enabled)=@$row;
- printf "%-25s %-20s %10s %11s %s\n",$job_name,fstag($path_path),humanSeconds($sum_stat_seconds),human($sum_stat_bytes_transferred),$job_enabled?'':'(disabled)';
+ $sth->execute(time - 7 * 24 * 60 * 60);
+ while (my $row = $sth->fetchrow_arrayref) {
+ my ($job_name, $path_path, $sum_stat_seconds, $sum_stat_bytes_transferred, $job_enabled) = @$row;
+ printf "%-25s %-20s %10s %11s %s\n", $job_name, fstag($path_path), humanSeconds($sum_stat_seconds), human($sum_stat_bytes_transferred), $job_enabled ? '' : '(disabled)';
}
- } elsif (@ARGV==1 && $ARGV[0] eq 'oversize') {
- my $cnt=0;
+ } elsif (@ARGV == 1 && $ARGV[0] eq 'oversize') {
+ my $cnt = 0;
- $sth=$dbh->prepare(<<"EOF");
-SELECT job_name,stat_du_bytes,path_path
-FROM $tabs,($last_stat)
-WHERE $join AND stat_du_bytes>? AND job_enabled=1 AND $join_last
+ $sth = $dbh->prepare(<<"EOF");
+SELECT job_name, stat_du_bytes, path_path
+FROM $tabs, ($last_stat)
+WHERE $join AND stat_du_bytes > ? AND job_enabled = 1 AND $join_last
ORDER BY stat_bytes DESC
EOF
- $sth->execute(1024*1024*1024*1024);
- while (my $row=$sth->fetchrow_arrayref) {
- my ($job_name,$stat_du_bytes,$path_path)=@$row;
- $job_name eq 'cfdl_imapspool_2' and $stat_du_bytes < 2.5*1024*1024*1024*1024 and next;
- $cnt==0 and print "The following backup jobs have a source size over the limit (usually 1TB):\n\n";
- printf "%-20s %-60s : %11s\n",$job_name,$path_path,human($stat_du_bytes||0);
+ $sth->execute(1024 * 1024 * 1024 * 1024);
+ while (my $row = $sth->fetchrow_arrayref) {
+ my ($job_name, $stat_du_bytes, $path_path) = @$row;
+ $job_name eq 'cfdl_imapspool_2' and $stat_du_bytes < 2.5 * 1024 * 1024 * 1024 * 1024 and next;
+ $cnt == 0 and print "The following backup jobs have a source size over the limit (usually 1TB):\n\n";
+ printf "%-20s %-60s : %11s\n", $job_name, $path_path, human($stat_du_bytes || 0);
$cnt++;
}
$cnt and exit 1;
print "No job has a source size over the limit (usually 1 TB).\n\n";
- } elsif (@ARGV==1 && $ARGV[0] eq 'dusize') {
- my $cnt=0;
+ } elsif (@ARGV == 1 && $ARGV[0] eq 'dusize') {
+ my $cnt = 0;
- $sth=$dbh->prepare(<<"EOF");
-SELECT job_name,stat_bytes,stat_du_bytes,path_path
+ $sth = $dbh->prepare(<<"EOF");
+SELECT job_name, stat_bytes, stat_du_bytes, path_path
FROM $tabs,($last_stat)
-WHERE $join AND job_enabled=1 AND $join_last
+WHERE $join AND job_enabled = 1 AND $join_last
ORDER BY stat_bytes DESC
EOF
$sth->execute();
- while (my $row=$sth->fetchrow_arrayref) {
- $cnt==0 and print "rsync size vs. du size:\n\n";
- my ($job_name,$stat_bytes,$stat_du_bytes,$path_path)=@$row;
+ while (my $row = $sth->fetchrow_arrayref) {
+ $cnt == 0 and print "rsync size vs. du size:\n\n";
+ my ($job_name, $stat_bytes, $stat_du_bytes, $path_path) = @$row;
defined $stat_du_bytes or next;
- printf "%-15d %-30s : RSYNC: %11s DU: %11s\n",$stat_du_bytes-$stat_bytes,$job_name,human($stat_bytes),human($stat_du_bytes||0);
+ printf "%-15d %-30s : RSYNC: %11s DU: %11s\n", $stat_du_bytes - $stat_bytes, $job_name, human($stat_bytes), human($stat_du_bytes || 0);
$cnt++;
}
}
@@ -1353,41 +1325,41 @@ EOF
}
sub cmd_job {
- @ARGV==1 or die USAGE;
- my ($job_name)=@_;
+ @ARGV == 1 or die USAGE;
+ my ($job_name) = @_;
$dbh->begin_work;
my $sth;
$sth=$dbh->prepare('
-SELECT job_id,job_name,job_ok,job_started,job_enabled,path_server,path_path,
-CASE WHEN job_ok THEN job_started-? ELSE JOB_STARTED-? END as due
+SELECT job_id, job_name, job_ok, job_started, job_enabled, path_server, path_path,
+CASE WHEN job_ok THEN job_started - ? ELSE JOB_STARTED - ? END as due
FROM job,path
-WHERE job_path_id=path_id
-AND job_name=?
+WHERE job_path_id = path_id
+AND job_name = ?
ORDER BY due
');
- $sth->execute(time-24*60*60,time-8*60*60,$job_name);
- while (my $row=$sth->fetchrow_arrayref) {
- my ($job_id,$job_name,$job_ok,$job_started,$job_enabled,$path_server,$path_path,$due)=@$row;
+ $sth->execute(time - 24 * 60 * 60, time - 8 * 60 * 60, $job_name);
+ while (my $row = $sth->fetchrow_arrayref) {
+ my ($job_id, $job_name, $job_ok, $job_started, $job_enabled, $path_server, $path_path, $due) = @$row;
- my $lpid=lck_get_pid("JOB.$job_name")||'';
+ my $lpid=lck_get_pid("JOB.$job_name") || '';
- my $state=$lpid?"PID $lpid":($job_ok?'ok':$job_started?'fail':'-');
- my $time=$job_started?shortDate($job_started):'never';
- my $src=$path_server.':'.$path_path;
+ my $state = $lpid ? "PID $lpid" : ($job_ok ? 'ok' : $job_started ? 'fail' : '-');
+ my $time = $job_started ? shortDate($job_started) : 'never';
+ my $src = $path_server . ':' . $path_path;
- printf("%19s %-9s %17s %-28s %s %s\n",($job_enabled?(humanDue($due)):'disabled'),$state,$time,$job_name,$src,$lpid);
+ printf("%19s %-9s %17s %-28s %s %s\n", ($job_enabled ? (humanDue($due)) : 'disabled'), $state, $time, $job_name, $src, $lpid);
}
print "\n";
- $sth=$dbh->prepare('select stat_type,job_name,stat_started,stat_du_bytes,stat_seconds,stat_bytes_transferred,path_server,path_path from job,stat,path where stat_job_id=job_id and job_path_id=path_id and job_name=? order by stat_started');
+ $sth=$dbh->prepare('select stat_type, job_name, stat_started, stat_du_bytes, stat_seconds, stat_bytes_transferred, path_server, path_path from job, stat, path where stat_job_id = job_id and job_path_id = path_id and job_name = ? order by stat_started');
$sth->execute($job_name);
- while (my $row=$sth->fetchrow_arrayref) {
- my ($stat_type,$job_name,$stat_started,$stat_du_bytes,$stat_seconds,$stat_bytes_transferred,$path_server,$path_path)=@$row;
- printf " %s %-21s %s %11s (%11s transferred in %11s) %s:%s\n",$stat_type,$job_name,shortDate($stat_started),human($stat_du_bytes||0),human($stat_bytes_transferred),humanSeconds($stat_seconds),$path_server,$path_path;
+ while (my $row = $sth->fetchrow_arrayref) {
+ my ($stat_type, $job_name, $stat_started, $stat_du_bytes, $stat_seconds, $stat_bytes_transferred, $path_server, $path_path) = @$row;
+ printf " %s %-21s %s %11s (%11s transferred in %11s) %s:%s\n", $stat_type, $job_name, shortDate($stat_started), human($stat_du_bytes || 0), human($stat_bytes_transferred), humanSeconds($stat_seconds), $path_server, $path_path;
}
$dbh->commit;
@@ -1397,18 +1369,18 @@ sub cmd_fix_stat {
upid_register_self();
- my $sth=$dbh->prepare('SELECT stat.rowid,stat_started,job_name,volume_path,path_path FROM stat,job,volume,path WHERE stat_du_bytes IS NULL AND stat_volume_id=volume_id AND stat_job_id=job_id AND job_path_id=path_id ORDER BY stat.rowid DESC');
+ my $sth = $dbh->prepare('SELECT stat.rowid, stat_started, job_name, volume_path, path_path FROM stat, job, volume, path WHERE stat_du_bytes IS NULL AND stat_volume_id = volume_id AND stat_job_id = job_id AND job_path_id = path_id ORDER BY stat.rowid DESC');
SCAN: while (1) {
upid_doing("du scan");
$dbh->begin_work();
$sth->execute();
- my $retry=0;
- while (my $row=$sth->fetchrow_arrayref) {
- my ($stat_rowid,$stat_started,$job_name,$volume_path,$path_path)=@$row;
- my $dir=sprintf '%s/%s/%s:%s:%s',$volume_path,$job_name,$job_name,timetag($stat_started),fstag($path_path);
- printf "%5d %s %-30s %s %s\n",$stat_rowid,shortDate($stat_started),$job_name,$volume_path,$dir;
- unless (lck_can_lock("JOB.$job_name",LCK_EX)) {
+ my $retry = 0;
+ while (my $row = $sth->fetchrow_arrayref) {
+ my ($stat_rowid, $stat_started, $job_name, $volume_path, $path_path) = @$row;
+ my $dir = sprintf '%s/%s/%s:%s:%s', $volume_path, $job_name, $job_name,timetag($stat_started), fstag($path_path);
+ printf "%5d %s %-30s %s %s\n", $stat_rowid, shortDate($stat_started), $job_name, $volume_path, $dir;
+ unless (lck_can_lock("JOB.$job_name", LCK_EX)) {
print " *** LOCKED ***\n";
$retry++;
next;
@@ -1417,15 +1389,15 @@ SCAN: while (1) {
print " *** missing : $dir\n";
next;
}
- lck_lock("JOB.$job_name",LCK_EX);
+ lck_lock("JOB.$job_name", LCK_EX);
upid_doing("du $dir");
$sth->finish();
$dbh->commit();
- my ($stat_du_bytes)=du_bytes("$dir");
- $dbh->do('UPDATE stat SET stat_du_bytes=? WHERE rowid=?',undef,$stat_du_bytes,$stat_rowid);
- print " ... ",human($stat_du_bytes),"\n";
- lck_unlock("JOB.$job_name",LCK_EX);
+ my ($stat_du_bytes) = du_bytes("$dir");
+ $dbh->do('UPDATE stat SET stat_du_bytes = ? WHERE rowid = ?', undef, $stat_du_bytes, $stat_rowid);
+ print " ... ", human($stat_du_bytes), "\n";
+ lck_unlock("JOB.$job_name", LCK_EX);
redo SCAN;
}
$dbh->commit();
@@ -1438,54 +1410,54 @@ SCAN: while (1) {
sub expire_one {
my ($want_job_name) = @_;
$dbh->begin_work();
- my $keep=time-$DAYS*24*60*60;
+ my $keep = time - $DAYS * 24 * 60 * 60;
my $sth;
if (defined $want_job_name) {
- $sth=$dbh->prepare(<<"EOF");
-SELECT stat.rowid,job_name,stat_started,path_path,volume_path
-FROM stat,job,path,volume,($COUNT_SQL)
-WHERE stat_volume_id=volume_id AND job_path_id=path_id AND stat_job_id=job_id AND job_id=count_job_id
-AND stat_started1
-AND job_name=?
+ $sth = $dbh->prepare(<<"EOF");
+SELECT stat.rowid, job_name, stat_started, path_path, volume_path
+FROM stat, job, path, volume, ($COUNT_SQL)
+WHERE stat_volume_id = volume_id AND job_path_id = path_id AND stat_job_id = job_id AND job_id = count_job_id
+AND stat_started < ?
+AND count_count > 1
+AND job_name = ?
ORDER BY stat_started
EOF
$sth->execute($keep, $want_job_name);
} else {
- $sth=$dbh->prepare(<<"EOF");
-SELECT stat.rowid,job_name,stat_started,path_path,volume_path
-FROM stat,job,path,volume,($COUNT_SQL)
-WHERE stat_volume_id=volume_id AND job_path_id=path_id AND stat_job_id=job_id AND job_id=count_job_id
-AND stat_started1
+ $sth = $dbh->prepare(<<"EOF");
+SELECT stat.rowid, job_name, stat_started, path_path, volume_path
+FROM stat, job, path, volume, ($COUNT_SQL)
+WHERE stat_volume_id = volume_id AND job_path_id = path_id AND stat_job_id = job_id AND job_id = count_job_id
+AND stat_started < ?
+AND count_count > 1
ORDER BY stat_started
EOF
$sth->execute($keep);
}
- while (my $row=$sth->fetchrow_arrayref) {
- my ($stat_rowid,$job_name,$stat_started,$path_path,$volume_path)=@$row;
- lck_can_lock("JOB.$job_name",LCK_EX) or next;
- lck_can_lock("EXPIRE.$volume_path",LCK_EX) or next;
- lck_can_lock("RUN.$stat_rowid",LCK_EX) or next;
-
- lck_lock("JOB.$job_name",LCK_EX);
- lck_lock("EXPIRE.$volume_path",LCK_EX);
- lck_lock("RUN.$stat_rowid",LCK_EX);
-
- my $timetag=timetag($stat_started);
- my $fstag=fstag($path_path);
+ while (my $row = $sth->fetchrow_arrayref) {
+ my ($stat_rowid, $job_name, $stat_started, $path_path, $volume_path) = @$row;
+ lck_can_lock("JOB.$job_name", LCK_EX) or next;
+ lck_can_lock("EXPIRE.$volume_path", LCK_EX) or next;
+ lck_can_lock("RUN.$stat_rowid", LCK_EX) or next;
+
+ lck_lock("JOB.$job_name", LCK_EX);
+ lck_lock("EXPIRE.$volume_path", LCK_EX);
+ lck_lock("RUN.$stat_rowid", LCK_EX);
+
+ my $timetag = timetag($stat_started);
+ my $fstag = fstag($path_path);
my $dir="$volume_path/$job_name/$job_name:$timetag:$fstag";
$sth->finish();
-d $dir and system "mv $dir $dir.EXPIRED" and exit 1;
- $dbh->do('DELETE FROM stat WHERE rowid=?',undef,$stat_rowid);
+ $dbh->do('DELETE FROM stat WHERE rowid=?', undef, $stat_rowid);
upid_doing("EXPIRE rm -rf $dir.EXPIRED");
$dbh->commit();
-d "$dir.EXPIRED" and system "rm -rf $dir.EXPIRED" and exit 1;
- lck_unlock("JOB.$job_name",LCK_EX);
- lck_unlock("EXPIRE.$volume_path",LCK_EX);
- lck_unlock("RUN.$stat_rowid",LCK_EX);
+ lck_unlock("JOB.$job_name", LCK_EX);
+ lck_unlock("EXPIRE.$volume_path", LCK_EX);
+ lck_unlock("RUN.$stat_rowid", LCK_EX);
return $job_name;
}
$dbh->commit();
@@ -1546,9 +1518,9 @@ sub cmd_expire {
sub cmd_move {
die "move code needs review w.r.t. stat_generation\n";
- my ($job_name,$to)=@_; # 'prj_AGHucho' , [ 'C4123' ]
+ my ($job_name, $to) = @_; # 'prj_AGHucho' , [ 'C4123' ]
upid_register_self();
- move_new($job_name,$to,'MOVE ');
+ move_new($job_name, $to, 'MOVE ');
}
sub cmd_do_jobs {
@@ -1557,85 +1529,81 @@ sub cmd_do_jobs {
do_jobs($ARGV[0]);
upid_deregister_self();
}
+
sub cmd_amd_scan {
amd_scan(1);
}
sub cmd_kill {
- @ARGV>=1 or die "usage: $0 kill pid...\n";
+ @ARGV >= 1 or die "usage: $0 kill pid...\n";
for my $pid (@ARGV) {
upid_kill($pid);
}
}
-our $READDIR_CACHED_DIR='';
+our $READDIR_CACHED_DIR = '';
our %READDIR_CACHED_ENTRIES;
sub file_exists_cached {
- my ($dir,$fn)=@_;
+ my ($dir, $fn)=@_;
if ($dir ne $READDIR_CACHED_DIR) {
- $READDIR_CACHED_DIR=$dir;
- %READDIR_CACHED_ENTRIES=();
+ $READDIR_CACHED_DIR = $dir;
+ %READDIR_CACHED_ENTRIES = ();
opendir D,$dir or return 0;
- $READDIR_CACHED_ENTRIES{$_}=1 for readdir D;
+ $READDIR_CACHED_ENTRIES{$_} = 1 for readdir D;
closedir D;
}
return exists $READDIR_CACHED_ENTRIES{$fn};
}
sub cmd_bfix {
- @ARGV==1 or die USAGE;
- my ($job_name)=@ARGV;
-
- my $sth=$dbh->prepare(<<'EOF');
-SELECT stat.rowid,stat_started,path_path,volume_path,job_name,volume_id
-FROM stat,job,volume,path
-WHERE stat_job_id=job_id
-AND stat_volume_id=volume_id
-AND job_path_id=path_id
-AND job_name=?
+ @ARGV == 1 or die USAGE;
+ my ($job_name) = @ARGV;
+
+ my $sth = $dbh->prepare(<<'EOF');
+SELECT stat.rowid, stat_started, path_path, volume_path, job_name, volume_id
+FROM stat, job, volume, path
+WHERE stat_job_id = job_id
+AND stat_volume_id = volume_id
+AND job_path_id = path_id
+AND job_name = ?
ORDER BY stat_started
EOF
$sth->execute($job_name);
- while (my $row=$sth->fetchrow_arrayref) {
- my ($stat_rowid,$stat_started,$path_path,$volume_path,$job_name,$volume_id)=@$row;
+ while (my $row = $sth->fetchrow_arrayref) {
+ my ($stat_rowid, $stat_started, $path_path, $volume_path, $job_name, $volume_id) = @$row;
- my $run_name=run_name($job_name,$stat_started,$path_path);
-
-# unless (file_exists_cached("$volume_path/$job_name",$run_name)) {
-# $e->("$volume_path/$job_name/$run_name : missing");
-
- printf "stat.rowid %-6d volume_id %-2d %s\n",$stat_rowid,$volume_id," $volume_path/$job_name/$run_name";
+ my $run_name = run_name($job_name, $stat_started, $path_path);
+ printf "stat.rowid %-6d volume_id %-2d %s\n", $stat_rowid, $volume_id, " $volume_path/$job_name/$run_name";
}
print "\n(command : pbackup sql 'update stat set stat_volume_id=VVV where rowid=RRR'\n";
-
}
sub cmd_verify {
upid_register_self();
upid_doing("VERIFY");
- my $err=0;
+ my $err = 0;
- my $e=sub {
+ my $e = sub {
$err++;
print "\n\t$_[0]";
};
- $|=1;
+ $| = 1;
print "every run has a directory ... ";
for (@{$dbh->selectall_arrayref(<<'EOF')}) {
-SELECT job_name,stat_started,path_path,volume_path
-FROM job,stat,path,volume
-WHERE stat_job_id=job_id AND job_path_id=path_id AND stat_volume_id=volume_id
+SELECT job_name, stat_started, path_path, volume_path
+FROM job, stat, path, volume
+WHERE stat_job_id = job_id AND job_path_id = path_id AND stat_volume_id = volume_id
ORDER by job_name
EOF
- my ($job_name,$stat_started,$path_path,$volume_path)=@$_;
- my $run_name=run_name($job_name,$stat_started,$path_path);
- unless (file_exists_cached("$volume_path/$job_name",$run_name)) {
+ my ($job_name, $stat_started, $path_path, $volume_path) = @$_;
+ my $run_name = run_name($job_name, $stat_started, $path_path);
+ unless (file_exists_cached("$volume_path/$job_name", $run_name)) {
$e->("$volume_path/$job_name/$run_name : missing");
}
}
@@ -1643,7 +1611,7 @@ EOF
print "every run directory is known ...";
for my $datadir (datadirs()) {
- my ($volume_id)=$dbh->selectrow_array('SELECT volume_id FROM volume WHERE volume_path=?',undef,$datadir);
+ my ($volume_id) = $dbh->selectrow_array('SELECT volume_id FROM volume WHERE volume_path = ?', undef, $datadir);
unless (defined $volume_id) {
$e->("datadir $datadir has no volume_id");
next;
@@ -1653,27 +1621,25 @@ EOF
$e->("$datadir: $!");
next;
}
- opendir D,'.';
- while ( defined (my $jobdir=readdir D) ) {
- $jobdir=~/^\.\.?$/ and next;
+ opendir D, '.';
+ while (defined (my $jobdir = readdir D)) {
+ $jobdir =~ /^\.\.?$/ and next;
if ($jobdir =~ /\.BEING_DELETED$/) {
$e->("$datadir/$jobdir : should be deleted");
next;
}
- ##print "$datadir/$jobdir ";
- unless (opendir R,$jobdir) {
+ unless (opendir R, $jobdir) {
$e->("$datadir/$jobdir: $!");
next;
}
- my $run_cnt=0;
- while ( defined (my $rundir=readdir R) ) {
- $rundir=~/^\.\.?$/ and next;
+ my $run_cnt = 0;
+ while (defined (my $rundir = readdir R)) {
+ $rundir =~ /^\.\.?$/ and next;
$rundir eq 'last' and next;
- ##print ".";
- my ($job_name,$timetag,$fstag)=parse_run_name($rundir);
+ my ($job_name, $timetag, $fstag) = parse_run_name($rundir);
unless (defined ($job_name)) {
$e->("$datadir/$jobdir/$rundir : invalid name");
next;
@@ -1684,10 +1650,10 @@ EOF
}
if ($timetag eq 'INCOMPLETE') {
# is this a valid and enabled job which eventually will be continued ?
- my $ok=0;
- for my $path (@{$dbh->selectcol_arrayref('SELECT path_path FROM job,path WHERE job_path_id=path_id AND job_name=? AND job_enabled=1',undef,$job_name)}) {
+ my $ok = 0;
+ for my $path (@{$dbh->selectcol_arrayref('SELECT path_path FROM job, path WHERE job_path_id = path_id AND job_name = ? AND job_enabled = 1', undef, $job_name)}) {
fstag($path) ne $fstag and next;
- $ok=1;
+ $ok = 1;
last;
}
unless ($ok) {
@@ -1697,20 +1663,20 @@ EOF
}
next;
}
- my $started=tag2time($timetag);
+ my $started = tag2time($timetag);
unless (defined $started) {
$e->("$datadir/$jobdir/$rundir : invalid timetag");
next;
}
- my $ok=0;
- my $sth=$dbh->prepare('SELECT stat_started,path_path FROM job,path,stat WHERE job_path_id=path_id AND stat_job_id=job_id AND job_name=? AND stat_volume_id=?');
- $sth->execute($job_name,$volume_id);
- while (my $row=$sth->fetchrow_arrayref) {
- my ($this_started,$path)=@$row;
+ my $ok = 0;
+ my $sth = $dbh->prepare('SELECT stat_started, path_path FROM job, path, stat WHERE job_path_id = path_id AND stat_job_id = job_id AND job_name = ? AND stat_volume_id = ?');
+ $sth->execute($job_name, $volume_id);
+ while (my $row = $sth->fetchrow_arrayref) {
+ my ($this_started, $path) = @$row;
fstag($path) eq $fstag or next;
timetag($this_started) eq $timetag or next;
- $ok=1;
+ $ok = 1;
last;
}
unless ($ok) {
@@ -1719,7 +1685,6 @@ EOF
$run_cnt++; # we count this as valid
}
}
- ##print "\n";
$run_cnt or $e->("$datadir/$jobdir : no valid runs");
}
}
@@ -1727,48 +1692,48 @@ EOF
}
sub purge_zombie_job {
- my ($serverpath)=@_; # god:/amd/god/X/X2042/project/toxdbweb
- my ($path_server,$path_path)=$serverpath=~/^([^:]+):(\S+)$/;
+ my ($serverpath) = @_; # god:/amd/god/X/X2042/project/toxdbweb
+ my ($path_server, $path_path) = $serverpath =~ /^([^:]+):(\S+)$/;
upid_register_self();
$dbh->begin_work();
- my ($job_id,$job_name,$job_enabled,$count_count) = $dbh->selectrow_array("SELECT job_id,job_name,job_enabled,count_count FROM job,path,stat,($COUNT_SQL) WHERE path_server=? AND path_path=? AND job_path_id=path_id AND stat_job_id=job_id AND stat_job_id=count_job_id",undef,$path_server,$path_path);
+ my ($job_id, $job_name, $job_enabled, $count_count) = $dbh->selectrow_array("SELECT job_id, job_name, job_enabled, count_count FROM job, path, stat, ($COUNT_SQL) WHERE path_server = ? AND path_path = ? AND job_path_id = path_id AND stat_job_id = job_id AND stat_job_id = count_job_id", undef, $path_server, $path_path);
defined $job_id or die "$serverpath: unknown in backup\n";
$job_enabled and die "$serverpath: job is enabled\n";
$count_count !=1 and die "$serverpath: $count_count instances\n";
- my ($stat_rowid,$stat_started,$volume_path) = $dbh->selectrow_array("SELECT stat.rowid,stat_started,volume_path FROM stat,volume where stat_job_id=? AND stat_volume_id=volume_id",undef,$job_id);
+ my ($stat_rowid, $stat_started, $volume_path) = $dbh->selectrow_array("SELECT stat.rowid, stat_started, volume_path FROM stat, volume where stat_job_id = ? AND stat_volume_id = volume_id", undef, $job_id);
- lck_can_lock("JOB.$job_name",LCK_EX) or die "job is locked\n";
- lck_can_lock("RUN.$stat_rowid",LCK_EX) or die "run is locked\n";
- lck_lock("JOB.$job_name",LCK_EX);
- lck_lock("RUN.$stat_rowid",LCK_EX);
+ lck_can_lock("JOB.$job_name", LCK_EX) or die "job is locked\n";
+ lck_can_lock("RUN.$stat_rowid", LCK_EX) or die "run is locked\n";
+ lck_lock("JOB.$job_name", LCK_EX);
+ lck_lock("RUN.$stat_rowid", LCK_EX);
- my $timetag=timetag($stat_started);
- my $fstag=fstag($path_path);
- my $dir="$volume_path/$job_name/$job_name:$timetag:$fstag";
+ my $timetag = timetag($stat_started);
+ my $fstag = fstag($path_path);
+ my $dir = "$volume_path/$job_name/$job_name:$timetag:$fstag";
-d $dir and system "mv $dir $dir.EXPIRED" and exit 1;
- $dbh->do('DELETE FROM stat WHERE rowid=?',undef,$stat_rowid);
+ $dbh->do('DELETE FROM stat WHERE rowid = ?', undef, $stat_rowid);
upid_doing("PURGE rm -rf $dir.EXPIRED");
$dbh->commit();
- -d "$dir.EXPIRED" and system "rm -rf $dir.EXPIRED" and exit 1;
+ -d "$dir.EXPIRED" and system "rm -rf $dir.EXPIRED" and exit 1;
unlink("$volume_path/$job_name/last");
rmdir("$volume_path/$job_name");
- lck_unlock("JOB.$job_name",LCK_EX);
- lck_unlock("RUN.$stat_rowid",LCK_EX);
+ lck_unlock("JOB.$job_name", LCK_EX);
+ lck_unlock("RUN.$stat_rowid", LCK_EX);
upid_deregister_self();
}
sub cmd_purge {
- @ARGV>=1 or die USAGE;
- my ($what)=$ARGV[0];
+ @ARGV >= 1 or die USAGE;
+ my ($what) = $ARGV[0];
if ($what eq 'zombie') {
- @ARGV==2 or die USAGE;
+ @ARGV == 2 or die USAGE;
purge_zombie_job($ARGV[1]);
} else {
die USAGE;
@@ -1776,14 +1741,14 @@ sub cmd_purge {
}
sub db_init {
- my ($cnt)=$dbh->selectrow_array('SELECT COUNT(*) FROM global');
+ my ($cnt) = $dbh->selectrow_array('SELECT COUNT(*) FROM global');
unless ($cnt) {
warn "initialze empty database\n";
$dbh->do('INSERT INTO global DEFAULT VALUES');
}
}
-@ARGV>=1 or die USAGE;
+@ARGV >= 1 or die USAGE;
my $cmd=shift;
# This command doesn't need local pbackup installation and database