diff --git a/mxmirror/mxmirror b/mxmirror/mxmirror new file mode 100755 index 0000000..999d263 --- /dev/null +++ b/mxmirror/mxmirror @@ -0,0 +1,1092 @@ +#!/usr/bin/perl + +# +# original is in /src/mariux/mariux/mxmirror/mxmirror +# + +use Socket; +use Sys::Hostname; +use Data::Dumper; +use Getopt::Long; + +use Date::Calc; + +use strict; +use warnings; + +$Data::Dumper::Sortkeys = "macheteinfach"; + + +#print Dumper $t1, $t2, $t3, $t4, $dt12, $dt21, $dt23, $dt34; + + + +my $JBODMAP='/etc/amd/amd.jbod'; +my $MIRRORMAP='/etc/mxmirrors'; + +my %CONFIG = ( + CMD => 'STRING', + START => '21:00', + END => '07:00', + STEP => '00:15' + ); + +my %opt = ( + from => undef, + to => undef, + check => undef, + verbose => 0, + debug => 0, + + ); + +my $fullhostname = hostname; +my ($hostname) = $fullhostname =~ /^(.*?)\./; +my $hostip = inet_ntoa(scalar gethostbyname($fullhostname)); + +my $jbodmap = read_jbodmap($JBODMAP); +my $mirrors = read_mirrormap($MIRRORMAP); +#my $timeslots = CalcTimeslots($mirrors); +my $timeslots; + +#print Dumper GroupMirrors($mirrors); + +#exit; + +sub GroupMirrors { + my $list = shift; + + my $nrofslots = 100; + + my @slots = (); + + my $nlist = {}; + + my @hosts = sort keys %{$list}; + + my $grpptr = {}; + + my $groups = []; + + my $ngrps=0; + + + foreach my $host (@hosts) { + my @dest = @{$list->{$host}}; + foreach my $dest (@dest) { + my $dhost = $dest->{destination}->{host}; + my ($grp, $dgrp); + + if(not exists $grpptr->{$host}) { + if(not exists $grpptr->{$dhost}) { + $grp = [ ]; + $grpptr->{$dhost} = $grp; + push @{$groups}, $grp; + } else { + $grp = $grpptr->{$dhost}; + } + $grpptr->{$host} = $grp; + } else { + $grp = $grpptr->{$host}; + + if(exists $grpptr->{$dhost}) { + $dgrp = $grpptr->{$dhost}; + + # merge groups if different + if($grp ne $dgrp) { + push @{$grp}, @{$dgrp}; + @{$groups} = grep { $_ ne $dgrp } @{$groups}; + } + } + $grpptr->{$dhost} = $grp; + } + + push @{$grp}, $dest; + + } + } + + print Dumper $grpptr; +# exit; + + for(my $i=0; $i < @{$groups}; $i++) { + print "group $i\n"; + + my $grp = $groups->[$i]; + + my %dummy = (); + + foreach my $m (@{$grp}) { + my $host = $m->{source}{host}; + my $dhost = $m->{destination}{host}; + push @{$dummy{"$host,$dhost"}}, $m; + print " $host -> $dhost\n"; + } + + + print Dumper \%dummy; + + @{$grp} = sort { $a->[0]{source}{host} cmp $b->[0]{source}{host} } values %dummy; +# @{$grp} = map { $dummy{$_} } sort keys %dummy; + } + + print Dumper $groups; + + exit; + return $groups; + +} + +sub GroupMirrors3 { + my $list = shift; + + my $nrofslots = 100; + + my @slots = (); + + my $nlist = {}; + + my @hosts = sort keys %{$list}; + + foreach my $host (@hosts) { + my @dest = @{$list->{$host}}; + foreach my $dest (@dest) { + my $dhost = $dest->{destination}->{host}; + + if(not exists $nlist->{$host}{$dhost} and $host ne $dhost) { + $nlist->{$host}{__cnt}++; + $nlist->{$dhost}{__cnt}++; + push @{$nlist->{$dhost}{__hosts}}, $host; + + + } + + $nlist->{$host}{$dhost} = 1; + +# push @{$nlist->{$host}{$dhost}{to}}, $dest; + } + } + + + foreach my $host (sort {$nlist->{$b}{__cnt} <=> $nlist->{$a}{__cnt} || $a cmp $b} keys %{$nlist}) { + + + my $cnt = $nlist->{$host}{__cnt}; + print "$host $cnt\n"; + my $ds = $cnt?$nrofslots/$cnt:1; + + my $s = 0; + + print "XXXX $host $nlist->{$host}{__cnt} $ds\n"; + + for($s=0; defined $slots[$s] and $s < $nrofslots; $s++) { + + } + + print " => first free slot: $s\n"; + + for(my $i=0; $i < $cnt; $i++) { + $slots[$s] = $host; + print " -> $s : $host\n"; + $s += $ds; + } + } + + print "RETURN.."; + return $nlist; + + +} +sub GroupMirrors2 { + my $list = shift; + + my $a = {}; + + my @hosts = sort keys %{$list}; + foreach my $h (@hosts) { + + unless(exists $a->{$h}) { + $a->{$h} = { + name => $h, + destinations => [], + sources => [] + } + } + my $this = $a->{$h}; + + foreach(@{$list->{$h}}) { + + my $d = $_->{destination}->{host}; + + unless(exists $a->{$d}) { + $a->{$d} = { + name => $d, + destinations => [], + sources => [] + } + } + + my $that = $a->{$d}; + + push @{$this->{destinations}}, $that; + push @{$that->{sources}}, $this; + + } + } + + foreach(keys %$a) { + unless(@{$a->{$_}->{destinations}}) { + printf "ENDPOINT: $_\n"; + delete $a->{$_} + } + } + + my $grps = {}; + + foreach(keys %$a) { + next unless defined $a->{$_}; + + my @g = check_a($a->{$_}); + + my @group = (); + + map { push @group, $a->{$_}; delete $a->{$_} } @g; + +# print "@g\n"; +# print Dumper \@group; + + $grps->{"@g"}->{group} = [ @group ]; + + } + + foreach my $g (keys %$grps) { + my $dest = {}; + foreach my $host (split / /, $g) { + next unless exists $list->{$host}; + printf "$g :: $host\n"; + foreach my $m (@{$list->{$host}}) { + $dest->{"$host $m->{destination}->{host}"} = [ $host, $m->{destination}->{host} ]; + + if($host eq $m->{destination}->{host}) { + $dest->{$host}->{"self-$m->{destination}->{host}"}->{self} = $host; + } else { + $dest->{$host}->{"to-$m->{destination}->{host}"}->{to} = $m->{destination}->{host}; + $dest->{$m->{destination}->{host}}->{"from-$host"}->{from} = $host; + } + } + } + + + + print Dumper $dest; + + } + + + + return $grps; + +} + + +sub check_a { + my $a = shift; + my $href = shift || {}; + + return if(exists $href->{$a->{name}}); + +# printf "checking $a->{name} .. \n"; + + $href->{$a->{name}} = 1; + + foreach(@{$a->{destinations}}, @{$a->{sources}}) { + next if(exists $href->{$_->{name}}); + check_a($_, $href); + } + + + + return sort keys %$href; + +} + +#print Dumper $timeslots; +#exit; + + +#print Dumper $mirrors; + +my $result = GetOptions( + "from:s" => \$opt{from}, + "to:s" => \$opt{to}, + "destinationhosts" => \$opt{destinationhosts}, + "sourcehosts" => \$opt{sourcehosts}, + "all" => \$opt{all}, + "quiet" => \$opt{quiet}, + "format|fmt:s" => \$opt{format}, + "execute" => \$opt{execute}, + "command|cmd:s" => \$opt{command}, + "check:s" => \$opt{check}, + "verbose" => \$opt{verbose}, + "debug" => \$opt{debug}, + "help" => \$opt{help} + ); + +if($opt{help}) { + print <<"EOF"; + +$0 [options] [action] + + actions: + + print mirror-cmds to mirror local filesystems + + --sourcehosts list all hosts to mirror + --destinationhosts list all mirror hosts + + --all list all defined mirrors + + --check check mirrorstatus of localhost + + options: + + --format= define output format + --execute execute if is not specified + + --command= execute for every host listed + + --config= read mirror-definitions from + --jbodmap= read amd-map for jbods from + + +EOF + exit 0; +} + + + +if(defined $opt{destinationhosts}) { + exit print_and_exec_hostlist(mirrors_reverse($mirrors)); +} + +if(defined $opt{sourcehosts}) { + exit print_and_exec_hostlist($mirrors); +} + +if(defined $opt{all}) { + exit print_and_exec_all($mirrors, $timeslots); +} + + + +if(defined $opt{check}) { + my $host = $opt{check} || $hostname; + + my $dest = mirrors_reverse($mirrors); + + foreach(@{$dest->{$host}}) { + printf "RULE: $_->{string}\n" if($opt{debug}); + print get_mirror_status($_); + printf " $_->{source}->{host}:$_->{source}->{hostpath} $_->{destination}->{host}:$_->{destination}->{hostpath} \t$_->{args}\n"; + } + + exit 0; +} + +my $err = 0; + +foreach(@{$mirrors->{$hostname}}) { + my $default_fmt = $CONFIG{CMD}; + my $fmt = (defined $opt{format})?$opt{format}:$default_fmt; + my $cmd = $opt{command}; + + if(not defined $cmd and $opt{execute}) { + $cmd = $fmt; + } + + if(not $fmt or (defined $cmd and not $cmd)) { + print "\n"; + print " default format: '$default_fmt'\n"; + print " default command: \n\n"; + print " format/command variables:\n"; + print " DSTHOST destination host\n"; + print " DSTPATH destination path\n"; + print " SRCHOST source host\n"; + print " SRCPATH source path\n"; + print " ARGS additional args\n"; + print " DESTINATION DSTHOST==SRCHOST : DSTHOST:DSTPATH \n"; + print " DSTHOST!=SRCHOST : DSTPATH \n\n"; + exit 1; + } + + print do_format($fmt, $_) . "\n"; + + if($cmd) { + my $command = do_format($cmd, $_); + if(system($command) != 0) { + print STDERR "mxmirror FAILED CMD: $command\n"; + $err++; + } + } +} + + +exit($err); + + +################################################################### + +sub print_and_exec_all { + my $list = shift; + my $ts = shift; + + my $default_fmt = "SRCHOST:SRCPATH DSTHOST:DSTPATH ARGS"; + my $fmt = (defined $opt{format})?$opt{format}:$default_fmt; + my $cmd = $opt{command}; + + if(not defined $cmd and $opt{execute}) { + $cmd = $fmt; + } + + if(not $fmt or (defined $cmd and not $cmd)) { + print "\n"; + print " default format: '$default_fmt'\n"; + print " default command: \n\n"; + print " format/command variables:\n"; + print " DSTHOST destination host\n"; + print " DSTPATH destination path\n"; + print " SRCHOST source host\n"; + print " SRCPATH source path\n"; + print " ARGS additional args\n"; + print " DESTINATION DSTHOST==SRCHOST : DSTHOST:DSTPATH \n"; + print " DSTHOST!=SRCHOST : DSTPATH \n\n"; + return 1; + } + + my @hosts = sort keys %{$list}; + + + foreach my $h (@hosts) { + my %vars = ( +# TIME => $ts->{hosts}->{$h}->{time_str} + ); + foreach(@{$list->{$h}}) { + + print do_format($fmt, $_, \%vars) . "\n" if($opt{format} or not $opt{command}); + + if($cmd) { + my $command = do_format($cmd, $_, \%vars); + print qx($command); + } + } + } + + return 0; +} + + + +sub print_and_exec_hostlist { + my $list = shift; + + my $default_fmt = "HOST"; + my $fmt = (defined $opt{format})?$opt{format}:$default_fmt; + my $cmd = $opt{command}; + + if(not defined $cmd and $opt{execute}) { + $cmd = $fmt; + } + + if(not $fmt or (defined $cmd and not $cmd)) { + print "\n"; + print " default format: '$default_fmt'\n"; + print " default command: \n\n"; + print " format/command variables:\n"; + print " HOST destination\n\n"; + return 1; + } + + foreach(sort keys %{$list}) { + print do_format($fmt, {}, {HOST => $_}) . "\n" if($opt{format} or not $opt{command}); + + if($cmd) { + my $command = do_format($cmd, {}, {HOST => $_}); + print qx($command); + } + } + + return 0; +} + + +sub mirrors_reverse { + my $mirrors = shift; + + my $r; + + foreach my $h (sort keys %{$mirrors}) { + foreach(@{$mirrors->{$h}}) { + push @{$r->{$_->{destination}->{host}}}, $_; + } + } + + return $r; +} + + +sub get_mirror_status { + my $mirror = shift; + + my $status; + my $days; + + return 'REMOTE' unless($hostname eq $mirror->{destination}->{host}); + + if(open(F, '<', "$mirror->{destination}->{hostpath}/.PMIRROR_STATUS")) { + my $line = ; + close F; + + ($status, $days) = $line =~ /^(\S+) (\d+) /; + + if($status eq 'OK') { + my ($year,$month,$day, $hour,$min,$sec, $doy,$dow,$dst) = Date::Calc::Gmtime($days); + + + $days = Date::Calc::Delta_Days($year,$month,$day, Date::Calc::Today()); + + return $days; + } + + + return $line; + + } + + return 'UNKNOWN'; +} + +#print Dumper \%opt; + +sub do_format { + my $fmt = shift; + my $data = shift; + my $vars = shift || {}; + + if(defined $data) { + my $dst = ($data->{source}->{host} eq $data->{destination}->{host})? + $data->{destination}->{hostpath} : + "$data->{destination}->{host}:$data->{destination}->{hostpath}"; + + $fmt =~ s/DESTINATION/$dst/g; + $fmt =~ s/SRCHOST/$data->{source}->{host}/g; + $fmt =~ s/DSTHOST/$data->{destination}->{host}/g; + $fmt =~ s/SRCPATH/$data->{source}->{hostpath}/g; + $fmt =~ s/DSTPATH/$data->{destination}->{hostpath}/g; + $fmt =~ s/ARGS/$data->{args}/g; + $fmt =~ s/STRING/$data->{string}/g; +# $fmt =~ s/TIME/$data->{timeslot}->{time_str}/g; + } + + foreach(keys %{$vars}) { + print "XXX" . $_; + $fmt =~ s/$_/$vars->{$_}/g; + } + + return $fmt; +} + + + +sub to_jbod_path { + my $host = shift; + my $jbod = shift; + my $path = shift; + + #return "/amd/${host}/X/${jbod}${prepend}${path}"; + + return "/jbod/${jbod}${path}"; +} + +sub to_amd_path { + my $host = shift; + my $partition = shift; + my $path = shift; + + return "/amd/${host}/${partition}${path}"; +} + +# from := : +# must be absolute or empty +# or must be set +# +# isset(): /amd// +# isempty(): + +# to := : +# must be absolute or empty +# or must be set +# +# isemtpy(): :/amd///mirror/(|/) +# isset(): :/amd// +# isempty(): : +# + + + +# from := +# /jbod/ + +# to := +# isset(): /jbod/ +# isjbod(): /jbod//mirror/ +# ishost(): /jbod//mirror// + +sub convert_to_sourcepath { + my $string = shift; + + my ($jbod, $path, $host, $hostpath, $partition); + + if(($jbod, $path) = $string =~ /^([XC]\d+)(.*?)$/) { + unless(defined $jbodmap->{$jbod}) { + printf STDERR "**ERROR: unknown jbod: $string"; + next; + } + + $host = $jbodmap->{$jbod}; + + $hostpath = to_jbod_path($host, $jbod, $path); + + return({ host => $host, + jbod => $jbod, + path => $path, + hostpath => $hostpath }); + } + + if(($host,$partition,$path) = $string =~ /^(.*?):(\d+)(.*?)$/) { + $hostpath = to_amd_path($host, $partition, $path); + + return({ host => $host, + partition => $partition, + path => $path, + hostpath => $hostpath }); + + } + + if(($host,$path) = $string =~ m(^(.*?):(/.+?)$) ) { + return({ host => $host, + path => $path, + hostpath => $path }); + + + + } + return; +} + +# from := : +# must be absolute or empty +# or must be set +# +# isset(): /amd// +# isempty(): +# +# to := : +# must be absolute or empty +# or must be set +# +# isemtpy(): :/amd///mirror/(|/) +# isset(): :/amd// +# isempty(): : +# +# from := +# /jbod/ +# +# to := +# isset(): /jbod/ +# isjbod(): /jbod//mirror/ +# ishost(): /jbod//mirror// + +sub convert_to_destinationpath { + my $string = shift; + my $source = shift; + + my ($jbod, $path, $host, $hostpath, $partition); + + if(($jbod, $path) = $string =~ /^([XC]\d+)(.*?)$/) { + unless(defined $jbodmap->{$jbod}) { + printf STDERR "**ERROR: unknown jbod: $string"; + next; + } + + $host = $jbodmap->{$jbod}; + + unless($path) { + if(defined $source->{jbod}) { + $path = "/mirror/$source->{jbod}$source->{path}"; + } elsif (defined $source->{partition}) { + $path = "/mirror/$source->{host}/$source->{partition}$source->{path}"; + } else { + $path = "/mirror/$source->{host}$source->{path}"; + } + } + + $hostpath = to_jbod_path($host, $jbod, $path); + + return({ host => $host, + jbod => $jbod, + path => $path, + hostpath => $hostpath }); + } + +# to := : +# must be absolute or empty +# or must be set +# +# isemtpy(): :/amd///mirror/(|/) +# isset(): :/amd// +# isempty(): : + + if(($host,$partition,$path) = $string =~ /^(.*?):(\d+)(.*?)$/) { + + + unless($path) { + if(defined $source->{jbod}) { + $path = "/mirror/$source->{jbod}$source->{path}"; + } else { + $path = "/mirror/$source->{host}/$source->{partition}$source->{path}"; + } + } + + $hostpath = to_amd_path($host, $partition, $path); + + return({ host => $host, + partition => $partition, + path => $path, + hostpath => $hostpath }); + + } + + if(($host,$path) = $string =~ m(^(.*?):(/.+?)$) ) { + return({ host => $host, + path => $path, + hostpath => $path }); + + } + + + + return; +} + + + + +#sub convert_to_path { +# my $string = shift; +# my $mirror = shift; +# +# my ($jbod, $path, $host, $hostpath, $partition); +# +# if(($jbod, $path) = $string =~ /^(X\d+)(.*?)$/) { +# unless(defined $jbodmap->{$jbod}) { +# printf STDERR "**ERROR: unknown jbod: $string"; +# next; +# } +# #sauerkraut:/amd/sauerkraut/X/X0053/mirror/balmer/1 +# +# $host = $jbodmap->{$jbod}; +# +# $hostpath = to_jbod_path($host, $jbod, ${path}, ${mirror}); +# +## printf " - jbod=${jbod} path=${path} => ${hostpath}\n"; +# +# return wantarray?($host, $hostpath):"${host}:${hostpath}"; +# } +# +# if(($host,$partition,$path) = $string =~ /^(.*?):(\d+)(.*?)$/) { +# $hostpath = to_amd_path($host, $partition, $path, $mirror); +# +## printf " - partition=${partition} path=${path} => ${hostpath}\n"; +# +# return wantarray?($host, $hostpath):"${host}:${hostpath}"; +# } +# +# +# +# +# return; +#} + +sub read_mirrormap { + my $file = shift; + + my $map = {}; + + my $string; + + my ($from, $to, $args, $host, $frompath, $topath); + + my ($source, $destination); + + open(F, '<', $file) or die "can't open $file: $?"; + + + + foreach() { + s/^\s+//; + s/\s+$//; + s/\s+/ /; + + next if($_ eq ""); + + if(/^#\s*(\S+)\s*=\s*(.*)/) { + $CONFIG{$1} = $2; + } + + next if(/^#/); + + $string = $_; + + if(($from, $to, $args) = /^(\S+) (\S+)\s*(.*)$/) { + + $source = convert_to_sourcepath($from); + $destination = convert_to_destinationpath($to, $source); + + + unless($source and $destination) { + print STDERR "**ERROR: CAN'T PARSE FROM/TO: $_\n"; + next; + } + + + push @{$map->{$source->{host}}}, {string => $string, source => $source, destination => $destination, args => $args}; + + } else { + print STDERR "**ERROR: CAN'T PARSE: $_\n"; + } + + } + + close F; + + return $map; +} + + + +sub read_jbodmap { + my $file = shift; + + open(F, '<', $file) or die "can't open $file: $?"; + + my $map = {}; + + foreach() { + next unless (m(/amd/(.*?)/[XC]/([XC]\d+))); + $map->{$2} = $1; + } + + close F; + + return $map; +} + + +sub DeltaTime { + my $t1 = shift; + my $t2 = shift; + + my $dm = TimeToSeconds($t2) - TimeToSeconds($t1); + + $dm = ($dm < 0) ? ($dm + 24*60*60) : $dm; + + my $dt = SecondsToTime($dm); + +# print Dumper ["delta", $t1, $t2, $dt, $dm]; + + return $dt; +} + +sub AddDeltaTime { + my $t1 = shift; + my $dt = shift; + + my $m = TimeToSeconds($t1) + TimeToSeconds($dt); + + my $t2 = SecondsToTime($m); + +# print Dumper ["add", $t1, $dt, $t2, $m]; + + return $t2; +} + +sub SubDeltaTime { + my $t1 = shift; + my $dt = shift; + + my $m = TimeToSeconds($t1) - TimeToSeconds($dt); + + my $t2 = SecondsToTime($m); + +# print Dumper ["add", $t1, $dt, $t2, $m]; + + return $t2; +} + + +sub TimeToMinutes { + my $t = shift; + my $m; + + $m = ($t->[0]*60 + $t->[1]) % (24*60); + +# print Dumper ["t2min", $t, $m]; + + return $m; + +} + +sub TimeToSeconds { + my $t = shift; + my $m; + + $m = ($t->[0]*60*60 + $t->[1]*60 + $t->[2]) % (24*60*60); + +# print Dumper ["t2min", $t, $m]; + + return $m; + +} + +sub SecondsToTime { + my $s = shift; + my $t = [0,0,0]; + + $t->[0] = int($s/(60*60)); + + $s -= ($t->[0] * 60*60); + + $t->[1] = int($s/60); + + $s -= ($t->[1] * 60); + + $t->[2] = $s; + + # this is my very first use of the %= statement EVER! + # YEAH! mx. + + $t->[0] %= 24; + +# print Dumper ["min2t", $m, $t]; + + return $t; + +} + + +sub MinutesToTime { + my $m = shift; + my $t = [0,0,0]; + + $t->[0] = int($m/60); + $t->[1] = $m - ($t->[0] * 60); + + # this is my very first use of the %= statement EVER! + # YEAH! mx. + + $t->[0] %= 24; + + +# print Dumper ["min2t", $m, $t]; + + return $t; + +} + +sub FormatTime { + my $t = shift; + + return sprintf("%02d:%02d:%02d", @{$t}); +} + + +sub ParseTime { + my $s = shift; + + my @t; + + if(@t = $s =~ /^(\d+):([0-5][0-9])$/) { + return [ @t, 0 ]; + } + + if(@t = $s =~ /^(\d+):([0-5][0-9]):([0-5][0-9])$/) { + return @t; + } + + warn "ParseTime invalid time '$s'"; + return [0,0]; +} + + + +sub CalcTimeslots { + my $list = shift; + + my @hosts = sort keys %{$list}; + + my $ts = {}; + my $t1 = ParseTime($CONFIG{START}); + my $t2 = ParseTime($CONFIG{END}); + my $step = ParseTime($CONFIG{STEP}); + + my $dt = DeltaTime($t1, $t2); + + my $s = TimeToSeconds($step); + my $dspan = SecondsToTime($s/2); + + $ts->{nslots} = int(TimeToSeconds($dt) / $s)+1; + $ts->{start} = $t1; + $ts->{end} = $t2; + $ts->{step} = $step; + $ts->{slots} = []; + $ts->{hosts} = {}; + + my $this = $ts->{start}; + + my $i = 0; + while($i++ < $ts->{nslots}) { + my $slot = {}; + + $slot->{time} = $this; + $slot->{timespan}->[0] = SubDeltaTime($this, $dspan); + $slot->{timespan}->[1] = AddDeltaTime($this, $dspan); + $slot->{time_str} = FormatTime($this); + + push @{$ts->{slots}}, $slot; + + $this = AddDeltaTime($this, $step); + } + + + $i = 0; + + foreach my $host (@hosts) { + my $slot = $ts->{slots}->[$i++ % $ts->{nslots}]; + + foreach(@{$list->{$host}}) { + $_->{timeslot} = $slot; + } + } + + return $ts; +} + + + +# read jbod map + +#eval $(cat /etc/amd/amd.jbod | sed -ne 's,^.*/amd/\(.*\)/X/\(X....\).*$,\2=\1,p') + + + +