From 44b957bda763cfac4209b72bf2fcc545bdda2621 Mon Sep 17 00:00:00 2001 From: Donald Buczek Date: Wed, 7 Dec 2022 08:24:44 +0100 Subject: [PATCH 1/7] mxproxmox: Add local_only argument to do_map --- mxproxmox/mxproxmox | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/mxproxmox/mxproxmox b/mxproxmox/mxproxmox index aafb077..57c0b40 100755 --- a/mxproxmox/mxproxmox +++ b/mxproxmox/mxproxmox @@ -57,7 +57,7 @@ sub save_qx { } sub do_map { - my ($prefix, $mapname) = @_; + my ($prefix, $mapname, $local_only) = @_; my @out; open my $in, "/etc/automount/$mapname" or die "/etc/automount/$mapname: $!\n"; while (<$in>) { @@ -70,7 +70,9 @@ sub do_map { @f = split ':', $hostname_and_path; @f == 2 or next; my ($host, $path) = @f; - push @out, [ "${prefix}_$key", $path ] if $host eq $hostname; + if (!$local_only || $host eq $hostname) { + push @out, [ "${prefix}_$key", $path ]; + } } return (@out); } @@ -210,12 +212,12 @@ sub cmd_local_backups { read_backup_list(); my @jobs = ([ "sys_$hostname", "/" ]) ; for my $p ( - do_map('home', 'auto.home'), - do_map('cfdl', 'auto.confidential'), - do_map('prj', 'auto.project'), - do_map('pkg', 'auto.package'), - do_map('cfdl', 'auto.confidential'), - do_map('src', 'auto.src'), + do_map('home', 'auto.home', 1), + do_map('cfdl', 'auto.confidential', 1), + do_map('prj', 'auto.project', 1), + do_map('pkg', 'auto.package', 1), + do_map('cfdl', 'auto.confidential', 1), + do_map('src', 'auto.src', 1), ) { my ($ident, $path) = @$p; push @jobs, $p if $ident =~ /^home_/; From 8d7e2ef79f500056287324050b1299f52cc78a26 Mon Sep 17 00:00:00 2001 From: Donald Buczek Date: Wed, 7 Dec 2022 08:40:11 +0100 Subject: [PATCH 2/7] mxproxmox: Move ua creation into subroutine We want to add a command which doesn't need to run with privileges to read the credential file. Initialization of the ua needs to read the credential file. Factor out the code to initialize the ua so that we can make that optional on a per-command basis. --- mxproxmox/mxproxmox | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/mxproxmox/mxproxmox b/mxproxmox/mxproxmox index 57c0b40..0a81bf7 100755 --- a/mxproxmox/mxproxmox +++ b/mxproxmox/mxproxmox @@ -37,7 +37,6 @@ sub read_cred_file { chomp($PBS_PASSWORD); $ENV{'PBS_PASSWORD'} = $PBS_PASSWORD; } -read_cred_file(); my $hostname = `hostname -s`;chomp $hostname; @@ -77,8 +76,11 @@ sub do_map { return (@out); } -my $ua = new LWP::UserAgent( ssl_opts => { verify_hostname => 0 } ); -$ua->default_header( Authorization => 'PBSAPIToken mariux64@pbs!backup:'.$PBS_PASSWORD ); +our $ua; +sub init_ua { + $ua = new LWP::UserAgent( ssl_opts => { verify_hostname => 0 } ); + $ua->default_header( Authorization => 'PBSAPIToken mariux64@pbs!backup:'.$PBS_PASSWORD ); +} sub get { my ($path) = @_; @@ -266,6 +268,9 @@ sub cmd_tclient { sys(@cmd); } +read_cred_file(); +init_ua(); + @ARGV >= 1 or die USAGE; my ($cmd,@rest) = shift; if ($cmd eq 'test') { From aa24bed99882df457d5e71db43da55a0725938ae Mon Sep 17 00:00:00 2001 From: Donald Buczek Date: Wed, 7 Dec 2022 08:42:56 +0100 Subject: [PATCH 3/7] mxproxmox: Remove empty command "test" --- mxproxmox/mxproxmox | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/mxproxmox/mxproxmox b/mxproxmox/mxproxmox index 0a81bf7..72913e9 100755 --- a/mxproxmox/mxproxmox +++ b/mxproxmox/mxproxmox @@ -11,7 +11,6 @@ usage: $0 cmd $0 backup ident path $0 local-backups : run local backups (if needed) - $0 test $0 api path : get from path, dump result $0 client [cmd...] : call proxmox-backup-client (add key file if required) $0 tclient [cmd...] : same with workaround for reiserfs /tmp @@ -151,9 +150,6 @@ sub latest_snapshot_age { return time-latest_snapshot($ident); } -sub cmd_test { -} - sub cmd_backup { my ($ident, $path, $limit) = @_; lstat $path; @@ -273,9 +269,7 @@ init_ua(); @ARGV >= 1 or die USAGE; my ($cmd,@rest) = shift; -if ($cmd eq 'test') { - cmd_test(@ARGV); -} elsif ($cmd eq 'api') { +if ($cmd eq 'api') { @ARGV==1 or die USAGE; cmd_api(@ARGV); } elsif ($cmd eq 'local-backups') { From 641f2ac47bded41e252ddc6437e2a58c6daa9280 Mon Sep 17 00:00:00 2001 From: Donald Buczek Date: Wed, 7 Dec 2022 08:50:38 +0100 Subject: [PATCH 4/7] mxproxmox: init credentials and api only as needed --- mxproxmox/mxproxmox | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/mxproxmox/mxproxmox b/mxproxmox/mxproxmox index 72913e9..a8437af 100755 --- a/mxproxmox/mxproxmox +++ b/mxproxmox/mxproxmox @@ -264,26 +264,32 @@ sub cmd_tclient { sys(@cmd); } -read_cred_file(); -init_ua(); - @ARGV >= 1 or die USAGE; my ($cmd,@rest) = shift; if ($cmd eq 'api') { @ARGV==1 or die USAGE; + read_cred_file(); + init_ua(); cmd_api(@ARGV); } elsif ($cmd eq 'local-backups') { @ARGV==0 or die USAGE; + read_cred_file(); + init_ua(); cmd_local_backups(@ARGV); } elsif ($cmd eq 'backup') { @ARGV==2 or die USAGE; + read_cred_file(); cmd_backup(@ARGV); } elsif ($cmd eq 'client') { + read_cred_file(); cmd_client(@ARGV); } elsif ($cmd eq 'tclient') { + read_cred_file(); cmd_tclient(@ARGV); } elsif ($cmd eq 'active-jobs') { @ARGV==0 or die USAGE; + read_cred_file(); + init_ua(); cmd_active_jobs(@ARGV); } else { die USAGE; From 1b5f675ba49843bbb72a539e8879c38faacc1972 Mon Sep 17 00:00:00 2001 From: Donald Buczek Date: Wed, 7 Dec 2022 09:08:33 +0100 Subject: [PATCH 5/7] mxproxmox: Add command `check-pbackup` Add new command `check-pbackup`, which makes suggestions, what jobs to add to /etc/mxproxmox/backup-list.dat based on pbackup configuration. The output consists of out-commented lines which can be added to /etc/mxproxmox/backup-list.dat and uncommented one by one to phase in new proxmox backup jobs. --- mxproxmox/mxproxmox | 53 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/mxproxmox/mxproxmox b/mxproxmox/mxproxmox index a8437af..f431a56 100755 --- a/mxproxmox/mxproxmox +++ b/mxproxmox/mxproxmox @@ -15,6 +15,7 @@ usage: $0 cmd $0 client [cmd...] : call proxmox-backup-client (add key file if required) $0 tclient [cmd...] : same with workaround for reiserfs /tmp $0 active-jobs : list jobs active on server + $0 check-pbackup : cross-check pbackup and mxproxmox jobs Examples: @@ -248,6 +249,55 @@ sub cmd_local_backups { } } +sub cmd_check_pbackup { + + read_backup_list(); + + my %jobs; + for my $p ( + do_map('home', 'auto.home', 0), + do_map('cfdl', 'auto.confidential', 0), + do_map('prj', 'auto.project', 0), + do_map('pkg', 'auto.package', 0), + do_map('cfdl', 'auto.confidential', 0), + do_map('src', 'auto.src', 0), + ) { + my ($ident, $path) = @$p; + $jobs{$ident} = $path if $ident =~ /^home_/; + $jobs{$ident} = $path if exists $INCLUDE{$ident}; + } + + my %pbackup_jobs; + open my $pipe, '-|', '/project/admin/pbackup/bin/pbackup', 'parse-jobs' or die "/project/admin/pbackup/bin/pbackup: $!\n"; + for (<$pipe>) { + chomp; + my ($ident, $path) = split " ", $_, 2; + next if $ident =~ /^sys_/; + $path =~ s/^[a-z0-9]+://i; + $pbackup_jobs{$ident} = $path; + } + close $pipe or exit 1; + + print "\n\n# The following output was generated by `mxproxmox check-pbackup`:\n\n"; + + for my $ident (sort keys %jobs) { + if (!exists $pbackup_jobs{$ident}) { + print "# consider to remove $ident, because it is not a a pbackup job\n"; + } + } + print "\n"; + + for my $ident (sort keys %pbackup_jobs) { + if (exists $jobs{$ident}) { + if ($pbackup_jobs{$ident} ne $jobs{$ident}) { + warn "$ident: path differ. pbackup: ", $pbackup_jobs{$ident}, " proxmox: ", $jobs{$ident}, "\n"; + } + } else { + printf "# %-30s # consider to add, because its a pbackup job\n", $ident; + } + } +} + sub cmd_client { my @cmd = ('proxmox-backup-client', @_); if ( $_[0] =~ /^(catalog|restore|mount|map|backup)$/ ) { @@ -291,6 +341,9 @@ if ($cmd eq 'api') { read_cred_file(); init_ua(); cmd_active_jobs(@ARGV); +} elsif ($cmd eq 'check-pbackup') { + @ARGV==0 or die USAGE; + cmd_check_pbackup(@ARGV); } else { die USAGE; } From 4cb9474a53872f23083b84e9cb5280939d30c9c6 Mon Sep 17 00:00:00 2001 From: Donald Buczek Date: Wed, 7 Dec 2022 11:51:00 +0100 Subject: [PATCH 6/7] mxproxmox: Remove redundant call to do_map for cfdl map --- mxproxmox/mxproxmox | 2 -- 1 file changed, 2 deletions(-) diff --git a/mxproxmox/mxproxmox b/mxproxmox/mxproxmox index f431a56..dc3cf0a 100755 --- a/mxproxmox/mxproxmox +++ b/mxproxmox/mxproxmox @@ -215,7 +215,6 @@ sub cmd_local_backups { do_map('cfdl', 'auto.confidential', 1), do_map('prj', 'auto.project', 1), do_map('pkg', 'auto.package', 1), - do_map('cfdl', 'auto.confidential', 1), do_map('src', 'auto.src', 1), ) { my ($ident, $path) = @$p; @@ -259,7 +258,6 @@ sub cmd_check_pbackup { do_map('cfdl', 'auto.confidential', 0), do_map('prj', 'auto.project', 0), do_map('pkg', 'auto.package', 0), - do_map('cfdl', 'auto.confidential', 0), do_map('src', 'auto.src', 0), ) { my ($ident, $path) = @$p; From bbd52311adeeab956f14e8e0e47b2285bab50603 Mon Sep 17 00:00:00 2001 From: Donald Buczek Date: Thu, 8 Dec 2022 19:31:34 +0100 Subject: [PATCH 7/7] mxproxmox: Remove trailing white space from backup-list lines Remove trailing whitespace from lines read from backup-list. If we keep trailing whitespace, the following split will keep the whitespace in $opt_limit. This is later checked with `if defined($limit)` , which is true. Still, the empty string will evaluate numerically to zero (with a warning). A limit zero is not what we want here. From `perldoc -f split`: When assigning to a list, if LIMIT is omitted, or zero, Perl supplies a LIMIT one larger than the number of variables in the list, to avoid unnecessary work This has this surprising effect: DB<1> x split " ","a " 0 'a' DB<2> @a = split " ","a " DB<3> x @s empty array DB<4> x @a 0 'a' DB<5> ($a, $b) = split " ","a " DB<6> x ($a, $b) 0 'a' 1 '' DB<7> --- mxproxmox/mxproxmox | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mxproxmox/mxproxmox b/mxproxmox/mxproxmox index dc3cf0a..49f9bf8 100755 --- a/mxproxmox/mxproxmox +++ b/mxproxmox/mxproxmox @@ -199,7 +199,7 @@ sub read_backup_list { open my $in, '<', '/etc/mxproxmox/backup-list.dat' or die "/etc/mxproxmox/backup-list.dat: $!\n"; while (<$in>) { chomp; - s/#.*//; + s/\s*#.*//; /\S/ or next; my ($project, $opt_limit) = split " "; $INCLUDE{$project} = $opt_limit;