mxproxmox

#! /usr/local/system/perl/bin/perl
use strict;
use warnings;
use JSON::PP;
use LWP::UserAgent;
use Data::Dumper;

sub USAGE {
    <<__EOF__;
usage: $0 cmd

    $0 backup ident path
    $0 local-backups      : run local backups (if needed)
    $0 api path           : get from path, dump result
    $0 client [cmd...]    : call proxmox-backup-client (add key file if required)
    $0 tclient [cmd...]   : same with workaround for reiserfs /tmp
    $0 active-jobs        : list jobs active on server
    $0 check-pbackup      : cross-check pbackup and mxproxmox jobs

Examples:

    $0 backup sys_theinternet /
    $0 backup home_buczek     /home/buczek"
__EOF__
}

our $PBS_BACKUP_KEY_FILE = "/etc/mxproxmox/proxmox-backup.key";
our $PBS_CRED_FILE = "/etc/mxproxmox/proxmox-backup.credentials";
our $PBS_PASSWORD;

$ENV{'PBS_REPOSITORY'} = 'mariux64@pbs!backup@nummer5.molgen.mpg.de:zpool2';
$ENV{'PBS_FINGERPRINT'} = '69:39:96:c0:ce:a0:02:00:a3:d9:17:62:35:44:0d:8f:f0:43:6f:7d:a9:9d:9a:3e:4b:94:de:e2:53:43:84:02';

sub read_cred_file {
    open my $fn, '<', $PBS_CRED_FILE or die "$PBS_CRED_FILE: $!\n";
    $PBS_PASSWORD = join('', <$fn>);
    chomp($PBS_PASSWORD);
    $ENV{'PBS_PASSWORD'} = $PBS_PASSWORD;
}

my $hostname = `hostname -s`;chomp $hostname;

sub sys {
    print join(' ', @_), "\n";
    system @_;
    $? and exit 1;
}

sub save_qx {
    open my $in, '-|', @_ or die "$!\n";
    my $ret=join ('', <$in>);
    close $in or die "$!\n";
    @? and exit 1;
    chomp($ret);
    return $ret;
}

sub do_map {
    my ($prefix, $mapname, $local_only) = @_;
    my @out;
    open my $in, "/etc/automount/$mapname" or die "/etc/automount/$mapname: $!\n";
    while (<$in>) {
        s /#.*//;
        my @f = split " ";
        @f or next;
        @f = grep !/^-/, @f;
        @f == 2 or next;         # key hostname:path
        my ($key, $hostname_and_path)=@f;
        @f = split ':', $hostname_and_path;
        @f == 2 or next;
        my ($host, $path) = @f;
        if (!$local_only || $host eq $hostname) {
            push @out, [ "${prefix}_$key", $path ];
        }
    }
    return (@out);
}

our $ua;
sub init_ua {
    $ua = new LWP::UserAgent( ssl_opts => { verify_hostname => 0 } );
    $ua->default_header( Authorization => 'PBSAPIToken mariux64@pbs!backup:'.$PBS_PASSWORD );
}

sub get {
    my ($path) = @_;
    my $response = $ua->get("https://nummer5.molgen.mpg.de:8007/api2/json/$path");
    $response->is_success or die $response->status_line;
    my $ref = decode_json($response->decoded_content)
}

sub cmd_api {
        my ($path)=@_;
        my $r = get($path);
        print Dumper(\$r);
}

sub server_available_bytes {
    my $r = get('status/datastore-usage');
    my $avail = $r->{data}[0]{avail};
}


# return true if there are less than 3 jobs running on the server
#
sub server_has_free_slots {
    my $r = get('nodes/localhost/tasks?running=1');
    return @{$r->{data}} < 3;
    for my $j ( @{$r->{data}} ) {
        print "active job: ",$j->{worker_id},"\n";
    }
}

sub cmd_active_jobs {
    my $r = get('nodes/localhost/tasks?running=1');
    if ( @{$r->{data}} ) {
        for my $j (@{$r->{data}} ) {
            print $j->{worker_id},"\n";
        }
    } else {
        print "no active jobs\n";
    }
}

sub get_snapshots {
    my ($ident) = @_;
    my $uri = URI->new('https://nummer5.molgen.mpg.de:8007/api2/json/admin/datastore/zpool2/snapshots');
    $uri->query_form('backup-id' => $ident);
    my $response = $ua->get($uri);
    $response->is_success or die $response->status_line;
    my $o = decode_json($response->decoded_content);
    my @times;
    for (@{$o->{data}}) {
        push @times, $_->{'backup-time'};
    }
    @times = sort { $b <=> $a } @times;
    return @times;
}

#
# return time of latest snapshot or 0 if none
#
sub latest_snapshot {
    my ($ident) = @_;
    my @snapshots = get_snapshots($ident);
    return @snapshots ? $snapshots[0] : 0;
}

sub latest_snapshot_age {
    my ($ident) =  @_;
    return time-latest_snapshot($ident);
}

sub cmd_backup {
    my ($ident, $path, $limit) =  @_;
    lstat $path;
    -e _ or die "$path: no such file or directory \n";
    -d _ or die "$path: not a directory \n";
    my $filesystem = save_qx('stat', '-f',  '-c', '%T', $path);
    $filesystem =~ /^(xfs|reiserfs)$/ or die "$path: filesystem $filesystem not known to be local\n";
    $> == 0 or die "Please be root\n";

    defined $limit or $limit = 1 * 1024*1024*1024;

    my @f = split " ",`du -ksx $path`;
    $? and exit 1;
    my $kb = $f[0];

    if ($kb > $limit) {
        warn(sprintf("WARNING: $ident size %d kB over limit %d kB. Skipped.\n", $kb, $limit));
        return;
    }

    sys(
        'proxmox-backup-client', 'backup',
        '--keyfile', $PBS_BACKUP_KEY_FILE,
        '--backup-id', $ident,
        "$ident.pxar:$path"
    );
}

sub lock {
    my ($lockfilename) = @_;
    open my $lockfile,">", $lockfilename or die "$lockfilename: $!\n";
    my $sts = flock($lockfile, 2 + 4);
    unless ($sts) {
        if ($! == 11) {                      # EWOULDBLOCK
            warn "INFO: Backup already running\n";
            exit 0;                            # success. Its a "try if not already running" operation
        }
        die "$lockfilename: $! ".($!+0)."\n";
    }
    return $lockfile;
}

our %INCLUDE;

sub read_backup_list {
    open my $in, '<', '/etc/mxproxmox/backup-list.dat' or die "/etc/mxproxmox/backup-list.dat: $!\n";
    while (<$in>) {
        chomp;
        s/\s*#.*//;
        /\S/ or next;
        my ($project, $opt_limit) = split " ";
        $INCLUDE{$project} = $opt_limit;
    }
}

sub cmd_local_backups {
    my $lockfile = lock('/run/proxmox-client.lock');
    read_backup_list();
    my @jobs = ([ "sys_$hostname", "/" ]) ;
    for my $p (
            do_map('home', 'auto.home', 1),
            do_map('cfdl', 'auto.confidential', 1),
            do_map('prj',  'auto.project', 1),
            do_map('pkg',  'auto.package', 1),
            do_map('src',  'auto.src', 1),
    ) {
        my ($ident, $path) = @$p;
        push @jobs, $p if $ident =~ /^home_/;
        push @jobs, [$ident, $path, $INCLUDE{$ident}] if exists $INCLUDE{$ident};
        if ($ident eq 'prj_acrbkps') {
            push @jobs, [ 'acrb_adchef',    "$path/Tmp/ADChef" ];                    #  68G
            push @jobs, [ 'acrb_winhome',   "$path/Tmp/WINHOME" ];                   # 108G
            push @jobs, [ 'acrb_winhype-a', "$path/Tmp/WINHYPE-A" , 2684354560 ];    # 1.9T -> 2.5T -> 2684354560k
            push @jobs, [ 'acrb_winhype-c', "$path/Tmp/WINHYPE-c" , 6979321856 ];    # 5.4T -> 6.5T -> 6979321856k
            push @jobs, [ 'acrb_wincert',   "$path/Tmp/WinCert" ];                   #  19G
            push @jobs, [ 'acrb_winsccm',   "$path/Tmp/WinSCCM" ];                   # 680G
        }

    }
    for my $p (@jobs) {
        my ($ident, $path, $opt_limit) = @$p;
        if (latest_snapshot_age($ident) < 7*24*60*60) {
            warn "INFO: $ident younger than 7 days. Skipped.\n";
            next;
        }
        while (1) {
            if ( server_available_bytes() < 1*1024*1024*1024*1024 ) {
                warn "waiting for free space on server\n";
                sleep 60;
                redo;
            }
            unless (server_has_free_slots()) {
                warn "waiting for free slots on server\n";
                sleep 60;
                redo;
            }
            last;
        }
        eval {
            cmd_backup($ident, $path, $opt_limit);
        };
        $@ and warn "$@\n";
    }
}

sub cmd_check_pbackup {

    read_backup_list();

    my %jobs;
    for my $p (
        do_map('home', 'auto.home', 0),
        do_map('cfdl', 'auto.confidential', 0),
        do_map('prj',  'auto.project', 0),
        do_map('pkg',  'auto.package', 0),
        do_map('src',  'auto.src', 0),
    ) {
        my ($ident, $path) = @$p;
        $jobs{$ident} = $path if $ident =~ /^home_/;
        $jobs{$ident} = $path if exists $INCLUDE{$ident};
    }

    my %pbackup_jobs;
    open my $pipe, '-|', '/project/admin/pbackup/bin/pbackup', 'parse-jobs' or die "/project/admin/pbackup/bin/pbackup: $!\n";
    for (<$pipe>) {
        chomp;
        my ($ident, $path) = split " ", $_, 2;
        next if $ident =~ /^sys_/;
        $path =~ s/^[a-z0-9]+://i;
        $pbackup_jobs{$ident} = $path;
    }
    close $pipe or exit 1;

    print "\n\n# The following output was generated by `mxproxmox check-pbackup`:\n\n";

    for my $ident (sort keys %jobs) {
        if (!exists $pbackup_jobs{$ident}) {
            print "# consider to remove $ident, because it is not a a pbackup job\n";
        }
    }
    print "\n";

    for my $ident (sort keys %pbackup_jobs) {
        if (exists $jobs{$ident}) {
            if ($pbackup_jobs{$ident} ne $jobs{$ident}) {
                warn "$ident: path differ. pbackup: ", $pbackup_jobs{$ident}, " proxmox: ", $jobs{$ident}, "\n";
            }
        } else {
            printf "# %-30s # consider to add, because its a pbackup job\n", $ident;
        }
    }
}

sub cmd_client {
    my @cmd = ('proxmox-backup-client', @_);
    if ( $_[0] =~ /^(catalog|restore|mount|map|backup)$/ ) {
        push @cmd, '--keyfile', $PBS_BACKUP_KEY_FILE;
    }
    sys(@cmd);
}

sub cmd_tclient {
    my @cmd = ('proxmox-backup-client.wrapper', @_);
    if ( $_[0] =~ /^(catalog|restore|mount|map|backup)$/ ) {
        push @cmd, '--keyfile', $PBS_BACKUP_KEY_FILE;
    }
    sys(@cmd);
}

@ARGV >= 1 or die USAGE;
my ($cmd,@rest) = shift;
if ($cmd eq 'api') {
    @ARGV==1 or die USAGE;
    read_cred_file();
    init_ua();
    cmd_api(@ARGV);
} elsif ($cmd eq 'local-backups') {
    @ARGV==0 or die USAGE;
    read_cred_file();
    init_ua();
    cmd_local_backups(@ARGV);
} elsif ($cmd eq 'backup') {
    @ARGV==2 or die USAGE;
    read_cred_file();
    cmd_backup(@ARGV);
} elsif ($cmd eq 'client') {
    read_cred_file();
    cmd_client(@ARGV);
} elsif ($cmd eq 'tclient') {
    read_cred_file();
    cmd_tclient(@ARGV);
} elsif ($cmd eq 'active-jobs') {
    @ARGV==0 or die USAGE;
    read_cred_file();
    init_ua();
    cmd_active_jobs(@ARGV);
} elsif ($cmd eq 'check-pbackup') {
    @ARGV==0 or die USAGE;
    cmd_check_pbackup(@ARGV);
} else {
    die USAGE;
}