mxraid_assemble

#! /usr/local/system/perl/bin/perl -w

use strict;

use Data::Dumper; $Data::Dumper::Sortkeys=1;
use Getopt::Std;  $Getopt::Std::STANDARD_HELP_VERSION=1;
sub HELP_MESSAGE{}
sub VERSION_MESSAGE{}
use Sys::Syslog;
use Time::HiRes qw(usleep);

import MxRaid::ConfData;
import MxRaid::HostData;
import MxRaid::Utils;
import MxRaid::Color;

my $VERSION = '1.1';

sub exec_usage {

  my ($prog) = $0 =~ m|([^/]+)$|;

  print <<"__HELP";

  $prog usage:

    $prog [options...]

    default is to print assembly commands when run w/o options.

    options:

      -a          assemble arrays when run as root
      -c          check database
      -d file     alternative database file
      -h          print this help and exit
      -l          list arrays on host, with -v will also show candidates
      -m          monochrome warnings (for the purists)
      -n label    print config records for new assemblies on host
      -q          be quiet
      -r l:d(:m)  config hints, if not a RAID 6 with 16 disks (level, disks, match)
      -v          be more verbose
      -V          print Version information and exit

    example (list state and assist in assembling a RAID1):

           mxraid_assemble -lv -n D0014 -r 1:2:flash


__HELP

  exit 0;
}

sub exec_version {
  my ($prog) = $0 =~ m|([^/]+)$|;
  print "  $prog $VERSION\n";
  print "  -h for usage\n\n";
  print "  Modules involved:\n";
  print "    MxRaid::ConfData $MxRaid::ConfData::VERSION\n";
  print "    MxRaid::HostData $MxRaid::HostData::VERSION\n";
  print "    MxRaid::Color    $MxRaid::Color::VERSION\n";
  print "\n";
  exit 0;
}

sub check_enclosures {

  # Slow down if known HBAs are in use, possibly there is a lazy enclosure attached.
  # This might be skipped if /proc/uptime is large enough?
  if (-e '/sys/module/aacraid/version' or -e '/sys/module/smartpqi/version') {
    usleep(0.3 * 1e6);
  }

  glob('/sys/class/enclosure/*') or return;

  my $cnt_0 = () = (glob('/sys/block/sd[a-z]'), glob('/sys/block/sd[a-z][a-z]'));
  usleep(0.3 * 1e6);
  my $cnt_1 = () = (glob('/sys/block/sd[a-z]'), glob('/sys/block/sd[a-z][a-z]'));

  if ( $cnt_0 != $cnt_1 ) { # then something still 'evolves', shout around and take a nap.
    my ($prog) = $0 =~ m|([^/]+)$|;
    openlog($prog,'pid','user');
    Sys::Syslog::setlogsock('unix'); # with 'native' we get EOLs in the logfile, option "noeol" doesn't work
    syslog('info', 'still discovering disks, diff is %d. Waiting 3 seconds ...', $cnt_1 - $cnt_0);
    sleep 3;
  }
}

my $ROOT=$<==0?1:0;

my %opts;
getopts('acd:hlmn:qr:vV', \%opts) or die "# ERROR: getopts failed, try -h.\n";  # Values in %opts
exec_usage if $opts{h};
exec_version if $opts{V};

my $VERBOSE=1;

$VERBOSE-=1 if $opts{q};
$VERBOSE+=1 if $opts{v};


my $MDADMCONF_DB =    '/etc/mxmd.conf';      # the 'database' with serials
if ($opts{d}) {
  my $db = $opts{d};
  if (-e $db) {
    $MDADMCONF_DB = $db;
  } else {
    die "# db file '$db' does not exist, stopped";
  }
}

my $MDADM_CONF_BASE = '/dev/shm/mdadm.conf'; # config for mdadm, created with information from above

my $MDADM_ASSEMBLE_OPTIONS = '';

my $cd = MxRaid::ConfData->new($MDADMCONF_DB); $cd->verbose($VERBOSE); $cd->load();
check_enclosures(); # idle a bit if enclosures are attached ...
my $hd = MxRaid::HostData->new($cd);           $hd->verbose($VERBOSE);
my $utils = MxRaid::Utils->new();

my $do_default_action = 1;

if ($opts{c}) {
  $do_default_action = 0;
  if ($VERBOSE >= 1) {
    printf "# Database file   :'%s'\n", $MDADMCONF_DB;
    printf "# Record count    : %d\n", $cd->number_of_db_records;
    printf "# Successor labels: %s\n", join(', ', sort(@{$cd->next_db_labels}));
    print  "\n";
  }
}

if ($opts{l}) {
  $do_default_action = 0;
  my @labels = sort @{$hd->raid_labels()};
  print "#  NOTE: no active raids found.\n" unless @labels;

  for my $label (@labels) {
    my %reg;
    print "$label:\n";
    for my $rec (@{$hd->configured_raids()->{$label}}) {
      my $md = $hd->mddev_by_member()->{$rec->[0]};
      $md = 'n/a' unless defined $md;
      $reg{$md}+=1;
      printf "     %-5s %-9s %-16s '%s'\n", $md, @$rec[0..2];
    }
    print "  Note: this doesn't look like a standard configuration.\n" if scalar keys %reg != 1;
    print "\n";
  }
  if ($VERBOSE >= 2) {
    my @mounts = split m/\n/, `cat /proc/self/mounts`;
    print "Disks not part of a configured SW-RAID:\n";
    for my $rec (@{$hd->non_raid_disks()}) {
      my $mnt = scalar(grep {m|^/dev/$rec->[0]\d*\b|} @mounts) ? ' (mounted)':'';
      printf "           %-9s %-16s '%s'%s\n", @$rec[0..2], $mnt;
    }
    print "\n";
  }
}

if ($opts{n}) {
  my $color = MxRaid::Color->new();
  $do_default_action = 0;

  my ($entry, $cnt, $size) = ('', 0, 0);
  my ($level, $num_wanted, $match) = (6, 16, undef);
  my $chunk_size = 512;
  my $dev_no = 0;
  my @mounts = split m/\n/, `cat /proc/self/mounts`;
  my @candidates;

  if ($opts{r}) {
    ($level,$num_wanted,$match) = split m/:/, $opts{r};
  }
  $entry .= $opts{n}.':';
  for my $rec (@{$hd->non_raid_disks()}) {
    my $dev = $rec->[0];
    next unless $dev =~ m/(sd[a-z]+|nvme\d+n\d+)/;
    if (grep {m|^/dev/$dev\d*\b|} @mounts) {
      warn "#  NOTE: /dev/$dev is mounted, skipping.\n" if $VERBOSE >= 2;
      next;
    }
    if (defined $match) {
      next unless ($rec->[1] =~ m/$match/i or $rec->[2] =~ m/$match/i);
    }
    push @candidates, $rec->[0];
    $entry .= ' '.$rec->[1];
    $size += $rec->[3];
    $cnt++;
  }

  if ($cnt != $num_wanted) {
    my $msg = "got $cnt disk(s), expected $num_wanted.";
    $msg = $color->t_red($msg) unless $opts{m};
    print "#  NOTE: $msg\n";
  }

  if ($opts{n} !~ m/^[CDM][\da-f]\d{3}$/) {
    my $msg = "label '$opts{n}' doesn't follow the CDM scheme.";
    $msg = $color->t_red($msg) unless $opts{m};
    print "#  NOTE: $msg\n";
  }

  if ($cnt) {
    if ($level == 0) {
      # do nothing
    } elsif ($level == 1) {
      $size /= 2;
    } elsif ($level == 5 and $cnt>1) {
      $size = int ((($cnt-1)/$cnt) * $size);
    } elsif ($level == 6 and $cnt>2) {
      $size = int ((($cnt-2)/$cnt) * $size);
    } else {
      print "#  RAID level $level not handled ($cnt disks)\n";
    }
    $size /= 1024**4; # TB


    # 100TB -> 512k, 50TB -> 256k
    while ( $size/$chunk_size < 0.115 and $chunk_size > 64) {
      $chunk_size/=2;
    }

    while (-e "/dev/md$dev_no") {
      $dev_no++;
      die "# ERROR: out of devices ($dev_no). Stopped", if $dev_no >= 128;
    }

    my $dev='md'.$dev_no;
    print "#  Record for '$MDADMCONF_DB'\n\n";
    print $entry;
    print "\n\n\n";
    print "#  Hint for creation with mdadm & mkfs.xfs:\n\n";
    printf "mdadm -C /dev/%s -l %d -n %d -N %s -c %s %s\n\n",
             $dev,
             $level, $cnt,
             $opts{n},
             $chunk_size,
             '/dev/' . join(' /dev/', @candidates);

    printf "mkfs.xfs -L %s /dev/%s\n\n", lc($opts{n}), $dev;


    printf "#  Note: size of array will be %.1f TB\n\n", $size;
  }
}

if ($do_default_action) {
  create_mdadm_configs();
}
exit;

# prepare to assemble for the complete host
sub create_mdadm_configs {

  my $do_assemble = $opts{a};

  # compare situation on host with database,
  # a replaced disk (different id) won't make it into '%RAIDS',
  # thus disk count in RAIDS is always <= disk count in DB
  # a db entry with too much or less disk entries will indeed spoil the lookup,
  # and assembly by mdadm will fail at the end.

  my @mdadm_conf;
  my $dev_no = 0;
  for my $rlabel (@{$hd->raid_labels()}) {
    my $condition_active = 0;
    my $condition_incomplete = 0;
    my $active_dev_no = -1;

    # this one triggers if serials are assigned to the wrong array.
    my $num_expected = scalar @{$cd->members($rlabel)};
    my $num_found = scalar @{$hd->member_serials($rlabel)};
    my $ratio = int(100*$num_found/($num_expected + 1e-6));
    if ($num_found != $num_expected) {
      warn "# ERROR: $rlabel, expected $num_expected members, but only $num_found disk(s) could be assigned, won't assemble.\n" if $VERBOSE >= 1;
      warn "#  NOTE: $rlabel, maybe the configuration contains disks from an array on a different host.\n" if $ratio<50 and $VERBOSE >= 1;
      warn "#  NOTE:        (More clearly, go and search for $rlabel somewhere else!)\n" if $ratio<50 and $VERBOSE >= 1;
      $condition_incomplete = 1;
    }
    my $res = $utils->list_differences( $cd->members($rlabel), $hd->member_serials($rlabel));
    if ($res) {
      # print Dumper $res;
      # second array in $res must be empty, first array contains missing one(s), make shure ;)
      die "# ERROR: $rlabel, possible software error, stopped" if scalar(@{$res->[1]});
      warn sprintf "#  NOTE: $rlabel, disk(s) missing: %s.\n", join(', ', @{$res->[0]}) if $VERBOSE >= 1;
      warn sprintf "#  NOTE: $rlabel, candidates available: %s\n", join(', ', map {"$_->[0]:$_->[1]"} @{$hd->non_raid_disks()}) if $VERBOSE >= 1;
    }

    # some more checks if some/all arrays run already
    if (scalar @{$hd->get_md_devices()}) {
      my %running;
      for my $dev (@{$hd->member_devs($rlabel)}) {
        my @sys_info = glob("/sys/block/$dev/holders/md*");
        next if scalar(@sys_info) == 0;
        die '# Fuck! Stopped' if scalar(@sys_info) > 1; # this will never happen ;)
        $sys_info[0] =~ m/(md(\d+))$/;
        my $md = $1;
        $active_dev_no = $2;
        push @{$running{$md}}, $dev; # should be the same key, otherwise the raid is scattered
      }

      if (scalar (keys %running) > 1) {
        warn "#  NOTE: $rlabel, the configured RAID spreads over multiple md-devices, skipped.\n" if $VERBOSE >= 1;
        next;
      }

      if (values %running) {
        $res = $utils->list_differences( $hd->member_devs($rlabel), values %running );
        if ($res) {
          warn "# Debug case.\n";
          warn "# That's embarrassing! (members that should be running / running but not a configured member)\n";
          warn Dumper $res;
          die "# ERROR: $rlabel, found a serious mismatch between disks running and stored configuration.\n";
        } else {
          if ($condition_incomplete) {
            warn "#  NOTE: $rlabel, Array is incomplete, but some disks are part of an active array, check config file ($MDADMCONF_DB)!\n" if $VERBOSE >= 1;
          } else {
            warn "#  NOTE: $rlabel, already active.\n" if $VERBOSE >= 2;
          }
          $condition_active = 1;
        }
      }
    }

    # ARRAY /dev/md0 devices=/dev/sdb,/dev/sdc,/dev/sdd,/dev/sde,/dev/sdf,/dev/sdg,/dev/sdh,/dev/sdi
    my @tmp = map { '/dev/' . $_ } @{$hd->member_devs($rlabel)}; # pull out devices
    my $conf_dev_no;
    # we increase $dev_no bottom up, whereas the ones from 'mdadm -A --scan' are populated top down
    if ($active_dev_no >= 0) {
      $conf_dev_no=$active_dev_no;
    } else {
      while (-e "/dev/md$dev_no") {
        $dev_no++;
        die "# ERROR: out of devices ($dev_no). Stopped", if $dev_no >= 128;
      }
      $conf_dev_no=$dev_no;
      $dev_no++;
    }
    push @mdadm_conf, [
                        $conf_dev_no , sprintf("ARRAY /dev/md%d devices=%s", $conf_dev_no, join ',', @tmp),
                        $rlabel , $condition_active, $condition_incomplete
                      ]; # -2-             -3-                -4-

    die "# ERROR: sane device limit (md127) blown, stopped" if $dev_no >= 128;
  }

  # got here? Then it's time to write config files ;)
  if (@mdadm_conf) {
    # Store files we've created, helps to avoid clobbering generated configs whithin one run.
    # This catches the case that $conf_dev_no isn't uniq (this may happen if running arrays are present *and* database errors are present)
    my %conf_files_created;
    my $bogus_count=1;
    for (@mdadm_conf) {
      my $dummy_config = $_->[3] + $_->[4];
      my $dev_no = $_->[0];
      my $rlabel = $_->[2];
      my $conf_fn = $MDADM_CONF_BASE;
      $conf_fn =~ s/conf$/$dev_no.conf/;
      while (exists $conf_files_created{$conf_fn}) {
        $conf_fn =~ s/.*\d+$//;
        $conf_fn = sprintf "%s.%03d", $conf_fn, $bogus_count;
        $bogus_count++;
        if ($bogus_count >= 100) {
          die "# ERROR: created 100+ config files due to config errors. Stopping now";
        }
      }
      $conf_files_created{$conf_fn}+=1;
      open(CNF, '>', $conf_fn) || die "# ERROR: failed to create '$conf_fn' ($!), stopped";
      if (!$dummy_config) {
        print CNF "# label: $rlabel\n";
        print CNF $_->[1], "\n";
      } else {
        if ($_->[4]) {
          print CNF "# $rlabel, Array is incomplete (found less drives than configured).\n" if $_->[4];
          print CNF "# Members are part of an active array, check your config!.\n" if $_->[3];
        } else {
          print CNF "# $rlabel, Array is already active.\n" if $_->[3];
        }
        print CNF "# ", $_->[1], "\n";
      }
      close CNF;
      warn sprintf("#  NOTE: $rlabel, created %s'%s'.\n", $dummy_config?'dummy config ':'',$conf_fn) if $VERBOSE >= 1;
      next if $dummy_config;

      if ($ROOT) {
        if ($do_assemble) {
          my @args = ('mdadm', '-A', '/dev/md'.$dev_no, '-c', $conf_fn);
          push @args, split m/\s+/, $MDADM_ASSEMBLE_OPTIONS if $MDADM_ASSEMBLE_OPTIONS;
          warn sprintf "#  NOTE: $rlabel, running '%s'\n", join(' ', @args) if $VERBOSE >= 2;
          system (@args) == 0 or warn "# Error system @args failed: $?"; # serious, always warn
        } else {
          printf "mdadm -A /dev/md%d -c %s\n", $dev_no, $conf_fn;
        }
        
      } else {
        printf "sudo mdadm -A /dev/md%d -c %s %s\n", $dev_no, $conf_fn, $MDADM_ASSEMBLE_OPTIONS;
      }
    }
  } else {
    warn "#  NOTE: No SW-RAIDS configured on this host.\n" if $VERBOSE >= 1;
  }

}



exit;


{ ####### MxRaid::ConfData
  package MxRaid::ConfData;
  BEGIN { $MxRaid::ConfData::VERSION = '0.01'; }

  use warnings;
  use strict;
  use Carp;
  use Data::Dumper; $Data::Dumper::Sortkeys=1;

  sub new {
    my($pkg, $config) = @_;
    my $self = bless {}, $pkg;
    $self->{config} = $config || return undef;
    $self->{verbose} = 0;
    $self->{db} = {};
    $self->{lookup} = undef;
    return $self;
  }

  sub load {
    my $self = shift;
    my @rec;
    my $array_name;
    my $lines=0;
    my @faulty_names;

    open(DB, '<', $self->{config}) or die "# ERROR: failed to open config '$self->{config}' ($!), stopped";
    while (<DB>) {
      $lines++;

      $_ =~ s/#.*$//;
      $_ =~ s/\s+$//;
      next unless $_;
      $_ =~ s/^\s+//;

      @rec = split m/\s+/, $_;

      if ($rec[0] !~ m/[CDM][\da-f]\d{3}:/) {
        warn "#  NOTE: raid name not in expected format (bad key '$rec[0]' at line $lines).\n" if $self->{verbose} >= 1;
        next;
      }

      $array_name = shift @rec;
      chop $array_name; # remove the ':'

      next unless @rec;
      if (exists $self->{db}{$array_name}) {
        warn "#  WARN: database contains same array name more than once ($array_name)!\n" if $self->{verbose} >= 1;
        push @faulty_names, $array_name;
      }
      $self->{db}{$array_name} = [ @rec ];
    }
    close DB;
    for (@faulty_names) {
      delete $self->{db}{$_};
      warn "#  NOTE: removed badly configured array '$_' from configuration.\n";
    }
    
    my %tmp; # see if this gets expensive ...
    for my $label (keys %{$self->{db}}) {
      for (@{$self->{db}{$label}}) {
        if (exists $tmp{$_}) {
          croak "# Error: database contains disk id more than once ($label: $_)! Bailing out";
        }
        $tmp{$_} = $label;
      }
    }
    $self->{lookup} = \%tmp;

    return scalar keys %{$self->{db}};
  }

  sub members {
    my $self = shift;
    my $label = shift || return undef;
    return $self->{db}{$label}
  }

#   sub serials_by_label {
#     my $self = shift;
#     my $label = shift || return undef;
#     return $self->{db}{$label}
#   }

  sub label_by_serial {
    my $self = shift;
    my $serial = shift || return undef;
    return $self->{lookup}{$serial}
  }

  sub verbose {
    my $self = shift;
    $self->{verbose} = shift if @_;
    $self->{verbose}
  }

  sub number_of_db_records {
    my $self = shift;
    scalar keys %{$self->{db}}
  }

  sub next_db_labels {
    my $self = shift;
    my %tmp;
    for (keys %{$self->{db}}) {
      my ($cat,$size,$num) = $_ =~ m/([CDM])([\da-f])(\d{3})/;
      next unless defined $num;
      $cat = '(C/M)' if $cat ne 'D';
      if (exists $tmp{"$cat$size"} and $tmp{"$cat$size"} > $num) {
        $num = $tmp{"$cat$size"};
      }
      $tmp{"$cat$size"} = $num;
    }
    my @next;
    for my $k (keys %tmp) {
      push @next, sprintf("%s%03d", $k, $tmp{$k}+1);
    }
    return \@next;
  }

}

{ ####### MxRaid::HostData
  package MxRaid::HostData;

  BEGIN { 
    $MxRaid::HostData::VERSION = '0.01'; 
    %MxRaid::HostData::BAD_MODELS = ( # and the cure...
       ST8000NM0065 => sub {substr $_[0], 0, 8}, # silly coding of serial
       ST8000NM0075 => sub {substr $_[0], 0, 8},
       ST8000NM001A => sub {substr $_[0], 0, 8},
       ST1000NM0045 => sub {substr $_[0], 0, 8},
    );
  }

  use warnings;
  use strict;
  use Carp;
  use Data::Dumper; $Data::Dumper::Sortkeys=1;

  sub new {
    my($pkg,$conf) = @_;
    my $self = bless {}, $pkg;
    $self->{config} = $conf;
    $self->{verbose} = 0;
    $self->{root_priv} = $<==0?1:0;
    $self->discover_basic();
    $self->discover_running();
    return $self;
  }

  sub discover_basic {
    my $self = shift;

    # take only whole disks, no partitions
    my @disks = grep {m:/(sd[a-z]+|nvme\d+n\d+)$:} (
                  glob('/sys/block/sd[a-z]*'), glob('/sys/block/nvme[0-9]*')
                );
    my $diskinfo = $self->get_hd_info(\@disks);
    my @non_raid;
    my %configured_raids;
    my %raid_labels;

    if (defined $self->{config}) {
      my $c = $self->{config};
      for my $d (@$diskinfo) {
        my $sn = $d->[1];
        if ( my $label = $c->label_by_serial($sn) ) {
          push @{$configured_raids{$label}}, $d;
          $raid_labels{$label}+=1;
        } else {
          #mdadm -E /dev/sdX -- use picky mode to catch stray raid members?
          push @non_raid, $d;
        }
      }
    }
    $self->configured_raids(\%configured_raids);
    $self->non_raid_disks(\@non_raid);
    $self->raid_labels( [ sort keys %raid_labels ] );
  }

  sub discover_running {
    my $self = shift;

    my @mds_found = ( 
       glob('/sys/block/md[0-9]'),            # upper limit 128, and no leading zeroes
       glob('/sys/block/md[0-9][0-9]'),       # but who knows
       glob('/sys/block/md[0-9][0-9][0-9]')); #

    my %hd2md;
    my %md2info;
    my %cross_lookup;
    my %md2hd; # scratch

    for my $m (@mds_found) {
      $m =~ m,([^/]+)$,;
      my $md = $1;
      # alternative : glob("/sys/block/$md/slaves/*");
      my @tmp = glob("/sys/block/*/holders/$md");
      for (@tmp) {
        $_ =~ m,/sys/block/(.+)/holders/(.+),;
        push @{$md2hd{$2}}, $1;
        die "# what, duplicate device?" if exists $hd2md{$1};
        $hd2md{$1} = $2;
      }
    }

    for my $k (keys %md2hd) {
      my @tmp = @{$md2hd{$k}};
      @tmp = map { '/sys/block/' . $_ } @tmp; # redo ...
      my $hd_info = $self->get_hd_info(\@tmp);
      # printf "%s: %s\n", $k, join(' ', @$hd_info);
      $md2info{$k} = [ @$hd_info ];

      for (@$hd_info) {
        $cross_lookup{s2d}->{$_->[1]} = $_->[0];
        $cross_lookup{d2s}->{$_->[0]} = $_->[1];
      }

    }


    $self->member_info_by_mddev(\%md2info);
    $self->mddev_by_member(\%hd2md);
    $self->member_dev_serial_cross_lookup(\%cross_lookup);

  }

  sub raid_labels {
    my $self = shift;
    $self->{raid_labels} = shift if @_;
    return $self->{raid_labels};
  }

  sub non_raid_disks() {
    my $self = shift;
    $self->{non_raid_disks} = shift if @_;
    return $self->{non_raid_disks};
  }

  sub configured_raids() {
    my $self = shift;
    $self->{configured_raids} = shift if @_;
    $self->{configured_raids};
  }

  sub member_serials {
    my $self = shift;
    my $label = shift;
    my $di = $self->{configured_raids}{$label};
    [ map {$_->[1]} @$di ];
  }

  sub member_devs {
    my $self = shift;
    my $label = shift;
    my $di = $self->{configured_raids}{$label};
    [ map {$_->[0]} @$di ];
  }

  sub member_info_by_mddev {
    my $self = shift;
    $self->{member_info_by_mddev} = shift if @_;
    return $self->{member_info_by_mddev};
  }

  sub mddev_by_member {
    my $self = shift;
    $self->{mddev_by_member} = shift if @_;
    return $self->{mddev_by_member};
  }

  sub member_dev_serial_cross_lookup {
    my $self = shift;
    $self->{member_dev_serial_cross_lookup} = shift if @_;
    return $self->{member_dev_serial_cross_lookup};
  }

  sub get_md_devices {
    my $self = shift;
    return [ sort keys %{$self->{member_info_by_mddev}} ];
  }

  # 'shortcut'
  sub get_hd_serial {
    my $self = shift;
    $self->get_hd_info($_[0], 1)
  }

  sub get_hd_info {
    my $self = shift;
    my $dev_list = shift;
    my $short_info = shift;

    my @ret;
    my $num = scalar @$dev_list;
    for (my $i=0; $i<$num; $i++) {
      my ($dk) = $dev_list->[$i] =~ m|([^/]+)$|;

      my $model = sys_fs_get_prop('/sys/block/'.$dk.'/device/model');
      my $sizeb = sys_fs_get_prop('/sys/block/'.$dk.'/size') * 512;

      # if (! -e '/sys/block/'.$dk.'/device/vpd_pg80') {
      #   warn "# Note failed to read serial via sysfs ($dk).\n" if $self->{verbose};
      #   next;
      # }

      my $serial_number;
      if (-e '/sys/block/'.$dk.'/device/vpd_pg80') { # exists even for ahci attached disks
        # Vital Product Data, page xyz
        my $vpd_pg80 = parse_vpd_pg80(sys_fs_get_prop('/sys/block/'.$dk.'/device/vpd_pg80'));
        if (defined $MxRaid::HostData::BAD_MODELS{uc($model)}) {
          $vpd_pg80 = $MxRaid::HostData::BAD_MODELS{uc($model)}($vpd_pg80);
        }
        $serial_number = $vpd_pg80;
      } elsif (-e '/sys/block/'.$dk.'/device/serial') { # nvme disks are supposed to have this
        $serial_number = sys_fs_get_prop('/sys/block/'.$dk.'/device/serial');
        $serial_number =~ s/^\s+//;
      } else {
        if ($self->{root_priv}) {
          $serial_number = smartctl_info($dev_list->[$i], 'Serial_Number');
        } else {
          warn "# You must be root to query '$dk'.\n" if $self->{verbose} >= 1;
        }
      }

      if (defined $serial_number) {
        if ($short_info) {
          push @ret, $serial_number;
        } else {
          push @ret, [$dk, $serial_number, $model, $sizeb];
        }
      }
    }
    return \@ret;
  }

  sub sys_fs_get_prop {
    my $fn = shift;
    my $ret = undef;
    if (-r $fn) {
      $ret = _slurp_file($fn);
      $ret =~ s/\s+$//;
    } else {
      warn "# ERROR: failed to read '$fn'\n";
    }
    return $ret;
  }

  # 'vpd_pg80'
  # Use sysfs attribute vpd_pg80 to read serial number
  # https://www.redhat.com/archives/dm-devel/2015-March/msg00156.html
  sub parse_vpd_pg80 {
    my $vpd_pg80 = shift;
    $vpd_pg80 =~ s/[^0-9A-Za-z]+/ /;
    $vpd_pg80 =~ s/ +/ /;
    $vpd_pg80 =~ s/^ //;
    $vpd_pg80 =~ s/ $//;
    return $vpd_pg80;
  }

  # reading must succeed, thus the croak/die
  sub _slurp_file {
    my $fn = shift;
    open my $F,'<',$fn or die "# Error ($fn): $!\n";
    return join ('',<$F>); # closes when scope is left 8)
  }


  # alternative if no vpd_pg80 avail
  sub smartctl_info {
    my $dev = shift;
    my @keys = @_;  
    # smartctl -i /dev/sda | grep 'Serial Number:'
    my @res = split m/\n/, `smartctl -i $dev`;

    my $parse = 0;
    my %smart_data;
    for (@res) {
      if ($_ =~ m/^===/) {
        $parse = 1;
        next;
      }
      if ($parse) {
        my ($key, $val) = $_ =~ m/^([^:]+):\s*(.+)$/;
        next unless defined $key and defined $val;
        $smart_data{$key} = $val;
      }
    }
    if (@keys) {
      if (scalar @keys == 1) {
        return $smart_data{$keys[0]};
      }
      my %ret;
      for (@keys) {
        $ret{$_} = $smart_data{$_};
      }
      return \%ret;
    }
    return \%smart_data;
  }

  sub verbose {
    my $self = shift;
    $self->{verbose} = shift if @_;
    $self->{verbose}
  }
}


{ ####### MxRaid::Utils
  package MxRaid::Utils;
  BEGIN { $MxRaid::Utils::VERSION = '0.01'; }

  use warnings;
  use strict;
  use Carp;
  use Data::Dumper; $Data::Dumper::Sortkeys=1;

  sub new {
    my($pkg) = @_;
    my $self = bless {}, $pkg;
    return $self;
  }

  sub get_xfs_label {
    my $self = shift;
    my $dev = shift;
    return undef unless $dev =~ m,^/dev/md\d+$,;
    my $label = `xfs_admin -l $dev`; chop($label);
    return $label;
  }

  sub list_differences {
    my $self = shift;
    my $la = shift;
    my $lb = shift;
    my $one_way = shift;

    # print Dumper $la, $lb; # die'';

    my %tmp;
    for (@$lb) {
      $tmp{$_}+=1;
    }

    my @not_in_b;
    for my $x (@$la) {
      push @not_in_b, $x if !exists $tmp{$x};
    }

    if ($one_way) {
      return [@not_in_b];
    }
    my @not_in_a = @{$self->list_differences($lb, $la, 1)};

    return 0 if scalar(@not_in_a) == 0 and scalar(@not_in_b) == 0;
    return [\@not_in_b, \@not_in_a];
  }


} ####### MxRaid::Utils

{
  package MxRaid::Color;
  BEGIN { $MxRaid::Color::VERSION = '0.01'; }


  # ABSTRACT: Perl module for simple color enriched printout

  use warnings;
  use strict;
  use Carp;
  use Data::Dumper; $Data::Dumper::Sortkeys=1;

  sub new {
    my($pkg)=@_;
    my $self=bless {}, $pkg;
    return $self;
  }

  # sub t_red { $_[0] || return ''; "*$_[0]*" } # monochrome
  sub t_red    { shift;  $_[0] || return ''; "\033[0;31m$_[0]\033[0;39m\033[0;22m" }
  sub t_green  { shift;  $_[0] || return ''; "\033[0;32m$_[0]\033[0;39m\033[0;22m" }
  sub t_purple { shift;  $_[0] || return ''; "\033[0;35m$_[0]\033[0;39m\033[0;22m" }

}