subagent-shell-base.functions

# vi:syntax=perl
use strict;
use warnings;
use Sys::Syslog qw(:standard :macros);
use Time::HiRes qw(time);
use IO::Socket;
use constant HAVE_SOCKET_INET6 => eval { require IO::Socket::INET6 };

our $config;

sub get_ntp_response {
# the function has been derived from Net-NTP module, http://search.cpan.org/~abh/Net-NTP-1.3/
  my $res = shift;
  my $host = shift || 'localhost';
  my $port = shift || 'ntp';
  my $rtimeout = shift || 10;

  use constant NTP_ADJ => 2208988800;

  my @ntp_packet_fields = (
    'Leap Indicator',
    'Version Number',
    'Mode',
    'Stratum',
    'Poll Interval',
    'Precision',
    'Root Delay',
    'Root Dispersion',
#      'Reference Clock Identifier', #the field is removed as not needed
    'Reference Timestamp',
    'Originate Timestamp',
    'Receive Timestamp',
    'Transmit Timestamp',
  );

  my $frac2bin = sub {
    my $bin  = '';
    my $frac = shift;
    while (length($bin) < 32) {
      $bin = $bin . int($frac * 2);
      $frac = ($frac * 2) - (int($frac * 2));
    }
    return $bin;
  };

  my $bin2frac = sub {
    my @bin = split '', shift;
    my $frac = 0;
    while (@bin) {
      $frac = ($frac + pop @bin) / 2;
    }
    return $frac;
  };

  my $sock;
  if (HAVE_SOCKET_INET6) {
    $sock = IO::Socket::INET6->new(Proto => 'udp', PeerHost => $host, PeerPort => $port );
  }
  else {
    $sock = IO::Socket::INET->new(Proto => 'udp', PeerHost => $host, PeerPort => $port );
  }
  return "$@" unless $sock;

  $sock->sockopt( SO_RCVTIMEO, pack('L!L!', $rtimeout, 0) );

  my %tmp_pkt;
  my %packet;
  my $data;

  my $client_localtime      = time;
  my $client_adj_localtime  = $client_localtime + NTP_ADJ;
  my $client_frac_localtime = $frac2bin->($client_adj_localtime);

  my $ntp_msg = pack("B8 C3 N10 B32", '00011011', (0) x 12, int($client_localtime), $client_frac_localtime);

  $sock->send($ntp_msg) or return "send() failed: $!";
  $sock->recv($data, 960) or return "recv() failed: $!";

  my $now_client_localtime = time;

  my @ntp_fields = qw/byte1 stratum poll precision/;
  push @ntp_fields, qw/delay delay_fb disp disp_fb ident/;
  push @ntp_fields, qw/ref_time ref_time_fb/;
  push @ntp_fields, qw/org_time org_time_fb/;
  push @ntp_fields, qw/recv_time recv_time_fb/;
  push @ntp_fields, qw/trans_time trans_time_fb/;

  @tmp_pkt{@ntp_fields} = unpack("a C3   n B16 n B16 H8   N B32 N B32   N B32 N B32", $data);

  @packet{@ntp_packet_fields} = (
    (unpack("C", $tmp_pkt{byte1} & "\xC0") >> 6),
    (unpack("C", $tmp_pkt{byte1} & "\x38") >> 3),
    (unpack("C", $tmp_pkt{byte1} & "\x07")),
    $tmp_pkt{stratum},
    (sprintf("%0.4f", $tmp_pkt{poll})),
    $tmp_pkt{precision} - 255,
    ($bin2frac->($tmp_pkt{delay_fb})),
    (sprintf("%0.4f", $tmp_pkt{disp})),
    (($tmp_pkt{ref_time}   += $bin2frac->($tmp_pkt{ref_time_fb}))   -= NTP_ADJ),
    (($tmp_pkt{org_time}   += $bin2frac->($tmp_pkt{org_time_fb}))),
    (($tmp_pkt{recv_time}  += $bin2frac->($tmp_pkt{recv_time_fb}))  -= NTP_ADJ),
    (($tmp_pkt{trans_time} += $bin2frac->($tmp_pkt{trans_time_fb})) -= NTP_ADJ)
  );
  $packet{'Delay'} = ($now_client_localtime-$client_localtime)-($packet{'Receive Timestamp'}-$packet{'Transmit Timestamp'});
  $packet{'Offset'} = (($packet{'Receive Timestamp'}-$client_localtime)+($packet{'Transmit Timestamp'}-$now_client_localtime))/2;
  $packet{'Duration'} = $now_client_localtime-$client_localtime;
  $packet{'Delta'} = $packet{'Receive Timestamp'}-$client_localtime-$packet{'Duration'}/2;
  %$res = %packet;
  return 0;
}

# apache httpd mod_status dashboard

#Total Accesses: 58760
#Total kBytes: 37099
#CPULoad: .0188875
#Uptime: 426632
#ReqPerSec: .13773
#BytesPerSec: 89.0448
#BytesPerReq: 646.518
#BusyWorkers: 1
#IdleWorkers: 99

#Scoreboard Key:
#   "_" Waiting for Connection, 
#   "W" Sending Reply, 
#   "R" Reading Request,
#   "K" KeepAlive (read), 
#   "D" DNS Lookup, 

sub apacheStatusMIB {
  my $o=shift;
  my $cfg=shift;
  my $mcall=shift;
  my $i=$$mcall;
  $$mcall=0; # the function support multicall

  my $cmd="curl -s -m $config->{'cmd_timeout'} ";
  my $r=0;
  my $t=gettimeofday();

  foreach my $a (@{$cfg->{'args'}}) {
    my $ts=gettimeofday();
    foreach (`$cmd $a->{'status_url'}`) {
      chomp;
      if ( /Scoreboard/ ) {
        my $s=(split )[1];
        $$o{"apsWaitingForConnection.$i"}=$s=~tr/_/_/; 
        $$o{"apsSendingReply.$i"}=$s=~tr/W/W/; 
        $$o{"apsReadingRequest.$i"}=$s=~tr/R/R/; 
        $$o{"apsKeepAlive.$i"}=$s=~tr/K/K/; 
        $$o{"apsOpenSlots.$i"}=$s=~tr/././; 
        $$o{"apsOtherSlots.$i"}=$s=~tr/DCLGIS//; 
      } elsif ( /^Total Accesses/ ) {
        $$o{"apsTotalAccesses.$i"} = ( split /: / )[1];
      } elsif ( /^CPULoad/ ) {
        $$o{"apsCpuLoad.$i"} = ( split /: / )[1];
      } elsif ( /^Uptime/ ) {
        $$o{"apsUptime.$i"} = ( split /: / )[1];
      } elsif ( /^BusyWorkers/ ) {
        $$o{"apsBusyWorkers.$i"} = ( split /: / )[1];
      } elsif ( /^IdleWorkers/ ) {
        $$o{"apsIdleWorkers.$i"} = ( split /: / )[1];
      } elsif ( /^ReqPerSec/ ) {
        $$o{"apsReqPersSec.$i"} = ( split /: / )[1];
      } elsif ( /^BytesPerSec/ ) {
        $$o{"apsBytesPerSec.$i"} = ( split /: / )[1];
      } elsif ( /^BytesPerReq/ ) {
        $$o{"apsBytesPerReq.$i"} = ( split /: / )[1];
      }
    }

    my $cmd_r = $? >> 8;
    while ( defined($$o{"apsIndex.$i"}) ) { $i++; } # detect if the function called previously and increase the index 
    $$o{"apsIndex.$i"} = $i;
    $$o{"apsCmdExecTime.$i"} = sprintf('%.3f',scalar gettimeofday() - $ts);
    $$o{"apsCmdExecStatus.$i"} = $cmd_r;
    $$o{"apsUrl.$i"} = (defined($a->{'description'})?$a->{'description'}:'') . "[$a->{'status_url'}]";
    if ( ! $cmd_r ) {
      my $allslots = $$o{"apsWaitingForConnection.$i"} + $$o{"apsSendingReply.$i"} + $$o{"apsReadingRequest.$i"} + $$o{"apsKeepAlive.$i"} + $$o{"apsOtherSlots.$i"} + $$o{"apsOpenSlots.$i"};
      my $busyslots = $$o{"apsSendingReply.$i"} + $$o{"apsReadingRequest.$i"} + $$o{"apsKeepAlive.$i"} + $$o{"apsOtherSlots.$i"};
      $$o{"apsSlotsUsage.$i"} = sprintf('%.2f', $busyslots*100/$allslots );
    }
    $r = $cmd_r if $cmd_r;
    $i++;
  }
  $$o{'apsFuncExecTime'} = sprintf('%.3f',scalar gettimeofday() - $t);
  $$o{'apsFuncExecStatus'} = $r;
  return  $r;
}

sub httpResponseMIB {
  no strict "refs";
  my $o=shift;
  my $cfg=shift;
  my $mcall=shift;
  my $i=$$mcall;
  $$mcall=0; # the function support multicall

  my $cmd="curl -v -s -m $config->{'cmd_timeout'} "; #don't use -v -s -m options for curl args
  my $r=0;
  my $t=gettimeofday();

  foreach my $a (@{$cfg->{'args'}}) {
    my $ts=gettimeofday();
    $$o{"httpRespMatched.$i"} = defined($a->{'match'})?2:0;
    $$o{"httpRespMatchString.$i"} = $a->{'match'} if defined($a->{'match'});
    foreach (`($cmd $a->{'curl'} | sed 's/^/STDOUT:/') 2>&1`) {
      chomp;
      if ( /^STDOUT/ and defined($a->{'match'}) and /$a->{'match'}/ ) {
        $$o{"httpRespMatched.$i"} = 1;
        for (my $m=1;$m <= $#+; $m++) {
          $$o{"httpRespMatchString.$i.$m"} = $$m;
        }
      }
      $$o{"httpRespCode.$i"} = ( split )[2] if ( /^< HTTP\// );
    }
    # don't return unmatch in the case of ERROR http code
    $$o{"httpRespMatched.$i"} = 0 if ( defined($$o{"httpRespCode.$i"}) and $$o{"httpRespCode.$i"} > 307 );
    my $cmd_r = $? >> 8;
    while ( defined($$o{"httpRespIndex.$i"}) ) { $i++; } # detect if the function called previously and increase the index 
    $$o{"httpRespIndex.$i"} = $i;
    $$o{"httpRespCmdExecTime.$i"} = sprintf('%.3f',scalar gettimeofday() - $ts);
    $$o{"httpRespCmdExecStatus.$i"} = $cmd_r;
    $$o{"httpRespCurlArgs.$i"} = (defined($a->{'description'})?$a->{'description'}:'') . "[$a->{'curl'}]";
    $$o{"httpRespMonitoringTemplate.$i"} = defined($a->{'monitoringTemplate'})?$a->{'monitoringTemplate'}:'';
    $r = $cmd_r if $cmd_r;
    $i++;
  }
  $$o{'httpRespFuncExecTime'} = sprintf('%.3f',scalar gettimeofday() - $t);
  $$o{'httpRespFuncExecStatus'} = $r;
  return  $r;
}

sub ntpSyncMIB {
  my $o=shift;
  my $cfg=shift;
  my %h;

  my $e=get_ntp_response( \%h, $cfg->{'args'}[0]->{'server'}, 'ntp', $config->{'cmd_timeout'} );
  if ( $e eq "0" ) {
    $$o{'ntpSyncStatus'} = sprintf('%.3f', $h{'Delta'});
    $$o{'ntpSyncServer'} = $cfg->{'args'}[0]->{'server'};
    $$o{'ntpSyncFuncExecTime'} = sprintf('%.3f',$h{'Duration'});
    $$o{'ntpSyncExecStatus'} = 0;
    return 0;
  } else {
    logMessage(LOG_WARNING, "ntpSyncMIB failed with $e");
    $$o{'ntpSyncServer'} = $cfg->{'args'}[0]->{'server'};
    $$o{'ntpSyncExecStatus'} = -1;
    return -1;
  }
}

sub ntpdateSyncMIB {
  my $o=shift;
  my $cfg=shift;
  my $t=gettimeofday();
  my $cmd="/usr/sbin/ntpdate -t $config->{'cmd_timeout'} -q4 $cfg->{'args'}[0]->{'server'}";
  my $v;

  foreach (`$cmd`) {
    chomp;
    $v=(split )[9] if ( /time server/ );
  }
  my $r = $? >> 8;
  $$o{'ntpSyncStatus'} = sprintf('%.3f', $v) if defined($v);
  $$o{'ntpSyncServer'} = $cfg->{'args'}[0]->{'server'};
  $$o{'ntpSyncFuncExecTime'} = sprintf('%.3f',scalar gettimeofday() - $t);
  $$o{'ntpSyncExecStatus'} = $r;
  return $r;
}

sub procCountMIB {
  my $o = shift;
  my $cfg = shift;
  my $t = gettimeofday();
  $$o{'procTotal'} =    0;
  $$o{'procRunning'} =  0;
  $$o{'procSleeping'} = 0;
  $$o{'procStopped'} =  0;
  $$o{'procZombie'} =   0;
  $$o{'procUninterruptible'} =   0;

  foreach(`ps ax -opid= -ovsize= -ostat=`) {
    chomp;
    my ($pid, $vsize, $stat) = split;
    if ( !defined($cfg->{'args'}[0]->{'extendedInfo'}) or $cfg->{'args'}[0]->{'extendedInfo'} eq 'on' ) {
      $$o{"procVirtualMemory.$pid"} = $vsize;
      $$o{"procOpenFdCount.$pid"} = 0;
      if ( opendir ( my $dh, "/proc/$pid/fd" ) ) {
        my $c = $#{[readdir $dh]};
        $$o{"procOpenFdCount.$pid"} = $c - 1 if $c>1;
        closedir $dh;
      } 
    }

    $$o{'procTotal'}++;
    $$o{'procRunning'}++  if $stat =~ /^R/;
    $$o{'procSleeping'}++ if $stat =~ /^S/;
    $$o{'procStopped'}++  if $stat =~ /^T/;
    $$o{'procZombie'}++   if $stat =~ /^Z/;
    $$o{'procUninterruptible'}++  if $stat =~ /^D/;
  }
  my $r = $? >> 8;
  $$o{'procFuncExecTime'} = sprintf('%.3f',scalar gettimeofday() - $t);
  $$o{'procExecStatus'} = $r;
  return $r;
}
  
sub ip_conntrackMIB {
  my $o = shift;
  my $t = gettimeofday();
  my ($cmd, $count, $max, $r);

  if (-f "/proc/sys/net/ipv4/ip_conntrack_max") {
    $cmd="cat /proc/sys/net/ipv4/netfilter/ip_conntrack_count /proc/sys/net/ipv4/ip_conntrack_max | tr '\n' ' '";
  } elsif (-f "/proc/sys/net/netfilter/nf_conntrack_max") {
    $cmd="cat /proc/sys/net/netfilter/nf_conntrack_count /proc/sys/net/netfilter/nf_conntrack_max | tr '\n' ' '";
  }
  
  if ( defined($cmd) ) {
    $$o{'ip-conntrackLoaded'} = 1;
    foreach (`$cmd`) {
      chomp;
      ($count, $max)=split;
    }
    $r = $? >> 8;
    if ( ! $r ) {
      $$o{'ip-conntrackCount'} = $count;
      $$o{'ip-conntrackMax'} = $max;
      $$o{'ip-conntrackUsage'} = sprintf('%.0f', 100*$count/$max);
    }
  } else {
    $$o{'ip-conntrackLoaded'} = 0;
    $r = 0;
  }

  $$o{'ip-conntrackFuncExecTime'} = sprintf('%.3f',scalar gettimeofday() - $t);
  $$o{'ip-conntrackExecStatus'} = $r;
  return $r;
}

sub dnsLookUpMIB {
  my $o=shift;
  my $cfg=shift;
  my $cmd="/usr/bin/host -W $config->{'cmd_timeout'}";
  my $i=1;
  my $r=0;
  my @nameservers;
  my $t=gettimeofday();

  foreach my $a (@{$cfg->{'args'}}) {
    my $srv='';
    my $ts=gettimeofday();
    if ( ! $a->{'fqdn'}) {$a->{'fqdn'} = hostname};
    
    if (defined($a->{'servers'})) {
      if ($a->{'servers'} =~/^\s*$/ ){  
        #get nameservers
        @nameservers = (); 
        open (ERC, '/etc/resolv.conf');
        while (<ERC>) {
          chomp;
          if (/nameserver (\S+)$/) {
            push @nameservers,$1; 
          }
        }
        close (ERC); 
      } else {
        @nameservers = split(/[,\s]*/,$a->{'servers'});
      }
    } 
    #next 'foreach' will be done at least once for each fqdn
    if ($#nameservers < 0) {push @nameservers, ' '};
    foreach $srv (@nameservers) {
      my $lookup_ips = '';
      foreach (`$cmd $a->{'fqdn'} $srv`) {
        chomp;
        my $s=$_;
        if ( /has address/ ) {
          $lookup_ips .=  (split / /,$s)[3] . ',';
        }
      }
      my $lookup_r = $? >> 8;
      $$o{"dnsIndex.$i"} = $i;
      $$o{"dnsNameLookUp.$i"} = $a->{'fqdn'}; $$o{"dnsNameLookUp.$i"} .= " at $srv" if ($srv =~ /\S/);
      $$o{"dnsNameResolvedIp.$i"} = $lookup_ips;
      $$o{"dnsLookUpExecTime.$i"} = sprintf('%.3f',scalar gettimeofday() - $ts);
      $$o{"dnsLookUpExecStatus.$i"} = $lookup_r;
      $r = $lookup_r if $lookup_r;
      $i++;
      
    }
  }
  $$o{'dnsLookUpFuncExecTime'} = sprintf('%.3f',scalar gettimeofday() - $t);
  $$o{'dnsLookUpFuncExecStatus'} = $r;
  return  $r;
}

sub osInfoMIB {
  my $o=shift;
  my $r;
  my $osRelease='';
  $$o{'osinfoRelease'} = `uname -rm`;
  $r = $? >> 8;
  return $r if $r;
  chomp $$o{'osinfoRelease'};
  if ( -f '/etc/redhat-release' ) {
    $osRelease = `head -1 /etc/redhat-release`;
    chomp $osRelease;
  } elsif ( -f '/etc/fedora-release' ) {
    $osRelease = `head -1 /etc/fedora-release`;
    chomp $osRelease;
  } elsif ( -f '/etc/debian_version' ) {
    $osRelease = `head -1 /etc/debian_version`;
    chomp $osRelease;
    $osRelease = "Debian $osRelease";
  }
  $r = $? >> 8;
  $$o{'osinfoRelease'} = "$osRelease $$o{'osinfoRelease'}";
  $$o{'osinfoSoftwareLastChange'} =  (stat('/var/lib/rpm/Packages'))[9] if ( -f '/var/lib/rpm/Packages' );
  return $r;  
}

sub lastLoginsMIB {
  my $o=shift;
  my $i=1;
  my $r;
  my $t=gettimeofday();

  foreach  (`w -h`) {
    chomp;
    $$o{"loggedInUserInfo.$i"} = $_;
    $i++;
  }
  $$o{'loggedInUsersCount'} = $i-1;

  $r = $? >> 8;

  $i=1;  
  my $cmd = ($^O =~ 'linux')?'last -20 -w':'last -20';
  foreach  (`$cmd`) {
    chomp;
    next if ( /^$/ );
    if ( /^wtmp begins/ ) {
      $$o{'wtmpBegins'} = $_;
      next;
    }

#    $$o{"lastUserIndex.$i"} = $i;
    $$o{"lastUserInfo.$i"} = $_;
    $i++;
  }
  $r = $r || ($? >> 8);
  return $r;
}

sub execCommandMIB {
  no strict "refs";
  my $o=shift;
  my $cfg=shift;
  my $mcall=shift; 
  my $i=$$mcall; 
  $$mcall=0; # the function support multicall
  my $r=0;
  my $t=gettimeofday();

  foreach my $a (@{$cfg->{'args'}}) {
    my $ts=gettimeofday();
    $$o{"cmdOutMatched.$i"} = defined($a->{'match'})?2:0;
    $$o{"cmdOutMatchString.$i"} = $a->{'match'} if defined($a->{'match'});

    foreach (`$a->{'cmd'}`) {
      chomp;
      if ( defined($a->{'match'}) and /$a->{'match'}/ ) {
        $$o{"cmdOutMatched.$i"} = 1;
        for (my $m=1;$m <= $#+; $m++) {
          $$o{"cmdOutMatchString.$i.$m"} = $$m;
        }
      }
    }

    my $cmd_r = $? >> 8;
    while ( defined($$o{"cmdIndex.$i"}) ) { $i++; } # detect if the function called previously and increase the index 
    $$o{"cmdIndex.$i"} = $i;
    $$o{"cmdExecTime.$i"} = sprintf('%.3f',scalar gettimeofday() - $ts);
    $$o{"cmdExecStatus.$i"} = $cmd_r;
    $$o{"cmdName.$i"} = (defined($a->{'description'})?$a->{'description'}:'') . "[$a->{'cmd'}]";
    $$o{"cmdMonitoringTemplate.$i"} = defined($a->{'monitoringTemplate'})?$a->{'monitoringTemplate'}:'';
    $r = $cmd_r if $cmd_r;
    $i++;
  }

  $$o{'cmdFuncExecTime'} = sprintf('%.3f',scalar gettimeofday() - $t);
  $$o{'cmdFuncExecStatus'} = $r;
  return $r;
}

sub rndcStatMIB {
  my $o=shift;
  my $cfg=shift;
  my $t=gettimeofday();
  my $r;
  if (! defined($cfg->{'args'}[0]->{'pre_cmd'}) ) {
    $cfg->{'args'}[0]->{'pre_cmd'}="rm -f $cfg->{'args'}[0]->{'stat_file'} && /usr/sbin/rndc stats";
  }
  `$cfg->{'args'}[0]->{'pre_cmd'}` unless ( $cfg->{'args'}[0]->{'pre_cmd'} eq '' );
  $r = $? >> 8;
  return $r if $r; 

  if (-f "$cfg->{'args'}[0]->{'stat_file'}") {
    foreach (`cat $cfg->{'args'}[0]->{'stat_file'}`) {
      chomp;
      if ( /(^success\s|successful answer)/ ) {
        $$o{"rndcSuccess"} = ( split /\s+/)[1];
      } elsif ( /in authoritative answer/ ) {
        $$o{"rndcAuthoritative"} = ( split /\s+/)[1];
      } elsif ( /in non authoritative answer/ ) {
        $$o{"rndcNonAuthoritative"} = ( split /\s+/)[1];
      } elsif ( /(^nxrrset\s|in nxrrset)/ ) {
        $$o{"rndcNxrrset"} = ( split /\s+/)[1];
      } elsif ( /(^failure\s|in SERVFAIL)/ ) {
        $$o{"rndcServfail"} = ( split /\s+/)[1];
      } elsif ( /(^nxdomain\s|in NXDOMAIN)/ ) {
        $$o{"rndcNxdomain"} = ( split /\s+/)[1];
      } elsif ( /(^recursion\s|caused recursion)/ ) {
        $$o{"rndcRecursion"} = ( split /\s+/)[1];
      } elsif ( /duplicate queries/ ) {
        $$o{"rndcDuplicate"} = ( split /\s+/)[1];
      } elsif ( /queries dropped/ ) {
        $$o{"rndcDropped"} = ( split /\s+/)[1];
      } elsif ( /other query failures/ ) {
        $$o{"rndcOtherFailures"} = ( split /\s+/)[1];
      }
    }
    $r = $? >> 8;
  } else {
    $r = -1;
  }
  $$o{"rndcAuthoritative"} = 0 unless defined($$o{"rndcAuthoritative"});
  $$o{"rndcNonAuthoritative"} = 0 unless defined($$o{"rndcNonAuthoritative"});
  $$o{"rndcDuplicate"} = 0 unless defined($$o{"rndcDuplicate"});
  $$o{"rndcDropped"} = 0 unless defined($$o{"rndcDropped"});
  $$o{"rndcOtherFailures"} = 0 unless defined($$o{"rndcOtherFailures"});

  $$o{'rndcFuncExecTime'} = sprintf('%.3f',scalar gettimeofday() - $t);
  $$o{'rndcExecStatus'} = $r;
  return $r;
}

sub mailqMIB {
  my $o=shift;
  my $c=0;
  my $r=0;
  my $t=gettimeofday();
  foreach (`mailq`) {
    chomp;
    last if ( /Mail queue is empty/ );
    $c++ if ( /^[\d[A-F]{10}/ );
  }
  $r = $? >> 8;
  $$o{'mailqCount'} = $c unless $r;
  $$o{'mailqFuncExecTime'} = sprintf('%.3f',scalar gettimeofday() - $t);
  $$o{'mailqExecStatus'} = $r;
  return $r;
}

sub vmstatMIB {
  my $o=shift;
  my $r=0;
  my $t=gettimeofday();

  my $c=`which vmstat`; chomp $c;
  
  if ( -x $c ) {
    foreach (`vmstat -s`) {
      chomp;
      s/^\s+//;
      my ($v,$c) = split / +/, $_, 2;
      if ( $c eq 'pages swapped in' ) {
        $$o{'vmstatPswpin'} = $v; 
      } elsif ( $c eq 'pages swapped out' ) {
        $$o{'vmstatPswpout'} = $v; 
      } elsif ( $c eq 'interrupts' ) {
        $$o{'vmstatInterrupts'} = $v; 
      } elsif ( $c eq 'CPU context switches' ) {
        $$o{'vmstatCntxSwitches'} = $v; 
      }     
    }

    $r = $? >> 8;
  } else {
    logMessage(LOG_WARNING, "vmstat is not found");
    $r = -1;
  }
  $$o{'vmstatFuncExecTime'} = sprintf('%.3f',scalar gettimeofday() - $t);
  $$o{'vmstatExecStatus'} = $r;
  return $r;
}

sub pingStatMIB {
  my $o = shift;
  my $cfg = shift;
  my $mcall=shift;
  my $i=$$mcall;
  $$mcall=0; # the function support multicall

  my $r = 0;
  my $t = gettimeofday();

  foreach my $a ( @{$cfg->{'args'}} ) {
    my $ts = gettimeofday();
    my $count = defined($a->{'count'})?$a->{'count'}:10;
    my $options = defined($a->{'options'})?$a->{'options'}:'';
    foreach (`ping $options -c $count $a->{'host'}`) {
      chomp;
      if ( /^(\d+).+transmitted.+ (\d+).+received.+ ([\d\.]+).+loss/ ) {
        $$o{"pingTransmitted.$i"} = $1;
        $$o{"pingReceived.$i"} = $2;
        $$o{"pingLoss.$i"} = $3;
      } elsif ( /^(rtt|round-trip).+= ([\d\.]+)\/([\d\.]+)\/([\d\.]+)/ ) {
        $$o{"pingRttMin.$i"} = $2;
        $$o{"pingRttAvg.$i"} = $3;
        $$o{"pingRttMax.$i"} = $4;
      }
    }

    my $cmd_r = $? >> 8;
    $$o{"pingIndex.$i"} = $i;
    $$o{"pingHost.$i"} = $a->{'host'} . ( defined($a->{'description'})?" \[$a->{'description'}\]":"" );
    $$o{"pingExecTime.$i"} = sprintf('%.3f',scalar gettimeofday() - $ts);
    $$o{"pingExecStatus.$i"} = $cmd_r;
    $r = $cmd_r if $cmd_r;
    $i++;
  }

  $$o{'pingFuncExecTime'} = sprintf('%.3f',scalar gettimeofday() - $t);
  $$o{'pingFuncExecStatus'} = $r;
  return $r;
}

my @diskStats;

sub diskStatsMIB {
  my $t = gettimeofday();
  my $o=shift;
  my $cfg=shift;
  my $i=1;
  my $r=0;


  my $get_diskstats_command = $cfg->{'args'}[0]->{'command'} || 'cat /proc/diskstats';
  my $match_re = $cfg->{'args'}[0]->{'match'} || '[hsv]d[a-z]+\\d*|c\\d+t\\d+d\\d+s\\d+$|cciss\/c\\d+d\\d+p\\d+|dm-\\d+';

  logMessage(LOG_DEBUG, "executing $get_diskstats_command");

  foreach (`$get_diskstats_command 2>/dev/null`) {
    # get contents of a file, split it and fill in the data
    # pretty simple, read mib for description
    # or look at https://www.kernel.org/doc/Documentation/iostats.txt
    # to read the same with more details
    my @block_devices_with_stats = split ;
    next unless $#block_devices_with_stats == 13;
    unless ( $block_devices_with_stats[2] =~ $match_re){
        next;
    }
    logMessage(LOG_DEBUG, "Block device $block_devices_with_stats[2] matched against $match_re");
    $$o{"diskStatsIndex.$i"}                    = $i;
    $$o{"diskStatsName.$i"}                     = $block_devices_with_stats[2];
    $$o{"diskStatsReadsCompleted.$i"}           = $block_devices_with_stats[3];
    $$o{"diskStatsReadsMerged.$i"}              = $block_devices_with_stats[4];
    $$o{"diskStatsSectorsRead.$i"}              = $block_devices_with_stats[5];
    $$o{"diskStatsMsSpentReading.$i"}           = $block_devices_with_stats[6];
    $$o{"diskStatsWritesCompleted.$i"}          = $block_devices_with_stats[7];
    $$o{"diskStatsWritesMerged.$i"}             = $block_devices_with_stats[8];
    $$o{"diskStatsSectorsWritten.$i"}           = $block_devices_with_stats[9];
    $$o{"diskStatsMsSpentWriting.$i"}           = $block_devices_with_stats[10];
    $$o{"diskStatsIOInProgress.$i"}             = $block_devices_with_stats[11];
    $$o{"diskStatsMsSpentInIO.$i"}              = $block_devices_with_stats[12];
    $$o{"diskStatsWeightedMsSpentInIO.$i"}      = $block_devices_with_stats[13];
    $$o{"diskStatsMeasurementTimestampMs.$i"}   = int(time*1000);
    if (exists($diskStats[$i])) {
        # time spent in io divided by overall time and multiplied by 100%
        # this displays percentage of time in io in the last cycle
        if ( $$o{"diskStatsMeasurementTimestampMs.$i"} - $diskStats[$i]{"diskStatsMeasurementTimestampMs.$i"} > 0 ) {
            $$o{"diskStatsUtilisation.$i"}      = ( $$o{"diskStatsMsSpentInIO.$i"}            - $diskStats[$i]{"diskStatsMsSpentInIO.$i"} )
                                                / ( $$o{"diskStatsMeasurementTimestampMs.$i"} - $diskStats[$i]{"diskStatsMeasurementTimestampMs.$i"} )
                                                * 100;
        } else {
            $$o{"diskStatsUtilisation.$i"}      = 0;
        }
        if ( $$o{"diskStatsReadsCompleted.$i"} + $$o{"diskStatsWritesCompleted.$i"} - $diskStats[$i]{"diskStatsTransfers.$i"}  > 0 ){
            $$o{"diskStatsAvgWait.$i"}          = ( $$o{"diskStatsMsSpentInIO.$i"}    - $diskStats[$i]{"diskStatsMsSpentInIO.$i"} )
                                                / ( $$o{"diskStatsReadsCompleted.$i"} + $$o{"diskStatsWritesCompleted.$i"} - $diskStats[$i]{"diskStatsTransfers.$i"} );
        } else {
            $$o{"diskStatsAvgWait.$i"}          = 0;
        }
    } else {
        $$o{"diskStatsUtilisation.$i"}          = 0;
        $$o{"diskStatsAvgWait.$i"}              = 0;
    };
    $diskStats[$i]{"diskStatsMsSpentInIO.$i"} = $$o{"diskStatsMsSpentInIO.$i"};
    $diskStats[$i]{"diskStatsMeasurementTimestampMs.$i"} = $$o{"diskStatsMeasurementTimestampMs.$i"};
    $diskStats[$i]{"diskStatsTransfers.$i"} = $$o{"diskStatsReadsCompleted.$i"} + $$o{"diskStatsWritesCompleted.$i"};
    $i++;
  }

  # prepare return value and fill in some statistics of execution
  $r = $? >> 8;
  $$o{'diskStatsCmdExecStatus'} = $r;
  $$o{'diskStatsCmdExecTime'} = sprintf('%.3f',scalar gettimeofday() - $t );

  return $r;

}

my $asvStats;
my $asvFirstRun;

sub asvMib {
  sub getnum {
    use POSIX qw(strtod);
    my $str = shift;
    $str =~ s/^\s+//;
    $str =~ s/\s+$//;
    $! = 0;
    my($num, $unparsed) = strtod($str);
    if (($str eq '') || ($unparsed != 0) || $!) {
      return undef;
    } else {
      return $num;
    }
  }
  sub numeric {
    my $check = shift;
    return undef if ( ! defined($check));
    return defined getnum($check)? $check : undef
  }

  my $t = gettimeofday();
  my $o=shift;
  my $cfg=shift;
  my $i=0;
  my $r=0;

  foreach my $a ( @{$cfg->{'args'}} ) {
    $i++;
    my $cmd = $a->{'cmd'};
    #next unless defined($cmd);

    my $re = $a->{'re'} || '\{\}';
    my @hosts = glob $a->{'hosts'};
    my $min;
    my $max;
    my $sum;
    my $host_num=0;
    $sum=0;

    $$o{"asvEntryIndex.$i"} = $i;
    $$o{"asvCmd.$i"} = $cmd;
   
    logMessage(LOG_DEBUG, "fetching data for '$cmd' from @hosts");

    foreach my $host (@hosts){
      $host_num++;
      my $val;
      my $current_cmd = $cmd;
      $current_cmd =~ s/$re/$host/g;
      my $output = `$current_cmd`;
      if( $output =~ /"?([0-9.]+)"?$/ ) {
        $val = numeric($1) ; 
      } else {
        undef $val;
      }
      if ( ! defined $val) {
        logMessage(LOG_WARNING, "Got no result from $host");
        undef $asvStats->{$cmd}[$i]->{$host};
        $r=1;
      } else {
        logMessage(LOG_NOTICE, "Got '$val' from $host");

        my $oldVal = 0;
        if (! (defined $a->{'gauge'} && $a->{'gauge'} =~ /^true$/)) {
          if (! defined $asvStats->{$cmd}[$i]->{$host}) {
            logMessage(LOG_NOTICE, "Old value for $host is undefined");
          }
          $oldVal = $asvStats->{$cmd}[$i]->{$host} || $val;
        }
        # difference is either the value itself or difference with previous value
        # if oldVal is undefined (a gauge) $val is unchanged
        my $difference = $val - $oldVal;
        # if old value was big and the current is small - there was at least $val events
        # if $val is less than 0, this still works as intended (e.g. if gauge)
        if ($difference < 0) { $difference = $val; }
        $min = defined $min && $min < $difference ? $min : $difference;
        $max = defined $max && $max > $difference ? $max : $difference;
        $sum = $sum + $difference;

        # if the cmd is marked as gauge don't calculate differences for it. 
        # use difference only for 'derive' like things (as per rrd)
        if (! (defined $a->{'gauge'} && $a->{'gauge'} =~ /^true$/)) {
          logMessage(LOG_DEBUG, "not a gauge value");
          $asvStats->{$cmd}[$i]->{$host} = $val
        };

        $$o{"asvHostIndex.$i.$host_num"} = $host_num;
        $$o{"asvHostName.$i.$host_num"} = $host;
        $$o{"asvHostValue.$i.$host_num"} = $val;
        logMessage(LOG_DEBUG, "Difference with old is $difference");
      }
    }
    if (! defined($asvFirstRun->{$cmd})) {
      logMessage(LOG_DEBUG, "first run for '$cmd'");
      $asvFirstRun->{$cmd} = 0;
    } else {
      $sum /= $config->{'poller_cycle'};
      $$o{"asvCmdMin.$i"} = $min;
      $$o{"asvCmdMax.$i"} = $max;
      $$o{"asvCmdSum.$i"} = $sum;
      $$o{"asvCmdAvg.$i"} = $sum/$host_num;
    }
  }

  $$o{'asvCmdExecStatus'} = $r;
  $$o{'asvCmdExecTime'} = sprintf('%.3f',scalar gettimeofday() - $t );

  return $r;

}

sub nginxStatusMIB {
  my $o=shift;
  my $cfg=shift;
  my $mcall=shift;
  my $i=$$mcall;
  $$mcall=0; # the function support multicall

  my $cmd="curl -s -m $config->{'cmd_timeout'} ";
  my $r=0;
  my $t=gettimeofday();

  foreach my $a (@{$cfg->{'args'}}) {
    my $ts=gettimeofday();
    foreach (`$cmd $a->{'status_url'}`) {
      chomp;
      if ( /^Active connections/ ) {
        $$o{"nginxActiveConnections.$i"} = ( split /: / )[1];
      } elsif ( /\d+\s+\d+\s+\d+/ ) {
        s/^\s+//;
        my @d = split;
        $$o{"nginxAccepts.$i"}  = $d[0]; 
        $$o{"nginxHandled.$i"}  = $d[1]; 
        $$o{"nginxRequests.$i"} = $d[2]; 
      } elsif ( /^Reading/ ) {
        my @d = split;
        $$o{"nginxReading.$i"} = $d[1]; 
        $$o{"nginxWriting.$i"} = $d[3]; 
        $$o{"nginxWaiting.$i"} = $d[5]; 
      }
    }

    my $cmd_r = $? >> 8;
    while ( defined($$o{"nginxIndex.$i"}) ) { $i++; } # detect if the function called previously and increase the index 
    $$o{"nginxIndex.$i"} = $i;
    $$o{"nginxCmdExecTime.$i"} = sprintf('%.3f',scalar gettimeofday() - $ts);
    $$o{"nginxCmdExecStatus.$i"} = $cmd_r;
    $$o{"nginxUrl.$i"} = (defined($a->{'description'})?$a->{'description'}:'') . "[$a->{'status_url'}]";
    $r = $cmd_r if $cmd_r;
    $i++;
  }
  $$o{'nginxFuncExecTime'} = sprintf('%.3f',scalar gettimeofday() - $t);
  $$o{'nginxFuncExecStatus'} = $r;
  return  $r;
}

return 1;