#!/usr/bin/perl
#++
#   $Id: bigstat,v 1.37 2010/02/03 02:51:50 az Exp $
# 
#   File:		bigstat
#   Date:		Sat Jul 10 20:21:23 2004
#   Author:		Alexander Zangerl (az)
# 
#   Abstract: statistics gatherer for mrtg
#	one script to rule them all, one script to bind them,
#	one script to bring them all and in the tempdir bind them
#	in the land of mrtg where there is no snmpd.
#	
#   License: GPL v1 or v2
#
#--
use strict;
use Getopt::Std;
use LWP::UserAgent;
use Net::Telnet;

my %conf;
chomp(my $hostname=`/bin/hostname`);

# args: which tests to enable
die "usage:\t$0 [-aclfmnsS] [-i if,if...] [-d diskspec,diskspec...]\n",
    "\t[-t targetdir] [-p persistdir] -H disk,disk\n",
    "[-P name:pathi:fieldnr:patho:fieldnr]\n",
    "[-D uname:passwd]\n",
    "[-x name:host:uname:passwd]\n",
    "diskspec is name=mountpoint,name=mp...\n\n",
"-a: apache access log for 2xx,3xx,401 vs. the rest
-c: cpu time user+nice vs. user+nice+system
-l: loadavg 1min vs. 15min
-f: iptables packet drops
-s: maillog good mail vs. spam
-n: newslog inbound vs. outbound articles (inaccurate)
-m: used memory vs. swap
-i: interface input bytes vs. output
-d: disk used blocks vs. inodes
-t: folder for current counter files
-p: folder for keeping offsat and status files
-T: netfilter tracked connections
-S: sensors, first two temperatures
-H: harddisk temperature, device as arg
-P: output name, 2x path to a file and field index (ws-separated)
-x: Thomson Adsl stats via telnet cli
-D: Internode quota
-I: inetd-mode, all output to stdout\n"
    if (!getopts("aclfmsni:d:t:p:TSH:IP:D:x:",\%conf));

my $targetdir=$conf{t}||"/tmp/bigstat";
if (!-d $targetdir)
{
    mkdir($targetdir,0700) or die "can't mkdir $targetdir: $!\n";
}
my $persistdir=$conf{p}||"/var/lib/bigstat";
if (!-d $persistdir)
{
    mkdir($persistdir,0700) or die "can't mkdir $persistdir: $!\n";
}

# get uptime
open(F,"/proc/uptime") or die "can't read /proc/uptime: $!\n";
my $uptime=int((split(/\s+/,<F>))[0]);
close F;
$uptime=sprintf("%d days, %d:%d:%d",$uptime/86400,
		$uptime%86400/3600,
		$uptime%86400%3600/60,
		$uptime%60);

if ($conf{a})
{
    my($in,$out)=(0) x 2;
    # apache access logs: logtail on /var/log/apache/access.log
    my $offsetfile="$persistdir/apache.offset";
    my $logfile="/var/log/apache/access.log";
    
    ($in,$out)=readstat("apache");
    foreach (logtail($logfile,$offsetfile))
    {
	# using this custom format:
    
	# heffalump.snafu.priv.at 210.8.28.121 - - [10/Jul/2004:17:19:41 +1000] 
	# "GET /mrtg/ HTTP/1.0" 304 - "-" "Googlebot/2.1 
	# (+http://www.googlebot.com/bot.html)"

	# considering 2xx, 3xx and 401 as ok

	if (/^\S+ \S+ \S+ \S+ \[[^\]]+\] \"[^\"]+\" (\d+)/)
	{
	    $1=~/^([23]..|401)$/? $in++: $out++;
	}
    }
    writestat("apache",$in,$out);
    writeout("apache",$in,$out);
}

if ($conf{c})
{
    my($in,$out)=(0)x2;
    # cpu state logging
    open(F,"/proc/stat") or die "can't open /proc/stat: $!\n";
    my $cpuline=<F>;
    close F;

    if ($cpuline=~/^cpu\s+(\d.*)$/)
    {
	# in 2.6: added columns iowait,irq,softirq,steal
	my ($user,$nice,$system,$idle,$io,$irq,$int,$steal)=split(/\s+/,$1);
	
	# we output user and nice and everything bar idle as total activity
	writeout("cpu",$user+$nice,$user+$nice+$system+$io+$irq+$int+$steal);
    }
}

# read files directly
if ($conf{P})
{
    my ($name,$fni,$fii,$fno,$fio)=split(/:/,$conf{P});
    my($in,$out)=(0)x2;
    
    open(F,$fni) or die "can't open $fni: $!\n";
    my $datai=<F>;
    close F;
    open(F,$fno) or die "can't open $fno: $!\n";
    my $datao=<F>;
    close F;
    $in=(split(/\s+/,$datai))[$fii];
    $out=(split(/\s+/,$datao))[$fio];
    writeout($name,$in,$out,"in $fni:$fii out $fno:$fio");
}

if($conf{T})
{
    my ($in,$out)=0;

    # old: read /proc/net/ip_conntrack or nf_conntrack, count lines, print
    # better: read count from /proc/sys/net/netfilter/nf_conntrack_count
    open(F,"/proc/sys/net/netfilter/nf_conntrack_count") 
	or die "can't open /proc/sys/net/netfilter/nf_conntrack_count: $!\n";
    my $conns=<F>;
    chomp $conns;
    close F;
    writeout("conns",$conns,$conns);
}


if ($conf{l})
{
    my($in,$out)=(0)x2;
    # cpu load logging
    open(F,"/proc/loadavg") or die "can't open /proc/loadavg: $!\n";
    my ($now,$five,$fifteen)=split(/\s+/,<F>);
    close F;

    writeout("loadavg",int($now*100),int($fifteen*100));
}

if ($conf{m})
{
    # free memory logging
    open(F,"/proc/meminfo") or die "cant open meminfo: $!\n";
    my (@max,@values,%data);
    foreach my $line (<F>)
    {
	if ($line =~ /^(Mem|Swap)(Total|Free):\s+(\d+)\s+kB/)
	{
	    $data{$1.$2}=$3*1024;
	}
    }
    close F;
    $values[0]=$data{MemTotal}-$data{MemFree};
    $values[1]=$data{SwapTotal}-$data{SwapFree};
    writeout("memory",@values,$data{MemTotal}." mem, "
	     .$data{SwapTotal}." swap");
}

if ($conf{d})
{
    # disk blocks, inodes logging
    # args: name=mp,name=mp...
    foreach my $set (split(/\s*,\s*/,$conf{d}))
    {
	my ($name,$fs)=split(/=/,$set);

	my (@max,@values);
	for my $cmd ("/bin/df -k","/bin/df -i")
	{
	    my $val=join(" ",(`$cmd $fs`)[1..2]);
	    my ($max,$used)=(split(/\s+/,$val))[1,2];
	    push @max,$max;
	    push @values,$used;
	}
	writeout("disk-$name",@values,"$max[0] kb, $max[1] inodes");
    }
}

if ($conf{H})
{
    # disk temperature via hddtemp -n
    # args: devnames without /dev/, comma separated
    for my $disk (split(/,/,$conf{H}))
    {
	my $temp=`/usr/sbin/hddtemp -n /dev/$disk`;
	chomp $temp;
	writeout("temp-$disk",$temp,0,"temperature $disk");
    }
}

if ($conf{S})
{
    # first two temperatures in sensors output
    my (@values, @names);
    for my $l (`sensors`)
    {
	chomp $l;
	if ($l=~/^(\S+ temp):\s*[+-]?(\d+\.\d+)/i)
	{
	    push @values,int($2+0.5);
	    push @names,$1;
	    last if (@values==2);
	}
    }
    writeout("temperature",@values,join(", ",@names));
}

if ($conf{f})
{
    # remember previous counts and detect wraps!
    my($old,undef)=readstat("firewall");
    # count dropped packets

    # Chain udp_in (1 references)
#    pkts      bytes target     prot opt in     out     source               destination         
#11232386 3228662572 ACCEPT     udp  --  any    any     anywhere             anywhere

    my ($new);
    open(F,"/sbin/iptables -n -x -v -L|") or die "can't fork iptables: $!\n";
    while(<F>)
    {
	if (/^\s*(\d+)\s+\d+\s+(TARPIT|DROP|REJECT)/)
	{
	    $new+=$1;
	}
    }
    close F;
    writestat("firewall",$new,$new);
    
    # wrapped? then forget the old value
    $old=$new if (!defined $old || $new<$old);
    # perhour doesn't work with absolute readings...
    $new=($new-$old)*3600;
    writeout("firewall",$new,$new);
}

if ($conf{i})
{
    # interface logging
    # args: if,if,if...
    my %ifs;
    foreach (split(/\s*,\s*/,$conf{i}))
    {
	$ifs{$_}=[readstat($_)];
    }
    open(F,"/proc/net/dev") or die "cant open /proc/net/dev: $!\n";
    foreach my $line (<F>)
    {
	# tosspot: has spaces after colon, cft and heffalump don't...
	if ($line =~s/^\s*(\S+):\s*(\S.+)$/$2/)
	{
	    my $ifname=$1;
	    next if (!$ifs{$ifname});
	    my ($in,$out)=(split(/\s+/,$2))[0,8];
	    writestat($ifname,$in,$out);

	    # detect reboot wraps
	    $ifs{$ifname}->[0]=$in
		if (!defined($ifs{$ifname}->[0]) || $ifs{$ifname}->[0]>$in);
	    $ifs{$ifname}->[1]=$out
		if (!defined($ifs{$ifname}->[1]) || $ifs{$ifname}->[1]>$out);
	    $ifs{$ifname}=[$in-$ifs{$ifname}->[0],$out-$ifs{$ifname}->[1]];
	}
    }
    close F;
    foreach (keys %ifs)
    {
        writeout("$_",$ifs{$_}->[0],$ifs{$_}->[1]);
    }
}

if ($conf{s})
{
    my($in,$out)=(0)x2;
    my($smtpin,$smtprej)=0;
    # email/spam stats: logtail on /var/log/maillog
    # good is $in, crap is $out
    # smtp sessions that reach data in $smtpin, early rejects are in $smtprej
    my $offsetfile="$persistdir/mail.offset";
    my $logfile="/var/log/maillog";
    
    ($in,$out)=readstat("mail");
    ($smtpin,$smtprej)=readstat("smtp");
    foreach (logtail($logfile,$offsetfile))
    {
	if (/sm-mta\[\d+\]: .+, stat=(virus|too much spam|sent)/i)
	{
	    (lc($1) eq "sent") and $in++ or $out++;
	}
	elsif (/mimedefang\[\d+\]: filter_(sender|recipient) (rejected|tempfailed)/)
	{
	    $smtprej++;
	}
	elsif (/sm-mta\[\d+\]: .+, nrcpts=(\d+)/)
	{
	    ($1 > 0) && $smtpin++;
	}
    }
    writestat("mail",$in,$out);
    writeout("mail",$in,$out);
    writestat("smtp",$smtpin,$smtprej);
    writeout("smtp",$smtpin,$smtprej);
}

if ($conf{n})
{
    my($in,$out)=(0)x2;
    # news stats: logtail on /var/log/news/news
    # inbound articles should be exact, outbound are the peers things 
    # are offered to. FIXME: parse innfeed info for that
    my $offsetfile="$persistdir/news.offset";
    my $logfile="/var/log/news/news";
    
    ($in,$out)=readstat("news");
    foreach (logtail($logfile,$offsetfile))
    {
	if (/^\w+\s+\d+\s+\d+:\d+:\d+\.\d+\s+(\S)\s+\S+\s+<[^>]+>\s+\d+\s+(.*)$/)
	{
	    my ($status,$cand)=($1,$2);

	    if ($status =~ /^[j+c]$/)
	    {
		$in++;
		my @all=split(/\s+/,$cand);
		$out+=@all;
	    }
	}
    }
    writestat("news",$in,$out);
    writeout("news",$in,$out);
}

# internode adsl usage, in MB
if ($conf{D})
{
    my ($uname,$pwd)=split(/:/,$conf{D},2);
    my($in,$out)=(0) x 2;
    my $url="https://customer-webtools-api.internode.on.net/cgi-bin/padsl-usage";
    
    my $ua=LWP::UserAgent->new;
    $ua->timeout(10);
    my $r=$ua->post($url,{username=>$uname,password=>$pwd});
    my $text=$r->content;
 
    # mb used, total, rollover and excess
    if ($text=~/(\d+\.\d+)\s+(\d+)\s+(.*)$/)
    {
	$in=int($1);
	$out=int($2);
	writeout("internode",$in,$out,$3);
    }
}

# read dsl stats from thomson modems, via telnet (*sigh*)
if ($conf{x})
{
    my ($name,$host,$uname,$pwd)=split(/:/,$conf{x},4);
    
    my $t=Net::Telnet->new(Timeout=>10,
			   Prompt=>"/\{$uname\}=>/");
    # $t->dump_log(*STDOUT);
    $t->open($host) || die "can't connect to $host: $!\n";
    $t->login($uname,$pwd) || die "can't login on host $host: $!\n";
    my @info = $t->cmd(":xdsl info expand=enabled counter_period_filter=current");
    $t->close;

    my ($thisuptime,$resets,$errorsecs,$down,$up,$attdn,$attup,$margdn,$margup);

    for my $l (@info)
    {
	if ($l=~/^\s*Up time \(Days hh:mm:ss\):\s*(\d+)\s+days?,\s+(\d+):(\d+):(\d+)\s*$/)
	{
	    my ($days,$hrs,$min,$secs)=($1,$2,$3,$4);
	    $thisuptime="${days} days, $hrs:$min:$secs";
	}
	elsif ($l=~/^\s*Number of reset:\s+(\d+)$/)
	{
	    $resets=$1;
	}
	elsif ($l=~/^\s*Error second \(ES\):\s*(\d+)\s*$/)
	{
	    $errorsecs=$1;

	}
	elsif ($l=~/^\s*Payload rate \[kbps\]:\s+(\d+)\s+(\d+)\s*$/)
	{
	    ($down,$up)=($1,$2);
	}
	elsif ($l=~/^\s*Attenuation \[dB\]:\s+(\d+\.\d+)\s+(\d+\.\d+)\s*$/)
	{
	    # rounding for mrtg
	    ($attdn,$attup)=(int($1+0.5),int($2+0.5));

	}
	elsif ($l=~/^\s*Margins \[dB\]:\s+(\d+\.\d+)\s+(\d+\.\d+)\s*$/)
	{
	    ($margdn,$margup)=(int($1+0.5),int($2+0.5));
	}
    }
    writeout("$name-errors",$errorsecs,$resets,undef,$thisuptime);
    writeout("$name-speed",$down,$up,undef,$thisuptime);
    writeout("$name-attenuation",$attdn,$attup,undef,$thisuptime);
    writeout("$name-margins",$margdn,$margup,undef,$thisuptime);
}

sub readstat
{
    my ($service)=@_;
    my($in,$out)=(0)x2;
    my $statfile="$persistdir/$service.status";

    if (!open(F,$statfile)) 
    {
	warn "can't open $statfile: $!\n";
	return undef;
    }
    else
    {
	my $laststatus=<F>;
	close F;
	($in,$out)=split(/\s+/,$laststatus);
    }
    return($in,$out);
}
    
sub writestat
{
    my ($service,$in,$out)=@_;
    my $statfile="$persistdir/$service.status";

    open(F,">$statfile") 
	or die "cant write to $statfile: $!\n";
    print F "$in\t$out\n";
    close F;
}

sub writeout
{
    my ($service,$in,$out,$extra,$thisuptime)=@_;

    $thisuptime||=$uptime;

    if (!$conf{I})
    {
	open(F, ">$targetdir/$service") 
	    or die "can't write $targetdir/$service: $!\n";
	print F "$in\n$out\n$thisuptime\n$hostname $service $extra\n";
	close F;
    }
    else
    {
	print "$service\n$in\n$out\n$thisuptime\n$hostname $service $extra\n";
    }
}


# code extracted/extended from logtail
# Author: Paul Slootman <paul@debian.org> 2001/03/14
# Licence: GPL
# args: logfile and optional offsetfile
# returns: array of loglines
# dies on errors
sub logtail
{
    my @result;
    my ($logfile, $offsetfile) = @_;
    die "File $logfile cannot be read.\n"
	if (! -f $logfile);
    # offsetfile not given, use .offset/$logfile in the same directory
    $offsetfile = $logfile . '.offset' unless ($offsetfile);

    die "File $logfile cannot be read.\n" if (!open(LOGFILE, $logfile));

    my ($inode, $offset) = (0, 0);
    if (open(OFFSET, $offsetfile)) 
    {
        chomp($inode=<OFFSET>);
	chomp($offset=<OFFSET>) if (defined $inode);
    }

    my ($ino, $size);
    die "Cannot get $logfile file size.\n" 
	unless ((undef,$ino,undef,undef,undef,undef,undef,$size) 
		= stat $logfile);
    
    if ($inode == $ino) 
    {
	return if $offset == $size; # short cut
	$offset = 0	if ($offset > $size); #  no warning here
    }
    $offset = 0 if ($inode != $ino || $offset > $size);
    
    seek(LOGFILE, $offset, 0);
    @result=<LOGFILE>;
    $size = tell LOGFILE;
    close LOGFILE;

    die "File $offsetfile cannot be created. Check your permissions.\n"
	if (!open(OFFSET, ">$offsetfile"));
    
    die "Cannot set permissions on file $offsetfile\n"
	if (!chmod 0600, $offsetfile);

    print OFFSET "$ino\n$size\n";
    close OFFSET;
    return @result;
}