bot-analyze

#!/usr/bin/perl
use strict;
my %agent;
my %action;
my %bandwidth;
my $actions;
my $hits;
my $bandwidth;
my $bot_bandwidth;
my $bot_hits;
while (<STDIN>) {
  s/\\"/'/g;
  my ($action, $code, $bytes, $referrer, $agent)
   = m/"(.*?)" (\d+) (\d+|-) .*?"(.*?)".*?"(.*?)"/ or die "Cannot parse:\n$_";
  $hits++;
  $bandwidth += $bytes;
  next unless $agent =~ /([a-z]*bot[a-z]*)/i;
  $agent =~ /http:\/\/([^ \/]+)/;
  my $key = $1 || 'no url';
  $agent{$key}++;
  $bandwidth{$key} += $bytes;
  $bot_hits++;
  $bot_bandwidth += $bytes;
  if ($action =~ /action=/i) {
    $actions++;
    $action{$key}++;
  }
}
my @result = sort {$agent{$b} <=> $agent{$a}} keys %agent;

print "    ----------------------------Bandwidth-------Hits-------Actions\n";
printf "%30s %9dM %10d\n", 'Everybody',
  $bandwidth / 1024 / 1024, $hits;
printf "%30s %9dM %10d   %3d%%   %3d%%\n", 'All Bots',
  $bot_bandwidth / 1024 / 1024, $bot_hits, 100, 100 * $actions / $bot_hits;
print "    --------------------------------------------------------------\n";
foreach my $key (@result) {
  printf "%30s %9dK %10d   %3d%%   %3d%%\n", $key,
    $bandwidth{$key} / 1024,
    $agent{$key},
    100 * $agent{$key} / $bot_hits,
    100 * $action{$key} / $agent{$key};
}