#!/usr/bin/perl -Tw
#-----------------------------------------------------------------------
# History
#-----------------------------------------------------------------------
# General house cleaning 2003-10-05 JR
#-----------------------------------------------------------------------
# Constants
#-----------------------------------------------------------------------
# Column info, used by both print_column_labels() and
# print_table_data() and porturl().
my ($PORT_WIDTH)=6;
my (@COLWIDTH) =
(-15, -15, -6, $PORT_WIDTH, $PORT_WIDTH,
8, 8, 8, 8, -13, -13, -6, -6);
my (@COLCOLOR) = (0, 1, 0, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1);
# Map protocol name to number
my %PROTO = ( 'tcp'=>6, 'udp'=>17, 'icmp'=>1);
# Map protocol number to name
my %PROTO_LABEL = (6=>'tcp', 17=>'udp', 1=>'icmp');
# Map talker number to label
my @TALKER_LABEL = (0,'L','R');
# Initialize values to prevent initialization error
# messages when print_form() is called (2003-02-20 JR)
my (%arg) = (
"qmin" => "",
"qmax" => "",
"ip_address" => "",
"local_port" => "",
"remote_port" => "",
"line_limit" => "",
"line_incr" => "",
"data_min" => "",
"data_max" => "",
"proto" => "any",
"first_talker" => "any",
"last_talker" => "any",
);
my (%search);
#-----------------------------------------------------------------------
# Initialize Directories, Modules, Output
#-----------------------------------------------------------------------
# make adjust-cgi will search and replace the below from
# what the configure script has detected as the ipaudit homedir.
BEGIN {
unshift (@INC,"/home/ipaudit/"); # Adjusted via adjust-cgi
}
# Set output to immediate flush
$| = 1;
# Print HTTP header now so subsequent output will be legal
print "Content-type: text/html\n\n";
use strict;
use ipaudit_config;
# Don't use advanced data parsing if not present
eval "use Time::ParseDate";
my ($Is_ParseDate_Found) = $@ eq "";
if ($Is_ParseDate_Found) {
use POSIX;
}
# Untaint environment
delete @ENV{'PATH', 'IFS', 'CDPATH', 'ENV', 'BASH_ENV'};
require 5.004; # needed for tie interface
#-----------------------------------------------------------------------
# Declare Variables
#-----------------------------------------------------------------------
my ($IP_DIR,$DATA_DIR,$ZCAT,$ZGREP,%conf);
my ($filemin,$filemax,
$min_date,$max_date,
$min_sec, $max_sec,
$data_max, $data_min,
$prefix_min, $prefix_max,
$ip_pattern,
$first_talker, $last_talker,
$protocol,
$line_limit, $line_incr, @files);
my (%portlist);
my ($cgidir, $t1,$t2);
my ($head,$msg);
#-----------------------------------------------------------------------
# Main
#-----------------------------------------------------------------------
# Starting time
$t1 = time();
# Read configuration file
%conf = &ipa_getconf();
# Initialize some dependent variables
$IP_DIR = &untaint ($conf{'IP_DIR'} ) || '/home/ipaudit';
$ZCAT = &untaint ($conf{'ZCAT'} ) || '/bin/zcat';
$ZGREP = &untaint ($conf{'ZGREP'} ) || '/usr/bin/zgrep';
$DATA_DIR = "$IP_DIR/data/30min";
# Cgi directory, used to call PortLookup script
($cgidir) = ($ENV{SCRIPT_NAME}=~/^(.*)\/([^\/]+)$/);
# Make sure that ZCAT and ZGREP aren't missing,
# this is a frequent source of errors.
if (not -x $ZCAT && -x $ZGREP ) {
$head = "ERROR: Script Misconfiguration";
$msg = "
ERROR
\n";
$msg .= "This script (~ipaudit/cgi-bin/SearchIpauditData) is misconfigured.
";
$msg .= "The ZCAT executable $ZCAT cannot be found. " if not -x $ZCAT;
$msg .= "The ZGREP executable $ZGREP cannot be found. " if not -x $ZGREP;
$msg .= <Be sure the following lines from the ipaudit-web.conf file
ZCAT=$ZCAT
ZGREP=$ZGREP
contain the correct file paths of the zcat and zgrep
utilities.
EOM
&croak($msg,$head);
}
# Read mapping of port number->name
&get_port_list();
# Print html header
&html_header();
# get list of all files in directory
@files = &get_files();
($min_date) = $files[ 0]=~/(\d{4}-\d{2}-\d{2}-\d{2}:*\d{2})/;
($max_date) = $files[-1]=~/(\d{4}-\d{2}-\d{2}-\d{2}:*\d{2})/;
# Print date range
printf "
Data Available from %s to %s.
\n",
$min_date, $max_date;
# Read form input
&read_arg;
# Format input stored %arg for redisplay in form.
&set_form_defaults();
# Set search parameters from from input
&set_search_param;
# Print search form
&print_form();
# If arguments present, do search
if($arg{submit} or $arg{date}) {
&read_files (@files);
}
# Print total processing time
$t2= time();
print "Total Processing time: ",$t2 - $t1, "seconds \n";
# Print footer
&html_footer();
exit;
#-----------------------------------------------------------------------
# Functions
#-----------------------------------------------------------------------
sub html_header() {
my $title = "IPAudit Log Search";
print<<"EOM";
$title
EOM
}
# Convert date in format "2002-02-13 10:30" or "2002-02-13"
# to seconds from Epoch
sub local_parsedate {
my ($date) = @_;
# use Time::ParseDate if available
if ($Is_ParseDate_Found) {
my ($sec) = eval 'parsedate($date, NO_RELATIVE => 1)';
return $sec;
}
my ($year,$month,$day);
my ($hour) = 0;
my ($min) = 0;
# Set isdst ("IS Daylight Savings Time, field 9) to
# "not available" (see 'man mktime')
if ($date=~m!(\d+)/(\d+)/(\d+)\s+(\d+):(\d+)!) {
return mktime (0,$5,$4,$3,$2-1,$1-1900,0,0,-1);
} elsif ($date=~m!(\d+)/(\d+)/(\d+)!) {
return mktime (0,0,0,$3,$2-1,$1-1900,0,0,-1);
} else {
&croak ("Cannot read date format (advanced date parsing not present)");
}
}
# Given a file name, returns the value as the unix time in seconds.
# Round down/up if optional round is negative/positive.
sub file2time {
my ($filename,$round) = @_;
# Change filename like "2002-11-07-07:30" to "2002/11/07 07:30"
if ($filename =~ /(\d{4})-(\d{2})-(\d{2})[ -](\d{2}:\d{2})/) {
$filename = "$1/$2/$3 $4";
# Change filename like "2002-11-07" to "2002/11/07 00:00"
} elsif ($filename =~ /(\d{4})-(\d{2})-(\d{2})/) {
$filename = "$1/$2/$3 00:00";
}
my ($sec) = local_parsedate ($filename);
# If round is negative, round down
if ($round < 0) {
$sec -= $sec % (-$round);
# If round is positive, round up
} elsif ($round > 0) {
$sec += ($round - $sec % $round);
}
return $sec;
}
# This gets the begining prefix - atleast so we know where to start looking.
sub time2file($ ) {
my $time_val = shift;
my ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst) = localtime($time_val);
# yay y2k!
$year += 1900;
return sprintf("%4d-%02d-%02d-%02d:%02d", $year, $mon + 1, $mday, $hour, $min);
}
# Get list of all data files in data directory
sub get_files() {
opendir(DIR, $DATA_DIR) or &croak ("Can't Open $DATA_DIR: $!\n");
# read in all the gzip files that start with a number and are really files
for (readdir(DIR)) {
next unless -f "$DATA_DIR/$_";
next unless /^(\d{4}-\d{2}-\d{2}-\d{2}:*\d{2}.txt.gz)/;
push @files, $1;
}
close(DIR);
return sort @files;
}
# Change human readable counts like '200m' or '200k' to numbers
sub str2bytes($ ) {
my $str = shift;
&croak ("str2bytes: no input") if !defined($str);
if($str =~ m/k$/i) {
$str =~ s/k$//gi;
return $str * 1024;
} elsif ($str =~ m/b$/i) {
$str =~ s/b$//gi;
return $str;
} elsif ($str =~ m/m$/i) {
$str =~ s/m$//gi;
return $str * 1024 * 1024;
} elsif ($str =~ m/g$/i) {
$str =~ s/g$//gi;
return $str * 1024 * 1024 * 1024;
} elsif ($str =~ m/^\d+$/) {
return $str;
}
&croak ("Couldn't convert str->bytes: $str\n");
}
# Convert numbers to human readable form like '200m' or '200k'
sub bytes2str($ ) {
my $bytes = shift;
&croak ("bytes2str: no input") if !defined($bytes);
if($bytes < 1024) {
return "$bytes";
} elsif ($bytes < 1048576) {
$bytes = $bytes / 1024;
return sprintf("%.1fk", $bytes);
} elsif ($bytes < 1073741824) {
$bytes = $bytes / 1048576;
return sprintf("%.2fM", $bytes);
}
$bytes = $bytes / 1073741824;
return sprintf("%.2dG", $bytes);
}
# Given search parameters, display corresponding 'connections'.
sub read_files {
my (@files) = @_;
my ($pid);
my ($line, $nprint, $nread, $nincr);
my ($datelen,$compare);
$datelen =
length($search{file_min}) > length($search{file_max}) ?
length($search{file_min}) :
length($search{file_max}) ;
# only get the files in our time range.
my (@file_list);
for (@files) {
$compare = substr($_,0,$datelen);
next unless
$search{file_min} le $compare &&
$search{file_max} ge $compare;
push @file_list, "$DATA_DIR/$_";
}
# saveing stderr - I don't care if the pipe breaks on zcat.
open(SAVERR, '>&STDERR');
open(STDERR, '>/dev/null');
if($search{ip_pattern}) {
$pid = open(FILE, "-|") or exec "$ZGREP" , $search{ip_pattern}, @file_list;
&croak ("$ZGREP exec failed: $!") unless defined($pid);
} else {
$pid = open(FILE, "-|") or exec "$ZCAT", @file_list;
&croak ("$ZCAT exec failed: $!") unless defined($pid);
}
# Print table header, column labels
&print_column_labels;
# Print data rows
$nprint = 0;
$nincr = 0;
$nread = 0;
###DEBUG #test
###DEBUG my ($k);
###DEBUG for $k (sort keys %arg) {
###DEBUG print "\$arg{$k} ($arg{$k}) \n";
###DEBUG }
###DEBUG for $k (sort keys %search) {
###DEBUG print "\$search{$k} ($search{$k}) \n";
###DEBUG }
###DEBUG #end
while(($nprint < $arg{line_limit}) && ($line = )) {
# # 0-Local IP 1-Remote IP
# # 2-Protocol (1=icmp, 6=tcp, 17=udp)
# # 3-Local Port 4-Remote Port
# # 5-Incoming (bytes) 6-Outgoing (bytes)
# # 7-Incoming (packets) 8-Outgoing (packets)
# # 9-First Packet time 10-Last Packet time
# # 11-First Packet source 12-Last Packet source (1=Local,2=Remote)
# Count number of lines read
$nread++;
# Test this input line for requested conditions
chomp $line;
my @data = split(/\s+/, $line, 13);
next if @data<13 && print "skipping (less than thirteen fields)";
# Test total byte min/max
next if ($search{data_min} > -1) && (($data[5] + $data[6]) < $search{data_min});
next if ($search{data_max} > -1) && (($data[5] + $data[6]) > $search{data_max});
if($search{lport_pattern}) {
next if $data[3] !~ m/^$search{lport_pattern}$/;
}
if($search{rport_pattern}) {
next if $data[4] !~ m/^$search{rport_pattern}$/;
}
# Select protocol
next if $search{protocol} && $data[2]!=$search{protocol};
# Compare first,last talker values
if($search{first_talker} ne "any") {
next if $data[11] ne $search{first_talker};
}
if($search{last_talker} ne "any") {
next if $data[12] ne $search{last_talker};
}
# Skip this eligible line until count of $incr is reached
next unless ++$nincr == $arg{line_incr};
$nincr = 0;
# Reformat ips
$data[0] = demunge_ip($data[0]);
$data[1] = demunge_ip($data[1]);
# Make bytes human readable
$data[5] = bytes2str($data[5]);
$data[6] = bytes2str($data[6]);
# Map protocols and talkers to their labels
$data[ 2] = $PROTO_LABEL {$data[ 2]};
$data[11] = $TALKER_LABEL[$data[11]];
$data[12] = $TALKER_LABEL[$data[12]];
# If only a one-way connection, set last field to '-'
if ($data[7]==0 || $data[8]==0) {
$data[12] = "-";
}
# Convert port numbers to port lookup url's
&porturl(\@data);
# Print this connection
&print_table_data($nprint,@data);
# Count number of lines printed
$nprint++;
}
# Close data table
&print_column_labels;
# Added by jh@dok.org
# Print Summary
print(" ");
print "** $nread lines read. \n";
if($nprint eq $arg{line_limit}) {
print
"** Max lines $nprint/$arg{line_limit} printed.";
} else {
print
"** $nprint lines printed.";
}
print("");
close(STDERR);
open(STDERR,">&SAVERR");
close(SAVERR);
}
# -- end of read files
sub print_column_labels {
my (@label) = (
"Local IP", "",
"Remote IP", "",
"Proto-", "col",
"Local", "Port",
"Remote", "Port",
"Incoming", "Bytes",
"Outgoing", "Bytes",
"Incoming", "Packets",
"Outgoing", "Packets",
"First Packet", "Time",
"Last Packet", "Time",
"First", "Talker",
"Last", "Talker"
);
my ($i,$l,$format,$line);
print "\n";
print "
";
for ($l=0;$l<2;$l++) {
for ($i=0;$i<13;$i++) {
$format = "%" . $COLWIDTH[$i] . "s";
$format = "$format"
if $COLCOLOR[$i];
printf " $format", $label[2*$i+$l];
}
print "\n";
}
print "
\n";
print "\n";
}
#
# Print data with alternate background line coloring and
# column keyed text colors
#
sub print_table_data {
my ($cnt,@data) = @_;
my ($i,$format);
if ($cnt % 2 == 0) {
print "
";
print "\n";
}
# converts an ip to ipaudit style
sub munge_ip($ ) {
my $ip = shift;
&croak ("munge_ip: No ip defined") unless defined($ip);
# taken almost directly from SearchIpauditData
return sprintf "%03d.%03d.%03d.%03d", ($ip =~/(\d{1,3})\.(\d{1,3}).(\d{1,3}).(\d{1,3})/);
}
# converts ip to "normal" form
sub demunge_ip {
my ($ip) = @_;
&croak ("demunge_ip: No ip defined") unless defined($ip);
return sprintf "%d.%d.%d.%d", ($ip =~/0*(\d+)\.0*(\d+).0*(\d+).0*(\d+)/);
}
# convert an ip fragment to ipaudit style
sub munge_ip_frag ($ ) {
return join ".", map ( sprintf ("%03d", $_) , split (/\./, shift) ) ;
}
# Print search form, setting all defaults
sub print_form() {
# Date format example depends on date subroutine used
my ($date_format_example) = $Is_ParseDate_Found ?
"Eg: yesterday, -2 days, last Wednesday, 2001-03-13-12:30" :
"Eg: 2002-03-13-12:30";
# Set defaults for SELECT boxes
my (%SELECTED) = ();
my ($key);
# Initialize values for %SELECT
$SELECTED{proto}{any} = "";
$SELECTED{proto}{tcp} = "";
$SELECTED{proto}{udp} = "";
$SELECTED{proto}{icmp} = "";
$SELECTED{first_talker}{any} = "";
$SELECTED{first_talker}{local} = "";
$SELECTED{first_talker}{remote} = "";
$SELECTED{last_talker}{any} = "";
$SELECTED{last_talker}{local} = "";
$SELECTED{last_talker}{remote} = "";
for $key (qw(proto first_talker last_talker)) {
$SELECTED{$key}{$arg{$key}} = "SELECTED";
}
print<<"EOM";
EOM
}
# HTML footer
sub html_footer() {
print "\n";
}
# Convert port list to regular expression
sub portlist2regex {
my ($list) = @_;
return undef if(!defined($list));
my $ret = $list;
$ret =~ s!,!|!g;
$ret = "($ret)";
return $ret;
}
# Read form input arguments
sub read_arg {
my ($key,$val);
# Parse $ENV{QUERY_STRING}
for (split /\&/, $ENV{QUERY_STRING}) {
($key,$val) = split /=/;
$val =~ s/\+/ /g;
$val =~ s/%([0-9a-fA-F]{2})/chr(hex($1))/ge;
$arg{$key} = $val;
}
}
# Set defaults on user form if not entered from previous call
sub set_form_defaults() {
$arg{line_limit} = 100 unless $arg{line_limit};
$arg{line_incr } = 1 unless $arg{line_incr };
# Set min and max times (qmin,qmax) if only date was entered.
if ($arg{date}) {
my $date = $arg{date};
# If date includes time,
# then min date/time is same as date,
# and max date/time is 30 minutes (1800 sec) later
if ($date =~ m/:/) {
# date includes time:
$arg{qmin} = $date;
$arg{qmax} = time2file(file2time($date,1800));
# If date does not include time,
# then min date/time is start of date at midnight,
# and max date/time next midnight
} elsif ($date =~ m/\d{4}-\d{2}-\d{2}/) {
$arg{qmin} = $date;
$arg{qmax} = time2file(86400 + file2time($date));
} else {
&croak ("Can't handle date=$date\n");
}
}
# $arg{ip} is used when script called from static page
# $arg{ip_address} is used when script calls itself
# Set ip_address if not set
if ($arg{ip}) {
my $ip = $arg{ip};
chomp($ip);
$arg{ip_address} = demunge_ip($ip);
}
} # set_form_defaults
# Re-format input parameters for use by data search function
sub set_search_param {
my ($key);
# yes this is weird. file2time_max handles the max range that a prefix would represent.
# basically - this defaults to only search the most recent file
$search{qmin} = defined $arg{qmin} ? $arg{qmin} : $max_date;
$search{qmax} = defined $arg{qmax} ? $arg{qmax} : time2file(file2time($max_date,1800));
# Find the min and max file to search from the requested time range
$search{file_min} = time2file(file2time($search{qmin},-1800));
$search{file_max} = time2file(file2time($search{qmax},-1800));
# Untaint ip_pattern
($search{ip_pattern}) = $arg{ip_address} =~/^(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})$/;
$search{ip_pattern} = munge_ip($search{ip_pattern});
# Set port and protocol search
$search{lp_query} = $arg{local_port } || "";
$search{rp_query} = $arg{remote_port} || "";
if (defined $arg{proto} && lc $arg{proto} ne "any") {
$search{protocol} = defined $PROTO{$arg{proto}} ? $PROTO{$arg{proto}} : $arg{proto};
}
# Set first and last talker search
for $key ('first_talker','last_talker') {
$search{$key} = $arg{$key} || "any";
if($search{$key} eq "remote") { $search{$key} = 2 };
if($search{$key} eq "local") { $search{$key} = 1 };
}
# Set minimum/maximum connection byte limit
for $key ('data_min','data_max') {
if ($arg{$key}) {
$search{$key} = str2bytes($arg{$key})
} else {
$search{$key} = -1;
}
}
# Build reqular expression to match ports
$search{lport_pattern} = portlist2regex($arg{local_port} ) if $arg{local_port};
$search{rport_pattern} = portlist2regex($arg{remote_port}) if $arg{remote_port};
} # &set_search_param
# Look up known ports, store in %port as "23u" "1024t"
# for ports 23/udp, 1024/tcp, etc.
sub get_port_list {
my ($scriptdir) = ($ENV{SCRIPT_FILENAME}=~/^(.*)\/([^\/]+)$/);
my (@F);
if (open F, "$scriptdir/port.lst") {
while () {
next if /^\s*#/;
@F=split(/\s*[\|]\s*/);
$F[0]=~s/^\s+//;
$portlist{$F[0]} = 1;
}
}
close F;
}
# Given port/protocol forms URL for link to CGI-SCRIPT "PortLookup"
sub porturl {
my ($data) = @_;
my ($port, $traffic, $portf, $index);
my (@temp) = (\$$data[3], $$data[6], \$$data[4], $$data[5]);
# Local udp/tcp port or outgoing ICMP code
while (@temp) {
$port = shift @temp;
$traffic = shift @temp;
$portf = sprintf "%${PORT_WIDTH}d", $$port;
$index = $$port . substr($$data[2],0,1);
if (defined $portlist{$index} && ($traffic || $$data[2] ne "icmp")) {
$$port =
"" .
$portf .
"";
} else {
$$port = $portf;
}
}
}
# Print HTML'ized message and die (cheap replacement for CGI::croak)
sub croak {
my ($msg, $head) = @_;
$head = $msg if not defined $head;
print<<"EOM";
$head