#!/usr/bin/perl -Tw #----------------------------------------------------------------------- # History #----------------------------------------------------------------------- # General house cleaning 2003-10-05 JR #----------------------------------------------------------------------- # Constants #----------------------------------------------------------------------- # Column info, used by both print_column_labels() and # print_table_data() and porturl(). my ($PORT_WIDTH)=6; my (@COLWIDTH) = (-15, -15, -6, $PORT_WIDTH, $PORT_WIDTH, 8, 8, 8, 8, -13, -13, -6, -6); my (@COLCOLOR) = (0, 1, 0, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1); # Map protocol name to number my %PROTO = ( 'tcp'=>6, 'udp'=>17, 'icmp'=>1); # Map protocol number to name my %PROTO_LABEL = (6=>'tcp', 17=>'udp', 1=>'icmp'); # Map talker number to label my @TALKER_LABEL = (0,'L','R'); # Initialize values to prevent initialization error # messages when print_form() is called (2003-02-20 JR) my (%arg) = ( "qmin" => "", "qmax" => "", "ip_address" => "", "local_port" => "", "remote_port" => "", "line_limit" => "", "line_incr" => "", "data_min" => "", "data_max" => "", "proto" => "any", "first_talker" => "any", "last_talker" => "any", ); my (%search); #----------------------------------------------------------------------- # Initialize Directories, Modules, Output #----------------------------------------------------------------------- # make adjust-cgi will search and replace the below from # what the configure script has detected as the ipaudit homedir. BEGIN { unshift (@INC,"/home/ipaudit/"); # Adjusted via adjust-cgi } # Set output to immediate flush $| = 1; # Print HTTP header now so subsequent output will be legal print "Content-type: text/html\n\n"; use strict; use ipaudit_config; # Don't use advanced data parsing if not present eval "use Time::ParseDate"; my ($Is_ParseDate_Found) = $@ eq ""; if ($Is_ParseDate_Found) { use POSIX; } # Untaint environment delete @ENV{'PATH', 'IFS', 'CDPATH', 'ENV', 'BASH_ENV'}; require 5.004; # needed for tie interface #----------------------------------------------------------------------- # Declare Variables #----------------------------------------------------------------------- my ($IP_DIR,$DATA_DIR,$ZCAT,$ZGREP,%conf); my ($filemin,$filemax, $min_date,$max_date, $min_sec, $max_sec, $data_max, $data_min, $prefix_min, $prefix_max, $ip_pattern, $first_talker, $last_talker, $protocol, $line_limit, $line_incr, @files); my (%portlist); my ($cgidir, $t1,$t2); my ($head,$msg); #----------------------------------------------------------------------- # Main #----------------------------------------------------------------------- # Starting time $t1 = time(); # Read configuration file %conf = &ipa_getconf(); # Initialize some dependent variables $IP_DIR = &untaint ($conf{'IP_DIR'} ) || '/home/ipaudit'; $ZCAT = &untaint ($conf{'ZCAT'} ) || '/bin/zcat'; $ZGREP = &untaint ($conf{'ZGREP'} ) || '/usr/bin/zgrep'; $DATA_DIR = "$IP_DIR/data/30min"; # Cgi directory, used to call PortLookup script ($cgidir) = ($ENV{SCRIPT_NAME}=~/^(.*)\/([^\/]+)$/); # Make sure that ZCAT and ZGREP aren't missing, # this is a frequent source of errors. if (not -x $ZCAT && -x $ZGREP ) { $head = "ERROR: Script Misconfiguration"; $msg = "

ERROR


\n"; $msg .= "This script (~ipaudit/cgi-bin/SearchIpauditData) is misconfigured.

"; $msg .= "The ZCAT executable $ZCAT cannot be found.
" if not -x $ZCAT; $msg .= "The ZGREP executable $ZGREP cannot be found.
" if not -x $ZGREP; $msg .= <Be sure the following lines from the ipaudit-web.conf file
ZCAT=$ZCAT
ZGREP=$ZGREP
contain the correct file paths of the zcat and zgrep utilities. EOM &croak($msg,$head); } # Read mapping of port number->name &get_port_list(); # Print html header &html_header(); # get list of all files in directory @files = &get_files(); ($min_date) = $files[ 0]=~/(\d{4}-\d{2}-\d{2}-\d{2}:*\d{2})/; ($max_date) = $files[-1]=~/(\d{4}-\d{2}-\d{2}-\d{2}:*\d{2})/; # Print date range printf "

Data Available from %s to %s.

\n", $min_date, $max_date; # Read form input &read_arg; # Format input stored %arg for redisplay in form. &set_form_defaults(); # Set search parameters from from input &set_search_param; # Print search form &print_form(); # If arguments present, do search if($arg{submit} or $arg{date}) { &read_files (@files); } # Print total processing time $t2= time(); print "
Total Processing time: ",$t2 - $t1, "seconds \n"; # Print footer &html_footer(); exit; #----------------------------------------------------------------------- # Functions #----------------------------------------------------------------------- sub html_header() { my $title = "IPAudit Log Search"; print<<"EOM"; $title

IPAudit - Log Search

Home  
 
   
EOM } # Convert date in format "2002-02-13 10:30" or "2002-02-13" # to seconds from Epoch sub local_parsedate { my ($date) = @_; # use Time::ParseDate if available if ($Is_ParseDate_Found) { my ($sec) = eval 'parsedate($date, NO_RELATIVE => 1)'; return $sec; } my ($year,$month,$day); my ($hour) = 0; my ($min) = 0; # Set isdst ("IS Daylight Savings Time, field 9) to # "not available" (see 'man mktime') if ($date=~m!(\d+)/(\d+)/(\d+)\s+(\d+):(\d+)!) { return mktime (0,$5,$4,$3,$2-1,$1-1900,0,0,-1); } elsif ($date=~m!(\d+)/(\d+)/(\d+)!) { return mktime (0,0,0,$3,$2-1,$1-1900,0,0,-1); } else { &croak ("Cannot read date format (advanced date parsing not present)"); } } # Given a file name, returns the value as the unix time in seconds. # Round down/up if optional round is negative/positive. sub file2time { my ($filename,$round) = @_; # Change filename like "2002-11-07-07:30" to "2002/11/07 07:30" if ($filename =~ /(\d{4})-(\d{2})-(\d{2})[ -](\d{2}:\d{2})/) { $filename = "$1/$2/$3 $4"; # Change filename like "2002-11-07" to "2002/11/07 00:00" } elsif ($filename =~ /(\d{4})-(\d{2})-(\d{2})/) { $filename = "$1/$2/$3 00:00"; } my ($sec) = local_parsedate ($filename); # If round is negative, round down if ($round < 0) { $sec -= $sec % (-$round); # If round is positive, round up } elsif ($round > 0) { $sec += ($round - $sec % $round); } return $sec; } # This gets the begining prefix - atleast so we know where to start looking. sub time2file($ ) { my $time_val = shift; my ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst) = localtime($time_val); # yay y2k! $year += 1900; return sprintf("%4d-%02d-%02d-%02d:%02d", $year, $mon + 1, $mday, $hour, $min); } # Get list of all data files in data directory sub get_files() { opendir(DIR, $DATA_DIR) or &croak ("Can't Open $DATA_DIR: $!\n"); # read in all the gzip files that start with a number and are really files for (readdir(DIR)) { next unless -f "$DATA_DIR/$_"; next unless /^(\d{4}-\d{2}-\d{2}-\d{2}:*\d{2}.txt.gz)/; push @files, $1; } close(DIR); return sort @files; } # Change human readable counts like '200m' or '200k' to numbers sub str2bytes($ ) { my $str = shift; &croak ("str2bytes: no input") if !defined($str); if($str =~ m/k$/i) { $str =~ s/k$//gi; return $str * 1024; } elsif ($str =~ m/b$/i) { $str =~ s/b$//gi; return $str; } elsif ($str =~ m/m$/i) { $str =~ s/m$//gi; return $str * 1024 * 1024; } elsif ($str =~ m/g$/i) { $str =~ s/g$//gi; return $str * 1024 * 1024 * 1024; } elsif ($str =~ m/^\d+$/) { return $str; } &croak ("Couldn't convert str->bytes: $str\n"); } # Convert numbers to human readable form like '200m' or '200k' sub bytes2str($ ) { my $bytes = shift; &croak ("bytes2str: no input") if !defined($bytes); if($bytes < 1024) { return "$bytes"; } elsif ($bytes < 1048576) { $bytes = $bytes / 1024; return sprintf("%.1fk", $bytes); } elsif ($bytes < 1073741824) { $bytes = $bytes / 1048576; return sprintf("%.2fM", $bytes); } $bytes = $bytes / 1073741824; return sprintf("%.2dG", $bytes); } # Given search parameters, display corresponding 'connections'. sub read_files { my (@files) = @_; my ($pid); my ($line, $nprint, $nread, $nincr); my ($datelen,$compare); $datelen = length($search{file_min}) > length($search{file_max}) ? length($search{file_min}) : length($search{file_max}) ; # only get the files in our time range. my (@file_list); for (@files) { $compare = substr($_,0,$datelen); next unless $search{file_min} le $compare && $search{file_max} ge $compare; push @file_list, "$DATA_DIR/$_"; } # saveing stderr - I don't care if the pipe breaks on zcat. open(SAVERR, '>&STDERR'); open(STDERR, '>/dev/null'); if($search{ip_pattern}) { $pid = open(FILE, "-|") or exec "$ZGREP" , $search{ip_pattern}, @file_list; &croak ("$ZGREP exec failed: $!") unless defined($pid); } else { $pid = open(FILE, "-|") or exec "$ZCAT", @file_list; &croak ("$ZCAT exec failed: $!") unless defined($pid); } # Print table header, column labels &print_column_labels; # Print data rows $nprint = 0; $nincr = 0; $nread = 0; ###DEBUG #test ###DEBUG my ($k); ###DEBUG for $k (sort keys %arg) { ###DEBUG print "\$arg{$k} ($arg{$k})
\n"; ###DEBUG } ###DEBUG for $k (sort keys %search) { ###DEBUG print "\$search{$k} ($search{$k})
\n"; ###DEBUG } ###DEBUG #end while(($nprint < $arg{line_limit}) && ($line = )) { # # 0-Local IP 1-Remote IP # # 2-Protocol (1=icmp, 6=tcp, 17=udp) # # 3-Local Port 4-Remote Port # # 5-Incoming (bytes) 6-Outgoing (bytes) # # 7-Incoming (packets) 8-Outgoing (packets) # # 9-First Packet time 10-Last Packet time # # 11-First Packet source 12-Last Packet source (1=Local,2=Remote) # Count number of lines read $nread++; # Test this input line for requested conditions chomp $line; my @data = split(/\s+/, $line, 13); next if @data<13 && print "skipping (less than thirteen fields)"; # Test total byte min/max next if ($search{data_min} > -1) && (($data[5] + $data[6]) < $search{data_min}); next if ($search{data_max} > -1) && (($data[5] + $data[6]) > $search{data_max}); if($search{lport_pattern}) { next if $data[3] !~ m/^$search{lport_pattern}$/; } if($search{rport_pattern}) { next if $data[4] !~ m/^$search{rport_pattern}$/; } # Select protocol next if $search{protocol} && $data[2]!=$search{protocol}; # Compare first,last talker values if($search{first_talker} ne "any") { next if $data[11] ne $search{first_talker}; } if($search{last_talker} ne "any") { next if $data[12] ne $search{last_talker}; } # Skip this eligible line until count of $incr is reached next unless ++$nincr == $arg{line_incr}; $nincr = 0; # Reformat ips $data[0] = demunge_ip($data[0]); $data[1] = demunge_ip($data[1]); # Make bytes human readable $data[5] = bytes2str($data[5]); $data[6] = bytes2str($data[6]); # Map protocols and talkers to their labels $data[ 2] = $PROTO_LABEL {$data[ 2]}; $data[11] = $TALKER_LABEL[$data[11]]; $data[12] = $TALKER_LABEL[$data[12]]; # If only a one-way connection, set last field to '-' if ($data[7]==0 || $data[8]==0) { $data[12] = "-"; } # Convert port numbers to port lookup url's &porturl(\@data); # Print this connection &print_table_data($nprint,@data); # Count number of lines printed $nprint++; } # Close data table &print_column_labels; # Added by jh@dok.org # Print Summary print("
 "); print "** $nread lines read.
\n"; if($nprint eq $arg{line_limit}) { print "** Max lines $nprint/$arg{line_limit} printed."; } else { print "** $nprint lines printed."; } print("
"); close(STDERR); open(STDERR,">&SAVERR"); close(SAVERR); } # -- end of read files sub print_column_labels { my (@label) = ( "Local IP", "", "Remote IP", "", "Proto-", "col", "Local", "Port", "Remote", "Port", "Incoming", "Bytes", "Outgoing", "Bytes", "Incoming", "Packets", "Outgoing", "Packets", "First Packet", "Time", "Last Packet", "Time", "First", "Talker", "Last", "Talker" ); my ($i,$l,$format,$line); print "\n"; print "
"; for ($l=0;$l<2;$l++) { for ($i=0;$i<13;$i++) { $format = "%" . $COLWIDTH[$i] . "s"; $format = "$format" if $COLCOLOR[$i]; printf " $format", $label[2*$i+$l]; } print "\n"; } print "
\n"; print "\n"; } # # Print data with alternate background line coloring and # column keyed text colors # sub print_table_data { my ($cnt,@data) = @_; my ($i,$format); if ($cnt % 2 == 0) { print "
"; } else { print "
"; } for ($i=0;$i<13;$i++) { $format = "%" . $COLWIDTH[$i] . "s"; $format = "$format" if $COLCOLOR[$i]; printf " $format", $data[$i]; } print "
"; print "\n"; } # converts an ip to ipaudit style sub munge_ip($ ) { my $ip = shift; &croak ("munge_ip: No ip defined") unless defined($ip); # taken almost directly from SearchIpauditData return sprintf "%03d.%03d.%03d.%03d", ($ip =~/(\d{1,3})\.(\d{1,3}).(\d{1,3}).(\d{1,3})/); } # converts ip to "normal" form sub demunge_ip { my ($ip) = @_; &croak ("demunge_ip: No ip defined") unless defined($ip); return sprintf "%d.%d.%d.%d", ($ip =~/0*(\d+)\.0*(\d+).0*(\d+).0*(\d+)/); } # convert an ip fragment to ipaudit style sub munge_ip_frag ($ ) { return join ".", map ( sprintf ("%03d", $_) , split (/\./, shift) ) ; } # Print search form, setting all defaults sub print_form() { # Date format example depends on date subroutine used my ($date_format_example) = $Is_ParseDate_Found ? "Eg: yesterday, -2 days, last Wednesday, 2001-03-13-12:30" : "Eg: 2002-03-13-12:30"; # Set defaults for SELECT boxes my (%SELECTED) = (); my ($key); # Initialize values for %SELECT $SELECTED{proto}{any} = ""; $SELECTED{proto}{tcp} = ""; $SELECTED{proto}{udp} = ""; $SELECTED{proto}{icmp} = ""; $SELECTED{first_talker}{any} = ""; $SELECTED{first_talker}{local} = ""; $SELECTED{first_talker}{remote} = ""; $SELECTED{last_talker}{any} = ""; $SELECTED{last_talker}{local} = ""; $SELECTED{last_talker}{remote} = ""; for $key (qw(proto first_talker last_talker)) { $SELECTED{$key}{$arg{$key}} = "SELECTED"; } print<<"EOM";
Search Form
Submit  
Start Date: $date_format_example
End Date:
IP Address:
Local Port: Eg: 21,23
Remote Port: Eg: 21,23
Max Lines Displayed: Eg: 200
Print Incr: Eg: 2
Min Session Size: Eg: 200, 2k, 1G
Max Session Size: Eg: 200, 2k, 1G
Protocol:
First Talker:
Last Talker:
EOM } # HTML footer sub html_footer() { print "\n"; } # Convert port list to regular expression sub portlist2regex { my ($list) = @_; return undef if(!defined($list)); my $ret = $list; $ret =~ s!,!|!g; $ret = "($ret)"; return $ret; } # Read form input arguments sub read_arg { my ($key,$val); # Parse $ENV{QUERY_STRING} for (split /\&/, $ENV{QUERY_STRING}) { ($key,$val) = split /=/; $val =~ s/\+/ /g; $val =~ s/%([0-9a-fA-F]{2})/chr(hex($1))/ge; $arg{$key} = $val; } } # Set defaults on user form if not entered from previous call sub set_form_defaults() { $arg{line_limit} = 100 unless $arg{line_limit}; $arg{line_incr } = 1 unless $arg{line_incr }; # Set min and max times (qmin,qmax) if only date was entered. if ($arg{date}) { my $date = $arg{date}; # If date includes time, # then min date/time is same as date, # and max date/time is 30 minutes (1800 sec) later if ($date =~ m/:/) { # date includes time: $arg{qmin} = $date; $arg{qmax} = time2file(file2time($date,1800)); # If date does not include time, # then min date/time is start of date at midnight, # and max date/time next midnight } elsif ($date =~ m/\d{4}-\d{2}-\d{2}/) { $arg{qmin} = $date; $arg{qmax} = time2file(86400 + file2time($date)); } else { &croak ("Can't handle date=$date\n"); } } # $arg{ip} is used when script called from static page # $arg{ip_address} is used when script calls itself # Set ip_address if not set if ($arg{ip}) { my $ip = $arg{ip}; chomp($ip); $arg{ip_address} = demunge_ip($ip); } } # set_form_defaults # Re-format input parameters for use by data search function sub set_search_param { my ($key); # yes this is weird. file2time_max handles the max range that a prefix would represent. # basically - this defaults to only search the most recent file $search{qmin} = defined $arg{qmin} ? $arg{qmin} : $max_date; $search{qmax} = defined $arg{qmax} ? $arg{qmax} : time2file(file2time($max_date,1800)); # Find the min and max file to search from the requested time range $search{file_min} = time2file(file2time($search{qmin},-1800)); $search{file_max} = time2file(file2time($search{qmax},-1800)); # Untaint ip_pattern ($search{ip_pattern}) = $arg{ip_address} =~/^(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})$/; $search{ip_pattern} = munge_ip($search{ip_pattern}); # Set port and protocol search $search{lp_query} = $arg{local_port } || ""; $search{rp_query} = $arg{remote_port} || ""; if (defined $arg{proto} && lc $arg{proto} ne "any") { $search{protocol} = defined $PROTO{$arg{proto}} ? $PROTO{$arg{proto}} : $arg{proto}; } # Set first and last talker search for $key ('first_talker','last_talker') { $search{$key} = $arg{$key} || "any"; if($search{$key} eq "remote") { $search{$key} = 2 }; if($search{$key} eq "local") { $search{$key} = 1 }; } # Set minimum/maximum connection byte limit for $key ('data_min','data_max') { if ($arg{$key}) { $search{$key} = str2bytes($arg{$key}) } else { $search{$key} = -1; } } # Build reqular expression to match ports $search{lport_pattern} = portlist2regex($arg{local_port} ) if $arg{local_port}; $search{rport_pattern} = portlist2regex($arg{remote_port}) if $arg{remote_port}; } # &set_search_param # Look up known ports, store in %port as "23u" "1024t" # for ports 23/udp, 1024/tcp, etc. sub get_port_list { my ($scriptdir) = ($ENV{SCRIPT_FILENAME}=~/^(.*)\/([^\/]+)$/); my (@F); if (open F, "$scriptdir/port.lst") { while () { next if /^\s*#/; @F=split(/\s*[\|]\s*/); $F[0]=~s/^\s+//; $portlist{$F[0]} = 1; } } close F; } # Given port/protocol forms URL for link to CGI-SCRIPT "PortLookup" sub porturl { my ($data) = @_; my ($port, $traffic, $portf, $index); my (@temp) = (\$$data[3], $$data[6], \$$data[4], $$data[5]); # Local udp/tcp port or outgoing ICMP code while (@temp) { $port = shift @temp; $traffic = shift @temp; $portf = sprintf "%${PORT_WIDTH}d", $$port; $index = $$port . substr($$data[2],0,1); if (defined $portlist{$index} && ($traffic || $$data[2] ne "icmp")) { $$port = "" . $portf . ""; } else { $$port = $portf; } } } # Print HTML'ized message and die (cheap replacement for CGI::croak) sub croak { my ($msg, $head) = @_; $head = $msg if not defined $head; print<<"EOM"; $head


$msg
EOM exit; } # Untaint variable sub untaint { $_[0] =~ /^(.*)$/; return $1; }