#!/usr/bin/perl -w

use strict;
use warnings;
use IO::Handle;
use File::Basename;
use Cwd qw(abs_path);
use Getopt::Long qw(:config no_ignore_case gnu_compat);
# DO NOT use Text::CSV, as it is too slow.
#use Text::CSV;


# set current path of this command, to refer other commands(ex. top2csv...)
my $curpath = dirname(Cwd::abs_path(__FILE__));

# for printing progression
require "$curpath/osw2csv_functions";


# option hash with initial value.
our %opts = ();


GetOptions(\%opts,
	'time=s',
	'timerange=s',
	'row=s',
	'col=s',
	'val=s',
	'valcmp=s',
	'diffcmp=s',
	'invert-match|v',
	'help|h',
	'quiet|q'
) || printUsageAndExit("Invalid options");

if ($opts{help}) {
	printUsageAndExit("help option specified.");
}

if (! -p STDIN && $#ARGV < 0) {
	printUsageAndExit("Invalid argument number.");
}
if (-p STDIN && $#ARGV < 0) {
	# set STDIN
	$ARGV[0] = "-";
}


# print error and exit
sub error_end {
	print STDERR join(' ', @_)."\n";
	exit 1;
}

# print processing message onto STDERR
sub print_mes {
	return if (defined($opts{quiet}));
	print STDERR join(' ', @_)."\n";
}


# print usage and exit
sub printUsageAndExit {
	print STDERR join("\n", @_)."\n";
        print STDERR <<"END_OF_USAGE";
usage:  # $0 [options] csv_files
description:
    This script just "grep" csv files. the options work in "and" condition.
options:
    --time <regex>  or  --row <regex>
        find out the times(1st column) which the regex matches
    --timerange <range>
        find out the times(1st column) which the range includes.
	range ex. "2016/05/23 1:23:45 - 2016/05/24 2:34:56"
    --col <regex>
        find out the columns(except 1st column) which the regex matches
    --val <regex>
        find out the columns/times which the regex matches
    --valcmp <(<|<=|==|=>|>)number>
        find out the columns/times which have the value matches the comparison
    --diffcmp <(<|<=|==|=>|>)number>
        find out the columns/times which have the difference matches the
	comparison with the value in previous row.
    --invert-match or -v
	output if the conditions do not match
    --quiet or -q
	quiet, suppress printing what this program is processing to STDERR.
ex.:
	find out the oracle process which increase/reduces VIRT mem over 1000kb
	between 00:12:1# to 00:12:3# from top-pVIRT.csv.

	# $0 --time "00:12:[1-3]" --col "oracle" --diffcmp ">=1000" top-pVIRT.csv
END_OF_USAGE
        exit 1;
}


# csvmatrix is a matrix which shows the CSV, is an array of string.
# Note: this is NOT an array of arrays, to shorten memory usage.
# csvmatrix = ( "10,2,5,3", "5,3,9,2", .... );
my @csvmatrix = ();

# generate CSV matrix(csvmatrix)
sub generateCSVMatrix {
	my @csvfiles = @_;

	my $csvheader;
	my $csvheadernum;

	initPrintProcessing();
	foreach my $csvfile (@csvfiles) {
		if ($csvfile eq "-") {
			print_mes "reading \"(STDIN)\" ...";
		} else {
			print_mes "reading \"$csvfile\" ...";
		}
		open my $fh, "<$csvfile" or error_end("$csvfile: $!");

		# check 1st line
		if (!($_ = <$fh>)) {
			close $fh;
			next;
		}
		chomp;
		if ($csvfile eq $csvfiles[0]) {
			# get csv header
			$csvheader = $_;
			# -1 is important for analyzing all null csv elements.
			my @ary = split(/,/, $csvheader, -1);
			$csvheadernum = $#ary;
			push(@csvmatrix, $csvheader);
		} elsif ($csvheader ne $_) {
			print_mes "$csvfile has different csv header, skip...";
			close $fh;
			next;
		}

		initPrintProcessing();
		# parse other lines
		while (<$fh>) {
			chomp;
#### remove this check as it is tooooo slow, 100 times slower to enable this.
####			my @row = split(/,/, $_, -1);
####			if ($#row != $csvheadernum) {
####				print_mes "row \"$row[0]\" has $#row columns but should be $csvheadernum, skip...";
####				next;
####			}
			push(@csvmatrix, $_);
			printProcessing();
		}
		close $fh;
		endPrintProcessing(" time samples found");
	}
}


# grep time from CSV matrix
sub grepTime {
	my $regex = $_[0];

	my $size = $#csvmatrix+1;
	my $removenum = 0;

	initPrintProcessing($size-1);
	if (!$opts{"invert-match"}) {
		# normarl grep
		for(my $i = $#csvmatrix; $i >= 1; $i--) {
			# substr() is faster than getting (split(/,/))[0];
			# ToDo: How about "(split(/,/, $csvmatrix[$i], 2))[0]"?
			my $time = substr($csvmatrix[$i],0,index($csvmatrix[$i],','));
			if ($time !~ /$regex/) {
				splice(@csvmatrix, $i, 1);
				$removenum++;
			}
			printProcessing(" $removenum removed");
		}
	} else {
		# invert grep, most are same to above, but different "!~"->"=~"
		for(my $i = $#csvmatrix; $i >= 1; $i--) {
			# substr() is faster than getting (split(/,/))[0];
			# ToDo: How about "(split(/,/, $csvmatrix[$i], 2))[0]"?
			my $time = substr($csvmatrix[$i],0,index($csvmatrix[$i],','));
			if ($time =~ /$regex/) {
				splice(@csvmatrix, $i, 1);
				$removenum++;
			}
			printProcessing(" $removenum removed");
		}
	}
	endPrintProcessing(" $removenum removed, $#csvmatrix remained");
}

# grep time range from CSV matrix
sub grepTimeRange {
	my $range = $_[0];
	# range should be several kind of strings:
	# "(from date(+time)) - (to date(+time))", for example:
	# "2017/03/01-",
	# "-2017/03/01",
	# "2017/03/01-2017/03/02",
	# "2017/03/01 12:13:14-",
	# "-2017/03/01 12:13:14",
	# "2017/03/01 - 2017/03/02 13:14:15",
	# "2017/03/01 12:13:14 - 2017/03/02 13:14:15" etc.
	# in case from/to is not set, it will be "1970/01/01 00:00:00 -
	# 2999/12:31 23:59:59"

	my ($from, $to) = split(/-/, $range);
	$from = "1970/01/01 00:00:00" if (!defined($from) || $from =~ /^\s*$/);
	$to   = "2999/12/31 23:59:59" if (!defined($to)   || $to   =~ /^\s*$/);
	$from =~ s/^\s*(.*?)\s*$/$1/;
	$to   =~ s/^\s*(.*?)\s*$/$1/;
	my ($from_date, $from_time) = split(/\s+/, $from);
	my ($to_date,   $to_time)   = split(/\s+/, $to);
	$from_time = "00:00:00" if (!defined($from_time));
	$to_time   = "23:59:59" if (!defined($to_time)  );
	if ($from_date !~ /^[0-9\/]+$/) {
		error_end("invalid timerange($range), from_date:\"$from_date\"");
	}
	if ($to_date !~ /^[0-9\/]+$/) {
		error_end("invalid timerange($range), to_date:\"$to_date\"");
	}
	if ($from_time !~ /^[0-9:]+$/) {
		error_end("invalid timerange($range), from_time:\"$from_time\"");
	}
	if ($to_time !~ /^[0-9:]+$/) {
		error_end("invalid timerange($range), to_time:\"$to_time\"");
	}

	$from = "$from_date $from_time";
	$to   = "$to_date $to_time";
	# now from/to should be "####/##/## ##:##:##"

	# add fixed timezone if available, and get second
	my $stz = (split(/,/, $csvmatrix[1], 3))[1];
	my $from_sec = parsedate("$from $stz");
	my $to_sec   = parsedate("$to   $stz");
	if (!defined($from_sec) || !defined($to_sec)) {
		error_end("invalid timerange($range, timezone=$stz)");
	}
	# now fromsec/tosec should be a second of "####/##/## ##:##:## TZ"

	if ($from_sec > $to_sec) {
		error_end("invalid timerange($range), from($from) is larger than to($to)");
	}

	my $removenum = 0;
	my $size = $#csvmatrix+1;
	initPrintProcessing($size-1);
	for(my $i = $#csvmatrix; $i >= 1; $i--) {
		# split(LIMIT=3) is important for performance
		my($time, $tz) = (split(/,/, $csvmatrix[$i], 3))[0,1];
		my ($sec, $err) = parsedate("$time $tz");
		if (!defined($sec)) {
			print STDERR "time \"$time $tz\" is invalid(err:$err), skip.\n";
			splice(@csvmatrix, $i, 1);
			$removenum++;
		}
		if ($sec < $from_sec || $to_sec < $sec) {
			splice(@csvmatrix, $i, 1);
			$removenum++;
		}
		printProcessing(" $removenum removed");
	}
	endPrintProcessing(" $removenum removed, $#csvmatrix remained");
}


# grep Columns with index array
sub grepColumn_with_indexary {
	my @grepcols = @_;
	my @colary = split(/,/, $csvmatrix[0], -1);
	my $colnum = $#colary;

	# unique_mize the @grepcols
	# index 0(=time column) is necessary.
	push(@grepcols, 0);
	my %uniqhash = map { $_, 1 } @grepcols;
	# sort is important, it is used below.
	@grepcols = sort { $a <=> $b } (keys %uniqhash);

	if ($opts{"invert-match"}) {
		# invert grep, realizing by invoking this function with
		# "non-matched index array"
		my @complement_grepcols = ();
		my $curidx = 0;
		foreach my $idx (@grepcols) {
			while ($curidx < $idx) {
				push(@complement_grepcols, $curidx++);
			}
			$curidx += 1; # skip $idx
		}
		for (; $curidx <= $colnum; $curidx++) {
			push(@complement_grepcols, $curidx);
		}
		# temporally disable "invert-match" option 
		$opts{"invert-match"} = 0;
		# invoke this function with "non-matched index array"
		grepColumn_with_indexary(@complement_grepcols);
		# revert "invert-match" option 
		$opts{"invert-match"} = 1;
		return;
	}

	# if the grepcols includes all of csvmatrix columns, do nothing
	if ($#grepcols >= $colnum) {
		return;
	}

	print_mes "    grepping ".($#grepcols+1)." columns ...";
	initPrintProcessing($#csvmatrix+1);

	for (my $i = $#csvmatrix; $i >= 0; $i--) {
###### this loop is so slow when $#{$csvmatrix[0]} is huge, like 130000.
###### Any ideas to get this speed up are appreciated.
######## multithreading...? Actually, multithreading has a significant issue,
######## splice() is not applicable for shared object!
		my @ary = split(',', $csvmatrix[$i], -1);
		my @result = map {defined($ary[$_]) ? $ary[$_] : ''} @grepcols;
		$csvmatrix[$i] = join(',', @result);
		printProcessing();
	}

	endPrintProcessing();
}


# grep column from CSV matrix
sub grepColumn {
	my $regex = $_[0];

	my @firstrow = split(/,/, $csvmatrix[0]);
	my @grepcols = ();
	initPrintProcessing($#firstrow); # no -1 is necessary
	for(my $i = $#firstrow; $i >= 1; $i--) {
		if ($firstrow[$i] =~ /$regex/) {
			push(@grepcols, $i);
		}
		printProcessing();
	}
	endPrintProcessing();
	grepColumn_with_indexary(@grepcols);
}


# grep value by comparison from CSV matrix
sub grepValueComparison {
	my $valstr = $_[0]; # "[<>=][\d\.]+"

	if ($valstr !~ /^[<=>]+\s*[\d\.]+$/) {
		error_end("invalid --val value(\"$valstr\").");
	}

	my $size = $#csvmatrix+1;
	initPrintProcessing($size-1);
	my %matchcols;
	for (my $i = $#csvmatrix; $i >= 1; $i--) {
		my $matched = 0;
		my @ary = split(/,/, $csvmatrix[$i], -1);
		for (my $j = $#ary; $j >= 1; $j--) {
			if ($ary[$j] ne "" && eval "$ary[$j] $valstr") {
				$matched = 1;
				$matchcols{$j} = 1;
			}
		}
		if (!$matched) { 
			# remove the row
			splice(@csvmatrix, $i, 1);
		}
		printProcessing();
	}
	endPrintProcessing();
	grepColumn_with_indexary(keys(%matchcols));
}


# grep value by regex from CSV matrix
sub grepValueRegex {
	my $regex = $_[0];

	my $size = $#csvmatrix+1;
	initPrintProcessing($size-1);
	my %matchcols;
	for (my $i = $#csvmatrix; $i >= 1; $i--) {
		my $matched = 0;
		my @ary = split(/,/, $csvmatrix[$i], -1);
		for (my $j = $#ary; $j >= 1; $j--) {
			if ($ary[$j] =~ /$regex/) {
				$matched = 1;
				$matchcols{$j} = 1;
			}
		}
		if (!$matched) {
			# remove the row
			splice(@csvmatrix, $i, 1);
		}
		printProcessing();
	}
	endPrintProcessing();
	grepColumn_with_indexary(keys(%matchcols));
}


# grep difference by comparison from CSV matrix
sub grepDifferenceComparison {
	my $diffstr = $_[0]; # "[<>=][\d\.]+"

	if ($diffstr !~ /^[<=>]+\s*[\d\.]+$/) {
		error_end("invalid --diff value(\"$diffstr\").");
	}

	my $size = $#csvmatrix+1;
	initPrintProcessing($size-2);
	my %matchcols;
	my (@ary, @prevary);
	@prevary = split(/,/,$csvmatrix[$#csvmatrix],-1) if ($#csvmatrix >= 0);
	for (my $i = $#csvmatrix; $i >= 2; $i--) {
		my $matched = 0;
		@ary = @prevary;
		@prevary = split(/,/, $csvmatrix[$i-1], -1);
		for (my $j = $#ary; $j >= 1; $j--) {
			my $prevval = $prevary[$j];
			my $val     = $ary[$j];
			$prevval = 0 if ($prevval eq "");
			$val     = 0 if ($val     eq "");
			if ($val !~ /^[\d.]+$/ || $prevval !~ /^[\d\.]+$/) {
				next;
			}
			my $diffval = abs($val - $prevval);
			if (eval "$diffval $diffstr") {
				$matched = 1;
				$matchcols{$j} = 1;
			}
		}
		if (!$matched) {
			# remove the row
			splice(@csvmatrix, $i, 1);
		}
		printProcessing();
	}
	endPrintProcessing();
	grepColumn_with_indexary(keys(%matchcols));
}


#######################################################################
# main routine

generateCSVMatrix(@ARGV);

# set --time if --row is specified.
if (defined($opts{row}) && !defined($opts{time})) {
	$opts{time} = $opts{row}
}

if (defined($opts{time})) {
	print_mes "processing --time ...";
	grepTime($opts{time});
}
if (defined($opts{timerange})) {
	print_mes "processing --timerange ...";
	grepTimeRange($opts{timerange});
}
if (defined($opts{col})) {
	print_mes "processing --col ...";
	grepColumn($opts{col});
}
if (defined($opts{val})) {
	print_mes "processing --val ...";
	grepValueRegex($opts{val});
}
if (defined($opts{valcmp})) {
	print_mes "processing --valcmp ...";
	grepValueComparison($opts{valcmp});
}
if (defined($opts{diffcmp})) {
	print_mes "processing --diffcmp ...";
	grepDifferenceComparison($opts{diffcmp});
}

# print the result
foreach my $rowstr (@csvmatrix) {
	print $rowstr."\n";
}

