#!/usr/bin/perl -w

use strict;
use warnings;
use Time::ParseDate;
use Getopt::Long qw(:config no_ignore_case gnu_compat);
use File::Temp qw(tempfile);

# for debug only, can be removed.
# use Data::Dumper;

# set autoflush on STDOUT, as this program needs it for showing the progression.
STDOUT->autoflush(1);


# option hash with initial value.
my %opts = ();


# exit with error message and code(1).
sub error_exit {
	print STDERR join(" ", @_)."\n";
	exit 1;
}


# current date, which is described at the 1st line of /var/log/sa/sar*
my $curdate = "1970/01/01";


# check if the line is empty, internally this sets $curdate if necessary.
# isEmptyLine($linestr)
sub isEmptyLine {
	$_ = $_[0];
	return 1 if (/^\s*$/);
	if (/^Linux /) {
		my @line = split(/\s+/, $_);
		# check "06/02/2016" or "2016-06-02" etc.
		my @dateary = grep(/[0-9]{2,4}[-\/][0-9]{2}[-\/][0-9]{2,4}/, @line);
		$curdate = $dateary[0] if ($#dateary == 0);
		return 1;
	}
	return 0;
}


# parse a line of "sar -A -f sa##", the line never be blank("")
# parseLine($file_handler)
sub parseLine {
	my $line = $_[0];
	my @parsed = split(/\s+/, $line);
	if ($#parsed >= 2 && $parsed[1] =~ /^[AP]M$/) {
		# exchange time string, ex. to "1:02:03 AM"
		my $timestr = shift @parsed;
		$parsed[0] = "$timestr $parsed[0]";
	}

	# exhange Japanese to one in English
	$parsed[0] =~ s/^平均値:$/Average/;
	$parsed[0] =~ s/^( ?[0-9]{1,2})時([0-9]{2})分([0-9]{2})秒$/$1:$2:$3/;
	$parsed[0] =~ s/^Average:$/Average/;

	if ($parsed[0] =~ /^ ?[0-9]{1,2}:[0-9]{2}:[0-9]{2}/ && $curdate ne "") {
		# this must be time, xfr to ex. "2016/05/22 17:30:25"
		my $nowsec = parsedate("$curdate $parsed[0]");
                my ($sec,$min,$hour,$mday,$mon,$year,,) = localtime($nowsec);
		$parsed[0] = sprintf("%04d/%02d/%02d %02d:%02d:%02d",
                      		  $year+1900,$mon+1,$mday,$hour,$min,$sec);
        }

	# check whether the $line includes ",", as it will be trouble to
	# generate CSV file(parsed by ",")
	for (my $i = 0; $i <= $#parsed; $i++) {
		if ($parsed[$i] =~ /,/) {
			print "Warning: the line '$line' includes \",\", replace it to \"_\"\n";
			$parsed[$i] =~ s/,/_/g;
		}
	}
	return @parsed;
}


# parse single line data, store the data onto %$datahash
# parseSingleLineData(@parsed_array)
sub parseSingleLineData {
	my ($fh, $datahash, $headerary) = @_;

	# push header
	push(@{$datahash->{Time}}, @$headerary[1..$#$headerary]);

	while(<$fh>) {
		return if isEmptyLine($_);
		my @parsed = parseLine($_);
		if ($#parsed != $#$headerary) {
			print STDERR "invalid element number, skip\n";
			print STDERR "   head:".join(',', @$headerary)."\n";
			print STDERR "   data:".join(',', @parsed)."\n";
			next;
		}
		my %elmhash;
		for (my $i=1; $i <= $#parsed; $i++) {
			$elmhash{$headerary->[$i]} = $parsed[$i];
		}
		# marge the hash
		if (!defined($datahash->{$parsed[0]})) {
			$datahash->{$parsed[0]} = \%elmhash;
		} else {
			$datahash->{$parsed[0]} = { %{$datahash->{$parsed[0]}},
						    %elmhash };
		}
	}
}


# parse multi line data(CPU,DEV,...), store the data onto %$datahash
# parseMultiLineData(@parsed_array)
sub parseMultiLineData {
	my ($fh, $datahash, $headerary) = @_;

	my @devary;
	while(<$fh>) {
		return if isEmptyLine($_);
		my @parsed = parseLine($_);
		if ($#parsed != $#$headerary) {
			print STDERR "invalid element number, skip\n";
			print STDERR "   head:".join(',', @$headerary)."\n";
			print STDERR "   data:".join(',', @parsed)."\n";
			next;
		}

		# push header
		my @curhead = map {"$_($headerary->[1]=$parsed[1])"} @$headerary;
		if (!grep($_ eq $parsed[1], @devary)) {
			push(@devary, $parsed[1]);
			if (!grep($curhead[2] eq $_, @{$datahash->{Time}})) {
				# add appropritate header
				push(@{$datahash->{Time}}, @curhead[2..$#curhead]);
			}
		}

		my %elmhash;
		for (my $i=2; $i <= $#parsed; $i++) {
			$elmhash{$curhead[$i]} = $parsed[$i];
		}
		# marge the hash
		if (!defined($datahash->{$parsed[0]})) {
			$datahash->{$parsed[0]} = \%elmhash;
		} else {
			$datahash->{$parsed[0]} = { %{$datahash->{$parsed[0]}},
						    %elmhash };

		}
	}
}


# read one sar file and generate %datahash
sub readOneSarFile {
	my $sarfile = $_[0];
	my $fh;
	if (!open($fh, "<$sarfile")) {
		error_exit "fail to open file \'$sarfile\'";
	}

	my %datahash = ( "Time" => [] );
	while(<$fh>) {
		# skip empty line
		next if isEmptyLine($_);

		my @headerary = parseLine($_);
		if ($#headerary <= 0) {
			print "Warning: cannot parse a line '$_', skip.\n";
			next;
		}

		# the line should be a header
		if ($headerary[1] =~ /^(CPU|TTY|DEV|IFACE)$/) {
			# push the header;
			parseMultiLineData($fh, \%datahash, \@headerary);
		} else {
			parseSingleLineData($fh, \%datahash, \@headerary);
		}
	}
	close($fh);
	return \%datahash;
}


# print usage and exit
sub printUsageAndExit {
	print join("\n", @_)."\n";
	print <<"END_OF_USAGE";
usage:  # $0 [options] sar_datadirs_or_sar_files
description:
	This script generates a csv file from sar text files(output of
	'sar -A -f /var/log/sa/sa01').
options:
	-o <output_csv_file>
	--output <output_csv_file>
		This specifies the csv file which is generated by this
		command. default = "sar-XXXX.csv"
		XXXX will be filled with unique characters not to overwrite
		existing files.
	--nonamecheck
		This specifies "do not check filename". By default, this
		script check filename whether it is "sar[0-9][0-9]" to
		decide if it is a valid sar file. However, with this
		option, this script just try to analyse any files as a
		sar file.
END_OF_USAGE
	exit 1;
}


# list files that are specified in @_, searched recursively in the directories.
sub listFiles {
	my (@args) = @_;

	my @ret = ();
	foreach my $arg (@args) {
		if (!-e $arg) {
			print STDERR "'$arg' does not exist, skip...\n";
		} elsif (-f $arg) {
			push(@ret, $arg);
		} elsif (-d $arg) {
			my @files = ();
			if (!opendir(DIRHANDLE, $arg)) {
				print STDERR "opendir($arg) failed. skip...\n";
				next;
			}
			foreach (readdir(DIRHANDLE)) {
				next if (/^\.\.?$/); # remove "." and ".."
				push(@files, "$arg/$_");
			}
			closedir(DIRHANDLE);
			push(@ret, listFiles(@files));
		}
	}
	return @ret;
}


# return files which has valid name(or all files if --nonamecheck is specified)
sub findFiles {
	my (@files) = @_;

	my @ret;
	# at first, pick the non-directory-files in @files;
	for (my $i=$#files; $i >= 0; $i--) {
		if (-f $files[$i]) {
			push(@ret, $files[$i]);
			splice(@files, $i, 1);
		}
	}

	my @allfiles = listFiles(@files);
	if ($opts{nonamecheck}) {
		return sort(@ret, @allfiles);
	}
	return sort(@ret, grep(/(^|\/)sar[0-9][0-9]$/, @allfiles))
}


# output %datahash go .csv file
sub outputDatahash {
	my $datahash = $_[0];

	# open output-file
	my ($ofname, $ofh);
	if ($opts{output}) {
		$ofname = $opts{output};
		open($ofh, ">$ofname") || error_exit("cannot open $ofname");
	} else {
		($ofh, $ofname) = tempfile("sar-XXXX", SUFFIX => '.csv');
		if (!$ofh) {
			error_exit("cannot open $ofname");
		}
	}
	print "Generating .csv file: \"$ofname\"\n";

	# @keys = key array for each line, @times = time array to be printed
	my @keys  = @{$$datahash{Time}};
	my @times = sort(keys(%$datahash));
	# remove "Average" from @times, as this could handle multiple files.
	@times = grep($_ ne "Time" && $_ ne "Average", @times);

	# at first, print header, which are in "Time"
	print $ofh "Time,".join(',', @keys)."\n";

	# next, print all lines begin with its time;
	foreach my $time (@times) {
		print $ofh "$time";
		foreach my $key (@keys) {
			print $ofh ',';
			# this "if" is necessary in case devices are
			# dynamically added/removed, like "vif*.*", as
			# %{$datahash} is sparse in the case.
			if (defined(${$datahash->{$time}}{$key})) {
				print $ofh ${$datahash->{$time}}{$key};
			}
		}
		print $ofh "\n";
	}
	close $ofh;
}


############################################################################
# main routine
############################################################################

if (!GetOptions(\%opts,
	'help|h',
	'output|o=s',
	'nonamecheck|n')) {
	printUsageAndExit("Invalid option, see usage.");
}

# check options/how to be booted
if ($opts{help}) {
	printUsageAndExit("help option specified.");
}
if (-p STDIN) {
	printUsageAndExit("Cannot be kicked as pipe command.");
}
if ($#ARGV < 0) {
	printUsageAndExit("Invalid argument number.");
}

# get file list.
my @sarfiles = findFiles(@ARGV);

if ($#sarfiles < 0) {
	error_exit("no sar files are found.");
}

# read and analyze files in @sarfiles
my $datahash = {};
# $datahash is a reference to all of analyzed sar data, like followings:
# %$datahash = {
#		  { Time => (%usr, %sys ...) }, # header information
#		  {"00:00:00" => { %usr => 98, %sys => 2, ...}, # actual data
#		  {"00:10:00" => { %usr => 97, %sys => 3, ...},
#		  {"00:20:00" => { %usr => 99, %sys => 1, ...},
#			:
# };
foreach my $sarfile (@sarfiles) {
	print "Processing sar file \"$sarfile\"...\n";
	my $part_of_datahash = readOneSarFile($sarfile);
	# this overrides the existing key, this is good enough as actually
	# the keys are unique except "Time".
	%$datahash = (%$datahash, %$part_of_datahash);
}

outputDatahash($datahash);

print "Done.\n";

exit 0;

