#!/usr/bin/perl -w

use strict;
use warnings;
use File::Basename;
use Cwd qw(abs_path);
use Getopt::Long qw(:config no_ignore_case gnu_compat);

# for debug only, can be removed.
# use Data::Dumper;

# set current path of this command
my $curpath = dirname(Cwd::abs_path(__FILE__));

my $CONFFILE = "/etc/osw2csv.conf";
if (-f "$curpath/".basename($CONFFILE)) {
	require "$curpath/".basename($CONFFILE);
} elsif (-f $CONFFILE) {
	require $CONFFILE;
}

require "$curpath/osw2csv_functions";


# topver: -1=notset, 0=RHEL5/6, 1=RHEL7
# top headers are different between versions, like:
# RHEL6: "Swap: 19531768k total,3435732k used,16096036k free,7002672k cached"
# RHEL7: "KiB Swap:    0 total,    0 free,    0 used. 15776332 avail Mem"
# RHEL7: "KiB Swap:11111+total,22222+free,33333+used. 44444444+avail Mem"
my $topver = -1;


sub printUsageAndExit {
	error(@_);
	print STDERR <<"END_OF_USAGE";
usage:	# cat topfile | $0 [option] > csvfile.csv
description:
	This script generate csv file from "top" data provided by OSWatcher
	or ExaWatcher.
	The top data is read from STDIN, and cvs is provided onto STDOUT.
	option:
	-a <NAME>, --analyze <NAME>
		each process analysis by NAME, like "VIRT", "RES", "%CPU"...
		"NUM" is special, which analyzes process number, needs -w.
	-w, --wrapup
		wrap-up mode, sum data of processes which has same name.
		This needs -a together.
END_OF_USAGE
	exit 1;
}


# belows are defined in $COMMONFUNCTIONS
our $sys_bootsecond ;  # set by $zzz_second and the 1st "top -" line.
our $zzz_second;  # set by "zzz ... " line. 0 means "there are no zzz line"


# get current second of this top
sub getCurrentSecondOfThisTop {
	my $line = $_[0];

	# analyze first "top - [\d:]+ up "
	$_ = $line;
	my ($nowhour, $nowmin, $nowsec) = /^top - (\d+):(\d+):(\d+) up/;
	return 0 if (!defined($nowhour));
	$_ =~ s/^top - [\d:]+ up\s*(.*)$/$1/;
	
	# get upday/uphour/upmin
	# there are several top format:
	# 1: top - ##:##:## up # min, # users? ...
	# 2: top - ##:##:## up ##:##,  # users? ...
	# 3: top - ##:##:## up # days, ##:##,  # users? ...
	# 4: top - ##:##:## up # days, # min,  # users? ...
	my ($upday, $uphour, $upmin) = (0, 0, 0);
	($upday) = /^(\d+)\s*day/;
	if (defined($upday)) {
		$_ =~ s/^\d+\s*days?,\s*(.*)$/$1/;
	} else {
		$upday = 0;
	}
	($uphour, $upmin) = /^(\d+):(\d+)/;
	if (!defined($uphour)) {
		$uphour = 0;
		($upmin) = /^(\d+)\s*min/;
		if (!defined($upmin)) {
			error_exit("fail to get uptime from 'top -' line:\n".
				   "$line");
		}
	}

	# set $up_second and $sys_bootsecond
	my $up_second = $upday*86400 + $uphour*3600 + $upmin*60;

	if ($zzz_second == 0) {
		my $datestr = sprintf("1970/01/01 %02d:%02d:%02d",
					$nowhour,$nowmin,$nowsec);
		my ($sec, $err) = getSecondsFromDateString($datestr);
		if (!$sec) {
			warning("'$datestr' cannot be analyzed, err: '$err'",
				"$zzz_second is set to 0.");
		}
		$zzz_second = +$sec;
		warning("no zzz line before first 'top -' line.".
			" Attempting the system is now at ".
			getDateStringFromSeconds($zzz_second));
	}
	$sys_bootsecond = $zzz_second - $up_second if ($sys_bootsecond == 0);
	
	# calcurate $ret(current second from 1970/01/01 00:00:00)
	my $cursec = $sys_bootsecond + $up_second;
	my ($sec,$min,$hour,$day,$mon,$year,) = localtime($cursec);
	$day++ if ($hour==23 && $nowhour==0); # could be 23:59:## -> 00:00:##
	my $curdatestr = sprintf("%04d/%02d/%02d %02d:%02d:%02d",
				$year+1900,$mon+1,$day,
				$nowhour,$nowmin,$nowsec);
	my ($ret, $err) = getSecondsFromDateString($curdatestr);
	if (!$ret) {
		warning("top - line cannot be parsed:",
			"$line\ncurdatestr: $curdatestr\nerr: $err");
	}
	# adjust again if across a day on 23:59:##
	$ret -= 86400 if ($cursec-$ret < -600);

	if ($cursec-$ret > 600) {
		warning("top - time diff is too much:",
			"$line",
			"curdatestr: $curdatestr",
			"retstr    : ".getDateStringFromSeconds($ret, $ENV{TZ}),
			"cursec-ret = ".($cursec-$ret));
	}
	$zzz_second = $ret;

	return $ret;
}


######################################################################
# regarding top header analysis

# cpu number that the top shows
my $cpunumber = 0;  # set the cpu number when /^%?Cpu\d\s*:/ exists in top output


# get top header, synchronized for analyzeTopLine();
sub getTopHeader {
	return "Time,Timezone,user(s),load avg1,load avg2,load avg3";
}

# analyze and return cvs-ed "top - " line
# using $zzz_second and $zzz_timezone to decide the actual time.
sub analyzeTopLine {
	my $line = $_[0];

	if ($line !~ /^top -/) {
		warning("Failed to analyze top line:", "$line");
		return ",,,";
	}
	my $sec = getCurrentSecondOfThisTop($line);
	$line =~ s/^top -.*?(\d+) users?,\s*load average:\s*([\d\.]+),\s*([\d\.]+),\s*([\d\.]+)$/$1,$2,$3,$4/;

	return getDateStringFromSeconds($sec).",$ENV{TZ},$line";
}


# get Task header, synchronized for analyzeTaskLine();
sub getTaskHeader {
	return "task total,task running,task sleeping,task stopped,task zombie";
}

# analyze and return cvs-ed "Task:" line
sub analyzeTaskLine {
	my $line = $_[0];
	if ($line !~ /^(Tasks|Threads):/) {
		warning("Failed to analyze Task line:", "$line");
		return ",,,,";
	}
	$line =~ s/^(Tasks|Threads):\s*(\d+) total,\s*(\d+) running,\s*(\d+) sleeping,\s*(\d+) stopped,\s*(\d+) zombie$/$1,$2,$3,$4,$5/;
	return $line;
}


# get Cpu(s) header, synchronized for analyzeTaskLine();
sub getCpuHeader {
	return "cpu %user,cpu %system,cpu %nice,cpu %idle,".
	       "cpu %iowait,cpu %hwirq,cpu %swirq,cpu %steal";
}

# analyze and return cvs-ed "Cpu(s):" line
sub analyzeCpuLine {
	my $line = $_[0];
	if ($line !~ /^%?Cpu\(s\)\s*:/) {
		warning("Failed to analyze Cpu(s) line:", "$line");
		return ",,,,,,,";
	}
	$line =~ s/^%?Cpu\(s\):\s*([\d\.]+)(%| )us,\s*([\d\.]+)(%| )sy,\s*([\d\.]+)(%| )ni,\s*([\d\.]+)(%| )id,\s*([\d\.]+)(%| )wa,\s*([\d\.]+)(%| )hi,\s*([\d\.]+)(%| )si,\s*([\d\.]+)(%| )st$/$1,$3,$5,$7,$9,$11,$13,$15/;
	return $line;
}


# get Cpu# header, synchronized for analyzeTaskLine();
sub getEachCpuHeader {
	my $ret = "";
	for (my $i = 0; $i < $cpunumber; $i++) {
		$ret .= "cpu$i %user,cpu$i %system,cpu$i %nice,cpu$i %idle,".
			"cpu$i %iowait,cpu$i %hwirq,cpu$i %swirq,cpu$i %steal";
		$ret .= "," if ($i < $cpunumber-1);
	}
	return $ret
}

# analyze and return cvs-ed "Cpu#:" line
sub analyzeEachCpuLine {
	my $line = $_[0];
	if ($line !~ /^%?Cpu\d+\s*:/) {
		warning("Failed to analyze Cpu# line:", "$line");
		return ",,,,,,,";
	}
	$line =~ s/^%?Cpu\d+\s*:\s*([\d\.]+)(%| )us,\s*([\d\.]+)(%| )sy,\s*([\d\.]+)(%| )ni,\s*([\d\.]+)(%| )id,\s*([\d\.]+)(%| )wa,\s*([\d\.]+)(%| )hi,\s*([\d\.]+)(%| )si,\s*([\d\.]+)(%| )st$/$1,$3,$5,$7,$9,$11,$13,$15/;
	return $line;
}


# get Mem: header, synchronized to analyzeMemLine();
sub getMemHeader {
	if ($topver <= 0) { # including -1
		return "mem total(k),mem used(k),mem free(k),mem buffers(k)";
	} else {
		return "mem total(k),mem used(k),mem free(k),mem buff/cache(k)";
	}
}

# print analyzed "Mem: " line
sub analyzeMemLine {
	my $line = $_[0];
	if ($topver < 0) {
		$topver = 0 if ($line =~ /^Mem:/);
		$topver = 1 if ($line =~ /^KiB Mem :/);
	}
	if ($topver == 0 && $line =~ /^Mem:/) {
		$line =~ s/^Mem:\s*(\d+)k total,\s*(\d+)k used,\s*(\d+)k free,\s*(\d+)k buffers$/$1,$2,$3,$4/;
	} elsif ($topver == 1 && $line =~ /^KiB Mem :/) {
		$line =~ s/^KiB Mem :\s*(\d+\+?) ?total,\s*(\d+\+?) ?free,\s*(\d+\+?) ?used,\s*(\d+\+?) ?buff\/cache$/$1,$3,$2,$4/;
	} else {
		warning("Failed to analyze Mem line(topver=$topver):", "$line");
		$line = ",,,";
	}
	return $line;
}

# get Swap: header, synchronized to analyzeSwapLine();
sub getSwapHeader {
	if ($topver <= 0) { # including -1
		return "swap total(k),swap used(k),swap free(k), mem cached(k)";
	} else {
		return "swap total(k),swap used(k),swap free(k), mem avail(k)";
	}
}

# print analyzed "Swap: " line
sub analyzeSwapLine {
	my $line = $_[0];
	if ($topver == 0 && $line =~ /^Swap:/) {
		$line =~ s/^Swap:\s*(\d+)k total,\s*(\d+)k used,\s*(\d+)k free,\s*(\d+)k cached$/$1,$2,$3,$4/;
	} elsif ($topver == 1 && $line =~ /^KiB Swap:/) {
		# Note: last \s* of the regex is necessary
		$line =~ s/^KiB Swap:\s*(\d+\+?) ?total,\s*(\d+\+?) ?free,\s*(\d+\+?) ?used[,\.]\s*(\d+\+?) ?avail Mem\s*$/$1,$3,$2,$4/;
	} else {
		warning("Failed to analyze Swap line(topver=$topver):", "$line");
		return ",,,";
	}
	return $line;
}


sub printTopCSVHeader {
	my $output;
	$output .= getTopHeader    .",";
	$output .= getTaskHeader   .",";
	$output .= getCpuHeader    ."," if ($cpunumber <= 0);
	$output .= getEachCpuHeader."," if ($cpunumber > 0);
	$output .= getMemHeader    .",";
	$output .= getSwapHeader;

	print $output."\n";
}


sub printTopCSVBody {
	my $header_printed = 0; # T/F: whether header colmun has been printed

	# read/check the first line
	return if (!defined($_ = readNextLineFromSTDIN()));
	while (defined($_)) {
		# set current time if there is "zzz ***" line
		if (setZZZLine($_)) {
			$_ = readNextLineFromSTDIN();
			next;
		}

		# ignore if the line is not /^top -/
		if ($_ !~ /^top -/) {
			$_ = readNextLineFromSTDIN();
			next;
		}
		# analyze "top - " line 
		my @output = ();
		push(@output, analyzeTopLine($_));
		$_ = readNextLineFromSTDIN();

		# analyze "(Tasks|Threads):" line
		push(@output, analyzeTaskLine($_));
		$_ = readNextLineFromSTDIN() if (/^(Tasks|Threads):/);

		# analyze "%?Cpu(s):" or "%?Cpu# :" line
		if (/^%?Cpu\(s\):/) {
			push(@output, analyzeCpuLine($_));
			$_ = readNextLineFromSTDIN();
		}
		if (/^%?Cpu\d+\s*:/) {
			my $j = 0;
			for (; !$header_printed || $j < $cpunumber; $j++) {
				if (/^%?Cpu$j\s*:/) {
					push(@output, analyzeEachCpuLine($_));
					$_ = readNextLineFromSTDIN();
				} elsif (/^%?Cpu\d+\s*:/) {
					# Cpu#s doesn't have continuous number
					push(@output, ",,,,,,,");
					# try again with current $_ and $j++
				} else {
					last if (!$header_printed);
					push(@output, analyzeEachCpuLine(""));
				}
			}
			$cpunumber = $j if (!$header_printed);
		}

		# analyze "Mem:" line
		push(@output, analyzeMemLine($_));
		$_ = readNextLineFromSTDIN() if (/^Mem:/ || /^KiB Mem :/);

		# analyze "Swap:" line
		push(@output, analyzeSwapLine($_));
		$_ = readNextLineFromSTDIN() if (/^Swap:/ || /^KiB Swap:/);

		if (!$header_printed) {
			printTopCSVHeader;
			$header_printed = 1;
		}
		print join(',',@output)."\n";
	}
}


######################################################################
# regarding each process analysis

# top header array(ex: PID,USER,PR,NI,VIRT,RES,SHR,S,%CPU,%MEM,TIME+,COMMAND)
my @topheader;
# cmd, pid and designated elm index
my ($cmdidx, $pididx, $elmidx) = (11, 0, -1);

# analyze and return cvs-ed "top - " line
# using $zzz_second and $zzz_timezone to decide the actual time.
sub analyzeTopLineAndGetSecond {
	my $line = $_[0];

	if ($line !~ /^top -/) {
		warning("Warning: Failed to analyze top line:", "$line");
		return 0;  # means "1970/01/01 00:00:00";
	}
	return getCurrentSecondOfThisTop($line);
	# note: this function does not return ENV{TZ}, different from others.
}


# check whether it is top process header "PID USER PR NI VIRT ..." 
sub checkTopHeader {
	my ($line, $elmname) = @_;
	# remove head-spaces
	$line =~ s/^\s+//;
	my @ary = split(/\s+/, $line);

	if (!grep ($_ eq "PID" || $_ eq "COMMAND", @ary) ||
	    $line !~ /^[%A-Z_\s\+-]+$/) {
		warning("Failed to analyze top header:", "$line");
		return 0;
	}
	if ($#topheader >= 0) {
		# topheader has been set;
# consider whether this check is necessary, as this takes time.
		if (join(',', @ary) ne join(',', @topheader)) {
			warning("Invalid top header, skip:", "$line\n");
			return 0;
		}
        } else {
		# this is the 1st time to get top header, need to store data.
		my ($tmppididx) = grep { $ary[$_] eq "PID" } 0..$#ary;
		my ($tmpcmdidx) = grep { $ary[$_] eq "COMMAND" } 0..$#ary;
		my ($tmpelmidx) = grep { $ary[$_] eq $elmname } 0..$#ary;
		if (!defined($tmpelmidx)) {
			error_exit("elmname($elmname) cannot be found in:\n".
				    join(',', @ary));
		}
		if (!defined($tmppididx) || !defined($tmpcmdidx)) {
			warning("Failed to analyze top header, skip:",
				"  line:$line",
				"  pididx=$tmppididx, cmdidx=$tmpcmdidx");
			return 0;
		}
		$pididx = $tmppididx;
		$cmdidx = $tmpcmdidx;
		$elmidx = $tmpelmidx;
		@topheader = @ary;
        }
        return 1;
}


# static value for analyzeTopProcessLine() ... "state" can not be used for hash
my %pows = ( 'm' => 1024, 'g' => 1024*1024, 't' => 1024*1024*1024);

# analyze and return cvs-ed process line
# ex. if $elmname=%CPU(=$elmidx=8)
#  57159 root      20   0 61488 1040  720 R 98.1  0.0   0:08.67 grep
# -> (57159,grep,98.1)
sub analyzeTopProcessLine {
	my $line = $_[0];

	# remove spaces on the top of the line.  spaces on the end of the line
	# will be removed by split()
	$line =~ s/^\s+//;

	# omitted, as this has been checked before invoking this function
	# if ($line !~ /^\d+ /) {
	#	warning("Failed to analyze Top-p line:", "$line");
	#	return ();
	#}

	my ($pid, $cmd, $ret) = (split(/\s+/, $line))[$pididx,$cmdidx,$elmidx];

	# meminfos need to be checked as VIRT/RES/SHR could include size
	# prefixes ex."10.5m", "2.3g" to be Kilo-Byte
	if ($ret =~ /^([0-9\.]+)([mgt])$/) {
		$ret = $1*$pows{$2};
	}
	return ($pid, $cmd, $ret);
}


# readlines from STDIN and construct %phash
sub getTopProcessHash {
	my $elmname = $_[0];
	# read/check the first line
	return if (!defined($_ = readNextLineFromSTDIN()));

	my %phash = (
		"prockey" => {},
		"procdata" => {}
	);
	initPrintProcessing();
	while (defined($_)) {
		# set current time if there is "zzz ***" line
		if (setZZZLine($_)) {
			$_ = readNextLineFromSTDIN();
			next;
		}

		# ignore if the line is not /^top -/
		if ($_ !~ /^top -/) {
			$_ = readNextLineFromSTDIN();
			next;
		}

		# analyze "top - " line 
		my $cursecond = analyzeTopLineAndGetSecond($_);
		printProcessing(", time:".getDateStringFromSeconds($cursecond));

		$_ = readNextLineFromSTDIN();

		# skip "(Tasks|Threads):" line
		$_ = readNextLineFromSTDIN() if (/^(Tasks|Threads):/);

		# skip "Cpu(s):" or "Cpu# :" line
		while (/^%?Cpu\(s\):/ || /^%?Cpu\d+\s*:/) {
			$_ = readNextLineFromSTDIN();
		}

		# skip "Mem:" line
		$_ = readNextLineFromSTDIN() if (/^Mem:/);

		# skip "Swap:" line
		$_ = readNextLineFromSTDIN() if (/^Swap:/);

		# check TopHeader("   PID USER      PR  NI ...") line
		$_ = readNextLineFromSTDIN() if (checkTopHeader($_, $elmname));

		# analyze process lines
		# FIXME: /^ *\d+\s+/ is a wrong coding as it assumes PID is 1st
		while (defined($_) && !isZZZLine($_) && /^ *\d+\s+/) {
			my ($pid, $cmd, $elmdat) = analyzeTopProcessLine($_);
			my $keystr = "$cmd:$pid";

			# $phash{prockey} was array but now hash because it is
			# twice faster than array.
			$phash{prockey}->{$keystr} = 1;
			# to reduce memory size, save only the necessary info.
			$phash{procdata}->{$cursecond}->{$keystr} = $elmdat;
			$_ = readNextLineFromSTDIN();
		}
	}
	endPrintProcessing();
	return \%phash;
}



######################################################################
# Main routine

# check the command option which as -p and/or -w whether it intends
# which information should be printed.

my %opts;
GetOptions(\%opts, 'help|h', "analyze|a=s", "wrapup|w");

if ($opts{help}) {
	printUsageAndExit("help eoption specified.");
}
if (!-p STDIN) {
	printUsageAndExit("run as piped command.");
}
if ($opts{wrapup} && !$opts{analyze}) {
	printUsageAndExit("-w needs -a option.");
}
if (!$opts{wrapup} && $opts{analyze} && $opts{analyze} eq "NUM") {
	printUsageAndExit("-a NUM needs -w option.");
}

if (!$opts{analyze}) {
	# for top heade analysis
	printTopCSVBody;
} else {
	# for each process analysis
	my $elmname = $opts{analyze};

	# $phash = process hash
	# Note that NUM is special, temporally change to "PID" here
	my $phash = getTopProcessHash($elmname eq "NUM" ? "PID" : $elmname);
	# $phash = { 
	#     # Note: prockey is a hash as it's faster than array to exclusive the keys
	#     prockey   => { "init:1" => 1, "oracle:5562" => 1, ... ];
	#     procdata  => { 2364349349(=second of "2016/07/08 10:50:01") => {
	#			"init:1" => 1002,
	#				# selected from VIRT,RES,SHR,%CPU,%MEM
	#			"oracle:5562" => 61488
	#				:
	#		   },
	#		   { 2364349359(=second of "2016/07/08 10:50:11") => {
	#				:
	#		   },
	#				:
	# }

	if (!$opts{wrapup}) {
		printProcessCSV($phash);
	} elsif ($elmname eq "NUM") {
		printWrapupedProcessCSV($phash, 1);
	} else {
		printWrapupedProcessCSV($phash, 0);
	}
}
exit 0;

