#!/usr/bin/perl -w

use strict;
use warnings;
use IO::Handle;
use File::Basename;
use File::Spec;
use File::Copy qw(move);
use File::Temp qw(tempdir tempfile);
use Cwd qw(abs_path getcwd);
use Getopt::Long qw(:config no_ignore_case gnu_compat);


# set path of this command, to refer other commands(ex. top2csv...)
my $cmdpath = dirname(Cwd::abs_path(__FILE__));

require "$cmdpath/osw2csv_functions";

my $TEMPDIR_TEMPLATE = "osw2csv_workdir_pid_".$$."_XXXXXXXX";


# set autoflush on STDOUT for showing the progression without 'newline'.
STDOUT->autoflush(1);


# SIGPIPE should be ignored, as the piped-child-command could exit abnormally.
my $got_sigpipe = 0;
$SIG{PIPE} = sub { $got_sigpipe = 1; };
# can not exit here even getting sigpipe, as $outputfile need to be removed.


# analysis structure
my %analysis = (
	meminfo => {
		converter  => "$cmdpath/meminfo2csv",
		searchdnam => ["oswmeminfo", "Meminfo.ExaWatcher", "MeminfoExaWatcher" ],
		searchfnam => ["Meminfo", "meminfo"],
		searchfext => ["dat"],
	},
	iostat => {
		converter  => "$cmdpath/iostat2csv",
		searchdnam => ["oswiostat", "Iostat.ExaWatcher", "IostatExaWatcher" ],
		searchfnam => ["Iostat", "iostat"],
		searchfext => ["dat"],
	},
	"netstat-s" => {
		converter  => "$cmdpath/netstat-s2csv",
		searchdnam => ["oswnetstat", "Netstat.ExaWatcher", "NetstatExaWatcher" ],
		searchfnam => ["Netstat", "netstat"],
		searchfext => ["dat"],
	},
	"netstat-i" => {
		converter  => "$cmdpath/netstat-i2csv",
		searchdnam => ["oswnetstat", "Netstat.ExaWatcher", "NetstatExaWatcher" ],
		searchfnam => ["Netstat", "netstat"],
		searchfext => ["dat"],
	},
	top => {
		converter  => "$cmdpath/top2csv",
		searchdnam => ["oswtop", "Top.ExaWatcher", "TopExaWatcher" ],
		searchfnam => ["Top", "top"],
		searchfext => ["dat"],
	},
	ps => {
		converter  => "$cmdpath/ps2csv",
		searchdnam => ["oswps", "Ps.ExaWatcher", "PsExaWatcher" ],
		searchfnam => ["Ps", "ps"],
		searchfnam_exc => ["Mpstat", "mpstat"],
		searchfext => ["dat"],
	},
	slabinfo => {
		converter  => "$cmdpath/slabinfo2csv",
		searchdnam => ["oswslabinfo", "Slabinfo.ExaWatcher", "SlabinfoExaWatcher" ],
		searchfnam => ["Slabinfo", "slabinfo"],
		searchfext => ["dat"],
	},
);

# option hash with initial value.
my $def_analysis = "meminfo,iostat,netstat-s,netstat-i,top";
my %opts = (analyze => $def_analysis, directory => ".", tempdir => "." );


# path for commands
my $CAT   = "/bin/cat";
my $GZIP  = "/bin/gzip";
my $BZIP2 = "/usr/bin/bzip2";
my $XZ    = "/usr/bin/xz";
my $TAR   = "/bin/tar";
my $UNZIP = "/usr/bin/unzip";


# compressed file name which ends with the words, and its extract command:
# Do not use IO::AnyUncompress or so, since EL5 does not have them.
#       .ext => "pipe command to decompress the file"
my %compress_extract = (
	".gz"  => "$GZIP -dc",
	".bz2" => "$BZIP2 -dc",
	".xz"  => "$XZ -dc"
);

# archive file name which ends with the words, and its extract command.
# Do not use Archive::Tar or so, since they are so slow(5 times slower!)
#       .ext => "command extracting files into current directory"
# Note that the commands should be "quiet" without errors(ex. unzip -q).
my %archive_extract = (
	".tar"     => "$TAR -xf",
	".tar.gz"  => "$TAR -xzf",
	".tgz"     => "$TAR -xzf",
	".tar.bz2" => "$TAR -xjf",
	".tbz2"    => "$TAR -xjf",
	".tar.xz"  => "$TAR -xJf",
	".txz"     => "$TAR -xJf",
	".zip"     => "$UNZIP -q"
);


# read configuration file
my $CONFFILE = "/etc/osw2csv.conf";
if (-f "$cmdpath/".basename($CONFFILE)) {
	require "$cmdpath/".basename($CONFFILE);
} elsif (-f $CONFFILE) {
	require $CONFFILE;
}



# check whether a file extension match in the list. case insensitive.
# matched_extension isExtMatch(filename, extlist)
sub isExtMatch
{
	my ($file, @extlist) = @_;
	foreach (@extlist) {
		if ($file =~ /$_$/i) {
			return $_;
		}
	}
	return "";
}


GetOptions(\%opts,
	'help|h',
	'analyze|a=s',
	'output|o=s',
	'directory|d=s',
	'nonamecheck',
	'wrapup|w',
	'dirnamecheck',
	'tempdir|t=s'
);


# print usage and exit
sub printUsageAndExit {
	error(@_);
        print STDERR <<"END_OF_USAGE";
usage:  # $0 [options] datadirs_or_datafiles_or_data_tar+gz/bz2/xz_files
description:
        This script generate csv files which are generated by OSWatcher/
	ExaWatcher in the designated directory, onto current dir.
	the name of the generated files will be "top.csv", "iostat.csv"...
	datafiles may be compressed with gzip(.gz)/bzip2(.bz2)/xz(.xz).
options:
	-a <list_of_analysis>
	--analyze <list_of_analysis>
		This specifies what analysis is necessary.
		currently the combination of followings connected whith ','.
		ex = "-a top,meminfo"
		default = "-a $def_analysis"
		list_of_analysis:
		    iostat    ... iostat info of whole system
		    meminfo   ... /proc/meminfo of whole system
		    netstat-s ... network info of whole system
		    netstat-i ... network info of each interfaces
		    top       ... top header info of whole system
		    slabinfo  ... slabinfo info of whole system
		    top<ELM>  ... shows <ELM> of each processes from top data.
		                  <ELM> is one of top header, like VIRT, RES,
		                  SHR, %CPU or %MEM etc. ex. topVIRT, top%CPU
		    topNUM    ... analyzes process number from top, needs -w.
		    ps<ELM>   ... shows <ELM> of each processes from ps data.
		                  <ELM> is one of ps header, like SZ, PID, PRI,
		                  WCHAN, or TIME etc.  ex. psSZ, ps%CPU
		    psNUM     ... analyzes process number from ps, needs -w.
	-w
	--wrapup
		Wrap up data of processes which has same name.
		This needs -a ps<ELM> or top<ELM>.
		This adds "wrapup" to output filename, like "top%CPUwrapup"
	-o <output_csv_file>
	--output <output_csv_file>
		This specifies the output csv file which is generated by
		this command.
		default: "top.csv" for top, "iostat.csv" for iostat...
		Note: this needs "-a <one_analyze>" option, as the
		output_csv_file can only be specified just one file.
	-d <output_directory>
	--directory <output_directory>
		This specifies the output directory.
		default: "./"(current directory).
	-n
	--nonamecheck
		This specifies "do not check filenames which include analysis
		type of strings".
		By default, this command check the filename, file_extension
		and directory name of the file to decide whether the files
		need to be analyzed.
		ex: "abc-iostat-def/ghi-iostat-jkl.dat" is only for checking
		iostat.
		Note: this needs "-a <one_analyze>" option, as the non-
		namecheck means one file could be handled by several analysis.
	--dirnamecheck
		This specifies "check directory names which include analysis
		type of strings". By default, directory name is never checked.
	-t <temporally directory>
	--tempdir <temporally directory>
		This specifies temporally directory which archived-files
		(ex. tar.gz) will be extracted into.
		default: "."(current directory).
END_OF_USAGE
        exit 1;
}


# extract archive file onto temporally directory(which will be removed
# automatically).
# I decide this is the best method as perl Archive::Tar is so slow...
# archive files in a archive file can be handled appropriately.
sub extractIfArchiveFile {
	my ($file) = @_;

	return if (! -f $file);
	my $ext = isExtMatch($file, keys(%archive_extract));
	# Note that this returns no file list if "it is not archive".
	return if (!$ext);
	
	# if it is an archive file(ex. tar.gz)
	my $dir = tempdir($opts{tempdir}."/".$TEMPDIR_TEMPLATE,
			  CLEANUP => 1);
	# error check is not necessary, as tempdir() exits this script
	# if it has any errors.
	message("Extracting $file into $dir/...");
	my $savedir = Cwd::getcwd();
	# do chdir, as "tar -C" or "unzip -d" is hard to use here
	chdir($dir);
	my $ret = system($archive_extract{$ext}." '$file'");
	chdir($savedir);
	if ($ret != 0) {
		warning("fail to extract $file, skip...");
		return;
	}
	return $dir;
}


# list files located under the args(=files/dirs/archives)
sub listFiles {
	my (@files_dirs_archives) = @_;

	my @ret = ();
	foreach (@files_dirs_archives) {
		$_ = File::Spec->rel2abs($_); # to abs_path
		if (/[';]/) {
			# remove files which have invalid characters to avoid
			# command injection
			warning("skip '$_' as it has invalid charactors");
		} elsif (!-r $_) {
			warning("skip '$_' as it is unreadable");
		} elsif (-f $_) {
			my $archive_dir = extractIfArchiveFile($_);
			if ($archive_dir) {
				# in case $_ is archive file(ex. tar.gz)
				push(@ret, listFiles($archive_dir));
			} else {
				# in case $_ is just a file
				push(@ret, $_);
			}
		} elsif (-d $_) {
			# in case $_ is a directory, handle all files in it
			if (opendir(my $dh, $_)) {
				my ($path, @files) = ($_, readdir($dh));
				closedir($dh);
				@files = grep(!/^\.{1,2}\/*$/, @files);
				@files = map { $_ = "$path/$_"; } @files;
				push(@ret, listFiles(@files));
			} else {
				warning("cannot opendir($_), skip...");
			}
		} else {
			warning("unhandled file '$_', skip...");
		}
	}

	return @ret;
}


# find "$key" data files, and return their list.
sub findFiles {
	my ($key, @files) = @_;

	return @files if ($opts{nonamecheck});

	my @ret = ();

	# pick up files which matches file extensions
	foreach my $fext (@{$analysis{$key}{searchfext}}) {
		push(@ret, grep(/$fext$/, @files));
		foreach my $cext (keys(%compress_extract)) {
			push(@ret, grep(/$fext$cext$/, @files));
		}
	}
	@files = @ret;

	@ret = ();
	# pick up files which have standard filenames
	# need to be 2 loops, as they are collected with "or" condition
	foreach my $file (@files) {
		my $hashp = $analysis{$key};
		FNAMLOOP: foreach my $fpart (@{$$hashp{searchfnam}}) {
			if ($file =~ /$fpart[^\/]*$/) {
				foreach my $exc (@{$$hashp{searchfnam_exc}}) {
					if ($file =~/$exc[^\/]*$/) {
						next FNAMLOOP;
					}
				}
				push(@ret, $file);
				last;
			}
		}
	}
	@files = @ret;

	if ($opts{dirnamecheck}) {
		@ret = ();
		# pick up standard dirnames
		foreach my $file (@files) {
			foreach my $fdir (@{$analysis{$key}{searchdnam}}) {
				if ($file !~ /\// || $file =~ /(^|\/)$fdir\//) {
					push(@ret, $file);
					last;
				}
			}
		}
	}

	# sort with file basename
	@ret = sort { basename($a) cmp basename($b) } @ret;

	return @ret;
}


# get the first line(without \n) of the file. return "" if the file is 0 byte.
sub getFirstLine {
	my $file = $_[0];
	my $fh;

	if (!open($fh, "<$file")) {
		error_exit("fail to open '$file' : $!");
	}
	my $firstline = eof($fh) ? "" : <$fh>;
	chomp($firstline);
	return $firstline;
}


# get the last line(without \n) of the file. return "" if the file is 0byte.
sub getLastLine {
	my $file = $_[0];
	my $fh;

	if (!open($fh, "<$file")) {
		error_exit("fail to open '$file' : $!");
	}
	if (!seek($fh, 0, 2)) {
		error_exit("fail to seek '$file' to the end of the file : $!");
	}
	my $lastline = "";
	my $fsize = tell($fh);
	# go back to the top of the "last line"
	for (my $idx = $fsize-1; $idx > 0 && tell($fh) == $fsize; $idx--) {
		if (!seek($fh, $idx, 0)) {
			error_exit("fail to seek '$file' to index=$idx : $!");
		}
		# dummy read to check whether tell() shows eof
		<$fh>;
	}
	# this should be the last line.
	$lastline = eof($fh) ? "" : <$fh>;
	chomp($lastline);
	close($fh);
	return $lastline;
}


# if the header need to be changed, "tail -n $file" shows the new header.
# This routine checks whether the header needs to be replaced
sub replaceHeaderIfNecessary {
	my $file = $_[0];
	my $lastline = getLastLine($file);
	return if ($lastline !~ /^Time,Timezone/);

	print STDERR "replacing header... ";
	my $fh;
	my ($th, $tmpfnam) = tempfile();
	if (!$th) {
		error_exit("fail to open temporaly file : $!");
	}	
	if (!open($fh, "<$file")) {
		error_exit("fail to open file: '$file' : $!");
	}
	print $th "$lastline\n";
	<$fh>; # skip 1st line
	while (my $line = <$fh>) {
		last if (eof($fh)); # skip lastline by checking eof()
		print $th $line;
	}
	close($fh);
	close($th);
	if (!move($tmpfnam, $file)) {
		error_exit("fail to move file from $tmpfnam to $file : $!");
	}
	print STDERR "done.\n";
}


# find files and run appropriate command
sub findFilesAndRunCommand {
	my ($key, $option, $ofilename, @files) = @_;
	my $outputfile = $opts{directory}."/$ofilename";
	$outputfile =~ s/.csv$/wrapup.csv/ if ($opts{wrapup});
	$outputfile = $opts{output} if (defined($opts{output}));
	my $command = $analysis{$key}{converter};

	if (-e $outputfile) {
		print "'$outputfile' exists, skip.\n";
		return;
	}
	message_nl("Searching $key data files ... ");
	@files = findFiles($key, @files);

	if ($#files < 0) {
		message("found 0 file, skip.");
		return;
	}
	message("found ".($#files+1)." file(s)");

	message("generating '$outputfile' by '$command $option' ... ");
	if (!open(OUTPIPE, "| $command $option > $outputfile")) {
		error_exit("fail to open pipe for '$command $option > $outputfile' : $!");
	}
	my $dir = "";
	foreach my $file (@files) {
		last if ($got_sigpipe);
		if ($dir eq "" || $dir ne dirname($file)) {
			$dir = dirname($file);
			message("  entering directory $dir/");
		}
		message("    ".basename($file));

		my $opencmd = "$CAT '$file' |";

		# consider whether the file is compressed
		my $ext = isExtMatch($file, keys(%compress_extract));
		if ($ext) {
			$opencmd .= " $compress_extract{$ext} |";
		}

		if (!open(DATAFILE, "$opencmd")) {
			warning("fail to open('$opencmd'), skip...");
			next;
		}
		# if the file is normal one(not compressed)
		while (<DATAFILE>) {
			print OUTPIPE $_;
		}
		close(DATAFILE);
		# put "\n" for incomplete endline
		print OUTPIPE "\n";
	}
	if (!close(OUTPIPE) || $got_sigpipe) {
		unlink($outputfile);
		error_exit("'$command' failed.");
	}

	# At last, check whether the head needs to be replaced to the last
	replaceHeaderIfNecessary($outputfile);
}


#########################################################################
# main routine


# option check
if ($opts{help}) {
	printUsageAndExit("help option specified.");
}
if (-p STDIN || $#ARGV < 0) {
	printUsageAndExit("Invalid argument number.");
}
if ($opts{wrapup} && !$opts{analyze}) {
	printUsageAndExit("-w needs -a.");
}
if ($opts{wrapup} && $opts{analyze} && !grep("NUM",split(/,/,$opts{analyze}))) {
	printUsageAndExit("-a NUM needs -w.");
}
if ($opts{analyze}) {
	my @analyze_ary = split(/\s*,\s*/, $opts{analyze});
	my $opt_analyze_num = scalar(@analyze_ary);
	if ($opt_analyze_num != 1 && ($opts{output} || $opts{nonamecheck})) {
		printUsageAndExit("option -a must be specified as one when -o or -n is specified.");
	}
}

my @files = listFiles(@ARGV); 


# analyze just only configured as analyzed.
foreach my $key (split(/,/, $opts{analyze})) {
	if ($key eq "ps") {
		warning("ps analysis needs specific ELM, skip.");
		next;
	}
	if ($key =~ /^(ps|top)(.+)$/) {
		# for special analysis with ps/top
		my($srckey, $elmname) = ($1, $2);
		# set appropriate option
		my $option = "-a $elmname";
		$option .= " -w" if ($opts{wrapup});
		findFilesAndRunCommand($srckey, $option, "$key.csv", @files)
	} else {
		# for normal analysis
		findFilesAndRunCommand($key, "", "$key.csv", @files)
	}
}
