#!/usr/bin/perl -w
#
# mocpro_level1.pl

# mocness tow data file processing routine - level 1

# 23June97/srg 
# Further modified August 11, 1997 rcg
# October 4, 1997, make into level 1.  rcg
# November 14, 1997 rcg Change output message to say displayed by.
# May 6, 1998 rcg Add new data location, /data6
# October 30, 1998 rcg.  Add new data location, /data3/gbdata
# December 3, 1998. rcg Add cruiseid and year to parameters for next level.
# December 17, 1998.  rcg Add remote host or number to mailed error message
# February 17, 1999.  Add /data6, /data7, /data8, /data9, /data10 rcg.
# March 23, 1999.  Add leading # to debug comment lines.  Fix year
#	adjustment to 4 digit.  rcg
# March 23, 1999.  Add brief_desc field name and value by reading data
#	from the inventory files.  Update sendmessage routine.rcg
# April 14, 1999.  Make Ashjian enhancements the real one.  Change script
#	 location back to /data/rgroman/ctd_mocness.  rcg
# June 4, 1999.  Use list rather than listgb program.  rcg
# June 21, 1999. V2.03 Add test for field named defined after EVENTLOG read.
#	Add autoflush.  Change debug output to STDERR.  RCG
# July 12, 1999. V2.04.  Change error code from %x to &x.  Have sendmessage
#	routine send mail AND output message to user.  rcg
# November 16, 1999. V2.05.  Also look in subdirectory mocness for relevant
#	files.  rcg
# January 28, 2000.  V2.06.  Fix Y2K computation.  rcg
# August 1, 2001. V2.07. Improve error handling during file open.  
#	Check for new line in filename and take file name after it.
#	May have been caused when /data6 died and had to be temporarily
#	put into another disk.  rcg
# October 3, 2001. V2.07so. Southern Ocean version of script used
#	originally in GB program.    Field names
#	also come from original location, but patterns are "local".
#	rcg
# December 24, 2001.  V1.08ZG Modified for ZooGene data.  It won't find the
#	cruises in the globec inventory.  Either update it or
#	make an inventory for the zoogene project.  For now,
#	don't access this information since we'll get an error
#	message. rcg
# March 6, 2003. V1.09ZG  Modified March 6, 2003.  Change location of
# 	script file and file names to be /data1/zoogene_data/mocness from
# 	/data1/zoogene/Scripts/mocness.  Change top_datadir to new location
#	as well.  rcg
# July 10, 2006  V1.10CM  Modified for CMarZ data  mda
# Dec. 12, 2011. V1.11BC Modified for BCO-DMO data. Scripts 
#       on dmoserv2 at /data/Scripts/ctd_mocness  mda + rcg

$version=" V1.11BC/Dec. 12, 2011";

# looks at all the MOC files in the specified directories 
# processing those with a "moc" subdir and then
# the files found with the file type specified at run-time.
#
# sets up info required for mocpro_level1.pl based
# on results of files lists and initial header data
# in the data files
#
# Passed parameter
#	$ARGV[0] = MOC1, or MOC10 or MOC.25 specifying which MOCNESS
#			data type to view
#	$ARGV[1] = file type to process, e.g. pro, tab, raw  (note:
#			the code will have to change to accommodate
#			other than pro format.)

#some definitions
$| = 1;
# top level data directories
@top_datadir=("/data302/torres/Antarctic");
$script_loc="/data/Scripts/ctd_mocness";   # script location dir
$i_am_level=1;
$script_name="mocpro_level2.pl";           
$fieldnames_level[0]="/data/Scripts/ctd_mocness/fieldnames_level0";
$fieldnames_level[1]="/data/Scripts/ctd_mocness/fieldnames_level1";
$fieldnames_level[2]="/data/Scripts/ctd_mocness/fieldnames_level2";
$fieldnames_level[3]="/data/Scripts/ctd_mocness/fieldnames_level3";
$moc1patterns="/data/Scripts/ctd_mocness/moc1patterns";
$moc10patterns="/data/Scripts/ctd_mocness/moc10patterns";
$mocpoint25patterns="/data/Scripts/ctd_mocness/mocpoint25patterns";

#$inventory_object = "/globec/gb/inventory";
#$inventory_object = "/globec/unaffiliated/inventory_lmg0602";

$error="&x";
$warning="#";

$datatype=uc $ARGV[0];
$filetype=lc $ARGV[1];
$filetypeuc=uc $filetype;

$| = 1;		#set to flush output

print STDOUT ("#Displayed by $0, $version\n",
	"#  For $datatype data, with filetype=$filetype\n");

if ($datatype eq "MOC10" ) {
	open PATTERN, $moc10patterns or 
		die "#$0 could not read $moc10patterns patterns file, $!";
	@pattern=<PATTERN>;
	chomp @pattern;
#	print STDOUT "#**debug, numb patterns=$#pattern, patterns=@pattern\n";
	close PATTERN;
}elsif ($datatype eq "MOC1" ) {
	open PATTERN, $moc1patterns or 
		die "#$0 could not read $moc1patterns patterns file, $!";
	@pattern=<PATTERN>;
	chomp @pattern;
#	print STDOUT "#**debug, numb patterns=$#pattern, patterns=@pattern\n";
	close PATTERN;
}elsif ($datatype eq "MOC.25" ) {
	open PATTERN, $mocpoint25patterns or 
		die "#$0 could not read $mocpoint25patterns patterns file, $!";
	@pattern=<PATTERN>;
	chomp @pattern;
#	print STDOUT "#**debug, numb patterns=$#pattern, patterns=@pattern\n";
	close PATTERN;
} else {
	print STDOUT "#$0 cannot recognize datatype=$datatype";
	exit 0;
}

if ($filetype eq "pro" ) {
#	yes, we can handle this format
}
else {
	print STDOUT "#Sorry, but I cannot handle filetype=$filetype";
}

$data = ();

foreach $top_datadir (@top_datadir) {
#	print STDOUT "\n#**debug, top_datadir=$top_datadir";
	chdir($top_datadir) or die "#Cannot change to $top_datadir directory";
	opendir DATADIR, $top_datadir or 
		die "#Cannot open directory $top_datadir";
	@cruise_dir = grep -d, readdir DATADIR;
	foreach $cruise_dir (@cruise_dir)   { # get filenames
		$cruise_id = uc $cruise_dir;   
#		print STDOUT "\n#**debug,cruise_dir=$cruise_dir\n";
		$moc_dir = $top_datadir . "/" . $cruise_dir . "/moc";
		if (! -e $moc_dir or ! -d $moc_dir )   {
			$moc_dir =~ s/\/moc$/\/mocness/;
			if (! -e $moc_dir or ! -d $moc_dir ) {next;}
		}
#		print STDOUT "#**debug,moc=$moc_dir\n";
		@datafile=<$moc_dir/*.$filetype>;
		push @datafile, <$moc_dir/*.$filetypeuc>;
		if ( $#datafile < 0 ) {next;}
		foreach $checkfile (@datafile) {
#			$temp=$checkfile;     #used with debug line only
#          		print STDOUT "#**debug, checkfile: $checkfile\n";
			foreach $pattern (@pattern) {
				$_=$checkfile;

				if ( m/$pattern/ ) {
					$datafile=$checkfile;
					if ( $datafile =~ m/\n/ ) {
						$datafile =~  s/.*\n(.*)/$1/;
					}
					goto FILEOKAY;
				}
			}
		}
#		print STDOUT "#**debug, checkfile=$temp does not match pattern\n";
		next;	#skip this directory, not of interest
FILEOKAY:	
		unless (open(PROFILE, $datafile) ) { 
			print STDOUT ("#***debug, datafile=$datafile\n");
			print STDOUT ("#Could not open data file $datafile",
				" error=$!\n");
			next;
		}
#		print STDOUT "\n#**debug, opening file=$datafile\n";
		
		$cruise=$cruise_id;
		$year_gmt="nd";
		$ship= substr $cruise_id, 0, 2;
READDATA:	while(<PROFILE>) {
			$record=$_;
			chomp ;
			if( m/Tow/ )  {
				@towline = split;
				$cruise = uc $towline[2];

			}
			elsif( m/Date:/ ) {
				if ((index $record, "/" ) > 0) {
					@dateline = split;
					@cruisedate = 
						split(/\//,$dateline[1]);
					$year_gmt = $cruisedate[2];
				}
				else {
					@cruisedate = 
						split /\s/, $record;
					$year_gmt = $cruisedate[$#cruisedate];
				}
				if (! defined $year_gmt) {$year_gmt="nd";}
				last READDATA;
			}
			elsif ( m/^\d\d.*/ )  {#old style format
				if ($ship eq "AL" or $ship eq "SJ") {
					$year_gmt=substr $cruise_id, 2, 2;
				}
				else {
					$year_gmt = "nd";
				}				
				last READDATA;
			}
			else {
				$message0= "Unexpected data format found" .
					" in file " . $datafile;
				#message1= "Record read=" . $_;
				&sendmessage ($warning, $message0, $message1);
				$year_gmt="nd";
				last READDATA;
			}
		}

		close(PROFILE);
		if (($ship eq "AL" or $ship eq "SJ") and $year_gmt eq "nd") {
			$year_gmt=substr $cruise_id, 2, 2;
		}
		if ($cruise ne $cruise_id) {
			$message0="Something wrong with file=" .
				 $datafile;
			$message1="Cruise id's do not match: " . 
				$cruise . " versus " .
				$cruise_id;
			&sendmessage ($warning, $message0, $message1);
			$cruise=$cruise_id; #Use value from directory name
		}
#		print STDOUT "\n#**debug ship=$ship\n";
#		print STDOUT "\n#**debug year_gmt=$year_gmt\n";
		if (($ship eq "AL" or $ship eq "SJ") and $year_gmt ne "nd") {
			if($year_gmt > 1000 ) {
				$testyear=$year_gmt - (int($year_gmt/100))*100;
			}
			else {
				$testyear=$year_gmt;
			}
			$yearfromdir=substr $cruise_id, 2, 2;
			if ($testyear != $yearfromdir) {
				$message0="Something wrong with the year " .
					"in " .
					$cruise . " for " .
				 	$datafile;
				 $message1="  Years do not match. " . 
				 	$testyear . " versus " .
					$yearfromdir;
				&sendmessage ($warning, $message0, $message1);
				$year_gmt=$yearfromdir;
			}
		}
		if ($year_gmt ne "nd" ) { 
			if ($year_gmt < 30 ) {
				$year_gmt+=2000;
			}
			elsif ($year_gmt < 100 ) {
				$year_gmt+=1900;
			}
		}	          
		$string = "(" . $script_loc . "/" . 
			$script_name . " " . $moc_dir . " " . $datatype . 
				" " . $filetype . " " . 
				" " . uc $cruise . " " . $year_gmt . ")";
		$field_values{"project"} = uc $cruise;
		$projection{"brief_desc"} = "nd";
#		&get_first_data_from_object ($inventory_object);
		$dataline = $cruise . "\t" . $year_gmt. "\t" . 
			 "nd\t" . $string . "\n";	
		push @data, $dataline;
    	}
}

# headers - 3 o/p by level0 Read from files

for ($i=$i_am_level; $i <= $#fieldnames_level; $i=$i+1) {
	open FIELDNAMES, $fieldnames_level[$i] or 
	   die "#Could not get $0 field names from $fieldnames_level[$i], $!";
		@fieldnames=<FIELDNAMES>;
	close FIELDNAMES;
	chomp (@fieldnames);
	foreach $variable (@fieldnames) {
		print STDOUT "$variable\t" unless (index $variable, "#") >= 0;
	}
	print STDOUT "\n";
}

foreach $line (sort @data) {
	print STDOUT $line;
}

undef $error;
undef $warning;
exit 0;

#---------------------------------------------------------------------------

sub sendmessage {

#Send a message to the user.
#The message sent will be in the strings $_[1] and $_[2]
#The prefix string is in $_[0]

my ( @args, $mailfile, $message0, $message1, $prefix, $who);
my ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst);

$prefix=$_[0];
$message0=$_[1];
$message1=$_[2];
($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst)=localtime(time);
if ($year >= 100 and $year <1000) { $year = $year + 1900; }
$mon++;
if ($mon < 10) { $mon = "0" . $mon; }
if ($mday < 10) {$mday = "0" . $mday; }
if ($hour < 10) { $hour = "0" . $hour; }
if ($min < 10) { $min = "0" . $min; }
if ($sec < 10) { $sec = "0" . $sec; }
$mailfile=">/tmp/sendmess" . $year . $yday . $hour . $min . $sec . ".tmp";

if ( open TEMPFILE, $mailfile) {
	if ( exists $ENV{'REMOTE_HOST'} ) {$who=$ENV{'REMOTE_HOST'} ; }
	elsif (exists $ENV{'REMOTE_ADDR'} ) {$who=$ENV{'REMOTE_ADDR'} ; }
	else {$who="not available"; }
	print TEMPFILE ("Message from $0\n");
	print TEMPFILE (" Date of message: $year/$mon/$mday $hour:$min\n");
	print TEMPFILE (" From: $who\n");
	print TEMPFILE (" $message0\n");
	print TEMPFILE (" $message1\n");
	close TEMPFILE;
	`/usr/bin/mail -w dmo\@globec.whoi.edu <$mailfile`;
	unlink $mailfile;
}
print STDOUT ($prefix,"Message from $0\n");
print STDOUT ($prefix," Date of message: $year/$mon/$mday $hour:$min\n");
print STDOUT ($prefix," $message0\n");
print STDOUT ($prefix," $message1\n");

return;
}


#---------------------------------------------------------------------------

sub get_first_data_from_object {

# Use the event log object to get at day and month for old format data
# Argument passed is:
#	$_[0] = jgofs object, e.g. /globec/gb/eventlogs

# In addition the following global hash arrays are available
#	%field_values, where
#		$field_values{"field_name"} = field_value  or
#		$field_values{"cruiseid"} = "AL9701"
# and
#	%projection, where
#		$projection{"field_name_desired"} = desired_field_value or
#		$projection{"inst"} = MOC1

my ( $field_name, @field_names, $field_value, $listgb, $listgb_options, 
	$message0, $message1, $object_spec, %returned_data );


#print STDERR ("#**debug, object=$_[0]\n");

my $inventory_file = "no";  #No inventory file exists for cmarz

#$listgb = "/data5/globec/bin/list ";
$listgb = "/data302/data_server/dmoserv3/bin/list ";
$listgb_options = "-c -n -b -t ";
$object_spec = "\"$_[0](";

#Create selection criteria
while (($field_name,$field_value) = each %field_values) {
	$object_spec = $object_spec . $field_name . "=" . $field_value . ",";
}

#Create projection criteria
@field_names = keys %projection;
foreach $field_name_desired ( @field_names) {
	$object_spec = $object_spec . $field_name_desired . ",";
}
$object_spec =~ s/(.*),$/$1/;
$object_spec = $object_spec . ")\"";

#print STDERR ("#**debug, list=$listgb\n");
#print STDERR ("#**debug, listgb_options=$listgb_options\n");
#print STDERR ("#**debug, object_spec=$object_spec\n");

$input = $listgb . $listgb_options . $object_spec . "|";

foreach $field_name_desired ( @field_names) {
	$projection{$field_name_desired} = "nd";
}

# return early since can't get data from non-existent cmarz inventory

return if $inventory_file eq "no";

unless ( open EVENTLOG, $input) {
	$message0="Could not open inventory to get cruise type";
	$message1="open=$input, error=$!";
	&sendmessage ($error, $message0, $message1);
	return;
}
# Get desired information
$k = -1;
while (<EVENTLOG>) {
	if ( m/^#/ ) { next; }
	chomp;
#	print STDERR ("#**debug, EVENTLOG=$_\n");
	@returned_data = split /\t/;
	unless (defined $returned_data[0] ) { $returned_data[0]="nd"; }
#	if ($returned_data[0] =~ m/Üd/i ) { $returned_data[0]="nd"; }
	unless ($returned_data[0] =~ m/\w+/ ) { $returned_data[0]="nd"; }
#	print STDERR ("#**debug, returned_data=@returned_data\n");
	if ($#returned_data < $#field_names) { 
		$message0="Could not get data from object $_[0]";
		$message1="Data returned=" . @returned_data;
		&sendmessage ($error, $message0, $message1);
		return;
		}
	foreach $field_name_desired ( @field_names) {
		$k = $k +1;
		$projection{$field_name_desired} = $returned_data[$k];
#		print STDERR ("#**debug, k=$k, ",
#			"projection{$field_name_desired} ",
#			"= $returned_data[$k]\n");
	}
	last;
}
close EVENTLOG;
return;
}