#!/usr/bin/perl -w
#
# mocpro_level3_conf.pl
# mocness tow data file manipulation routine - level 3
# For version, see $version

# July 10, 2006 V1.54CM  Modified for CMarZ data.  mda
# July 22, 2008. V1.55  Modified to test for lat and lon values
#	equal to -999.000000 and set to "nd", and for temp or
#	sal equal to 50.000 and set to "nd".  If temp is bad,
#	then set values for theta, sigma and sal to 'nd'.  If
#	sal bad, then set sigma to 'nd'.  Warning:  Code depends
#	on the order of the data input to be in a "good"
#	order, otherwise bad values could be output before
#	they are set to "nd".  rcg
# July 25, 2008. V1.56.  Fix coding so that if temp value is bad
#	the sal value is also set as bad.  rcg
# Jan. 30, 2012. V2.0BC Modified for BCO-DMO data and to use 
#       configuration file as one of inputs. Scripts 
#       on dmoserv2 at /data/Scripts/ctd_mocness  mda + rcg
# February 13, 2012. V2.01BC Divide comment so it fits on two lines. rcg
# April 11, 2014. V2.20. Add require statement for make_log_entry routine. rcg
#
# Looks at all the file $ARGV[3] and outputs level 3 data

# Passed parameters:
#	$ARGV[0] - configuration file
#	$ARGV[1] - data type
#	$ARGV[2] - file type
#	$ARGV[3] - data file to process
#	$ARGV[4] - cruiseid

$version="V2.20/April 11, 2014";

unless ( defined $ARGV[0] ) {
	print STDOUT ("$error Displayed by $0 \n#  Version: $version\n",
		"$error Required configuration file not specified.\n");
	exit;
}

use lib '/data1/www/bco-dmo/perlmodule';
	require ('sendmessage.pl');
	require ('get_config_file_hash.pl');
	require ('make_log_entry.pl');

$configuration_file = $ARGV[0];

$i_am_level = 3;

@required = (
	'fieldnames_level_0', 'fieldnames_level_1', 'fieldnames_level_2',
        'fieldnames_level_3', 'level_3_script', 'data_directories',
	 'moc10patterns', 'moc1patterns', 'mocpoint25patterns'

);

$config_param_ref=&get_config_file_hash($configuration_file);
unless(defined $config_param_ref){
	print STDOUT ("#**error, could not get configuration file contents\n");
	exit;
}

%config_param=%{$config_param_ref};

$status = ' ';
foreach (@required) {
	unless (exists $config_param{$_} and defined $config_param{$_} ) {
		$status = $status . ' ' . $_;
	}
}

unless ($status eq ' ') {
	&make_log_entry(
		"Required configuration parameters not defined=$status");
	&sendmessage($error, "Required configuration parameters $status",
		"not defined.");
	exit;
}

$debug='no';
if (exists $config_param{'debug'} and defined $config_param{'debug'}) {
	if ($debug =~ m/^y/i or $debug =~ m/^1/) {$debug='yes'}
}

print STDOUT "#Displayed by $0 \n#  Version: $version\n";

$error="&x";
$warning="#";

$| = 1;


$fieldnames_level[3] = $config_param{'fieldnames_level_3'};

$datafile=$ARGV[3];

print STDOUT "#  Using $datafile\n";
print STDOUT "# \n";

# Set up fieldname lists
for ($j=0; $j<=9; $j++) {
	$level = 'fieldnames_level_'.$j;
	unless(exists ($config_param{$level}) and
		defined($config_param{$level})) {
		last;
	}
	$fieldnames_level[$j]=$config_param{$level};
	print STDOUT ("# **debug, fieldnames_level[$j]=$fieldnames_level[$j] \n \n")
		if $debug eq 'yes';
}

# Read fieldnames allowed and save for later use
if ($i_am_level != $#fieldnames_level) {
	$message0 = "At level $i_am_level rather than $#fieldnames_level";
	$message1 = "\tin routine $0.  Cannot continue.";
	&sendmessage ($error, $message0, $message1);
	exit 0;
}

# The following code assumes that we are at the last level (Level 3).
#   If this is wrong, Dicky will buy Bob a box of cookies.

@fieldnames=split /,/, $fieldnames_level[$#fieldnames_level];
chomp (@fieldnames);

# Remove field name attributes if present
for ($i=0; $i <= $#fieldnames; $i=$i+1) {
	if (index ($fieldnames[$i], "#") >= 0 ) {  #comment line, drop out
		$fieldnames[$i]="";
		next;
	}
	$qualifier[$i]=$fieldnames[$i];
	$fieldnames[$i]=~ s/(^.*)\[.*/$1/;
	$qualifier[$i]=~ s/^.*(\[.*)/$1/;
	if ($fieldnames[$i] eq $qualifier[$i]) {$qualifier[$i]="";}
	print STDOUT ("#**debug, fieldnames[$i]=",$fieldnames[$i], 
		", qualifier[$i]=",$qualifier[$i], "\n")
		if $debug eq 'yes';
}
# Read data file

open MOCDATA, $datafile or
	die "#  Could not open CTD from MOCNESS data file $datafile, $!";
$_=<MOCDATA>;
chomp;
@notuse=();
push @save_for_comments, $_ ;
print STDOUT " **debug, array save_for_comments=@save_for_comments\n"
	if $debug eq 'yes';
print STDOUT "#**debug, before old/new determination _=$_\n"
	if $debug eq 'yes';

if ( m/^\d{1,}.*/ ) {
#	old form found, with one field or value per line
	$format='old';
	$numbfields=$_ - 1;
	print  STDOUT "# **debug, numbfields=$numbfields\n"
		if $debug eq 'yes';
	for ($k=0; $k <= $numbfields; $k++) {
		$fields_in_data[$k] = <MOCDATA>;
		chomp $fields_in_data[$k];
		print STDOUT ("# **debug, fields_in_data[",
			$k,"]=$fields_in_data[$k]\n")
			if $debug eq 'yes';
	}
	@save_for_comments=("#Note: local times may be in error.  12/8/98 rcg",
		"Thinks there are field names here");
}
else {
# 	new format with field names and data on per line basis
	$format="new";
	$numbfields=0;
	while ($numbfields == 0) {
		$_=<MOCDATA>;
		unless (defined $_ ) {
			$message0 = "Reached end of file in $datafile without";
			$message1 = "finding list of variables expected.";
			&sendmessage ($error, $message0, $message1);
			exit 0;
		}
		chomp;
		s/^ {0,1}% {0,1}//;
		print STDOUT ("# **debug, _=$_\n")
			if $debug eq 'yes';
		push @save_for_comments, $_;
		@fields_in_data = split /\s+/, $_ ;
		for ($i=0; $i <= $#fields_in_data; $i++) {
			$fields_in_data[$i] =~ s/ //g;
			print STDOUT 
			   ("**debug, fields_in_data[$i]=$fields_in_data[$i]\n")
				if $debug eq 'yes';
		}
		if ($fields_in_data[0] eq "" ) {shift @fields_in_data; }
		if ($fields_in_data[0] eq "time" ) {
			$numbfields = <MOCDATA>;
			chomp $numbfields;
			$numbfields =~ s/^% {0,1}//;
			$numbfields = $numbfields -1;
			if ($numbfields != $#fields_in_data) {
				$message0 = "Datafile $datafile had $#fields_in_data fields.";
				$message1 = "Number of fields expected was $numbfields.";
				&sendmessage ($error, $message0, $message1);
				exit 0;
			}
		}
	}
}

for ($i=0; $i <= $#fields_in_data; $i=$i+1) {
	$fields_in_data[$i] =~ s/\s//g;
	if ( $fields_in_data[$i] eq "time" and $format eq "new") { 
		$fields_in_data[$i] = "yrday_local"; }
	elsif ( $fields_in_data[$i] eq "time" and $format eq "old") {
		$fields_in_data[$i] = "time_local_alt"; }
	elsif ( $fields_in_data[$i] eq "pres" ) {
		$fields_in_data[$i] = "press"; }
	print STDOUT ("#**debug, redefine variable names, ", 
		"fields_in_data[$i]=",$fields_in_data[$i], "\n")
			if $debug eq 'yes';	
	
}

# Output file comments if available (from new format only)
if ($#save_for_comments >= 0 ) {
	pop @save_for_comments;	#Remove line of field names
	foreach $comment (@save_for_comments) {
		print STDOUT ("# **debug, comment=$comment\n")
			if $debug eq 'yes';
		print STDOUT ("#",$comment,"\n");
	}
}

# Determine field names that I know about and provide feedback if missing some
NAMES:
for ($i=0; $i <= $#fields_in_data; $i=$i+1) {
	foreach $name (@fieldnames) {
		print STDOUT 
		   ("#**debug, compare # $i, $name with $fields_in_data[$i]\n")
			if $debug eq 'yes';
		next NAMES if $name eq "";
		next NAMES if $name eq $fields_in_data[$i];
	}
	$message0 = "Cannot use field name $fields_in_data[$i].";
	$message1 = "The Data Management Office has been contacted.";
	&sendmessage ($warning, $message0, $message1);
	push @notuse, $i;
}
push @notuse, -1;	#In case all field names are okay

# Output valid field names
SKIP1:
for ($i=0; $i <= $#fields_in_data; $i++) {
	foreach $j (@notuse) {
		if ($j == $i) {next SKIP1; }
		print STDOUT ($fields_in_data[$i]);
		if ( defined $qualifier[$i] ) {print STDOUT ($qualifier[$i]); }
		print STDOUT ("\t");
		last;		#printed out valid field name, end inner loop
	}
}
print STDOUT "\n";

#Read data from file and output
$set_skip_internal_header = 0;

while (<MOCDATA>) {
	$temp_bad = 'no';
	$sal_bad = 'no';
	chomp;
	if ( m/^#/ ) { next; }
	if ( $set_skip_internal_header > 0) {
		$set_skip_internal_header = $set_skip_internal_header - 1;
		next;
	}
#	unless ( m/\w/ ) { next; }
	print STDOUT ("#**debug, format=$format, next data read=$_\n")
			if $debug eq 'yes';
	if ($format eq "old") {
		$data[0]=$_;
		$data[0] =~ s/\s//g;
		for ($i=1; $i <= $numbfields; $i=$i+1) {
			$data[$i]=<MOCDATA>;
			$data[$i] =~ s/\s//g;
			print STDOUT ("#**debug -old fmt, data[$i]=$data[$i]\n")
				if $debug eq 'yes';
		}
	}
	else {		# newformat
		if ( m/Tow:/ ) {
			$set_skip_internal_header = 6;
			print STDOUT ("#Internal header records ",
				"skipped in ", $datafile, "\n");
			next;
		}
		@data = split /\s+/, $_;
		for ($i=0; $i <= $#data; $i=$i+1) {
			$data[$i] =~ s/\s//g;
			print STDOUT ("#**debug -new fmt, data[$i]=$data[$i]\n")
				if $debug eq 'yes';
		}
		if ($data[0] eq "" ) {shift @data; }		
	}
	
#	output only valid field values
SKIP2:	for ($i=0; $i <= $#data; $i=$i+1) {
		print STDOUT ("\n ***Debug, i=$i\n")
			if $debug eq 'yes';
		foreach $j (@notuse) {
			next SKIP2 if $j == $i ;
			print STDOUT ("   ***Debug, j=$j\n") if $debug eq 'yes';
			print STDOUT ("   ***Debug, fields_in_data[$i]=$fields_in_data[$i]\n")
				if $debug eq 'yes';
			if ($fields_in_data[$i] eq "time_local_alt" and  
				$format eq "old") {
				$hour = int($data[$i]);
				if ( $hour < 1 ) {$hour='00'; }
				elsif ($hour < 10)  {$hour='0' . $hour; }
				$min = ($data[$i] - $hour)*60 ;
				$min = sprintf "%6.2f", $min;
				$min =~ s/ //g;
				if ( $min < 10 ) {$min='0' . $min; } #has leading 0
				print STDOUT ("#**debug, data[$i]=$data[$i]\n")
					if $debug eq 'yes';
				$data[$i] = $hour . $min ;
				print STDOUT ('#**debug, hour=', $hour,
					' min=', $min, " data[$i]=", 
					$data[$i], "\n") if $debug eq 'yes';
				print STDOUT "$data[$i]\t";
			}
			elsif ($fields_in_data[$i] eq "lat" or
					$fields_in_data[$i] eq "lon") {
				$data[$i] =~ s/-999.000000/nd/;
				print STDOUT "$data[$i]\t";
			}
			elsif ($fields_in_data[$i] eq "temp") {
				if ($data[$i] =~ m/50.000/) {
					$data[$i] = 'nd';;
					$temp_bad = 'yes';
				}
				print STDOUT "$data[$i]\t";
			}
			elsif ($fields_in_data[$i] eq "sal") {
				if ($data[$i] =~ m/50.000/ or $temp_bad eq 'yes') {
					$data[$i] = 'nd';
					$sal_bad = 'yes';
				}
				print STDOUT "$data[$i]\t";
			}
			elsif ($fields_in_data[$i] eq "sigma" and
					$sal_bad eq 'yes') {
				$data[$i] = 'nd';
				print STDOUT "$data[$i]\t";
			}
			elsif ($temp_bad eq 'yes' and 
					($fields_in_data[$i] eq "theta" or 
					$fields_in_data[$i] eq "sal" or
					$fields_in_data[$i] eq "sigma") ) {
				$data[$i] = 'nd';
				print STDOUT "$data[$i]\t";
			}
			else { print STDOUT "$data[$i]\t"; }
			print STDOUT ("       ***debug, data[$i]=$data[$i]\n")
				if $debug eq 'yes';
			last;
		}
	}
	print STDOUT "\n";
}

close MOCDATA;

undef $error;
undef $warning;

exit 0;