#!/usr/bin/perl -w # process_email.pl # Program to read in data sent in from the FleetLink project and update # the necessary data files with the new data. # All necessary run-time and configuration values are read in from$config_param{'send_mail_option'} # a configuration file specified as the first argument in the command # line. # Assumptions: # 1. Two level 0 files already exist as specified in the # configuration file. I plan to remove this assumption. $version="V1.43 February 7, 2001"; # February 7, 2001. V1.43. Fix the mailing of warning and error messages to the dmo. # February 1, 2001. V1.42. Fix NWS output data field names and add tab between value and # file name. # January 29, 2001. V1.41. Add archiving of old hourly NWS data to separate archive files. # Change structure of NWS data tree to accommodate separate files for each vessel. # Add debug run time parameter so can output test lines if set to yes or 1. # Remove F/V and R/V from in front of vessel names. Add test of alpha character in # test for valid number. Replace vessel names with vessel codes using the input # file to get the codes. Add test for "_" character in valid number routine. Add # test if configuration file is specified so don't get warning message. Add # underscore to athena output file name after vessel code. Fix misspelling in # number test for humidity values (used wind_speed instead). # October 2, 2000. V1.40. Add vessel name to NWS data. Fix test for finding record # with least number of nd's. # September 29, 2000. V1.39. Add output for National Weather Service data pickup. # Fix typo in error message when opening comments data file. # September 28, 2000. V1.38. Modify validity check for numbers to include at least one # digit. # September 14, 2000. V1.37. Add check that degrees and minutes are defined before # testing them; fix misspelling in log message; add better data validity checks; # remove creation of old athena data files when duplicate entry is found; # modify log message accordingly for athena data files accordingly; add # Adventurer as new vessel; ignor line found in config file # when no equal sign present; improve Y2K test in Athena data section; # fix logic of find duplicate entries in Athena data file. RCG # August 17, 2000. V1.36a. Add check for null output in Athena data which can happen after # removing all white spaces. # August 17, 2000. V1.36. Use variables for max and min values. Add tests for non- # numerical values for known field names. Add range test for lat and lon. Add # test for undefined configuration file name. Add code for a "TEST" cruise to # play with dummy data. # August 16, 2000. V1.35. Lots of changes as follows: # fix unitialized variable in ATHENA_VESSEL reading section, # add routine to determine cruise name, and use in various places, # fix output to comments vessel file so doesn't try to output to closed file, # add range checks for air temp, humidity, air pressure, wind speed, wind direction # May 30, 2000. V1.34. Fix email bugs. Add test for defined variables and change # location of mail utility. Fix COMMENTS_DATE code to place output within open section. # May 25, 2000. V1.33. Tweak testing for ways of spelling the vessel name. Add # test for Susan and Caitlyn name without R/V. # May 9, 2000. V1.32. Check for valid date in comments section and use current date if # not defined. Use translate_vessel_name subroutine in comments section and # add test for & to and replacement. Remove this elsewhere. Ditto for blanks # replaced by underscore. Suppress multiline output of comments of nd's. # May 5, 2000. V1.31.2. Turn e-mail back off for error messages. # May 5, 2000. V1.31.1. Turn e-mail back on for error messages. Sort file names # before processing. Remove extra return in Athena data files. # May 1, 2000. V1.30. Change to log fact that no files were processed. # April 14, 2000. V1.29. Correct processing of Athena position information. Input records # may need to have trailing carriage return character removed. # April 13, 2000. V1.28. Handle missing time (either/or hours, minutes or seconds) # in Athena data. # April 3, 2000. V1.27 Finish coding for comment line processing. Fix bug that # caused reprocessing of already viewed (uuencoded) files. Remove leading # and trailing whitespace in lines with =: and chomp value. Check for legal # time values in Athena data. Check for defined time in tow data and # set to nd if not defined. # March 31, 2000. V1.26 Add output of comment lines to their own set of files. # March 30, 2000. V1.25 Add emailing of comment lines. # March 29, 2000. V1.24 Fix multiple problems with not processing data, # duplicate file names. Add processing of comments. rcg # March 23, 2000. V1.23 Add ability to handle athena file names of the form # .z<##>. Add flag for whether data were extracted. rcg # March 20, 2000. V1.22 Check for undefined file_header when only Athena data are # sent to avoid warning about undefined variable. rcg # March 17, 2000. V1.21 Archive Athena data when only Athena data are processed. # March 16, 2000. V1.20 Allow for phi symbol to separate degrees and minutes values # in Athena data. Add code to more accurately document undefined file type. rcg # March 14, 2000. V1.19 Change data files to be stored in subdirectories # based on ship name code. Add code to uudecode and unzip incoming # files. rcg # March 14, 2000. V1.18 Work on problem of replacing Athena file. rcg # March 13, 2000. V1.17 Fix logic in hourly Athena data routine, again. rcg # March 10, 2000. V1.16 Fix logic in hourly Athena data routine. rcg # March 9, 2000. V1.15 Fix missing > symbol in tow file and missing # (empty) Athena hourly data file. rcg # March 8, 2000. V1.14 Fix bugs: using wrong Athena file name; # and add additional log entries. rcg # March 3,, 2000. V1.13 Change name of saved old Athena data file name. # Add error status values to messages. Use first vessel name found # whenever possible. Return to catch format with semicolons. Try # to truncate time strings limiting to hundrendths of minutes. rcg # March 2, 2000. V1.12 Fix output of field names, handle "bad" athena records. rcg # February 11, 2000. V1.11 Further edits to handle more variable input data formats. # February 4, 2000. V1.10 Change to handle email containing both ATHENA and # underway data. rcg # October 27, 1999. V1.01 Change output to replace blanks with underscores in data # so that JGOFS selection works as expected. rcg # October 22, 1999. V1.00 R. Groman $| = 1; $error = "&x"; $warning = "#"; $athena_key_phrase = "ATHENA V"; $cdate_fields = "year\tmonth\tday\t"; $imet_fields = "wind_speed\twind_direction\tpress_bar\thumidity\ttemp_air\t"; #Set maximum and minimum values $min{'bar'} = 700; $max{'bar'} = 1300; $min{'humidity'} = 0; $max{'humidity'} = 100; $min{'temp'} = -60; #air temperature $max{'temp'} = 120; $min{'wind_speed'} = 0; $max{'wind_speed'} = 200; $min{'wind_direction'} = 0; $max{'wind_direction'} = 360; $min{'lat'} = -90; $max{'lat'} = 90; $min{'lon'} = -90; $max{'lon'} = 90; $date = scalar localtime; $day_today = (localtime)[6]; $day_today++; $month_today = (localtime)[4]; $month_today++; $year_today = (localtime)[5]; if ($year_today < 1000) {$year_today = $year_today + 1900;} $hour_today = (localtime)[2]; if ($hour_today < 10 ) {$hour_today = "0" . $hour_today;} $minute_today = (localtime)[1]; if ($minute_today < 10 ) {$minute_today = "0" . $minute_today;} $time_today = $hour_today . $minute_today; $time_stamp = $year_today . $month_today . $day_today . $time_today; $configuration_file = $ARGV[0]; print STDOUT ("\n Program $0 Version: $version\n"); print STDOUT (" configuration file is $configuration_file\n") if defined $configuration_file; print STDOUT (" Date of run: ", $date, "\n\n"); @required = ( "archive_data_directory", "athena_data_field_names", "athena_data_file_root", "athena_hourly_root", "athena_hourly_field_names", "athena_vessel_field_names", "athena_vessel_file", "catch_info_field_names", "catch_info_file", "comments_vessel_file", "comments_vessel_field_names", "comments_date_file", "comments_date_field_names", "comments_file", "comments_field_names", "email_command_string", "new_data_directory", "tow_info_field_names", "tow_info_file", "vessel_info_field_names", "vessel_info_file", "vessel_stats_field_names", "vessel_stats_file", "vessel_code_file"); &read_configuration_file($configuration_file); if (exists $config_param{'debug'} and defined $config_param{'debug'} ) { if ($config_param{'debug'} =~ m/^y/i or $config_param{'debug'} =~ m/^1/ ) {; $debug = "yes"; } else { $debug = "no"; } } &make_log_entry (">>>>>>>Begin Program=$0", "Date of run=$date\tVersion=$version", "Configuration file=$configuration_file"); $okay = "yes"; for ($i=0; $i<=$#required; $i++) { unless (exists $config_param{$required[$i]} ) { $okay="no"; &sendmessage ($error, "$required[$i] is missing from configuration ", "file=$configuration_file"); &make_log_entry ("$required[$i] is missing from configuration file", "file=$configuration_file"); } else { # print STDOUT ("# $required[$i]=$config_param{$required[$i]}\n"); &make_log_entry ("$required[$i]=$config_param{$required[$i]}"); } } if ( $okay eq "no") { &sendmessage ($error, "One or more parameters are missing from the configuration file", "Cannot continue."); &make_log_entry ( "One or more parameters missing from configuration file=$configuration_file"); exit; } $status = &load_platform_hash; unless ($status =~ m/^okay/i ) { &sendmessage ($error, "Could not load the platform hash table, status=$status", "Cannot continue."); &make_log_entry ( "Could not load the platform hash table=$status"); exit; } &process_data; undef $error; undef $warning; undef $version; undef $time_stamp; &make_log_entry ("<<<<<<) { chomp; s/\s//g; if (m/^#/) { next;} unless ( m/\S+/ ) {next;} unless ( m/=/ ) { &sendmessage ($warning, "No equal sign in line of configuration file=$filename", "Line is=$_"); next; } # print STDOUT ("***debug, config input line=$_\n"); ($parameter, $value) = split /=/; $config_param{$parameter} = $value; # print STDOUT ("#**debug, config_param{$parameter}=", # $config_param{$parameter}, "\n"); } close CONFIG_FILE; } #------------------------------------------------------------------------ sub sendmessage { #Send a message to the user. #The message sent will be in the strings $_[0] and $_[1] my ( @args, $mailfile, $message0, $message1, $prefix, $who); my ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst); $prefix=$_[0]; $message0=$_[1]; $message1=$_[2]; ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst)=localtime(time); if ($year >= 100 and $year <= 1000) {$year = $year + 1900} if ($year < 1900) { $year = $year + 2000; } $mon++; if ($mon < 10) { $mon = "0" . $mon; } if ($mday < 10) {$mday = "0" . $mday; } if ($hour < 10) { $hour = "0" . $hour; } if ($min < 10) { $min = "0" . $min; } if ($sec < 10) { $sec = "0" . $sec; } undef $wday; undef $isdst; $mailfile="/tmp/sendmess" . $year . $yday . $hour . $min . $sec . ".tmp"; if ( -e $mailfile) { unlink $mailfile } if ( open TEMPFILE, ">$mailfile") { print TEMPFILE ("Message from $0\n"); if ( exists $ENV{'REMOTE_HOST'} ) {$who=$ENV{'REMOTE_HOST'} ; } elsif (exists $ENV{'REMOTE_ADDR'} ) {$who=$ENV{'REMOTE_ADDR'} ; } else {$who="not available"; } print TEMPFILE (" Date of message: $year/$mon/$mday $hour:$min\n"); print TEMPFILE (" From: $who\n"); print TEMPFILE (" $message0\n"); print TEMPFILE (" $message1\n"); close TEMPFILE; `/bin/mail -s "Probem with $0" dmo\@globec.whoi.edu <$mailfile`; unlink $mailfile; } print STDOUT ($prefix," $message0\n"); print STDOUT ($prefix," $message1\n"); print STDOUT ($prefix," Above message from $0\n"); print STDOUT ($prefix," Date of message: $year/$mon/$mday $hour:$min\n"); return 0; } #---------------------------------------------------------------------------- sub process_data { # Process the new data from the file directory specified in # $config_param{new_data_directory}. Processed data are moved to # $config_param{archive_data_directory}. # Assumptions: # 1. Hash %config_param contains the necessary run-time parameters. my ($catch_counter, $comments_counter, $field_name, $file_header_counter, $file, @files_to_process, $full_file_name, $new_file, $new_full_file_name, $status, $value); unless (chdir $config_param{'new_data_directory'} ) { $err = $!; &sendmessage ($error, "Could not change directory=$config_param{'new_data_directory'}", "Error code=$err. Cannot continue."); &make_log_entry( "Could not change directory=$config_param{'new_data_directory'}", "Error code=$err"); return; } # Make list of files to process. If new data are added during this # processing run, newer files are skipped until next run of the program. unless ( opendir DATADIR, $config_param{'new_data_directory'} ) { $err = $!; &sendmessage ($error, 'Could not open data directory for uuencoded files.', "$config_param{'new_data_directory'}. Error code=$err. Cannot continue."); &make_log_entry( "Could not open data directory for uuencoded files=$config_param{'new_data_directory'}", "Error=$err"); return; } @files_to_process = grep !/^\./, readdir DATADIR; close DATADIR; unless ($#files_to_process >= 0) { &sendmessage ($warning, "No files to process this time.", "Data directory was $config_param{'new_data_directory'}"); &make_log_entry ("No files to process", "Data directory=$config_param{'new_data_directory'}"); return; } #First need to uudecode and move encoded file to archive. foreach $file (sort @files_to_process) { $full_file_name = $config_param{'new_data_directory'} . "/" . $file; print STDOUT ("\tProcessing $full_file_name as uuencoded file ...\n"); `/usr/bin/uudecode $full_file_name >& /dev/null`; $status = $?; unless ($status == 0) { &make_log_entry ("Could not uudecode file=$full_file_name", "File will remain for further processing"); } else { &make_log_entry ("Successfully uudecoded the file=$full_file_name"); &archive_data ($full_file_name); } } unless ( opendir DATADIR, $config_param{'new_data_directory'} ) { $err = $!; &sendmessage ($error, 'Could not open data directory for zipped files.', "$config_param{'new_data_directory'}. Error code=$err. Cannot continue."); &make_log_entry( "Could not open data directory for zipped files=$config_param{'new_data_directory'}", "Error=$err"); return; } # Now unzip the resulting files as long as file type is ".z" just in case new # files have arrived in the interim. @files_to_process = grep !/^\./, readdir DATADIR; close DATADIR; foreach $file (sort @files_to_process) { $full_file_name = $config_param{'new_data_directory'} . "/" . $file; print STDOUT ("\tProcessing $full_file_name as zipped file ...\n"); $new_file = $file; if ($file =~ m/\.z\d+$/i ) { $new_file =~ s/^(.*)(\.z)(\d+)$/$1_$3$2/i; $new_full_file_name = $config_param{'new_data_directory'} . "/" . $new_file; if ( (rename $full_file_name, $new_full_file_name) == 0) { &make_log_entry ("Could not rename file=$full_file_name", "To file=$new_full_file_name"); next; } else { &make_log_entry ("Renamed file=$full_file_name", "To new file=$new_full_file_name"); } $full_file_name = $new_full_file_name; } unless ($new_file =~ m/\.z$/i ) { &make_log_entry ("Could not gunzip file=$full_file_name", "File does not have .z file type", "File will remain for further processing"); next; } &make_log_entry ("About to gunzip file=$full_file_name"); `/bin/gunzip $full_file_name >& /dev/null`; $status = $?; unless ($status == 0) { &sendmessage ($warning, "Could not gunzip $full_file_name\n$warning Status=$status", "File will be moved to archive without processing"); &make_log_entry ("Could not gunzip file=$full_file_name", "Status=$status", "File will be moved to archive without processing"); } else { &make_log_entry ("Successfully unziped file=$full_file_name") } } unless ( opendir DATADIR, $config_param{'new_data_directory'} ) { $err = $!; &sendmessage ($error, 'Could not open data directory.', "$config_param{'new_data_directory'}. Error code=$err. Cannot continue."); &make_log_entry( "Could not open data directory=$config_param{'new_data_directory'}", "Error=$err"); return; } @files_to_process = grep !/^\./, readdir DATADIR; close DATADIR; # Start reading files; determine the type of file it is and go from there # Store field names and data in the hash array %data, but watch out for # duplicates. foreach $file (sort @files_to_process) { # print STDOUT ("#**Debug, file=$file\n"); $data_extracted_flag = "no"; $full_file_name = $config_param{'new_data_directory'} . "/" . $file; print STDOUT ("\tProcessing $full_file_name as data ...\n"); &make_log_entry ("Processing data file=$full_file_name"); unless (open DATA, $full_file_name) { $err = $!; &sendmessage ($warning, "Could not open data file $full_file_name", "Error code=$err. Continuing to next file"); &make_log_entry ("Could not open data file=$full_file_name", "Error=$err"); next; } undef %data; $catch_counter = -1; $file_header_counter = -1; $comments_counter = -1; # Read contents of file. Skip records unless they contain a # a =: string. This gets around other stuff for now. # A special case are the Athena data files. Check for those # and process them. Assumes ATHENA key phrase # appears only once in a data file. foreach $record () { if ($record =~ m/$athena_key_phrase/ ) { $status = &add_athena_data ($full_file_name); &make_log_entry ("Processed athena file=$full_file_name"); # Cannot archive at this point since file may contain more data next; } unless ($record =~ m/=:/ ) { next; } chomp $record; $record =~ tr /\x0D//d; # print STDOUT ("#**debug, record=$record\n"); ($field_name, $value) = split /=:/, $record; unless ( defined $value) { $value = 'nd' } $value =~ s/\s+$//; #strip trailing whitespace $field_name = lc $field_name; $field_name =~ s/\s+$//g; $field_name =~ s/^\s+//; #strip leading whitespace $field_name =~ s/ /_/g; #Replace embedded blanks with _ $field_name =~ s/\//_/g; #Replace slashes with underscores $value =~ s/^\s+//; #strip leading whitespace if ($value eq "" or (not ($value =~ m/\S/)) ) {$value='nd' } # print STDOUT ("#**debug, field_name=$field_name, value=$value\n"); if ((exists $data{$field_name} ) and $field_name eq "catch") { $catch_counter++; $key = "catch_" . $catch_counter; $data{$key} = $value; } elsif ((exists $data{$field_name} ) and $field_name eq "file_header") { $file_header_counter++; $key = "file_header_" . $file_header_counter; $data{$key} = $value; } elsif ((exists $data{$field_name} ) and $field_name=~ m/comments/i) { $comments_counter++; $key = "comments_" . $comments_counter; $data{$key} = $value; } elsif ((exists $data{$field_name} ) and $data{$field_name} ne $value ) { &make_log_entry ( "Duplicate field name found for field name=$field_name", "Old=$data{$field_name}", "New (ignored)=$value"); } else { $data{$field_name} = $value; } } $status = "okay"; if (defined $data{'file_header'} ) { $status = &add_catch_data ($data{'file_header'}); } unless ($status =~ m/^failed/i and $data_extracted_flag eq "no") { &archive_data ($full_file_name) } &output_comments_prn; &email_comments_prn; undef %data; close DATA; } } #------------------------------------------------------------------------- sub add_athena_data { # Read in this data file as athena data and write it out to the data file(s). # First passed parameter contains full file name of data. my ($athena_in_file, @athena_data_field_names, $athena_data_file, $athena_data_file_root, @athena_hourly_field_names, @athena_vessel_field_names, $athena_vessel_file, $here, $info_file, @info_field_names, @local_field_names, $previous_vessel_name, $vessel_code, $vessel_name, $vessel_file ); my ($year_start, $month_start , $day_start, $time_start) ; $here = "add_athena_data"; $athena_in_file = $_[0]; $athena_vessel_file = $config_param{'athena_vessel_file'}; @athena_vessel_field_names = split /,/, $config_param{'athena_vessel_field_names'}; $athena_data_file_root = $config_param{'athena_data_file_root'}; @athena_data_field_names = split /,/, $config_param{'athena_data_field_names'}; $athena_hourly_root = $config_param{'athena_hourly_root'}; @athena_hourly_field_names = split /,/, $config_param{'athena_hourly_field_names'}; unless (open ATHENA_IN, $athena_in_file) { $err = $!; &sendmessage ($warning, "Could not open athena input file $athena_in_file", "Error code=$err. Cannot process this file."); &make_log_entry ("Could not open athena input file=$athena_in_file", "Error=$err"); return "Failed, could not open athena input file"; } $status = &get_athena_header_information; #print STDOUT ("***debug $here, after call to get athena header information athena header=", # @athena_header, "\n"); close ATHENA_IN; if ($status =~ m/failed/i ) { &sendmessage ($error, "Could not get athena header information from file $athena_in_file", "Status is $status."); &make_log_entry ("Could not get athena header information from file=$athena_in_file", "Status=$status"); return "Failed, could not get athena header information"; } ($vessel_name, $year_start, $month_start, $day_start, $time_start) = @athena_header; $vessel_name = &translate_vessel_name ( $vessel_name ); $previous_vessel_name = &determine_vessel_name; if ($vessel_name ne $previous_vessel_name ) { &make_log_entry ("Using vessel name from Athena=$vessel_name", "Although name previously known as=$previous_vessel_name"); } # If vessel name known via athena use it everywhere from now on if ($vessel_name ne "nd" ) { $data{'vessel_name'} = $vessel_name; } # Need to know the vessel code in order to construct the athena file names. $vessel_code = &getvesselcode ($vessel_name); $athena_hourly_file = $athena_hourly_root . $vessel_code . "_" . $year_start . $month_start . $day_start . $time_start . ".dat"; $athena_data_file = $athena_data_file_root . $vessel_code . ".dat"; unless (-e $athena_vessel_file) { unless (open ATHENA_VESSEL, ">$athena_vessel_file") { $err = $! &sendmessage ($warning, "Could not open new athena vessel file $athena_vessel_file", "Error code=$err. Cannot process data."); &make_log_entry ("Could not open new athena vessel file=$athena_vessel_file", "Error=$err"); return "Failed, could not open new athena vessel file"; } print ATHENA_VESSEL ("#Athena vessel file created ", $date, "\n"); foreach (@athena_vessel_field_names) { s/\s//g; print ATHENA_VESSEL ($_,"\t"); } print ATHENA_VESSEL (">\n"); foreach (@athena_data_field_names) { s/\s//g; print ATHENA_VESSEL ($_,"\t"); } print ATHENA_VESSEL (">\n"); foreach (@athena_hourly_field_names) { s/\s//g; if ( m/IMET/ ) { print ATHENA_VESSEL ($imet_fields); } elsif ( m/CDATE/ ) { print ATHENA_VESSEL ($cdate_fields); } else { print ATHENA_VESSEL ($_,"\t"); } } print ATHENA_VESSEL ("\n"); close ATHENA_VESSEL; } #print STDOUT ("***debug $here, should open new hourly file\n"); #print STDOUT ("***debug $here, athena_data_file=$athena_data_file\n"); unless (-e $athena_data_file) { unless (open ATHENA_DATA, ">$athena_data_file") { $err = $!; &sendmessage ($error, "Could not open new athena data file $athena_data_file", "Error code=$err. Cannot process data."); &make_log_entry ("Could not open new athena data file=$athena_data_file", "Error=$err"); return "Failed, could not open new athena data file"; } print ATHENA_DATA ("#Athena data file created ", $date, "\n"); foreach (@athena_data_field_names) { s/\s//g; print ATHENA_DATA ($_,"\t"); } print ATHENA_DATA (">\n"); foreach (@athena_hourly_field_names) { s/\s//g; if ( m/IMET/ ) { print ATHENA_DATA ($imet_fields); } elsif ( m/CDATE/ ) { print ATHENA_DATA ($cdate_fields); } else { print ATHENA_DATA ($_,"\t"); } } print ATHENA_DATA ("\n"); close ATHENA_DATA; } $status = &add_to_athena_vessel_file ($athena_vessel_file, $vessel_name, $athena_data_file); if ($status =~ m/Failed/i ) { return $status;} $status = &add_to_athena_data_file ($athena_data_file, $athena_hourly_file, @athena_header); if ($status =~ m/Failed/i ) { return $status} $status = &create_athena_hourly_data ($athena_in_file, $athena_hourly_file); if ($status =~ m/Failed/i ) { return $status} if ($status =~ m/Warning/i ) { make_log_entry ("Warning retured from routine=create_athena_hourly_data", "Status=$status"); } &make_log_entry ("Athena data added from input file=$athena_in_file"); undef $here; return "Okay, athena data processed"; } #------------------------------------------------------------------------- sub add_catch_data { # Read in data and write it out to the appropriate data file(s). # Passed parameter: # $_[0] = file header string (defines file type), e.g. # vessel_stats (aka catch data) or tow_info # Determine if data exist. If not, issue warning message and return. my ($counter, $file_name, $file_type, $key, $status); #print STDOUT ("#**debug, add_catch_data routine, \$_[0]=$_[0]\n"); if (defined $_[0] ) { $file_type = $_[0]; $file_type =~ s/\s//g; # print STDOUT ("***debug, file_type=$file_type\n"); } else { &make_log_entry("Routine=add_catch_data", "File type not defined", "Probably only athena data sent"); $status = "Warning, undefined data type"; return $status; } if (exists $config_param{$file_type . "_file"} and defined $config_param{$file_type . "_file"} ) { $file_name = $config_param{$file_type . "_file"}; } else { &make_log_entry("Routine=add_catch_data", "File name cannot be resolved", "File_type=$file_type"); $status = "Error, unresolved file type, $file_type"; return $status; } $status = "ok"; # Update data files as necessary. Note that data files may contain more # than one file type, except for Athena data, which are handled separately. # If other than vessel_stats file header is first, this logic won't capture # vessel stats information. if ($file_type eq "vessel_stats" ) { $status = &update_vessel_info_file; if ($status =~ m/^Failed/i ) { return $status } $status = &update_vessel_stats_file ($file_name); if ($status =~ m/^Failed/i ) { return $status } $counter = 0; $key = "file_header_" . $counter; while ( exists $data{$key} ) { if ( $data{$key} eq "tow_info" ) { &do_tow_file ('nd') } $counter++; $key = "file_header_" . $counter; } } elsif ($file_type eq "trip_info" ) { $status = &update_vessel_stats_file ($file_name); if ($status =~ m/^Failed/i ) { return $status } $counter = 0; $key = "file_header_" . $counter; while ( exists $data{$key} ) { if ( $data{$key} eq "tow_info" ) { &do_tow_file ('nd') } $counter++; $key = "file_header_" . $counter; } } elsif ($file_type eq "tow_info") { &do_tow_file ('nd'); } else { &sendmessage( $warning, "Could not update for file type $file_type data.", "File name was $file_name. Continue processing."); &make_log_entry("add_catch_data cannot process file type=$file_type", "File name=$file_name", "Message=unknown data type"); $status = "Failed, unknown data type"; } return $status; } #--------------------------------------------------------------------- sub update_vessel_info_file { # Update vessel info file specified by $config_param{"vessel_info_file"} # with the available information. my ($field_name, @field_names, $file_name, $temp, $vessel_name); $file_name = $config_param{"vessel_info_file"}; chomp $file_name; #print STDOUT ("#**debug, update_vessel_info_file routine, file_name=$file_name\n"); $vessel_name = &determine_vessel_name; if ($vessel_name eq "nd" ) { &sendmessage ($warning, "Vessel name not defined.", "Cannot update vessel info file."); &make_log_entry("Vessel name not defined for file=$file_name"); return "Failed, vessel name not defined"; } @field_names = split /,/, $config_param{"vessel_info_field_names"}; if ( -e $file_name) { unless (open VESSELINFO, "$file_name") { $err = $!; &sendmessage ($warning, "Could not open vessel info file $file_name for reading.", "Error=$! Cannot update with $vessel_name. Continue processing."); &make_log_entry("Cannot open vessel info file for reading=$file_name", "Error=$err"); return "Failed, cannot open vessel info file for reading"; } } else { unless (open VESSELINFO, ">$file_name") { $err = $!; &sendmessage ($error, "Could not open new vessel info file $file_name", "Error=$err Cannot open new vessel info file $vessel_name."); &make_log_entry("Cannot open new vessel info file=$file_name", "Error=$err"); return "Failed, cannot create new vessel info file"; } print VESSELINFO ("#Vessel info file created ", $date, "\n"); foreach (@field_names) { print VESSELINFO ($_, "\t"); } print VESSELINFO ("\n"); close VESSELINFO; open VESSELINFO, "$file_name"; } # Check if record already exists. while ( ) { if (m/^#/o ) { next } if (m/$vessel_name/ ) { close VESSELINFO; &make_log_entry("Vessel info record exists for vessel=$vessel_name"); return "Okay, record exixts"; } } close VESSELINFO; unless (open VESSELINFO, ">>$file_name") { $err = $!; &sendmessage ($warning, "Could not open vessel info file $file_name for appending.", "Error=$err. Cannot update with $vessel_name. Continue processing."); &make_log_entry("Cannot open vessel info file for appending=$file_name", "Error status=$err"); return "Failed, cannot append to vessel info file"; } foreach $field_name (@field_names) { if ( exists $data{$field_name} and defined $data{$field_name} ) { $temp = $data{$field_name}; if ($temp =~ m/ /o and $temp !~ m/^'/ ) { $temp =~ s/ /_/go; # $temp = "'" . $temp . "'"; } print VESSELINFO ( $temp, "\t"); } else { print VESSELINFO ( "nd\t"); } } print VESSELINFO ( "\n"); close VESSELINFO; $data_extracted_flag = "yes"; &make_log_entry ("Vessel info data added for=$vessel_name"); return "Okay, vessel info added"; } #------------------------------------------------------------------------ sub update_vessel_stats_file { # Update vessel stats file $_[0]. # Must check to see if vessel stats file has existing record # for this trip before adding new one. my (@date, $file_name, @field_names, @catch_field_names, @tow_field_names, $temp, $vessel, $vessel_code); $file_name = $_[0]; #print STDOUT ("***debug, update_vessel_stats_file routine, file_name=$file_name\n"); @field_names = split /,/, $config_param{'vessel_stats_field_names'}; @catch_field_names = split /,/, $config_param{'catch_info_field_names'}; @tow_field_names = split /,/, $config_param{'tow_info_field_names'}; $vessel = &determine_vessel_name; if ($vessel eq "nd") { &sendmessage ($warning, "Vessel name not defined.", "\tCould not update vessel stats file. Continue processing."); &make_log_entry("Vessel name not defined in update vessel stats file", "Update vessel stats file=$file_name"); return "Failed, vessel name not defined"; } if ( -e $file_name) { unless (open DATAFILEOUT, "$file_name") { $err = $!; &sendmessage ($warning, "Could not open vessel stats file $file_name for reading.", "Error=$err\n. Continue processing."); &make_log_entry("Could not open vessel stats for reading file=$file_name", "Error=$err"); return "Failed, could not open vessel stats for reading"; } # print STDOUT ("\n***debug, for vessel=$vessel\n"); while () { if ( m/^#/o ) { next } # print STDOUT ("***debug, record from file=$_\n"); if ( m/$vessel/ ) { close DATAFILEOUT; &make_log_entry("Vessel stats file contains entry for vessel=$vessel"); return "Okay, record exists"; } } } else { unless (open DATAFILEOUT, ">$file_name") { $err = $!; &sendmessage ($warning, "Could not create vessel stats file $file_name.", "Error=$err\n."); &make_log_entry("Could not create vessel stats file=$file_name", "Error=$err"); return "Failed, could not create vessel stats"; } print DATAFILEOUT ("#Vessel stats file created $date\n"); foreach (@field_names) { print DATAFILEOUT ($_, "\t"); } print DATAFILEOUT (">\n"); foreach (@tow_field_names) { print DATAFILEOUT ($_, "\t"); } print DATAFILEOUT (">\n"); foreach (@catch_field_names) { print DATAFILEOUT ($_, "\t"); } print DATAFILEOUT ("\n"); } close DATAFILEOUT; unless (open DATAFILEOUT, ">>$file_name") { $err = $!; &sendmessage ($warning, "Could not open vessel stats file $file_name for appending.", "Error=$err\n. Continue processing."); &make_log_entry("Could not open vessel stats for appending file=$file_name", "Error=$err"); return "Failed, could not open vessel stats for appending"; } $vessel_code = &getvesselcode ($vessel); chomp $vessel_code; if (defined $data{"time_date_of_departure"} ) { @date = split / /, $data{"time_date_of_departure"}; } else { # print STDOUT ("***debug, date string=$date\n"); @date = split /\s+/, $date; if ( $date[2] < 9 ) { $date[2] = "0" . $date[2]; } } #print STDOUT ("***debug, vessel_code=$vessel_code\n"); #print STDOUT ("***debug, date=$date[4], $date[1], $date[2]\n"); $tow_filename = $vessel_code . "_" . $date[4] . $date[1] . $date[2] . ".dat"; #print STDOUT ("***debug, tow_filename=$tow_filename\n"); foreach $field_name (@field_names) { if ( exists $data{$field_name} and defined $data{$field_name} ) { $temp = $data{$field_name}; if ($temp =~ m/ / and $temp !~ m/^'/ ) { $temp =~ s/ /_/g; # $temp = "'" . $temp . "'"; } print DATAFILEOUT ( $temp, "\t"); } else { print DATAFILEOUT ( "nd\t"); } } print DATAFILEOUT ( $config_param{"tow_info_file"}, $tow_filename, "\n"); close DATAFILEOUT; $data_extracted_flag = "yes"; &make_log_entry ("Vessel stats file=$file_name", "Vessel stats data added vessel=$data{'vessel_name'}", "Tow info file=$tow_filename"); return "Okay, vessel stats data added"; } #-------------------------------------------------------------------- sub do_tow_file { # Create tow file. Passed parameter, if present, is the full file spec # of the tow file to create. If set to "nd" file name will be determined. # Possible problem: may also need to add entry in vessel stat file if nd. my (@catch_field_names, $catch_file_name, @date, $day, $day_of_week, $degrees, @existing_tow_data, $exists, @field_names, $field_name, $minutes, $month, $output_record, $sign, %sign_convention, $time, $tow_file, $vessel_code, $vessel_name, $year); %sign_convention = ( 'N', 1, 'S', -1, 'E', 1, 'W', -1); @existing_tow_data = []; $vessel_name = &determine_vessel_name; if ($vessel_name ne "nd" ) { $vessel_code = &getvesselcode ( $vessel_name ); } else { $vessel_code = "NoVesselCode"; } chomp $vessel_code; @field_names = split /,/, $config_param{'tow_info_field_names'}; #print STDOUT ("\n**debug, tow info field names=@field_names\n"); @catch_field_names = split /,/, $config_param{'catch_info_field_names'}; if (defined $data{"time_date_of_departure"} ) { @date = split / /, $data{"time_date_of_departure"}; } else { @date = split /\s+/, $date; if ( $date[2] < 9 ) { $date[2] = "0" . $date[2]; } } if (defined $_[0] and $_[0] ne "nd" ) { $tow_file = $_[0]; } else { $tow_file = $config_param{'tow_info_file'} . $vessel_code . "_" . $date[4] . $date[1] . $date[2] . ".dat"; } chomp $tow_file; $tow_file =~ s/'//g; #print STDOUT ("#**debug, tow_file=$tow_file\n"); if ( -e $tow_file) { unless (open TOWFILE, "$tow_file") { $err = $!; &sendmessage ($warning, "Could not open tow file $tow_file for reading", "Error=$err"); &make_log_entry ("Could not open tow file for reading=$tow_file", "Error=$err"); return "Failed, could not open tow file for reading"; } @existing_tow_data = ; close TOWFILE; unless (open TOWFILE, ">>$tow_file" ) { $err = $!; &sendmessage ($warning, "Could not open tow file $tow_file for appending", "Error=$err"); &make_log_entry ("Could not open tow file for appending=$tow_file", "Error=$err"); return "Failed, could not open tow file for appending"; } } else { unless ( open TOWFILE, ">$tow_file" ) { $err = $!; &sendmessage ($warning, "Could not open new tow file $tow_file", "Error=$err"); &make_log_entry ("Could not open new tow file=$tow_file", "Error=$err"); return "Failed, could not open new tow file"; } print TOWFILE ("#Tow file created ", $date, "\n"); foreach (@field_names) { print TOWFILE ($_, "\t"); } print TOWFILE (">\n"); foreach (@catch_field_names) { print TOWFILE ($_, "\t"); } print TOWFILE ("\n"); } $output_record = ""; foreach $field_name (@field_names) { if ( exists $data{$field_name} and defined $data{$field_name} ) { # print STDOUT ("**debug, data{$field_name}"); # print STDOUT ("=$data{$field_name}\n"); $temp = $data{$field_name}; $temp =~ s/\cA//g; unless ( $temp =~ m/\w+/ ) { $temp = "nd"; } undef $sign; # print STDOUT ("***debug, data{$field_name}=$data{$field_name}\n"); # print STDOUT ("***debug, temp=$temp\n"); if ($temp eq "nd" ) { # Cannot do anything further in this section } elsif ($field_name =~ m/latitude/i or $field_name =~ m/longitude/i) { ($degrees, $sign) = split /,/, $temp; unless (defined $degrees and defined $sign ) { $temp = "nd"; } else { $temp =~ s/(\d{2,3})(\d\d\.\d+)/$1/; ($degrees, $minutes) = ($1, $2); # print STDOUT ("***debug, degrees=$degrees\n"); # print STDOUT ("***debug, minutes=$minutes\n"); if ( defined $sign ) { $sign = uc $sign } # print STDOUT ("***debug, sign_convention{$sign}=$sign_convention{$sign}\n"); if ( defined $sign and defined $sign_convention{$sign} ) { $temp = $sign_convention{$sign} * ($degrees + ($minutes/60) ); $temp = sprintf "%9.4f", $temp; $temp =~ s/ //g; # print STDOUT ("***debug, after truncation temp=$temp\n"); } else { $temp = "nd"; } } } elsif ($field_name =~ m/date/i ) { ($day_of_week, $month, $day, $time, $year) = split /\s/, $temp; undef $day_of_week; if (defined $time and &validate_colon_time ($time) ne "nd") { ($hour, $minute, $second) = split /:/, $time; $time = $hour . $minute; $time = $time + ($second/60); $time =~ s/(\d{1,4}.?\d{0,2}).*/$1/; } else { $time = "nd"; } $temp = $year . ' ' . $month . ' ' . $day . ' ' . $time; } unless (defined $temp) { $temp = "nd"; } if ($temp =~ m/ / and $temp !~ m/^'/ ) { $temp =~ s/ /_/g; } $output_record = $output_record . $temp . "\t"; } else { $output_record = $output_record . $temp . "\t"; } } $catch_file_name = $vessel_code . "_" . $date[4] . $date[1] . $date[2] . ".dat"; $catch_file_name =~ s/'//g; $output_record = $output_record . $config_param{'catch_info_file'} . $catch_file_name . "\n"; #Check if record already exists $exists = "no"; foreach (@existing_tow_data) { if ($output_record eq $_ ) { $exists = "yes"; last; } } if ($exists eq "no" ) { print TOWFILE ( $output_record); &make_log_entry ("Tow data added for=$vessel_code", "Tow file=$tow_file"); } else { &make_log_entry ("Tow data already exists for=$vessel_code", "Tow file=$tow_file"); } close TOWFILE; # Create catch file, assuming there are data $status = &create_catch_file($config_param{'catch_info_file'} . $catch_file_name); if ($status =~ m/Failed/i ) { return $status; } else { return "Okay, tow and catch data added"; } } #------------------------------------------------------------------ sub create_catch_file { # Create catch file. Passed parameter, if present, is the full file spec # of the catch file to create. If set to "nd" will be defined. my ($catch_counter, $catch_file, @data_record, @date, $vessel_code, $vessel_name); $vessel_name = &determine_vessel_name; if ( $vessel_name ne "nd" ) { $vessel_code = &getvesselcode ( $data{'vessel_name'} ); } else { $vessel_code = "NoVesselCode"; } if (defined $data{"time_date_of_departure"} ) { @date = split / /, $data{"time_date_of_departure"}; } else { @date = split /\s+/, $date; if ( $date[2] < 9 ) { $date[2] = "0" . $date[2]; } } if (defined $_[0] and $_[0] ne "nd" ) { $catch_file = $_[0]; } else { $catch_file = $config_param{'catch_info_file'} . $vessel_code . "_" . $date[4] . $date[1] . $date[2] . ".dat"; } chomp $catch_file; $catch_file =~ s/'//g; #print STDOUT ("#**debug, catch_file=$catch_file\n"); unless ($data{'catch'} ) { &make_log_entry("No catch data to write to file=$catch_file"); return "Okay, no catch data to write"; } @field_names = split /,/, $config_param{"catch_info_field_names"}; if (-e $catch_file) { unless (open CATCHFILE, ">>$catch_file" ) { $err = $!; &sendmessage ($warning, "Could not open catch file $catch_file", " for appending. Error=$err\nContinuing."); &make_log_entry("Could not open catch file for appending=$catch_file", "Error=$err"); return "Failed, could not open catch file for appending"; } } else { unless ( open CATCHFILE, ">$catch_file" ) { $err = $!; &sendmesage ($warning, "Could not open new catch file $catch_file, ", "Error=$err\nWill continue."); &make_log_entry("Could not open new catch file=$catch_file", "Error=$err"); return "Failed, could not open new catch file"; } print CATCHFILE ("#Catch file created $date\n"); foreach $field_name (@field_names) { $field_name =~ s/\s//go; print CATCHFILE ( $field_name, "\t"); } print CATCHFILE ("\n"); } # Need to loop for each catch record available in this tow #print STDOUT ("#**debug, data{catch}=$data{'catch'}\n"); @data_record = split /;/, $data{'catch'}; #assumes fields separated by semicolons # Alternative approach assuming no semicolons. Not tested. # Ignoring units if present #@data_record = $data{'catch'} =~ s/^([a-zA-Z\/ \-]+) ([A-Z]+) (\d+).*$/$&/; unless ($#data_record == $#field_names) { &sendmessage ($warning,"Catch data record=@data_record", "Does not match field names=@field_names"); &make_log_entry("Number of field names ($#field_names) does not match data=@data_record"); } else { foreach (@data_record) { s/^\s+//o; s/\s+$//o; if ( m/ /o and $_ !~ m/^'/o ) { $_ =~ s/ /_/g; # $_ = "'" . $_ . "'"; } s/_pounds//i; print CATCHFILE ( $_, "\t"); } print CATCHFILE ("\n"); } $catch_counter=-1; OUTPUT_CATCH: $catch_counter ++; if (exists $data{"catch_$catch_counter"} ) { # print STDOUT ("#**debug, data{catch}=$data{'catch'}\n"); @data_record = split /;/, $data{"catch_$catch_counter"}; unless ($#data_record == $#field_names) { &sendmessage ($warning,"Catch data record=@data_record", "Does not match field names=@field_names"); &make_log_entry("Number of field names ($#field_names) does not match data=@data_record"); } else { foreach (@data_record) { s/^\s+//; s/\s+$//; if ( m/ / and $_ !~ m/^'/ ) { $_ =~ s/ /_/g; # $_ = "'" . $_ . "'"; } s/_pounds//i; print CATCHFILE ( $_, "\t"); } print CATCHFILE ("\n"); } } else { close CATCHFILE; $data_extracted_flag = "yes"; &make_log_entry ("Catch data added for=$vessel_code\tCatch file=$catch_file"); return "Okay, catch data added"; } goto OUTPUT_CATCH; } #---------------------------------------------------------------------- sub getvesselcode { # Looks up the vessel code for $_[0] if defined otherwise it uses # $config_param{"vessel_code_file"} if defined. # Otherwise, returns the value for "nd". Return is via return argument. # Do check with both blanks and underscores in place of blanks. my ($code, $default, $platform, $vessel_alt, $vessel_name); #print STDOUT ("#**debug, getvesselcode, _[0]=", # $_[0], "\n"); #print STDOUT ("#**debug, getvesselcode, data{vessel_name}=", # $data{'vessel_name'}, "\n"); if (defined $_[0] ) { $vessel_name = $_[0]; } else { $vessel_name = &determine_vessel_name;; } $vessel_name = lc $vessel_name; $code = $vessel_name_file_entries{$vessel_name}; foreach $platform (keys %vessel_name_file_entries) { $platform =~ s/%/ /g ; # print STDOUT ("#**debug, vessel record=$_\n"); $code = $vessel_name_file_entries{$platform}; $platform = lc $platform; $vessel_alt = $platform; $vessel_alt =~ s/ /_/g; if ( $vessel_name =~ m/$platform/ or $vessel_name =~ m/$vessel_alt/) { if ( $code eq "nd" ) {$code = $default; } # print STDOUT ("#**debug, platform=$platform, code=$code\n"); close VESSELCODE; return $code; } } undef $platform; close VESSELCODE; return $default; } #----------------------------------------------------------------------- sub make_log_entry { # Make a log entry with @_ if the log file $config_param{"logfile"} # exists. Each entry in @_ are separated by tabs. my ($login, @month, $temp); my ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst); ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst)=localtime(time); if ($year >= 100 and $year <= 1000) {$year = $year + 1900} if ($year < 1900) { $year = $year + 2000; } $mon++; if ($mon < 10) { $mon = "0" . $mon; } if ($mday < 10) {$mday = "0" . $mday; } if ($hour < 10) { $hour = "0" . $hour; } if ($min < 10) { $min = "0" . $min; } if ($sec < 10) { $sec = "0" . $sec; } if (exists $config_param{"logfile"} ) { $temp = ">>" . $config_param{'logfile'}; unless ( open LOGFILE, $temp ) { &sendmessage ($warning, "Could not open logfile=$temp", "Error code=$!"); } else { $login = getlogin || (getpwuid($<))[0] || "Intruder!!"; $login="\tusername=" . $login; print LOGFILE ($year, "/" . $mon . "/" . $mday . ":" . $hour . $min . "." . $sec, $login); foreach (@_) { print LOGFILE ("\t", $_); } print LOGFILE ("\n"); close LOGFILE; } } return; } #---------------------------------------------------------------------- sub archive_data { # Archive the file specified at _[0] to $config_param{'archive_data_directory'} my ($counter, $new_location, $number); my $full_file_name = $_[0]; $counter = 0; $new_location = $full_file_name; $new_location =~ s/.*\/(.*)/$1/; $new_location = $config_param{'archive_data_directory'} . "/" . $new_location; UNIQUE_NAME: if (-e $new_location ) { # &sendmessage ($warning, "Tried to archive to $new_location", # "but file already exists. Will come up with new name."); $new_location = $full_file_name; $new_location =~ s/.*\/(.*)/$1/; $new_location = $config_param{'archive_data_directory'} . "/" . $new_location; $new_location = $new_location . "_" . $counter; $counter++; goto UNIQUE_NAME; } unless (open IN, $full_file_name) { $err = $!; &sendmessage ($warning, "Could not open input file $full_file_name for archiving.", "Error=$err File not archived."); &make_log_entry ("Could not open input archiving file=$full_file_name", "Error=$err"); return; } unless (open OUT, ">$new_location") { $err = $!; &sendmessage ($warning, "Could not open output file $new_location for archiving.", "Error=$err File not archived."); &make_log_entry ("Could not open output archiving file=$new_location", "Error=$err"); return; } while () { print OUT ($_); } &make_log_entry ("Output archive file created=$new_location"); close IN; close OUT; $number = unlink $full_file_name; if ( $number < 1 ) { &sendmessage ($warning, "Could not remove original data file $full_file_name", "Check if archive file at $new_location okay."); &make_log_entry ("Could not remove original data file=$full_file_name", "Check if archive file okay at=$new_location"); } else { &make_log_entry ("Input file=$full_file_name\tMoved to=$new_location"); } return; } #------------------------------------------------------------------- sub get_athena_header_information { # Read in the Athena data file via ATHENA_IN handle (which was already opened). # Return the following information in @athena_header: # $vessel_name - vessel name # $year_start - start year # $month_start - start month # $day_start - start day # $time_start - start time my ( $day_start, $hour, $i, $max_stack, $minute, $month_start, $second, @stack, $time_start, $year_start, $vessel_name); $max_stack = 3; $i = 0; while () { chomp; tr /\x0D//d; if (m/ATHENA V/ ) { #Got the information. Extract and return. if ($i < 3 ) { &sendmessage ($error, "Too few records in Athena header.", "Cannot continue."); &make_log_entry ("Too few records in athena header=$i"); return "Failed, too few records in Athena header"; } $vessel_name = &determine_vessel_name; if ($vessel_name eq "nd") { $vessel_name = $stack[$i-3]; $vessel_name =~ s/(.*) DATA LOG .*/$1/; $vessel_name =~ s!\w/\w (.*)!$1!; $vessel_name =~ &translate_vessel_name ($vessel_name); $data{'vessel_name'} = $vessel_name; } # print STDOUT ("***debug, header, vessel_name=$vessel_name\n"); $year_start = $stack[$i-1]; $year_start =~ s!(\d\d)/(\d\d)/(\d+)(.*)!$3!; if ($year_start >= 100 and $year_start < 1000) { $year_start = $year_start + 1900; } if ($year_start < 1900 ) { $year_start = $year_start + 2000; } $month_start = $1; $day_start = $2; ($hour, $minute, $second) = split /:/, $4; $second = $second/60; $time_start = $hour . $minute . "." . $second; $time_start =~ s/\.0\./\./; $time_start =~ s/\s//g; $time_start =~ s/(\d{1,4}.?\d{0,2}).*/$1/; @athena_header = ($vessel_name, $year_start, $month_start, $day_start, $time_start); # print STDOUT ("***debug, athena_header=@athena_header\n"); return "Okay"; } $i++; $stack[$i] = $_; if ( $i > $max_stack ) { shift @stack; $i--; } } return "Failed, could not extract out Athena header information"; } #------------------------------------------------------------------- sub add_to_athena_vessel_file { # Add Athena vessel data from passed parameters to specified data file. # If entry already exists, make no changes to the file. Assumes Athena # vessel file exists. # athena_vessel_file - Athena output vessel file name # vessel_name - vessel name # athena_data_file - Athena data file specification where next # data level resides my ($athena_data_file, $athena_vessel_file, @record, $vessel_name); ($athena_vessel_file, $vessel_name, $athena_data_file) = @_; #print STDOUT ("***debug, add_to_athena_vessel_file\n"); #print STDOUT ("***debug, athena_data_file=$athena_data_file\n"); #print STDOUT ("***debug, athena_vessel_file=$athena_vessel_file\n"); #print STDOUT ("***debug, athena_data_file=$athena_data_file\n"); unless ( open ATHENA_VESSEL, "$athena_vessel_file") { $err = $!; &sendmessage ($error, "Could not open existing Athena vessel file $athena_vessel_file", "Error=$err\nCannot process data"); &make_log_entry ("Could not open existing athena vessel file=$athena_vessel_file", "Error=$err"); return "Failed, could not open existing Athena vessel file $athena_vessel_file"; } while () { if ( m/^#/ ) {next } chomp; tr /\x0D//d; @record = split /\s/; if ( $#record >= 0 and $record[0] eq $vessel_name ) { close ATHENA_VESSEL; return "Okay"; } } close ATHENA_VESSEL; unless ( open ATHENA_VESSEL, ">>$athena_vessel_file") { $err = $!; &sendmessage ($error, "Could not append to Athena vessel file $athena_vessel_file", "Error=$err\nCannot process data"); &make_log_entry ("Could not append to existing athena vessel file=$athena_vessel_file", "Error=$err"); return "Failed, could not append to existing Athena vessel file $athena_vessel_file"; } print ATHENA_VESSEL ($vessel_name, "\t", $athena_data_file, "\n"); close ATHENA_VESSEL; &make_log_entry ("Athena vessel file=$athena_vessel_file", "Updated with vessel name=$vessel_name"); return "Okay"; } #--------------------------------------------------------------------- sub add_to_athena_data_file { # Add to the Athena data file specified in the passed parameter list using the rest of # the data present. However, check for duplicates first. # athena_data_file - Athena data file specification # athena_hourly_file - Athena hourly file name # athena_header - rest of incoming array containing data to output starting # with vessel name (which isn't needed) my (@arg, $athena_data_file, $athena_hourly_file, $err, $here, $i, $j, $k, $new_record, @record, $update_existing_record); my ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst); $here = "add_to_athena_data_file"; $athena_data_file = $_[0]; $athena_hourly_file = $_[1]; for ($i=3; $i <= $#_; $i++) { $new_record[$i-3] =$_[$i]; } #print STDOUT ("***debug $here, add_to_athena_data_file\n"); #print STDOUT ("***debug $here, athena_data_file=$athena_data_file\n"); #print STDOUT ("***debug $here, athena_hourly_file=$athena_hourly_file\n"); #print STDOUT ("***debug $here, new_record=@new_record\n"); unless ( open ATHENA_DATA, "$athena_data_file") { $err = $!; &sendmessage ($error, "Could not open existing Athena data file $athena_data_file", "Error=$err\nCannot process data"); &make_log_entry ("Could not open existing athena data file=$athena_data_file", "Error=$err"); return "Failed, could not open existing Athena data file $athena_data_file"; } $update_existing_record = "no"; $i=-1; FINDDUP: while () { $i++; if ( m/^#/ ) {next } chomp; tr /\x0D//d; @record = split /\t/; if ($#record == -1) {next} for ($j=0; $j < $#new_record; $j++) { # print ("***debug $here, record[$j]=$record[$j], new_record[$j]=$new_record[$j]\n"); if ($record[$j] eq $new_record[$j] ) { if ( $j == ($#new_record - 1) ) { $update_existing_record = "yes"; last FINDDUP; } } else { next FINDDUP; } } } close ATHENA_DATA; #print ("***debug $here, update_existing_record=$update_existing_record\n"); unless ($update_existing_record eq "yes") { unless ( open ATHENA_DATA, ">>$athena_data_file") { $err = $!; &sendmessage ($error, "Could not append to Athena data file $athena_data_file", "Error=$err\nCannot process data"); &make_log_entry ("Could not append to existing athena data file=$athena_data_file", "Error=$err"); return "Failed, could not append to existing Athena data file $athena_data_file"; } for ($j=0; $j <= $#new_record; $j++) { print ATHENA_DATA ($new_record[$j], "\t"); } print ATHENA_DATA ($athena_hourly_file,"\n"); &make_log_entry ("Sucessfully updated Athena data file=$athena_data_file"); } else { &make_log_entry ("Duplicate record so no need to upate Athena data file=$athena_data_file"); } undef $here; return "Okay, successfully processed athena data file entry"; } #-------------------------------------------------------------- sub create_athena_hourly_data { # Create Athena hourly data file using input data file specified. # athena_data_file - input Athena data file # athena_hourly_file - new Athena hourly data file to be created # Assumptions: # 1. If appending to an existing hourly data file, we assume that the expected # field names have not changed since the time the file was created, presumably # earlier in the day. The files should change daily so this assumption should # not be broken too often. # 2. If the valid field names do not match the expected field descriptions # or don't match the output fields then there will be missing data. Note # however, the field numbers can vary from file to file. # Field numbers Descriptions # 002 CDATE (Computer date) # 003 CTIME (Computer time) # 013 GPS_UTC (GPS University Time) # 015 GPS_LAT (GPS Latitude) # 016 GPS_LON (GPS Longitude) # 020 IMET (IMET data {Wnd, Bar, Hum, SWR, Prc}) # 030 SSTMP (Sea surface temperature) my ($athena_data_file, %athena_field_numbers_from_file, @athena_field_numbers_from_file, $athena_hourly_file, $cdate_field_number, $code, @data_array, @expected_field_names, $filestatus, $header_found, $i, $i1, $imet_field_number, $k, $key, $key1, @legal_output_numbers, $new_line, @nws_array, @print_data, @print_fields, $print_fields, $record, $temp, $value ); my ($day, $month, $year, $time, $hour, $minute, $second); ($athena_data_file, $athena_hourly_file) = @_; @nws_array = (); $nws_record =""; #print STDOUT ("***debug, athena_data_file=$athena_data_file\n"); #print STDOUT ("***debug, athena_hourly_file=$athena_hourly_file\n"); if (-e $athena_data_file) { unless (open ATHENA_DATA_LOCAL, "$athena_data_file") { $err = $!; &sendmessage ($error, "Could not open Athena data file $athena_data_file", "Error=$err\nCannot process data."); &make_log_entry ("Could not open Athena data file=$athena_data_file", "Error=$err"); return "Failed, could not open Athena data file"; } } else { &sendmessage ($error, "Expect Athena data file $athena_data_file", "to exist. But does not. Cannot process data."); &make_log_entry ("Athena data file does not exist, file=$athena_data_file"); return "Failed, Athena data file does not exist"; } #print STDOUT ("***debug, trying to add to Athena hourly file=$athena_hourly_file\n"); if (-e $athena_hourly_file) { $filestatus = "appended"; unless (open ATHENA_HOURLY, ">>$athena_hourly_file") { $err = $!; &sendmessage ($error, "Could not append to Athena hourly file $athena_hourly_file", "Error=$err."); &make_log_entry ("Could not append to Athena hourly file=$athena_data_file", "Error=$err"); return "Failed, could not append to Athena hourly file"; } } else { $filestatus = "created"; unless (open ATHENA_HOURLY, ">$athena_hourly_file") { $err = $!; &sendmessage ($error, "Could not open new Athena hourly file $athena_hourly_file", "Error=$err."); &make_log_entry ("Could not open new Athena hourly file=$athena_data_file", "Error=$err"); return "Failed, could not open new Athena hourly file"; } } $header_found = "no"; #print STDOUT ("***debug, at start of OUTERLOOP\n"); OUTERLOOP: $record = ; unless (defined $record) { close ATHENA_DATA_LOCAL; close ATHENA_HOURLY; &make_log_entry ("No Athena data found in file=$athena_data_file"); return "Warning, could not find any Athena data"; } chomp $record; $record =~ tr /\x0D//d; #print STDOUT ("***debug, record=$record\n"); if ($header_found eq "yes" ) { $header_found = "completed"; unless (defined $record) { &sendmessage ($error, "Could not determine Athena header data in $athena_data_file", "Cannot process data."); &make_log_entry ("Could not determine Athena header data file=$athena_data_file"); close ATHENA_DATA_LOCAL; close ATHENA_HOURLY; return "Failed, could not determine Athena header data"; } # print STDOUT ("***debug, header found section, record=$record\n"); unless ($record=~ m/\w+/ ) { $record = ; } while ( $record =~ m/^ *\d+/ ) { $temp = $record; chomp $temp; $temp =~ s/ *(\d+)\|\s+(\w+)\s+.*/$2/; $value = $1; $temp =~ s/\s//g; $athena_field_numbers_from_file{$temp} = $value; $athena_field_numbers_from_file[$value] = $temp; # print STDOUT ("***debug, athena_field_numbers_from_file{$temp} =", # $athena_field_numbers_from_file{$temp}, "\n"); $record = ; # print STDOUT ("***debug, inside while, record=$record\n"); unless (defined $record) { &sendmessage ($error, "Could not determine Athena header data in $athena_data_file", "Cannot process data."); &make_log_entry ("Could not determine Athena header data file=$athena_data_file"); close ATHENA_DATA_LOCAL; close ATHENA_HOURLY; return "Failed, could not determine Athena header data"; } } # Write variable list for hourly data based on what is available from data # Only the CDATE and IMET field names get special treatment @expected_field_names = split /,/, $config_param{"athena_hourly_field_names"}; undef @legal_output_numbers; undef $cdate_field_number; undef $imet_field_number; print ATHENA_HOURLY ("#Athena hourly file $filestatus $date\n"); $print_fields = ""; foreach $key1 (sort keys %athena_field_numbers_from_file) { $key = $key1; $key =~ s/\s//g; # print STDOUT ("***debug, key=$key\n"); foreach $i1 (@expected_field_names) { $i = $i1; $i =~ s/\s//g; # print STDOUT ("***debug, i=$i\n"); if ($key eq $i) { push @legal_output_numbers, $athena_field_numbers_from_file{$key}; if (uc $key eq 'CDATE') { $print_fields = $print_fields . "\t" . $cdate_fields; $cdate_field_number = $athena_field_numbers_from_file{$key}; } elsif (uc $key eq 'IMET') { $print_fields = $print_fields . "\t" . $imet_fields; $imet_field_number = $athena_field_numbers_from_file{$key}; } else { $print_fields = $print_fields . "\t" . $key ; } last; } } } chomp $print_fields; $print_fields =~ tr /\x0D//d; $print_fields =~ s/ //g; $print_fields =~ s/^\t{0,}(.*)/$1/; $print_fields =~ s/\t{2,}/\t/g; # print STDOUT ("***debug, print_fields=$print_fields\n"); print ATHENA_HOURLY ($print_fields, "\n") unless $filestatus eq "appended"; $data_extracted_flag = "yes"; @print_fields = split /\t/, $print_fields; # print STDOUT ("***debug, array print_fields=@print_fields\n"); } # Main loop to extract data elsif ($header_found eq "completed") { $new_line = "no"; for ($k=0; $k <= $#print_fields; $k++) { $print_data[$k]="nd"; } while ( ) { if ( m/\|/ ) { $record = $_; unless (defined $_ ) { &sendmessage ($error, "Reading past Athena EOF in ATHENA_DATA_LOCAL", "Cannot continue"); &make_log_entry ( "Reading past Athena EOF in ATHENA_DATA_LOCAL"); return "Failed, reading past Athena EOF"; } chomp $record; $record =~ tr/\x0D//d; ($code, $value) = split /\| /, $record; unless ( defined $code and $code =~ m/\d+/ ) { next; } unless (defined $value) {$value = "nd"; } # print STDOUT ("***debug, legal_output_numbers=",@legal_output_numbers,"\n"); foreach $i (@legal_output_numbers) { if ($i == $code) { $new_line = "yes"; # print STDOUT ("***debug, inside foreach loop, i=$i, code=$code\n"); if ($i == $cdate_field_number ) { ($month, $day, $year) = split /\//, $value; unless (defined $year) { $year = "nd"; } if ( $year =~ m/[a-zA-Z]/ ) { $year = "nd"; $day = "nd"; $month = "nd"; } if ($year ne "nd" and $year >= 100 and $year <= 1000 ) { $year = $year + 1900; } if ($year ne "nd" and $year < 100 ) {$year = $year + 2000; } for ($k=0; $k <= $#print_fields; $k++) { if ($print_fields[$k] eq 'year' ) { $print_data[$k] = $year; # print STDOUT ("***debug, year: print_data[$k]=", # "$print_data[$k]\n"); } elsif ($print_fields[$k] eq 'month' ) { $print_data[$k] = $month; # print STDOUT ("***debug, month: print_data[$k]=", # "$print_data[$k]\n"); } elsif ($print_fields[$k] eq 'day' ) { $print_data[$k] = $day; # print STDOUT ("***debug, day: print_data[$k]=", # "$print_data[$k]\n"); } } } elsif ($i == $imet_field_number) { ($wind, $bar, $hum, $stuff) = split /;/, $value; if (defined $wind) { ($wind_speed, $wind_direction) = split /\s+/, $wind; } else { $wind = "nd"; $wind_speed = "nd"; $wind_direction = "nd"; } if (defined $hum ) { $hum =~ s/\s{0,}(.*)/$1/; ($humidity, $temp, $stuff) = split /\s+/, $hum; } else { $hum = "nd"; $humidity = "nd"; $temp = "nd"; } undef $stuff; if (defined $wind_speed and &valid_number($wind_speed) ) { if ( $wind_speed < $min{'wind_speed'} or $wind_speed > $max{'wind_speed'}) { &make_log_entry("Out of range, wind speed=$wind_speed"); $wind_speed = "nd"; } } else { $wind_speed = "nd"; } if (defined $wind_direction and &valid_number($wind_direction)) { if ($wind_direction < $min{'wind_direction'} or $wind_direction > $max{'wind_direction'} ) { &make_log_entry("Out of range wind direction=$wind_direction"); $wind_direction = "nd" } } else { $wind_direction = "nd"; } if (defined $bar and &valid_number($bar) ) { if ($bar < $min{'bar'} or $bar > $max{'bar'} ) { &make_log_entry("Out of range bar=$bar"); $bar = "nd"; } } else { $bar = "nd"; } if (defined $humidity ) { $humidity =~ s/\s//g; if ( &valid_number($humidity) ) { if ($humidity < $min{'humidity'} or $humidity > $max{'humidity'} ) { &make_log_entry("Out of range humidity=$humidity"); $humidity = "nd"; } } else { $humidity = "nd"; } } else { $humidity = "nd"; } if (defined $temp ) { $temp =~ s/\s//g; if ( &valid_number($temp) ) { if ($temp < $min{'temp'} or $temp > $max{'temp'}) { &make_log_entry("Out of range temp=$temp"); $temp = "nd"; } } else { $temp = "nd"; } } else { $temp = "nd"; } for ($k=0; $k <= $#print_fields; $k++) { if ($print_fields[$k] eq 'wind_speed' ) { $print_data[$k] = $wind_speed; # print STDOUT ("***debug, wind speed: print_data[$k]=", # "$print_data[$k]\n"); } elsif ($print_fields[$k] eq 'wind_direction' ) { $print_data[$k] = $wind_direction; # print STDOUT ("***debug, wind dir: print_data[$k]=", # "$print_data[$k]\n"); } elsif ($print_fields[$k] eq 'press_bar' ) { $print_data[$k] = $bar; # print STDOUT ("***debug, press bar: print_data[$k]=", # "$print_data[$k]\n"); } elsif ($print_fields[$k] eq 'humidity' ) { $print_data[$k] = $humidity; # print STDOUT ("***debug, humidity: print_data[$k]=", # "$print_data[$k]\n"); } elsif ($print_fields[$k] eq 'temp_air' ) { $print_data[$k] = $temp; # print STDOUT ("***debug, temp air: print_data[$k]=", # "$print_data[$k]\n"); } } } elsif ($athena_field_numbers_from_file[$i] =~ m/LAT/i or $athena_field_numbers_from_file[$i] =~ m/LON/i ) { for ($k=0; $k <= $#print_fields; $k++) { if ($print_fields[$k] eq $athena_field_numbers_from_file[$i] ) { $print_data[$k] = &reformat_athena_lat_lon ($value); # print STDOUT ("***debug, lat/lon: print_data[$k]=", # "$print_data[$k]\n"); last; } } } else { $value =~ s/\s//g; if ($value eq "") { $value = "nd"; }; if ($value eq "-") { $value = "nd"; }; if ($value =~ m/[a-zA-Z\(\)_]/ ) { $value = "nd"}; if ($value =~ m/:/ ) { #Assume this is a time value, check if okay $value = &validate_colon_time ($value); unless ($value =~ m/:\d/ ) { $value = "nd"}; } # print STDOUT ("***debug, athena_field_numbers_from_file[$i]=", # $athena_field_numbers_from_file[$i], "\n"); for ($k=0; $k <= $#print_fields; $k++) { if ($print_fields[$k] eq $athena_field_numbers_from_file[$i] ) { $print_data[$k] = $value; # print STDOUT ("***debug, print_data[$k]=$print_data[$k]\n"); # print STDOUT ("***debug, value=$value\n"); last; } } } } } } else { if ( $new_line eq "yes" ) { # print ATHENA_HOURLY ("\n"); #appears to be an extra return for ($k=0; $k <= $#print_fields; $k++) { print ATHENA_HOURLY ("$print_data[$k]\t"); $nws_record = $nws_record . "\t" . $print_data[$k]; # print STDOUT ("***debug, print_data[$k] ($print_fields[$k])=$print_data[$k]\n"); $print_data[$k] = "nd"; } print ATHENA_HOURLY ("\n"); shift @nws_array if $#nws_array > 4; #Save 5 most recent values push @nws_array, $nws_record; $nws_record = ""; $new_line="no"; } } } close ATHENA_DATA_LOCAL; close ATHENA_HOURLY; &output_nws_data ($print_fields, @nws_array); return "Okay, done processing Athena data"; } elsif ($record =~ m/$athena_key_phrase/ ) { $header_found = "yes"; } goto OUTERLOOP; } #--------------------------------------------------------------- sub reformat_athena_lat_lon { # Reformat the latitude or longitude value provided as $_[0] in a decimal value # with the correct sign. Return the value computed or "nd" if it is not a legal # value my ($degrees, $minutes, $sign, %sign_convention, $value); %sign_convention = ( 'N', 1, 'S', -1, 'E', 1, 'W', -1); #print STDOUT ("***debug, sign_convention{'W'} =", $sign_convention{'W'}, "\n"); #print STDOUT ("***debug, convert Athena lat/lon=$_[0]\n"); if (defined $_[0] ) { $value = $_[0]; # print STDOUT ("***debug, value=$value\n"); ($degrees, $sign) = split /,/, $value; if (defined $degrees and defined $sign ) { $degrees =~ s/ø/_/; ($degrees, $minutes) = split /_/, $degrees; # print STDOUT ("***debug, (degrees, minutes)=($degrees, $minutes)\n"); $sign = uc $sign; # print STDOUT ("***debug, sign=$sign\n"); # print STDOUT ("***debug, sign_convention{$sign}=", $sign_convention{$sign}, "\n"); if ( ! defined $degrees or ! defined $minutes or $degrees !~ m/\d+/ or $minutes !~ m/\d+\.\d+/ ) { $value = "nd"; } elsif ( defined $sign and defined $sign_convention{$sign} ) { $value = $sign_convention{$sign} * ($degrees + ($minutes/60) ); $value = sprintf "%9.4f", $value; $value =~ s/ //g; } else { $value = "nd"; } } else { $value = "nd"; } } else { $value = "nd"; } #Check for valid range. unless ($value eq "nd" ) { if ( $sign =~ m/[NS] / ) { if ($value < $min{'lat'} or $value > $max{'lat'} ) { $value = "nd"; } elsif ($value < $min{'lon'} or $value > $max{'lon'} ) { $value = "nd"; } } } #print STDOUT ("***debug, convert Athena lat/lon value=$value\n"); return $value; } #--------------------------------------------------------------- sub translate_vessel_name { # Translate the vessel name given in $_[0] and return the translated name # If no translation exists, return the "nd" translated name. # Assumptions: # 1. The hash %vessel_name_file_entries already is populated with the # key being the vessel name and the value being the translated name. my ($key, $vessel_name); #print STDOUT ("***debug, translate_vessel_name ,vessel_name=$_[0]\n"); if (defined $_[0] ) { $vessel_name = lc $_[0]; $vessel_name =~ s/&/and/g; $vessel_name =~ s/ /_/g; $vessel_name =~ s/^F\/V_//i; $vessel_name =~ s/^R\/V_//i; if (exists $vessel_name_file_entries{$vessel_name} and defined $vessel_name_file_entries{$vessel_name} ) { return $vessel_name_file_entries{$vessel_name}; } } else { return $vessel_name_file_entries{"nd"}; } #Try to pattern match to see if we can find something close # to it. foreach $key (keys %vessel_name_file_entries) { if ($key =~ m/$vessel_name/i ) { return $vessel_name_file_entries{$key}; } if ($vessel_name =~ m/$key/i ) { return $vessel_name_file_entries{$key}; } } &sendmessage ($warning, "Could not find vessel name $vessel_name", "Using nd value"); &make_log_entry ( "Could not find vessel name=$vessel_name"); return $vessel_name_file_entries{"nd"}; } #------------------------------------------------------------------- sub email_comments_prn { #email comments if any comments exists and if the Email "command" string # is included in the comment line. In this case, the email is sent to the person # specified in the command string line after the Email "command" string. my ( $comment_counter, $comment, $email_command_string, $flag, $message, $sendee); $message = ""; if (exists $config_param{'email_command_string'} and defined $config_param{'email_command_string'} ) { $email_command_string = $config_param{'email_command_string'}; } else { &sendmessage ($warning, "Cannot process comment lines for email", "Email command string not defined"); &make_log_entry("Cannot process comment lines for email", "Email command string not defined"); return "Warning, email command strind not defined"; } $flag = "no"; if (exists $data{'comments'} and defined $data{'comments'} ) { $comment = $data{'comments'}; } else { &make_log_entry ("No comments to e-mail"); return "Okay, no comments to e-mail"; } if ($comment =~ m/$email_command_string/ ) { $message = $comment . "\n"; $flag = "yes"; } $comments_counter = 0; while (defined $data{'comments_comments_counter'} ) { $comment = $data{'comments_comments_counter'}; $comments_counter++; if ( ($comment =~ m/$email_command_string/i) or ($flag eq "yes") ) { $flag = "yes"; $message = $message . $comment . "\n"; } } if ($flag eq "yes" ) { $sendee = $message; $sendee =~ s/.*$email_command_string\s+(.*)\@(.*)\s+.*/$1\@$2/i; &mailmessage($sendee, $message) ; } make_log_entry("Comments mailed=$flag"); return "Comments mailed-$flag"; } #----------------------------------------------------------------- sub mailmessage { # Send an email message to $_[0]. For now, also send message to rgroman. # $_[0] - sendee (address to whom message is to be sent) # $_[1] - message (multiple lines of text) my ( $mailfile, $message, $who); my ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst); $sendee = $_[0]; $sendee = 'rgroman@whoi.edu,' . $sendee; $message = $_[1]; ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst)=localtime(time); if ( $year < 1000 ) {$year = $year + 1900; } $mon++; if ($mon < 10) { $mon = "0" . $mon; } if ($mday < 10) {$mday = "0" . $mday; } if ($hour < 10) { $hour = "0" . $hour; } if ($min < 10) { $min = "0" . $min; } if ($sec < 10) { $sec = "0" . $sec; } $mailfile=">/tmp/sendmess" . $year . $yday . $hour . $min . $sec . ".tmp"; if ( open TEMPFILE, $mailfile) { if ( exists $ENV{'REMOTE_HOST'} ) {$who=$ENV{'REMOTE_HOST'} ; } elsif (exists $ENV{'REMOTE_ADDR'} ) {$who=$ENV{'REMOTE_ADDR'} ; } else {$who="not available"; } print TEMPFILE ("Message from $0"); print TEMPFILE (" Date of message: $year/$mon/$mday $hour:$min\n"); print TEMPFILE (" From: $data{'vessel_name'}\n"); print TEMPFILE (" From host: $who\n"); unless (exists $data{'vessel_name'} and defined $data{'vessel_name'} ) { if (exists $data{'vessel'} and defined $data{'vessel'} ) { $data{'vessel_name'} = $data{'vessel'}; } } if (exists $data{'vessel_name'} and defined $data{'vessel_name'} ) { print TEMPFILE (" Subject: Message from $data{'vessel_name'}\n"); } print TEMPFILE (" Message text:\n$message\n"); close TEMPFILE; $sendee =~ s/\@/\\\@/; `/bin/mail -s \"FleetLink message\" $sendee <$mailfile`; unlink $mailfile; } return; } #------------------------------------------------------------------ sub output_comments_prn { # Output comments if they exist to appropriate comments data file. my ( $comment_counter, $comment, $comments_vessel_file, @comments_vessel_field_names, $comments_date_file, @comments_date_field_names, $comments_file, @comments_field_names, @date, $message, $record, $record_exists, $record_to_add, $vessel, $vessel_code); $message = ""; if (exists $data{'comments'} and defined $data{'comments'} ) { $comment = $data{'comments'}; } else { &make_log_entry("No comments to output"); return "Okay, no comments to output"; } $vessel = &determine_vessel_name; if ($vessel eq "nd") { &make_log_entry ("No vessel name defined for comment output", "Will use nd as name"); } $vessel_code = &getvesselcode ($vessel); $comments_vessel_file = $config_param{'comments_vessel_file'}; @comments_vessel_field_names = split /,/, $config_param{'comments_vessel_field_names'}; $comments_date_file = $config_param{'comments_date_file'} . $vessel_code . ".dat"; @comments_date_field_names = split /,/, $config_param{'comments_date_field_names'}; if (defined $data{"time_date_of_departure"} ) { @date = split / /, $data{"time_date_of_departure"}; unless (defined $date[4] and defined $date[1] and defined $date[2]) { @date = split /\s+/, $date; if ( $date[2] < 9 ) { $date[2] = "0" . $date[2]; } } } else { # print STDOUT ("***debug, date string=$date\n"); @date = split /\s+/, $date; if ( $date[2] < 9 ) { $date[2] = "0" . $date[2]; } } $comments_file = $config_param{'comments_file'} . $vessel_code . "_" . $date[4] . $date[1] . $date[2] . ".dat"; @comments_field_names = split /,/, $config_param{'comments_field_names'}; #Add to comments vessel file. Create if necessary. unless (-e $comments_vessel_file) { unless (open COMMENTS_VESSEL, ">$comments_vessel_file") { $err = $! &sendmessage ($warning, "Could not open new comments vessel file $comments_vessel_file", "Error code=$err. Cannot process data."); &make_log_entry ( "Could not open new comments vessel file=$comments_vessel_file", "Error=$err"); return "Failed, could not open new comments vessel file"; } print COMMENTS_VESSEL ("#Comments vessel file created ", $date, "\n"); foreach (@comments_vessel_field_names) { s/\s//g; print COMMENTS_VESSEL ($_,"\t"); } print COMMENTS_VESSEL (">\n"); foreach (@comments_date_field_names) { s/\s//g; print COMMENTS_VESSEL ($_,"\t"); } print COMMENTS_VESSEL (">\n"); foreach (@comments_field_names) { s/\s//g; print COMMENTS_VESSEL ($_,"\t"); } print COMMENTS_VESSEL ("\n"); close COMMENTS_VESSEL; } # Read comments vessel file and see if record exists. If not, append the new one. unless (open COMMENTS_VESSEL, "$comments_vessel_file") { $err = $! &sendmessage ($warning, "Could not open comments vessel file for reading, $comments_vessel_file", "Error code=$err. Cannot process data."); &make_log_entry ( "Could not open for reading, comments vessel file=$comments_vessel_file", "Error=$err"); return "Failed, could not open comments vessel file for reading"; } $record_exists = "no"; while () { if ( m/$vessel/i ) { &make_log_entry ( "Comments vessel file already contains vessel=$vessel"); $record_exists = "yes"; last; } } close COMMENTS_VESSEL; # Add new comment vessel record if necessary if ($record_exists eq "no" ) { unless (open COMMENTS_VESSEL, ">>$comments_vessel_file" ) { $err=$!; &sendmessage ($warning, "Could not open comments vessel file $comments_vessel_file", "for appending, for vessel=$vessel. Error=$err"); &make_log_entry( "Could not open for appending comments vessel file=$comments_vessel_file", "For vessel=$vessel", "Error=$err"); return "Failed, could not open comments vessel file for appending"; } $record_to_add = $vessel . "\t" . $comments_date_file . "\n"; print COMMENTS_VESSEL ( $record_to_add); close COMMENTS_VESSEL; } # Now update, if necessary, the comments date file. Create if necessary unless (-e $comments_date_file) { unless (open COMMENTS_DATE, ">$comments_date_file") { $err = $! &sendmessage ($warning, "Could not open new comments date file $comments_date_file", "Error code=$err. Cannot process data."); &make_log_entry ( "Could not open new comments date file=$comments_date_file", "Error=$err"); return "Failed, could not open new comments date file"; } print COMMENTS_DATE ("#Comments date file created ", $date, "\n"); foreach (@comments_date_field_names) { s/\s//g; print COMMENTS_DATE ($_,"\t"); } print COMMENTS_DATE (">\n"); foreach (@comments_field_names) { s/\s//g; print COMMENTS_DATE ($_,"\t"); } print COMMENTS_DATE ("\n"); close COMMENTS_DATE; } # Read comments date file and see if record exists. If not, append the new one. unless (open COMMENTS_DATE, "$comments_date_file") { $err = $! &sendmessage ($warning, "Could not open comments date file for reading, $comments_date_file", "Error code=$err. Cannot process data."); &make_log_entry ( "Could not open for reading, comments date file=$comments_date_file", "Error=$err"); return "Failed, could not open comments date file for reading"; } $record_to_add = $date[4] . "\t" . $date[1] . "\t" . $date[2] . "\t" . $comments_file . "\n"; $record_exists = "no"; while () { if ( m/$record_to_add/i ) { &make_log_entry ( "Comments date file already contains date=$date[4]/$date[1]/$date[2]"); $record_exists = "yes"; last; } } close COMMENTS_DATE; # Add new comment date record if necessary if ($record_exists eq "no" ) { unless (open COMMENTS_DATE, ">>$comments_date_file" ) { $err=$!; &sendmessage ($warning, "Could not open comments date file $comments_vessel_file", "for appending, for vessel=$vessel. Error=$err"); &make_log_entry( "Could not open for appending comments date file=$comments_date_file", "For vessel=$vessel", "Error=$err"); return "Failed, could not open comments date file for appending"; } print COMMENTS_DATE ($record_to_add); close COMMENTS_DATE; } #Add to comments file. Create if necessary. unless (-e $comments_file) { unless (open COMMENTS, ">$comments_file") { $err = $! &sendmessage ($warning, "Could not open new comments file $comments_file", "Error code=$err. Cannot process data."); &make_log_entry ( "Could not open new comments file=$comments_file", "Error=$err"); return "Failed, could not open new comments file"; } print COMMENTS ("#Comments file created ", $date, "\n"); foreach (@comments_field_names) { s/\s//g; print COMMENTS ($_,"\t"); } print COMMENTS ("\n"); close COMMENTS; } unless (open COMMENTS, ">>$comments_file" ) { $err=$!; &sendmessage ($warning, "Could not open comments data file $comments_file", "for appending, for vessel=$vessel. Error=$err"); &make_log_entry( "Could not open for appending comments data file=$comments_file", "For vessel=$vessel", "Error=$err"); return "Failed, could not open comments date file for appending"; } #output comments print COMMENTS ("\'$comment\'\n"); $comments_counter = 0; while (defined $data{"comments_$comments_counter"}) { $comment = $data{"comments_$comments_counter"}; $comments_counter++; print COMMENTS ("\'$comment\'\n") unless $comment eq "nd"; } close COMMENTS; make_log_entry("Comments output to file=$comments_file", "For vessel=$vessel"); return "Okay, comments output to file $comments_file"; } #------------------------------------------------------------- sub validate_colon_time { # Validate the time passed as $_[0]. Assumes it should look like xx:yy:zz as # hours:minutes:seconds. Returned value is either what was passed in, # if valid, or "nd". my ($hours, $minutes, $seconds, $value); if (defined $_[0]) { $value = $_[0]; } else { return "nd"; } unless ($value =~ m/\d\d:\d\d:\d\d/ ) { return "nd"; } ($hours, $minutes, $seconds) = split /:/, $value; if ($hours < 0 or $hours >23 or $minutes < 0 or $minutes > 59 or $seconds < 0 or $ seconds > 60) { return "nd"; } return $value; } #---------------------------------------------------------- sub determine_vessel_name { #Checks the $data{'vessel_name'} and $data{'vessel'} variables for vessel name, #and uses the translate_vessel_name routine to obtain the "real" vessel name. #On return, the vessel name, if known, is returned. If not known, the value returned #is "nd". my ($vessel); unless (exists $data{'vessel_name'} and defined $data{'vessel_name'} ) { if ( exists $data{'vessel'} and defined $data{'vessel'} ) { $data{'vessel_name'} = $data{'vessel'}; } else { return "nd"; } } $vessel = &translate_vessel_name ( $data{'vessel_name'} ); return $vessel; } #------------------------------------------------------------------- sub valid_number { # Check $_[0] as a valid number. It could have a leading blank, + or - or # start with digits, followed by 0 or 1 decimal points, followed by 0 or more # digits, followed by e or E followed by an optional sign followed by 0 or # more digits. Add check for at least one digit. if ($_[0] =~ m/[ +-]{0,1}\d{0,}\.{0,1}\d{0,}E{0,1}[+-]{0,1}\d{0,}/i and $_[0] =~ m/\d+/ and $_[0] !~ m/[abcdfghijklmnopqrstuvwxyz_,]/i) { return 1; } &make_log_entry("Bad numerical value found with value =$_[0]"); return 0; } #------------------------------------------------------------------- sub output_nws_data { # Generates (or updates) the National Weather Service output file specified by # $config_param{'NWS_output_file'} when defined. If not defined, log message # is written, but not considered an error. # Passed parameters: # $_[0] -$print_fields tab delimited list of Athena hourly data field names # $_[1], $_[2], ... - tab delimited list of Athena data records. # Routine outputs the latest entry in the list of data records with the least number # of "nd"'s. That is the one with the most useful data, hopefully. # Originally used single file for data from all vessels but can't do this since field # names may vary too much. Change to separate file for each vessel. Also change # archiving scheme. my ($err, $file, $found, $i, $least_index, $least_value, $max_number_of_records, $nd_counter, @nws_data, $print_fields, $top_level_file, $vessel, $vessel_code); $print_fields = shift @_; @nws_data = @_; $max_number_of_records = 9999; $vessel = &determine_vessel_name; $vessel_code = &getvesselcode ($vessel); unless (defined $config_param{'NWS_output_root'} ) { &make_log_entry("NWS output root not defined in config file for vessel=$vessel", "For vessel code=$vessel_code", "NWS update made=no"); return; } $file = $config_param{'NWS_output_root'} . "/" . $vessel_code . "_" . $config_param{'NWS_output_filename'} . "." . $config_param{'NWS_output_extension'}; $top_level_file = $config_param{'NWS_output_vessel_name_file'}; # Check if vessel code is in top level file. If not, add it. unless (-e $top_level_file) { unless (open NWS, ">$top_level_file") { $err = $! &sendmessage ($error, "Could not open new top level NWS data file $top_level_file", "for vessel code=$vessel_code. Error=$err"); &make_log_entry( "Could not open new NWS data file=$top_level_file", "For vessel=$vessel", "Error=$err"); return; } # Include vessel name in next file too so NWS can more easily know where data # comes from. print NWS ("#NWS top level file created $date\n", "vessel_code\t>\n", "vessel\n", $vessel_code, "\t", $file, "\n"); close NWS; &make_log_entry("Created new top level NWS data file=$top_level_file", "For vessel=$vessel", "For vessel code=$vessel_code"); } else { # Open top level file for append and add new record if necessary. # Assumes same file naming convention unless (open NWS, "$top_level_file") { $err = $! &sendmessage ($error, "Could not open existing top level NWS data file $top_level_file", "for vessel code=$vessel_code. Error=$err"); &make_log_entry( "Could not open existing NWS data file=$top_level_file", "For vessel=$vessel", "For vessel code=$vessel_code", "Error=$err"); return; } $found = "no"; while () { if ( m/^#/ ) {next} if ( m/$vessel_code/ ) { $found = "yes"; last; } } close NWS; if ($found eq "no") { # Append latest vessel record unless (open NWS, ">>$top_level_file") { $err = $! &sendmessage ($error, "Could not append to top level NWS data file $top_level_file", "for vessel code=$vessel_code. Error=$err"); &make_log_entry( "Could not append to NWS data file=$top_level_file", "For vessel=$vessel", "For vessel_code=$vessel_code", "Error=$err"); return; } print NWS ($vessel_code, "\t", $file, "\n"); close NWS; &make_log_entry("Add to top level NWS data file=$top_level_file", "Vessel=$vessel", "Vessel code=$vessel_code"); } } #Create new NWS vessel data file along with comment line and field names # but archive old one first, if necessary if (-e $file) { &archive_nws_data ($file); } unless (open NWS_VESSEL, ">$file") { $err = $! &sendmessage ($error, "Could not open new NWS vessel data file $file", "for vessel code=$vessel_code. Error=$err"); &make_log_entry( "Could not open new NWS vessel data file=$file", "For vessel=$vessel", "For vessel code=$vessel_code", "Error=$err"); return; } #Include vessel name (as field name "vessel" in this file too so NWS can more easily # know what vessel the data comes from. print NWS_VESSEL ("#NWS vessel file created $date\n", "vessel\t", $print_fields, "\n"); #Determine best record to output $least_index = $max_number_of_records; $least_value = $max_number_of_records ; if ($max_number_of_records == $#nws_data) { &sendmessage ($warning, "Number of records passed for NWS data=$#nws_data", "Maximum number of records expected=$max_number_of_record"); &make_log_entry( "Number of records passed for NWS data=$#nws_data", "Maximum number of records expected=$max_number_of_record", "Could not update for vessel=$vessel", "For vessel code=$vessel_code"); close NWS_VESSEL; return; } for ($i=0; $i<=$#nws_data; $i++) { $nd_counter = $nws_data[$i] =~ tr/nd/nd/; if ($nd_counter <= $least_value) { #want the latest values $least_index = $i; $least_value = $nd_counter; } } if ($least_index < $max_number_of_records) { print NWS_VESSEL ($vessel_code, "\t", $nws_data[$least_index], "\n"); &make_log_entry("NWS vessel data updated in file=$file", "For vessel=$vessel", "For vessel code=$vessel_code"); } else { &make_log_entry("NWS vessel data not updated in file=$file", "Maximum number of records exceeded=$max_number_of_records"); } close NWS_VESSEL; } #----------------------------------------------------------- sub archive_nws_data { # Archive the old information in the NWS vessel file given as $_[1]. # File is renamed to have next higher number available. my ($counter, $err, $new_location, $number); my $full_file_name = $_[0]; $counter = 0; $new_location = $full_file_name; $new_location =~ s/.*\/(.*)/$1/; $new_location = $config_param{'NWS_output_archive_directory'} . "/" . $new_location; UNIQUE_NAME: if (-e $new_location ) { # &sendmessage ($warning, "Tried to archive to $new_location", # "but file already exists. Will come up with new name."); $new_location = $full_file_name; $new_location =~ s/.*\/(.*)/$1/; $new_location = $config_param{'NWS_output_archive_directory'} . "/" . $new_location; $new_location = $new_location . "_" . $counter; $counter++; goto UNIQUE_NAME; } unless (open IN, $full_file_name) { $err = $!; &sendmessage ($warning, "Could not open input file $full_file_name for archiving.", "Error=$err File not archived."); &make_log_entry ("Could not open input archiving file=$full_file_name", "Error=$err"); return; } unless (open OUT, ">$new_location") { $err = $!; &sendmessage ($warning, "Could not open output file $new_location for archiving.", "Error=$err File not archived."); &make_log_entry ("Could not open output archiving file=$new_location", "Error=$err"); return; } while () { print OUT ($_); } &make_log_entry ("Output archive NWS vessel file created=$new_location"); close IN; close OUT; $number = unlink $full_file_name; if ( $number < 1 ) { &sendmessage ($warning, "Could not remove original NWS vessel data file $full_file_name", "Check if archive file at $new_location okay."); &make_log_entry ("Could not remove original NWS vessel data file=$full_file_name", "Check if archive file okay at=$new_location"); } else { &make_log_entry ("Input file=$full_file_name\tMoved to=$new_location"); } return; } #--------------------------------------------------------------------- sub load_platform_hash { # Input data into the platform hash table, %vessel_name_file_entries my ($code, $err, $platform); unless (open VESSELCODE, $config_param{'vessel_code_file'} ) { $err = $!; &sendmessage ($warning, "Could not open platform/vessel code file Error=$err", "File name is $config_param{'vessel_code_file'}"); &make_log_entry("Could not open platform/vessel code file=$config_param{'vessel_code_file'}", "Error=$err"); return "Error, Could not open platform/vessel code file, error=$err"; } while () { if ( m/^#/ ) { next; } # print STDOUT ("#**debug, vessel record=$_\n"); chomp; s/%/_/g ; unless (m/\t/) {next; } ($platform, $code) = split /\t/; $platform = lc $platform; $platform =~ s/ /_/g; unless (defined $code) { $code = "A10002"; } $vessel_name_file_entries{$platform} = $code; } close VESSELCODE; return "okay"; } #----------------------------------------------------------------------