#!/usr/bin/perl -w

# Make metadata from Aquarius house-keeping telemetry data to allow the rapid
# determination of alarm status of a particular telemetry parameter for an
# arbitrary time period in the mission.

# Norman Kuring		26-Feb-2010	Original development
# Norman Kuring		22-Mar-2010	Original development
# Norman Kuring		14-Apr-2010	I forgot to post_process the final days.
# Norman Kuring		 1-Jul-2010	Reorganization for new telemetry
#					grouping.
# Norman Kuring		15-Jul-2010	Generate monthly metadata.
# Norman Kuring		 5-Aug-2010	Consolidate metadata into a single file
#					per year.
# Norman Kuring		18-Nov-2010	Make this work with the simulated
#					telemetry data naming conventions.
# Norman Kuring		28-Dec-2010	Rather than generating all of the
#					metadata every time, just update the
#					existing metadata with the latest
#					new telemetry alarms.
# Norman Kuring		 3-Jan-2011	Unpack alarm values with template "c"
#					instead of with template "a".
# Norman Kuring		12-Jan-2011	Fix 1-off error in day-of-month index.
# Norman Kuring		12-Jan-2011	Change one "unpack 'C'" to "unpack 'c'"
#					to maintain consistency in the code.
# Norman Kuring		13-Jan-2011	Exit program if no new telemetry data
#					are available.
# Norman Kuring		19-Apr-2011	Make this work for service platform
#					telemetry files too.
# Norman Kuring		21-Apr-2011	Allow comments in parameter index files.
# Norman Kuring		 6-Jun-2011	Change to naming convention.  Might
#					be more coming....
# Norman Kuring		15-Jul-2011	Apply leap-second correction to GPS
#					times and filter out bogus time stamps.
# Norman Kuring		 8-Aug-2011	Include system loads in diagnostic
#					output.
# Norman Kuring		16-Aug-2011	Change logic to reflect new reality
#					of telemetry files from CONAE that can
#					be up to 3 days long.
# Norman Kuring		30-Sep-2011	Use read_dir and isleap from Utility
#					module and get_param_specs from
#					ReadTelemMeta module.
# Norman Kuring		 4-Oct-2011	Change S/P file-matching regular
#					expression to also include daily
#					merged GSFC files.
# Norman Kuring		12-Oct-2011	Limit values of zero were being mis-
#					interpreted as no limit because a value
#					of zero is interpreted as "false".
#					This has been fixed in the alarm_state
#					subroutine.
# Norman Kuring		 3-Jan-2013	Fix logic error that discarded fine-
#					scale threshold-exceedance information
#					whenever all the maximum alarm levels
#					in a longer time aggregate were equal
#					to each other.
# Norman Kuring		 3-Jan-2013	I am also changing the name of this
#					script from aquarius_hkt_meta.sim.pl
#					to make_telem_exceedance_meta.pl .

use strict;
use Time::Local;
use File::Path;
use lib '/web/aquarius/scripts';
use GpsToUtc qw(&read_leap_secs &gps2utc);
use Utility qw(read_dir isleap);
use ReadTelemMeta qw(get_param_specs);

my $mission_start = 991_754_785;	# 10 June 2011 15:26:10 UTC
my $now = $^T - timegm(0,0,0,6,0,80);	# I'm ignoring leap seconds when I set
					# the current time to the GPS epoch.

# The following variables will control which set of telemetry values
# are read and output in summary during different iterations of the main
# processing loop.
my @outtyp = qw(aq sp);
my @datadir = qw(
/web/data/Aquarius/DOWNLINKS/L1
/web/data/Aquarius/HKT
);
my @fpat = qw(
^Q\d{8}_\d{6}_T\d{8}_\d{6}_\d{8}_\d{6}\.AQ_\w+\.txt$
^(?:CGSS|SDMOC|GSFC)_\d{8}(?:_\d{6}_\d{17})?_SACD_HKTM[SR][TG]\.bin\.SP_T\d{8}_\d{6}_\d{8}_\d{6}\.SP_\w+\.txt$
);
my @gpat = qw(
^Q\d{8}_\d{6}_T\d{8}_\d{6}_\d{8}_\d{6}\.AQ_(\w+)\.txt$
^(?:CGSS|SDMOC|GSFC)_\d{8}(?:_\d{6}_\d{17})?_SACD_HKTM[SR][TG]\.bin\.SP_T\d{8}_\d{6}_\d{8}_\d{6}\.SP_(\w+)\.txt$
);

my $metadir = '/web/aquarius/metadata/alarms';

# Initialize the GPS to UTC conversion function.
read_leap_secs;

for(my $typ=0; $typ<@outtyp; $typ++){

  # Get parameter information as a hash of arrays.
  #                  [    0   ,     1      ,  2  , 3, 4, 5, 6,    7      ]
  # $pindex{PARAM_ID}[GROUP_ID,GROUP_COLUMN,UNITS,RL,YL,YH,RH,DESCRIPTION]
  my %pindex = get_param_specs($outtyp[$typ]);

  my @params = sort keys %pindex;
  my %params;
  my %ctg;
  for(my $i=0; $i<@params; $i++){
    $params{$params[$i]} = $i;
    push @{ $ctg{ $pindex{ $params[$i] }[0] } }, $params[$i];
  }
  foreach my $g (keys %ctg){
    # Put the parameters in each category in the same order as they
    # appear in the data files for that category.
    @{ $ctg{$g} } = sort { $pindex{$a}[1] <=> $pindex{$b}[1] } @{ $ctg{$g} };
  }

  my %alarmbuf;
  my %alarminfo;
  my %meta;
  my %modtime;

  my @years = read_dir($datadir[$typ],'^\d{4}$');
  foreach my $year (@years){
    print STDERR "[",scalar localtime,
                 "] Reading $outtyp[$typ] metadata for $year.\n";
    ($meta{$year},$modtime{$year}) = read_meta($year,$outtyp[$typ]);
  }
  foreach my $year (@years){
    my @mmdd = read_dir("$datadir[$typ]/$year",'^\d{4}$');
    foreach my $mmdd (@mmdd){
      my $modt = $^T - 86400*(-M "$datadir[$typ]/$year/$mmdd");

      # Skip days that we've already read.  This test compares the
      # modification time of the previously written version of the
      # metadata produced by this script with the modification times
      # of the directories containing new telemetry data.  The assumption
      # is that the Data System will change the modification times of
      # the data directories whenever new data are added and that this
      # script will itself not take too long to run. (I currently allow
      # a grace period of two hours [7200 seconds].)
      next if $modtime{$year} - $modt > 7200;

      my @files = read_dir("$datadir[$typ]/$year/$mmdd",$fpat[$typ]);
      @files or next;

      my $day = mmdd2doy($year,$mmdd);
      print STDERR "[",scalar localtime,"] $year $day ($mmdd)\n";
      my $nominal_day = timegm(0,0,0,1,0,$year-1900)/86400 + $day - 1;

      # Because we get files from CONAE that contain up to three days' worth
      # of telemetry, and because outputs from Liang's analysis tool are
      # placed in directories according to the earliest data they contain,
      # I need to prepare alarm buffers for four full days starting with
      # the day indicated by the current directory name.
      for(my $d=$nominal_day; $d<=$nominal_day+3; $d++){
        unless(defined $alarmbuf{$d}){
          # Each alarm buffer holds alarm statuses for all of the
          # telemetry parameters for every second of a day.  There
          # are four 2-bit statuses per byte in the buffer.  The buffer
          # initially gets filled with zeros which signify missing data.
          $alarmbuf{$d} = chr(0) x (86400*@params/4);

          # If I'm at the beginning or end of a year, then the current
          # day's year may be different from $year, so I get the year
          # from the day.
          my $dyr = (gmtime 86400*$d)[5] + 1900;

          if($meta{$dyr}){
            # If there are metadata for this year
            # then convert those metadata into a bitmap of sorts;
            # otherwise, just keep the default, zeroed-out alarmbuf
            # created above.
            $alarmbuf{$d} = daymeta2buf($d,$meta{$dyr});
          }
        }
      }

      foreach my $d (keys %alarmbuf){
        # Post process all of the alarm buffers that are earlier than
        # the current nominal day, and then delete each
        # post-processed buffer to free up memory.
        if($d < $nominal_day){
          $alarminfo{$d} = post_process(\@params,$alarmbuf{$d});
          delete $alarmbuf{$d};
        }
      }

      foreach my $f (@files){
        my $file = "$datadir[$typ]/$year/$mmdd/$f";
        my ($ctg) = $f =~ /$gpat[$typ]/;
        defined $ctg{$ctg} or next;
        my $params_in_ctg = @{ $ctg{$ctg} };
        open F,$file or die "Could not open $file ($!)";
        my $recno = 0;
        while(<F>){
          $recno++;
          my ($ts,@v) = split ' ';

          # Filter out data with bogus time stamps.
          next if $ts < $mission_start || $ts > $now;

          my $nv = @v;

          $nv == $params_in_ctg or die <<"          --";
          Expected $params_in_ctg telemetry values but found $nv
          on record $recno of $file.
          --

          my $t_sec  = int(gps2utc($ts) + 0.5);
          my $t_day  = int($t_sec/86400);

          my $si = $t_sec % 86400;		# second index

          for(my $i=0; $i<$params_in_ctg; $i++){
            my $prm = $ctg{$ctg}[$i];
            my $val = $v[$i];
            my $pi = $params{$prm};		# parameter index
            my $index = 86400*$pi + $si;	# index into alarm array

            # I'm not checking for any previously set alarm value here
            # because I assume that there should not be more than one
            # alarm value per parameter per second; the last value read
            # in is the one that gets used.  (This could be considered
            # a feature.)
            vec($alarmbuf{$t_day},$index,2)
            = alarm_state($val,@{$pindex{$prm}}[3..6]);
          }
        } # end while(<F>)
        close F;
      } # end foreach my $f (@files)
    } # end foreach my $mmdd (@mmdd)
  } # end foreach my $year (@years)
  unless(keys %alarmbuf){
    warn "No new $outtyp[$typ] data available under $datadir[$typ].\n";
    next;
  }
  foreach my $d (keys %alarmbuf){
    # Post process the remaining alarm buffers, and then delete each
    # post-processed buffer to free up memory.
    $alarminfo{$d} = post_process(\@params,$alarmbuf{$d});
    delete $alarmbuf{$d};
  }

  # Gather the daily alarm states according to month and year.
  my ($sd,$ed) = (sort {$a <=> $b} keys %alarminfo)[0,-1];
  my ($sy,$ey) = ( (gmtime 86400*$sd)[5], (gmtime 86400*$ed)[5] );
  my %yearinfo;
  my (@y,%y);
  for(my $y=$sy;$y<=$ey;$y++){	# For each year...

    my $year = $y + 1900;	# Convert gmtime year to more intuitive value.

    my @mdays = (31,28,31,30,31,30,31,31,30,31,30,31);
    isleap($year) and $mdays[1]++;

    for(my $p=0; $p<@params; $p++){	# For each parameter...
      my (@m,%m);
      for(my $m=0; $m<12; $m++){	# For each month of the year...
        my (@d,%d);
        for(my $date=0; $date<$mdays[$m]; $date++){ # For each day of the month
          my $d = timegm(0,0,0,$date + 1,$m,$y)/86400;

          if( defined $alarminfo{$d}[$p] ){
            # There are new metadata for this day.

            push @d,$alarminfo{$d}[$p];
            $d{ $alarminfo{$d}[$p][0] }++;

          }
          elsif( $meta{$year} ){
            # There are previously created metadata for this year.

            if( $meta{$year}[$p][1] ){
              # There's a previous monthly breakdown for this parameter.

              if( $meta{$year}[$p][1][$m][1] ){
                # There's a previous daily breakdown for this month.

                push @d,$meta{$year}[$p][1][$m][1][$date];
                $d{ $meta{$year}[$p][1][$m][1][$date][0] }++;
              }
              else{
                # Store the previous monthly metadata for the day.

                push @d,$meta{$year}[$p][1][$m];
                $d{ $meta{$year}[$p][1][$m][0] }++;
              }
            }
            else{
              # Store the previous yearly metadata for the day.

              push @d,$meta{$year}[$p];
              $d{ $meta{$year}[$p][0] }++;
            }
          }
          else{
            push @d, [0,undef];
            $d{ 0 }++;
          }
        } # end for(my $date=0; $date<$mdays[$m]; $date++)
        my @states = sort {$b <=> $a} keys %d;
        my $components;
        if(@states > 1 or grep defined $_->[1],@d){
          # If there was more than one alarm value during this month
          # for this parameter or if any of the component days have
          # sub-components, then keep the component daily alarm
          # breakdown; otherwise, leave the components undefined.
          $components = \@d;
        }
        my $alarm_info = [$states[0],$components];
        push @m,$alarm_info;
        # Put the highest alarm status of the month into the month hash.
        $m{$states[0]}++;
      } # end for(my $m=0; $m<12; $m++)
      my @states = sort {$b <=> $a} keys %m;
      my $components;
      if(@states > 1 or grep defined $_->[1],@m){
        # If there was more than one alarm value during this year
        # for this parameter or if any of the component months have
        # sub-components, then keep the component monthly alarm
        # breakdown; otherwise, leave the components undefined.
        $components = \@m;
      }
      $yearinfo{$year}[$p] = [$states[0],$components];
    } # end for(my $p=0; $p<@params; $p++)
    write_alarm_info($year,$yearinfo{$year},scalar(@params),$outtyp[$typ]);
  } # end for(my $y=$sy;$y<=$ey;$y++)
  my $uptime = `/usr/bin/uptime`;
  $uptime =~ s/^\s*\d+:\d+:\d+\s*//;
  my $hostname = `/bin/hostname`;
  chop $hostname;
  print STDERR "[",scalar localtime,"] $hostname $uptime";
  print STDERR "[",scalar localtime,"] Done.\n";
} # end for(my $typ=0; $typ<@outtyp; $typ++)


sub mmdd2doy{
  my ($y,$mmdd) = @_;
  my ($m,$d) = unpack "a2a2",$mmdd;
  sprintf "%03d", (gmtime timegm(0,0,0,$d,$m-1,$y-1900))[7] + 1;
}

sub doy2mmdd{
  my ($y,$doy) = @_;
  my ($d,$m) = (gmtime timegm(0,0,0,1,0,$y-1900) + 86400*($doy - 1))[3,4];
  sprintf "%02d%02d",$m+1,$d;
}

sub post_process{
  my ($params,$buf) = @_;
  my @alarminfo;
  for(my $p=0; $p<@$params; $p++){	# For each parameter...
    my (@h,%h);
    for(my $h=0; $h<24; $h++){		# For each hour of the day...
      my (@m,%m);
      for(my $m=0; $m<60; $m++){	# For each minute of that hour...
        my (@s,%s);
        for(my $s=0; $s<60; $s++){	# For each second of that minute...

          # Retrieve the alarm state for this second from the buffer.
          my $index = 86400*$p + 3600*$h + 60*$m + $s;
          my $alarm = vec $buf,$index,2;

          # Store the alarm state.
          push @s,$alarm;
          $s{$alarm}++;
        }
        # Put the alarm states for this minute into descending order.
        # The possible alarm states are:
        #   0 - missing telemetry value   (gray)
        #   1 - nominal telemetry value   (green)
        #   2 - yellow limit reached      (yellow)
        #   3 -    red limit reached      (red)
        my @states = sort {$b <=> $a} keys %s;

        my $components;
        if(@states > 1 and $states[1] > 0){
          # Remember the components if more than one non-missing
          # alarm state was found during this minute.
          #
          # Because many of the telemetry parameters are not reported
          # for every second, I am not going to let alarm values of 0
          # (i.e. "missing") count when deciding whether or not to keep
          # track of all of the alarm values within this minute.
          $components = \@s;
        }

        # Package up the highest alarm state of this minute with
        # a pointer to the component second states if necessary,
        # and put the information into the minute array.
        my $alarm_info = [$states[0],$components];
        push @m,$alarm_info;
        # Store the highest alarm state of the minute in this hash.
        $m{$states[0]}++;
      } # end for(my $m=0; $m<60; $m++)

      my @states = sort {$b <=> $a} keys %m;
      my $components;
      if(@states > 1 or grep defined $_->[1],@m){
        # At the minute level and above I always count "missing" as
        # a valid alarm state, so it is sufficient to check for more
        # than one state without worrying about what the states are.
        # Any minute having sub-components is also grounds for putting
        # the pointers to the minutes into the hour array.
        $components = \@m;
      }
      my $alarm_info = [$states[0],$components];
      push @h,$alarm_info;
      # Put the highest alarm status of the hour into the hour hash.
      $h{$states[0]}++;
    } # end for(my $h=0; $h<24; $h++)

    my @states = sort {$b <=> $a} keys %h;
    my $components;
    if(@states > 1 or grep defined $_->[1],@h){
      $components = \@h;
    }
    $alarminfo[$p] = [$states[0],$components];
  } # end for(my $p=0; $p<@$params; $p++)
  \@alarminfo;
}

sub write_alarm_info{
  my ($year,$alarminfo,$numparams,$typ) = @_;

  # Do not write out a yearly bin file for any year that has no alarm
  # status that is greater than 0 (i.e. don't write a file for a year
  # which has no data).
  all_empty($alarminfo) and return;

  my $file = "$metadir/Q$year.$typ.bin";

  open F,'>',$file or die "Could not create $file ($!)";

  print STDERR "[",scalar localtime,"] Writing $file\n";

  print F $year;
  print F pack("L",$numparams);

  # Initialize the offset at 8 bytes (for the year and
  # parameter count I just printed to the metadata file)
  # plus 5 additional bytes for each year alarm (1 byte)
  # plus month offset (4 bytes) that I will write out below.
  my $offset = 4 + 4 + (1 + 4)*$numparams;

  # Output the yearly alarm values.
  print STDERR "[",scalar localtime,"] Output yearly alarm values.\n";
  for(my $p=0; $p<$numparams; $p++){
    # Write out the year alarm for this parameter.
    print F chr($alarminfo->[$p][0]);

    if($alarminfo->[$p][1]){
      # If the alarms vary across the months then write out
      # the offset to the monthly data.
      print F pack("L",$offset);
      $offset += 12*(1 + 4);
    }
    else{
      # Otherwise, write a null offset.
      print F pack("L",0);
    }
  }

  # Output the monthly alarm values.
  print STDERR "[",scalar localtime,"] Output monthly alarm values.\n";
  for(my $p=0; $p<$numparams; $p++){
    if($alarminfo->[$p][1]){
      # If there is more than one alarm state
      # for this parameter for this year...
      foreach my $mo ( @{ $alarminfo->[$p][1] } ){
        print F chr($mo->[0]);
        if($mo->[1]){
          print F pack("L",$offset);
          my $numdays = @{ $mo->[1] };
          $offset += $numdays*(1 + 4);
        }
        else{
          print F pack("L",0);
        }
      }
    }
  }

  # Output the daily alarm values.
  print STDERR "[",scalar localtime,"] Output daily alarm values.\n";
  for(my $p=0; $p<$numparams; $p++){
    if($alarminfo->[$p][1]){
      foreach my $mo ( @{ $alarminfo->[$p][1] } ){
        if($mo->[1]){
          # If there is more than one alarm state
          # for this parameter for this month...
          foreach my $d ( @{ $mo->[1] } ){
            print F chr($d->[0]);
            if($d->[1]){
              print F pack("L",$offset);
              $offset += 24*(1 + 4);
            }
            else{
              print F pack("L",0);
            }
          }
        }
      }
    }
  }

  # Output the hourly alarm values.
  print STDERR "[",scalar localtime,"] Output hourly alarm values.\n";
  for(my $p=0; $p<$numparams; $p++){
    if($alarminfo->[$p][1]){
      foreach my $mo ( @{ $alarminfo->[$p][1] } ){
        if($mo->[1]){
          foreach my $d ( @{ $mo->[1] } ){
            if($d->[1]){
              foreach my $h ( @{ $d->[1] } ){
                print F chr($h->[0]);
                if($h->[1]){
                  print F pack("L",$offset);
                  $offset += 60*(1 + 4);
                }
                else{
                  print F pack("L",0);
                }
              }
            }
          }
        }
      }
    }
  }

  # Output the minute-by-minute alarm values.
  print STDERR "[",scalar localtime,"] Output minute-by-minute alarm values.\n";
  for(my $p=0; $p<$numparams; $p++){
    if($alarminfo->[$p][1]){
      foreach my $mo ( @{ $alarminfo->[$p][1] } ){
        if($mo->[1]){
          foreach my $d ( @{ $mo->[1] } ){
            if($d->[1]){
              foreach my $h ( @{ $d->[1] } ){
                if($h->[1]){
                  foreach my $m ( @{ $h->[1] } ){
                    print F chr($m->[0]);
                    if($m->[1]){
                      print F pack("L",$offset);
                      $offset += 60;
                    }
                    else{
                      print F pack("L",0);
                    }
                  }
                }
              }
            }
          }
        }
      }
    }
  }

  # Output the second-by-second alarm values.
  print STDERR "[",scalar localtime,"] Output second-by-second alarm values.\n";
  for(my $p=0; $p<$numparams; $p++){
    if($alarminfo->[$p][1]){
      foreach my $mo ( @{ $alarminfo->[$p][1] } ){
        if($mo->[1]){
          foreach my $d ( @{ $mo->[1] } ){
            if($d->[1]){
              foreach my $h ( @{ $d->[1] } ){
                if($h->[1]){
                  foreach my $m ( @{ $h->[1] } ){
                    if($m->[1]){
                      print F pack('c' x 60, @{ $m->[1] });
                    }
                  }
                }
              }
            }
          }
        }
      }
    }
  }
  close F;
}

sub all_empty{
  my $meta = shift;
  for(my $p=0; $p<@$meta; $p++){
    $meta->[$p][0] and return 0;
  }
  return 1;
}

sub alarm_state{
  my ($v,$rl,$yl,$yh,$rh) = @_;
  $v  =~ /^[+\-]?(?:\d+\.?\d*|\.\d+)$/ or return 0;	# Require numeric value
  $rl =~ /^[+\-]?(?:\d+\.?\d*|\.\d+)$/ or $rl = undef;	# Not all parameters
  $yl =~ /^[+\-]?(?:\d+\.?\d*|\.\d+)$/ or $yl = undef;	# have been assigned
  $yh =~ /^[+\-]?(?:\d+\.?\d*|\.\d+)$/ or $yh = undef;	# limits; those cases
  $rh =~ /^[+\-]?(?:\d+\.?\d*|\.\d+)$/ or $rh = undef;	# are denoted by N/A.

  if(defined $rl and $v <= $rl
  or defined $rh and $v >= $rh){ return 3 }			# red alarm

  if(defined $yl and $v <= $yl
  or defined $yh and $v >= $yh){ return 2 }			# yellow alarm

  return 1;							# green
}

sub read_meta{
  my ($year,$typ) = @_;
  my @dpm = (31,28,31,30,31,30,31,31,30,31,30,31);
  $dpm[1] = 29 if isleap($year);
  my $file = "$metadir/Q$year.$typ.bin";
  open F,'<',$file or return(0,0);
  my $ts = $^T - 86400*(-M $file);
  my $offset = 0;
  my $bytes;
  read(F,$bytes,4) == 4 or die "Error reading $file ($!)";
  $bytes eq $year or die "Unexpected year ($bytes) in $file\n";
  $offset += 4;
  read(F,$bytes,4) == 4 or die "Error reading $file ($!)";
  my $numparams = unpack "L",$bytes;
  $offset += 4;
  # Read the yearly metadata for each parameter.
  read(F,$bytes,5*$numparams) == 5*$numparams or die "Error reading $file ($!)";
  my @meta = map {[unpack "cL",$_]} unpack 'a5' x $numparams, $bytes;
  $offset += 5*$numparams;
  for(my $p=0; $p<@meta; $p++){
    next if $meta[$p][1] == 0;
    if($meta[$p][1] != $offset){
      die "Unexpected format in $file ($meta[$p][1] != $offset)";
    }
    # Read the monthly metadata.
    read(F,$bytes,5*12) == 5*12 or die "Error reading $file ($!)";
    $meta[$p][1] = [map {[unpack "cL",$_]} unpack 'a5' x 12, $bytes];
    $offset += 5*12;
  }
  for(my $p=0; $p<@meta; $p++){
    next if $meta[$p][1] == 0;
    for(my $o=0; $o<12; $o++){
      next if $meta[$p][1][$o][1] == 0;
      if($meta[$p][1][$o][1] != $offset){
        die "Unexpected format in $file ($meta[$p][1][$o][1] != $offset)";
      }
      # Read the daily metadata.
      read(F,$bytes,5*$dpm[$o]) == 5*$dpm[$o] or die "Error reading $file ($!)";

      $meta[$p][1][$o][1] =
      [map {[unpack "cL",$_]} unpack 'a5' x $dpm[$o], $bytes];

      $offset += 5*$dpm[$o];
    }
  }
  for(my $p=0; $p<@meta; $p++){
    next if $meta[$p][1] == 0;
    for(my $o=0; $o<12; $o++){
      next if $meta[$p][1][$o][1] == 0;
      for(my $d=0; $d<$dpm[$o]; $d++){
        next if $meta[$p][1][$o][1][$d][1] == 0;
        if($meta[$p][1][$o][1][$d][1] != $offset){
          die "Unexpected format in $file ",
              "($meta[$p][1][$o][1][$d][1] != $offset)";
        }
        # Read the hourly metadata.
        read(F,$bytes,5*24) == 5*24 or die "Error reading $file ($!)";

        $meta[$p][1][$o][1][$d][1] =
        [map {[unpack "cL",$_]} unpack 'a5' x 24, $bytes];

        $offset += 5*24;
      }
    }
  }
  for(my $p=0; $p<@meta; $p++){
    next if $meta[$p][1] == 0;
    for(my $o=0; $o<12; $o++){
      next if $meta[$p][1][$o][1] == 0;
      for(my $d=0; $d<$dpm[$o]; $d++){
        next if $meta[$p][1][$o][1][$d][1] == 0;
        for(my $h=0; $h<24; $h++){
          next if $meta[$p][1][$o][1][$d][1][$h][1] == 0;
          if($meta[$p][1][$o][1][$d][1][$h][1] != $offset){
            die "Unexpected format in $file ",
                "($meta[$p][1][$o][1][$d][1][$h][1] != $offset)";
          }
          # Read the minute-by-minute metadata.
          read(F,$bytes,5*60) == 5*60 or die "Error reading $file ($!)";

          $meta[$p][1][$o][1][$d][1][$h][1] =
          [map {[unpack "cL",$_]} unpack 'a5' x 60, $bytes];

          $offset += 5*60;
        }
      }
    }
  }
  for(my $p=0; $p<@meta; $p++){
    next if $meta[$p][1] == 0;
    for(my $o=0; $o<12; $o++){
      next if $meta[$p][1][$o][1] == 0;
      for(my $d=0; $d<$dpm[$o]; $d++){
        next if $meta[$p][1][$o][1][$d][1] == 0;
        for(my $h=0; $h<24; $h++){
          next if $meta[$p][1][$o][1][$d][1][$h][1] == 0;
          for(my $m=0; $m<60; $m++){
            next if $meta[$p][1][$o][1][$d][1][$h][1][$m][1] == 0;
            if($meta[$p][1][$o][1][$d][1][$h][1][$m][1] != $offset){
              die "Unexpected format in $file ",
                  "($meta[$p][1][$o][1][$d][1][$h][1][$m][1] != $offset)";
            }
            # Read the individual second alarms.
            read(F,$bytes,60) == 60 or die "Error reading $file ($!)";

            $meta[$p][1][$o][1][$d][1][$h][1][$m][1] =
            [unpack 'c' x 60, $bytes];

            $offset += 60;
          }
        }
      }
    }
  }
  (\@meta,$ts);
}

sub daymeta2buf{
  my ($day,$meta) = @_;
  my ($d,$o) = (gmtime 86400*$day)[3,4];
  $d--;

  # Each alarm buffer holds alarm statuses for all of the
  # telemetry parameters for every second of a day.  There
  # are four 2-bit statuses per byte in the buffer.  The buffer
  # initially gets filled with zeros which signify missing data.
  my $buf = chr(0) x (86400*@$meta/4);

  for(my $p=0; $p<@$meta; $p++){
    my ($pa,$pp) = @{$meta->[$p]};
    if($pp){
      my ($oa,$op) = @{$pp->[$o]};
      if($op){
        my ($da,$dp) = @{$op->[$d]};
        if($dp){
          for(my $h=0; $h<24; $h++){
            my ($ha,$hp) = @{$dp->[$h]};
            if($hp){
              for(my $m=0; $m<60; $m++){
                my ($ma,$mp) = @{$hp->[$m]};
                if($mp){
                  for(my $s=0; $s<60; $s++){
                    my $i = 86400*$p + 3600*$h + 60*$m + $s;
                    vec($buf,$i,2) = $mp->[$s];
                  }
                }
                elsif($ma > 0){
                  for(my $s=0; $s<60; $s++){
                    my $i = 86400*$p + 3600*$h + 60*$m + $s;
                    vec($buf,$i,2) = $ma;
                  }
                }
              }
            }
            elsif($ha > 0){
              for(my $s=0; $s<3600; $s++){
                my $i = 86400*$p + 3600*$h + $s;
                vec($buf,$i,2) = $ha;
              }
            }
          }
        }
        elsif($da > 0){
          for(my $s=0; $s<86400; $s++){
            my $i = 86400*$p + $s;
            vec($buf,$i,2) = $da;
          }
        }
      }
      elsif($oa > 0){
        for(my $s=0; $s<86400; $s++){
          my $i = 86400*$p + $s;
          vec($buf,$i,2) = $oa;
        }
      }
    }
    elsif($pa > 0){
      for(my $s=0; $s<86400; $s++){
        my $i = 86400*$p + $s;
        vec($buf,$i,2) = $pa;
      }
    }
  }
  $buf;
}
