Subs::Filter (version 2.0)


package Subs::Filter;
##############################################################################
#
# DESCRIPTION: Apply filtering criteria - spacial, temporal, or
# DESCRIPTION: event selection - to the event data.
# DESCRIPTION:
#
# HISTORY:
# HISTORY: $Log: Filter.pm,v $
# HISTORY: Revision 1.60  2016/11/23 15:05:07  apsop
# HISTORY: Processing of unfiltered XRT event files continues even if a cleaned event
# HISTORY: file is not produced by xrtscreen (provided xrtscreen does not exit with
# HISTORY: an error).
# HISTORY:
# HISTORY: Revision 1.59  2015/09/28 15:28:55  apsop
# HISTORY: Updated to HEASoft 6.17 and applied XRT, UVOT, and clock CALDB patches. Modified XRT event file processing to avoid further processing after an error occurs.
# HISTORY:
# HISTORY: Revision 1.58  2009/06/05 18:27:41  apsop
# HISTORY: Added column TLMPHAS to delete
# HISTORY:
# HISTORY: Revision 1.57  2008/06/30 17:30:38  apsop
# HISTORY: Guard against another posibble writing of GOOD_ATT keyword to nonexistent extension.
# HISTORY:
# HISTORY: Revision 1.56  2008/06/26 14:26:25  apsop
# HISTORY: Guard against writing GOOD_ATT keyword to nonexistent extension.
# HISTORY:
# HISTORY: Revision 1.55  2008/06/23 14:09:12  apsop
# HISTORY: When moving uvot images to damaged file, check if image has already been moved.  Check of events in xrt files before running xrtscreen.
# HISTORY:
# HISTORY: Revision 1.54  2008/06/02 13:28:55  apsop
# HISTORY: Change D_REASON keyword to DMG_STAT.
# HISTORY:
# HISTORY: Revision 1.53  2008/05/16 14:19:15  apsop
# HISTORY: New method for screening uvot images and moving bad ones out of the way.  Rework uvot event screening, but disabled for now.
# HISTORY:
# HISTORY: Revision 1.52  2007/11/08 17:13:35  apsop
# HISTORY: Do not try and get calibration events from an empty xrt event list.
# HISTORY:
# HISTORY: Revision 1.51  2007/09/27 17:26:01  apsop
# HISTORY: The column name PHAS0 should be PHASO, with an O rather than a zero.
# HISTORY:
# HISTORY: Revision 1.50  2007/09/25 20:10:43  apsop
# HISTORY: Remove PHAS0 column from the xrt cleaned event lists.
# HISTORY:
# HISTORY: Revision 1.49  2007/09/11 18:15:27  apsop
# HISTORY: Move production of filter file to a separate class.  Clean up obsolete subroutines.
# HISTORY:
# HISTORY: Revision 1.48  2006/09/10 20:05:16  apsop
# HISTORY: Remove explicit mode parameter setting from xrtscreen.
# HISTORY:
# HISTORY: Revision 1.47  2006/08/09 14:53:32  apsop
# HISTORY: Sort the xrt header file before using in making the filter file.
# HISTORY:
# HISTORY: Revision 1.46  2006/08/08 14:52:47  apsop
# HISTORY: Fix bugs in sorting of the DAP files.
# HISTORY:
# HISTORY: Revision 1.45  2006/08/04 13:36:00  apsop
# HISTORY: Check for duplicate time rows in dap files and remove.
# HISTORY:
# HISTORY: Revision 1.44  2006/05/10 15:20:13  apsop
# HISTORY: Remove RAWXTL column from filtered event lists.
# HISTORY:
# HISTORY: Revision 1.43  2006/04/27 16:21:59  apsop
# HISTORY: Comment out leapsec and tprec parameters from newmakefilter call. Allow for either INDEF or NULL values returned from FITSfile class.
# HISTORY:
# HISTORY: Revision 1.42  2006/01/31 16:44:48  apsop
# HISTORY: Switch to using snapshot gti to filter mkf file.
# HISTORY:
# HISTORY: Revision 1.41  2005/12/02 15:42:09  apsop
# HISTORY: Going back to using newmakefilter.
# HISTORY:
# HISTORY: Revision 1.40  2005/12/01 20:54:20  apsop
# HISTORY: Replace newmakefilter task with makefilter task.
# HISTORY:
# HISTORY: Revision 1.39  2005/11/20 20:31:18  apsop
# HISTORY: Check for presence of ACS_DATA extension in attitude file.
# HISTORY:
# HISTORY: Revision 1.38  2005/11/17 16:57:39  apsop
# HISTORY: Fix another typo, and clean up temporary files.
# HISTORY:
# HISTORY: Revision 1.37  2005/11/17 14:14:56  apsop
# HISTORY: Fix typo error.
# HISTORY:
# HISTORY: Revision 1.36  2005/11/08 17:38:57  apsop
# HISTORY: Fix bugs in gett s/c hk and bat dap hk files. Obtain min/max/median values for proper dap hk columns.  Make xrt event files of calibration only sources.
# HISTORY:
# HISTORY: Revision 1.35  2005/08/30 14:00:10  apsop
# HISTORY: Switch to using newmakefilter. Add proper NULL values for unsigned integer columns. Append used GTI to mkf file.
# HISTORY:
# HISTORY: Revision 1.34  2005/07/29 14:06:12  apsop
# HISTORY: remove bias map entries from xrt hd before using to make a filter file.
# HISTORY:
# HISTORY: Revision 1.33  2005/05/31 20:04:40  apsop
# HISTORY: Perform GTI filtering on .mkf file.  Derive min/max time using stats
# HISTORY: instead of dumping entire file.
# HISTORY:
# HISTORY: Revision 1.32  2005/05/04 15:19:14  apsop
# HISTORY: Give up on old algorithm and test for columns in cleaned xrt event list before deleting.
# HISTORY:
# HISTORY: Revision 1.31  2005/05/03 15:43:04  apsop
# HISTORY: Change globs for xrt modes that have b? instead of w?.
# HISTORY:
# HISTORY: Revision 1.30  2005/04/29 15:41:26  apsop
# HISTORY: Large change in setting of xrtscreen parameters. uvotscreen disabled. Remove unneeded columns from screened xrt event lists.
# HISTORY:
# HISTORY: Revision 1.29  2005/04/06 15:45:03  apsop
# HISTORY: Change to using CALDB for cal parameters.
# HISTORY:
# HISTORY: Revision 1.28  2005/03/25 19:55:03  apsop
# HISTORY: Fix bug in merging of HK data into mkf file.
# HISTORY:
# HISTORY: Revision 1.27  2005/03/15 19:57:56  apsop
# HISTORY: Fix bug in call for getting eng hk file names.  Write DATE-* keywords to mkf primary header.
# HISTORY:
# HISTORY: Revision 1.27  2005/03/15 19:03:26  apsop
# HISTORY: Check for presence of xrt no position message when making tdrss cat file.
# HISTORY:
# HISTORY: Revision 1.26  2005/02/08 18:18:38  apsop
# HISTORY: Move calculation of attorb file to Filter class. Split making of attitude flag columns in to two parts.
# HISTORY:
# HISTORY: Revision 1.25  2005/01/24 18:58:11  apsop
# HISTORY:
# HISTORY: Add support for SAFEHOLD column to makefilter file.
# HISTORY:
# HISTORY: Revision 1.24  2005/01/12 17:21:02  apsop
# HISTORY: Incorporate removal of _PNT suffix from output columns of prefilter
# HISTORY:
# HISTORY: Revision 1.23  2004/11/19 21:46:48  apsop
# HISTORY: New version of xrt2fits.
# HISTORY:
# HISTORY: Revision 1.22  2004/10/12 16:19:49  apsop
# HISTORY: Turn on data filtering to produce cleaned event lists
# HISTORY:
# HISTORY: Revision 1.21  2004/08/30 13:21:51  apsop
# HISTORY: Bobs new version of module, but comment out actual screening for now.
# HISTORY:
# HISTORY: Revision 1.20  2004/08/27 18:38:13  apsop
# HISTORY: Modified to use Ed's code to determine the filtering expressions, but
# HISTORY: the instrument tools for the actual filtering.
# HISTORY:
# HISTORY: Revision 1.19  2004/05/06 20:02:34  dah
# HISTORY: Add version number back into the header comments.
# HISTORY:
# HISTORY: Revision 1.18  2004/04/16 20:21:18  dah
# HISTORY: Begin using embedded history records
# HISTORY:
#
# VERSION: 2.0
#
#
##############################################################################


use Subs::SwiftSub;
use Util::GTIfile;

@ISA = ("Subs::SwiftSub");
use strict;

sub new {
    my $proto=shift;
    my $self=$proto->SUPER::new();

    $self->{DESCRIPTION}="Cleaning and filtering the event files";

    return $self;
}

##################
# METHODS:
##################

sub body {
    my $self=shift;

    my $log     =$self->log();
    my $filename=$self->filename();
    my $procpar =$self->procpar();
    my $jobpar  =$self->jobpar();

    ##################################################
    # plot interesting quantities in the filter file
    ##################################################
  #  $self->make_plots();

    ##################################################
    # screen out bad event data
    ##################################################
    $self->filter_data();


} # end of body method


##############################################################################
# Make a number of plots of interesting quantities in the filter file
##############################################################################
sub make_plots {
    my $self = shift;

    my $log     =$self->log();
    my $filename=$self->filename();
    my $procpar =$self->procpar();
    my $jobpar  =$self->jobpar();

    ########################
    # get the filter file
    ########################
    my $filter = $filename->get("filter", "s");
    unless( -f $filter) { return; }

    my $fits_filter = Util::FITSfile->new($filter, 1);

    $log->entry("Plotting interesting things in the filter file");

    ######################################
    # set up for plotting
    ######################################
    my %yparm=(attd=>"RA DEC ROLL ANG_DIST",
               weel=>"FWHEEL"              ,
               bat1=>"BDIHKGRBFOLLOW BDIHKSLEWING BDIHKSAA BDIHKVALID",
               bat2=>"BDIHKNGOODDETS BCAHKDOINGBLK BBRHKTRIGNUM",
               saaa=>"ORBIT_SAA ACS_SAA BDIHKSAA"     );

    my $command_tmp=$self->temp_file("plot_commands");

    my $fplot=Util::Ftool->new("fplot")
                         ->params({infile=>$filter,
                                   xparm   => "TIME",
                                   rows    => "-",
                                   device  => "/cps",
                                   pltcmd => "\@$command_tmp",
                                   offset  => "yes"});

    my $plot_output="pgplot.ps";


    my $mode;
    foreach $mode (keys %yparm) {
        #################################################
        # make sure the filter file has all the columns
        # that we need for this plot
        #################################################
        my $good=1;
        foreach my $col (split /\s+/, $yparm{$mode}) {
            unless( $fits_filter->find_column($col)) {
                $log->entry("$filter is missing $col");
                $good=0;
                last;
            }
        }
        
        unless($good) {
            $log->entry("Skipping $mode plot");
            next;
        }

        #######################################
        # copy the plot command file and 
        # insert the file name for "MKF"
        #######################################
        my $template = $self->proctop()."/lists/filter_plot.$mode";
        open IN, "<$template";
        open OUT, ">$command_tmp";
        while (<IN>) {

            s/MKF/$filter/;
            print OUT;
        }
        close(OUT);
        close(IN);

        ##################################
        # make the plot
        ##################################
        unlink $plot_output;
        $fplot->params({yparm=>$yparm{$mode}})
              ->run();

        ############################################
        # save the plot with the correct file name
        ############################################
        if( -s $plot_output ) {
            rename $plot_output,
                   $filename->get("filterplot","s",$mode,0);
        } else {
            $log->error(2,"$plot_output not created");
        }
    } # end of loop over plot types


} # end of make_plots method


###########################################################################
# apply standard filters to the event data
###########################################################################
sub filter_data {

    my $self = shift;

    my $log     =$self->log();
    my $filename=$self->filename();
    my $procpar =$self->procpar();
    my $jobpar  =$self->jobpar();

    $self->xrtscreen();
    $self->xrt_cal_events();

    $self->uvot_event_screen;
    $self->uvot_image_screen;

} # end of filter_data



sub uvot_image_screen {

  my $self = shift;

  my $log     = $self->log();
  my $filename= $self->filename();
  my $procpar = $self->procpar();
  my $jobpar  = $self->jobpar();

  $log->entry("Screen uvot images.");

  my $cat_file = $filename->get('hk', 'uvot', 'ct', '*');
  #####################################
  # No cat file indicates no uvot data
  #####################################
  return unless -f $cat_file;

  #####################################
  # Put needed columns into arrays
  #####################################
  my %columns;
  my $cat_fits = Util::FITSfile->new($cat_file, 1);
  foreach my $column_name (qw(IFILEREF IMENAME ESTART ESTOP Image Event)){
    $cat_fits->cols($column_name);
    $columns{$column_name} = [$cat_fits->table()];
  }

  #####################################
  # Loop over catalog entries
  #####################################
  my @cat_rows_to_delete;
  my %examined_extensions;
  my $cat_rows = $cat_fits->nrows();
  for(my $cat_row_cnt=0; $cat_row_cnt<$cat_rows; $cat_row_cnt++){
    #####################################
    # Only process entries with images
    #####################################
    next unless $columns{Image}->[$cat_row_cnt] != 0;
    my $extension_name = $columns{IMENAME}->[$cat_row_cnt];
    $log->entry('Examining exposure '. $extension_name);

    my $good_attitude_fraction = $self->good_attitude_fraction($columns{ESTART}->[$cat_row_cnt], 
							       $columns{ESTOP}->[$cat_row_cnt]);

    my $exposure_image_fits = Util::FITSfile->new($columns{IFILEREF}->[$cat_row_cnt], $extension_name);
    unless( grep(/${extension_name}/, keys(%examined_extensions)) ){
      $exposure_image_fits->keyword('GOOD_ATT', $good_attitude_fraction,
				    'Fraction of exposure interval with good attitude info');
    }
    $log->entry("Good attitude fraction is $good_attitude_fraction .");

    #######################################################################
    # Check if we need to treat this image as damaged.
    #######################################################################
    if( $good_attitude_fraction < $jobpar->read('uvot_att_damage_thresh') ){
      unless( grep(/${extension_name}/, keys(%examined_extensions)) ){
	$exposure_image_fits->keyword('DMG_STAT', 'Bad_Attitude', 
				      'If or why the image tagged as damaged');
      }
      ########################################################################
      # Create damaged image file if necessary, with empty catalog extension.
      ########################################################################
      my $damage_image_file = $filename->get('rawimage', 'uvot', 'di', '0');
      unless( -f $damage_image_file ){
	Util::HEAdas->new('ftcopy')
	            ->params({infile => $cat_file . '[EXPCATALOG][#row==0]',
			      outfile => $damage_image_file})
		    ->run();
      }
      #################################################################################
      # Move damaged image to the damaged image file.
      # If this extension has already been examined, then image has alreay been moved.
      #################################################################################
      unless( grep(/${extension_name}/, keys(%examined_extensions)) ){
        $log->entry('Move '. $exposure_image_fits->fullname() .' to damaged image file');

        Util::HEAdas->new('ftappend')
	            ->params({infile => $exposure_image_fits->fullname(),
		              outfile => $damage_image_file})
                    ->run();

        Util::HEAdas->new('ftdelhdu')
	            ->params({infile => $exposure_image_fits->fullname(),
		       	      outfile => 'none',
			      confirm => 'yes'})
		    ->run();
      }
      ##############################################
      # Copy exposure record to damaged image file.
      ##############################################
      my $temp_file = 'merged_file.tmp';
      $cat_fits->specs('[#row == ' . ($cat_row_cnt+1) .']');
      my $merge = Util::HEAdas->new('ftmerge')
                              ->params({infile => "$damage_image_file\[EXPCATALOG\], " . 
					$cat_fits->fullname(),
					copyall => 'yes',
					outfile => $temp_file})
			      ->run();

      unless( $merge->had_error() ){
	rename $temp_file, $damage_image_file;
      }else{
	unlink $temp_file;
      }
      $cat_fits->specs('');

      ################################################
      # Check if this is an image and event exposure.
      ################################################
      if( $columns{Event}->[$cat_row_cnt] == 1 ){
	############################################
	# Set image flag to zero in catalog record.
	############################################
	Util::HEAdas->new('ftedit')
	            ->params({infile => $cat_file,
			      column => 'Image',
			      row => $cat_row_cnt+1})
		    ->run();
      }else{
	#################################################
	# Add this row to the list of rows to be deleted
	#################################################
	push @cat_rows_to_delete, $cat_row_cnt+1;
      } 
    }
    ############################################
    # Keep track of extensions already examined
    ############################################
    $examined_extensions{$extension_name} = 1;
  }# End loop over catalog entries.

  ###########################################
  # Delete tagged rows from the catalog file
  ###########################################
  if(@cat_rows_to_delete){
    Util::HEAdas->new('ftdelrow')
	        ->params({infile => $cat_fits->fullname(),
			  rows => join(',', @cat_rows_to_delete),
			  outfile => 'none',
			  confirm => 'yes'})
	        ->run();
  }
}

sub uvot_event_screen {

    my $self = shift;

    my $log     = $self->log();
    my $filename= $self->filename();
    my $procpar = $self->procpar();
    my $jobpar  = $self->jobpar();

#    my $option = $jobpar->read('opt_uvot_screen');
#    if ($option =~ /^n/) {
#        $log->entry('UVOT event screening disabled');
#        return;
#    }

    my $gti = $filename->get('gti', 's', 'at');
    my $attorb = $filename->get('attorb', 's');
    my $temp_list = 'events.tmp';

    my $uscreen = Util::HEAdas->new('uvotscreen')
                     ->params({
                        attorbfile => $attorb,
                        aoexpr => 'ANG_DIST.lt.100',
                        evexpr => 'QUALITY==0',
                        badpixfile => 'CALDB'
                     })
           		->is_script(1);

    my $copy = Util::HEAdas->new('ftcopy')
                     ->params({ history => 'yes',
				copyall => 'yes' });

    # iterate over UVOT unfiltered event files
    foreach my $unfFile ($filename->get('unfiltered', 'uvot', '*', '*')) {
      my $expression = "[col *; QUALITY = gtifilter(\'$gti\') ? QUALITY : QUALITY+256]";
      $copy->params({infile => $unfFile . '[EVENTS]' . $expression,
                     outfile => $temp_list})
           ->run();

      unless( $copy->had_error() ){
        rename $temp_list, $unfFile;
      }else{
	unlink $temp_list;
      }

##      my $evtFile = $filename->corresponding('unfiltered', 'event', $unfFile);
##      $log->entry("filtering $unfFile");
##
##      $uscreen->params({infile => $unfFile,
##                        outfile => $evtFile})
##              ->run();
    }

} # end of uvotscreen


sub xrtscreen {
    my $self = shift;

    my $log     = $self->log();
    my $filename= $self->filename();
    my $procpar = $self->procpar();
    my $jobpar  = $self->jobpar();

    #######################################
    # find the filter file
    #######################################
    my $filter = $filename->get('filter', 'x');
	if (not -f $filter) {
		$log->entry("No filter file, so can't do screening");
		return
	}

    my $xrtscreen = Util::HEAdas->new('xrtscreen')
				->params({
                                          mkffile => $filter,
					  gtiexpr => 'DEFAULT',
					  exprgrade => 'DEFAULT',
					  expr => 'DEFAULT',
					  clobber => 'no',
					  chatter => 3,
					  history => 'yes',
					  createattgti => 'yes',
					  createinstrgti => 'yes',
					  gtiscreen => 'yes',
					  evtscreen => 'yes',
					  obsmodescreen => 'yes',
					  gtifile => 'xrtscreen_gti.tmp',
					  usrgtifile => 'NONE',
					  hkrangefile => 'CALDB',
					  timecol => 'TIME',
					  outfile => 'DEFAULT',
					  gtiext => 'GTI',
					  evtrangefile => 'CALDB',
					  cleanup => 'yes'
				 })
				->is_script(1);

    ################################################
    # determine the modes which should be filtered
    ################################################
    my @modes= qw(pcw?po wtw?po pub?po lrb?po pcw?s[lt] wtw?s[lt] pub?s[lt] lrb?s[lt]);

    my %failed;
    Subs::XrtEvents::loadFailedEventFiles($self, \%failed);

    my $inst = 'xrt';
    ########################
    # loop over modes
    ########################
    foreach my $mode (@modes) {

        my $mo = substr($mode,0,2);

        #################################################
        # determine what kid of files we should use as
        # input for filtering
        ################################################
        my $unfiltered_type="unfiltered";
        if ($mo ne "pc") {
            ########################################################
            # for the XRT use the reconstructed event files unless
            # this is photon counting  mode
            ########################################################
            $unfiltered_type = "reconst";
        }

	if($mode =~ /s.lt./){
	  $xrtscreen->params({createattgti => 'no'});
	}else{
	  $xrtscreen->params({createattgti => 'yes'});
	}

        #########################################
        # get the unfiltered files
        #########################################
        my @unfs = $filename->get($unfiltered_type, $inst, $mode, '*');

        #############################################
        # check if there are any files in this mode
        #############################################
        if (@unfs) {
            $log->entry("Mode $mode");
        } else {
            $log->entry("No unfiltered files for mode $mode\n");
            next;
        }

        #############################
        # loop over unfiltered files
        #############################
        foreach my $unf (@unfs) {
	    #########################################
	    # Check if there is anything in the file
	    #########################################
	    my $new_fits = Util::FITSfile->new($unf, 'EVENTS');
	    next if( $new_fits->keyword('NAXIS2') == 0 );
            if ($failed{$unf}) {
                $log->entry("Skipping extraction from $unf");
                next;
            }

            ##################################################
            # get the name of the corresponding filtered file
            ##################################################
            my $evt = $filename->corresponding($unfiltered_type, 'event', $unf);

            $log->entry("Extracting $evt from $unf");

            ###########################
            # run xrtscreen
            ###########################
	    $xrtscreen->params({
				infile => $unf,
				outfile => $evt,
			       })
	               ->run;

	    unlink(qw(xselect.hty defaults.def xrtscreen_gti.tmp));

	    if ($xrtscreen->had_error) {
	      $failed{$unf} = 1;
	      $log->entry("Will avoid further processing of $unf");
	      next;
	    }

	    if (not -f $evt) {
	      $log->entry("Missing $evt");
	      next;
	    }

            ####################################################
            # make sure the screened file has some events in it
            ####################################################
	    my $fits = Util::FITSfile->new($evt,"EVENTS");
            my $nevents = $fits->keyword("NAXIS2");
            if($nevents) {
                $log->entry("$nevents filtered events");

		#############################
		# Delete unnecessary columns
		#############################
		my @cols = qw(CCDFrame PHAS PHASO Amp PixsAbove ROTIME OFFSET RAWXTL TLMPHAS);
		my $specs='[col ';
		foreach my $cname (@cols){
		  $specs .= "-$cname; " 
		    if $fits->find_column($cname)
		}
		$specs .= ']';

	        $fits->specs($specs);
		$fits->copy('xrt_events.tmp');
		unlink $evt;
		rename 'xrt_events.tmp', $evt;
            } else {
                $log->entry("Deleting $evt since it has $nevents events");
                unlink $evt;
                next;
            }

	  } # end foreach unfiltered

	} # end foreach mode

    Subs::XrtEvents::saveFailedEventFiles($self, \%failed);

} # end of xrtscreen

sub fixNulls {
  ############################################################
  # Set TNULL values for columns with unsigned integer types.
  # Need this as makefilter uses an inappropriate default 
  # TNULL = -max for these columns
  ############################################################
  my ($self, $file, $ext) = @_[0,1,2];

  my $fits = Util::FITSfile->new($file, $ext);
  my %keywords = $fits->keywords();
  my %max = ('B' => 256,
	     'I' => 32768,
	     'J' => 2147483648);

  $fits->begin_many_keywords();
  my $n=0;
  foreach my $tform (grep /^TFORM/, keys %keywords){
    next unless $keywords{$tform} =~ /([IJ])/;
    my $type = $1;
    my $num = $tform;
    $num =~ s/TFORM//;

    next unless $keywords{"TZERO$num"} = $max{$type};

    unless( $keywords{"TNULL$num"} ){
      $fits->keyword("TNULL$num", $max{$type}-1);
      $n++;
    }
  }
  $fits->end_many_keywords() if $n;
}


sub xrt_cal_events {
  my $self = shift;

  my $log     = $self->log();
  my $filename= $self->filename();
  my $procpar = $self->procpar();
  my $jobpar  = $self->jobpar();

  my %failed;
  Subs::XrtEvents::loadFailedEventFiles($self, \%failed);

  #####################################################
  # Select out the events from the calibration sources
  #####################################################
  my $spec = '[(STATUS & b1111000000000000) > b0]';
  foreach my $win ('w2', 'w3'){
    my @phot = $filename->get('unfiltered', 'x', "pc${win}*", '*');
    my $xrtcal = $filename->get('xrtcal', 'x', "pc${win}", 0);
    my @more_files;
    my $i=0;
    foreach my $file (@phot) {
      if ($failed{$file}) {
         $log->entry("Skipping $file due to previous error");
         next;
      }
      my $fits_file = Util::FITSfile->new($file, 'EVENTS', $spec);
      next unless $fits_file->nrows();
      if( -f $xrtcal ){
        my $tmp = 'xrt_cal_' . $i++ . '.tmp';
        $fits_file->copy($tmp);
        push @more_files, $tmp .'[EVENTS]';
      }else{
        $fits_file->copy($xrtcal);
      }
    }

    ###############################################################
    # If there is more than one file per window type, then need to
    # merge them together and update timing keywords.
    ###############################################################
    if(@more_files){
      my $merge_file = 'xrtcal_merge.tmp';
      Util::HEAdas->new('ftmerge')
                  ->params({infile => $xrtcal .','. join(',', @more_files),
                            outfile => $merge_file})
                  ->run();

      my $del = Util::HEAdas->new('ftdelhdu')
                            ->params({outfile => 'none',
                                      confirm => 'yes',
                                      clobber => 'yes'});

      $del->params({infile => $merge_file .'[GTI]'})
          ->run();
      $del->params({infile => $merge_file .'[BADPIX]'})
          ->run();

      my $stats=Util::HEAdas->new("ftstat") 
                            ->params({infile  => $merge_file .'[EVENTS][col TIME]',
	                              centroid => 'no'})
		                ->verbose(0)
				->run();

      my $parfile = $stats->parfile();
      my $tmin = $parfile->read('min');
      my $tmax = $parfile->read('max');

      my $merge_fits = Util::FITSfile->new($merge_file, 'EVENTS');
      $merge_fits->keyword('TSTART', $tmin);
      $merge_fits->keyword('TSTOP', $tmax);

      $merge_fits->ext(0);
      $merge_fits->keyword('TSTART', $tmin);
      $merge_fits->keyword('TSTOP', $tmax);

      my $first = Util::Date->new($tmin);
      my $last  = Util::Date->new($tmax);
      $merge_fits->keyword('DATE-OBS', $first->date().'T'.$first->time() );
      $merge_fits->keyword('DATE-END', $last->date().'T'.$last->time() );

      unlink $xrtcal;
      rename $merge_file, $xrtcal;
    }

    unlink (grep s/^(xrt_cal.*\.tmp).*$/$1/, @more_files);

    if( -f $xrtcal){
      my $rows = Util::FITSfile->new($xrtcal, 'EVENTS')->nrows();
      unlink $xrtcal unless $rows;
    }
  }
}

1;