Subs::SW0WrapUp (version 0.0)


package Subs::SW0WrapUp;
##############################################################################
#
# DESCRIPTION: This subroutine class handles a number of cleanup tasks which 
# DESCRIPTION: need to be done at the end of each processing run,
# DESCRIPTION: including creating output catalog files and verifying
# DESCRIPTION: the product files. Each of these functions is handled
# DESCRIPTION: by a separate method to make it easier to write a sub-class
# DESCRIPTION: which overriseds the default body method.
# DESCRIPTION: 
# DESCRIPTION: Each processing script should run this subroutine class
# DESCRIPTION: (or a decendant of it) last. 
#
# HISTORY: 
# HISTORY: $Log: SW0WrapUp.pm,v $
# HISTORY: Revision 1.59  2007/09/28 17:14:48  apsop
# HISTORY: Change writing of ATTFLAG keyword so that it is a string rather than an integer.
# HISTORY:
# HISTORY: Revision 1.58  2007/09/11 18:01:26  apsop
# HISTORY: Look for and use ATTFLAG value in primary header of files before using default value.
# HISTORY:
# HISTORY: Revision 1.57  2007/07/25 20:17:57  apsop
# HISTORY: Write REPROC keyword if needed.
# HISTORY:
# HISTORY: Revision 1.56  2007/06/28 20:56:37  apsop
# HISTORY: Don;t write standard keywords to scaled map files.
# HISTORY:
# HISTORY: Revision 1.55  2007/04/02 15:03:47  apsop
# HISTORY: Uncomment cleanup_files(), which was accidentally comment out.
# HISTORY:
# HISTORY: Revision 1.54  2007/04/01 19:11:30  apsop
# HISTORY: Move calculation of version numbers to SwiftSub.pm
# HISTORY:
# HISTORY: Revision 1.53  2007/03/15 21:25:57  apsop
# HISTORY: Remove hard coding of cycle number.
# HISTORY:
# HISTORY: Revision 1.52  2007/02/27 19:36:37  apsop
# HISTORY: Get trigger time from jobpar burst_time param.
# HISTORY:
# HISTORY: Revision 1.51  2007/02/13 15:38:01  apsop
# HISTORY: Remove ATTSTATU keyword from *sat.fits file.
# HISTORY:
# HISTORY: Revision 1.50  2007/02/08 21:54:54  apsop
# HISTORY: Fix bug in getting ATTSTATU value.
# HISTORY:
# HISTORY: Revision 1.49  2007/02/01 15:01:53  apsop
# HISTORY: Do not write UTCFINIT to attitdue files.  Write attitude status keyword to files.
# HISTORY:
# HISTORY: Revision 1.48  2006/09/20 20:32:26  apsop
# HISTORY: Remove trailing _S from swift version string, if present.
# HISTORY:
# HISTORY: Revision 1.47  2006/09/10 20:00:38  apsop
# HISTORY: Make parsing of ftools version compatible with ape. Call swiftversion as a generic tool.
# HISTORY:
# HISTORY: Revision 1.46  2006/07/03 01:44:59  apsop
# HISTORY: Fix bug in detecting 900+ segments.
# HISTORY:
# HISTORY: Revision 1.45  2006/06/28 19:05:16  apsop
# HISTORY: For 900 sequences, change the root when searching for ack messages.  
# HISOTRY: Set error condition if any level 2 errors occur on the final processing.
# HISTORY:
# HISTORY: Revision 1.44  2006/05/02 13:13:21  apsop
# HISTORY: Do not try and get EXTNAME from hdu 0.
# HISTORY:
# HISTORY: Revision 1.43  2006/04/27 16:02:48  apsop
# HISTORY: Add message for tracking keyword updates in log file.
# HISTORY:
# HISTORY: Revision 1.42  2006/01/12 17:05:42  apsop
# HISTORY: Remove repository bgaoff and bdetflag files in clean up.
# HISTORY:
# HISTORY: Revision 1.41  2006/01/05 23:26:06  apsop
# HISTORY: Implemented merging of BAT detector enable/disable and gain/offset maps
# HISTORY: for the observation and pruned the work-around that saved them with
# HISTORY: index numbers.
# HISTORY:
# HISTORY: Revision 1.40  2005/12/19 16:17:10  apsop
# HISTORY: Export bat catalog files to repository.  Remove any leftover bat tdrss 
# HISTORY: attitude data files.
# HISTORY:
# HISTORY: Revision 1.39  2005/11/20 20:36:11  apsop
# HISTORY: Fix bug in renaming of bgaoff type files.
# HISTORY:
# HISTORY: Revision 1.37  2005/11/08 19:22:28  apsop
# HISTORY: Populate the TIMELIST and DATALIST hashes. Used to be an SWCheckInput.
# HISTORY:
# HISTORY: Revision 1.36  2005/10/28 18:16:49  apsop
# HISTORY: Fix bug in renaming of gain/off and detector flag files.
# HISTORY:
# HISTORY: Revision 1.35  2005/10/26 17:30:08  apsop
# HISTORY: New subroutine for cleaning up files that will not be catalogued. Quick fix to 
# HISTORY: rename gain/offset and det enable bat files.
# HISTORY:
# HISTORY: Revision 1.34  2005/08/31 16:05:23  apsop
# HISTORY: Add TRIGTIME keyword to all non-trdss products if available.
# HISTORY:
# HISTORY: Revision 1.33  2005/07/29 14:20:10  apsop
# HISTORY: Fix bug in running local version of fverify. Switch to ftverify.
# HISTORY:
# HISTORY: Revision 1.32  2005/04/19 15:34:25  apsop
# HISTORY: Add in CALDBVER keyword. Fix up some keyword comments.
# HISTORY:
# HISTORY: Revision 1.31  2005/03/18 20:17:48  apsop
# HISTORY: Put orbit files back into the distribution.
# HISTORY:
# HISTORY: Revision 1.30  2005/03/17 16:50:06  apsop
# HISTORY: Remove TLE file from distribution.
# HISTORY:
# HISTORY: Revision 1.29  2005/03/15 19:54:54  apsop
# HISTORY: Add in code taken from StartEndTimes for writing UTCFINIT keyword.
# HISTORY:
# HISTORY:
# HISTORY: Revision 1.28  2005/03/02 18:19:16  apsop
# HISTORY: Delete any sha1 checksum files still around.
# HISTORY:
# HISTORY: Revision 1.27  2004/12/22 18:11:57  apsop
# HISTORY: Fix up setting of TIERRELA TIERABSO values so that they have decimal points.
# HISTORY:
# HISTORY: Revision 1.26  2004/12/10 03:36:30  apsop
# HISTORY: Do not write _PNT keywords to tdrss files.
# HISTORY:
# HISTORY: Revision 1.25  2004/12/03 13:35:37  apsop
# HISTORY: Make compatible with test pipeline: st versus sw
# HISTORY:
# HISTORY: Revision 1.24  2004/11/29 15:25:03  apsop
# HISTORY: Fix format of TARG_ID and SEG_NUM keywords
# HISTORY:
# HISTORY: Revision 1.22  2004/11/19 21:46:48  apsop
# HISTORY: New version of xrt2fits.
# HISTORY:
# HISTORY: Revision 1.21  2004/11/16 15:35:58  apsop
# HISTORY: Call swiftversion as a real headas tool.
# HISTORY:
# HISTORY: Revision 1.20  2004/11/09 23:42:29  apsop
# HISTORY: Remove telemetry files before making catalogues.
# HISTORY:
# HISTORY: Revision 1.19  2004/11/02 21:20:20  apsop
# HISTORY: Write DATE* keywords into tapecat file.
# HISTORY:
# HISTORY: Revision 1.18  2004/10/25 14:52:07  apsop
# HISTORY: Write timing keywords to primary hdu. Some general reformatting.
# HISTORY:
# HISTORY: Revision 1.17  2004/10/13 19:53:42  apsop
# HISTORY: Remove checksum file from future_files call.
# HISTORY:
# HISTORY: Revision 1.16  2004/10/12 16:22:21  apsop
# HISTORY: Override and do not make the checksum file. Override verify method and turn 
# HISTORY: off testdata option in fverify to speed this up. Add decimal points where 
# HISTORY: needed in two keyowrd values.
# HISTORY:
# HISTORY: Revision 1.15  2004/09/14 18:54:32  apsop
# HISTORY: Do not write SEQPNUM or (default) OBS_ID into tdrss messages.
# HISTORY:
# HISTORY: Revision 1.14  2004/09/02 00:06:22  apsop
# HISTORY: Store software version and cycle in job.par file.
# HISTORY:
# HISTORY: Revision 1.13  2004/08/17 13:57:11  apsop
# HISTORY: Write keywords for software version and for burst in bat source catalogue.
# HISTORY:
# HISTORY: Revision 1.12  2004/07/06 20:02:57  apsop
# HISTORY: Keep track of calibration files and delete them at the end.
# HISTORY:
# HISTORY: Revision 1.11  2004/06/10 19:57:06  apsop
# HISTORY: Add in writing of OBJ, TARG_ID, SEG_NUM and timing keywords.
# HISTORY:
# HISTORY: Revision 1.10  2004/06/07 17:03:44  apsop
# HISTORY: Switch to using _PNT instead of _NOM for ra, dec and roll parameters.
# HISTORY:
# HISTORY: Revision 1.9  2004/05/28 19:44:06  apsop
# HISTORY: Make sure leading zeroes are included in OBS_ID
# HISTORY:
# HISTORY: Revision 1.8  2004/05/06 20:02:34  dah
# HISTORY: Add version number back into the header comments.
# HISTORY:
# HISTORY: Revision 1.7  2004/04/28 13:51:10  dah
# HISTORY: Change SERQNUM keyword to OBS_ID, and remove leading zeroes from SEQPNUM.
# HISTORY:
# HISTORY: Revision 1.6  2004/04/16 20:21:18  dah
# HISTORY: Begin using embedded history records
# HISTORY:
#
# VERSION: 0.0
#
#
##############################################################################

use Subs::Sub;
use Subs::WrapUp;
use Util::SWCatalogue;
@ISA = ("Subs::WrapUp");
use strict;


#########################################
# constructor
#########################################
sub new {
    my $proto=shift;
    my $self=$proto->SUPER::new();

    $self->{DESCRIPTION}="Doing final wrapup of all swift files";

    $self->{FILES}=();
    $self->{FITS_FILES}=();

    $self->{CHECKSUM_TYPE}="checksum";
    $self->{HTML_CAT_TYPE}="fileinfo";

    return $self;
}

#########################################################################
# add standard keywords to the headers of all the FITS files.
#########################################################################
sub write_standard_keywords {

    my $self=shift;

    my $log      = $self->log();
    my $filename = $self->filename();
    my $jobpar   = $self->jobpar();
    my $procpar  = $self->procpar();

    $log->entry("Writing standard keywords to all FITS files");

    #######################
    # Setup for UTCF value
    #######################
    my $time_file = $filename->get('timedata', 'swift', '', 0);
    my (@utcf_times, @utcf);
    if( -f $time_file ){
      my $time_fits = Util::FITSfile->new($time_file, 'UTCF');
      @utcf_times = $time_fits->cols('TIME')->table();
      @utcf = $time_fits->cols('UTCF')->table();
    }

    my $soft_version = $jobpar->read('softver');
    my $cal_version = $jobpar->read('caldbver');
    my $reprocess = $jobpar->read('reprocess');

    ############################
    # Trigger time, if relevant
    ############################
    my $trigtime = $jobpar->read('burst_time');

    ####################################
    # loop over all FITS files
    ####################################
    my $file;
    foreach $file (Util::SWCatalogue->fits_files() ) {
        $log->entry("Doing file $file.");
      
        my $fits = Util::FITSfile->new($file);
	my $is_tdrss = $file =~ /s[wt][t\d]\d{10}ms/;
	my $is_batevt = $file =~ /s[wt][t\d]\d{10}bev/;
	my $is_eng = $file =~ /s[wt][t\d]\d{10}.*en\.hk$/;
	my $is_att = $file =~ /s[wt][t\d]\d{10}.at\.fits$/;
	my $is_sm = $file =~ /s[wt][t\d]\d{10}bsmcb\.fits$/;

	####################################
	# Which attitude file is being used
	####################################
	my $attflag = '100';

        ##################################
        # loop over HDUs in the FITS file
        ##################################
        my $nhdus = $fits->nhdus();
	unless ($nhdus) {
	  $log->error(2, "Cannot get number of HDUs for FITS file $file.");
	  next;
	}

        my $hdu;
        for($hdu=0; $hdu<$nhdus; $hdu++) {
            $fits->ext($hdu);
	    my $extname = $hdu==0 ? '' : $fits->keyword('EXTNAME');

            ################################
            # write keywords to the file
            ################################
            $fits->begin_many_keywords();

            $fits->keyword('PROCVER', $procpar->read('version'), 
                           'Processing script version' );

	    unless( $is_eng || ($extname && $extname=~/GTI/) ){
	      $fits->keyword('SOFTVER', $soft_version, 'HEASOFT and Swift versions');
	      $fits->keyword('CALDBVER', $cal_version, 'CALDB index versions used');
	    }	    

	    if($hdu==0){
	      $fits->keyword('TIMESYS', 'TT', 'time system');
	      $fits->keyword('MJDREFI', 51910, 'MJD reference day 01 Jan 2001 00:00:00');
	      $fits->keyword('MJDREFF', 7.428703700000000E-04, 
			     'MJD reference (fraction of day) 01 Jan 2001 00:00:00');
	      $fits->keyword('CLOCKAPP', 'F', 'If clock correction are applied (F/T)');
	      $fits->keyword('TIMEUNIT', 's', 'Time unit for timing header keywords');
	      
	      $fits->keyword('REPROC', 'T', 'Is this from a bulk reprocessing run?') if $reprocess eq 'yes';

	      my $att0 = $fits->keyword('ATTFLAG');
	      $attflag = $att0 if $att0;
	    }else{
	      $fits->keyword('TIERRELA', '1.0E-8', '[s/s] relative errors expressed as rate');
	      $fits->keyword('TIERABSO', '1.0', '[s] timing  precision  in seconds');
	    }

	    unless($is_tdrss || $is_sm){
              $fits->keyword('OBS_ID', sprintf("'%011d'", $jobpar->read('sequence')), 
                             'Observation ID' );

              $fits->keyword('SEQPNUM', int($jobpar->read('seqprocnum')), 
                             'Number of times the dataset processed' );
	      $fits->keyword('TARG_ID', int(sprintf("%d",$jobpar->read('target'))),
			     'Target ID');
	      $fits->keyword('SEG_NUM', int(sprintf("%d",$jobpar->read('obs'))),
			     'Segment number');

	      $fits->keyword('OBJECT', $jobpar->read('object'), 'Object name');
	      $fits->keyword('RA_OBJ', $jobpar->read('burst_ra'), '[deg] R.A. Object');
	      $fits->keyword('DEC_OBJ', $jobpar->read('burst_dec'), '[deg] Dec Object');

	      $fits->keyword('RA_PNT', $jobpar->read('ra'), '[deg] RA pointing');
	      $fits->keyword('DEC_PNT', $jobpar->read('dec'), '[deg] Dec pointing');
	      $fits->keyword('PA_PNT', $jobpar->read('roll'), '[deg] Position angle (roll)');

	      $fits->keyword('TRIGTIME', $trigtime, '[s] MET TRIGger Time for Automatic Target') 
		if $trigtime;
	    }

	    if($is_batevt){
	      $fits->keyword('CATSRC',
			     $jobpar->read('burst_cat_src') eq 'yes' ? 'T' : 'F'); 
	    }

	    if( $is_att ){
	      $fits->keyword('-UTCFINIT', ' ');
	      $fits->keyword('-ATTSTATU', ' ');
	    }else{
	      $fits->keyword('ATTFLAG', "'$attflag'", 'Orgin of attitude information');

	      #############################
	      # Determine and set UTCFINIT
	      #############################
	      my $tstart = $fits->keyword('TSTART');
	      unless( $tstart ){
		my $date = $fits->keyword('DATE-OBS');
		if( $date ){
		  my $start = Util::Date->new($date);
		  $tstart = $start->seconds();
		}
	      }

	      if( $tstart && @utcf ){
		my $itime = 0;
		while( $tstart > $utcf_times[$itime] && 
		       $itime < $#utcf_times){ $itime++ }
		$fits->keyword('UTCFINIT', $utcf[$itime], '[s] UTCF at TSTART');
	      }
	    }

            $fits->end_many_keywords();
	  }

      }

} # end of write_standard_keywords method

###########################################################################
# make all FITS catalogs and the HTML file catalog
###########################################################################
sub make_catalogs {
    my $self=shift;

    my $log      = $self->log();
    my $jobpar   = $self->jobpar();
    my $filename=$self->filename();

    $filename->export_to_repository('bcatalog', 'bat');
    $filename->export_to_repository('rcatalog', 'bat');

    $self->cleanup_files();

    ######################################################
    # get names of catalogs, and assume the HTML catalog 
    # has the same files as the first FITS catalog
    ######################################################
    my @fits_catalogs=$filename->catalog_types();
    my $html_cat_is_like=$fits_catalogs[0];

    ##########################################################
    # register the files we have yet to make with the 
    # Catalog class
    ##########################################################
    Util::SWCatalogue->filename($filename);
    Util::SWCatalogue->future_files((map     {$_=>'FITS'} @fits_catalogs),
				    'fileinfo' =>'HTML');

    ###############################################
    # create FITS catalogs
    ###############################################
    my $type;
    foreach $type (@fits_catalogs ) {

        Util::SWCatalogue->new($type)
	                 ->make()
                         ->register_in_parfile();
    }
    
    my $date = Util::Date->new();
    my $attitude = $filename->get('attitude', 's');
    my $attstatus;
    if( -f $attitude ){
      $attstatus = Util::FITSfile->new($attitude, 0)->keyword('ATTSTATU');
    }else{
      $attstatus = '200';
    }

    my $fitscat = Util::FITSfile->new($filename->get('tapecat', 'proc', '', 0));
    foreach my $ext (0,1){
      $fitscat->ext($ext);
      $fitscat->keyword('DATE-OBS', $jobpar->read('obsdate') .'T'. $jobpar->read('obstime'));
      $fitscat->keyword('DATE-END', $jobpar->read('enddate') .'T'. $jobpar->read('endtime'));
      $fitscat->keyword('DATE', $date->date() .'T'. $date->time());
      $fitscat->keyword('ATTSTATU', $attstatus, 'Status of corrected attitude files');
    }



    ########################################
    # create the HTML catalog
    ########################################
    Util::HTMLcatalog->new($html_cat_is_like,'fileinfo')
                     ->make();

} # end of make_catalogs method

########################################################
# Don't make a checksum file.  This is done by the DTS
########################################################
sub make_checksum_file {
    my $self=shift;

    my $log=$self->log();

    $log->entry('Not making a checksum file');

} # end of make_checksums method


#####################################
# run fverify on all the FITS files
#####################################
sub verify {
    my $self=shift;

    my $jobpar   = $self->jobpar();
    my $log=$self->log();

    $log->entry('Running ftverify on all FITS files');

    #######################################################
    # set up fverify. Set testdata='no' to speed things up
    #######################################################
    my $fverify=Util::HEAdas->new('ftverify')
                           ->params({outfile=>'STDOUT',
                                     prhead=>'no',
                                     testdata=>'no'})
                           ->verbose(0);

    ######################################
    # loop over all FITS files
    ######################################
    my $file;
    foreach $file (Util::Catalog->fits_files() ) {
        $fverify->params({infile => $file})
	        ->run();
    }

    #############################################
    # If this is a final processing, and level 2
    # errors are bad
    #############################################
    if( $jobpar->{TIMELIST}->{final} && $jobpar->read('nprocerrors') ){
      $jobpar->set({proc_error=>'yes'});
    }
 
} # end of verify method

sub cleanup_files {
    my $self=shift;

    my $log      = $self->log();
    my $jobpar   = $self->jobpar();
    my $filename=$self->filename();

    $log->entry("Cleaning up temporary files.");

    unlink @{$filename->{CAL_FILES}};
    unlink $filename->any('telemetry');
    unlink $filename->get('tdrsslcatt', 'bat', '', '*');
    ###    unlink $filename->fetch_orbit();

    delete $filename->{INFO}->{b}->{bdetflag}->{repository};
    unlink $filename->get('bdetflag', 'bat', '*', '*');
    delete $filename->{INFO}->{b}->{bgaoff}->{repository};
    unlink $filename->get('bgaoff', 'bat', '*', '*');

    unlink glob '*.sha1';
    unlink glob '*.lock';
    unlink glob '*.flag';
    unlink glob '*.queue';
    unlink glob '*.front_end_id';

    unlink $filename->get('hk', 'proc', 'badb', 0);
}