Subs::SW0WrapUp (version 0.0)


package Subs::SW0WrapUp;
##############################################################################
#
# DESCRIPTION: This subroutine class handles a number of cleanup tasks which 
# DESCRIPTION: need to be done at the end of each processing run,
# DESCRIPTION: including creating output catalog files and verifying
# DESCRIPTION: the product files. Each of these functions is handled
# DESCRIPTION: by a separate method to make it easier to write a sub-class
# DESCRIPTION: which overriseds the default body method.
# DESCRIPTION: 
# DESCRIPTION: Each processing script should run this subroutine class
# DESCRIPTION: (or a decendant of it) last. 
#
# HISTORY: 
# HISTORY: $Log: SW0WrapUp.pm,v $
# HISTORY: Revision 1.42  2006/01/12 17:05:42  apsop
# HISTORY: Remove repository bgaoff and bdetflag files in clean up.
# HISTORY:
# HISTORY: Revision 1.41  2006/01/05 23:26:06  apsop
# HISTORY: Implemented merging of BAT detector enable/disable and gain/offset maps
# HISTORY: for the observation and pruned the work-around that saved them with
# HISTORY: index numbers.
# HISTORY:
# HISTORY: Revision 1.40  2005/12/19 16:17:10  apsop
# HISTORY: Export bat catalog files to repository.  Remove any leftover bat tdrss attitude data files.
# HISTORY:
# HISTORY: Revision 1.39  2005/11/20 20:36:11  apsop
# HISTORY: Fix bug in renaming of bgaoff type files.
# HISTORY:
# HISTORY: Revision 1.37  2005/11/08 19:22:28  apsop
# HISTORY: Populate the TIMELIST and DATALIST hashes. Used to be an SWCheckInput.
# HISTORY:
# HISTORY: Revision 1.36  2005/10/28 18:16:49  apsop
# HISTORY: Fix bug in renaming of gain/off and detector flag files.
# HISTORY:
# HISTORY: Revision 1.35  2005/10/26 17:30:08  apsop
# HISTORY: New subroutine for cleaning up files that will not be catalogued. Quick fix to rename gain/offset and det enable bat files.
# HISTORY:
# HISTORY: Revision 1.34  2005/08/31 16:05:23  apsop
# HISTORY: Add TRIGTIME keyword to all non-trdss products if available.
# HISTORY:
# HISTORY: Revision 1.33  2005/07/29 14:20:10  apsop
# HISTORY: Fix bug in running local version of fverify. Switch to ftverify.
# HISTORY:
# HISTORY: Revision 1.32  2005/04/19 15:34:25  apsop
# HISTORY: Add in CALDBVER keyword. Fix up some keyword comments.
# HISTORY:
# HISTORY: Revision 1.31  2005/03/18 20:17:48  apsop
# HISTORY: Put orbit files back into the distribution.
# HISTORY:
# HISTORY: Revision 1.30  2005/03/17 16:50:06  apsop
# HISTORY: Remove TLE file from distribution.
# HISTORY:
# HISTORY: Revision 1.29  2005/03/15 19:54:54  apsop
# HISTORY: Add in code taken from StartEndTimes for writing UTCFINIT keyword.
# HISTORY:
# HISTORY:
# HISTORY: Revision 1.28  2005/03/02 18:19:16  apsop
# HISTORY: Delete any sha1 checksum files still around.
# HISTORY:
# HISTORY: Revision 1.27  2004/12/22 18:11:57  apsop
# HISTORY: Fix up setting of TIERRELA TIERABSO values so that they have decimal points.
# HISTORY:
# HISTORY: Revision 1.26  2004/12/10 03:36:30  apsop
# HISTORY: Do not write _PNT keywords to tdrss files.
# HISTORY:
# HISTORY: Revision 1.25  2004/12/03 13:35:37  apsop
# HISTORY: Make compatible with test pipeline: st versus sw
# HISTORY:
# HISTORY: Revision 1.24  2004/11/29 15:25:03  apsop
# HISTORY: Fix format of TARG_ID and SEG_NUM keywords
# HISTORY:
# HISTORY: Revision 1.22  2004/11/19 21:46:48  apsop
# HISTORY: New version of xrt2fits.
# HISTORY:
# HISTORY: Revision 1.21  2004/11/16 15:35:58  apsop
# HISTORY: Call swiftversion as a real headas tool.
# HISTORY:
# HISTORY: Revision 1.20  2004/11/09 23:42:29  apsop
# HISTORY: Remove telemetry files before making catalogues.
# HISTORY:
# HISTORY: Revision 1.19  2004/11/02 21:20:20  apsop
# HISTORY: Write DATE* keywords into tapecat file.
# HISTORY:
# HISTORY: Revision 1.18  2004/10/25 14:52:07  apsop
# HISTORY: Write timing keywords to primary hdu. Some general reformatting.
# HISTORY:
# HISTORY: Revision 1.17  2004/10/13 19:53:42  apsop
# HISTORY: Remove checksum file from future_files call.
# HISTORY:
# HISTORY: Revision 1.16  2004/10/12 16:22:21  apsop
# HISTORY: Override and do not make the checksum file. Override verify method and turn off testdata option in fverify to speed this up. Add decimal points where needed in two keyowrd values.
# HISTORY:
# HISTORY: Revision 1.15  2004/09/14 18:54:32  apsop
# HISTORY: Do not write SEQPNUM or (default) OBS_ID into tdrss messages.
# HISTORY:
# HISTORY: Revision 1.14  2004/09/02 00:06:22  apsop
# HISTORY: Store software version and cycle in job.par file.
# HISTORY:
# HISTORY: Revision 1.13  2004/08/17 13:57:11  apsop
# HISTORY: Write keywords for software version and for burst in bat source catalogue.
# HISTORY:
# HISTORY: Revision 1.12  2004/07/06 20:02:57  apsop
# HISTORY: Keep track of calibration files and delete them at the end.
# HISTORY:
# HISTORY: Revision 1.11  2004/06/10 19:57:06  apsop
# HISTORY: Add in writing of OBJ, TARG_ID, SEG_NUM and timing keywords.
# HISTORY:
# HISTORY: Revision 1.10  2004/06/07 17:03:44  apsop
# HISTORY: Switch to using _PNT instead of _NOM for ra, dec and roll parameters.
# HISTORY:
# HISTORY: Revision 1.9  2004/05/28 19:44:06  apsop
# HISTORY: Make sure leading zeroes are included in OBS_ID
# HISTORY:
# HISTORY: Revision 1.8  2004/05/06 20:02:34  dah
# HISTORY: Add version number back into the header comments.
# HISTORY:
# HISTORY: Revision 1.7  2004/04/28 13:51:10  dah
# HISTORY: Change SERQNUM keyword to OBS_ID, and remove leading zeroes from SEQPNUM.
# HISTORY:
# HISTORY: Revision 1.6  2004/04/16 20:21:18  dah
# HISTORY: Begin using embedded history records
# HISTORY:
#
# VERSION: 0.0
#
#
##############################################################################

use Subs::Sub;
use Subs::WrapUp;
use Util::SWCatalogue;
@ISA = ("Subs::WrapUp");
use strict;


#########################################
# constructor
#########################################
sub new {
    my $proto=shift;
    my $self=$proto->SUPER::new();

    $self->{DESCRIPTION}="Doing final wrapup of all swift files";

    $self->{FILES}=();
    $self->{FITS_FILES}=();

    $self->{CHECKSUM_TYPE}="checksum";
    $self->{HTML_CAT_TYPE}="fileinfo";

    return $self;
}

#########################################################################
# add standard keywords to the headers of all the FITS files.
#########################################################################
sub write_standard_keywords {

    my $self=shift;

    my $log      = $self->log();
    my $filename = $self->filename();
    my $jobpar   = $self->jobpar();
    my $procpar  = $self->procpar();

    $log->entry("Writing standard keywords to all FITS files");

    #######################
    # Setup for UTCF value
    #######################
    my $time_file = $filename->get('timedata', 'swift', '', 0);
    my (@utcf_times, @utcf);
    if( -f $time_file ){
      my $time_fits = Util::FITSfile->new($time_file, 'UTCF');
      @utcf_times = $time_fits->cols('TIME')->table();
      @utcf = $time_fits->cols('UTCF')->table();
    }

    ####################################
    # Get software versions
    ####################################
    my $fversion;
    open VER, $procpar->read('ftools').'/syspfiles/ftools.par';
    while (<VER>){
      $fversion = $1 if /version,s,h,"([^"]*)"/;
    }
    close VER;

    my $sversion = Util::HEAdas->new('swiftversion')
                               ->is_script(1)
                               ->run();

    my $soft_version = 'Hea_' . $fversion . '_' . $sversion->stdout();
    chomp $soft_version;

    $jobpar->set({softver => $soft_version,
		  cycle   => 1});

    #####################
    # Get caldb versions
    #####################
    my $caldb = $procpar->read('caldb');
    my @cal_vers;
    foreach my $inst qw(bat uvota xrt mis){
      my $fits = Util::FITSfile->new("$caldb/data/swift/$inst/caldb.indx");
      my $ckey = $fits->keyword('CALDBVER');
      $ckey = '?' unless $ckey;
      push @cal_vers, substr($inst, 0, 1) . $ckey;
    }
    my $cal_version = join('_', @cal_vers);

    ############################
    # Trigger time, if relevant
    ############################
    my $root = $filename->sequence_specific();
    my @ack_files = $filename->get('tdmess', 'bat', 'ce', '*');
    @ack_files = grep /^${root}/, @ack_files;   
    my $trigtime;
    if( @ack_files ){
      my $batack = Util::FITSfile->new($ack_files[0], 0);
      $trigtime = $batack->keyword('TRIGTIME');
    }

    ####################################
    # loop over all FITS files
    ####################################
    my $file;
    foreach $file (Util::SWCatalogue->fits_files() ) {

        my $fits = Util::FITSfile->new($file);
	my $is_tdrss = $file =~ /s[wt][t\d]\d{10}ms/;
	my $is_batevt = $file =~ /s[wt][t\d]\d{10}bev/;
	my $is_eng = $file =~ /s[wt][t\d]\d{10}.*en\.hk$/;

        ##################################
        # loop over HDUs in the FITS file
        ##################################
        my $nhdus = $fits->nhdus();
	unless ($nhdus) {
	  $self->log()->error(2, "Cannot get number of HDUs for FITS file $file.");
	  next;
	}

        my $hdu;
        for($hdu=0; $hdu<$nhdus; $hdu++) {
            $fits->ext($hdu);
	    my $extname = $fits->keyword('EXTNAME');

            ################################
            # write keywords to the file
            ################################
            $fits->begin_many_keywords();

            $fits->keyword('PROCVER', $procpar->read('version'), 
                           'Processing script version' );

	    unless( $is_eng || ($extname && $extname=~/GTI/) ){
	      $fits->keyword('SOFTVER', $soft_version, 'HEASOFT and Swift versions');
	      $fits->keyword('CALDBVER', $cal_version, 'CALDB index versions used');
	    }	    

	    if($hdu==0){
	      $fits->keyword('TIMESYS', 'TT', 'time system');
	      $fits->keyword('MJDREFI', 51910, 'MJD reference day 01 Jan 2001 00:00:00');
	      $fits->keyword('MJDREFF', 7.428703700000000E-04, 
			     'MJD reference (fraction of day) 01 Jan 2001 00:00:00');
	      $fits->keyword('CLOCKAPP', 'F', 'If clock correction are applied (F/T)');
	      $fits->keyword('TIMEUNIT', 's', 'Time unit for timing header keywords');
	    }else{
	      $fits->keyword('TIERRELA', '1.0E-8', '[s/s] relative errors expressed as rate');
	      $fits->keyword('TIERABSO', '1.0', '[s] timing  precision  in seconds');
	    }

	    unless($is_tdrss){
              $fits->keyword('OBS_ID', sprintf("'%011d'", $jobpar->read('sequence')), 
                             'Observation ID' );

              $fits->keyword('SEQPNUM', int($jobpar->read('seqprocnum')), 
                             'Number of times the dataset processed' );

	      $fits->keyword('TARG_ID', int(sprintf("%d",$jobpar->read('target'))),
			     'Target ID');
	      $fits->keyword('SEG_NUM', int(sprintf("%d",$jobpar->read('obs'))),
			     'Segment number');

	      $fits->keyword('OBJECT', $jobpar->read('object'), 'Object name');
	      $fits->keyword('RA_OBJ', $jobpar->read('burst_ra'), '[deg] R.A. Object');
	      $fits->keyword('DEC_OBJ', $jobpar->read('burst_dec'), '[deg] Dec Object');

	      $fits->keyword('RA_PNT', $jobpar->read('ra'), '[deg] RA pointing');
	      $fits->keyword('DEC_PNT', $jobpar->read('dec'), '[deg] Dec pointing');
	      $fits->keyword('PA_PNT', $jobpar->read('roll'), '[deg] Position angle (roll)');

	      $fits->keyword('TRIGTIME', $trigtime, '[s] MET TRIGger Time for Automatic Target') 
		if $trigtime;
	    }

	    if($is_batevt){
	      $fits->keyword('CATSRC',
			     $jobpar->read('burst_cat_src') eq 'yes' ? 'T' : 'F'); 
	    }

            #############################
            # Determine and set UTCFINIT
            #############################
            my $tstart = $fits->keyword('TSTART');
            unless( $tstart ){
              my $date = $fits->keyword('DATE-OBS');
              if( $date ){
                my $start = Util::Date->new($date);
                $tstart = $start->seconds();
              }
            }

            if( $tstart && @utcf ){
	      my $itime = 0;
	      while( $tstart > $utcf_times[$itime] && 
                     $itime < $#utcf_times){ $itime++ }
	      $fits->keyword('UTCFINIT', $utcf[$itime], '[s] UTCF at TSTART');
            }

            $fits->end_many_keywords();
        }

    }

} # end of write_standard_keywords method

###########################################################################
# make all FITS catalogs and the HTML file catalog
###########################################################################
sub make_catalogs {
    my $self=shift;

    my $log      = $self->log();
    my $jobpar   = $self->jobpar();
    my $filename=$self->filename();

    $filename->export_to_repository('bcatalog', 'bat');
    $filename->export_to_repository('rcatalog', 'bat');

    $self->cleanup_files();

    ######################################################
    # get names of catalogs, and assume the HTML catalog 
    # has the same files as the first FITS catalog
    ######################################################
    my @fits_catalogs=$filename->catalog_types();
    my $html_cat_is_like=$fits_catalogs[0];

    ##########################################################
    # register the files we have yet to make with the 
    # Catalog class
    ##########################################################
    Util::SWCatalogue->filename($filename);
    Util::SWCatalogue->future_files((map     {$_=>'FITS'} @fits_catalogs),
				    'fileinfo' =>'HTML');

    ###############################################
    # create FITS catalogs
    ###############################################
    my $type;
    foreach $type (@fits_catalogs ) {

        Util::SWCatalogue->new($type)
	                 ->make()
                         ->register_in_parfile();
    }
    
    my $date = Util::Date->new();
    my $fitscat = Util::FITSfile->new($filename->get('tapecat', 'proc', '', 0));
    foreach my $ext (0,1){
      $fitscat->ext($ext);
      $fitscat->keyword('DATE-OBS', $jobpar->read('obsdate') .'T'. $jobpar->read('obstime'));
      $fitscat->keyword('DATE-END', $jobpar->read('enddate') .'T'. $jobpar->read('endtime'));
      $fitscat->keyword('DATE', $date->date() .'T'. $date->time());
    }



    ########################################
    # create the HTML catalog
    ########################################
    Util::HTMLcatalog->new($html_cat_is_like,'fileinfo')
                     ->make();

} # end of make_catalogs method

########################################################
# Don't make a checksum file.  This is done by the DTS
########################################################
sub make_checksum_file {
    my $self=shift;

    my $log=$self->log();

    $log->entry('Not making a checksum file');

} # end of make_checksums method


#####################################
# run fverify on all the FITS files
#####################################
sub verify {
    my $self=shift;

    my $log=$self->log();

    $log->entry('Running ftverify on all FITS files');

    #######################################################
    # set up fverify. Set testdata='no' to speed things up
    #######################################################
    my $fverify=Util::HEAdas->new('ftverify')
                           ->params({outfile=>'STDOUT',
                                     prhead=>'no',
                                     testdata=>'no'})
                           ->verbose(0);

    ######################################
    # loop over all FITS files
    ######################################
    my $file;
    foreach $file (Util::Catalog->fits_files() ) {
        $fverify->params({infile => $file})
	        ->run();
    }

 
} # end of verify method

sub cleanup_files {
    my $self=shift;

    my $log      = $self->log();
    my $jobpar   = $self->jobpar();
    my $filename=$self->filename();

    $log->entry("Cleaning up temporary files.");

    unlink @{$filename->{CAL_FILES}};
    unlink $filename->any('telemetry');
    unlink $filename->get('tdrsslcatt', 'bat', '', '*');
    ###    unlink $filename->fetch_orbit();

    delete $filename->{INFO}->{b}->{bdetflag}->{repository};
    unlink $filename->get('bdetflag', 'bat', '*', '*');
    delete $filename->{INFO}->{b}->{bgaoff}->{repository};
    unlink $filename->get('bgaoff', 'bat', '*', '*');

    unlink glob '*.sha1';
    unlink glob '*.lock';
    unlink glob '*.flag';
    unlink glob '*.queue';
    unlink glob '*.front_end_id';

    unlink $filename->get('hk', 'proc', 'badb', 0);
}