package Subs::SW0WrapUp;
##############################################################################
#
# DESCRIPTION: This subroutine class handles a number of cleanup tasks which
# DESCRIPTION: need to be done at the end of each processing run,
# DESCRIPTION: including creating output catalog files and verifying
# DESCRIPTION: the product files. Each of these functions is handled
# DESCRIPTION: by a separate method to make it easier to write a sub-class
# DESCRIPTION: which overriseds the default body method.
# DESCRIPTION:
# DESCRIPTION: Each processing script should run this subroutine class
# DESCRIPTION: (or a decendant of it) last.
#
# HISTORY: $Log: SW0WrapUp.pm,v $
# HISTORY: Revision 1.70 2016/11/04 12:56:36 apsop
# HISTORY: Do not remove UTCFINIT from attitude files since it is used by prefilter to accomodate Swift clock drift.
# HISTORY:
# HISTORY: Revision 1.69 2015/09/28 15:28:55 apsop
# HISTORY: Updated to HEASoft 6.17 and applied XRT, UVOT, and clock CALDB patches. Modified XRT event file processing to avoid further processing after an error occurs.
# HISTORY:
# HISTORY: Revision 1.68 2014/02/28 10:44:37 apsop
# HISTORY: Major changes to how ATTFLAG keywords are set: Only for BAT, UVOT,
# HISTORY: and XRT files, and dependent on instrument and whether it's a
# HISTORY: FINAL run. ATTFLAG values are logged. Previously set ATTFLAGs
# HISTORY: are not overwritten; ones that disagree or are unexpected are
# HISTORY: emailed to watchers. SOFTVER keyword is now too long for the
# HISTORY: comment string to be useful.
# HISTORY:
# HISTORY: Revision 1.67 2013/05/30 07:52:51 apsop
# HISTORY: Log exports to BAT repository
# HISTORY:
# HISTORY: Revision 1.66 2012/01/12 06:52:03 apsop
# HISTORY: Changes going to proc3.15.03
# HISTORY:
# HISTORY: 2011-09-29 JRG as apsop: New ximage built, so revert ftverify.
# HISTORY:
# HISTORY: 2011-09-19 JRG as apsop: Instead, call ftverify with errreport='s',
# HISTORY: (only check severe errors), for "*xpc_ex.img" and "*xwt_ex.img"
# HISTORY: files only, so that WCSAXES location is not checked for those
# HISTORY: files. REVERT TO NORMAL VERIFY WHEN XIMAGE IS FIXED TO POSITION
# HISTORY: WCSAXES CORRECTLY.
# HISTORY:
# HISTORY: 2011-09-13 by JRG as apsop: in verify, don't run ftverify on
# HISTORY: xpc_ex.img files b/c they fail due to mislocated WCSAXES
# HISTORY: keyword. NOTE: Should revert to old behavior if that error
# HISTORY: is eventually checked for elsewhere!
# HISTORY:
# HISTORY: Revision 1.65 2011/01/20 17:45:54 apsop
# HISTORY: Added code to determine the value of ATTFlag based on which
# HISTORY: attitude file has been used
# HISTORY:
# HISTORY: Revision 1.64 2008/12/10 13:57:02 apsop
# HISTORY: Previous commit was bogus. This is the fix.
# HISTORY:
# HISTORY: Revision 1.63 2008/12/10 13:51:35 apsop
# HISTORY: Fix problem with multiple copies of GTI extensions.
# HISTORY:
# HISTORY: Revision 1.62 2008/06/23 16:40:34 apsop
# HISTORY: Previous checkin was a mistake. Fix.
# HISTORY:
# HISTORY: Revision 1.61 2008/06/23 14:12:40 apsop
# HISTORY: Check for existence of files before applying aspect correction.
# HISTORY:
# HISTORY: Revision 1.60 2007/11/20 15:55:30 apsop
# HISTORY: Fix typo in calling cleanup_files.
# HISTORY:
# HISTORY: Revision 1.59 2007/09/28 17:14:48 apsop
# HISTORY: Change writing of ATTFLAG keyword so that it is a string rather than an integer.
# HISTORY:
# HISTORY: Revision 1.58 2007/09/11 18:01:26 apsop
# HISTORY: Look for and use ATTFLAG value in primary header of files before using default value.
# HISTORY:
# HISTORY: Revision 1.57 2007/07/25 20:17:57 apsop
# HISTORY: Write REPROC keyword if needed.
# HISTORY:
# HISTORY: Revision 1.56 2007/06/28 20:56:37 apsop
# HISTORY: Don;t write standard keywords to scaled map files.
# HISTORY:
# HISTORY: Revision 1.55 2007/04/02 15:03:47 apsop
# HISTORY: Uncomment cleanup_files(), which was accidentally comment out.
# HISTORY:
# HISTORY: Revision 1.54 2007/04/01 19:11:30 apsop
# HISTORY: Move calculation of version numbers to SwiftSub.pm
# HISTORY:
# HISTORY: Revision 1.53 2007/03/15 21:25:57 apsop
# HISTORY: Remove hard coding of cycle number.
# HISTORY:
# HISTORY: Revision 1.52 2007/02/27 19:36:37 apsop
# HISTORY: Get trigger time from jobpar burst_time param.
# HISTORY:
# HISTORY: Revision 1.51 2007/02/13 15:38:01 apsop
# HISTORY: Remove ATTSTATU keyword from *sat.fits file.
# HISTORY:
# HISTORY: Revision 1.50 2007/02/08 21:54:54 apsop
# HISTORY: Fix bug in getting ATTSTATU value.
# HISTORY:
# HISTORY: Revision 1.49 2007/02/01 15:01:53 apsop
# HISTORY: Do not write UTCFINIT to attitdue files. Write attitude status keyword to files.
# HISTORY:
# HISTORY: Revision 1.48 2006/09/20 20:32:26 apsop
# HISTORY: Remove trailing _S from swift version string, if present.
# HISTORY:
# HISTORY: Revision 1.47 2006/09/10 20:00:38 apsop
# HISTORY: Make parsing of ftools version compatible with ape. Call swiftversion as a generic tool.
# HISTORY:
# HISTORY: Revision 1.46 2006/07/03 01:44:59 apsop
# HISTORY: Fix bug in detecting 900+ segments.
# HISTORY:
# HISTORY: Revision 1.45 2006/06/28 19:05:16 apsop
# HISTORY: For 900 sequences, change the root when searching for ack messages.
# HISOTRY: Set error condition if any level 2 errors occur on the final processing.
# HISTORY:
# HISTORY: Revision 1.44 2006/05/02 13:13:21 apsop
# HISTORY: Do not try and get EXTNAME from hdu 0.
# HISTORY:
# HISTORY: Revision 1.43 2006/04/27 16:02:48 apsop
# HISTORY: Add message for tracking keyword updates in log file.
# HISTORY:
# HISTORY: Revision 1.42 2006/01/12 17:05:42 apsop
# HISTORY: Remove repository bgaoff and bdetflag files in clean up.
# HISTORY:
# HISTORY: Revision 1.41 2006/01/05 23:26:06 apsop
# HISTORY: Implemented merging of BAT detector enable/disable and gain/offset maps
# HISTORY: for the observation and pruned the work-around that saved them with
# HISTORY: index numbers.
# HISTORY:
# HISTORY: Revision 1.40 2005/12/19 16:17:10 apsop
# HISTORY: Export bat catalog files to repository. Remove any leftover bat tdrss
# HISTORY: attitude data files.
# HISTORY:
# HISTORY: Revision 1.39 2005/11/20 20:36:11 apsop
# HISTORY: Fix bug in renaming of bgaoff type files.
# HISTORY:
# HISTORY: Revision 1.37 2005/11/08 19:22:28 apsop
# HISTORY: Populate the TIMELIST and DATALIST hashes. Used to be an SWCheckInput.
# HISTORY:
# HISTORY: Revision 1.36 2005/10/28 18:16:49 apsop
# HISTORY: Fix bug in renaming of gain/off and detector flag files.
# HISTORY:
# HISTORY: Revision 1.35 2005/10/26 17:30:08 apsop
# HISTORY: New subroutine for cleaning up files that will not be catalogued. Quick fix to
# HISTORY: rename gain/offset and det enable bat files.
# HISTORY:
# HISTORY: Revision 1.34 2005/08/31 16:05:23 apsop
# HISTORY: Add TRIGTIME keyword to all non-trdss products if available.
# HISTORY:
# HISTORY: Revision 1.33 2005/07/29 14:20:10 apsop
# HISTORY: Fix bug in running local version of fverify. Switch to ftverify.
# HISTORY:
# HISTORY: Revision 1.32 2005/04/19 15:34:25 apsop
# HISTORY: Add in CALDBVER keyword. Fix up some keyword comments.
# HISTORY:
# HISTORY: Revision 1.31 2005/03/18 20:17:48 apsop
# HISTORY: Put orbit files back into the distribution.
# HISTORY:
# HISTORY: Revision 1.30 2005/03/17 16:50:06 apsop
# HISTORY: Remove TLE file from distribution.
# HISTORY:
# HISTORY: Revision 1.29 2005/03/15 19:54:54 apsop
# HISTORY: Add in code taken from StartEndTimes for writing UTCFINIT keyword.
# HISTORY:
# HISTORY:
# HISTORY: Revision 1.28 2005/03/02 18:19:16 apsop
# HISTORY: Delete any sha1 checksum files still around.
# HISTORY:
# HISTORY: Revision 1.27 2004/12/22 18:11:57 apsop
# HISTORY: Fix up setting of TIERRELA TIERABSO values so that they have decimal points.
# HISTORY:
# HISTORY: Revision 1.26 2004/12/10 03:36:30 apsop
# HISTORY: Do not write _PNT keywords to tdrss files.
# HISTORY:
# HISTORY: Revision 1.25 2004/12/03 13:35:37 apsop
# HISTORY: Make compatible with test pipeline: st versus sw
# HISTORY:
# HISTORY: Revision 1.24 2004/11/29 15:25:03 apsop
# HISTORY: Fix format of TARG_ID and SEG_NUM keywords
# HISTORY:
# HISTORY: Revision 1.22 2004/11/19 21:46:48 apsop
# HISTORY: New version of xrt2fits.
# HISTORY:
# HISTORY: Revision 1.21 2004/11/16 15:35:58 apsop
# HISTORY: Call swiftversion as a real headas tool.
# HISTORY:
# HISTORY: Revision 1.20 2004/11/09 23:42:29 apsop
# HISTORY: Remove telemetry files before making catalogues.
# HISTORY:
# HISTORY: Revision 1.19 2004/11/02 21:20:20 apsop
# HISTORY: Write DATE* keywords into tapecat file.
# HISTORY:
# HISTORY: Revision 1.18 2004/10/25 14:52:07 apsop
# HISTORY: Write timing keywords to primary hdu. Some general reformatting.
# HISTORY:
# HISTORY: Revision 1.17 2004/10/13 19:53:42 apsop
# HISTORY: Remove checksum file from future_files call.
# HISTORY:
# HISTORY: Revision 1.16 2004/10/12 16:22:21 apsop
# HISTORY: Override and do not make the checksum file. Override verify method and turn
# HISTORY: off testdata option in fverify to speed this up. Add decimal points where
# HISTORY: needed in two keyowrd values.
# HISTORY:
# HISTORY: Revision 1.15 2004/09/14 18:54:32 apsop
# HISTORY: Do not write SEQPNUM or (default) OBS_ID into tdrss messages.
# HISTORY:
# HISTORY: Revision 1.14 2004/09/02 00:06:22 apsop
# HISTORY: Store software version and cycle in job.par file.
# HISTORY:
# HISTORY: Revision 1.13 2004/08/17 13:57:11 apsop
# HISTORY: Write keywords for software version and for burst in bat source catalogue.
# HISTORY:
# HISTORY: Revision 1.12 2004/07/06 20:02:57 apsop
# HISTORY: Keep track of calibration files and delete them at the end.
# HISTORY:
# HISTORY: Revision 1.11 2004/06/10 19:57:06 apsop
# HISTORY: Add in writing of OBJ, TARG_ID, SEG_NUM and timing keywords.
# HISTORY:
# HISTORY: Revision 1.10 2004/06/07 17:03:44 apsop
# HISTORY: Switch to using _PNT instead of _NOM for ra, dec and roll parameters.
# HISTORY:
# HISTORY: Revision 1.9 2004/05/28 19:44:06 apsop
# HISTORY: Make sure leading zeroes are included in OBS_ID
# HISTORY:
# HISTORY: Revision 1.8 2004/05/06 20:02:34 dah
# HISTORY: Add version number back into the header comments.
# HISTORY:
# HISTORY: Revision 1.7 2004/04/28 13:51:10 dah
# HISTORY: Change SERQNUM keyword to OBS_ID, and remove leading zeroes from SEQPNUM.
# HISTORY:
# HISTORY: Revision 1.6 2004/04/16 20:21:18 dah
# HISTORY: Begin using embedded history records
# HISTORY:
#
# VERSION: $Revision: 1.70 $
#
##############################################################################
use Subs::Sub;
use Subs::WrapUp;
use Util::SWCatalogue;
use Util::Email;
@ISA = ("Subs::WrapUp");
use strict;
#########################################
# constructor
#########################################
sub new {
my $proto=shift;
my $self=$proto->SUPER::new();
$self->{DESCRIPTION}="Doing final wrapup of all swift files";
$self->{FILES}=();
$self->{FITS_FILES}=();
$self->{CHECKSUM_TYPE}="checksum";
$self->{HTML_CAT_TYPE}="fileinfo";
return $self;
}
#########################################################################
# add standard keywords to the headers of all the FITS files.
#########################################################################
sub write_standard_keywords {
my $self = shift;
my $log = $self->log();
my $filename = $self->filename();
my $jobpar = $self->jobpar();
my $procpar = $self->procpar();
$log->entry("Writing standard keywords to all FITS files");
Util::SW0Filename::write_standard_keywords($self,
{ ATTFLAG => 1 },
Util::SWCatalogue->fits_files());
} # end of write_standard_keywords method
###########################################################################
# make all FITS catalogs and the HTML file catalog
###########################################################################
sub make_catalogs {
my $self=shift;
my $log = $self->log();
my $jobpar = $self->jobpar();
my $filename=$self->filename();
$log->entry("SW0WrapUp.pm exporting to the shared repositories " .
"bcatalog and rcatalog");
$filename->export_to_repository('bcatalog', 'bat');
$filename->export_to_repository('rcatalog', 'bat');
$self->cleanup_files();
######################################################
# get names of catalogs, and assume the HTML catalog
# has the same files as the first FITS catalog
######################################################
my @fits_catalogs=$filename->catalog_types();
my $html_cat_is_like=$fits_catalogs[0];
##########################################################
# register the files we have yet to make with the
# Catalog class
##########################################################
Util::SWCatalogue->filename($filename);
Util::SWCatalogue->future_files((map {$_=>'FITS'} @fits_catalogs),
'fileinfo' =>'HTML');
###############################################
# create FITS catalogs
###############################################
my $type;
foreach $type (@fits_catalogs ) {
Util::SWCatalogue->new($type)
->make()
->register_in_parfile();
}
my $date = Util::Date->new();
my $attitude = $filename->get('attitude', 's');
my $attstatus;
if( -f $attitude ){
$attstatus = Util::FITSfile->new($attitude, 0)->keyword('ATTSTATU');
}else{
$attstatus = '200';
}
my $fitscat = Util::FITSfile->new($filename->get('tapecat', 'proc', '', 0));
foreach my $ext (0,1){
$fitscat->ext($ext);
$fitscat->keyword('DATE-OBS', $jobpar->read('obsdate') .'T'. $jobpar->read('obstime'));
$fitscat->keyword('DATE-END', $jobpar->read('enddate') .'T'. $jobpar->read('endtime'));
$fitscat->keyword('DATE', $date->date() .'T'. $date->time());
$fitscat->keyword('ATTSTATU', $attstatus, 'Status of corrected attitude files');
}
########################################
# create the HTML catalog
########################################
Util::HTMLcatalog->new($html_cat_is_like,'fileinfo')
->make();
} # end of make_catalogs method
########################################################
# Don't make a checksum file. This is done by the DTS
########################################################
sub make_checksum_file {
my $self=shift;
my $log=$self->log();
$log->entry('Not making a checksum file');
} # end of make_checksums method
#####################################
# run fverify on all the FITS files
#####################################
sub verify {
my $self=shift;
my $jobpar = $self->jobpar();
my $log=$self->log();
$log->entry('Running ftverify on all FITS files');
#######################################################
# set up fverify. Set testdata='no' to speed things up
#######################################################
my $fverify=Util::HEAdas->new('ftverify')
->params({outfile=>'STDOUT',
prhead=>'no',
testdata=>'no'})
->verbose(0);
######################################
# loop over all FITS files
######################################
my $file;
foreach $file (Util::Catalog->fits_files() ) {
$fverify->params({infile => $file})
->run();
# TO WORKAROUND XIMAGE WCSAXES BUG, COMMENT OUT ABOVE fverify
# AND UNCOMMENT THIS BLOCK
# # Only report severe errors for xpc_ex.img and xwt_ex.img files,
# # because they have the WCSAXES keyword in the wrong location.
# if ( ($file =~ /xpc_ex\.img/) or ($file =~ /xwt_ex\.img/) ) {
# $fverify->params({infile => $file,
# errreport=>'s'})
# ->run();
# } else {
# $fverify->params({infile => $file})
# ->run();
# }
}
#############################################
# If this is a final processing, and level 2
# errors are bad
#############################################
if( $jobpar->{TIMELIST}->{final} && $jobpar->read('nprocerrors') ){
$jobpar->set({proc_error=>'yes'});
}
} # end of verify method
sub cleanup_files {
my $self=shift;
my $log = $self->log();
my $jobpar = $self->jobpar();
my $filename=$self->filename();
$log->entry("Cleaning up temporary files.");
unlink @{$filename->{CAL_FILES}};
unlink $filename->any('telemetry');
unlink $filename->get('tdrsslcatt', 'bat', '', '*');
### unlink $filename->fetch_orbit();
delete $filename->{INFO}->{b}->{bdetflag}->{repository};
unlink $filename->get('bdetflag', 'bat', '*', '*');
delete $filename->{INFO}->{b}->{bgaoff}->{repository};
unlink $filename->get('bgaoff', 'bat', '*', '*');
unlink glob '*.sha1';
unlink glob '*.lock';
unlink glob '*.flag';
unlink glob '*.queue';
unlink glob '*.front_end_id';
unlink $filename->get('hk', 'proc', 'badb', 0);
}