package Subs::XrtEvents; ############################################################################## # # DESCRIPTION: This subroutine runs the XRT tools delivered in build 1 # # HISTORY: # HISTORY: $Log: XrtEvents.pm,v $ # HISTORY: Revision 1.43 2015/09/28 15:28:55 apsop # HISTORY: Updated to HEASoft 6.17 and applied XRT, UVOT, and clock CALDB patches. Modified XRT event file processing to avoid further processing after an error occurs. # HISTORY: # HISTORY: Revision 1.42 2014/08/28 10:20:03 apsop # HISTORY: Not calling xrtpcbias for PC Slew and Settling event files can # HISTORY: sometimes cause other problems later, because the files then do # HISTORY: not have PHAS0 columns or [BIASDIFF] HDUs. (Eg., Filter.pm # HISTORY: xrt_cal_events may not be able merge the [EVENTS] HDUs when # HISTORY: creating the xpc*cb_uf file.) So, go back to calling xrtpcbias # HISTORY: (with srcdetx=srcdety=300) for these files. # HISTORY: # HISTORY: Revision 1.41 2014/08/14 10:37:10 apsop # HISTORY: Skip xrtpcbias for PC Slew and Settling events. # HISTORY: # HISTORY: Revision 1.40 2014/02/27 10:01:40 apsop # HISTORY: Don't use uat attitude file, at request of XRT Team. # HISTORY: # HISTORY: Revision 1.39 2013/05/30 07:47:33 apsop # HISTORY: Log when using srcdetx/y=300 in xrtpcbias/xrtpccorr # HISTORY: # HISTORY: Revision 1.38 2013/05/24 02:42:53 apsop # HISTORY: Add srcdetx=300, srcdety=300 to xrtpcbias call when # HISTORY: spacecraft is in SLEW or SETTLING mode. # HISTORY: # HISTORY: Revision 1.37 2011/01/20 18:52:43 apsop # HISTORY: Added code to get ra and dec for source GRB in the same way as done # HISTORY: in other place. # HISTORY: Added code to use UVOT attitude file if available # HISTORY: # HISTORY: Revision 1.36 2007/09/11 17:56:32 apsop # HISTORY: Add xrtpcbias task, and update parameters for build 21.1. # HISTORY: # HISTORY: Revision 1.35 2007/01/31 16:46:33 apsop # HISTORY: Change to xrtwtcorr parameters. # HISTORY: # HISTORY: Revision 1.34 2006/10/06 18:15:14 apsop # HISTORY: Update keywords in header file after running xrthkproc. # HISTORY: # HISTORY: Revision 1.33 2006/09/25 13:30:57 apsop # HISTORY: Remove code for xrthkproc workaround. # HISTORY: # HISTORY: Revision 1.32 2006/09/20 20:35:32 apsop # HISTORY: Temporary fix, which removes manual mode zero rows from the xrt header file before processing. # HISTORY: # HISTORY: Revision 1.31 2006/07/03 19:33:20 apsop # HISTORY: Delete reconstructed event files if they are empty after filtering. # HISTORY: # HISTORY: Revision 1.30 2006/05/17 19:07:55 apsop # HISTORY: Fix up creation of uf ufre event files for wt mode. # HISTORY: # HISTORY: Revision 1.29 2006/05/10 15:16:31 apsop # HISTORY: Change transition from unfiltered to reconstructed event list in a window timing mode. # HISTORY: # HISTORY: Revision 1.28 2006/04/26 20:44:21 apsop # HISTORY: Change ordering of when xrtflagpix is called. Trap case where # HISTORY: EVTPHA column does not exist in lt and wt files. # HISTORY: # HISTORY: Revision 1.27 2005/11/08 20:08:37 apsop # HISTORY: Add in xrtwtcorr task, and update a couple of parameters. # HISTORY: # HISTORY: Revision 1.25 2005/10/04 13:35:27 apsop # HISTORY: Pass XRT housekeeping trailer packets to xrttimetag. # HISTORY: # HISTORY: Revision 1.24 2005/05/26 13:00:06 apsop # HISTORY: Param change for xrtpdcorr due to new version of xrt2fits. # HISTORY: # HISTORY: Revision 1.23 2005/05/02 13:49:34 apsop # HISTORY: Fix bug in name of column that is deleted from reconstructed files (EVTPHA). # HISTORY: # HISTORY: Revision 1.22 2005/04/29 20:33:04 apsop # HISTORY: Bug fixes for previous version in use of pifile.tmp. # HISTORY: # HISTORY: Revision 1.21 2005/04/29 15:52:00 apsop # HISTORY: Several parameter changes. Remove EVPHA columns after processing. # HISTORY: # HISTORY: Revision 1.20 2005/04/19 15:10:10 apsop # HISTORY: Changes for build14 tasks, many now use header file. Stop if not header file. # HISTORY: # HISTORY: Revision 1.19 2005/04/06 15:46:19 apsop # HISTORY: Change to using CALDB for cal parameters. # HISTORY: # HISTORY: Revision 1.18 2005/02/25 20:29:40 apsop # HISTORY: Explicitly set method parameter in xrtpdcorr to SG. # HISTORY: # HISTORY: Revision 1.17 2004/11/16 15:32:09 apsop # HISTORY: Changes to fix handling of window timing events. # HISTORY: # HISTORY: Revision 1.16 2004/11/09 23:54:56 apsop # HISTORY: Rework order of tasks to make proper split between level 1 and level1a. # HISTORY: # HISTORY: Revision 1.15 2004/11/02 21:25:16 apsop # HISTORY: Rearrange calling sequence and production of level 1a files. # HISTORY: # HISTORY: Revision 1.14 2004/08/30 13:18:37 apsop # HISTORY: Changes for build 9. NONE does not work, rearrangement for bias info. # HISTORY: # HISTORY: Revision 1.13 2004/07/23 16:04:13 apsop # HISTORY: Fix stupid bug in _fetching_ attitude file. Should be _getting_. # HISTORY: # HISTORY: Revision 1.12 2004/07/19 16:04:55 apsop # HISTORY: Fix bug in using fetch_cal() instead of fetch() to get attitude file. # HISTORY: # HISTORY: Revision 1.11 2004/07/11 20:43:51 apsop # HISTORY: Turn chatter down on timetag and hkproc in order to reduce output to reasonable level. # HISTORY: # HISTORY: Revision 1.10 2004/06/29 14:35:33 apsop # HISTORY: Substantial changes to support build 8. # HISTORY: # HISTORY: Revision 1.9 2004/05/06 20:02:34 dah # HISTORY: Add version number back into the header comments. # HISTORY: # HISTORY: Revision 1.8 2004/05/04 16:31:47 dah # HISTORY: Test for presence of header file before processing it. # HISTORY: # HISTORY: Revision 1.7 2004/04/28 13:56:43 dah # HISTORY: Save reconstructed event lists in seperate files. Change order in which xrtpdcorr # HISTORY: is called. Fix bug in calling xrthkproc. # HISTORY: # HISTORY: Revision 1.6 2004/04/16 20:21:18 dah # HISTORY: Begin using embedded history records # HISTORY: # # VERSION: $Revision: 1.43 $ # # ############################################################################## use Subs::Sub; use Util::SwiftTags; @ISA = ("Subs::Sub"); use strict; sub new { my $proto=shift; my $self=$proto->SUPER::new(); $self->{DESCRIPTION}="Running XRT tasks for event list processing"; return $self; } ################## # METHODS: ################## sub body { my $self=shift; my $log =$self->log(); my $filename=$self->filename(); my $procpar =$self->procpar(); my $jobpar =$self->jobpar(); my ($srcra, $srcdec) = $self->Subs::XrtProducts::GetRaDec(); if (!defined $srcra or !defined $srcdec) { $log->error(1, "Unable to XrtEvents since coordinates are not fully defined"); return; } #################################### # Get lists of event types #################################### my @phot = $filename->get('unfiltered', 'x', 'pc*', '*'); my @diod = ($filename->get('unfiltered', 'x', 'lr*', '*'), $filename->get('unfiltered', 'x', 'pu*', '*')); my @wind = $filename->get('unfiltered', 'x', 'wt*', '*'); my (@wind1a, @diod1a); ############################################ # get attitude file, for xrt only pat or sat ############################################ my $attitude = $filename->get('attcorr', 'p'); if (-e $attitude) { $log->entry("Got pat attitude file $attitude"); } else { $log->error(1, "$attitude pat attitude file does not exist, trying sat file"); $attitude = $filename->get('attitude', 's'); if (-e $attitude) { $log->entry("Got sat attitude file $attitude"); } else { $log->error(1, "Unable to find attitude file, bailing"); return; } } my $filter = $filename->get('filter', 'x'); if (not -f "$filter") { $log->error(1, "Unable to find xrt filter file $filter, bailing"); return; } ############################# # xrthkproc ############################# my $head = $filename->get('hk', 'x', 'hd', '*'); my $trailer = $filename->get('hk', 'x', 'tr', '*'); unless(-f $head){ $log->error([ 1, XRT_NO_HDFILE ], "Unable to find xrt header file $head, cannot continue."); return; } ############################################# # First we have to fix up the hk header file ############################################# my $xrthkproc = Util::HEAdas->new('xrthkproc'); $log->entry("Running ".$xrthkproc->name()." on $head"); $xrthkproc->params({hdfile => $head, outfile => 'hkout.tmp', attfile => $attitude, srcdetx => 300, srcdety => 300, srcra => $srcra, srcdec => $srcdec, ranom => $jobpar->read('ra'), decnom => $jobpar->read('dec'), teldef => 'CALDB', chatter => 3, clobber => 'yes', history => 'yes'}) ->run(); unless( $xrthkproc->had_error() ){ unlink $head; rename 'hkout.tmp', $head; } ####################### # update time keywords ####################### my $head_fits = Util::FITSfile->new($head); my $start = $head_fits->keyword('TSTART'); my $stop = $head_fits->keyword('TSTOP'); my $start_date = Util::Date->new($start); my $stop_date = Util::Date->new($stop); $head_fits->keyword('DATE-OBS', $start_date->date().'T'.$start_date->time() ); $head_fits->keyword('DATE-END', $stop_date->date().'T'.$stop_date->time() ); $head_fits->ext(0); $head_fits->keyword('DATE-OBS', $start_date->date().'T'.$start_date->time() ); $head_fits->keyword('DATE-END', $stop_date->date().'T'.$stop_date->time() ); $head_fits->keyword('TSTART', $start); $head_fits->keyword('TSTOP', $stop); my %failed; # event files which have encountered errors so avoid further processing unless (-f $trailer) { if (@wind or @diod) { my $stuck = join(' ', @wind, @diod); $log->error(1, "Unable to find XRT trailer file $trailer needed for $stuck"); } foreach my $unf (@wind, @diod) { $failed{$unf} = 'missing trailer file'; } } ############# # xrttimetag ############# my $timetag = Util::HEAdas->new('xrttimetag'); my $timefile = 'timefile.tmp'; $timetag->params({outfile => $timefile, hdfile => $head, trfile => $trailer, usehkkey => 'no', attfile => $attitude, usesrcdet => 'no', srcra => $srcra, srcdec => $srcdec, ranom => $jobpar->read('ra'), decnom => $jobpar->read('dec'), teldef => 'CALDB', colfile => 'CALDB', chatter => 3, clobber => 'yes', history => 'yes'}); my $unf; foreach $unf (@wind){ if ($failed{$unf}) { # may not be possible $log->entry("Skipping xrttimetag on $unf"); next; } $log->entry("Running ".$timetag->name()." on $unf"); $timetag->params({infile => $unf}) ->run(); if ($timetag->had_error()) { $failed{$unf} = 1; } else { unlink $unf; rename $timefile, $unf; } } foreach $unf (@diod){ if ($failed{$unf}) { # may not be possible $log->entry("Skipping xrttimetag on $unf"); next; } $log->entry("Running ".$timetag->name()." on $unf"); $timetag->params({infile => $unf}) ->run(); if ($timetag->had_error()) { unlink($timefile); $failed{$unf} = 1; } else { unlink $unf; rename $timefile, $unf; ############################# # Filter using generated GTI ############################# my $difile = $filename->corresponding('unfiltered', 'reconst', $unf); my $fits = Util::FITSfile->new($unf, 'EVENTS', '[gtifilter()]'); $fits->copy($difile); my $new_fits = Util::FITSfile->new($difile, 'EVENTS'); if( $new_fits->keyword('NAXIS2') > 0 ){ push @diod1a, $difile; }else{ $log->entry("File $difile is empty. Deleting."); unlink $difile; } } } ############# # xrtpcbias # Run on each Photon Counting-mode event file (except SLEW and SETTLING # modes). If successful, replace the original file with the # bias-corrected one. ############# foreach $unf (@phot){ if ($failed{$unf}) { $log->entry("Skipping xrtpcbias on $unf"); next; } # initialize $pcbias here since otherwise srcdetx/y override # applies to all subsequent files my $pcbias = Util::HEAdas->new('xrtpcbias'); $pcbias->params({outfile => 'biasfile.tmp', teldef => 'CALDB', attfile => $attitude, mkffile => $filter, hdfile => $head, thrfile => 'CALDB', srcra => $srcra, srcdec => $srcdec, chatter => 3, clobber => 'yes', history => 'yes'}); $log->entry("Running ".$pcbias->name()." on $unf"); # 1.38: When the spacecraft is slewing or settling and XRT is in Photon # Counting mode, xrtpccorr (which xrtpcbias calls) cannot calculate # the detector source position, failing with "The .mkf file is not # appropriate for the ... events file", and causing xrtpcbias to # fail with an "E2: Unable to correct PHAS values" error. # Specifying srcdetx=300 and srcdety=300 in this situation should # fix this, per Matteo Perri, 2013-04-24. # # 1.41: However this didn't always work, so Matteo has recommended # instead just skipping xrtpcbias in this case, since the # associated event files are not used for scientific purposes # anyway (email 2014-05-31). # # 1.42: Unfortunately that turned out to sometimes cause other # problems down the line, because the *xpc*sl* and *xpc*st* # [EVENTS] HDUs then don't have PHAS0 columns, and the files don't # have [BIASDIFF] HDUs. For instance, Filter.pm::xrt_cal_events # could error trying to merge the [EVENTS] HDUs when creating the # xpc*cb_uf files. So, we're going back to calling xrtpcbias with # srcdetx=srcdety=300, and living with the occasional remaining # ".mkf not appropriate" error. my $unf_fits = Util::FITSfile->new($unf, 'EVENTS'); my $obs_mode = $unf_fits->keyword('OBS_MODE'); # NB:keyword() trims ws $log->entry("obs mode = $obs_mode"); if ( defined($obs_mode) && (($obs_mode eq 'SLEW') || ($obs_mode eq 'SETTLING')) ) { # [To skip calling xrtpcbias, use these lines instead:] # $log->entry("skipping xrtpcbias for $obs_mode mode"); # next; # [These lines to call xrtpcbias/xttpccorr with # srcdetx=srcdety=300 for these files:] $pcbias->params({ srcdetx => 300, srcdety => 300 }); $log->entry("using srcdetx=srcdety=300 in xrtpccorr " . "for $obs_mode mode"); } $pcbias->params({infile => $unf}) ->run(); if ($pcbias->had_error()) { $failed{$unf} = 1; } else { unlink $unf; rename 'biasfile.tmp', $unf; } } ############# # xrtflagpix ############# my $flagpix = Util::HEAdas->new('xrtflagpix'); $flagpix->params({outfile => 'flagfile.tmp', hdfile => $head, bpfile => 'CALDB', bptable => 'CALDB', srcfile => 'CALDB', thrfile => 'CALDB', phas1thr => 80, maxtemp => 0, userbpfile => 'NONE', outbpfile => 'NONE', overstatus => 'yes', chatter => 3, clobber => 'yes', history => 'yes'}); foreach $unf (@phot){ if ($failed{$unf}) { $log->entry("Skipping xrtflagpix on $unf"); next; } $log->entry("Running ".$flagpix->name()." on $unf"); $flagpix->params({infile => $unf}) ->run(); if ($flagpix->had_error()) { unlink 'flagfile.tmp'; $failed{$unf} = 1; } else { unlink $unf; rename 'flagfile.tmp', $unf; } } foreach $unf (@wind){ if ($failed{$unf}) { $log->entry("Skipping xrtflagpix on $unf"); next; } $log->entry("Running ".$flagpix->name()." on $unf"); $flagpix->params({infile => $unf}) ->run(); if( $flagpix->had_error() ){ $failed{$unf} = 1; unlink 'flagfile.tmp'; }else{ unlink $unf; rename 'flagfile.tmp', $unf; ############################# # Filter using generated GTI ############################# my $wtfile = $filename->corresponding('unfiltered', 'reconst', $unf); my $fits = Util::FITSfile->new($unf, 'EVENTS', '[gtifilter()]'); $fits->copy($wtfile); my $new_fits = Util::FITSfile->new($wtfile, 'EVENTS'); if( $new_fits->keyword('NAXIS2') > 0 ){ push @wind1a, $wtfile; }else{ $log->entry("File $wtfile is empty. Deleting."); unlink $wtfile; } } } ############################### # xrtwtcorr ############################### my $wtcorr = Util::HEAdas->new('xrtwtcorr'); $wtcorr->params({outfile => 'wtfile.tmp', hdfile => $head, trfile => $trailer, colfile => 'CALDB', npixels => 20, biasth => 200, thrfile => 'CALDB', history => 'yes', nevents => 20, biasdiff => 2, nframe => 20, biasmode => 'M20P', clobber => 'yes', chatter => 5}); foreach $unf (@wind1a) { if ($failed{$unf}) { $log->entry("Skipping xrtwtcorr on $unf"); next; } $log->entry("Running ".$wtcorr->name()." on $unf"); $wtcorr->params({infile => $unf}) ->run(); if ($wtcorr->had_error()) { $failed{$unf} = 1; } else { unlink $unf; rename 'wtfile.tmp', $unf; } } ############################### # xrtpdcorr ############################### my $pdcorr = Util::HEAdas->new('xrtpdcorr'); $pdcorr->params({outfile => 'pdfile.tmp', biasfile => 'CALDB', thrfile => 'CALDB', hdfile => $head, method => 'MN', bias => -1, biasth => 300, biasdiff => 4095, history => 'yes', clobber => 'yes', chatter => 5}); foreach $unf (@diod1a) { if ($failed{$unf}) { $log->entry("Skipping xrtpdcorr on $unf"); next; } $log->entry("Running ".$pdcorr->name()." on $unf"); $pdcorr->params({infile => $unf}) ->run(); if ($pdcorr->had_error()) { $failed{$unf} = 1; } else { unlink $unf; rename 'pdfile.tmp', $unf; } } ############################# # xrtevtrec ############################# my $evtrec = Util::HEAdas->new('xrtevtrec'); $evtrec->params({outfile => 'recfile.tmp', hdfile => $head, gradefile => 'CALDB', thrfile => 'CALDB', addcol => 'no', delnull => 'yes', event => 80, split => 80, flagneigh => 'yes', chatter => 5, clobber => 'yes', history => 'yes'}); foreach $unf (@diod1a, @wind1a){ if ($failed{$unf}) { $log->entry("Skipping xrtevtrec on $unf"); next; } $log->entry("Running ".$evtrec->name()." on $unf"); $evtrec->params({infile => $unf}) ->run(); if ($evtrec->had_error()) { $failed{$unf} = 1; } else { unlink $unf; rename 'recfile.tmp', $unf; } } ############################### # xrtpcgrade ############################### my $ph2br = Util::HEAdas->new('xrtpcgrade'); $ph2br->params({outfile => 'gradefile.tmp', hdfile => $head, split => 40, gradefile => 'CALDB', thrfile => 'CALDB', ascagrade => 'no', history => 'yes', clobber => 'yes', chatter => 5 }); foreach $unf (@phot) { if ($failed{$unf}) { $log->entry("Skipping xrtpcgrade on $unf"); next; } $log->entry("Running ".$ph2br->name()." on $unf"); $ph2br->params({infile => $unf}) ->run(); if ($ph2br->had_error()) { $failed{$unf} = 1; } else { unlink $unf; rename 'gradefile.tmp', $unf; } } ############################### # xrthotpix ############################### my $hotpix = Util::HEAdas->new('xrthotpix'); $hotpix->params({outfile => 'pixfile.tmp', outbpfile => 'NONE', phamax => 4095, iterate => 'yes', gradeiterate => 'yes', overstatus => 'yes', cellsize => 3, hotneigh => 'no', history => 'yes', clobber => 'yes', chatter => 5}); foreach $unf (@phot) { if ($failed{$unf}) { $log->entry("Skipping xrthotpix on $unf"); next; } $log->entry("Running ".$hotpix->name()." on $unf"); $hotpix->params({infile => $unf}) ->run(); if ($hotpix->had_error()) { $failed{$unf} = 1; } else { unlink $unf; rename 'pixfile.tmp', $unf; } } #################################### # set up xrtcalcpi #################################### my $pha2pi = Util::HEAdas->new('xrtcalcpi'); $pha2pi->params({outfile => 'pifile.tmp', hdfile => $head, gainfile => 'CALDB', gainnom => -99.0, # uses default from cal file offset => 0.0, randomflag => 'yes', seed => -1457, chatter => 2, clobber => 'yes', history => 'yes' }); foreach $unf (@phot) { if ($failed{$unf}) { $log->entry("Skipping xrtcalcpi on $unf"); next; } my ($inst, $mode, $index) = $filename->parse($unf, 'unfiltered'); $log->entry("Running ".$pha2pi->name()." on $unf"); $pha2pi->params({infile => $unf}) ->run(); if ($pha2pi->had_error()) { $failed{$unf} = 1; } else { unlink $unf; rename 'pifile.tmp', $unf; } } foreach $unf (@wind1a, @diod1a) { if ($failed{$unf}) { $log->entry("Skipping xrtcalcpi on $unf"); next; } my ($inst, $mode, $index) = $filename->parse($unf, 'reconst'); $log->entry("Running ".$pha2pi->name()." on $unf"); $pha2pi->params({infile => $unf}) ->run(); if ($pha2pi->had_error()) { $failed{$unf} = 1; } else { unlink $unf; my $copy = Util::HEAdas->new('ftcopy') ->params({infile => 'pifile.tmp[EVENTS][col -EVTPHA]', outfile => $unf}); $copy->seriousness(1); $copy->run(); if( $copy->had_error() ){ rename 'pifile.tmp', $unf; }else{ unlink 'pifile.tmp'; } } } # save list of files with errors to avoid future processing saveFailedEventFiles($self, \%failed); } # end of body method sub saveFailedEventFiles { my ($sub, $failed) = @_; my $log = $sub->log; $log->entry("saving failed XRT event files"); if (open(FAILED, '>xrtevent.failed')) { foreach my $key (sort(keys(%$failed))) { print FAILED "$key\n"; $log->entry("saved '$key'"); } close(FAILED); } else { $log->error(1, "unable to create xrtevent.failed: $!"); } } sub loadFailedEventFiles { my ($sub, $failed) = @_; my $log = $sub->log; $log->entry("loading failed XRT event files"); if (open(FAILED, 'xrtevent.failed')) { while (<FAILED>) { chomp; $log->entry("found '$_'"); $failed->{$_} = 1; } close(FAILED); } else { $log->error(1, "unable to read xrtevent.failed: $!"); } } 1;