##################### generated by xml-casa (v2) from concateovsa.xml ###############
##################### cdf0275a4ed32b7fdcd90e799b22f760 ##############################
from __future__ import absolute_import
from casashell.private.stack_manip import find_local as __sf__
from casashell.private.stack_manip import find_frame as _find_frame
from casatools.typecheck import validator as _pc
from casatools.coercetype import coerce as _coerce
from suncasatasks import concateovsa as _concateovsa_t
from collections import OrderedDict
import numpy
import sys
import os
import shutil
[docs]
def static_var(varname, value):
def decorate(func):
setattr(func, varname, value)
return func
return decorate
[docs]
class _concateovsa:
"""
concateovsa ---- Concatenate several EOVSA visibility data sets.
This is a EOVSA version of CASA concat task.
The list of data sets given in the vis argument are chronologically concatenated
into an output data set in concatvis, i.e. the data sets in vis are first ordered
by the time of their earliest integration and then concatenated.
If there are fields whose direction agrees within the direction tolerance
(parameter dirtol), the actual direction in the resulting, merged output field
will be the one from the chronologically first input MS.
If concatvis already exists (e.g., it is the same as the first input data set),
then the other input data sets will be appended to the concatvis data set.
There is no limit to the number of input data sets.
If none of the input data sets have any scratch columns (model and corrected
columns), none are created in the concatvis. Otherwise these columns are
created on output and initialized to their default value (1 in model column,
data in corrected column) for those data with no input columns.
Spectral windows for each data set with the same chanelization, and within a
specified frequency tolerance of another data set will be combined into one
spectral window.
A field position in one data set that is within a specified direction tolerance
of another field position in any other data set will be combined into one
field. The field names need not be the same---only their position is used.
Each appended dataset is assigned a new observation id (provided the entries
in the observation table are indeed different).
Keyword arguments:
vis -- Name of input visibility files to be combined
default: none; example: vis = ['src2.ms','ngc5921.ms','ngc315.ms']
concatvis -- Name of visibility file that will contain the concatenated data
note: if this file exits on disk then the input files are
added to this file. Otherwise the new file contains
the concatenated data. Be careful here when concatenating to
an existing file.
default: none; example: concatvis='src2.ms'
example: concatvis='outvis.ms'
datacolumn -- Which data column to use for processing (case-insensitive).
default: 'corrected'; example: datacolumn='data'
options: 'data', 'corrected'.
freqtol -- Frequency shift tolerance for considering data to be in the same
spwid. The number of channels must also be the same.
default: '' == 1 Hz
example: freqtol='10MHz' will not combine spwid unless they are
within 10 MHz.
Note: This option is useful to combine spectral windows with very slight
frequency differences caused by Doppler tracking, for example.
dirtol -- Direction shift tolerance for considering data as the same field
default: '' == 1 mas (milliarcsec)
example: dirtol='1arcsec' will not combine data for a field unless
their phase center differ by less than 1 arcsec. If the field names
are different in the input data sets, the name in the output data
set will be the first relevant data set in the list.
respectname -- If true, fields with a different name are not merged even if their
direction agrees (within dirtol)
default: False
timesort -- If true, the output visibility table will be sorted in time.
default: false. Data in order as read in.
example: timesort=true
Note: There is no constraint on data that is simultaneously observed for
more than one field; for example multi-source correlation of VLBA data.
copypointing -- Make a proper copy of the POINTING subtable (can be time consuming).
If False, the result is an empty POINTING table.
default: True
visweightscale -- The weights of the individual MSs will be scaled in the concatenated
output MS by the factors in this list. SIGMA will be scaled by 1/sqrt(factor).
Useful for handling heterogeneous arrays.
Use plotms to inspect the "Wt" column as a reference for determining the scaling
factors. See the cookbook for more details.
example: [1.,3.,3.] - scale the weights of the second and third MS by a factor 3
and the SIGMA column of these MS by a factor 1/sqrt(3).
default: [] (empty list) - no scaling
forcesingleephemfield -- By default, concat will only merge two ephemeris fields if
the first ephemeris covers the time range of the second. Otherwise, two separate
fields with separate ephemerides are placed in the output MS.
In order to override this behaviour and make concat merge the non-overlapping
or only partially overlapping input ephemerides, the name or id of the field
in question needs to be placed into the list in parameter 'forcesingleephemfield'.
example: ['Neptune'] - will make sure that there is only one joint ephemeris for
field Neptune in the output MS
default: '' - standard treatment of all ephemeris fields
--------- parameter descriptions ---------------------------------------------
vis Name of input visibility files to be concatenated
concatvis Name of output visibility file
datacolumn Which data column(s) to concatenate
keep_orig_ms If false, input vis files will be removed
cols2rm Columns in concatvis to be removed to slim the concatvis
freqtol Frequency shift tolerance for considering data as the same spwid
dirtol Direction shift tolerance for considering data as the same field
respectname If true, fields with a different name are not merged even if their direction agrees
timesort If true, sort by TIME in ascending order
copypointing Copy all rows of the POINTING table.
visweightscale List of the weight scaling factors to be applied to the individual MSs
forcesingleephemfield make sure that there is only one joint ephemeris for every field in this list
--------- examples -----------------------------------------------------------
concateovsa(vis=['UDB20180102161402.ms','UDB20180102173518.ms'], concatvis='UDB20180102_allday.ms')
will concatenate 'UDB20180102161402.ms' and 'UDB20180102173518.ms' into 'UDB20180102_allday.ms'
"""
[docs]
_info_group_ = """utility, manipulation"""
[docs]
_info_desc_ = """Concatenate several EOVSA visibility data sets."""
[docs]
__schema = {'vis': {'anyof': [{'type': 'cStr', 'coerce': _coerce.to_str}, {'type': 'cStrVec', 'coerce': [_coerce.to_list,_coerce.to_strvec]}]}, 'concatvis': {'type': 'cStr', 'coerce': _coerce.to_str}, 'datacolumn': {'type': 'cStr', 'coerce': _coerce.to_str, 'allowed': [ 'corrected', 'data', 'CORRECTED', 'DATA' ]}, 'keep_orig_ms': {'type': 'cBool'}, 'cols2rm': {'type': 'cStr', 'coerce': _coerce.to_str, 'allowed': [ 'model', 'corrected', 'CORRECTED', 'MODEL,CORRECTED', 'model,corrected', 'MODEL' ]}, 'freqtol': {'type': 'cVariant', 'coerce': [_coerce.to_variant]}, 'dirtol': {'type': 'cVariant', 'coerce': [_coerce.to_variant]}, 'respectname': {'type': 'cBool'}, 'timesort': {'type': 'cBool'}, 'copypointing': {'type': 'cBool'}, 'visweightscale': {'type': 'cFloatVec', 'coerce': [_coerce.to_list,_coerce.to_floatvec]}, 'forcesingleephemfield': {'type': 'cVariant', 'coerce': [_coerce.to_variant]}}
def __init__(self):
[docs]
self.__root_frame_ = None
[docs]
def __globals_(self):
if self.__root_frame_ is None:
self.__root_frame_ = _find_frame( )
assert self.__root_frame_ is not None, "could not find CASAshell global frame"
return self.__root_frame_
[docs]
def __to_string_(self,value):
if type(value) is str:
return "'%s'" % value
else:
return str(value)
[docs]
def __validate_(self,doc,schema):
return _pc.validate(doc,schema)
[docs]
def __do_inp_output(self,param_prefix,description_str,formatting_chars):
out = self.__stdout or sys.stdout
description = description_str.split( )
prefix_width = 23 + 21 + 4
output = [ ]
addon = ''
first_addon = True
while len(description) > 0:
## starting a new line.....................................................................
if len(output) == 0:
## for first line add parameter information............................................
if len(param_prefix)-formatting_chars > prefix_width - 1:
output.append(param_prefix)
continue
addon = param_prefix + ' #'
first_addon = True
addon_formatting = formatting_chars
else:
## for subsequent lines space over prefix width........................................
addon = (' ' * prefix_width) + '#'
first_addon = False
addon_formatting = 0
## if first word of description puts us over the screen width, bail........................
if len(addon + description[0]) - addon_formatting + 1 > self.term_width:
## if we're doing the first line make sure it's output.................................
if first_addon: output.append(addon)
break
while len(description) > 0:
## if the next description word puts us over break for the next line...................
if len(addon + description[0]) - addon_formatting + 1 > self.term_width: break
addon = addon + ' ' + description[0]
description.pop(0)
output.append(addon)
out.write('\n'.join(output) + '\n')
#--------- return nonsubparam values ----------------------------------------------
[docs]
def __forcesingleephemfield_dflt( self, glb ):
return ''
[docs]
def __forcesingleephemfield( self, glb ):
if 'forcesingleephemfield' in glb: return glb['forcesingleephemfield']
return ''
[docs]
def __vis_dflt( self, glb ):
return ''
[docs]
def __vis( self, glb ):
if 'vis' in glb: return glb['vis']
return ''
[docs]
def __dirtol_dflt( self, glb ):
return ''
[docs]
def __dirtol( self, glb ):
if 'dirtol' in glb: return glb['dirtol']
return ''
[docs]
def __timesort_dflt( self, glb ):
return True
[docs]
def __timesort( self, glb ):
if 'timesort' in glb: return glb['timesort']
return True
[docs]
def __datacolumn_dflt( self, glb ):
return 'corrected'
[docs]
def __datacolumn( self, glb ):
if 'datacolumn' in glb: return glb['datacolumn']
return 'corrected'
[docs]
def __respectname_dflt( self, glb ):
return False
[docs]
def __respectname( self, glb ):
if 'respectname' in glb: return glb['respectname']
return False
[docs]
def __keep_orig_ms_dflt( self, glb ):
return True
[docs]
def __keep_orig_ms( self, glb ):
if 'keep_orig_ms' in glb: return glb['keep_orig_ms']
return True
[docs]
def __visweightscale_dflt( self, glb ):
return [ ]
[docs]
def __visweightscale( self, glb ):
if 'visweightscale' in glb: return glb['visweightscale']
return [ ]
[docs]
def __concatvis_dflt( self, glb ):
return ''
[docs]
def __concatvis( self, glb ):
if 'concatvis' in glb: return glb['concatvis']
return ''
[docs]
def __copypointing_dflt( self, glb ):
return True
[docs]
def __copypointing( self, glb ):
if 'copypointing' in glb: return glb['copypointing']
return True
[docs]
def __cols2rm_dflt( self, glb ):
return 'model,corrected'
[docs]
def __cols2rm( self, glb ):
if 'cols2rm' in glb: return glb['cols2rm']
return 'model,corrected'
[docs]
def __freqtol_dflt( self, glb ):
return ''
[docs]
def __freqtol( self, glb ):
if 'freqtol' in glb: return glb['freqtol']
return ''
#--------- return inp/go default --------------------------------------------------
#--------- return subparam values -------------------------------------------------
#--------- subparam inp output ----------------------------------------------------
[docs]
def __vis_inp(self):
description = ''
value = self.__vis( self.__globals_( ) )
(pre,post) = ('','') if self.__validate_({'vis': value},{'vis': self.__schema['vis']}) else ('\x1B[91m','\x1B[0m')
self.__do_inp_output('%-21.21s = %s%-23s%s' % ('vis',pre,self.__to_string_(value),post),description,0+len(pre)+len(post))
[docs]
def __concatvis_inp(self):
description = ''
value = self.__concatvis( self.__globals_( ) )
(pre,post) = ('','') if self.__validate_({'concatvis': value},{'concatvis': self.__schema['concatvis']}) else ('\x1B[91m','\x1B[0m')
self.__do_inp_output('%-21.21s = %s%-23s%s' % ('concatvis',pre,self.__to_string_(value),post),description,0+len(pre)+len(post))
[docs]
def __datacolumn_inp(self):
description = ''
value = self.__datacolumn( self.__globals_( ) )
(pre,post) = ('','') if self.__validate_({'datacolumn': value},{'datacolumn': self.__schema['datacolumn']}) else ('\x1B[91m','\x1B[0m')
self.__do_inp_output('%-21.21s = %s%-23s%s' % ('datacolumn',pre,self.__to_string_(value),post),description,0+len(pre)+len(post))
[docs]
def __keep_orig_ms_inp(self):
description = ''
value = self.__keep_orig_ms( self.__globals_( ) )
(pre,post) = ('','') if self.__validate_({'keep_orig_ms': value},{'keep_orig_ms': self.__schema['keep_orig_ms']}) else ('\x1B[91m','\x1B[0m')
self.__do_inp_output('%-21.21s = %s%-23s%s' % ('keep_orig_ms',pre,self.__to_string_(value),post),description,0+len(pre)+len(post))
[docs]
def __cols2rm_inp(self):
description = ''
value = self.__cols2rm( self.__globals_( ) )
(pre,post) = ('','') if self.__validate_({'cols2rm': value},{'cols2rm': self.__schema['cols2rm']}) else ('\x1B[91m','\x1B[0m')
self.__do_inp_output('%-21.21s = %s%-23s%s' % ('cols2rm',pre,self.__to_string_(value),post),description,0+len(pre)+len(post))
[docs]
def __freqtol_inp(self):
description = ''
value = self.__freqtol( self.__globals_( ) )
(pre,post) = ('','') if self.__validate_({'freqtol': value},{'freqtol': self.__schema['freqtol']}) else ('\x1B[91m','\x1B[0m')
self.__do_inp_output('%-21.21s = %s%-23s%s' % ('freqtol',pre,self.__to_string_(value),post),description,0+len(pre)+len(post))
[docs]
def __dirtol_inp(self):
description = ''
value = self.__dirtol( self.__globals_( ) )
(pre,post) = ('','') if self.__validate_({'dirtol': value},{'dirtol': self.__schema['dirtol']}) else ('\x1B[91m','\x1B[0m')
self.__do_inp_output('%-21.21s = %s%-23s%s' % ('dirtol',pre,self.__to_string_(value),post),description,0+len(pre)+len(post))
[docs]
def __respectname_inp(self):
description = ''
value = self.__respectname( self.__globals_( ) )
(pre,post) = ('','') if self.__validate_({'respectname': value},{'respectname': self.__schema['respectname']}) else ('\x1B[91m','\x1B[0m')
self.__do_inp_output('%-21.21s = %s%-23s%s' % ('respectname',pre,self.__to_string_(value),post),description,0+len(pre)+len(post))
[docs]
def __timesort_inp(self):
description = ''
value = self.__timesort( self.__globals_( ) )
(pre,post) = ('','') if self.__validate_({'timesort': value},{'timesort': self.__schema['timesort']}) else ('\x1B[91m','\x1B[0m')
self.__do_inp_output('%-21.21s = %s%-23s%s' % ('timesort',pre,self.__to_string_(value),post),description,0+len(pre)+len(post))
[docs]
def __copypointing_inp(self):
description = ''
value = self.__copypointing( self.__globals_( ) )
(pre,post) = ('','') if self.__validate_({'copypointing': value},{'copypointing': self.__schema['copypointing']}) else ('\x1B[91m','\x1B[0m')
self.__do_inp_output('%-21.21s = %s%-23s%s' % ('copypointing',pre,self.__to_string_(value),post),description,0+len(pre)+len(post))
[docs]
def __visweightscale_inp(self):
description = ''
value = self.__visweightscale( self.__globals_( ) )
(pre,post) = ('','') if self.__validate_({'visweightscale': value},{'visweightscale': self.__schema['visweightscale']}) else ('\x1B[91m','\x1B[0m')
self.__do_inp_output('%-21.21s = %s%-23s%s' % ('visweightscale',pre,self.__to_string_(value),post),description,0+len(pre)+len(post))
[docs]
def __forcesingleephemfield_inp(self):
description = ''
value = self.__forcesingleephemfield( self.__globals_( ) )
(pre,post) = ('','') if self.__validate_({'forcesingleephemfield': value},{'forcesingleephemfield': self.__schema['forcesingleephemfield']}) else ('\x1B[91m','\x1B[0m')
self.__do_inp_output('%-21.21s = %s%-23s%s' % ('forcesingleephemfield',pre,self.__to_string_(value),post),description,0+len(pre)+len(post))
#--------- global default implementation-------------------------------------------
@static_var('state', __sf__('casa_inp_go_state'))
[docs]
def set_global_defaults(self):
self.set_global_defaults.state['last'] = self
glb = self.__globals_( )
if 'dirtol' in glb: del glb['dirtol']
if 'concatvis' in glb: del glb['concatvis']
if 'cols2rm' in glb: del glb['cols2rm']
if 'datacolumn' in glb: del glb['datacolumn']
if 'keep_orig_ms' in glb: del glb['keep_orig_ms']
if 'freqtol' in glb: del glb['freqtol']
if 'vis' in glb: del glb['vis']
if 'forcesingleephemfield' in glb: del glb['forcesingleephemfield']
if 'visweightscale' in glb: del glb['visweightscale']
if 'timesort' in glb: del glb['timesort']
if 'respectname' in glb: del glb['respectname']
if 'copypointing' in glb: del glb['copypointing']
#--------- inp function -----------------------------------------------------------
[docs]
def inp(self):
print("# concateovsa -- %s" % self._info_desc_)
self.term_width, self.term_height = shutil.get_terminal_size(fallback=(80, 24))
self.__vis_inp( )
self.__concatvis_inp( )
self.__datacolumn_inp( )
self.__keep_orig_ms_inp( )
self.__cols2rm_inp( )
self.__freqtol_inp( )
self.__dirtol_inp( )
self.__respectname_inp( )
self.__timesort_inp( )
self.__copypointing_inp( )
self.__visweightscale_inp( )
self.__forcesingleephemfield_inp( )
#--------- tget function ----------------------------------------------------------
@static_var('state', __sf__('casa_inp_go_state'))
[docs]
def tget(self,file=None):
from casashell.private.stack_manip import find_frame
from runpy import run_path
filename = None
if file is None:
if os.path.isfile("concateovsa.last"):
filename = "concateovsa.last"
elif isinstance(file, str):
if os.path.isfile(file):
filename = file
if filename is not None:
glob = find_frame( )
newglob = run_path( filename, init_globals={ } )
for i in newglob:
glob[i] = newglob[i]
self.tget.state['last'] = self
else:
print("could not find last file, setting defaults instead...")
self.set_global_defaults( )
[docs]
def __call__( self, vis=None, concatvis=None, datacolumn=None, keep_orig_ms=None, cols2rm=None, freqtol=None, dirtol=None, respectname=None, timesort=None, copypointing=None, visweightscale=None, forcesingleephemfield=None ):
def noobj(s):
if s.startswith('<') and s.endswith('>'):
return "None"
else:
return s
_prefile = os.path.realpath('concateovsa.pre')
_postfile = os.path.realpath('concateovsa.last')
_return_result_ = None
_arguments = [vis,concatvis,datacolumn,keep_orig_ms,cols2rm,freqtol,dirtol,respectname,timesort,copypointing,visweightscale,forcesingleephemfield]
_invocation_parameters = OrderedDict( )
if any(map(lambda x: x is not None,_arguments)):
# invoke python style
# set the non sub-parameters that are not None
local_global = { }
if vis is not None: local_global['vis'] = vis
if concatvis is not None: local_global['concatvis'] = concatvis
if datacolumn is not None: local_global['datacolumn'] = datacolumn
if keep_orig_ms is not None: local_global['keep_orig_ms'] = keep_orig_ms
if cols2rm is not None: local_global['cols2rm'] = cols2rm
if freqtol is not None: local_global['freqtol'] = freqtol
if dirtol is not None: local_global['dirtol'] = dirtol
if respectname is not None: local_global['respectname'] = respectname
if timesort is not None: local_global['timesort'] = timesort
if copypointing is not None: local_global['copypointing'] = copypointing
if visweightscale is not None: local_global['visweightscale'] = visweightscale
if forcesingleephemfield is not None: local_global['forcesingleephemfield'] = forcesingleephemfield
# the invocation parameters for the non-subparameters can now be set - this picks up those defaults
_invocation_parameters['vis'] = self.__vis( local_global )
_invocation_parameters['concatvis'] = self.__concatvis( local_global )
_invocation_parameters['datacolumn'] = self.__datacolumn( local_global )
_invocation_parameters['keep_orig_ms'] = self.__keep_orig_ms( local_global )
_invocation_parameters['cols2rm'] = self.__cols2rm( local_global )
_invocation_parameters['freqtol'] = self.__freqtol( local_global )
_invocation_parameters['dirtol'] = self.__dirtol( local_global )
_invocation_parameters['respectname'] = self.__respectname( local_global )
_invocation_parameters['timesort'] = self.__timesort( local_global )
_invocation_parameters['copypointing'] = self.__copypointing( local_global )
_invocation_parameters['visweightscale'] = self.__visweightscale( local_global )
_invocation_parameters['forcesingleephemfield'] = self.__forcesingleephemfield( local_global )
# the sub-parameters can then be set. Use the supplied value if not None, else the function, which gets the appropriate default
else:
# invoke with inp/go semantics
_invocation_parameters['vis'] = self.__vis( self.__globals_( ) )
_invocation_parameters['concatvis'] = self.__concatvis( self.__globals_( ) )
_invocation_parameters['datacolumn'] = self.__datacolumn( self.__globals_( ) )
_invocation_parameters['keep_orig_ms'] = self.__keep_orig_ms( self.__globals_( ) )
_invocation_parameters['cols2rm'] = self.__cols2rm( self.__globals_( ) )
_invocation_parameters['freqtol'] = self.__freqtol( self.__globals_( ) )
_invocation_parameters['dirtol'] = self.__dirtol( self.__globals_( ) )
_invocation_parameters['respectname'] = self.__respectname( self.__globals_( ) )
_invocation_parameters['timesort'] = self.__timesort( self.__globals_( ) )
_invocation_parameters['copypointing'] = self.__copypointing( self.__globals_( ) )
_invocation_parameters['visweightscale'] = self.__visweightscale( self.__globals_( ) )
_invocation_parameters['forcesingleephemfield'] = self.__forcesingleephemfield( self.__globals_( ) )
try:
with open(_prefile,'w') as _f:
for _i in _invocation_parameters:
_f.write("%-21s = %s\n" % (_i,noobj(repr(_invocation_parameters[_i]))))
_f.write("#concateovsa( ")
count = 0
for _i in _invocation_parameters:
_f.write("%s=%s" % (_i,noobj(repr(_invocation_parameters[_i]))))
count += 1
if count < len(_invocation_parameters): _f.write(",")
_f.write(" )\n")
except: pass
try:
_return_result_ = _concateovsa_t( _invocation_parameters['vis'],_invocation_parameters['concatvis'],_invocation_parameters['datacolumn'],_invocation_parameters['keep_orig_ms'],_invocation_parameters['cols2rm'],_invocation_parameters['freqtol'],_invocation_parameters['dirtol'],_invocation_parameters['respectname'],_invocation_parameters['timesort'],_invocation_parameters['copypointing'],_invocation_parameters['visweightscale'],_invocation_parameters['forcesingleephemfield'] )
except Exception as e:
from traceback import format_exc
from casatasks import casalog
casalog.origin('concateovsa')
casalog.post("Exception Reported: Error in concateovsa: %s" % str(e),'SEVERE')
casalog.post(format_exc( ))
_return_result_ = False
try:
os.rename(_prefile,_postfile)
except: pass
return _return_result_
[docs]
concateovsa = _concateovsa( )