pylal.printutils

Source Code for Module pylal.printutils

1 # 2 # ============================================================================= 3 # 4 # Preamble 5 # 6 # ============================================================================= 7 # 8 9 ''' 10 A collection of utilities to assist in printing out information from an xmldoc. 11 ''' 12 13 import sys, re, math 14 import time, datetime 15 16 from glue import iterutils 17 from glue.ligolw.utils import print_tables 18 from glue.ligolw import ligolw 19 from glue.ligolw import table 20 from glue.ligolw import lsctables 21 from glue import git_version 22 23 from pylal.xlal.date import XLALGPSToUTC 24 try: 25 from pylal.xlal.datatypes.ligotimegps import LIGOTimeGPS 26 except ImportError: 27 # s6 code 28 from pylal.xlal.date import LIGOTimeGPS 29 30 31 __author__ = "Collin Capano <cdcapano@physics.syr.edu>" 32 __version__ = git_version.id

33 34 35 # ============================================================================= 36 # 37 # Utilities 38 # 39 # ============================================================================= 40 41 -def generic_get_pyvalue(obj):

42 if obj.value is None: 43 return None 44 return ligolwtypes.ToPyType[obj.type or "lstring"](obj.value)

45

46 47 -def get_columns_to_print(xmldoc, tableName, with_sngl = False):

48 """ 49 Retrieves canonical columns to print for the given tableName. 50 Returns a columnList, row_span and rspan_break lists. 51 52 @with_sngl: for the loudest_events table, if with_sngl turned on, will print 53 sngl_ifo end_times 54 """ 55 tableName = table.Table.TableName(tableName) 56 summTable = table.get_table(xmldoc, tableName ) 57 # get rankname 58 rankname = [col.getAttribute("Name").split(":")[-1] 59 for col in summTable.getElementsByTagName(u'Column') if "rank" in col.getAttribute("Name")][0] 60 61 if tableName == "loudest_events" and not with_sngl: 62 durname = [col.getAttribute("Name").split(":")[-1] 63 for col in summTable.getElementsByTagName(u'Column') if "_duration__Px_" in col.getAttribute("Name") 64 and not col.getAttribute("Name").split(":")[-1].startswith('sngl_')][0] 65 columnList = [ 66 rankname, 67 'combined_far', 68 'fap', 69 'fap_1yr', 70 'snr', 71 'end_time', 72 'gps_time_utc__Px_click_for_daily_ihope_xP_', 73 'ifos__Px_click_for_elog_xP_', 74 'instruments_on', 75 'mass', 76 'mchirp', 77 'mini_followup', 78 'omega_scan', 79 durname] 80 row_span_columns = rspan_break_columns = [durname] 81 elif tableName == "loudest_events" and with_sngl: 82 durname = [col.getAttribute("Name").split(":")[-1] 83 for col in summTable.getElementsByTagName(u'Column') if "_duration__Px_" in col.getAttribute("Name") 84 and not col.getAttribute("Name").split(":")[-1].startswith(u'sngl_')][0] 85 columnList = [ 86 rankname, 87 'combined_far', 88 'fap', 89 'fap_1yr', 90 'snr', 91 'mass', 92 'mchirp', 93 'instruments_on', 94 'sngl_ifo__Px_click_for_elog_xP_', 95 'sngl_end_time', 96 'sngl_event_time_utc__Px_click_for_daily_ihope_xP_', 97 'mini_followup', 98 'omega_scan', 99 durname] 100 row_span_columns = rspan_break_columns = \ 101 [col for col in summTable.columnnames if not col.startswith('sngl_')] 102 elif tableName == "selected_found_injections": 103 durname = [col.getAttribute("Name").split(":")[-1] 104 for col in summTable.getElementsByTagName(u'Column') if "_duration__Px_" in col.getAttribute("Name")][0] 105 columnList = [ 106 rankname, 107 'injected_gps_time', 108 'injected_event_time_utc__Px_click_for_daily_ihope_xP_', 109 'elogs', 110 'mini_followup', 111 'omega_scan', 112 'sim_tag', 113 'injected_decisive_distance', 114 'injected_mchirp', 115 'injected_mass1', 116 'injected_mass2', 117 'recovered_match_rank', 118 'recovered_ifos', 119 'recovered_combined_far', 120 'recovered_fap', 121 'recovered_fap_1yr', 122 'recovered_snr', 123 'recovered_gps_time', 124 'recovered_mchirp', 125 'recovered_mass'] 126 row_span_columns = rspan_break_columns = [ 127 rankname, 128 'injected_gps_time', 129 'injected_event_time_utc__Px_click_for_daily_ihope_xP_', 130 'elogs', 131 'mini_followup', 132 'omega_scan', 133 'sim_tag', 134 'injected_decisive_distance', 135 'injected_mchirp', 136 'injected_mass1', 137 'injected_mass2'] 138 if with_sngl: 139 row_span_columns.extend( [col for col in summTable.columnnames if col.startswith('recovered_')] ) 140 elif tableName == "close_missed_injections": 141 columnList = [ 142 'rank', 143 'decisive_distance', 144 'gps_time', 145 'injection_time_utc__Px_click_for_daily_ihope_xP_', 146 'elogs', 147 'mchirp', 148 'mass1', 149 'mass2', 150 'eff_dist_h', 151 'eff_dist_l', 152 'eff_dist_v', 153 'sim_tag', 154 'mini_followup', 155 'omega_scan' 156 ] 157 row_span_columns = rspan_break_columns = [] 158 else: 159 # unrecognized table, just return all the columns in the table 160 columnList = [col.getAttribute("Name").split(":")[-1] for col in summTable.getElementsByTagName(u'Column')] 161 row_span_columns = rspan_break_columns = [] 162 163 return columnList, row_span_columns, rspan_break_columns

164

165 166 # 167 # Some helper functions for manipulating times 168 # 169 170 -def get_dst_start_end(ifo, year):

171 """ 172 Figures out what dates daylight savings time starts and ends at a given site on a given year. 173 """ 174 # in the United Stats, prior to 2007, DST began on the first Sunday in April 175 # and ended on the last Sunday in October (http://aa.usno.navy.mil/faq/docs/daylight_time.php) 176 if ("H" in ifo or "L" in ifo) and year < 2007: 177 for ii in range(1,28): 178 dst_start = datetime.datetime(year, 4, ii, 2, 0, 0) 179 if dst_start.strftime('%A') == 'Sunday': 180 break 181 for ii in range(31,0,-1): 182 dst_end = datetime.datetime(year, 10, ii, 2, 0, 0) 183 if dst_end.strftime('%A') == 'Sunday': 184 break 185 # in the US, starting in 2007, DST begins on the second Sunday in March and ends on the first 186 # Sunday in November 187 elif ("H" in ifo or "L" in ifo) and year >= 2007: 188 nn = 1 189 for ii in range(1,31): 190 dst_start = datetime.datetime(year, 3, ii, 2, 0, 0) 191 if dst_start.strftime('%A') == 'Sunday' and nn == 2: 192 break 193 elif dst_start.strftime('%A') == 'Sunday': 194 nn += 1 195 for ii in range(1,28): 196 dst_end = datetime.datetime(year, 11, ii, 2, 0, 0) 197 if dst_end.strftime('%A') == 'Sunday': 198 break 199 # in Europe, DST begins on the last Sunday of March and ends on the last Sunday of October 200 # source: http://www.timeanddate.com/news/time/europe-dst-starts-march-29-2009.html 201 elif ("V" in ifo or "G" in ifo): 202 for ii in range(31,0,-1): 203 dst_start = datetime.datetime(year, 3, ii, 2, 0, 0) 204 if dst_start.strftime('%A') == 'Sunday': 205 break 206 for ii in range(31,0,-1): 207 dst_end = datetime.datetime(year, 10, ii, 2, 0, 0) 208 if dst_end.strftime('%A') == 'Sunday': 209 break 210 else: 211 raise ValueError, "unrecognized ifo %s" % ifo 212 213 return dst_start, dst_end

214

215 216 -def get_sitelocaltime_from_gps(ifo, gpstime):

217 # get the utc time in datetime.datetime format 218 utctime = XLALGPSToUTC(LIGOTimeGPS(gpstime, 0)) 219 utctime = datetime.datetime(utctime[0],utctime[1],utctime[2],utctime[3],utctime[4],utctime[5],utctime[6]) 220 # figure out if daylight savings time was on or not 221 dst_start, dst_end = get_dst_start_end(ifo, utctime.year) 222 # figure out the appropriate time offset 223 if "H" in ifo: 224 toffset = datetime.timedelta(hours=-7) 225 elif "L" in ifo: 226 toffset = datetime.timedelta(hours=-5) 227 elif ("V" in ifo or "G" in ifo): 228 toffset = datetime.timedelta(hours=+2) 229 # apply the dst time offset to see if daylight savings was on; if not, adjust the toffset 230 if not (utctime + toffset >= dst_start and utctime + toffset < dst_end): 231 toffset = toffset + datetime.timedelta(hours=-1) 232 233 return utctime + toffset

234

235 236 -def format_end_time_in_utc(gps_sec):

237 return time.strftime("%a %d %b %Y %H:%M:%S", XLALGPSToUTC(LIGOTimeGPS(gps_sec, 0)))

238

239 240 -def get_elog_page(ifo, gpstime):

241 # set site_address 242 if "H" in ifo: 243 site_address = "http://ilog.ligo-wa.caltech.edu/ilog/pub/ilog.cgi?group=detector" 244 elif "L" in ifo: 245 site_address = "http://ilog.ligo-la.caltech.edu/ilog/pub/ilog.cgi?group=detector" 246 elif "V" in ifo: 247 #FIXME: What's the site address and format for Virgo log book? 248 site_address = "https://pub3.ego-gw.it/logbook/" 249 # get local time at the site 250 site_localtime = get_sitelocaltime_from_gps(ifo, gpstime) 251 # set the address 252 if "H" in ifo or "L" in ifo: 253 site_address = "%s&date_to_view=%s" % ( site_address, site_localtime.strftime("%m/%d/%Y") ) 254 255 return site_address

256

257 -def get_daily_ihope_page(gpstime, pages_location = "https://ldas-jobs.ligo.caltech.edu/~cbc/ihope_daily"):

258 utctime = XLALGPSToUTC(LIGOTimeGPS(gpstime, 0)) 259 return "%s/%s/%s/" %(pages_location, time.strftime("%Y%m", utctime), time.strftime("%Y%m%d", utctime))

260

261 262 -def create_hyperlink(address, link, external=True):

263 return '<a href="%s"%s>%s</a>' % (address, external and ' rel="external"' or '', link)

264

265 266 -def create_filter( connection, tableName, param_name = None, param_ranges = None, 267 exclude_coincs = None, include_only_coincs = None, sim_tag = 'ALLINJ', verbose = False):

268 """ 269 Strings together param_name, param_ranges, exclude/include_only_coincs, and 270 sim_tag options into a filter string that can be stuck in a sqlite WHERE clause. 271 """ 272 from pylal import ligolw_sqlutils as sqlutils 273 274 in_this_filter = '' 275 276 # Get param and param-ranges if specified 277 if param_name is not None: 278 param_name = sqlutils.validate_option(param_name) 279 param_filters = sqlutils.parse_param_ranges( tableName, param_name, 280 param_ranges, verbose = verbose ).get_param_filters() 281 # since want triggers that fall within all the parameters, concatenate 282 # all param ranges 283 param_filters = '\n\t\tOR '.join( param_filters ) 284 in_this_filter = ''.join([ in_this_filter, '\n\tAND (\n\t\t', param_filters, '\n\t)' ]) 285 286 # Get exclude_coincs list if specified 287 if exclude_coincs is not None: 288 exclude_coinc_filters = sqlutils.parse_coinc_options( exclude_coincs, 289 verbose = verbose ).get_coinc_filters( coinc_instruments_table = tableName ) 290 # concatenate exclude_coinc_filters 291 exclude_coinc_filters = '\n\t\tOR '.join( exclude_coinc_filters ) 292 # add to in_this_filter 293 in_this_filter = ''.join([ in_this_filter, '\n\tAND NOT (\n\t\t', exclude_coinc_filters, '\n\t)' ]) 294 295 # Get include_only_coincs list if specified 296 if include_only_coincs is not None: 297 include_coinc_filters = sqlutils.parse_coinc_options( include_only_coincs, 298 verbose = verbose ).get_coinc_filters( coinc_instruments_table = tableName ) 299 # concatenate include_coinc_filters 300 include_coinc_filters = '\n\t\tOR '.join( include_coinc_filters ) 301 # add to in_this_filter 302 in_this_filter = ''.join([ in_this_filter, '\n\tAND (\n\t\t', include_coinc_filters, '\n\t)' ]) 303 304 # if sim-tag specified add the sim-tag to the filter 305 if sim_tag != 'ALLINJ': 306 # create a map between sim_proc_id and sim-tag and a function to parse it 307 sim_map = sqlutils.sim_tag_proc_id_mapper( connection ) 308 connection.create_function( 'get_sim_tag', 1, sim_map.get_sim_tag ) 309 # parse the sim_tag for multiple sims; then cycle over and add to the filter 310 sim_filter = '' 311 sim_list = sim_tag.split('+') 312 for sim_tag in sim_list: 313 # check that sim_tag is in the the map 314 sim_tag = sqlutils.validate_option(sim_tag, lower = False).upper() 315 if sim_tag not in sim_map.tag_id_map.keys(): 316 raise ValueError, "sim-tag %s not found in database" % sim_tag 317 # create the filter 318 sim_filter = '\n\t\tOR '.join([ sim_filter, ''.join(['get_sim_tag(experiment_summary.sim_proc_id) == "', sim_tag, '"' ]) ]) 319 320 # strip the leading OR and add to in_this_filter 321 sim_filter = re.sub(r'OR ', '', sim_filter, 1) 322 in_this_filter = ''.join([ in_this_filter, '\n\tAND (', sim_filter, '\n\t)' ]) 323 324 # remove the leading AND 325 in_this_filter = re.sub(r'\n\tAND', '', in_this_filter, 1) 326 327 return in_this_filter

328

329 330 # ============================================================================= 331 # 332 # Library API 333 # 334 # ============================================================================= 335 336 -def printsims(connection, simulation_table, recovery_table, map_label, ranking_stat, rank_by, comparison_datatype, 337 sort_by = 'rank', param_name = None, param_ranges = None, exclude_coincs = None, include_only_coincs = None, 338 sim_tag = 'ALLINJ', rank_range = None, convert_durations = 's', 339 daily_ihope_pages_location = 'https://ldas-jobs.ligo.caltech.edu/~cbc/ihope_daily', verbose = False):

340 341 from pylal import ligolw_sqlutils as sqlutils 342 343 # check and format options appropriately 344 simulation_table = sqlutils.validate_option(simulation_table) 345 recovery_table = sqlutils.validate_option(recovery_table) 346 ranking_stat = sqlutils.validate_option(ranking_stat) 347 rank_by = sqlutils.validate_option(rank_by, lower = False).upper() 348 comparison_datatype = sqlutils.validate_option(comparison_datatype) 349 convert_durations = sqlutils.validate_option(convert_durations) 350 351 if rank_by == 'MIN': 352 rank_by = 'ASC' 353 elif rank_by == 'MAX': 354 rank_by = 'DESC' 355 elif rank_by != 'ASC' or rank_by != 'DESC': 356 raise ValueError, 'rank_by must be MAX (or DESC) or MIN (or ASC)' 357 358 if not ranking_stat.startswith(recovery_table): 359 ranking_stat = '.'.join([recovery_table, ranking_stat]) 360 361 # 362 # Set up sim_rec_map table 363 # 364 if 'sim_rec_map' not in sqlutils.get_tables_in_database( connection ): 365 sqlutils.create_sim_rec_map_table(connection, simulation_table, recovery_table, map_label, ranking_stat) 366 367 # 368 # Initialize ranking. Statistics for ranking are collected from non-injections 369 # in the recovery table. 370 # 371 if verbose: 372 print >> sys.stderr, "Getting statistics for ranking..." 373 ranker = sqlutils.rank_stats(recovery_table, ranking_stat, rank_by) 374 # add requirement that stats not be found in the sim_rec_table to in_this_filter 375 rank_filter = ''.join([ 376 recovery_table, '''.coinc_event_id NOT IN ( 377 SELECT 378 rec_id 379 FROM 380 sim_rec_map) 381 AND 382 experiment_summary.datatype == "''', comparison_datatype, '"']) 383 384 # add to the rank filter any other desired filters; note that sim-tag is forced to ALLINJ if not comparing 385 # to simulation datatype 386 in_this_filter = create_filter(connection, recovery_table, param_name = param_name, param_ranges = param_ranges, 387 exclude_coincs = exclude_coincs, include_only_coincs = include_only_coincs, 388 sim_tag = comparison_datatype == 'simulation' and sim_tag or 'ALLINJ', 389 verbose = False) 390 if in_this_filter != '': 391 rank_filter = '\n\tAND '.join([ in_this_filter, rank_filter ]) 392 393 rank_filter = '\n\t'.join([ sqlutils.join_experiment_tables_to_coinc_table(recovery_table), 'WHERE', rank_filter ]) 394 395 ranker.populate_stats_list(connection, limit = None, filter = rank_filter) 396 connection.create_function( 'rank', 1, ranker.get_rank ) 397 398 # 399 # Set recovery table filters 400 # 401 in_this_filter = create_filter(connection, recovery_table, param_name = param_name, param_ranges = param_ranges, 402 exclude_coincs = exclude_coincs, include_only_coincs = include_only_coincs, sim_tag = sim_tag, 403 verbose = verbose) 404 405 # Now apply the filter to the sim_rec_map table: this will delete all sim/rec maps where the simulation id is 406 # mapped to a recovered event that falls outside the filter, even if that particular sim/rec map is in the 407 # filter. For example, if the filter is recovery_table.combined_far != 0., and there are two entries in the 408 # sim_rec_map table sharing the same sim_id: 409 # sim_id:0 | rec_id:0 | rec_id's combined_far = 0. 410 # sim_id:0 | rec_id:1 | rec_id's combined_far = 1.2 411 # both entries will get deleted even though the second entry's combined_far is not 0. 412 if in_this_filter != '': 413 # join the needed tables to in_this_filter 414 in_this_filter = ''.join([ sqlutils.join_experiment_tables_to_coinc_table(recovery_table), "\n WHERE\n\t", in_this_filter ]) 415 sqlscript = ''.join([ """ 416 CREATE TEMP TABLE del_ids AS 417 SELECT 418 sim_id AS del_id 419 FROM 420 sim_rec_map 421 WHERE 422 rec_id NOT IN ( 423 SELECT 424 """, recovery_table, """.coinc_event_id 425 FROM 426 """, recovery_table, """ 427 """, in_this_filter, """ 428 AND experiment_summary.datatype == "simulation" 429 ); 430 431 DELETE FROM 432 sim_rec_map 433 WHERE 434 sim_id IN ( 435 SELECT 436 del_id 437 FROM 438 del_ids ); 439 440 DROP TABLE del_ids;""" ]) 441 connection.cursor().executescript(sqlscript) 442 443 # 444 # Set other needed functions 445 # 446 447 # establish what units will be converting duration to 448 def convert_duration( duration ): 449 return sqlutils.convert_duration( duration, convert_durations )

450 connection.create_function( 'convert_duration', 1, convert_duration ) 451 452 # create the get_sim_tag function 453 sim_map = sqlutils.sim_tag_proc_id_mapper( connection ) 454 connection.create_function( 'get_sim_tag', 1, sim_map.get_sim_tag ) 455 456 # Get range ranks 457 if rank_range is not None: 458 rank_range_parser = sqlutils.parse_param_ranges( 'rank(sim_rec_map', 'ranking_stat)', 459 rank_range, verbose = verbose ) 460 461 # 462 # Create and prepare the SelectedFoundTable to store summary information 463 # 464 465 # Get recovery table and simulation table column names from database 466 simulation_table_columns = sqlutils.get_column_names_from_table( connection, simulation_table ) 467 recovery_table_columns = sqlutils.get_column_names_from_table( connection, recovery_table ) 468 469 # Get list of column name for injected parameters 470 injected_cols = [] 471 for col in simulation_table_columns: 472 if col == 'simulation_id': 473 injected_cols.append('simulation_id') 474 else: 475 injected_cols.append('injected_'+col) 476 injected_cols.extend(['injected_decisive_distance','injected_gps_time', 'injected_gps_time_ns', 'injected_event_time_utc__Px_click_for_daily_ihope_xP_']) 477 478 # Get list of column names from the recovery table 479 recovered_cols = [] 480 for col in recovery_table_columns: 481 if col == 'coinc_event_id': 482 recovered_cols.append(u'coinc_event_id') 483 else: 484 recovered_cols.append(u'recovered_'+col) 485 # generate list of column names for the summary table 486 column_names = injected_cols + recovered_cols 487 # add instruments on, duration, mini_followups 488 rankname = 'rank_in_' + comparison_datatype.strip().lower() + '_using_' + ranking_stat.split('.')[-1] 489 durname = ''.join([ 'simulation', u'_duration__Px_', convert_durations, '_xP_' ]) 490 column_names.extend( [ rankname, 'recovered_match_rank', 'instruments_on', 'elogs', durname, 'mini_followup','omega_scan', 'sim_tag' ] ) 491 492 # 493 # define needed tables 494 # 495 class tmpSimTable(table.Table): 496 tableName = "tmp_sim_table" 497 validcolumns = dict([ [col, sqlutils.get_col_type(simulation_table, col)] for col in simulation_table_columns ]) 498 class tmpSim(object): 499 __slots__ = tmpSimTable.validcolumns.keys() 500 def get_gps_time(self, site): 501 if '%s_start_time' % site in self.__slots__: 502 return LIGOTimeGPS(getattr(self, '%s_start_time' % site ), getattr(self, '%s_start_time_ns' % site)) 503 elif '%s_end_time' % site in self.__slots__: 504 return LIGOTimeGPS(getattr(self, '%s_end_time' % site ), getattr(self, '%s_end_time_ns' % site)) 505 else: 506 raise AttributeError, "could not find an injected %s_start_time nor an injected %s_end_time" %(site,site) 507 def get_pyvalue(self): 508 return generic_get_pyvalue(self) 509 510 class tmpRecTable(table.Table): 511 tableName = "tmp_rec_table" 512 validcolumns = dict([ [col, sqlutils.get_col_type(recovery_table, col)] for col in recovery_table_columns ]) 513 class tmpRec(object): 514 __slots__ = tmpRecTable.validcolumns.keys() 515 def get_gps_time(self): 516 if 'start_time' in self.__slots__: 517 return LIGOTimeGPS(self.start_time, self.start_time_ns) 518 elif 'end_time' in self.__slots__: 519 return LIGOTimeGPS(self.end_time, self.end_time_ns) 520 else: 521 raise AttributeError, "could not find a recovered start_time or recovered end_time" 522 def get_pyvalue(self): 523 return generic_get_pyvalue(self) 524 525 class SelectedFoundTable(table.Table): 526 tableName = "selected_found_injections" 527 validcolumns = {} 528 for col_name in column_names: 529 if 'rank_in_' in col_name: 530 validcolumns[col_name] = "int_4u" 531 elif '_duration_' in col_name: 532 validcolumns[col_name] = "real_8" 533 elif 'instruments_on' == col_name: 534 validcolumns[col_name] = lsctables.ExperimentTable.validcolumns['instruments'] 535 elif col_name == 'injected_start_time' or col_name == 'injected_start_time_ns': 536 validcolumns[col_name] = "int_4s" 537 elif col_name == 'injected_gps_time' or col_name == 'injected_gps_time_ns': 538 validcolumns[col_name] = "int_4s" 539 elif col_name == 'injected_decisive_distance': 540 validcolumns[col_name] = "real_8" 541 elif 'injected_' in col_name or col_name == 'simulation_id': 542 validcolumns[col_name] = sqlutils.get_col_type(simulation_table, re.sub('injected_', '', col_name)) 543 elif 'recovered_' in col_name or col_name == 'coinc_event_id': 544 validcolumns[col_name] = sqlutils.get_col_type(recovery_table, re.sub('recovered_', '', col_name)) 545 # if custom columns exist in the database, just set them to lstrings 546 else: 547 validcolumns[col_name] = "lstring" 548 # add FAP columns 549 validcolumns['recovered_fap'] = "real_8" 550 validcolumns['recovered_fap_1yr'] = "real_8" 551 552 class SelectedFound(object): 553 __slots__ = SelectedFoundTable.validcolumns.keys() 554 555 def set_recovered_gps_time(self, gps_time): 556 if 'recovered_start_time' in self.__slots__: 557 self.recovered_start_time = gps_time.seconds 558 self.recovered_start_time_ns = gps_time.nanoseconds 559 elif 'recovered_end_time' in self.__slots__: 560 self.recovered_end_time = gps_time.seconds 561 self.recovered_end_time_ns = gps_time.nanoseconds 562 else: 563 raise AttributeError, "could not find a recovered_start_time or recovered_end_time" 564 565 def get_recovered_gps_time(self): 566 if 'recovered_start_time' in self.__slots__: 567 return LIGOTimeGPS(self.recovered_start_time, self.recovered_start_time_ns) 568 elif 'recovered_end_time' in self.__slots__: 569 return LIGOTimeGPS(self.recovered_end_time, self.recovered_end_time_ns) 570 else: 571 raise AttributeError, "could not find a recovered_start_time or recovered_end_time" 572 573 def set_injected_gps_time(self, site, gps_time): 574 if 'injected_%s_start_time' % site in self.__slots__: 575 setattr(self, 'injected_%s_start_time' % site, gps_time.seconds) 576 setattr(self, 'injected_%s_start_time_ns' % site, gps_time.nanoseconds) 577 elif 'injected_%s_end_time' % site in self.__slots__: 578 setattr(self, 'injected_%s_end_time' % site, gps_time.seconds) 579 setattr(self, 'injected_%s_end_time_ns' % site, gps_time.nanoseconds) 580 else: 581 raise AttributeError, "could not find an injected_%s_start_time nor an injected_%s_end_time" %(site,site) 582 583 def get_injected_gps_time(self, site): 584 if 'injected_%s_start_time' % site in self.__slots__: 585 return LIGOTimeGPS(getattr(self, 'injected_%s_start_time' % site ), getattr(self, 'injected_%s_start_time_ns' % site)) 586 elif 'injected_%s_end_time' % site in self.__slots__: 587 return LIGOTimeGPS(getattr(self, 'injected_%s_end_time' % site ), getattr(self, 'injected_%s_end_time_ns' % site)) 588 else: 589 raise AttributeError, "could not find an injected_%s_start_time nor an injected_%s_end_time" %(site,site) 590 591 def get_pyvalue(self): 592 return generic_get_pyvalue(self) 593 594 # connect the rows to the tables 595 tmpSimTable.RowType = tmpSim 596 tmpRecTable.RowType = tmpRec 597 SelectedFoundTable.RowType = SelectedFound 598 599 # 600 # Get the Data 601 # 602 tmp_sim_table = lsctables.New(tmpSimTable) 603 tmp_rec_table = lsctables.New(tmpRecTable) 604 tmp_sftable = lsctables.New(SelectedFoundTable) 605 prior_sim_id = '' 606 group_rank = None 607 current_match_rank = 1 608 sqlquery = ''.join([""" 609 SELECT 610 """, simulation_table, """.*, 611 """, recovery_table, """.*, 612 get_sim_tag(experiment_summary.sim_proc_id), 613 rank(sim_rec_map.ranking_stat), 614 NULL AS match_rank, 615 experiment.instruments, 616 convert_duration(experiment_summary.duration), 617 NULL AS mini_followup 618 FROM 619 sim_rec_map 620 JOIN 621 """, ', '.join([simulation_table, recovery_table]), """, experiment, experiment_summary, experiment_map ON ( 622 sim_rec_map.sim_id == """, simulation_table, """.simulation_id AND 623 sim_rec_map.rec_id == """, recovery_table, """.coinc_event_id AND 624 sim_rec_map.rec_id == experiment_map.coinc_event_id AND 625 experiment_map.experiment_summ_id == experiment_summary.experiment_summ_id AND 626 experiment_summary.experiment_id == experiment.experiment_id) 627 ORDER BY 628 sim_rec_map.sim_id, sim_rec_map.ranking_stat """, rank_by]) 629 630 if verbose: 631 print >> sys.stderr, "Getting coincs..." 632 print >> sys.stderr, "SQLite query used is:" 633 print >> sys.stderr, sqlquery 634 635 for values in connection.cursor().execute( sqlquery ).fetchall(): 636 # sort the data 637 tmp_sim_row = tmpSim() 638 tmp_rec_row = tmpRec() 639 [ setattr(tmp_sim_row, column, values[ii]) for ii, column in enumerate(simulation_table_columns) ] 640 [ setattr(tmp_rec_row, column, values[ii+1+jj]) for jj, column in enumerate(recovery_table_columns) ] 641 # figure out the rank 642 this_inj_rank = values[-5] 643 this_sim_id = tmp_sim_row.simulation_id 644 this_ranking_stat = getattr(tmp_rec_row, ranking_stat.split('.')[-1]) 645 if this_sim_id == prior_sim_id and this_ranking_stat != prior_ranking_stat: 646 current_match_rank += 1 647 elif this_sim_id != prior_sim_id: 648 current_match_rank = 1 649 group_rank = this_inj_rank 650 prior_sim_id = this_sim_id 651 prior_ranking_stat = this_ranking_stat 652 # only store data if the group_rank falls in the desired rank ranges 653 if rank_range and rank_range_parser.group_by_param_range(group_rank) is None: 654 continue 655 on_instruments = lsctables.instrument_set_from_ifos(values[-3]) 656 duration = values[-2] 657 # now that have all the information from this row, create a row for the selected found table 658 sfrow = SelectedFound() 659 # set the ranks 660 setattr(sfrow, rankname, group_rank) 661 sfrow.recovered_match_rank = current_match_rank 662 # set the injected parameters 663 use_this_site = sorted(on_instruments)[0][0].lower() 664 injected_time = tmp_sim_row.get_gps_time( use_this_site ) 665 sfrow.injected_gps_time = injected_time.seconds 666 sfrow.injected_gps_time_ns = injected_time.nanoseconds 667 for col in simulation_table_columns: 668 if col == "simulation_id": 669 sfrow.simulation_id = tmp_sim_row.simulation_id 670 else: 671 setattr(sfrow, 'injected_'+col, getattr( tmp_sim_row, col) ) 672 # set the recovered parameters 673 for col in recovery_table_columns: 674 if col == "coinc_event_id": 675 sfrow.coinc_event_id = tmp_rec_row.coinc_event_id 676 else: 677 setattr(sfrow, 'recovered_'+col, getattr( tmp_rec_row, col) ) 678 # calculate and add faps 679 if sfrow.recovered_combined_far is not None: 680 t_in_s = float(values[-2]) / sqlutils.convert_duration(1, convert_durations) 681 sfrow.recovered_fap = 1 - math.exp(-sqlutils.convert_duration(t_in_s, 'yr') * sfrow.recovered_combined_far) 682 sfrow.recovered_fap_1yr = 1 - math.exp(-sfrow.recovered_combined_far) 683 else: 684 sfrow.recovered_fap = None 685 sfrow.recovered_fap_1yr = None 686 # set elog page 687 elog_pages = [(ifo, get_elog_page(ifo, sfrow.injected_gps_time)) for ifo in on_instruments] 688 sfrow.elogs = ','.join([ create_hyperlink(elog[1], elog[0]) for elog in sorted(elog_pages) ]) 689 # set daily_ihope page 690 event_time_utc = format_end_time_in_utc( sfrow.injected_gps_time ) 691 daily_ihope_address = get_daily_ihope_page(sfrow.injected_gps_time, pages_location = daily_ihope_pages_location) 692 sfrow.injected_event_time_utc__Px_click_for_daily_ihope_xP_ = create_hyperlink( daily_ihope_address, event_time_utc ) 693 # set any other info 694 sfrow.instruments_on = ','.join(sorted(on_instruments)) 695 sfrow.injected_decisive_distance = sorted([getattr(sfrow, 'injected_eff_dist_%s' % ifo[0].lower()) for ifo in on_instruments])[1] 696 sfrow.mini_followup = None 697 sfrow.omega_scan = None 698 sfrow.sim_tag = values[-6] 699 setattr(sfrow, durname, duration) 700 701 # add the row 702 tmp_sftable.append(sfrow) 703 704 # Re-sort the sftable by rank, recovered_match_rank 705 sftable = lsctables.New(SelectedFoundTable) 706 for sfrow in sorted([ row for row in tmp_sftable ], key = lambda row: getattr(row, sort_by == 'rank' and rankname or sort_by) ): 707 if sfrow.simulation_id not in [row.simulation_id for row in sftable]: 708 sftable.append(sfrow) 709 sftable.extend(sub_row for sub_row in sorted([row for row in tmp_sftable 710 if row.simulation_id == sfrow.simulation_id 711 and row.coinc_event_id != sfrow.coinc_event_id], 712 key = lambda row: row.recovered_match_rank)) 713 714 # drop the sim_rec_map table 715 connection.cursor().execute("DROP TABLE sim_rec_map") 716 717 return sftable 718

719 720 721 -def printmissed(connection, simulation_table, recovery_table, map_label, livetime_program, 722 param_name = None, param_ranges = None, exclude_coincs = None, include_only_coincs = None, sim_tag = 'ALLINJ', 723 limit = None, daily_ihope_pages_location = 'https://ldas-jobs.ligo.caltech.edu/~cbc/ihope_daily', verbose = False):

724 725 from pylal import ligolw_sqlutils as sqlutils 726 from pylal import ligolw_cbc_compute_durations as compute_dur 727 from glue import segments 728 from glue.ligolw import dbtables 729 730 # Get simulation/recovery tables 731 simulation_table = sqlutils.validate_option(simulation_table) 732 recovery_table = sqlutils.validate_option(recovery_table) 733 734 # create the get_sim_tag function 735 sim_map = sqlutils.sim_tag_proc_id_mapper( connection ) 736 connection.create_function( 'get_sim_tag', 1, sim_map.get_sim_tag ) 737 738 # 739 # Create and prepare the CloseMissedTable to store summary information 740 # 741 742 # Get simulation table column names from database 743 simulation_table_columns = sqlutils.get_column_names_from_table( connection, simulation_table ) 744 column_names = simulation_table_columns + \ 745 ['rank', 'decisive_distance', 'gps_time', 'gps_time_ns', 'injection_time_utc__Px_click_for_daily_ihope_xP_', 'elogs', 'instruments_on', 'veto_def_name', 'mini_followup','omega_scan', 'sim_tag'] 746 747 748 # define needed tables 749 class CloseMissedTable(table.Table): 750 tableName = "close_missed_injections" 751 validcolumns = {} 752 for col_name in column_names: 753 if 'rank' in col_name: 754 validcolumns[col_name] = "int_4u" 755 elif 'instruments_on' == col_name: 756 validcolumns[col_name] = lsctables.ExperimentTable.validcolumns['instruments'] 757 elif 'veto_def_name' == col_name: 758 validcolumns[col_name] = lsctables.ExperimentSummaryTable.validcolumns['veto_def_name'] 759 elif 'decisive_distance' == col_name: 760 validcolumns[col_name] = sqlutils.get_col_type(simulation_table, 'eff_dist_h') 761 elif 'gps_time' == col_name or 'gps_time_ns' == col_name: 762 validcolumns[col_name] = "int_4s" 763 elif 'sim_tag' == col_name: 764 validcolumns[col_name] = "lstring" 765 else: 766 validcolumns[col_name] = sqlutils.get_col_type(simulation_table, col_name, default = 'lstring')

767 768 class CloseMissed(object): 769 __slots__ = CloseMissedTable.validcolumns.keys() 770 771 def get_pyvalue(self): 772 return generic_get_pyvalue(self) 773 774 # connect the rows to the tables 775 CloseMissedTable.RowType = CloseMissed 776 777 # create the table 778 cmtable = lsctables.New(CloseMissedTable) 779 780 # set up sim_rec_map table 781 sqlutils.create_sim_rec_map_table(connection, simulation_table, recovery_table, map_label, None) 782 783 # 784 # Set table filters 785 # 786 787 # we force the include/exclude filters to None; will check for excluded/included ifo time 788 # when cycling through the ifo times 789 filter = """ 790 WHERE 791 simulation_id NOT IN ( 792 SELECT 793 sim_id 794 FROM 795 sim_rec_map )""" 796 af = create_filter( connection, simulation_table, param_name = param_name, param_ranges = param_ranges, 797 exclude_coincs = None, include_only_coincs = None, sim_tag = sim_tag, verbose = verbose) 798 af = re.sub(r'experiment_summary[.]sim_proc_id', 'process_id', af) 799 if af != '': 800 filter = '\n'.join([ filter, """ 801 AND""", af]) 802 # get desired instrument times 803 if include_only_coincs is not None: 804 include_times = [on_instruments for on_instruments, type in 805 sqlutils.parse_coinc_options( include_only_coincs, verbose = verbose ).get_coinc_types().items() 806 if 'ALL' in type] 807 if exclude_coincs is not None: 808 exclude_times = [on_instruments for on_instruments, type in 809 sqlutils.parse_coinc_options( exclude_coincs, verbose = verbose ).get_coinc_types().items() 810 if 'ALL' in type] 811 812 # get the usertags of inspiral jobs in this database 813 sqlquery = """ 814 SELECT value 815 FROM process_params 816 WHERE param == "-userTag" 817 GROUP BY value 818 """ 819 usertags = set(usertag[0] for usertag in connection.cursor().execute(sqlquery) ) 820 821 # Get the single-ifo science segments after CAT-1 vetoes 822 try: 823 if "FULL_DATA" in usertags: 824 tag = "FULL_DATA" 825 else: 826 tag = list(usertags)[0] 827 except IndexError: 828 # This is hacky anyway, so let's just take a guess 829 tag = "FULL_DATA" 830 ifo_segments = compute_dur.get_single_ifo_segments(connection, program_name = livetime_program, usertag = tag) 831 if ifo_segments == {}: 832 raise ValueError, "Cannot find any analysis segments using %s as a livetime program; cannot get missed injections." % livetime_program 833 834 if verbose: 835 print >> sys.stderr, "Getting all veto category names from the experiment_summary table..." 836 837 xmldoc = dbtables.get_xml(connection) 838 # get veto_segments 839 veto_segments = compute_dur.get_veto_segments(xmldoc, verbose) 840 841 # make a dictionary of zerolag "shifts" needed for the get_coinc_segments function 842 zerolag_dict = {} 843 for ifo in ifo_segments: 844 zerolag_dict[ifo] = 0.0 845 846 # Cycle over available veto categories 847 for veto_def_name, veto_seg_dict in veto_segments.items(): 848 post_vetoes_ifosegs = ifo_segments - veto_seg_dict 849 850 # make a dictionary of coincident segments by exclusive on-ifos 851 coinc_segs = compute_dur.get_coinc_segments(post_vetoes_ifosegs, zerolag_dict) 852 853 # 854 # Get all the on_instrument times and cycle over them 855 # 856 sqlquery = """ 857 SELECT DISTINCT experiment.instruments 858 FROM experiment 859 JOIN experiment_summary ON ( 860 experiment.experiment_id == experiment_summary.experiment_id ) 861 WHERE experiment_summary.veto_def_name == :1 862 """ 863 for on_instruments in connection.cursor().execute(sqlquery, (veto_def_name,)).fetchall(): 864 on_instruments = lsctables.instrument_set_from_ifos(on_instruments[0]) 865 866 # check if this on_instruments is desired; if not, skip 867 if include_only_coincs is not None and frozenset(on_instruments) not in include_times: 868 continue 869 if exclude_coincs is not None and frozenset(on_instruments) in exclude_times: 870 continue 871 872 on_times = coinc_segs[','.join(sorted(on_instruments))] 873 874 def is_in_on_time(gps_time, gps_time_ns): 875 return LIGOTimeGPS(gps_time, gps_time_ns) in on_times 876 877 connection.create_function('is_in_on_time', 2, is_in_on_time) 878 879 # add the check for on time to the filter 880 in_this_filter = filter 881 # figure out if simulation_table has end_times or start_times 882 end_or_start = any('end_time' in c for c in simulation_table_columns) and '_end_time' or '_start_time' 883 for instrument in on_instruments: 884 inst_time = instrument.lower()[0] + end_or_start 885 inst_time_ns = inst_time + '_ns' 886 in_this_filter = ''.join([ in_this_filter, 887 '\n\tAND is_in_on_time(', inst_time, ',', inst_time_ns, ')' ]) 888 889 # 890 # Set up decisive distance argument 891 # 892 893 def get_decisive_distance( *args ): 894 return sorted(args)[1] 895 896 connection.create_function('get_decisive_distance', len(on_instruments), get_decisive_distance) 897 decisive_distance = ''.join(['get_decisive_distance(', ','.join(['eff_dist_'+inst.lower()[0] for inst in on_instruments]), ')' ]) 898 899 # 900 # Initialize ranking. Statistics for ranking are based on decisive distance 901 # 902 if verbose: 903 print >> sys.stderr, "Getting statistics for ranking..." 904 ranker = sqlutils.rank_stats(simulation_table, decisive_distance, 'ASC') 905 # add requirement that stats not be found in the sim_rec_table to in_this_filter 906 ranker.populate_stats_list(connection, limit = limit, filter = in_this_filter) 907 connection.create_function( 'rank', 1, ranker.get_rank ) 908 909 # 910 # Get the Data 911 # 912 sqlquery = ''.join([""" 913 SELECT 914 *, 915 get_sim_tag(process_id), 916 """, decisive_distance, """, 917 rank(""", decisive_distance, """) 918 FROM 919 """, simulation_table, """ 920 """, in_this_filter, """ 921 %s""" % (limit is not None and ''.join(['AND rank(', decisive_distance, ') <= ', str(limit)]) or ''), """ 922 ORDER BY 923 rank(""", decisive_distance, """) ASC 924 """]) 925 926 if verbose: 927 print >> sys.stderr, "Getting injections..." 928 print >> sys.stderr, "SQLite query used is:" 929 print >> sys.stderr, sqlquery 930 931 for values in connection.cursor().execute( sqlquery ).fetchall(): 932 cmrow = CloseMissed() 933 [ setattr(cmrow, column, values[ii]) for ii, column in enumerate(simulation_table_columns) ] 934 cmrow.decisive_distance = values[-2] 935 cmrow.rank = values[-1] 936 cmrow.instruments_on = lsctables.ifos_from_instrument_set(on_instruments) 937 cmrow.veto_def_name = veto_def_name 938 cmrow.sim_tag = values[-3] 939 cmrow.mini_followup = None 940 cmrow.omega_scan = None 941 cmrow.gps_time = getattr(cmrow, sorted(on_instruments)[0][0].lower() + end_or_start) 942 cmrow.gps_time_ns = getattr(cmrow, sorted(on_instruments)[0][0].lower() + end_or_start + '_ns') 943 # set elog page 944 elog_pages = [(ifo, get_elog_page(ifo, cmrow.gps_time)) for ifo in on_instruments] 945 cmrow.elogs = ','.join([ create_hyperlink(elog[1], elog[0]) for elog in sorted(elog_pages) ]) 946 # set daily_ihope page 947 injection_time_utc = format_end_time_in_utc( cmrow.gps_time ) 948 daily_ihope_address = get_daily_ihope_page(cmrow.gps_time, pages_location = daily_ihope_pages_location) 949 cmrow.injection_time_utc__Px_click_for_daily_ihope_xP_ = create_hyperlink( daily_ihope_address, injection_time_utc ) 950 951 # add the row 952 cmtable.append(cmrow) 953 954 # drop the sim_rec_map table 955 connection.cursor().execute("DROP TABLE sim_rec_map") 956 957 return cmtable 958

959 -def get_sngl_info(connection, summary_table, sngl_table, daily_ihope_pages_location = 'https://ldas-jobs.ligo.caltech.edu/~cbc/ihope_daily', verbose = False):

960 961 from pylal import ligolw_sqlutils as sqlutils 962 963 if verbose: 964 print >> sys.stderr, "Getting Sngl table info..." 965 966 # index the summary table 967 lcindex = dict([ [row.coinc_event_id, row] for row in summary_table ]) 968 969 # get sngl table columns 970 sngl_table_cols = sqlutils.get_column_names_from_table( connection, sngl_table ) 971 972 # create an object to store the info 973 import copy 974 class MergedTable(table.Table): 975 tableName = summary_table.tableName 976 validcolumns = copy.deepcopy(summary_table.validcolumns) 977 for sngl_col in sngl_table_cols: 978 validcolumns['sngl_'+sngl_col] = sqlutils.get_col_type(sngl_table, sngl_col) 979 validcolumns['sngl_event_time_utc__Px_click_for_daily_ihope_xP_'] = 'lstring' 980 validcolumns['sngl_ifo__Px_click_for_elog_xP_'] = 'lstring'

981 982 class Merged(object): 983 __slots__ = MergedTable.validcolumns.keys() 984 _effective_snr = None 985 _new_snr = None 986 987 def get_sngl_gps_time(self): 988 if 'sngl_start_time' in self.__slots__: 989 return self.sngl_start_time 990 elif 'sngl_end_time' in self.__slots__: 991 return self.sngl_end_time 992 else: 993 raise AttributeError, "could not find a sngl_start_time or sngl_end_time" 994 995 @property 996 def new_snr(self): 997 if not self._new_snr: 998 self._new_snr = lsctables.SnglInspiral.get_new_snr(self) 999 return self._new_snr 1000 1001 @new_snr.setter 1002 def new_snr(self): 1003 err_msg = "The new_snr property cannot be set directly." 1004 raise ValueError(err_msg) 1005 1006 @property 1007 def effective_snr(self): 1008 if not self._effective_snr: 1009 self._effective_snr = \ 1010 lsctables.SnglInspiral.get_effective_snr(self) 1011 return self._new_snr 1012 1013 @new_snr.setter 1014 def effective_snr(self): 1015 err_msg = "The effective_snr property cannot be set directly." 1016 raise ValueError(err_msg) 1017 1018 def get_pyvalue(self): 1019 return printutils.generic_get_pyvalue() 1020 1021 # connect the row to the table 1022 MergedTable.RowType = Merged 1023 1024 # 1025 # create a document 1026 # 1027 mtable = lsctables.New(MergedTable) 1028 1029 sqlquery = "CREATE TEMP TABLE select_ceids (coinc_event_id)" 1030 connection.cursor().execute(sqlquery) 1031 sqlquery = "INSERT INTO select_ceids (coinc_event_id) VALUES (?)" 1032 connection.cursor().executemany(sqlquery, [(row.coinc_event_id,) for row in summary_table]) 1033 1034 sqlquery = ''.join([""" 1035 SELECT 1036 select_ceids.coinc_event_id, 1037 """, sngl_table, """.* 1038 FROM 1039 """, sngl_table, ''' 1040 JOIN 1041 select_ceids, coinc_event_map 1042 ON 1043 select_ceids.coinc_event_id == coinc_event_map.coinc_event_id AND 1044 coinc_event_map.table_name == "''', sngl_table, """" AND 1045 coinc_event_map.event_id == """, sngl_table, """.event_id 1046 """]) 1047 if verbose: 1048 print >> sys.stderr, "SQLite Query Used is:" 1049 print >> sys.stderr, sqlquery 1050 for data in connection.cursor().execute(sqlquery).fetchall(): 1051 mrow = Merged() 1052 data = list(data) 1053 mrow.coinc_event_id = data.pop(0) 1054 # set sngl info 1055 for sngl_col, value in zip(sngl_table_cols, data): 1056 setattr(mrow, 'sngl_'+sngl_col, value) 1057 # set coinc info 1058 [setattr(mrow, col, getattr(lcindex[mrow.coinc_event_id], col)) 1059 for col in lcindex[mrow.coinc_event_id].__slots__ if col != 'coinc_event_id'] 1060 # set sngl end time utc 1061 gps_time_utc = format_end_time_in_utc( mrow.get_sngl_gps_time() ) 1062 daily_ihope_address = get_daily_ihope_page(mrow.get_sngl_gps_time(), pages_location = daily_ihope_pages_location) 1063 mrow.sngl_event_time_utc__Px_click_for_daily_ihope_xP_ = create_hyperlink( daily_ihope_address, gps_time_utc ) 1064 # set elog page 1065 mrow.sngl_ifo__Px_click_for_elog_xP_ = create_hyperlink( get_elog_page(mrow.sngl_ifo, mrow.get_sngl_gps_time()), mrow.sngl_ifo ) 1066 # add the row 1067 mtable.append(mrow) 1068 1069 # delete the select_ceids table 1070 connection.cursor().execute('DROP TABLE select_ceids') 1071 1072 return mtable 1073