pyerrors.input.sfcf
1import os 2import fnmatch 3import re 4import numpy as np # Thinly-wrapped numpy 5from ..obs import Obs 6from .utils import sort_names, check_idl 7import itertools 8 9 10sep = "/" 11 12 13def read_sfcf(path, prefix, name, quarks='.*', corr_type="bi", noffset=0, wf=0, wf2=0, version="1.0c", cfg_separator="n", silent=False, **kwargs): 14 """Read sfcf files from given folder structure. 15 16 Parameters 17 ---------- 18 path : str 19 Path to the sfcf files. 20 prefix : str 21 Prefix of the sfcf files. 22 name : str 23 Name of the correlation function to read. 24 quarks : str 25 Label of the quarks used in the sfcf input file. e.g. "quark quark" 26 for version 0.0 this does NOT need to be given with the typical " - " 27 that is present in the output file, 28 this is done automatically for this version 29 corr_type : str 30 Type of correlation function to read. Can be 31 - 'bi' for boundary-inner 32 - 'bb' for boundary-boundary 33 - 'bib' for boundary-inner-boundary 34 noffset : int 35 Offset of the source (only relevant when wavefunctions are used) 36 wf : int 37 ID of wave function 38 wf2 : int 39 ID of the second wavefunction 40 (only relevant for boundary-to-boundary correlation functions) 41 im : bool 42 if True, read imaginary instead of real part 43 of the correlation function. 44 names : list 45 Alternative labeling for replicas/ensembles. 46 Has to have the appropriate length 47 ens_name : str 48 replaces the name of the ensemble 49 version: str 50 version of SFCF, with which the measurement was done. 51 if the compact output option (-c) was specified, 52 append a "c" to the version (e.g. "1.0c") 53 if the append output option (-a) was specified, 54 append an "a" to the version 55 cfg_separator : str 56 String that separates the ensemble identifier from the configuration number (default 'n'). 57 replica: list 58 list of replica to be read, default is all 59 files: list 60 list of files to be read per replica, default is all. 61 for non-compact output format, hand the folders to be read here. 62 check_configs: list[list[int]] 63 list of list of supposed configs, eg. [range(1,1000)] 64 for one replicum with 1000 configs 65 66 Returns 67 ------- 68 result: list[Obs] 69 list of Observables with length T, observable per timeslice. 70 bb-type correlators have length 1. 71 """ 72 ret = read_sfcf_multi(path, prefix, [name], quarks_list=[quarks], corr_type_list=[corr_type], 73 noffset_list=[noffset], wf_list=[wf], wf2_list=[wf2], version=version, 74 cfg_separator=cfg_separator, silent=silent, **kwargs) 75 return ret[name][quarks][str(noffset)][str(wf)][str(wf2)] 76 77 78def read_sfcf_multi(path, prefix, name_list, quarks_list=['.*'], corr_type_list=['bi'], noffset_list=[0], wf_list=[0], wf2_list=[0], version="1.0c", cfg_separator="n", silent=False, keyed_out=False, **kwargs): 79 """Read sfcf files from given folder structure. 80 81 Parameters 82 ---------- 83 path : str 84 Path to the sfcf files. 85 prefix : str 86 Prefix of the sfcf files. 87 name : str 88 Name of the correlation function to read. 89 quarks_list : list[str] 90 Label of the quarks used in the sfcf input file. e.g. "quark quark" 91 for version 0.0 this does NOT need to be given with the typical " - " 92 that is present in the output file, 93 this is done automatically for this version 94 corr_type_list : list[str] 95 Type of correlation function to read. Can be 96 - 'bi' for boundary-inner 97 - 'bb' for boundary-boundary 98 - 'bib' for boundary-inner-boundary 99 noffset_list : list[int] 100 Offset of the source (only relevant when wavefunctions are used) 101 wf_list : int 102 ID of wave function 103 wf2_list : list[int] 104 ID of the second wavefunction 105 (only relevant for boundary-to-boundary correlation functions) 106 im : bool 107 if True, read imaginary instead of real part 108 of the correlation function. 109 names : list 110 Alternative labeling for replicas/ensembles. 111 Has to have the appropriate length 112 ens_name : str 113 replaces the name of the ensemble 114 version: str 115 version of SFCF, with which the measurement was done. 116 if the compact output option (-c) was specified, 117 append a "c" to the version (e.g. "1.0c") 118 if the append output option (-a) was specified, 119 append an "a" to the version 120 cfg_separator : str 121 String that separates the ensemble identifier from the configuration number (default 'n'). 122 replica: list 123 list of replica to be read, default is all 124 files: list[list[int]] 125 list of files to be read per replica, default is all. 126 for non-compact output format, hand the folders to be read here. 127 check_configs: list[list[int]] 128 list of list of supposed configs, eg. [range(1,1000)] 129 for one replicum with 1000 configs 130 131 Returns 132 ------- 133 result: dict[list[Obs]] 134 dict with one of the following properties: 135 if keyed_out: 136 dict[key] = list[Obs] 137 where key has the form name/quarks/offset/wf/wf2 138 if not keyed_out: 139 dict[name][quarks][offset][wf][wf2] = list[Obs] 140 """ 141 142 if kwargs.get('im'): 143 im = 1 144 part = 'imaginary' 145 else: 146 im = 0 147 part = 'real' 148 149 known_versions = ["0.0", "1.0", "2.0", "1.0c", "2.0c", "1.0a", "2.0a"] 150 151 if version not in known_versions: 152 raise Exception("This version is not known!") 153 if (version[-1] == "c"): 154 appended = False 155 compact = True 156 version = version[:-1] 157 elif (version[-1] == "a"): 158 appended = True 159 compact = False 160 version = version[:-1] 161 else: 162 compact = False 163 appended = False 164 ls = [] 165 if "replica" in kwargs: 166 ls = kwargs.get("replica") 167 else: 168 for (dirpath, dirnames, filenames) in os.walk(path): 169 if not appended: 170 ls.extend(dirnames) 171 else: 172 ls.extend(filenames) 173 break 174 if not ls: 175 raise Exception('Error, directory not found') 176 # Exclude folders with different names 177 for exc in ls: 178 if not fnmatch.fnmatch(exc, prefix + '*'): 179 ls = list(set(ls) - set([exc])) 180 181 if not appended: 182 ls = sort_names(ls) 183 replica = len(ls) 184 185 else: 186 replica = len([file.split(".")[-1] for file in ls]) // len(set([file.split(".")[-1] for file in ls])) 187 if replica == 0: 188 raise Exception('No replica found in directory') 189 if not silent: 190 print('Read', part, 'part of', name_list, 'from', prefix[:-1], ',', replica, 'replica') 191 192 if 'names' in kwargs: 193 new_names = kwargs.get('names') 194 if len(new_names) != len(set(new_names)): 195 raise Exception("names are not unique!") 196 if len(new_names) != replica: 197 raise Exception('names should have the length', replica) 198 199 else: 200 ens_name = kwargs.get("ens_name") 201 if not appended: 202 new_names = _get_rep_names(ls, ens_name) 203 else: 204 new_names = _get_appended_rep_names(ls, prefix, name_list[0], ens_name) 205 new_names = sort_names(new_names) 206 207 idl = [] 208 209 noffset_list = [str(x) for x in noffset_list] 210 wf_list = [str(x) for x in wf_list] 211 wf2_list = [str(x) for x in wf2_list] 212 213 # setup dict structures 214 intern = {} 215 for name, corr_type in zip(name_list, corr_type_list): 216 intern[name] = {} 217 b2b, single = _extract_corr_type(corr_type) 218 intern[name]["b2b"] = b2b 219 intern[name]["single"] = single 220 intern[name]["spec"] = {} 221 for quarks in quarks_list: 222 intern[name]["spec"][quarks] = {} 223 for off in noffset_list: 224 intern[name]["spec"][quarks][off] = {} 225 for w in wf_list: 226 intern[name]["spec"][quarks][off][w] = {} 227 for w2 in wf2_list: 228 intern[name]["spec"][quarks][off][w][w2] = {} 229 intern[name]["spec"][quarks][off][w][w2]["pattern"] = _make_pattern(version, name, off, w, w2, intern[name]['b2b'], quarks) 230 231 internal_ret_dict = {} 232 needed_keys = _lists2key(name_list, quarks_list, noffset_list, wf_list, wf2_list) 233 for key in needed_keys: 234 internal_ret_dict[key] = [] 235 236 if not appended: 237 for i, item in enumerate(ls): 238 rep_path = path + '/' + item 239 if "files" in kwargs: 240 files = kwargs.get("files") 241 if isinstance(files, list): 242 if all(isinstance(f, list) for f in files): 243 files = files[i] 244 elif all(isinstance(f, str) for f in files): 245 files = files 246 else: 247 raise TypeError("files has to be of type list[list[str]] or list[str]!") 248 else: 249 raise TypeError("files has to be of type list[list[str]] or list[str]!") 250 251 else: 252 files = [] 253 sub_ls = _find_files(rep_path, prefix, compact, files) 254 rep_idl = [] 255 no_cfg = len(sub_ls) 256 for cfg in sub_ls: 257 try: 258 if compact: 259 rep_idl.append(int(cfg.split(cfg_separator)[-1])) 260 else: 261 rep_idl.append(int(cfg[3:])) 262 except Exception: 263 raise Exception("Couldn't parse idl from directory, problem with file " + cfg) 264 rep_idl.sort() 265 # maybe there is a better way to print the idls 266 if not silent: 267 print(item, ':', no_cfg, ' configurations') 268 idl.append(rep_idl) 269 # here we have found all the files we need to look into. 270 if i == 0: 271 if version != "0.0" and compact: 272 file = path + '/' + item + '/' + sub_ls[0] 273 for name in name_list: 274 if version == "0.0" or not compact: 275 file = path + '/' + item + '/' + sub_ls[0] + '/' + name 276 for key in _lists2key(quarks_list, noffset_list, wf_list, wf2_list): 277 specs = _key2specs(key) 278 quarks = specs[0] 279 off = specs[1] 280 w = specs[2] 281 w2 = specs[3] 282 # here, we want to find the place within the file, 283 # where the correlator we need is stored. 284 # to do so, the pattern needed is put together 285 # from the input values 286 start_read, T = _find_correlator(file, version, intern[name]["spec"][quarks][str(off)][str(w)][str(w2)]["pattern"], intern[name]['b2b'], silent=silent) 287 intern[name]["spec"][quarks][str(off)][str(w)][str(w2)]["start"] = start_read 288 intern[name]["T"] = T 289 # preparing the datastructure 290 # the correlators get parsed into... 291 deltas = [] 292 for j in range(intern[name]["T"]): 293 deltas.append([]) 294 internal_ret_dict[sep.join([name, key])] = deltas 295 296 if compact: 297 rep_deltas = _read_compact_rep(path, item, sub_ls, intern, needed_keys, im) 298 for key in needed_keys: 299 name = _key2specs(key)[0] 300 for t in range(intern[name]["T"]): 301 internal_ret_dict[key][t].append(rep_deltas[key][t]) 302 else: 303 for key in needed_keys: 304 rep_data = [] 305 name = _key2specs(key)[0] 306 for subitem in sub_ls: 307 cfg_path = path + '/' + item + '/' + subitem 308 file_data = _read_o_file(cfg_path, name, needed_keys, intern, version, im) 309 rep_data.append(file_data) 310 for t in range(intern[name]["T"]): 311 internal_ret_dict[key][t].append([]) 312 for cfg in range(no_cfg): 313 internal_ret_dict[key][t][i].append(rep_data[cfg][key][t]) 314 else: 315 for key in needed_keys: 316 specs = _key2specs(key) 317 name = specs[0] 318 quarks = specs[1] 319 off = specs[2] 320 w = specs[3] 321 w2 = specs[4] 322 if "files" in kwargs: 323 if isinstance(kwargs.get("files"), list) and all(isinstance(f, str) for f in kwargs.get("files")): 324 name_ls = kwargs.get("files") 325 else: 326 raise TypeError("In append mode, files has to be of type list[str]!") 327 else: 328 name_ls = ls 329 for exc in name_ls: 330 if not fnmatch.fnmatch(exc, prefix + '*.' + name): 331 name_ls = list(set(name_ls) - set([exc])) 332 name_ls = sort_names(name_ls) 333 pattern = intern[name]['spec'][quarks][off][w][w2]['pattern'] 334 deltas = [] 335 for rep, file in enumerate(name_ls): 336 rep_idl = [] 337 filename = path + '/' + file 338 T, rep_idl, rep_data = _read_append_rep(filename, pattern, intern[name]['b2b'], cfg_separator, im, intern[name]['single']) 339 if rep == 0: 340 intern[name]['T'] = T 341 for t in range(intern[name]['T']): 342 deltas.append([]) 343 for t in range(intern[name]['T']): 344 deltas[t].append(rep_data[t]) 345 internal_ret_dict[key] = deltas 346 if name == name_list[0]: 347 idl.append(rep_idl) 348 349 if kwargs.get("check_configs") is True: 350 if not silent: 351 print("Checking for missing configs...") 352 che = kwargs.get("check_configs") 353 if not (len(che) == len(idl)): 354 raise Exception("check_configs has to be the same length as replica!") 355 for r in range(len(idl)): 356 if not silent: 357 print("checking " + new_names[r]) 358 check_idl(idl[r], che[r]) 359 if not silent: 360 print("Done") 361 362 result_dict = {} 363 if keyed_out: 364 for key in needed_keys: 365 result = [] 366 for t in range(intern[name]["T"]): 367 result.append(Obs(internal_ret_dict[key][t], new_names, idl=idl)) 368 result_dict[key] = result 369 else: 370 for name in name_list: 371 result_dict[name] = {} 372 for quarks in quarks_list: 373 result_dict[name][quarks] = {} 374 for off in noffset_list: 375 result_dict[name][quarks][off] = {} 376 for w in wf_list: 377 result_dict[name][quarks][off][w] = {} 378 for w2 in wf2_list: 379 key = _specs2key(name, quarks, off, w, w2) 380 result = [] 381 for t in range(intern[name]["T"]): 382 result.append(Obs(internal_ret_dict[key][t], new_names, idl=idl)) 383 result_dict[name][quarks][str(off)][str(w)][str(w2)] = result 384 return result_dict 385 386 387def _lists2key(*lists): 388 keys = [] 389 for tup in itertools.product(*lists): 390 keys.append(sep.join(tup)) 391 return keys 392 393 394def _key2specs(key): 395 return key.split(sep) 396 397 398def _specs2key(*specs): 399 return sep.join(specs) 400 401 402def _read_o_file(cfg_path, name, needed_keys, intern, version, im): 403 return_vals = {} 404 for key in needed_keys: 405 file = cfg_path + '/' + name 406 specs = _key2specs(key) 407 if specs[0] == name: 408 with open(file) as fp: 409 lines = fp.readlines() 410 quarks = specs[1] 411 off = specs[2] 412 w = specs[3] 413 w2 = specs[4] 414 T = intern[name]["T"] 415 start_read = intern[name]["spec"][quarks][off][w][w2]["start"] 416 deltas = [] 417 for line in lines[start_read:start_read + T]: 418 floats = list(map(float, line.split())) 419 if version == "0.0": 420 deltas.append(floats[im - intern[name]["single"]]) 421 else: 422 deltas.append(floats[1 + im - intern[name]["single"]]) 423 return_vals[key] = deltas 424 return return_vals 425 426 427def _extract_corr_type(corr_type): 428 if corr_type == 'bb': 429 b2b = True 430 single = True 431 elif corr_type == 'bib': 432 b2b = True 433 single = False 434 else: 435 b2b = False 436 single = False 437 return b2b, single 438 439 440def _find_files(rep_path, prefix, compact, files=[]): 441 sub_ls = [] 442 if not files == []: 443 files.sort(key=lambda x: int(re.findall(r'\d+', x)[-1])) 444 else: 445 for (dirpath, dirnames, filenames) in os.walk(rep_path): 446 if compact: 447 sub_ls.extend(filenames) 448 else: 449 sub_ls.extend(dirnames) 450 break 451 if compact: 452 for exc in sub_ls: 453 if not fnmatch.fnmatch(exc, prefix + '*'): 454 sub_ls = list(set(sub_ls) - set([exc])) 455 sub_ls.sort(key=lambda x: int(re.findall(r'\d+', x)[-1])) 456 else: 457 for exc in sub_ls: 458 if not fnmatch.fnmatch(exc, 'cfg*'): 459 sub_ls = list(set(sub_ls) - set([exc])) 460 sub_ls.sort(key=lambda x: int(x[3:])) 461 files = sub_ls 462 if len(files) == 0: 463 raise FileNotFoundError("Did not find files in", rep_path, "with prefix", prefix, "and the given structure.") 464 return files 465 466 467def _make_pattern(version, name, noffset, wf, wf2, b2b, quarks): 468 if version == "0.0": 469 pattern = "# " + name + " : offset " + str(noffset) + ", wf " + str(wf) 470 if b2b: 471 pattern += ", wf_2 " + str(wf2) 472 qs = quarks.split(" ") 473 pattern += " : " + qs[0] + " - " + qs[1] 474 else: 475 pattern = 'name ' + name + '\nquarks ' + quarks + '\noffset ' + str(noffset) + '\nwf ' + str(wf) 476 if b2b: 477 pattern += '\nwf_2 ' + str(wf2) 478 return pattern 479 480 481def _find_correlator(file_name, version, pattern, b2b, silent=False): 482 T = 0 483 484 with open(file_name, "r") as my_file: 485 486 content = my_file.read() 487 match = re.search(pattern, content) 488 if match: 489 if version == "0.0": 490 start_read = content.count('\n', 0, match.start()) + 1 491 T = content.count('\n', start_read) 492 else: 493 start_read = content.count('\n', 0, match.start()) + 5 + b2b 494 end_match = re.search(r'\n\s*\n', content[match.start():]) 495 T = content[match.start():].count('\n', 0, end_match.start()) - 4 - b2b 496 if not T > 0: 497 raise ValueError("Correlator with pattern\n" + pattern + "\nis empty!") 498 if not silent: 499 print(T, 'entries, starting to read in line', start_read) 500 501 else: 502 raise ValueError('Correlator with pattern\n' + pattern + '\nnot found.') 503 504 return start_read, T 505 506 507def _read_compact_file(rep_path, cfg_file, intern, needed_keys, im): 508 return_vals = {} 509 with open(rep_path + cfg_file) as fp: 510 lines = fp.readlines() 511 for key in needed_keys: 512 keys = _key2specs(key) 513 name = keys[0] 514 quarks = keys[1] 515 off = keys[2] 516 w = keys[3] 517 w2 = keys[4] 518 519 T = intern[name]["T"] 520 start_read = intern[name]["spec"][quarks][off][w][w2]["start"] 521 # check, if the correlator is in fact 522 # printed completely 523 if (start_read + T + 1 > len(lines)): 524 raise Exception("EOF before end of correlator data! Maybe " + rep_path + cfg_file + " is corrupted?") 525 corr_lines = lines[start_read - 6: start_read + T] 526 t_vals = [] 527 528 if corr_lines[1 - intern[name]["b2b"]].strip() != 'name ' + name: 529 raise Exception('Wrong format in file', cfg_file) 530 531 for k in range(6, T + 6): 532 floats = list(map(float, corr_lines[k].split())) 533 t_vals.append(floats[-2:][im]) 534 return_vals[key] = t_vals 535 return return_vals 536 537 538def _read_compact_rep(path, rep, sub_ls, intern, needed_keys, im): 539 rep_path = path + '/' + rep + '/' 540 no_cfg = len(sub_ls) 541 542 return_vals = {} 543 for key in needed_keys: 544 name = _key2specs(key)[0] 545 deltas = [] 546 for t in range(intern[name]["T"]): 547 deltas.append(np.zeros(no_cfg)) 548 return_vals[key] = deltas 549 550 for cfg in range(no_cfg): 551 cfg_file = sub_ls[cfg] 552 cfg_data = _read_compact_file(rep_path, cfg_file, intern, needed_keys, im) 553 for key in needed_keys: 554 name = _key2specs(key)[0] 555 for t in range(intern[name]["T"]): 556 return_vals[key][t][cfg] = cfg_data[key][t] 557 return return_vals 558 559 560def _read_chunk(chunk, gauge_line, cfg_sep, start_read, T, corr_line, b2b, pattern, im, single): 561 try: 562 idl = int(chunk[gauge_line].split(cfg_sep)[-1]) 563 except Exception: 564 raise Exception("Couldn't parse idl from directory, problem with chunk around line ", gauge_line) 565 566 found_pat = "" 567 data = [] 568 for li in chunk[corr_line + 1:corr_line + 6 + b2b]: 569 found_pat += li 570 if re.search(pattern, found_pat): 571 for t, line in enumerate(chunk[start_read:start_read + T]): 572 floats = list(map(float, line.split())) 573 data.append(floats[im + 1 - single]) 574 return idl, data 575 576 577def _read_append_rep(filename, pattern, b2b, cfg_separator, im, single): 578 with open(filename, 'r') as fp: 579 content = fp.readlines() 580 data_starts = [] 581 for linenumber, line in enumerate(content): 582 if "[run]" in line: 583 data_starts.append(linenumber) 584 if len(set([data_starts[i] - data_starts[i - 1] for i in range(1, len(data_starts))])) > 1: 585 raise Exception("Irregularities in file structure found, not all runs have the same output length") 586 chunk = content[:data_starts[1]] 587 for linenumber, line in enumerate(chunk): 588 if line.startswith("gauge_name"): 589 gauge_line = linenumber 590 elif line.startswith("[correlator]"): 591 corr_line = linenumber 592 found_pat = "" 593 for li in chunk[corr_line + 1: corr_line + 6 + b2b]: 594 found_pat += li 595 if re.search(pattern, found_pat): 596 start_read = corr_line + 7 + b2b 597 break 598 else: 599 raise ValueError("Did not find pattern\n", pattern, "\nin\n", filename) 600 endline = corr_line + 6 + b2b 601 while not chunk[endline] == "\n": 602 endline += 1 603 T = endline - start_read 604 605 # all other chunks should follow the same structure 606 rep_idl = [] 607 rep_data = [] 608 609 for cnfg in range(len(data_starts)): 610 start = data_starts[cnfg] 611 stop = start + data_starts[1] 612 chunk = content[start:stop] 613 idl, data = _read_chunk(chunk, gauge_line, cfg_separator, start_read, T, corr_line, b2b, pattern, im, single) 614 rep_idl.append(idl) 615 rep_data.append(data) 616 617 data = [] 618 619 for t in range(T): 620 data.append([]) 621 for c in range(len(rep_data)): 622 data[t].append(rep_data[c][t]) 623 return T, rep_idl, data 624 625 626def _get_rep_names(ls, ens_name=None): 627 new_names = [] 628 for entry in ls: 629 try: 630 idx = entry.index('r') 631 except Exception: 632 raise Exception("Automatic recognition of replicum failed, please enter the key word 'names'.") 633 634 if ens_name: 635 new_names.append('ens_name' + '|' + entry[idx:]) 636 else: 637 new_names.append(entry[:idx] + '|' + entry[idx:]) 638 return new_names 639 640 641def _get_appended_rep_names(ls, prefix, name, ens_name=None): 642 new_names = [] 643 for exc in ls: 644 if not fnmatch.fnmatch(exc, prefix + '*.' + name): 645 ls = list(set(ls) - set([exc])) 646 ls.sort(key=lambda x: int(re.findall(r'\d+', x)[-1])) 647 for entry in ls: 648 myentry = entry[:-len(name) - 1] 649 try: 650 idx = myentry.index('r') 651 except Exception: 652 raise Exception("Automatic recognition of replicum failed, please enter the key word 'names'.") 653 654 if ens_name: 655 new_names.append('ens_name' + '|' + entry[idx:]) 656 else: 657 new_names.append(myentry[:idx] + '|' + myentry[idx:]) 658 return new_names
sep =
'/'
def
read_sfcf( path, prefix, name, quarks='.*', corr_type='bi', noffset=0, wf=0, wf2=0, version='1.0c', cfg_separator='n', silent=False, **kwargs):
14def read_sfcf(path, prefix, name, quarks='.*', corr_type="bi", noffset=0, wf=0, wf2=0, version="1.0c", cfg_separator="n", silent=False, **kwargs): 15 """Read sfcf files from given folder structure. 16 17 Parameters 18 ---------- 19 path : str 20 Path to the sfcf files. 21 prefix : str 22 Prefix of the sfcf files. 23 name : str 24 Name of the correlation function to read. 25 quarks : str 26 Label of the quarks used in the sfcf input file. e.g. "quark quark" 27 for version 0.0 this does NOT need to be given with the typical " - " 28 that is present in the output file, 29 this is done automatically for this version 30 corr_type : str 31 Type of correlation function to read. Can be 32 - 'bi' for boundary-inner 33 - 'bb' for boundary-boundary 34 - 'bib' for boundary-inner-boundary 35 noffset : int 36 Offset of the source (only relevant when wavefunctions are used) 37 wf : int 38 ID of wave function 39 wf2 : int 40 ID of the second wavefunction 41 (only relevant for boundary-to-boundary correlation functions) 42 im : bool 43 if True, read imaginary instead of real part 44 of the correlation function. 45 names : list 46 Alternative labeling for replicas/ensembles. 47 Has to have the appropriate length 48 ens_name : str 49 replaces the name of the ensemble 50 version: str 51 version of SFCF, with which the measurement was done. 52 if the compact output option (-c) was specified, 53 append a "c" to the version (e.g. "1.0c") 54 if the append output option (-a) was specified, 55 append an "a" to the version 56 cfg_separator : str 57 String that separates the ensemble identifier from the configuration number (default 'n'). 58 replica: list 59 list of replica to be read, default is all 60 files: list 61 list of files to be read per replica, default is all. 62 for non-compact output format, hand the folders to be read here. 63 check_configs: list[list[int]] 64 list of list of supposed configs, eg. [range(1,1000)] 65 for one replicum with 1000 configs 66 67 Returns 68 ------- 69 result: list[Obs] 70 list of Observables with length T, observable per timeslice. 71 bb-type correlators have length 1. 72 """ 73 ret = read_sfcf_multi(path, prefix, [name], quarks_list=[quarks], corr_type_list=[corr_type], 74 noffset_list=[noffset], wf_list=[wf], wf2_list=[wf2], version=version, 75 cfg_separator=cfg_separator, silent=silent, **kwargs) 76 return ret[name][quarks][str(noffset)][str(wf)][str(wf2)]
Read sfcf files from given folder structure.
Parameters
- path (str): Path to the sfcf files.
- prefix (str): Prefix of the sfcf files.
- name (str): Name of the correlation function to read.
- quarks (str): Label of the quarks used in the sfcf input file. e.g. "quark quark" for version 0.0 this does NOT need to be given with the typical " - " that is present in the output file, this is done automatically for this version
- corr_type (str):
Type of correlation function to read. Can be
- 'bi' for boundary-inner
- 'bb' for boundary-boundary
- 'bib' for boundary-inner-boundary
- noffset (int): Offset of the source (only relevant when wavefunctions are used)
- wf (int): ID of wave function
- wf2 (int): ID of the second wavefunction (only relevant for boundary-to-boundary correlation functions)
- im (bool): if True, read imaginary instead of real part of the correlation function.
- names (list): Alternative labeling for replicas/ensembles. Has to have the appropriate length
- ens_name (str): replaces the name of the ensemble
- version (str): version of SFCF, with which the measurement was done. if the compact output option (-c) was specified, append a "c" to the version (e.g. "1.0c") if the append output option (-a) was specified, append an "a" to the version
- cfg_separator (str): String that separates the ensemble identifier from the configuration number (default 'n').
- replica (list): list of replica to be read, default is all
- files (list): list of files to be read per replica, default is all. for non-compact output format, hand the folders to be read here.
- check_configs (list[list[int]]): list of list of supposed configs, eg. [range(1,1000)] for one replicum with 1000 configs
Returns
- result (list[Obs]): list of Observables with length T, observable per timeslice. bb-type correlators have length 1.
def
read_sfcf_multi( path, prefix, name_list, quarks_list=['.*'], corr_type_list=['bi'], noffset_list=[0], wf_list=[0], wf2_list=[0], version='1.0c', cfg_separator='n', silent=False, keyed_out=False, **kwargs):
79def read_sfcf_multi(path, prefix, name_list, quarks_list=['.*'], corr_type_list=['bi'], noffset_list=[0], wf_list=[0], wf2_list=[0], version="1.0c", cfg_separator="n", silent=False, keyed_out=False, **kwargs): 80 """Read sfcf files from given folder structure. 81 82 Parameters 83 ---------- 84 path : str 85 Path to the sfcf files. 86 prefix : str 87 Prefix of the sfcf files. 88 name : str 89 Name of the correlation function to read. 90 quarks_list : list[str] 91 Label of the quarks used in the sfcf input file. e.g. "quark quark" 92 for version 0.0 this does NOT need to be given with the typical " - " 93 that is present in the output file, 94 this is done automatically for this version 95 corr_type_list : list[str] 96 Type of correlation function to read. Can be 97 - 'bi' for boundary-inner 98 - 'bb' for boundary-boundary 99 - 'bib' for boundary-inner-boundary 100 noffset_list : list[int] 101 Offset of the source (only relevant when wavefunctions are used) 102 wf_list : int 103 ID of wave function 104 wf2_list : list[int] 105 ID of the second wavefunction 106 (only relevant for boundary-to-boundary correlation functions) 107 im : bool 108 if True, read imaginary instead of real part 109 of the correlation function. 110 names : list 111 Alternative labeling for replicas/ensembles. 112 Has to have the appropriate length 113 ens_name : str 114 replaces the name of the ensemble 115 version: str 116 version of SFCF, with which the measurement was done. 117 if the compact output option (-c) was specified, 118 append a "c" to the version (e.g. "1.0c") 119 if the append output option (-a) was specified, 120 append an "a" to the version 121 cfg_separator : str 122 String that separates the ensemble identifier from the configuration number (default 'n'). 123 replica: list 124 list of replica to be read, default is all 125 files: list[list[int]] 126 list of files to be read per replica, default is all. 127 for non-compact output format, hand the folders to be read here. 128 check_configs: list[list[int]] 129 list of list of supposed configs, eg. [range(1,1000)] 130 for one replicum with 1000 configs 131 132 Returns 133 ------- 134 result: dict[list[Obs]] 135 dict with one of the following properties: 136 if keyed_out: 137 dict[key] = list[Obs] 138 where key has the form name/quarks/offset/wf/wf2 139 if not keyed_out: 140 dict[name][quarks][offset][wf][wf2] = list[Obs] 141 """ 142 143 if kwargs.get('im'): 144 im = 1 145 part = 'imaginary' 146 else: 147 im = 0 148 part = 'real' 149 150 known_versions = ["0.0", "1.0", "2.0", "1.0c", "2.0c", "1.0a", "2.0a"] 151 152 if version not in known_versions: 153 raise Exception("This version is not known!") 154 if (version[-1] == "c"): 155 appended = False 156 compact = True 157 version = version[:-1] 158 elif (version[-1] == "a"): 159 appended = True 160 compact = False 161 version = version[:-1] 162 else: 163 compact = False 164 appended = False 165 ls = [] 166 if "replica" in kwargs: 167 ls = kwargs.get("replica") 168 else: 169 for (dirpath, dirnames, filenames) in os.walk(path): 170 if not appended: 171 ls.extend(dirnames) 172 else: 173 ls.extend(filenames) 174 break 175 if not ls: 176 raise Exception('Error, directory not found') 177 # Exclude folders with different names 178 for exc in ls: 179 if not fnmatch.fnmatch(exc, prefix + '*'): 180 ls = list(set(ls) - set([exc])) 181 182 if not appended: 183 ls = sort_names(ls) 184 replica = len(ls) 185 186 else: 187 replica = len([file.split(".")[-1] for file in ls]) // len(set([file.split(".")[-1] for file in ls])) 188 if replica == 0: 189 raise Exception('No replica found in directory') 190 if not silent: 191 print('Read', part, 'part of', name_list, 'from', prefix[:-1], ',', replica, 'replica') 192 193 if 'names' in kwargs: 194 new_names = kwargs.get('names') 195 if len(new_names) != len(set(new_names)): 196 raise Exception("names are not unique!") 197 if len(new_names) != replica: 198 raise Exception('names should have the length', replica) 199 200 else: 201 ens_name = kwargs.get("ens_name") 202 if not appended: 203 new_names = _get_rep_names(ls, ens_name) 204 else: 205 new_names = _get_appended_rep_names(ls, prefix, name_list[0], ens_name) 206 new_names = sort_names(new_names) 207 208 idl = [] 209 210 noffset_list = [str(x) for x in noffset_list] 211 wf_list = [str(x) for x in wf_list] 212 wf2_list = [str(x) for x in wf2_list] 213 214 # setup dict structures 215 intern = {} 216 for name, corr_type in zip(name_list, corr_type_list): 217 intern[name] = {} 218 b2b, single = _extract_corr_type(corr_type) 219 intern[name]["b2b"] = b2b 220 intern[name]["single"] = single 221 intern[name]["spec"] = {} 222 for quarks in quarks_list: 223 intern[name]["spec"][quarks] = {} 224 for off in noffset_list: 225 intern[name]["spec"][quarks][off] = {} 226 for w in wf_list: 227 intern[name]["spec"][quarks][off][w] = {} 228 for w2 in wf2_list: 229 intern[name]["spec"][quarks][off][w][w2] = {} 230 intern[name]["spec"][quarks][off][w][w2]["pattern"] = _make_pattern(version, name, off, w, w2, intern[name]['b2b'], quarks) 231 232 internal_ret_dict = {} 233 needed_keys = _lists2key(name_list, quarks_list, noffset_list, wf_list, wf2_list) 234 for key in needed_keys: 235 internal_ret_dict[key] = [] 236 237 if not appended: 238 for i, item in enumerate(ls): 239 rep_path = path + '/' + item 240 if "files" in kwargs: 241 files = kwargs.get("files") 242 if isinstance(files, list): 243 if all(isinstance(f, list) for f in files): 244 files = files[i] 245 elif all(isinstance(f, str) for f in files): 246 files = files 247 else: 248 raise TypeError("files has to be of type list[list[str]] or list[str]!") 249 else: 250 raise TypeError("files has to be of type list[list[str]] or list[str]!") 251 252 else: 253 files = [] 254 sub_ls = _find_files(rep_path, prefix, compact, files) 255 rep_idl = [] 256 no_cfg = len(sub_ls) 257 for cfg in sub_ls: 258 try: 259 if compact: 260 rep_idl.append(int(cfg.split(cfg_separator)[-1])) 261 else: 262 rep_idl.append(int(cfg[3:])) 263 except Exception: 264 raise Exception("Couldn't parse idl from directory, problem with file " + cfg) 265 rep_idl.sort() 266 # maybe there is a better way to print the idls 267 if not silent: 268 print(item, ':', no_cfg, ' configurations') 269 idl.append(rep_idl) 270 # here we have found all the files we need to look into. 271 if i == 0: 272 if version != "0.0" and compact: 273 file = path + '/' + item + '/' + sub_ls[0] 274 for name in name_list: 275 if version == "0.0" or not compact: 276 file = path + '/' + item + '/' + sub_ls[0] + '/' + name 277 for key in _lists2key(quarks_list, noffset_list, wf_list, wf2_list): 278 specs = _key2specs(key) 279 quarks = specs[0] 280 off = specs[1] 281 w = specs[2] 282 w2 = specs[3] 283 # here, we want to find the place within the file, 284 # where the correlator we need is stored. 285 # to do so, the pattern needed is put together 286 # from the input values 287 start_read, T = _find_correlator(file, version, intern[name]["spec"][quarks][str(off)][str(w)][str(w2)]["pattern"], intern[name]['b2b'], silent=silent) 288 intern[name]["spec"][quarks][str(off)][str(w)][str(w2)]["start"] = start_read 289 intern[name]["T"] = T 290 # preparing the datastructure 291 # the correlators get parsed into... 292 deltas = [] 293 for j in range(intern[name]["T"]): 294 deltas.append([]) 295 internal_ret_dict[sep.join([name, key])] = deltas 296 297 if compact: 298 rep_deltas = _read_compact_rep(path, item, sub_ls, intern, needed_keys, im) 299 for key in needed_keys: 300 name = _key2specs(key)[0] 301 for t in range(intern[name]["T"]): 302 internal_ret_dict[key][t].append(rep_deltas[key][t]) 303 else: 304 for key in needed_keys: 305 rep_data = [] 306 name = _key2specs(key)[0] 307 for subitem in sub_ls: 308 cfg_path = path + '/' + item + '/' + subitem 309 file_data = _read_o_file(cfg_path, name, needed_keys, intern, version, im) 310 rep_data.append(file_data) 311 for t in range(intern[name]["T"]): 312 internal_ret_dict[key][t].append([]) 313 for cfg in range(no_cfg): 314 internal_ret_dict[key][t][i].append(rep_data[cfg][key][t]) 315 else: 316 for key in needed_keys: 317 specs = _key2specs(key) 318 name = specs[0] 319 quarks = specs[1] 320 off = specs[2] 321 w = specs[3] 322 w2 = specs[4] 323 if "files" in kwargs: 324 if isinstance(kwargs.get("files"), list) and all(isinstance(f, str) for f in kwargs.get("files")): 325 name_ls = kwargs.get("files") 326 else: 327 raise TypeError("In append mode, files has to be of type list[str]!") 328 else: 329 name_ls = ls 330 for exc in name_ls: 331 if not fnmatch.fnmatch(exc, prefix + '*.' + name): 332 name_ls = list(set(name_ls) - set([exc])) 333 name_ls = sort_names(name_ls) 334 pattern = intern[name]['spec'][quarks][off][w][w2]['pattern'] 335 deltas = [] 336 for rep, file in enumerate(name_ls): 337 rep_idl = [] 338 filename = path + '/' + file 339 T, rep_idl, rep_data = _read_append_rep(filename, pattern, intern[name]['b2b'], cfg_separator, im, intern[name]['single']) 340 if rep == 0: 341 intern[name]['T'] = T 342 for t in range(intern[name]['T']): 343 deltas.append([]) 344 for t in range(intern[name]['T']): 345 deltas[t].append(rep_data[t]) 346 internal_ret_dict[key] = deltas 347 if name == name_list[0]: 348 idl.append(rep_idl) 349 350 if kwargs.get("check_configs") is True: 351 if not silent: 352 print("Checking for missing configs...") 353 che = kwargs.get("check_configs") 354 if not (len(che) == len(idl)): 355 raise Exception("check_configs has to be the same length as replica!") 356 for r in range(len(idl)): 357 if not silent: 358 print("checking " + new_names[r]) 359 check_idl(idl[r], che[r]) 360 if not silent: 361 print("Done") 362 363 result_dict = {} 364 if keyed_out: 365 for key in needed_keys: 366 result = [] 367 for t in range(intern[name]["T"]): 368 result.append(Obs(internal_ret_dict[key][t], new_names, idl=idl)) 369 result_dict[key] = result 370 else: 371 for name in name_list: 372 result_dict[name] = {} 373 for quarks in quarks_list: 374 result_dict[name][quarks] = {} 375 for off in noffset_list: 376 result_dict[name][quarks][off] = {} 377 for w in wf_list: 378 result_dict[name][quarks][off][w] = {} 379 for w2 in wf2_list: 380 key = _specs2key(name, quarks, off, w, w2) 381 result = [] 382 for t in range(intern[name]["T"]): 383 result.append(Obs(internal_ret_dict[key][t], new_names, idl=idl)) 384 result_dict[name][quarks][str(off)][str(w)][str(w2)] = result 385 return result_dict
Read sfcf files from given folder structure.
Parameters
- path (str): Path to the sfcf files.
- prefix (str): Prefix of the sfcf files.
- name (str): Name of the correlation function to read.
- quarks_list (list[str]): Label of the quarks used in the sfcf input file. e.g. "quark quark" for version 0.0 this does NOT need to be given with the typical " - " that is present in the output file, this is done automatically for this version
- corr_type_list (list[str]):
Type of correlation function to read. Can be
- 'bi' for boundary-inner
- 'bb' for boundary-boundary
- 'bib' for boundary-inner-boundary
- noffset_list (list[int]): Offset of the source (only relevant when wavefunctions are used)
- wf_list (int): ID of wave function
- wf2_list (list[int]): ID of the second wavefunction (only relevant for boundary-to-boundary correlation functions)
- im (bool): if True, read imaginary instead of real part of the correlation function.
- names (list): Alternative labeling for replicas/ensembles. Has to have the appropriate length
- ens_name (str): replaces the name of the ensemble
- version (str): version of SFCF, with which the measurement was done. if the compact output option (-c) was specified, append a "c" to the version (e.g. "1.0c") if the append output option (-a) was specified, append an "a" to the version
- cfg_separator (str): String that separates the ensemble identifier from the configuration number (default 'n').
- replica (list): list of replica to be read, default is all
- files (list[list[int]]): list of files to be read per replica, default is all. for non-compact output format, hand the folders to be read here.
- check_configs (list[list[int]]): list of list of supposed configs, eg. [range(1,1000)] for one replicum with 1000 configs
Returns
- result (dict[list[Obs]]): dict with one of the following properties: if keyed_out: dict[key] = list[Obs] where key has the form name/quarks/offset/wf/wf2 if not keyed_out: dict[name][quarks][offset][wf][wf2] = list[Obs]