diff --git a/docs/pyerrors/input/sfcf.html b/docs/pyerrors/input/sfcf.html index ac65fd41..c7a60bcc 100644 --- a/docs/pyerrors/input/sfcf.html +++ b/docs/pyerrors/input/sfcf.html @@ -391,356 +391,355 @@ 307 cfg_path = path + '/' + item + '/' + subitem 308 file_data = _read_o_file(cfg_path, name, needed_keys, intern, version, im) 309 rep_data.append(file_data) -310 print(rep_data) -311 for t in range(intern[name]["T"]): -312 internal_ret_dict[key][t].append([]) -313 for cfg in range(no_cfg): -314 internal_ret_dict[key][t][i].append(rep_data[cfg][key][t]) -315 else: -316 for key in needed_keys: -317 specs = _key2specs(key) -318 name = specs[0] -319 quarks = specs[1] -320 off = specs[2] -321 w = specs[3] -322 w2 = specs[4] -323 if "files" in kwargs: -324 if isinstance(kwargs.get("files"), list) and all(isinstance(f, str) for f in kwargs.get("files")): -325 name_ls = kwargs.get("files") -326 else: -327 raise TypeError("In append mode, files has to be of type list[str]!") -328 else: -329 name_ls = ls -330 for exc in name_ls: -331 if not fnmatch.fnmatch(exc, prefix + '*.' + name): -332 name_ls = list(set(name_ls) - set([exc])) -333 name_ls = sort_names(name_ls) -334 pattern = intern[name]['spec'][quarks][off][w][w2]['pattern'] -335 deltas = [] -336 for rep, file in enumerate(name_ls): -337 rep_idl = [] -338 filename = path + '/' + file -339 T, rep_idl, rep_data = _read_append_rep(filename, pattern, intern[name]['b2b'], cfg_separator, im, intern[name]['single']) -340 if rep == 0: -341 intern[name]['T'] = T -342 for t in range(intern[name]['T']): -343 deltas.append([]) -344 for t in range(intern[name]['T']): -345 deltas[t].append(rep_data[t]) -346 internal_ret_dict[key] = deltas -347 if name == name_list[0]: -348 idl.append(rep_idl) -349 -350 if kwargs.get("check_configs") is True: -351 if not silent: -352 print("Checking for missing configs...") -353 che = kwargs.get("check_configs") -354 if not (len(che) == len(idl)): -355 raise Exception("check_configs has to be the same length as replica!") -356 for r in range(len(idl)): -357 if not silent: -358 print("checking " + new_names[r]) -359 check_idl(idl[r], che[r]) -360 if not silent: -361 print("Done") -362 -363 result_dict = {} -364 if keyed_out: -365 for key in needed_keys: -366 result = [] -367 for t in range(intern[name]["T"]): -368 result.append(Obs(internal_ret_dict[key][t], new_names, idl=idl)) -369 result_dict[key] = result -370 else: -371 for name in name_list: -372 result_dict[name] = {} -373 for quarks in quarks_list: -374 result_dict[name][quarks] = {} -375 for off in noffset_list: -376 result_dict[name][quarks][off] = {} -377 for w in wf_list: -378 result_dict[name][quarks][off][w] = {} -379 for w2 in wf2_list: -380 key = _specs2key(name, quarks, off, w, w2) -381 result = [] -382 for t in range(intern[name]["T"]): -383 result.append(Obs(internal_ret_dict[key][t], new_names, idl=idl)) -384 result_dict[name][quarks][str(off)][str(w)][str(w2)] = result -385 return result_dict +310 for t in range(intern[name]["T"]): +311 internal_ret_dict[key][t].append([]) +312 for cfg in range(no_cfg): +313 internal_ret_dict[key][t][i].append(rep_data[cfg][key][t]) +314 else: +315 for key in needed_keys: +316 specs = _key2specs(key) +317 name = specs[0] +318 quarks = specs[1] +319 off = specs[2] +320 w = specs[3] +321 w2 = specs[4] +322 if "files" in kwargs: +323 if isinstance(kwargs.get("files"), list) and all(isinstance(f, str) for f in kwargs.get("files")): +324 name_ls = kwargs.get("files") +325 else: +326 raise TypeError("In append mode, files has to be of type list[str]!") +327 else: +328 name_ls = ls +329 for exc in name_ls: +330 if not fnmatch.fnmatch(exc, prefix + '*.' + name): +331 name_ls = list(set(name_ls) - set([exc])) +332 name_ls = sort_names(name_ls) +333 pattern = intern[name]['spec'][quarks][off][w][w2]['pattern'] +334 deltas = [] +335 for rep, file in enumerate(name_ls): +336 rep_idl = [] +337 filename = path + '/' + file +338 T, rep_idl, rep_data = _read_append_rep(filename, pattern, intern[name]['b2b'], cfg_separator, im, intern[name]['single']) +339 if rep == 0: +340 intern[name]['T'] = T +341 for t in range(intern[name]['T']): +342 deltas.append([]) +343 for t in range(intern[name]['T']): +344 deltas[t].append(rep_data[t]) +345 internal_ret_dict[key] = deltas +346 if name == name_list[0]: +347 idl.append(rep_idl) +348 +349 if kwargs.get("check_configs") is True: +350 if not silent: +351 print("Checking for missing configs...") +352 che = kwargs.get("check_configs") +353 if not (len(che) == len(idl)): +354 raise Exception("check_configs has to be the same length as replica!") +355 for r in range(len(idl)): +356 if not silent: +357 print("checking " + new_names[r]) +358 check_idl(idl[r], che[r]) +359 if not silent: +360 print("Done") +361 +362 result_dict = {} +363 if keyed_out: +364 for key in needed_keys: +365 result = [] +366 for t in range(intern[name]["T"]): +367 result.append(Obs(internal_ret_dict[key][t], new_names, idl=idl)) +368 result_dict[key] = result +369 else: +370 for name in name_list: +371 result_dict[name] = {} +372 for quarks in quarks_list: +373 result_dict[name][quarks] = {} +374 for off in noffset_list: +375 result_dict[name][quarks][off] = {} +376 for w in wf_list: +377 result_dict[name][quarks][off][w] = {} +378 for w2 in wf2_list: +379 key = _specs2key(name, quarks, off, w, w2) +380 result = [] +381 for t in range(intern[name]["T"]): +382 result.append(Obs(internal_ret_dict[key][t], new_names, idl=idl)) +383 result_dict[name][quarks][str(off)][str(w)][str(w2)] = result +384 return result_dict +385 386 -387 -388def _lists2key(*lists): -389 keys = [] -390 for tup in itertools.product(*lists): -391 keys.append(sep.join(tup)) -392 return keys +387def _lists2key(*lists): +388 keys = [] +389 for tup in itertools.product(*lists): +390 keys.append(sep.join(tup)) +391 return keys +392 393 -394 -395def _key2specs(key): -396 return key.split(sep) +394def _key2specs(key): +395 return key.split(sep) +396 397 -398 -399def _specs2key(*specs): -400 return sep.join(specs) +398def _specs2key(*specs): +399 return sep.join(specs) +400 401 -402 -403def _read_o_file(cfg_path, name, needed_keys, intern, version, im): -404 return_vals = {} -405 for key in needed_keys: -406 file = cfg_path + '/' + name -407 specs = _key2specs(key) -408 if specs[0] == name: -409 with open(file) as fp: -410 lines = fp.readlines() -411 quarks = specs[1] -412 off = specs[2] -413 w = specs[3] -414 w2 = specs[4] -415 T = intern[name]["T"] -416 start_read = intern[name]["spec"][quarks][off][w][w2]["start"] -417 deltas = [] -418 for line in lines[start_read:start_read + T]: -419 floats = list(map(float, line.split())) -420 if version == "0.0": -421 deltas.append(floats[im - intern[name]["single"]]) -422 else: -423 deltas.append(floats[1 + im - intern[name]["single"]]) -424 return_vals[key] = deltas -425 return return_vals +402def _read_o_file(cfg_path, name, needed_keys, intern, version, im): +403 return_vals = {} +404 for key in needed_keys: +405 file = cfg_path + '/' + name +406 specs = _key2specs(key) +407 if specs[0] == name: +408 with open(file) as fp: +409 lines = fp.readlines() +410 quarks = specs[1] +411 off = specs[2] +412 w = specs[3] +413 w2 = specs[4] +414 T = intern[name]["T"] +415 start_read = intern[name]["spec"][quarks][off][w][w2]["start"] +416 deltas = [] +417 for line in lines[start_read:start_read + T]: +418 floats = list(map(float, line.split())) +419 if version == "0.0": +420 deltas.append(floats[im - intern[name]["single"]]) +421 else: +422 deltas.append(floats[1 + im - intern[name]["single"]]) +423 return_vals[key] = deltas +424 return return_vals +425 426 -427 -428def _extract_corr_type(corr_type): -429 if corr_type == 'bb': -430 b2b = True -431 single = True -432 elif corr_type == 'bib': -433 b2b = True -434 single = False -435 else: -436 b2b = False -437 single = False -438 return b2b, single +427def _extract_corr_type(corr_type): +428 if corr_type == 'bb': +429 b2b = True +430 single = True +431 elif corr_type == 'bib': +432 b2b = True +433 single = False +434 else: +435 b2b = False +436 single = False +437 return b2b, single +438 439 -440 -441def _find_files(rep_path, prefix, compact, files=[]): -442 sub_ls = [] -443 if not files == []: -444 files.sort(key=lambda x: int(re.findall(r'\d+', x)[-1])) -445 else: -446 for (dirpath, dirnames, filenames) in os.walk(rep_path): -447 if compact: -448 sub_ls.extend(filenames) -449 else: -450 sub_ls.extend(dirnames) -451 break -452 if compact: -453 for exc in sub_ls: -454 if not fnmatch.fnmatch(exc, prefix + '*'): -455 sub_ls = list(set(sub_ls) - set([exc])) -456 sub_ls.sort(key=lambda x: int(re.findall(r'\d+', x)[-1])) -457 else: -458 for exc in sub_ls: -459 if not fnmatch.fnmatch(exc, 'cfg*'): -460 sub_ls = list(set(sub_ls) - set([exc])) -461 sub_ls.sort(key=lambda x: int(x[3:])) -462 files = sub_ls -463 if len(files) == 0: -464 raise FileNotFoundError("Did not find files in", rep_path, "with prefix", prefix, "and the given structure.") -465 return files +440def _find_files(rep_path, prefix, compact, files=[]): +441 sub_ls = [] +442 if not files == []: +443 files.sort(key=lambda x: int(re.findall(r'\d+', x)[-1])) +444 else: +445 for (dirpath, dirnames, filenames) in os.walk(rep_path): +446 if compact: +447 sub_ls.extend(filenames) +448 else: +449 sub_ls.extend(dirnames) +450 break +451 if compact: +452 for exc in sub_ls: +453 if not fnmatch.fnmatch(exc, prefix + '*'): +454 sub_ls = list(set(sub_ls) - set([exc])) +455 sub_ls.sort(key=lambda x: int(re.findall(r'\d+', x)[-1])) +456 else: +457 for exc in sub_ls: +458 if not fnmatch.fnmatch(exc, 'cfg*'): +459 sub_ls = list(set(sub_ls) - set([exc])) +460 sub_ls.sort(key=lambda x: int(x[3:])) +461 files = sub_ls +462 if len(files) == 0: +463 raise FileNotFoundError("Did not find files in", rep_path, "with prefix", prefix, "and the given structure.") +464 return files +465 466 -467 -468def _make_pattern(version, name, noffset, wf, wf2, b2b, quarks): -469 if version == "0.0": -470 pattern = "# " + name + " : offset " + str(noffset) + ", wf " + str(wf) -471 if b2b: -472 pattern += ", wf_2 " + str(wf2) -473 qs = quarks.split(" ") -474 pattern += " : " + qs[0] + " - " + qs[1] -475 else: -476 pattern = 'name ' + name + '\nquarks ' + quarks + '\noffset ' + str(noffset) + '\nwf ' + str(wf) -477 if b2b: -478 pattern += '\nwf_2 ' + str(wf2) -479 return pattern +467def _make_pattern(version, name, noffset, wf, wf2, b2b, quarks): +468 if version == "0.0": +469 pattern = "# " + name + " : offset " + str(noffset) + ", wf " + str(wf) +470 if b2b: +471 pattern += ", wf_2 " + str(wf2) +472 qs = quarks.split(" ") +473 pattern += " : " + qs[0] + " - " + qs[1] +474 else: +475 pattern = 'name ' + name + '\nquarks ' + quarks + '\noffset ' + str(noffset) + '\nwf ' + str(wf) +476 if b2b: +477 pattern += '\nwf_2 ' + str(wf2) +478 return pattern +479 480 -481 -482def _find_correlator(file_name, version, pattern, b2b, silent=False): -483 T = 0 -484 -485 with open(file_name, "r") as my_file: -486 -487 content = my_file.read() -488 match = re.search(pattern, content) -489 if match: -490 if version == "0.0": -491 start_read = content.count('\n', 0, match.start()) + 1 -492 T = content.count('\n', start_read) -493 else: -494 start_read = content.count('\n', 0, match.start()) + 5 + b2b -495 end_match = re.search(r'\n\s*\n', content[match.start():]) -496 T = content[match.start():].count('\n', 0, end_match.start()) - 4 - b2b -497 if not T > 0: -498 raise ValueError("Correlator with pattern\n" + pattern + "\nis empty!") -499 if not silent: -500 print(T, 'entries, starting to read in line', start_read) -501 -502 else: -503 raise ValueError('Correlator with pattern\n' + pattern + '\nnot found.') -504 -505 return start_read, T +481def _find_correlator(file_name, version, pattern, b2b, silent=False): +482 T = 0 +483 +484 with open(file_name, "r") as my_file: +485 +486 content = my_file.read() +487 match = re.search(pattern, content) +488 if match: +489 if version == "0.0": +490 start_read = content.count('\n', 0, match.start()) + 1 +491 T = content.count('\n', start_read) +492 else: +493 start_read = content.count('\n', 0, match.start()) + 5 + b2b +494 end_match = re.search(r'\n\s*\n', content[match.start():]) +495 T = content[match.start():].count('\n', 0, end_match.start()) - 4 - b2b +496 if not T > 0: +497 raise ValueError("Correlator with pattern\n" + pattern + "\nis empty!") +498 if not silent: +499 print(T, 'entries, starting to read in line', start_read) +500 +501 else: +502 raise ValueError('Correlator with pattern\n' + pattern + '\nnot found.') +503 +504 return start_read, T +505 506 -507 -508def _read_compact_file(rep_path, cfg_file, intern, needed_keys, im): -509 return_vals = {} -510 with open(rep_path + cfg_file) as fp: -511 lines = fp.readlines() -512 for key in needed_keys: -513 keys = _key2specs(key) -514 name = keys[0] -515 quarks = keys[1] -516 off = keys[2] -517 w = keys[3] -518 w2 = keys[4] -519 -520 T = intern[name]["T"] -521 start_read = intern[name]["spec"][quarks][off][w][w2]["start"] -522 # check, if the correlator is in fact -523 # printed completely -524 if (start_read + T + 1 > len(lines)): -525 raise Exception("EOF before end of correlator data! Maybe " + rep_path + cfg_file + " is corrupted?") -526 corr_lines = lines[start_read - 6: start_read + T] -527 t_vals = [] -528 -529 if corr_lines[1 - intern[name]["b2b"]].strip() != 'name ' + name: -530 raise Exception('Wrong format in file', cfg_file) -531 -532 for k in range(6, T + 6): -533 floats = list(map(float, corr_lines[k].split())) -534 t_vals.append(floats[-2:][im]) -535 return_vals[key] = t_vals -536 return return_vals +507def _read_compact_file(rep_path, cfg_file, intern, needed_keys, im): +508 return_vals = {} +509 with open(rep_path + cfg_file) as fp: +510 lines = fp.readlines() +511 for key in needed_keys: +512 keys = _key2specs(key) +513 name = keys[0] +514 quarks = keys[1] +515 off = keys[2] +516 w = keys[3] +517 w2 = keys[4] +518 +519 T = intern[name]["T"] +520 start_read = intern[name]["spec"][quarks][off][w][w2]["start"] +521 # check, if the correlator is in fact +522 # printed completely +523 if (start_read + T + 1 > len(lines)): +524 raise Exception("EOF before end of correlator data! Maybe " + rep_path + cfg_file + " is corrupted?") +525 corr_lines = lines[start_read - 6: start_read + T] +526 t_vals = [] +527 +528 if corr_lines[1 - intern[name]["b2b"]].strip() != 'name ' + name: +529 raise Exception('Wrong format in file', cfg_file) +530 +531 for k in range(6, T + 6): +532 floats = list(map(float, corr_lines[k].split())) +533 t_vals.append(floats[-2:][im]) +534 return_vals[key] = t_vals +535 return return_vals +536 537 -538 -539def _read_compact_rep(path, rep, sub_ls, intern, needed_keys, im): -540 rep_path = path + '/' + rep + '/' -541 no_cfg = len(sub_ls) -542 -543 return_vals = {} -544 for key in needed_keys: -545 name = _key2specs(key)[0] -546 deltas = [] -547 for t in range(intern[name]["T"]): -548 deltas.append(np.zeros(no_cfg)) -549 return_vals[key] = deltas -550 -551 for cfg in range(no_cfg): -552 cfg_file = sub_ls[cfg] -553 cfg_data = _read_compact_file(rep_path, cfg_file, intern, needed_keys, im) -554 for key in needed_keys: -555 name = _key2specs(key)[0] -556 for t in range(intern[name]["T"]): -557 return_vals[key][t][cfg] = cfg_data[key][t] -558 return return_vals +538def _read_compact_rep(path, rep, sub_ls, intern, needed_keys, im): +539 rep_path = path + '/' + rep + '/' +540 no_cfg = len(sub_ls) +541 +542 return_vals = {} +543 for key in needed_keys: +544 name = _key2specs(key)[0] +545 deltas = [] +546 for t in range(intern[name]["T"]): +547 deltas.append(np.zeros(no_cfg)) +548 return_vals[key] = deltas +549 +550 for cfg in range(no_cfg): +551 cfg_file = sub_ls[cfg] +552 cfg_data = _read_compact_file(rep_path, cfg_file, intern, needed_keys, im) +553 for key in needed_keys: +554 name = _key2specs(key)[0] +555 for t in range(intern[name]["T"]): +556 return_vals[key][t][cfg] = cfg_data[key][t] +557 return return_vals +558 559 -560 -561def _read_chunk(chunk, gauge_line, cfg_sep, start_read, T, corr_line, b2b, pattern, im, single): -562 try: -563 idl = int(chunk[gauge_line].split(cfg_sep)[-1]) -564 except Exception: -565 raise Exception("Couldn't parse idl from directory, problem with chunk around line ", gauge_line) -566 -567 found_pat = "" -568 data = [] -569 for li in chunk[corr_line + 1:corr_line + 6 + b2b]: -570 found_pat += li -571 if re.search(pattern, found_pat): -572 for t, line in enumerate(chunk[start_read:start_read + T]): -573 floats = list(map(float, line.split())) -574 data.append(floats[im + 1 - single]) -575 return idl, data +560def _read_chunk(chunk, gauge_line, cfg_sep, start_read, T, corr_line, b2b, pattern, im, single): +561 try: +562 idl = int(chunk[gauge_line].split(cfg_sep)[-1]) +563 except Exception: +564 raise Exception("Couldn't parse idl from directory, problem with chunk around line ", gauge_line) +565 +566 found_pat = "" +567 data = [] +568 for li in chunk[corr_line + 1:corr_line + 6 + b2b]: +569 found_pat += li +570 if re.search(pattern, found_pat): +571 for t, line in enumerate(chunk[start_read:start_read + T]): +572 floats = list(map(float, line.split())) +573 data.append(floats[im + 1 - single]) +574 return idl, data +575 576 -577 -578def _read_append_rep(filename, pattern, b2b, cfg_separator, im, single): -579 with open(filename, 'r') as fp: -580 content = fp.readlines() -581 data_starts = [] -582 for linenumber, line in enumerate(content): -583 if "[run]" in line: -584 data_starts.append(linenumber) -585 if len(set([data_starts[i] - data_starts[i - 1] for i in range(1, len(data_starts))])) > 1: -586 raise Exception("Irregularities in file structure found, not all runs have the same output length") -587 chunk = content[:data_starts[1]] -588 for linenumber, line in enumerate(chunk): -589 if line.startswith("gauge_name"): -590 gauge_line = linenumber -591 elif line.startswith("[correlator]"): -592 corr_line = linenumber -593 found_pat = "" -594 for li in chunk[corr_line + 1: corr_line + 6 + b2b]: -595 found_pat += li -596 if re.search(pattern, found_pat): -597 start_read = corr_line + 7 + b2b -598 break -599 else: -600 raise ValueError("Did not find pattern\n", pattern, "\nin\n", filename) -601 endline = corr_line + 6 + b2b -602 while not chunk[endline] == "\n": -603 endline += 1 -604 T = endline - start_read -605 -606 # all other chunks should follow the same structure -607 rep_idl = [] -608 rep_data = [] -609 -610 for cnfg in range(len(data_starts)): -611 start = data_starts[cnfg] -612 stop = start + data_starts[1] -613 chunk = content[start:stop] -614 idl, data = _read_chunk(chunk, gauge_line, cfg_separator, start_read, T, corr_line, b2b, pattern, im, single) -615 rep_idl.append(idl) -616 rep_data.append(data) -617 -618 data = [] -619 -620 for t in range(T): -621 data.append([]) -622 for c in range(len(rep_data)): -623 data[t].append(rep_data[c][t]) -624 return T, rep_idl, data +577def _read_append_rep(filename, pattern, b2b, cfg_separator, im, single): +578 with open(filename, 'r') as fp: +579 content = fp.readlines() +580 data_starts = [] +581 for linenumber, line in enumerate(content): +582 if "[run]" in line: +583 data_starts.append(linenumber) +584 if len(set([data_starts[i] - data_starts[i - 1] for i in range(1, len(data_starts))])) > 1: +585 raise Exception("Irregularities in file structure found, not all runs have the same output length") +586 chunk = content[:data_starts[1]] +587 for linenumber, line in enumerate(chunk): +588 if line.startswith("gauge_name"): +589 gauge_line = linenumber +590 elif line.startswith("[correlator]"): +591 corr_line = linenumber +592 found_pat = "" +593 for li in chunk[corr_line + 1: corr_line + 6 + b2b]: +594 found_pat += li +595 if re.search(pattern, found_pat): +596 start_read = corr_line + 7 + b2b +597 break +598 else: +599 raise ValueError("Did not find pattern\n", pattern, "\nin\n", filename) +600 endline = corr_line + 6 + b2b +601 while not chunk[endline] == "\n": +602 endline += 1 +603 T = endline - start_read +604 +605 # all other chunks should follow the same structure +606 rep_idl = [] +607 rep_data = [] +608 +609 for cnfg in range(len(data_starts)): +610 start = data_starts[cnfg] +611 stop = start + data_starts[1] +612 chunk = content[start:stop] +613 idl, data = _read_chunk(chunk, gauge_line, cfg_separator, start_read, T, corr_line, b2b, pattern, im, single) +614 rep_idl.append(idl) +615 rep_data.append(data) +616 +617 data = [] +618 +619 for t in range(T): +620 data.append([]) +621 for c in range(len(rep_data)): +622 data[t].append(rep_data[c][t]) +623 return T, rep_idl, data +624 625 -626 -627def _get_rep_names(ls, ens_name=None): -628 new_names = [] -629 for entry in ls: -630 try: -631 idx = entry.index('r') -632 except Exception: -633 raise Exception("Automatic recognition of replicum failed, please enter the key word 'names'.") -634 -635 if ens_name: -636 new_names.append('ens_name' + '|' + entry[idx:]) -637 else: -638 new_names.append(entry[:idx] + '|' + entry[idx:]) -639 return new_names +626def _get_rep_names(ls, ens_name=None): +627 new_names = [] +628 for entry in ls: +629 try: +630 idx = entry.index('r') +631 except Exception: +632 raise Exception("Automatic recognition of replicum failed, please enter the key word 'names'.") +633 +634 if ens_name: +635 new_names.append('ens_name' + '|' + entry[idx:]) +636 else: +637 new_names.append(entry[:idx] + '|' + entry[idx:]) +638 return new_names +639 640 -641 -642def _get_appended_rep_names(ls, prefix, name, ens_name=None): -643 new_names = [] -644 for exc in ls: -645 if not fnmatch.fnmatch(exc, prefix + '*.' + name): -646 ls = list(set(ls) - set([exc])) -647 ls.sort(key=lambda x: int(re.findall(r'\d+', x)[-1])) -648 for entry in ls: -649 myentry = entry[:-len(name) - 1] -650 try: -651 idx = myentry.index('r') -652 except Exception: -653 raise Exception("Automatic recognition of replicum failed, please enter the key word 'names'.") -654 -655 if ens_name: -656 new_names.append('ens_name' + '|' + entry[idx:]) -657 else: -658 new_names.append(myentry[:idx] + '|' + myentry[idx:]) -659 return new_names +641def _get_appended_rep_names(ls, prefix, name, ens_name=None): +642 new_names = [] +643 for exc in ls: +644 if not fnmatch.fnmatch(exc, prefix + '*.' + name): +645 ls = list(set(ls) - set([exc])) +646 ls.sort(key=lambda x: int(re.findall(r'\d+', x)[-1])) +647 for entry in ls: +648 myentry = entry[:-len(name) - 1] +649 try: +650 idx = myentry.index('r') +651 except Exception: +652 raise Exception("Automatic recognition of replicum failed, please enter the key word 'names'.") +653 +654 if ens_name: +655 new_names.append('ens_name' + '|' + entry[idx:]) +656 else: +657 new_names.append(myentry[:idx] + '|' + myentry[idx:]) +658 return new_names @@ -1144,82 +1143,81 @@ bb-type correlators have length 1. 308 cfg_path = path + '/' + item + '/' + subitem 309 file_data = _read_o_file(cfg_path, name, needed_keys, intern, version, im) 310 rep_data.append(file_data) -311 print(rep_data) -312 for t in range(intern[name]["T"]): -313 internal_ret_dict[key][t].append([]) -314 for cfg in range(no_cfg): -315 internal_ret_dict[key][t][i].append(rep_data[cfg][key][t]) -316 else: -317 for key in needed_keys: -318 specs = _key2specs(key) -319 name = specs[0] -320 quarks = specs[1] -321 off = specs[2] -322 w = specs[3] -323 w2 = specs[4] -324 if "files" in kwargs: -325 if isinstance(kwargs.get("files"), list) and all(isinstance(f, str) for f in kwargs.get("files")): -326 name_ls = kwargs.get("files") -327 else: -328 raise TypeError("In append mode, files has to be of type list[str]!") -329 else: -330 name_ls = ls -331 for exc in name_ls: -332 if not fnmatch.fnmatch(exc, prefix + '*.' + name): -333 name_ls = list(set(name_ls) - set([exc])) -334 name_ls = sort_names(name_ls) -335 pattern = intern[name]['spec'][quarks][off][w][w2]['pattern'] -336 deltas = [] -337 for rep, file in enumerate(name_ls): -338 rep_idl = [] -339 filename = path + '/' + file -340 T, rep_idl, rep_data = _read_append_rep(filename, pattern, intern[name]['b2b'], cfg_separator, im, intern[name]['single']) -341 if rep == 0: -342 intern[name]['T'] = T -343 for t in range(intern[name]['T']): -344 deltas.append([]) -345 for t in range(intern[name]['T']): -346 deltas[t].append(rep_data[t]) -347 internal_ret_dict[key] = deltas -348 if name == name_list[0]: -349 idl.append(rep_idl) -350 -351 if kwargs.get("check_configs") is True: -352 if not silent: -353 print("Checking for missing configs...") -354 che = kwargs.get("check_configs") -355 if not (len(che) == len(idl)): -356 raise Exception("check_configs has to be the same length as replica!") -357 for r in range(len(idl)): -358 if not silent: -359 print("checking " + new_names[r]) -360 check_idl(idl[r], che[r]) -361 if not silent: -362 print("Done") -363 -364 result_dict = {} -365 if keyed_out: -366 for key in needed_keys: -367 result = [] -368 for t in range(intern[name]["T"]): -369 result.append(Obs(internal_ret_dict[key][t], new_names, idl=idl)) -370 result_dict[key] = result -371 else: -372 for name in name_list: -373 result_dict[name] = {} -374 for quarks in quarks_list: -375 result_dict[name][quarks] = {} -376 for off in noffset_list: -377 result_dict[name][quarks][off] = {} -378 for w in wf_list: -379 result_dict[name][quarks][off][w] = {} -380 for w2 in wf2_list: -381 key = _specs2key(name, quarks, off, w, w2) -382 result = [] -383 for t in range(intern[name]["T"]): -384 result.append(Obs(internal_ret_dict[key][t], new_names, idl=idl)) -385 result_dict[name][quarks][str(off)][str(w)][str(w2)] = result -386 return result_dict +311 for t in range(intern[name]["T"]): +312 internal_ret_dict[key][t].append([]) +313 for cfg in range(no_cfg): +314 internal_ret_dict[key][t][i].append(rep_data[cfg][key][t]) +315 else: +316 for key in needed_keys: +317 specs = _key2specs(key) +318 name = specs[0] +319 quarks = specs[1] +320 off = specs[2] +321 w = specs[3] +322 w2 = specs[4] +323 if "files" in kwargs: +324 if isinstance(kwargs.get("files"), list) and all(isinstance(f, str) for f in kwargs.get("files")): +325 name_ls = kwargs.get("files") +326 else: +327 raise TypeError("In append mode, files has to be of type list[str]!") +328 else: +329 name_ls = ls +330 for exc in name_ls: +331 if not fnmatch.fnmatch(exc, prefix + '*.' + name): +332 name_ls = list(set(name_ls) - set([exc])) +333 name_ls = sort_names(name_ls) +334 pattern = intern[name]['spec'][quarks][off][w][w2]['pattern'] +335 deltas = [] +336 for rep, file in enumerate(name_ls): +337 rep_idl = [] +338 filename = path + '/' + file +339 T, rep_idl, rep_data = _read_append_rep(filename, pattern, intern[name]['b2b'], cfg_separator, im, intern[name]['single']) +340 if rep == 0: +341 intern[name]['T'] = T +342 for t in range(intern[name]['T']): +343 deltas.append([]) +344 for t in range(intern[name]['T']): +345 deltas[t].append(rep_data[t]) +346 internal_ret_dict[key] = deltas +347 if name == name_list[0]: +348 idl.append(rep_idl) +349 +350 if kwargs.get("check_configs") is True: +351 if not silent: +352 print("Checking for missing configs...") +353 che = kwargs.get("check_configs") +354 if not (len(che) == len(idl)): +355 raise Exception("check_configs has to be the same length as replica!") +356 for r in range(len(idl)): +357 if not silent: +358 print("checking " + new_names[r]) +359 check_idl(idl[r], che[r]) +360 if not silent: +361 print("Done") +362 +363 result_dict = {} +364 if keyed_out: +365 for key in needed_keys: +366 result = [] +367 for t in range(intern[name]["T"]): +368 result.append(Obs(internal_ret_dict[key][t], new_names, idl=idl)) +369 result_dict[key] = result +370 else: +371 for name in name_list: +372 result_dict[name] = {} +373 for quarks in quarks_list: +374 result_dict[name][quarks] = {} +375 for off in noffset_list: +376 result_dict[name][quarks][off] = {} +377 for w in wf_list: +378 result_dict[name][quarks][off][w] = {} +379 for w2 in wf2_list: +380 key = _specs2key(name, quarks, off, w, w2) +381 result = [] +382 for t in range(intern[name]["T"]): +383 result.append(Obs(internal_ret_dict[key][t], new_names, idl=idl)) +384 result_dict[name][quarks][str(off)][str(w)][str(w2)] = result +385 return result_dict