pyerrors.input.sfcf

  1import fnmatch
  2import itertools
  3import os
  4import re
  5import warnings
  6
  7import numpy as np  # Thinly-wrapped numpy
  8
  9from ..obs import Obs
 10from .utils import check_idl, sort_names
 11
 12sep = "/"
 13
 14
 15def read_sfcf(path, prefix, name, quarks='.*', corr_type="bi", noffset=0, wf=0, wf2=0, version="1.0c", cfg_separator="n", cfg_func=None, silent=False, **kwargs):
 16    """Read sfcf files from given folder structure.
 17
 18    Parameters
 19    ----------
 20    path : str
 21        Path to the sfcf files.
 22    prefix : str
 23        Prefix of the sfcf files.
 24    name : str
 25        Name of the correlation function to read.
 26    quarks : str
 27        Label of the quarks used in the sfcf input file. e.g. "quark quark"
 28        for version 0.0 this does NOT need to be given with the typical " - "
 29        that is present in the output file,
 30        this is done automatically for this version
 31    corr_type : str
 32        Type of correlation function to read. Can be
 33        - 'bi' for boundary-inner
 34        - 'bb' for boundary-boundary
 35        - 'bib' for boundary-inner-boundary
 36    noffset : int
 37        Offset of the source (only relevant when wavefunctions are used)
 38    wf : int
 39        ID of wave function
 40    wf2 : int
 41        ID of the second wavefunction
 42        (only relevant for boundary-to-boundary correlation functions)
 43    im : bool
 44        if True, read imaginary instead of real part
 45        of the correlation function.
 46    names : list
 47        Alternative labeling for replicas/ensembles.
 48        Has to have the appropriate length
 49    ens_name : str
 50        replaces the name of the ensemble
 51    version: str
 52        version of SFCF, with which the measurement was done.
 53        if the compact output option (-c) was specified,
 54        append a "c" to the version (e.g. "1.0c")
 55        if the append output option (-a) was specified,
 56        append an "a" to the version
 57    cfg_separator : str
 58        String that separates the ensemble identifier from the configuration number (default 'n').
 59    replica: list
 60        list of replica to be read, default is all
 61    files: list
 62        list of files to be read per replica, default is all.
 63        for non-compact output format, hand the folders to be read here.
 64    check_configs: list[list[int]]
 65        list of list of supposed configs, eg. [range(1,1000)]
 66        for one replicum with 1000 configs
 67
 68    Returns
 69    -------
 70    result: list[Obs]
 71        list of Observables with length T, observable per timeslice.
 72        bb-type correlators have length 1.
 73    """
 74    ret = read_sfcf_multi(path, prefix, [name], quarks_list=[quarks], corr_type_list=[corr_type],
 75                          noffset_list=[noffset], wf_list=[wf], wf2_list=[wf2], version=version,
 76                          cfg_separator=cfg_separator, cfg_func=cfg_func, silent=silent, **kwargs)
 77    return ret[name][quarks][str(noffset)][str(wf)][str(wf2)]
 78
 79
 80def read_sfcf_multi(path, prefix, name_list, quarks_list=None, corr_type_list=None, noffset_list=None, wf_list=None, wf2_list=None, version="1.0c", cfg_separator="n", cfg_func=None, silent=False, keyed_out=False, **kwargs):
 81    """Read sfcf files from given folder structure.
 82
 83    Parameters
 84    ----------
 85    path : str
 86        Path to the sfcf files.
 87    prefix : str
 88        Prefix of the sfcf files.
 89    name : str
 90        Name of the correlation function to read.
 91    quarks_list : list[str]
 92        Label of the quarks used in the sfcf input file. e.g. "quark quark"
 93        for version 0.0 this does NOT need to be given with the typical " - "
 94        that is present in the output file,
 95        this is done automatically for this version
 96    corr_type_list : list[str]
 97        Type of correlation function to read. Can be
 98        - 'bi' for boundary-inner
 99        - 'bb' for boundary-boundary
100        - 'bib' for boundary-inner-boundary
101    noffset_list : list[int]
102        Offset of the source (only relevant when wavefunctions are used)
103    wf_list : int
104        ID of wave function
105    wf2_list : list[int]
106        ID of the second wavefunction
107        (only relevant for boundary-to-boundary correlation functions)
108    im : bool
109        if True, read imaginary instead of real part
110        of the correlation function.
111    names : list
112        Alternative labeling for replicas/ensembles.
113        Has to have the appropriate length
114    ens_name : str
115        replaces the name of the ensemble
116    version: str
117        version of SFCF, with which the measurement was done.
118        if the compact output option (-c) was specified,
119        append a "c" to the version (e.g. "1.0c")
120        if the append output option (-a) was specified,
121        append an "a" to the version
122    cfg_separator : str
123        String that separates the ensemble identifier from the configuration number (default 'n').
124    replica: list
125        list of replica to be read, default is all
126    files: list[list[int]]
127        list of files to be read per replica, default is all.
128        for non-compact output format, hand the folders to be read here.
129    check_configs: list[list[int]]
130        list of list of supposed configs, eg. [range(1,1000)]
131        for one replicum with 1000 configs
132    rep_string: str
133        Separator of ensemble name and replicum. Example: In "ensAr0", "r" would be the separator string.
134    Returns
135    -------
136    result: dict[list[Obs]]
137        dict with one of the following properties:
138        if keyed_out:
139            dict[key] = list[Obs]
140            where key has the form name/quarks/offset/wf/wf2
141        if not keyed_out:
142            dict[name][quarks][offset][wf][wf2] = list[Obs]
143    """
144
145    if quarks_list is None:
146        quarks_list = ['.*']
147    if corr_type_list is None:
148        corr_type_list = ['bi']
149    if noffset_list is None:
150        noffset_list = [0]
151    if wf_list is None:
152        wf_list = [0]
153    if wf2_list is None:
154        wf2_list = [0]
155
156    if kwargs.get('im'):
157        im = 1
158        part = 'imaginary'
159    else:
160        im = 0
161        part = 'real'
162
163    known_versions = ["0.0", "1.0", "2.0", "1.0c", "2.0c", "1.0a", "2.0a"]
164
165    if version not in known_versions:
166        raise ValueError("This version is not known!")
167    if (version[-1] == "c"):
168        appended = False
169        compact = True
170        version = version[:-1]
171    elif (version[-1] == "a"):
172        appended = True
173        compact = False
174        version = version[:-1]
175    else:
176        compact = False
177        appended = False
178    ls = []
179    if "replica" in kwargs:
180        ls = kwargs.get("replica")
181    else:
182        for (_dirpath, dirnames, filenames) in os.walk(path):
183            if not appended:
184                ls.extend(dirnames)
185            else:
186                ls.extend(filenames)
187            break
188        if not ls:
189            raise FileNotFoundError('Error, directory not found')
190        # Exclude folders with different names
191        for exc in ls:
192            if not fnmatch.fnmatch(exc, prefix + '*'):
193                ls = list(set(ls) - set([exc]))
194
195    if not appended:
196        ls = sort_names(ls)
197        replica = len(ls)
198
199    else:
200        replica = len([file.split(".")[-1] for file in ls]) // len(set([file.split(".")[-1] for file in ls]))
201    if replica == 0:
202        raise FileNotFoundError('No replica found in directory')
203    if not silent:
204        print('Read', part, 'part of', name_list, 'from', prefix[:-1], ',', replica, 'replica')
205
206    if 'names' in kwargs:
207        new_names = kwargs.get('names')
208        if len(new_names) != len(set(new_names)):
209            raise ValueError("names are not unique!")
210        if len(new_names) != replica:
211            raise ValueError(f'names should have the length {replica}')
212
213    else:
214        ens_name = kwargs.get("ens_name")
215        if not appended:
216            new_names = _get_rep_names(ls, ens_name, rep_sep=(kwargs.get('rep_string', 'r')))
217        else:
218            new_names = _get_appended_rep_names(ls, prefix, name_list[0], ens_name, rep_sep=(kwargs.get('rep_string', 'r')))
219        new_names = sort_names(new_names)
220
221    idl = []
222
223    noffset_list = [str(x) for x in noffset_list]
224    wf_list = [str(x) for x in wf_list]
225    wf2_list = [str(x) for x in wf2_list]
226
227    # setup dict structures
228    intern = {}
229    for name, corr_type in zip(name_list, corr_type_list, strict=True):
230        intern[name] = {}
231        b2b, single = _extract_corr_type(corr_type)
232        intern[name]["b2b"] = b2b
233        intern[name]["single"] = single
234        intern[name]["spec"] = {}
235        for quarks in quarks_list:
236            intern[name]["spec"][quarks] = {}
237            for off in noffset_list:
238                intern[name]["spec"][quarks][off] = {}
239                for w in wf_list:
240                    intern[name]["spec"][quarks][off][w] = {}
241                    if b2b:
242                        for w2 in wf2_list:
243                            intern[name]["spec"][quarks][off][w][w2] = {}
244                            intern[name]["spec"][quarks][off][w][w2]["pattern"] = _make_pattern(version, name, off, w, w2, intern[name]['b2b'], quarks)
245                    else:
246                        intern[name]["spec"][quarks][off][w]["0"] = {}
247                        intern[name]["spec"][quarks][off][w]["0"]["pattern"] = _make_pattern(version, name, off, w, 0, intern[name]['b2b'], quarks)
248
249    internal_ret_dict = {}
250    needed_keys = []
251    for name, corr_type in zip(name_list, corr_type_list, strict=True):
252        b2b, single = _extract_corr_type(corr_type)
253        if b2b:
254            needed_keys.extend(_lists2key([name], quarks_list, noffset_list, wf_list, wf2_list))
255        else:
256            needed_keys.extend(_lists2key([name], quarks_list, noffset_list, wf_list, ["0"]))
257
258    for key in needed_keys:
259        internal_ret_dict[key] = []
260
261    def _default_idl_func(cfg_string, cfg_sep):
262        return int(cfg_string.split(cfg_sep)[-1])
263
264    if cfg_func is None:
265        print("Default idl function in use.")
266        cfg_func = _default_idl_func
267        cfg_func_args = [cfg_separator]
268    else:
269        cfg_func_args = kwargs.get("cfg_func_args", [])
270
271    if not appended:
272        for i, item in enumerate(ls):
273            rep_path = path + '/' + item
274            if "files" in kwargs:
275                files = kwargs.get("files")
276                if isinstance(files, list):
277                    if all(isinstance(f, list) for f in files):
278                        files = files[i]
279                    elif not all(isinstance(f, str) for f in files):
280                        raise TypeError("files has to be of type list[list[str]] or list[str]!")
281                else:
282                    raise TypeError("files has to be of type list[list[str]] or list[str]!")
283
284            else:
285                files = []
286            sub_ls = _find_files(rep_path, prefix, compact, files)
287            rep_idl = []
288            no_cfg = len(sub_ls)
289            for cfg in sub_ls:
290                try:
291                    if compact:
292                        rep_idl.append(cfg_func(cfg, *cfg_func_args))
293                    else:
294                        rep_idl.append(int(cfg[3:]))
295                except Exception as err:
296                    raise Exception("Couldn't parse idl from directory, problem with file " + cfg) from err
297            rep_idl.sort()
298            # maybe there is a better way to print the idls
299            if not silent:
300                print(item, ':', no_cfg, ' configurations')
301            idl.append(rep_idl)
302            # here we have found all the files we need to look into.
303            if i == 0:
304                if version != "0.0" and compact:
305                    file = path + '/' + item + '/' + sub_ls[0]
306                for name_index, name in enumerate(name_list):
307                    if version == "0.0" or not compact:
308                        file = path + '/' + item + '/' + sub_ls[0] + '/' + name
309                    if corr_type_list[name_index] == 'bi':
310                        name_keys = _lists2key(quarks_list, noffset_list, wf_list, ["0"])
311                    else:
312                        name_keys = _lists2key(quarks_list, noffset_list, wf_list, wf2_list)
313                    for key in name_keys:
314                        specs = _key2specs(key)
315                        quarks = specs[0]
316                        off = specs[1]
317                        w = specs[2]
318                        w2 = specs[3]
319                        # here, we want to find the place within the file,
320                        # where the correlator we need is stored.
321                        # to do so, the pattern needed is put together
322                        # from the input values
323                        start_read, T = _find_correlator(file, version, intern[name]["spec"][quarks][str(off)][str(w)][str(w2)]["pattern"], intern[name]['b2b'], silent=silent)
324                        intern[name]["spec"][quarks][str(off)][str(w)][str(w2)]["start"] = start_read
325                        intern[name]["T"] = T
326                        # preparing the datastructure
327                        # the correlators get parsed into...
328                        deltas = []
329                        for _j in range(intern[name]["T"]):
330                            deltas.append([])
331                        internal_ret_dict[sep.join([name, key])] = deltas
332
333            if compact:
334                rep_deltas = _read_compact_rep(path, item, sub_ls, intern, needed_keys, im)
335                for key in needed_keys:
336                    name = _key2specs(key)[0]
337                    for t in range(intern[name]["T"]):
338                        internal_ret_dict[key][t].append(rep_deltas[key][t])
339            else:
340                for key in needed_keys:
341                    rep_data = []
342                    name = _key2specs(key)[0]
343                    for subitem in sub_ls:
344                        cfg_path = path + '/' + item + '/' + subitem
345                        file_data = _read_o_file(cfg_path, name, needed_keys, intern, version, im)
346                        rep_data.append(file_data)
347                    for t in range(intern[name]["T"]):
348                        internal_ret_dict[key][t].append([])
349                        for cfg in range(no_cfg):
350                            internal_ret_dict[key][t][i].append(rep_data[cfg][key][t])
351    else:
352        for key in needed_keys:
353            specs = _key2specs(key)
354            name = specs[0]
355            quarks = specs[1]
356            off = specs[2]
357            w = specs[3]
358            w2 = specs[4]
359            if "files" in kwargs:
360                if isinstance(kwargs.get("files"), list) and all(isinstance(f, str) for f in kwargs.get("files")):
361                    name_ls = kwargs.get("files")
362                else:
363                    raise TypeError("In append mode, files has to be of type list[str]!")
364            else:
365                name_ls = ls
366                for exc in name_ls:
367                    if not fnmatch.fnmatch(exc, prefix + '*.' + name):
368                        name_ls = list(set(name_ls) - set([exc]))
369            name_ls = sort_names(name_ls)
370            pattern = intern[name]['spec'][quarks][off][w][w2]['pattern']
371            deltas = []
372            for rep, file in enumerate(name_ls):
373                rep_idl = []
374                filename = path + '/' + file
375                T, rep_idl, rep_data = _read_append_rep(filename, pattern, intern[name]['b2b'], im, intern[name]['single'], cfg_func, cfg_func_args)
376                if rep == 0:
377                    intern[name]['T'] = T
378                    for _ in range(intern[name]['T']):
379                        deltas.append([])
380                for t in range(intern[name]['T']):
381                    deltas[t].append(rep_data[t])
382                internal_ret_dict[key] = deltas
383                if name == name_list[0]:
384                    idl.append(rep_idl)
385
386    if kwargs.get("check_configs") is True:
387        if not silent:
388            print("Checking for missing configs...")
389        che = kwargs.get("check_configs")
390        if not (len(che) == len(idl)):
391            raise ValueError("check_configs has to be the same length as replica!")
392        for r in range(len(idl)):
393            if not silent:
394                print("checking " + new_names[r])
395            check_idl(idl[r], che[r])
396        if not silent:
397            print("Done")
398
399    result_dict = {}
400    if keyed_out:
401        for key in needed_keys:
402            name = _key2specs(key)[0]
403            result = []
404            for t in range(intern[name]["T"]):
405                result.append(Obs(internal_ret_dict[key][t], new_names, idl=idl))
406            result_dict[key] = result
407    else:
408        for name, corr_type in zip(name_list, corr_type_list, strict=True):
409            result_dict[name] = {}
410            for quarks in quarks_list:
411                result_dict[name][quarks] = {}
412                for off in noffset_list:
413                    result_dict[name][quarks][off] = {}
414                    for w in wf_list:
415                        result_dict[name][quarks][off][w] = {}
416                        if corr_type != 'bi':
417                            for w2 in wf2_list:
418                                key = _specs2key(name, quarks, off, w, w2)
419                                result = []
420                                for t in range(intern[name]["T"]):
421                                    result.append(Obs(internal_ret_dict[key][t], new_names, idl=idl))
422                                result_dict[name][quarks][str(off)][str(w)][str(w2)] = result
423                        else:
424                            key = _specs2key(name, quarks, off, w, "0")
425                            result = []
426                            for t in range(intern[name]["T"]):
427                                result.append(Obs(internal_ret_dict[key][t], new_names, idl=idl))
428                            result_dict[name][quarks][str(off)][str(w)][str(0)] = result
429    return result_dict
430
431
432def _lists2key(*lists):
433    keys = []
434    for tup in itertools.product(*lists):
435        keys.append(sep.join(tup))
436    return keys
437
438
439def _key2specs(key):
440    return key.split(sep)
441
442
443def _specs2key(*specs):
444    return sep.join(specs)
445
446
447def _read_o_file(cfg_path, name, needed_keys, intern, version, im):
448    return_vals = {}
449    for key in needed_keys:
450        file = cfg_path + '/' + name
451        specs = _key2specs(key)
452        if specs[0] == name:
453            with open(file) as fp:
454                lines = fp.readlines()
455                quarks = specs[1]
456                off = specs[2]
457                w = specs[3]
458                w2 = specs[4]
459                T = intern[name]["T"]
460                start_read = intern[name]["spec"][quarks][off][w][w2]["start"]
461                deltas = []
462                for line in lines[start_read:start_read + T]:
463                    floats = list(map(float, line.split()))
464                    if version == "0.0":
465                        deltas.append(floats[im - intern[name]["single"]])
466                    else:
467                        deltas.append(floats[1 + im - intern[name]["single"]])
468                return_vals[key] = deltas
469    return return_vals
470
471
472def _extract_corr_type(corr_type):
473    if corr_type == 'bb':
474        b2b = True
475        single = True
476    elif corr_type == 'bib':
477        b2b = True
478        single = False
479    else:
480        b2b = False
481        single = False
482    return b2b, single
483
484
485def _find_files(rep_path, prefix, compact, files=None):
486    if files is None:
487        files = []
488    sub_ls = []
489    if not files == []:
490        files.sort(key=lambda x: int(re.findall(r'\d+', x)[-1]))
491    else:
492        for (_dirpath, dirnames, filenames) in os.walk(rep_path):
493            if compact:
494                sub_ls.extend(filenames)
495            else:
496                sub_ls.extend(dirnames)
497            break
498        if compact:
499            for exc in sub_ls:
500                if not fnmatch.fnmatch(exc, prefix + '*'):
501                    sub_ls = list(set(sub_ls) - set([exc]))
502            sub_ls.sort(key=lambda x: int(re.findall(r'\d+', x)[-1]))
503        else:
504            for exc in sub_ls:
505                if not fnmatch.fnmatch(exc, 'cfg*'):
506                    sub_ls = list(set(sub_ls) - set([exc]))
507            sub_ls.sort(key=lambda x: int(x[3:]))
508        files = sub_ls
509    if len(files) == 0:
510        raise FileNotFoundError("Did not find files in", rep_path, "with prefix", prefix, "and the given structure.")
511    return files
512
513
514def _make_pattern(version, name, noffset, wf, wf2, b2b, quarks):
515    if version == "0.0":
516        pattern = "# " + name + " : offset " + str(noffset) + ", wf " + str(wf)
517        if b2b:
518            pattern += ", wf_2 " + str(wf2)
519        qs = quarks.split(" ")
520        pattern += " : " + qs[0] + " - " + qs[1]
521    else:
522        pattern = 'name      ' + name + '\nquarks    ' + quarks + '\noffset    ' + str(noffset) + '\nwf        ' + str(wf)
523        if b2b:
524            pattern += '\nwf_2      ' + str(wf2)
525    return pattern
526
527
528def _find_correlator(file_name, version, pattern, b2b, silent=False):
529    T = 0
530
531    with open(file_name) as my_file:
532
533        content = my_file.read()
534        match = re.search(pattern, content)
535        if match:
536            if version == "0.0":
537                start_read = content.count('\n', 0, match.start()) + 1
538                T = content.count('\n', start_read)
539            else:
540                start_read = content.count('\n', 0, match.start()) + 5 + b2b
541                end_match = re.search(r'\n\s*\n', content[match.start():])
542                T = content[match.start():].count('\n', 0, end_match.start()) - 4 - b2b
543            if not T > 0:
544                raise ValueError("Correlator with pattern\n" + pattern + "\nis empty!")
545            if not silent:
546                print(T, 'entries, starting to read in line', start_read)
547
548        else:
549            raise ValueError('Correlator with pattern\n' + pattern + '\nnot found.')
550
551    return start_read, T
552
553
554def _read_compact_file(rep_path, cfg_file, intern, needed_keys, im):
555    return_vals = {}
556    with open(rep_path + cfg_file) as fp:
557        lines = fp.readlines()
558        for key in needed_keys:
559            keys = _key2specs(key)
560            name = keys[0]
561            quarks = keys[1]
562            off = keys[2]
563            w = keys[3]
564            w2 = keys[4]
565
566            T = intern[name]["T"]
567            start_read = intern[name]["spec"][quarks][off][w][w2]["start"]
568            # check, if the correlator is in fact
569            # printed completely
570            if (start_read + T + 1 > len(lines)):
571                raise Exception("EOF before end of correlator data! Maybe " + rep_path + cfg_file + " is corrupted?")
572            corr_lines = lines[start_read - 6: start_read + T]
573            t_vals = []
574
575            if corr_lines[1 - intern[name]["b2b"]].strip() != 'name      ' + name:
576                raise Exception('Wrong format in file', cfg_file)
577
578            for k in range(6, T + 6):
579                floats = list(map(float, corr_lines[k].split()))
580                t_vals.append(floats[-2:][im])
581            return_vals[key] = t_vals
582    return return_vals
583
584
585def _read_compact_rep(path, rep, sub_ls, intern, needed_keys, im):
586    rep_path = path + '/' + rep + '/'
587    no_cfg = len(sub_ls)
588
589    return_vals = {}
590    for key in needed_keys:
591        name = _key2specs(key)[0]
592        deltas = []
593        for _ in range(intern[name]["T"]):
594            deltas.append(np.zeros(no_cfg))
595        return_vals[key] = deltas
596
597    for cfg in range(no_cfg):
598        cfg_file = sub_ls[cfg]
599        cfg_data = _read_compact_file(rep_path, cfg_file, intern, needed_keys, im)
600        for key in needed_keys:
601            name = _key2specs(key)[0]
602            for t in range(intern[name]["T"]):
603                return_vals[key][t][cfg] = cfg_data[key][t]
604    return return_vals
605
606
607def _read_chunk_data(chunk, start_read, T, corr_line, b2b, pattern, im, single):
608    found_pat = ""
609    data = []
610    for li in chunk[corr_line + 1:corr_line + 6 + b2b]:
611        found_pat += li
612    if re.search(pattern, found_pat):
613        for _t, line in enumerate(chunk[start_read:start_read + T]):
614            floats = list(map(float, line.split()))
615            data.append(floats[im + 1 - single])
616    return data
617
618
619def _check_append_rep(content, start_list):
620    data_len_list = []
621    header_len_list = []
622    has_regular_len_heads = True
623    for chunk_num in range(len(start_list)):
624        start = start_list[chunk_num]
625        if chunk_num == len(start_list) - 1:
626            stop = len(content)
627        else:
628            stop = start_list[chunk_num + 1]
629        chunk = content[start:stop]
630        for linenumber, line in enumerate(chunk):
631            if line.startswith("[correlator]"):
632                header_len = linenumber
633                break
634        header_len_list.append(header_len)
635        data_len_list.append(len(chunk) - header_len)
636
637    if len(set(header_len_list)) > 1:
638        warnings.warn("Not all headers have the same length. Data parts do.", stacklevel=2)
639        has_regular_len_heads = False
640
641    if len(set(data_len_list)) > 1:
642        raise Exception("Irregularities in file structure found, not all run data are of the same output length")
643    return has_regular_len_heads
644
645
646def _read_chunk_structure(chunk, pattern, b2b):
647    start_read = 0
648    for linenumber, line in enumerate(chunk):
649        if line.startswith("gauge_name"):
650            gauge_line = linenumber
651        elif line.startswith("[correlator]"):
652            corr_line = linenumber
653            found_pat = ""
654            for li in chunk[corr_line + 1: corr_line + 6 + b2b]:
655                found_pat += li
656            if re.search(pattern, found_pat):
657                start_read = corr_line + 7 + b2b
658                break
659    if start_read == 0:
660        raise ValueError("Did not find pattern\n", pattern)
661    endline = corr_line + 6 + b2b
662    while not chunk[endline] == "\n":
663        endline += 1
664    T = endline - start_read
665    return gauge_line, corr_line, start_read, T
666
667
668def _read_append_rep(filename, pattern, b2b, im, single, idl_func, cfg_func_args):
669    with open(filename) as fp:
670        content = fp.readlines()
671        chunk_start_lines = []
672        for linenumber, line in enumerate(content):
673            if "[run]" in line:
674                chunk_start_lines.append(linenumber)
675        has_regular_len_heads = _check_append_rep(content, chunk_start_lines)
676        if has_regular_len_heads:
677            chunk = content[:chunk_start_lines[1]]
678            try:
679                gauge_line, corr_line, start_read, T = _read_chunk_structure(chunk, pattern, b2b)
680            except ValueError as err:
681                raise ValueError("Did not find pattern\n", pattern, "\nin\n", filename, "lines", 1, "to", chunk_start_lines[1] + 1) from err
682        # if has_regular_len_heads is true, all other chunks should follow the same structure
683        rep_idl = []
684        rep_data = []
685
686        for chunk_num in range(len(chunk_start_lines)):
687            start = chunk_start_lines[chunk_num]
688            if chunk_num == len(chunk_start_lines) - 1:
689                stop = len(content)
690            else:
691                stop = chunk_start_lines[chunk_num + 1]
692            chunk = content[start:stop]
693            if not has_regular_len_heads:
694                gauge_line, corr_line, start_read, T = _read_chunk_structure(chunk, pattern, b2b)
695            try:
696                idl = idl_func(chunk[gauge_line], *cfg_func_args)
697            except Exception as err:
698                raise Exception("Couldn't parse idl from file", filename, ", problem with chunk of lines", start + 1, "to", stop + 1) from err
699            data = _read_chunk_data(chunk, start_read, T, corr_line, b2b, pattern, im, single)
700            rep_idl.append(idl)
701            rep_data.append(data)
702
703        data = []
704
705        for t in range(T):
706            data.append([])
707            for c in range(len(rep_data)):
708                data[t].append(rep_data[c][t])
709        return T, rep_idl, data
710
711
712def _get_rep_names(ls, ens_name=None, rep_sep='r'):
713    new_names = []
714    for entry in ls:
715        try:
716            idx = entry.index(rep_sep)
717        except Exception as err:
718            raise Exception("Automatic recognition of replicum failed, please enter the key word 'names'.") from err
719
720        if ens_name:
721            new_names.append(ens_name + '|' + entry[idx:])
722        else:
723            new_names.append(entry[:idx] + '|' + entry[idx:])
724    return new_names
725
726
727def _get_appended_rep_names(ls, prefix, name, ens_name=None, rep_sep='r'):
728    new_names = []
729    for exc in ls:
730        if not fnmatch.fnmatch(exc, prefix + '*.' + name):
731            ls = list(set(ls) - set([exc]))
732    ls.sort(key=lambda x: int(re.findall(r'\d+', x)[-1]))
733    for entry in ls:
734        myentry = entry[:-len(name) - 1]
735        try:
736            idx = myentry.index(rep_sep)
737        except Exception as err:
738            raise Exception("Automatic recognition of replicum failed, please enter the key word 'names'.") from err
739
740        if ens_name:
741            new_names.append(ens_name + '|' + entry[idx:])
742        else:
743            new_names.append(myentry[:idx] + '|' + myentry[idx:])
744    return new_names