pyerrors.input.sfcf

  1import os
  2import fnmatch
  3import re
  4import numpy as np  # Thinly-wrapped numpy
  5from ..obs import Obs
  6from .utils import sort_names, check_idl
  7import itertools
  8import warnings
  9
 10
 11sep = "/"
 12
 13
 14def read_sfcf(path, prefix, name, quarks='.*', corr_type="bi", noffset=0, wf=0, wf2=0, version="1.0c", cfg_separator="n", cfg_func=None, silent=False, **kwargs):
 15    """Read sfcf files from given folder structure.
 16
 17    Parameters
 18    ----------
 19    path : str
 20        Path to the sfcf files.
 21    prefix : str
 22        Prefix of the sfcf files.
 23    name : str
 24        Name of the correlation function to read.
 25    quarks : str
 26        Label of the quarks used in the sfcf input file. e.g. "quark quark"
 27        for version 0.0 this does NOT need to be given with the typical " - "
 28        that is present in the output file,
 29        this is done automatically for this version
 30    corr_type : str
 31        Type of correlation function to read. Can be
 32        - 'bi' for boundary-inner
 33        - 'bb' for boundary-boundary
 34        - 'bib' for boundary-inner-boundary
 35    noffset : int
 36        Offset of the source (only relevant when wavefunctions are used)
 37    wf : int
 38        ID of wave function
 39    wf2 : int
 40        ID of the second wavefunction
 41        (only relevant for boundary-to-boundary correlation functions)
 42    im : bool
 43        if True, read imaginary instead of real part
 44        of the correlation function.
 45    names : list
 46        Alternative labeling for replicas/ensembles.
 47        Has to have the appropriate length
 48    ens_name : str
 49        replaces the name of the ensemble
 50    version: str
 51        version of SFCF, with which the measurement was done.
 52        if the compact output option (-c) was specified,
 53        append a "c" to the version (e.g. "1.0c")
 54        if the append output option (-a) was specified,
 55        append an "a" to the version
 56    cfg_separator : str
 57        String that separates the ensemble identifier from the configuration number (default 'n').
 58    replica: list
 59        list of replica to be read, default is all
 60    files: list
 61        list of files to be read per replica, default is all.
 62        for non-compact output format, hand the folders to be read here.
 63    check_configs: list[list[int]]
 64        list of list of supposed configs, eg. [range(1,1000)]
 65        for one replicum with 1000 configs
 66
 67    Returns
 68    -------
 69    result: list[Obs]
 70        list of Observables with length T, observable per timeslice.
 71        bb-type correlators have length 1.
 72    """
 73    ret = read_sfcf_multi(path, prefix, [name], quarks_list=[quarks], corr_type_list=[corr_type],
 74                          noffset_list=[noffset], wf_list=[wf], wf2_list=[wf2], version=version,
 75                          cfg_separator=cfg_separator, cfg_func=cfg_func, silent=silent, **kwargs)
 76    return ret[name][quarks][str(noffset)][str(wf)][str(wf2)]
 77
 78
 79def read_sfcf_multi(path, prefix, name_list, quarks_list=['.*'], corr_type_list=['bi'], noffset_list=[0], wf_list=[0], wf2_list=[0], version="1.0c", cfg_separator="n", cfg_func=None, silent=False, keyed_out=False, **kwargs):
 80    """Read sfcf files from given folder structure.
 81
 82    Parameters
 83    ----------
 84    path : str
 85        Path to the sfcf files.
 86    prefix : str
 87        Prefix of the sfcf files.
 88    name : str
 89        Name of the correlation function to read.
 90    quarks_list : list[str]
 91        Label of the quarks used in the sfcf input file. e.g. "quark quark"
 92        for version 0.0 this does NOT need to be given with the typical " - "
 93        that is present in the output file,
 94        this is done automatically for this version
 95    corr_type_list : list[str]
 96        Type of correlation function to read. Can be
 97        - 'bi' for boundary-inner
 98        - 'bb' for boundary-boundary
 99        - 'bib' for boundary-inner-boundary
100    noffset_list : list[int]
101        Offset of the source (only relevant when wavefunctions are used)
102    wf_list : int
103        ID of wave function
104    wf2_list : list[int]
105        ID of the second wavefunction
106        (only relevant for boundary-to-boundary correlation functions)
107    im : bool
108        if True, read imaginary instead of real part
109        of the correlation function.
110    names : list
111        Alternative labeling for replicas/ensembles.
112        Has to have the appropriate length
113    ens_name : str
114        replaces the name of the ensemble
115    version: str
116        version of SFCF, with which the measurement was done.
117        if the compact output option (-c) was specified,
118        append a "c" to the version (e.g. "1.0c")
119        if the append output option (-a) was specified,
120        append an "a" to the version
121    cfg_separator : str
122        String that separates the ensemble identifier from the configuration number (default 'n').
123    replica: list
124        list of replica to be read, default is all
125    files: list[list[int]]
126        list of files to be read per replica, default is all.
127        for non-compact output format, hand the folders to be read here.
128    check_configs: list[list[int]]
129        list of list of supposed configs, eg. [range(1,1000)]
130        for one replicum with 1000 configs
131    rep_string: str
132        Separator of ensemble name and replicum. Example: In "ensAr0", "r" would be the separator string.
133    Returns
134    -------
135    result: dict[list[Obs]]
136        dict with one of the following properties:
137        if keyed_out:
138            dict[key] = list[Obs]
139            where key has the form name/quarks/offset/wf/wf2
140        if not keyed_out:
141            dict[name][quarks][offset][wf][wf2] = list[Obs]
142    """
143
144    if kwargs.get('im'):
145        im = 1
146        part = 'imaginary'
147    else:
148        im = 0
149        part = 'real'
150
151    known_versions = ["0.0", "1.0", "2.0", "1.0c", "2.0c", "1.0a", "2.0a"]
152
153    if version not in known_versions:
154        raise Exception("This version is not known!")
155    if (version[-1] == "c"):
156        appended = False
157        compact = True
158        version = version[:-1]
159    elif (version[-1] == "a"):
160        appended = True
161        compact = False
162        version = version[:-1]
163    else:
164        compact = False
165        appended = False
166    ls = []
167    if "replica" in kwargs:
168        ls = kwargs.get("replica")
169    else:
170        for (dirpath, dirnames, filenames) in os.walk(path):
171            if not appended:
172                ls.extend(dirnames)
173            else:
174                ls.extend(filenames)
175            break
176        if not ls:
177            raise Exception('Error, directory not found')
178        # Exclude folders with different names
179        for exc in ls:
180            if not fnmatch.fnmatch(exc, prefix + '*'):
181                ls = list(set(ls) - set([exc]))
182
183    if not appended:
184        ls = sort_names(ls)
185        replica = len(ls)
186
187    else:
188        replica = len([file.split(".")[-1] for file in ls]) // len(set([file.split(".")[-1] for file in ls]))
189    if replica == 0:
190        raise Exception('No replica found in directory')
191    if not silent:
192        print('Read', part, 'part of', name_list, 'from', prefix[:-1], ',', replica, 'replica')
193
194    if 'names' in kwargs:
195        new_names = kwargs.get('names')
196        if len(new_names) != len(set(new_names)):
197            raise Exception("names are not unique!")
198        if len(new_names) != replica:
199            raise Exception('names should have the length', replica)
200
201    else:
202        ens_name = kwargs.get("ens_name")
203        if not appended:
204            new_names = _get_rep_names(ls, ens_name, rep_sep=(kwargs.get('rep_string', 'r')))
205        else:
206            new_names = _get_appended_rep_names(ls, prefix, name_list[0], ens_name, rep_sep=(kwargs.get('rep_string', 'r')))
207        new_names = sort_names(new_names)
208
209    idl = []
210
211    noffset_list = [str(x) for x in noffset_list]
212    wf_list = [str(x) for x in wf_list]
213    wf2_list = [str(x) for x in wf2_list]
214
215    # setup dict structures
216    intern = {}
217    for name, corr_type in zip(name_list, corr_type_list):
218        intern[name] = {}
219        b2b, single = _extract_corr_type(corr_type)
220        intern[name]["b2b"] = b2b
221        intern[name]["single"] = single
222        intern[name]["spec"] = {}
223        for quarks in quarks_list:
224            intern[name]["spec"][quarks] = {}
225            for off in noffset_list:
226                intern[name]["spec"][quarks][off] = {}
227                for w in wf_list:
228                    intern[name]["spec"][quarks][off][w] = {}
229                    if b2b:
230                        for w2 in wf2_list:
231                            intern[name]["spec"][quarks][off][w][w2] = {}
232                            intern[name]["spec"][quarks][off][w][w2]["pattern"] = _make_pattern(version, name, off, w, w2, intern[name]['b2b'], quarks)
233                    else:
234                        intern[name]["spec"][quarks][off][w]["0"] = {}
235                        intern[name]["spec"][quarks][off][w]["0"]["pattern"] = _make_pattern(version, name, off, w, 0, intern[name]['b2b'], quarks)
236
237    internal_ret_dict = {}
238    needed_keys = []
239    for name, corr_type in zip(name_list, corr_type_list):
240        b2b, single = _extract_corr_type(corr_type)
241        if b2b:
242            needed_keys.extend(_lists2key([name], quarks_list, noffset_list, wf_list, wf2_list))
243        else:
244            needed_keys.extend(_lists2key([name], quarks_list, noffset_list, wf_list, ["0"]))
245
246    for key in needed_keys:
247        internal_ret_dict[key] = []
248
249    def _default_idl_func(cfg_string, cfg_sep):
250        return int(cfg_string.split(cfg_sep)[-1])
251
252    if cfg_func is None:
253        print("Default idl function in use.")
254        cfg_func = _default_idl_func
255        cfg_func_args = [cfg_separator]
256    else:
257        cfg_func_args = kwargs.get("cfg_func_args", [])
258
259    if not appended:
260        for i, item in enumerate(ls):
261            rep_path = path + '/' + item
262            if "files" in kwargs:
263                files = kwargs.get("files")
264                if isinstance(files, list):
265                    if all(isinstance(f, list) for f in files):
266                        files = files[i]
267                    elif all(isinstance(f, str) for f in files):
268                        files = files
269                    else:
270                        raise TypeError("files has to be of type list[list[str]] or list[str]!")
271                else:
272                    raise TypeError("files has to be of type list[list[str]] or list[str]!")
273
274            else:
275                files = []
276            sub_ls = _find_files(rep_path, prefix, compact, files)
277            rep_idl = []
278            no_cfg = len(sub_ls)
279            for cfg in sub_ls:
280                try:
281                    if compact:
282                        rep_idl.append(cfg_func(cfg, *cfg_func_args))
283                    else:
284                        rep_idl.append(int(cfg[3:]))
285                except Exception:
286                    raise Exception("Couldn't parse idl from directory, problem with file " + cfg)
287            rep_idl.sort()
288            # maybe there is a better way to print the idls
289            if not silent:
290                print(item, ':', no_cfg, ' configurations')
291            idl.append(rep_idl)
292            # here we have found all the files we need to look into.
293            if i == 0:
294                if version != "0.0" and compact:
295                    file = path + '/' + item + '/' + sub_ls[0]
296                for name_index, name in enumerate(name_list):
297                    if version == "0.0" or not compact:
298                        file = path + '/' + item + '/' + sub_ls[0] + '/' + name
299                    if corr_type_list[name_index] == 'bi':
300                        name_keys = _lists2key(quarks_list, noffset_list, wf_list, ["0"])
301                    else:
302                        name_keys = _lists2key(quarks_list, noffset_list, wf_list, wf2_list)
303                    for key in name_keys:
304                        specs = _key2specs(key)
305                        quarks = specs[0]
306                        off = specs[1]
307                        w = specs[2]
308                        w2 = specs[3]
309                        # here, we want to find the place within the file,
310                        # where the correlator we need is stored.
311                        # to do so, the pattern needed is put together
312                        # from the input values
313                        start_read, T = _find_correlator(file, version, intern[name]["spec"][quarks][str(off)][str(w)][str(w2)]["pattern"], intern[name]['b2b'], silent=silent)
314                        intern[name]["spec"][quarks][str(off)][str(w)][str(w2)]["start"] = start_read
315                        intern[name]["T"] = T
316                        # preparing the datastructure
317                        # the correlators get parsed into...
318                        deltas = []
319                        for j in range(intern[name]["T"]):
320                            deltas.append([])
321                        internal_ret_dict[sep.join([name, key])] = deltas
322
323            if compact:
324                rep_deltas = _read_compact_rep(path, item, sub_ls, intern, needed_keys, im)
325                for key in needed_keys:
326                    name = _key2specs(key)[0]
327                    for t in range(intern[name]["T"]):
328                        internal_ret_dict[key][t].append(rep_deltas[key][t])
329            else:
330                for key in needed_keys:
331                    rep_data = []
332                    name = _key2specs(key)[0]
333                    for subitem in sub_ls:
334                        cfg_path = path + '/' + item + '/' + subitem
335                        file_data = _read_o_file(cfg_path, name, needed_keys, intern, version, im)
336                        rep_data.append(file_data)
337                    for t in range(intern[name]["T"]):
338                        internal_ret_dict[key][t].append([])
339                        for cfg in range(no_cfg):
340                            internal_ret_dict[key][t][i].append(rep_data[cfg][key][t])
341    else:
342        for key in needed_keys:
343            specs = _key2specs(key)
344            name = specs[0]
345            quarks = specs[1]
346            off = specs[2]
347            w = specs[3]
348            w2 = specs[4]
349            if "files" in kwargs:
350                if isinstance(kwargs.get("files"), list) and all(isinstance(f, str) for f in kwargs.get("files")):
351                    name_ls = kwargs.get("files")
352                else:
353                    raise TypeError("In append mode, files has to be of type list[str]!")
354            else:
355                name_ls = ls
356                for exc in name_ls:
357                    if not fnmatch.fnmatch(exc, prefix + '*.' + name):
358                        name_ls = list(set(name_ls) - set([exc]))
359            name_ls = sort_names(name_ls)
360            pattern = intern[name]['spec'][quarks][off][w][w2]['pattern']
361            deltas = []
362            for rep, file in enumerate(name_ls):
363                rep_idl = []
364                filename = path + '/' + file
365                T, rep_idl, rep_data = _read_append_rep(filename, pattern, intern[name]['b2b'], im, intern[name]['single'], cfg_func, cfg_func_args)
366                if rep == 0:
367                    intern[name]['T'] = T
368                    for t in range(intern[name]['T']):
369                        deltas.append([])
370                for t in range(intern[name]['T']):
371                    deltas[t].append(rep_data[t])
372                internal_ret_dict[key] = deltas
373                if name == name_list[0]:
374                    idl.append(rep_idl)
375
376    if kwargs.get("check_configs") is True:
377        if not silent:
378            print("Checking for missing configs...")
379        che = kwargs.get("check_configs")
380        if not (len(che) == len(idl)):
381            raise Exception("check_configs has to be the same length as replica!")
382        for r in range(len(idl)):
383            if not silent:
384                print("checking " + new_names[r])
385            check_idl(idl[r], che[r])
386        if not silent:
387            print("Done")
388
389    result_dict = {}
390    if keyed_out:
391        for key in needed_keys:
392            name = _key2specs(key)[0]
393            result = []
394            for t in range(intern[name]["T"]):
395                result.append(Obs(internal_ret_dict[key][t], new_names, idl=idl))
396            result_dict[key] = result
397    else:
398        for name, corr_type in zip(name_list, corr_type_list):
399            result_dict[name] = {}
400            for quarks in quarks_list:
401                result_dict[name][quarks] = {}
402                for off in noffset_list:
403                    result_dict[name][quarks][off] = {}
404                    for w in wf_list:
405                        result_dict[name][quarks][off][w] = {}
406                        if corr_type != 'bi':
407                            for w2 in wf2_list:
408                                key = _specs2key(name, quarks, off, w, w2)
409                                result = []
410                                for t in range(intern[name]["T"]):
411                                    result.append(Obs(internal_ret_dict[key][t], new_names, idl=idl))
412                                result_dict[name][quarks][str(off)][str(w)][str(w2)] = result
413                        else:
414                            key = _specs2key(name, quarks, off, w, "0")
415                            result = []
416                            for t in range(intern[name]["T"]):
417                                result.append(Obs(internal_ret_dict[key][t], new_names, idl=idl))
418                            result_dict[name][quarks][str(off)][str(w)][str(0)] = result
419    return result_dict
420
421
422def _lists2key(*lists):
423    keys = []
424    for tup in itertools.product(*lists):
425        keys.append(sep.join(tup))
426    return keys
427
428
429def _key2specs(key):
430    return key.split(sep)
431
432
433def _specs2key(*specs):
434    return sep.join(specs)
435
436
437def _read_o_file(cfg_path, name, needed_keys, intern, version, im):
438    return_vals = {}
439    for key in needed_keys:
440        file = cfg_path + '/' + name
441        specs = _key2specs(key)
442        if specs[0] == name:
443            with open(file) as fp:
444                lines = fp.readlines()
445                quarks = specs[1]
446                off = specs[2]
447                w = specs[3]
448                w2 = specs[4]
449                T = intern[name]["T"]
450                start_read = intern[name]["spec"][quarks][off][w][w2]["start"]
451                deltas = []
452                for line in lines[start_read:start_read + T]:
453                    floats = list(map(float, line.split()))
454                    if version == "0.0":
455                        deltas.append(floats[im - intern[name]["single"]])
456                    else:
457                        deltas.append(floats[1 + im - intern[name]["single"]])
458                return_vals[key] = deltas
459    return return_vals
460
461
462def _extract_corr_type(corr_type):
463    if corr_type == 'bb':
464        b2b = True
465        single = True
466    elif corr_type == 'bib':
467        b2b = True
468        single = False
469    else:
470        b2b = False
471        single = False
472    return b2b, single
473
474
475def _find_files(rep_path, prefix, compact, files=[]):
476    sub_ls = []
477    if not files == []:
478        files.sort(key=lambda x: int(re.findall(r'\d+', x)[-1]))
479    else:
480        for (dirpath, dirnames, filenames) in os.walk(rep_path):
481            if compact:
482                sub_ls.extend(filenames)
483            else:
484                sub_ls.extend(dirnames)
485            break
486        if compact:
487            for exc in sub_ls:
488                if not fnmatch.fnmatch(exc, prefix + '*'):
489                    sub_ls = list(set(sub_ls) - set([exc]))
490            sub_ls.sort(key=lambda x: int(re.findall(r'\d+', x)[-1]))
491        else:
492            for exc in sub_ls:
493                if not fnmatch.fnmatch(exc, 'cfg*'):
494                    sub_ls = list(set(sub_ls) - set([exc]))
495            sub_ls.sort(key=lambda x: int(x[3:]))
496        files = sub_ls
497    if len(files) == 0:
498        raise FileNotFoundError("Did not find files in", rep_path, "with prefix", prefix, "and the given structure.")
499    return files
500
501
502def _make_pattern(version, name, noffset, wf, wf2, b2b, quarks):
503    if version == "0.0":
504        pattern = "# " + name + " : offset " + str(noffset) + ", wf " + str(wf)
505        if b2b:
506            pattern += ", wf_2 " + str(wf2)
507        qs = quarks.split(" ")
508        pattern += " : " + qs[0] + " - " + qs[1]
509    else:
510        pattern = 'name      ' + name + '\nquarks    ' + quarks + '\noffset    ' + str(noffset) + '\nwf        ' + str(wf)
511        if b2b:
512            pattern += '\nwf_2      ' + str(wf2)
513    return pattern
514
515
516def _find_correlator(file_name, version, pattern, b2b, silent=False):
517    T = 0
518
519    with open(file_name, "r") as my_file:
520
521        content = my_file.read()
522        match = re.search(pattern, content)
523        if match:
524            if version == "0.0":
525                start_read = content.count('\n', 0, match.start()) + 1
526                T = content.count('\n', start_read)
527            else:
528                start_read = content.count('\n', 0, match.start()) + 5 + b2b
529                end_match = re.search(r'\n\s*\n', content[match.start():])
530                T = content[match.start():].count('\n', 0, end_match.start()) - 4 - b2b
531            if not T > 0:
532                raise ValueError("Correlator with pattern\n" + pattern + "\nis empty!")
533            if not silent:
534                print(T, 'entries, starting to read in line', start_read)
535
536        else:
537            raise ValueError('Correlator with pattern\n' + pattern + '\nnot found.')
538
539    return start_read, T
540
541
542def _read_compact_file(rep_path, cfg_file, intern, needed_keys, im):
543    return_vals = {}
544    with open(rep_path + cfg_file) as fp:
545        lines = fp.readlines()
546        for key in needed_keys:
547            keys = _key2specs(key)
548            name = keys[0]
549            quarks = keys[1]
550            off = keys[2]
551            w = keys[3]
552            w2 = keys[4]
553
554            T = intern[name]["T"]
555            start_read = intern[name]["spec"][quarks][off][w][w2]["start"]
556            # check, if the correlator is in fact
557            # printed completely
558            if (start_read + T + 1 > len(lines)):
559                raise Exception("EOF before end of correlator data! Maybe " + rep_path + cfg_file + " is corrupted?")
560            corr_lines = lines[start_read - 6: start_read + T]
561            t_vals = []
562
563            if corr_lines[1 - intern[name]["b2b"]].strip() != 'name      ' + name:
564                raise Exception('Wrong format in file', cfg_file)
565
566            for k in range(6, T + 6):
567                floats = list(map(float, corr_lines[k].split()))
568                t_vals.append(floats[-2:][im])
569            return_vals[key] = t_vals
570    return return_vals
571
572
573def _read_compact_rep(path, rep, sub_ls, intern, needed_keys, im):
574    rep_path = path + '/' + rep + '/'
575    no_cfg = len(sub_ls)
576
577    return_vals = {}
578    for key in needed_keys:
579        name = _key2specs(key)[0]
580        deltas = []
581        for t in range(intern[name]["T"]):
582            deltas.append(np.zeros(no_cfg))
583        return_vals[key] = deltas
584
585    for cfg in range(no_cfg):
586        cfg_file = sub_ls[cfg]
587        cfg_data = _read_compact_file(rep_path, cfg_file, intern, needed_keys, im)
588        for key in needed_keys:
589            name = _key2specs(key)[0]
590            for t in range(intern[name]["T"]):
591                return_vals[key][t][cfg] = cfg_data[key][t]
592    return return_vals
593
594
595def _read_chunk_data(chunk, start_read, T, corr_line, b2b, pattern, im, single):
596    found_pat = ""
597    data = []
598    for li in chunk[corr_line + 1:corr_line + 6 + b2b]:
599        found_pat += li
600    if re.search(pattern, found_pat):
601        for t, line in enumerate(chunk[start_read:start_read + T]):
602            floats = list(map(float, line.split()))
603            data.append(floats[im + 1 - single])
604    return data
605
606
607def _check_append_rep(content, start_list):
608    data_len_list = []
609    header_len_list = []
610    has_regular_len_heads = True
611    for chunk_num in range(len(start_list)):
612        start = start_list[chunk_num]
613        if chunk_num == len(start_list) - 1:
614            stop = len(content)
615        else:
616            stop = start_list[chunk_num + 1]
617        chunk = content[start:stop]
618        for linenumber, line in enumerate(chunk):
619            if line.startswith("[correlator]"):
620                header_len = linenumber
621                break
622        header_len_list.append(header_len)
623        data_len_list.append(len(chunk) - header_len)
624
625    if len(set(header_len_list)) > 1:
626        warnings.warn("Not all headers have the same length. Data parts do.")
627        has_regular_len_heads = False
628
629    if len(set(data_len_list)) > 1:
630        raise Exception("Irregularities in file structure found, not all run data are of the same output length")
631    return has_regular_len_heads
632
633
634def _read_chunk_structure(chunk, pattern, b2b):
635    start_read = 0
636    for linenumber, line in enumerate(chunk):
637        if line.startswith("gauge_name"):
638            gauge_line = linenumber
639        elif line.startswith("[correlator]"):
640            corr_line = linenumber
641            found_pat = ""
642            for li in chunk[corr_line + 1: corr_line + 6 + b2b]:
643                found_pat += li
644            if re.search(pattern, found_pat):
645                start_read = corr_line + 7 + b2b
646                break
647    if start_read == 0:
648        raise ValueError("Did not find pattern\n", pattern)
649    endline = corr_line + 6 + b2b
650    while not chunk[endline] == "\n":
651        endline += 1
652    T = endline - start_read
653    return gauge_line, corr_line, start_read, T
654
655
656def _read_append_rep(filename, pattern, b2b, im, single, idl_func, cfg_func_args):
657    with open(filename, 'r') as fp:
658        content = fp.readlines()
659        chunk_start_lines = []
660        for linenumber, line in enumerate(content):
661            if "[run]" in line:
662                chunk_start_lines.append(linenumber)
663        has_regular_len_heads = _check_append_rep(content, chunk_start_lines)
664        if has_regular_len_heads:
665            chunk = content[:chunk_start_lines[1]]
666            try:
667                gauge_line, corr_line, start_read, T = _read_chunk_structure(chunk, pattern, b2b)
668            except ValueError:
669                raise ValueError("Did not find pattern\n", pattern, "\nin\n", filename, "lines", 1, "to", chunk_start_lines[1] + 1)
670        # if has_regular_len_heads is true, all other chunks should follow the same structure
671        rep_idl = []
672        rep_data = []
673
674        for chunk_num in range(len(chunk_start_lines)):
675            start = chunk_start_lines[chunk_num]
676            if chunk_num == len(chunk_start_lines) - 1:
677                stop = len(content)
678            else:
679                stop = chunk_start_lines[chunk_num + 1]
680            chunk = content[start:stop]
681            if not has_regular_len_heads:
682                gauge_line, corr_line, start_read, T = _read_chunk_structure(chunk, pattern, b2b)
683            try:
684                idl = idl_func(chunk[gauge_line], *cfg_func_args)
685            except Exception:
686                raise Exception("Couldn't parse idl from file", filename, ", problem with chunk of lines", start + 1, "to", stop + 1)
687            data = _read_chunk_data(chunk, start_read, T, corr_line, b2b, pattern, im, single)
688            rep_idl.append(idl)
689            rep_data.append(data)
690
691        data = []
692
693        for t in range(T):
694            data.append([])
695            for c in range(len(rep_data)):
696                data[t].append(rep_data[c][t])
697        return T, rep_idl, data
698
699
700def _get_rep_names(ls, ens_name=None, rep_sep='r'):
701    new_names = []
702    for entry in ls:
703        try:
704            idx = entry.index(rep_sep)
705        except Exception:
706            raise Exception("Automatic recognition of replicum failed, please enter the key word 'names'.")
707
708        if ens_name:
709            new_names.append(ens_name + '|' + entry[idx:])
710        else:
711            new_names.append(entry[:idx] + '|' + entry[idx:])
712    return new_names
713
714
715def _get_appended_rep_names(ls, prefix, name, ens_name=None, rep_sep='r'):
716    new_names = []
717    for exc in ls:
718        if not fnmatch.fnmatch(exc, prefix + '*.' + name):
719            ls = list(set(ls) - set([exc]))
720    ls.sort(key=lambda x: int(re.findall(r'\d+', x)[-1]))
721    for entry in ls:
722        myentry = entry[:-len(name) - 1]
723        try:
724            idx = myentry.index(rep_sep)
725        except Exception:
726            raise Exception("Automatic recognition of replicum failed, please enter the key word 'names'.")
727
728        if ens_name:
729            new_names.append(ens_name + '|' + entry[idx:])
730        else:
731            new_names.append(myentry[:idx] + '|' + myentry[idx:])
732    return new_names