# pyerrors.input.json

View Source

      1import rapidjson as json
      2import gzip
      3import getpass
      4import socket
      5import datetime
      6import platform
      7import warnings
      8import re
      9import numpy as np
     10from ..obs import Obs
     11from ..covobs import Covobs
     12from ..correlators import Corr
     13from ..misc import _assert_equal_properties
     14from .. import version as pyerrorsversion
     15
     16
     17def create_json_string(ol, description='', indent=1):
     18    """Generate the string for the export of a list of Obs or structures containing Obs
     19    to a .json(.gz) file
     20
     21    Parameters
     22    ----------
     23    ol : list
     24        List of objects that will be exported. At the moment, these objects can be
     25        either of: Obs, list, numpy.ndarray, Corr.
     26        All Obs inside a structure have to be defined on the same set of configurations.
     27    description : str
     28        Optional string that describes the contents of the json file.
     29    indent : int
     30        Specify the indentation level of the json file. None or 0 is permissible and
     31        saves disk space.
     32
     33    Returns
     34    -------
     35    json_string : str
     36        String for export to .json(.gz) file
     37    """
     38
     39    def _gen_data_d_from_list(ol):
     40        dl = []
     41        No = len(ol)
     42        for name in ol[0].mc_names:
     43            ed = {}
     44            ed['id'] = name
     45            ed['replica'] = []
     46            for r_name in ol[0].e_content[name]:
     47                rd = {}
     48                rd['name'] = r_name
     49                rd['deltas'] = []
     50                offsets = [o.r_values[r_name] - o.value for o in ol]
     51                deltas = np.column_stack([ol[oi].deltas[r_name] + offsets[oi] for oi in range(No)])
     52                for i in range(len(ol[0].idl[r_name])):
     53                    rd['deltas'].append([ol[0].idl[r_name][i]])
     54                    rd['deltas'][-1] += deltas[i].tolist()
     55                ed['replica'].append(rd)
     56            dl.append(ed)
     57        return dl
     58
     59    def _gen_cdata_d_from_list(ol):
     60        dl = []
     61        for name in ol[0].cov_names:
     62            ed = {}
     63            ed['id'] = name
     64            ed['layout'] = str(ol[0].covobs[name].cov.shape).lstrip('(').rstrip(')').rstrip(',')
     65            ed['cov'] = list(np.ravel(ol[0].covobs[name].cov))
     66            ncov = ol[0].covobs[name].cov.shape[0]
     67            ed['grad'] = []
     68            for i in range(ncov):
     69                ed['grad'].append([])
     70                for o in ol:
     71                    ed['grad'][-1].append(o.covobs[name].grad[i][0])
     72            dl.append(ed)
     73        return dl
     74
     75    def write_Obs_to_dict(o):
     76        d = {}
     77        d['type'] = 'Obs'
     78        d['layout'] = '1'
     79        if o.tag:
     80            d['tag'] = [o.tag]
     81        if o.reweighted:
     82            d['reweighted'] = o.reweighted
     83        d['value'] = [o.value]
     84        data = _gen_data_d_from_list([o])
     85        if len(data) > 0:
     86            d['data'] = data
     87        cdata = _gen_cdata_d_from_list([o])
     88        if len(cdata) > 0:
     89            d['cdata'] = cdata
     90        return d
     91
     92    def write_List_to_dict(ol):
     93        _assert_equal_properties(ol)
     94        d = {}
     95        d['type'] = 'List'
     96        d['layout'] = '%d' % len(ol)
     97        taglist = [o.tag for o in ol]
     98        if np.any([tag is not None for tag in taglist]):
     99            d['tag'] = taglist
    100        if ol[0].reweighted:
    101            d['reweighted'] = ol[0].reweighted
    102        d['value'] = [o.value for o in ol]
    103        data = _gen_data_d_from_list(ol)
    104        if len(data) > 0:
    105            d['data'] = data
    106        cdata = _gen_cdata_d_from_list(ol)
    107        if len(cdata) > 0:
    108            d['cdata'] = cdata
    109        return d
    110
    111    def write_Array_to_dict(oa):
    112        ol = np.ravel(oa)
    113        _assert_equal_properties(ol)
    114        d = {}
    115        d['type'] = 'Array'
    116        d['layout'] = str(oa.shape).lstrip('(').rstrip(')').rstrip(',')
    117        taglist = [o.tag for o in ol]
    118        if np.any([tag is not None for tag in taglist]):
    119            d['tag'] = taglist
    120        if ol[0].reweighted:
    121            d['reweighted'] = ol[0].reweighted
    122        d['value'] = [o.value for o in ol]
    123        data = _gen_data_d_from_list(ol)
    124        if len(data) > 0:
    125            d['data'] = data
    126        cdata = _gen_cdata_d_from_list(ol)
    127        if len(cdata) > 0:
    128            d['cdata'] = cdata
    129        return d
    130
    131    def _nan_Obs_like(obs):
    132        samples = []
    133        names = []
    134        idl = []
    135        for key, value in obs.idl.items():
    136            samples.append([np.nan] * len(value))
    137            names.append(key)
    138            idl.append(value)
    139        my_obs = Obs(samples, names, idl)
    140        my_obs._covobs = obs._covobs
    141        for name in obs._covobs:
    142            my_obs.names.append(name)
    143        my_obs.reweighted = obs.reweighted
    144        return my_obs
    145
    146    def write_Corr_to_dict(my_corr):
    147        first_not_none = next(i for i, j in enumerate(my_corr.content) if np.all(j))
    148        dummy_array = np.empty((my_corr.N, my_corr.N), dtype=object)
    149        dummy_array[:] = _nan_Obs_like(my_corr.content[first_not_none].ravel()[0])
    150        content = [o if o is not None else dummy_array for o in my_corr.content]
    151        dat = write_Array_to_dict(np.array(content, dtype=object))
    152        dat['type'] = 'Corr'
    153        corr_meta_data = str(my_corr.tag)
    154        if 'tag' in dat.keys():
    155            dat['tag'].append(corr_meta_data)
    156        else:
    157            dat['tag'] = [corr_meta_data]
    158        taglist = dat['tag']
    159        dat['tag'] = {}  # tag is now a dictionary, that contains the previous taglist in the key "tag"
    160        dat['tag']['tag'] = taglist
    161        if my_corr.prange is not None:
    162            dat['tag']['prange'] = my_corr.prange
    163        return dat
    164
    165    if not isinstance(ol, list):
    166        ol = [ol]
    167
    168    d = {}
    169    d['program'] = 'pyerrors %s' % (pyerrorsversion.__version__)
    170    d['version'] = '1.1'
    171    d['who'] = getpass.getuser()
    172    d['date'] = datetime.datetime.now().astimezone().strftime('%Y-%m-%d %H:%M:%S %z')
    173    d['host'] = socket.gethostname() + ', ' + platform.platform()
    174
    175    if description:
    176        d['description'] = description
    177
    178    d['obsdata'] = []
    179    for io in ol:
    180        if isinstance(io, Obs):
    181            d['obsdata'].append(write_Obs_to_dict(io))
    182        elif isinstance(io, list):
    183            d['obsdata'].append(write_List_to_dict(io))
    184        elif isinstance(io, np.ndarray):
    185            d['obsdata'].append(write_Array_to_dict(io))
    186        elif isinstance(io, Corr):
    187            d['obsdata'].append(write_Corr_to_dict(io))
    188        else:
    189            raise Exception("Unkown datatype.")
    190
    191    def _jsonifier(o):
    192        if isinstance(o, np.int64):
    193            return int(o)
    194        raise TypeError('%r is not JSON serializable' % o)
    195
    196    if indent:
    197        return json.dumps(d, indent=indent, ensure_ascii=False, default=_jsonifier, write_mode=json.WM_SINGLE_LINE_ARRAY)
    198    else:
    199        return json.dumps(d, indent=indent, ensure_ascii=False, default=_jsonifier, write_mode=json.WM_COMPACT)
    200
    201
    202def dump_to_json(ol, fname, description='', indent=1, gz=True):
    203    """Export a list of Obs or structures containing Obs to a .json(.gz) file
    204
    205    Parameters
    206    ----------
    207    ol : list
    208        List of objects that will be exported. At the moment, these objects can be
    209        either of: Obs, list, numpy.ndarray, Corr.
    210        All Obs inside a structure have to be defined on the same set of configurations.
    211    fname : str
    212        Filename of the output file.
    213    description : str
    214        Optional string that describes the contents of the json file.
    215    indent : int
    216        Specify the indentation level of the json file. None or 0 is permissible and
    217        saves disk space.
    218    gz : bool
    219        If True, the output is a gzipped json. If False, the output is a json file.
    220
    221    Returns
    222    -------
    223    Null
    224    """
    225
    226    jsonstring = create_json_string(ol, description, indent)
    227
    228    if not fname.endswith('.json') and not fname.endswith('.gz'):
    229        fname += '.json'
    230
    231    if gz:
    232        if not fname.endswith('.gz'):
    233            fname += '.gz'
    234
    235        fp = gzip.open(fname, 'wb')
    236        fp.write(jsonstring.encode('utf-8'))
    237    else:
    238        fp = open(fname, 'w', encoding='utf-8')
    239        fp.write(jsonstring)
    240    fp.close()
    241
    242
    243def _parse_json_dict(json_dict, verbose=True, full_output=False):
    244    """Reconstruct a list of Obs or structures containing Obs from a dict that
    245    was built out of a json string.
    246
    247    The following structures are supported: Obs, list, numpy.ndarray, Corr
    248    If the list contains only one element, it is unpacked from the list.
    249
    250    Parameters
    251    ----------
    252    json_string : str
    253        json string containing the data.
    254    verbose : bool
    255        Print additional information that was written to the file.
    256    full_output : bool
    257        If True, a dict containing auxiliary information and the data is returned.
    258        If False, only the data is returned.
    259
    260    Returns
    261    -------
    262    result : list[Obs]
    263        reconstructed list of observables from the json string
    264    or
    265    result : Obs
    266        only one observable if the list only has one entry
    267    or
    268    result : dict
    269        if full_output=True
    270    """
    271
    272    def _gen_obsd_from_datad(d):
    273        retd = {}
    274        if d:
    275            retd['names'] = []
    276            retd['idl'] = []
    277            retd['deltas'] = []
    278            for ens in d:
    279                for rep in ens['replica']:
    280                    rep_name = rep['name']
    281                    if len(rep_name) > len(ens["id"]):
    282                        if rep_name[len(ens["id"])] != "|":
    283                            tmp_list = list(rep_name)
    284                            tmp_list = tmp_list[:len(ens["id"])] + ["|"] + tmp_list[len(ens["id"]):]
    285                            rep_name = ''.join(tmp_list)
    286                    retd['names'].append(rep_name)
    287                    retd['idl'].append([di[0] for di in rep['deltas']])
    288                    retd['deltas'].append(np.array([di[1:] for di in rep['deltas']]))
    289        return retd
    290
    291    def _gen_covobsd_from_cdatad(d):
    292        retd = {}
    293        for ens in d:
    294            retl = []
    295            name = ens['id']
    296            layouts = ens.get('layout', '1').strip()
    297            layout = [int(ls.strip()) for ls in layouts.split(',') if len(ls) > 0]
    298            cov = np.reshape(ens['cov'], layout)
    299            grad = ens['grad']
    300            nobs = len(grad[0])
    301            for i in range(nobs):
    302                retl.append({'name': name, 'cov': cov, 'grad': [g[i] for g in grad]})
    303            retd[name] = retl
    304        return retd
    305
    306    def get_Obs_from_dict(o):
    307        layouts = o.get('layout', '1').strip()
    308        if layouts != '1':
    309            raise Exception("layout is %s has to be 1 for type Obs." % (layouts), RuntimeWarning)
    310
    311        values = o['value']
    312        od = _gen_obsd_from_datad(o.get('data', {}))
    313        cd = _gen_covobsd_from_cdatad(o.get('cdata', {}))
    314
    315        if od:
    316            ret = Obs([[ddi[0] + values[0] for ddi in di] for di in od['deltas']], od['names'], idl=od['idl'])
    317            ret._value = values[0]
    318        else:
    319            ret = Obs([], [], means=[])
    320            ret._value = values[0]
    321        for name in cd:
    322            co = cd[name][0]
    323            ret._covobs[name] = Covobs(None, co['cov'], co['name'], grad=co['grad'])
    324            ret.names.append(co['name'])
    325
    326        ret.reweighted = o.get('reweighted', False)
    327        ret.tag = o.get('tag', [None])[0]
    328        return ret
    329
    330    def get_List_from_dict(o):
    331        layouts = o.get('layout', '1').strip()
    332        layout = int(layouts)
    333        values = o['value']
    334        od = _gen_obsd_from_datad(o.get('data', {}))
    335        cd = _gen_covobsd_from_cdatad(o.get('cdata', {}))
    336
    337        ret = []
    338        taglist = o.get('tag', layout * [None])
    339        for i in range(layout):
    340            if od:
    341                ret.append(Obs([list(di[:, i] + values[i]) for di in od['deltas']], od['names'], idl=od['idl']))
    342                ret[-1]._value = values[i]
    343            else:
    344                ret.append(Obs([], [], means=[]))
    345                ret[-1]._value = values[i]
    346                print('Created Obs with means= ', values[i])
    347            for name in cd:
    348                co = cd[name][i]
    349                ret[-1]._covobs[name] = Covobs(None, co['cov'], co['name'], grad=co['grad'])
    350                ret[-1].names.append(co['name'])
    351
    352            ret[-1].reweighted = o.get('reweighted', False)
    353            ret[-1].tag = taglist[i]
    354        return ret
    355
    356    def get_Array_from_dict(o):
    357        layouts = o.get('layout', '1').strip()
    358        layout = [int(ls.strip()) for ls in layouts.split(',') if len(ls) > 0]
    359        N = np.prod(layout)
    360        values = o['value']
    361        od = _gen_obsd_from_datad(o.get('data', {}))
    362        cd = _gen_covobsd_from_cdatad(o.get('cdata', {}))
    363
    364        ret = []
    365        taglist = o.get('tag', N * [None])
    366        for i in range(N):
    367            if od:
    368                ret.append(Obs([di[:, i] + values[i] for di in od['deltas']], od['names'], idl=od['idl']))
    369                ret[-1]._value = values[i]
    370            else:
    371                ret.append(Obs([], [], means=[]))
    372                ret[-1]._value = values[i]
    373            for name in cd:
    374                co = cd[name][i]
    375                ret[-1]._covobs[name] = Covobs(None, co['cov'], co['name'], grad=co['grad'])
    376                ret[-1].names.append(co['name'])
    377            ret[-1].reweighted = o.get('reweighted', False)
    378            ret[-1].tag = taglist[i]
    379        return np.reshape(ret, layout)
    380
    381    def get_Corr_from_dict(o):
    382        if isinstance(o.get('tag'), list):  # supports the old way
    383            taglist = o.get('tag')  # This had to be modified to get the taglist from the dictionary
    384            temp_prange = None
    385        elif isinstance(o.get('tag'), dict):
    386            tagdic = o.get('tag')
    387            taglist = tagdic['tag']
    388            if 'prange' in tagdic:
    389                temp_prange = tagdic['prange']
    390            else:
    391                temp_prange = None
    392        else:
    393            raise Exception("The tag is not a list or dict")
    394
    395        corr_tag = taglist[-1]
    396        tmp_o = o
    397        tmp_o['tag'] = taglist[:-1]
    398        if len(tmp_o['tag']) == 0:
    399            del tmp_o['tag']
    400        dat = get_Array_from_dict(tmp_o)
    401        my_corr = Corr([None if np.isnan(o.ravel()[0].value) else o for o in list(dat)])
    402        if corr_tag != 'None':
    403            my_corr.tag = corr_tag
    404
    405        my_corr.prange = temp_prange
    406        return my_corr
    407
    408    prog = json_dict.get('program', '')
    409    version = json_dict.get('version', '')
    410    who = json_dict.get('who', '')
    411    date = json_dict.get('date', '')
    412    host = json_dict.get('host', '')
    413    if prog and verbose:
    414        print('Data has been written using %s.' % (prog))
    415    if version and verbose:
    416        print('Format version %s' % (version))
    417    if np.any([who, date, host] and verbose):
    418        print('Written by %s on %s on host %s' % (who, date, host))
    419    description = json_dict.get('description', '')
    420    if description and verbose:
    421        print()
    422        print('Description: ', description)
    423    obsdata = json_dict['obsdata']
    424    ol = []
    425    for io in obsdata:
    426        if io['type'] == 'Obs':
    427            ol.append(get_Obs_from_dict(io))
    428        elif io['type'] == 'List':
    429            ol.append(get_List_from_dict(io))
    430        elif io['type'] == 'Array':
    431            ol.append(get_Array_from_dict(io))
    432        elif io['type'] == 'Corr':
    433            ol.append(get_Corr_from_dict(io))
    434        else:
    435            raise Exception("Unknown datatype.")
    436
    437    if full_output:
    438        retd = {}
    439        retd['program'] = prog
    440        retd['version'] = version
    441        retd['who'] = who
    442        retd['date'] = date
    443        retd['host'] = host
    444        retd['description'] = description
    445        retd['obsdata'] = ol
    446
    447        return retd
    448    else:
    449        if len(obsdata) == 1:
    450            ol = ol[0]
    451
    452        return ol
    453
    454
    455def import_json_string(json_string, verbose=True, full_output=False):
    456    """Reconstruct a list of Obs or structures containing Obs from a json string.
    457
    458    The following structures are supported: Obs, list, numpy.ndarray, Corr
    459    If the list contains only one element, it is unpacked from the list.
    460
    461    Parameters
    462    ----------
    463    json_string : str
    464        json string containing the data.
    465    verbose : bool
    466        Print additional information that was written to the file.
    467    full_output : bool
    468        If True, a dict containing auxiliary information and the data is returned.
    469        If False, only the data is returned.
    470
    471    Returns
    472    -------
    473    result : list[Obs]
    474        reconstructed list of observables from the json string
    475    or
    476    result : Obs
    477        only one observable if the list only has one entry
    478    or
    479    result : dict
    480        if full_output=True
    481    """
    482
    483    return _parse_json_dict(json.loads(json_string), verbose, full_output)
    484
    485
    486def load_json(fname, verbose=True, gz=True, full_output=False):
    487    """Import a list of Obs or structures containing Obs from a .json(.gz) file.
    488
    489    The following structures are supported: Obs, list, numpy.ndarray, Corr
    490    If the list contains only one element, it is unpacked from the list.
    491
    492    Parameters
    493    ----------
    494    fname : str
    495        Filename of the input file.
    496    verbose : bool
    497        Print additional information that was written to the file.
    498    gz : bool
    499        If True, assumes that data is gzipped. If False, assumes JSON file.
    500    full_output : bool
    501        If True, a dict containing auxiliary information and the data is returned.
    502        If False, only the data is returned.
    503
    504    Returns
    505    -------
    506    result : list[Obs]
    507        reconstructed list of observables from the json string
    508    or
    509    result : Obs
    510        only one observable if the list only has one entry
    511    or
    512    result : dict
    513        if full_output=True
    514    """
    515    if not fname.endswith('.json') and not fname.endswith('.gz'):
    516        fname += '.json'
    517    if gz:
    518        if not fname.endswith('.gz'):
    519            fname += '.gz'
    520        with gzip.open(fname, 'r') as fin:
    521            d = json.load(fin)
    522    else:
    523        if fname.endswith('.gz'):
    524            warnings.warn("Trying to read from %s without unzipping!" % fname, UserWarning)
    525        with open(fname, 'r', encoding='utf-8') as fin:
    526            d = json.loads(fin.read())
    527
    528    return _parse_json_dict(d, verbose, full_output)
    529
    530
    531def _ol_from_dict(ind, reps='DICTOBS'):
    532    """Convert a dictionary of Obs objects to a list and a dictionary that contains
    533    placeholders instead of the Obs objects.
    534
    535    Parameters
    536    ----------
    537    ind : dict
    538        Dict of JSON valid structures and objects that will be exported.
    539        At the moment, these object can be either of: Obs, list, numpy.ndarray, Corr.
    540        All Obs inside a structure have to be defined on the same set of configurations.
    541    reps : str
    542        Specify the structure of the placeholder in exported dict to be reps[0-9]+.
    543    """
    544
    545    obstypes = (Obs, Corr, np.ndarray)
    546
    547    if not reps.isalnum():
    548        raise Exception('Placeholder string has to be alphanumeric!')
    549    ol = []
    550    counter = 0
    551
    552    def dict_replace_obs(d):
    553        nonlocal ol
    554        nonlocal counter
    555        x = {}
    556        for k, v in d.items():
    557            if isinstance(v, dict):
    558                v = dict_replace_obs(v)
    559            elif isinstance(v, list) and all([isinstance(o, Obs) for o in v]):
    560                v = obslist_replace_obs(v)
    561            elif isinstance(v, list):
    562                v = list_replace_obs(v)
    563            elif isinstance(v, obstypes):
    564                ol.append(v)
    565                v = reps + '%d' % (counter)
    566                counter += 1
    567            elif isinstance(v, str):
    568                if bool(re.match(r'%s[0-9]+' % (reps), v)):
    569                    raise Exception('Dict contains string %s that matches the placeholder! %s Cannot be savely exported.' % (v, reps))
    570            x[k] = v
    571        return x
    572
    573    def list_replace_obs(li):
    574        nonlocal ol
    575        nonlocal counter
    576        x = []
    577        for e in li:
    578            if isinstance(e, list):
    579                e = list_replace_obs(e)
    580            elif isinstance(e, list) and all([isinstance(o, Obs) for o in e]):
    581                e = obslist_replace_obs(e)
    582            elif isinstance(e, dict):
    583                e = dict_replace_obs(e)
    584            elif isinstance(e, obstypes):
    585                ol.append(e)
    586                e = reps + '%d' % (counter)
    587                counter += 1
    588            elif isinstance(e, str):
    589                if bool(re.match(r'%s[0-9]+' % (reps), e)):
    590                    raise Exception('Dict contains string %s that matches the placeholder! %s Cannot be savely exported.' % (e, reps))
    591            x.append(e)
    592        return x
    593
    594    def obslist_replace_obs(li):
    595        nonlocal ol
    596        nonlocal counter
    597        il = []
    598        for e in li:
    599            il.append(e)
    600
    601        ol.append(il)
    602        x = reps + '%d' % (counter)
    603        counter += 1
    604        return x
    605
    606    nd = dict_replace_obs(ind)
    607
    608    return ol, nd
    609
    610
    611def dump_dict_to_json(od, fname, description='', indent=1, reps='DICTOBS', gz=True):
    612    """Export a dict of Obs or structures containing Obs to a .json(.gz) file
    613
    614    Parameters
    615    ----------
    616    od : dict
    617        Dict of JSON valid structures and objects that will be exported.
    618        At the moment, these objects can be either of: Obs, list, numpy.ndarray, Corr.
    619        All Obs inside a structure have to be defined on the same set of configurations.
    620    fname : str
    621        Filename of the output file.
    622    description : str
    623        Optional string that describes the contents of the json file.
    624    indent : int
    625        Specify the indentation level of the json file. None or 0 is permissible and
    626        saves disk space.
    627    reps : str
    628        Specify the structure of the placeholder in exported dict to be reps[0-9]+.
    629    gz : bool
    630        If True, the output is a gzipped json. If False, the output is a json file.
    631
    632    Returns
    633    -------
    634    None
    635    """
    636
    637    if not isinstance(od, dict):
    638        raise Exception('od has to be a dictionary. Did you want to use dump_to_json?')
    639
    640    infostring = ('This JSON file contains a python dictionary that has been parsed to a list of structures. '
    641                  'OBSDICT contains the dictionary, where Obs or other structures have been replaced by '
    642                  '' + reps + '[0-9]+. The field description contains the additional description of this JSON file. '
    643                  'This file may be parsed to a dict with the pyerrors routine load_json_dict.')
    644
    645    desc_dict = {'INFO': infostring, 'OBSDICT': {}, 'description': description}
    646    ol, desc_dict['OBSDICT'] = _ol_from_dict(od, reps=reps)
    647
    648    dump_to_json(ol, fname, description=desc_dict, indent=indent, gz=gz)
    649
    650
    651def _od_from_list_and_dict(ol, ind, reps='DICTOBS'):
    652    """Parse a list of Obs or structures containing Obs and an accompanying
    653    dict, where the structures have been replaced by placeholders to a
    654    dict that contains the structures.
    655
    656    The following structures are supported: Obs, list, numpy.ndarray, Corr
    657
    658    Parameters
    659    ----------
    660    ol : list
    661        List of objects -
    662        At the moment, these objects can be either of: Obs, list, numpy.ndarray, Corr.
    663        All Obs inside a structure have to be defined on the same set of configurations.
    664    ind : dict
    665        Dict that defines the structure of the resulting dict and contains placeholders
    666    reps : str
    667        Specify the structure of the placeholder in imported dict to be reps[0-9]+.
    668    """
    669    if not reps.isalnum():
    670        raise Exception('Placeholder string has to be alphanumeric!')
    671
    672    counter = 0
    673
    674    def dict_replace_string(d):
    675        nonlocal counter
    676        nonlocal ol
    677        x = {}
    678        for k, v in d.items():
    679            if isinstance(v, dict):
    680                v = dict_replace_string(v)
    681            elif isinstance(v, list):
    682                v = list_replace_string(v)
    683            elif isinstance(v, str) and bool(re.match(r'%s[0-9]+' % (reps), v)):
    684                index = int(v[len(reps):])
    685                v = ol[index]
    686                counter += 1
    687            x[k] = v
    688        return x
    689
    690    def list_replace_string(li):
    691        nonlocal counter
    692        nonlocal ol
    693        x = []
    694        for e in li:
    695            if isinstance(e, list):
    696                e = list_replace_string(e)
    697            elif isinstance(e, dict):
    698                e = dict_replace_string(e)
    699            elif isinstance(e, str) and bool(re.match(r'%s[0-9]+' % (reps), e)):
    700                index = int(e[len(reps):])
    701                e = ol[index]
    702                counter += 1
    703            x.append(e)
    704        return x
    705
    706    nd = dict_replace_string(ind)
    707
    708    if counter == 0:
    709        raise Exception('No placeholder has been replaced! Check if reps is set correctly.')
    710
    711    return nd
    712
    713
    714def load_json_dict(fname, verbose=True, gz=True, full_output=False, reps='DICTOBS'):
    715    """Import a dict of Obs or structures containing Obs from a .json(.gz) file.
    716
    717    The following structures are supported: Obs, list, numpy.ndarray, Corr
    718
    719    Parameters
    720    ----------
    721    fname : str
    722        Filename of the input file.
    723    verbose : bool
    724        Print additional information that was written to the file.
    725    gz : bool
    726        If True, assumes that data is gzipped. If False, assumes JSON file.
    727    full_output : bool
    728        If True, a dict containing auxiliary information and the data is returned.
    729        If False, only the data is returned.
    730    reps : str
    731        Specify the structure of the placeholder in imported dict to be reps[0-9]+.
    732
    733    Returns
    734    -------
    735    data : Obs / list / Corr
    736        Read data
    737    or
    738    data : dict
    739        Read data and meta-data
    740    """
    741    indata = load_json(fname, verbose=verbose, gz=gz, full_output=True)
    742    description = indata['description']['description']
    743    indict = indata['description']['OBSDICT']
    744    ol = indata['obsdata']
    745    od = _od_from_list_and_dict(ol, indict, reps=reps)
    746
    747    if full_output:
    748        indata['description'] = description
    749        indata['obsdata'] = od
    750        return indata
    751    else:
    752        return od

def create_json_string(ol, description='', indent=1): View Source

     18def create_json_string(ol, description='', indent=1):
     19    """Generate the string for the export of a list of Obs or structures containing Obs
     20    to a .json(.gz) file
     21
     22    Parameters
     23    ----------
     24    ol : list
     25        List of objects that will be exported. At the moment, these objects can be
     26        either of: Obs, list, numpy.ndarray, Corr.
     27        All Obs inside a structure have to be defined on the same set of configurations.
     28    description : str
     29        Optional string that describes the contents of the json file.
     30    indent : int
     31        Specify the indentation level of the json file. None or 0 is permissible and
     32        saves disk space.
     33
     34    Returns
     35    -------
     36    json_string : str
     37        String for export to .json(.gz) file
     38    """
     39
     40    def _gen_data_d_from_list(ol):
     41        dl = []
     42        No = len(ol)
     43        for name in ol[0].mc_names:
     44            ed = {}
     45            ed['id'] = name
     46            ed['replica'] = []
     47            for r_name in ol[0].e_content[name]:
     48                rd = {}
     49                rd['name'] = r_name
     50                rd['deltas'] = []
     51                offsets = [o.r_values[r_name] - o.value for o in ol]
     52                deltas = np.column_stack([ol[oi].deltas[r_name] + offsets[oi] for oi in range(No)])
     53                for i in range(len(ol[0].idl[r_name])):
     54                    rd['deltas'].append([ol[0].idl[r_name][i]])
     55                    rd['deltas'][-1] += deltas[i].tolist()
     56                ed['replica'].append(rd)
     57            dl.append(ed)
     58        return dl
     59
     60    def _gen_cdata_d_from_list(ol):
     61        dl = []
     62        for name in ol[0].cov_names:
     63            ed = {}
     64            ed['id'] = name
     65            ed['layout'] = str(ol[0].covobs[name].cov.shape).lstrip('(').rstrip(')').rstrip(',')
     66            ed['cov'] = list(np.ravel(ol[0].covobs[name].cov))
     67            ncov = ol[0].covobs[name].cov.shape[0]
     68            ed['grad'] = []
     69            for i in range(ncov):
     70                ed['grad'].append([])
     71                for o in ol:
     72                    ed['grad'][-1].append(o.covobs[name].grad[i][0])
     73            dl.append(ed)
     74        return dl
     75
     76    def write_Obs_to_dict(o):
     77        d = {}
     78        d['type'] = 'Obs'
     79        d['layout'] = '1'
     80        if o.tag:
     81            d['tag'] = [o.tag]
     82        if o.reweighted:
     83            d['reweighted'] = o.reweighted
     84        d['value'] = [o.value]
     85        data = _gen_data_d_from_list([o])
     86        if len(data) > 0:
     87            d['data'] = data
     88        cdata = _gen_cdata_d_from_list([o])
     89        if len(cdata) > 0:
     90            d['cdata'] = cdata
     91        return d
     92
     93    def write_List_to_dict(ol):
     94        _assert_equal_properties(ol)
     95        d = {}
     96        d['type'] = 'List'
     97        d['layout'] = '%d' % len(ol)
     98        taglist = [o.tag for o in ol]
     99        if np.any([tag is not None for tag in taglist]):
    100            d['tag'] = taglist
    101        if ol[0].reweighted:
    102            d['reweighted'] = ol[0].reweighted
    103        d['value'] = [o.value for o in ol]
    104        data = _gen_data_d_from_list(ol)
    105        if len(data) > 0:
    106            d['data'] = data
    107        cdata = _gen_cdata_d_from_list(ol)
    108        if len(cdata) > 0:
    109            d['cdata'] = cdata
    110        return d
    111
    112    def write_Array_to_dict(oa):
    113        ol = np.ravel(oa)
    114        _assert_equal_properties(ol)
    115        d = {}
    116        d['type'] = 'Array'
    117        d['layout'] = str(oa.shape).lstrip('(').rstrip(')').rstrip(',')
    118        taglist = [o.tag for o in ol]
    119        if np.any([tag is not None for tag in taglist]):
    120            d['tag'] = taglist
    121        if ol[0].reweighted:
    122            d['reweighted'] = ol[0].reweighted
    123        d['value'] = [o.value for o in ol]
    124        data = _gen_data_d_from_list(ol)
    125        if len(data) > 0:
    126            d['data'] = data
    127        cdata = _gen_cdata_d_from_list(ol)
    128        if len(cdata) > 0:
    129            d['cdata'] = cdata
    130        return d
    131
    132    def _nan_Obs_like(obs):
    133        samples = []
    134        names = []
    135        idl = []
    136        for key, value in obs.idl.items():
    137            samples.append([np.nan] * len(value))
    138            names.append(key)
    139            idl.append(value)
    140        my_obs = Obs(samples, names, idl)
    141        my_obs._covobs = obs._covobs
    142        for name in obs._covobs:
    143            my_obs.names.append(name)
    144        my_obs.reweighted = obs.reweighted
    145        return my_obs
    146
    147    def write_Corr_to_dict(my_corr):
    148        first_not_none = next(i for i, j in enumerate(my_corr.content) if np.all(j))
    149        dummy_array = np.empty((my_corr.N, my_corr.N), dtype=object)
    150        dummy_array[:] = _nan_Obs_like(my_corr.content[first_not_none].ravel()[0])
    151        content = [o if o is not None else dummy_array for o in my_corr.content]
    152        dat = write_Array_to_dict(np.array(content, dtype=object))
    153        dat['type'] = 'Corr'
    154        corr_meta_data = str(my_corr.tag)
    155        if 'tag' in dat.keys():
    156            dat['tag'].append(corr_meta_data)
    157        else:
    158            dat['tag'] = [corr_meta_data]
    159        taglist = dat['tag']
    160        dat['tag'] = {}  # tag is now a dictionary, that contains the previous taglist in the key "tag"
    161        dat['tag']['tag'] = taglist
    162        if my_corr.prange is not None:
    163            dat['tag']['prange'] = my_corr.prange
    164        return dat
    165
    166    if not isinstance(ol, list):
    167        ol = [ol]
    168
    169    d = {}
    170    d['program'] = 'pyerrors %s' % (pyerrorsversion.__version__)
    171    d['version'] = '1.1'
    172    d['who'] = getpass.getuser()
    173    d['date'] = datetime.datetime.now().astimezone().strftime('%Y-%m-%d %H:%M:%S %z')
    174    d['host'] = socket.gethostname() + ', ' + platform.platform()
    175
    176    if description:
    177        d['description'] = description
    178
    179    d['obsdata'] = []
    180    for io in ol:
    181        if isinstance(io, Obs):
    182            d['obsdata'].append(write_Obs_to_dict(io))
    183        elif isinstance(io, list):
    184            d['obsdata'].append(write_List_to_dict(io))
    185        elif isinstance(io, np.ndarray):
    186            d['obsdata'].append(write_Array_to_dict(io))
    187        elif isinstance(io, Corr):
    188            d['obsdata'].append(write_Corr_to_dict(io))
    189        else:
    190            raise Exception("Unkown datatype.")
    191
    192    def _jsonifier(o):
    193        if isinstance(o, np.int64):
    194            return int(o)
    195        raise TypeError('%r is not JSON serializable' % o)
    196
    197    if indent:
    198        return json.dumps(d, indent=indent, ensure_ascii=False, default=_jsonifier, write_mode=json.WM_SINGLE_LINE_ARRAY)
    199    else:
    200        return json.dumps(d, indent=indent, ensure_ascii=False, default=_jsonifier, write_mode=json.WM_COMPACT)

Generate the string for the export of a list of Obs or structures containing Obs to a .json(.gz) file

###### Parameters

- ol (list): List of objects that will be exported. At the moment, these objects can be either of: Obs, list, numpy.ndarray, Corr. All Obs inside a structure have to be defined on the same set of configurations.
- description (str): Optional string that describes the contents of the json file.
- indent (int): Specify the indentation level of the json file. None or 0 is permissible and saves disk space.

###### Returns

- json_string (str): String for export to .json(.gz) file

def dump_to_json(ol, fname, description='', indent=1, gz=True): View Source

    203def dump_to_json(ol, fname, description='', indent=1, gz=True):
    204    """Export a list of Obs or structures containing Obs to a .json(.gz) file
    205
    206    Parameters
    207    ----------
    208    ol : list
    209        List of objects that will be exported. At the moment, these objects can be
    210        either of: Obs, list, numpy.ndarray, Corr.
    211        All Obs inside a structure have to be defined on the same set of configurations.
    212    fname : str
    213        Filename of the output file.
    214    description : str
    215        Optional string that describes the contents of the json file.
    216    indent : int
    217        Specify the indentation level of the json file. None or 0 is permissible and
    218        saves disk space.
    219    gz : bool
    220        If True, the output is a gzipped json. If False, the output is a json file.
    221
    222    Returns
    223    -------
    224    Null
    225    """
    226
    227    jsonstring = create_json_string(ol, description, indent)
    228
    229    if not fname.endswith('.json') and not fname.endswith('.gz'):
    230        fname += '.json'
    231
    232    if gz:
    233        if not fname.endswith('.gz'):
    234            fname += '.gz'
    235
    236        fp = gzip.open(fname, 'wb')
    237        fp.write(jsonstring.encode('utf-8'))
    238    else:
    239        fp = open(fname, 'w', encoding='utf-8')
    240        fp.write(jsonstring)
    241    fp.close()

Export a list of Obs or structures containing Obs to a .json(.gz) file

###### Parameters

- ol (list): List of objects that will be exported. At the moment, these objects can be either of: Obs, list, numpy.ndarray, Corr. All Obs inside a structure have to be defined on the same set of configurations.
- fname (str): Filename of the output file.
- description (str): Optional string that describes the contents of the json file.
- indent (int): Specify the indentation level of the json file. None or 0 is permissible and saves disk space.
- gz (bool): If True, the output is a gzipped json. If False, the output is a json file.

###### Returns

- Null

def import_json_string(json_string, verbose=True, full_output=False): View Source

    456def import_json_string(json_string, verbose=True, full_output=False):
    457    """Reconstruct a list of Obs or structures containing Obs from a json string.
    458
    459    The following structures are supported: Obs, list, numpy.ndarray, Corr
    460    If the list contains only one element, it is unpacked from the list.
    461
    462    Parameters
    463    ----------
    464    json_string : str
    465        json string containing the data.
    466    verbose : bool
    467        Print additional information that was written to the file.
    468    full_output : bool
    469        If True, a dict containing auxiliary information and the data is returned.
    470        If False, only the data is returned.
    471
    472    Returns
    473    -------
    474    result : list[Obs]
    475        reconstructed list of observables from the json string
    476    or
    477    result : Obs
    478        only one observable if the list only has one entry
    479    or
    480    result : dict
    481        if full_output=True
    482    """
    483
    484    return _parse_json_dict(json.loads(json_string), verbose, full_output)

Reconstruct a list of Obs or structures containing Obs from a json string.

The following structures are supported: Obs, list, numpy.ndarray, Corr If the list contains only one element, it is unpacked from the list.

###### Parameters

- json_string (str): json string containing the data.
- verbose (bool): Print additional information that was written to the file.
- full_output (bool): If True, a dict containing auxiliary information and the data is returned. If False, only the data is returned.

###### Returns

- result (list[Obs]): reconstructed list of observables from the json string
- or
- result (Obs): only one observable if the list only has one entry
- or
- result (dict): if full_output=True

def load_json(fname, verbose=True, gz=True, full_output=False): View Source

    487def load_json(fname, verbose=True, gz=True, full_output=False):
    488    """Import a list of Obs or structures containing Obs from a .json(.gz) file.
    489
    490    The following structures are supported: Obs, list, numpy.ndarray, Corr
    491    If the list contains only one element, it is unpacked from the list.
    492
    493    Parameters
    494    ----------
    495    fname : str
    496        Filename of the input file.
    497    verbose : bool
    498        Print additional information that was written to the file.
    499    gz : bool
    500        If True, assumes that data is gzipped. If False, assumes JSON file.
    501    full_output : bool
    502        If True, a dict containing auxiliary information and the data is returned.
    503        If False, only the data is returned.
    504
    505    Returns
    506    -------
    507    result : list[Obs]
    508        reconstructed list of observables from the json string
    509    or
    510    result : Obs
    511        only one observable if the list only has one entry
    512    or
    513    result : dict
    514        if full_output=True
    515    """
    516    if not fname.endswith('.json') and not fname.endswith('.gz'):
    517        fname += '.json'
    518    if gz:
    519        if not fname.endswith('.gz'):
    520            fname += '.gz'
    521        with gzip.open(fname, 'r') as fin:
    522            d = json.load(fin)
    523    else:
    524        if fname.endswith('.gz'):
    525            warnings.warn("Trying to read from %s without unzipping!" % fname, UserWarning)
    526        with open(fname, 'r', encoding='utf-8') as fin:
    527            d = json.loads(fin.read())
    528
    529    return _parse_json_dict(d, verbose, full_output)

Import a list of Obs or structures containing Obs from a .json(.gz) file.

The following structures are supported: Obs, list, numpy.ndarray, Corr If the list contains only one element, it is unpacked from the list.

###### Parameters

- fname (str): Filename of the input file.
- verbose (bool): Print additional information that was written to the file.
- gz (bool): If True, assumes that data is gzipped. If False, assumes JSON file.
- full_output (bool): If True, a dict containing auxiliary information and the data is returned. If False, only the data is returned.

###### Returns

- result (list[Obs]): reconstructed list of observables from the json string
- or
- result (Obs): only one observable if the list only has one entry
- or
- result (dict): if full_output=True

def dump_dict_to_json(od, fname, description='', indent=1, reps='DICTOBS', gz=True): View Source

    612def dump_dict_to_json(od, fname, description='', indent=1, reps='DICTOBS', gz=True):
    613    """Export a dict of Obs or structures containing Obs to a .json(.gz) file
    614
    615    Parameters
    616    ----------
    617    od : dict
    618        Dict of JSON valid structures and objects that will be exported.
    619        At the moment, these objects can be either of: Obs, list, numpy.ndarray, Corr.
    620        All Obs inside a structure have to be defined on the same set of configurations.
    621    fname : str
    622        Filename of the output file.
    623    description : str
    624        Optional string that describes the contents of the json file.
    625    indent : int
    626        Specify the indentation level of the json file. None or 0 is permissible and
    627        saves disk space.
    628    reps : str
    629        Specify the structure of the placeholder in exported dict to be reps[0-9]+.
    630    gz : bool
    631        If True, the output is a gzipped json. If False, the output is a json file.
    632
    633    Returns
    634    -------
    635    None
    636    """
    637
    638    if not isinstance(od, dict):
    639        raise Exception('od has to be a dictionary. Did you want to use dump_to_json?')
    640
    641    infostring = ('This JSON file contains a python dictionary that has been parsed to a list of structures. '
    642                  'OBSDICT contains the dictionary, where Obs or other structures have been replaced by '
    643                  '' + reps + '[0-9]+. The field description contains the additional description of this JSON file. '
    644                  'This file may be parsed to a dict with the pyerrors routine load_json_dict.')
    645
    646    desc_dict = {'INFO': infostring, 'OBSDICT': {}, 'description': description}
    647    ol, desc_dict['OBSDICT'] = _ol_from_dict(od, reps=reps)
    648
    649    dump_to_json(ol, fname, description=desc_dict, indent=indent, gz=gz)

Export a dict of Obs or structures containing Obs to a .json(.gz) file

###### Parameters

- od (dict): Dict of JSON valid structures and objects that will be exported. At the moment, these objects can be either of: Obs, list, numpy.ndarray, Corr. All Obs inside a structure have to be defined on the same set of configurations.
- fname (str): Filename of the output file.
- description (str): Optional string that describes the contents of the json file.
- indent (int): Specify the indentation level of the json file. None or 0 is permissible and saves disk space.
- reps (str): Specify the structure of the placeholder in exported dict to be reps[0-9]+.
- gz (bool): If True, the output is a gzipped json. If False, the output is a json file.

###### Returns

- None

def load_json_dict(fname, verbose=True, gz=True, full_output=False, reps='DICTOBS'): View Source

    715def load_json_dict(fname, verbose=True, gz=True, full_output=False, reps='DICTOBS'):
    716    """Import a dict of Obs or structures containing Obs from a .json(.gz) file.
    717
    718    The following structures are supported: Obs, list, numpy.ndarray, Corr
    719
    720    Parameters
    721    ----------
    722    fname : str
    723        Filename of the input file.
    724    verbose : bool
    725        Print additional information that was written to the file.
    726    gz : bool
    727        If True, assumes that data is gzipped. If False, assumes JSON file.
    728    full_output : bool
    729        If True, a dict containing auxiliary information and the data is returned.
    730        If False, only the data is returned.
    731    reps : str
    732        Specify the structure of the placeholder in imported dict to be reps[0-9]+.
    733
    734    Returns
    735    -------
    736    data : Obs / list / Corr
    737        Read data
    738    or
    739    data : dict
    740        Read data and meta-data
    741    """
    742    indata = load_json(fname, verbose=verbose, gz=gz, full_output=True)
    743    description = indata['description']['description']
    744    indict = indata['description']['OBSDICT']
    745    ol = indata['obsdata']
    746    od = _od_from_list_and_dict(ol, indict, reps=reps)
    747
    748    if full_output:
    749        indata['description'] = description
    750        indata['obsdata'] = od
    751        return indata
    752    else:
    753        return od

Import a dict of Obs or structures containing Obs from a .json(.gz) file.

The following structures are supported: Obs, list, numpy.ndarray, Corr

###### Parameters

- fname (str): Filename of the input file.
- verbose (bool): Print additional information that was written to the file.
- gz (bool): If True, assumes that data is gzipped. If False, assumes JSON file.
- full_output (bool): If True, a dict containing auxiliary information and the data is returned. If False, only the data is returned.
- reps (str): Specify the structure of the placeholder in imported dict to be reps[0-9]+.

###### Returns

- data (Obs / list / Corr): Read data
- or
- data (dict): Read data and meta-data