# pyerrors.input.json

View Source

      0import rapidjson as json
      1import gzip
      2import getpass
      3import socket
      4import datetime
      5import platform
      6import warnings
      7import re
      8import numpy as np
      9from ..obs import Obs
     10from ..covobs import Covobs
     11from ..correlators import Corr
     12from ..misc import _assert_equal_properties
     13from .. import version as pyerrorsversion
     14
     15
     16def create_json_string(ol, description='', indent=1):
     17    """Generate the string for the export of a list of Obs or structures containing Obs
     18    to a .json(.gz) file
     19
     20    Parameters
     21    ----------
     22    ol : list
     23        List of objects that will be exported. At the moment, these objects can be
     24        either of: Obs, list, numpy.ndarray, Corr.
     25        All Obs inside a structure have to be defined on the same set of configurations.
     26    description : str
     27        Optional string that describes the contents of the json file.
     28    indent : int
     29        Specify the indentation level of the json file. None or 0 is permissible and
     30        saves disk space.
     31    """
     32
     33    def _gen_data_d_from_list(ol):
     34        dl = []
     35        for name in ol[0].mc_names:
     36            ed = {}
     37            ed['id'] = name
     38            ed['replica'] = []
     39            for r_name in ol[0].e_content[name]:
     40                rd = {}
     41                rd['name'] = r_name
     42                if ol[0].is_merged.get(r_name, False):
     43                    rd['is_merged'] = True
     44                rd['deltas'] = []
     45                for i in range(len(ol[0].idl[r_name])):
     46                    rd['deltas'].append([ol[0].idl[r_name][i]])
     47                    for o in ol:
     48                        rd['deltas'][-1].append(o.deltas[r_name][i])
     49                ed['replica'].append(rd)
     50            dl.append(ed)
     51        return dl
     52
     53    def _gen_cdata_d_from_list(ol):
     54        dl = []
     55        for name in ol[0].cov_names:
     56            ed = {}
     57            ed['id'] = name
     58            ed['layout'] = str(ol[0].covobs[name].cov.shape).lstrip('(').rstrip(')').rstrip(',')
     59            ed['cov'] = list(np.ravel(ol[0].covobs[name].cov))
     60            ncov = ol[0].covobs[name].cov.shape[0]
     61            ed['grad'] = []
     62            for i in range(ncov):
     63                ed['grad'].append([])
     64                for o in ol:
     65                    ed['grad'][-1].append(o.covobs[name].grad[i][0])
     66            dl.append(ed)
     67        return dl
     68
     69    def write_Obs_to_dict(o):
     70        d = {}
     71        d['type'] = 'Obs'
     72        d['layout'] = '1'
     73        if o.tag:
     74            d['tag'] = [o.tag]
     75        if o.reweighted:
     76            d['reweighted'] = o.reweighted
     77        d['value'] = [o.value]
     78        data = _gen_data_d_from_list([o])
     79        if len(data) > 0:
     80            d['data'] = data
     81        cdata = _gen_cdata_d_from_list([o])
     82        if len(cdata) > 0:
     83            d['cdata'] = cdata
     84        return d
     85
     86    def write_List_to_dict(ol):
     87        _assert_equal_properties(ol)
     88        d = {}
     89        d['type'] = 'List'
     90        d['layout'] = '%d' % len(ol)
     91        taglist = [o.tag for o in ol]
     92        if np.any([tag is not None for tag in taglist]):
     93            d['tag'] = taglist
     94        if ol[0].reweighted:
     95            d['reweighted'] = ol[0].reweighted
     96        d['value'] = [o.value for o in ol]
     97        data = _gen_data_d_from_list(ol)
     98        if len(data) > 0:
     99            d['data'] = data
    100        cdata = _gen_cdata_d_from_list(ol)
    101        if len(cdata) > 0:
    102            d['cdata'] = cdata
    103        return d
    104
    105    def write_Array_to_dict(oa):
    106        ol = np.ravel(oa)
    107        _assert_equal_properties(ol)
    108        d = {}
    109        d['type'] = 'Array'
    110        d['layout'] = str(oa.shape).lstrip('(').rstrip(')').rstrip(',')
    111        taglist = [o.tag for o in ol]
    112        if np.any([tag is not None for tag in taglist]):
    113            d['tag'] = taglist
    114        if ol[0].reweighted:
    115            d['reweighted'] = ol[0].reweighted
    116        d['value'] = [o.value for o in ol]
    117        data = _gen_data_d_from_list(ol)
    118        if len(data) > 0:
    119            d['data'] = data
    120        cdata = _gen_cdata_d_from_list(ol)
    121        if len(cdata) > 0:
    122            d['cdata'] = cdata
    123        return d
    124
    125    def _nan_Obs_like(obs):
    126        samples = []
    127        names = []
    128        idl = []
    129        for key, value in obs.idl.items():
    130            samples.append([np.nan] * len(value))
    131            names.append(key)
    132            idl.append(value)
    133        my_obs = Obs(samples, names, idl)
    134        my_obs._covobs = obs._covobs
    135        for name in obs._covobs:
    136            my_obs.names.append(name)
    137        my_obs.reweighted = obs.reweighted
    138        my_obs.is_merged = obs.is_merged
    139        return my_obs
    140
    141    def write_Corr_to_dict(my_corr):
    142        first_not_none = next(i for i, j in enumerate(my_corr.content) if np.all(j))
    143        dummy_array = np.empty((my_corr.N, my_corr.N), dtype=object)
    144        dummy_array[:] = _nan_Obs_like(my_corr.content[first_not_none].ravel()[0])
    145        content = [o if o is not None else dummy_array for o in my_corr.content]
    146        dat = write_Array_to_dict(np.array(content, dtype=object))
    147        dat['type'] = 'Corr'
    148        corr_meta_data = str(my_corr.tag)
    149        if 'tag' in dat.keys():
    150            dat['tag'].append(corr_meta_data)
    151        else:
    152            dat['tag'] = [corr_meta_data]
    153        taglist = dat['tag']
    154        dat['tag'] = {}  # tag is now a dictionary, that contains the previous taglist in the key "tag"
    155        dat['tag']['tag'] = taglist
    156        if my_corr.prange is not None:
    157            dat['tag']['prange'] = my_corr.prange
    158        return dat
    159
    160    if not isinstance(ol, list):
    161        ol = [ol]
    162
    163    d = {}
    164    d['program'] = 'pyerrors %s' % (pyerrorsversion.__version__)
    165    d['version'] = '1.0'
    166    d['who'] = getpass.getuser()
    167    d['date'] = datetime.datetime.now().astimezone().strftime('%Y-%m-%d %H:%M:%S %z')
    168    d['host'] = socket.gethostname() + ', ' + platform.platform()
    169
    170    if description:
    171        d['description'] = description
    172
    173    d['obsdata'] = []
    174    for io in ol:
    175        if isinstance(io, Obs):
    176            d['obsdata'].append(write_Obs_to_dict(io))
    177        elif isinstance(io, list):
    178            d['obsdata'].append(write_List_to_dict(io))
    179        elif isinstance(io, np.ndarray):
    180            d['obsdata'].append(write_Array_to_dict(io))
    181        elif isinstance(io, Corr):
    182            d['obsdata'].append(write_Corr_to_dict(io))
    183        else:
    184            raise Exception("Unkown datatype.")
    185
    186    if indent:
    187        return json.dumps(d, indent=indent, ensure_ascii=False, write_mode=json.WM_SINGLE_LINE_ARRAY)
    188    else:
    189        return json.dumps(d, indent=indent, ensure_ascii=False, write_mode=json.WM_COMPACT)
    190
    191
    192def dump_to_json(ol, fname, description='', indent=1, gz=True):
    193    """Export a list of Obs or structures containing Obs to a .json(.gz) file
    194
    195    Parameters
    196    ----------
    197    ol : list
    198        List of objects that will be exported. At the moment, these objects can be
    199        either of: Obs, list, numpy.ndarray, Corr.
    200        All Obs inside a structure have to be defined on the same set of configurations.
    201    fname : str
    202        Filename of the output file.
    203    description : str
    204        Optional string that describes the contents of the json file.
    205    indent : int
    206        Specify the indentation level of the json file. None or 0 is permissible and
    207        saves disk space.
    208    gz : bool
    209        If True, the output is a gzipped json. If False, the output is a json file.
    210    """
    211
    212    jsonstring = create_json_string(ol, description, indent)
    213
    214    if not fname.endswith('.json') and not fname.endswith('.gz'):
    215        fname += '.json'
    216
    217    if gz:
    218        if not fname.endswith('.gz'):
    219            fname += '.gz'
    220
    221        fp = gzip.open(fname, 'wb')
    222        fp.write(jsonstring.encode('utf-8'))
    223    else:
    224        fp = open(fname, 'w', encoding='utf-8')
    225        fp.write(jsonstring)
    226    fp.close()
    227
    228
    229def _parse_json_dict(json_dict, verbose=True, full_output=False):
    230    """Reconstruct a list of Obs or structures containing Obs from a dict that
    231    was built out of a json string.
    232
    233    The following structures are supported: Obs, list, numpy.ndarray, Corr
    234    If the list contains only one element, it is unpacked from the list.
    235
    236    Parameters
    237    ----------
    238    json_string : str
    239        json string containing the data.
    240    verbose : bool
    241        Print additional information that was written to the file.
    242    full_output : bool
    243        If True, a dict containing auxiliary information and the data is returned.
    244        If False, only the data is returned.
    245    """
    246
    247    def _gen_obsd_from_datad(d):
    248        retd = {}
    249        if d:
    250            retd['names'] = []
    251            retd['idl'] = []
    252            retd['deltas'] = []
    253            retd['is_merged'] = {}
    254            for ens in d:
    255                for rep in ens['replica']:
    256                    rep_name = rep['name']
    257                    if len(rep_name) > len(ens["id"]):
    258                        if rep_name[len(ens["id"])] != "|":
    259                            tmp_list = list(rep_name)
    260                            tmp_list = tmp_list[:len(ens["id"])] + ["|"] + tmp_list[len(ens["id"]):]
    261                            rep_name = ''.join(tmp_list)
    262                    retd['names'].append(rep_name)
    263                    retd['idl'].append([di[0] for di in rep['deltas']])
    264                    retd['deltas'].append(np.array([di[1:] for di in rep['deltas']]))
    265                    retd['is_merged'][rep_name] = rep.get('is_merged', False)
    266        return retd
    267
    268    def _gen_covobsd_from_cdatad(d):
    269        retd = {}
    270        for ens in d:
    271            retl = []
    272            name = ens['id']
    273            layouts = ens.get('layout', '1').strip()
    274            layout = [int(ls.strip()) for ls in layouts.split(',') if len(ls) > 0]
    275            cov = np.reshape(ens['cov'], layout)
    276            grad = ens['grad']
    277            nobs = len(grad[0])
    278            for i in range(nobs):
    279                retl.append({'name': name, 'cov': cov, 'grad': [g[i] for g in grad]})
    280            retd[name] = retl
    281        return retd
    282
    283    def get_Obs_from_dict(o):
    284        layouts = o.get('layout', '1').strip()
    285        if layouts != '1':
    286            raise Exception("layout is %s has to be 1 for type Obs." % (layouts), RuntimeWarning)
    287
    288        values = o['value']
    289        od = _gen_obsd_from_datad(o.get('data', {}))
    290        cd = _gen_covobsd_from_cdatad(o.get('cdata', {}))
    291
    292        if od:
    293            ret = Obs([[ddi[0] + values[0] for ddi in di] for di in od['deltas']], od['names'], idl=od['idl'])
    294            ret.is_merged = od['is_merged']
    295        else:
    296            ret = Obs([], [], means=[])
    297            ret._value = values[0]
    298        for name in cd:
    299            co = cd[name][0]
    300            ret._covobs[name] = Covobs(None, co['cov'], co['name'], grad=co['grad'])
    301            ret.names.append(co['name'])
    302
    303        ret.reweighted = o.get('reweighted', False)
    304        ret.tag = o.get('tag', [None])[0]
    305        return ret
    306
    307    def get_List_from_dict(o):
    308        layouts = o.get('layout', '1').strip()
    309        layout = int(layouts)
    310        values = o['value']
    311        od = _gen_obsd_from_datad(o.get('data', {}))
    312        cd = _gen_covobsd_from_cdatad(o.get('cdata', {}))
    313
    314        ret = []
    315        taglist = o.get('tag', layout * [None])
    316        for i in range(layout):
    317            if od:
    318                ret.append(Obs([list(di[:, i] + values[i]) for di in od['deltas']], od['names'], idl=od['idl']))
    319                ret[-1].is_merged = od['is_merged']
    320            else:
    321                ret.append(Obs([], [], means=[]))
    322                ret[-1]._value = values[i]
    323                print('Created Obs with means= ', values[i])
    324            for name in cd:
    325                co = cd[name][i]
    326                ret[-1]._covobs[name] = Covobs(None, co['cov'], co['name'], grad=co['grad'])
    327                ret[-1].names.append(co['name'])
    328
    329            ret[-1].reweighted = o.get('reweighted', False)
    330            ret[-1].tag = taglist[i]
    331        return ret
    332
    333    def get_Array_from_dict(o):
    334        layouts = o.get('layout', '1').strip()
    335        layout = [int(ls.strip()) for ls in layouts.split(',') if len(ls) > 0]
    336        N = np.prod(layout)
    337        values = o['value']
    338        od = _gen_obsd_from_datad(o.get('data', {}))
    339        cd = _gen_covobsd_from_cdatad(o.get('cdata', {}))
    340
    341        ret = []
    342        taglist = o.get('tag', N * [None])
    343        for i in range(N):
    344            if od:
    345                ret.append(Obs([di[:, i] + values[i] for di in od['deltas']], od['names'], idl=od['idl']))
    346                ret[-1].is_merged = od['is_merged']
    347            else:
    348                ret.append(Obs([], [], means=[]))
    349                ret[-1]._value = values[i]
    350            for name in cd:
    351                co = cd[name][i]
    352                ret[-1]._covobs[name] = Covobs(None, co['cov'], co['name'], grad=co['grad'])
    353                ret[-1].names.append(co['name'])
    354            ret[-1].reweighted = o.get('reweighted', False)
    355            ret[-1].tag = taglist[i]
    356        return np.reshape(ret, layout)
    357
    358    def get_Corr_from_dict(o):
    359        if isinstance(o.get('tag'), list):  # supports the old way
    360            taglist = o.get('tag')  # This had to be modified to get the taglist from the dictionary
    361            temp_prange = None
    362        elif isinstance(o.get('tag'), dict):
    363            tagdic = o.get('tag')
    364            taglist = tagdic['tag']
    365            if 'prange' in tagdic:
    366                temp_prange = tagdic['prange']
    367            else:
    368                temp_prange = None
    369        else:
    370            raise Exception("The tag is not a list or dict")
    371
    372        corr_tag = taglist[-1]
    373        tmp_o = o
    374        tmp_o['tag'] = taglist[:-1]
    375        if len(tmp_o['tag']) == 0:
    376            del tmp_o['tag']
    377        dat = get_Array_from_dict(tmp_o)
    378        my_corr = Corr([None if np.isnan(o.ravel()[0].value) else o for o in list(dat)])
    379        if corr_tag != 'None':
    380            my_corr.tag = corr_tag
    381
    382        my_corr.prange = temp_prange
    383        return my_corr
    384
    385    prog = json_dict.get('program', '')
    386    version = json_dict.get('version', '')
    387    who = json_dict.get('who', '')
    388    date = json_dict.get('date', '')
    389    host = json_dict.get('host', '')
    390    if prog and verbose:
    391        print('Data has been written using %s.' % (prog))
    392    if version and verbose:
    393        print('Format version %s' % (version))
    394    if np.any([who, date, host] and verbose):
    395        print('Written by %s on %s on host %s' % (who, date, host))
    396    description = json_dict.get('description', '')
    397    if description and verbose:
    398        print()
    399        print('Description: ', description)
    400    obsdata = json_dict['obsdata']
    401    ol = []
    402    for io in obsdata:
    403        if io['type'] == 'Obs':
    404            ol.append(get_Obs_from_dict(io))
    405        elif io['type'] == 'List':
    406            ol.append(get_List_from_dict(io))
    407        elif io['type'] == 'Array':
    408            ol.append(get_Array_from_dict(io))
    409        elif io['type'] == 'Corr':
    410            ol.append(get_Corr_from_dict(io))
    411        else:
    412            raise Exception("Unkown datatype.")
    413
    414    if full_output:
    415        retd = {}
    416        retd['program'] = prog
    417        retd['version'] = version
    418        retd['who'] = who
    419        retd['date'] = date
    420        retd['host'] = host
    421        retd['description'] = description
    422        retd['obsdata'] = ol
    423
    424        return retd
    425    else:
    426        if len(obsdata) == 1:
    427            ol = ol[0]
    428
    429        return ol
    430
    431
    432def import_json_string(json_string, verbose=True, full_output=False):
    433    """Reconstruct a list of Obs or structures containing Obs from a json string.
    434
    435    The following structures are supported: Obs, list, numpy.ndarray, Corr
    436    If the list contains only one element, it is unpacked from the list.
    437
    438    Parameters
    439    ----------
    440    json_string : str
    441        json string containing the data.
    442    verbose : bool
    443        Print additional information that was written to the file.
    444    full_output : bool
    445        If True, a dict containing auxiliary information and the data is returned.
    446        If False, only the data is returned.
    447    """
    448
    449    return _parse_json_dict(json.loads(json_string), verbose, full_output)
    450
    451
    452def load_json(fname, verbose=True, gz=True, full_output=False):
    453    """Import a list of Obs or structures containing Obs from a .json(.gz) file.
    454
    455    The following structures are supported: Obs, list, numpy.ndarray, Corr
    456    If the list contains only one element, it is unpacked from the list.
    457
    458    Parameters
    459    ----------
    460    fname : str
    461        Filename of the input file.
    462    verbose : bool
    463        Print additional information that was written to the file.
    464    gz : bool
    465        If True, assumes that data is gzipped. If False, assumes JSON file.
    466    full_output : bool
    467        If True, a dict containing auxiliary information and the data is returned.
    468        If False, only the data is returned.
    469    """
    470    if not fname.endswith('.json') and not fname.endswith('.gz'):
    471        fname += '.json'
    472    if gz:
    473        if not fname.endswith('.gz'):
    474            fname += '.gz'
    475        with gzip.open(fname, 'r') as fin:
    476            d = json.load(fin)
    477    else:
    478        if fname.endswith('.gz'):
    479            warnings.warn("Trying to read from %s without unzipping!" % fname, UserWarning)
    480        with open(fname, 'r', encoding='utf-8') as fin:
    481            d = json.loads(fin.read())
    482
    483    return _parse_json_dict(d, verbose, full_output)
    484
    485
    486def _ol_from_dict(ind, reps='DICTOBS'):
    487    """Convert a dictionary of Obs objects to a list and a dictionary that contains
    488    placeholders instead of the Obs objects.
    489
    490    Parameters
    491    ----------
    492    ind : dict
    493        Dict of JSON valid structures and objects that will be exported.
    494        At the moment, these object can be either of: Obs, list, numpy.ndarray, Corr.
    495        All Obs inside a structure have to be defined on the same set of configurations.
    496    reps : str
    497        Specify the structure of the placeholder in exported dict to be reps[0-9]+.
    498    """
    499
    500    obstypes = (Obs, Corr, np.ndarray)
    501
    502    if not reps.isalnum():
    503        raise Exception('Placeholder string has to be alphanumeric!')
    504    ol = []
    505    counter = 0
    506
    507    def dict_replace_obs(d):
    508        nonlocal ol
    509        nonlocal counter
    510        x = {}
    511        for k, v in d.items():
    512            if isinstance(v, dict):
    513                v = dict_replace_obs(v)
    514            elif isinstance(v, list) and all([isinstance(o, Obs) for o in v]):
    515                v = obslist_replace_obs(v)
    516            elif isinstance(v, list):
    517                v = list_replace_obs(v)
    518            elif isinstance(v, obstypes):
    519                ol.append(v)
    520                v = reps + '%d' % (counter)
    521                counter += 1
    522            elif isinstance(v, str):
    523                if bool(re.match(r'%s[0-9]+' % (reps), v)):
    524                    raise Exception('Dict contains string %s that matches the placeholder! %s Cannot be savely exported.' % (v, reps))
    525            x[k] = v
    526        return x
    527
    528    def list_replace_obs(li):
    529        nonlocal ol
    530        nonlocal counter
    531        x = []
    532        for e in li:
    533            if isinstance(e, list):
    534                e = list_replace_obs(e)
    535            elif isinstance(e, list) and all([isinstance(o, Obs) for o in e]):
    536                e = obslist_replace_obs(e)
    537            elif isinstance(e, dict):
    538                e = dict_replace_obs(e)
    539            elif isinstance(e, obstypes):
    540                ol.append(e)
    541                e = reps + '%d' % (counter)
    542                counter += 1
    543            elif isinstance(e, str):
    544                if bool(re.match(r'%s[0-9]+' % (reps), e)):
    545                    raise Exception('Dict contains string %s that matches the placeholder! %s Cannot be savely exported.' % (e, reps))
    546            x.append(e)
    547        return x
    548
    549    def obslist_replace_obs(li):
    550        nonlocal ol
    551        nonlocal counter
    552        il = []
    553        for e in li:
    554            il.append(e)
    555
    556        ol.append(il)
    557        x = reps + '%d' % (counter)
    558        counter += 1
    559        return x
    560
    561    nd = dict_replace_obs(ind)
    562
    563    return ol, nd
    564
    565
    566def dump_dict_to_json(od, fname, description='', indent=1, reps='DICTOBS', gz=True):
    567    """Export a dict of Obs or structures containing Obs to a .json(.gz) file
    568
    569    Parameters
    570    ----------
    571    od : dict
    572        Dict of JSON valid structures and objects that will be exported.
    573        At the moment, these objects can be either of: Obs, list, numpy.ndarray, Corr.
    574        All Obs inside a structure have to be defined on the same set of configurations.
    575    fname : str
    576        Filename of the output file.
    577    description : str
    578        Optional string that describes the contents of the json file.
    579    indent : int
    580        Specify the indentation level of the json file. None or 0 is permissible and
    581        saves disk space.
    582    reps : str
    583        Specify the structure of the placeholder in exported dict to be reps[0-9]+.
    584    gz : bool
    585        If True, the output is a gzipped json. If False, the output is a json file.
    586    """
    587
    588    if not isinstance(od, dict):
    589        raise Exception('od has to be a dictionary. Did you want to use dump_to_json?')
    590
    591    infostring = ('This JSON file contains a python dictionary that has been parsed to a list of structures. '
    592                  'OBSDICT contains the dictionary, where Obs or other structures have been replaced by '
    593                  '' + reps + '[0-9]+. The field description contains the additional description of this JSON file. '
    594                  'This file may be parsed to a dict with the pyerrors routine load_json_dict.')
    595
    596    desc_dict = {'INFO': infostring, 'OBSDICT': {}, 'description': description}
    597    ol, desc_dict['OBSDICT'] = _ol_from_dict(od, reps=reps)
    598
    599    dump_to_json(ol, fname, description=desc_dict, indent=indent, gz=gz)
    600
    601
    602def _od_from_list_and_dict(ol, ind, reps='DICTOBS'):
    603    """Parse a list of Obs or structures containing Obs and an accompanying
    604    dict, where the structures have been replaced by placeholders to a
    605    dict that contains the structures.
    606
    607    The following structures are supported: Obs, list, numpy.ndarray, Corr
    608
    609    Parameters
    610    ----------
    611    ol : list
    612        List of objects -
    613        At the moment, these objects can be either of: Obs, list, numpy.ndarray, Corr.
    614        All Obs inside a structure have to be defined on the same set of configurations.
    615    ind : dict
    616        Dict that defines the structure of the resulting dict and contains placeholders
    617    reps : str
    618        Specify the structure of the placeholder in imported dict to be reps[0-9]+.
    619    """
    620    if not reps.isalnum():
    621        raise Exception('Placeholder string has to be alphanumeric!')
    622
    623    counter = 0
    624
    625    def dict_replace_string(d):
    626        nonlocal counter
    627        nonlocal ol
    628        x = {}
    629        for k, v in d.items():
    630            if isinstance(v, dict):
    631                v = dict_replace_string(v)
    632            elif isinstance(v, list):
    633                v = list_replace_string(v)
    634            elif isinstance(v, str) and bool(re.match(r'%s[0-9]+' % (reps), v)):
    635                index = int(v[len(reps):])
    636                v = ol[index]
    637                counter += 1
    638            x[k] = v
    639        return x
    640
    641    def list_replace_string(li):
    642        nonlocal counter
    643        nonlocal ol
    644        x = []
    645        for e in li:
    646            if isinstance(e, list):
    647                e = list_replace_string(e)
    648            elif isinstance(e, dict):
    649                e = dict_replace_string(e)
    650            elif isinstance(e, str) and bool(re.match(r'%s[0-9]+' % (reps), e)):
    651                index = int(e[len(reps):])
    652                e = ol[index]
    653                counter += 1
    654            x.append(e)
    655        return x
    656
    657    nd = dict_replace_string(ind)
    658
    659    if counter == 0:
    660        raise Exception('No placeholder has been replaced! Check if reps is set correctly.')
    661
    662    return nd
    663
    664
    665def load_json_dict(fname, verbose=True, gz=True, full_output=False, reps='DICTOBS'):
    666    """Import a dict of Obs or structures containing Obs from a .json(.gz) file.
    667
    668    The following structures are supported: Obs, list, numpy.ndarray, Corr
    669
    670    Parameters
    671    ----------
    672    fname : str
    673        Filename of the input file.
    674    verbose : bool
    675        Print additional information that was written to the file.
    676    gz : bool
    677        If True, assumes that data is gzipped. If False, assumes JSON file.
    678    full_output : bool
    679        If True, a dict containing auxiliary information and the data is returned.
    680        If False, only the data is returned.
    681    reps : str
    682        Specify the structure of the placeholder in imported dict to be reps[0-9]+.
    683    """
    684    indata = load_json(fname, verbose=verbose, gz=gz, full_output=True)
    685    description = indata['description']['description']
    686    indict = indata['description']['OBSDICT']
    687    ol = indata['obsdata']
    688    od = _od_from_list_and_dict(ol, indict, reps=reps)
    689
    690    if full_output:
    691        indata['description'] = description
    692        indata['obsdata'] = od
    693        return indata
    694    else:
    695        return od

def create_json_string(ol, description='', indent=1):
View Source

     17def create_json_string(ol, description='', indent=1):
     18    """Generate the string for the export of a list of Obs or structures containing Obs
     19    to a .json(.gz) file
     20
     21    Parameters
     22    ----------
     23    ol : list
     24        List of objects that will be exported. At the moment, these objects can be
     25        either of: Obs, list, numpy.ndarray, Corr.
     26        All Obs inside a structure have to be defined on the same set of configurations.
     27    description : str
     28        Optional string that describes the contents of the json file.
     29    indent : int
     30        Specify the indentation level of the json file. None or 0 is permissible and
     31        saves disk space.
     32    """
     33
     34    def _gen_data_d_from_list(ol):
     35        dl = []
     36        for name in ol[0].mc_names:
     37            ed = {}
     38            ed['id'] = name
     39            ed['replica'] = []
     40            for r_name in ol[0].e_content[name]:
     41                rd = {}
     42                rd['name'] = r_name
     43                if ol[0].is_merged.get(r_name, False):
     44                    rd['is_merged'] = True
     45                rd['deltas'] = []
     46                for i in range(len(ol[0].idl[r_name])):
     47                    rd['deltas'].append([ol[0].idl[r_name][i]])
     48                    for o in ol:
     49                        rd['deltas'][-1].append(o.deltas[r_name][i])
     50                ed['replica'].append(rd)
     51            dl.append(ed)
     52        return dl
     53
     54    def _gen_cdata_d_from_list(ol):
     55        dl = []
     56        for name in ol[0].cov_names:
     57            ed = {}
     58            ed['id'] = name
     59            ed['layout'] = str(ol[0].covobs[name].cov.shape).lstrip('(').rstrip(')').rstrip(',')
     60            ed['cov'] = list(np.ravel(ol[0].covobs[name].cov))
     61            ncov = ol[0].covobs[name].cov.shape[0]
     62            ed['grad'] = []
     63            for i in range(ncov):
     64                ed['grad'].append([])
     65                for o in ol:
     66                    ed['grad'][-1].append(o.covobs[name].grad[i][0])
     67            dl.append(ed)
     68        return dl
     69
     70    def write_Obs_to_dict(o):
     71        d = {}
     72        d['type'] = 'Obs'
     73        d['layout'] = '1'
     74        if o.tag:
     75            d['tag'] = [o.tag]
     76        if o.reweighted:
     77            d['reweighted'] = o.reweighted
     78        d['value'] = [o.value]
     79        data = _gen_data_d_from_list([o])
     80        if len(data) > 0:
     81            d['data'] = data
     82        cdata = _gen_cdata_d_from_list([o])
     83        if len(cdata) > 0:
     84            d['cdata'] = cdata
     85        return d
     86
     87    def write_List_to_dict(ol):
     88        _assert_equal_properties(ol)
     89        d = {}
     90        d['type'] = 'List'
     91        d['layout'] = '%d' % len(ol)
     92        taglist = [o.tag for o in ol]
     93        if np.any([tag is not None for tag in taglist]):
     94            d['tag'] = taglist
     95        if ol[0].reweighted:
     96            d['reweighted'] = ol[0].reweighted
     97        d['value'] = [o.value for o in ol]
     98        data = _gen_data_d_from_list(ol)
     99        if len(data) > 0:
    100            d['data'] = data
    101        cdata = _gen_cdata_d_from_list(ol)
    102        if len(cdata) > 0:
    103            d['cdata'] = cdata
    104        return d
    105
    106    def write_Array_to_dict(oa):
    107        ol = np.ravel(oa)
    108        _assert_equal_properties(ol)
    109        d = {}
    110        d['type'] = 'Array'
    111        d['layout'] = str(oa.shape).lstrip('(').rstrip(')').rstrip(',')
    112        taglist = [o.tag for o in ol]
    113        if np.any([tag is not None for tag in taglist]):
    114            d['tag'] = taglist
    115        if ol[0].reweighted:
    116            d['reweighted'] = ol[0].reweighted
    117        d['value'] = [o.value for o in ol]
    118        data = _gen_data_d_from_list(ol)
    119        if len(data) > 0:
    120            d['data'] = data
    121        cdata = _gen_cdata_d_from_list(ol)
    122        if len(cdata) > 0:
    123            d['cdata'] = cdata
    124        return d
    125
    126    def _nan_Obs_like(obs):
    127        samples = []
    128        names = []
    129        idl = []
    130        for key, value in obs.idl.items():
    131            samples.append([np.nan] * len(value))
    132            names.append(key)
    133            idl.append(value)
    134        my_obs = Obs(samples, names, idl)
    135        my_obs._covobs = obs._covobs
    136        for name in obs._covobs:
    137            my_obs.names.append(name)
    138        my_obs.reweighted = obs.reweighted
    139        my_obs.is_merged = obs.is_merged
    140        return my_obs
    141
    142    def write_Corr_to_dict(my_corr):
    143        first_not_none = next(i for i, j in enumerate(my_corr.content) if np.all(j))
    144        dummy_array = np.empty((my_corr.N, my_corr.N), dtype=object)
    145        dummy_array[:] = _nan_Obs_like(my_corr.content[first_not_none].ravel()[0])
    146        content = [o if o is not None else dummy_array for o in my_corr.content]
    147        dat = write_Array_to_dict(np.array(content, dtype=object))
    148        dat['type'] = 'Corr'
    149        corr_meta_data = str(my_corr.tag)
    150        if 'tag' in dat.keys():
    151            dat['tag'].append(corr_meta_data)
    152        else:
    153            dat['tag'] = [corr_meta_data]
    154        taglist = dat['tag']
    155        dat['tag'] = {}  # tag is now a dictionary, that contains the previous taglist in the key "tag"
    156        dat['tag']['tag'] = taglist
    157        if my_corr.prange is not None:
    158            dat['tag']['prange'] = my_corr.prange
    159        return dat
    160
    161    if not isinstance(ol, list):
    162        ol = [ol]
    163
    164    d = {}
    165    d['program'] = 'pyerrors %s' % (pyerrorsversion.__version__)
    166    d['version'] = '1.0'
    167    d['who'] = getpass.getuser()
    168    d['date'] = datetime.datetime.now().astimezone().strftime('%Y-%m-%d %H:%M:%S %z')
    169    d['host'] = socket.gethostname() + ', ' + platform.platform()
    170
    171    if description:
    172        d['description'] = description
    173
    174    d['obsdata'] = []
    175    for io in ol:
    176        if isinstance(io, Obs):
    177            d['obsdata'].append(write_Obs_to_dict(io))
    178        elif isinstance(io, list):
    179            d['obsdata'].append(write_List_to_dict(io))
    180        elif isinstance(io, np.ndarray):
    181            d['obsdata'].append(write_Array_to_dict(io))
    182        elif isinstance(io, Corr):
    183            d['obsdata'].append(write_Corr_to_dict(io))
    184        else:
    185            raise Exception("Unkown datatype.")
    186
    187    if indent:
    188        return json.dumps(d, indent=indent, ensure_ascii=False, write_mode=json.WM_SINGLE_LINE_ARRAY)
    189    else:
    190        return json.dumps(d, indent=indent, ensure_ascii=False, write_mode=json.WM_COMPACT)

Generate the string for the export of a list of Obs or structures containing Obs to a .json(.gz) file

###### Parameters

- ol (list): List of objects that will be exported. At the moment, these objects can be either of: Obs, list, numpy.ndarray, Corr. All Obs inside a structure have to be defined on the same set of configurations.
- description (str): Optional string that describes the contents of the json file.
- indent (int): Specify the indentation level of the json file. None or 0 is permissible and saves disk space.

def dump_to_json(ol, fname, description='', indent=1, gz=True):
View Source

    193def dump_to_json(ol, fname, description='', indent=1, gz=True):
    194    """Export a list of Obs or structures containing Obs to a .json(.gz) file
    195
    196    Parameters
    197    ----------
    198    ol : list
    199        List of objects that will be exported. At the moment, these objects can be
    200        either of: Obs, list, numpy.ndarray, Corr.
    201        All Obs inside a structure have to be defined on the same set of configurations.
    202    fname : str
    203        Filename of the output file.
    204    description : str
    205        Optional string that describes the contents of the json file.
    206    indent : int
    207        Specify the indentation level of the json file. None or 0 is permissible and
    208        saves disk space.
    209    gz : bool
    210        If True, the output is a gzipped json. If False, the output is a json file.
    211    """
    212
    213    jsonstring = create_json_string(ol, description, indent)
    214
    215    if not fname.endswith('.json') and not fname.endswith('.gz'):
    216        fname += '.json'
    217
    218    if gz:
    219        if not fname.endswith('.gz'):
    220            fname += '.gz'
    221
    222        fp = gzip.open(fname, 'wb')
    223        fp.write(jsonstring.encode('utf-8'))
    224    else:
    225        fp = open(fname, 'w', encoding='utf-8')
    226        fp.write(jsonstring)
    227    fp.close()

Export a list of Obs or structures containing Obs to a .json(.gz) file

###### Parameters

- ol (list): List of objects that will be exported. At the moment, these objects can be either of: Obs, list, numpy.ndarray, Corr. All Obs inside a structure have to be defined on the same set of configurations.
- fname (str): Filename of the output file.
- description (str): Optional string that describes the contents of the json file.
- indent (int): Specify the indentation level of the json file. None or 0 is permissible and saves disk space.
- gz (bool): If True, the output is a gzipped json. If False, the output is a json file.

def import_json_string(json_string, verbose=True, full_output=False):
View Source

    433def import_json_string(json_string, verbose=True, full_output=False):
    434    """Reconstruct a list of Obs or structures containing Obs from a json string.
    435
    436    The following structures are supported: Obs, list, numpy.ndarray, Corr
    437    If the list contains only one element, it is unpacked from the list.
    438
    439    Parameters
    440    ----------
    441    json_string : str
    442        json string containing the data.
    443    verbose : bool
    444        Print additional information that was written to the file.
    445    full_output : bool
    446        If True, a dict containing auxiliary information and the data is returned.
    447        If False, only the data is returned.
    448    """
    449
    450    return _parse_json_dict(json.loads(json_string), verbose, full_output)

Reconstruct a list of Obs or structures containing Obs from a json string.

The following structures are supported: Obs, list, numpy.ndarray, Corr If the list contains only one element, it is unpacked from the list.

###### Parameters

- json_string (str): json string containing the data.
- verbose (bool): Print additional information that was written to the file.
- full_output (bool): If True, a dict containing auxiliary information and the data is returned. If False, only the data is returned.

def load_json(fname, verbose=True, gz=True, full_output=False):
View Source

    453def load_json(fname, verbose=True, gz=True, full_output=False):
    454    """Import a list of Obs or structures containing Obs from a .json(.gz) file.
    455
    456    The following structures are supported: Obs, list, numpy.ndarray, Corr
    457    If the list contains only one element, it is unpacked from the list.
    458
    459    Parameters
    460    ----------
    461    fname : str
    462        Filename of the input file.
    463    verbose : bool
    464        Print additional information that was written to the file.
    465    gz : bool
    466        If True, assumes that data is gzipped. If False, assumes JSON file.
    467    full_output : bool
    468        If True, a dict containing auxiliary information and the data is returned.
    469        If False, only the data is returned.
    470    """
    471    if not fname.endswith('.json') and not fname.endswith('.gz'):
    472        fname += '.json'
    473    if gz:
    474        if not fname.endswith('.gz'):
    475            fname += '.gz'
    476        with gzip.open(fname, 'r') as fin:
    477            d = json.load(fin)
    478    else:
    479        if fname.endswith('.gz'):
    480            warnings.warn("Trying to read from %s without unzipping!" % fname, UserWarning)
    481        with open(fname, 'r', encoding='utf-8') as fin:
    482            d = json.loads(fin.read())
    483
    484    return _parse_json_dict(d, verbose, full_output)

Import a list of Obs or structures containing Obs from a .json(.gz) file.

The following structures are supported: Obs, list, numpy.ndarray, Corr If the list contains only one element, it is unpacked from the list.

###### Parameters

- fname (str): Filename of the input file.
- verbose (bool): Print additional information that was written to the file.
- gz (bool): If True, assumes that data is gzipped. If False, assumes JSON file.
- full_output (bool): If True, a dict containing auxiliary information and the data is returned. If False, only the data is returned.

def dump_dict_to_json(od, fname, description='', indent=1, reps='DICTOBS', gz=True):
View Source

    567def dump_dict_to_json(od, fname, description='', indent=1, reps='DICTOBS', gz=True):
    568    """Export a dict of Obs or structures containing Obs to a .json(.gz) file
    569
    570    Parameters
    571    ----------
    572    od : dict
    573        Dict of JSON valid structures and objects that will be exported.
    574        At the moment, these objects can be either of: Obs, list, numpy.ndarray, Corr.
    575        All Obs inside a structure have to be defined on the same set of configurations.
    576    fname : str
    577        Filename of the output file.
    578    description : str
    579        Optional string that describes the contents of the json file.
    580    indent : int
    581        Specify the indentation level of the json file. None or 0 is permissible and
    582        saves disk space.
    583    reps : str
    584        Specify the structure of the placeholder in exported dict to be reps[0-9]+.
    585    gz : bool
    586        If True, the output is a gzipped json. If False, the output is a json file.
    587    """
    588
    589    if not isinstance(od, dict):
    590        raise Exception('od has to be a dictionary. Did you want to use dump_to_json?')
    591
    592    infostring = ('This JSON file contains a python dictionary that has been parsed to a list of structures. '
    593                  'OBSDICT contains the dictionary, where Obs or other structures have been replaced by '
    594                  '' + reps + '[0-9]+. The field description contains the additional description of this JSON file. '
    595                  'This file may be parsed to a dict with the pyerrors routine load_json_dict.')
    596
    597    desc_dict = {'INFO': infostring, 'OBSDICT': {}, 'description': description}
    598    ol, desc_dict['OBSDICT'] = _ol_from_dict(od, reps=reps)
    599
    600    dump_to_json(ol, fname, description=desc_dict, indent=indent, gz=gz)

Export a dict of Obs or structures containing Obs to a .json(.gz) file

###### Parameters

- od (dict): Dict of JSON valid structures and objects that will be exported. At the moment, these objects can be either of: Obs, list, numpy.ndarray, Corr. All Obs inside a structure have to be defined on the same set of configurations.
- fname (str): Filename of the output file.
- description (str): Optional string that describes the contents of the json file.
- indent (int): Specify the indentation level of the json file. None or 0 is permissible and saves disk space.
- reps (str): Specify the structure of the placeholder in exported dict to be reps[0-9]+.
- gz (bool): If True, the output is a gzipped json. If False, the output is a json file.

def load_json_dict(fname, verbose=True, gz=True, full_output=False, reps='DICTOBS'):
View Source

    666def load_json_dict(fname, verbose=True, gz=True, full_output=False, reps='DICTOBS'):
    667    """Import a dict of Obs or structures containing Obs from a .json(.gz) file.
    668
    669    The following structures are supported: Obs, list, numpy.ndarray, Corr
    670
    671    Parameters
    672    ----------
    673    fname : str
    674        Filename of the input file.
    675    verbose : bool
    676        Print additional information that was written to the file.
    677    gz : bool
    678        If True, assumes that data is gzipped. If False, assumes JSON file.
    679    full_output : bool
    680        If True, a dict containing auxiliary information and the data is returned.
    681        If False, only the data is returned.
    682    reps : str
    683        Specify the structure of the placeholder in imported dict to be reps[0-9]+.
    684    """
    685    indata = load_json(fname, verbose=verbose, gz=gz, full_output=True)
    686    description = indata['description']['description']
    687    indict = indata['description']['OBSDICT']
    688    ol = indata['obsdata']
    689    od = _od_from_list_and_dict(ol, indict, reps=reps)
    690
    691    if full_output:
    692        indata['description'] = description
    693        indata['obsdata'] = od
    694        return indata
    695    else:
    696        return od

Import a dict of Obs or structures containing Obs from a .json(.gz) file.

The following structures are supported: Obs, list, numpy.ndarray, Corr

###### Parameters

- fname (str): Filename of the input file.
- verbose (bool): Print additional information that was written to the file.
- gz (bool): If True, assumes that data is gzipped. If False, assumes JSON file.
- full_output (bool): If True, a dict containing auxiliary information and the data is returned. If False, only the data is returned.
- reps (str): Specify the structure of the placeholder in imported dict to be reps[0-9]+.