# pyerrors.input.json

View Source

      1import rapidjson as json
      2import gzip
      3import getpass
      4import socket
      5import datetime
      6import platform
      7import warnings
      8import re
      9import numpy as np
     10from ..obs import Obs
     11from ..covobs import Covobs
     12from ..correlators import Corr
     13from ..misc import _assert_equal_properties
     14from .. import version as pyerrorsversion
     15
     16
     17def create_json_string(ol, description='', indent=1):
     18    """Generate the string for the export of a list of Obs or structures containing Obs
     19    to a .json(.gz) file
     20
     21    Parameters
     22    ----------
     23    ol : list
     24        List of objects that will be exported. At the moment, these objects can be
     25        either of: Obs, list, numpy.ndarray, Corr.
     26        All Obs inside a structure have to be defined on the same set of configurations.
     27    description : str
     28        Optional string that describes the contents of the json file.
     29    indent : int
     30        Specify the indentation level of the json file. None or 0 is permissible and
     31        saves disk space.
     32    """
     33
     34    def _gen_data_d_from_list(ol):
     35        dl = []
     36        No = len(ol)
     37        for name in ol[0].mc_names:
     38            ed = {}
     39            ed['id'] = name
     40            ed['replica'] = []
     41            for r_name in ol[0].e_content[name]:
     42                rd = {}
     43                rd['name'] = r_name
     44                if ol[0].is_merged.get(r_name, False):
     45                    rd['is_merged'] = True
     46                rd['deltas'] = []
     47                offsets = [o.r_values[r_name] - o.value for o in ol]
     48                deltas = np.column_stack([ol[oi].deltas[r_name] + offsets[oi] for oi in range(No)])
     49                for i in range(len(ol[0].idl[r_name])):
     50                    rd['deltas'].append([ol[0].idl[r_name][i]])
     51                    rd['deltas'][-1] += deltas[i].tolist()
     52                ed['replica'].append(rd)
     53            dl.append(ed)
     54        return dl
     55
     56    def _gen_cdata_d_from_list(ol):
     57        dl = []
     58        for name in ol[0].cov_names:
     59            ed = {}
     60            ed['id'] = name
     61            ed['layout'] = str(ol[0].covobs[name].cov.shape).lstrip('(').rstrip(')').rstrip(',')
     62            ed['cov'] = list(np.ravel(ol[0].covobs[name].cov))
     63            ncov = ol[0].covobs[name].cov.shape[0]
     64            ed['grad'] = []
     65            for i in range(ncov):
     66                ed['grad'].append([])
     67                for o in ol:
     68                    ed['grad'][-1].append(o.covobs[name].grad[i][0])
     69            dl.append(ed)
     70        return dl
     71
     72    def write_Obs_to_dict(o):
     73        d = {}
     74        d['type'] = 'Obs'
     75        d['layout'] = '1'
     76        if o.tag:
     77            d['tag'] = [o.tag]
     78        if o.reweighted:
     79            d['reweighted'] = o.reweighted
     80        d['value'] = [o.value]
     81        data = _gen_data_d_from_list([o])
     82        if len(data) > 0:
     83            d['data'] = data
     84        cdata = _gen_cdata_d_from_list([o])
     85        if len(cdata) > 0:
     86            d['cdata'] = cdata
     87        return d
     88
     89    def write_List_to_dict(ol):
     90        _assert_equal_properties(ol)
     91        d = {}
     92        d['type'] = 'List'
     93        d['layout'] = '%d' % len(ol)
     94        taglist = [o.tag for o in ol]
     95        if np.any([tag is not None for tag in taglist]):
     96            d['tag'] = taglist
     97        if ol[0].reweighted:
     98            d['reweighted'] = ol[0].reweighted
     99        d['value'] = [o.value for o in ol]
    100        data = _gen_data_d_from_list(ol)
    101        if len(data) > 0:
    102            d['data'] = data
    103        cdata = _gen_cdata_d_from_list(ol)
    104        if len(cdata) > 0:
    105            d['cdata'] = cdata
    106        return d
    107
    108    def write_Array_to_dict(oa):
    109        ol = np.ravel(oa)
    110        _assert_equal_properties(ol)
    111        d = {}
    112        d['type'] = 'Array'
    113        d['layout'] = str(oa.shape).lstrip('(').rstrip(')').rstrip(',')
    114        taglist = [o.tag for o in ol]
    115        if np.any([tag is not None for tag in taglist]):
    116            d['tag'] = taglist
    117        if ol[0].reweighted:
    118            d['reweighted'] = ol[0].reweighted
    119        d['value'] = [o.value for o in ol]
    120        data = _gen_data_d_from_list(ol)
    121        if len(data) > 0:
    122            d['data'] = data
    123        cdata = _gen_cdata_d_from_list(ol)
    124        if len(cdata) > 0:
    125            d['cdata'] = cdata
    126        return d
    127
    128    def _nan_Obs_like(obs):
    129        samples = []
    130        names = []
    131        idl = []
    132        for key, value in obs.idl.items():
    133            samples.append([np.nan] * len(value))
    134            names.append(key)
    135            idl.append(value)
    136        my_obs = Obs(samples, names, idl)
    137        my_obs._covobs = obs._covobs
    138        for name in obs._covobs:
    139            my_obs.names.append(name)
    140        my_obs.reweighted = obs.reweighted
    141        my_obs.is_merged = obs.is_merged
    142        return my_obs
    143
    144    def write_Corr_to_dict(my_corr):
    145        first_not_none = next(i for i, j in enumerate(my_corr.content) if np.all(j))
    146        dummy_array = np.empty((my_corr.N, my_corr.N), dtype=object)
    147        dummy_array[:] = _nan_Obs_like(my_corr.content[first_not_none].ravel()[0])
    148        content = [o if o is not None else dummy_array for o in my_corr.content]
    149        dat = write_Array_to_dict(np.array(content, dtype=object))
    150        dat['type'] = 'Corr'
    151        corr_meta_data = str(my_corr.tag)
    152        if 'tag' in dat.keys():
    153            dat['tag'].append(corr_meta_data)
    154        else:
    155            dat['tag'] = [corr_meta_data]
    156        taglist = dat['tag']
    157        dat['tag'] = {}  # tag is now a dictionary, that contains the previous taglist in the key "tag"
    158        dat['tag']['tag'] = taglist
    159        if my_corr.prange is not None:
    160            dat['tag']['prange'] = my_corr.prange
    161        return dat
    162
    163    if not isinstance(ol, list):
    164        ol = [ol]
    165
    166    d = {}
    167    d['program'] = 'pyerrors %s' % (pyerrorsversion.__version__)
    168    d['version'] = '1.1'
    169    d['who'] = getpass.getuser()
    170    d['date'] = datetime.datetime.now().astimezone().strftime('%Y-%m-%d %H:%M:%S %z')
    171    d['host'] = socket.gethostname() + ', ' + platform.platform()
    172
    173    if description:
    174        d['description'] = description
    175
    176    d['obsdata'] = []
    177    for io in ol:
    178        if isinstance(io, Obs):
    179            d['obsdata'].append(write_Obs_to_dict(io))
    180        elif isinstance(io, list):
    181            d['obsdata'].append(write_List_to_dict(io))
    182        elif isinstance(io, np.ndarray):
    183            d['obsdata'].append(write_Array_to_dict(io))
    184        elif isinstance(io, Corr):
    185            d['obsdata'].append(write_Corr_to_dict(io))
    186        else:
    187            raise Exception("Unkown datatype.")
    188
    189    def _jsonifier(o):
    190        if isinstance(o, np.int64):
    191            return int(o)
    192        raise TypeError('%r is not JSON serializable' % o)
    193
    194    if indent:
    195        return json.dumps(d, indent=indent, ensure_ascii=False, default=_jsonifier, write_mode=json.WM_SINGLE_LINE_ARRAY)
    196    else:
    197        return json.dumps(d, indent=indent, ensure_ascii=False, default=_jsonifier, write_mode=json.WM_COMPACT)
    198
    199
    200def dump_to_json(ol, fname, description='', indent=1, gz=True):
    201    """Export a list of Obs or structures containing Obs to a .json(.gz) file
    202
    203    Parameters
    204    ----------
    205    ol : list
    206        List of objects that will be exported. At the moment, these objects can be
    207        either of: Obs, list, numpy.ndarray, Corr.
    208        All Obs inside a structure have to be defined on the same set of configurations.
    209    fname : str
    210        Filename of the output file.
    211    description : str
    212        Optional string that describes the contents of the json file.
    213    indent : int
    214        Specify the indentation level of the json file. None or 0 is permissible and
    215        saves disk space.
    216    gz : bool
    217        If True, the output is a gzipped json. If False, the output is a json file.
    218    """
    219
    220    jsonstring = create_json_string(ol, description, indent)
    221
    222    if not fname.endswith('.json') and not fname.endswith('.gz'):
    223        fname += '.json'
    224
    225    if gz:
    226        if not fname.endswith('.gz'):
    227            fname += '.gz'
    228
    229        fp = gzip.open(fname, 'wb')
    230        fp.write(jsonstring.encode('utf-8'))
    231    else:
    232        fp = open(fname, 'w', encoding='utf-8')
    233        fp.write(jsonstring)
    234    fp.close()
    235
    236
    237def _parse_json_dict(json_dict, verbose=True, full_output=False):
    238    """Reconstruct a list of Obs or structures containing Obs from a dict that
    239    was built out of a json string.
    240
    241    The following structures are supported: Obs, list, numpy.ndarray, Corr
    242    If the list contains only one element, it is unpacked from the list.
    243
    244    Parameters
    245    ----------
    246    json_string : str
    247        json string containing the data.
    248    verbose : bool
    249        Print additional information that was written to the file.
    250    full_output : bool
    251        If True, a dict containing auxiliary information and the data is returned.
    252        If False, only the data is returned.
    253    """
    254
    255    def _gen_obsd_from_datad(d):
    256        retd = {}
    257        if d:
    258            retd['names'] = []
    259            retd['idl'] = []
    260            retd['deltas'] = []
    261            retd['is_merged'] = {}
    262            for ens in d:
    263                for rep in ens['replica']:
    264                    rep_name = rep['name']
    265                    if len(rep_name) > len(ens["id"]):
    266                        if rep_name[len(ens["id"])] != "|":
    267                            tmp_list = list(rep_name)
    268                            tmp_list = tmp_list[:len(ens["id"])] + ["|"] + tmp_list[len(ens["id"]):]
    269                            rep_name = ''.join(tmp_list)
    270                    retd['names'].append(rep_name)
    271                    retd['idl'].append([di[0] for di in rep['deltas']])
    272                    retd['deltas'].append(np.array([di[1:] for di in rep['deltas']]))
    273                    retd['is_merged'][rep_name] = rep.get('is_merged', False)
    274        return retd
    275
    276    def _gen_covobsd_from_cdatad(d):
    277        retd = {}
    278        for ens in d:
    279            retl = []
    280            name = ens['id']
    281            layouts = ens.get('layout', '1').strip()
    282            layout = [int(ls.strip()) for ls in layouts.split(',') if len(ls) > 0]
    283            cov = np.reshape(ens['cov'], layout)
    284            grad = ens['grad']
    285            nobs = len(grad[0])
    286            for i in range(nobs):
    287                retl.append({'name': name, 'cov': cov, 'grad': [g[i] for g in grad]})
    288            retd[name] = retl
    289        return retd
    290
    291    def get_Obs_from_dict(o):
    292        layouts = o.get('layout', '1').strip()
    293        if layouts != '1':
    294            raise Exception("layout is %s has to be 1 for type Obs." % (layouts), RuntimeWarning)
    295
    296        values = o['value']
    297        od = _gen_obsd_from_datad(o.get('data', {}))
    298        cd = _gen_covobsd_from_cdatad(o.get('cdata', {}))
    299
    300        if od:
    301            ret = Obs([[ddi[0] + values[0] for ddi in di] for di in od['deltas']], od['names'], idl=od['idl'])
    302            ret._value = values[0]
    303            ret.is_merged = od['is_merged']
    304        else:
    305            ret = Obs([], [], means=[])
    306            ret._value = values[0]
    307        for name in cd:
    308            co = cd[name][0]
    309            ret._covobs[name] = Covobs(None, co['cov'], co['name'], grad=co['grad'])
    310            ret.names.append(co['name'])
    311
    312        ret.reweighted = o.get('reweighted', False)
    313        ret.tag = o.get('tag', [None])[0]
    314        return ret
    315
    316    def get_List_from_dict(o):
    317        layouts = o.get('layout', '1').strip()
    318        layout = int(layouts)
    319        values = o['value']
    320        od = _gen_obsd_from_datad(o.get('data', {}))
    321        cd = _gen_covobsd_from_cdatad(o.get('cdata', {}))
    322
    323        ret = []
    324        taglist = o.get('tag', layout * [None])
    325        for i in range(layout):
    326            if od:
    327                ret.append(Obs([list(di[:, i] + values[i]) for di in od['deltas']], od['names'], idl=od['idl']))
    328                ret[-1]._value = values[i]
    329                ret[-1].is_merged = od['is_merged']
    330            else:
    331                ret.append(Obs([], [], means=[]))
    332                ret[-1]._value = values[i]
    333                print('Created Obs with means= ', values[i])
    334            for name in cd:
    335                co = cd[name][i]
    336                ret[-1]._covobs[name] = Covobs(None, co['cov'], co['name'], grad=co['grad'])
    337                ret[-1].names.append(co['name'])
    338
    339            ret[-1].reweighted = o.get('reweighted', False)
    340            ret[-1].tag = taglist[i]
    341        return ret
    342
    343    def get_Array_from_dict(o):
    344        layouts = o.get('layout', '1').strip()
    345        layout = [int(ls.strip()) for ls in layouts.split(',') if len(ls) > 0]
    346        N = np.prod(layout)
    347        values = o['value']
    348        od = _gen_obsd_from_datad(o.get('data', {}))
    349        cd = _gen_covobsd_from_cdatad(o.get('cdata', {}))
    350
    351        ret = []
    352        taglist = o.get('tag', N * [None])
    353        for i in range(N):
    354            if od:
    355                ret.append(Obs([di[:, i] + values[i] for di in od['deltas']], od['names'], idl=od['idl']))
    356                ret[-1]._value = values[i]
    357                ret[-1].is_merged = od['is_merged']
    358            else:
    359                ret.append(Obs([], [], means=[]))
    360                ret[-1]._value = values[i]
    361            for name in cd:
    362                co = cd[name][i]
    363                ret[-1]._covobs[name] = Covobs(None, co['cov'], co['name'], grad=co['grad'])
    364                ret[-1].names.append(co['name'])
    365            ret[-1].reweighted = o.get('reweighted', False)
    366            ret[-1].tag = taglist[i]
    367        return np.reshape(ret, layout)
    368
    369    def get_Corr_from_dict(o):
    370        if isinstance(o.get('tag'), list):  # supports the old way
    371            taglist = o.get('tag')  # This had to be modified to get the taglist from the dictionary
    372            temp_prange = None
    373        elif isinstance(o.get('tag'), dict):
    374            tagdic = o.get('tag')
    375            taglist = tagdic['tag']
    376            if 'prange' in tagdic:
    377                temp_prange = tagdic['prange']
    378            else:
    379                temp_prange = None
    380        else:
    381            raise Exception("The tag is not a list or dict")
    382
    383        corr_tag = taglist[-1]
    384        tmp_o = o
    385        tmp_o['tag'] = taglist[:-1]
    386        if len(tmp_o['tag']) == 0:
    387            del tmp_o['tag']
    388        dat = get_Array_from_dict(tmp_o)
    389        my_corr = Corr([None if np.isnan(o.ravel()[0].value) else o for o in list(dat)])
    390        if corr_tag != 'None':
    391            my_corr.tag = corr_tag
    392
    393        my_corr.prange = temp_prange
    394        return my_corr
    395
    396    prog = json_dict.get('program', '')
    397    version = json_dict.get('version', '')
    398    who = json_dict.get('who', '')
    399    date = json_dict.get('date', '')
    400    host = json_dict.get('host', '')
    401    if prog and verbose:
    402        print('Data has been written using %s.' % (prog))
    403    if version and verbose:
    404        print('Format version %s' % (version))
    405    if np.any([who, date, host] and verbose):
    406        print('Written by %s on %s on host %s' % (who, date, host))
    407    description = json_dict.get('description', '')
    408    if description and verbose:
    409        print()
    410        print('Description: ', description)
    411    obsdata = json_dict['obsdata']
    412    ol = []
    413    for io in obsdata:
    414        if io['type'] == 'Obs':
    415            ol.append(get_Obs_from_dict(io))
    416        elif io['type'] == 'List':
    417            ol.append(get_List_from_dict(io))
    418        elif io['type'] == 'Array':
    419            ol.append(get_Array_from_dict(io))
    420        elif io['type'] == 'Corr':
    421            ol.append(get_Corr_from_dict(io))
    422        else:
    423            raise Exception("Unkown datatype.")
    424
    425    if full_output:
    426        retd = {}
    427        retd['program'] = prog
    428        retd['version'] = version
    429        retd['who'] = who
    430        retd['date'] = date
    431        retd['host'] = host
    432        retd['description'] = description
    433        retd['obsdata'] = ol
    434
    435        return retd
    436    else:
    437        if len(obsdata) == 1:
    438            ol = ol[0]
    439
    440        return ol
    441
    442
    443def import_json_string(json_string, verbose=True, full_output=False):
    444    """Reconstruct a list of Obs or structures containing Obs from a json string.
    445
    446    The following structures are supported: Obs, list, numpy.ndarray, Corr
    447    If the list contains only one element, it is unpacked from the list.
    448
    449    Parameters
    450    ----------
    451    json_string : str
    452        json string containing the data.
    453    verbose : bool
    454        Print additional information that was written to the file.
    455    full_output : bool
    456        If True, a dict containing auxiliary information and the data is returned.
    457        If False, only the data is returned.
    458    """
    459
    460    return _parse_json_dict(json.loads(json_string), verbose, full_output)
    461
    462
    463def load_json(fname, verbose=True, gz=True, full_output=False):
    464    """Import a list of Obs or structures containing Obs from a .json(.gz) file.
    465
    466    The following structures are supported: Obs, list, numpy.ndarray, Corr
    467    If the list contains only one element, it is unpacked from the list.
    468
    469    Parameters
    470    ----------
    471    fname : str
    472        Filename of the input file.
    473    verbose : bool
    474        Print additional information that was written to the file.
    475    gz : bool
    476        If True, assumes that data is gzipped. If False, assumes JSON file.
    477    full_output : bool
    478        If True, a dict containing auxiliary information and the data is returned.
    479        If False, only the data is returned.
    480    """
    481    if not fname.endswith('.json') and not fname.endswith('.gz'):
    482        fname += '.json'
    483    if gz:
    484        if not fname.endswith('.gz'):
    485            fname += '.gz'
    486        with gzip.open(fname, 'r') as fin:
    487            d = json.load(fin)
    488    else:
    489        if fname.endswith('.gz'):
    490            warnings.warn("Trying to read from %s without unzipping!" % fname, UserWarning)
    491        with open(fname, 'r', encoding='utf-8') as fin:
    492            d = json.loads(fin.read())
    493
    494    return _parse_json_dict(d, verbose, full_output)
    495
    496
    497def _ol_from_dict(ind, reps='DICTOBS'):
    498    """Convert a dictionary of Obs objects to a list and a dictionary that contains
    499    placeholders instead of the Obs objects.
    500
    501    Parameters
    502    ----------
    503    ind : dict
    504        Dict of JSON valid structures and objects that will be exported.
    505        At the moment, these object can be either of: Obs, list, numpy.ndarray, Corr.
    506        All Obs inside a structure have to be defined on the same set of configurations.
    507    reps : str
    508        Specify the structure of the placeholder in exported dict to be reps[0-9]+.
    509    """
    510
    511    obstypes = (Obs, Corr, np.ndarray)
    512
    513    if not reps.isalnum():
    514        raise Exception('Placeholder string has to be alphanumeric!')
    515    ol = []
    516    counter = 0
    517
    518    def dict_replace_obs(d):
    519        nonlocal ol
    520        nonlocal counter
    521        x = {}
    522        for k, v in d.items():
    523            if isinstance(v, dict):
    524                v = dict_replace_obs(v)
    525            elif isinstance(v, list) and all([isinstance(o, Obs) for o in v]):
    526                v = obslist_replace_obs(v)
    527            elif isinstance(v, list):
    528                v = list_replace_obs(v)
    529            elif isinstance(v, obstypes):
    530                ol.append(v)
    531                v = reps + '%d' % (counter)
    532                counter += 1
    533            elif isinstance(v, str):
    534                if bool(re.match(r'%s[0-9]+' % (reps), v)):
    535                    raise Exception('Dict contains string %s that matches the placeholder! %s Cannot be savely exported.' % (v, reps))
    536            x[k] = v
    537        return x
    538
    539    def list_replace_obs(li):
    540        nonlocal ol
    541        nonlocal counter
    542        x = []
    543        for e in li:
    544            if isinstance(e, list):
    545                e = list_replace_obs(e)
    546            elif isinstance(e, list) and all([isinstance(o, Obs) for o in e]):
    547                e = obslist_replace_obs(e)
    548            elif isinstance(e, dict):
    549                e = dict_replace_obs(e)
    550            elif isinstance(e, obstypes):
    551                ol.append(e)
    552                e = reps + '%d' % (counter)
    553                counter += 1
    554            elif isinstance(e, str):
    555                if bool(re.match(r'%s[0-9]+' % (reps), e)):
    556                    raise Exception('Dict contains string %s that matches the placeholder! %s Cannot be savely exported.' % (e, reps))
    557            x.append(e)
    558        return x
    559
    560    def obslist_replace_obs(li):
    561        nonlocal ol
    562        nonlocal counter
    563        il = []
    564        for e in li:
    565            il.append(e)
    566
    567        ol.append(il)
    568        x = reps + '%d' % (counter)
    569        counter += 1
    570        return x
    571
    572    nd = dict_replace_obs(ind)
    573
    574    return ol, nd
    575
    576
    577def dump_dict_to_json(od, fname, description='', indent=1, reps='DICTOBS', gz=True):
    578    """Export a dict of Obs or structures containing Obs to a .json(.gz) file
    579
    580    Parameters
    581    ----------
    582    od : dict
    583        Dict of JSON valid structures and objects that will be exported.
    584        At the moment, these objects can be either of: Obs, list, numpy.ndarray, Corr.
    585        All Obs inside a structure have to be defined on the same set of configurations.
    586    fname : str
    587        Filename of the output file.
    588    description : str
    589        Optional string that describes the contents of the json file.
    590    indent : int
    591        Specify the indentation level of the json file. None or 0 is permissible and
    592        saves disk space.
    593    reps : str
    594        Specify the structure of the placeholder in exported dict to be reps[0-9]+.
    595    gz : bool
    596        If True, the output is a gzipped json. If False, the output is a json file.
    597    """
    598
    599    if not isinstance(od, dict):
    600        raise Exception('od has to be a dictionary. Did you want to use dump_to_json?')
    601
    602    infostring = ('This JSON file contains a python dictionary that has been parsed to a list of structures. '
    603                  'OBSDICT contains the dictionary, where Obs or other structures have been replaced by '
    604                  '' + reps + '[0-9]+. The field description contains the additional description of this JSON file. '
    605                  'This file may be parsed to a dict with the pyerrors routine load_json_dict.')
    606
    607    desc_dict = {'INFO': infostring, 'OBSDICT': {}, 'description': description}
    608    ol, desc_dict['OBSDICT'] = _ol_from_dict(od, reps=reps)
    609
    610    dump_to_json(ol, fname, description=desc_dict, indent=indent, gz=gz)
    611
    612
    613def _od_from_list_and_dict(ol, ind, reps='DICTOBS'):
    614    """Parse a list of Obs or structures containing Obs and an accompanying
    615    dict, where the structures have been replaced by placeholders to a
    616    dict that contains the structures.
    617
    618    The following structures are supported: Obs, list, numpy.ndarray, Corr
    619
    620    Parameters
    621    ----------
    622    ol : list
    623        List of objects -
    624        At the moment, these objects can be either of: Obs, list, numpy.ndarray, Corr.
    625        All Obs inside a structure have to be defined on the same set of configurations.
    626    ind : dict
    627        Dict that defines the structure of the resulting dict and contains placeholders
    628    reps : str
    629        Specify the structure of the placeholder in imported dict to be reps[0-9]+.
    630    """
    631    if not reps.isalnum():
    632        raise Exception('Placeholder string has to be alphanumeric!')
    633
    634    counter = 0
    635
    636    def dict_replace_string(d):
    637        nonlocal counter
    638        nonlocal ol
    639        x = {}
    640        for k, v in d.items():
    641            if isinstance(v, dict):
    642                v = dict_replace_string(v)
    643            elif isinstance(v, list):
    644                v = list_replace_string(v)
    645            elif isinstance(v, str) and bool(re.match(r'%s[0-9]+' % (reps), v)):
    646                index = int(v[len(reps):])
    647                v = ol[index]
    648                counter += 1
    649            x[k] = v
    650        return x
    651
    652    def list_replace_string(li):
    653        nonlocal counter
    654        nonlocal ol
    655        x = []
    656        for e in li:
    657            if isinstance(e, list):
    658                e = list_replace_string(e)
    659            elif isinstance(e, dict):
    660                e = dict_replace_string(e)
    661            elif isinstance(e, str) and bool(re.match(r'%s[0-9]+' % (reps), e)):
    662                index = int(e[len(reps):])
    663                e = ol[index]
    664                counter += 1
    665            x.append(e)
    666        return x
    667
    668    nd = dict_replace_string(ind)
    669
    670    if counter == 0:
    671        raise Exception('No placeholder has been replaced! Check if reps is set correctly.')
    672
    673    return nd
    674
    675
    676def load_json_dict(fname, verbose=True, gz=True, full_output=False, reps='DICTOBS'):
    677    """Import a dict of Obs or structures containing Obs from a .json(.gz) file.
    678
    679    The following structures are supported: Obs, list, numpy.ndarray, Corr
    680
    681    Parameters
    682    ----------
    683    fname : str
    684        Filename of the input file.
    685    verbose : bool
    686        Print additional information that was written to the file.
    687    gz : bool
    688        If True, assumes that data is gzipped. If False, assumes JSON file.
    689    full_output : bool
    690        If True, a dict containing auxiliary information and the data is returned.
    691        If False, only the data is returned.
    692    reps : str
    693        Specify the structure of the placeholder in imported dict to be reps[0-9]+.
    694    """
    695    indata = load_json(fname, verbose=verbose, gz=gz, full_output=True)
    696    description = indata['description']['description']
    697    indict = indata['description']['OBSDICT']
    698    ol = indata['obsdata']
    699    od = _od_from_list_and_dict(ol, indict, reps=reps)
    700
    701    if full_output:
    702        indata['description'] = description
    703        indata['obsdata'] = od
    704        return indata
    705    else:
    706        return od

def create_json_string(ol, description='', indent=1): View Source

     18def create_json_string(ol, description='', indent=1):
     19    """Generate the string for the export of a list of Obs or structures containing Obs
     20    to a .json(.gz) file
     21
     22    Parameters
     23    ----------
     24    ol : list
     25        List of objects that will be exported. At the moment, these objects can be
     26        either of: Obs, list, numpy.ndarray, Corr.
     27        All Obs inside a structure have to be defined on the same set of configurations.
     28    description : str
     29        Optional string that describes the contents of the json file.
     30    indent : int
     31        Specify the indentation level of the json file. None or 0 is permissible and
     32        saves disk space.
     33    """
     34
     35    def _gen_data_d_from_list(ol):
     36        dl = []
     37        No = len(ol)
     38        for name in ol[0].mc_names:
     39            ed = {}
     40            ed['id'] = name
     41            ed['replica'] = []
     42            for r_name in ol[0].e_content[name]:
     43                rd = {}
     44                rd['name'] = r_name
     45                if ol[0].is_merged.get(r_name, False):
     46                    rd['is_merged'] = True
     47                rd['deltas'] = []
     48                offsets = [o.r_values[r_name] - o.value for o in ol]
     49                deltas = np.column_stack([ol[oi].deltas[r_name] + offsets[oi] for oi in range(No)])
     50                for i in range(len(ol[0].idl[r_name])):
     51                    rd['deltas'].append([ol[0].idl[r_name][i]])
     52                    rd['deltas'][-1] += deltas[i].tolist()
     53                ed['replica'].append(rd)
     54            dl.append(ed)
     55        return dl
     56
     57    def _gen_cdata_d_from_list(ol):
     58        dl = []
     59        for name in ol[0].cov_names:
     60            ed = {}
     61            ed['id'] = name
     62            ed['layout'] = str(ol[0].covobs[name].cov.shape).lstrip('(').rstrip(')').rstrip(',')
     63            ed['cov'] = list(np.ravel(ol[0].covobs[name].cov))
     64            ncov = ol[0].covobs[name].cov.shape[0]
     65            ed['grad'] = []
     66            for i in range(ncov):
     67                ed['grad'].append([])
     68                for o in ol:
     69                    ed['grad'][-1].append(o.covobs[name].grad[i][0])
     70            dl.append(ed)
     71        return dl
     72
     73    def write_Obs_to_dict(o):
     74        d = {}
     75        d['type'] = 'Obs'
     76        d['layout'] = '1'
     77        if o.tag:
     78            d['tag'] = [o.tag]
     79        if o.reweighted:
     80            d['reweighted'] = o.reweighted
     81        d['value'] = [o.value]
     82        data = _gen_data_d_from_list([o])
     83        if len(data) > 0:
     84            d['data'] = data
     85        cdata = _gen_cdata_d_from_list([o])
     86        if len(cdata) > 0:
     87            d['cdata'] = cdata
     88        return d
     89
     90    def write_List_to_dict(ol):
     91        _assert_equal_properties(ol)
     92        d = {}
     93        d['type'] = 'List'
     94        d['layout'] = '%d' % len(ol)
     95        taglist = [o.tag for o in ol]
     96        if np.any([tag is not None for tag in taglist]):
     97            d['tag'] = taglist
     98        if ol[0].reweighted:
     99            d['reweighted'] = ol[0].reweighted
    100        d['value'] = [o.value for o in ol]
    101        data = _gen_data_d_from_list(ol)
    102        if len(data) > 0:
    103            d['data'] = data
    104        cdata = _gen_cdata_d_from_list(ol)
    105        if len(cdata) > 0:
    106            d['cdata'] = cdata
    107        return d
    108
    109    def write_Array_to_dict(oa):
    110        ol = np.ravel(oa)
    111        _assert_equal_properties(ol)
    112        d = {}
    113        d['type'] = 'Array'
    114        d['layout'] = str(oa.shape).lstrip('(').rstrip(')').rstrip(',')
    115        taglist = [o.tag for o in ol]
    116        if np.any([tag is not None for tag in taglist]):
    117            d['tag'] = taglist
    118        if ol[0].reweighted:
    119            d['reweighted'] = ol[0].reweighted
    120        d['value'] = [o.value for o in ol]
    121        data = _gen_data_d_from_list(ol)
    122        if len(data) > 0:
    123            d['data'] = data
    124        cdata = _gen_cdata_d_from_list(ol)
    125        if len(cdata) > 0:
    126            d['cdata'] = cdata
    127        return d
    128
    129    def _nan_Obs_like(obs):
    130        samples = []
    131        names = []
    132        idl = []
    133        for key, value in obs.idl.items():
    134            samples.append([np.nan] * len(value))
    135            names.append(key)
    136            idl.append(value)
    137        my_obs = Obs(samples, names, idl)
    138        my_obs._covobs = obs._covobs
    139        for name in obs._covobs:
    140            my_obs.names.append(name)
    141        my_obs.reweighted = obs.reweighted
    142        my_obs.is_merged = obs.is_merged
    143        return my_obs
    144
    145    def write_Corr_to_dict(my_corr):
    146        first_not_none = next(i for i, j in enumerate(my_corr.content) if np.all(j))
    147        dummy_array = np.empty((my_corr.N, my_corr.N), dtype=object)
    148        dummy_array[:] = _nan_Obs_like(my_corr.content[first_not_none].ravel()[0])
    149        content = [o if o is not None else dummy_array for o in my_corr.content]
    150        dat = write_Array_to_dict(np.array(content, dtype=object))
    151        dat['type'] = 'Corr'
    152        corr_meta_data = str(my_corr.tag)
    153        if 'tag' in dat.keys():
    154            dat['tag'].append(corr_meta_data)
    155        else:
    156            dat['tag'] = [corr_meta_data]
    157        taglist = dat['tag']
    158        dat['tag'] = {}  # tag is now a dictionary, that contains the previous taglist in the key "tag"
    159        dat['tag']['tag'] = taglist
    160        if my_corr.prange is not None:
    161            dat['tag']['prange'] = my_corr.prange
    162        return dat
    163
    164    if not isinstance(ol, list):
    165        ol = [ol]
    166
    167    d = {}
    168    d['program'] = 'pyerrors %s' % (pyerrorsversion.__version__)
    169    d['version'] = '1.1'
    170    d['who'] = getpass.getuser()
    171    d['date'] = datetime.datetime.now().astimezone().strftime('%Y-%m-%d %H:%M:%S %z')
    172    d['host'] = socket.gethostname() + ', ' + platform.platform()
    173
    174    if description:
    175        d['description'] = description
    176
    177    d['obsdata'] = []
    178    for io in ol:
    179        if isinstance(io, Obs):
    180            d['obsdata'].append(write_Obs_to_dict(io))
    181        elif isinstance(io, list):
    182            d['obsdata'].append(write_List_to_dict(io))
    183        elif isinstance(io, np.ndarray):
    184            d['obsdata'].append(write_Array_to_dict(io))
    185        elif isinstance(io, Corr):
    186            d['obsdata'].append(write_Corr_to_dict(io))
    187        else:
    188            raise Exception("Unkown datatype.")
    189
    190    def _jsonifier(o):
    191        if isinstance(o, np.int64):
    192            return int(o)
    193        raise TypeError('%r is not JSON serializable' % o)
    194
    195    if indent:
    196        return json.dumps(d, indent=indent, ensure_ascii=False, default=_jsonifier, write_mode=json.WM_SINGLE_LINE_ARRAY)
    197    else:
    198        return json.dumps(d, indent=indent, ensure_ascii=False, default=_jsonifier, write_mode=json.WM_COMPACT)

Generate the string for the export of a list of Obs or structures containing Obs to a .json(.gz) file

###### Parameters

- ol (list): List of objects that will be exported. At the moment, these objects can be either of: Obs, list, numpy.ndarray, Corr. All Obs inside a structure have to be defined on the same set of configurations.
- description (str): Optional string that describes the contents of the json file.
- indent (int): Specify the indentation level of the json file. None or 0 is permissible and saves disk space.

def dump_to_json(ol, fname, description='', indent=1, gz=True): View Source

    201def dump_to_json(ol, fname, description='', indent=1, gz=True):
    202    """Export a list of Obs or structures containing Obs to a .json(.gz) file
    203
    204    Parameters
    205    ----------
    206    ol : list
    207        List of objects that will be exported. At the moment, these objects can be
    208        either of: Obs, list, numpy.ndarray, Corr.
    209        All Obs inside a structure have to be defined on the same set of configurations.
    210    fname : str
    211        Filename of the output file.
    212    description : str
    213        Optional string that describes the contents of the json file.
    214    indent : int
    215        Specify the indentation level of the json file. None or 0 is permissible and
    216        saves disk space.
    217    gz : bool
    218        If True, the output is a gzipped json. If False, the output is a json file.
    219    """
    220
    221    jsonstring = create_json_string(ol, description, indent)
    222
    223    if not fname.endswith('.json') and not fname.endswith('.gz'):
    224        fname += '.json'
    225
    226    if gz:
    227        if not fname.endswith('.gz'):
    228            fname += '.gz'
    229
    230        fp = gzip.open(fname, 'wb')
    231        fp.write(jsonstring.encode('utf-8'))
    232    else:
    233        fp = open(fname, 'w', encoding='utf-8')
    234        fp.write(jsonstring)
    235    fp.close()

Export a list of Obs or structures containing Obs to a .json(.gz) file

###### Parameters

- ol (list): List of objects that will be exported. At the moment, these objects can be either of: Obs, list, numpy.ndarray, Corr. All Obs inside a structure have to be defined on the same set of configurations.
- fname (str): Filename of the output file.
- description (str): Optional string that describes the contents of the json file.
- indent (int): Specify the indentation level of the json file. None or 0 is permissible and saves disk space.
- gz (bool): If True, the output is a gzipped json. If False, the output is a json file.

def import_json_string(json_string, verbose=True, full_output=False): View Source

    444def import_json_string(json_string, verbose=True, full_output=False):
    445    """Reconstruct a list of Obs or structures containing Obs from a json string.
    446
    447    The following structures are supported: Obs, list, numpy.ndarray, Corr
    448    If the list contains only one element, it is unpacked from the list.
    449
    450    Parameters
    451    ----------
    452    json_string : str
    453        json string containing the data.
    454    verbose : bool
    455        Print additional information that was written to the file.
    456    full_output : bool
    457        If True, a dict containing auxiliary information and the data is returned.
    458        If False, only the data is returned.
    459    """
    460
    461    return _parse_json_dict(json.loads(json_string), verbose, full_output)

Reconstruct a list of Obs or structures containing Obs from a json string.

The following structures are supported: Obs, list, numpy.ndarray, Corr If the list contains only one element, it is unpacked from the list.

###### Parameters

- json_string (str): json string containing the data.
- verbose (bool): Print additional information that was written to the file.
- full_output (bool): If True, a dict containing auxiliary information and the data is returned. If False, only the data is returned.

def load_json(fname, verbose=True, gz=True, full_output=False): View Source

    464def load_json(fname, verbose=True, gz=True, full_output=False):
    465    """Import a list of Obs or structures containing Obs from a .json(.gz) file.
    466
    467    The following structures are supported: Obs, list, numpy.ndarray, Corr
    468    If the list contains only one element, it is unpacked from the list.
    469
    470    Parameters
    471    ----------
    472    fname : str
    473        Filename of the input file.
    474    verbose : bool
    475        Print additional information that was written to the file.
    476    gz : bool
    477        If True, assumes that data is gzipped. If False, assumes JSON file.
    478    full_output : bool
    479        If True, a dict containing auxiliary information and the data is returned.
    480        If False, only the data is returned.
    481    """
    482    if not fname.endswith('.json') and not fname.endswith('.gz'):
    483        fname += '.json'
    484    if gz:
    485        if not fname.endswith('.gz'):
    486            fname += '.gz'
    487        with gzip.open(fname, 'r') as fin:
    488            d = json.load(fin)
    489    else:
    490        if fname.endswith('.gz'):
    491            warnings.warn("Trying to read from %s without unzipping!" % fname, UserWarning)
    492        with open(fname, 'r', encoding='utf-8') as fin:
    493            d = json.loads(fin.read())
    494
    495    return _parse_json_dict(d, verbose, full_output)

Import a list of Obs or structures containing Obs from a .json(.gz) file.

The following structures are supported: Obs, list, numpy.ndarray, Corr If the list contains only one element, it is unpacked from the list.

###### Parameters

- fname (str): Filename of the input file.
- verbose (bool): Print additional information that was written to the file.
- gz (bool): If True, assumes that data is gzipped. If False, assumes JSON file.
- full_output (bool): If True, a dict containing auxiliary information and the data is returned. If False, only the data is returned.

def dump_dict_to_json(od, fname, description='', indent=1, reps='DICTOBS', gz=True): View Source

    578def dump_dict_to_json(od, fname, description='', indent=1, reps='DICTOBS', gz=True):
    579    """Export a dict of Obs or structures containing Obs to a .json(.gz) file
    580
    581    Parameters
    582    ----------
    583    od : dict
    584        Dict of JSON valid structures and objects that will be exported.
    585        At the moment, these objects can be either of: Obs, list, numpy.ndarray, Corr.
    586        All Obs inside a structure have to be defined on the same set of configurations.
    587    fname : str
    588        Filename of the output file.
    589    description : str
    590        Optional string that describes the contents of the json file.
    591    indent : int
    592        Specify the indentation level of the json file. None or 0 is permissible and
    593        saves disk space.
    594    reps : str
    595        Specify the structure of the placeholder in exported dict to be reps[0-9]+.
    596    gz : bool
    597        If True, the output is a gzipped json. If False, the output is a json file.
    598    """
    599
    600    if not isinstance(od, dict):
    601        raise Exception('od has to be a dictionary. Did you want to use dump_to_json?')
    602
    603    infostring = ('This JSON file contains a python dictionary that has been parsed to a list of structures. '
    604                  'OBSDICT contains the dictionary, where Obs or other structures have been replaced by '
    605                  '' + reps + '[0-9]+. The field description contains the additional description of this JSON file. '
    606                  'This file may be parsed to a dict with the pyerrors routine load_json_dict.')
    607
    608    desc_dict = {'INFO': infostring, 'OBSDICT': {}, 'description': description}
    609    ol, desc_dict['OBSDICT'] = _ol_from_dict(od, reps=reps)
    610
    611    dump_to_json(ol, fname, description=desc_dict, indent=indent, gz=gz)

Export a dict of Obs or structures containing Obs to a .json(.gz) file

###### Parameters

- od (dict): Dict of JSON valid structures and objects that will be exported. At the moment, these objects can be either of: Obs, list, numpy.ndarray, Corr. All Obs inside a structure have to be defined on the same set of configurations.
- fname (str): Filename of the output file.
- description (str): Optional string that describes the contents of the json file.
- indent (int): Specify the indentation level of the json file. None or 0 is permissible and saves disk space.
- reps (str): Specify the structure of the placeholder in exported dict to be reps[0-9]+.
- gz (bool): If True, the output is a gzipped json. If False, the output is a json file.

def load_json_dict(fname, verbose=True, gz=True, full_output=False, reps='DICTOBS'): View Source

    677def load_json_dict(fname, verbose=True, gz=True, full_output=False, reps='DICTOBS'):
    678    """Import a dict of Obs or structures containing Obs from a .json(.gz) file.
    679
    680    The following structures are supported: Obs, list, numpy.ndarray, Corr
    681
    682    Parameters
    683    ----------
    684    fname : str
    685        Filename of the input file.
    686    verbose : bool
    687        Print additional information that was written to the file.
    688    gz : bool
    689        If True, assumes that data is gzipped. If False, assumes JSON file.
    690    full_output : bool
    691        If True, a dict containing auxiliary information and the data is returned.
    692        If False, only the data is returned.
    693    reps : str
    694        Specify the structure of the placeholder in imported dict to be reps[0-9]+.
    695    """
    696    indata = load_json(fname, verbose=verbose, gz=gz, full_output=True)
    697    description = indata['description']['description']
    698    indict = indata['description']['OBSDICT']
    699    ol = indata['obsdata']
    700    od = _od_from_list_and_dict(ol, indict, reps=reps)
    701
    702    if full_output:
    703        indata['description'] = description
    704        indata['obsdata'] = od
    705        return indata
    706    else:
    707        return od

Import a dict of Obs or structures containing Obs from a .json(.gz) file.

The following structures are supported: Obs, list, numpy.ndarray, Corr

###### Parameters

- fname (str): Filename of the input file.
- verbose (bool): Print additional information that was written to the file.
- gz (bool): If True, assumes that data is gzipped. If False, assumes JSON file.
- full_output (bool): If True, a dict containing auxiliary information and the data is returned. If False, only the data is returned.
- reps (str): Specify the structure of the placeholder in imported dict to be reps[0-9]+.