# pyerrors.input.json

View Source

      1import rapidjson as json
      2import gzip
      3import getpass
      4import socket
      5import datetime
      6import platform
      7import warnings
      8import re
      9import numpy as np
     10from ..obs import Obs
     11from ..covobs import Covobs
     12from ..correlators import Corr
     13from ..misc import _assert_equal_properties
     14from .. import version as pyerrorsversion
     15
     16
     17def create_json_string(ol, description='', indent=1):
     18    """Generate the string for the export of a list of Obs or structures containing Obs
     19    to a .json(.gz) file
     20
     21    Parameters
     22    ----------
     23    ol : list
     24        List of objects that will be exported. At the moment, these objects can be
     25        either of: Obs, list, numpy.ndarray, Corr.
     26        All Obs inside a structure have to be defined on the same set of configurations.
     27    description : str
     28        Optional string that describes the contents of the json file.
     29    indent : int
     30        Specify the indentation level of the json file. None or 0 is permissible and
     31        saves disk space.
     32    """
     33
     34    def _gen_data_d_from_list(ol):
     35        dl = []
     36        No = len(ol)
     37        for name in ol[0].mc_names:
     38            ed = {}
     39            ed['id'] = name
     40            ed['replica'] = []
     41            for r_name in ol[0].e_content[name]:
     42                rd = {}
     43                rd['name'] = r_name
     44                if ol[0].is_merged.get(r_name, False):
     45                    rd['is_merged'] = True
     46                rd['deltas'] = []
     47                offsets = [o.r_values[r_name] - o.value for o in ol]
     48                deltas = np.column_stack([ol[oi].deltas[r_name] + offsets[oi] for oi in range(No)])
     49                for i in range(len(ol[0].idl[r_name])):
     50                    rd['deltas'].append([ol[0].idl[r_name][i]])
     51                    rd['deltas'][-1] += deltas[i].tolist()
     52                ed['replica'].append(rd)
     53            dl.append(ed)
     54        return dl
     55
     56    def _gen_cdata_d_from_list(ol):
     57        dl = []
     58        for name in ol[0].cov_names:
     59            ed = {}
     60            ed['id'] = name
     61            ed['layout'] = str(ol[0].covobs[name].cov.shape).lstrip('(').rstrip(')').rstrip(',')
     62            ed['cov'] = list(np.ravel(ol[0].covobs[name].cov))
     63            ncov = ol[0].covobs[name].cov.shape[0]
     64            ed['grad'] = []
     65            for i in range(ncov):
     66                ed['grad'].append([])
     67                for o in ol:
     68                    ed['grad'][-1].append(o.covobs[name].grad[i][0])
     69            dl.append(ed)
     70        return dl
     71
     72    def write_Obs_to_dict(o):
     73        d = {}
     74        d['type'] = 'Obs'
     75        d['layout'] = '1'
     76        if o.tag:
     77            d['tag'] = [o.tag]
     78        if o.reweighted:
     79            d['reweighted'] = o.reweighted
     80        d['value'] = [o.value]
     81        data = _gen_data_d_from_list([o])
     82        if len(data) > 0:
     83            d['data'] = data
     84        cdata = _gen_cdata_d_from_list([o])
     85        if len(cdata) > 0:
     86            d['cdata'] = cdata
     87        return d
     88
     89    def write_List_to_dict(ol):
     90        _assert_equal_properties(ol)
     91        d = {}
     92        d['type'] = 'List'
     93        d['layout'] = '%d' % len(ol)
     94        taglist = [o.tag for o in ol]
     95        if np.any([tag is not None for tag in taglist]):
     96            d['tag'] = taglist
     97        if ol[0].reweighted:
     98            d['reweighted'] = ol[0].reweighted
     99        d['value'] = [o.value for o in ol]
    100        data = _gen_data_d_from_list(ol)
    101        if len(data) > 0:
    102            d['data'] = data
    103        cdata = _gen_cdata_d_from_list(ol)
    104        if len(cdata) > 0:
    105            d['cdata'] = cdata
    106        return d
    107
    108    def write_Array_to_dict(oa):
    109        ol = np.ravel(oa)
    110        _assert_equal_properties(ol)
    111        d = {}
    112        d['type'] = 'Array'
    113        d['layout'] = str(oa.shape).lstrip('(').rstrip(')').rstrip(',')
    114        taglist = [o.tag for o in ol]
    115        if np.any([tag is not None for tag in taglist]):
    116            d['tag'] = taglist
    117        if ol[0].reweighted:
    118            d['reweighted'] = ol[0].reweighted
    119        d['value'] = [o.value for o in ol]
    120        data = _gen_data_d_from_list(ol)
    121        if len(data) > 0:
    122            d['data'] = data
    123        cdata = _gen_cdata_d_from_list(ol)
    124        if len(cdata) > 0:
    125            d['cdata'] = cdata
    126        return d
    127
    128    def _nan_Obs_like(obs):
    129        samples = []
    130        names = []
    131        idl = []
    132        for key, value in obs.idl.items():
    133            samples.append([np.nan] * len(value))
    134            names.append(key)
    135            idl.append(value)
    136        my_obs = Obs(samples, names, idl)
    137        my_obs._covobs = obs._covobs
    138        for name in obs._covobs:
    139            my_obs.names.append(name)
    140        my_obs.reweighted = obs.reweighted
    141        my_obs.is_merged = obs.is_merged
    142        return my_obs
    143
    144    def write_Corr_to_dict(my_corr):
    145        first_not_none = next(i for i, j in enumerate(my_corr.content) if np.all(j))
    146        dummy_array = np.empty((my_corr.N, my_corr.N), dtype=object)
    147        dummy_array[:] = _nan_Obs_like(my_corr.content[first_not_none].ravel()[0])
    148        content = [o if o is not None else dummy_array for o in my_corr.content]
    149        dat = write_Array_to_dict(np.array(content, dtype=object))
    150        dat['type'] = 'Corr'
    151        corr_meta_data = str(my_corr.tag)
    152        if 'tag' in dat.keys():
    153            dat['tag'].append(corr_meta_data)
    154        else:
    155            dat['tag'] = [corr_meta_data]
    156        taglist = dat['tag']
    157        dat['tag'] = {}  # tag is now a dictionary, that contains the previous taglist in the key "tag"
    158        dat['tag']['tag'] = taglist
    159        if my_corr.prange is not None:
    160            dat['tag']['prange'] = my_corr.prange
    161        return dat
    162
    163    if not isinstance(ol, list):
    164        ol = [ol]
    165
    166    d = {}
    167    d['program'] = 'pyerrors %s' % (pyerrorsversion.__version__)
    168    d['version'] = '1.1'
    169    d['who'] = getpass.getuser()
    170    d['date'] = datetime.datetime.now().astimezone().strftime('%Y-%m-%d %H:%M:%S %z')
    171    d['host'] = socket.gethostname() + ', ' + platform.platform()
    172
    173    if description:
    174        d['description'] = description
    175
    176    d['obsdata'] = []
    177    for io in ol:
    178        if isinstance(io, Obs):
    179            d['obsdata'].append(write_Obs_to_dict(io))
    180        elif isinstance(io, list):
    181            d['obsdata'].append(write_List_to_dict(io))
    182        elif isinstance(io, np.ndarray):
    183            d['obsdata'].append(write_Array_to_dict(io))
    184        elif isinstance(io, Corr):
    185            d['obsdata'].append(write_Corr_to_dict(io))
    186        else:
    187            raise Exception("Unkown datatype.")
    188
    189    if indent:
    190        return json.dumps(d, indent=indent, ensure_ascii=False, write_mode=json.WM_SINGLE_LINE_ARRAY)
    191    else:
    192        return json.dumps(d, indent=indent, ensure_ascii=False, write_mode=json.WM_COMPACT)
    193
    194
    195def dump_to_json(ol, fname, description='', indent=1, gz=True):
    196    """Export a list of Obs or structures containing Obs to a .json(.gz) file
    197
    198    Parameters
    199    ----------
    200    ol : list
    201        List of objects that will be exported. At the moment, these objects can be
    202        either of: Obs, list, numpy.ndarray, Corr.
    203        All Obs inside a structure have to be defined on the same set of configurations.
    204    fname : str
    205        Filename of the output file.
    206    description : str
    207        Optional string that describes the contents of the json file.
    208    indent : int
    209        Specify the indentation level of the json file. None or 0 is permissible and
    210        saves disk space.
    211    gz : bool
    212        If True, the output is a gzipped json. If False, the output is a json file.
    213    """
    214
    215    jsonstring = create_json_string(ol, description, indent)
    216
    217    if not fname.endswith('.json') and not fname.endswith('.gz'):
    218        fname += '.json'
    219
    220    if gz:
    221        if not fname.endswith('.gz'):
    222            fname += '.gz'
    223
    224        fp = gzip.open(fname, 'wb')
    225        fp.write(jsonstring.encode('utf-8'))
    226    else:
    227        fp = open(fname, 'w', encoding='utf-8')
    228        fp.write(jsonstring)
    229    fp.close()
    230
    231
    232def _parse_json_dict(json_dict, verbose=True, full_output=False):
    233    """Reconstruct a list of Obs or structures containing Obs from a dict that
    234    was built out of a json string.
    235
    236    The following structures are supported: Obs, list, numpy.ndarray, Corr
    237    If the list contains only one element, it is unpacked from the list.
    238
    239    Parameters
    240    ----------
    241    json_string : str
    242        json string containing the data.
    243    verbose : bool
    244        Print additional information that was written to the file.
    245    full_output : bool
    246        If True, a dict containing auxiliary information and the data is returned.
    247        If False, only the data is returned.
    248    """
    249
    250    def _gen_obsd_from_datad(d):
    251        retd = {}
    252        if d:
    253            retd['names'] = []
    254            retd['idl'] = []
    255            retd['deltas'] = []
    256            retd['is_merged'] = {}
    257            for ens in d:
    258                for rep in ens['replica']:
    259                    rep_name = rep['name']
    260                    if len(rep_name) > len(ens["id"]):
    261                        if rep_name[len(ens["id"])] != "|":
    262                            tmp_list = list(rep_name)
    263                            tmp_list = tmp_list[:len(ens["id"])] + ["|"] + tmp_list[len(ens["id"]):]
    264                            rep_name = ''.join(tmp_list)
    265                    retd['names'].append(rep_name)
    266                    retd['idl'].append([di[0] for di in rep['deltas']])
    267                    retd['deltas'].append(np.array([di[1:] for di in rep['deltas']]))
    268                    retd['is_merged'][rep_name] = rep.get('is_merged', False)
    269        return retd
    270
    271    def _gen_covobsd_from_cdatad(d):
    272        retd = {}
    273        for ens in d:
    274            retl = []
    275            name = ens['id']
    276            layouts = ens.get('layout', '1').strip()
    277            layout = [int(ls.strip()) for ls in layouts.split(',') if len(ls) > 0]
    278            cov = np.reshape(ens['cov'], layout)
    279            grad = ens['grad']
    280            nobs = len(grad[0])
    281            for i in range(nobs):
    282                retl.append({'name': name, 'cov': cov, 'grad': [g[i] for g in grad]})
    283            retd[name] = retl
    284        return retd
    285
    286    def get_Obs_from_dict(o):
    287        layouts = o.get('layout', '1').strip()
    288        if layouts != '1':
    289            raise Exception("layout is %s has to be 1 for type Obs." % (layouts), RuntimeWarning)
    290
    291        values = o['value']
    292        od = _gen_obsd_from_datad(o.get('data', {}))
    293        cd = _gen_covobsd_from_cdatad(o.get('cdata', {}))
    294
    295        if od:
    296            ret = Obs([[ddi[0] + values[0] for ddi in di] for di in od['deltas']], od['names'], idl=od['idl'])
    297            ret._value = values[0]
    298            ret.is_merged = od['is_merged']
    299        else:
    300            ret = Obs([], [], means=[])
    301            ret._value = values[0]
    302        for name in cd:
    303            co = cd[name][0]
    304            ret._covobs[name] = Covobs(None, co['cov'], co['name'], grad=co['grad'])
    305            ret.names.append(co['name'])
    306
    307        ret.reweighted = o.get('reweighted', False)
    308        ret.tag = o.get('tag', [None])[0]
    309        return ret
    310
    311    def get_List_from_dict(o):
    312        layouts = o.get('layout', '1').strip()
    313        layout = int(layouts)
    314        values = o['value']
    315        od = _gen_obsd_from_datad(o.get('data', {}))
    316        cd = _gen_covobsd_from_cdatad(o.get('cdata', {}))
    317
    318        ret = []
    319        taglist = o.get('tag', layout * [None])
    320        for i in range(layout):
    321            if od:
    322                ret.append(Obs([list(di[:, i] + values[i]) for di in od['deltas']], od['names'], idl=od['idl']))
    323                ret[-1]._value = values[i]
    324                ret[-1].is_merged = od['is_merged']
    325            else:
    326                ret.append(Obs([], [], means=[]))
    327                ret[-1]._value = values[i]
    328                print('Created Obs with means= ', values[i])
    329            for name in cd:
    330                co = cd[name][i]
    331                ret[-1]._covobs[name] = Covobs(None, co['cov'], co['name'], grad=co['grad'])
    332                ret[-1].names.append(co['name'])
    333
    334            ret[-1].reweighted = o.get('reweighted', False)
    335            ret[-1].tag = taglist[i]
    336        return ret
    337
    338    def get_Array_from_dict(o):
    339        layouts = o.get('layout', '1').strip()
    340        layout = [int(ls.strip()) for ls in layouts.split(',') if len(ls) > 0]
    341        N = np.prod(layout)
    342        values = o['value']
    343        od = _gen_obsd_from_datad(o.get('data', {}))
    344        cd = _gen_covobsd_from_cdatad(o.get('cdata', {}))
    345
    346        ret = []
    347        taglist = o.get('tag', N * [None])
    348        for i in range(N):
    349            if od:
    350                ret.append(Obs([di[:, i] + values[i] for di in od['deltas']], od['names'], idl=od['idl']))
    351                ret[-1]._value = values[i]
    352                ret[-1].is_merged = od['is_merged']
    353            else:
    354                ret.append(Obs([], [], means=[]))
    355                ret[-1]._value = values[i]
    356            for name in cd:
    357                co = cd[name][i]
    358                ret[-1]._covobs[name] = Covobs(None, co['cov'], co['name'], grad=co['grad'])
    359                ret[-1].names.append(co['name'])
    360            ret[-1].reweighted = o.get('reweighted', False)
    361            ret[-1].tag = taglist[i]
    362        return np.reshape(ret, layout)
    363
    364    def get_Corr_from_dict(o):
    365        if isinstance(o.get('tag'), list):  # supports the old way
    366            taglist = o.get('tag')  # This had to be modified to get the taglist from the dictionary
    367            temp_prange = None
    368        elif isinstance(o.get('tag'), dict):
    369            tagdic = o.get('tag')
    370            taglist = tagdic['tag']
    371            if 'prange' in tagdic:
    372                temp_prange = tagdic['prange']
    373            else:
    374                temp_prange = None
    375        else:
    376            raise Exception("The tag is not a list or dict")
    377
    378        corr_tag = taglist[-1]
    379        tmp_o = o
    380        tmp_o['tag'] = taglist[:-1]
    381        if len(tmp_o['tag']) == 0:
    382            del tmp_o['tag']
    383        dat = get_Array_from_dict(tmp_o)
    384        my_corr = Corr([None if np.isnan(o.ravel()[0].value) else o for o in list(dat)])
    385        if corr_tag != 'None':
    386            my_corr.tag = corr_tag
    387
    388        my_corr.prange = temp_prange
    389        return my_corr
    390
    391    prog = json_dict.get('program', '')
    392    version = json_dict.get('version', '')
    393    who = json_dict.get('who', '')
    394    date = json_dict.get('date', '')
    395    host = json_dict.get('host', '')
    396    if prog and verbose:
    397        print('Data has been written using %s.' % (prog))
    398    if version and verbose:
    399        print('Format version %s' % (version))
    400    if np.any([who, date, host] and verbose):
    401        print('Written by %s on %s on host %s' % (who, date, host))
    402    description = json_dict.get('description', '')
    403    if description and verbose:
    404        print()
    405        print('Description: ', description)
    406    obsdata = json_dict['obsdata']
    407    ol = []
    408    for io in obsdata:
    409        if io['type'] == 'Obs':
    410            ol.append(get_Obs_from_dict(io))
    411        elif io['type'] == 'List':
    412            ol.append(get_List_from_dict(io))
    413        elif io['type'] == 'Array':
    414            ol.append(get_Array_from_dict(io))
    415        elif io['type'] == 'Corr':
    416            ol.append(get_Corr_from_dict(io))
    417        else:
    418            raise Exception("Unkown datatype.")
    419
    420    if full_output:
    421        retd = {}
    422        retd['program'] = prog
    423        retd['version'] = version
    424        retd['who'] = who
    425        retd['date'] = date
    426        retd['host'] = host
    427        retd['description'] = description
    428        retd['obsdata'] = ol
    429
    430        return retd
    431    else:
    432        if len(obsdata) == 1:
    433            ol = ol[0]
    434
    435        return ol
    436
    437
    438def import_json_string(json_string, verbose=True, full_output=False):
    439    """Reconstruct a list of Obs or structures containing Obs from a json string.
    440
    441    The following structures are supported: Obs, list, numpy.ndarray, Corr
    442    If the list contains only one element, it is unpacked from the list.
    443
    444    Parameters
    445    ----------
    446    json_string : str
    447        json string containing the data.
    448    verbose : bool
    449        Print additional information that was written to the file.
    450    full_output : bool
    451        If True, a dict containing auxiliary information and the data is returned.
    452        If False, only the data is returned.
    453    """
    454
    455    return _parse_json_dict(json.loads(json_string), verbose, full_output)
    456
    457
    458def load_json(fname, verbose=True, gz=True, full_output=False):
    459    """Import a list of Obs or structures containing Obs from a .json(.gz) file.
    460
    461    The following structures are supported: Obs, list, numpy.ndarray, Corr
    462    If the list contains only one element, it is unpacked from the list.
    463
    464    Parameters
    465    ----------
    466    fname : str
    467        Filename of the input file.
    468    verbose : bool
    469        Print additional information that was written to the file.
    470    gz : bool
    471        If True, assumes that data is gzipped. If False, assumes JSON file.
    472    full_output : bool
    473        If True, a dict containing auxiliary information and the data is returned.
    474        If False, only the data is returned.
    475    """
    476    if not fname.endswith('.json') and not fname.endswith('.gz'):
    477        fname += '.json'
    478    if gz:
    479        if not fname.endswith('.gz'):
    480            fname += '.gz'
    481        with gzip.open(fname, 'r') as fin:
    482            d = json.load(fin)
    483    else:
    484        if fname.endswith('.gz'):
    485            warnings.warn("Trying to read from %s without unzipping!" % fname, UserWarning)
    486        with open(fname, 'r', encoding='utf-8') as fin:
    487            d = json.loads(fin.read())
    488
    489    return _parse_json_dict(d, verbose, full_output)
    490
    491
    492def _ol_from_dict(ind, reps='DICTOBS'):
    493    """Convert a dictionary of Obs objects to a list and a dictionary that contains
    494    placeholders instead of the Obs objects.
    495
    496    Parameters
    497    ----------
    498    ind : dict
    499        Dict of JSON valid structures and objects that will be exported.
    500        At the moment, these object can be either of: Obs, list, numpy.ndarray, Corr.
    501        All Obs inside a structure have to be defined on the same set of configurations.
    502    reps : str
    503        Specify the structure of the placeholder in exported dict to be reps[0-9]+.
    504    """
    505
    506    obstypes = (Obs, Corr, np.ndarray)
    507
    508    if not reps.isalnum():
    509        raise Exception('Placeholder string has to be alphanumeric!')
    510    ol = []
    511    counter = 0
    512
    513    def dict_replace_obs(d):
    514        nonlocal ol
    515        nonlocal counter
    516        x = {}
    517        for k, v in d.items():
    518            if isinstance(v, dict):
    519                v = dict_replace_obs(v)
    520            elif isinstance(v, list) and all([isinstance(o, Obs) for o in v]):
    521                v = obslist_replace_obs(v)
    522            elif isinstance(v, list):
    523                v = list_replace_obs(v)
    524            elif isinstance(v, obstypes):
    525                ol.append(v)
    526                v = reps + '%d' % (counter)
    527                counter += 1
    528            elif isinstance(v, str):
    529                if bool(re.match(r'%s[0-9]+' % (reps), v)):
    530                    raise Exception('Dict contains string %s that matches the placeholder! %s Cannot be savely exported.' % (v, reps))
    531            x[k] = v
    532        return x
    533
    534    def list_replace_obs(li):
    535        nonlocal ol
    536        nonlocal counter
    537        x = []
    538        for e in li:
    539            if isinstance(e, list):
    540                e = list_replace_obs(e)
    541            elif isinstance(e, list) and all([isinstance(o, Obs) for o in e]):
    542                e = obslist_replace_obs(e)
    543            elif isinstance(e, dict):
    544                e = dict_replace_obs(e)
    545            elif isinstance(e, obstypes):
    546                ol.append(e)
    547                e = reps + '%d' % (counter)
    548                counter += 1
    549            elif isinstance(e, str):
    550                if bool(re.match(r'%s[0-9]+' % (reps), e)):
    551                    raise Exception('Dict contains string %s that matches the placeholder! %s Cannot be savely exported.' % (e, reps))
    552            x.append(e)
    553        return x
    554
    555    def obslist_replace_obs(li):
    556        nonlocal ol
    557        nonlocal counter
    558        il = []
    559        for e in li:
    560            il.append(e)
    561
    562        ol.append(il)
    563        x = reps + '%d' % (counter)
    564        counter += 1
    565        return x
    566
    567    nd = dict_replace_obs(ind)
    568
    569    return ol, nd
    570
    571
    572def dump_dict_to_json(od, fname, description='', indent=1, reps='DICTOBS', gz=True):
    573    """Export a dict of Obs or structures containing Obs to a .json(.gz) file
    574
    575    Parameters
    576    ----------
    577    od : dict
    578        Dict of JSON valid structures and objects that will be exported.
    579        At the moment, these objects can be either of: Obs, list, numpy.ndarray, Corr.
    580        All Obs inside a structure have to be defined on the same set of configurations.
    581    fname : str
    582        Filename of the output file.
    583    description : str
    584        Optional string that describes the contents of the json file.
    585    indent : int
    586        Specify the indentation level of the json file. None or 0 is permissible and
    587        saves disk space.
    588    reps : str
    589        Specify the structure of the placeholder in exported dict to be reps[0-9]+.
    590    gz : bool
    591        If True, the output is a gzipped json. If False, the output is a json file.
    592    """
    593
    594    if not isinstance(od, dict):
    595        raise Exception('od has to be a dictionary. Did you want to use dump_to_json?')
    596
    597    infostring = ('This JSON file contains a python dictionary that has been parsed to a list of structures. '
    598                  'OBSDICT contains the dictionary, where Obs or other structures have been replaced by '
    599                  '' + reps + '[0-9]+. The field description contains the additional description of this JSON file. '
    600                  'This file may be parsed to a dict with the pyerrors routine load_json_dict.')
    601
    602    desc_dict = {'INFO': infostring, 'OBSDICT': {}, 'description': description}
    603    ol, desc_dict['OBSDICT'] = _ol_from_dict(od, reps=reps)
    604
    605    dump_to_json(ol, fname, description=desc_dict, indent=indent, gz=gz)
    606
    607
    608def _od_from_list_and_dict(ol, ind, reps='DICTOBS'):
    609    """Parse a list of Obs or structures containing Obs and an accompanying
    610    dict, where the structures have been replaced by placeholders to a
    611    dict that contains the structures.
    612
    613    The following structures are supported: Obs, list, numpy.ndarray, Corr
    614
    615    Parameters
    616    ----------
    617    ol : list
    618        List of objects -
    619        At the moment, these objects can be either of: Obs, list, numpy.ndarray, Corr.
    620        All Obs inside a structure have to be defined on the same set of configurations.
    621    ind : dict
    622        Dict that defines the structure of the resulting dict and contains placeholders
    623    reps : str
    624        Specify the structure of the placeholder in imported dict to be reps[0-9]+.
    625    """
    626    if not reps.isalnum():
    627        raise Exception('Placeholder string has to be alphanumeric!')
    628
    629    counter = 0
    630
    631    def dict_replace_string(d):
    632        nonlocal counter
    633        nonlocal ol
    634        x = {}
    635        for k, v in d.items():
    636            if isinstance(v, dict):
    637                v = dict_replace_string(v)
    638            elif isinstance(v, list):
    639                v = list_replace_string(v)
    640            elif isinstance(v, str) and bool(re.match(r'%s[0-9]+' % (reps), v)):
    641                index = int(v[len(reps):])
    642                v = ol[index]
    643                counter += 1
    644            x[k] = v
    645        return x
    646
    647    def list_replace_string(li):
    648        nonlocal counter
    649        nonlocal ol
    650        x = []
    651        for e in li:
    652            if isinstance(e, list):
    653                e = list_replace_string(e)
    654            elif isinstance(e, dict):
    655                e = dict_replace_string(e)
    656            elif isinstance(e, str) and bool(re.match(r'%s[0-9]+' % (reps), e)):
    657                index = int(e[len(reps):])
    658                e = ol[index]
    659                counter += 1
    660            x.append(e)
    661        return x
    662
    663    nd = dict_replace_string(ind)
    664
    665    if counter == 0:
    666        raise Exception('No placeholder has been replaced! Check if reps is set correctly.')
    667
    668    return nd
    669
    670
    671def load_json_dict(fname, verbose=True, gz=True, full_output=False, reps='DICTOBS'):
    672    """Import a dict of Obs or structures containing Obs from a .json(.gz) file.
    673
    674    The following structures are supported: Obs, list, numpy.ndarray, Corr
    675
    676    Parameters
    677    ----------
    678    fname : str
    679        Filename of the input file.
    680    verbose : bool
    681        Print additional information that was written to the file.
    682    gz : bool
    683        If True, assumes that data is gzipped. If False, assumes JSON file.
    684    full_output : bool
    685        If True, a dict containing auxiliary information and the data is returned.
    686        If False, only the data is returned.
    687    reps : str
    688        Specify the structure of the placeholder in imported dict to be reps[0-9]+.
    689    """
    690    indata = load_json(fname, verbose=verbose, gz=gz, full_output=True)
    691    description = indata['description']['description']
    692    indict = indata['description']['OBSDICT']
    693    ol = indata['obsdata']
    694    od = _od_from_list_and_dict(ol, indict, reps=reps)
    695
    696    if full_output:
    697        indata['description'] = description
    698        indata['obsdata'] = od
    699        return indata
    700    else:
    701        return od

def create_json_string(ol, description='', indent=1): View Source

     18def create_json_string(ol, description='', indent=1):
     19    """Generate the string for the export of a list of Obs or structures containing Obs
     20    to a .json(.gz) file
     21
     22    Parameters
     23    ----------
     24    ol : list
     25        List of objects that will be exported. At the moment, these objects can be
     26        either of: Obs, list, numpy.ndarray, Corr.
     27        All Obs inside a structure have to be defined on the same set of configurations.
     28    description : str
     29        Optional string that describes the contents of the json file.
     30    indent : int
     31        Specify the indentation level of the json file. None or 0 is permissible and
     32        saves disk space.
     33    """
     34
     35    def _gen_data_d_from_list(ol):
     36        dl = []
     37        No = len(ol)
     38        for name in ol[0].mc_names:
     39            ed = {}
     40            ed['id'] = name
     41            ed['replica'] = []
     42            for r_name in ol[0].e_content[name]:
     43                rd = {}
     44                rd['name'] = r_name
     45                if ol[0].is_merged.get(r_name, False):
     46                    rd['is_merged'] = True
     47                rd['deltas'] = []
     48                offsets = [o.r_values[r_name] - o.value for o in ol]
     49                deltas = np.column_stack([ol[oi].deltas[r_name] + offsets[oi] for oi in range(No)])
     50                for i in range(len(ol[0].idl[r_name])):
     51                    rd['deltas'].append([ol[0].idl[r_name][i]])
     52                    rd['deltas'][-1] += deltas[i].tolist()
     53                ed['replica'].append(rd)
     54            dl.append(ed)
     55        return dl
     56
     57    def _gen_cdata_d_from_list(ol):
     58        dl = []
     59        for name in ol[0].cov_names:
     60            ed = {}
     61            ed['id'] = name
     62            ed['layout'] = str(ol[0].covobs[name].cov.shape).lstrip('(').rstrip(')').rstrip(',')
     63            ed['cov'] = list(np.ravel(ol[0].covobs[name].cov))
     64            ncov = ol[0].covobs[name].cov.shape[0]
     65            ed['grad'] = []
     66            for i in range(ncov):
     67                ed['grad'].append([])
     68                for o in ol:
     69                    ed['grad'][-1].append(o.covobs[name].grad[i][0])
     70            dl.append(ed)
     71        return dl
     72
     73    def write_Obs_to_dict(o):
     74        d = {}
     75        d['type'] = 'Obs'
     76        d['layout'] = '1'
     77        if o.tag:
     78            d['tag'] = [o.tag]
     79        if o.reweighted:
     80            d['reweighted'] = o.reweighted
     81        d['value'] = [o.value]
     82        data = _gen_data_d_from_list([o])
     83        if len(data) > 0:
     84            d['data'] = data
     85        cdata = _gen_cdata_d_from_list([o])
     86        if len(cdata) > 0:
     87            d['cdata'] = cdata
     88        return d
     89
     90    def write_List_to_dict(ol):
     91        _assert_equal_properties(ol)
     92        d = {}
     93        d['type'] = 'List'
     94        d['layout'] = '%d' % len(ol)
     95        taglist = [o.tag for o in ol]
     96        if np.any([tag is not None for tag in taglist]):
     97            d['tag'] = taglist
     98        if ol[0].reweighted:
     99            d['reweighted'] = ol[0].reweighted
    100        d['value'] = [o.value for o in ol]
    101        data = _gen_data_d_from_list(ol)
    102        if len(data) > 0:
    103            d['data'] = data
    104        cdata = _gen_cdata_d_from_list(ol)
    105        if len(cdata) > 0:
    106            d['cdata'] = cdata
    107        return d
    108
    109    def write_Array_to_dict(oa):
    110        ol = np.ravel(oa)
    111        _assert_equal_properties(ol)
    112        d = {}
    113        d['type'] = 'Array'
    114        d['layout'] = str(oa.shape).lstrip('(').rstrip(')').rstrip(',')
    115        taglist = [o.tag for o in ol]
    116        if np.any([tag is not None for tag in taglist]):
    117            d['tag'] = taglist
    118        if ol[0].reweighted:
    119            d['reweighted'] = ol[0].reweighted
    120        d['value'] = [o.value for o in ol]
    121        data = _gen_data_d_from_list(ol)
    122        if len(data) > 0:
    123            d['data'] = data
    124        cdata = _gen_cdata_d_from_list(ol)
    125        if len(cdata) > 0:
    126            d['cdata'] = cdata
    127        return d
    128
    129    def _nan_Obs_like(obs):
    130        samples = []
    131        names = []
    132        idl = []
    133        for key, value in obs.idl.items():
    134            samples.append([np.nan] * len(value))
    135            names.append(key)
    136            idl.append(value)
    137        my_obs = Obs(samples, names, idl)
    138        my_obs._covobs = obs._covobs
    139        for name in obs._covobs:
    140            my_obs.names.append(name)
    141        my_obs.reweighted = obs.reweighted
    142        my_obs.is_merged = obs.is_merged
    143        return my_obs
    144
    145    def write_Corr_to_dict(my_corr):
    146        first_not_none = next(i for i, j in enumerate(my_corr.content) if np.all(j))
    147        dummy_array = np.empty((my_corr.N, my_corr.N), dtype=object)
    148        dummy_array[:] = _nan_Obs_like(my_corr.content[first_not_none].ravel()[0])
    149        content = [o if o is not None else dummy_array for o in my_corr.content]
    150        dat = write_Array_to_dict(np.array(content, dtype=object))
    151        dat['type'] = 'Corr'
    152        corr_meta_data = str(my_corr.tag)
    153        if 'tag' in dat.keys():
    154            dat['tag'].append(corr_meta_data)
    155        else:
    156            dat['tag'] = [corr_meta_data]
    157        taglist = dat['tag']
    158        dat['tag'] = {}  # tag is now a dictionary, that contains the previous taglist in the key "tag"
    159        dat['tag']['tag'] = taglist
    160        if my_corr.prange is not None:
    161            dat['tag']['prange'] = my_corr.prange
    162        return dat
    163
    164    if not isinstance(ol, list):
    165        ol = [ol]
    166
    167    d = {}
    168    d['program'] = 'pyerrors %s' % (pyerrorsversion.__version__)
    169    d['version'] = '1.1'
    170    d['who'] = getpass.getuser()
    171    d['date'] = datetime.datetime.now().astimezone().strftime('%Y-%m-%d %H:%M:%S %z')
    172    d['host'] = socket.gethostname() + ', ' + platform.platform()
    173
    174    if description:
    175        d['description'] = description
    176
    177    d['obsdata'] = []
    178    for io in ol:
    179        if isinstance(io, Obs):
    180            d['obsdata'].append(write_Obs_to_dict(io))
    181        elif isinstance(io, list):
    182            d['obsdata'].append(write_List_to_dict(io))
    183        elif isinstance(io, np.ndarray):
    184            d['obsdata'].append(write_Array_to_dict(io))
    185        elif isinstance(io, Corr):
    186            d['obsdata'].append(write_Corr_to_dict(io))
    187        else:
    188            raise Exception("Unkown datatype.")
    189
    190    if indent:
    191        return json.dumps(d, indent=indent, ensure_ascii=False, write_mode=json.WM_SINGLE_LINE_ARRAY)
    192    else:
    193        return json.dumps(d, indent=indent, ensure_ascii=False, write_mode=json.WM_COMPACT)

Generate the string for the export of a list of Obs or structures containing Obs to a .json(.gz) file

###### Parameters

- ol (list): List of objects that will be exported. At the moment, these objects can be either of: Obs, list, numpy.ndarray, Corr. All Obs inside a structure have to be defined on the same set of configurations.
- description (str): Optional string that describes the contents of the json file.
- indent (int): Specify the indentation level of the json file. None or 0 is permissible and saves disk space.

def dump_to_json(ol, fname, description='', indent=1, gz=True): View Source

    196def dump_to_json(ol, fname, description='', indent=1, gz=True):
    197    """Export a list of Obs or structures containing Obs to a .json(.gz) file
    198
    199    Parameters
    200    ----------
    201    ol : list
    202        List of objects that will be exported. At the moment, these objects can be
    203        either of: Obs, list, numpy.ndarray, Corr.
    204        All Obs inside a structure have to be defined on the same set of configurations.
    205    fname : str
    206        Filename of the output file.
    207    description : str
    208        Optional string that describes the contents of the json file.
    209    indent : int
    210        Specify the indentation level of the json file. None or 0 is permissible and
    211        saves disk space.
    212    gz : bool
    213        If True, the output is a gzipped json. If False, the output is a json file.
    214    """
    215
    216    jsonstring = create_json_string(ol, description, indent)
    217
    218    if not fname.endswith('.json') and not fname.endswith('.gz'):
    219        fname += '.json'
    220
    221    if gz:
    222        if not fname.endswith('.gz'):
    223            fname += '.gz'
    224
    225        fp = gzip.open(fname, 'wb')
    226        fp.write(jsonstring.encode('utf-8'))
    227    else:
    228        fp = open(fname, 'w', encoding='utf-8')
    229        fp.write(jsonstring)
    230    fp.close()

Export a list of Obs or structures containing Obs to a .json(.gz) file

###### Parameters

- ol (list): List of objects that will be exported. At the moment, these objects can be either of: Obs, list, numpy.ndarray, Corr. All Obs inside a structure have to be defined on the same set of configurations.
- fname (str): Filename of the output file.
- description (str): Optional string that describes the contents of the json file.
- indent (int): Specify the indentation level of the json file. None or 0 is permissible and saves disk space.
- gz (bool): If True, the output is a gzipped json. If False, the output is a json file.

def import_json_string(json_string, verbose=True, full_output=False): View Source

    439def import_json_string(json_string, verbose=True, full_output=False):
    440    """Reconstruct a list of Obs or structures containing Obs from a json string.
    441
    442    The following structures are supported: Obs, list, numpy.ndarray, Corr
    443    If the list contains only one element, it is unpacked from the list.
    444
    445    Parameters
    446    ----------
    447    json_string : str
    448        json string containing the data.
    449    verbose : bool
    450        Print additional information that was written to the file.
    451    full_output : bool
    452        If True, a dict containing auxiliary information and the data is returned.
    453        If False, only the data is returned.
    454    """
    455
    456    return _parse_json_dict(json.loads(json_string), verbose, full_output)

Reconstruct a list of Obs or structures containing Obs from a json string.

The following structures are supported: Obs, list, numpy.ndarray, Corr If the list contains only one element, it is unpacked from the list.

###### Parameters

- json_string (str): json string containing the data.
- verbose (bool): Print additional information that was written to the file.
- full_output (bool): If True, a dict containing auxiliary information and the data is returned. If False, only the data is returned.

def load_json(fname, verbose=True, gz=True, full_output=False): View Source

    459def load_json(fname, verbose=True, gz=True, full_output=False):
    460    """Import a list of Obs or structures containing Obs from a .json(.gz) file.
    461
    462    The following structures are supported: Obs, list, numpy.ndarray, Corr
    463    If the list contains only one element, it is unpacked from the list.
    464
    465    Parameters
    466    ----------
    467    fname : str
    468        Filename of the input file.
    469    verbose : bool
    470        Print additional information that was written to the file.
    471    gz : bool
    472        If True, assumes that data is gzipped. If False, assumes JSON file.
    473    full_output : bool
    474        If True, a dict containing auxiliary information and the data is returned.
    475        If False, only the data is returned.
    476    """
    477    if not fname.endswith('.json') and not fname.endswith('.gz'):
    478        fname += '.json'
    479    if gz:
    480        if not fname.endswith('.gz'):
    481            fname += '.gz'
    482        with gzip.open(fname, 'r') as fin:
    483            d = json.load(fin)
    484    else:
    485        if fname.endswith('.gz'):
    486            warnings.warn("Trying to read from %s without unzipping!" % fname, UserWarning)
    487        with open(fname, 'r', encoding='utf-8') as fin:
    488            d = json.loads(fin.read())
    489
    490    return _parse_json_dict(d, verbose, full_output)

Import a list of Obs or structures containing Obs from a .json(.gz) file.

The following structures are supported: Obs, list, numpy.ndarray, Corr If the list contains only one element, it is unpacked from the list.

###### Parameters

- fname (str): Filename of the input file.
- verbose (bool): Print additional information that was written to the file.
- gz (bool): If True, assumes that data is gzipped. If False, assumes JSON file.
- full_output (bool): If True, a dict containing auxiliary information and the data is returned. If False, only the data is returned.

def dump_dict_to_json(od, fname, description='', indent=1, reps='DICTOBS', gz=True): View Source

    573def dump_dict_to_json(od, fname, description='', indent=1, reps='DICTOBS', gz=True):
    574    """Export a dict of Obs or structures containing Obs to a .json(.gz) file
    575
    576    Parameters
    577    ----------
    578    od : dict
    579        Dict of JSON valid structures and objects that will be exported.
    580        At the moment, these objects can be either of: Obs, list, numpy.ndarray, Corr.
    581        All Obs inside a structure have to be defined on the same set of configurations.
    582    fname : str
    583        Filename of the output file.
    584    description : str
    585        Optional string that describes the contents of the json file.
    586    indent : int
    587        Specify the indentation level of the json file. None or 0 is permissible and
    588        saves disk space.
    589    reps : str
    590        Specify the structure of the placeholder in exported dict to be reps[0-9]+.
    591    gz : bool
    592        If True, the output is a gzipped json. If False, the output is a json file.
    593    """
    594
    595    if not isinstance(od, dict):
    596        raise Exception('od has to be a dictionary. Did you want to use dump_to_json?')
    597
    598    infostring = ('This JSON file contains a python dictionary that has been parsed to a list of structures. '
    599                  'OBSDICT contains the dictionary, where Obs or other structures have been replaced by '
    600                  '' + reps + '[0-9]+. The field description contains the additional description of this JSON file. '
    601                  'This file may be parsed to a dict with the pyerrors routine load_json_dict.')
    602
    603    desc_dict = {'INFO': infostring, 'OBSDICT': {}, 'description': description}
    604    ol, desc_dict['OBSDICT'] = _ol_from_dict(od, reps=reps)
    605
    606    dump_to_json(ol, fname, description=desc_dict, indent=indent, gz=gz)

Export a dict of Obs or structures containing Obs to a .json(.gz) file

###### Parameters

- od (dict): Dict of JSON valid structures and objects that will be exported. At the moment, these objects can be either of: Obs, list, numpy.ndarray, Corr. All Obs inside a structure have to be defined on the same set of configurations.
- fname (str): Filename of the output file.
- description (str): Optional string that describes the contents of the json file.
- indent (int): Specify the indentation level of the json file. None or 0 is permissible and saves disk space.
- reps (str): Specify the structure of the placeholder in exported dict to be reps[0-9]+.
- gz (bool): If True, the output is a gzipped json. If False, the output is a json file.

def load_json_dict(fname, verbose=True, gz=True, full_output=False, reps='DICTOBS'): View Source

    672def load_json_dict(fname, verbose=True, gz=True, full_output=False, reps='DICTOBS'):
    673    """Import a dict of Obs or structures containing Obs from a .json(.gz) file.
    674
    675    The following structures are supported: Obs, list, numpy.ndarray, Corr
    676
    677    Parameters
    678    ----------
    679    fname : str
    680        Filename of the input file.
    681    verbose : bool
    682        Print additional information that was written to the file.
    683    gz : bool
    684        If True, assumes that data is gzipped. If False, assumes JSON file.
    685    full_output : bool
    686        If True, a dict containing auxiliary information and the data is returned.
    687        If False, only the data is returned.
    688    reps : str
    689        Specify the structure of the placeholder in imported dict to be reps[0-9]+.
    690    """
    691    indata = load_json(fname, verbose=verbose, gz=gz, full_output=True)
    692    description = indata['description']['description']
    693    indict = indata['description']['OBSDICT']
    694    ol = indata['obsdata']
    695    od = _od_from_list_and_dict(ol, indict, reps=reps)
    696
    697    if full_output:
    698        indata['description'] = description
    699        indata['obsdata'] = od
    700        return indata
    701    else:
    702        return od

Import a dict of Obs or structures containing Obs from a .json(.gz) file.

The following structures are supported: Obs, list, numpy.ndarray, Corr

###### Parameters

- fname (str): Filename of the input file.
- verbose (bool): Print additional information that was written to the file.
- gz (bool): If True, assumes that data is gzipped. If False, assumes JSON file.
- full_output (bool): If True, a dict containing auxiliary information and the data is returned. If False, only the data is returned.
- reps (str): Specify the structure of the placeholder in imported dict to be reps[0-9]+.