pyerrors.input.json

  1import rapidjson as json
  2import gzip
  3import getpass
  4import socket
  5import datetime
  6import platform
  7import warnings
  8import re
  9import numpy as np
 10from ..obs import Obs
 11from ..covobs import Covobs
 12from ..correlators import Corr
 13from ..misc import _assert_equal_properties
 14from .. import version as pyerrorsversion
 15
 16
 17def create_json_string(ol, description='', indent=1):
 18    """Generate the string for the export of a list of Obs or structures containing Obs
 19    to a .json(.gz) file
 20
 21    Parameters
 22    ----------
 23    ol : list
 24        List of objects that will be exported. At the moment, these objects can be
 25        either of: Obs, list, numpy.ndarray, Corr.
 26        All Obs inside a structure have to be defined on the same set of configurations.
 27    description : str
 28        Optional string that describes the contents of the json file.
 29    indent : int
 30        Specify the indentation level of the json file. None or 0 is permissible and
 31        saves disk space.
 32    """
 33
 34    def _gen_data_d_from_list(ol):
 35        dl = []
 36        No = len(ol)
 37        for name in ol[0].mc_names:
 38            ed = {}
 39            ed['id'] = name
 40            ed['replica'] = []
 41            for r_name in ol[0].e_content[name]:
 42                rd = {}
 43                rd['name'] = r_name
 44                if ol[0].is_merged.get(r_name, False):
 45                    rd['is_merged'] = True
 46                rd['deltas'] = []
 47                offsets = [o.r_values[r_name] - o.value for o in ol]
 48                deltas = np.column_stack([ol[oi].deltas[r_name] + offsets[oi] for oi in range(No)])
 49                for i in range(len(ol[0].idl[r_name])):
 50                    rd['deltas'].append([ol[0].idl[r_name][i]])
 51                    rd['deltas'][-1] += deltas[i].tolist()
 52                ed['replica'].append(rd)
 53            dl.append(ed)
 54        return dl
 55
 56    def _gen_cdata_d_from_list(ol):
 57        dl = []
 58        for name in ol[0].cov_names:
 59            ed = {}
 60            ed['id'] = name
 61            ed['layout'] = str(ol[0].covobs[name].cov.shape).lstrip('(').rstrip(')').rstrip(',')
 62            ed['cov'] = list(np.ravel(ol[0].covobs[name].cov))
 63            ncov = ol[0].covobs[name].cov.shape[0]
 64            ed['grad'] = []
 65            for i in range(ncov):
 66                ed['grad'].append([])
 67                for o in ol:
 68                    ed['grad'][-1].append(o.covobs[name].grad[i][0])
 69            dl.append(ed)
 70        return dl
 71
 72    def write_Obs_to_dict(o):
 73        d = {}
 74        d['type'] = 'Obs'
 75        d['layout'] = '1'
 76        if o.tag:
 77            d['tag'] = [o.tag]
 78        if o.reweighted:
 79            d['reweighted'] = o.reweighted
 80        d['value'] = [o.value]
 81        data = _gen_data_d_from_list([o])
 82        if len(data) > 0:
 83            d['data'] = data
 84        cdata = _gen_cdata_d_from_list([o])
 85        if len(cdata) > 0:
 86            d['cdata'] = cdata
 87        return d
 88
 89    def write_List_to_dict(ol):
 90        _assert_equal_properties(ol)
 91        d = {}
 92        d['type'] = 'List'
 93        d['layout'] = '%d' % len(ol)
 94        taglist = [o.tag for o in ol]
 95        if np.any([tag is not None for tag in taglist]):
 96            d['tag'] = taglist
 97        if ol[0].reweighted:
 98            d['reweighted'] = ol[0].reweighted
 99        d['value'] = [o.value for o in ol]
100        data = _gen_data_d_from_list(ol)
101        if len(data) > 0:
102            d['data'] = data
103        cdata = _gen_cdata_d_from_list(ol)
104        if len(cdata) > 0:
105            d['cdata'] = cdata
106        return d
107
108    def write_Array_to_dict(oa):
109        ol = np.ravel(oa)
110        _assert_equal_properties(ol)
111        d = {}
112        d['type'] = 'Array'
113        d['layout'] = str(oa.shape).lstrip('(').rstrip(')').rstrip(',')
114        taglist = [o.tag for o in ol]
115        if np.any([tag is not None for tag in taglist]):
116            d['tag'] = taglist
117        if ol[0].reweighted:
118            d['reweighted'] = ol[0].reweighted
119        d['value'] = [o.value for o in ol]
120        data = _gen_data_d_from_list(ol)
121        if len(data) > 0:
122            d['data'] = data
123        cdata = _gen_cdata_d_from_list(ol)
124        if len(cdata) > 0:
125            d['cdata'] = cdata
126        return d
127
128    def _nan_Obs_like(obs):
129        samples = []
130        names = []
131        idl = []
132        for key, value in obs.idl.items():
133            samples.append([np.nan] * len(value))
134            names.append(key)
135            idl.append(value)
136        my_obs = Obs(samples, names, idl)
137        my_obs._covobs = obs._covobs
138        for name in obs._covobs:
139            my_obs.names.append(name)
140        my_obs.reweighted = obs.reweighted
141        my_obs.is_merged = obs.is_merged
142        return my_obs
143
144    def write_Corr_to_dict(my_corr):
145        first_not_none = next(i for i, j in enumerate(my_corr.content) if np.all(j))
146        dummy_array = np.empty((my_corr.N, my_corr.N), dtype=object)
147        dummy_array[:] = _nan_Obs_like(my_corr.content[first_not_none].ravel()[0])
148        content = [o if o is not None else dummy_array for o in my_corr.content]
149        dat = write_Array_to_dict(np.array(content, dtype=object))
150        dat['type'] = 'Corr'
151        corr_meta_data = str(my_corr.tag)
152        if 'tag' in dat.keys():
153            dat['tag'].append(corr_meta_data)
154        else:
155            dat['tag'] = [corr_meta_data]
156        taglist = dat['tag']
157        dat['tag'] = {}  # tag is now a dictionary, that contains the previous taglist in the key "tag"
158        dat['tag']['tag'] = taglist
159        if my_corr.prange is not None:
160            dat['tag']['prange'] = my_corr.prange
161        return dat
162
163    if not isinstance(ol, list):
164        ol = [ol]
165
166    d = {}
167    d['program'] = 'pyerrors %s' % (pyerrorsversion.__version__)
168    d['version'] = '1.1'
169    d['who'] = getpass.getuser()
170    d['date'] = datetime.datetime.now().astimezone().strftime('%Y-%m-%d %H:%M:%S %z')
171    d['host'] = socket.gethostname() + ', ' + platform.platform()
172
173    if description:
174        d['description'] = description
175
176    d['obsdata'] = []
177    for io in ol:
178        if isinstance(io, Obs):
179            d['obsdata'].append(write_Obs_to_dict(io))
180        elif isinstance(io, list):
181            d['obsdata'].append(write_List_to_dict(io))
182        elif isinstance(io, np.ndarray):
183            d['obsdata'].append(write_Array_to_dict(io))
184        elif isinstance(io, Corr):
185            d['obsdata'].append(write_Corr_to_dict(io))
186        else:
187            raise Exception("Unkown datatype.")
188
189    if indent:
190        return json.dumps(d, indent=indent, ensure_ascii=False, write_mode=json.WM_SINGLE_LINE_ARRAY)
191    else:
192        return json.dumps(d, indent=indent, ensure_ascii=False, write_mode=json.WM_COMPACT)
193
194
195def dump_to_json(ol, fname, description='', indent=1, gz=True):
196    """Export a list of Obs or structures containing Obs to a .json(.gz) file
197
198    Parameters
199    ----------
200    ol : list
201        List of objects that will be exported. At the moment, these objects can be
202        either of: Obs, list, numpy.ndarray, Corr.
203        All Obs inside a structure have to be defined on the same set of configurations.
204    fname : str
205        Filename of the output file.
206    description : str
207        Optional string that describes the contents of the json file.
208    indent : int
209        Specify the indentation level of the json file. None or 0 is permissible and
210        saves disk space.
211    gz : bool
212        If True, the output is a gzipped json. If False, the output is a json file.
213    """
214
215    jsonstring = create_json_string(ol, description, indent)
216
217    if not fname.endswith('.json') and not fname.endswith('.gz'):
218        fname += '.json'
219
220    if gz:
221        if not fname.endswith('.gz'):
222            fname += '.gz'
223
224        fp = gzip.open(fname, 'wb')
225        fp.write(jsonstring.encode('utf-8'))
226    else:
227        fp = open(fname, 'w', encoding='utf-8')
228        fp.write(jsonstring)
229    fp.close()
230
231
232def _parse_json_dict(json_dict, verbose=True, full_output=False):
233    """Reconstruct a list of Obs or structures containing Obs from a dict that
234    was built out of a json string.
235
236    The following structures are supported: Obs, list, numpy.ndarray, Corr
237    If the list contains only one element, it is unpacked from the list.
238
239    Parameters
240    ----------
241    json_string : str
242        json string containing the data.
243    verbose : bool
244        Print additional information that was written to the file.
245    full_output : bool
246        If True, a dict containing auxiliary information and the data is returned.
247        If False, only the data is returned.
248    """
249
250    def _gen_obsd_from_datad(d):
251        retd = {}
252        if d:
253            retd['names'] = []
254            retd['idl'] = []
255            retd['deltas'] = []
256            retd['is_merged'] = {}
257            for ens in d:
258                for rep in ens['replica']:
259                    rep_name = rep['name']
260                    if len(rep_name) > len(ens["id"]):
261                        if rep_name[len(ens["id"])] != "|":
262                            tmp_list = list(rep_name)
263                            tmp_list = tmp_list[:len(ens["id"])] + ["|"] + tmp_list[len(ens["id"]):]
264                            rep_name = ''.join(tmp_list)
265                    retd['names'].append(rep_name)
266                    retd['idl'].append([di[0] for di in rep['deltas']])
267                    retd['deltas'].append(np.array([di[1:] for di in rep['deltas']]))
268                    retd['is_merged'][rep_name] = rep.get('is_merged', False)
269        return retd
270
271    def _gen_covobsd_from_cdatad(d):
272        retd = {}
273        for ens in d:
274            retl = []
275            name = ens['id']
276            layouts = ens.get('layout', '1').strip()
277            layout = [int(ls.strip()) for ls in layouts.split(',') if len(ls) > 0]
278            cov = np.reshape(ens['cov'], layout)
279            grad = ens['grad']
280            nobs = len(grad[0])
281            for i in range(nobs):
282                retl.append({'name': name, 'cov': cov, 'grad': [g[i] for g in grad]})
283            retd[name] = retl
284        return retd
285
286    def get_Obs_from_dict(o):
287        layouts = o.get('layout', '1').strip()
288        if layouts != '1':
289            raise Exception("layout is %s has to be 1 for type Obs." % (layouts), RuntimeWarning)
290
291        values = o['value']
292        od = _gen_obsd_from_datad(o.get('data', {}))
293        cd = _gen_covobsd_from_cdatad(o.get('cdata', {}))
294
295        if od:
296            ret = Obs([[ddi[0] + values[0] for ddi in di] for di in od['deltas']], od['names'], idl=od['idl'])
297            ret.is_merged = od['is_merged']
298        else:
299            ret = Obs([], [], means=[])
300            ret._value = values[0]
301        for name in cd:
302            co = cd[name][0]
303            ret._covobs[name] = Covobs(None, co['cov'], co['name'], grad=co['grad'])
304            ret.names.append(co['name'])
305
306        ret.reweighted = o.get('reweighted', False)
307        ret.tag = o.get('tag', [None])[0]
308        return ret
309
310    def get_List_from_dict(o):
311        layouts = o.get('layout', '1').strip()
312        layout = int(layouts)
313        values = o['value']
314        od = _gen_obsd_from_datad(o.get('data', {}))
315        cd = _gen_covobsd_from_cdatad(o.get('cdata', {}))
316
317        ret = []
318        taglist = o.get('tag', layout * [None])
319        for i in range(layout):
320            if od:
321                ret.append(Obs([list(di[:, i] + values[i]) for di in od['deltas']], od['names'], idl=od['idl']))
322                ret[-1].is_merged = od['is_merged']
323            else:
324                ret.append(Obs([], [], means=[]))
325                ret[-1]._value = values[i]
326                print('Created Obs with means= ', values[i])
327            for name in cd:
328                co = cd[name][i]
329                ret[-1]._covobs[name] = Covobs(None, co['cov'], co['name'], grad=co['grad'])
330                ret[-1].names.append(co['name'])
331
332            ret[-1].reweighted = o.get('reweighted', False)
333            ret[-1].tag = taglist[i]
334        return ret
335
336    def get_Array_from_dict(o):
337        layouts = o.get('layout', '1').strip()
338        layout = [int(ls.strip()) for ls in layouts.split(',') if len(ls) > 0]
339        N = np.prod(layout)
340        values = o['value']
341        od = _gen_obsd_from_datad(o.get('data', {}))
342        cd = _gen_covobsd_from_cdatad(o.get('cdata', {}))
343
344        ret = []
345        taglist = o.get('tag', N * [None])
346        for i in range(N):
347            if od:
348                ret.append(Obs([di[:, i] + values[i] for di in od['deltas']], od['names'], idl=od['idl']))
349                ret[-1].is_merged = od['is_merged']
350            else:
351                ret.append(Obs([], [], means=[]))
352                ret[-1]._value = values[i]
353            for name in cd:
354                co = cd[name][i]
355                ret[-1]._covobs[name] = Covobs(None, co['cov'], co['name'], grad=co['grad'])
356                ret[-1].names.append(co['name'])
357            ret[-1].reweighted = o.get('reweighted', False)
358            ret[-1].tag = taglist[i]
359        return np.reshape(ret, layout)
360
361    def get_Corr_from_dict(o):
362        if isinstance(o.get('tag'), list):  # supports the old way
363            taglist = o.get('tag')  # This had to be modified to get the taglist from the dictionary
364            temp_prange = None
365        elif isinstance(o.get('tag'), dict):
366            tagdic = o.get('tag')
367            taglist = tagdic['tag']
368            if 'prange' in tagdic:
369                temp_prange = tagdic['prange']
370            else:
371                temp_prange = None
372        else:
373            raise Exception("The tag is not a list or dict")
374
375        corr_tag = taglist[-1]
376        tmp_o = o
377        tmp_o['tag'] = taglist[:-1]
378        if len(tmp_o['tag']) == 0:
379            del tmp_o['tag']
380        dat = get_Array_from_dict(tmp_o)
381        my_corr = Corr([None if np.isnan(o.ravel()[0].value) else o for o in list(dat)])
382        if corr_tag != 'None':
383            my_corr.tag = corr_tag
384
385        my_corr.prange = temp_prange
386        return my_corr
387
388    prog = json_dict.get('program', '')
389    version = json_dict.get('version', '')
390    who = json_dict.get('who', '')
391    date = json_dict.get('date', '')
392    host = json_dict.get('host', '')
393    if prog and verbose:
394        print('Data has been written using %s.' % (prog))
395    if version and verbose:
396        print('Format version %s' % (version))
397    if np.any([who, date, host] and verbose):
398        print('Written by %s on %s on host %s' % (who, date, host))
399    description = json_dict.get('description', '')
400    if description and verbose:
401        print()
402        print('Description: ', description)
403    obsdata = json_dict['obsdata']
404    ol = []
405    for io in obsdata:
406        if io['type'] == 'Obs':
407            ol.append(get_Obs_from_dict(io))
408        elif io['type'] == 'List':
409            ol.append(get_List_from_dict(io))
410        elif io['type'] == 'Array':
411            ol.append(get_Array_from_dict(io))
412        elif io['type'] == 'Corr':
413            ol.append(get_Corr_from_dict(io))
414        else:
415            raise Exception("Unkown datatype.")
416
417    if full_output:
418        retd = {}
419        retd['program'] = prog
420        retd['version'] = version
421        retd['who'] = who
422        retd['date'] = date
423        retd['host'] = host
424        retd['description'] = description
425        retd['obsdata'] = ol
426
427        return retd
428    else:
429        if len(obsdata) == 1:
430            ol = ol[0]
431
432        return ol
433
434
435def import_json_string(json_string, verbose=True, full_output=False):
436    """Reconstruct a list of Obs or structures containing Obs from a json string.
437
438    The following structures are supported: Obs, list, numpy.ndarray, Corr
439    If the list contains only one element, it is unpacked from the list.
440
441    Parameters
442    ----------
443    json_string : str
444        json string containing the data.
445    verbose : bool
446        Print additional information that was written to the file.
447    full_output : bool
448        If True, a dict containing auxiliary information and the data is returned.
449        If False, only the data is returned.
450    """
451
452    return _parse_json_dict(json.loads(json_string), verbose, full_output)
453
454
455def load_json(fname, verbose=True, gz=True, full_output=False):
456    """Import a list of Obs or structures containing Obs from a .json(.gz) file.
457
458    The following structures are supported: Obs, list, numpy.ndarray, Corr
459    If the list contains only one element, it is unpacked from the list.
460
461    Parameters
462    ----------
463    fname : str
464        Filename of the input file.
465    verbose : bool
466        Print additional information that was written to the file.
467    gz : bool
468        If True, assumes that data is gzipped. If False, assumes JSON file.
469    full_output : bool
470        If True, a dict containing auxiliary information and the data is returned.
471        If False, only the data is returned.
472    """
473    if not fname.endswith('.json') and not fname.endswith('.gz'):
474        fname += '.json'
475    if gz:
476        if not fname.endswith('.gz'):
477            fname += '.gz'
478        with gzip.open(fname, 'r') as fin:
479            d = json.load(fin)
480    else:
481        if fname.endswith('.gz'):
482            warnings.warn("Trying to read from %s without unzipping!" % fname, UserWarning)
483        with open(fname, 'r', encoding='utf-8') as fin:
484            d = json.loads(fin.read())
485
486    return _parse_json_dict(d, verbose, full_output)
487
488
489def _ol_from_dict(ind, reps='DICTOBS'):
490    """Convert a dictionary of Obs objects to a list and a dictionary that contains
491    placeholders instead of the Obs objects.
492
493    Parameters
494    ----------
495    ind : dict
496        Dict of JSON valid structures and objects that will be exported.
497        At the moment, these object can be either of: Obs, list, numpy.ndarray, Corr.
498        All Obs inside a structure have to be defined on the same set of configurations.
499    reps : str
500        Specify the structure of the placeholder in exported dict to be reps[0-9]+.
501    """
502
503    obstypes = (Obs, Corr, np.ndarray)
504
505    if not reps.isalnum():
506        raise Exception('Placeholder string has to be alphanumeric!')
507    ol = []
508    counter = 0
509
510    def dict_replace_obs(d):
511        nonlocal ol
512        nonlocal counter
513        x = {}
514        for k, v in d.items():
515            if isinstance(v, dict):
516                v = dict_replace_obs(v)
517            elif isinstance(v, list) and all([isinstance(o, Obs) for o in v]):
518                v = obslist_replace_obs(v)
519            elif isinstance(v, list):
520                v = list_replace_obs(v)
521            elif isinstance(v, obstypes):
522                ol.append(v)
523                v = reps + '%d' % (counter)
524                counter += 1
525            elif isinstance(v, str):
526                if bool(re.match(r'%s[0-9]+' % (reps), v)):
527                    raise Exception('Dict contains string %s that matches the placeholder! %s Cannot be savely exported.' % (v, reps))
528            x[k] = v
529        return x
530
531    def list_replace_obs(li):
532        nonlocal ol
533        nonlocal counter
534        x = []
535        for e in li:
536            if isinstance(e, list):
537                e = list_replace_obs(e)
538            elif isinstance(e, list) and all([isinstance(o, Obs) for o in e]):
539                e = obslist_replace_obs(e)
540            elif isinstance(e, dict):
541                e = dict_replace_obs(e)
542            elif isinstance(e, obstypes):
543                ol.append(e)
544                e = reps + '%d' % (counter)
545                counter += 1
546            elif isinstance(e, str):
547                if bool(re.match(r'%s[0-9]+' % (reps), e)):
548                    raise Exception('Dict contains string %s that matches the placeholder! %s Cannot be savely exported.' % (e, reps))
549            x.append(e)
550        return x
551
552    def obslist_replace_obs(li):
553        nonlocal ol
554        nonlocal counter
555        il = []
556        for e in li:
557            il.append(e)
558
559        ol.append(il)
560        x = reps + '%d' % (counter)
561        counter += 1
562        return x
563
564    nd = dict_replace_obs(ind)
565
566    return ol, nd
567
568
569def dump_dict_to_json(od, fname, description='', indent=1, reps='DICTOBS', gz=True):
570    """Export a dict of Obs or structures containing Obs to a .json(.gz) file
571
572    Parameters
573    ----------
574    od : dict
575        Dict of JSON valid structures and objects that will be exported.
576        At the moment, these objects can be either of: Obs, list, numpy.ndarray, Corr.
577        All Obs inside a structure have to be defined on the same set of configurations.
578    fname : str
579        Filename of the output file.
580    description : str
581        Optional string that describes the contents of the json file.
582    indent : int
583        Specify the indentation level of the json file. None or 0 is permissible and
584        saves disk space.
585    reps : str
586        Specify the structure of the placeholder in exported dict to be reps[0-9]+.
587    gz : bool
588        If True, the output is a gzipped json. If False, the output is a json file.
589    """
590
591    if not isinstance(od, dict):
592        raise Exception('od has to be a dictionary. Did you want to use dump_to_json?')
593
594    infostring = ('This JSON file contains a python dictionary that has been parsed to a list of structures. '
595                  'OBSDICT contains the dictionary, where Obs or other structures have been replaced by '
596                  '' + reps + '[0-9]+. The field description contains the additional description of this JSON file. '
597                  'This file may be parsed to a dict with the pyerrors routine load_json_dict.')
598
599    desc_dict = {'INFO': infostring, 'OBSDICT': {}, 'description': description}
600    ol, desc_dict['OBSDICT'] = _ol_from_dict(od, reps=reps)
601
602    dump_to_json(ol, fname, description=desc_dict, indent=indent, gz=gz)
603
604
605def _od_from_list_and_dict(ol, ind, reps='DICTOBS'):
606    """Parse a list of Obs or structures containing Obs and an accompanying
607    dict, where the structures have been replaced by placeholders to a
608    dict that contains the structures.
609
610    The following structures are supported: Obs, list, numpy.ndarray, Corr
611
612    Parameters
613    ----------
614    ol : list
615        List of objects -
616        At the moment, these objects can be either of: Obs, list, numpy.ndarray, Corr.
617        All Obs inside a structure have to be defined on the same set of configurations.
618    ind : dict
619        Dict that defines the structure of the resulting dict and contains placeholders
620    reps : str
621        Specify the structure of the placeholder in imported dict to be reps[0-9]+.
622    """
623    if not reps.isalnum():
624        raise Exception('Placeholder string has to be alphanumeric!')
625
626    counter = 0
627
628    def dict_replace_string(d):
629        nonlocal counter
630        nonlocal ol
631        x = {}
632        for k, v in d.items():
633            if isinstance(v, dict):
634                v = dict_replace_string(v)
635            elif isinstance(v, list):
636                v = list_replace_string(v)
637            elif isinstance(v, str) and bool(re.match(r'%s[0-9]+' % (reps), v)):
638                index = int(v[len(reps):])
639                v = ol[index]
640                counter += 1
641            x[k] = v
642        return x
643
644    def list_replace_string(li):
645        nonlocal counter
646        nonlocal ol
647        x = []
648        for e in li:
649            if isinstance(e, list):
650                e = list_replace_string(e)
651            elif isinstance(e, dict):
652                e = dict_replace_string(e)
653            elif isinstance(e, str) and bool(re.match(r'%s[0-9]+' % (reps), e)):
654                index = int(e[len(reps):])
655                e = ol[index]
656                counter += 1
657            x.append(e)
658        return x
659
660    nd = dict_replace_string(ind)
661
662    if counter == 0:
663        raise Exception('No placeholder has been replaced! Check if reps is set correctly.')
664
665    return nd
666
667
668def load_json_dict(fname, verbose=True, gz=True, full_output=False, reps='DICTOBS'):
669    """Import a dict of Obs or structures containing Obs from a .json(.gz) file.
670
671    The following structures are supported: Obs, list, numpy.ndarray, Corr
672
673    Parameters
674    ----------
675    fname : str
676        Filename of the input file.
677    verbose : bool
678        Print additional information that was written to the file.
679    gz : bool
680        If True, assumes that data is gzipped. If False, assumes JSON file.
681    full_output : bool
682        If True, a dict containing auxiliary information and the data is returned.
683        If False, only the data is returned.
684    reps : str
685        Specify the structure of the placeholder in imported dict to be reps[0-9]+.
686    """
687    indata = load_json(fname, verbose=verbose, gz=gz, full_output=True)
688    description = indata['description']['description']
689    indict = indata['description']['OBSDICT']
690    ol = indata['obsdata']
691    od = _od_from_list_and_dict(ol, indict, reps=reps)
692
693    if full_output:
694        indata['description'] = description
695        indata['obsdata'] = od
696        return indata
697    else:
698        return od
def create_json_string(ol, description='', indent=1)
 18def create_json_string(ol, description='', indent=1):
 19    """Generate the string for the export of a list of Obs or structures containing Obs
 20    to a .json(.gz) file
 21
 22    Parameters
 23    ----------
 24    ol : list
 25        List of objects that will be exported. At the moment, these objects can be
 26        either of: Obs, list, numpy.ndarray, Corr.
 27        All Obs inside a structure have to be defined on the same set of configurations.
 28    description : str
 29        Optional string that describes the contents of the json file.
 30    indent : int
 31        Specify the indentation level of the json file. None or 0 is permissible and
 32        saves disk space.
 33    """
 34
 35    def _gen_data_d_from_list(ol):
 36        dl = []
 37        No = len(ol)
 38        for name in ol[0].mc_names:
 39            ed = {}
 40            ed['id'] = name
 41            ed['replica'] = []
 42            for r_name in ol[0].e_content[name]:
 43                rd = {}
 44                rd['name'] = r_name
 45                if ol[0].is_merged.get(r_name, False):
 46                    rd['is_merged'] = True
 47                rd['deltas'] = []
 48                offsets = [o.r_values[r_name] - o.value for o in ol]
 49                deltas = np.column_stack([ol[oi].deltas[r_name] + offsets[oi] for oi in range(No)])
 50                for i in range(len(ol[0].idl[r_name])):
 51                    rd['deltas'].append([ol[0].idl[r_name][i]])
 52                    rd['deltas'][-1] += deltas[i].tolist()
 53                ed['replica'].append(rd)
 54            dl.append(ed)
 55        return dl
 56
 57    def _gen_cdata_d_from_list(ol):
 58        dl = []
 59        for name in ol[0].cov_names:
 60            ed = {}
 61            ed['id'] = name
 62            ed['layout'] = str(ol[0].covobs[name].cov.shape).lstrip('(').rstrip(')').rstrip(',')
 63            ed['cov'] = list(np.ravel(ol[0].covobs[name].cov))
 64            ncov = ol[0].covobs[name].cov.shape[0]
 65            ed['grad'] = []
 66            for i in range(ncov):
 67                ed['grad'].append([])
 68                for o in ol:
 69                    ed['grad'][-1].append(o.covobs[name].grad[i][0])
 70            dl.append(ed)
 71        return dl
 72
 73    def write_Obs_to_dict(o):
 74        d = {}
 75        d['type'] = 'Obs'
 76        d['layout'] = '1'
 77        if o.tag:
 78            d['tag'] = [o.tag]
 79        if o.reweighted:
 80            d['reweighted'] = o.reweighted
 81        d['value'] = [o.value]
 82        data = _gen_data_d_from_list([o])
 83        if len(data) > 0:
 84            d['data'] = data
 85        cdata = _gen_cdata_d_from_list([o])
 86        if len(cdata) > 0:
 87            d['cdata'] = cdata
 88        return d
 89
 90    def write_List_to_dict(ol):
 91        _assert_equal_properties(ol)
 92        d = {}
 93        d['type'] = 'List'
 94        d['layout'] = '%d' % len(ol)
 95        taglist = [o.tag for o in ol]
 96        if np.any([tag is not None for tag in taglist]):
 97            d['tag'] = taglist
 98        if ol[0].reweighted:
 99            d['reweighted'] = ol[0].reweighted
100        d['value'] = [o.value for o in ol]
101        data = _gen_data_d_from_list(ol)
102        if len(data) > 0:
103            d['data'] = data
104        cdata = _gen_cdata_d_from_list(ol)
105        if len(cdata) > 0:
106            d['cdata'] = cdata
107        return d
108
109    def write_Array_to_dict(oa):
110        ol = np.ravel(oa)
111        _assert_equal_properties(ol)
112        d = {}
113        d['type'] = 'Array'
114        d['layout'] = str(oa.shape).lstrip('(').rstrip(')').rstrip(',')
115        taglist = [o.tag for o in ol]
116        if np.any([tag is not None for tag in taglist]):
117            d['tag'] = taglist
118        if ol[0].reweighted:
119            d['reweighted'] = ol[0].reweighted
120        d['value'] = [o.value for o in ol]
121        data = _gen_data_d_from_list(ol)
122        if len(data) > 0:
123            d['data'] = data
124        cdata = _gen_cdata_d_from_list(ol)
125        if len(cdata) > 0:
126            d['cdata'] = cdata
127        return d
128
129    def _nan_Obs_like(obs):
130        samples = []
131        names = []
132        idl = []
133        for key, value in obs.idl.items():
134            samples.append([np.nan] * len(value))
135            names.append(key)
136            idl.append(value)
137        my_obs = Obs(samples, names, idl)
138        my_obs._covobs = obs._covobs
139        for name in obs._covobs:
140            my_obs.names.append(name)
141        my_obs.reweighted = obs.reweighted
142        my_obs.is_merged = obs.is_merged
143        return my_obs
144
145    def write_Corr_to_dict(my_corr):
146        first_not_none = next(i for i, j in enumerate(my_corr.content) if np.all(j))
147        dummy_array = np.empty((my_corr.N, my_corr.N), dtype=object)
148        dummy_array[:] = _nan_Obs_like(my_corr.content[first_not_none].ravel()[0])
149        content = [o if o is not None else dummy_array for o in my_corr.content]
150        dat = write_Array_to_dict(np.array(content, dtype=object))
151        dat['type'] = 'Corr'
152        corr_meta_data = str(my_corr.tag)
153        if 'tag' in dat.keys():
154            dat['tag'].append(corr_meta_data)
155        else:
156            dat['tag'] = [corr_meta_data]
157        taglist = dat['tag']
158        dat['tag'] = {}  # tag is now a dictionary, that contains the previous taglist in the key "tag"
159        dat['tag']['tag'] = taglist
160        if my_corr.prange is not None:
161            dat['tag']['prange'] = my_corr.prange
162        return dat
163
164    if not isinstance(ol, list):
165        ol = [ol]
166
167    d = {}
168    d['program'] = 'pyerrors %s' % (pyerrorsversion.__version__)
169    d['version'] = '1.1'
170    d['who'] = getpass.getuser()
171    d['date'] = datetime.datetime.now().astimezone().strftime('%Y-%m-%d %H:%M:%S %z')
172    d['host'] = socket.gethostname() + ', ' + platform.platform()
173
174    if description:
175        d['description'] = description
176
177    d['obsdata'] = []
178    for io in ol:
179        if isinstance(io, Obs):
180            d['obsdata'].append(write_Obs_to_dict(io))
181        elif isinstance(io, list):
182            d['obsdata'].append(write_List_to_dict(io))
183        elif isinstance(io, np.ndarray):
184            d['obsdata'].append(write_Array_to_dict(io))
185        elif isinstance(io, Corr):
186            d['obsdata'].append(write_Corr_to_dict(io))
187        else:
188            raise Exception("Unkown datatype.")
189
190    if indent:
191        return json.dumps(d, indent=indent, ensure_ascii=False, write_mode=json.WM_SINGLE_LINE_ARRAY)
192    else:
193        return json.dumps(d, indent=indent, ensure_ascii=False, write_mode=json.WM_COMPACT)

Generate the string for the export of a list of Obs or structures containing Obs to a .json(.gz) file

Parameters
  • ol (list): List of objects that will be exported. At the moment, these objects can be either of: Obs, list, numpy.ndarray, Corr. All Obs inside a structure have to be defined on the same set of configurations.
  • description (str): Optional string that describes the contents of the json file.
  • indent (int): Specify the indentation level of the json file. None or 0 is permissible and saves disk space.
def dump_to_json(ol, fname, description='', indent=1, gz=True)
196def dump_to_json(ol, fname, description='', indent=1, gz=True):
197    """Export a list of Obs or structures containing Obs to a .json(.gz) file
198
199    Parameters
200    ----------
201    ol : list
202        List of objects that will be exported. At the moment, these objects can be
203        either of: Obs, list, numpy.ndarray, Corr.
204        All Obs inside a structure have to be defined on the same set of configurations.
205    fname : str
206        Filename of the output file.
207    description : str
208        Optional string that describes the contents of the json file.
209    indent : int
210        Specify the indentation level of the json file. None or 0 is permissible and
211        saves disk space.
212    gz : bool
213        If True, the output is a gzipped json. If False, the output is a json file.
214    """
215
216    jsonstring = create_json_string(ol, description, indent)
217
218    if not fname.endswith('.json') and not fname.endswith('.gz'):
219        fname += '.json'
220
221    if gz:
222        if not fname.endswith('.gz'):
223            fname += '.gz'
224
225        fp = gzip.open(fname, 'wb')
226        fp.write(jsonstring.encode('utf-8'))
227    else:
228        fp = open(fname, 'w', encoding='utf-8')
229        fp.write(jsonstring)
230    fp.close()

Export a list of Obs or structures containing Obs to a .json(.gz) file

Parameters
  • ol (list): List of objects that will be exported. At the moment, these objects can be either of: Obs, list, numpy.ndarray, Corr. All Obs inside a structure have to be defined on the same set of configurations.
  • fname (str): Filename of the output file.
  • description (str): Optional string that describes the contents of the json file.
  • indent (int): Specify the indentation level of the json file. None or 0 is permissible and saves disk space.
  • gz (bool): If True, the output is a gzipped json. If False, the output is a json file.
def import_json_string(json_string, verbose=True, full_output=False)
436def import_json_string(json_string, verbose=True, full_output=False):
437    """Reconstruct a list of Obs or structures containing Obs from a json string.
438
439    The following structures are supported: Obs, list, numpy.ndarray, Corr
440    If the list contains only one element, it is unpacked from the list.
441
442    Parameters
443    ----------
444    json_string : str
445        json string containing the data.
446    verbose : bool
447        Print additional information that was written to the file.
448    full_output : bool
449        If True, a dict containing auxiliary information and the data is returned.
450        If False, only the data is returned.
451    """
452
453    return _parse_json_dict(json.loads(json_string), verbose, full_output)

Reconstruct a list of Obs or structures containing Obs from a json string.

The following structures are supported: Obs, list, numpy.ndarray, Corr If the list contains only one element, it is unpacked from the list.

Parameters
  • json_string (str): json string containing the data.
  • verbose (bool): Print additional information that was written to the file.
  • full_output (bool): If True, a dict containing auxiliary information and the data is returned. If False, only the data is returned.
def load_json(fname, verbose=True, gz=True, full_output=False)
456def load_json(fname, verbose=True, gz=True, full_output=False):
457    """Import a list of Obs or structures containing Obs from a .json(.gz) file.
458
459    The following structures are supported: Obs, list, numpy.ndarray, Corr
460    If the list contains only one element, it is unpacked from the list.
461
462    Parameters
463    ----------
464    fname : str
465        Filename of the input file.
466    verbose : bool
467        Print additional information that was written to the file.
468    gz : bool
469        If True, assumes that data is gzipped. If False, assumes JSON file.
470    full_output : bool
471        If True, a dict containing auxiliary information and the data is returned.
472        If False, only the data is returned.
473    """
474    if not fname.endswith('.json') and not fname.endswith('.gz'):
475        fname += '.json'
476    if gz:
477        if not fname.endswith('.gz'):
478            fname += '.gz'
479        with gzip.open(fname, 'r') as fin:
480            d = json.load(fin)
481    else:
482        if fname.endswith('.gz'):
483            warnings.warn("Trying to read from %s without unzipping!" % fname, UserWarning)
484        with open(fname, 'r', encoding='utf-8') as fin:
485            d = json.loads(fin.read())
486
487    return _parse_json_dict(d, verbose, full_output)

Import a list of Obs or structures containing Obs from a .json(.gz) file.

The following structures are supported: Obs, list, numpy.ndarray, Corr If the list contains only one element, it is unpacked from the list.

Parameters
  • fname (str): Filename of the input file.
  • verbose (bool): Print additional information that was written to the file.
  • gz (bool): If True, assumes that data is gzipped. If False, assumes JSON file.
  • full_output (bool): If True, a dict containing auxiliary information and the data is returned. If False, only the data is returned.
def dump_dict_to_json(od, fname, description='', indent=1, reps='DICTOBS', gz=True)
570def dump_dict_to_json(od, fname, description='', indent=1, reps='DICTOBS', gz=True):
571    """Export a dict of Obs or structures containing Obs to a .json(.gz) file
572
573    Parameters
574    ----------
575    od : dict
576        Dict of JSON valid structures and objects that will be exported.
577        At the moment, these objects can be either of: Obs, list, numpy.ndarray, Corr.
578        All Obs inside a structure have to be defined on the same set of configurations.
579    fname : str
580        Filename of the output file.
581    description : str
582        Optional string that describes the contents of the json file.
583    indent : int
584        Specify the indentation level of the json file. None or 0 is permissible and
585        saves disk space.
586    reps : str
587        Specify the structure of the placeholder in exported dict to be reps[0-9]+.
588    gz : bool
589        If True, the output is a gzipped json. If False, the output is a json file.
590    """
591
592    if not isinstance(od, dict):
593        raise Exception('od has to be a dictionary. Did you want to use dump_to_json?')
594
595    infostring = ('This JSON file contains a python dictionary that has been parsed to a list of structures. '
596                  'OBSDICT contains the dictionary, where Obs or other structures have been replaced by '
597                  '' + reps + '[0-9]+. The field description contains the additional description of this JSON file. '
598                  'This file may be parsed to a dict with the pyerrors routine load_json_dict.')
599
600    desc_dict = {'INFO': infostring, 'OBSDICT': {}, 'description': description}
601    ol, desc_dict['OBSDICT'] = _ol_from_dict(od, reps=reps)
602
603    dump_to_json(ol, fname, description=desc_dict, indent=indent, gz=gz)

Export a dict of Obs or structures containing Obs to a .json(.gz) file

Parameters
  • od (dict): Dict of JSON valid structures and objects that will be exported. At the moment, these objects can be either of: Obs, list, numpy.ndarray, Corr. All Obs inside a structure have to be defined on the same set of configurations.
  • fname (str): Filename of the output file.
  • description (str): Optional string that describes the contents of the json file.
  • indent (int): Specify the indentation level of the json file. None or 0 is permissible and saves disk space.
  • reps (str): Specify the structure of the placeholder in exported dict to be reps[0-9]+.
  • gz (bool): If True, the output is a gzipped json. If False, the output is a json file.
def load_json_dict(fname, verbose=True, gz=True, full_output=False, reps='DICTOBS')
669def load_json_dict(fname, verbose=True, gz=True, full_output=False, reps='DICTOBS'):
670    """Import a dict of Obs or structures containing Obs from a .json(.gz) file.
671
672    The following structures are supported: Obs, list, numpy.ndarray, Corr
673
674    Parameters
675    ----------
676    fname : str
677        Filename of the input file.
678    verbose : bool
679        Print additional information that was written to the file.
680    gz : bool
681        If True, assumes that data is gzipped. If False, assumes JSON file.
682    full_output : bool
683        If True, a dict containing auxiliary information and the data is returned.
684        If False, only the data is returned.
685    reps : str
686        Specify the structure of the placeholder in imported dict to be reps[0-9]+.
687    """
688    indata = load_json(fname, verbose=verbose, gz=gz, full_output=True)
689    description = indata['description']['description']
690    indict = indata['description']['OBSDICT']
691    ol = indata['obsdata']
692    od = _od_from_list_and_dict(ol, indict, reps=reps)
693
694    if full_output:
695        indata['description'] = description
696        indata['obsdata'] = od
697        return indata
698    else:
699        return od

Import a dict of Obs or structures containing Obs from a .json(.gz) file.

The following structures are supported: Obs, list, numpy.ndarray, Corr

Parameters
  • fname (str): Filename of the input file.
  • verbose (bool): Print additional information that was written to the file.
  • gz (bool): If True, assumes that data is gzipped. If False, assumes JSON file.
  • full_output (bool): If True, a dict containing auxiliary information and the data is returned. If False, only the data is returned.
  • reps (str): Specify the structure of the placeholder in imported dict to be reps[0-9]+.