pyerrors.input.json

  1import rapidjson as json
  2import gzip
  3import getpass
  4import socket
  5import datetime
  6import platform
  7import warnings
  8import re
  9import numpy as np
 10from ..obs import Obs
 11from ..covobs import Covobs
 12from ..correlators import Corr
 13from ..misc import _assert_equal_properties
 14from .. import version as pyerrorsversion
 15
 16
 17def create_json_string(ol, description='', indent=1):
 18    """Generate the string for the export of a list of Obs or structures containing Obs
 19    to a .json(.gz) file
 20
 21    Parameters
 22    ----------
 23    ol : list
 24        List of objects that will be exported. At the moment, these objects can be
 25        either of: Obs, list, numpy.ndarray, Corr.
 26        All Obs inside a structure have to be defined on the same set of configurations.
 27    description : str
 28        Optional string that describes the contents of the json file.
 29    indent : int
 30        Specify the indentation level of the json file. None or 0 is permissible and
 31        saves disk space.
 32    """
 33
 34    def _gen_data_d_from_list(ol):
 35        dl = []
 36        No = len(ol)
 37        for name in ol[0].mc_names:
 38            ed = {}
 39            ed['id'] = name
 40            ed['replica'] = []
 41            for r_name in ol[0].e_content[name]:
 42                rd = {}
 43                rd['name'] = r_name
 44                rd['deltas'] = []
 45                offsets = [o.r_values[r_name] - o.value for o in ol]
 46                deltas = np.column_stack([ol[oi].deltas[r_name] + offsets[oi] for oi in range(No)])
 47                for i in range(len(ol[0].idl[r_name])):
 48                    rd['deltas'].append([ol[0].idl[r_name][i]])
 49                    rd['deltas'][-1] += deltas[i].tolist()
 50                ed['replica'].append(rd)
 51            dl.append(ed)
 52        return dl
 53
 54    def _gen_cdata_d_from_list(ol):
 55        dl = []
 56        for name in ol[0].cov_names:
 57            ed = {}
 58            ed['id'] = name
 59            ed['layout'] = str(ol[0].covobs[name].cov.shape).lstrip('(').rstrip(')').rstrip(',')
 60            ed['cov'] = list(np.ravel(ol[0].covobs[name].cov))
 61            ncov = ol[0].covobs[name].cov.shape[0]
 62            ed['grad'] = []
 63            for i in range(ncov):
 64                ed['grad'].append([])
 65                for o in ol:
 66                    ed['grad'][-1].append(o.covobs[name].grad[i][0])
 67            dl.append(ed)
 68        return dl
 69
 70    def write_Obs_to_dict(o):
 71        d = {}
 72        d['type'] = 'Obs'
 73        d['layout'] = '1'
 74        if o.tag:
 75            d['tag'] = [o.tag]
 76        if o.reweighted:
 77            d['reweighted'] = o.reweighted
 78        d['value'] = [o.value]
 79        data = _gen_data_d_from_list([o])
 80        if len(data) > 0:
 81            d['data'] = data
 82        cdata = _gen_cdata_d_from_list([o])
 83        if len(cdata) > 0:
 84            d['cdata'] = cdata
 85        return d
 86
 87    def write_List_to_dict(ol):
 88        _assert_equal_properties(ol)
 89        d = {}
 90        d['type'] = 'List'
 91        d['layout'] = '%d' % len(ol)
 92        taglist = [o.tag for o in ol]
 93        if np.any([tag is not None for tag in taglist]):
 94            d['tag'] = taglist
 95        if ol[0].reweighted:
 96            d['reweighted'] = ol[0].reweighted
 97        d['value'] = [o.value for o in ol]
 98        data = _gen_data_d_from_list(ol)
 99        if len(data) > 0:
100            d['data'] = data
101        cdata = _gen_cdata_d_from_list(ol)
102        if len(cdata) > 0:
103            d['cdata'] = cdata
104        return d
105
106    def write_Array_to_dict(oa):
107        ol = np.ravel(oa)
108        _assert_equal_properties(ol)
109        d = {}
110        d['type'] = 'Array'
111        d['layout'] = str(oa.shape).lstrip('(').rstrip(')').rstrip(',')
112        taglist = [o.tag for o in ol]
113        if np.any([tag is not None for tag in taglist]):
114            d['tag'] = taglist
115        if ol[0].reweighted:
116            d['reweighted'] = ol[0].reweighted
117        d['value'] = [o.value for o in ol]
118        data = _gen_data_d_from_list(ol)
119        if len(data) > 0:
120            d['data'] = data
121        cdata = _gen_cdata_d_from_list(ol)
122        if len(cdata) > 0:
123            d['cdata'] = cdata
124        return d
125
126    def _nan_Obs_like(obs):
127        samples = []
128        names = []
129        idl = []
130        for key, value in obs.idl.items():
131            samples.append([np.nan] * len(value))
132            names.append(key)
133            idl.append(value)
134        my_obs = Obs(samples, names, idl)
135        my_obs._covobs = obs._covobs
136        for name in obs._covobs:
137            my_obs.names.append(name)
138        my_obs.reweighted = obs.reweighted
139        return my_obs
140
141    def write_Corr_to_dict(my_corr):
142        first_not_none = next(i for i, j in enumerate(my_corr.content) if np.all(j))
143        dummy_array = np.empty((my_corr.N, my_corr.N), dtype=object)
144        dummy_array[:] = _nan_Obs_like(my_corr.content[first_not_none].ravel()[0])
145        content = [o if o is not None else dummy_array for o in my_corr.content]
146        dat = write_Array_to_dict(np.array(content, dtype=object))
147        dat['type'] = 'Corr'
148        corr_meta_data = str(my_corr.tag)
149        if 'tag' in dat.keys():
150            dat['tag'].append(corr_meta_data)
151        else:
152            dat['tag'] = [corr_meta_data]
153        taglist = dat['tag']
154        dat['tag'] = {}  # tag is now a dictionary, that contains the previous taglist in the key "tag"
155        dat['tag']['tag'] = taglist
156        if my_corr.prange is not None:
157            dat['tag']['prange'] = my_corr.prange
158        return dat
159
160    if not isinstance(ol, list):
161        ol = [ol]
162
163    d = {}
164    d['program'] = 'pyerrors %s' % (pyerrorsversion.__version__)
165    d['version'] = '1.1'
166    d['who'] = getpass.getuser()
167    d['date'] = datetime.datetime.now().astimezone().strftime('%Y-%m-%d %H:%M:%S %z')
168    d['host'] = socket.gethostname() + ', ' + platform.platform()
169
170    if description:
171        d['description'] = description
172
173    d['obsdata'] = []
174    for io in ol:
175        if isinstance(io, Obs):
176            d['obsdata'].append(write_Obs_to_dict(io))
177        elif isinstance(io, list):
178            d['obsdata'].append(write_List_to_dict(io))
179        elif isinstance(io, np.ndarray):
180            d['obsdata'].append(write_Array_to_dict(io))
181        elif isinstance(io, Corr):
182            d['obsdata'].append(write_Corr_to_dict(io))
183        else:
184            raise Exception("Unkown datatype.")
185
186    def _jsonifier(o):
187        if isinstance(o, np.int64):
188            return int(o)
189        raise TypeError('%r is not JSON serializable' % o)
190
191    if indent:
192        return json.dumps(d, indent=indent, ensure_ascii=False, default=_jsonifier, write_mode=json.WM_SINGLE_LINE_ARRAY)
193    else:
194        return json.dumps(d, indent=indent, ensure_ascii=False, default=_jsonifier, write_mode=json.WM_COMPACT)
195
196
197def dump_to_json(ol, fname, description='', indent=1, gz=True):
198    """Export a list of Obs or structures containing Obs to a .json(.gz) file
199
200    Parameters
201    ----------
202    ol : list
203        List of objects that will be exported. At the moment, these objects can be
204        either of: Obs, list, numpy.ndarray, Corr.
205        All Obs inside a structure have to be defined on the same set of configurations.
206    fname : str
207        Filename of the output file.
208    description : str
209        Optional string that describes the contents of the json file.
210    indent : int
211        Specify the indentation level of the json file. None or 0 is permissible and
212        saves disk space.
213    gz : bool
214        If True, the output is a gzipped json. If False, the output is a json file.
215    """
216
217    jsonstring = create_json_string(ol, description, indent)
218
219    if not fname.endswith('.json') and not fname.endswith('.gz'):
220        fname += '.json'
221
222    if gz:
223        if not fname.endswith('.gz'):
224            fname += '.gz'
225
226        fp = gzip.open(fname, 'wb')
227        fp.write(jsonstring.encode('utf-8'))
228    else:
229        fp = open(fname, 'w', encoding='utf-8')
230        fp.write(jsonstring)
231    fp.close()
232
233
234def _parse_json_dict(json_dict, verbose=True, full_output=False):
235    """Reconstruct a list of Obs or structures containing Obs from a dict that
236    was built out of a json string.
237
238    The following structures are supported: Obs, list, numpy.ndarray, Corr
239    If the list contains only one element, it is unpacked from the list.
240
241    Parameters
242    ----------
243    json_string : str
244        json string containing the data.
245    verbose : bool
246        Print additional information that was written to the file.
247    full_output : bool
248        If True, a dict containing auxiliary information and the data is returned.
249        If False, only the data is returned.
250    """
251
252    def _gen_obsd_from_datad(d):
253        retd = {}
254        if d:
255            retd['names'] = []
256            retd['idl'] = []
257            retd['deltas'] = []
258            for ens in d:
259                for rep in ens['replica']:
260                    rep_name = rep['name']
261                    if len(rep_name) > len(ens["id"]):
262                        if rep_name[len(ens["id"])] != "|":
263                            tmp_list = list(rep_name)
264                            tmp_list = tmp_list[:len(ens["id"])] + ["|"] + tmp_list[len(ens["id"]):]
265                            rep_name = ''.join(tmp_list)
266                    retd['names'].append(rep_name)
267                    retd['idl'].append([di[0] for di in rep['deltas']])
268                    retd['deltas'].append(np.array([di[1:] for di in rep['deltas']]))
269        return retd
270
271    def _gen_covobsd_from_cdatad(d):
272        retd = {}
273        for ens in d:
274            retl = []
275            name = ens['id']
276            layouts = ens.get('layout', '1').strip()
277            layout = [int(ls.strip()) for ls in layouts.split(',') if len(ls) > 0]
278            cov = np.reshape(ens['cov'], layout)
279            grad = ens['grad']
280            nobs = len(grad[0])
281            for i in range(nobs):
282                retl.append({'name': name, 'cov': cov, 'grad': [g[i] for g in grad]})
283            retd[name] = retl
284        return retd
285
286    def get_Obs_from_dict(o):
287        layouts = o.get('layout', '1').strip()
288        if layouts != '1':
289            raise Exception("layout is %s has to be 1 for type Obs." % (layouts), RuntimeWarning)
290
291        values = o['value']
292        od = _gen_obsd_from_datad(o.get('data', {}))
293        cd = _gen_covobsd_from_cdatad(o.get('cdata', {}))
294
295        if od:
296            ret = Obs([[ddi[0] + values[0] for ddi in di] for di in od['deltas']], od['names'], idl=od['idl'])
297            ret._value = values[0]
298        else:
299            ret = Obs([], [], means=[])
300            ret._value = values[0]
301        for name in cd:
302            co = cd[name][0]
303            ret._covobs[name] = Covobs(None, co['cov'], co['name'], grad=co['grad'])
304            ret.names.append(co['name'])
305
306        ret.reweighted = o.get('reweighted', False)
307        ret.tag = o.get('tag', [None])[0]
308        return ret
309
310    def get_List_from_dict(o):
311        layouts = o.get('layout', '1').strip()
312        layout = int(layouts)
313        values = o['value']
314        od = _gen_obsd_from_datad(o.get('data', {}))
315        cd = _gen_covobsd_from_cdatad(o.get('cdata', {}))
316
317        ret = []
318        taglist = o.get('tag', layout * [None])
319        for i in range(layout):
320            if od:
321                ret.append(Obs([list(di[:, i] + values[i]) for di in od['deltas']], od['names'], idl=od['idl']))
322                ret[-1]._value = values[i]
323            else:
324                ret.append(Obs([], [], means=[]))
325                ret[-1]._value = values[i]
326                print('Created Obs with means= ', values[i])
327            for name in cd:
328                co = cd[name][i]
329                ret[-1]._covobs[name] = Covobs(None, co['cov'], co['name'], grad=co['grad'])
330                ret[-1].names.append(co['name'])
331
332            ret[-1].reweighted = o.get('reweighted', False)
333            ret[-1].tag = taglist[i]
334        return ret
335
336    def get_Array_from_dict(o):
337        layouts = o.get('layout', '1').strip()
338        layout = [int(ls.strip()) for ls in layouts.split(',') if len(ls) > 0]
339        N = np.prod(layout)
340        values = o['value']
341        od = _gen_obsd_from_datad(o.get('data', {}))
342        cd = _gen_covobsd_from_cdatad(o.get('cdata', {}))
343
344        ret = []
345        taglist = o.get('tag', N * [None])
346        for i in range(N):
347            if od:
348                ret.append(Obs([di[:, i] + values[i] for di in od['deltas']], od['names'], idl=od['idl']))
349                ret[-1]._value = values[i]
350            else:
351                ret.append(Obs([], [], means=[]))
352                ret[-1]._value = values[i]
353            for name in cd:
354                co = cd[name][i]
355                ret[-1]._covobs[name] = Covobs(None, co['cov'], co['name'], grad=co['grad'])
356                ret[-1].names.append(co['name'])
357            ret[-1].reweighted = o.get('reweighted', False)
358            ret[-1].tag = taglist[i]
359        return np.reshape(ret, layout)
360
361    def get_Corr_from_dict(o):
362        if isinstance(o.get('tag'), list):  # supports the old way
363            taglist = o.get('tag')  # This had to be modified to get the taglist from the dictionary
364            temp_prange = None
365        elif isinstance(o.get('tag'), dict):
366            tagdic = o.get('tag')
367            taglist = tagdic['tag']
368            if 'prange' in tagdic:
369                temp_prange = tagdic['prange']
370            else:
371                temp_prange = None
372        else:
373            raise Exception("The tag is not a list or dict")
374
375        corr_tag = taglist[-1]
376        tmp_o = o
377        tmp_o['tag'] = taglist[:-1]
378        if len(tmp_o['tag']) == 0:
379            del tmp_o['tag']
380        dat = get_Array_from_dict(tmp_o)
381        my_corr = Corr([None if np.isnan(o.ravel()[0].value) else o for o in list(dat)])
382        if corr_tag != 'None':
383            my_corr.tag = corr_tag
384
385        my_corr.prange = temp_prange
386        return my_corr
387
388    prog = json_dict.get('program', '')
389    version = json_dict.get('version', '')
390    who = json_dict.get('who', '')
391    date = json_dict.get('date', '')
392    host = json_dict.get('host', '')
393    if prog and verbose:
394        print('Data has been written using %s.' % (prog))
395    if version and verbose:
396        print('Format version %s' % (version))
397    if np.any([who, date, host] and verbose):
398        print('Written by %s on %s on host %s' % (who, date, host))
399    description = json_dict.get('description', '')
400    if description and verbose:
401        print()
402        print('Description: ', description)
403    obsdata = json_dict['obsdata']
404    ol = []
405    for io in obsdata:
406        if io['type'] == 'Obs':
407            ol.append(get_Obs_from_dict(io))
408        elif io['type'] == 'List':
409            ol.append(get_List_from_dict(io))
410        elif io['type'] == 'Array':
411            ol.append(get_Array_from_dict(io))
412        elif io['type'] == 'Corr':
413            ol.append(get_Corr_from_dict(io))
414        else:
415            raise Exception("Unknown datatype.")
416
417    if full_output:
418        retd = {}
419        retd['program'] = prog
420        retd['version'] = version
421        retd['who'] = who
422        retd['date'] = date
423        retd['host'] = host
424        retd['description'] = description
425        retd['obsdata'] = ol
426
427        return retd
428    else:
429        if len(obsdata) == 1:
430            ol = ol[0]
431
432        return ol
433
434
435def import_json_string(json_string, verbose=True, full_output=False):
436    """Reconstruct a list of Obs or structures containing Obs from a json string.
437
438    The following structures are supported: Obs, list, numpy.ndarray, Corr
439    If the list contains only one element, it is unpacked from the list.
440
441    Parameters
442    ----------
443    json_string : str
444        json string containing the data.
445    verbose : bool
446        Print additional information that was written to the file.
447    full_output : bool
448        If True, a dict containing auxiliary information and the data is returned.
449        If False, only the data is returned.
450    """
451
452    return _parse_json_dict(json.loads(json_string), verbose, full_output)
453
454
455def load_json(fname, verbose=True, gz=True, full_output=False):
456    """Import a list of Obs or structures containing Obs from a .json(.gz) file.
457
458    The following structures are supported: Obs, list, numpy.ndarray, Corr
459    If the list contains only one element, it is unpacked from the list.
460
461    Parameters
462    ----------
463    fname : str
464        Filename of the input file.
465    verbose : bool
466        Print additional information that was written to the file.
467    gz : bool
468        If True, assumes that data is gzipped. If False, assumes JSON file.
469    full_output : bool
470        If True, a dict containing auxiliary information and the data is returned.
471        If False, only the data is returned.
472    """
473    if not fname.endswith('.json') and not fname.endswith('.gz'):
474        fname += '.json'
475    if gz:
476        if not fname.endswith('.gz'):
477            fname += '.gz'
478        with gzip.open(fname, 'r') as fin:
479            d = json.load(fin)
480    else:
481        if fname.endswith('.gz'):
482            warnings.warn("Trying to read from %s without unzipping!" % fname, UserWarning)
483        with open(fname, 'r', encoding='utf-8') as fin:
484            d = json.loads(fin.read())
485
486    return _parse_json_dict(d, verbose, full_output)
487
488
489def _ol_from_dict(ind, reps='DICTOBS'):
490    """Convert a dictionary of Obs objects to a list and a dictionary that contains
491    placeholders instead of the Obs objects.
492
493    Parameters
494    ----------
495    ind : dict
496        Dict of JSON valid structures and objects that will be exported.
497        At the moment, these object can be either of: Obs, list, numpy.ndarray, Corr.
498        All Obs inside a structure have to be defined on the same set of configurations.
499    reps : str
500        Specify the structure of the placeholder in exported dict to be reps[0-9]+.
501    """
502
503    obstypes = (Obs, Corr, np.ndarray)
504
505    if not reps.isalnum():
506        raise Exception('Placeholder string has to be alphanumeric!')
507    ol = []
508    counter = 0
509
510    def dict_replace_obs(d):
511        nonlocal ol
512        nonlocal counter
513        x = {}
514        for k, v in d.items():
515            if isinstance(v, dict):
516                v = dict_replace_obs(v)
517            elif isinstance(v, list) and all([isinstance(o, Obs) for o in v]):
518                v = obslist_replace_obs(v)
519            elif isinstance(v, list):
520                v = list_replace_obs(v)
521            elif isinstance(v, obstypes):
522                ol.append(v)
523                v = reps + '%d' % (counter)
524                counter += 1
525            elif isinstance(v, str):
526                if bool(re.match(r'%s[0-9]+' % (reps), v)):
527                    raise Exception('Dict contains string %s that matches the placeholder! %s Cannot be savely exported.' % (v, reps))
528            x[k] = v
529        return x
530
531    def list_replace_obs(li):
532        nonlocal ol
533        nonlocal counter
534        x = []
535        for e in li:
536            if isinstance(e, list):
537                e = list_replace_obs(e)
538            elif isinstance(e, list) and all([isinstance(o, Obs) for o in e]):
539                e = obslist_replace_obs(e)
540            elif isinstance(e, dict):
541                e = dict_replace_obs(e)
542            elif isinstance(e, obstypes):
543                ol.append(e)
544                e = reps + '%d' % (counter)
545                counter += 1
546            elif isinstance(e, str):
547                if bool(re.match(r'%s[0-9]+' % (reps), e)):
548                    raise Exception('Dict contains string %s that matches the placeholder! %s Cannot be savely exported.' % (e, reps))
549            x.append(e)
550        return x
551
552    def obslist_replace_obs(li):
553        nonlocal ol
554        nonlocal counter
555        il = []
556        for e in li:
557            il.append(e)
558
559        ol.append(il)
560        x = reps + '%d' % (counter)
561        counter += 1
562        return x
563
564    nd = dict_replace_obs(ind)
565
566    return ol, nd
567
568
569def dump_dict_to_json(od, fname, description='', indent=1, reps='DICTOBS', gz=True):
570    """Export a dict of Obs or structures containing Obs to a .json(.gz) file
571
572    Parameters
573    ----------
574    od : dict
575        Dict of JSON valid structures and objects that will be exported.
576        At the moment, these objects can be either of: Obs, list, numpy.ndarray, Corr.
577        All Obs inside a structure have to be defined on the same set of configurations.
578    fname : str
579        Filename of the output file.
580    description : str
581        Optional string that describes the contents of the json file.
582    indent : int
583        Specify the indentation level of the json file. None or 0 is permissible and
584        saves disk space.
585    reps : str
586        Specify the structure of the placeholder in exported dict to be reps[0-9]+.
587    gz : bool
588        If True, the output is a gzipped json. If False, the output is a json file.
589    """
590
591    if not isinstance(od, dict):
592        raise Exception('od has to be a dictionary. Did you want to use dump_to_json?')
593
594    infostring = ('This JSON file contains a python dictionary that has been parsed to a list of structures. '
595                  'OBSDICT contains the dictionary, where Obs or other structures have been replaced by '
596                  '' + reps + '[0-9]+. The field description contains the additional description of this JSON file. '
597                  'This file may be parsed to a dict with the pyerrors routine load_json_dict.')
598
599    desc_dict = {'INFO': infostring, 'OBSDICT': {}, 'description': description}
600    ol, desc_dict['OBSDICT'] = _ol_from_dict(od, reps=reps)
601
602    dump_to_json(ol, fname, description=desc_dict, indent=indent, gz=gz)
603
604
605def _od_from_list_and_dict(ol, ind, reps='DICTOBS'):
606    """Parse a list of Obs or structures containing Obs and an accompanying
607    dict, where the structures have been replaced by placeholders to a
608    dict that contains the structures.
609
610    The following structures are supported: Obs, list, numpy.ndarray, Corr
611
612    Parameters
613    ----------
614    ol : list
615        List of objects -
616        At the moment, these objects can be either of: Obs, list, numpy.ndarray, Corr.
617        All Obs inside a structure have to be defined on the same set of configurations.
618    ind : dict
619        Dict that defines the structure of the resulting dict and contains placeholders
620    reps : str
621        Specify the structure of the placeholder in imported dict to be reps[0-9]+.
622    """
623    if not reps.isalnum():
624        raise Exception('Placeholder string has to be alphanumeric!')
625
626    counter = 0
627
628    def dict_replace_string(d):
629        nonlocal counter
630        nonlocal ol
631        x = {}
632        for k, v in d.items():
633            if isinstance(v, dict):
634                v = dict_replace_string(v)
635            elif isinstance(v, list):
636                v = list_replace_string(v)
637            elif isinstance(v, str) and bool(re.match(r'%s[0-9]+' % (reps), v)):
638                index = int(v[len(reps):])
639                v = ol[index]
640                counter += 1
641            x[k] = v
642        return x
643
644    def list_replace_string(li):
645        nonlocal counter
646        nonlocal ol
647        x = []
648        for e in li:
649            if isinstance(e, list):
650                e = list_replace_string(e)
651            elif isinstance(e, dict):
652                e = dict_replace_string(e)
653            elif isinstance(e, str) and bool(re.match(r'%s[0-9]+' % (reps), e)):
654                index = int(e[len(reps):])
655                e = ol[index]
656                counter += 1
657            x.append(e)
658        return x
659
660    nd = dict_replace_string(ind)
661
662    if counter == 0:
663        raise Exception('No placeholder has been replaced! Check if reps is set correctly.')
664
665    return nd
666
667
668def load_json_dict(fname, verbose=True, gz=True, full_output=False, reps='DICTOBS'):
669    """Import a dict of Obs or structures containing Obs from a .json(.gz) file.
670
671    The following structures are supported: Obs, list, numpy.ndarray, Corr
672
673    Parameters
674    ----------
675    fname : str
676        Filename of the input file.
677    verbose : bool
678        Print additional information that was written to the file.
679    gz : bool
680        If True, assumes that data is gzipped. If False, assumes JSON file.
681    full_output : bool
682        If True, a dict containing auxiliary information and the data is returned.
683        If False, only the data is returned.
684    reps : str
685        Specify the structure of the placeholder in imported dict to be reps[0-9]+.
686    """
687    indata = load_json(fname, verbose=verbose, gz=gz, full_output=True)
688    description = indata['description']['description']
689    indict = indata['description']['OBSDICT']
690    ol = indata['obsdata']
691    od = _od_from_list_and_dict(ol, indict, reps=reps)
692
693    if full_output:
694        indata['description'] = description
695        indata['obsdata'] = od
696        return indata
697    else:
698        return od
def create_json_string(ol, description='', indent=1):
 18def create_json_string(ol, description='', indent=1):
 19    """Generate the string for the export of a list of Obs or structures containing Obs
 20    to a .json(.gz) file
 21
 22    Parameters
 23    ----------
 24    ol : list
 25        List of objects that will be exported. At the moment, these objects can be
 26        either of: Obs, list, numpy.ndarray, Corr.
 27        All Obs inside a structure have to be defined on the same set of configurations.
 28    description : str
 29        Optional string that describes the contents of the json file.
 30    indent : int
 31        Specify the indentation level of the json file. None or 0 is permissible and
 32        saves disk space.
 33    """
 34
 35    def _gen_data_d_from_list(ol):
 36        dl = []
 37        No = len(ol)
 38        for name in ol[0].mc_names:
 39            ed = {}
 40            ed['id'] = name
 41            ed['replica'] = []
 42            for r_name in ol[0].e_content[name]:
 43                rd = {}
 44                rd['name'] = r_name
 45                rd['deltas'] = []
 46                offsets = [o.r_values[r_name] - o.value for o in ol]
 47                deltas = np.column_stack([ol[oi].deltas[r_name] + offsets[oi] for oi in range(No)])
 48                for i in range(len(ol[0].idl[r_name])):
 49                    rd['deltas'].append([ol[0].idl[r_name][i]])
 50                    rd['deltas'][-1] += deltas[i].tolist()
 51                ed['replica'].append(rd)
 52            dl.append(ed)
 53        return dl
 54
 55    def _gen_cdata_d_from_list(ol):
 56        dl = []
 57        for name in ol[0].cov_names:
 58            ed = {}
 59            ed['id'] = name
 60            ed['layout'] = str(ol[0].covobs[name].cov.shape).lstrip('(').rstrip(')').rstrip(',')
 61            ed['cov'] = list(np.ravel(ol[0].covobs[name].cov))
 62            ncov = ol[0].covobs[name].cov.shape[0]
 63            ed['grad'] = []
 64            for i in range(ncov):
 65                ed['grad'].append([])
 66                for o in ol:
 67                    ed['grad'][-1].append(o.covobs[name].grad[i][0])
 68            dl.append(ed)
 69        return dl
 70
 71    def write_Obs_to_dict(o):
 72        d = {}
 73        d['type'] = 'Obs'
 74        d['layout'] = '1'
 75        if o.tag:
 76            d['tag'] = [o.tag]
 77        if o.reweighted:
 78            d['reweighted'] = o.reweighted
 79        d['value'] = [o.value]
 80        data = _gen_data_d_from_list([o])
 81        if len(data) > 0:
 82            d['data'] = data
 83        cdata = _gen_cdata_d_from_list([o])
 84        if len(cdata) > 0:
 85            d['cdata'] = cdata
 86        return d
 87
 88    def write_List_to_dict(ol):
 89        _assert_equal_properties(ol)
 90        d = {}
 91        d['type'] = 'List'
 92        d['layout'] = '%d' % len(ol)
 93        taglist = [o.tag for o in ol]
 94        if np.any([tag is not None for tag in taglist]):
 95            d['tag'] = taglist
 96        if ol[0].reweighted:
 97            d['reweighted'] = ol[0].reweighted
 98        d['value'] = [o.value for o in ol]
 99        data = _gen_data_d_from_list(ol)
100        if len(data) > 0:
101            d['data'] = data
102        cdata = _gen_cdata_d_from_list(ol)
103        if len(cdata) > 0:
104            d['cdata'] = cdata
105        return d
106
107    def write_Array_to_dict(oa):
108        ol = np.ravel(oa)
109        _assert_equal_properties(ol)
110        d = {}
111        d['type'] = 'Array'
112        d['layout'] = str(oa.shape).lstrip('(').rstrip(')').rstrip(',')
113        taglist = [o.tag for o in ol]
114        if np.any([tag is not None for tag in taglist]):
115            d['tag'] = taglist
116        if ol[0].reweighted:
117            d['reweighted'] = ol[0].reweighted
118        d['value'] = [o.value for o in ol]
119        data = _gen_data_d_from_list(ol)
120        if len(data) > 0:
121            d['data'] = data
122        cdata = _gen_cdata_d_from_list(ol)
123        if len(cdata) > 0:
124            d['cdata'] = cdata
125        return d
126
127    def _nan_Obs_like(obs):
128        samples = []
129        names = []
130        idl = []
131        for key, value in obs.idl.items():
132            samples.append([np.nan] * len(value))
133            names.append(key)
134            idl.append(value)
135        my_obs = Obs(samples, names, idl)
136        my_obs._covobs = obs._covobs
137        for name in obs._covobs:
138            my_obs.names.append(name)
139        my_obs.reweighted = obs.reweighted
140        return my_obs
141
142    def write_Corr_to_dict(my_corr):
143        first_not_none = next(i for i, j in enumerate(my_corr.content) if np.all(j))
144        dummy_array = np.empty((my_corr.N, my_corr.N), dtype=object)
145        dummy_array[:] = _nan_Obs_like(my_corr.content[first_not_none].ravel()[0])
146        content = [o if o is not None else dummy_array for o in my_corr.content]
147        dat = write_Array_to_dict(np.array(content, dtype=object))
148        dat['type'] = 'Corr'
149        corr_meta_data = str(my_corr.tag)
150        if 'tag' in dat.keys():
151            dat['tag'].append(corr_meta_data)
152        else:
153            dat['tag'] = [corr_meta_data]
154        taglist = dat['tag']
155        dat['tag'] = {}  # tag is now a dictionary, that contains the previous taglist in the key "tag"
156        dat['tag']['tag'] = taglist
157        if my_corr.prange is not None:
158            dat['tag']['prange'] = my_corr.prange
159        return dat
160
161    if not isinstance(ol, list):
162        ol = [ol]
163
164    d = {}
165    d['program'] = 'pyerrors %s' % (pyerrorsversion.__version__)
166    d['version'] = '1.1'
167    d['who'] = getpass.getuser()
168    d['date'] = datetime.datetime.now().astimezone().strftime('%Y-%m-%d %H:%M:%S %z')
169    d['host'] = socket.gethostname() + ', ' + platform.platform()
170
171    if description:
172        d['description'] = description
173
174    d['obsdata'] = []
175    for io in ol:
176        if isinstance(io, Obs):
177            d['obsdata'].append(write_Obs_to_dict(io))
178        elif isinstance(io, list):
179            d['obsdata'].append(write_List_to_dict(io))
180        elif isinstance(io, np.ndarray):
181            d['obsdata'].append(write_Array_to_dict(io))
182        elif isinstance(io, Corr):
183            d['obsdata'].append(write_Corr_to_dict(io))
184        else:
185            raise Exception("Unkown datatype.")
186
187    def _jsonifier(o):
188        if isinstance(o, np.int64):
189            return int(o)
190        raise TypeError('%r is not JSON serializable' % o)
191
192    if indent:
193        return json.dumps(d, indent=indent, ensure_ascii=False, default=_jsonifier, write_mode=json.WM_SINGLE_LINE_ARRAY)
194    else:
195        return json.dumps(d, indent=indent, ensure_ascii=False, default=_jsonifier, write_mode=json.WM_COMPACT)

Generate the string for the export of a list of Obs or structures containing Obs to a .json(.gz) file

Parameters
  • ol (list): List of objects that will be exported. At the moment, these objects can be either of: Obs, list, numpy.ndarray, Corr. All Obs inside a structure have to be defined on the same set of configurations.
  • description (str): Optional string that describes the contents of the json file.
  • indent (int): Specify the indentation level of the json file. None or 0 is permissible and saves disk space.
def dump_to_json(ol, fname, description='', indent=1, gz=True):
198def dump_to_json(ol, fname, description='', indent=1, gz=True):
199    """Export a list of Obs or structures containing Obs to a .json(.gz) file
200
201    Parameters
202    ----------
203    ol : list
204        List of objects that will be exported. At the moment, these objects can be
205        either of: Obs, list, numpy.ndarray, Corr.
206        All Obs inside a structure have to be defined on the same set of configurations.
207    fname : str
208        Filename of the output file.
209    description : str
210        Optional string that describes the contents of the json file.
211    indent : int
212        Specify the indentation level of the json file. None or 0 is permissible and
213        saves disk space.
214    gz : bool
215        If True, the output is a gzipped json. If False, the output is a json file.
216    """
217
218    jsonstring = create_json_string(ol, description, indent)
219
220    if not fname.endswith('.json') and not fname.endswith('.gz'):
221        fname += '.json'
222
223    if gz:
224        if not fname.endswith('.gz'):
225            fname += '.gz'
226
227        fp = gzip.open(fname, 'wb')
228        fp.write(jsonstring.encode('utf-8'))
229    else:
230        fp = open(fname, 'w', encoding='utf-8')
231        fp.write(jsonstring)
232    fp.close()

Export a list of Obs or structures containing Obs to a .json(.gz) file

Parameters
  • ol (list): List of objects that will be exported. At the moment, these objects can be either of: Obs, list, numpy.ndarray, Corr. All Obs inside a structure have to be defined on the same set of configurations.
  • fname (str): Filename of the output file.
  • description (str): Optional string that describes the contents of the json file.
  • indent (int): Specify the indentation level of the json file. None or 0 is permissible and saves disk space.
  • gz (bool): If True, the output is a gzipped json. If False, the output is a json file.
def import_json_string(json_string, verbose=True, full_output=False):
436def import_json_string(json_string, verbose=True, full_output=False):
437    """Reconstruct a list of Obs or structures containing Obs from a json string.
438
439    The following structures are supported: Obs, list, numpy.ndarray, Corr
440    If the list contains only one element, it is unpacked from the list.
441
442    Parameters
443    ----------
444    json_string : str
445        json string containing the data.
446    verbose : bool
447        Print additional information that was written to the file.
448    full_output : bool
449        If True, a dict containing auxiliary information and the data is returned.
450        If False, only the data is returned.
451    """
452
453    return _parse_json_dict(json.loads(json_string), verbose, full_output)

Reconstruct a list of Obs or structures containing Obs from a json string.

The following structures are supported: Obs, list, numpy.ndarray, Corr If the list contains only one element, it is unpacked from the list.

Parameters
  • json_string (str): json string containing the data.
  • verbose (bool): Print additional information that was written to the file.
  • full_output (bool): If True, a dict containing auxiliary information and the data is returned. If False, only the data is returned.
def load_json(fname, verbose=True, gz=True, full_output=False):
456def load_json(fname, verbose=True, gz=True, full_output=False):
457    """Import a list of Obs or structures containing Obs from a .json(.gz) file.
458
459    The following structures are supported: Obs, list, numpy.ndarray, Corr
460    If the list contains only one element, it is unpacked from the list.
461
462    Parameters
463    ----------
464    fname : str
465        Filename of the input file.
466    verbose : bool
467        Print additional information that was written to the file.
468    gz : bool
469        If True, assumes that data is gzipped. If False, assumes JSON file.
470    full_output : bool
471        If True, a dict containing auxiliary information and the data is returned.
472        If False, only the data is returned.
473    """
474    if not fname.endswith('.json') and not fname.endswith('.gz'):
475        fname += '.json'
476    if gz:
477        if not fname.endswith('.gz'):
478            fname += '.gz'
479        with gzip.open(fname, 'r') as fin:
480            d = json.load(fin)
481    else:
482        if fname.endswith('.gz'):
483            warnings.warn("Trying to read from %s without unzipping!" % fname, UserWarning)
484        with open(fname, 'r', encoding='utf-8') as fin:
485            d = json.loads(fin.read())
486
487    return _parse_json_dict(d, verbose, full_output)

Import a list of Obs or structures containing Obs from a .json(.gz) file.

The following structures are supported: Obs, list, numpy.ndarray, Corr If the list contains only one element, it is unpacked from the list.

Parameters
  • fname (str): Filename of the input file.
  • verbose (bool): Print additional information that was written to the file.
  • gz (bool): If True, assumes that data is gzipped. If False, assumes JSON file.
  • full_output (bool): If True, a dict containing auxiliary information and the data is returned. If False, only the data is returned.
def dump_dict_to_json(od, fname, description='', indent=1, reps='DICTOBS', gz=True):
570def dump_dict_to_json(od, fname, description='', indent=1, reps='DICTOBS', gz=True):
571    """Export a dict of Obs or structures containing Obs to a .json(.gz) file
572
573    Parameters
574    ----------
575    od : dict
576        Dict of JSON valid structures and objects that will be exported.
577        At the moment, these objects can be either of: Obs, list, numpy.ndarray, Corr.
578        All Obs inside a structure have to be defined on the same set of configurations.
579    fname : str
580        Filename of the output file.
581    description : str
582        Optional string that describes the contents of the json file.
583    indent : int
584        Specify the indentation level of the json file. None or 0 is permissible and
585        saves disk space.
586    reps : str
587        Specify the structure of the placeholder in exported dict to be reps[0-9]+.
588    gz : bool
589        If True, the output is a gzipped json. If False, the output is a json file.
590    """
591
592    if not isinstance(od, dict):
593        raise Exception('od has to be a dictionary. Did you want to use dump_to_json?')
594
595    infostring = ('This JSON file contains a python dictionary that has been parsed to a list of structures. '
596                  'OBSDICT contains the dictionary, where Obs or other structures have been replaced by '
597                  '' + reps + '[0-9]+. The field description contains the additional description of this JSON file. '
598                  'This file may be parsed to a dict with the pyerrors routine load_json_dict.')
599
600    desc_dict = {'INFO': infostring, 'OBSDICT': {}, 'description': description}
601    ol, desc_dict['OBSDICT'] = _ol_from_dict(od, reps=reps)
602
603    dump_to_json(ol, fname, description=desc_dict, indent=indent, gz=gz)

Export a dict of Obs or structures containing Obs to a .json(.gz) file

Parameters
  • od (dict): Dict of JSON valid structures and objects that will be exported. At the moment, these objects can be either of: Obs, list, numpy.ndarray, Corr. All Obs inside a structure have to be defined on the same set of configurations.
  • fname (str): Filename of the output file.
  • description (str): Optional string that describes the contents of the json file.
  • indent (int): Specify the indentation level of the json file. None or 0 is permissible and saves disk space.
  • reps (str): Specify the structure of the placeholder in exported dict to be reps[0-9]+.
  • gz (bool): If True, the output is a gzipped json. If False, the output is a json file.
def load_json_dict(fname, verbose=True, gz=True, full_output=False, reps='DICTOBS'):
669def load_json_dict(fname, verbose=True, gz=True, full_output=False, reps='DICTOBS'):
670    """Import a dict of Obs or structures containing Obs from a .json(.gz) file.
671
672    The following structures are supported: Obs, list, numpy.ndarray, Corr
673
674    Parameters
675    ----------
676    fname : str
677        Filename of the input file.
678    verbose : bool
679        Print additional information that was written to the file.
680    gz : bool
681        If True, assumes that data is gzipped. If False, assumes JSON file.
682    full_output : bool
683        If True, a dict containing auxiliary information and the data is returned.
684        If False, only the data is returned.
685    reps : str
686        Specify the structure of the placeholder in imported dict to be reps[0-9]+.
687    """
688    indata = load_json(fname, verbose=verbose, gz=gz, full_output=True)
689    description = indata['description']['description']
690    indict = indata['description']['OBSDICT']
691    ol = indata['obsdata']
692    od = _od_from_list_and_dict(ol, indict, reps=reps)
693
694    if full_output:
695        indata['description'] = description
696        indata['obsdata'] = od
697        return indata
698    else:
699        return od

Import a dict of Obs or structures containing Obs from a .json(.gz) file.

The following structures are supported: Obs, list, numpy.ndarray, Corr

Parameters
  • fname (str): Filename of the input file.
  • verbose (bool): Print additional information that was written to the file.
  • gz (bool): If True, assumes that data is gzipped. If False, assumes JSON file.
  • full_output (bool): If True, a dict containing auxiliary information and the data is returned. If False, only the data is returned.
  • reps (str): Specify the structure of the placeholder in imported dict to be reps[0-9]+.