From b7bc508dcfed0ae537cbc0a23dfec1858940e04e Mon Sep 17 00:00:00 2001 From: Simon Kuberski Date: Tue, 14 Dec 2021 15:47:14 +0100 Subject: [PATCH 1/4] Added covobs to json file format --- pyerrors/input/json.py | 55 +++++++++++++++++++++++++++++++++++++++++- tests/io_test.py | 7 +++++- 2 files changed, 60 insertions(+), 2 deletions(-) diff --git a/pyerrors/input/json.py b/pyerrors/input/json.py index 4bd7031b..3f900e0a 100644 --- a/pyerrors/input/json.py +++ b/pyerrors/input/json.py @@ -7,6 +7,7 @@ import datetime import platform import warnings from ..obs import Obs +from ..covobs import Covobs from .. import version as pyerrorsversion @@ -50,7 +51,7 @@ def create_json_string(ol, description='', indent=1): def _gen_data_d_from_list(ol): dl = [] - for name in ol[0].e_names: + for name in ol[0].mc_names: ed = {} ed['id'] = name ed['replica'] = [] @@ -69,6 +70,22 @@ def create_json_string(ol, description='', indent=1): dl.append(ed) return dl + def _gen_cdata_d_from_list(ol): + dl = [] + for name in ol[0].cov_names: + ed = {} + ed['id'] = name + ed['layout'] = str(ol[0].covobs[name].cov.shape).lstrip('(').rstrip(')').rstrip(',') + ed['cov'] = list(np.ravel(ol[0].covobs[name].cov)) + ncov = ol[0].covobs[name].cov.shape[0] + ed['grad'] = [] + for i in range(ncov): + ed['grad'].append([]) + for o in ol: + ed['grad'][-1].append(o.covobs[name].grad[i][0]) + dl.append(ed) + return dl + def _assert_equal_properties(ol, otype=Obs): for o in ol: if not isinstance(o, otype): @@ -93,6 +110,7 @@ def create_json_string(ol, description='', indent=1): d['reweighted'] = o.reweighted d['value'] = [o.value] d['data'] = _gen_data_d_from_list([o]) + d['cdata'] = _gen_cdata_d_from_list([o]) return d def write_List_to_dict(ol): @@ -107,6 +125,7 @@ def create_json_string(ol, description='', indent=1): d['reweighted'] = ol[0].reweighted d['value'] = [o.value for o in ol] d['data'] = _gen_data_d_from_list(ol) + d['cdata'] = _gen_cdata_d_from_list(ol) return d def write_Array_to_dict(oa): @@ -122,6 +141,7 @@ def create_json_string(ol, description='', indent=1): d['reweighted'] = ol[0].reweighted d['value'] = [o.value for o in ol] d['data'] = _gen_data_d_from_list(ol) + d['cdata'] = _gen_cdata_d_from_list(ol) return d if not isinstance(ol, list): @@ -234,6 +254,22 @@ def import_json_string(json_string, verbose=True, full_output=False): retd['is_merged'][rep['name']] = rep.get('is_merged', False) return retd + def _gen_covobsd_from_cdatad(d): + retd = {} + for ens in d: + retl = [] + name = ens['id'] + layouts = ens.get('layout', '1').strip() + layout = [int(ls.strip()) for ls in layouts.split(',') if len(ls) > 0] + cov = np.reshape(ens['cov'], layout) + grad = ens['grad'] + nobs = len(grad[0]) + print(nobs, grad) + for i in range(nobs): + retl.append({'name': name, 'cov': cov, 'grad': [g[i] for g in grad]}) + retd[name] = retl + return retd + def get_Obs_from_dict(o): layouts = o.get('layout', '1').strip() if layouts != '1': @@ -241,8 +277,14 @@ def import_json_string(json_string, verbose=True, full_output=False): values = o['value'] od = _gen_obsd_from_datad(o['data']) + cd = _gen_covobsd_from_cdatad(o['cdata']) ret = Obs([[ddi[0] + values[0] for ddi in di] for di in od['deltas']], od['names'], idl=od['idl']) + for name in cd: + co = cd[name][0] + ret._covobs[name] = Covobs(None, co['cov'], co['name'], grad=co['grad']) + ret.names.append(co['name']) + ret.reweighted = o.get('reweighted', False) ret.is_merged = od['is_merged'] ret.tag = o.get('tag', [None])[0] @@ -253,11 +295,17 @@ def import_json_string(json_string, verbose=True, full_output=False): layout = int(layouts) values = o['value'] od = _gen_obsd_from_datad(o['data']) + cd = _gen_covobsd_from_cdatad(o['cdata']) ret = [] taglist = o.get('tag', layout * [None]) for i in range(layout): ret.append(Obs([list(di[:, i] + values[i]) for di in od['deltas']], od['names'], idl=od['idl'])) + for name in cd: + co = cd[name][i] + ret[-1]._covobs[name] = Covobs(None, co['cov'], co['name'], grad=co['grad']) + ret[-1].names.append(co['name']) + ret[-1].reweighted = o.get('reweighted', False) ret[-1].is_merged = od['is_merged'] ret[-1].tag = taglist[i] @@ -269,11 +317,16 @@ def import_json_string(json_string, verbose=True, full_output=False): N = np.prod(layout) values = o['value'] od = _gen_obsd_from_datad(o['data']) + cd = _gen_covobsd_from_cdatad(o['cdata']) ret = [] taglist = o.get('tag', N * [None]) for i in range(N): ret.append(Obs([di[:, i] + values[i] for di in od['deltas']], od['names'], idl=od['idl'])) + for name in cd: + co = cd[name][i] + ret[-1]._covobs[name] = Covobs(None, co['cov'], co['name'], grad=co['grad']) + ret[-1].names.append(co['name']) ret[-1].reweighted = o.get('reweighted', False) ret[-1].is_merged = od['is_merged'] ret[-1].tag = taglist[i] diff --git a/tests/io_test.py b/tests/io_test.py index a14ed52d..7b64b2ff 100644 --- a/tests/io_test.py +++ b/tests/io_test.py @@ -13,10 +13,15 @@ def test_jsonio(): otag = 'This has been merged!' o4.tag = otag do = o - .2 * o4 + co1 = pe.cov_Obs(1., .123, 'cov1') + do *= co1 do.tag = {'A': 2} o5 = pe.pseudo_Obs(0.8, .1, 'two|r2') - o5.tag = 2*otag + co2 = pe.cov_Obs([1, 2], [[.12, .004], [.004, .02]], 'cov2') + o5 /= co2[0] + o3 /= co2[1] + o5.tag = 2 * otag testl = [o3, o5] arr = np.array([o3, o5]) From 76b483d7307425c9a9dfaa16698bd91117615d36 Mon Sep 17 00:00:00 2001 From: Simon Kuberski Date: Tue, 14 Dec 2021 15:54:52 +0100 Subject: [PATCH 2/4] Pretty output for cov and grad in json --- pyerrors/input/json.py | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/pyerrors/input/json.py b/pyerrors/input/json.py index 3f900e0a..b3f20768 100644 --- a/pyerrors/input/json.py +++ b/pyerrors/input/json.py @@ -49,6 +49,22 @@ def create_json_string(ol, description='', indent=1): def __str__(self): return self.__repr__() + class Floatlist: + def __init__(self, li): + self.li = list(li) + + def __repr__(self): + s = '[' + for i in range(len(self.li)): + if i > 0: + s += ', ' + s += '%1.15e' % (self.li[i]) + s += ']' + return s + + def __str__(self): + return self.__repr__() + def _gen_data_d_from_list(ol): dl = [] for name in ol[0].mc_names: @@ -76,13 +92,14 @@ def create_json_string(ol, description='', indent=1): ed = {} ed['id'] = name ed['layout'] = str(ol[0].covobs[name].cov.shape).lstrip('(').rstrip(')').rstrip(',') - ed['cov'] = list(np.ravel(ol[0].covobs[name].cov)) + ed['cov'] = Floatlist(np.ravel(ol[0].covobs[name].cov)) ncov = ol[0].covobs[name].cov.shape[0] ed['grad'] = [] for i in range(ncov): ed['grad'].append([]) for o in ol: ed['grad'][-1].append(o.covobs[name].grad[i][0]) + ed['grad'][-1] = Floatlist(ed['grad'][-1]) dl.append(ed) return dl @@ -174,9 +191,9 @@ def create_json_string(ol, description='', indent=1): deltas = False split = s.split('\n') for i in range(len(split)): - if '"deltas":' in split[i]: + if '"deltas":' in split[i] or '"cov":' in split[i] or '"grad":' in split[i]: deltas = True - elif deltas: + if deltas: split[i] = split[i].replace('"[', '[').replace(']"', ']') if split[i][-1] == ']': deltas = False From 533898206009c5e0736b239cdb0a02f2539cad9e Mon Sep 17 00:00:00 2001 From: Simon Kuberski Date: Tue, 14 Dec 2021 16:27:05 +0100 Subject: [PATCH 3/4] Allow Obs in json format to be pure Covobs, i.e., to have empty deltas --- pyerrors/input/json.py | 83 +++++++++++++++++++++++++++--------------- tests/io_test.py | 4 +- 2 files changed, 56 insertions(+), 31 deletions(-) diff --git a/pyerrors/input/json.py b/pyerrors/input/json.py index b3f20768..131dccf3 100644 --- a/pyerrors/input/json.py +++ b/pyerrors/input/json.py @@ -126,8 +126,12 @@ def create_json_string(ol, description='', indent=1): if o.reweighted: d['reweighted'] = o.reweighted d['value'] = [o.value] - d['data'] = _gen_data_d_from_list([o]) - d['cdata'] = _gen_cdata_d_from_list([o]) + data = _gen_data_d_from_list([o]) + if len(data) > 0: + d['data'] = data + cdata = _gen_cdata_d_from_list([o]) + if len(cdata) > 0: + d['cdata'] = cdata return d def write_List_to_dict(ol): @@ -141,8 +145,12 @@ def create_json_string(ol, description='', indent=1): if ol[0].reweighted: d['reweighted'] = ol[0].reweighted d['value'] = [o.value for o in ol] - d['data'] = _gen_data_d_from_list(ol) - d['cdata'] = _gen_cdata_d_from_list(ol) + data = _gen_data_d_from_list(ol) + if len(data) > 0: + d['data'] = data + cdata = _gen_cdata_d_from_list(ol) + if len(cdata) > 0: + d['cdata'] = cdata return d def write_Array_to_dict(oa): @@ -157,8 +165,12 @@ def create_json_string(ol, description='', indent=1): if ol[0].reweighted: d['reweighted'] = ol[0].reweighted d['value'] = [o.value for o in ol] - d['data'] = _gen_data_d_from_list(ol) - d['cdata'] = _gen_cdata_d_from_list(ol) + data = _gen_data_d_from_list(ol) + if len(data) > 0: + d['data'] = data + cdata = _gen_cdata_d_from_list(ol) + if len(cdata) > 0: + d['cdata'] = cdata return d if not isinstance(ol, list): @@ -259,16 +271,17 @@ def import_json_string(json_string, verbose=True, full_output=False): def _gen_obsd_from_datad(d): retd = {} - retd['names'] = [] - retd['idl'] = [] - retd['deltas'] = [] - retd['is_merged'] = {} - for ens in d: - for rep in ens['replica']: - retd['names'].append(rep['name']) - retd['idl'].append([di[0] for di in rep['deltas']]) - retd['deltas'].append(np.array([di[1:] for di in rep['deltas']])) - retd['is_merged'][rep['name']] = rep.get('is_merged', False) + if d: + retd['names'] = [] + retd['idl'] = [] + retd['deltas'] = [] + retd['is_merged'] = {} + for ens in d: + for rep in ens['replica']: + retd['names'].append(rep['name']) + retd['idl'].append([di[0] for di in rep['deltas']]) + retd['deltas'].append(np.array([di[1:] for di in rep['deltas']])) + retd['is_merged'][rep['name']] = rep.get('is_merged', False) return retd def _gen_covobsd_from_cdatad(d): @@ -281,7 +294,6 @@ def import_json_string(json_string, verbose=True, full_output=False): cov = np.reshape(ens['cov'], layout) grad = ens['grad'] nobs = len(grad[0]) - print(nobs, grad) for i in range(nobs): retl.append({'name': name, 'cov': cov, 'grad': [g[i] for g in grad]}) retd[name] = retl @@ -293,17 +305,21 @@ def import_json_string(json_string, verbose=True, full_output=False): raise Exception("layout is %s has to be 1 for type Obs." % (layouts), RuntimeWarning) values = o['value'] - od = _gen_obsd_from_datad(o['data']) - cd = _gen_covobsd_from_cdatad(o['cdata']) + od = _gen_obsd_from_datad(o.get('data', {})) + cd = _gen_covobsd_from_cdatad(o.get('cdata', {})) - ret = Obs([[ddi[0] + values[0] for ddi in di] for di in od['deltas']], od['names'], idl=od['idl']) + if od: + ret = Obs([[ddi[0] + values[0] for ddi in di] for di in od['deltas']], od['names'], idl=od['idl']) + ret.is_merged = od['is_merged'] + else: + ret = Obs([], []) + ret._value = values[0] for name in cd: co = cd[name][0] ret._covobs[name] = Covobs(None, co['cov'], co['name'], grad=co['grad']) ret.names.append(co['name']) ret.reweighted = o.get('reweighted', False) - ret.is_merged = od['is_merged'] ret.tag = o.get('tag', [None])[0] return ret @@ -311,20 +327,25 @@ def import_json_string(json_string, verbose=True, full_output=False): layouts = o.get('layout', '1').strip() layout = int(layouts) values = o['value'] - od = _gen_obsd_from_datad(o['data']) - cd = _gen_covobsd_from_cdatad(o['cdata']) + od = _gen_obsd_from_datad(o.get('data', {})) + cd = _gen_covobsd_from_cdatad(o.get('cdata', {})) ret = [] taglist = o.get('tag', layout * [None]) for i in range(layout): - ret.append(Obs([list(di[:, i] + values[i]) for di in od['deltas']], od['names'], idl=od['idl'])) + if od: + ret.append(Obs([list(di[:, i] + values[i]) for di in od['deltas']], od['names'], idl=od['idl'])) + ret[-1].is_merged = od['is_merged'] + else: + ret.append(Obs([], [])) + ret[-1]._value = values[i] + print('Created Obs with means= ', values[i]) for name in cd: co = cd[name][i] ret[-1]._covobs[name] = Covobs(None, co['cov'], co['name'], grad=co['grad']) ret[-1].names.append(co['name']) ret[-1].reweighted = o.get('reweighted', False) - ret[-1].is_merged = od['is_merged'] ret[-1].tag = taglist[i] return ret @@ -333,19 +354,23 @@ def import_json_string(json_string, verbose=True, full_output=False): layout = [int(ls.strip()) for ls in layouts.split(',') if len(ls) > 0] N = np.prod(layout) values = o['value'] - od = _gen_obsd_from_datad(o['data']) - cd = _gen_covobsd_from_cdatad(o['cdata']) + od = _gen_obsd_from_datad(o.get('data', {})) + cd = _gen_covobsd_from_cdatad(o.get('cdata', {})) ret = [] taglist = o.get('tag', N * [None]) for i in range(N): - ret.append(Obs([di[:, i] + values[i] for di in od['deltas']], od['names'], idl=od['idl'])) + if od: + ret.append(Obs([di[:, i] + values[i] for di in od['deltas']], od['names'], idl=od['idl'])) + ret[-1].is_merged = od['is_merged'] + else: + ret.append(Obs([], [])) + ret[-1]._value = values[i] for name in cd: co = cd[name][i] ret[-1]._covobs[name] = Covobs(None, co['cov'], co['name'], grad=co['grad']) ret[-1].names.append(co['name']) ret[-1].reweighted = o.get('reweighted', False) - ret[-1].is_merged = od['is_merged'] ret[-1].tag = taglist[i] return np.reshape(ret, layout) diff --git a/tests/io_test.py b/tests/io_test.py index 7b64b2ff..151aede9 100644 --- a/tests/io_test.py +++ b/tests/io_test.py @@ -38,7 +38,7 @@ def test_jsonio(): tt.tag = 'Test Obs: Ä' - ol = [o4, do, testl, mat, arr, np.array([o]), np.array([tt, tt]), [tt, tt]] + ol = [o4, do, testl, mat, arr, np.array([o]), np.array([tt, tt]), [tt, tt], co1, co2, np.array(co2)] fname = 'test_rw' jsonio.dump_to_json(ol, fname, indent=1, description='[I am a tricky description]') @@ -50,7 +50,7 @@ def test_jsonio(): for o, r in zip(ol, rl): assert np.all(o == r) - for i in range(len(rl)): + for i in range(len(ol)): if isinstance(ol[i], pe.Obs): o = ol[i] - rl[i] assert(o.is_zero()) From a21554841ed4e4e6aa840a0d92c2d2521d358920 Mon Sep 17 00:00:00 2001 From: Simon Kuberski Date: Tue, 14 Dec 2021 16:31:33 +0100 Subject: [PATCH 4/4] Added tests for json --- tests/io_test.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/io_test.py b/tests/io_test.py index 151aede9..92781785 100644 --- a/tests/io_test.py +++ b/tests/io_test.py @@ -14,7 +14,8 @@ def test_jsonio(): o4.tag = otag do = o - .2 * o4 co1 = pe.cov_Obs(1., .123, 'cov1') - do *= co1 + co3 = pe.cov_Obs(4., .1 ** 2, 'cov3') + do *= co1 / co3 do.tag = {'A': 2} o5 = pe.pseudo_Obs(0.8, .1, 'two|r2') @@ -38,7 +39,7 @@ def test_jsonio(): tt.tag = 'Test Obs: Ä' - ol = [o4, do, testl, mat, arr, np.array([o]), np.array([tt, tt]), [tt, tt], co1, co2, np.array(co2)] + ol = [o4, do, testl, mat, arr, np.array([o]), np.array([tt, tt]), [tt, tt], co1, co2, np.array(co2), co1 / co2[0]] fname = 'test_rw' jsonio.dump_to_json(ol, fname, indent=1, description='[I am a tricky description]')