pyerrors.input.json
1import rapidjson as json 2import gzip 3import getpass 4import socket 5import datetime 6import platform 7import warnings 8import re 9import numpy as np 10from ..obs import Obs 11from ..covobs import Covobs 12from ..correlators import Corr 13from ..misc import _assert_equal_properties 14from .. import version as pyerrorsversion 15 16 17def create_json_string(ol, description='', indent=1): 18 """Generate the string for the export of a list of Obs or structures containing Obs 19 to a .json(.gz) file 20 21 Parameters 22 ---------- 23 ol : list 24 List of objects that will be exported. At the moment, these objects can be 25 either of: Obs, list, numpy.ndarray, Corr. 26 All Obs inside a structure have to be defined on the same set of configurations. 27 description : str 28 Optional string that describes the contents of the json file. 29 indent : int 30 Specify the indentation level of the json file. None or 0 is permissible and 31 saves disk space. 32 33 Returns 34 ------- 35 json_string : str 36 String for export to .json(.gz) file 37 """ 38 39 def _gen_data_d_from_list(ol): 40 dl = [] 41 No = len(ol) 42 for name in ol[0].mc_names: 43 ed = {} 44 ed['id'] = name 45 ed['replica'] = [] 46 for r_name in ol[0].e_content[name]: 47 rd = {} 48 rd['name'] = r_name 49 rd['deltas'] = [] 50 offsets = [o.r_values[r_name] - o.value for o in ol] 51 deltas = np.column_stack([ol[oi].deltas[r_name] + offsets[oi] for oi in range(No)]) 52 for i in range(len(ol[0].idl[r_name])): 53 rd['deltas'].append([ol[0].idl[r_name][i]]) 54 rd['deltas'][-1] += deltas[i].tolist() 55 ed['replica'].append(rd) 56 dl.append(ed) 57 return dl 58 59 def _gen_cdata_d_from_list(ol): 60 dl = [] 61 for name in ol[0].cov_names: 62 ed = {} 63 ed['id'] = name 64 ed['layout'] = str(ol[0].covobs[name].cov.shape).lstrip('(').rstrip(')').rstrip(',') 65 ed['cov'] = list(np.ravel(ol[0].covobs[name].cov)) 66 ncov = ol[0].covobs[name].cov.shape[0] 67 ed['grad'] = [] 68 for i in range(ncov): 69 ed['grad'].append([]) 70 for o in ol: 71 ed['grad'][-1].append(o.covobs[name].grad[i][0]) 72 dl.append(ed) 73 return dl 74 75 def write_Obs_to_dict(o): 76 d = {} 77 d['type'] = 'Obs' 78 d['layout'] = '1' 79 if o.tag: 80 d['tag'] = [o.tag] 81 if o.reweighted: 82 d['reweighted'] = o.reweighted 83 d['value'] = [o.value] 84 data = _gen_data_d_from_list([o]) 85 if len(data) > 0: 86 d['data'] = data 87 cdata = _gen_cdata_d_from_list([o]) 88 if len(cdata) > 0: 89 d['cdata'] = cdata 90 return d 91 92 def write_List_to_dict(ol): 93 _assert_equal_properties(ol) 94 d = {} 95 d['type'] = 'List' 96 d['layout'] = '%d' % len(ol) 97 taglist = [o.tag for o in ol] 98 if np.any([tag is not None for tag in taglist]): 99 d['tag'] = taglist 100 if ol[0].reweighted: 101 d['reweighted'] = ol[0].reweighted 102 d['value'] = [o.value for o in ol] 103 data = _gen_data_d_from_list(ol) 104 if len(data) > 0: 105 d['data'] = data 106 cdata = _gen_cdata_d_from_list(ol) 107 if len(cdata) > 0: 108 d['cdata'] = cdata 109 return d 110 111 def write_Array_to_dict(oa): 112 ol = np.ravel(oa) 113 _assert_equal_properties(ol) 114 d = {} 115 d['type'] = 'Array' 116 d['layout'] = str(oa.shape).lstrip('(').rstrip(')').rstrip(',') 117 taglist = [o.tag for o in ol] 118 if np.any([tag is not None for tag in taglist]): 119 d['tag'] = taglist 120 if ol[0].reweighted: 121 d['reweighted'] = ol[0].reweighted 122 d['value'] = [o.value for o in ol] 123 data = _gen_data_d_from_list(ol) 124 if len(data) > 0: 125 d['data'] = data 126 cdata = _gen_cdata_d_from_list(ol) 127 if len(cdata) > 0: 128 d['cdata'] = cdata 129 return d 130 131 def _nan_Obs_like(obs): 132 samples = [] 133 names = [] 134 idl = [] 135 for key, value in obs.idl.items(): 136 samples.append([np.nan] * len(value)) 137 names.append(key) 138 idl.append(value) 139 my_obs = Obs(samples, names, idl) 140 my_obs._covobs = obs._covobs 141 for name in obs._covobs: 142 my_obs.names.append(name) 143 my_obs.reweighted = obs.reweighted 144 return my_obs 145 146 def write_Corr_to_dict(my_corr): 147 first_not_none = next(i for i, j in enumerate(my_corr.content) if np.all(j)) 148 dummy_array = np.empty((my_corr.N, my_corr.N), dtype=object) 149 dummy_array[:] = _nan_Obs_like(my_corr.content[first_not_none].ravel()[0]) 150 content = [o if o is not None else dummy_array for o in my_corr.content] 151 dat = write_Array_to_dict(np.array(content, dtype=object)) 152 dat['type'] = 'Corr' 153 corr_meta_data = str(my_corr.tag) 154 if 'tag' in dat.keys(): 155 dat['tag'].append(corr_meta_data) 156 else: 157 dat['tag'] = [corr_meta_data] 158 taglist = dat['tag'] 159 dat['tag'] = {} # tag is now a dictionary, that contains the previous taglist in the key "tag" 160 dat['tag']['tag'] = taglist 161 if my_corr.prange is not None: 162 dat['tag']['prange'] = my_corr.prange 163 return dat 164 165 if not isinstance(ol, list): 166 ol = [ol] 167 168 d = {} 169 d['program'] = 'pyerrors %s' % (pyerrorsversion.__version__) 170 d['version'] = '1.1' 171 d['who'] = getpass.getuser() 172 d['date'] = datetime.datetime.now().astimezone().strftime('%Y-%m-%d %H:%M:%S %z') 173 d['host'] = socket.gethostname() + ', ' + platform.platform() 174 175 if description: 176 d['description'] = description 177 178 d['obsdata'] = [] 179 for io in ol: 180 if isinstance(io, Obs): 181 d['obsdata'].append(write_Obs_to_dict(io)) 182 elif isinstance(io, list): 183 d['obsdata'].append(write_List_to_dict(io)) 184 elif isinstance(io, np.ndarray): 185 d['obsdata'].append(write_Array_to_dict(io)) 186 elif isinstance(io, Corr): 187 d['obsdata'].append(write_Corr_to_dict(io)) 188 else: 189 raise Exception("Unkown datatype.") 190 191 def _jsonifier(o): 192 if isinstance(o, np.int64): 193 return int(o) 194 raise TypeError('%r is not JSON serializable' % o) 195 196 if indent: 197 return json.dumps(d, indent=indent, ensure_ascii=False, default=_jsonifier, write_mode=json.WM_SINGLE_LINE_ARRAY) 198 else: 199 return json.dumps(d, indent=indent, ensure_ascii=False, default=_jsonifier, write_mode=json.WM_COMPACT) 200 201 202def dump_to_json(ol, fname, description='', indent=1, gz=True): 203 """Export a list of Obs or structures containing Obs to a .json(.gz) file 204 205 Parameters 206 ---------- 207 ol : list 208 List of objects that will be exported. At the moment, these objects can be 209 either of: Obs, list, numpy.ndarray, Corr. 210 All Obs inside a structure have to be defined on the same set of configurations. 211 fname : str 212 Filename of the output file. 213 description : str 214 Optional string that describes the contents of the json file. 215 indent : int 216 Specify the indentation level of the json file. None or 0 is permissible and 217 saves disk space. 218 gz : bool 219 If True, the output is a gzipped json. If False, the output is a json file. 220 221 Returns 222 ------- 223 Null 224 """ 225 226 jsonstring = create_json_string(ol, description, indent) 227 228 if not fname.endswith('.json') and not fname.endswith('.gz'): 229 fname += '.json' 230 231 if gz: 232 if not fname.endswith('.gz'): 233 fname += '.gz' 234 235 fp = gzip.open(fname, 'wb') 236 fp.write(jsonstring.encode('utf-8')) 237 else: 238 fp = open(fname, 'w', encoding='utf-8') 239 fp.write(jsonstring) 240 fp.close() 241 242 243def _parse_json_dict(json_dict, verbose=True, full_output=False): 244 """Reconstruct a list of Obs or structures containing Obs from a dict that 245 was built out of a json string. 246 247 The following structures are supported: Obs, list, numpy.ndarray, Corr 248 If the list contains only one element, it is unpacked from the list. 249 250 Parameters 251 ---------- 252 json_string : str 253 json string containing the data. 254 verbose : bool 255 Print additional information that was written to the file. 256 full_output : bool 257 If True, a dict containing auxiliary information and the data is returned. 258 If False, only the data is returned. 259 260 Returns 261 ------- 262 result : list[Obs] 263 reconstructed list of observables from the json string 264 or 265 result : Obs 266 only one observable if the list only has one entry 267 or 268 result : dict 269 if full_output=True 270 """ 271 272 def _gen_obsd_from_datad(d): 273 retd = {} 274 if d: 275 retd['names'] = [] 276 retd['idl'] = [] 277 retd['deltas'] = [] 278 for ens in d: 279 for rep in ens['replica']: 280 rep_name = rep['name'] 281 if len(rep_name) > len(ens["id"]): 282 if rep_name[len(ens["id"])] != "|": 283 tmp_list = list(rep_name) 284 tmp_list = tmp_list[:len(ens["id"])] + ["|"] + tmp_list[len(ens["id"]):] 285 rep_name = ''.join(tmp_list) 286 retd['names'].append(rep_name) 287 retd['idl'].append([di[0] for di in rep['deltas']]) 288 retd['deltas'].append(np.array([di[1:] for di in rep['deltas']])) 289 return retd 290 291 def _gen_covobsd_from_cdatad(d): 292 retd = {} 293 for ens in d: 294 retl = [] 295 name = ens['id'] 296 layouts = ens.get('layout', '1').strip() 297 layout = [int(ls.strip()) for ls in layouts.split(',') if len(ls) > 0] 298 cov = np.reshape(ens['cov'], layout) 299 grad = ens['grad'] 300 nobs = len(grad[0]) 301 for i in range(nobs): 302 retl.append({'name': name, 'cov': cov, 'grad': [g[i] for g in grad]}) 303 retd[name] = retl 304 return retd 305 306 def get_Obs_from_dict(o): 307 layouts = o.get('layout', '1').strip() 308 if layouts != '1': 309 raise Exception("layout is %s has to be 1 for type Obs." % (layouts), RuntimeWarning) 310 311 values = o['value'] 312 od = _gen_obsd_from_datad(o.get('data', {})) 313 cd = _gen_covobsd_from_cdatad(o.get('cdata', {})) 314 315 if od: 316 ret = Obs([[ddi[0] + values[0] for ddi in di] for di in od['deltas']], od['names'], idl=od['idl']) 317 ret._value = values[0] 318 else: 319 ret = Obs([], [], means=[]) 320 ret._value = values[0] 321 for name in cd: 322 co = cd[name][0] 323 ret._covobs[name] = Covobs(None, co['cov'], co['name'], grad=co['grad']) 324 ret.names.append(co['name']) 325 326 ret.reweighted = o.get('reweighted', False) 327 ret.tag = o.get('tag', [None])[0] 328 return ret 329 330 def get_List_from_dict(o): 331 layouts = o.get('layout', '1').strip() 332 layout = int(layouts) 333 values = o['value'] 334 od = _gen_obsd_from_datad(o.get('data', {})) 335 cd = _gen_covobsd_from_cdatad(o.get('cdata', {})) 336 337 ret = [] 338 taglist = o.get('tag', layout * [None]) 339 for i in range(layout): 340 if od: 341 ret.append(Obs([list(di[:, i] + values[i]) for di in od['deltas']], od['names'], idl=od['idl'])) 342 ret[-1]._value = values[i] 343 else: 344 ret.append(Obs([], [], means=[])) 345 ret[-1]._value = values[i] 346 print('Created Obs with means= ', values[i]) 347 for name in cd: 348 co = cd[name][i] 349 ret[-1]._covobs[name] = Covobs(None, co['cov'], co['name'], grad=co['grad']) 350 ret[-1].names.append(co['name']) 351 352 ret[-1].reweighted = o.get('reweighted', False) 353 ret[-1].tag = taglist[i] 354 return ret 355 356 def get_Array_from_dict(o): 357 layouts = o.get('layout', '1').strip() 358 layout = [int(ls.strip()) for ls in layouts.split(',') if len(ls) > 0] 359 N = np.prod(layout) 360 values = o['value'] 361 od = _gen_obsd_from_datad(o.get('data', {})) 362 cd = _gen_covobsd_from_cdatad(o.get('cdata', {})) 363 364 ret = [] 365 taglist = o.get('tag', N * [None]) 366 for i in range(N): 367 if od: 368 ret.append(Obs([di[:, i] + values[i] for di in od['deltas']], od['names'], idl=od['idl'])) 369 ret[-1]._value = values[i] 370 else: 371 ret.append(Obs([], [], means=[])) 372 ret[-1]._value = values[i] 373 for name in cd: 374 co = cd[name][i] 375 ret[-1]._covobs[name] = Covobs(None, co['cov'], co['name'], grad=co['grad']) 376 ret[-1].names.append(co['name']) 377 ret[-1].reweighted = o.get('reweighted', False) 378 ret[-1].tag = taglist[i] 379 return np.reshape(ret, layout) 380 381 def get_Corr_from_dict(o): 382 if isinstance(o.get('tag'), list): # supports the old way 383 taglist = o.get('tag') # This had to be modified to get the taglist from the dictionary 384 temp_prange = None 385 elif isinstance(o.get('tag'), dict): 386 tagdic = o.get('tag') 387 taglist = tagdic['tag'] 388 if 'prange' in tagdic: 389 temp_prange = tagdic['prange'] 390 else: 391 temp_prange = None 392 else: 393 raise Exception("The tag is not a list or dict") 394 395 corr_tag = taglist[-1] 396 tmp_o = o 397 tmp_o['tag'] = taglist[:-1] 398 if len(tmp_o['tag']) == 0: 399 del tmp_o['tag'] 400 dat = get_Array_from_dict(tmp_o) 401 my_corr = Corr([None if np.isnan(o.ravel()[0].value) else o for o in list(dat)]) 402 if corr_tag != 'None': 403 my_corr.tag = corr_tag 404 405 my_corr.prange = temp_prange 406 return my_corr 407 408 prog = json_dict.get('program', '') 409 version = json_dict.get('version', '') 410 who = json_dict.get('who', '') 411 date = json_dict.get('date', '') 412 host = json_dict.get('host', '') 413 if prog and verbose: 414 print('Data has been written using %s.' % (prog)) 415 if version and verbose: 416 print('Format version %s' % (version)) 417 if np.any([who, date, host] and verbose): 418 print('Written by %s on %s on host %s' % (who, date, host)) 419 description = json_dict.get('description', '') 420 if description and verbose: 421 print() 422 print('Description: ', description) 423 obsdata = json_dict['obsdata'] 424 ol = [] 425 for io in obsdata: 426 if io['type'] == 'Obs': 427 ol.append(get_Obs_from_dict(io)) 428 elif io['type'] == 'List': 429 ol.append(get_List_from_dict(io)) 430 elif io['type'] == 'Array': 431 ol.append(get_Array_from_dict(io)) 432 elif io['type'] == 'Corr': 433 ol.append(get_Corr_from_dict(io)) 434 else: 435 raise Exception("Unknown datatype.") 436 437 if full_output: 438 retd = {} 439 retd['program'] = prog 440 retd['version'] = version 441 retd['who'] = who 442 retd['date'] = date 443 retd['host'] = host 444 retd['description'] = description 445 retd['obsdata'] = ol 446 447 return retd 448 else: 449 if len(obsdata) == 1: 450 ol = ol[0] 451 452 return ol 453 454 455def import_json_string(json_string, verbose=True, full_output=False): 456 """Reconstruct a list of Obs or structures containing Obs from a json string. 457 458 The following structures are supported: Obs, list, numpy.ndarray, Corr 459 If the list contains only one element, it is unpacked from the list. 460 461 Parameters 462 ---------- 463 json_string : str 464 json string containing the data. 465 verbose : bool 466 Print additional information that was written to the file. 467 full_output : bool 468 If True, a dict containing auxiliary information and the data is returned. 469 If False, only the data is returned. 470 471 Returns 472 ------- 473 result : list[Obs] 474 reconstructed list of observables from the json string 475 or 476 result : Obs 477 only one observable if the list only has one entry 478 or 479 result : dict 480 if full_output=True 481 """ 482 return _parse_json_dict(json.loads(json_string), verbose, full_output) 483 484 485def load_json(fname, verbose=True, gz=True, full_output=False): 486 """Import a list of Obs or structures containing Obs from a .json(.gz) file. 487 488 The following structures are supported: Obs, list, numpy.ndarray, Corr 489 If the list contains only one element, it is unpacked from the list. 490 491 Parameters 492 ---------- 493 fname : str 494 Filename of the input file. 495 verbose : bool 496 Print additional information that was written to the file. 497 gz : bool 498 If True, assumes that data is gzipped. If False, assumes JSON file. 499 full_output : bool 500 If True, a dict containing auxiliary information and the data is returned. 501 If False, only the data is returned. 502 503 Returns 504 ------- 505 result : list[Obs] 506 reconstructed list of observables from the json string 507 or 508 result : Obs 509 only one observable if the list only has one entry 510 or 511 result : dict 512 if full_output=True 513 """ 514 if not fname.endswith('.json') and not fname.endswith('.gz'): 515 fname += '.json' 516 if gz: 517 if not fname.endswith('.gz'): 518 fname += '.gz' 519 with gzip.open(fname, 'r') as fin: 520 d = json.load(fin) 521 else: 522 if fname.endswith('.gz'): 523 warnings.warn("Trying to read from %s without unzipping!" % fname, UserWarning) 524 with open(fname, 'r', encoding='utf-8') as fin: 525 d = json.loads(fin.read()) 526 527 return _parse_json_dict(d, verbose, full_output) 528 529 530def _ol_from_dict(ind, reps='DICTOBS'): 531 """Convert a dictionary of Obs objects to a list and a dictionary that contains 532 placeholders instead of the Obs objects. 533 534 Parameters 535 ---------- 536 ind : dict 537 Dict of JSON valid structures and objects that will be exported. 538 At the moment, these object can be either of: Obs, list, numpy.ndarray, Corr. 539 All Obs inside a structure have to be defined on the same set of configurations. 540 reps : str 541 Specify the structure of the placeholder in exported dict to be reps[0-9]+. 542 """ 543 544 obstypes = (Obs, Corr, np.ndarray) 545 546 if not reps.isalnum(): 547 raise Exception('Placeholder string has to be alphanumeric!') 548 ol = [] 549 counter = 0 550 551 def dict_replace_obs(d): 552 nonlocal ol 553 nonlocal counter 554 x = {} 555 for k, v in d.items(): 556 if isinstance(v, dict): 557 v = dict_replace_obs(v) 558 elif isinstance(v, list) and all([isinstance(o, Obs) for o in v]): 559 v = obslist_replace_obs(v) 560 elif isinstance(v, list): 561 v = list_replace_obs(v) 562 elif isinstance(v, obstypes): 563 ol.append(v) 564 v = reps + '%d' % (counter) 565 counter += 1 566 elif isinstance(v, str): 567 if bool(re.match(r'%s[0-9]+' % (reps), v)): 568 raise Exception('Dict contains string %s that matches the placeholder! %s Cannot be savely exported.' % (v, reps)) 569 x[k] = v 570 return x 571 572 def list_replace_obs(li): 573 nonlocal ol 574 nonlocal counter 575 x = [] 576 for e in li: 577 if isinstance(e, list): 578 e = list_replace_obs(e) 579 elif isinstance(e, list) and all([isinstance(o, Obs) for o in e]): 580 e = obslist_replace_obs(e) 581 elif isinstance(e, dict): 582 e = dict_replace_obs(e) 583 elif isinstance(e, obstypes): 584 ol.append(e) 585 e = reps + '%d' % (counter) 586 counter += 1 587 elif isinstance(e, str): 588 if bool(re.match(r'%s[0-9]+' % (reps), e)): 589 raise Exception('Dict contains string %s that matches the placeholder! %s Cannot be savely exported.' % (e, reps)) 590 x.append(e) 591 return x 592 593 def obslist_replace_obs(li): 594 nonlocal ol 595 nonlocal counter 596 il = [] 597 for e in li: 598 il.append(e) 599 600 ol.append(il) 601 x = reps + '%d' % (counter) 602 counter += 1 603 return x 604 605 nd = dict_replace_obs(ind) 606 607 return ol, nd 608 609 610def dump_dict_to_json(od, fname, description='', indent=1, reps='DICTOBS', gz=True): 611 """Export a dict of Obs or structures containing Obs to a .json(.gz) file 612 613 Parameters 614 ---------- 615 od : dict 616 Dict of JSON valid structures and objects that will be exported. 617 At the moment, these objects can be either of: Obs, list, numpy.ndarray, Corr. 618 All Obs inside a structure have to be defined on the same set of configurations. 619 fname : str 620 Filename of the output file. 621 description : str 622 Optional string that describes the contents of the json file. 623 indent : int 624 Specify the indentation level of the json file. None or 0 is permissible and 625 saves disk space. 626 reps : str 627 Specify the structure of the placeholder in exported dict to be reps[0-9]+. 628 gz : bool 629 If True, the output is a gzipped json. If False, the output is a json file. 630 631 Returns 632 ------- 633 None 634 """ 635 636 if not isinstance(od, dict): 637 raise Exception('od has to be a dictionary. Did you want to use dump_to_json?') 638 639 infostring = ('This JSON file contains a python dictionary that has been parsed to a list of structures. ' 640 'OBSDICT contains the dictionary, where Obs or other structures have been replaced by ' 641 '' + reps + '[0-9]+. The field description contains the additional description of this JSON file. ' 642 'This file may be parsed to a dict with the pyerrors routine load_json_dict.') 643 644 desc_dict = {'INFO': infostring, 'OBSDICT': {}, 'description': description} 645 ol, desc_dict['OBSDICT'] = _ol_from_dict(od, reps=reps) 646 647 dump_to_json(ol, fname, description=desc_dict, indent=indent, gz=gz) 648 649 650def _od_from_list_and_dict(ol, ind, reps='DICTOBS'): 651 """Parse a list of Obs or structures containing Obs and an accompanying 652 dict, where the structures have been replaced by placeholders to a 653 dict that contains the structures. 654 655 The following structures are supported: Obs, list, numpy.ndarray, Corr 656 657 Parameters 658 ---------- 659 ol : list 660 List of objects - 661 At the moment, these objects can be either of: Obs, list, numpy.ndarray, Corr. 662 All Obs inside a structure have to be defined on the same set of configurations. 663 ind : dict 664 Dict that defines the structure of the resulting dict and contains placeholders 665 reps : str 666 Specify the structure of the placeholder in imported dict to be reps[0-9]+. 667 """ 668 if not reps.isalnum(): 669 raise Exception('Placeholder string has to be alphanumeric!') 670 671 counter = 0 672 673 def dict_replace_string(d): 674 nonlocal counter 675 nonlocal ol 676 x = {} 677 for k, v in d.items(): 678 if isinstance(v, dict): 679 v = dict_replace_string(v) 680 elif isinstance(v, list): 681 v = list_replace_string(v) 682 elif isinstance(v, str) and bool(re.match(r'%s[0-9]+' % (reps), v)): 683 index = int(v[len(reps):]) 684 v = ol[index] 685 counter += 1 686 x[k] = v 687 return x 688 689 def list_replace_string(li): 690 nonlocal counter 691 nonlocal ol 692 x = [] 693 for e in li: 694 if isinstance(e, list): 695 e = list_replace_string(e) 696 elif isinstance(e, dict): 697 e = dict_replace_string(e) 698 elif isinstance(e, str) and bool(re.match(r'%s[0-9]+' % (reps), e)): 699 index = int(e[len(reps):]) 700 e = ol[index] 701 counter += 1 702 x.append(e) 703 return x 704 705 nd = dict_replace_string(ind) 706 707 if counter == 0: 708 raise Exception('No placeholder has been replaced! Check if reps is set correctly.') 709 710 return nd 711 712 713def load_json_dict(fname, verbose=True, gz=True, full_output=False, reps='DICTOBS'): 714 """Import a dict of Obs or structures containing Obs from a .json(.gz) file. 715 716 The following structures are supported: Obs, list, numpy.ndarray, Corr 717 718 Parameters 719 ---------- 720 fname : str 721 Filename of the input file. 722 verbose : bool 723 Print additional information that was written to the file. 724 gz : bool 725 If True, assumes that data is gzipped. If False, assumes JSON file. 726 full_output : bool 727 If True, a dict containing auxiliary information and the data is returned. 728 If False, only the data is returned. 729 reps : str 730 Specify the structure of the placeholder in imported dict to be reps[0-9]+. 731 732 Returns 733 ------- 734 data : Obs / list / Corr 735 Read data 736 or 737 data : dict 738 Read data and meta-data 739 """ 740 indata = load_json(fname, verbose=verbose, gz=gz, full_output=True) 741 description = indata['description']['description'] 742 indict = indata['description']['OBSDICT'] 743 ol = indata['obsdata'] 744 od = _od_from_list_and_dict(ol, indict, reps=reps) 745 746 if full_output: 747 indata['description'] = description 748 indata['obsdata'] = od 749 return indata 750 else: 751 return od
18def create_json_string(ol, description='', indent=1): 19 """Generate the string for the export of a list of Obs or structures containing Obs 20 to a .json(.gz) file 21 22 Parameters 23 ---------- 24 ol : list 25 List of objects that will be exported. At the moment, these objects can be 26 either of: Obs, list, numpy.ndarray, Corr. 27 All Obs inside a structure have to be defined on the same set of configurations. 28 description : str 29 Optional string that describes the contents of the json file. 30 indent : int 31 Specify the indentation level of the json file. None or 0 is permissible and 32 saves disk space. 33 34 Returns 35 ------- 36 json_string : str 37 String for export to .json(.gz) file 38 """ 39 40 def _gen_data_d_from_list(ol): 41 dl = [] 42 No = len(ol) 43 for name in ol[0].mc_names: 44 ed = {} 45 ed['id'] = name 46 ed['replica'] = [] 47 for r_name in ol[0].e_content[name]: 48 rd = {} 49 rd['name'] = r_name 50 rd['deltas'] = [] 51 offsets = [o.r_values[r_name] - o.value for o in ol] 52 deltas = np.column_stack([ol[oi].deltas[r_name] + offsets[oi] for oi in range(No)]) 53 for i in range(len(ol[0].idl[r_name])): 54 rd['deltas'].append([ol[0].idl[r_name][i]]) 55 rd['deltas'][-1] += deltas[i].tolist() 56 ed['replica'].append(rd) 57 dl.append(ed) 58 return dl 59 60 def _gen_cdata_d_from_list(ol): 61 dl = [] 62 for name in ol[0].cov_names: 63 ed = {} 64 ed['id'] = name 65 ed['layout'] = str(ol[0].covobs[name].cov.shape).lstrip('(').rstrip(')').rstrip(',') 66 ed['cov'] = list(np.ravel(ol[0].covobs[name].cov)) 67 ncov = ol[0].covobs[name].cov.shape[0] 68 ed['grad'] = [] 69 for i in range(ncov): 70 ed['grad'].append([]) 71 for o in ol: 72 ed['grad'][-1].append(o.covobs[name].grad[i][0]) 73 dl.append(ed) 74 return dl 75 76 def write_Obs_to_dict(o): 77 d = {} 78 d['type'] = 'Obs' 79 d['layout'] = '1' 80 if o.tag: 81 d['tag'] = [o.tag] 82 if o.reweighted: 83 d['reweighted'] = o.reweighted 84 d['value'] = [o.value] 85 data = _gen_data_d_from_list([o]) 86 if len(data) > 0: 87 d['data'] = data 88 cdata = _gen_cdata_d_from_list([o]) 89 if len(cdata) > 0: 90 d['cdata'] = cdata 91 return d 92 93 def write_List_to_dict(ol): 94 _assert_equal_properties(ol) 95 d = {} 96 d['type'] = 'List' 97 d['layout'] = '%d' % len(ol) 98 taglist = [o.tag for o in ol] 99 if np.any([tag is not None for tag in taglist]): 100 d['tag'] = taglist 101 if ol[0].reweighted: 102 d['reweighted'] = ol[0].reweighted 103 d['value'] = [o.value for o in ol] 104 data = _gen_data_d_from_list(ol) 105 if len(data) > 0: 106 d['data'] = data 107 cdata = _gen_cdata_d_from_list(ol) 108 if len(cdata) > 0: 109 d['cdata'] = cdata 110 return d 111 112 def write_Array_to_dict(oa): 113 ol = np.ravel(oa) 114 _assert_equal_properties(ol) 115 d = {} 116 d['type'] = 'Array' 117 d['layout'] = str(oa.shape).lstrip('(').rstrip(')').rstrip(',') 118 taglist = [o.tag for o in ol] 119 if np.any([tag is not None for tag in taglist]): 120 d['tag'] = taglist 121 if ol[0].reweighted: 122 d['reweighted'] = ol[0].reweighted 123 d['value'] = [o.value for o in ol] 124 data = _gen_data_d_from_list(ol) 125 if len(data) > 0: 126 d['data'] = data 127 cdata = _gen_cdata_d_from_list(ol) 128 if len(cdata) > 0: 129 d['cdata'] = cdata 130 return d 131 132 def _nan_Obs_like(obs): 133 samples = [] 134 names = [] 135 idl = [] 136 for key, value in obs.idl.items(): 137 samples.append([np.nan] * len(value)) 138 names.append(key) 139 idl.append(value) 140 my_obs = Obs(samples, names, idl) 141 my_obs._covobs = obs._covobs 142 for name in obs._covobs: 143 my_obs.names.append(name) 144 my_obs.reweighted = obs.reweighted 145 return my_obs 146 147 def write_Corr_to_dict(my_corr): 148 first_not_none = next(i for i, j in enumerate(my_corr.content) if np.all(j)) 149 dummy_array = np.empty((my_corr.N, my_corr.N), dtype=object) 150 dummy_array[:] = _nan_Obs_like(my_corr.content[first_not_none].ravel()[0]) 151 content = [o if o is not None else dummy_array for o in my_corr.content] 152 dat = write_Array_to_dict(np.array(content, dtype=object)) 153 dat['type'] = 'Corr' 154 corr_meta_data = str(my_corr.tag) 155 if 'tag' in dat.keys(): 156 dat['tag'].append(corr_meta_data) 157 else: 158 dat['tag'] = [corr_meta_data] 159 taglist = dat['tag'] 160 dat['tag'] = {} # tag is now a dictionary, that contains the previous taglist in the key "tag" 161 dat['tag']['tag'] = taglist 162 if my_corr.prange is not None: 163 dat['tag']['prange'] = my_corr.prange 164 return dat 165 166 if not isinstance(ol, list): 167 ol = [ol] 168 169 d = {} 170 d['program'] = 'pyerrors %s' % (pyerrorsversion.__version__) 171 d['version'] = '1.1' 172 d['who'] = getpass.getuser() 173 d['date'] = datetime.datetime.now().astimezone().strftime('%Y-%m-%d %H:%M:%S %z') 174 d['host'] = socket.gethostname() + ', ' + platform.platform() 175 176 if description: 177 d['description'] = description 178 179 d['obsdata'] = [] 180 for io in ol: 181 if isinstance(io, Obs): 182 d['obsdata'].append(write_Obs_to_dict(io)) 183 elif isinstance(io, list): 184 d['obsdata'].append(write_List_to_dict(io)) 185 elif isinstance(io, np.ndarray): 186 d['obsdata'].append(write_Array_to_dict(io)) 187 elif isinstance(io, Corr): 188 d['obsdata'].append(write_Corr_to_dict(io)) 189 else: 190 raise Exception("Unkown datatype.") 191 192 def _jsonifier(o): 193 if isinstance(o, np.int64): 194 return int(o) 195 raise TypeError('%r is not JSON serializable' % o) 196 197 if indent: 198 return json.dumps(d, indent=indent, ensure_ascii=False, default=_jsonifier, write_mode=json.WM_SINGLE_LINE_ARRAY) 199 else: 200 return json.dumps(d, indent=indent, ensure_ascii=False, default=_jsonifier, write_mode=json.WM_COMPACT)
Generate the string for the export of a list of Obs or structures containing Obs to a .json(.gz) file
Parameters
- ol (list): List of objects that will be exported. At the moment, these objects can be either of: Obs, list, numpy.ndarray, Corr. All Obs inside a structure have to be defined on the same set of configurations.
- description (str): Optional string that describes the contents of the json file.
- indent (int): Specify the indentation level of the json file. None or 0 is permissible and saves disk space.
Returns
- json_string (str): String for export to .json(.gz) file
203def dump_to_json(ol, fname, description='', indent=1, gz=True): 204 """Export a list of Obs or structures containing Obs to a .json(.gz) file 205 206 Parameters 207 ---------- 208 ol : list 209 List of objects that will be exported. At the moment, these objects can be 210 either of: Obs, list, numpy.ndarray, Corr. 211 All Obs inside a structure have to be defined on the same set of configurations. 212 fname : str 213 Filename of the output file. 214 description : str 215 Optional string that describes the contents of the json file. 216 indent : int 217 Specify the indentation level of the json file. None or 0 is permissible and 218 saves disk space. 219 gz : bool 220 If True, the output is a gzipped json. If False, the output is a json file. 221 222 Returns 223 ------- 224 Null 225 """ 226 227 jsonstring = create_json_string(ol, description, indent) 228 229 if not fname.endswith('.json') and not fname.endswith('.gz'): 230 fname += '.json' 231 232 if gz: 233 if not fname.endswith('.gz'): 234 fname += '.gz' 235 236 fp = gzip.open(fname, 'wb') 237 fp.write(jsonstring.encode('utf-8')) 238 else: 239 fp = open(fname, 'w', encoding='utf-8') 240 fp.write(jsonstring) 241 fp.close()
Export a list of Obs or structures containing Obs to a .json(.gz) file
Parameters
- ol (list): List of objects that will be exported. At the moment, these objects can be either of: Obs, list, numpy.ndarray, Corr. All Obs inside a structure have to be defined on the same set of configurations.
- fname (str): Filename of the output file.
- description (str): Optional string that describes the contents of the json file.
- indent (int): Specify the indentation level of the json file. None or 0 is permissible and saves disk space.
- gz (bool): If True, the output is a gzipped json. If False, the output is a json file.
Returns
- Null
456def import_json_string(json_string, verbose=True, full_output=False): 457 """Reconstruct a list of Obs or structures containing Obs from a json string. 458 459 The following structures are supported: Obs, list, numpy.ndarray, Corr 460 If the list contains only one element, it is unpacked from the list. 461 462 Parameters 463 ---------- 464 json_string : str 465 json string containing the data. 466 verbose : bool 467 Print additional information that was written to the file. 468 full_output : bool 469 If True, a dict containing auxiliary information and the data is returned. 470 If False, only the data is returned. 471 472 Returns 473 ------- 474 result : list[Obs] 475 reconstructed list of observables from the json string 476 or 477 result : Obs 478 only one observable if the list only has one entry 479 or 480 result : dict 481 if full_output=True 482 """ 483 return _parse_json_dict(json.loads(json_string), verbose, full_output)
Reconstruct a list of Obs or structures containing Obs from a json string.
The following structures are supported: Obs, list, numpy.ndarray, Corr If the list contains only one element, it is unpacked from the list.
Parameters
- json_string (str): json string containing the data.
- verbose (bool): Print additional information that was written to the file.
- full_output (bool): If True, a dict containing auxiliary information and the data is returned. If False, only the data is returned.
Returns
- result (list[Obs]): reconstructed list of observables from the json string
- or
- result (Obs): only one observable if the list only has one entry
- or
- result (dict): if full_output=True
486def load_json(fname, verbose=True, gz=True, full_output=False): 487 """Import a list of Obs or structures containing Obs from a .json(.gz) file. 488 489 The following structures are supported: Obs, list, numpy.ndarray, Corr 490 If the list contains only one element, it is unpacked from the list. 491 492 Parameters 493 ---------- 494 fname : str 495 Filename of the input file. 496 verbose : bool 497 Print additional information that was written to the file. 498 gz : bool 499 If True, assumes that data is gzipped. If False, assumes JSON file. 500 full_output : bool 501 If True, a dict containing auxiliary information and the data is returned. 502 If False, only the data is returned. 503 504 Returns 505 ------- 506 result : list[Obs] 507 reconstructed list of observables from the json string 508 or 509 result : Obs 510 only one observable if the list only has one entry 511 or 512 result : dict 513 if full_output=True 514 """ 515 if not fname.endswith('.json') and not fname.endswith('.gz'): 516 fname += '.json' 517 if gz: 518 if not fname.endswith('.gz'): 519 fname += '.gz' 520 with gzip.open(fname, 'r') as fin: 521 d = json.load(fin) 522 else: 523 if fname.endswith('.gz'): 524 warnings.warn("Trying to read from %s without unzipping!" % fname, UserWarning) 525 with open(fname, 'r', encoding='utf-8') as fin: 526 d = json.loads(fin.read()) 527 528 return _parse_json_dict(d, verbose, full_output)
Import a list of Obs or structures containing Obs from a .json(.gz) file.
The following structures are supported: Obs, list, numpy.ndarray, Corr If the list contains only one element, it is unpacked from the list.
Parameters
- fname (str): Filename of the input file.
- verbose (bool): Print additional information that was written to the file.
- gz (bool): If True, assumes that data is gzipped. If False, assumes JSON file.
- full_output (bool): If True, a dict containing auxiliary information and the data is returned. If False, only the data is returned.
Returns
- result (list[Obs]): reconstructed list of observables from the json string
- or
- result (Obs): only one observable if the list only has one entry
- or
- result (dict): if full_output=True
611def dump_dict_to_json(od, fname, description='', indent=1, reps='DICTOBS', gz=True): 612 """Export a dict of Obs or structures containing Obs to a .json(.gz) file 613 614 Parameters 615 ---------- 616 od : dict 617 Dict of JSON valid structures and objects that will be exported. 618 At the moment, these objects can be either of: Obs, list, numpy.ndarray, Corr. 619 All Obs inside a structure have to be defined on the same set of configurations. 620 fname : str 621 Filename of the output file. 622 description : str 623 Optional string that describes the contents of the json file. 624 indent : int 625 Specify the indentation level of the json file. None or 0 is permissible and 626 saves disk space. 627 reps : str 628 Specify the structure of the placeholder in exported dict to be reps[0-9]+. 629 gz : bool 630 If True, the output is a gzipped json. If False, the output is a json file. 631 632 Returns 633 ------- 634 None 635 """ 636 637 if not isinstance(od, dict): 638 raise Exception('od has to be a dictionary. Did you want to use dump_to_json?') 639 640 infostring = ('This JSON file contains a python dictionary that has been parsed to a list of structures. ' 641 'OBSDICT contains the dictionary, where Obs or other structures have been replaced by ' 642 '' + reps + '[0-9]+. The field description contains the additional description of this JSON file. ' 643 'This file may be parsed to a dict with the pyerrors routine load_json_dict.') 644 645 desc_dict = {'INFO': infostring, 'OBSDICT': {}, 'description': description} 646 ol, desc_dict['OBSDICT'] = _ol_from_dict(od, reps=reps) 647 648 dump_to_json(ol, fname, description=desc_dict, indent=indent, gz=gz)
Export a dict of Obs or structures containing Obs to a .json(.gz) file
Parameters
- od (dict): Dict of JSON valid structures and objects that will be exported. At the moment, these objects can be either of: Obs, list, numpy.ndarray, Corr. All Obs inside a structure have to be defined on the same set of configurations.
- fname (str): Filename of the output file.
- description (str): Optional string that describes the contents of the json file.
- indent (int): Specify the indentation level of the json file. None or 0 is permissible and saves disk space.
- reps (str): Specify the structure of the placeholder in exported dict to be reps[0-9]+.
- gz (bool): If True, the output is a gzipped json. If False, the output is a json file.
Returns
- None
714def load_json_dict(fname, verbose=True, gz=True, full_output=False, reps='DICTOBS'): 715 """Import a dict of Obs or structures containing Obs from a .json(.gz) file. 716 717 The following structures are supported: Obs, list, numpy.ndarray, Corr 718 719 Parameters 720 ---------- 721 fname : str 722 Filename of the input file. 723 verbose : bool 724 Print additional information that was written to the file. 725 gz : bool 726 If True, assumes that data is gzipped. If False, assumes JSON file. 727 full_output : bool 728 If True, a dict containing auxiliary information and the data is returned. 729 If False, only the data is returned. 730 reps : str 731 Specify the structure of the placeholder in imported dict to be reps[0-9]+. 732 733 Returns 734 ------- 735 data : Obs / list / Corr 736 Read data 737 or 738 data : dict 739 Read data and meta-data 740 """ 741 indata = load_json(fname, verbose=verbose, gz=gz, full_output=True) 742 description = indata['description']['description'] 743 indict = indata['description']['OBSDICT'] 744 ol = indata['obsdata'] 745 od = _od_from_list_and_dict(ol, indict, reps=reps) 746 747 if full_output: 748 indata['description'] = description 749 indata['obsdata'] = od 750 return indata 751 else: 752 return od
Import a dict of Obs or structures containing Obs from a .json(.gz) file.
The following structures are supported: Obs, list, numpy.ndarray, Corr
Parameters
- fname (str): Filename of the input file.
- verbose (bool): Print additional information that was written to the file.
- gz (bool): If True, assumes that data is gzipped. If False, assumes JSON file.
- full_output (bool): If True, a dict containing auxiliary information and the data is returned. If False, only the data is returned.
- reps (str): Specify the structure of the placeholder in imported dict to be reps[0-9]+.
Returns
- data (Obs / list / Corr): Read data
- or
- data (dict): Read data and meta-data