add ability to only write to temp files when reading many measurements

corrlib/meas_io.py

@@ -17,7 +17,7 @@ from .tracker import get, save, unlock
 CACHE_DIR = ".cache"
 
 
-def write_measurement(path: Path, ensemble: str, measurement: dict[str, dict[str, dict[str, Any]]], uuid: str, code: str, parameter_file: str | None) -> None:
+def write_measurement(path: Path, ensemble: str, measurement: dict[str, dict[str, dict[str, Any]]], uuid: str, code: str, parameter_file: str | None, final_write: dict[str, bool]) -> None:
     """
     Write a measurement to the backlog.
     If the file for the measurement already exists, update the measurement.
@@ -36,6 +36,8 @@ def write_measurement(path: Path, ensemble: str, measurement: dict[str, dict[str
         Name of the code that was used for the project.
     parameter_file: str
         The parameter file used for the measurement.
+    final_write: bool
+        Determmines whether this is the final ime the file is touched during the current import.
     """
     path = Path(path)
     db_file = get_db_file(path)
@@ -52,12 +54,16 @@ def write_measurement(path: Path, ensemble: str, measurement: dict[str, dict[str
     for corr in measurement.keys():
         file_in_archive = Path('.') / 'archive' / ensemble / corr / str(uuid + '.json.gz')
         file = Path(path) / file_in_archive
-        known_meas = {}
+        tmp_file_in_archive = Path('.') / 'archive' / ensemble / corr / (str(uuid) + ".p")
+        tmp_file = Path(path) / tmp_file_in_archive
+        known_meas: dict[str, Any] = {}
         if not os.path.exists(path / 'archive' / ensemble / corr):
             os.makedirs(path / 'archive' / ensemble / corr)
             files_to_save.append(file_in_archive)
         else:
-            if os.path.exists(file):
+            if os.path.exists(tmp_file):
+                known_meas = load_object(str(tmp_file))
+            elif os.path.exists(file):
                 if file not in files_to_save:
                     unlock(path, file_in_archive)
                     files_to_save.append(file_in_archive)
@@ -132,7 +138,12 @@ def write_measurement(path: Path, ensemble: str, measurement: dict[str, dict[str
                 c.execute("INSERT INTO backlogs (name, ensemble, code, path, project, parameters, parameter_file, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, datetime('now'), datetime('now'))",
                                                 (corr, ensemble, code, meas_path, uuid, pars[subkey], parameter_file))
         conn.commit()
-        pj.dump_dict_to_json(known_meas, str(file))
+        if final_write[str(file)]:
+            pj.dump_dict_to_json(known_meas, str(file))
+            if os.path.exists(tmp_file):
+                os.remove(tmp_file)
+        else:
+            dump_object(known_meas, str(tmp_file)[:-2])
     conn.close()
     save(path, message="Add measurements to database", files=files_to_save)
     return

corrlib/toml.py

@@ -192,7 +192,6 @@ def import_toml(path: Path, file: str, copy_file: bool=True) -> None:
     mname_list = list(measurements.keys())
     for mname in mname_list:
         md = measurements[mname]
-        print(f"Import measurement {imeas}/{nmeas}: {mname}")
         ensemble = md['ensemble']
         if project['code'] == 'sfcf':
             param = sfcf.read_param(path, uuid, md['param_file'])
@@ -204,9 +203,12 @@ def import_toml(path: Path, file: str, copy_file: bool=True) -> None:
                 affected_by_meas = affected_files(param['type'], ensemble, uuid)
             elif md['measurement'] == 't1':
                 affected_by_meas = affected_files(param['type'], ensemble, uuid)
-        affected_file_d[mname] = affected_by_meas
-    discard_after = step_differences(mname_list, affected_file_d)
-
+        affected_file_d[mname] = [str(path / f) for f in affected_by_meas]
+    future_affected_file_d = {}
+    for i,mname in enumerate(mname_list):
+        future_affected_file_d[mname] = []
+        for mname2 in mname_list[i+1:]:
+            future_affected_file_d[mname].extend(affected_file_d[mname2])
     for mname in mname_list:
         md = measurements[mname]
         print(f"Import measurement {imeas}/{nmeas}: {mname}")
@@ -267,7 +269,12 @@ def import_toml(path: Path, file: str, copy_file: bool=True) -> None:
                 measurement = openQCD.extract_t1(path, uuid, md['path'], ensemble, param, str(md["prefix"]), int(md["dtr_read"]), int(md["xmin"]), int(md["spatial_extent"]),
                                                  fit_range=int(md.get('fit_range', 5)), postfix=str(md.get('postfix', '')), names=md.get('names', []), files=md.get('files', []),
                                                  r_start=md.get('r_start', []), r_stop=md.get('r_stop', []), r_step=md.get('r_step', 1))
-        write_measurement(path, ensemble, measurement, uuid, project['code'], (md['param_file'] if 'param_file' in md else None))
+        final_write = {}
+        for file in affected_file_d[mname]:
+            final_write[str(file)] = True
+            if str(file) in future_affected_file_d[mname]:
+                final_write[str(file)] = False
+        write_measurement(path, ensemble, measurement, uuid, project['code'], (md['param_file'] if 'param_file' in md else None), final_write)
         imeas += 1
         print(mname + " imported.")
 

corrlib/version.py

@@ -3,12 +3,12 @@
 from __future__ import annotations
 
 __all__ = [
-    "__commit_id__",
     "__version__",
     "__version_tuple__",
-    "commit_id",
     "version",
     "version_tuple",
+    "__commit_id__",
+    "commit_id",
 ]
 
 version: str
@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '0.3.1.dev22+g4b1c21309.d20260701'
-__version_tuple__ = version_tuple = (0, 3, 1, 'dev22', 'g4b1c21309.d20260701')
+__version__ = version = '0.3.1.dev32+g906a2bdf3.d20260710'
+__version_tuple__ = version_tuple = (0, 3, 1, 'dev32', 'g906a2bdf3.d20260710')
 
-__commit_id__ = commit_id = 'g4b1c21309'
+__commit_id__ = commit_id = 'g906a2bdf3'