diff --git a/corrlib/cache_io.py b/corrlib/cache_io.py new file mode 100644 index 0000000..4d1d632 --- /dev/null +++ b/corrlib/cache_io.py @@ -0,0 +1,33 @@ +from typing import Union, Optional +import os +import shutil + + +def drop_cache_files(path: str, fs: Optional[list[str]]=None): + cache_dir = os.path.join(path, ".cache") + if fs is None: + fs = os.listdir(cache_dir) + for f in fs: + shutil.rmtree(os.path.join(cache_dir, f)) + + +def cache_dir(path, file): + cache_path_list = [path] + cache_path_list.append(".cache") + cache_path_list.extend(file.split("/")[1:]) + cache_path = "/".join(cache_path_list) + return cache_path + + +def cache_path(path, file, hash, key): + cache_path = os.path.join(cache_dir(path, file), hash, key) + return cache_path + +def is_in_cache(path, record, hash): + + if os.file.exists(cache_path(path, file, hash, key)): + return True + else: + return False + + \ No newline at end of file diff --git a/corrlib/meas_io.py b/corrlib/meas_io.py index fca5671..ad9a6e8 100644 --- a/corrlib/meas_io.py +++ b/corrlib/meas_io.py @@ -4,14 +4,15 @@ import datalad.api as dl import sqlite3 from .input import sfcf,openQCD import json -from typing import Union +from typing import Union, Optional from pyerrors import Obs, Corr, dump_object, load_object from hashlib import sha256, sha1 -from .tools import cached +from .tools import cached, record2name_key import shutil +from .caching import cache_path, cache_dir -def write_measurement(path, ensemble, measurement, uuid, code, parameter_file=None): +def write_measurement(path, ensemble, measurement, uuid, code, parameter_file: Optional[str]=None): """ Write a measurement to the backlog. If the file for the measurement already exists, update the measurement. @@ -115,7 +116,7 @@ def load_record(path: str, meas_path: str): return load_records(path, [meas_path])[0] -def load_records(path: str, meas_paths: list[str], preloaded = {}) -> list[Union[Corr, Obs]]: +def load_records(path: str, record_paths: list[str], preloaded = {}) -> list[Union[Corr, Obs]]: """ Load a list of records by their paths. @@ -131,11 +132,10 @@ def load_records(path: str, meas_paths: list[str], preloaded = {}) -> list[Union List """ needed_data: dict[str, list[str]] = {} - for mpath in meas_paths: - file = mpath.split("::")[0] + for rpath in record_paths: + file, key = record2name_key(rpath) if file not in needed_data.keys(): needed_data[file] = [] - key = mpath.split("::")[1] needed_data[file].append(key) returned_data: list = [] for file in needed_data.keys(): @@ -153,19 +153,6 @@ def load_records(path: str, meas_paths: list[str], preloaded = {}) -> list[Union return returned_data -def cache_dir(path, file): - cache_path_list = [path] - cache_path_list.append(".cache") - cache_path_list.extend(file.split("/")[1:]) - cache_path = "/".join(cache_path_list) - return cache_path - - -def cache_path(path, file, key): - cache_path = os.path.join(cache_dir(path, file), key) - return cache_path - - def preload(path: str, file: str): dl.get(os.path.join(path, file), dataset=path) filedict = pj.load_json_dict(os.path.join(path, file)) @@ -196,7 +183,3 @@ def drop_record(path: str, meas_path: str): else: raise ValueError("This measurement does not exist as a file!") -def drop_cache(path: str): - cache_dir = os.path.join(path, ".cache") - for f in os.listdir(cache_dir): - shutil.rmtree(os.path.join(cache_dir, f)) diff --git a/corrlib/tools.py b/corrlib/tools.py index 3ac8bfe..44697cc 100644 --- a/corrlib/tools.py +++ b/corrlib/tools.py @@ -16,3 +16,9 @@ def m2k(m): def k2m(k): return (1/(2*k))-4 + + +def record2name_key(record_path: str): + file = record_path.split("::")[0] + key = record_path.split("::")[1] + return file, key \ No newline at end of file