refactor/data_backend #12

Merged
jkuhl merged 36 commits from refactor/data_backend into develop 2025-12-04 15:47:45 +01:00
4 changed files with 13 additions and 12 deletions
Showing only changes of commit 2537fea06c - Show all commits

roll out save replacement
Some checks failed
Pytest / pytest (3.13) (push) Has been cancelled
Pytest / pytest (3.14) (push) Has been cancelled
Ruff / ruff (push) Waiting to run
Mypy / mypy (push) Successful in 45s
Pytest / pytest (3.12) (push) Has been cancelled

Justus Kuhlmann 2025-12-04 12:29:31 +01:00
Signed by: jkuhl
GPG key ID: 00ED992DD79B85A6

View file

@ -1,5 +1,5 @@
import os
import datalad.api as dl
from .tracker import save
import git
GITMODULES_FILE = '.gitmodules'
@ -40,5 +40,6 @@ def move_submodule(repo_path: str, old_path: str, new_path: str) -> None:
repo = git.Repo(repo_path)
repo.git.add('.gitmodules')
# save new state of the dataset
dl.save(repo_path, message=f"Move module from {old_path} to {new_path}", dataset=repo_path)
save(repo_path, message=f"Move module from {old_path} to {new_path}", files=['.gitmodules', repo_path])
return

View file

@ -6,7 +6,7 @@ from .git_tools import move_submodule
import shutil
from .find import _project_lookup_by_id
from .tools import list2str, str2list
from .tracker import get
from .tracker import get, save
from typing import Union, Optional
@ -43,7 +43,7 @@ def create_project(path: str, uuid: str, owner: Union[str, None]=None, tags: Uni
c.execute("INSERT INTO projects (id, aliases, customTags, owner, code, created_at, updated_at) VALUES (?, ?, ?, ?, ?, datetime('now'), datetime('now'))", (uuid, alias_str, tag_str, owner, code))
conn.commit()
conn.close()
dl.save(db, message="Added entry for project " + uuid + " to database", dataset=path)
save(path, message="Added entry for project " + uuid + " to database", files=["backlogger.db"])
def update_project_data(path: str, uuid: str, prop: str, value: Union[str, None] = None) -> None:
@ -79,7 +79,7 @@ def update_aliases(path: str, uuid: str, aliases: list[str]) -> None:
alias_str = list2str(new_alias_list)
dl.unlock(db, dataset=path)
update_project_data(path, uuid, "aliases", alias_str)
dl.save(db, dataset=path)
save(path, message="Updated aliases for project " + uuid, files=["backlogger.db"])
return
@ -109,11 +109,11 @@ def import_project(path: str, url: str, owner: Union[str, None]=None, tags: Opti
in order to receive a uuid and have a consistent interface.
"""
tmp_path = path + '/projects/tmp'
tmp_path = os.path.join(path, 'projects/tmp')
if not isDataset:
dl.create(tmp_path, dataset=path)
shutil.copytree(url + "/*", path + '/projects/tmp/')
dl.save(tmp_path, dataset=path)
save(path, message="Created temporary project dataset", files=['projects/tmp'])
else:
dl.install(path=tmp_path, source=url, dataset=path)
tmp_ds = dl.Dataset(tmp_path)
@ -128,7 +128,7 @@ def import_project(path: str, url: str, owner: Union[str, None]=None, tags: Opti
create_project(path, uuid, owner, tags, aliases, code)
move_submodule(path, 'projects/tmp', 'projects/' + uuid)
os.mkdir(path + '/import_scripts/' + uuid)
dl.save([db, path + '/projects/' + uuid], message="Import project from " + url, dataset=path)
save(path, message="Import project from " + url, files=['projects/' + uuid, 'backlogger.db'])
else:
dl.drop(tmp_path, reckless='kill')
shutil.rmtree(tmp_path)

View file

@ -8,7 +8,7 @@ from typing import Union
from pyerrors import Obs, Corr, dump_object, load_object
from hashlib import sha256
from .tools import cached
from .tracker import get
from .tracker import get, save
import shutil
from typing import Any
@ -96,7 +96,7 @@ def write_measurement(path: str, ensemble: str, measurement: dict[str, dict[str,
pj.dump_dict_to_json(known_meas, file)
files.append(path + '/backlogger.db')
conn.close()
dl.save(files, message="Add measurements to database", dataset=path)
save(path, message="Add measurements to database", files=files)
def load_record(path: str, meas_path: str) -> Union[Corr, Obs]:
@ -196,7 +196,7 @@ def drop_record(path: str, meas_path: str) -> None:
del known_meas[sub_key]
dl.unlock(file, dataset=path)
pj.dump_dict_to_json(known_meas, file)
dl.save([db, file], message="Drop measurements to database", dataset=path)
save(path, message="Drop measurements to database", files=[db, file])
return
else:
raise ValueError("This measurement does not exist as a file!")

View file

@ -39,7 +39,7 @@ def init(path: str) -> None:
if tracker == 'datalad':
dl.create(path)
elif tracker == 'None':
os.path.makedirs(path, exist_ok=True)
os.makedirs(path, exist_ok=True)
else:
raise ValueError(f"Tracker {tracker} is not supported.")
return