Merge pull request 'refactor/data_backend' (#12) from refactor/data_backend into develop

Reviewed-on: https://www.kuhl-mann.de/git/git/jkuhl/corrlib/pulls/12

.github/workflows/mypy.yaml

@@ -0,0 +1,30 @@
+name: Mypy
+
+on:
+  push:
+  pull_request:
+  workflow_dispatch:
+
+jobs:
+  mypy:
+    runs-on: ubuntu-latest
+    env:
+      UV_CACHE_DIR: /tmp/.uv-cache
+    steps:
+      - name: Install git-annex
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y git-annex
+      - name: Check out the repository
+        uses: https://github.com/RouxAntoine/checkout@v4.1.8
+        with:
+          show-progress: true
+      - name: Install uv
+        uses: astral-sh/setup-uv@v7
+        with:
+          python-version: ${{ matrix.python-version }}
+          enable-cache: true
+      - name: Install corrlib
+        run: uv sync --locked --all-extras --dev --python "3.12"
+      - name: Run tests
+        run: uv run mypy corrlib

.github/workflows/pytest.yaml

@@ -0,0 +1,39 @@
+name: Pytest
+
+on:
+  push:
+  pull_request:
+  workflow_dispatch:
+  schedule:
+    - cron: '0 4 1 * *'
+
+jobs:
+  pytest:
+    strategy:
+      matrix:
+        python-version:
+          - "3.12"
+          - "3.13"
+          - "3.14"
+
+    runs-on: ubuntu-latest
+    env:
+      UV_CACHE_DIR: /tmp/.uv-cache
+    steps:
+      - name: Install git-annex
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y git-annex
+      - name: Check out the repository
+        uses: https://github.com/RouxAntoine/checkout@v4.1.8
+        with:
+          show-progress: true
+      - name: Install uv
+        uses: astral-sh/setup-uv@v7
+        with:
+          python-version: ${{ matrix.python-version }}
+          enable-cache: true
+      - name: Install corrlib
+        run: uv sync --locked --all-extras --dev --python ${{ matrix.python-version }}
+      - name: Run tests
+        run: uv run pytest --cov=corrlib tests

.github/workflows/ruff.yaml

@@ -0,0 +1,30 @@
+name: Ruff
+
+on:
+  push:
+  pull_request:
+  workflow_dispatch:
+
+jobs:
+  ruff:
+
+    runs-on: ubuntu-latest
+    env:
+      UV_CACHE_DIR: /tmp/.uv-cache
+    steps:
+      - name: Install git-annex
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y git-annex
+      - name: Check out the repository
+        uses: https://github.com/RouxAntoine/checkout@v4.1.8
+        with:
+          show-progress: true
+      - name: Install uv
+        uses: astral-sh/setup-uv@v7
+        with:
+          enable-cache: true
+      - name: Install corrlib
+        run: uv sync --locked --all-extras --dev --python "3.12"
+      - name: Run tests
+        run: uv run ruff check corrlib

.gitignore

@@ -2,3 +2,7 @@ pyerrors_corrlib.egg-info
 __pycache__
 *.egg-info
 test.ipynb
+.vscode
+.venv
+.pytest_cache
+.coverage

.gitmodules

@@ -1,5 +0,0 @@
-[submodule "projects/tmp"]
-	path = projects/tmp
-	url = git@kuhl-mann.de:lattice/charm_SF_data.git
-	datalad-id = 5f402163-77f2-470e-b6f1-64d7bf9f87d4
-	datalad-url = git@kuhl-mann.de:lattice/charm_SF_data.git

corrlib/__init__.py

@@ -15,9 +15,10 @@ For now, we are interested in collecting primary IObservables only, as these are
 
 __app_name__ = "corrlib"
 
-from .main import *
 from .import input as input
-from .initialization import *
-from .meas_io import *
-from .find import *
-from .version import __version__
+from .initialization import create as create
+from .meas_io import load_record as load_record
+from .meas_io import load_records as load_records
+from .find import find_project as find_project
+from .find import find_record as find_record
+from .find import list_projects as list_projects

corrlib/__main__.py

@@ -1,8 +1,9 @@
 from corrlib import cli, __app_name__
 
 
-def main():
+def main() -> None:
     cli.app(prog_name=__app_name__)
+    return
 
 
 if __name__ == "__main__":

corrlib/cli.py

@@ -1,6 +1,6 @@
 from typing import Optional
 import typer
-from corrlib import __app_name__, __version__
+from corrlib import __app_name__
 from .initialization import create
 from .toml import import_tomls, update_project, reimport_project
 from .find import find_record, list_projects
@@ -8,6 +8,7 @@ from .tools import str2list
 from .main import update_aliases
 from .meas_io import drop_cache as mio_drop_cache
 import os
+from importlib.metadata import version
 
 
 app = typer.Typer()
@@ -15,7 +16,7 @@ app = typer.Typer()
 
 def _version_callback(value: bool) -> None:
     if value:
-        typer.echo(f"{__app_name__} v{__version__}")
+        print(__app_name__, version(__app_name__))
         raise typer.Exit()
 
 
@@ -25,9 +26,9 @@ def update(
         str('./corrlib'),
         "--dataset",
         "-d",
-    ),
+        ),
     uuid: str = typer.Argument(),
-) -> None:
+    ) -> None:
     """
     Update a project by it's UUID.
     """
@@ -42,7 +43,7 @@ def list(
         "-d",
     ),
     entities: str = typer.Argument('ensembles'),
-) -> None:
+    ) -> None:
     """
     List entities. (ensembles, projects)
     """
@@ -71,10 +72,10 @@ def alias_add(
         str('./corrlib'),
         "--dataset",
         "-d",
-    ),
+        ),
     uuid: str = typer.Argument(),
     alias: str = typer.Argument(),
-) -> None:
+    ) -> None:
     """
     Add an alias to a project UUID.
     """
@@ -89,11 +90,11 @@ def find(
         str('./corrlib'),
         "--dataset",
         "-d",
-    ),
+        ),
     ensemble: str = typer.Argument(),
     corr: str = typer.Argument(),
     code: str = typer.Argument(),
-) -> None:
+    ) -> None:
     """
     Find a record in the backlog at hand. Through specifying it's ensemble and the measured correlator.
     """
@@ -107,15 +108,15 @@ def importer(
         str('./corrlib'),
         "--dataset",
         "-d",
-    ),
+        ),
     files: str = typer.Argument(
-    ),
+        ),
     copy_file: bool = typer.Option(
         bool(True),
         "--save",
         "-s",
-    ),
-) -> None:
+        ),
+    ) -> None:
     """
     Import a project from a .toml-file via CLI.
     """
@@ -151,12 +152,17 @@ def init(
         str('./corrlib'),
         "--dataset",
         "-d",
-    ),
-) -> None:
+        ),
+    tracker: str = typer.Option(
+        str('datalad'),
+        "--tracker",
+        "-t",
+        ),
+    ) -> None:
     """
     Initialize a new backlog-database.
     """
-    create(path)
+    create(path, tracker)
     return
 
 
@@ -166,8 +172,8 @@ def drop_cache(
         str('./corrlib'),
         "--dataset",
         "-d",
-    ),
-) -> None:
+        ),
+    ) -> None:
     """
     Drop the currect cache directory of the dataset.
     """
@@ -184,6 +190,6 @@ def main(
         help="Show the application's version and exit.",
         callback=_version_callback,
         is_eager=True,
-    )
-) -> None:
+        )
+    ) -> None:
     return

corrlib/find.py

@@ -1,29 +1,31 @@
 import sqlite3
-import datalad.api as dl
 import os
 import json
 import pandas as pd
 import numpy as np
 from .input.implementations import codes
-from .tools import k2m, get_file
+from .tools import k2m, get_db_file
+from .tracker import get
+from typing import Any, Optional
+
 # this will implement the search functionality
 
 
-def _project_lookup_by_alias(db, alias):
+def _project_lookup_by_alias(db: str, alias: str) -> str:
     # this will lookup the project name based on the alias
     conn = sqlite3.connect(db)
     c = conn.cursor()
     c.execute(f"SELECT * FROM 'projects' WHERE alias = '{alias}'")
     results = c.fetchall()
     conn.close()
-    if len(results) > 1:
+    if len(results)>1:
         print("Error: multiple projects found with alias " + alias)
     elif len(results) == 0:
         raise Exception("Error: no project found with alias " + alias)
-    return results[0][0]
+    return str(results[0][0])
 
 
-def _project_lookup_by_id(db, uuid):
+def _project_lookup_by_id(db: str, uuid: str) -> list[tuple[str, str]]:
     conn = sqlite3.connect(db)
     c = conn.cursor()
     c.execute(f"SELECT * FROM 'projects' WHERE id = '{uuid}'")
@@ -32,7 +34,8 @@ def _project_lookup_by_id(db, uuid):
     return results
 
 
-def _db_lookup(db, ensemble, correlator_name,code, project=None,  parameters=None, created_before=None, created_after=None, updated_before=None, updated_after=None, revision=None):
+def _db_lookup(db: str, ensemble: str, correlator_name: str, code: str, project: Optional[str]=None,  parameters: Optional[str]=None,
+               created_before: Optional[str]=None, created_after: Optional[Any]=None, updated_before: Optional[Any]=None, updated_after: Optional[Any]=None) -> pd.DataFrame:
     project_str = project
 
     search_expr = f"SELECT * FROM 'backlogs' WHERE name = '{correlator_name}' AND ensemble = '{ensemble}'"
@@ -56,7 +59,7 @@ def _db_lookup(db, ensemble, correlator_name,code, project=None,  parameters=Non
     return results
 
 
-def sfcf_filter(results, **kwargs):
+def sfcf_filter(results: pd.DataFrame, **kwargs: Any) -> pd.DataFrame:
     drops = []
     for ind in range(len(results)):
         result = results.iloc[ind]
@@ -139,27 +142,30 @@ def sfcf_filter(results, **kwargs):
     return results.drop(drops)
 
 
-def find_record(path, ensemble, correlator_name, code, project=None, parameters=None, created_before=None, created_after=None, updated_before=None, updated_after=None, revision=None, **kwargs):
-    db = path + '/backlogger.db'
+def find_record(path: str, ensemble: str, correlator_name: str, code: str, project: Optional[str]=None, parameters: Optional[str]=None,
+                created_before: Optional[str]=None, created_after: Optional[str]=None, updated_before: Optional[str]=None, updated_after: Optional[str]=None, revision: Optional[str]=None, **kwargs: Any) -> pd.DataFrame:
+    db_file = get_db_file(path)
+    db = os.path.join(path, db_file)
     if code not in codes:
         raise ValueError("Code " + code + "unknown, take one of the following:" + ", ".join(codes))
-    get_file(path, "backlogger.db")
-    results = _db_lookup(db, ensemble, correlator_name,code, project, parameters=parameters, created_before=created_before, created_after=created_after, updated_before=updated_before, updated_after=updated_after, revision=revision)
+    get(path, db_file)
+    results = _db_lookup(db, ensemble, correlator_name,code, project, parameters=parameters, created_before=created_before, created_after=created_after, updated_before=updated_before, updated_after=updated_after)
     if code == "sfcf":
         results = sfcf_filter(results, **kwargs)
     print("Found " + str(len(results)) + " result" + ("s" if len(results)>1 else ""))
     return results.reset_index()
 
 
-def find_project(path, name):
-    get_file(path, "backlogger.db")
-    return _project_lookup_by_alias(os.path.join(path, "backlogger.db"), name)
+def find_project(path: str, name: str) -> str:
+    db_file = get_db_file(path)
+    get(path, db_file)
+    return _project_lookup_by_alias(os.path.join(path, db_file), name)
 
 
-def list_projects(path):
-    db = path + '/backlogger.db'
-    get_file(path, "backlogger.db")
-    conn = sqlite3.connect(db)
+def list_projects(path: str) -> list[tuple[str, str]]:
+    db_file = get_db_file(path)
+    get(path, db_file)
+    conn = sqlite3.connect(os.path.join(path, db_file))
     c = conn.cursor()
     c.execute("SELECT id,aliases FROM projects")
     results = c.fetchall()

corrlib/git_tools.py

@@ -1,11 +1,11 @@
 import os
-import datalad.api as dl
+from .tracker import save
 import git
 
 GITMODULES_FILE = '.gitmodules'
 
 
-def move_submodule(repo_path, old_path, new_path):
+def move_submodule(repo_path: str, old_path: str, new_path: str) -> None:
     """
     Move a submodule to a new location.
 
@@ -40,4 +40,6 @@ def move_submodule(repo_path, old_path, new_path):
     repo = git.Repo(repo_path)
     repo.git.add('.gitmodules')
     # save new state of the dataset
-    dl.save(repo_path, message=f"Move module from {old_path} to {new_path}", dataset=repo_path)
+    save(repo_path, message=f"Move module from {old_path} to {new_path}", files=['.gitmodules', repo_path])
+
+    return

corrlib/initialization.py

@@ -1,9 +1,10 @@
+from configparser import ConfigParser
 import sqlite3
-import datalad.api as dl
 import os
+from .tracker import save, init
 
 
-def _create_db(db):
+def _create_db(db: str) -> None:
     """
     Create the database file and the table.
 
@@ -32,21 +33,55 @@ def _create_db(db):
                   updated_at TEXT)''')
     conn.commit()
     conn.close()
+    return
 
 
-def create(path):
+def _create_config(path: str, tracker: str, cached: bool) -> ConfigParser:
+    """
+    Create the config file for backlogger.
+
+    """
+    config = ConfigParser()
+    config['core'] = {
+        'version': '1.0',
+        'tracker': tracker,
+        'cached': str(cached),
+    }
+    config['paths'] = {
+        'db': 'backlogger.db',
+        'projects_path': 'projects',
+        'archive_path': 'archive',
+        'toml_imports_path': 'toml_imports',
+        'import_scripts_path': 'import_scripts',
+    }
+    return config
+
+
+def _write_config(path: str, config: ConfigParser) -> None:
+    """
+    Write the config file to disk.
+    """
+    with open(os.path.join(path, '.corrlib'), 'w') as configfile:
+        config.write(configfile)
+    return
+
+
+def create(path: str, tracker: str = 'datalad', cached: bool = True) -> None:
     """
     Create folder of backlogs.
 
     """
-    dl.create(path)
-    _create_db(path + '/backlogger.db')
-    os.chmod(path + '/backlogger.db', 0o666)  # why does this not work?
-    os.makedirs(path + '/projects')
-    os.makedirs(path + '/archive')
-    os.makedirs(path + '/toml_imports')
-    os.makedirs(path + '/import_scripts/template.py')
-    with open(path + "/.gitignore", "w") as fp:
+    config = _create_config(path, tracker, cached)
+    init(path, tracker)
+    _write_config(path, config)
+    _create_db(os.path.join(path, config['paths']['db']))
+    os.chmod(os.path.join(path, config['paths']['db']), 0o666)
+    os.makedirs(os.path.join(path, config['paths']['projects_path']))
+    os.makedirs(os.path.join(path, config['paths']['archive_path']))
+    os.makedirs(os.path.join(path, config['paths']['toml_imports_path']))
+    os.makedirs(os.path.join(path, config['paths']['import_scripts_path'], 'template.py'))
+    with open(os.path.join(path, ".gitignore"), "w") as fp:
         fp.write(".cache")
         fp.close()
-    dl.save(path, dataset=path, message="Initialize backlogger directory.")
+    save(path, message="Initialized correlator library")
+    return

corrlib/input/__init__.py

@@ -2,6 +2,6 @@
 Import functions for different codes.
 """
 
-from . import sfcf
-from . import openQCD
-from . import implementations
+from . import sfcf as sfcf
+from . import openQCD as openQCD
+from . import implementations as implementations

corrlib/input/openQCD.py

@@ -2,7 +2,7 @@ import pyerrors.input.openQCD as input
 import datalad.api as dl
 import os
 import fnmatch
-from typing import Any
+from typing import Any, Optional
 
 
 def read_ms1_param(path: str, project: str, file_in_project: str) -> dict[str, Any]:
@@ -67,7 +67,7 @@ def read_ms3_param(path: str, project: str, file_in_project: str) -> dict[str, A
     return param
 
 
-def read_rwms(path: str, project: str, dir_in_project: str, param: dict[str, Any], prefix: str, postfix: str="ms1", version: str='2.0', names: list[str]=None, files: list[str]=None) -> dict[str, Any]:
+def read_rwms(path: str, project: str, dir_in_project: str, param: dict[str, Any], prefix: str, postfix: str="ms1", version: str='2.0', names: Optional[list[str]]=None, files: Optional[list[str]]=None) -> dict[str, Any]:
     dataset = os.path.join(path, "projects", project)
     directory = os.path.join(dataset, dir_in_project)
     if files is None:
@@ -94,7 +94,7 @@ def read_rwms(path: str, project: str, dir_in_project: str, param: dict[str, Any
     return rw_dict
 
 
-def extract_t0(path: str, project: str, dir_in_project: str, param: dict[str, Any], prefix: str, dtr_read: int, xmin: int, spatial_extent: int, fit_range: int = 5, postfix: str=None, names: list[str]=None, files: list[str]=None) -> dict[str, Any]:
+def extract_t0(path: str, project: str, dir_in_project: str, param: dict[str, Any], prefix: str, dtr_read: int, xmin: int, spatial_extent: int, fit_range: int = 5, postfix: str="", names: Optional[list[str]]=None, files: Optional[list[str]]=None) -> dict[str, Any]:
     dataset = os.path.join(path, "projects", project)
     directory = os.path.join(dataset, dir_in_project)
     if files is None:
@@ -132,7 +132,7 @@ def extract_t0(path: str, project: str, dir_in_project: str, param: dict[str, An
     return t0_dict
 
 
-def extract_t1(path: str, project: str, dir_in_project: str, param: dict[str, Any], prefix: str, dtr_read: int, xmin: int, spatial_extent: int, fit_range: int = 5, postfix: str = None, names: list[str]=None, files: list[str]=None) -> dict[str, Any]:
+def extract_t1(path: str, project: str, dir_in_project: str, param: dict[str, Any], prefix: str, dtr_read: int, xmin: int, spatial_extent: int, fit_range: int = 5, postfix: str = "", names: Optional[list[str]]=None, files: Optional[list[str]]=None) -> dict[str, Any]:
     directory = os.path.join(path, "projects", project, dir_in_project)
     if files is None:
         files = []

corrlib/input/sfcf.py

@@ -5,7 +5,7 @@ import os
 from typing import Any
 
 
-bi_corrs: list = ["f_P", "fP", "f_p",
+bi_corrs: list[str] = ["f_P", "fP", "f_p",
             "g_P", "gP", "g_p",
             "fA0", "f_A", "f_a",
             "gA0", "g_A", "g_a",
@@ -43,7 +43,7 @@ bi_corrs: list = ["f_P", "fP", "f_p",
             "l3A2", "l3_A2", "g_av23",
             ]
 
-bb_corrs: list = [
+bb_corrs: list[str] = [
             'F1',
             'F_1',
             'f_1',
@@ -64,7 +64,7 @@ bb_corrs: list = [
             'F_sPdP_d',
             ]
 
-bib_corrs: list = [
+bib_corrs: list[str] = [
             'F_V0',
             'K_V0',
             ]
@@ -184,7 +184,7 @@ def read_param(path: str, project: str, file_in_project: str) -> dict[str, Any]:
     return params
 
 
-def _map_params(params: dict, spec_list: list) -> dict[str, Any]:
+def _map_params(params: dict[str, Any], spec_list: list[str]) -> dict[str, Any]:
     """
     Map the extracted parameters to the extracted data.
 
@@ -228,7 +228,7 @@ def _map_params(params: dict, spec_list: list) -> dict[str, Any]:
     return new_specs
 
 
-def get_specs(key, parameters, sep='/') -> str:
+def get_specs(key: str, parameters: dict[str, Any], sep: str = '/') -> str:
     key_parts = key.split(sep)
     if corr_types[key_parts[0]] == 'bi':
         param = _map_params(parameters, key_parts[1:-1])
@@ -238,7 +238,7 @@ def get_specs(key, parameters, sep='/') -> str:
     return s
 
 
-def read_data(path, project, dir_in_project, prefix, param, version='1.0c', cfg_seperator='n', sep='/', **kwargs) -> dict:
+def read_data(path: str, project: str, dir_in_project: str, prefix: str, param: dict[str, Any], version: str = '1.0c', cfg_seperator: str = 'n', sep: str = '/', **kwargs: Any) -> dict[str, Any]:
     """
     Extract the data from the sfcf file.
 

corrlib/main.py

@@ -5,11 +5,12 @@ import os
 from .git_tools import move_submodule
 import shutil
 from .find import _project_lookup_by_id
-from .tools import list2str, str2list, get_file
-from typing import Union
+from .tools import list2str, str2list, get_db_file
+from .tracker import get, save, unlock, clone, drop
+from typing import Union, Optional
 
 
-def create_project(path: str, uuid: str, owner: Union[str, None]=None, tags: Union[str, None]=None, aliases: Union[str, None]=None, code: Union[str, None]=None):
+def create_project(path: str, uuid: str, owner: Union[str, None]=None, tags: Union[list[str], None]=None, aliases: Union[list[str], None]=None, code: Union[str, None]=None) -> None:
     """
     Create a new project entry in the database.
 
@@ -24,30 +25,32 @@ def create_project(path: str, uuid: str, owner: Union[str, None]=None, tags: Uni
     code: str (optional)
         The code that was used to create the measurements.
     """
-    db = path + "/backlogger.db"
-    get_file(path, "backlogger.db")
+    db_file = get_db_file(path)
+    db = os.path.join(path, db_file)
+    get(path, db_file)
     conn = sqlite3.connect(db)
     c = conn.cursor()
     known_projects = c.execute("SELECT * FROM projects WHERE id=?", (uuid,))
     if known_projects.fetchone():
         raise ValueError("Project already imported, use update_project() instead.")
 
-    dl.unlock(db, dataset=path)
-    alias_str = None
+    unlock(path, db_file)
+    alias_str = ""
     if aliases is not None:
         alias_str = list2str(aliases)
-    tag_str = None
+    tag_str = ""
     if tags is not None:
         tag_str = list2str(tags)
     c.execute("INSERT INTO projects (id, aliases, customTags, owner, code, created_at, updated_at) VALUES (?, ?, ?, ?, ?, datetime('now'), datetime('now'))", (uuid, alias_str, tag_str, owner, code))
     conn.commit()
     conn.close()
-    dl.save(db, message="Added entry for project " + uuid + " to database", dataset=path)
+    save(path, message="Added entry for project " + uuid + " to database", files=[db_file])
 
 
-def update_project_data(path, uuid, prop, value = None):
-    get_file(path, "backlogger.db")
-    conn = sqlite3.connect(os.path.join(path, "backlogger.db"))
+def update_project_data(path: str, uuid: str, prop: str, value: Union[str, None] = None) -> None:
+    db_file = get_db_file(path)
+    get(path, db_file)
+    conn = sqlite3.connect(os.path.join(path, db_file))
     c = conn.cursor()
     c.execute(f"UPDATE projects SET '{prop}' = '{value}' WHERE id == '{uuid}'")
     conn.commit()
@@ -55,9 +58,10 @@ def update_project_data(path, uuid, prop, value = None):
     return
 
 
-def update_aliases(path: str, uuid: str, aliases: list[str]):
-    db = os.path.join(path, "backlogger.db")
-    get_file(path, "backlogger.db")
+def update_aliases(path: str, uuid: str, aliases: list[str]) -> None:
+    db_file = get_db_file(path)
+    db = os.path.join(path, db_file)
+    get(path, db_file)
     known_data = _project_lookup_by_id(db, uuid)[0]
     known_aliases = known_data[1]
 
@@ -76,13 +80,13 @@ def update_aliases(path: str, uuid: str, aliases: list[str]):
 
     if not len(new_alias_list) == len(known_alias_list):
         alias_str = list2str(new_alias_list)
-        dl.unlock(db, dataset=path)
+        unlock(path, db_file)
         update_project_data(path, uuid, "aliases", alias_str)
-        dl.save(db, dataset=path)
+        save(path, message="Updated aliases for project " + uuid, files=[db_file])
     return
 
 
-def import_project(path: str, url: str, owner: Union[str, None]=None, tags: Union[str, None]=None, aliases: Union[str, None]=None, code: Union[str, None]=None, isDataset: bool=True):
+def import_project(path: str, url: str, owner: Union[str, None]=None, tags: Optional[list[str]]=None, aliases: Optional[list[str]]=None, code: Optional[str]=None, isDataset: bool=True) -> str:
     """
     Parameters
     ----------
@@ -108,26 +112,21 @@ def import_project(path: str, url: str, owner: Union[str, None]=None, tags: Unio
         in order to receive a uuid and have a consistent interface.
 
     """
-    tmp_path = path + '/projects/tmp'
-    if not isDataset:
-        dl.create(tmp_path, dataset=path)
-        shutil.copytree(url + "/*", path + '/projects/tmp/')
-        dl.save(tmp_path, dataset=path)
-    else:
-        dl.install(path=tmp_path, source=url, dataset=path)
+    tmp_path = os.path.join(path, 'projects/tmp')
+    clone(path, source=url, target=tmp_path)
     tmp_ds = dl.Dataset(tmp_path)
     conf = dlc.ConfigManager(tmp_ds)
-    uuid = conf.get("datalad.dataset.id")
+    uuid = str(conf.get("datalad.dataset.id"))
     if not uuid:
         raise ValueError("The dataset does not have a uuid!")
     if not os.path.exists(path + "/projects/" + uuid):
-        db = path + "/backlogger.db"
-        get_file(path, "backlogger.db")
-        dl.unlock(db, dataset=path)
+        db_file = get_db_file(path)
+        get(path, db_file)
+        unlock(path, db_file)
         create_project(path, uuid, owner, tags, aliases, code)
         move_submodule(path, 'projects/tmp', 'projects/' + uuid)
         os.mkdir(path + '/import_scripts/' + uuid)
-        dl.save([db, path + '/projects/' + uuid], message="Import project from " + url, dataset=path)
+        save(path, message="Import project from " + url, files=['projects/' + uuid, db_file])
     else:
         dl.drop(tmp_path, reckless='kill')
         shutil.rmtree(tmp_path)
@@ -142,9 +141,10 @@ def import_project(path: str, url: str, owner: Union[str, None]=None, tags: Unio
     return uuid
 
 
-def drop_project_data(path: str, uuid: str, path_in_project: str = ""):
+def drop_project_data(path: str, uuid: str, path_in_project: str = "") -> None:
     """
-    Drop (parts of) a prject to free up diskspace
+    Drop (parts of) a project to free up diskspace
     """
-    dl.drop(path + "/projects/" + uuid + "/" + path_in_project)
+    drop(path + "/projects/" + uuid + "/" + path_in_project)
+    return
 

corrlib/meas_io.py

@@ -1,17 +1,18 @@
 from pyerrors.input import json as pj
 import os
-import datalad.api as dl
 import sqlite3
 from .input import sfcf,openQCD
 import json
 from typing import Union
 from pyerrors import Obs, Corr, dump_object, load_object
 from hashlib import sha256
-from .tools import cached, get_file
+from .tools import get_db_file, cache_enabled
+from .tracker import get, save, unlock
 import shutil
+from typing import Any
 
 
-def write_measurement(path, ensemble, measurement, uuid, code, parameter_file=None):
+def write_measurement(path: str, ensemble: str, measurement: dict[str, dict[str, dict[str, Any]]], uuid: str, code: str, parameter_file: str) -> None:
     """
     Write a measurement to the backlog.
     If the file for the measurement already exists, update the measurement.
@@ -27,9 +28,10 @@ def write_measurement(path, ensemble, measurement, uuid, code, parameter_file=No
     uuid: str
         The uuid of the project.
     """
-    db = os.path.join(path, 'backlogger.db')
-    get_file(path, "backlogger.db")
-    dl.unlock(db, dataset=path)
+    db_file = get_db_file(path)
+    db = os.path.join(path, db_file)
+    get(path, db_file)
+    unlock(path, db_file)
     conn = sqlite3.connect(db)
     c = conn.cursor()
     files = []
@@ -42,7 +44,7 @@ def write_measurement(path, ensemble, measurement, uuid, code, parameter_file=No
             os.makedirs(os.path.join(path, '.', 'archive', ensemble, corr))
         else:
             if os.path.exists(file):
-                dl.unlock(file, dataset=path)
+                unlock(path, file_in_archive)
                 known_meas = pj.load_json_dict(file)
         if code == "sfcf":
             parameters = sfcf.read_param(path, uuid, parameter_file)
@@ -92,12 +94,12 @@ def write_measurement(path, ensemble, measurement, uuid, code, parameter_file=No
                                                 (corr, ensemble, code, meas_path, uuid, pars[subkey], parameter_file))
         conn.commit()
         pj.dump_dict_to_json(known_meas, file)
-    files.append(path + '/backlogger.db')
+    files.append(os.path.join(path, db_file))
     conn.close()
-    dl.save(files, message="Add measurements to database", dataset=path)
+    save(path, message="Add measurements to database", files=files)
 
 
-def load_record(path: str, meas_path: str):
+def load_record(path: str, meas_path: str) -> Union[Corr, Obs]:
     """
     Load a list of records by their paths.
 
@@ -116,7 +118,7 @@ def load_record(path: str, meas_path: str):
     return load_records(path, [meas_path])[0]
 
 
-def load_records(path: str, meas_paths: list[str], preloaded = {}) -> list[Union[Corr, Obs]]:
+def load_records(path: str, meas_paths: list[str], preloaded: dict[str, Any] = {}) -> list[Union[Corr, Obs]]:
     """
     Load a list of records by their paths.
 
@@ -138,7 +140,7 @@ def load_records(path: str, meas_paths: list[str], preloaded = {}) -> list[Union
             needed_data[file] = []
         key = mpath.split("::")[1]
         needed_data[file].append(key)
-    returned_data: list = []
+    returned_data: list[Any] = []
     for file in needed_data.keys():
         for key in list(needed_data[file]):
             if os.path.exists(cache_path(path, file, key) + ".p"):
@@ -147,14 +149,14 @@ def load_records(path: str, meas_paths: list[str], preloaded = {}) -> list[Union
                 if file not in preloaded:
                     preloaded[file] = preload(path, file)
                 returned_data.append(preloaded[file][key])
-                if cached:
+                if cache_enabled(path):
                     if not os.path.exists(cache_dir(path, file)):
                         os.makedirs(cache_dir(path, file))
                     dump_object(preloaded[file][key], cache_path(path, file, key))
     return returned_data
 
 
-def cache_dir(path, file):
+def cache_dir(path: str, file: str) -> str:
     cache_path_list = [path]
     cache_path_list.append(".cache")
     cache_path_list.extend(file.split("/")[1:])
@@ -162,25 +164,26 @@ def cache_dir(path, file):
     return cache_path
 
 
-def cache_path(path, file, key):
+def cache_path(path: str, file: str, key: str) -> str:
     cache_path = os.path.join(cache_dir(path, file), key)
     return cache_path
 
 
-def preload(path: str, file: str):
-    get_file(path, file)
-    filedict = pj.load_json_dict(os.path.join(path, file))
+def preload(path: str, file: str) -> dict[str, Any]:
+    get(path, file)
+    filedict: dict[str, Any] = pj.load_json_dict(os.path.join(path, file))
     print("> read file")
     return filedict
 
 
-def drop_record(path: str, meas_path: str):
+def drop_record(path: str, meas_path: str) -> None:
     file_in_archive = meas_path.split("::")[0]
     file = os.path.join(path, file_in_archive)
-    db = os.path.join(path, 'backlogger.db')
-    get_file(path, 'backlogger.db')
+    db_file = get_db_file(path)
+    db = os.path.join(path, db_file)
+    get(path, db_file)
     sub_key = meas_path.split("::")[1]
-    dl.unlock(db, dataset=path)
+    unlock(path, db_file)
     conn = sqlite3.connect(db)
     c = conn.cursor()
     if c.execute("SELECT * FROM backlogs WHERE path = ?", (meas_path, )).fetchone() is not None:
@@ -192,14 +195,16 @@ def drop_record(path: str, meas_path: str):
     known_meas = pj.load_json_dict(file)
     if sub_key in known_meas:
         del known_meas[sub_key]
-        dl.unlock(file, dataset=path)
+        unlock(path, file_in_archive)
         pj.dump_dict_to_json(known_meas, file)
-        dl.save([db, file], message="Drop measurements to database", dataset=path)
+        save(path, message="Drop measurements to database", files=[db, file])
         return
     else:
         raise ValueError("This measurement does not exist as a file!")
 
-def drop_cache(path: str):
+
+def drop_cache(path: str) -> None:
     cache_dir = os.path.join(path, ".cache")
     for f in os.listdir(cache_dir):
         shutil.rmtree(os.path.join(cache_dir, f))
+    return

corrlib/toml.py

@@ -10,21 +10,26 @@ the import of projects via TOML.
 
 import tomllib as toml
 import shutil
+
+import datalad.api as dl
+from .tracker import save
 from .input import sfcf, openQCD
 from .main import import_project, update_aliases
 from .meas_io import write_measurement
-import datalad.api as dl
 import os
 from .input.implementations import codes as known_codes
+from typing import Any
 
-def replace_string(string: str, name: str, val: str):
+
+def replace_string(string: str, name: str, val: str) -> str:
     if '{' + name + '}' in string:
         n = string.replace('{' + name + '}', val)
         return n
     else:
         return string
 
-def replace_in_meas(measurements: dict, vars: dict[str, str]):
+
+def replace_in_meas(measurements: dict[str, dict[str, Any]], vars: dict[str, str]) -> dict[str, dict[str, Any]]:
     # replace global variables
     for name, value in vars.items():
         for m in measurements.keys():
@@ -36,7 +41,8 @@ def replace_in_meas(measurements: dict, vars: dict[str, str]):
                         measurements[m][key][i] = replace_string(measurements[m][key][i], name, value)
     return measurements
 
-def fill_cons(measurements, constants):
+
+def fill_cons(measurements: dict[str, dict[str, Any]], constants: dict[str, str]) -> dict[str, dict[str, Any]]:
     for m in measurements.keys():
         for name, val in constants.items():
             if name not in measurements[m].keys():
@@ -44,7 +50,7 @@ def fill_cons(measurements, constants):
     return measurements
 
 
-def check_project_data(d: dict) -> None:
+def check_project_data(d: dict[str, dict[str, str]]) -> None:
     if 'project' not in d.keys() or 'measurements' not in d.keys() or len(list(d.keys())) > 4:
         raise ValueError('There should only be maximally be four keys on the top level, "project" and "measurements" are mandatory, "contants" is optional!')
     project_data = d['project']
@@ -57,7 +63,7 @@ def check_project_data(d: dict) -> None:
     return
 
 
-def check_measurement_data(measurements: dict, code: str) -> None:
+def check_measurement_data(measurements: dict[str, dict[str, str]], code: str) -> None:
     var_names: list[str] = []
     if code == "sfcf":
         var_names = ["path", "ensemble", "param_file", "version", "prefix", "cfg_seperator", "names"]
@@ -91,14 +97,14 @@ def import_toml(path: str, file: str, copy_file: bool=True) -> None:
     with open(file, 'rb') as fp:
         toml_dict = toml.load(fp)
     check_project_data(toml_dict)
-    project: dict = toml_dict['project']
+    project: dict[str, Any] = toml_dict['project']
     if project['code'] not in known_codes:
         raise ValueError('Code' + project['code'] + 'has no import implementation!')
-    measurements: dict = toml_dict['measurements']
+    measurements: dict[str, dict[str, Any]] = toml_dict['measurements']
     measurements = fill_cons(measurements, toml_dict['constants'] if 'constants' in toml_dict else {})
     measurements = replace_in_meas(measurements, toml_dict['replace'] if 'replace' in toml_dict else {})
     check_measurement_data(measurements, project['code'])
-    aliases = project.get('aliases', None)
+    aliases = project.get('aliases', [])
     uuid = project.get('uuid', None)
     if uuid is not None:
         if not os.path.exists(path + "/projects/" + uuid):
@@ -133,29 +139,29 @@ def import_toml(path: str, file: str, copy_file: bool=True) -> None:
                     for rwp in ["integrator", "eps", "ntot", "dnms"]:
                         param[rwp] = "Unknown"
                 param['type'] = 't0'
-                measurement = openQCD.extract_t0(path, uuid, md['path'], param, md["prefix"], md["dtr_read"], md["xmin"], md["spatial_extent"],
-                                                 fit_range=md.get('fit_range', 5), postfix=md.get('postfix', None), names=md.get('names', None), files=md.get('files', None))
+                measurement = openQCD.extract_t0(path, uuid, md['path'], param, str(md["prefix"]), int(md["dtr_read"]), int(md["xmin"]), int(md["spatial_extent"]),
+                                                 fit_range=int(md.get('fit_range', 5)), postfix=str(md.get('postfix', '')), names=md.get('names', []), files=md.get('files', []))
             elif md['measurement'] == 't1':
                 if 'param_file' in md:
                     param = openQCD.read_ms3_param(path, uuid, md['param_file'])
                 param['type'] = 't1'
-                measurement = openQCD.extract_t1(path, uuid, md['path'], param, md["prefix"], md["dtr_read"], md["xmin"], md["spatial_extent"],
-                                                 fit_range=md.get('fit_range', 5), postfix=md.get('postfix', None), names=md.get('names', None), files=md.get('files', None))
+                measurement = openQCD.extract_t1(path, uuid, md['path'], param, str(md["prefix"]), int(md["dtr_read"]), int(md["xmin"]), int(md["spatial_extent"]),
+                                                 fit_range=int(md.get('fit_range', 5)), postfix=str(md.get('postfix', '')), names=md.get('names', []), files=md.get('files', []))
 
-        write_measurement(path, ensemble, measurement, uuid, project['code'], (md['param_file'] if 'param_file' in md else None))
+        write_measurement(path, ensemble, measurement, uuid, project['code'], (md['param_file'] if 'param_file' in md else ''))
 
     if not os.path.exists(os.path.join(path, "toml_imports", uuid)):
         os.makedirs(os.path.join(path, "toml_imports", uuid))
     if copy_file:
         import_file = os.path.join(path, "toml_imports", uuid, file.split("/")[-1])
         shutil.copy(file, import_file)
-        dl.save(import_file, message="Import using " + import_file, dataset=path)
+        save(path, files=[import_file], message="Import using " + import_file)
         print("File copied to " + import_file)
     print("Imported project.")
     return
 
 
-def reimport_project(path, uuid):
+def reimport_project(path: str, uuid: str) -> None:
     """
     Reimport an existing project using the files that are already available for this project.
 
@@ -173,6 +179,7 @@ def reimport_project(path, uuid):
     return
 
 
-def update_project(path, uuid):
+def update_project(path: str, uuid: str) -> None:
     dl.update(how='merge', follow='sibling', dataset=os.path.join(path, "projects", uuid))
     # reimport_project(path, uuid)
+    return

corrlib/tools.py

@@ -1,29 +1,54 @@
 import os
-import datalad.api as dl
+from configparser import ConfigParser
+from typing import Any
+
+CONFIG_FILENAME = ".corrlib"
 
 
-def str2list(string):
+def str2list(string: str) -> list[str]:
     return string.split(",")
 
-def list2str(mylist):
+def list2str(mylist: list[str]) -> str:
     s = ",".join(mylist)
     return s
 
-cached = True
+cached: bool = True
 
-def m2k(m):
+def m2k(m: float) -> float:
     return 1/(2*m+8)
 
 
-def k2m(k):
+def k2m(k: float) -> float:
     return (1/(2*k))-4
 
 
-def get_file(path, file):
-    if file == "backlogger.db":
-        print("Downloading database...")
-    else:
-        print("Downloading data...")
-    dl.get(os.path.join(path, file), dataset=path)
-    print("> downloaded file")
+def set_config(path: str, section: str, option: str, value: Any) -> None:
+    config_path = os.path.join(path, '.corrlib')
+    config = ConfigParser()
+    if os.path.exists(config_path):
+        config.read(config_path)
+    if not config.has_section(section):
+        config.add_section(section)
+    config.set(section, option, value)
+    with open(config_path, 'w') as configfile:
+        config.write(configfile)
+    return
 
+
+def get_db_file(path: str) -> str:
+    config_path = os.path.join(path, CONFIG_FILENAME)
+    config = ConfigParser()
+    if os.path.exists(config_path):
+        config.read(config_path)
+    db_file = config.get('paths', 'db', fallback='backlogger.db')
+    return db_file
+
+
+def cache_enabled(path: str) -> bool:
+    config_path = os.path.join(path, CONFIG_FILENAME)
+    config = ConfigParser()
+    if os.path.exists(config_path):
+        config.read(config_path)
+    cached_str = config.get('core', 'cached', fallback='True')
+    cached_bool = cached_str == ('True')
+    return cached_bool

corrlib/tracker.py

@@ -0,0 +1,169 @@
+import os
+from configparser import ConfigParser
+import datalad.api as dl
+from typing import Optional
+import shutil
+from .tools import get_db_file
+
+
+def get_tracker(path: str) -> str:
+    """
+    Get the tracker used in the dataset located at path.
+
+    Parameters
+    ----------
+    path: str
+        The path to the backlogger folder.
+
+    Returns
+    -------
+    tracker: str
+        The tracker used in the dataset.
+    """
+    config_path = os.path.join(path, '.corrlib')
+    config = ConfigParser()
+    if os.path.exists(config_path):
+        config.read(config_path)
+    else:
+        raise FileNotFoundError(f"No config file found in {path}.")
+    tracker = config.get('core', 'tracker', fallback='datalad')
+    return tracker
+
+
+def get(path: str, file: str) -> None:
+    """
+    Wrapper function to get a file from the dataset located at path with the specified tracker.
+
+    Parameters
+    ----------
+    path: str
+        The path to the backlogger folder.
+    file: str
+        The file to get.
+    """
+    tracker = get_tracker(path)
+    if tracker == 'datalad':
+        if file == get_db_file(path):
+            print("Downloading database...")
+        else:
+            print("Downloading data...")
+        dl.get(os.path.join(path, file), dataset=path)
+        print("> downloaded file")
+    elif tracker == 'None':
+        pass
+    else:
+        raise ValueError(f"Tracker {tracker} is not supported.")
+    return
+
+
+def save(path: str, message: str, files: Optional[list[str]]=None) -> None:
+    """
+    Wrapper function to save a file to the dataset located at path with the specified tracker.
+
+    Parameters
+    ----------
+    path: str
+        The path to the backlogger folder.
+    message: str
+        The commit message.
+    files: list[str], optional
+        The files to save. If None, all changes are saved.
+    """
+    tracker = get_tracker(path)
+    if tracker == 'datalad':
+        if files is not None:
+            files = [os.path.join(path, f) for f in files]
+        dl.save(files, message=message, dataset=path)
+    elif tracker == 'None':
+        Warning("Tracker 'None' does not implement save.")
+        pass
+    else:
+        raise ValueError(f"Tracker {tracker} is not supported.")
+
+
+def init(path: str, tracker: str='datalad') -> None:
+    """
+    Initialize a dataset at the specified path with the specified tracker.
+
+    Parameters
+    ----------
+    path: str
+        The path to initialize the dataset.
+    tracker: str
+        The tracker to use. Currently only 'datalad' and 'None' are supported.
+    """
+    if tracker == 'datalad':
+        dl.create(path)
+    elif tracker == 'None':
+        os.makedirs(path, exist_ok=True)
+    else:
+        raise ValueError(f"Tracker {tracker} is not supported.")
+    return
+
+
+def unlock(path: str, file: str) -> None:
+    """
+    Wrapper function to unlock a file in the dataset located at path with the specified tracker.
+
+    Parameters
+    ----------
+    path    : str
+        The path to the backlogger folder.
+    file    : str
+        The file to unlock.
+    """
+    tracker = get_tracker(path)
+    if tracker == 'datalad':
+        dl.unlock(file, dataset=path)
+    elif tracker == 'None':
+        Warning("Tracker 'None' does not implement unlock.")
+        pass
+    else:
+        raise ValueError(f"Tracker {tracker} is not supported.")
+    return
+
+
+def clone(path: str, source: str, target: str) -> None:
+    """
+    Wrapper function to clone a dataset from source to target with the specified tracker.
+    Parameters
+    ----------
+    path: str
+        The path to the backlogger folder.
+    source: str
+        The source dataset to clone.
+    target: str
+        The target path to clone the dataset to.
+    """
+    tracker = get_tracker(path)
+    if tracker == 'datalad':
+        dl.clone(target=target, source=source, dataset=path)
+    elif tracker == 'None':
+        os.makedirs(path, exist_ok=True)
+        # Implement a simple clone by copying files
+        shutil.copytree(source, target, dirs_exist_ok=False)
+    else:
+        raise ValueError(f"Tracker {tracker} is not supported.")
+    return
+
+
+def drop(path: str, reckless: Optional[str]=None) -> None:
+    """
+    Wrapper function to drop data from a dataset located at path with the specified tracker.
+
+    Parameters
+    ----------
+    path: str
+        The path to the backlogger folder.
+    reckless: Optional[str]
+        The datalad's reckless option for dropping data.
+    """
+    tracker = get_tracker(path)
+    if tracker == 'datalad':
+        dl.drop(path, reckless=reckless)
+    elif tracker == 'None':
+        Warning("Tracker 'None' does not implement drop.")
+        pass
+    else:
+        raise ValueError(f"Tracker {tracker} is not supported.")
+    return

corrlib/version.py

@@ -1 +1,34 @@
-__version__ = "0.2.3"
+# file generated by setuptools-scm
+# don't change, don't track in version control
+
+__all__ = [
+    "__version__",
+    "__version_tuple__",
+    "version",
+    "version_tuple",
+    "__commit_id__",
+    "commit_id",
+]
+
+TYPE_CHECKING = False
+if TYPE_CHECKING:
+    from typing import Tuple
+    from typing import Union
+
+    VERSION_TUPLE = Tuple[Union[int, str], ...]
+    COMMIT_ID = Union[str, None]
+else:
+    VERSION_TUPLE = object
+    COMMIT_ID = object
+
+version: str
+__version__: str
+__version_tuple__: VERSION_TUPLE
+version_tuple: VERSION_TUPLE
+commit_id: COMMIT_ID
+__commit_id__: COMMIT_ID
+
+__version__ = version = '0.2.4.dev14+g602324f84.d20251202'
+__version_tuple__ = version_tuple = (0, 2, 4, 'dev14', 'g602324f84.d20251202')
+
+__commit_id__ = commit_id = 'g602324f84'

pyproject.toml

@@ -1,6 +1,52 @@
 [build-system]
-requires = ["setuptools >= 63.0.0", "wheel"]
+requires = ["setuptools >= 63.0.0", "wheel", "setuptools-scm"]
 build-backend = "setuptools.build_meta"
 
+[project]
+requires-python = ">=3.10"
+name = "corrlib"
+dynamic = ["version"]
+dependencies = [
+    "gitpython>=3.1.45",
+    'pyerrors>=2.11.1',
+    "datalad>=1.1.0",
+    'typer>=0.12.5',
+]
+description = "Python correlation library"
+authors = [
+  { name = 'Justus Kuhlmann', email = 'j_kuhl19@uni-muenster.de'}
+]
+
+[project.scripts]
+pcl = "corrlib.cli:app"
+
+[tool.setuptools.packages.find]
+include = ["corrlib", "corrlib.*"]
+
+[tool.setuptools_scm]
+write_to = "corrlib/version.py"
+
 [tool.ruff.lint]
-ignore = ["F403"]
+ignore = ["E501"]
+extend-select = [
+  "YTT",
+  "E",
+  "W",
+  "F",
+]
+
+[tool.mypy]
+strict = true
+implicit_reexport = false
+follow_untyped_imports = false
+ignore_missing_imports = true
+
+[dependency-groups]
+dev = [
+    "mypy>=1.19.0",
+    "pandas-stubs>=2.3.3.251201",
+    "pytest>=9.0.1",
+    "pytest-cov>=7.0.0",
+    "pytest-pretty>=1.3.0",
+    "ruff>=0.14.7",
+]

setup.py

@@ -1,18 +0,0 @@
-from setuptools import setup
-from distutils.util import convert_path
-
-
-version = {}
-with open(convert_path('corrlib/version.py')) as ver_file:
-    exec(ver_file.read(), version)
-
-setup(name='pycorrlib',
-      version=version['__version__'],
-      author='Justus Kuhlmann',
-      author_email='j_kuhl19@uni-muenster.de',
-      install_requires=['pyerrors>=2.11.1', 'datalad>=1.1.0', 'typer>=0.12.5'],
-      entry_points = {
-        'console_scripts': ['pcl=corrlib.cli:app'],
-        },
-      packages=['corrlib', 'corrlib.input']
-      )

tests/cli_test.py

@@ -0,0 +1,91 @@
+from typer.testing import CliRunner
+from corrlib.cli import app
+import os
+import sqlite3 as sql
+
+
+runner = CliRunner()
+
+
+def test_version():
+    result = runner.invoke(app, ["--version"])
+    assert result.exit_code == 0
+    assert "corrlib" in result.output
+
+
+def test_init_folders(tmp_path):
+    dataset_path = tmp_path / "test_dataset"
+    result = runner.invoke(app, ["init", "--dataset", str(dataset_path)])
+    assert result.exit_code == 0
+    assert os.path.exists(str(dataset_path))
+    assert os.path.exists(str(dataset_path / "backlogger.db"))
+
+
+def test_init_db(tmp_path):
+    dataset_path = tmp_path / "test_dataset"
+    result = runner.invoke(app, ["init", "--dataset", str(dataset_path)])
+    assert result.exit_code == 0
+    assert os.path.exists(str(dataset_path / "backlogger.db"))
+    conn = sql.connect(str(dataset_path / "backlogger.db"))
+    cursor = conn.cursor()
+    cursor.execute("SELECT name FROM sqlite_master WHERE type='table';")
+    tables = cursor.fetchall()
+    expected_tables = [
+        'projects',
+        'backlogs',
+        ]
+    table_names = [table[0] for table in tables]
+    for expected_table in expected_tables:
+        assert expected_table in table_names
+    
+    cursor.execute("SELECT * FROM projects;")
+    projects = cursor.fetchall()
+    assert len(projects) == 0
+
+    cursor.execute("SELECT * FROM backlogs;")
+    backlogs = cursor.fetchall()
+    assert len(backlogs) == 0
+
+    cursor.execute("PRAGMA table_info('projects');")
+    project_columns = cursor.fetchall()
+    expected_project_columns = [
+        "id",
+        "aliases",
+        "customTags",
+        "owner",
+        "code",
+        "created_at",
+        "updated_at"
+        ]
+    project_column_names = [col[1] for col in project_columns]
+    for expected_col in expected_project_columns:
+        assert expected_col in project_column_names
+    
+    cursor.execute("PRAGMA table_info('backlogs');")
+    backlog_columns = cursor.fetchall()
+    expected_backlog_columns = [
+        "id",
+        "name",
+        "ensemble",
+        "code",
+        "path",
+        "project",
+        "customTags",
+        "parameters",
+        "parameter_file",
+        "created_at",
+        "updated_at"
+        ]
+    backlog_column_names = [col[1] for col in backlog_columns]
+    for expected_col in expected_backlog_columns:
+        assert expected_col in backlog_column_names
+
+
+def test_list(tmp_path):
+    dataset_path = tmp_path / "test_dataset"
+    result = runner.invoke(app, ["init", "--dataset", str(dataset_path)])
+    assert result.exit_code == 0
+    result = runner.invoke(app, ["list", "--dataset", str(dataset_path), "ensembles"])
+    assert result.exit_code == 0
+    result = runner.invoke(app, ["list", "--dataset", str(dataset_path), "projects"])
+    assert result.exit_code == 0

tests/import_project_test.py

@@ -14,4 +14,4 @@ def test_toml_check_measurement_data():
                             "names": ['list', 'of', 'names']
                         }
                     }
-    t.check_measurement_data(measurements)
+    t.check_measurement_data(measurements, "sfcf")

tests/test_initialization.py

@@ -0,0 +1,93 @@
+import corrlib.initialization as init
+import os
+import sqlite3 as sql
+
+def test_init_folders(tmp_path):
+    dataset_path = tmp_path / "test_dataset"
+    init.create(str(dataset_path))
+    assert os.path.exists(str(dataset_path))
+    assert os.path.exists(str(dataset_path / "backlogger.db"))
+
+
+def test_init_folders_no_tracker(tmp_path):
+    dataset_path = tmp_path / "test_dataset"
+    init.create(str(dataset_path), tracker="None")
+    assert os.path.exists(str(dataset_path))
+    assert os.path.exists(str(dataset_path / "backlogger.db"))
+
+
+def test_init_config(tmp_path):
+    dataset_path = tmp_path / "test_dataset"
+    init.create(str(dataset_path), tracker="None")
+    config_path = dataset_path / ".corrlib"
+    assert os.path.exists(str(config_path))
+    from configparser import ConfigParser
+    config = ConfigParser()
+    config.read(str(config_path))
+    assert config.get("core", "tracker") == "None"
+    assert config.get("core", "version") == "1.0"
+    assert config.get("core", "cached") == "True"
+    assert config.get("paths", "db") == "backlogger.db"
+    assert config.get("paths", "projects_path") == "projects"
+    assert config.get("paths", "archive_path") == "archive"
+    assert config.get("paths", "toml_imports_path") == "toml_imports"
+    assert config.get("paths", "import_scripts_path") == "import_scripts"
+
+
+def test_init_db(tmp_path):
+    dataset_path = tmp_path / "test_dataset"
+    init.create(str(dataset_path))
+    assert os.path.exists(str(dataset_path / "backlogger.db"))
+    conn = sql.connect(str(dataset_path / "backlogger.db"))
+    cursor = conn.cursor()
+    cursor.execute("SELECT name FROM sqlite_master WHERE type='table';")
+    tables = cursor.fetchall()
+    expected_tables = [
+        'projects',
+        'backlogs',
+        ]
+    table_names = [table[0] for table in tables]
+    for expected_table in expected_tables:
+        assert expected_table in table_names
+
+    cursor.execute("SELECT * FROM projects;")
+    projects = cursor.fetchall()
+    assert len(projects) == 0
+
+    cursor.execute("SELECT * FROM backlogs;")
+    backlogs = cursor.fetchall()
+    assert len(backlogs) == 0
+
+    cursor.execute("PRAGMA table_info('projects');")
+    project_columns = cursor.fetchall()
+    expected_project_columns = [
+        "id",
+        "aliases",
+        "customTags",
+        "owner",
+        "code",
+        "created_at",
+        "updated_at"
+        ]
+    project_column_names = [col[1] for col in project_columns]
+    for expected_col in expected_project_columns:
+        assert expected_col in project_column_names
+
+    cursor.execute("PRAGMA table_info('backlogs');")
+    backlog_columns = cursor.fetchall()
+    expected_backlog_columns = [
+        "id",
+        "name",
+        "ensemble",
+        "code",
+        "path",
+        "project",
+        "customTags",
+        "parameters",
+        "parameter_file",
+        "created_at",
+        "updated_at"
+        ]
+    backlog_column_names = [col[1] for col in backlog_columns]
+    for expected_col in expected_backlog_columns:
+        assert expected_col in backlog_column_names

tests/tools_test.py

@@ -0,0 +1,31 @@
+
+
+from corrlib import tools as tl
+
+
+def test_m2k():
+    for m in [0.1, 0.5, 1.0]:
+        expected_k = 1 / (2 * m + 8)
+        assert tl.m2k(m) == expected_k
+
+
+def test_k2m():
+    for m in [0.1, 0.5, 1.0]:
+        assert tl.k2m(m) == (1/(2*m))-4
+
+
+def test_k2m_m2k():
+    for m in [0.1, 0.5, 1.0]:
+        k = tl.m2k(m)
+        m_converted = tl.k2m(k)
+        assert abs(m - m_converted) < 1e-9
+
+
+def test_str2list():
+    assert tl.str2list("a,b,c") == ["a", "b", "c"]
+    assert tl.str2list("1,2,3") == ["1", "2", "3"]
+
+
+def test_list2str():
+    assert tl.list2str(["a", "b", "c"]) == "a,b,c"
+    assert tl.list2str(["1", "2", "3"]) == "1,2,3"