Merge pull request 'feat/path_check' (#40) from feat/path_check into develop

Reviewed-on: #40

corrlib/cli.py

@@ -4,7 +4,7 @@ from corrlib import __app_name__
 
 from .initialization import create
 from .toml import import_tomls, update_project, reimport_project
-from .find import find_record, list_projects
+from .find import find_record, list_projects, list_ensembles
 from .tools import str2list
 from .main import update_aliases
 from .meas_io import drop_cache as mio_drop_cache
@@ -56,15 +56,15 @@ def lister(
     """
     if entities in ['ensembles', 'Ensembles','ENSEMBLES']:
         print("Ensembles:")
-        for item in os.listdir(path / "archive"):
+        ensemble_results = list_ensembles(path)
-            if os.path.isdir(path / "archive" / item):
+        for e in ensemble_results:
-                print(item)
+            print(e)
     elif entities == 'projects':
-        results = list_projects(path)
+        project_results = list_projects(path)
         print("Projects:")
         header = "UUID".ljust(37) + "| Aliases"
         print(header)
-        for project in results:
+        for project in project_results:
             if project[1] is not None:
                 aliases = " | ".join(str2list(project[1]))
             else:

corrlib/find.py

@@ -382,3 +382,10 @@ def list_projects(path: Path) -> list[tuple[str, str]]:
     conn.close()
     return results
 
+
+def list_ensembles(path: Path) -> list[str]:
+    res = []
+    for item in os.listdir(path / "archive"):
+        if os.path.isdir(path / "archive" / item):
+            res.append(item)
+    return res

corrlib/integrity.py

@@ -7,7 +7,6 @@ from .tracker import get
 import pyerrors.input.json as pj
 import os
 from configparser import ConfigParser
-
 from typing import Any
 
 
@@ -61,7 +60,69 @@ def are_keys_unique(db: Path, table: str, col: str) -> bool:
     c.execute(f"SELECT COUNT( DISTINCT CAST({col} AS nvarchar(4000))), COUNT({col}) FROM {table};")
     results = c.fetchall()[0]
     conn.close()
-    return bool(results[0] == results[1])
+    res = bool(results[0] == results[1])
+    if not res:
+        print("Unique:", results[0], "All:", results[1])
+    return res
+
+
+def _list_projects(path: Path) -> list[tuple[str, str]]:
+    """
+    List all projects known to the library.
+
+    Parameters
+    ----------
+    path: str
+        The path of the library.
+
+    Returns
+    -------
+    results: list[Any]
+        The projects known to the library.
+    """
+    db_file = get_db_file(path)
+    get(path, db_file)
+    conn = sqlite3.connect(os.path.join(path, db_file))
+    c = conn.cursor()
+    c.execute("SELECT id,aliases FROM projects")
+    results = c.fetchall()
+    conn.close()
+    return results
+
+
+def _list_ensembles(path: Path) -> list[str]:
+    res = []
+    for item in os.listdir(path / "archive"):
+        if os.path.isdir(path / "archive" / item):
+            res.append(item)
+    return res
+
+
+def check_path_format(result: pd.Series, ensembles: list[str], projects: list[str]) -> None:
+    """
+    Check whether the path of the given result has the right format.
+
+    Parameters
+    ----------
+    result: pd.Series
+        The result to be checked.
+    """
+    p = result['path']
+    if not p.startswith('archive'):
+        raise ValueError(f'The path {p} does not start correctly')
+
+    meas_key = p.split('::')[1]
+    ensemble = p.split('/')[1]
+    project = p.split('/')[3].split('.')[0]
+    if not len(meas_key) == 64:
+        raise ValueError(f'meas_key of {p} is scrambled')
+    if ensemble not in ensembles:
+        raise ValueError(f'meas_key of {p} points to an unknown ensemble')
+    if project not in projects:
+        raise ValueError(f'meas_key of {p} points to an unknown project id ({project})')
+    if not ensemble == result['ensemble']:
+        raise ValueError(f'Ensemble in database and file does not match for path {p}.')
+
 
 
 def check_db_integrity(path: Path) -> None:
@@ -82,10 +143,13 @@ def check_db_integrity(path: Path) -> None:
     search_expr = "SELECT * FROM 'backlogs'"
     conn = sqlite3.connect(path / db)
     results = pd.read_sql(search_expr, conn)
+    ensembles = _list_ensembles(path)
+    projects = [p[0] for p in _list_projects(path)]
 
     for _, result in results.iterrows():
         if not has_valid_times(result):
             raise ValueError(f"Result with id {result[id]} has wrong time signatures.")
+        check_path_format(result, ensembles, projects)
     return
 
 

tests/integrity_test.py

@@ -0,0 +1,189 @@
+import corrlib.integrity as integ
+import corrlib.find as find
+import datalad.api as dl
+import corrlib.initialization as cinit
+import sqlite3
+from pathlib import Path
+import os
+import pandas as pd
+import datetime as dt
+import pytest
+
+
+def test_list_ensembles(tmp_path: Path) -> None:
+    """
+    Check against the implementation in find to check if they are the same.
+    """
+    os.mkdir(tmp_path / 'archive')
+    os.mkdir(tmp_path / 'archive' / 'A')
+    os.mkdir(tmp_path / 'archive' / 'B')
+    os.mkdir(tmp_path / 'archive' / 'C')
+    integ_results = integ._list_ensembles(tmp_path)
+    assert len(integ_results) == 3
+    find_results = find.list_ensembles(tmp_path)
+    assert len(find_results) == 3
+    for f,i in zip(find_results, integ_results):
+        assert f == i
+
+
+def test_list_projects(tmp_path: Path) -> None:
+    cinit.create(tmp_path)
+    db = tmp_path / "backlogger.db"
+    dl.unlock(str(db), dataset=str(tmp_path))
+    conn = sqlite3.connect(db)
+    c = conn.cursor()
+
+    customTags = ""
+    owner = "owner"
+    code = "sfcf"
+    created_at = "today"
+    updated_at = "today"
+
+    id = "asdf1"
+    aliases = "a1,s1,d1,f1"
+    c.execute("INSERT INTO projects (id, aliases, customTags, owner, code, created_at, updated_at) VALUES (?,?,?,?,?,?,?)", (id, aliases, customTags, owner, code , created_at, updated_at))
+    id = "asdf2"
+    aliases = "a2,s2,d2,f2"
+    c.execute("INSERT INTO projects (id, aliases, customTags, owner, code, created_at, updated_at) VALUES (?,?,?,?,?,?,?)", (id, aliases, customTags, owner, code , created_at, updated_at))
+    id = "asdf3"
+    aliases = "a3,s3,d3,f3"
+    c.execute("INSERT INTO projects (id, aliases, customTags, owner, code, created_at, updated_at) VALUES (?,?,?,?,?,?,?)", (id, aliases, customTags, owner, code , created_at, updated_at))
+    conn.commit()
+    conn.close
+    integ_results = integ._list_projects(tmp_path)
+    assert len(integ_results) == 3
+    find_results = find.list_projects(tmp_path)
+    assert len(find_results) == 3
+    for f,i in zip(find_results, integ_results):
+        assert f == i
+
+
+def test_has_valid_time() -> None:
+    record_A = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf0", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in",
+            '2025-03-26 12:55:18.229966', '2025-03-26 12:55:18.229966'] # only created
+    record_B = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf1", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in",
+                '2025-03-26 12:55:18.229966', '2025-04-26 12:55:18.229966'] # created and updated
+    record_C = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf2", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in",
+                '2026-03-26 12:55:18.229966', '2026-04-14 12:55:18.229966'] # created and updated later
+    record_D = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf3", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in",
+                '2026-03-26 12:55:18.229966', '2026-03-27 12:55:18.229966']
+    record_E = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf4", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in",
+                '2024-03-26 12:55:18.229966', '2024-03-26 12:55:18.229966'] # only created, earlier
+    record_F = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf5", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in",
+                '2026-03-26 12:55:18.229966', '2024-03-26 12:55:18.229966'] # this is invalid...
+    record_G = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf2", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in",
+                '2026-03-26 12:55:18.229966', str(dt.datetime.now() + dt.timedelta(days=2, hours=3, minutes=5, seconds=30))] # created and updated later
+    cols = ["name",
+            "ensemble",
+            "code",
+            "path",
+            "project",
+            "parameters",
+            "parameter_file",
+            "created_at",
+            "updated_at"]
+    data = [record_A, record_B, record_C, record_D, record_E]
+
+    df = pd.DataFrame(data,columns=cols)
+    for _, result in df.iterrows():
+        assert integ.has_valid_times(result)
+    data = [record_F, record_G]
+    df = pd.DataFrame(data,columns=cols)
+    for _, result in df.iterrows():
+        assert not integ.has_valid_times(result)
+
+
+def test_are_keys_unique(tmp_path: Path) -> None:
+    db = tmp_path / 'test_success.db'
+
+    record_A = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf0", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in",
+            '2025-03-26 12:55:18.229966', '2025-03-26 12:55:18.229966'] # only created
+    record_B = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf1", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in",
+                '2025-03-26 12:55:18.229966', '2025-04-26 12:55:18.229966'] # created and updated
+    record_C = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf2", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in",
+                '2026-03-26 12:55:18.229966', '2026-04-14 12:55:18.229966'] # created and updated later
+    record_D = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf3", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in",
+                '2026-03-26 12:55:18.229966', '2026-03-27 12:55:18.229966']
+    record_E = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf4", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in",
+                '2024-03-26 12:55:18.229966', '2024-03-26 12:55:18.229966'] # only created, earlier
+    record_F = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf5", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in",
+                '2026-03-26 12:55:18.229966', '2024-03-26 12:55:18.229966'] # this is invalid...
+    record_G = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf2", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in",
+                '2026-03-26 12:55:18.229966', str(dt.datetime.now() + dt.timedelta(days=2, hours=3, minutes=5, seconds=30))] # created and updated later
+
+    cols = ["name",
+            "ensemble",
+            "code",
+            "path",
+            "project",
+            "parameters",
+            "parameter_file",
+            "created_at",
+            "updated_at"]
+
+    data = [record_A, record_B, record_C, record_D, record_E, record_F]
+    df = pd.DataFrame(data,columns=cols)
+    conn = sqlite3.connect(db)
+    df.to_sql('backlogs', conn)
+    conn.close()
+    assert integ.are_keys_unique(db, 'backlogs', 'path')
+
+    db = tmp_path / 'test_fail.db'
+    data = [record_A, record_B, record_C, record_D, record_E, record_F, record_G]
+
+    df = pd.DataFrame(data,columns=cols)
+    conn = sqlite3.connect(db)
+    df.to_sql('backlogs', conn)
+    conn.close()
+    assert not integ.are_keys_unique(db, 'backlogs', 'path')
+
+
+def test_check_path_format() -> None:
+    record_A = ["f_A", "ensA", "sfcf", "archive/ensA/f_A/Project_A.json.gz::asdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdf", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in",
+            '2025-03-26 12:55:18.229966', '2025-03-26 12:55:18.229966'] # only created
+    record_B = ["f_A", "ensA", "sfcf", "archive/ensA/f_A/Project_B.json.gz::asdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdf", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in",
+                '2025-03-26 12:55:18.229966', '2025-04-26 12:55:18.229966'] # created and updated
+    record_C = ["f_A", "ensA", "sfcf", "archive/ensA/f_A/Project_A.json.gz::asdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdf", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in",
+                '2026-03-26 12:55:18.229966', '2026-04-14 12:55:18.229966'] # created and updated later
+    record_D = ["f_A", "ensA", "sfcf", "archive/ensA/f_A/Project_B.json.gz::asdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdf", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in",
+                '2026-03-26 12:55:18.229966', '2026-03-27 12:55:18.229966']
+    record_E = ["f_A", "ensA", "sfcf", "archive/ensA/f_A/Project_A.json.gz::asdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdf", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in",
+                '2024-03-26 12:55:18.229966', '2024-03-26 12:55:18.229966'] # only created, earlier
+    record_F = ["f_A", "ensA", "sfcf", "archive/ensA/f_A/Project_B.json.gz::asdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdf", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in",
+                '2026-03-26 12:55:18.229966', '2024-03-26 12:55:18.229966'] # this is invalid...
+    record_G = ["f_A", "ensA", "sfcf", "archive/ensA/f_A/Project_A.json.gz::asdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfas", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in",
+                '2026-03-26 12:55:18.229966', str(dt.datetime.now() + dt.timedelta(days=2, hours=3, minutes=5, seconds=30))] # created and updated later
+
+    projects = ['Project_A', 'Project_B']
+    ensembles = ['ensA']
+    cols = ["name",
+            "ensemble",
+            "code",
+            "path",
+            "project",
+            "parameters",
+            "parameter_file",
+            "created_at",
+            "updated_at"]
+
+    data = [record_A, record_B, record_C, record_D, record_E, record_F]
+    df = pd.DataFrame(data,columns=cols)
+    for _, result in df.iterrows():
+        integ.check_path_format(result, ensembles, projects)
+
+    projects = ['Project_A', 'Project_B']
+    ensembles = ['ensB']
+    for _, result in df.iterrows():
+        with pytest.raises(ValueError):
+            integ.check_path_format(result, ensembles, projects)
+
+    projects = ['Project_A', 'Project_B']
+    ensembles = ['ensA', 'ensB']
+    for _, result in df.iterrows():
+        integ.check_path_format(result, ensembles, projects)
+
+    data = [record_G]
+    df = pd.DataFrame(data,columns=cols)
+    for _, result in df.iterrows():
+        with pytest.raises(ValueError):
+            integ.check_path_format(result, ensembles, projects)