From 553e0b31fef1197d507e3c738d31a8325e4aa9da Mon Sep 17 00:00:00 2001 From: danlooo Date: Tue, 6 Jan 2026 14:38:54 +0100 Subject: [PATCH 1/4] Add nested traits --- pathtraits/db.py | 38 +++++++++++++++++++++++++++++++------- 1 file changed, 31 insertions(+), 7 deletions(-) diff --git a/pathtraits/db.py b/pathtraits/db.py index de64628..169ef90 100644 --- a/pathtraits/db.py +++ b/pathtraits/db.py @@ -5,6 +5,7 @@ import logging import sqlite3 import os +from collections.abc import MutableMapping import yaml from pathtraits.pathpair import PathPair @@ -62,6 +63,25 @@ def merge_rows(rows: list): res = {k: sorted(v, key=str) if len(v) > 1 else v[0] for k, v in res.items()} return res + @staticmethod + def flatten_dict(dictionary: dict, root_key: str = "", separator: str = "/"): + """ + Docstring for flatten_dict + + :param d: Description + :type d: dict + """ + items = [] + for key, value in dictionary.items(): + new_key = root_key + separator + key if root_key else key + if isinstance(value, MutableMapping): + items.extend( + TraitsDB.flatten_dict(value, new_key, separator=separator).items() + ) + else: + items.append((new_key, value)) + return dict(items) + def __init__(self, db_path): db_path = os.path.join(db_path) self.cursor = sqlite3.connect(db_path, autocommit=True).cursor() @@ -189,15 +209,15 @@ def put(self, table, condition=None, update=True, **kwargs): # update values = " , ".join([f"{k}={v}" for (k, v) in escaped_kwargs.items()]) if condition: - update_query = f"UPDATE {table} SET {values} WHERE {condition};" + update_query = f"UPDATE [{table}] SET {values} WHERE {condition};" else: - update_query = f"UPDATE {table} SET {values};" + update_query = f"UPDATE [{table}] SET {values};" self.execute(update_query) else: # insert keys = " , ".join(escaped_kwargs.keys()) values = " , ".join([str(x) for x in escaped_kwargs.values()]) - insert_query = f"INSERT INTO {table} ({keys}) VALUES ({values});" + insert_query = f"INSERT INTO [{table}] ({keys}) VALUES ({values});" self.execute(insert_query) def put_data_view(self): @@ -209,7 +229,7 @@ def put_data_view(self): if self.traits: join_query = " ".join( [ - f"LEFT JOIN {x} ON {x}.path = path.id" + f"LEFT JOIN [{x}] ON [{x}].path = path.id \n" for x in self.traits if x != "path" ] @@ -217,7 +237,7 @@ def put_data_view(self): create_view_query = f""" CREATE VIEW data AS - SELECT path.path, {', '.join(self.traits)} + SELECT path.path, [{'], ['.join(self.traits)}] FROM path {join_query}; """ @@ -263,9 +283,9 @@ def create_trait_table(self, trait_name, value_type): return sql_type = TraitsDB.sql_type(value_type) add_table_query = f""" - CREATE TABLE {trait_name} ( + CREATE TABLE [{trait_name}] ( path INTEGER, - {trait_name} {sql_type}, + [{trait_name}] {sql_type}, FOREIGN KEY(path) REFERENCES path(id) ); """ @@ -303,6 +323,8 @@ def add_pathpair(self, pair: PathPair): if not isinstance(traits, dict): return + traits = TraitsDB.flatten_dict(traits) + # put path in db only if there are traits path_id = self.put_path_id(os.path.abspath(pair.object_path)) for k, v in traits.items(): @@ -317,8 +339,10 @@ def add_pathpair(self, pair: PathPair): t = type(v[0]) if isinstance(v, list) else type(v) self.create_trait_table(k, t) if k in self.traits: + # add to list if isinstance(v, list): for vv in v: self.put_trait(path_id, k, vv, update=False) + # overwrite scalar else: self.put_trait(path_id, k, v) From 40cefa7ed0fc0c137cc977ef99e6ca2908f44577 Mon Sep 17 00:00:00 2001 From: danlooo Date: Tue, 6 Jan 2026 14:53:22 +0100 Subject: [PATCH 2/4] Add insert nested traits --- pathtraits/db.py | 2 +- test/example/EU/meta.yml | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/pathtraits/db.py b/pathtraits/db.py index 169ef90..86705f3 100644 --- a/pathtraits/db.py +++ b/pathtraits/db.py @@ -215,7 +215,7 @@ def put(self, table, condition=None, update=True, **kwargs): self.execute(update_query) else: # insert - keys = " , ".join(escaped_kwargs.keys()) + keys = "[" + "], [".join(escaped_kwargs.keys()) + "]" values = " , ".join([str(x) for x in escaped_kwargs.values()]) insert_query = f"INSERT INTO [{table}] ({keys}) VALUES ({values});" self.execute(insert_query) diff --git a/test/example/EU/meta.yml b/test/example/EU/meta.yml index 249d569..d31042c 100644 --- a/test/example/EU/meta.yml +++ b/test/example/EU/meta.yml @@ -3,3 +3,8 @@ users: - dloos - fgans score: 3.5 +foo: + bar: + a: 1 + b: 2 + c: [1, 2, 3] From 33d36221ee038a7188944b6e8ae04729b5b7c707 Mon Sep 17 00:00:00 2001 From: danlooo Date: Tue, 6 Jan 2026 14:58:45 +0100 Subject: [PATCH 3/4] Unify type path suffix --- pathtraits/db.py | 6 +++--- test/test.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pathtraits/db.py b/pathtraits/db.py index 86705f3..0ce6a92 100644 --- a/pathtraits/db.py +++ b/pathtraits/db.py @@ -34,12 +34,12 @@ def row_factory(cursor, row): if v is None: continue # sqlite don't know bool - if k.endswith("_BOOL"): + if k.endswith("/BOOL"): v = v > 0 if isinstance(v, float): v_int = int(v) v = v_int if v_int == v else v - k = k.removesuffix("_TEXT").removesuffix("_REAL").removesuffix("_BOOL") + k = k.removesuffix("/TEXT").removesuffix("/REAL").removesuffix("/BOOL") res[k] = v return res @@ -334,7 +334,7 @@ def add_pathpair(self, pair: PathPair): # get element type for list # add: handle lists with mixed element type t = type(v[0]) if isinstance(v, list) else type(v) - k = f"{k}_{TraitsDB.sql_type(t)}" + k = f"{k}/{TraitsDB.sql_type(t)}" if k not in self.traits: t = type(v[0]) if isinstance(v, list) else type(v) self.create_trait_table(k, t) diff --git a/test/test.py b/test/test.py index baba261..25070cb 100644 --- a/test/test.py +++ b/test/test.py @@ -63,7 +63,7 @@ def test_example(self): def test_data_view(self): source = len(self.db.execute("SELECT * FROM data;").fetchall()) - target = 6 + target = 10 self.assertEqual(source, target) From d5c3189af0ea0d92b9ee4e86d2341dc05a32cbb5 Mon Sep 17 00:00:00 2001 From: danlooo Date: Tue, 6 Jan 2026 15:39:12 +0100 Subject: [PATCH 4/4] Add get nested trait --- pathtraits/access.py | 27 +++++++++++++++++++++++++++ pathtraits/db.py | 7 ++++--- test/test.py | 1 + 3 files changed, 32 insertions(+), 3 deletions(-) diff --git a/pathtraits/access.py b/pathtraits/access.py index 75ed324..0300ae6 100644 --- a/pathtraits/access.py +++ b/pathtraits/access.py @@ -11,6 +11,32 @@ logger = logging.getLogger(__name__) +def nest_dict(flat_dict, delimiter="/"): + """ + Transforms a flat dictionary with path-like keys into a nested dictionary. + + :param flat_dict: The flat dictionary with path-like keys. + :param delimiter: The delimiter used in the keys (default is '/'). + :return: A nested dictionary. + """ + nested_dict = {} + + for path, value in flat_dict.items(): + keys = path.split(delimiter) + current = nested_dict + + for key in keys[:-1]: + # If the key doesn't exist or is not a dictionary, create/overwrite it as a dictionary + if key not in current or not isinstance(current[key], dict): + current[key] = {} + current = current[key] + + # Set the value at the final key + current[keys[-1]] = value + + return nested_dict + + def get_dict(self, path): """ Get traits for a path as a Python dictionary @@ -40,6 +66,7 @@ def get_dict(self, path): if not (v and k != "path"): continue res[k] = v + res = nest_dict(res) return res diff --git a/pathtraits/db.py b/pathtraits/db.py index 0ce6a92..8dd9608 100644 --- a/pathtraits/db.py +++ b/pathtraits/db.py @@ -54,10 +54,11 @@ def merge_rows(rows: list): for row in rows: for k, v in row.items(): # pylint: disable=C0201 - if k in res.keys() and v not in res[k]: + if not k in res.keys(): + res[k] = [] + if not v in res[k]: res[k].append(v) - else: - res[k] = [v] + # simplify lists with just one element # ensure fixed order of list entries res = {k: sorted(v, key=str) if len(v) > 1 else v[0] for k, v in res.items()} diff --git a/test/test.py b/test/test.py index 25070cb..4af892c 100644 --- a/test/test.py +++ b/test/test.py @@ -47,6 +47,7 @@ def test_eu(self): "is_example": True, "score": 3.5, "users": ["dloos", "fgans"], + "foo": {"bar": {"a": 1, "b": 2, "c": [1, 2, 3]}}, } for k, v in target.items(): self.assertEqual(source[k], v)