[AI-generated, unverified] SQLAlchemy 2.1, rdflib 7 graph-aware store, publish as ifcopenshell-rdflib-sqlalchemy

.github/workflows/python-package.yml

@@ -16,17 +16,25 @@ jobs:
 
     runs-on: ubuntu-latest
     strategy:
+      fail-fast: false
       matrix:
-        python-version: [3.7, 3.8, 3.9, '3.10']
+        python-version: ['3.9', '3.10', '3.11', '3.12', '3.13']
+        sqlalchemy: ['2.0.*', '2.1.*']
         database:
         - pgsql
         - mysql
         - sqlite
+        exclude:
+        # SQLAlchemy 2.1 requires Python 3.11.
+        - python-version: '3.9'
+          sqlalchemy: '2.1.*'
+        - python-version: '3.10'
+          sqlalchemy: '2.1.*'
     # There's some recursion error here (See #76)
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python-version }}
     - name: Install dependencies
@@ -37,6 +45,7 @@ jobs:
         python -m pip install --upgrade wheel # stop pip complaining about a setup.py install
         sudo apt-get install python3-dev
         pip install .
+        pip install "sqlalchemy==${{ matrix.sqlalchemy }}"
     - name: Test with tox
       env:
         DB: ${{ matrix.database }}
@@ -74,23 +83,3 @@ jobs:
           # Maps tcp port 3306 on service container to the host
           - 3306:3306
 
-  dev-deploy:
-    name: Deploy Dev Package to PyPI
-    needs: build
-    if: github.event_name == 'push'
-      && (github.ref == 'refs/heads/develop'
-          || startsWith(github.ref, 'refs/heads/release/'))
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - name: Set up Python 3
-        uses: actions/setup-python@v2
-        with:
-          python-version: 3
-      - name: Install Deploy Dependencies
-        run: pip install wheel twine
-      - name: Deploy
-        run: ./deploy.sh
-        env:
-          TWINE_USERNAME: __token__
-          TWINE_PASSWORD: ${{ secrets.TWINE_PASSWORD }}

README.md

@@ -1,6 +1,17 @@
 RDFLib-SQLAlchemy
 =================
 
+This is IfcOpenShell's fork of [RDFLib/rdflib-sqlalchemy](https://github.com/RDFLib/rdflib-sqlalchemy),
+published on PyPI as `ifcopenshell-rdflib-sqlalchemy`, so that Bonsai's Brick support has a store that
+works with current SQLAlchemy and rdflib. It carries upstream's `develop` plus what upstream has not
+released or merged: SQLAlchemy 2.x support (including the `.subquery()` 2.1 needs), the where-clause
+guards from the `brickschema-rdflib-sqlalchemy` fork by Gabe Fierro, `importlib.metadata` instead of
+`pkg_resources`, and a graph-aware store as rdflib 7's Dataset requires. Fixes are offered upstream
+(RDFLib/rdflib-sqlalchemy#116, #117). The module name stays `rdflib_sqlalchemy` and the rdflib store
+plugin stays `SQLAlchemy`, so it is a drop-in replacement; do not install it alongside another
+`rdflib-sqlalchemy` distribution.
+
+
 A SQLAlchemy-backed, formula-aware RDFLib Store. It stores its triples
 in the following partitions:
 

rdflib_sqlalchemy/__init__.py

@@ -1,10 +1,14 @@
 # -*- coding: utf-8 -*-
 """SQLAlchemy Store plugin for RDFLib."""
 import logging
-from pkg_resources import get_distribution
+import sys
 
+if sys.version_info >= (3, 8):
+    import importlib.metadata as importlib_metadata
+else:
+    import importlib_metadata
 
-__version__ = get_distribution("rdflib_sqlalchemy").version
+__version__ = importlib_metadata.version("ifcopenshell-rdflib-sqlalchemy")
 
 
 class NullHandler(logging.Handler):

rdflib_sqlalchemy/sql.py

@@ -62,13 +62,15 @@ def union_select(select_components, distinct=False, select_type=TRIPLE_SELECT):
             else:
                 raise ValueError('Unrecognized table type {}'.format(tableType))
             select_clause = expression.select(*[functions.count().label('aCount')]).select_from(
-                expression.select(*cols).where(whereClause).distinct().select_from(table))
+                expression.select(*cols).where(whereClause).distinct().select_from(table).subquery())
         elif select_type == CONTEXT_SELECT:
             select_clause = expression.select(table.c.context)
             if whereClause is not None:
                 select_clause = expression.select(table.c.context).where(whereClause)
         elif tableType in FULL_TRIPLE_PARTITIONS:
-            select_clause = table.select().where(whereClause)
+            select_clause = table.select()
+            if whereClause is not None:
+                select_clause = select_clause.where(whereClause)
         elif tableType == ASSERTED_TYPE_PARTITION:
             select_clause = expression.select(
                 *[table.c.id.label("id"),
@@ -78,8 +80,9 @@ def union_select(select_components, distinct=False, select_type=TRIPLE_SELECT):
                  table.c.context.label("context"),
                  table.c.termComb.label("termcomb"),
                  expression.literal_column("NULL").label("objlanguage"),
-                 expression.literal_column("NULL").label("objdatatype")]).where(
-                whereClause)
+                 expression.literal_column("NULL").label("objdatatype")])
+            if whereClause is not None:
+                select_clause = select_clause.where(whereClause)
         elif tableType == ASSERTED_NON_TYPE_PARTITION:
             all_table_columns = [c for c in table.columns] + \
                                 [expression.literal_column("NULL").label("objlanguage"),

rdflib_sqlalchemy/store.py

@@ -30,6 +30,7 @@
 from rdflib_sqlalchemy.tables import (
     create_asserted_statements_table,
     create_literal_statements_table,
+    create_contexts_table,
     create_namespace_binds_table,
     create_quoted_statements_table,
     create_type_statements_table,
@@ -92,6 +93,7 @@ class SQLAlchemy(Store, SQLGeneratorMixin, StatisticsMixin):
     context_aware = True
     formula_aware = True
     transaction_aware = True
+    graph_aware = True
     regex_matching = PYTHON_REGEX
     configuration = Literal("sqlite://")
 
@@ -395,7 +397,8 @@ def remove(self, triple, context):
                     if not self.STRONGLY_TYPED_TERMS or isinstance(obj, Literal):
                         # remove literal triple
                         clause = self.build_clause(literal_table, subject, predicate, obj, context)
-                        connection.execute(literal_table.delete().where(clause))
+                        query = literal_table.delete().where(clause) if clause is not None else literal_table.delete()
+                        connection.execute(query)
 
                     for table in [quoted_table, asserted_table]:
                         # If asserted non rdf:type table and obj is Literal,
@@ -404,16 +407,21 @@ def remove(self, triple, context):
                             continue
                         else:
                             clause = self.build_clause(table, subject, predicate, obj, context)
-                            connection.execute(table.delete().where(clause))
+                            query = table.delete().where(clause) if clause is not None else table.delete()
+                            connection.execute(query)
 
                 if predicate == RDF.type or predicate is None:
                     # Need to check rdf:type and quoted partitions (in addition
                     # perhaps)
                     clause = self.build_clause(asserted_type_table, subject, RDF.type, obj, context, True)
-                    connection.execute(asserted_type_table.delete().where(clause))
+                    query = asserted_type_table.delete()
+                    if clause is not None:
+                        query = query.where(clause)
+                    connection.execute(query)
 
                     clause = self.build_clause(quoted_table, subject, predicate, obj, context)
-                    connection.execute(quoted_table.delete().where(clause))
+                    query = quoted_table.delete().where(clause) if clause is not None else quoted_table.delete()
+                    connection.execute(query)
             except Exception:
                 _logger.exception("Removal failed.")
                 raise
@@ -628,8 +636,34 @@ def contexts(self, triple=None):
         with self.engine.connect() as connection:
             res = connection.execute(q)
             rt = res.fetchall()
-        for context in [rtTuple[0] for rtTuple in rt]:
-            yield URIRef(context)
+            recorded = []
+            if triple is None:
+                contexts_table = self.tables["contexts"]
+                recorded = connection.execute(expression.select(contexts_table.c.context)).fetchall()
+        seen = set()
+        for context in [row[0] for row in rt] + [row[0] for row in recorded]:
+            if context not in seen:
+                seen.add(context)
+                yield URIRef(context)
+
+    # Graph-aware store interface
+
+    def add_graph(self, graph):
+        """Record a named graph, so that contexts() lists it while it is empty."""
+        contexts_table = self.tables["contexts"]
+        with self.engine.begin() as connection:
+            known = connection.execute(
+                expression.select(contexts_table.c.id).where(contexts_table.c.context == graph.identifier)
+            ).first()
+            if known is None:
+                connection.execute(contexts_table.insert().values(context=graph.identifier))
+
+    def remove_graph(self, graph):
+        """Remove a named graph and every triple in it."""
+        self.remove((None, None, None), graph)
+        contexts_table = self.tables["contexts"]
+        with self.engine.begin() as connection:
+            connection.execute(contexts_table.delete().where(contexts_table.c.context == graph.identifier))
 
     # Namespace persistence interface implementation
 
@@ -692,6 +726,7 @@ def _create_table_definitions(self):
             "literal_statements": create_literal_statements_table(self._interned_id, self.metadata),
             "quoted_statements": create_quoted_statements_table(self._interned_id, self.metadata),
             "namespace_binds": create_namespace_binds_table(self._interned_id, self.metadata),
+            "contexts": create_contexts_table(self._interned_id, self.metadata),
         }
 
     def _get_build_command(self, triple, context=None, quoted=False):

rdflib_sqlalchemy/tables.py

@@ -7,6 +7,7 @@
 
 TABLE_NAME_TEMPLATES = [
     "{interned_id}_asserted_statements",
+    "{interned_id}_contexts",
     "{interned_id}_literal_statements",
     "{interned_id}_namespace_binds",
     "{interned_id}_quoted_statements",
@@ -211,3 +212,24 @@ def create_namespace_binds_table(interned_id, metadata):
             mysql_length=MYSQL_MAX_INDEX_LENGTH,
         )
     )
+
+
+def create_contexts_table(interned_id, metadata):
+    """The named graphs the store knows, whether or not they hold triples.
+
+    A graph-aware store (rdflib's Dataset needs one) lists an added graph
+    while it is empty and forgets it on remove_graph(); the statement tables
+    only know a context once a triple is in it.
+    """
+    return Table(
+        "{interned_id}_contexts".format(interned_id=interned_id),
+        metadata,
+        Column("id", types.Integer, nullable=False, primary_key=True),
+        Column("context", TermType, nullable=False),
+        Index(
+            "{interned_id}_context_index".format(interned_id=interned_id),
+            "context",
+            unique=True,
+            mysql_length=MYSQL_MAX_INDEX_LENGTH,
+        ),
+    )

setup.py

@@ -1,19 +1,19 @@
 #!/usr/bin/env python
 from setuptools import setup
 
-project = "rdflib-sqlalchemy"
-version = "0.5.5.dev0"
+project = "ifcopenshell-rdflib-sqlalchemy"
+version = "0.7.0"
 
 
 setup(
     name=project,
     version=version,
     description="rdflib extension adding SQLAlchemy as an AbstractSQLStore back-end store",
-    author="Graham Higgins, Adam Ever-Hadani",
-    author_email="gjhiggins@gmail.com, adamhadani@globality.com",
-    url="http://github.com/RDFLib/rdflib-sqlalchemy",
+    author="Graham Higgins, Adam Ever-Hadani, IfcOpenShell contributors",
+    author_email="gjhiggins@gmail.com, adamhadani@globality.com, dion@thinkmoult.com",
+    url="https://github.com/IfcOpenShell/rdflib-sqlalchemy",
     packages=["rdflib_sqlalchemy"],
-    download_url="https://github.com/RDFLib/rdflib-sqlalchemy/zipball/master",
+    download_url="https://github.com/IfcOpenShell/rdflib-sqlalchemy/zipball/develop",
     license="BSD",
     platforms=["any"],
     long_description="""
@@ -47,6 +47,7 @@
         "rdflib>=6,<8",
         "six>=1.10.0",
         "SQLAlchemy>=2.0.23",
+        "importlib-metadata; python_version < '3.8'",
     ],
     entry_points={
         'rdf.plugins.store': [

test/context_case.py

@@ -173,6 +173,26 @@ def cid(c):
         self.assertIn(self.c1, contextList)
         self.assertIn(self.c2, contextList)
 
+    def testGraphAware(self):
+        # rdflib's Dataset needs a graph-aware store: a graph added through
+        # add_graph() is listed by contexts() while it is still empty, and
+        # remove_graph() takes it out together with its triples.
+        store = self.graph.store
+        self.assertTrue(store.graph_aware)
+        empty = URIRef("urn:x-rdflib:empty")
+        store.add_graph(Graph(store, empty))
+        store.add_graph(Graph(store, empty))
+        self.assertIn(empty, list(store.contexts()))
+        self.assertEqual(list(store.contexts()).count(empty), 1)
+        graph = Graph(store, self.c1)
+        graph.add((self.michel, self.likes, self.pizza))
+        self.assertIn(self.c1, list(store.contexts()))
+        store.remove_graph(graph)
+        self.assertNotIn(self.c1, list(store.contexts()))
+        self.assertEqual(len(list(graph.triples((None, None, None)))), 0)
+        store.remove_graph(Graph(store, empty))
+        self.assertNotIn(empty, list(store.contexts()))
+
     def testRemoveContext(self):
         c1 = self.c1
 

tox.ini

@@ -1,11 +1,10 @@
 [tox]
 envlist =
-    py37,py38,py39,py310,lint
+    py39,py310,py311,py312,py313,lint
 
 [testenv]
 passenv = DB,DBURI
 commands =
-    {envpython} setup.py clean --all
     pytest --cov=rdflib_sqlalchemy
 
 deps =
@@ -21,10 +20,11 @@ deps =
 
 [gh-actions]
 python =
-    3.7: py37, lint
-    3.8: py38
-    3.9: py39
+    3.9: py39, lint
     3.10: py310
+    3.11: py311
+    3.12: py312
+    3.13: py313
 
 [flake8]
 max-line-length = 120