Test every document metadata field through the existing Prism API

.github/workflows/document-metadata.yaml

@@ -0,0 +1,123 @@
+name: Document metadata integration
+
+on:
+  push:
+  pull_request:
+  workflow_dispatch:
+    inputs:
+      db_ref:
+        description: Polypheny-DB revision containing document metadata support
+        required: false
+
+permissions:
+  contents: read
+
+jobs:
+  build-polypheny:
+    timeout-minutes: 30
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          repository: polypheny/Polypheny-DB
+          ref: ${{ inputs.db_ref || vars.POLYPHENY_DB_REF || '60148752327124249d5263ffaab15fdfc54f4517' }}
+
+      - uses: actions/setup-java@v4
+        with:
+          distribution: temurin
+          java-version: '17'
+
+      - name: Build server and plugins and validate Java metadata
+        run: >-
+          ./gradlew :dbms:shadowJar assemblePlugins
+          :plugins:prism-interface:test
+          :plugins:mongodb-adapter:test --tests '*MongoIndexMetadataTest'
+          -PwithoutpullingUi=true --console=plain
+
+      - name: Produce final executable server JAR
+        # dbms:jar and dbms:shadowJar currently share an output path. Run the
+        # shadow task last so the uploaded artifact contains runtime classes.
+        run: ./gradlew :dbms:shadowJar -PwithoutpullingUi=true --console=plain
+
+      - name: Select executable server JAR
+        run: |
+          python - <<'PY'
+          from pathlib import Path
+          import shutil
+          import zipfile
+          candidates = [path for path in Path('dbms/build/libs').glob('dbms-*.jar')
+                        if not path.stem.endswith(('-tests', '-sources', '-javadoc'))]
+          assert len(candidates) == 1, f'Expected one executable JAR, found {candidates}'
+          with zipfile.ZipFile(candidates[0]) as archive:
+              assert 'org/slf4j/Logger.class' in archive.namelist(), 'Server JAR lacks runtime dependencies'
+          shutil.copyfile(candidates[0], 'polypheny.jar')
+          PY
+
+      - uses: actions/upload-artifact@v4
+        with:
+          name: polypheny-jar
+          path: polypheny.jar
+
+  document-metadata:
+    timeout-minutes: 15
+    runs-on: ubuntu-latest
+    needs: build-polypheny
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+
+      - uses: actions/setup-java@v4
+        with:
+          distribution: temurin
+          java-version: '17'
+
+      - name: Install test dependencies
+        run: python -m pip install -r requirements.txt pymongo==4.10.1
+
+      - name: Start MongoDB replica set
+        run: |
+          docker run --detach --name prism-metadata-mongo --publish 27017:27017 mongo:7.0 --replSet prism --bind_ip_all
+          python - <<'PY'
+          import time
+          from pymongo import MongoClient
+          from pymongo.errors import PyMongoError
+          client = MongoClient('127.0.0.1', 27017, directConnection=True, serverSelectionTimeoutMS=1000)
+          deadline = time.monotonic() + 60
+          while True:
+              try:
+                  client.admin.command('ping')
+                  break
+              except PyMongoError:
+                  if time.monotonic() >= deadline:
+                      raise
+                  time.sleep(1)
+          client.admin.command('replSetInitiate', {'_id': 'prism', 'members': [{'_id': 0, 'host': '127.0.0.1:27017'}]})
+          while not client.admin.command('hello').get('isWritablePrimary'):
+              if time.monotonic() >= deadline:
+                  raise RuntimeError('MongoDB did not elect a primary')
+              time.sleep(0.2)
+          client.close()
+          PY
+
+      - uses: actions/download-artifact@v4
+        with:
+          name: polypheny-jar
+
+      - name: Run live document and index metadata suites
+        uses: polypheny/GitHub-Action-Run-Polypheny@v0.2.0
+        with:
+          cmd: python -m pytest -v tests/document_metadata.py tests/document_metadata_mongo.py
+          jar: polypheny.jar
+          autodocker: false
+        timeout-minutes: 10
+
+      - name: MongoDB logs
+        if: failure()
+        run: docker logs prism-metadata-mongo
+
+      - name: Stop MongoDB
+        if: always()
+        run: docker rm --force prism-metadata-mongo

.github/workflows/test.yaml

@@ -6,12 +6,40 @@ on:
 
 jobs:
   build-polypheny:
-    timeout-minutes: 15
+    timeout-minutes: 30
     runs-on: ubuntu-latest
     steps:
-      - uses: polypheny/GitHub-Action-Build-Polypheny@main
+      - uses: actions/checkout@v4
         with:
-          branch: master
+          repository: polypheny/Polypheny-DB
+          ref: ${{ vars.POLYPHENY_DB_REF || '60148752327124249d5263ffaab15fdfc54f4517' }}
+
+      - uses: actions/setup-java@v4
+        with:
+          distribution: temurin
+          java-version: '17'
+
+      - name: Build executable server and plugins
+        run: ./gradlew :dbms:shadowJar assemblePlugins -PwithoutpullingUi=true --console=plain
+
+      - name: Select executable server JAR
+        run: |
+          python - <<'PY'
+          from pathlib import Path
+          import shutil
+          import zipfile
+          candidates = [path for path in Path('dbms/build/libs').glob('dbms-*.jar')
+                        if not path.stem.endswith(('-tests', '-sources', '-javadoc'))]
+          assert len(candidates) == 1, f'Expected one executable JAR, found {candidates}'
+          with zipfile.ZipFile(candidates[0]) as archive:
+              assert 'org/slf4j/Logger.class' in archive.namelist(), 'Server JAR lacks runtime dependencies'
+          shutil.copyfile(candidates[0], 'polypheny.jar')
+          PY
+
+      - uses: actions/upload-artifact@v4
+        with:
+          name: polypheny-jar
+          path: polypheny.jar
 
   test-system-python:
     timeout-minutes: 10
@@ -20,10 +48,31 @@ jobs:
       matrix:
         version: ['3.9', '3.10', '3.11', '3.12', '3.13']
         os: [ubuntu-latest, macos-latest, windows-latest]
+        # Python 3.9/3.10 on macOS require the Intel distribution.
+        exclude:
+          - {version: '3.9', os: macos-latest}
+          - {version: '3.10', os: macos-latest}
+        include:
+          - {version: '3.9', os: macos-15-intel}
+          - {version: '3.10', os: macos-15-intel}
     runs-on: ${{ matrix.os }}
     needs: build-polypheny
     steps:
+      # setup-python has no Python 3.9 build for Ubuntu 26.04.
+      - uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3
+        if: matrix.os == 'ubuntu-latest' && matrix.version == '3.9'
+        with:
+          version: '0.13.8'
+          enable-cache: false
+
+      - name: Set up Python 3.9 on Ubuntu
+        if: matrix.os == 'ubuntu-latest' && matrix.version == '3.9'
+        run: |
+          uv venv --python 3.9.25 --managed-python --seed "$RUNNER_TEMP/polypheny-python"
+          echo "$RUNNER_TEMP/polypheny-python/bin" >> "$GITHUB_PATH"
+
       - uses: actions/setup-python@v5
+        if: matrix.os != 'ubuntu-latest' || matrix.version != '3.9'
         with:
           python-version: ${{ matrix.version }}
 
@@ -74,7 +123,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        adapter: [ mongodb, hsqldb, monetdb, postgresql, file, cottontail, neo4j ]
+        adapter: [ mongodb, hsqldb, monetdb, postgresql, file, neo4j ]
     runs-on: ubuntu-latest
     needs: build-polypheny
     steps:

README.md

@@ -25,6 +25,10 @@ cd /path/to/polyphney-connector-python-x.y.z/
 python setup.py install
 ```
 
+The connector uses the Prism 2.2 client definitions and connects to servers
+that report a compatible Prism 2.x API. The minor versions do not have to match.
+Document collection metadata requires server-side support introduced in Prism 2.2.
+
 
 ## Getting Started
 
@@ -83,7 +87,6 @@ Run the tests with coverage report:
 coverage run --source polypheny -m pytest && coverage report -m
 ```
 
-
 ## Roadmap
 See the [open issues](https://github.com/polypheny/Polypheny-DB/labels/A-python) for a list of proposed features (and known issues).
 

polypheny/serialize.py

@@ -121,12 +121,6 @@ def proto2py(value):
     elif name == "list":
         return list(map(lambda e: proto2py(e), value.list.values))
     elif name == "document":
-        res = {}
-        for entry in value.document.entries:
-            k = proto2py(entry.key)
-            assert isinstance(k, str)  # TODO: Correct?
-            v = proto2py(entry.value)
-            res[k] = v
-        return res
+        return {key: proto2py(entry) for key, entry in value.document.entries.items()}
     else:
         raise RuntimeError("Unhandled value type")

requirements.txt

@@ -1,4 +1,4 @@
-polypheny-prism-api==1.9
+polypheny-prism-api==2.2
 pytest
 coverage
 Sphinx

setup.py

@@ -77,6 +77,6 @@ def readme():
     },
     python_requires=">=3.8",
     install_requires=[
-        "polypheny-prism-api==1.9",
+        "polypheny-prism-api==2.2",
     ],
 )