From f8bafb20b734ebb1b35c812fa723dbfb04257f73 Mon Sep 17 00:00:00 2001 From: Taewoon Kim Date: Sat, 14 Nov 2026 15:27:36 +0900 Subject: [PATCH 1/5] Tests first: native count() builds no result object for a row it counts (refs #388 and #315) count() walked the whole result with `for _ in self`: one NativeResult per row, built and thrown away. These tests pin the number it returns (the rows the same query gives through to_list(), at sizes around the 256-row first reply and the 2048-row later replies, for typed and wide rows, aggregates, a Cypher result and a command), the state it leaves the result in (only the rows not yet read are counted, an empty or exhausted result counts 0, a closed result or database raises), the error an engine failure met after the first reply raises, and that no NativeResult is built for a counted row. The last one fails on the parent. Co-Authored-By: Claude Sonnet 5.5 --- tests/conftest.py | 2 + tests/test_native_count.py | 187 +++++++++++++++++++++++++++++++++++++ 2 files changed, 189 insertions(+) create mode 100644 tests/test_native_count.py diff --git a/tests/conftest.py b/tests/conftest.py index 22b929f..a06e590 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -120,6 +120,8 @@ collect_ignore.append("test_native_vertex_vector_crossings.py") # the native columnar fast path (arcadedb._native.bulk.insert_columns_fast) with a rows-dropped fault in it, found by the ingest-integrity check (#336) collect_ignore.append("test_native_ingest_integrity_faults.py") + # count() of a native result set counts the rows without building a result object for each (and, in the library, without encoding them) + collect_ignore.append("test_native_count.py") if not os.path.isfile( os.path.join(os.path.dirname(os.path.dirname(__file__)), "docs", "index.md") ): diff --git a/tests/test_native_count.py b/tests/test_native_count.py new file mode 100644 index 0000000..c914b7a --- /dev/null +++ b/tests/test_native_count.py @@ -0,0 +1,187 @@ +"""``count()`` on the native build builds no result object for the rows it counts (#388, #315). + +``NativeResultSet.count()`` used to walk the whole result with ``for _ in self``: one ``NativeResult`` was built per row (a tuple popped +from the buffer, an object with its own dict and meta) just to be counted and thrown away. It now takes the rows of a reply as ``len``. + +The counts here are objects built, not seconds: deterministic. The behaviour tests pin what ``count()`` is for (the number of rows still +to be read, the state the result is left in, the errors) at sizes around the batch limits (256 rows in the first reply, 2048 per later +reply), so a cheaper count cannot buy a different answer. The oracle for the number is the rows the same query gives through +``to_list()``. + +Needs the native library: conftest.py does not collect this file on the JVM build. +""" + +from datetime import date, datetime +from decimal import Decimal + +import pytest + +from arcadedb import ArcadeDBError +from arcadedb._native import database, lib + +ROWS = 5000 # 256 in the first reply, then 2048 + 2048 + 648 +EDGE_SIZES = [0, 1, 255, 256, 257, 2303, 2304, 2305, ROWS] + + +@pytest.fixture +def db(temp_db): + temp_db.command( + "sql", "CREATE DOCUMENT TYPE R BUCKETS 1" + ) # one bucket: a scan is in insertion order + temp_db.command("sql", "CREATE PROPERTY R.id INTEGER") + temp_db.command("sql", "CREATE PROPERTY R.txt STRING") + temp_db.command("sql", "CREATE PROPERTY R.g INTEGER") + temp_db.insert_many( + "R", [{"id": i, "txt": f"t{i}", "g": i % 7} for i in range(ROWS)] + ) + lib.flush_closes() # a cursor closed earlier is not part of a call measured below + return temp_db + + +@pytest.fixture +def built(monkeypatch): + """The number of ``NativeResult`` objects built while it is installed.""" + seen = [] + real = database.NativeResult + + def counting(*args, **kwargs): + seen.append(1) + return real(*args, **kwargs) + + monkeypatch.setattr(database, "NativeResult", counting) + return seen + + +def _state(rs): + return rs._closed, rs._exhausted, rs._more + + +# -- the number: the same as the rows the same query gives + + +@pytest.mark.parametrize("size", EDGE_SIZES) +def test_count_is_the_number_of_rows_to_list_gives(db, size): + query = f"SELECT id, txt FROM R LIMIT {size}" + rows = db.query("sql", query).to_list() + assert len(rows) == size + assert db.query("sql", query).count() == size + + +def test_count_of_typed_wide_rows_is_the_number_of_rows(db): + db.command("sql", "CREATE DOCUMENT TYPE W") + n = 2500 + rows = [ + { + "i": i, + "d": date(2020, 1, 1 + i % 28), + "dt": datetime(2021, 2, 3, 4, 5, i % 60), + "dec": Decimal(f"{i}.25"), + "b": bytes([i % 256, 7]), + "big": 2**60 + i, + "v": [float(i), 1.5, 2.5], + "m": {"a": i, "n": [i, {"z": None}]}, + "n": None, + } + for i in range(n) + ] + db.insert_many("W", rows) + want = len(db.query("sql", "SELECT FROM W").to_list()) + assert want == n + assert db.query("sql", "SELECT FROM W").count() == n + + +def test_count_of_aggregates_and_projections(db): + assert db.query("sql", "SELECT count(*) AS c FROM R").count() == 1 + assert db.query("sql", "SELECT g, count(*) AS c FROM R GROUP BY g").count() == 7 + assert ( + db.query("sql", "SELECT @rid AS rid, @type AS t FROM R WHERE id < 3000").count() + == 3000 + ) + + +def test_count_of_a_cypher_result(db): + db.command("sql", "CREATE VERTEX TYPE V") + db.insert_many("V", [{"k": i} for i in range(2600)]) + assert db.query("opencypher", "MATCH (n:V) RETURN n.k AS k").count() == 2600 + + +def test_count_of_a_command_drains_it_and_the_statement_ran_to_the_end(db): + """A command's rows are pulled by the library when the statement runs and read late: counting them runs the statement through.""" + db.command("sql", "CREATE DOCUMENT TYPE B") + with db.transaction(): + rs = db.command("sql", "INSERT INTO B FROM SELECT FROM R") + assert rs.count() == ROWS + assert db.query("sql", "SELECT count(*) AS c FROM B").first().get("c") == ROWS + + +# -- the state a result is left in is the one iteration leaves it in + + +def test_count_counts_only_the_rows_not_yet_read(db): + rs = db.query("sql", "SELECT id FROM R") + assert [next(rs).get("id") for _ in range(3)] == [0, 1, 2] + assert rs.count() == ROWS - 3 + assert _state(rs) == (True, True, False) + assert rs.count() == 0 # exhausted: nothing remains + assert list(rs) == [] and rs.to_list() == [] and rs.first() is None + + +def test_count_reads_across_the_first_reply_after_rows_were_taken_from_it(db): + rs = db.query("sql", "SELECT id FROM R") + for _ in range( + 300 + ): # past the 256 rows of the first reply: the cursor was read once + next(rs) + assert rs.count() == ROWS - 300 + + +def test_count_of_an_empty_result_leaves_it_exhausted(db): + rs = db.query("sql", "SELECT FROM R WHERE id < 0") + assert rs.count() == 0 + assert _state(rs) == (True, True, False) + assert rs.count() == 0 + + +def test_count_after_the_result_was_closed_early_raises(db): + rs = db.query("sql", "SELECT FROM R") + rs.close() + with pytest.raises(ArcadeDBError, match="closed before all its rows were read"): + rs.count() + + +def test_count_on_a_closed_database_raises(temp_db): + temp_db.command("sql", "CREATE DOCUMENT TYPE T") + temp_db.insert_many("T", [{"k": i} for i in range(600)]) + rs = temp_db.query("sql", "SELECT FROM T") + temp_db.close() + with pytest.raises(ArcadeDBError, match="Database is closed"): + rs.count() + + +def test_count_raises_an_engine_error_met_after_the_first_reply_and_closes_the_result( + db, +): + # the first 256 rows are fine; the division by zero is at id 2999, in a later reply + rs = db.query("sql", "SELECT 1 / (id - 2999) AS f FROM R") + with pytest.raises(ArcadeDBError) as info: + rs.count() + assert str(info.value).startswith("Reading the result set failed: ") + assert "Exception: " in str(info.value) + assert _state(rs) == (True, True, False) + assert list(rs) == [] + # the cursor is closed on the Java side, not left open: the database still reads and drops + assert db.query("sql", "SELECT count(*) AS c FROM R").first().get("c") == ROWS + + +# -- objects: a row that is only counted is not turned into a result object + + +def test_count_builds_no_result_object(db, built): + rs = db.query("sql", "SELECT id, txt FROM R") + assert rs.count() == ROWS + assert built == [] + rs = db.query("sql", "SELECT id, txt FROM R") + next(rs) + assert len(built) == 1 # the spy is in place: a row read by next() is one object + assert rs.count() == ROWS - 1 + assert len(built) == 1 From 2e382ed3292a4316fa5bb9bd2566990de2ccb66b Mon Sep 17 00:00:00 2001 From: Taewoon Kim Date: Sat, 14 Nov 2026 15:28:14 +0900 Subject: [PATCH 2/5] Native count(): count the rows of a reply as len, build no NativeResult (refs #388 and #315) count() read the result with `for _ in self`, so every counted row was a tuple popped from the buffer and a NativeResult built to be thrown away. It now adds len() of the buffer and of each reply the cursor gives, through _fill(into) as to_list() does. The replies are still read and parsed (the library encodes every row), which is what the next commit removes. Checked without the library: the old and the new count return the same number and leave the result in the same state (closed, exhausted, no cursor) for 0, 1, 255, 256, 257, 2303, 2304, 2305 and 5000 rows and for an error met after the first reply; the new one builds 0 NativeResult where the old one built one per row. Not measured: the speed claim waits for a timed A/B. Co-Authored-By: Claude Sonnet 5.5 --- src/arcadedb/_native/database.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/src/arcadedb/_native/database.py b/src/arcadedb/_native/database.py index 68fe2f1..c518e37 100644 --- a/src/arcadedb/_native/database.py +++ b/src/arcadedb/_native/database.py @@ -644,9 +644,19 @@ def iter_chunks(self, size: int = 1000, convert_types: bool = True): yield chunk def count(self) -> int: - n = 0 - for _ in self: - n += 1 + """The number of rows still to be read; the result is finished. A reply's rows are counted as ``len`` of the batch: no + ``NativeResult`` and no tuple per row is built for a row that is only counted. + """ + if not self._readable(): + return 0 + buf = self._buf + n = len(buf) + buf.clear() + batch: list = [] + while self._fill(batch): + n += len(batch) + batch.clear() + self._finish() return n def first(self) -> Optional[NativeResult]: From c2596b04588c0663d7cae46db75a68684a56f7d4 Mon Sep 17 00:00:00 2001 From: Taewoon Kim Date: Sat, 14 Nov 2026 15:28:25 +0900 Subject: [PATCH 3/5] Tests first: a native count() with rows in the library is one foreign call, arcadedb_n_count (refs #388 and #315) After the previous commit count() still reads every reply past the first one as a JSON text (arcadedb_n_next_bin: Java encodes each row, Python parses it) just to take len(). These tests pin the target: a result with rows left in the library is counted with one call of the entry point arcadedb_n_count and no other (the library drains and closes the cursor, so no second close is queued), for a query cursor and for a command cursor; a result whose first reply holds every row, and an exhausted one, make no crossing at all; no reply text is read and no NativeResult is built. They fail on the previous commit (the calls are arcadedb_n_next_bin, and the entry point does not exist). The behaviour tests of the first commit stay the oracle for the number, the state and the errors. Co-Authored-By: Claude Sonnet 5.5 --- tests/test_native_count.py | 96 +++++++++++++++++++++++++++++++++++--- 1 file changed, 89 insertions(+), 7 deletions(-) diff --git a/tests/test_native_count.py b/tests/test_native_count.py index c914b7a..f556d13 100644 --- a/tests/test_native_count.py +++ b/tests/test_native_count.py @@ -1,12 +1,15 @@ -"""``count()`` on the native build builds no result object for the rows it counts (#388, #315). +"""``count()`` on the native build counts the rows in the library and builds nothing for them (#388, #315). -``NativeResultSet.count()`` used to walk the whole result with ``for _ in self``: one ``NativeResult`` was built per row (a tuple popped -from the buffer, an object with its own dict and meta) just to be counted and thrown away. It now takes the rows of a reply as ``len``. +``NativeResultSet.count()`` used to walk the whole result with ``for _ in self``: every batch past the first reply crossed into the +library as a JSON text (``arcadedb_n_next_bin``), Java encoded every row into it, Python parsed it into one dict per row, and one +``NativeResult`` was built per row just to be thrown away. It now takes the rows already held in the first reply as ``len`` and asks +the library for the rest in one crossing, ``arcadedb_n_count``, which returns a long: the cursor is drained and closed on the Java side +and no row is encoded. A result with nothing left in the library makes no crossing at all. -The counts here are objects built, not seconds: deterministic. The behaviour tests pin what ``count()`` is for (the number of rows still -to be read, the state the result is left in, the errors) at sizes around the batch limits (256 rows in the first reply, 2048 per later -reply), so a cheaper count cannot buy a different answer. The oracle for the number is the rows the same query gives through -``to_list()``. +The counts here are crossings and objects built, not seconds: deterministic. The behaviour tests pin what ``count()`` is for (the +number of rows still to be read, the state the result is left in, the errors) at sizes around the batch limits (256 rows in the first +reply, 2048 per later reply), so a lower count of crossings cannot buy a different answer. The oracle for the number is the rows the +same query gives through ``to_list()``. Needs the native library: conftest.py does not collect this file on the JVM build. """ @@ -38,6 +41,35 @@ def db(temp_db): return temp_db +class _ForeignCalls: + """The entry points of the library called while it is installed, by name (a proxy of the loaded library object).""" + + def __init__(self, monkeypatch): + self.names = [] + names = self.names + real_lib = lib._lib + + class _Proxy: + def __getattr__(self, attr): + function = getattr(real_lib, attr) + + def counted(*args): + names.append(attr) + return function(*args) + + return counted + + monkeypatch.setattr(lib, "_lib", _Proxy()) + + def reset(self): + del self.names[:] + + +@pytest.fixture +def foreign(db, monkeypatch): + return _ForeignCalls(monkeypatch) + + @pytest.fixture def built(monkeypatch): """The number of ``NativeResult`` objects built while it is installed.""" @@ -185,3 +217,53 @@ def test_count_builds_no_result_object(db, built): assert len(built) == 1 # the spy is in place: a row read by next() is one object assert rs.count() == ROWS - 1 assert len(built) == 1 + + +# -- crossings: the rows are counted in the library, not carried over + + +def test_count_of_a_result_with_rows_in_the_library_is_one_foreign_call(db, foreign): + rs = db.query("sql", "SELECT id, txt FROM R") + assert len(rs._buf) == database._FIRST_BATCH and rs._more + foreign.reset() + assert rs.count() == ROWS + assert foreign.names == ["arcadedb_n_count"] + lib.flush_closes() + assert ( + "arcadedb_n_rs_close_many" not in foreign.names + ) # the library closed the cursor: no second close is queued + + +def test_count_of_a_command_cursor_is_one_foreign_call(db, foreign): + with db.transaction(): + rs = db.command("sql", "UPDATE R SET touched = 1 RETURN AFTER WHERE id < 3000") + foreign.reset() + assert rs.count() == 3000 + assert foreign.names == ["arcadedb_n_count"] + + +def test_count_of_a_result_the_first_reply_holds_in_full_makes_no_crossing(db, foreign): + rs = db.query("sql", "SELECT id FROM R LIMIT 200") + assert not rs._more + foreign.reset() + assert rs.count() == 200 + assert foreign.names == [] + + +def test_count_of_an_exhausted_result_makes_no_crossing(db, foreign): + rs = db.query("sql", "SELECT id FROM R") + rs.to_list() + foreign.reset() + assert rs.count() == 0 + assert foreign.names == [] + + +def test_count_reads_no_reply_text_and_builds_no_result_object(db, monkeypatch, built): + taken = [] + real = lib._take + monkeypatch.setattr(lib, "_take", lambda *a: taken.append(1) or real(*a)) + rs = db.query("sql", "SELECT id, txt FROM R") + taken.clear() + assert rs.count() == ROWS + assert taken == [] # no batch came back as text + assert built == [] # and no NativeResult was built for a row From c1f73e47d6a1a6d1ff414093918838dd690249c5 Mon Sep 17 00:00:00 2001 From: Taewoon Kim Date: Sat, 14 Nov 2026 15:29:44 +0900 Subject: [PATCH 4/5] Native count(): arcadedb_n_count returns the number of rows still in the library as one long (refs #388 and #315) After the previous commit count() still asked the library for every reply past the first one: Java encoded each row into a JSON text (arcadedb_n_next_bin), Python parsed it into one dict per row, and the dicts were only measured with len(). The new entry point arcadedb_n_count(thread, cursor) drains the cursor in Java (hasNext and next, no row encoded) and closes it, and returns the count as a long; -1 is an error whose text arcadedb_n_last_error holds, built by the same errorJson as the "rerr" of a JSON reply, so the exception count() raises, and the state it leaves the result in, do not change. Like arcadedb_n_tx and arcadedb_n_cmd it returns no reply text: one foreign call, no malloc, no arcadedb_free, no decode. count() asks it only when the cursor still has rows: a result whose first reply holds every row, and an exhausted one, make no crossing. The rows are no longer encoded to be counted, so a row the codec could not encode no longer fails a count(); that was an accident of the old route, not a behaviour. Not measured: the speed and memory claims wait for a timed A/B against 26.11.1.post3. Co-Authored-By: Claude Sonnet 5.5 --- .../org/humemai/nativelib/ArcadeNative.java | 30 +++++++++++++++++++ src/arcadedb/_native/database.py | 21 +++++++++---- src/arcadedb/_native/lib.py | 20 ++++++++++--- tests/test_native_count.py | 4 ++- 4 files changed, 64 insertions(+), 11 deletions(-) diff --git a/native/src/main/java/org/humemai/nativelib/ArcadeNative.java b/native/src/main/java/org/humemai/nativelib/ArcadeNative.java index b1d3e7f..65471d3 100644 --- a/native/src/main/java/org/humemai/nativelib/ArcadeNative.java +++ b/native/src/main/java/org/humemai/nativelib/ArcadeNative.java @@ -371,6 +371,36 @@ private static CCharPointer frameOut(final int length, final byte[] b) { return p; } + /** + * The number of rows an open cursor still holds, as one long: the cursor is drained here (the rows are pulled from the engine but + * not encoded) and closed. Rows a cursor already gave out in an earlier reply are not counted. -1 on error: the error text is then + * read with arcadedb_n_last_error on the same thread, as for arcadedb_n_cmd, built by the same errorJson as the "rerr" of a JSON + * reply, so the exception the caller raises does not change. The cursor is closed on an error too. No reply text, so no malloc, no + * arcadedb_free and no JSON decode in the caller, and no row is built as text. + */ + @CEntryPoint(name = "arcadedb_n_count") + public static long count(final IsolateThread thread, final long rsId) { + ResultSet rs = null; + try { + rs = RESULT_SETS.get(rsId); + if (rs == null) + throw new IllegalArgumentException("unknown result set " + rsId); + long n = 0; + while (rs.hasNext()) { + rs.next(); + n++; + } + RESULT_SETS.remove(rsId); + closeQuietly(rs); + return n; + } catch (final Throwable t) { + RESULT_SETS.remove(rsId); + closeQuietly(rs); + LAST_ERROR.set(errorJson(t)); + return -1; + } + } + @CEntryPoint(name = "arcadedb_n_rs_close") public static CCharPointer rsClose(final IsolateThread thread, final long rsId) { closeQuietly(RESULT_SETS.remove(rsId)); diff --git a/src/arcadedb/_native/database.py b/src/arcadedb/_native/database.py index c518e37..5fbe916 100644 --- a/src/arcadedb/_native/database.py +++ b/src/arcadedb/_native/database.py @@ -644,18 +644,27 @@ def iter_chunks(self, size: int = 1000, convert_types: bool = True): yield chunk def count(self) -> int: - """The number of rows still to be read; the result is finished. A reply's rows are counted as ``len`` of the batch: no - ``NativeResult`` and no tuple per row is built for a row that is only counted. + """The number of rows still to be read; the result is finished. The rows already in this object (the first reply) are counted + as ``len``, the rows still in the library by one crossing, ``arcadedb_n_count``, which returns a long: the library drains and + closes the cursor and encodes no row, and no ``NativeResult`` or dict is built for a counted row. """ if not self._readable(): return 0 buf = self._buf n = len(buf) buf.clear() - batch: list = [] - while self._fill(batch): - n += len(batch) - batch.clear() + if self._pending_error is not None: + self._fill() # the error of the first reply: raised, as reading the rows before it would + if self._more and self._rs: + rows = lib.count(self._rs) + # the library closed the cursor, on success and on an error alike: this object holds no cursor any more + self._more = False + self._rs = 0 + if rows < 0: + self._closed = True + self._exhausted = True + raise _read_error(json.loads(lib.last_error())) + n += rows self._finish() return n diff --git a/src/arcadedb/_native/lib.py b/src/arcadedb/_native/lib.py index 96fbc6e..ec9e9e6 100644 --- a/src/arcadedb/_native/lib.py +++ b/src/arcadedb/_native/lib.py @@ -246,6 +246,7 @@ class _Params(ctypes.Structure): "arcadedb_n_next_col": [P, L, N, S, N], "arcadedb_n_rs_close": [P, L], "arcadedb_n_facets": [P, L], + "arcadedb_n_count": [P, L], "arcadedb_n_cmd": [P, L, N, S, S, S, S, L, N], "arcadedb_n_last_error": [P], "arcadedb_n_rs_close_many": [P, P, N], @@ -272,9 +273,10 @@ class _Params(ctypes.Structure): "arcadedb_n_set_upcall": [P, P], "arcadedb_free": [P, P], } - # the entry points that answer with a number, not with the address of a reply text: a cursor id (arcadedb_n_cmd) and the - # transaction state (arcadedb_n_tx); -1 is an error for both, its text is read with arcadedb_n_last_error - returns_long = ("arcadedb_n_cmd", "arcadedb_n_tx") + # the entry points that answer with a number, not with the address of a reply text: a cursor id (arcadedb_n_cmd), the + # transaction state (arcadedb_n_tx) and a row count (arcadedb_n_count); -1 is an error for all, its text is read with + # arcadedb_n_last_error + returns_long = ("arcadedb_n_cmd", "arcadedb_n_tx", "arcadedb_n_count") for name, argtypes in sigs.items(): fn = getattr(lib, name, None) if fn is None: # a library built before this entry point existed @@ -529,8 +531,18 @@ def tx(handle, op) -> int: return lib.arcadedb_n_tx(th, handle, op) +def count(rs_id: int) -> int: + """``arcadedb_n_count``: the number of rows the cursor still holds (it is drained and closed in the library), or -1 for an error (read + it with :func:`last_error`; the cursor is closed then too). One foreign call: no reply text to copy, free or decode, no row encoded. + """ + lib = _lib or _load() + t = _local.__dict__.get("thread") + th = t.handle if t is not None else _thread_handle() + return lib.arcadedb_n_count(th, rs_id) + + def last_error() -> str: - """The error of the last failed :func:`cmd` or :func:`tx` on this thread, as the JSON text of an error reply.""" + """The error of the last failed :func:`cmd`, :func:`tx` or :func:`count` on this thread, as the JSON text of an error reply.""" return call("arcadedb_n_last_error") diff --git a/tests/test_native_count.py b/tests/test_native_count.py index f556d13..4d73da1 100644 --- a/tests/test_native_count.py +++ b/tests/test_native_count.py @@ -93,7 +93,9 @@ def _state(rs): @pytest.mark.parametrize("size", EDGE_SIZES) def test_count_is_the_number_of_rows_to_list_gives(db, size): - query = f"SELECT id, txt FROM R LIMIT {size}" + query = ( + f"SELECT id, txt FROM R LIMIT {size}" # nosec B608 - an int from the list above + ) rows = db.query("sql", query).to_list() assert len(rows) == size assert db.query("sql", query).count() == size From 3574efc6dd4bf0ca187655f5c0cc3efa908477a2 Mon Sep 17 00:00:00 2001 From: Taewoon Kim Date: Fri, 20 Nov 2026 13:31:05 +0900 Subject: [PATCH 5/5] Native count(): a statement that mentions FACETS is counted row by row, so its facets survive (refs #388 and #315) Main now appends the facets of a FACETS query to the reply that drains the cursor, and a facets() after the rows are read returns them. arcadedb_n_count drains and closes the cursor in the library and carries no facets, so count() followed by facets() returned None on the rebased branch. A result records at creation whether its statement text holds the word FACETS (any case, a string literal included, so a false positive costs speed only); count() reads such a result with the iterating route of main and uses the new entry for every other result. Tests: count() then facets() equals facets() after to_list() on an engine that has the clause (skipped before 26.12.1), a statement that mentions the word makes no arcadedb_n_count call, and a statement without it still makes the single call. Co-Authored-By: Claude Sonnet 5.5 --- src/arcadedb/_native/database.py | 29 ++++++++++++++++-- tests/test_native_count.py | 51 ++++++++++++++++++++++++++++++++ 2 files changed, 77 insertions(+), 3 deletions(-) diff --git a/src/arcadedb/_native/database.py b/src/arcadedb/_native/database.py index 5fbe916..d53fce4 100644 --- a/src/arcadedb/_native/database.py +++ b/src/arcadedb/_native/database.py @@ -199,9 +199,21 @@ def __repr__(self) -> str: ) +_FACETS_WORD = re.compile(r"\bfacets\b", re.IGNORECASE) + + +def _mentions_facets(command) -> bool: + """Whether a statement's text holds the word FACETS (any case, a string literal included: a false positive only costs speed).""" + return isinstance(command, str) and _FACETS_WORD.search(command) is not None + + class NativeResultSet: """Cursor over a statement's rows, fetched in batches through ``arcadedb_n_next``.""" + # a result of a statement that may have a FACETS clause: the facets ride on the reply that drains the cursor, so count() reads + # such a result row by row and does not drain it in the library with arcadedb_n_count (set by Database._exec) + _may_have_facets = False + def __init__(self, database, reply: dict, tagged: bool = True): self._database = database # strong reference: a result keeps its database alive # whether a reply of this result carried a restored tagged value (codec.loads_tagged), so a row may hold a value that is not @@ -646,8 +658,15 @@ def iter_chunks(self, size: int = 1000, convert_types: bool = True): def count(self) -> int: """The number of rows still to be read; the result is finished. The rows already in this object (the first reply) are counted as ``len``, the rows still in the library by one crossing, ``arcadedb_n_count``, which returns a long: the library drains and - closes the cursor and encodes no row, and no ``NativeResult`` or dict is built for a counted row. + closes the cursor and encodes no row, and no ``NativeResult`` or dict is built for a counted row. A result of a statement that + may have a FACETS clause is counted row by row instead: its facets come with the reply that drains the cursor, and a drain + in the library would lose them. """ + if self._may_have_facets: + n = 0 + for _ in self: + n += 1 + return n if not self._readable(): return 0 buf = self._buf @@ -831,7 +850,9 @@ def _exec( ) if rid < 0: raise _error(prefix, json.loads(lib.last_error())) - return NativeResultSet._lazy(self, rid) + rs = NativeResultSet._lazy(self, rid) + rs._may_have_facets = _mentions_facets(command) + return rs # phase 7 (arcadedb_n_exec flags): 2 = a command's rows are pulled but encoded only when read, 4 = float arrays come back as a float32 tail if flags & 1 and _LAZY_COMMAND_ROWS and not eager: flags |= 2 @@ -873,7 +894,9 @@ def _exec( reply, tagged = loads_tagged(raw, floats) if not reply.get("ok"): raise _error(prefix, reply) - return NativeResultSet(self, reply, tagged) + rs = NativeResultSet(self, reply, tagged) + rs._may_have_facets = _mentions_facets(command) + return rs def query(self, language: str, command: str, *args) -> NativeResultSet: """Run a read statement. Parameters bind as on the JPype path (``?`` positional, ``:name`` from one dict).""" diff --git a/tests/test_native_count.py b/tests/test_native_count.py index 4d73da1..25ca31e 100644 --- a/tests/test_native_count.py +++ b/tests/test_native_count.py @@ -269,3 +269,54 @@ def test_count_reads_no_reply_text_and_builds_no_result_object(db, monkeypatch, assert rs.count() == ROWS assert taken == [] # no batch came back as text assert built == [] # and no NativeResult was built for a row + + +# -- facets: a FACETS statement is counted row by row, because its facets ride on the reply that drains the cursor + +FACETS_Q = "SELECT id FROM R ORDER BY id FACETS g TERMS TOP 10" + + +def _facets_after_to_list(db): + """The facets of FACETS_Q after its rows were read with to_list(), or a skip on an engine that predates the FACETS clause.""" + rs = db.query("sql", FACETS_Q) + try: + rows = rs.to_list() + except ArcadeDBError as e: + if "syntax error" in str(e).lower(): + pytest.skip("the engine predates the FACETS clause (26.12.1)") + raise + assert len(rows) == ROWS + facets = rs.facets() + assert facets is not None + return facets + + +def test_count_then_facets_of_a_facets_query_returns_the_facets(db, foreign): + expected = _facets_after_to_list(db) + rs = db.query("sql", FACETS_Q) + assert ( + len(rs._buf) == database._FIRST_BATCH and rs._more + ) # rows still in the library + foreign.reset() + assert rs.count() == ROWS + assert "arcadedb_n_count" not in foreign.names # the statement was read row by row + assert rs.facets() == expected + + +def test_a_statement_that_mentions_facets_is_counted_row_by_row(db, foreign): + """Any text with the word, in any case and inside a literal too: a false positive costs speed, never a wrong number.""" + rs = db.query("sql", "SELECT id, 'Facets' AS w FROM R") + assert rs._more + foreign.reset() + assert rs.count() == ROWS + assert "arcadedb_n_count" not in foreign.names + assert rs._closed and rs._exhausted + + +def test_a_statement_without_the_word_facets_still_uses_the_counting_entry(db, foreign): + for query in ("SELECT id, txt FROM R", "SELECT id AS facetsum FROM R"): + rs = db.query("sql", query) + assert rs._more and not rs._may_have_facets + foreign.reset() + assert rs.count() == ROWS + assert foreign.names == ["arcadedb_n_count"]