diff --git a/native/src/main/java/org/humemai/nativelib/ArcadeNative.java b/native/src/main/java/org/humemai/nativelib/ArcadeNative.java index b1d3e7f..65471d3 100644 --- a/native/src/main/java/org/humemai/nativelib/ArcadeNative.java +++ b/native/src/main/java/org/humemai/nativelib/ArcadeNative.java @@ -371,6 +371,36 @@ private static CCharPointer frameOut(final int length, final byte[] b) { return p; } + /** + * The number of rows an open cursor still holds, as one long: the cursor is drained here (the rows are pulled from the engine but + * not encoded) and closed. Rows a cursor already gave out in an earlier reply are not counted. -1 on error: the error text is then + * read with arcadedb_n_last_error on the same thread, as for arcadedb_n_cmd, built by the same errorJson as the "rerr" of a JSON + * reply, so the exception the caller raises does not change. The cursor is closed on an error too. No reply text, so no malloc, no + * arcadedb_free and no JSON decode in the caller, and no row is built as text. + */ + @CEntryPoint(name = "arcadedb_n_count") + public static long count(final IsolateThread thread, final long rsId) { + ResultSet rs = null; + try { + rs = RESULT_SETS.get(rsId); + if (rs == null) + throw new IllegalArgumentException("unknown result set " + rsId); + long n = 0; + while (rs.hasNext()) { + rs.next(); + n++; + } + RESULT_SETS.remove(rsId); + closeQuietly(rs); + return n; + } catch (final Throwable t) { + RESULT_SETS.remove(rsId); + closeQuietly(rs); + LAST_ERROR.set(errorJson(t)); + return -1; + } + } + @CEntryPoint(name = "arcadedb_n_rs_close") public static CCharPointer rsClose(final IsolateThread thread, final long rsId) { closeQuietly(RESULT_SETS.remove(rsId)); diff --git a/src/arcadedb/_native/database.py b/src/arcadedb/_native/database.py index 68fe2f1..d53fce4 100644 --- a/src/arcadedb/_native/database.py +++ b/src/arcadedb/_native/database.py @@ -199,9 +199,21 @@ def __repr__(self) -> str: ) +_FACETS_WORD = re.compile(r"\bfacets\b", re.IGNORECASE) + + +def _mentions_facets(command) -> bool: + """Whether a statement's text holds the word FACETS (any case, a string literal included: a false positive only costs speed).""" + return isinstance(command, str) and _FACETS_WORD.search(command) is not None + + class NativeResultSet: """Cursor over a statement's rows, fetched in batches through ``arcadedb_n_next``.""" + # a result of a statement that may have a FACETS clause: the facets ride on the reply that drains the cursor, so count() reads + # such a result row by row and does not drain it in the library with arcadedb_n_count (set by Database._exec) + _may_have_facets = False + def __init__(self, database, reply: dict, tagged: bool = True): self._database = database # strong reference: a result keeps its database alive # whether a reply of this result carried a restored tagged value (codec.loads_tagged), so a row may hold a value that is not @@ -644,9 +656,35 @@ def iter_chunks(self, size: int = 1000, convert_types: bool = True): yield chunk def count(self) -> int: - n = 0 - for _ in self: - n += 1 + """The number of rows still to be read; the result is finished. The rows already in this object (the first reply) are counted + as ``len``, the rows still in the library by one crossing, ``arcadedb_n_count``, which returns a long: the library drains and + closes the cursor and encodes no row, and no ``NativeResult`` or dict is built for a counted row. A result of a statement that + may have a FACETS clause is counted row by row instead: its facets come with the reply that drains the cursor, and a drain + in the library would lose them. + """ + if self._may_have_facets: + n = 0 + for _ in self: + n += 1 + return n + if not self._readable(): + return 0 + buf = self._buf + n = len(buf) + buf.clear() + if self._pending_error is not None: + self._fill() # the error of the first reply: raised, as reading the rows before it would + if self._more and self._rs: + rows = lib.count(self._rs) + # the library closed the cursor, on success and on an error alike: this object holds no cursor any more + self._more = False + self._rs = 0 + if rows < 0: + self._closed = True + self._exhausted = True + raise _read_error(json.loads(lib.last_error())) + n += rows + self._finish() return n def first(self) -> Optional[NativeResult]: @@ -812,7 +850,9 @@ def _exec( ) if rid < 0: raise _error(prefix, json.loads(lib.last_error())) - return NativeResultSet._lazy(self, rid) + rs = NativeResultSet._lazy(self, rid) + rs._may_have_facets = _mentions_facets(command) + return rs # phase 7 (arcadedb_n_exec flags): 2 = a command's rows are pulled but encoded only when read, 4 = float arrays come back as a float32 tail if flags & 1 and _LAZY_COMMAND_ROWS and not eager: flags |= 2 @@ -854,7 +894,9 @@ def _exec( reply, tagged = loads_tagged(raw, floats) if not reply.get("ok"): raise _error(prefix, reply) - return NativeResultSet(self, reply, tagged) + rs = NativeResultSet(self, reply, tagged) + rs._may_have_facets = _mentions_facets(command) + return rs def query(self, language: str, command: str, *args) -> NativeResultSet: """Run a read statement. Parameters bind as on the JPype path (``?`` positional, ``:name`` from one dict).""" diff --git a/src/arcadedb/_native/lib.py b/src/arcadedb/_native/lib.py index 96fbc6e..ec9e9e6 100644 --- a/src/arcadedb/_native/lib.py +++ b/src/arcadedb/_native/lib.py @@ -246,6 +246,7 @@ class _Params(ctypes.Structure): "arcadedb_n_next_col": [P, L, N, S, N], "arcadedb_n_rs_close": [P, L], "arcadedb_n_facets": [P, L], + "arcadedb_n_count": [P, L], "arcadedb_n_cmd": [P, L, N, S, S, S, S, L, N], "arcadedb_n_last_error": [P], "arcadedb_n_rs_close_many": [P, P, N], @@ -272,9 +273,10 @@ class _Params(ctypes.Structure): "arcadedb_n_set_upcall": [P, P], "arcadedb_free": [P, P], } - # the entry points that answer with a number, not with the address of a reply text: a cursor id (arcadedb_n_cmd) and the - # transaction state (arcadedb_n_tx); -1 is an error for both, its text is read with arcadedb_n_last_error - returns_long = ("arcadedb_n_cmd", "arcadedb_n_tx") + # the entry points that answer with a number, not with the address of a reply text: a cursor id (arcadedb_n_cmd), the + # transaction state (arcadedb_n_tx) and a row count (arcadedb_n_count); -1 is an error for all, its text is read with + # arcadedb_n_last_error + returns_long = ("arcadedb_n_cmd", "arcadedb_n_tx", "arcadedb_n_count") for name, argtypes in sigs.items(): fn = getattr(lib, name, None) if fn is None: # a library built before this entry point existed @@ -529,8 +531,18 @@ def tx(handle, op) -> int: return lib.arcadedb_n_tx(th, handle, op) +def count(rs_id: int) -> int: + """``arcadedb_n_count``: the number of rows the cursor still holds (it is drained and closed in the library), or -1 for an error (read + it with :func:`last_error`; the cursor is closed then too). One foreign call: no reply text to copy, free or decode, no row encoded. + """ + lib = _lib or _load() + t = _local.__dict__.get("thread") + th = t.handle if t is not None else _thread_handle() + return lib.arcadedb_n_count(th, rs_id) + + def last_error() -> str: - """The error of the last failed :func:`cmd` or :func:`tx` on this thread, as the JSON text of an error reply.""" + """The error of the last failed :func:`cmd`, :func:`tx` or :func:`count` on this thread, as the JSON text of an error reply.""" return call("arcadedb_n_last_error") diff --git a/tests/conftest.py b/tests/conftest.py index 22b929f..a06e590 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -120,6 +120,8 @@ collect_ignore.append("test_native_vertex_vector_crossings.py") # the native columnar fast path (arcadedb._native.bulk.insert_columns_fast) with a rows-dropped fault in it, found by the ingest-integrity check (#336) collect_ignore.append("test_native_ingest_integrity_faults.py") + # count() of a native result set counts the rows without building a result object for each (and, in the library, without encoding them) + collect_ignore.append("test_native_count.py") if not os.path.isfile( os.path.join(os.path.dirname(os.path.dirname(__file__)), "docs", "index.md") ): diff --git a/tests/test_native_count.py b/tests/test_native_count.py new file mode 100644 index 0000000..25ca31e --- /dev/null +++ b/tests/test_native_count.py @@ -0,0 +1,322 @@ +"""``count()`` on the native build counts the rows in the library and builds nothing for them (#388, #315). + +``NativeResultSet.count()`` used to walk the whole result with ``for _ in self``: every batch past the first reply crossed into the +library as a JSON text (``arcadedb_n_next_bin``), Java encoded every row into it, Python parsed it into one dict per row, and one +``NativeResult`` was built per row just to be thrown away. It now takes the rows already held in the first reply as ``len`` and asks +the library for the rest in one crossing, ``arcadedb_n_count``, which returns a long: the cursor is drained and closed on the Java side +and no row is encoded. A result with nothing left in the library makes no crossing at all. + +The counts here are crossings and objects built, not seconds: deterministic. The behaviour tests pin what ``count()`` is for (the +number of rows still to be read, the state the result is left in, the errors) at sizes around the batch limits (256 rows in the first +reply, 2048 per later reply), so a lower count of crossings cannot buy a different answer. The oracle for the number is the rows the +same query gives through ``to_list()``. + +Needs the native library: conftest.py does not collect this file on the JVM build. +""" + +from datetime import date, datetime +from decimal import Decimal + +import pytest + +from arcadedb import ArcadeDBError +from arcadedb._native import database, lib + +ROWS = 5000 # 256 in the first reply, then 2048 + 2048 + 648 +EDGE_SIZES = [0, 1, 255, 256, 257, 2303, 2304, 2305, ROWS] + + +@pytest.fixture +def db(temp_db): + temp_db.command( + "sql", "CREATE DOCUMENT TYPE R BUCKETS 1" + ) # one bucket: a scan is in insertion order + temp_db.command("sql", "CREATE PROPERTY R.id INTEGER") + temp_db.command("sql", "CREATE PROPERTY R.txt STRING") + temp_db.command("sql", "CREATE PROPERTY R.g INTEGER") + temp_db.insert_many( + "R", [{"id": i, "txt": f"t{i}", "g": i % 7} for i in range(ROWS)] + ) + lib.flush_closes() # a cursor closed earlier is not part of a call measured below + return temp_db + + +class _ForeignCalls: + """The entry points of the library called while it is installed, by name (a proxy of the loaded library object).""" + + def __init__(self, monkeypatch): + self.names = [] + names = self.names + real_lib = lib._lib + + class _Proxy: + def __getattr__(self, attr): + function = getattr(real_lib, attr) + + def counted(*args): + names.append(attr) + return function(*args) + + return counted + + monkeypatch.setattr(lib, "_lib", _Proxy()) + + def reset(self): + del self.names[:] + + +@pytest.fixture +def foreign(db, monkeypatch): + return _ForeignCalls(monkeypatch) + + +@pytest.fixture +def built(monkeypatch): + """The number of ``NativeResult`` objects built while it is installed.""" + seen = [] + real = database.NativeResult + + def counting(*args, **kwargs): + seen.append(1) + return real(*args, **kwargs) + + monkeypatch.setattr(database, "NativeResult", counting) + return seen + + +def _state(rs): + return rs._closed, rs._exhausted, rs._more + + +# -- the number: the same as the rows the same query gives + + +@pytest.mark.parametrize("size", EDGE_SIZES) +def test_count_is_the_number_of_rows_to_list_gives(db, size): + query = ( + f"SELECT id, txt FROM R LIMIT {size}" # nosec B608 - an int from the list above + ) + rows = db.query("sql", query).to_list() + assert len(rows) == size + assert db.query("sql", query).count() == size + + +def test_count_of_typed_wide_rows_is_the_number_of_rows(db): + db.command("sql", "CREATE DOCUMENT TYPE W") + n = 2500 + rows = [ + { + "i": i, + "d": date(2020, 1, 1 + i % 28), + "dt": datetime(2021, 2, 3, 4, 5, i % 60), + "dec": Decimal(f"{i}.25"), + "b": bytes([i % 256, 7]), + "big": 2**60 + i, + "v": [float(i), 1.5, 2.5], + "m": {"a": i, "n": [i, {"z": None}]}, + "n": None, + } + for i in range(n) + ] + db.insert_many("W", rows) + want = len(db.query("sql", "SELECT FROM W").to_list()) + assert want == n + assert db.query("sql", "SELECT FROM W").count() == n + + +def test_count_of_aggregates_and_projections(db): + assert db.query("sql", "SELECT count(*) AS c FROM R").count() == 1 + assert db.query("sql", "SELECT g, count(*) AS c FROM R GROUP BY g").count() == 7 + assert ( + db.query("sql", "SELECT @rid AS rid, @type AS t FROM R WHERE id < 3000").count() + == 3000 + ) + + +def test_count_of_a_cypher_result(db): + db.command("sql", "CREATE VERTEX TYPE V") + db.insert_many("V", [{"k": i} for i in range(2600)]) + assert db.query("opencypher", "MATCH (n:V) RETURN n.k AS k").count() == 2600 + + +def test_count_of_a_command_drains_it_and_the_statement_ran_to_the_end(db): + """A command's rows are pulled by the library when the statement runs and read late: counting them runs the statement through.""" + db.command("sql", "CREATE DOCUMENT TYPE B") + with db.transaction(): + rs = db.command("sql", "INSERT INTO B FROM SELECT FROM R") + assert rs.count() == ROWS + assert db.query("sql", "SELECT count(*) AS c FROM B").first().get("c") == ROWS + + +# -- the state a result is left in is the one iteration leaves it in + + +def test_count_counts_only_the_rows_not_yet_read(db): + rs = db.query("sql", "SELECT id FROM R") + assert [next(rs).get("id") for _ in range(3)] == [0, 1, 2] + assert rs.count() == ROWS - 3 + assert _state(rs) == (True, True, False) + assert rs.count() == 0 # exhausted: nothing remains + assert list(rs) == [] and rs.to_list() == [] and rs.first() is None + + +def test_count_reads_across_the_first_reply_after_rows_were_taken_from_it(db): + rs = db.query("sql", "SELECT id FROM R") + for _ in range( + 300 + ): # past the 256 rows of the first reply: the cursor was read once + next(rs) + assert rs.count() == ROWS - 300 + + +def test_count_of_an_empty_result_leaves_it_exhausted(db): + rs = db.query("sql", "SELECT FROM R WHERE id < 0") + assert rs.count() == 0 + assert _state(rs) == (True, True, False) + assert rs.count() == 0 + + +def test_count_after_the_result_was_closed_early_raises(db): + rs = db.query("sql", "SELECT FROM R") + rs.close() + with pytest.raises(ArcadeDBError, match="closed before all its rows were read"): + rs.count() + + +def test_count_on_a_closed_database_raises(temp_db): + temp_db.command("sql", "CREATE DOCUMENT TYPE T") + temp_db.insert_many("T", [{"k": i} for i in range(600)]) + rs = temp_db.query("sql", "SELECT FROM T") + temp_db.close() + with pytest.raises(ArcadeDBError, match="Database is closed"): + rs.count() + + +def test_count_raises_an_engine_error_met_after_the_first_reply_and_closes_the_result( + db, +): + # the first 256 rows are fine; the division by zero is at id 2999, in a later reply + rs = db.query("sql", "SELECT 1 / (id - 2999) AS f FROM R") + with pytest.raises(ArcadeDBError) as info: + rs.count() + assert str(info.value).startswith("Reading the result set failed: ") + assert "Exception: " in str(info.value) + assert _state(rs) == (True, True, False) + assert list(rs) == [] + # the cursor is closed on the Java side, not left open: the database still reads and drops + assert db.query("sql", "SELECT count(*) AS c FROM R").first().get("c") == ROWS + + +# -- objects: a row that is only counted is not turned into a result object + + +def test_count_builds_no_result_object(db, built): + rs = db.query("sql", "SELECT id, txt FROM R") + assert rs.count() == ROWS + assert built == [] + rs = db.query("sql", "SELECT id, txt FROM R") + next(rs) + assert len(built) == 1 # the spy is in place: a row read by next() is one object + assert rs.count() == ROWS - 1 + assert len(built) == 1 + + +# -- crossings: the rows are counted in the library, not carried over + + +def test_count_of_a_result_with_rows_in_the_library_is_one_foreign_call(db, foreign): + rs = db.query("sql", "SELECT id, txt FROM R") + assert len(rs._buf) == database._FIRST_BATCH and rs._more + foreign.reset() + assert rs.count() == ROWS + assert foreign.names == ["arcadedb_n_count"] + lib.flush_closes() + assert ( + "arcadedb_n_rs_close_many" not in foreign.names + ) # the library closed the cursor: no second close is queued + + +def test_count_of_a_command_cursor_is_one_foreign_call(db, foreign): + with db.transaction(): + rs = db.command("sql", "UPDATE R SET touched = 1 RETURN AFTER WHERE id < 3000") + foreign.reset() + assert rs.count() == 3000 + assert foreign.names == ["arcadedb_n_count"] + + +def test_count_of_a_result_the_first_reply_holds_in_full_makes_no_crossing(db, foreign): + rs = db.query("sql", "SELECT id FROM R LIMIT 200") + assert not rs._more + foreign.reset() + assert rs.count() == 200 + assert foreign.names == [] + + +def test_count_of_an_exhausted_result_makes_no_crossing(db, foreign): + rs = db.query("sql", "SELECT id FROM R") + rs.to_list() + foreign.reset() + assert rs.count() == 0 + assert foreign.names == [] + + +def test_count_reads_no_reply_text_and_builds_no_result_object(db, monkeypatch, built): + taken = [] + real = lib._take + monkeypatch.setattr(lib, "_take", lambda *a: taken.append(1) or real(*a)) + rs = db.query("sql", "SELECT id, txt FROM R") + taken.clear() + assert rs.count() == ROWS + assert taken == [] # no batch came back as text + assert built == [] # and no NativeResult was built for a row + + +# -- facets: a FACETS statement is counted row by row, because its facets ride on the reply that drains the cursor + +FACETS_Q = "SELECT id FROM R ORDER BY id FACETS g TERMS TOP 10" + + +def _facets_after_to_list(db): + """The facets of FACETS_Q after its rows were read with to_list(), or a skip on an engine that predates the FACETS clause.""" + rs = db.query("sql", FACETS_Q) + try: + rows = rs.to_list() + except ArcadeDBError as e: + if "syntax error" in str(e).lower(): + pytest.skip("the engine predates the FACETS clause (26.12.1)") + raise + assert len(rows) == ROWS + facets = rs.facets() + assert facets is not None + return facets + + +def test_count_then_facets_of_a_facets_query_returns_the_facets(db, foreign): + expected = _facets_after_to_list(db) + rs = db.query("sql", FACETS_Q) + assert ( + len(rs._buf) == database._FIRST_BATCH and rs._more + ) # rows still in the library + foreign.reset() + assert rs.count() == ROWS + assert "arcadedb_n_count" not in foreign.names # the statement was read row by row + assert rs.facets() == expected + + +def test_a_statement_that_mentions_facets_is_counted_row_by_row(db, foreign): + """Any text with the word, in any case and inside a literal too: a false positive costs speed, never a wrong number.""" + rs = db.query("sql", "SELECT id, 'Facets' AS w FROM R") + assert rs._more + foreign.reset() + assert rs.count() == ROWS + assert "arcadedb_n_count" not in foreign.names + assert rs._closed and rs._exhausted + + +def test_a_statement_without_the_word_facets_still_uses_the_counting_entry(db, foreign): + for query in ("SELECT id, txt FROM R", "SELECT id AS facetsum FROM R"): + rs = db.query("sql", query) + assert rs._more and not rs._may_have_facets + foreign.reset() + assert rs.count() == ROWS + assert foreign.names == ["arcadedb_n_count"]