From 684656cb267e2a85d0903ca8a2a63760614ad2e2 Mon Sep 17 00:00:00 2001 From: Lukas Geiger Date: Sat, 14 Nov 2026 22:55:01 +0100 Subject: [PATCH 1/3] fix(sql-tab): cap query results instead of loading every row The table view respects a row limit, but the SQL tab used fetchall(): "SELECT * FROM big_table" loaded every row into memory and the Treeview. Measured on 200,000 rows: 2.7 s and 200,000 tree rows, growing linearly - millions of rows freeze the app for minutes. The SQL tab now fetches at most SQL_RESULT_MAX_ROWS (10,000) rows plus one to detect truncation and says so in the status line ("erste 10000 Zeilen ... mit LIMIT/WHERE eingrenzen"); the export context records the limit. Same query after the fix: 0.29 s, 10,000 rows. Test red before the fix; real Tk run on a 200,000-row database compared against master; pytest 173 passed. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01THB2EzUDHN8Rq5fvjMqkBL --- SQLiteViewer.py | 16 +++++++++++++--- tests/test_bugsweep_20261114.py | 21 +++++++++++++++++++++ 2 files changed, 34 insertions(+), 3 deletions(-) create mode 100644 tests/test_bugsweep_20261114.py diff --git a/SQLiteViewer.py b/SQLiteViewer.py index 701efb3..cc0fb40 100644 --- a/SQLiteViewer.py +++ b/SQLiteViewer.py @@ -33,6 +33,9 @@ APP_TITLE = "SQLite Viewer Pro" APP_VERSION = "2.1.0" DEFAULT_LIMIT = 1000 +# Obergrenze für SQL-Tab-Ergebnisse: fetchall() lud bei „SELECT * FROM große_tabelle“ sonst Millionen +# Zeilen in Speicher und Treeview und fror die App ein. +SQL_RESULT_MAX_ROWS = 10_000 MAX_CELL_DISPLAY_LEN = 500 APP_ICON_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), "SQLiteViewer.ico") _CSV_FORMULA_PREFIXES = ("=", "+", "-", "@") @@ -1080,7 +1083,9 @@ def execute_sql(self): # Das ist robuster als ein Prefix-Check und fängt auch Kommentare # vor SELECT/WITH/PRAGMA/EXPLAIN korrekt ab. if cur.description is not None: - rows = cur.fetchall() + rows = cur.fetchmany(SQL_RESULT_MAX_ROWS + 1) + truncated = len(rows) > SQL_RESULT_MAX_ROWS + rows = rows[:SQL_RESULT_MAX_ROWS] cols = [desc[0] for desc in cur.description] SqlViewer._set_sql_result_state( self, @@ -1089,7 +1094,7 @@ def execute_sql(self): view="query", table=self.table_var.get() or None, query=sql, - row_limit=None, + row_limit=SQL_RESULT_MAX_ROWS if truncated else None, search_term=None, sort_column=None, sort_descending=None, @@ -1097,7 +1102,12 @@ def execute_sql(self): self._populate_sql_result(cols, rows) SqlViewer._update_export_actions(self) elapsed = (datetime.now() - start_time).total_seconds() - if rows: + if truncated: + self.sql_status.config( + text=f"✓ erste {len(rows)} Zeilen (Ergebnis hat {SQL_RESULT_MAX_ROWS}+; " + f"mit LIMIT/WHERE eingrenzen) in {elapsed:.3f}s" + ) + elif rows: self.sql_status.config(text=f"✓ {len(rows)} Zeilen in {elapsed:.3f}s") else: self.sql_status.config(text="✓ Keine Ergebnisse") diff --git a/tests/test_bugsweep_20261114.py b/tests/test_bugsweep_20261114.py new file mode 100644 index 0000000..640e003 --- /dev/null +++ b/tests/test_bugsweep_20261114.py @@ -0,0 +1,21 @@ +"""Regressionstests — Bugsweep SQLiteViewer 2026-11-14.""" +from __future__ import annotations + +import SQLiteViewer +from test_execute_sql import _build_fake_viewer + + +def test_sql_tab_does_not_load_an_unbounded_result(): + """Die Tabellenansicht hält sich an ein Zeilenlimit, der SQL-Tab holte per fetchall() alles: + „SELECT * FROM große_tabelle“ lud Millionen Zeilen in Speicher und Treeview — App friert ein.""" + fake, conn, status, calls = _build_fake_viewer("SELECT * FROM items") + limit = SQLiteViewer.SQL_RESULT_MAX_ROWS + conn.executemany("INSERT INTO items (name) VALUES (?)", [(f"n{i}",) for i in range(limit + 500)]) + conn.commit() + try: + SQLiteViewer.SqlViewer.execute_sql(fake) + finally: + conn.close() + columns, rows = calls["populate"] + assert len(rows) == limit + assert str(limit) in status.get("text", "") and "+" in status.get("text", ""), status From 1addde608a7296df83aa4ee8011cd6f08beb2a4e Mon Sep 17 00:00:00 2001 From: Lukas Geiger Date: Sat, 14 Nov 2026 22:57:48 +0100 Subject: [PATCH 2/3] fix(search): table search finds umlauts regardless of case MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit SQLite LIKE folds case for ASCII only: searching "übersicht" never found "Übersicht", "ärzte" not "Ärzte" (same defect as ProSync and DokuZen today). Terms with non-ASCII characters are now compared via a casefold() SQL function on both sides; pure ASCII terms keep the fast native LIKE because the search runs on every key press. Wildcards stay escaped and the export context keeps the term as typed. Tests red before the fix; real run on 200,000 rows: umlaut search 0.5-0.8 s with the hit found, ASCII search 0.18 s as before; pytest 175 passed. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01THB2EzUDHN8Rq5fvjMqkBL --- SQLiteViewer.py | 19 +++++++++++++++-- tests/test_bugsweep_20261114.py | 36 +++++++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+), 2 deletions(-) diff --git a/SQLiteViewer.py b/SQLiteViewer.py index cc0fb40..d67bcda 100644 --- a/SQLiteViewer.py +++ b/SQLiteViewer.py @@ -42,6 +42,11 @@ _CSV_IGNORABLE_PREFIX = "".join(chr(code) for code in range(0x21)) + "\ufeff" +def _casefold_sql(value: Any): + """SQL-Funktion casefold(): Text Unicode-gefaltet, andere Typen unverändert (LIKE wandelt sie selbst).""" + return value.casefold() if isinstance(value, str) else value + + def _csv_safe_value(value: Any): """Neutralisiert Stringwerte, die Tabellenkalkulationen als Formel deuten können.""" if not isinstance(value, str): @@ -1487,13 +1492,23 @@ def _search_data(self): if not cols: return - conditions = " OR ".join([f"{self._ident(col)} LIKE ? ESCAPE '\\'" for col in cols]) + # SQLite-LIKE ignoriert Groß/Klein nur bei ASCII: „übersicht“ fand „Übersicht“ nicht. Für + # Begriffe mit Nicht-ASCII beide Seiten per Python-casefold vergleichen; reine ASCII-Begriffe + # bleiben beim schnellen nativen LIKE (Suche läuft bei jedem Tastendruck). + if search_term.isascii(): + column_expr, like_term = "{}", search_term + else: + self.conn.create_function("casefold", 1, _casefold_sql, deterministic=True) + column_expr, like_term = "casefold({})", search_term.casefold() + conditions = " OR ".join( + [f"{column_expr.format(self._ident(col))} LIKE ? ESCAPE '\\'" for col in cols] + ) order_clause = "" if self.sort_column and self.sort_column in cols: direction = "DESC" if self.sort_reverse else "ASC" order_clause = f" ORDER BY {self._ident(self.sort_column)} {direction}" query = f"SELECT * FROM {self._ident(table)} WHERE {conditions}{order_clause} LIMIT ?" - escaped_term = self._escape_like_pattern(search_term) + escaped_term = self._escape_like_pattern(like_term) params = [f"%{escaped_term}%" for _ in cols] params.append(limit) diff --git a/tests/test_bugsweep_20261114.py b/tests/test_bugsweep_20261114.py index 640e003..a31f0e7 100644 --- a/tests/test_bugsweep_20261114.py +++ b/tests/test_bugsweep_20261114.py @@ -19,3 +19,39 @@ def test_sql_tab_does_not_load_an_unbounded_result(): columns, rows = calls["populate"] assert len(rows) == limit assert str(limit) in status.get("text", "") and "+" in status.get("text", ""), status + + +def test_table_search_ignores_case_of_umlauts(): + """SQLite-LIKE ignoriert Groß/Klein nur bei ASCII: „übersicht“ fand „Übersicht“ nicht, „ärzte“ nicht + „Ärzte“ — im Deutschen stehen Ä/Ö/Ü oft am Wortanfang (gleiches Muster wie ProSync/DokuZen).""" + from test_bugsweep_20260627 import _build_fake_for_search + + fake, conn, status, tree = _build_fake_for_search(["id", "name"], search_term="übersicht") + conn.executemany("INSERT INTO items (name) VALUES (?)", [("Übersicht 2026",), ("Ärzte",), ("Notiz",)]) + conn.commit() + try: + SQLiteViewer.SqlViewer._search_data(fake) + found = [row[1] for row in fake.current_data] + fake.search_var.set("ärzte") + SQLiteViewer.SqlViewer._search_data(fake) + found_aerzte = [row[1] for row in fake.current_data] + fake.search_var.set("50%") # Wildcards bleiben maskiert + SQLiteViewer.SqlViewer._search_data(fake) + found_percent = fake.current_data + finally: + conn.close() + assert "Suchfehler" not in status.get("text", ""), status + assert found == ["Übersicht 2026"] + assert found_aerzte == ["Ärzte"] + assert found_percent == [] + + +def test_search_export_context_keeps_the_typed_term(): + from test_bugsweep_20260627 import _build_fake_for_search + + fake, conn, status, tree = _build_fake_for_search(["id", "name"], search_term="Übersicht") + try: + SQLiteViewer.SqlViewer._search_data(fake) + finally: + conn.close() + assert fake.export_context["search_term"] == "Übersicht" From 6499a7f4bd4be26f7e649f4d49c1e039958e9c2e Mon Sep 17 00:00:00 2001 From: Lukas Geiger Date: Sat, 14 Nov 2026 23:03:17 +0100 Subject: [PATCH 3/3] perf(search): debounce the table search while typing The table search ran on every key release as a full table scan - on 200,000 rows 0.2-0.8 s of lag per key. Typing now schedules the search (250 ms, same after()/after_cancel() pattern as the SQL highlighting); Enter searches immediately and drops a pending run; closing the window cancels a pending search. Test red before the change; real Tk run with five quick key events: one search instead of five on master; pytest 176 passed. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01THB2EzUDHN8Rq5fvjMqkBL --- SQLiteViewer.py | 25 ++++++++++++++++++-- tests/test_bugsweep_20261114.py | 41 +++++++++++++++++++++++++++++++++ 2 files changed, 64 insertions(+), 2 deletions(-) diff --git a/SQLiteViewer.py b/SQLiteViewer.py index d67bcda..6b552d4 100644 --- a/SQLiteViewer.py +++ b/SQLiteViewer.py @@ -36,6 +36,7 @@ # Obergrenze für SQL-Tab-Ergebnisse: fetchall() lud bei „SELECT * FROM große_tabelle“ sonst Millionen # Zeilen in Speicher und Treeview und fror die App ein. SQL_RESULT_MAX_ROWS = 10_000 +SEARCH_DEBOUNCE_MS = 250 # Tabellensuche beim Tippen erst nach kurzer Pause starten MAX_CELL_DISPLAY_LEN = 500 APP_ICON_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), "SQLiteViewer.ico") _CSV_FORMULA_PREFIXES = ("=", "+", "-", "@") @@ -610,8 +611,8 @@ def _build_toolbar(self): self.search_var = tk.StringVar() self.search_entry = ttk.Entry(bar, textvariable=self.search_var, width=20) self.search_entry.pack(side=tk.LEFT, padx=4) - self.search_entry.bind("", lambda e: self._search_data()) - self.search_entry.bind("", lambda e: self._search_data()) + self.search_entry.bind("", lambda e: SqlViewer._search_now(self)) + self.search_entry.bind("", lambda e: SqlViewer._schedule_search(self, e)) self.search_entry.bind("", self._clear_search) add_tip(self.search_entry, "Volltextsuche in aktiver Tabelle (Eingabe / Escape zum Leeren)") @@ -1552,6 +1553,25 @@ def _search_data(self): except Exception as e: self._set_status(f"Suchfehler: {e}") + def _cancel_scheduled_search(self): + if getattr(self, "_search_job", None) is not None: + try: + self.after_cancel(self._search_job) + except Exception: + pass + self._search_job = None + + def _schedule_search(self, _event=None): + """Tippen: Suche entprellt einplanen statt bei jedem Tastendruck die ganze Tabelle zu scannen + (200 000 Zeilen: 0,2–0,8 s pro Taste).""" + SqlViewer._cancel_scheduled_search(self) + self._search_job = self.after(SEARCH_DEBOUNCE_MS, lambda: SqlViewer._search_now(self)) + + def _search_now(self): + """Enter bzw. abgelaufene Entprellzeit: geplante Suche verwerfen und sofort suchen.""" + SqlViewer._cancel_scheduled_search(self) + self._search_data() + def _focus_search(self): self.search_entry.focus_set() self.search_entry.select_range(0, tk.END) @@ -1681,6 +1701,7 @@ def _on_close(self): except Exception: pass self._sql_highlight_job = None + SqlViewer._cancel_scheduled_search(self) self.close_db() self.destroy() diff --git a/tests/test_bugsweep_20261114.py b/tests/test_bugsweep_20261114.py index a31f0e7..f6f1ce3 100644 --- a/tests/test_bugsweep_20261114.py +++ b/tests/test_bugsweep_20261114.py @@ -55,3 +55,44 @@ def test_search_export_context_keeps_the_typed_term(): finally: conn.close() assert fake.export_context["search_term"] == "Übersicht" + + +def test_search_typing_is_debounced(): + """Die Tabellensuche lief bei jedem Tastendruck () als voller Tabellenscan — bei + 200 000 Zeilen 0,2–0,8 s Verzögerung pro Taste. Tippen plant die Suche jetzt entprellt ein, + Enter sucht sofort, Schließen bricht eine geplante Suche ab.""" + from types import SimpleNamespace + + scheduled, cancelled, searched = [], [], [] + + class MockApp: + _search_job = None + _sql_highlight_job = None + + def after(self, ms, callback): + scheduled.append((ms, callback)) + return f"job_{len(scheduled)}" + + def after_cancel(self, job_id): + cancelled.append(job_id) + + def _search_data(self): + searched.append(True) + + def close_db(self): + pass + + def destroy(self): + pass + + app = MockApp() + key = SimpleNamespace(keysym="a") + SQLiteViewer.SqlViewer._schedule_search(app, key) + SQLiteViewer.SqlViewer._schedule_search(app, key) + assert searched == [] and cancelled == ["job_1"] and app._search_job == "job_2" + assert scheduled[-1][0] == SQLiteViewer.SEARCH_DEBOUNCE_MS + SQLiteViewer.SqlViewer._search_now(app) # Enter + assert searched == [True] and "job_2" in cancelled and app._search_job is None + SQLiteViewer.SqlViewer._schedule_search(app, key) + SQLiteViewer.SqlViewer._on_close(app) + assert "job_3" in cancelled