From ac4f98a23885a239bf7ea1c4536f21c9b8961983 Mon Sep 17 00:00:00 2001 From: Abdullah Farooqui Date: Wed, 1 Apr 2026 01:17:09 +0200 Subject: [PATCH 1/4] added exa_statistics --- .claude/settings.local.json | 8 ++ doc/exa_statistics/auditing.rst | 130 +++++++++++++++++++++++++ doc/exa_statistics/index.rst | 36 +++++++ doc/exa_statistics/overview.rst | 102 +++++++++++++++++++ doc/exa_statistics/query_analytics.rst | 124 +++++++++++++++++++++++ doc/exa_statistics/system_health.rst | 124 +++++++++++++++++++++++ doc/index.rst | 1 + 7 files changed, 525 insertions(+) create mode 100644 .claude/settings.local.json create mode 100644 doc/exa_statistics/auditing.rst create mode 100644 doc/exa_statistics/index.rst create mode 100644 doc/exa_statistics/overview.rst create mode 100644 doc/exa_statistics/query_analytics.rst create mode 100644 doc/exa_statistics/system_health.rst diff --git a/.claude/settings.local.json b/.claude/settings.local.json new file mode 100644 index 0000000..9498faa --- /dev/null +++ b/.claude/settings.local.json @@ -0,0 +1,8 @@ +{ + "permissions": { + "allow": [ + "Bash(claude plugin:*)", + "Bash(cp:*)" + ] + } +} diff --git a/doc/exa_statistics/auditing.rst b/doc/exa_statistics/auditing.rst new file mode 100644 index 0000000..772054d --- /dev/null +++ b/doc/exa_statistics/auditing.rst @@ -0,0 +1,130 @@ +====================== +Auditing & Compliance +====================== + +The ``EXA_DBA_AUDIT_*`` tables provide a full audit trail of SQL execution and user sessions. +They are designed for security investigations, compliance reporting, and operational support. + +.. note:: + All tables in this section require the ``SELECT ANY DICTIONARY`` system privilege. + See :ref:`exa-statistics-access-control` for details on granting this privilege. + +EXA_DBA_AUDIT_SQL +================== + +Records every SQL statement executed in the database. Key columns: + +.. list-table:: + :header-rows: 1 + :widths: 30 70 + + * - Column + - Description + * - ``USER_NAME`` + - Database user who executed the statement + * - ``SESSION_ID`` + - Session identifier + * - ``STMT_ID`` + - Statement identifier within the session + * - ``COMMAND_NAME`` + - SQL command type (e.g., ``SELECT``, ``CREATE TABLE``) + * - ``COMMAND_CLASS`` + - Broad category: ``DQL``, ``DML``, ``DDL``, ``DCL``, ``TCL`` + * - ``SQL_TEXT`` + - Full SQL text (up to 2,000,000 characters) + * - ``SUCCESS`` + - ``TRUE`` if the statement completed without error + * - ``ERROR_CODE`` / ``ERROR_TEXT`` + - Error details for failed statements + * - ``DURATION`` + - Execution time in seconds + * - ``STMT_START_TIME`` + - Timestamp when the statement began executing + +EXA_DBA_AUDIT_SESSIONS +======================== + +Records every database session, including login and logout events. Key columns: + +.. list-table:: + :header-rows: 1 + :widths: 30 70 + + * - Column + - Description + * - ``SESSION_ID`` + - Unique session identifier + * - ``USER_NAME`` + - Database user who opened the session + * - ``OS_USER`` + - Operating system user on the client machine + * - ``HOST`` + - Client host address + * - ``LOGIN_TIME`` + - Session start timestamp + * - ``LOGOUT_TIME`` + - Session end timestamp (``NULL`` if session is still active) + * - ``SUCCESS`` + - ``TRUE`` for successful logins; ``FALSE`` for failed login attempts + * - ``ENCRYPTED`` + - ``TRUE`` if the connection was encrypted + +Managing Audit Log Size +======================== + +Audit tables grow continuously. Remove old records while retaining recent history with +``TRUNCATE AUDIT LOGS``: + +.. code-block:: sql + + -- Keep the last 30 days; remove everything older + TRUNCATE AUDIT LOGS KEEP FROM DAYS=30; + +.. warning:: + ``TRUNCATE AUDIT LOGS`` permanently deletes the removed records. This action cannot be undone. + +Recipes +======= + +Find All Failed Statements with Error Details +---------------------------------------------- + +.. code-block:: sql + + SELECT USER_NAME, COMMAND_NAME, SQL_TEXT, + ERROR_CODE, ERROR_TEXT, STMT_START_TIME + FROM EXA_DBA_AUDIT_SQL + WHERE SUCCESS = FALSE + ORDER BY STMT_START_TIME DESC + LIMIT 50; + +Track Login History for a Specific User +----------------------------------------- + +.. code-block:: sql + + SELECT SESSION_ID, LOGIN_TIME, LOGOUT_TIME, + HOST, OS_USER, ENCRYPTED + FROM EXA_DBA_AUDIT_SESSIONS + WHERE USER_NAME = 'MY_USER' + ORDER BY LOGIN_TIME DESC; + +List All DDL Statements Executed Today +---------------------------------------- + +.. code-block:: sql + + SELECT USER_NAME, COMMAND_NAME, SQL_TEXT, STMT_START_TIME + FROM EXA_DBA_AUDIT_SQL + WHERE COMMAND_CLASS = 'DDL' + AND CAST(STMT_START_TIME AS DATE) = CURRENT_DATE + ORDER BY STMT_START_TIME DESC; + +Remove Audit Logs Older Than 30 Days +-------------------------------------- + +.. code-block:: sql + + TRUNCATE AUDIT LOGS KEEP FROM DAYS=30; + +**Further reading:** `EXA_DBA_AUDIT_SQL `_ · `EXA_DBA_AUDIT_SESSIONS `_ · `Auditing Concepts `_ diff --git a/doc/exa_statistics/index.rst b/doc/exa_statistics/index.rst new file mode 100644 index 0000000..e6fb146 --- /dev/null +++ b/doc/exa_statistics/index.rst @@ -0,0 +1,36 @@ +========================================== +EXA_STATISTICS: Database Monitoring & Audit +========================================== + +``EXA_STATISTICS`` is a built-in system schema present in every Exasol database. It automatically +collects and stores historical data about query activity, resource usage, database size, and user +sessions. Use it to monitor performance, plan capacity, and maintain a full audit trail — without +installing any additional tools. + +.. list-table:: Table Groups at a Glance + :header-rows: 1 + :widths: 25 50 25 + + * - Group + - Purpose + - Access + * - SQL Activity (``EXA_SQL_*``) + - Track executed statements, execution modes, duration, and CPU usage + - All users + * - System Monitor (``EXA_MONITOR_*``) + - Monitor CPU, memory, swap, I/O, and network metrics + - All users + * - Database Size (``EXA_DB_SIZE_*``) + - Track storage volume, compression ratios, and recommended RAM + - All users + * - Audit (``EXA_DBA_AUDIT_*``) + - Full SQL and session audit trail for compliance and security + - ``SELECT ANY DICTIONARY`` required + +.. toctree:: + :maxdepth: 2 + + overview + query_analytics + system_health + auditing diff --git a/doc/exa_statistics/overview.rst b/doc/exa_statistics/overview.rst new file mode 100644 index 0000000..f1c17c7 --- /dev/null +++ b/doc/exa_statistics/overview.rst @@ -0,0 +1,102 @@ +======== +Overview +======== + +What is EXA_STATISTICS +======================= + +``EXA_STATISTICS`` is a system schema built into every Exasol database. It continuously collects +statistics about SQL execution, hardware resource usage, storage, and user sessions. Data is updated +automatically every minute and is immediately available for querying — no configuration required. + +The schema covers four functional areas: + +* **SQL Activity** — historical record of every executed statement +* **System Monitor** — CPU, memory, I/O, and network metrics over time +* **Database Size** — storage volume, compression ratios, and RAM recommendations +* **Audit** — full audit trail of SQL statements and user sessions + +Namespace Integration +===================== + +``EXA_STATISTICS`` and ``SYS`` are automatically integrated into Exasol's namespace. You can query +statistical tables directly without specifying the schema name: + +.. code-block:: sql + + -- Both are equivalent: + SELECT * FROM EXA_SQL_LAST_DAY; + SELECT * FROM EXA_STATISTICS.EXA_SQL_LAST_DAY; + +.. _exa-statistics-access-control: + +Access Control +============== + +Most statistical tables are readable by all database users. Tables with ``DBA`` in their name +contain sensitive data and require the ``SELECT ANY DICTIONARY`` system privilege. + +.. list-table:: + :header-rows: 1 + :widths: 45 55 + + * - Table Group + - Required Privilege + * - ``EXA_SQL_*``, ``EXA_MONITOR_*``, ``EXA_DB_SIZE_*`` + - None — any connected user + * - ``EXA_DBA_AUDIT_*`` + - ``SELECT ANY DICTIONARY`` + +To grant the privilege to a user: + +.. code-block:: sql + + GRANT SELECT ANY DICTIONARY TO my_user; + +Aggregation Levels +================== + +Each metric category is available at multiple time granularities: + +.. list-table:: + :header-rows: 1 + :widths: 20 80 + + * - Suffix + - Description + * - ``_LAST_DAY`` + - Rolling 24-hour window — one row per measurement interval + * - ``_HOURLY`` + - Aggregated per hour + * - ``_DAILY`` + - Aggregated per calendar day + * - ``_MONTHLY`` + - Aggregated per calendar month + +Use ``_LAST_DAY`` tables for real-time investigation and the aggregated tables for trend analysis +and capacity planning. + +Time Zone +========= + +All timestamps in ``EXA_STATISTICS`` are stored in the database's configured time zone +(``DBTIMEZONE``). To check your current setting: + +.. code-block:: sql + + SELECT DBTIMEZONE; + +Refreshing Statistics +===================== + +Statistics are updated automatically every minute. To force an immediate refresh: + +.. code-block:: sql + + FLUSH STATISTICS; + COMMIT; + +.. note:: + Open a new transaction after flushing to see the latest data reflected in your queries. + +**Further reading:** `Statistical System Tables `_ · `FLUSH STATISTICS `_ diff --git a/doc/exa_statistics/query_analytics.rst b/doc/exa_statistics/query_analytics.rst new file mode 100644 index 0000000..87e25b6 --- /dev/null +++ b/doc/exa_statistics/query_analytics.rst @@ -0,0 +1,124 @@ +=========================== +Query Performance Analytics +=========================== + +The ``EXA_SQL_*`` tables record every SQL statement that Exasol successfully compiles and executes. +Use them to identify slow queries, measure CPU consumption, and understand workload patterns over time. + +EXA_SQL_LAST_DAY +================ + +Contains one row per executed statement within the rolling 24-hour window. Key columns: + +.. list-table:: + :header-rows: 1 + :widths: 30 70 + + * - Column + - Description + * - ``SESSION_ID`` + - Identifier of the session that ran the statement + * - ``STMT_ID`` + - Statement identifier within the session + * - ``COMMAND_NAME`` + - SQL command type (e.g., ``SELECT``, ``INSERT``, ``COMMIT``) + * - ``EXECUTION_MODE`` + - How the statement was handled — see Execution Modes below + * - ``DURATION`` + - Execution time in seconds + * - ``CPU`` + - CPU usage percentage during execution + * - ``SUCCESS`` + - ``TRUE`` if the statement completed without error + * - ``START_TIME`` / ``STOP_TIME`` + - Execution window timestamps + * - ``TEMP_DB_RAM_PEAK`` + - Peak temporary memory used (bytes) + +.. note:: + ``EXA_SQL_LAST_DAY`` does not record the SQL text or the executing user name. For the full audit + trail including SQL text, see ``EXA_DBA_AUDIT_SQL`` in :doc:`auditing`. + +EXA_SQL_DAILY and EXA_SQL_HOURLY +================================= + +Aggregated views of SQL activity. Each row represents a command type (``SELECT``, ``MERGE``, etc.) +within a time period. Key aggregated columns include ``COUNT``, ``DURATION_AVG``, ``DURATION_MAX``, +``CPU_AVG``, and ``CPU_MAX``. + +Use these tables to spot workload trends without scanning per-statement detail. + +Execution Modes +=============== + +The ``EXECUTION_MODE`` column classifies how each statement was handled: + +.. list-table:: + :header-rows: 1 + :widths: 20 80 + + * - Mode + - Meaning + * - ``EXECUTE`` + - Statement was fully compiled and executed + * - ``PREPARE`` + - Statement was prepared but not yet executed + * - ``CACHED`` + - Result was served from the query cache — no recomputation + * - ``CRASHED`` + - Statement failed during execution + +Recipes +======= + +Find the Slowest Queries (Last 24 Hours) +----------------------------------------- + +.. code-block:: sql + + SELECT SESSION_ID, STMT_ID, COMMAND_NAME, DURATION, SUCCESS + FROM EXA_SQL_LAST_DAY + ORDER BY DURATION DESC + LIMIT 20; + +Identify CPU-Intensive Statements +----------------------------------- + +.. code-block:: sql + + SELECT SESSION_ID, STMT_ID, COMMAND_NAME, CPU, DURATION + FROM EXA_SQL_LAST_DAY + WHERE CPU > 50 + ORDER BY CPU DESC; + +Check Cache Hit Rate +--------------------- + +A high proportion of ``CACHED`` executions indicates effective use of Exasol's query cache. + +.. code-block:: sql + + WITH totals AS ( + SELECT EXECUTION_MODE, COUNT(*) AS statement_count + FROM EXA_SQL_LAST_DAY + GROUP BY EXECUTION_MODE + ) + SELECT + EXECUTION_MODE, + statement_count, + ROUND(statement_count * 100.0 / SUM(statement_count) OVER (), 1) AS pct + FROM totals + ORDER BY statement_count DESC; + +Daily Workload Summary by Command Type +---------------------------------------- + +.. code-block:: sql + + SELECT COMMAND_NAME, COUNT AS statement_count, + DURATION_AVG, DURATION_MAX + FROM EXA_SQL_DAILY + WHERE SUCCESS = TRUE + ORDER BY DURATION_MAX DESC; + +**Further reading:** `EXA_SQL_LAST_DAY `_ · `EXA_SQL_DAILY `_ diff --git a/doc/exa_statistics/system_health.rst b/doc/exa_statistics/system_health.rst new file mode 100644 index 0000000..79534e8 --- /dev/null +++ b/doc/exa_statistics/system_health.rst @@ -0,0 +1,124 @@ +================================= +System Health & Capacity Planning +================================= + +The ``EXA_MONITOR_*`` and ``EXA_DB_SIZE_*`` tables provide a continuous record of hardware resource +usage and storage consumption. Use them to detect performance bottlenecks, identify memory pressure, +and plan infrastructure growth. + +EXA_MONITOR_LAST_DAY +===================== + +Contains one row per measurement interval (approximately every minute) for the past 24 hours. +Key columns: + +.. list-table:: + :header-rows: 1 + :widths: 30 70 + + * - Column + - Description + * - ``MEASURE_TIME`` + - Timestamp of the measurement + * - ``CPU`` + - Average CPU usage across all cluster nodes (%) + * - ``LOAD`` + - System load average + * - ``TEMP_DB_RAM`` + - Temporary DB RAM in use (MiB) + * - ``PERSISTENT_DB_RAM`` + - Persistent DB RAM in use (MiB) + * - ``SWAP`` + - Swap space in use — values above 0 indicate memory pressure (MiB) + * - ``HDD_READ`` / ``HDD_WRITE`` + - Disk read/write throughput (MiB/s) + * - ``NET`` + - Network traffic (MiB/s) + +EXA_MONITOR_DAILY +================== + +Aggregates monitor data per calendar day. Each metric has both an ``_AVG`` and ``_MAX`` variant +(e.g., ``CPU_AVG``, ``CPU_MAX``). Use this table to identify days with unusual load or to compare +weekly performance trends. + +EXA_DB_SIZE_LAST_DAY +===================== + +Tracks storage volume metrics at the cluster level, measured approximately every hour. Key columns: + +.. list-table:: + :header-rows: 1 + :widths: 40 60 + + * - Column + - Description + * - ``SIZE_RAW_COMPRESSED`` + - Compressed data volume (MiB) + * - ``SIZE_UNCOMPRESSED`` + - Uncompressed (raw) data volume (MiB) + * - ``RECOMMENDED_DB_RAM_SIZE`` + - RAM size Exasol recommends for optimal query performance (MiB) + * - ``SIZE_PERSISTENT_VOLUME`` + - Total persistent storage allocated (MiB) + * - ``SIZE_PERSISTENT_VOLUME_PERCENTAGE`` + - Percentage of persistent volume currently used + +EXA_DB_SIZE_DAILY +================== + +Daily aggregation of database size metrics. Use this table to track storage growth over weeks and +months and to feed capacity planning reports. + +Recipes +======= + +Snapshot Current System Resource Usage +---------------------------------------- + +.. code-block:: sql + + SELECT MEASURE_TIME, CPU, LOAD, TEMP_DB_RAM, PERSISTENT_DB_RAM, SWAP + FROM EXA_MONITOR_LAST_DAY + ORDER BY MEASURE_TIME DESC + LIMIT 10; + +Detect Memory Pressure +----------------------- + +Swap usage above zero suggests the database is under memory pressure. Sustained swap values +warrant a RAM review or workload optimisation. + +.. code-block:: sql + + SELECT MEASURE_TIME, SWAP, CPU, TEMP_DB_RAM + FROM EXA_MONITOR_LAST_DAY + WHERE SWAP > 0 + ORDER BY MEASURE_TIME DESC; + +Check Storage Utilization and Recommended RAM +---------------------------------------------- + +.. code-block:: sql + + SELECT MEASURE_TIME, + SIZE_RAW_COMPRESSED, + SIZE_PERSISTENT_VOLUME, + SIZE_PERSISTENT_VOLUME_PERCENTAGE, + RECOMMENDED_DB_RAM_SIZE + FROM EXA_DB_SIZE_LAST_DAY + ORDER BY MEASURE_TIME DESC + LIMIT 1; + +Track Storage Growth Over the Last Two Weeks +--------------------------------------------- + +.. code-block:: sql + + SELECT MEASURE_TIME, + SIZE_RAW_COMPRESSED + FROM EXA_DB_SIZE_DAILY + ORDER BY MEASURE_TIME DESC + LIMIT 14; + +**Further reading:** `EXA_MONITOR_LAST_DAY `_ · `EXA_MONITOR_DAILY `_ · `EXA_DB_SIZE_LAST_DAY `_ diff --git a/doc/index.rst b/doc/index.rst index 873fe9b..6503b97 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -11,6 +11,7 @@ Documentation and resources for data scientists and programmatic users to perfor data_ingestion UDF/index.rst data_science/index.rst + exa_statistics/index gen_ai/index.rst connect_to_exasol/index.rst examples/index.rst From 43b02058dd548b2b549ad61b5616755e1c334631 Mon Sep 17 00:00:00 2001 From: Abdullah Farooqui Date: Wed, 1 Apr 2026 01:24:48 +0200 Subject: [PATCH 2/4] Fix RST title overline length in exa_statistics/index.rst --- doc/exa_statistics/index.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/exa_statistics/index.rst b/doc/exa_statistics/index.rst index e6fb146..e267d81 100644 --- a/doc/exa_statistics/index.rst +++ b/doc/exa_statistics/index.rst @@ -1,6 +1,6 @@ -========================================== +=========================================== EXA_STATISTICS: Database Monitoring & Audit -========================================== +=========================================== ``EXA_STATISTICS`` is a built-in system schema present in every Exasol database. It automatically collects and stores historical data about query activity, resource usage, database size, and user From 9fcad9265ac2d47b760676682486b17b44774dd2 Mon Sep 17 00:00:00 2001 From: Abdullah Farooqui Date: Wed, 1 Apr 2026 02:21:20 +0200 Subject: [PATCH 3/4] fixed short overline --- .claude/settings.local.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.claude/settings.local.json b/.claude/settings.local.json index 9498faa..a124f7e 100644 --- a/.claude/settings.local.json +++ b/.claude/settings.local.json @@ -2,7 +2,8 @@ "permissions": { "allow": [ "Bash(claude plugin:*)", - "Bash(cp:*)" + "Bash(cp:*)", + "Bash(poetry run:*)" ] } } From 9a4d270f064d96cc3ce7fd99a236f6ff5f94f8d4 Mon Sep 17 00:00:00 2001 From: Abdullah Farooqui Date: Wed, 1 Apr 2026 02:28:42 +0200 Subject: [PATCH 4/4] fixed redirected links --- doc/getting_started.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/getting_started.rst b/doc/getting_started.rst index 1ab210b..66b8167 100644 --- a/doc/getting_started.rst +++ b/doc/getting_started.rst @@ -15,7 +15,7 @@ Setting up the db ----------------- Our `main -documentation `__ +documentation `__ shows you how to create a SaaS account and setup your first database and cluster. @@ -43,7 +43,7 @@ Connecting to an Exasol database is performed via: It is slightly different when connecting to a SaaS database as we need a `personal access -token `__: +token `__: .. code:: python