From b5a4abb3e71126fce9b1307e40f02e8badc22dcf Mon Sep 17 00:00:00 2001 From: anumathad-o Date: Mon, 21 Sep 2026 19:22:29 +0200 Subject: [PATCH] Fixes for v1.0.4 --- CHANGELOG.rst | 14 + SECURITY.md | 2 +- docs/Makefile | 20 - docs/make.bat | 35 - docs/source/api.rst | 110 -- docs/source/conf.py | 33 - docs/source/index.rst | 44 - docs/source/installation.rst | 15 - docs/source/quickstart.rst | 109 -- docs/source/rest_api.rst | 2328 ------------------------ src/oracle_vecdb/vecdb_exception.py | 166 +- src/oracle_vecdb/version.py | 2 +- tests/services/test_ords_exceptions.py | 206 +++ 13 files changed, 373 insertions(+), 2711 deletions(-) delete mode 100644 docs/Makefile delete mode 100644 docs/make.bat delete mode 100644 docs/source/api.rst delete mode 100644 docs/source/conf.py delete mode 100644 docs/source/index.rst delete mode 100644 docs/source/installation.rst delete mode 100644 docs/source/quickstart.rst delete mode 100755 docs/source/rest_api.rst diff --git a/CHANGELOG.rst b/CHANGELOG.rst index bc29560..7288dc2 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -6,6 +6,20 @@ All notable changes to this project will be documented in this file. The format is based on the `Keep a Changelog `__, and this project adheres to `Semantic Versioning `__. +1.0.4 - 2026-09-21 +------------------ + +Added +~~~~~ + +- Exception diagnostic redaction of sensitive values in nested arguments. + +Changed +~~~~~~~ + +- Removed the obsolete Sphinx documentation tree; the repository README and + published reference sources remain the documentation entry points. + 1.0.3 - 2026-09-07 ------------------ diff --git a/SECURITY.md b/SECURITY.md index 2ca8102..fb42c94 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -35,4 +35,4 @@ sufficiently hardened for production use. [1]: mailto:secalert_us@oracle.com [2]: https://www.oracle.com/corporate/security-practices/assurance/vulnerability/reporting.html [3]: https://www.oracle.com/security-alerts/encryptionkey.html -[4]: https://www.oracle.com/security-alerts/ +[4]: https://www.oracle.com/security-alerts/ \ No newline at end of file diff --git a/docs/Makefile b/docs/Makefile deleted file mode 100644 index d0c3cbf..0000000 --- a/docs/Makefile +++ /dev/null @@ -1,20 +0,0 @@ -# Minimal makefile for Sphinx documentation -# - -# You can set these variables from the command line, and also -# from the environment for the first two. -SPHINXOPTS ?= -SPHINXBUILD ?= sphinx-build -SOURCEDIR = source -BUILDDIR = build - -# Put it first so that "make" without argument is like "make help". -help: - @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) - -.PHONY: help Makefile - -# Catch-all target: route all unknown targets to Sphinx using the new -# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). -%: Makefile - @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/docs/make.bat b/docs/make.bat deleted file mode 100644 index 747ffb7..0000000 --- a/docs/make.bat +++ /dev/null @@ -1,35 +0,0 @@ -@ECHO OFF - -pushd %~dp0 - -REM Command file for Sphinx documentation - -if "%SPHINXBUILD%" == "" ( - set SPHINXBUILD=sphinx-build -) -set SOURCEDIR=source -set BUILDDIR=build - -%SPHINXBUILD% >NUL 2>NUL -if errorlevel 9009 ( - echo. - echo.The 'sphinx-build' command was not found. Make sure you have Sphinx - echo.installed, then set the SPHINXBUILD environment variable to point - echo.to the full path of the 'sphinx-build' executable. Alternatively you - echo.may add the Sphinx directory to PATH. - echo. - echo.If you don't have Sphinx installed, grab it from - echo.https://www.sphinx-doc.org/ - exit /b 1 -) - -if "%1" == "" goto help - -%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% -goto end - -:help -%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% - -:end -popd diff --git a/docs/source/api.rst b/docs/source/api.rst deleted file mode 100644 index 25492c2..0000000 --- a/docs/source/api.rst +++ /dev/null @@ -1,110 +0,0 @@ -Python API Reference -==================== - -.. currentmodule:: oracle_vecdb.client - -OracleVecDB Client ------------------- - -.. autoclass:: OracleVecDB - :no-members: - -Public Response Models ----------------------- - -Oracle VecDB facade methods return typed SDK response models. For new code, -prefer importing them from ``oracle_vecdb.data_types``. - -- ``QueryResponse`` exposes search hits under ``items``. -- ``RerankResponse`` exposes rerank scores under ``items``. -- Use response attributes directly when application code needs a few fields. -- Use ``to_json()`` only when printing or writing a complete response, and - ``to_dict()`` only when a complete Python dictionary is needed. Both create - additional data proportional to the response size. - -Large response guidance -~~~~~~~~~~~~~~~~~~~~~~~ - -Avoid serializing large responses unless they cross an application boundary. -Use pagination for collection methods, a bounded ``top_k`` for ``query``, -``output_selector`` for the metadata fields you need, and leave -``include_vectors=False`` unless vector values are required. - -Models ------- - -- :meth:`OracleVecDB.list_models` -- :meth:`OracleVecDB.load_model` -- :meth:`OracleVecDB.describe_model` -- :meth:`OracleVecDB.drop_model` - -.. automethod:: OracleVecDB.list_models -.. automethod:: OracleVecDB.load_model -.. automethod:: OracleVecDB.describe_model -.. automethod:: OracleVecDB.drop_model - -Vector Tables -------------- - -- :meth:`OracleVecDB.describe_vector_database` -- :meth:`OracleVecDB.list_vector_tables` -- :meth:`OracleVecDB.create_vector_table` -- :meth:`OracleVecDB.describe_vector_table` -- :meth:`OracleVecDB.drop_vector_table` -- :meth:`OracleVecDB.update_vector_table_annotation` - -.. automethod:: OracleVecDB.describe_vector_database -.. automethod:: OracleVecDB.list_vector_tables -.. automethod:: OracleVecDB.create_vector_table -.. automethod:: OracleVecDB.describe_vector_table -.. automethod:: OracleVecDB.drop_vector_table -.. automethod:: OracleVecDB.update_vector_table_annotation - -Manage Data ------------ - -- :meth:`OracleVecDB.generate_embedding` -- :meth:`OracleVecDB.upsert_vectors` -- :meth:`OracleVecDB.list_vectors` -- :meth:`OracleVecDB.delete_vectors` -- :meth:`OracleVecDB.load_vectors` -- :meth:`OracleVecDB.list_vector_load_jobs` -- :meth:`OracleVecDB.describe_vector_load_job` -- :meth:`OracleVecDB.get_vector_load_job_log` - -.. automethod:: OracleVecDB.generate_embedding -.. automethod:: OracleVecDB.upsert_vectors -.. automethod:: OracleVecDB.list_vectors -.. automethod:: OracleVecDB.delete_vectors -.. automethod:: OracleVecDB.load_vectors -.. automethod:: OracleVecDB.list_vector_load_jobs -.. automethod:: OracleVecDB.describe_vector_load_job -.. automethod:: OracleVecDB.get_vector_load_job_log - -Search Data ------------ - -- :meth:`OracleVecDB.query` -- :meth:`OracleVecDB.rerank` - -.. automethod:: OracleVecDB.query -.. automethod:: OracleVecDB.rerank - -Indexes -------- - -- :meth:`OracleVecDB.create_index` -- :meth:`OracleVecDB.list_index_jobs` -- :meth:`OracleVecDB.describe_index_job` -- :meth:`OracleVecDB.get_index_job_log` -- :meth:`OracleVecDB.rebuild_index` -- :meth:`OracleVecDB.describe_index` -- :meth:`OracleVecDB.drop_index` - -.. automethod:: OracleVecDB.create_index -.. automethod:: OracleVecDB.list_index_jobs -.. automethod:: OracleVecDB.describe_index_job -.. automethod:: OracleVecDB.get_index_job_log -.. automethod:: OracleVecDB.rebuild_index -.. automethod:: OracleVecDB.describe_index -.. automethod:: OracleVecDB.drop_index diff --git a/docs/source/conf.py b/docs/source/conf.py deleted file mode 100644 index 8b0ee35..0000000 --- a/docs/source/conf.py +++ /dev/null @@ -1,33 +0,0 @@ -import os -import sys -# Point to the src directory -sys.path.insert(0, os.path.abspath('../../src')) - -project = 'oracle-vecdb' -copyright = '2026, Oracle' -author = 'Tanmay Bagaria' - -extensions = [ - 'sphinx.ext.autodoc', - 'sphinx.ext.napoleon', - 'sphinx.ext.viewcode', - 'sphinx_autodoc_typehints', -] - -templates_path = ['_templates'] -exclude_patterns = [] - -html_theme = 'sphinx_rtd_theme' -html_static_path = ['_static'] - -autodoc_default_options = { - 'members': True, - 'member-order': 'bysource', - 'special-members': '__init__', - 'undoc-members': False, - 'exclude-members': '__weakref__' -} - -napoleon_google_docstring = True -napoleon_numpy_docstring = True -napoleon_include_init_with_doc = True \ No newline at end of file diff --git a/docs/source/index.rst b/docs/source/index.rst deleted file mode 100644 index b61e540..0000000 --- a/docs/source/index.rst +++ /dev/null @@ -1,44 +0,0 @@ -Oracle VecDB Python SDK Docs -============================ - -Welcome to the home of the Oracle VecDB Python SDK documentation. Use this site to install the SDK, walk through quickstarts, and discover the full API surface for table/index management, vector search, models, and inference operations in Oracle AI Database (26ai+). - -.. toctree:: - :maxdepth: 2 - :caption: Contents - - installation - quickstart - api - rest_api - -Highlights -========== - -- πŸ”‘ Simple configuration helpers to connect to Oracle VecDB endpoints -- πŸ“¦ Programmatic control over vector tables, indexes, metadata, and annotations -- 🧠 Embedding, inference, and search workflows powered by Oracle AI Database models -- πŸ” Support for upsert, delete, filter, and similarity/rerank queries - -Getting started -=============== - -If you're new to the SDK, follow these steps: - -1. :doc:`installation` β€” set up prerequisites and install the package. -2. :doc:`quickstart` β€” configure a client, create a table, load vectors, and run a query. -3. :doc:`api` β€” explore every endpoint, request/response schema, and usage pattern. - -Looking for examples? ---------------------- - -- Browse the `sample applications `_ for full-stack reference implementations. -- Use the `sample notebooks `_ for interactive tutorials and experiments. - -Feedback & contributions -======================== - -- Issues and feature requests: use the SDK repository's issue tracker. -- Contributions are welcomeβ€”see :doc:`installation` for setup details and read `CONTRIBUTING <../CONTRIBUTING>`_. - -Happy building! πŸ˜„ \ No newline at end of file diff --git a/docs/source/installation.rst b/docs/source/installation.rst deleted file mode 100644 index 52f1a25..0000000 --- a/docs/source/installation.rst +++ /dev/null @@ -1,15 +0,0 @@ -Installation -============ - -Using pip ---------- - -.. code-block:: bash - - pip install oracle-vecdb - - -Requirements ------------- - -- Python 3.10+ \ No newline at end of file diff --git a/docs/source/quickstart.rst b/docs/source/quickstart.rst deleted file mode 100644 index 177024d..0000000 --- a/docs/source/quickstart.rst +++ /dev/null @@ -1,109 +0,0 @@ -Quick Start Guide -================= - -This quickstart walks through installing the SDK, configuring a client, creating a table, loading sample vectors, and running a similarity query. Use it to sanity-check your Oracle VecDB (26ai+) environment before building more advanced apps. - -Requirements ------------- - -- Python 3.10+ -- Access to an Oracle VecDB endpoint (hosted through Oracle AI Database 26ai+) -- Authentication (either bearer token or username/password) - -Installation ------------- - -.. code-block:: bash - - pip install oracle-vecdb - -.. note:: - Hosts typically look like ``https://:/ords//_/db-api/stable/vecdb/``. Ensure TLS (HTTPS) is enabled and the URL is reachable from your environment. - -1. Configure the client ------------------------ - -.. code-block:: python - - from oracle_vecdb import OracleVecDB, Configuration - - config = Configuration( - rest_url="https://:/ords//_/db-api/stable/vecdb/", - # choose one auth method - access_token="", - # or username="", password="", - ) - - vecdb = OracleVecDB(config) - -2. Create a table ------------------ - -.. code-block:: python - - vecdb.create_vector_table(name="demo") - -3. Load example vectors ------------------------ - -.. code-block:: python - - vecdb.upsert_vectors( - table_name="demo", - vectors=[ - {"id": "1", "dense_vector": [0.1, 0.1], "metadata": {"genre": "comedy"}}, - {"id": "2", "dense_vector": [0.2, 0.2], "metadata": {"genre": "drama"}}, - ], - ) - -Large inline datasets are automatically split into requests below the service -limit:: - - large_vector_list = [...] # the complete inline dataset - response = vecdb.upsert_vectors( - table_name="demo", - vectors=large_vector_list, - ) - print(response.upserted_count) - -The SDK preserves vector order and aggregates the batch counts. Batching does -not deduplicate IDs or guarantee that service rate limits will not be reached; -duplicate IDs and transient HTTP 429 responses remain service conditions. A -single vector larger than 32 MiB is rejected before any request is sent. - -4. Run a similarity query -------------------------- - -.. code-block:: python - - results = vecdb.query( - table_name="demo", - query_by={"vector": [0.15, 0.1]}, - top_k=1, - ) - - for index in range(len(results)): - item = results[index] - row = item if isinstance(item, dict) else item.model_dump() - print(row["id"], row["distance"], row["metadata"]) - -Sample output:: - - 2 0.08 {'genre': 'drama'} - -.. note:: - Most SDK methods return typed response models rather than raw JSON strings. - Access response attributes directly when you need a few values. Use - ``response.model_dump()`` or ``response.to_json()`` only when printing or writing a complete response, - and ``response.to_dict()`` only when a complete Python dictionary is - required; both create additional data proportional to the response size. - Keep ``top_k`` small, leave ``include_vectors=False`` unless needed, and use - ``output_selector`` to reduce large query responses. Query result rows - support list-style indexing. - -Next steps ----------- - -- Explore the :doc:`installation` guide for environment-specific setup. -- Follow the :doc:`api` reference to discover table/index operations, search options, and model endpoints. -- Try the `sample applications `_ or `sample notebooks `_ to see full-stack and notebook workflows. diff --git a/docs/source/rest_api.rst b/docs/source/rest_api.rst deleted file mode 100755 index 649ac77..0000000 --- a/docs/source/rest_api.rst +++ /dev/null @@ -1,2328 +0,0 @@ -REST API Reference -=================== - -.. contents:: - :local: - :depth: 1 - -Authentication & Base URL --------------------------- - -- Host template: ``https://:/ords//_/db-api/stable/vecdb/`` -- Headers: ``Authorization: Bearer `` (Bearer or Basic), ``Content-Type: application/json`` -- Most POST/PATCH operations accept optional ``debugFlags`` objects that raise VecDB tracing levels; when omitted, standard logging is used. - -.. note:: - - **debugFlags** is accepted by POST/PATCH endpoints. Each key accepts ``"low"``, ``"medium"``, or ``"high"``. - Available keys: ``VECTOR_INDEX``, ``VECTOR_INDEX_NEIGHBOR_GRAPH``, ``VECTOR_INDEX_NEIGHBOR_GRAPH_BUILD``, - ``VECTOR_INDEX_NEIGHBOR_GRAPH_MEM``, ``VECTOR_INDEX_NEIGHBOR_GRAPH_SEARCH``, - ``VECTOR_INDEX_NEIGHBOR_GRAPH_APPCHNG``, ``VECTOR_INDEX_NEIGHBOR_GRAPH_STATS``, - ``VECTOR_INDEX_NEIGHBOR_PARTITIONS``, ``VECTOR_INDEX_FIXED_VIEW``, ``VECIDX_TRANS``, - ``VECIDX_TRANS_COM``, ``VECIDX_TRANS_PJ``, ``VECIDX_TRANS_PJ_DWNGRD``, ``VECIDX_TRANS_PJ_GROW``, - ``VECIDX_TRANS_SJ``, ``VECIDX_TRANS_SJ_BG``, ``VEC_INDEX_CALIBRATION``, ``VECTOR_TRACE``. - -Distance Metric Guidance -------------------------- - -Use the following guidance anywhere a request accepts ``distance_metric``. - -.. list-table:: - :header-rows: 1 - :widths: 20 40 40 - - * - Metric - - Summary - - When to use - * - ``COSINE`` - - Compares vector direction and ignores magnitude. - - Best default for semantic search, text embeddings, and normalized vectors. - * - ``DOT`` - - Uses dot-product similarity, so both direction and magnitude can affect ranking. - - Use when the model was trained with dot-product scoring or magnitude carries meaning. - * - ``EUCLIDEAN`` - - Measures straight-line distance between vectors. - - Use for geometric or spatial comparisons where actual distance matters. - * - ``EUCLIDEAN_SQUARED / L2_SQUARED`` - - Euclidean distance without the square root. - - Faster distance calculations where exact distance magnitude is unimportant. - * - ``MANHATTAN`` - - Sums absolute differences across vector dimensions. - - Use for sparse vectors or when you want less sensitivity to large single-dimension differences. - * - ``HAMMING`` - - Counts how many positions differ between two vectors. - - Use for binary embeddings, hashes, bit vectors, or yes/no feature encodings. - * - ``JACCARD`` - - Compares shared features against total features present across both vectors. - - Use for set-style similarity, binary tags, and sparse binary feature data. - -Models -------- - -**GET /vecdb/models/** - -List all loaded embedding and reranking models. - -Returns information about all models available for use in the database, -limited to models currently loaded in the service. - -**Returns** - -JSON response containing an array of models with model names, types (embedding, reranking), algorithms, creation timestamps, and attributes and parameters. - -.. code-block:: json - :caption: Example 200 response - - { - "items": [ - { - "model_name": "all-MiniLM-L6-v2", - "algorithm": "ONNX", - "mining_function": "EMBEDDING", - "creation_date": "2026-01-14T09:32:11Z", - "attributes": [ - { - "name": "NUM_DIMENSIONS", - "value": "384", - "data_type": "NUMBER", - "data_length": 22, - "vector_info": null - } - ] - } - ], - "hasMore": false, - "limit": 25, - "offset": 0, - "count": 1, - "links": [] - } - -.. code-block:: bash - :caption: Example curl request - - curl -X GET \ - "https://:/ords//_/db-api/stable/vecdb/models/" \ - -H "Accept: application/json" \ - # Choose ONE authentication method: - - # Option 1: Basic authentication - -u ":" - - # Option 2: OAuth Bearer token - # -H "Authorization: Bearer " - ----- - -**POST /vecdb/models/** - -Load an embedding or reranking model into the database. - -Imports a model from object storage (ONNX format) for use in embedding -generation and reranking operations. Once loaded, the model can be used -for integrated table embeddings or standalone inference. - -**Request body** - -.. list-table:: - :header-rows: 1 - :widths: 25 15 15 45 - - * - Parameter - - Type - - Required - - Description - * - ``modelName`` - - string - - Yes - - Unique name to assign to the loaded model. - * - ``url`` - - string - - Yes - - Object storage URL where the model file is located. Supports Oracle Object Storage URLs and public URLs. - * - ``modelParams`` - - object - - No - - Model loading parameters. Example: ``{'provider': 'database', 'credential': 'OCI_CRED', 'metadata': {...}}``. - * - ``debugFlags`` - - object - - No - - See debugFlags note above. - -**Returns** - -JSON response confirming the model was loaded successfully. - -.. code-block:: json - :caption: Example 201 response - - { - "model_name": "all-MiniLM-L6-v2", - "algorithm": "ONNX", - "mining_function": "EMBEDDING", - "creation_date": "2026-02-01T18:45:09Z", - "attributes": [] - } - -.. code-block:: bash - :caption: Example curl request - - curl -X POST \ - "https://:/ords//_/db-api/stable/vecdb/models/" \ - -H "Content-Type: application/json" \ - -H "Accept: application/json" \ - # Choose ONE authentication method: - - # Option 1: Basic authentication - -u ":" \ - - # Option 2: OAuth Bearer token - # -H "Authorization: Bearer " \ - - -d '{ - "modelName": "all-MiniLM-L6-v2", - "url": "https://objectstorage.us-phoenix-1.oraclecloud.com/n/namespace/b/bucket/o/model.onnx", - "modelParams": { - "provider": "database", - "credential": "OCI_CREDENTIAL" - } - }' - -**Response 400** – the request body included invalid parameters. - -**Response 404** – model not found. - ----- - -**GET /vecdb/models/{model_name}** - -Retrieve detailed metadata for a loaded model. - -Returns comprehensive information about the model including its type, -parameters, attributes, and usage statistics. - -**Path parameter** - -.. list-table:: - :header-rows: 1 - :widths: 25 15 15 45 - - * - Parameter - - Type - - Required - - Description - * - ``model_name`` - - string - - Yes - - Name of the model to describe. - -**Returns** - -JSON response containing the model name and type, algorithm and mining function, input/output attributes, creation timestamp, and model parameters. - -.. code-block:: json - :caption: Example 200 response - - { - "model_name": "all-MiniLM-L6-v2", - "algorithm": "ONNX", - "mining_function": "EMBEDDING", - "creation_date": "2026-01-14T09:32:11Z", - "attributes": [ - { - "name": "NUM_DIMENSIONS", - "value": "384", - "data_type": "NUMBER", - "data_length": 22, - "vector_info": null - } - ] - } - -.. code-block:: bash - :caption: Example curl request - - curl -X GET \ - "https://:/ords//_/db-api/stable/vecdb/models/all-MiniLM-L6-v2" \ - -H "Accept: application/json" \ - # Choose ONE authentication method: - - # Option 1: Basic authentication - -u ":" - - # Option 2: OAuth Bearer token - # -H "Authorization: Bearer " - -**Response 404** – model not found. - ----- - -**DELETE /vecdb/models/{model_name}** - -Remove a loaded model from the database. - -Drops the specified embedding or reranking model. Models currently in use -by vector tables cannot be dropped and will throw an error. - -**Path parameter** - -.. list-table:: - :header-rows: 1 - :widths: 25 15 15 45 - - * - Parameter - - Type - - Required - - Description - * - ``model_name`` - - string - - Yes - - Name of the model to drop. - -**Returns** - -JSON response confirming model deletion. - -.. code-block:: json - :caption: Example 200 response - - { - "dropped": "YES", - "message": "Model all-MiniLM-L6-v2 removed" - } - -.. code-block:: bash - :caption: Example curl request - - curl -X DELETE \ - "https://:/ords//_/db-api/stable/vecdb/models/all-MiniLM-L6-v2" \ - -H "Accept: application/json" \ - # Choose ONE authentication method: - - # Option 1: Basic authentication - -u ":" - - # Option 2: OAuth Bearer token - # -H "Authorization: Bearer " - -**Response 400** – the model is used in a vector table and cannot be deleted. - -**Response 404** – model not found. - ----- - -Vector Tables --------------- - -**GET /vecdb/summary** - -Get summary statistics for the entire vector database service. - - -**Returns** - -JSON response with database-level statistics including total tables, models, and vectors. - -.. code-block:: json - :caption: Example 200 response - - { - "total_tables": 4, - "total_vectors": 125000, - "total_models": 2 - } - -.. code-block:: bash - :caption: Example curl request - - curl -X GET \ - "https://:/ords//_/db-api/stable/vecdb/summary" \ - -H "Accept: application/json" \ - # Choose ONE authentication method: - - # Option 1: Basic authentication - -u ":" - - # Option 2: OAuth Bearer token - # -H "Authorization: Bearer " - ----- - -**GET /vecdb/vector-tables/** - -List all vector tables in the database. - -Returns a list of all vector tables accessible to the current user, including -their names and basic configuration details. - -**Returns** - -JSON response containing an array of table information including table names, descriptions, vector types, row counts, and creation timestamps. - -.. code-block:: json - :caption: Example 200 response - - { - "items": [ - { - "table_name": "documents", - "vector_type": "dense", - "status": "READY", - "annotations": {"department": "knowledge"} - }, - { - "table_name": "product_vectors", - "vector_type": "dense", - "status": "READY" - } - ] - } - -Each entry contains ``table_name``, ``vector_type``, ``status``, -``index_params`` (if defined), and annotations. - -.. code-block:: bash - :caption: Example curl request - - curl -X GET \ - "https://:/ords//_/db-api/stable/vecdb/vector-tables/" \ - -H "Accept: application/json" \ - # Choose ONE authentication method: - - # Option 1: Basic authentication - -u ":" - - # Option 2: OAuth Bearer token - # -H "Authorization: Bearer " - ----- - -**POST /vecdb/vector-tables/** - -Create a new vector table for storing vector embeddings. - -Creates a vector table with a fixed schema optimized for vector search. The table -includes columns for ID, vector data, and JSON metadata. You can configure automatic -ID generation, embedding integration, and index parameters during creation. - -**Request body** - -.. list-table:: - :header-rows: 1 - :widths: 25 15 15 45 - - * - Parameter - - Type - - Required - - Description - * - ``name`` - - string - - Yes - - Name of the vector table to create. Must be unique within the database. - * - ``comment`` - - string - - No - - Table comment. - * - ``annotations`` - - object - - No - - Key-value pairs added to the vector table metadata. Example: ``{"application": "chatbot", "department": "sales"}``. - * - ``tableParams`` - - object - - No - - Table creation controls. Use ``{"auto_generate_id": true}`` when the ID column should default to ``SYS_GUID()``. - * - ``embedParams`` - - object - - No - - Configuration for integrated embedding model. If provided, the table will automatically generate embeddings on insert: - - .. code-block:: json - - { - "model": "", - "embed_metadata_jsonpath": "" - } - - * - ``indexParams`` - - object - - No - - Nested 26.2 vector and metadata index configuration: - - .. code-block:: json - - { - "vector_index_params": { - "auto_index": true, - "organization": "", - "distance_metric": "", - "accuracy": 90, - "online_build": true, - "quantization_type": "", - "compression_ratio": 4, - "distribute_params": { - "distribute_method": "" - }, - "advanced_params": { - "partitions": 16, - "neighbors": 32, - "efConstruction": 128, - "rescore_factor": 10, - "algorithm": "uniform_quantization" - } - }, - "metadata_index_params": { - "auto_index": true, - "include_paths": ["tenant", "category"], - "exclude_paths": ["body"] - }, - "parallel_creation": 4 - } - - ``partitions`` applies to ``PARTITIONS`` organization; - ``neighbors`` and ``efConstruction`` apply to ``INMEMORY GRAPH`` organization. - * - ``debugFlags`` - - object - - No - - See debugFlags note above. - -**Returns** - -JSON response containing the created table details and status. - -.. code-block:: json - :caption: Example 201 response - - { - "table_name": "product_vectors", - "vector_type": "dense", - "auto_generate_id": true, - "vector_table_type": "BYOV", - "index_params": { - "vector_index_params": { - "auto_index": true, - "organization": "PARTITIONS" - }, - "parallel_creation": 4 - }, - "annotations": {"application": "catalog"} - } - -.. code-block:: bash - :caption: Example curl request – Create table for pre-computed vectors - - curl -X POST \ - "https://:/ords//_/db-api/stable/vecdb/vector-tables/" \ - -H "Content-Type: application/json" \ - -H "Accept: application/json" \ - # Choose ONE authentication method: - - # Option 1: Basic authentication - -u ":" \ - - # Option 2: OAuth Bearer token - # -H "Authorization: Bearer " \ - - -d '{ - "name": "product_vectors", - "comment": "Product embeddings", - "tableParams": { - "auto_generate_id": true - }, - "indexParams": { - "vector_index_params": { - "auto_index": true, - "organization": "PARTITIONS", - "distance_metric": "COSINE" - }, - "parallel_creation": 4 - }, - "annotations": {"application": "catalog"} - }' - -.. code-block:: bash - :caption: Example curl request – Create table with integrated embedding - - curl -X POST \ - "https://:/ords//_/db-api/stable/vecdb/vector-tables/" \ - -H "Content-Type: application/json" \ - -H "Accept: application/json" \ - # Choose ONE authentication method: - - # Option 1: Basic authentication - -u ":" \ - - # Option 2: OAuth Bearer token - # -H "Authorization: Bearer " \ - - -d '{ - "name": "documents", - "tableParams": { - "auto_generate_id": true - }, - "embedParams": { - "model": "all_MiniLM_L12_v2", - "embed_metadata_jsonpath": "content" - } - }' - -.. code-block:: bash - :caption: Example curl request – Create table for bring-your-own vectors with manual indexing - - curl -X POST \ - "https://:/ords//_/db-api/stable/vecdb/vector-tables/" \ - -H "Content-Type: application/json" \ - -H "Accept: application/json" \ - # Choose ONE authentication method: - - # Option 1: Basic authentication - -u ":" \ - - # Option 2: OAuth Bearer token - # -H "Authorization: Bearer " \ - - -d '{ - "name": "customer_vectors", - "comment": "Manually managed vector table", - "indexParams": { - "vector_index_params": { - "auto_index": false - } - } - }' - -**Response 400** – invalid or missing parameters (for example, missing ``name``). - -**Response 409** – a table with that name already exists. - ----- - -**GET /vecdb/vector-tables/{vector_table_name}** - -Retrieve detailed configuration and metadata for a vector table. - -Returns comprehensive information about the specified table including its schema, -index configuration, embedding settings, row count, and creation timestamp. - -**Path parameter** - -.. list-table:: - :header-rows: 1 - :widths: 25 15 15 45 - - * - Parameter - - Type - - Required - - Description - * - ``vector_table_name`` - - string - - Yes - - Name of the vector table to describe. - -**Returns** - -JSON response describing the table including table name and description, vector type and dimensions, index parameters and status, embedding model configuration (if applicable), row count and storage statistics, and annotations and metadata. - -.. code-block:: json - :caption: Example 200 response - - { - "table_name": "product_vectors", - "description": "Product embeddings", - "vector_type": "dense", - "index_params": { - "distance_metric": "COSINE", - "organization": "PARTITIONS" - }, - "embed_params": null, - "annotations": {"version": "1.0"}, - "created": "2026-01-31T09:12:42Z" - } - -Key fields: - -- ``table_name``: Vector table identifier. -- ``index_params``: Active index metadata (if configured). -- ``annotations``: Custom metadata supplied at creation/update time. - -.. code-block:: bash - :caption: Example curl request - - curl -X GET \ - "https://:/ords//_/db-api/stable/vecdb/vector-tables/product_vectors" \ - -H "Accept: application/json" \ - # Choose ONE authentication method: - - # Option 1: Basic authentication - -u ":" - - # Option 2: OAuth Bearer token - # -H "Authorization: Bearer " - -**Response 404** – not found or precondition failed. - ----- - -**PATCH /vecdb/vector-tables/{vector_table_name}** - -Update the description and annotations for an existing vector table. - -Modifies the metadata and configuration of a vector table without affecting -the stored data. Can update description, annotations, and index parameters. - -.. note:: - - Annotations are replaced entirely, not merged. To preserve existing - annotations, include them in the update request. - -**Path parameter** - -.. list-table:: - :header-rows: 1 - :widths: 25 15 15 45 - - * - Parameter - - Type - - Required - - Description - * - ``vector_table_name`` - - string - - Yes - - Name of the vector table to update. - -**Request body** - -.. list-table:: - :header-rows: 1 - :widths: 25 15 15 45 - - * - Parameter - - Type - - Required - - Description - * - ``description`` - - string - - Yes - - New description for the table. - * - ``annotations`` - - object - - No - - New annotations to replace existing ones. Annotations are completely replaced, not merged. - * - ``indexParams`` - - object - - No - - Updated index configuration parameters. - * - ``debugFlags`` - - object - - No - - See debugFlags note above. - -**Returns** - -JSON response confirming the update. - -.. code-block:: json - :caption: Example 202 response - - { - "status": "ACCEPTED", - "tableName": "products", - "requestId": "c9b8f6a2-...", - "message": "Update scheduled" - } - -.. code-block:: bash - :caption: Example curl request - - curl -X PATCH \ - "https://:/ords//_/db-api/stable/vecdb/vector-tables/products" \ - -H "Content-Type: application/json" \ - -H "Accept: application/json" \ - # Choose ONE authentication method: - - # Option 1: Basic authentication - -u ":" \ - - # Option 2: OAuth Bearer token - # -H "Authorization: Bearer " \ - - -d '{ - "description": "Updated product embeddings", - "annotations": { - "version": "2.0", - "updated": "2026-02-10" - } - }' - -**Response 400** – invalid JSON, invalid types, or missing required fields. - -**Response 404** – not found or precondition failed. - ----- - -**DELETE /vecdb/vector-tables/{vector_table_name}** - -Permanently delete a vector table and all its data. - -Drops the specified vector table, including all vectors, metadata, and associated -indexes. This operation cannot be undone. - -.. warning:: - - This operation is irreversible. All data in the table will be permanently deleted. - -**Path parameter** - -.. list-table:: - :header-rows: 1 - :widths: 25 15 15 45 - - * - Parameter - - Type - - Required - - Description - * - ``vector_table_name`` - - string - - Yes - - Name of the vector table to drop. - -**Returns** - -JSON response confirming the table was dropped successfully. - -.. code-block:: json - :caption: Example 200 response - - { - "status": "SUCCEEDED", - "message": "Table dropped successfully", - "tableName": "old_vectors" - } - -.. code-block:: bash - :caption: Example curl request - - curl -X DELETE \ - "https://:/ords//_/db-api/stable/vecdb/vector-tables/old_vectors" \ - -H "Accept: application/json" \ - # Choose ONE authentication method: - - # Option 1: Basic authentication - -u ":" - - # Option 2: OAuth Bearer token - # -H "Authorization: Bearer " - -**Response 404** – table not found. - -Manage Data ------------- - -**POST /vecdb/embed** - -Generate vector embeddings for text inputs using a loaded model. - -Converts text into dense vector representations using the specified embedding -model. The model must be loaded in the database using ``POST /vecdb/models/`` first. - -**Request body** - -.. list-table:: - :header-rows: 1 - :widths: 25 15 15 45 - - * - Parameter - - Type - - Required - - Description - * - ``modelName`` - - string - - Yes - - Name of the loaded embedding model to use. - * - ``inputs`` - - array - - Yes - - Array of input objects to embed. Each entry contains ``text`` (string, required). Example: ``[{"text": "text1"}, {"text": "text2"}]``. - * - ``debugFlags`` - - object - - No - - See debugFlags note above. - -**Returns** - -JSON response containing the generated embeddings β€” an array of vectors corresponding to each input. - -.. code-block:: json - :caption: Example 200 response - - { - "data": [ - { - "embedding": [0.1, 0.2, 0.3, 0.4], - "text": "Wireless headphones" - }, - { - "embedding": [0.5, 0.4, 0.3, 0.2], - "text": "Ergonomic office chair" - } - ] - } - -.. code-block:: bash - :caption: Example curl request - - curl -X POST \ - "https://:/ords//_/db-api/stable/vecdb/embed" \ - -H "Content-Type: application/json" \ - -H "Accept: application/json" \ - # Choose ONE authentication method: - - # Option 1: Basic authentication - -u ":" \ - - # Option 2: OAuth Bearer token - # -H "Authorization: Bearer " \ - - -d '{ - "modelName": "all-MiniLM-L6-v2", - "inputs": [ - {"text": "Wireless noise-cancelling headphones"}, - {"text": "Ergonomic office chair with lumbar support"} - ] - }' - -**Response 400** – the request body included invalid parameters. - ----- - -**POST /vecdb/vector-tables/{vector_table_name}/upsert** - -Insert or update vectors in a table. - -Upserts vectors into the specified table. If a vector with the same ID already -exists, it will be updated with the new values. Otherwise, a new record is inserted. - -If the table is configured with ``autoGenerateID: true``, you don't need to provide -``id`` as part of the upsert object. For tables with integrated embedding models, you -can provide text in metadata β€” embeddings will be generated automatically. - -**Path parameter** - -.. list-table:: - :header-rows: 1 - :widths: 25 15 15 45 - - * - Parameter - - Type - - Required - - Description - * - ``vector_table_name`` - - string - - Yes - - Name of the vector table. - -**Request body** - -.. list-table:: - :header-rows: 1 - :widths: 25 15 15 45 - - * - Parameter - - Type - - Required - - Description - * - ``vectors`` - - array - - Yes - - Array of vector objects to upsert. Each entry contains ``id``, ``dense_vector``, and ``metadata``. For tables with embedding models, ``dense_vector`` can be omitted and the embedding will be generated automatically from the configured metadata field. - * - ``debugFlags`` - - object - - No - - See debugFlags note above. - -**Returns** - -JSON response confirming upsert with count of inserted/updated vectors. - -.. code-block:: json - :caption: Example 201 response - - { - "upserted_count": 2 - } - -.. code-block:: bash - :caption: Example curl request – Upsert pre-computed vectors - - curl -X POST \ - "https://:/ords//_/db-api/stable/vecdb/vector-tables/products/upsert" \ - -H "Content-Type: application/json" \ - -H "Accept: application/json" \ - # Choose ONE authentication method: - - # Option 1: Basic authentication - -u ":" \ - - # Option 2: OAuth Bearer token - # -H "Authorization: Bearer " \ - - -d '{ - "vectors": [ - { - "id": "prod_1", - "dense_vector": [0.1, 0.2, 0.3, 0.4, 0.5], - "metadata": { - "name": "Wireless Headphones", - "category": "electronics", - "price": 99.99 - } - }, - { - "id": "prod_2", - "dense_vector": [0.2, 0.3, 0.1, 0.5, 0.4], - "metadata": { - "name": "Smart Watch", - "category": "electronics", - "price": 199.99 - } - } - ] - }' - -.. code-block:: bash - :caption: Example curl request – Upsert with automatic embedding (table must have embed_params configured) - - curl -X POST \ - "https://:/ords//_/db-api/stable/vecdb/vector-tables/documents/upsert" \ - -H "Content-Type: application/json" \ - -H "Accept: application/json" \ - -u ":" \ - -d '{ - "vectors": [ - { - "id": "doc_1", - "metadata": { - "content": "Machine learning is transforming healthcare", - "category": "AI", - "author": "John Doe" - } - }, - { - "id": "doc_2", - "metadata": { - "content": "Vector databases enable semantic search", - "category": "Database", - "author": "Jane Smith" - } - } - ] - }' - -**Response 400** – the request body included invalid parameters. - -**Response 404** – the specified vector table does not exist. - ----- - -**POST /vecdb/vector-tables/{vector_table_name}/list** - -Retrieve vectors from a table by their IDs. - -Lists vector records with their IDs, embeddings, and metadata. Supports -pagination for large result sets. - -**Path parameter** - -.. list-table:: - :header-rows: 1 - :widths: 25 15 15 45 - - * - Parameter - - Type - - Required - - Description - * - ``vector_table_name`` - - string - - Yes - - Name of the vector table. - -**Request body** - -.. list-table:: - :header-rows: 1 - :widths: 25 15 15 45 - - * - Parameter - - Type - - Required - - Description - * - ``ids`` - - array of string - - No - - List of vector IDs to retrieve. Example: ``["id1", "id2", "id3"]``. - * - ``limit`` - - number - - No - - Maximum number of results to return. Defaults to ``15``. - * - ``offset`` - - number - - No - - Number of records to skip for pagination. - * - ``debugFlags`` - - object - - No - - See debugFlags note above. - -**Returns** - -JSON response containing matching vectors including IDs, dense vectors, and metadata. - -.. code-block:: json - :caption: Example 200 response - - { - "items": [ - { - "id": "prod_1", - "dense_vector": [0.1, 0.2, 0.3, 0.4, 0.5], - "metadata": {"name": "Wireless Headphones", "category": "electronics", "price": 99.99} - }, - { - "id": "prod_2", - "dense_vector": [0.2, 0.3, 0.1, 0.5, 0.4], - "metadata": {"name": "Smart Watch", "category": "electronics", "price": 199.99} - } - ], - "limit": 15, - "offset": 0, - "count": 2 - } - -.. code-block:: bash - :caption: Example curl request – Get specific vectors by ID - - curl -X POST \ - "https://:/ords//_/db-api/stable/vecdb/vector-tables/products/list" \ - -H "Content-Type: application/json" \ - -H "Accept: application/json" \ - # Choose ONE authentication method: - - # Option 1: Basic authentication - -u ":" \ - - # Option 2: OAuth Bearer token - # -H "Authorization: Bearer " \ - - -d '{ - "ids": ["prod_1", "prod_2", "prod_3"], - "limit": 15, - "offset": 0 - }' - -.. code-block:: bash - :caption: Example curl request – Paginate through results (First 10 results) - - curl -X POST \ - "https://:/ords//_/db-api/stable/vecdb/vector-tables/products/list" \ - -H "Content-Type: application/json" \ - -H "Accept: application/json" \ - -u ":" \ - -d '{ - "limit": 10, - "offset": 0 - }' - -.. code-block:: bash - :caption: Example curl request – Next 10 results - - curl -X POST \ - "https://:/ords//_/db-api/stable/vecdb/vector-tables/products/list" \ - -H "Content-Type: application/json" \ - -H "Accept: application/json" \ - -u ":" \ - -d '{ - "limit": 10, - "offset": 10 - }' - -**Response 400** – the request body included invalid parameters. - -**Response 404** – the specified vector table does not exist. - ----- - -**POST /vecdb/vector-tables/{vector_table_name}/delete** - -Delete vectors from a table by their IDs. - -Removes the specified vectors and their associated metadata from the table. - -**Path parameter** - -.. list-table:: - :header-rows: 1 - :widths: 25 15 15 45 - - * - Parameter - - Type - - Required - - Description - * - ``vector_table_name`` - - string - - Yes - - Name of the vector table. - -**Request body** - -.. list-table:: - :header-rows: 1 - :widths: 25 15 15 45 - - * - Parameter - - Type - - Required - - Description - * - ``ids`` - - array of string - - Yes - - List of vector IDs to delete. Example: ``["id1", "id2", "id3"]``. - * - ``debugFlags`` - - object - - No - - See debugFlags note above. - -**Returns** - -JSON response confirming deletion with count of deleted vectors. - -.. code-block:: json - :caption: Example 200 response - - { - "message": "Vectors removed successfully." - } - -.. code-block:: bash - :caption: Example curl request - - curl -X POST \ - "https://:/ords//_/db-api/stable/vecdb/vector-tables/products/delete" \ - -H "Content-Type: application/json" \ - -H "Accept: application/json" \ - # Choose ONE authentication method: - - # Option 1: Basic authentication - -u ":" \ - - # Option 2: OAuth Bearer token - # -H "Authorization: Bearer " \ - - -d '{ - "ids": ["prod_old_1", "prod_old_2", "prod_old_3"] - }' - -**Response 400** – the request body included invalid parameters. - -**Response 404** – the specified vector table does not exist. - ----- - -**POST /vecdb/load** - -Bulk load vectors from a CSV file in object storage. - -Loads a large dataset from object storage into an existing vector table -asynchronously. If the specified table does not exist, the service returns a -not-found error. If the table exists, the new vectors are appended. - -The object storage URL should point to a CSV file with the following format: - -.. code-block:: text - - id,dense_vector,metadata - id1,[0.1, 0.2, 0.3],{"field1": "value1", "field2": "value2"} - id2,[0.4, 0.5, 0.6],{"field1": "value3", "field2": "value4"} - -**Request body** - -.. list-table:: - :header-rows: 1 - :widths: 25 15 15 45 - - * - Parameter - - Type - - Required - - Description - * - ``tableName`` - - string - - Yes - - Name of the existing target vector table. - * - ``url`` - - string - - Yes - - Object storage URL pointing to the CSV file containing vectors. - * - ``params`` - - object - - No - - Optional parameters for the load operation. Example: ``{'credential': 'OCI_CREDENTIAL'}`` if the object storage URL requires authentication. - * - ``debugFlags`` - - object - - No - - See debugFlags note above. - -**Returns** - -JSON response containing the load job ID and initial status. - -.. code-block:: json - :caption: Example 200 response - - { - "job_name": "LOAD_PRODUCTS_20260210", - "job_creator": "VECDB_USER", - "job_type": "SCHEDULED", - "operation": "LOAD_CSV", - "state": "SCHEDULED", - "start_date": null, - "links": [ - {"rel": "job", "href": "/vecdb/load/jobs/LOAD_PRODUCTS_20260210/"} - ] - } - -.. code-block:: bash - :caption: Example curl request - - curl -X POST \ - "https://:/ords//_/db-api/stable/vecdb/load" \ - -H "Content-Type: application/json" \ - -H "Accept: application/json" \ - # Choose ONE authentication method: - - # Option 1: Basic authentication - -u ":" \ - - # Option 2: OAuth Bearer token - # -H "Authorization: Bearer " \ - - -d '{ - "tableName": "products", - "url": "https://objectstorage.region.oraclecloud.com/.../vectors.csv", - "params": { - "credential": "OCI_CREDENTIAL" - } - }' - -**Response 400** – the request body included invalid parameters. - ----- - -**GET /vecdb/load/jobs/** - -List all bulk load operations. - -Returns metadata for all load jobs including their states and progress. - -**Returns** - -JSON response containing an array of load jobs with job names, owners, states, and timestamps. - -.. code-block:: json - :caption: Example 200 response - - { - "items": [ - { - "job_name": "LOAD_PRODUCTS_20260210", - "job_creator": "VECDB_USER", - "job_type": "SCHEDULED", - "operation": "LOAD_CSV", - "state": "SUCCEEDED", - "start_date": "2026-02-10T19:00:00Z", - "links": [ - {"rel": "self", "href": "/vecdb/load/jobs/LOAD_PRODUCTS_20260210/"} - ] - } - ], - "hasMore": false, - "limit": 25, - "offset": 0, - "count": 1, - "links": [] - } - -.. code-block:: bash - :caption: Example curl request - - curl -X GET \ - "https://:/ords//_/db-api/stable/vecdb/load/jobs/" \ - -H "Accept: application/json" \ - # Choose ONE authentication method: - - # Option 1: Basic authentication - -u ":" - - # Option 2: OAuth Bearer token - # -H "Authorization: Bearer " - ----- - -**GET /vecdb/load/jobs/{load_job_name}/** - -Get the status of a bulk load operation. - -Returns details about an asynchronous load job initiated by ``POST /vecdb/load``. - -**Path parameter** - -.. list-table:: - :header-rows: 1 - :widths: 25 15 15 45 - - * - Parameter - - Type - - Required - - Description - * - ``load_job_name`` - - string - - Yes - - Name of the load job to describe. - -**Returns** - -JSON response containing job status, progress, and statistics. - -.. code-block:: json - :caption: Example 200 response - - { - "job_name": "LOAD_PRODUCTS_20260210", - "job_creator": "VECDB_USER", - "job_type": "SCHEDULED", - "operation": "LOAD_CSV", - "state": "SUCCEEDED", - "start_date": "2026-02-10T19:00:00Z", - "links": [ - {"rel": "self", "href": "/vecdb/load/jobs/LOAD_PRODUCTS_20260210/"}, - {"rel": "jobfile", "href": "/vecdb/load/jobs/LOAD_PRODUCTS_20260210/jobfile"} - ] - } - -.. code-block:: bash - :caption: Example curl request - - curl -X GET \ - "https://:/ords//_/db-api/stable/vecdb/load/jobs/LOAD_JOB_67890/" \ - -H "Accept: application/json" \ - # Choose ONE authentication method: - - # Option 1: Basic authentication - -u ":" - - # Option 2: OAuth Bearer token - # -H "Authorization: Bearer " - ----- - -**GET /vecdb/load/jobs/{load_job_name}/jobfile** - -Retrieve the log output for a bulk load job. - -**Path parameter** - -.. list-table:: - :header-rows: 1 - :widths: 25 15 15 45 - - * - Parameter - - Type - - Required - - Description - * - ``load_job_name`` - - string - - Yes - - Name of the load job. - -**Returns** - -JSON response containing log file metadata and contents. - -.. code-block:: json - :caption: Example 200 response - - { - "log_date": "2026-02-10T19:12:45Z", - "job_name": "LOAD_PRODUCTS_20260210", - "status": "INFO", - "error#": 0, - "additional_info": "Loaded 500 rows", - "actual_start_date": "2026-02-10T19:00:05Z", - "run_duration": "+000 00:12:40.000", - "links": [] - } - -.. code-block:: bash - :caption: Example curl request - - curl -X GET \ - "https://:/ords//_/db-api/stable/vecdb/load/jobs/LOAD_JOB_67890/jobfile" \ - -H "Accept: application/json" \ - # Choose ONE authentication method: - - # Option 1: Basic authentication - -u ":" - - # Option 2: OAuth Bearer token - # -H "Authorization: Bearer " - -Search Data ------------- - -**POST /vecdb/vector-tables/{vector_table_name}/query** - -Perform a vector similarity search using text, a vector, or an ID. - -Performs similarity search to find the most similar vectors in the table. -Supports filtering by metadata and various distance metrics. Retrieves the -IDs, metadata, vectors and similarity scores of the most similar items. - -**Path parameter** - -.. list-table:: - :header-rows: 1 - :widths: 25 15 15 45 - - * - Parameter - - Type - - Required - - Description - * - ``vector_table_name`` - - string - - Yes - - Name of the vector table to search. - -**Request body** - -.. list-table:: - :header-rows: 1 - :widths: 25 15 15 45 - - * - Parameter - - Type - - Required - - Description - * - ``queryBy`` - - object - - Yes - - Query specification. One of: ``{'vector': [0.1, 0.2, ...]}`` (search by vector), ``{'text': 'query text'}`` (search by text, requires table with embedding model), ``{'id': 'vector_id'}`` (find similar vectors to an existing record). - * - ``topK`` - - number - - Yes - - Number of most similar results to return. - * - ``includeVectors`` - - boolean - - No - - Include vector values in response. Defaults to ``false`` to minimize response size. - * - ``filters`` - - object - - No - - Metadata filters to narrow search results. Supported operators: ``$eq``, ``$ne``, ``$gt``, ``$gte``, ``$lt``, ``$lte``, ``$in``, ``$nin``, ``$and``, ``$or``, ``$exists``. Example: ``{'category': {'$eq': 'electronics'}, 'price': {'$lt': 100}}``. - * - ``advancedOptions`` - - object - - No - - Search tuning parameters: - - - ``distance_metric``: Override the default metric. Supported values are - ``COSINE``, ``MANHATTAN``, ``HAMMING``, ``JACCARD``, ``DOT``, - ``EUCLIDEAN``, ``L2_SQUARED``, and ``EUCLIDEAN_SQUARED``. Refer to - the Distance Metric Guidance table above for metric summaries and - usage guidance. - - ``accuracy``: Target accuracy (0–100). Higher values provide better recall but slower search; ``100`` approximates an exact search. - - ``idx_parameters``: Index-specific overrides. Supported keys: - - - ``efsearch``: HNSW beam width controlling recall. Use this instead of ``accuracy`` to specify the maximum number of candidates considered while probing the index. Higher values provide better accuracy. - - ``neighbor partition probes``: IVF partition probes controlling how many inverted lists are scanned. - * - ``debugFlags`` - - object - - No - - See debugFlags note above. - -**Returns** - -JSON response containing matching results β€” an array of results with IDs, distances, and metadata. Distance scores (lower is more similar for most metrics). Vector values are included if ``includeVectors`` is ``true``. - -.. code-block:: json - :caption: Example 200 response - - [ - { - "id": "prod_123", - "metadata": {"name": "Wireless Headphones"}, - "distance": 0.12 - } - ] - -.. code-block:: bash - :caption: Example curl request – Search by query vector - - curl -X POST \ - "https://:/ords//_/db-api/stable/vecdb/vector-tables/products/query" \ - -H "Content-Type: application/json" \ - -H "Accept: application/json" \ - # Choose ONE authentication method: - - # Option 1: Basic authentication - -u ":" \ - - # Option 2: OAuth Bearer token - # -H "Authorization: Bearer " \ - - -d '{ - "queryBy": {"vector": [0.1, 0.2, 0.3]}, - "topK": 10 - }' - -.. code-block:: bash - :caption: Example curl request – Search by text with filtering - - curl -X POST \ - "https://:/ords//_/db-api/stable/vecdb/vector-tables/products/query" \ - -H "Content-Type: application/json" \ - -H "Accept: application/json" \ - -u ":" \ - -d '{ - "queryBy": {"text": "wireless headphones"}, - "topK": 5, - "filters": { - "$and": [ - {"category": {"$eq": "electronics"}}, - {"price": {"$lt": 200}} - ] - } - }' - -.. code-block:: bash - :caption: Example curl request – Find similar items to an existing product - - curl -X POST \ - "https://:/ords//_/db-api/stable/vecdb/vector-tables/products/query" \ - -H "Content-Type: application/json" \ - -H "Accept: application/json" \ - -u ":" \ - -d '{ - "queryBy": {"id": "prod_12345"}, - "topK": 10, - "filters": {"category": {"$eq": "electronics"}} - }' - -.. code-block:: bash - :caption: Example curl request – Search with custom distance metric - - curl -X POST \ - "https://:/ords//_/db-api/stable/vecdb/vector-tables/products/query" \ - -H "Content-Type: application/json" \ - -H "Accept: application/json" \ - -u ":" \ - -d '{ - "queryBy": {"text": "laptop"}, - "topK": 10, - "advancedOptions": { - "distance_metric": "EUCLIDEAN", - "accuracy": 95, - "idx_parameters": { - "efsearch": 128, - "neighbor partition probes": 4 - } - }, - "includeVectors": true - }' - -**Response 400** – the request body included invalid parameters. - -**Response 404** – the specified vector table does not exist. - ----- - -**POST /vecdb/rerank** - -Re-rank search results based on relevance to a query. - -Uses a reranking model to score and reorder documents relative to a query. -This improves search quality by performing a more detailed comparison between -the query and each candidate document. - -**Request body** - -.. list-table:: - :header-rows: 1 - :widths: 25 15 15 45 - - * - Parameter - - Type - - Required - - Description - * - ``query`` - - string - - Yes - - The search query text. - * - ``documents`` - - array of string - - Yes - - List of documents to rerank. Typically the results from ``POST /query``. Minimum 1 item. - * - ``modelName`` - - string - - Yes - - Name of the loaded reranking model. - * - ``modelParams`` - - object - - No - - Model configuration. Example: ``{"top_n": 5}`` to return only top 5 reranked results. - * - ``debugFlags`` - - object - - No - - See debugFlags note above. - -**Returns** - -JSON response containing reranked documents with relevance scores. - -.. code-block:: json - :caption: Example 200 response - - [ - { - "text": "Machine learning for health", - "index": 0, - "score": 0.82 - } - ] - -.. code-block:: bash - :caption: Example curl request - - curl -X POST \ - "https://:/ords//_/db-api/stable/vecdb/rerank" \ - -H "Content-Type: application/json" \ - -H "Accept: application/json" \ - # Choose ONE authentication method: - - # Option 1: Basic authentication - -u ":" \ - - # Option 2: OAuth Bearer token - # -H "Authorization: Bearer " \ - - -d '{ - "modelName": "cohere-rerank-3.5", - "query": "machine learning applications in healthcare", - "documents": [ - "Machine learning is transforming healthcare", - "Vector databases enable semantic search", - "Deep learning models for medical imaging" - ], - "modelParams": { - "top_n": 5 - } - }' - -**Response 400** – the request body included invalid parameters. - -Indexes --------- - -**POST /vecdb/vector-indexes/** - -Create a vector index on a table to enable fast similarity search. - -Creates an index for efficient approximate nearest neighbor (ANN) search. -The index creation runs asynchronously as a background job. Use ``GET /vecdb/vector-indexes/{vector_table_name}`` -or ``GET /vecdb/vector-indexes/jobs/{index_job_name}/`` to monitor progress. - -Supports IVF (Inverted File) and HNSW (Hierarchical Navigable Small World) indexes. -When ``indexParams`` are omitted, ORDS creates an index using server-side -defaults. ORDS 26.2 represents vector and metadata settings as nested -``vector_index_params`` and ``metadata_index_params`` objects. - -**Request body** - -.. list-table:: - :header-rows: 1 - :widths: 25 15 15 45 - - * - Parameter - - Type - - Required - - Description - * - ``tableName`` - - string - - Yes - - Name of the vector table to index. - * - ``indexParams`` - - object - - No - - Nested 26.2 index configuration. If not specified, ORDS uses server-side defaults: - - .. code-block:: json - - { - "vector_index_params": { - "auto_index": true, - "organization": "", - "distance_metric": "", - "quantization_type": "", - "compression_ratio": 4, - "advanced_params": { - "partitions": 16, - "neighbors": 32, - "efConstruction": 128 - } - }, - "metadata_index_params": { - "auto_index": true, - "include_paths": ["tenant"], - "exclude_paths": ["body"] - }, - "parallel_creation": 4 - } - - ``partitions`` applies to ``PARTITIONS`` (IVF) organization; - ``neighbors`` and ``efConstruction`` apply to ``INMEMORY GRAPH`` (HNSW) organization. - * - ``debugFlags`` - - object - - No - - See debugFlags note above. - -**Returns** - -JSON response containing the index job ID and status. - -.. code-block:: json - :caption: Example 200 response - - { - "job_name": "IDX_PRODUCTS_20260210", - "job_creator": "VECDB_USER", - "job_type": "SCHEDULED", - "operation": "CREATE_INDEX", - "state": "SCHEDULED", - "start_date": null, - "links": [ - {"rel": "job", "href": "/vecdb/vector-indexes/jobs/IDX_PRODUCTS_20260210/"} - ] - } - -.. code-block:: bash - :caption: Example curl request – Create index with default IVF settings - - curl -X POST \ - "https://:/ords//_/db-api/stable/vecdb/vector-indexes/" \ - -H "Content-Type: application/json" \ - -H "Accept: application/json" \ - # Choose ONE authentication method: - - # Option 1: Basic authentication - -u ":" \ - - # Option 2: OAuth Bearer token - # -H "Authorization: Bearer " \ - - -d '{ - "tableName": "products" - }' - -.. code-block:: bash - :caption: Example curl request – Create HNSW index with custom parameters - - curl -X POST \ - "https://:/ords//_/db-api/stable/vecdb/vector-indexes/" \ - -H "Content-Type: application/json" \ - -H "Accept: application/json" \ - -u ":" \ - -d '{ - "tableName": "products", - "indexParams": { - "vector_index_params": { - "auto_index": true, - "organization": "INMEMORY GRAPH", - "distance_metric": "COSINE", - "quantization_type": "SCALAR", - "compression_ratio": 4, - "distribute_params": { - "distribute_method": "AUTO" - }, - "advanced_params": { - "neighbors": 32, - "efConstruction": 128, - "rescore_factor": 10, - "algorithm": "uniform_quantization" - } - }, - "metadata_index_params": { - "auto_index": true, - "include_paths": ["tenant", "category"], - "exclude_paths": ["body"] - }, - "parallel_creation": 4 - } - }' - -.. code-block:: bash - :caption: Example curl request – Create IVF index with explicit defaults - - curl -X POST \ - "https://:/ords//_/db-api/stable/vecdb/vector-indexes/" \ - -H "Content-Type: application/json" \ - -H "Accept: application/json" \ - -u ":" \ - -d '{ - "tableName": "products", - "indexParams": { - "vector_index_params": { - "organization": "PARTITIONS", - "distance_metric": "COSINE", - "advanced_params": { - "partitions": 16 - } - } - } - }' - -**Response 400** – the request body included invalid parameters. - -**Response 404** – the vector table was not found. - ----- - -**GET /vecdb/vector-indexes/jobs/** - -List all index build and rebuild jobs. - -Returns metadata for all CREATE and REBUILD index operations, including -their current states and progress. - -**Returns** - -JSON response containing an array of index jobs with job names, owners, states (SCHEDULED, RUNNING, SUCCEEDED, FAILED), start/end timestamps, and log file paths. - -.. code-block:: json - :caption: Example 200 response - - { - "items": [ - { - "job_name": "IDX_PRODUCTS_20260210", - "job_creator": "VECDB_USER", - "job_type": "SCHEDULED", - "operation": "CREATE_INDEX", - "state": "SUCCEEDED", - "start_date": "2026-02-10T20:00:00Z", - "links": [ - {"rel": "self", "href": "/vecdb/vector-indexes/jobs/IDX_PRODUCTS_20260210/"} - ] - } - ], - "hasMore": false, - "limit": 25, - "offset": 0, - "count": 1, - "links": [] - } - -.. code-block:: bash - :caption: Example curl request - - curl -X GET \ - "https://:/ords//_/db-api/stable/vecdb/vector-indexes/jobs/" \ - -H "Accept: application/json" \ - # Choose ONE authentication method: - - # Option 1: Basic authentication - -u ":" - - # Option 2: OAuth Bearer token - # -H "Authorization: Bearer " - ----- - -**GET /vecdb/vector-indexes/jobs/{index_job_name}/** - -Retrieve metadata and status for a specific index build job. - -Returns details about an asynchronous index creation or rebuild job, including -its current state, progress, owner, and log file location. - -**Path parameter** - -.. list-table:: - :header-rows: 1 - :widths: 25 15 15 45 - - * - Parameter - - Type - - Required - - Description - * - ``index_job_name`` - - string - - Yes - - Name of the index job to describe. - -**Returns** - -JSON response describing the index job including job name and owner, job state (SCHEDULED, RUNNING, SUCCEEDED, FAILED), start and end timestamps, log file path, and error messages (if failed). - -.. code-block:: json - :caption: Example 200 response - - { - "job_name": "IDX_PRODUCTS_20260210", - "job_creator": "VECDB_USER", - "job_type": "SCHEDULED", - "operation": "CREATE_INDEX", - "state": "SUCCEEDED", - "start_date": "2026-02-10T20:00:00Z", - "links": [ - {"rel": "self", "href": "/vecdb/vector-indexes/jobs/IDX_PRODUCTS_20260210/"}, - {"rel": "jobfile", "href": "/vecdb/vector-indexes/jobs/IDX_PRODUCTS_20260210/jobfile"} - ] - } - -.. code-block:: bash - :caption: Example curl request - - curl -X GET \ - "https://:/ords//_/db-api/stable/vecdb/vector-indexes/jobs/INX_JOB_12345/" \ - -H "Accept: application/json" \ - # Choose ONE authentication method: - - # Option 1: Basic authentication - -u ":" - - # Option 2: OAuth Bearer token - # -H "Authorization: Bearer " - ----- - -**GET /vecdb/vector-indexes/jobs/{index_job_name}/jobfile** - -Retrieve the log output for an index build job. - -Returns the detailed log file contents for diagnosing index creation issues -or monitoring progress. - -**Path parameter** - -.. list-table:: - :header-rows: 1 - :widths: 25 15 15 45 - - * - Parameter - - Type - - Required - - Description - * - ``index_job_name`` - - string - - Yes - - Name of the index job. - -**Returns** - -JSON response containing log file metadata and contents. - -.. code-block:: json - :caption: Example 200 response - - { - "log_date": "2026-02-10T20:15:00Z", - "job_name": "IDX_PRODUCTS_20260210", - "status": "INFO", - "error#": 0, - "additional_info": "Index created successfully", - "actual_start_date": "2026-02-10T20:00:05Z", - "run_duration": "+000 00:14:55.000", - "links": [] - } - -.. code-block:: bash - :caption: Example curl request - - curl -X GET \ - "https://:/ords//_/db-api/stable/vecdb/vector-indexes/jobs/INX_JOB_12345/jobfile" \ - -H "Accept: application/json" \ - # Choose ONE authentication method: - - # Option 1: Basic authentication - -u ":" - - # Option 2: OAuth Bearer token - # -H "Authorization: Bearer " - ----- - -**POST /vecdb/vector-indexes/{vector_table_name}** - -Rebuild an existing vector index with updated parameters. - -Recreates the index, optionally with new configuration parameters. Useful for -optimizing search performance or adjusting to changed data distributions. -The rebuild runs asynchronously as a background job. - -**Path parameter** - -.. list-table:: - :header-rows: 1 - :widths: 25 15 15 45 - - * - Parameter - - Type - - Required - - Description - * - ``vector_table_name`` - - string - - Yes - - Name of the vector table whose index will be rebuilt. - -**Request body** (optional) - -.. list-table:: - :header-rows: 1 - :widths: 25 15 15 45 - - * - Parameter - - Type - - Required - - Description - * - ``indexParams`` - - object - - No - - New 26.2 index configuration. Use ``index_type`` to scope the rebuild to - ``vector``, ``metadata``, or ``all``: - - .. code-block:: json - - { - "index_type": "", - "vector_index_params": { - "organization": "", - "distance_metric": "" - }, - "metadata_index_params": { - "include_paths": ["tenant"] - }, - "parallel_creation": 4 - } - - * - ``debugFlags`` - - object - - No - - See debugFlags note above. - -**Returns** - -JSON response containing the rebuild job ID and status. - -.. code-block:: json - :caption: Example 200 response - - { - "job_name": "REBUILD_IDX_PRODUCTS_20260210", - "job_creator": "VECDB_USER", - "job_type": "SCHEDULED", - "operation": "REBUILD_INDEX", - "state": "SCHEDULED", - "start_date": null, - "links": [ - {"rel": "job", "href": "/vecdb/vector-indexes/jobs/REBUILD_IDX_PRODUCTS_20260210/"} - ] - } - -.. code-block:: bash - :caption: Example curl request - - curl -X POST \ - "https://:/ords//_/db-api/stable/vecdb/vector-indexes/products" \ - -H "Content-Type: application/json" \ - -H "Accept: application/json" \ - # Choose ONE authentication method: - - # Option 1: Basic authentication - -u ":" \ - - # Option 2: OAuth Bearer token - # -H "Authorization: Bearer " \ - - -d '{ - "indexParams": { - "index_type": "all", - "parallel_creation": 4 - } - }' - -**Response 400** – the request body included invalid parameters. - -**Response 404** – the vector table was not found. - ----- - -**GET /vecdb/vector-indexes/{vector_table_name}** - -Get the current status and configuration of a vector table's index. - -Returns detailed information about the index including its type, parameters, -build status, and statistics. - -**Path parameter** - -.. list-table:: - :header-rows: 1 - :widths: 25 15 15 45 - - * - Parameter - - Type - - Required - - Description - * - ``vector_table_name`` - - string - - Yes - - Name of the vector table whose index to describe. - -**Returns** - -JSON response containing the index type (IVF, HNSW), build status (BUILDING, READY, FAILED), index parameters, and statistics (indexed vectors, memory usage). - -.. code-block:: json - :caption: Example 200 response - - { - "Index Status": "VALID" - } - -.. code-block:: bash - :caption: Example curl request - - curl -X GET \ - "https://:/ords//_/db-api/stable/vecdb/vector-indexes/products" \ - -H "Accept: application/json" \ - # Choose ONE authentication method: - - # Option 1: Basic authentication - -u ":" - - # Option 2: OAuth Bearer token - # -H "Authorization: Bearer " - -**Response 400** – the request body included invalid parameters. - -**Response 404** – the vector table was not found. - ----- - -**POST /vecdb/vector-indexes/{vector_table_name}/delete** - -Drop the vector index from a table. - -Removes the index while preserving the table and its data. Queries will -fall back to exact search until a new index is created. - -**Path parameter** - -.. list-table:: - :header-rows: 1 - :widths: 25 15 15 45 - - * - Parameter - - Type - - Required - - Description - * - ``vector_table_name`` - - string - - Yes - - Name of the vector table whose index to drop. - -**Request body** - -.. list-table:: - :header-rows: 1 - :widths: 25 15 15 45 - - * - Parameter - - Type - - Required - - Description - * - ``indexParams`` - - object - - No - - Use ``index_type`` to drop ``vector``, ``metadata``, or ``all`` indexes. - Metadata drops can include ``metadata_index_params.include_paths`` to - select paths. - * - ``debugFlags`` - - object - - No - - See debugFlags note above. - -**Returns** - -JSON response confirming index deletion. - -.. code-block:: json - :caption: Example 200 response - - { - "name": "IDX_PRODUCTS_IVF", - "message": "Index drop request submitted." - } - -.. code-block:: bash - :caption: Example curl request – Drop all indexes - - curl -X POST \ - "https://:/ords//_/db-api/stable/vecdb/vector-indexes/products/delete" \ - -H "Content-Type: application/json" \ - -H "Accept: application/json" \ - # Choose ONE authentication method: - - # Option 1: Basic authentication - -u ":" \ - - # Option 2: OAuth Bearer token - # -H "Authorization: Bearer " \ - - -d '{ - "indexParams": { - "index_type": "all" - } - }' - -.. code-block:: bash - :caption: Example curl request – Drop selected metadata indexes - - curl -X POST \ - "https://:/ords//_/db-api/stable/vecdb/vector-indexes/products/delete" \ - -H "Content-Type: application/json" \ - -H "Accept: application/json" \ - -u ":" \ - -d '{ - "indexParams": { - "index_type": "metadata", - "metadata_index_params": { - "include_paths": ["tenant"] - } - } - }' - -**Response 400** – the request body included invalid parameters. - -**Response 404** – the vector table was not found. - -Error Catalogue ----------------- - -.. list-table:: - :header-rows: 1 - :widths: 20 80 - - * - Status Code - - Description - * - ``400 Bad Request`` - - Validation problems, incompatible payloads, or attempts to drop models still referenced by tables. - * - ``404 Not Found`` - - Target model, table, index, or job does not exist. - * - ``409 Conflict`` - - Resource already exists (for example, attempting to create a duplicate table). diff --git a/src/oracle_vecdb/vecdb_exception.py b/src/oracle_vecdb/vecdb_exception.py index 30547a1..afc9e34 100644 --- a/src/oracle_vecdb/vecdb_exception.py +++ b/src/oracle_vecdb/vecdb_exception.py @@ -16,8 +16,9 @@ _REDACTED = "" # These names identify values that must not survive in an exception object. -# Keep this list exact (after normalization) so useful fields such as -# ``request_id``, ``connection_id``, and ``query_id`` remain available. +# Keep exact identifiers such as ``request_id``, ``connection_id``, and +# ``query_id`` available while also recognizing sensitive composite names such +# as ``token_value`` inside otherwise safe nested request fields. _SENSITIVE_VALUE_KEYS = { "authorization", "proxy_authorization", @@ -69,15 +70,58 @@ "url", } +_SENSITIVE_KEY_FRAGMENTS = { + "api_key", + "apikey", + "auth", + "authorization", + "cookie", + "credential", + "credentials", + "header", + "headers", + "password", + "passwd", + "private_key", + "pwd", + "secret", + "token", +} + +_SAFE_ANNOTATION_KEYS = { + "application", + "department", + "dimension", + "metric", + "owner", + "tier", + "updated", + "version", +} + +_RAW_KEY_VALUE_PATTERN = re.compile( + r"(?P[\"']?)(?P[A-Za-z][A-Za-z0-9_.-]*)" + r"(?P[\"']?)(?P\s*)" + r"(?P[:=])(?P\s*)" + r"(?:(?P[\"'])(?P[^\"']*)" + r"(?P=value_quote)|(?P[^,\s}\]]+))" +) + def _normalized_key(value: Any) -> str: - return re.sub(r"[^a-z0-9]+", "_", str(value).lower()).strip("_") + text = re.sub(r"([A-Z]+)([A-Z][a-z])", r"\1_\2", str(value)) + text = re.sub(r"([a-z0-9])([A-Z])", r"\1_\2", text) + return re.sub(r"[^a-z0-9]+", "_", text.lower()).strip("_") def _is_sensitive_key(value: Any) -> bool: normalized = _normalized_key(value) - return normalized in _SENSITIVE_VALUE_KEYS or normalized.endswith( - ("_token", "_secret", "_password", "_credential", "_api_key") + return normalized in _SENSITIVE_VALUE_KEYS or any( + normalized == fragment + or normalized.startswith(f"{fragment}_") + or normalized.endswith(f"_{fragment}") + or f"_{fragment}_" in normalized + for fragment in _SENSITIVE_KEY_FRAGMENTS ) @@ -101,6 +145,10 @@ def __str__(self) -> str: class VecDBException(Exception): """Base exception with normalized service error details.""" + # Keep the public exception concise by default. Integration tests can + # enable this when the complete ORDSErrorResponse is needed for triage. + include_full_ords_response = False + def __init__( self, status: Any = None, @@ -307,6 +355,13 @@ def _sanitize_arguments(arguments: Any) -> Any: "vector_index_params", "debug_flags", } + normalized_safe_keys = { + _normalized_key(safe_key) for safe_key in safe_keys + } + normalized_safe_annotation_keys = { + _normalized_key(annotation_key) + for annotation_key in _SAFE_ANNOTATION_KEYS + } def sanitize( value: Any, key: str = "", safe_context: bool = False @@ -314,10 +369,24 @@ def sanitize( key_normalized = _normalized_key(key) if _is_sensitive_key(key): return _REDACTED - is_safe_key = safe_context or key_normalized in { - _normalized_key(safe_key) for safe_key in safe_keys - } - if isinstance(value, dict): + is_safe_key = safe_context or key_normalized in normalized_safe_keys + if key_normalized == "annotations": + # Do not propagate safe_context into arbitrary annotation extensions. + if value is None: + return None + if not isinstance(value, Mapping): + return _REDACTED + return { + item_key: ( + VecDBException._redact_value(item_value, str(item_key)) + if isinstance(item_value, (str, int, float, bool)) + else _REDACTED + ) + for item_key, item_value in value.items() + if _normalized_key(item_key) + in normalized_safe_annotation_keys + } + if isinstance(value, Mapping): if not is_safe_key and key_normalized not in { "", "args", @@ -451,10 +520,26 @@ def _sync_exception_fields(self) -> None: def vecdb_message(self) -> str: return f"VECDB-{getattr(self, 'error_code', 'HTTP-unknown')}: {getattr(self, 'error_message', 'Request failed')}" - def format(self, *, include_trace: bool = False) -> str: - """Format the error; include the traceback only when requested.""" + def format( + self, + *, + include_trace: bool = False, + include_full_ords_response: Optional[bool] = None, + ) -> str: + """Format the error with optional service diagnostics. + + ``include_full_ords_response`` overrides the class-level setting for + one formatting call. When omitted, the class-level setting controls + whether all fields returned by ORDS are rendered. + """ + if include_full_ords_response is None: + include_full_ords_response = bool( + getattr(self, "include_full_ords_response", False) + ) if self.service_error is not None: - original = self._format_service_error() + original = self._format_service_error( + include_full_ords_response=include_full_ords_response + ) message = ( f"\nOperation - {self.operation}\n" f"Request Data/Parameters - {self.arguments}\n" @@ -482,7 +567,9 @@ def format(self, *, include_trace: bool = False) -> str: message += f"\nAction: {self.action}" return self._redact_diagnostic_text(message) - def _format_service_error(self) -> str: + def _format_service_error( + self, *, include_full_ords_response: bool = False + ) -> str: """Render the original ORDS error once, without generated duplication.""" error = self.service_error name = self.service_error_class_name or type(error).__name__ @@ -520,6 +607,13 @@ def _format_service_error(self) -> str: # validation error exposes an incompatible errors() API. pass payload = self._response_payload(body, data) + if include_full_ords_response and isinstance(payload, dict): + return ( + message + + "\n" + + json.dumps(self._redact_value(payload), indent=4, default=str) + ) + response_message = ( payload.get("message") if isinstance(payload, dict) else None ) @@ -605,6 +699,12 @@ def _redact_diagnostic_text(value: Any) -> Any: return value redacted = re.sub(r"(?i)(bearer\s+)[^\s\"',]+", r"\1", value) + redacted = re.sub( + r"(?i)((?:proxy-)?authorization\s*=\s*(?:basic|bearer)\s+)" + r"[^\s\"',]+", + r"\1", + redacted, + ) redacted = re.sub( r"(?i)((?:proxy-)?authorization\s*:\s*(?:basic|bearer)\s+)" r"[^\s\"',]+", @@ -629,7 +729,8 @@ def _redact_diagnostic_text(value: Any) -> Any: ) redacted = re.sub( r"(?i)([?&](?:access[_-]?token|token|secret|signature|" - r"credential|password|x-amz-signature|x-amz-credential)=)" + r"credential|password|api[_-]?key|x-amz-signature|" + r"x-amz-credential)=)" r"[^&#\s\"']+", r"\1", redacted, @@ -642,7 +743,8 @@ def _redact_diagnostic_text(value: Any) -> Any: redacted, ) redacted = re.sub( - r"(?i)([\"']?(?:password|passwd|credential|username)[\"']?" + r"(?i)([\"']?(?:password|passwd|pwd|secret|token|credential|" + r"username|auth|api[-_]?key)[\"']?" r"\s*[:=]\s*[\"']?)[^\s,;\"'}]+", r"\1", redacted, @@ -669,6 +771,40 @@ def _redact_diagnostic_text(value: Any) -> Any: r"\1", redacted, ) + + def redact_raw_key_value(match: re.Match[str]) -> str: + key = match.group("key") + raw_value = ( + match.group("quoted_value") or match.group("bare_value") or "" + ) + if _normalized_key(key) in { + "authorization", + "proxy_authorization", + } and raw_value.lower() in {"basic", "bearer"}: + return match.group(0) + if not _is_sensitive_key(key): + return match.group(0) + value_quote = match.group("value_quote") or "" + return ( + f"{match.group('key_prefix')}{key}{match.group('key_suffix')}" + f"{match.group('before_separator')}{match.group('separator')}" + f"{match.group('after_separator')}{value_quote}" + f"{_REDACTED}{value_quote}" + ) + + redacted = _RAW_KEY_VALUE_PATTERN.sub(redact_raw_key_value, redacted) + # Unstructured transport errors have no field name to guide the + # recursive redactor. Remove opaque values that explicitly identify + # themselves as secrets, while preserving ordinary diagnostics such + # as request IDs, ORA codes, and connection failures. + redacted = re.sub( + r"(?i)\b(?:(?:[a-z0-9.-]+[_-])?(?:secret|password|passwd|" + r"credential|api[-_]?key)[_-][a-z0-9.-]+|" + r"(?:[a-z0-9.-]+[_-])token(?:[_-][a-z0-9.-]+)?|" + r"token[_-][a-z0-9.-]+)\b", + "", + redacted, + ) return redacted def is_original_exception( diff --git a/src/oracle_vecdb/version.py b/src/oracle_vecdb/version.py index d538cfd..aacfcea 100644 --- a/src/oracle_vecdb/version.py +++ b/src/oracle_vecdb/version.py @@ -1,4 +1,4 @@ """Single source of truth for SDK and generated ORDS versions.""" -SDK_VERSION = "1.0.3" +SDK_VERSION = "1.0.4" ORDS_RELEASE_VERSION = "26.2.2" diff --git a/tests/services/test_ords_exceptions.py b/tests/services/test_ords_exceptions.py index cd89a29..05ae633 100644 --- a/tests/services/test_ords_exceptions.py +++ b/tests/services/test_ords_exceptions.py @@ -108,6 +108,70 @@ def test_service_error_uses_captured_original_class_name(): assert '"message": "Other failure"' in str(error) # nosec B101 +def test_service_error_can_render_the_full_ords_response(monkeypatch): + error = VecDBException.from_service_error( + "create_vector_table", + {"kwargs": {"name": "DOCS"}}, + "ORDSService", + ServiceError( + status=500, + reason="Internal Server Error", + body=json.dumps( + { + "code": "InternalServerError", + "message": "Internal Server Error", + "type": "tag:oracle.com,2020:error/InternalServerError", + "instance": "tag:oracle.com,2020:ecid/test", + "diagnosticTrace": "database diagnostic details", + "stackTrace": "database stack details", + } + ), + ), + ) + + monkeypatch.setattr(VecDBException, "include_full_ords_response", True) + + rendered = str(error) + + assert ( + '"type": "tag:oracle.com,2020:error/InternalServerError"' in rendered + ) # nosec B101 + assert ( + '"diagnosticTrace": "database diagnostic details"' in rendered + ) # nosec B101 + assert '"stackTrace": "database stack details"' in rendered # nosec B101 + + +def test_full_ords_response_format_override_is_per_call(monkeypatch): + error = VecDBException.from_service_error( + "query", + {}, + "ORDSService", + ServiceError( + status=500, + body=json.dumps( + { + "code": "InternalServerError", + "message": "Internal Server Error", + "type": "tag:oracle.com,2020:error/InternalServerError", + "instance": "tag:oracle.com,2020:ecid/test", + } + ), + ), + ) + monkeypatch.setattr(VecDBException, "include_full_ords_response", True) + + concise = error.format(include_full_ords_response=False) + full = error.format(include_full_ords_response=True) + + assert ( + '"type": "tag:oracle.com,2020:error/InternalServerError"' not in concise + ) # nosec B101 + assert ( + '"type": "tag:oracle.com,2020:error/InternalServerError"' in full + ) # nosec B101 + + @pytest.mark.parametrize( "canary", [ @@ -257,6 +321,29 @@ def test_service_error_redacts_search_text_and_renders_safe_arguments(): assert "ANNOTATION_SECRET" not in rendered # nosec B101 +def test_service_error_redacts_composite_sensitive_keys_in_safe_arguments(): + error = VecDBException.from_service_error( + "create_vector_table", + { + "kwargs": { + "annotations": { + "token_value": "ANNOTATION_TOKEN_SECRET", # nosec B105 + "api_secret_value": "ANNOTATION_API_SECRET", # nosec B105 + "apiSecretValue": "ANNOTATION_CAMEL_SECRET", # nosec B105 + } + } + }, + "ORDSService", + ServiceError(status=500, reason="Internal Server Error"), + ) + + rendered = str(error) + assert "ANNOTATION_TOKEN_SECRET" not in rendered # nosec B101 + assert "ANNOTATION_API_SECRET" not in rendered # nosec B101 + assert "ANNOTATION_CAMEL_SECRET" not in rendered # nosec B101 + assert error.arguments["kwargs"]["annotations"] == {} # nosec B101 + + def test_generic_vecdb_exception_can_be_reused_by_another_service(): error = VecDBException( status=422, @@ -334,6 +421,71 @@ def model_dump(self): VecDBException._redact_value(BrokenSerializer()) +@pytest.mark.parametrize( + "key", + [ + "accessToken", + "access_token", + "access-token", + "ACCESS.TOKEN", + "JSONToken", + ], +) +def test_exception_redaction_normalizes_sensitive_key_variants(key): + value = "TOP_SECRET_VALUE" # nosec B105 + + sanitized = VecDBException._redact_value({key: value}) + + assert sanitized == {key: ""} # nosec B101 + + +@pytest.mark.parametrize( + ("diagnostic", "secret"), + [ + ('{"refreshToken":"REFRESH_SECRET"}', "REFRESH_SECRET"), + ("clientSecret=CLIENT_SECRET", "CLIENT_SECRET"), + ("privateKey: PRIVATE_KEY_SECRET", "PRIVATE_KEY_SECRET"), + ("databaseName=DATABASE_SECRET", "DATABASE_SECRET"), + ("connectionString: CONNECTION_SECRET", "CONNECTION_SECRET"), + ("authorizationToken=AUTHORIZATION_SECRET", "AUTHORIZATION_SECRET"), + ], +) +def test_exception_redaction_handles_json_and_raw_sensitive_key_forms( + diagnostic, secret +): + sanitized = VecDBException._redact_value(diagnostic) + + assert secret not in sanitized # nosec B101 + assert "" in sanitized # nosec B101 + + +def test_service_error_does_not_retain_unknown_nested_annotation_values(): + error = VecDBException.from_service_error( + "create_vector_table", + { + "kwargs": { + "annotations": { + "tier": "gold", + "credentialValue": "ANNOTATION_SECRET", # nosec B105 + "customExtension": { + "nestedValue": "NESTED_ANNOTATION_SECRET" # nosec B105 + }, + } + } + }, + "ORDSService", + ServiceError(status=500, reason="Internal Server Error"), + ) + + annotations = error.arguments["kwargs"]["annotations"] + rendered = f"{error}\n{error!r}\n{vars(error)!r}" + + assert annotations == {"tier": "gold"} # nosec B101 + assert "ANNOTATION_SECRET" not in rendered # nosec B101 + assert "NESTED_ANNOTATION_SECRET" not in rendered # nosec B101 + assert "customExtension" not in rendered # nosec B101 + + def test_exception_formats_pydantic_validation_details(): class RequiredModel(BaseModel): value: int @@ -426,6 +578,29 @@ class ProtocolError(Exception): assert "connection reset by peer" in str(error) # nosec B101 +def test_unstructured_error_redacts_marked_secrets_but_preserves_diagnostics(): + class UnstructuredError(Exception): + def __str__(self): + return ( + "backend diagnostic TOP_SECRET_VALUE; " + "request_id=req-123; connection reset by peer" + ) + + error = VecDBException.from_service_error( + "query", {}, "ORDSService", UnstructuredError() + ) + rendered = ( + f"{error}\n{error!r}\n{error.format(include_trace=True)}\n" + f"{vars(error)!r}" + ) + + assert "TOP_SECRET_VALUE" not in rendered # nosec B101 + assert "backend diagnostic" in rendered # nosec B101 + assert "request_id=req-123" in rendered # nosec B101 + assert "connection reset by peer" in rendered # nosec B101 + assert "UnstructuredError" in rendered # nosec B101 + + def test_nested_harness_context_does_not_replace_not_found_message(): error = VecDBException.from_service_error( "drop_vector_table", @@ -598,3 +773,34 @@ def test_redaction_preserves_actionable_service_diagnostics(): assert "customer_database" not in repr(vars(error)) # nosec B101 assert "request-123" in repr(error.data) # nosec B101 assert error.headers["X-Request-ID"] == "request-123" # nosec B101 + + +def test_redaction_preserves_ordinary_credential_error_diagnostics(): + error = VecDBException.from_service_error( + "describe_vector_database", + {}, + "ORDSService", + ServiceError( + status=574, + body=json.dumps( + { + "code": "DatabaseCredentialError", + "title": "Database Credential Error", + "message": ( + "ORDS was unable to make a connection to the database. " + "The username or password of the database user is invalid. " + "ORA-01017: invalid username or not authorized; logon denied" + ), + "type": "tag:oracle.com,2020:error/DatabaseCredentialError", + "instance": "ecid-credential-diagnostic", + } + ), + ), + ) + + rendered = error.format(include_full_ords_response=True) + assert '"title": "Database Credential Error"' in rendered # nosec B101 + assert "username or password of the database user" in rendered # nosec B101 + assert ( + "ORA-01017: invalid username or not authorized" in rendered + ) # nosec B101