From 5391ab5c26b67bff456992cf6b6d7267e05ac931 Mon Sep 17 00:00:00 2001 From: messere1 Date: Fri, 4 Sep 2026 11:35:46 +0800 Subject: [PATCH 1/2] docs: add pg_tokenizer ecosystem integration --- CN/modules/ROOT/nav.adoc | 3 +- .../ecosystem_overview.adoc | 1 + .../ecosystem_components/pg_tokenizer.adoc | 154 ++++++++++++++++++ EN/modules/ROOT/nav.adoc | 1 + .../ecosystem_overview.adoc | 1 + .../ecosystem_components/pg_tokenizer.adoc | 154 ++++++++++++++++++ 6 files changed, 313 insertions(+), 1 deletion(-) create mode 100644 CN/modules/ROOT/pages/master/ecosystem_components/pg_tokenizer.adoc create mode 100644 EN/modules/ROOT/pages/master/ecosystem_components/pg_tokenizer.adoc diff --git a/CN/modules/ROOT/nav.adoc b/CN/modules/ROOT/nav.adoc index b62fe4a5..18b9a376 100644 --- a/CN/modules/ROOT/nav.adoc +++ b/CN/modules/ROOT/nav.adoc @@ -75,7 +75,8 @@ *** xref:master/ecosystem_components/pg_readonly.adoc[pg_readonly] *** xref:master/ecosystem_components/zhparser.adoc[zhparser] *** xref:master/ecosystem_components/pgbackrest.adoc[pgBackRest] -*** xref:master/ecosystem_components/set_user.adoc[set_user] +*** xref:master/ecosystem_components/set_user.adoc[set_user] +*** xref:master/ecosystem_components/pg_tokenizer.adoc[pg_tokenizer] * 监控运维 ** xref:master/getting-started/daily_monitoring.adoc[日常监控] ** xref:master/getting-started/daily_maintenance.adoc[日常维护] diff --git a/CN/modules/ROOT/pages/master/ecosystem_components/ecosystem_overview.adoc b/CN/modules/ROOT/pages/master/ecosystem_components/ecosystem_overview.adoc index 074ce67c..68cf8857 100644 --- a/CN/modules/ROOT/pages/master/ecosystem_components/ecosystem_overview.adoc +++ b/CN/modules/ROOT/pages/master/ecosystem_components/ecosystem_overview.adoc @@ -41,6 +41,7 @@ IvorySQL 作为一款兼容 Oracle 且基于 PostgreSQL 的高级开源数据库 | 28 | xref:master/ecosystem_components/zhparser.adoc[zhparser] | master branch | 用于中文全文搜索的PostgreSQL插件，基于SCWS（即：简易中文分词系统）实现了一个中文解析器 | 搜索引擎、关键字提取 | 29 | xref:master/ecosystem_components/pgbackrest.adoc[pgBackRest] | 2.58.0 | 可靠的 PostgreSQL 备份和恢复解决方案 | 容灾备份、大库备份、异地/多层容灾 | 30 | xref:master/ecosystem_components/set_user.adoc[set_user] | REL4_2_0 | PostgreSQL 安全审计扩展，可控角色切换，支持白名单、强制审计、拦截高危操作 | 可控角色切换、权限管理、审计日志 +| 31 | xref:master/ecosystem_components/pg_tokenizer.adoc[pg_tokenizer] | 0.1.1 | 提供可配置的数据库内文本分析器和分词器，支持多语言处理和触发器自动生成词元 | 全文搜索、RAG 预处理、多语言分词 |==== 这些插件均经过 IvorySQL 团队的测试和适配，确保在 IvorySQL 环境下稳定运行。用户可以根据业务需求选择合适的插件，进一步提升数据库系统的能力和灵活性。 diff --git a/CN/modules/ROOT/pages/master/ecosystem_components/pg_tokenizer.adoc b/CN/modules/ROOT/pages/master/ecosystem_components/pg_tokenizer.adoc new file mode 100644 index 00000000..b7f77a74 --- /dev/null +++ b/CN/modules/ROOT/pages/master/ecosystem_components/pg_tokenizer.adoc @@ -0,0 +1,154 @@ +:sectnums: +:sectnumlevels: 5 + += pg_tokenizer + +== 概述 + +pg_tokenizer 是基于 Rust 开发的 PostgreSQL 扩展，在数据库内提供可配置的分词器和文本分析器。处理流水线可以组合预分词器、字符过滤器、词元过滤器和模型，还可通过表触发器自动生成词元数组。 + +项目地址： + +测试版本：0.1.1 + +许可证：Apache-2.0 + +== 兼容性 + +pg_tokenizer 0.1.1 已在官方 IvorySQL 5.4 x86_64 Linux 镜像（PostgreSQL 18.4）中完成源码编译和验证。测试覆盖中文 Jieba 分词、英文词元处理、自定义模型触发器、扩展生命周期，以及 IvorySQL 两种兼容模式。 + +== 编译和安装 + +安装 Rust、Clang、CMake 和 IvorySQL 开发文件。pg_tokenizer 0.1.1 使用 pgrx 0.16.1。 + +[source,bash] +---- +git clone https://github.com/tensorchord/pg_tokenizer.git +cd pg_tokenizer +git checkout 0.1.1 + +rustup default stable +cargo install cargo-pgrx --version 0.16.1 --locked +cargo pgrx init --pg18=/usr/ivory-5/bin/pg_config + +cargo build --lib --release --features pg18 +cargo pgrx schema --features pg18 --out target/schema.sql +sed 's/@CARGO_VERSION@/0.1.1/g' pg_tokenizer.control \ + > target/pg_tokenizer.control + +sudo install -m 755 target/release/libpg_tokenizer.so \ + "$(/usr/ivory-5/bin/pg_config --pkglibdir)/pg_tokenizer.so" +sudo install -m 644 target/pg_tokenizer.control \ + "$(/usr/ivory-5/bin/pg_config --sharedir)/extension/pg_tokenizer.control" +sudo install -m 644 target/schema.sql \ + "$(/usr/ivory-5/bin/pg_config --sharedir)/extension/pg_tokenizer--0.1.1.sql" +---- + +[NOTE] +0.1.1 标签的控制文件仍包含包版本占位符。请按上面的命令替换，使 `CREATE EXTENSION` 正确安装 0.1.1 版本。 + +在已有预加载列表中加入 pg_tokenizer，然后重启 IvorySQL： + +[source,ini] +---- +shared_preload_libraries = 'gb18030_2022,liboracle_parser,ivorysql_ora,pg_tokenizer' +---- + +[source,sql] +---- +CREATE EXTENSION pg_tokenizer CASCADE; +SELECT extversion FROM pg_extension WHERE extname = 'pg_tokenizer'; +---- + +== 文本分析 + +创建 Jieba 搜索模式分析器并处理中文文本： + +[source,sql] +---- +SET search_path TO "$user", public, tokenizer_catalog; + +SELECT create_text_analyzer('jieba_search', $$ +[pre_tokenizer.jieba] +mode = "search" +$$); + +SELECT apply_text_analyzer('我们中出了一个叛徒', 'jieba_search'); +-- {我们,中出,了,一个,叛徒} +---- + +分词器可以组合模型和过滤器： + +[source,sql] +---- +SELECT create_tokenizer('english_stem', $$ +model = "bert_base_uncased" +pre_tokenizer.regex = '(?u)\b\w\w+\b' +[[character_filters]] +to_lowercase = {} +[[token_filters]] +stopwords = "nltk_english" +[[token_filters]] +stemmer = "english_porter2" +$$); + +SELECT tokenize( + 'PostgreSQL is a powerful open-source database system.', + 'english_stem' +); +---- + +== 使用触发器自动分词 + +[source,sql] +---- +CREATE TABLE tokenizer_documents ( + id integer GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + passage text NOT NULL, + embedding integer[] +); + +SELECT create_custom_model_tokenizer_and_trigger( + tokenizer_name => 'document_tokenizer', + model_name => 'document_model', + text_analyzer_name => 'jieba_search', + table_name => 'tokenizer_documents', + source_column => 'passage', + target_column => 'embedding' +); + +INSERT INTO tokenizer_documents (passage) +VALUES ('IvorySQL 支持 PostgreSQL 和 Oracle 双模式。'); + +SELECT id, cardinality(embedding) > 0 AS tokens_created +FROM tokenizer_documents; +---- + +== Oracle 兼容模式 + +文本分析器和分词器可以直接在 Oracle 模式中使用。自定义模型触发器生成的 SQL 会访问小写内部目录对象，因此需使用普通标识符折叠规则： + +[source,sql] +---- +SET ivorysql.compatible_mode = oracle; +SET ivorysql.identifier_case_switch = normal; + +SELECT 1 FROM dual; +SELECT apply_text_analyzer('南京市长江大桥', 'jieba_search'); + +INSERT INTO tokenizer_documents (passage) +VALUES ('中文分词在 Oracle 兼容模式下正常工作。'); + +SELECT id, cardinality(embedding) > 0 AS tokens_created +FROM tokenizer_documents ORDER BY id; +---- + +[IMPORTANT] +使用 Oracle 模式默认的 `ivorysql.identifier_case_switch = interchange` 时，自定义模型触发器可能报错：`relation "tokenizer_catalog.MODEL_DOCUMENT_MODEL" does not exist`。创建或调用这类触发器的会话应把该参数设为 `normal`。 + +== 运维说明 + +* 必须把 pg_tokenizer 加入 `shared_preload_libraries`，修改后需要重启数据库。 +* 多语言文本建议使用 UTF-8 数据库编码。 +* 源码构建会下载并编译 Rust 依赖，耗时和内存占用高于普通 PGXS 扩展。 +* 上线前应使用有代表性的业务文本验证具体的分析器和模型配置。 diff --git a/EN/modules/ROOT/nav.adoc b/EN/modules/ROOT/nav.adoc index e4221c78..c464789c 100644 --- a/EN/modules/ROOT/nav.adoc +++ b/EN/modules/ROOT/nav.adoc @@ -76,6 +76,7 @@ *** xref:master/ecosystem_components/zhparser_en.adoc[zhparser] *** xref:master/ecosystem_components/pgbackrest.adoc[pgBackRest] *** xref:master/ecosystem_components/set_user.adoc[set_user] +*** xref:master/ecosystem_components/pg_tokenizer.adoc[pg_tokenizer] * Monitor and O&M ** xref:master/getting-started/daily_monitoring.adoc[Monitoring] ** xref:master/getting-started/daily_maintenance.adoc[Maintenance] diff --git a/EN/modules/ROOT/pages/master/ecosystem_components/ecosystem_overview.adoc b/EN/modules/ROOT/pages/master/ecosystem_components/ecosystem_overview.adoc index 4e3270c9..c95cb6b5 100644 --- a/EN/modules/ROOT/pages/master/ecosystem_components/ecosystem_overview.adoc +++ b/EN/modules/ROOT/pages/master/ecosystem_components/ecosystem_overview.adoc @@ -42,6 +42,7 @@ IvorySQL, as an advanced open-source database compatible with Oracle and based o |*28*| xref:master/ecosystem_components/zhparser_en.adoc[zhparser] | master branch | PostgreSQL extension for full-text search of Chinese language (Mandarin Chinese). It implements a Chinese language parser base on the | Search engine、keyword extraction |*29*| xref:master/ecosystem_components/pgbackrest.adoc[pgBackRest] | 2.58.0 | pgBackRest is a reliable backup and restore solution for PostgreSQL that seamlessly scales up to the largest databases and workloads | Disaster recovery backup, large database backup, off-site/multi-tier disaster recovery | *30* | xref:master/ecosystem_components/set_user.adoc[set_user] | REL4_2_0 | PostgreSQL security auditing extension with controlled role switching, supporting allowlists, enforced auditing, and blocking of high-risk operations | Controlled role switching, privilege management, audit logging +| *31* | xref:master/ecosystem_components/pg_tokenizer.adoc[pg_tokenizer] | 0.1.1 | Configurable in-database text analyzers and tokenizers with multilingual processing and trigger-based token generation | Full-text search, RAG preprocessing, multilingual tokenization |==== These plugins have all been tested and adapted by the IvorySQL team to ensure stable operation in the IvorySQL environment. Users can select appropriate plugins based on business needs to further enhance the capabilities and flexibility of the database system. diff --git a/EN/modules/ROOT/pages/master/ecosystem_components/pg_tokenizer.adoc b/EN/modules/ROOT/pages/master/ecosystem_components/pg_tokenizer.adoc new file mode 100644 index 00000000..d02856dd --- /dev/null +++ b/EN/modules/ROOT/pages/master/ecosystem_components/pg_tokenizer.adoc @@ -0,0 +1,154 @@ +:sectnums: +:sectnumlevels: 5 + += pg_tokenizer + +== Overview + +pg_tokenizer is a Rust-based PostgreSQL extension that provides configurable tokenizers and text analyzers inside the database. Pipelines can combine pre-tokenizers, character filters, token filters, and models, and can populate token arrays automatically with table triggers. + +Project page: + +Tested version: 0.1.1 + +License: Apache-2.0 + +== Compatibility + +pg_tokenizer 0.1.1 was built from source and validated on the official IvorySQL 5.4 x86_64 Linux image (PostgreSQL 18.4). The tests covered Chinese Jieba analysis, English tokenization, custom-model triggers, extension lifecycle, and both IvorySQL compatibility modes. + +== Building and Installing + +Install Rust, Clang, CMake, and the IvorySQL development files. pg_tokenizer 0.1.1 uses pgrx 0.16.1. + +[source,bash] +---- +git clone https://github.com/tensorchord/pg_tokenizer.git +cd pg_tokenizer +git checkout 0.1.1 + +rustup default stable +cargo install cargo-pgrx --version 0.16.1 --locked +cargo pgrx init --pg18=/usr/ivory-5/bin/pg_config + +cargo build --lib --release --features pg18 +cargo pgrx schema --features pg18 --out target/schema.sql +sed 's/@CARGO_VERSION@/0.1.1/g' pg_tokenizer.control \ + > target/pg_tokenizer.control + +sudo install -m 755 target/release/libpg_tokenizer.so \ + "$(/usr/ivory-5/bin/pg_config --pkglibdir)/pg_tokenizer.so" +sudo install -m 644 target/pg_tokenizer.control \ + "$(/usr/ivory-5/bin/pg_config --sharedir)/extension/pg_tokenizer.control" +sudo install -m 644 target/schema.sql \ + "$(/usr/ivory-5/bin/pg_config --sharedir)/extension/pg_tokenizer--0.1.1.sql" +---- + +[NOTE] +Tag 0.1.1 retains a package-version placeholder in its control file. Replace it as shown above so that `CREATE EXTENSION` installs version 0.1.1 correctly. + +Add pg_tokenizer to the existing preload list and restart IvorySQL: + +[source,ini] +---- +shared_preload_libraries = 'gb18030_2022,liboracle_parser,ivorysql_ora,pg_tokenizer' +---- + +[source,sql] +---- +CREATE EXTENSION pg_tokenizer CASCADE; +SELECT extversion FROM pg_extension WHERE extname = 'pg_tokenizer'; +---- + +== Text Analysis + +Create a Jieba search-mode analyzer and apply it to Chinese text: + +[source,sql] +---- +SET search_path TO "$user", public, tokenizer_catalog; + +SELECT create_text_analyzer('jieba_search', $$ +[pre_tokenizer.jieba] +mode = "search" +$$); + +SELECT apply_text_analyzer('我们中出了一个叛徒', 'jieba_search'); +-- {我们,中出,了,一个,叛徒} +---- + +Tokenizers can combine a model and filters: + +[source,sql] +---- +SELECT create_tokenizer('english_stem', $$ +model = "bert_base_uncased" +pre_tokenizer.regex = '(?u)\b\w\w+\b' +[[character_filters]] +to_lowercase = {} +[[token_filters]] +stopwords = "nltk_english" +[[token_filters]] +stemmer = "english_porter2" +$$); + +SELECT tokenize( + 'PostgreSQL is a powerful open-source database system.', + 'english_stem' +); +---- + +== Automatic Tokenization with a Trigger + +[source,sql] +---- +CREATE TABLE tokenizer_documents ( + id integer GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + passage text NOT NULL, + embedding integer[] +); + +SELECT create_custom_model_tokenizer_and_trigger( + tokenizer_name => 'document_tokenizer', + model_name => 'document_model', + text_analyzer_name => 'jieba_search', + table_name => 'tokenizer_documents', + source_column => 'passage', + target_column => 'embedding' +); + +INSERT INTO tokenizer_documents (passage) +VALUES ('IvorySQL 支持 PostgreSQL 和 Oracle 双模式。'); + +SELECT id, cardinality(embedding) > 0 AS tokens_created +FROM tokenizer_documents; +---- + +== Oracle Compatibility Mode + +Text analyzers and tokenizers work directly in Oracle mode. For custom-model triggers, use normal identifier folding because the extension's generated SQL addresses lower-case internal catalog objects: + +[source,sql] +---- +SET ivorysql.compatible_mode = oracle; +SET ivorysql.identifier_case_switch = normal; + +SELECT 1 FROM dual; +SELECT apply_text_analyzer('南京市长江大桥', 'jieba_search'); + +INSERT INTO tokenizer_documents (passage) +VALUES ('中文分词在 Oracle 兼容模式下正常工作。'); + +SELECT id, cardinality(embedding) > 0 AS tokens_created +FROM tokenizer_documents ORDER BY id; +---- + +[IMPORTANT] +With Oracle mode's default `ivorysql.identifier_case_switch = interchange`, a custom-model trigger can fail with `relation "tokenizer_catalog.MODEL_DOCUMENT_MODEL" does not exist`. Set the parameter to `normal` for sessions that create or invoke these triggers. + +== Operational Notes + +* pg_tokenizer must be present in `shared_preload_libraries`; changing the list requires a server restart. +* Use UTF-8 database encoding for multilingual text. +* Building from source downloads and compiles Rust dependencies and therefore takes more time and memory than a typical PGXS extension. +* Test application-specific analyzer and model configurations with representative text before production use. From a733ba9679b27c515447c6e8c831090280ad7f62 Mon Sep 17 00:00:00 2001 From: messere1 Date: Fri, 4 Sep 2026 11:44:15 +0800 Subject: [PATCH 2/2] docs: address pg_tokenizer review comments --- CN/modules/ROOT/nav.adoc | 4 ++-- .../ROOT/pages/master/ecosystem_components/pg_tokenizer.adoc | 2 +- .../pages/master/ecosystem_components/ecosystem_overview.adoc | 2 +- .../ROOT/pages/master/ecosystem_components/pg_tokenizer.adoc | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/CN/modules/ROOT/nav.adoc b/CN/modules/ROOT/nav.adoc index 18b9a376..ee19362d 100644 --- a/CN/modules/ROOT/nav.adoc +++ b/CN/modules/ROOT/nav.adoc @@ -75,8 +75,8 @@ *** xref:master/ecosystem_components/pg_readonly.adoc[pg_readonly] *** xref:master/ecosystem_components/zhparser.adoc[zhparser] *** xref:master/ecosystem_components/pgbackrest.adoc[pgBackRest] -*** xref:master/ecosystem_components/set_user.adoc[set_user] -*** xref:master/ecosystem_components/pg_tokenizer.adoc[pg_tokenizer] +*** xref:master/ecosystem_components/set_user.adoc[set_user] +*** xref:master/ecosystem_components/pg_tokenizer.adoc[pg_tokenizer] * 监控运维 ** xref:master/getting-started/daily_monitoring.adoc[日常监控] ** xref:master/getting-started/daily_maintenance.adoc[日常维护] diff --git a/CN/modules/ROOT/pages/master/ecosystem_components/pg_tokenizer.adoc b/CN/modules/ROOT/pages/master/ecosystem_components/pg_tokenizer.adoc index b7f77a74..de48853e 100644 --- a/CN/modules/ROOT/pages/master/ecosystem_components/pg_tokenizer.adoc +++ b/CN/modules/ROOT/pages/master/ecosystem_components/pg_tokenizer.adoc @@ -51,7 +51,7 @@ sudo install -m 644 target/schema.sql \ [source,ini] ---- -shared_preload_libraries = 'gb18030_2022,liboracle_parser,ivorysql_ora,pg_tokenizer' +shared_preload_libraries = 'gb18030_2022, liboracle_parser, ivorysql_ora, pg_tokenizer' ---- [source,sql] diff --git a/EN/modules/ROOT/pages/master/ecosystem_components/ecosystem_overview.adoc b/EN/modules/ROOT/pages/master/ecosystem_components/ecosystem_overview.adoc index c95cb6b5..9241efea 100644 --- a/EN/modules/ROOT/pages/master/ecosystem_components/ecosystem_overview.adoc +++ b/EN/modules/ROOT/pages/master/ecosystem_components/ecosystem_overview.adoc @@ -42,7 +42,7 @@ IvorySQL, as an advanced open-source database compatible with Oracle and based o |*28*| xref:master/ecosystem_components/zhparser_en.adoc[zhparser] | master branch | PostgreSQL extension for full-text search of Chinese language (Mandarin Chinese). It implements a Chinese language parser base on the | Search engine、keyword extraction |*29*| xref:master/ecosystem_components/pgbackrest.adoc[pgBackRest] | 2.58.0 | pgBackRest is a reliable backup and restore solution for PostgreSQL that seamlessly scales up to the largest databases and workloads | Disaster recovery backup, large database backup, off-site/multi-tier disaster recovery | *30* | xref:master/ecosystem_components/set_user.adoc[set_user] | REL4_2_0 | PostgreSQL security auditing extension with controlled role switching, supporting allowlists, enforced auditing, and blocking of high-risk operations | Controlled role switching, privilege management, audit logging -| *31* | xref:master/ecosystem_components/pg_tokenizer.adoc[pg_tokenizer] | 0.1.1 | Configurable in-database text analyzers and tokenizers with multilingual processing and trigger-based token generation | Full-text search, RAG preprocessing, multilingual tokenization +|*31*| xref:master/ecosystem_components/pg_tokenizer.adoc[pg_tokenizer] | 0.1.1 | Configurable in-database text analyzers and tokenizers with multilingual processing and trigger-based token generation | Full-text search, RAG preprocessing, multilingual tokenization |==== These plugins have all been tested and adapted by the IvorySQL team to ensure stable operation in the IvorySQL environment. Users can select appropriate plugins based on business needs to further enhance the capabilities and flexibility of the database system. diff --git a/EN/modules/ROOT/pages/master/ecosystem_components/pg_tokenizer.adoc b/EN/modules/ROOT/pages/master/ecosystem_components/pg_tokenizer.adoc index d02856dd..2ee10a32 100644 --- a/EN/modules/ROOT/pages/master/ecosystem_components/pg_tokenizer.adoc +++ b/EN/modules/ROOT/pages/master/ecosystem_components/pg_tokenizer.adoc @@ -51,7 +51,7 @@ Add pg_tokenizer to the existing preload list and restart IvorySQL: [source,ini] ---- -shared_preload_libraries = 'gb18030_2022,liboracle_parser,ivorysql_ora,pg_tokenizer' +shared_preload_libraries = 'gb18030_2022, liboracle_parser, ivorysql_ora, pg_tokenizer' ---- [source,sql]