Feat: Add Apache Doris engine adapter with OLAP and Iceberg multi-catalog support

Closes #5875

Co-authored-by: xinge.ji <xinge.ji@outlook.com>
Signed-off-by: Bruce Arctor <5032356+brucearctor@users.noreply.github.com>

Author: brucearctor

.github/scripts/install-prerequisites.sh

@@ -28,6 +28,10 @@ elif [ "$ENGINE" == "fabric" ]; then
     rm packages-microsoft-prod.deb
 
     ENGINE_DEPENDENCIES="msodbcsql18"
+elif [ "$ENGINE" == "doris" ]; then
+    ENGINE_DEPENDENCIES="mysql-client"
+    sudo sysctl -w vm.max_map_count=2000000
+    sudo swapoff -a || true
 fi
 
 ALL_DEPENDENCIES="$COMMON_DEPENDENCIES $ENGINE_DEPENDENCIES"

.github/scripts/wait-for-db.sh

@@ -104,6 +104,34 @@ db2_ready() {
     done
 }
 
+doris_ready() {
+    probe_port 9030 || return 1
+    HOSTNAME=${DOCKER_HOSTNAME:-127.0.0.1}
+    if [ "$HOSTNAME" = "localhost" ]; then
+        HOSTNAME="127.0.0.1"
+    fi
+    INIT_SQL="ADMIN SET FRONTEND CONFIG ('default_replication_num' = '1'); ADMIN SET FRONTEND CONFIG ('force_olap_table_replication_num' = '1'); ADMIN SET FRONTEND CONFIG ('enable_unicode_name_support' = 'true');"
+    if command -v mysql > /dev/null 2>&1; then
+        mysql -h "$HOSTNAME" -P 9030 -u root -e "SHOW BACKENDS\G" 2>/dev/null | grep -q "Alive: true" && \
+        mysql -h "$HOSTNAME" -P 9030 -u root -e "$INIT_SQL" 2>/dev/null
+    else
+        python3 -c "
+import pymysql, sys
+conn = pymysql.connect(host='$HOSTNAME', port=9030, user='root', autocommit=True)
+with conn.cursor() as cur:
+    cur.execute('SHOW BACKENDS')
+    rows = cur.fetchall()
+    cols = [d[0] for d in cur.description]
+    alive_idx = cols.index('Alive')
+    if not rows or not any(str(r[alive_idx]).lower() == 'true' for r in rows):
+        sys.exit(1)
+    for cfg in ('default_replication_num', 'force_olap_table_replication_num'):
+        cur.execute(f\"ADMIN SET FRONTEND CONFIG ('{cfg}' = '1')\")
+    cur.execute(\"ADMIN SET FRONTEND CONFIG ('enable_unicode_name_support' = 'true')\")
+" 2>/dev/null
+    fi
+}
+
 echo "Waiting for $ENGINE to be ready..."
 
 READINESS_FUNC="${ENGINE}_ready"

.github/workflows/pr.yaml

@@ -269,6 +269,7 @@ jobs:
             clickhouse,
             risingwave,
             starrocks,
+            doris,
           ]
     env:
       PYTEST_XDIST_AUTO_NUM_WORKERS: 2

.readthedocs.yaml

@@ -6,7 +6,7 @@ build:
     python: "3.10"
   jobs:
     pre_build:
-      - pip install -e ".[athena,azuresql,bigframes,bigquery,clickhouse,databricks,dbt,dlt,gcppostgres,github,llm,mssql,mysql,mwaa,postgres,redshift,slack,snowflake,starrocks,trino,web,risingwave]"
+      - pip install -e ".[athena,azuresql,bigframes,bigquery,clickhouse,databricks,dbt,dlt,doris,gcppostgres,github,llm,mssql,mysql,mwaa,postgres,redshift,slack,snowflake,starrocks,trino,web,risingwave]"
       - make api-docs
 
 mkdocs:

Makefile

@@ -127,9 +127,9 @@ dev-publish: ui-build clean-build publish
 jupyter-example:
 	jupyter lab tests/slows/jupyter/example_outputs.ipynb
 
-engine-up: engine-clickhouse-up engine-mssql-up engine-mysql-up engine-postgres-up engine-spark-up engine-trino-up
+engine-up: engine-clickhouse-up engine-doris-up engine-mssql-up engine-mysql-up engine-postgres-up engine-spark-up engine-trino-up
 
-engine-down: engine-clickhouse-down engine-mssql-down engine-mysql-down engine-postgres-down engine-spark-down engine-trino-down
+engine-down: engine-clickhouse-down engine-doris-down engine-mssql-down engine-mysql-down engine-postgres-down engine-spark-down engine-trino-down
 
 fast-test:
 	pytest -n auto -m "fast and not cicdonly and not isolated" --junitxml=test-results/junit-fast-test.xml && pytest -m "isolated and not slow and not fork_isolated" && pytest tests/test_forking.py::test_parallel_load -m "isolated and fork_isolated and not slow" && pytest tests/test_forking.py::test_parallel_load_multi_repo -m "isolated and fork_isolated and not slow" && pytest -m "registry_isolation" && pytest -m "dialect_isolated"
@@ -226,6 +226,9 @@ starrocks-test: engine-starrocks-up
 db2-test: engine-db2-up
 	pytest -n auto -m "db2" --reruns 3 --junitxml=test-results/junit-db2.xml
 
+doris-test: engine-doris-up
+	pytest -n auto -m "doris" --reruns 3 --junitxml=test-results/junit-doris.xml
+
 #################
 # Cloud Engines #
 #################

docs/guides/configuration.md

@@ -933,6 +933,7 @@ These pages describe the connection configuration options for each execution eng
 * [BigQuery](../integrations/engines/bigquery.md)
 * [ClickHouse](../integrations/engines/clickhouse.md)
 * [Databricks](../integrations/engines/databricks.md)
+* [Doris](../integrations/engines/doris.md)
 * [DuckDB](../integrations/engines/duckdb.md)
 * [Fabric](../integrations/engines/fabric.md)
 * [MotherDuck](../integrations/engines/motherduck.md)
@@ -978,6 +979,7 @@ Unsupported state engines, even for development:
 * [Spark](../integrations/engines/spark.md)
 * [StarRocks](../integrations/engines/starrocks.md)
 * [Trino](../integrations/engines/trino.md)
+* [Doris](../integrations/engines/doris.md)
 
 This example gateway configuration uses Snowflake for the data warehouse connection and Postgres for the state backend connection:
 

docs/guides/connections.md

@@ -85,6 +85,7 @@ default_gateway: local_db
 * [ClickHouse](../integrations/engines/clickhouse.md)
 * [Databricks](../integrations/engines/databricks.md)
 * [Db2](../integrations/engines/db2.md)
+* [Doris](../integrations/engines/doris.md)
 * [DuckDB](../integrations/engines/duckdb.md)
 * [Fabric](../integrations/engines/fabric.md)
 * [MotherDuck](../integrations/engines/motherduck.md)

docs/installation.md

@@ -54,6 +54,7 @@ Other extras are required to use specific SQL engines, like Bigquery or Postgres
     | Bigquery      | `pip install "sqlmesh[bigquery]"`    |
     | ClickHouse    | `pip install "sqlmesh[clickhouse]"`  |
     | Databricks    | `pip install "sqlmesh[databricks]"`  |
+    | Doris         | `pip install "sqlmesh[doris]"`       |
     | Fabric        | `pip install "sqlmesh[fabric]"`      |
     | GCP Postgres  | `pip install "sqlmesh[gcppostgres]"` |
     | MS SQL Server | `pip install "sqlmesh[mssql]"`       |