Feat: Add Apache Doris engine adapter with OLAP and Iceberg multi-catalog support

.github/scripts/install-prerequisites.sh

@@ -28,6 +28,8 @@ elif [ "$ENGINE" == "fabric" ]; then
     rm packages-microsoft-prod.deb
 
     ENGINE_DEPENDENCIES="msodbcsql18"
+elif [ "$ENGINE" == "doris" ]; then
+    sudo sysctl -w vm.max_map_count=2000000
 fi
 
 ALL_DEPENDENCIES="$COMMON_DEPENDENCIES $ENGINE_DEPENDENCIES"

.github/scripts/wait-for-db.sh

@@ -104,6 +104,46 @@ db2_ready() {
     done
 }
 
+doris_ready() {
+    probe_port 9030
+
+    echo "Waiting for Doris FE to accept SQL connections..."
+    while true; do
+        if docker exec -i doris-fe mysql -h127.0.0.1 -P9030 -uroot -e "SHOW FRONTENDS\G" 2>/dev/null | grep -q "Alive: true"; then
+            echo "Doris FE is alive"
+            break
+        fi
+        sleep 2
+    done
+
+    docker exec -i doris-fe mysql -h127.0.0.1 -P9030 -uroot -e "ALTER SYSTEM ADD BACKEND '172.30.80.3:9050';" 2>/dev/null || true
+
+    echo "Checking for 1 alive Doris backends..."
+    while true; do
+        echo "Checking Doris backends..."
+        ALIVE_BACKENDS=$(docker exec -i doris-fe mysql -h127.0.0.1 -P9030 -uroot -e "show backends \G" 2>/dev/null | grep -c "^ *Alive: true *$" || true)
+
+        # fallback value if failed to get number
+        if ! [[ "$ALIVE_BACKENDS" =~ ^[0-9]+$ ]]; then
+            echo "WARN: Unable to parse number of alive backends, got: '$ALIVE_BACKENDS'"
+            ALIVE_BACKENDS=0
+        fi
+
+        echo "Found $ALIVE_BACKENDS alive backends"
+
+        if [ "$ALIVE_BACKENDS" -ge 1 ]; then
+            echo "Doris has 1 or more alive backends"
+            break
+        fi
+
+        echo "Waiting for more backends to become alive..."
+        sleep 3
+    done
+
+    # set default replication num to 1 (there is only one be in the docker compose file)
+    docker exec -i doris-fe mysql -h127.0.0.1 -P9030 -uroot -e "ADMIN SET FRONTEND CONFIG ('force_olap_table_replication_num' = '1');"
+}
+
 echo "Waiting for $ENGINE to be ready..."
 
 READINESS_FUNC="${ENGINE}_ready"

.github/workflows/pr.yaml

@@ -269,6 +269,7 @@ jobs:
             clickhouse,
             risingwave,
             starrocks,
+            doris,
           ]
     env:
       PYTEST_XDIST_AUTO_NUM_WORKERS: 2

.readthedocs.yaml

@@ -6,7 +6,7 @@ build:
     python: "3.10"
   jobs:
     pre_build:
-      - pip install -e ".[athena,azuresql,bigframes,bigquery,clickhouse,databricks,dbt,dlt,gcppostgres,github,llm,mssql,mysql,mwaa,postgres,redshift,slack,snowflake,starrocks,trino,web,risingwave]"
+      - pip install -e ".[athena,azuresql,bigframes,bigquery,clickhouse,databricks,dbt,dlt,doris,gcppostgres,github,llm,mssql,mysql,mwaa,postgres,redshift,slack,snowflake,starrocks,trino,web,risingwave]"
       - make api-docs
 
 mkdocs:

Makefile

@@ -226,6 +226,9 @@ starrocks-test: engine-starrocks-up
 db2-test: engine-db2-up
 	pytest -n auto -m "db2" --reruns 3 --junitxml=test-results/junit-db2.xml
 
+doris-test: engine-doris-up
+	pytest -n auto -m "doris" --reruns 3 --junitxml=test-results/junit-doris.xml
+
 #################
 # Cloud Engines #
 #################

docs/concepts/models/overview.md

@@ -181,6 +181,7 @@ This table lists each engine's support for `TABLE` and `VIEW` object comments:
 | BigQuery      | Y                | Y               |
 | ClickHouse    | Y                | Y               |
 | Databricks    | Y                | Y               |
+| Doris         | Y                | Y               |
 | DuckDB <=0.9  | N                | N               |
 | DuckDB >=0.10 | Y                | Y               |
 | MySQL         | Y                | Y               |

docs/guides/configuration.md

@@ -933,6 +933,7 @@ These pages describe the connection configuration options for each execution eng
 * [BigQuery](../integrations/engines/bigquery.md)
 * [ClickHouse](../integrations/engines/clickhouse.md)
 * [Databricks](../integrations/engines/databricks.md)
+* [Doris](../integrations/engines/doris.md)
 * [DuckDB](../integrations/engines/duckdb.md)
 * [Fabric](../integrations/engines/fabric.md)
 * [MotherDuck](../integrations/engines/motherduck.md)
@@ -975,6 +976,7 @@ Other state engines with fast and reliable database transactions (less tested th
 Unsupported state engines, even for development:
 
 * [ClickHouse](../integrations/engines/clickhouse.md)
+* [Doris](../integrations/engines/doris.md)
 * [Spark](../integrations/engines/spark.md)
 * [StarRocks](../integrations/engines/starrocks.md)
 * [Trino](../integrations/engines/trino.md)

docs/guides/connections.md

@@ -85,6 +85,7 @@ default_gateway: local_db
 * [ClickHouse](../integrations/engines/clickhouse.md)
 * [Databricks](../integrations/engines/databricks.md)
 * [Db2](../integrations/engines/db2.md)
+* [Doris](../integrations/engines/doris.md)
 * [DuckDB](../integrations/engines/duckdb.md)
 * [Fabric](../integrations/engines/fabric.md)
 * [MotherDuck](../integrations/engines/motherduck.md)

docs/installation.md

@@ -54,6 +54,7 @@ Other extras are required to use specific SQL engines, like Bigquery or Postgres
     | Bigquery      | `pip install "sqlmesh[bigquery]"`    |
     | ClickHouse    | `pip install "sqlmesh[clickhouse]"`  |
     | Databricks    | `pip install "sqlmesh[databricks]"`  |
+    | Doris         | `pip install "sqlmesh[doris]"`       |
     | Fabric        | `pip install "sqlmesh[fabric]"`      |
     | GCP Postgres  | `pip install "sqlmesh[gcppostgres]"` |
     | MS SQL Server | `pip install "sqlmesh[mssql]"`       |