Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 16 additions & 1 deletion api/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,8 @@ This module provides components organized into two main areas aligned with the [

### Unified Language Specification

- **`UnifiedQueryPlanner`**: Accepts PPL (Piped Processing Language) or SQL queries and returns Calcite `RelNode` logical plans as intermediate representation.
- **`UnifiedQueryParser`**: Parses PPL (Piped Processing Language) or SQL queries and returns the native parse result (`UnresolvedPlan` for PPL, `SqlNode` for Calcite SQL).
- **`UnifiedQueryPlanner`**: Accepts PPL or SQL queries and returns Calcite `RelNode` logical plans as intermediate representation.
- **`UnifiedQueryTranspiler`**: Converts Calcite logical plans (`RelNode`) into SQL strings for various target databases using different SQL dialects.

### Unified Execution Runtime
Expand Down Expand Up @@ -42,6 +43,20 @@ UnifiedQueryContext context = UnifiedQueryContext.builder()
.build();
```

### UnifiedQueryParser

Use `UnifiedQueryParser` to parse queries into their native parse tree. The parser is owned by `UnifiedQueryContext` and returns the native parse result for each language.

```java
// PPL parsing
UnresolvedPlan ast = (UnresolvedPlan) context.getParser().parse("source = logs | where status = 200");

// SQL parsing (with QueryType.SQL context)
SqlNode sqlNode = (SqlNode) sqlContext.getParser().parse("SELECT * FROM logs WHERE status = 200");
```

Callers can then use each language's native visitor infrastructure (`AbstractNodeVisitor` for PPL, `SqlBasicVisitor` for Calcite SQL) on the typed result for further analysis.

### UnifiedQueryPlanner

Use `UnifiedQueryPlanner` to parse and analyze PPL or SQL queries into Calcite logical plans. The planner accepts a `UnifiedQueryContext` and can be reused for multiple queries.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,8 @@
import java.util.Objects;
import java.util.Optional;
import java.util.concurrent.Callable;
import lombok.Value;
import lombok.AllArgsConstructor;
import lombok.Getter;
import org.apache.calcite.avatica.util.Casing;
import org.apache.calcite.jdbc.CalciteSchema;
import org.apache.calcite.plan.RelTraitDef;
Expand All @@ -26,6 +27,9 @@
import org.apache.calcite.tools.FrameworkConfig;
import org.apache.calcite.tools.Frameworks;
import org.apache.calcite.tools.Programs;
import org.opensearch.sql.api.parser.CalciteSqlQueryParser;
import org.opensearch.sql.api.parser.PPLQueryParser;
import org.opensearch.sql.api.parser.UnifiedQueryParser;
import org.opensearch.sql.calcite.CalcitePlanContext;
import org.opensearch.sql.calcite.SysLimit;
import org.opensearch.sql.common.setting.Settings;
Expand All @@ -40,14 +44,18 @@
* centralizes configuration for catalog schemas, query type, execution limits, and other settings,
* enabling consistent behavior across all unified query operations.
*/
@Value
@AllArgsConstructor
@Getter
public class UnifiedQueryContext implements AutoCloseable {

/** CalcitePlanContext containing Calcite framework configuration and query type. */
CalcitePlanContext planContext;
private final CalcitePlanContext planContext;

/** Settings containing execution limits and feature flags used by parsers and planners. */
Settings settings;
private final Settings settings;

/** Query parser created eagerly from this context's configuration. */
private final UnifiedQueryParser<?> parser;

/**
* Returns the profiling result. Call after query execution to retrieve collected metrics. Returns
Expand Down Expand Up @@ -202,7 +210,14 @@ public UnifiedQueryContext build() {
CalcitePlanContext.create(
buildFrameworkConfig(), SysLimit.fromSettings(settings), queryType);
QueryProfiling.activate(profiling);
return new UnifiedQueryContext(planContext, settings);
return new UnifiedQueryContext(planContext, settings, createParser(planContext, settings));
}

private UnifiedQueryParser<?> createParser(CalcitePlanContext planContext, Settings settings) {
return switch (queryType) {
case PPL -> new PPLQueryParser(settings);
case SQL -> new CalciteSqlQueryParser(planContext);
};
}

private Settings buildSettings() {
Expand Down
35 changes: 10 additions & 25 deletions api/src/main/java/org/opensearch/sql/api/UnifiedQueryPlanner.java
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@
import static org.opensearch.sql.monitor.profile.MetricName.ANALYZE;

import lombok.RequiredArgsConstructor;
import org.antlr.v4.runtime.tree.ParseTree;
import org.apache.calcite.rel.RelCollation;
import org.apache.calcite.rel.RelCollations;
import org.apache.calcite.rel.RelNode;
Expand All @@ -18,15 +17,11 @@
import org.apache.calcite.sql.SqlNode;
import org.apache.calcite.tools.Frameworks;
import org.apache.calcite.tools.Planner;
import org.opensearch.sql.ast.statement.Query;
import org.opensearch.sql.ast.statement.Statement;
import org.opensearch.sql.api.parser.UnifiedQueryParser;
import org.opensearch.sql.ast.tree.UnresolvedPlan;
import org.opensearch.sql.calcite.CalciteRelNodeVisitor;
import org.opensearch.sql.common.antlr.SyntaxCheckException;
import org.opensearch.sql.executor.QueryType;
import org.opensearch.sql.ppl.antlr.PPLSyntaxParser;
import org.opensearch.sql.ppl.parser.AstBuilder;
import org.opensearch.sql.ppl.parser.AstStatementBuilder;

/**
* {@code UnifiedQueryPlanner} provides a high-level API for parsing and analyzing queries using the
Expand Down Expand Up @@ -93,36 +88,26 @@ public RelNode plan(String query) throws Exception {
}
}

/** AST-based planning via ANTLR parser → UnresolvedPlan → CalciteRelNodeVisitor. */
@RequiredArgsConstructor
/** AST-based planning via context-owned parser → UnresolvedPlan → CalciteRelNodeVisitor. */
private static class CustomVisitorStrategy implements PlanningStrategy {
private final UnifiedQueryContext context;
private final PPLSyntaxParser parser = new PPLSyntaxParser();
private final UnifiedQueryParser<UnresolvedPlan> parser;
private final CalciteRelNodeVisitor relNodeVisitor =
new CalciteRelNodeVisitor(new EmptyDataSourceService());

@SuppressWarnings("unchecked")
CustomVisitorStrategy(UnifiedQueryContext context) {
this.context = context;
this.parser = (UnifiedQueryParser<UnresolvedPlan>) context.getParser();
}

@Override
public RelNode plan(String query) {
UnresolvedPlan ast = parse(query);
UnresolvedPlan ast = parser.parse(query);
RelNode logical = relNodeVisitor.analyze(ast, context.getPlanContext());
return preserveCollation(logical);
}

private UnresolvedPlan parse(String query) {
ParseTree cst = parser.parse(query);
AstStatementBuilder astStmtBuilder =
new AstStatementBuilder(
new AstBuilder(query, context.getSettings()),
AstStatementBuilder.StatementBuilderContext.builder().build());
Statement statement = cst.accept(astStmtBuilder);

if (statement instanceof Query) {
return ((Query) statement).getPlan();
}
throw new UnsupportedOperationException(
"Only query statements are supported but got " + statement.getClass().getSimpleName());
}

private RelNode preserveCollation(RelNode logical) {
RelCollation collation = logical.getTraitSet().getCollation();
if (!(logical instanceof Sort) && collation != RelCollations.EMPTY) {
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
/*
* Copyright OpenSearch Contributors
* SPDX-License-Identifier: Apache-2.0
*/

package org.opensearch.sql.api.parser;

import lombok.RequiredArgsConstructor;
import org.apache.calcite.sql.SqlNode;
import org.apache.calcite.sql.parser.SqlParseException;
import org.apache.calcite.sql.parser.SqlParser;
import org.opensearch.sql.calcite.CalcitePlanContext;
import org.opensearch.sql.common.antlr.SyntaxCheckException;

/** Calcite SQL query parser that produces {@link SqlNode} as the native parse result. */
@RequiredArgsConstructor
public class CalciteSqlQueryParser implements UnifiedQueryParser<SqlNode> {

/** Calcite plan context providing parser configuration (e.g., case sensitivity, conformance). */
private final CalcitePlanContext planContext;

@Override
public SqlNode parse(String query) {
try {
SqlParser parser = SqlParser.create(query, planContext.config.getParserConfig());
return parser.parseQuery();
} catch (SqlParseException e) {
throw new SyntaxCheckException("Failed to parse SQL query: " + e.getMessage());
}
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
/*
* Copyright OpenSearch Contributors
* SPDX-License-Identifier: Apache-2.0
*/

package org.opensearch.sql.api.parser;

import lombok.RequiredArgsConstructor;
import org.antlr.v4.runtime.tree.ParseTree;
import org.opensearch.sql.ast.statement.Query;
import org.opensearch.sql.ast.statement.Statement;
import org.opensearch.sql.ast.tree.UnresolvedPlan;
import org.opensearch.sql.common.setting.Settings;
import org.opensearch.sql.ppl.antlr.PPLSyntaxParser;
import org.opensearch.sql.ppl.parser.AstBuilder;
import org.opensearch.sql.ppl.parser.AstStatementBuilder;

/** PPL query parser that produces {@link UnresolvedPlan} as the native parse result. */
@RequiredArgsConstructor
public class PPLQueryParser implements UnifiedQueryParser<UnresolvedPlan> {

/** Settings containing execution limits and feature flags used by AST builders. */
private final Settings settings;

/** Reusable ANTLR-based PPL syntax parser. Stateless and thread-safe. */
private final PPLSyntaxParser syntaxParser = new PPLSyntaxParser();

@Override
public UnresolvedPlan parse(String query) {
ParseTree cst = syntaxParser.parse(query);
AstStatementBuilder astStmtBuilder =
new AstStatementBuilder(
new AstBuilder(query, settings),
AstStatementBuilder.StatementBuilderContext.builder().build());
Statement statement = cst.accept(astStmtBuilder);

if (statement instanceof Query) {
return ((Query) statement).getPlan();
}
throw new UnsupportedOperationException(
"Only query statements are supported but got " + statement.getClass().getSimpleName());
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
/*
* Copyright OpenSearch Contributors
* SPDX-License-Identifier: Apache-2.0
*/

package org.opensearch.sql.api.parser;

/**
* Language-neutral query parser interface. Returns the native parse result for the language (e.g.,
* {@code UnresolvedPlan} for PPL, {@code SqlNode} for Calcite SQL).
*
* @param <T> the native parse result type for this language
*/
public interface UnifiedQueryParser<T> {

/**
* Parses the query and returns the native parse result.
*
* @param query the raw query string
* @return the native parse result
*/
T parse(String query);
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,115 @@
/*
* Copyright OpenSearch Contributors
* SPDX-License-Identifier: Apache-2.0
*/

package org.opensearch.sql.api.parser;

import org.apache.calcite.sql.parser.SqlParserFixture;
import org.junit.Test;
import org.opensearch.sql.api.UnifiedQueryTestBase;
import org.opensearch.sql.executor.QueryType;

/**
* SQL parser tests using Calcite's {@link SqlParserFixture} for idiomatic parse-unparse assertions.
* Parser config is read from {@link org.opensearch.sql.api.UnifiedQueryContext} to stay in sync
* with production.
*/
public class UnifiedQueryParserSqlTest extends UnifiedQueryTestBase {

@Override
protected QueryType queryType() {
return QueryType.SQL;
}

@Test
public void testParseSelectStar() {
sql("SELECT * FROM catalog.employees")
.ok(
"""
SELECT *
FROM `catalog`.`employees`\
""");
}

@Test
public void testParseSelectColumns() {
sql("SELECT id, name FROM catalog.employees")
.ok(
"""
SELECT `id`, `name`
FROM `catalog`.`employees`\
""");
}

@Test
public void testParseFilter() {
sql("""
SELECT name
FROM catalog.employees
WHERE age > 30\
""")
.ok(
"""
SELECT `name`
FROM `catalog`.`employees`
WHERE (`age` > 30)\
""");
}

@Test
public void testParseAggregate() {
sql("""
SELECT department, count(*) AS cnt
FROM catalog.employees
GROUP BY department\
""")
.ok(
"""
SELECT `department`, COUNT(*) AS `cnt`
FROM `catalog`.`employees`
GROUP BY `department`\
""");
}

@Test
public void testParseOrderBy() {
sql("""
SELECT name
FROM catalog.employees
ORDER BY age DESC\
""")
.ok(
"""
SELECT `name`
FROM `catalog`.`employees`
ORDER BY `age` DESC\
""");
}

@Test
public void testParseJoin() {
sql("""
SELECT a.id, b.name
FROM catalog.employees a
JOIN catalog.employees b ON a.id = b.age\
""")
.ok(
"""
SELECT `a`.`id`, `b`.`name`
FROM `catalog`.`employees` AS `a`
INNER JOIN `catalog`.`employees` AS `b` ON (`a`.`id` = `b`.`age`)\
""");
}

@Test
public void testSyntaxErrorFails() {
sql("SELECT ^FROM^").fails("(?s).*Incorrect syntax near the keyword 'FROM'.*");
}

private SqlParserFixture sql(String sql) {
return SqlParserFixture.DEFAULT
.withConfig(c -> context.getPlanContext().config.getParserConfig())
.sql(sql);
}
}
Loading
Loading