Skip to content

IGNITE-28864 Add technical columns to Calcite engin SQL scan #13343

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
vldpyatkov wants to merge 1 commit into
base: master
Choose a base branch
from
Open

IGNITE-28864 Add technical columns to Calcite engin SQL scan #13343

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .../main/java/org/apache/ignite/internal/processors/query/calcite/prepare/IgnitePlanner.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -365,7 +365,7 @@ private static boolean isAsCall(SqlNode node) {
}

CalciteCatalogReader catalogReader = this.catalogReader.withSchemaPath(schemaPath);
SqlValidator validator = new IgniteSqlValidator(operatorTbl, catalogReader, typeFactory, validatorCfg, ctx.parameters());
IgniteSqlValidator validator = new IgniteSqlValidator(operatorTbl, catalogReader, typeFactory, validatorCfg, ctx.parameters());
SqlToRelConverter sqlToRelConverter = sqlToRelConverter(validator, catalogReader, sqlToRelConverterCfg);
RelRoot root = sqlToRelConverter.convertQuery(sqlNode, true, false);
root = root.withRel(sqlToRelConverter.decorrelate(sqlNode, root.rel));
Expand Down
79 changes: 58 additions & 21 deletions .../java/org/apache/ignite/internal/processors/query/calcite/prepare/IgniteSqlValidator.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -136,16 +136,32 @@ public IgniteSqlValidator(
nullType = typeFactory.createSqlType(SqlTypeName.NULL);
}


/** {@inheritDoc} */
@Override public void validateInsert(SqlInsert insert) {
validateTableModify(insert.getTargetTable());

if (insert.getTargetColumnList() == null)
insert.setOperand(3, inferColumnList(insert));
else
validateInsertTargets(insert);

super.validateInsert(insert);
}

/**
* Validates insert target columns to ensure they do not contain any technical columns.
*
* @param insert Insert statement.
*/
private void validateInsertTargets(SqlInsert insert) {
for (SqlNode node : insert.getTargetColumnList()) {
SqlIdentifier id = (SqlIdentifier)node;

validateTechnicalColumnDmlTarget(id);
}
}

/** {@inheritDoc} */
@Override public void validateUpdate(SqlUpdate call) {
validateTableModify(call.getTargetTable());
Expand Down Expand Up @@ -198,10 +214,19 @@ private void validateTableModify(SqlNode table) {
SqlIdentifier alias = call.getAlias() != null ? call.getAlias() :
new SqlIdentifier(deriveAlias(targetTable, 0), SqlParserPos.ZERO);

table.unwrap(IgniteTable.class).descriptor().selectForUpdateRowType((IgniteTypeFactory)typeFactory)
.getFieldNames().stream()
.map(name -> alias.plus(name, SqlParserPos.ZERO))
.forEach(selectList::add);
RelDataType updateRowType = table.unwrap(IgniteTable.class).descriptor()
.selectForUpdateRowType(typeFactory());

for (RelDataTypeField field : updateRowType.getFieldList()) {
if (QueryUtils.isTechnicalFieldName(field.getName())) {
selectList.add(SqlValidatorUtil.addAlias(
SqlLiteral.createNull(SqlParserPos.ZERO),
"__IGNITE_DML" + field.getName()
));
}
else
selectList.add(alias.plus(field.getName(), SqlParserPos.ZERO));
}

int ordinal = 0;
// Force unique aliases to avoid a duplicate for Y with SET X=Y
Expand Down Expand Up @@ -247,6 +272,7 @@ private void validateTableModify(SqlNode table) {
super.validateSelect(select, targetRowType);
}


/** {@inheritDoc} */
@Override protected void validateNamespace(SqlValidatorNamespace namespace, RelDataType targetRowType) {
SqlValidatorTable table = namespace.getTable();
Expand Down Expand Up @@ -294,7 +320,7 @@ else if (n instanceof SqlDynamicParam) {
if (call.getKind() == SqlKind.AS) {
final String alias = deriveAlias(call, 0);

if (isSystemFieldName(alias))
if (QueryUtils.isReservedFieldName(alias))
throw newValidationError(call, IgniteResource.INSTANCE.illegalAlias(alias));
}
else if (call.getKind() == SqlKind.CAST) {
Expand Down Expand Up @@ -419,18 +445,26 @@ private SqlNode rewriteTableToQuery(SqlNode from) {
SelectScope scope,
boolean includeSysVars
) {
if (!includeSysVars && exp.getKind() == SqlKind.IDENTIFIER && isSystemFieldName(deriveAlias(exp, 0))) {
SqlQualified qualified = scope.fullyQualify((SqlIdentifier)exp);
if (!includeSysVars && exp.getKind() == SqlKind.IDENTIFIER) {
String alias = deriveAlias(exp, 0);

if (qualified.namespace == null)
// Technical columns are never exposed via SELECT *.
if (QueryUtils.isTechnicalFieldNameIgnoreCase(alias))
return;

if (qualified.namespace.getTable() != null) {
// If child is table and has only system fields, expand star to these fields.
// Otherwise, expand star to non-system fields only.
for (RelDataTypeField fld : qualified.namespace.getRowType().getFieldList()) {
if (!isSystemField(fld))
return;
if (QueryUtils.isReservedFieldName(alias)) {
SqlQualified qualified = scope.fullyQualify((SqlIdentifier)exp);

if (qualified.namespace == null)
return;

if (qualified.namespace.getTable() != null) {
// If child is table and has only system fields, expand star to these fields.
// Otherwise, expand star to non-system fields only.
for (RelDataTypeField fld : qualified.namespace.getRowType().getFieldList()) {
if (!isSystemField(fld))
return;
}
}
}
}
Expand All @@ -440,7 +474,7 @@ private SqlNode rewriteTableToQuery(SqlNode from) {

/** {@inheritDoc} */
@Override public boolean isSystemField(RelDataTypeField field) {
return isSystemFieldName(field.getName());
return QueryUtils.isReservedFieldName(field.getName());
}

/** */
Expand Down Expand Up @@ -524,6 +558,7 @@ private void validateUpdateFields(SqlUpdate call) {

for (SqlNode node : call.getTargetColumnList()) {
SqlIdentifier id = (SqlIdentifier)node;
validateTechnicalColumnDmlTarget(id);

RelDataTypeField target = SqlValidatorUtil.getTargetField(
baseType, typeFactory(), id, getCatalogReader(), relOptTable);
Expand Down Expand Up @@ -566,12 +601,6 @@ private IgniteTypeFactory typeFactory() {
return (IgniteTypeFactory)typeFactory;
}

/** */
private boolean isSystemFieldName(String alias) {
return QueryUtils.KEY_FIELD_NAME.equalsIgnoreCase(alias)
|| QueryUtils.VAL_FIELD_NAME.equalsIgnoreCase(alias);
}

/** {@inheritDoc} */
@Override public RelDataType deriveType(SqlValidatorScope scope, SqlNode expr) {
if (expr instanceof SqlDynamicParam) {
Expand All @@ -584,6 +613,14 @@ private boolean isSystemFieldName(String alias) {
return super.deriveType(scope, expr);
}

/** */
private void validateTechnicalColumnDmlTarget(SqlIdentifier id) {
String fieldName = id.names.get(id.names.size() - 1);

if (QueryUtils.isTechnicalFieldNameIgnoreCase(fieldName))
throw newValidationError(id, IgniteResource.INSTANCE.cannotModifyTechnicalColumn(id.toString()));
}

/** @return A derived type or {@code null} if unable to determine. */
@Nullable private RelDataType deriveDynamicParameterType(SqlDynamicParam node, RelDataType nullValType) {
RelDataType type = getValidatedNodeTypeIfKnown(node);
Expand Down
116 changes: 111 additions & 5 deletions .../org/apache/ignite/internal/processors/query/calcite/schema/CacheTableDescriptorImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,7 @@
import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState;
import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionTopology;
import org.apache.ignite.internal.processors.cache.persistence.CacheDataRow;
import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
import org.apache.ignite.internal.processors.query.GridQueryProperty;
import org.apache.ignite.internal.processors.query.GridQueryTypeDescriptor;
import org.apache.ignite.internal.processors.query.QueryUtils;
Expand Down Expand Up @@ -123,7 +124,7 @@ public CacheTableDescriptorImpl(GridCacheContextInfo<?, ?> cacheInfo, GridQueryT

Set<String> fields = this.typeDesc.fields().keySet();

List<CacheColumnDescriptor> descriptors = new ArrayList<>(fields.size() + 2);
List<CacheColumnDescriptor> descriptors = new ArrayList<>(fields.size() + 4);

// A _key/_val field is virtual in case there is an alias or a property(es) mapped to the _key/_val field.
BitSet virtualFields = new BitSet();
Expand Down Expand Up @@ -177,6 +178,28 @@ else if (Objects.equals(field, typeDesc.valueFieldAlias())) {
}
}

virtualFields.set(descriptors.size());
descriptors.add(new TechnicalDescriptor(QueryUtils.VER_FIELD_NAME, GridCacheVersion.class, descriptors.size()) {
@Override public Object value(ExecutionContext<?> ectx, GridCacheContext<?, ?> cctx, CacheDataRow src)
throws IgniteCheckedException {
GridCacheVersion ver = src.version();

if (ver != null)
return ver;

CacheDataRow row = cctx.offheap().read(cctx, src.key());

return row == null ? null : row.version();
}
});

virtualFields.set(descriptors.size());
descriptors.add(new TechnicalDescriptor(QueryUtils.SRC_FIELD_NAME, Integer.class, descriptors.size()) {
@Override public Object value(ExecutionContext<?> ectx, GridCacheContext<?, ?> cctx, CacheDataRow src) {
return cctx.cacheId();
}
});

Map<String, CacheColumnDescriptor> descriptorsMap = U.newHashMap(descriptors.size());
for (CacheColumnDescriptor descriptor : descriptors)
descriptorsMap.put(descriptor.name(), descriptor);
Expand Down Expand Up @@ -278,6 +301,9 @@ else if (affFields.isEmpty())
@Override public boolean isUpdateAllowed(RelOptTable tbl, int colIdx) {
final CacheColumnDescriptor desc = descriptors[colIdx];

if (QueryUtils.isTechnicalFieldName(desc.name()))
return false;

return !desc.key() && (desc.field() || QueryUtils.isSqlType(desc.storageType()));
}

Expand Down Expand Up @@ -382,9 +408,12 @@ private <Row> Object insertVal(Row row, ExecutionContext<Row> ectx) throws Ignit
for (int i = 2; i < descriptors.length; i++) {
final CacheColumnDescriptor desc = descriptors[i];

if (!desc.field() || desc.key())
continue;

Object fieldVal = hnd.get(i, row);

if (desc.field() && !desc.key() && fieldVal != null)
if (fieldVal != null)
desc.set(val, TypeUtils.fromInternal(ectx, fieldVal, desc.storageType()));
}
}
Expand Down Expand Up @@ -442,14 +471,18 @@ private <Row> ModifyTuple mergeTuple(Row row, List<String> updateColList, Execut

int rowColumnsCnt = hnd.columnCount(row);

if (rowColumnsCnt == descriptors.length)
// An empty update column list unambiguously means there is no WHEN MATCHED clause at all (a MERGE
// statement always has at least one WHEN clause), so the row can only originate from the INSERT
// section. Note: the row width alone can't be used to detect this case, since, depending on the
// number of updated columns, it may coincide with the width of a WHEN MATCHED-only row.
if (updateColList.isEmpty())
return insertTuple(row, ectx); // Only WHEN NOT MATCHED clause in MERGE.
else if (rowColumnsCnt == descriptors.length + updateColList.size())
return updateTuple(row, updateColList, 0, ectx); // Only WHEN MATCHED clause in MERGE.
else {
// Both WHEN MATCHED and WHEN NOT MATCHED clauses in MERGE.
assert rowColumnsCnt == descriptors.length * 2 + updateColList.size() : "Unexpected columns count: " +
rowColumnsCnt;
assert rowColumnsCnt == 2 * descriptors.length + updateColList.size() : "Unexpected columns count: "
+ rowColumnsCnt;

int updateOffset = descriptors.length; // Offset of fields for update statement.

Expand Down Expand Up @@ -805,6 +838,79 @@ private FieldDescriptor(GridQueryProperty desc, int fieldIdx) {
}
}

/** */
private abstract static class TechnicalDescriptor implements CacheColumnDescriptor {
/** */
private final String name;

/** */
private final Class<?> storageType;

/** */
private final int fieldIdx;

/** */
private volatile RelDataType logicalType;

/** */
private TechnicalDescriptor(String name, Class<?> storageType, int fieldIdx) {
this.name = name;
this.storageType = storageType;
this.fieldIdx = fieldIdx;
}

/** {@inheritDoc} */
@Override public boolean field() {
return false;
}

/** {@inheritDoc} */
@Override public boolean key() {
return false;
}

/** {@inheritDoc} */
@Override public boolean hasDefaultValue() {
return false;
}

/** {@inheritDoc} */
@Override public Object defaultValue() {
throw new AssertionError();
}

/** {@inheritDoc} */
@Override public String name() {
return name;
}

/** {@inheritDoc} */
@Override public int fieldIndex() {
return fieldIdx;
}

/** {@inheritDoc} */
@Override public RelDataType logicalType(IgniteTypeFactory f) {
if (logicalType == null) {
logicalType = storageType == GridCacheVersion.class
? f.createTypeWithNullability(f.createJavaType(storageType), true)
: TypeUtils.sqlType(f, storageType, PRECISION_NOT_SPECIFIED, SCALE_NOT_SPECIFIED, true);
}

return logicalType;
}

/** {@inheritDoc} */
@Override public Class<?> storageType() {
return storageType;
}

/** {@inheritDoc} */
@Override public void set(Object dst, Object val) {
throw new AssertionError();
}
}

/** {@inheritDoc} */
@Override public GridQueryTypeDescriptor typeDescription() {
return typeDesc;
Expand Down
4 changes: 4 additions & 0 deletions ...rc/main/java/org/apache/ignite/internal/processors/query/calcite/util/IgniteResource.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,10 @@ public interface IgniteResource {
@Resources.BaseMessage("Cannot update field \"{0}\". You cannot update key, key fields or val field in case the val is a complex type.")
Resources.ExInst<SqlValidatorException> cannotUpdateField(String field);

/** */
@Resources.BaseMessage("Cannot modify technical column \"{0}\".")
Resources.ExInst<SqlValidatorException> cannotModifyTechnicalColumn(String field);

/** */
@Resources.BaseMessage("Illegal aggregate function. {0} is unsupported at the moment.")
Resources.ExInst<SqlValidatorException> unsupportedAggregationFunction(String a0);
Expand Down
Loading
Loading