diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/schema/IoTDBMetadataFetchIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/schema/IoTDBMetadataFetchIT.java index 1b6818d39f0a5..d7a62ffa1dc62 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/schema/IoTDBMetadataFetchIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/schema/IoTDBMetadataFetchIT.java @@ -494,6 +494,25 @@ public void showCountTimeSeries() throws SQLException { } } + @Test + public void showCountTimeSeriesExcludeInternalDatabaseAndIncludeView() throws SQLException { + try (Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement()) { + final long baseVisibleCount = queryCount(statement, "COUNT TIMESERIES root.ln*.**"); + statement.execute("CREATE DATABASE root.count_it"); + statement.execute( + "CREATE TIMESERIES root.count_it.src.s1 WITH DATATYPE = INT32, ENCODING = PLAIN"); + statement.execute( + "CREATE TIMESERIES root.count_it.src.s2 WITH DATATYPE = INT32, ENCODING = PLAIN"); + statement.execute("CREATE VIEW root.count_it.dst.v1 AS SELECT s1 FROM root.count_it.src;"); + + final long localCount = queryCount(statement, "COUNT TIMESERIES root.count_it.**"); + assertEquals(3L, localCount); + assertEquals( + baseVisibleCount + localCount, queryCount(statement, "COUNT TIMESERIES root.**")); + } + } + @Test public void showCountTimeSeriesWithTag() throws SQLException { try (Connection connection = EnvFactory.getEnv().getConnection(); @@ -865,4 +884,11 @@ public void showDeadbandInfo() throws SQLException { } } } + + private long queryCount(final Statement statement, final String sql) throws SQLException { + try (ResultSet resultSet = statement.executeQuery(sql)) { + Assert.assertTrue(resultSet.next()); + return resultSet.getLong(1); + } + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/CountGroupByLevelScanOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/CountGroupByLevelScanOperator.java index 69f36680fb747..35c57dafa4d15 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/CountGroupByLevelScanOperator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/CountGroupByLevelScanOperator.java @@ -27,6 +27,7 @@ import org.apache.iotdb.db.queryengine.execution.operator.OperatorContext; import org.apache.iotdb.db.queryengine.execution.operator.schema.source.ISchemaSource; import org.apache.iotdb.db.queryengine.execution.operator.source.SourceOperator; +import org.apache.iotdb.db.schemaengine.schemaregion.ISchemaRegion; import org.apache.iotdb.db.schemaengine.schemaregion.read.resp.info.ISchemaInfo; import org.apache.iotdb.db.schemaengine.schemaregion.read.resp.reader.ISchemaReader; @@ -95,6 +96,10 @@ public OperatorContext getOperatorContext() { return operatorContext; } + private ISchemaRegion getSchemaRegion() { + return ((SchemaDriverContext) operatorContext.getDriverContext()).getSchemaRegion(); + } + @Override public ListenableFuture isBlocked() { if (isBlocked == null) { @@ -109,6 +114,11 @@ public ListenableFuture isBlocked() { */ private ListenableFuture tryGetNext() { if (schemaReader == null) { + if (schemaSource.shouldSkipSchemaRegion(getSchemaRegion())) { + next = null; + isFinished = true; + return NOT_BLOCKED; + } schemaReader = createTimeSeriesReader(); } while (true) { @@ -172,15 +182,14 @@ public TsBlock next() throws Exception { @Override public boolean hasNext() throws Exception { isBlocked().get(); // wait for the next TsBlock - if (!schemaReader.isSuccess()) { + if (schemaReader != null && !schemaReader.isSuccess()) { throw new SchemaExecutionException(schemaReader.getFailure()); } return next != null; } public ISchemaReader createTimeSeriesReader() { - return schemaSource.getSchemaReader( - ((SchemaDriverContext) operatorContext.getDriverContext()).getSchemaRegion()); + return schemaSource.getSchemaReader(getSchemaRegion()); } private TsBlock constructTsBlockAndClearMap(Map countMap) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/SchemaCountOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/SchemaCountOperator.java index 9b2884233c13d..4cafe40c36b1c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/SchemaCountOperator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/SchemaCountOperator.java @@ -100,6 +100,10 @@ public ListenableFuture isBlocked() { */ private ListenableFuture tryGetNext() { ISchemaRegion schemaRegion = getSchemaRegion(); + if (schemaSource.shouldSkipSchemaRegion(schemaRegion)) { + next = constructTsBlock(0); + return NOT_BLOCKED; + } if (schemaSource.hasSchemaStatistic(schemaRegion)) { long statisticCount = schemaSource.getSchemaStatistic(schemaRegion); // Check if database path itself is counted as a device (bug fix) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/ISchemaSource.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/ISchemaSource.java index 41bd5f3fbe0c1..4417203fdc863 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/ISchemaSource.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/ISchemaSource.java @@ -57,6 +57,10 @@ void transformToTsBlockColumns( long getSchemaStatistic(final ISchemaRegion schemaRegion); + default boolean shouldSkipSchemaRegion(final ISchemaRegion schemaRegion) { + return false; + } + default boolean checkRegionDatabaseIncluded(final ISchemaRegion schemaRegion) { return true; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/SchemaSourceFactory.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/SchemaSourceFactory.java index 87560ad47be59..f45dc6a9b061a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/SchemaSourceFactory.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/SchemaSourceFactory.java @@ -48,7 +48,7 @@ public static ISchemaSource getTimeSeriesSchemaCountSourc Map templateMap, PathPatternTree scope) { return new TimeSeriesSchemaSource( - pathPattern, isPrefixMatch, 0, 0, schemaFilter, templateMap, false, scope, null); + pathPattern, isPrefixMatch, 0, 0, schemaFilter, templateMap, false, true, scope, null); } // show time series @@ -69,6 +69,7 @@ public static ISchemaSource getTimeSeriesSchemaScanSource schemaFilter, templateMap, true, + false, scope, timeseriesOrdering); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/TimeSeriesSchemaSource.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/TimeSeriesSchemaSource.java index a56cfa228bfb8..5ac6d59b0897c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/TimeSeriesSchemaSource.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/TimeSeriesSchemaSource.java @@ -27,6 +27,7 @@ import org.apache.iotdb.commons.schema.column.ColumnHeader; import org.apache.iotdb.commons.schema.column.ColumnHeaderConstant; import org.apache.iotdb.commons.schema.filter.SchemaFilter; +import org.apache.iotdb.commons.schema.table.Audit; import org.apache.iotdb.commons.schema.template.Template; import org.apache.iotdb.commons.schema.view.ViewType; import org.apache.iotdb.db.queryengine.plan.statement.component.Ordering; @@ -55,6 +56,7 @@ public class TimeSeriesSchemaSource implements ISchemaSource templateMap; private final boolean needViewDetail; + private final boolean excludeInternalDatabase; private final Ordering timeseriesOrdering; TimeSeriesSchemaSource( @@ -65,6 +67,7 @@ public class TimeSeriesSchemaSource implements ISchemaSource templateMap, boolean needViewDetail, + boolean excludeInternalDatabase, PathPatternTree scope, Ordering timeseriesOrdering) { this.pathPattern = pathPattern; @@ -74,6 +77,7 @@ public class TimeSeriesSchemaSource implements ISchemaSource map) { if (map == null || map.isEmpty()) { return null; diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/schema/SchemaCountOperatorTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/schema/SchemaCountOperatorTest.java index b59689a8deb86..3cd50ebf51314 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/schema/SchemaCountOperatorTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/schema/SchemaCountOperatorTest.java @@ -114,6 +114,82 @@ public void testSchemaCountOperator() throws Exception { } } + @Test + public void testSchemaCountOperatorSkipSchemaRegion() throws Exception { + ExecutorService instanceNotificationExecutor = + IoTDBThreadPoolFactory.newFixedThreadPool(1, "test-instance-notification"); + try { + QueryId queryId = new QueryId("stub_query"); + FragmentInstanceId instanceId = + new FragmentInstanceId(new PlanFragmentId(queryId, 0), "stub-instance"); + FragmentInstanceStateMachine stateMachine = + new FragmentInstanceStateMachine(instanceId, instanceNotificationExecutor); + FragmentInstanceContext fragmentInstanceContext = + createFragmentInstanceContext(instanceId, stateMachine); + DriverContext driverContext = new DriverContext(fragmentInstanceContext, 0); + PlanNodeId planNodeId = queryId.genPlanNodeId(); + ISchemaRegion schemaRegion = Mockito.mock(ISchemaRegion.class); + OperatorContext operatorContext = + driverContext.addOperatorContext( + 1, planNodeId, SchemaCountOperator.class.getSimpleName()); + operatorContext.setDriverContext( + new SchemaDriverContext(fragmentInstanceContext, schemaRegion, 0)); + ISchemaSource schemaSource = Mockito.mock(ISchemaSource.class); + Mockito.when(schemaSource.shouldSkipSchemaRegion(schemaRegion)).thenReturn(true); + + SchemaCountOperator schemaCountOperator = + new SchemaCountOperator<>( + planNodeId, driverContext.getOperatorContexts().get(0), schemaSource); + + assertTrue(schemaCountOperator.hasNext()); + TsBlock tsBlock = schemaCountOperator.next(); + assertEquals(0, tsBlock.getColumn(0).getLong(0)); + assertTrue(schemaCountOperator.isFinished()); + Mockito.verify(schemaSource, Mockito.never()).getSchemaReader(schemaRegion); + } finally { + instanceNotificationExecutor.shutdown(); + } + } + + @Test + public void testSchemaCountOperatorUseSchemaStatistic() throws Exception { + ExecutorService instanceNotificationExecutor = + IoTDBThreadPoolFactory.newFixedThreadPool(1, "test-instance-notification"); + try { + QueryId queryId = new QueryId("stub_query"); + FragmentInstanceId instanceId = + new FragmentInstanceId(new PlanFragmentId(queryId, 0), "stub-instance"); + FragmentInstanceStateMachine stateMachine = + new FragmentInstanceStateMachine(instanceId, instanceNotificationExecutor); + FragmentInstanceContext fragmentInstanceContext = + createFragmentInstanceContext(instanceId, stateMachine); + DriverContext driverContext = new DriverContext(fragmentInstanceContext, 0); + PlanNodeId planNodeId = queryId.genPlanNodeId(); + ISchemaRegion schemaRegion = Mockito.mock(ISchemaRegion.class); + OperatorContext operatorContext = + driverContext.addOperatorContext( + 1, planNodeId, SchemaCountOperator.class.getSimpleName()); + operatorContext.setDriverContext( + new SchemaDriverContext(fragmentInstanceContext, schemaRegion, 0)); + ISchemaSource schemaSource = Mockito.mock(ISchemaSource.class); + Mockito.when(schemaSource.hasSchemaStatistic(schemaRegion)).thenReturn(true); + Mockito.when(schemaSource.getSchemaStatistic(schemaRegion)).thenReturn(7L); + Mockito.when(schemaSource.checkRegionDatabaseIncluded(schemaRegion)).thenReturn(true); + + SchemaCountOperator schemaCountOperator = + new SchemaCountOperator<>( + planNodeId, driverContext.getOperatorContexts().get(0), schemaSource); + + assertTrue(schemaCountOperator.hasNext()); + TsBlock tsBlock = schemaCountOperator.next(); + assertEquals(7, tsBlock.getColumn(0).getLong(0)); + Mockito.verify(schemaSource).getSchemaStatistic(schemaRegion); + Mockito.verify(schemaSource, Mockito.never()).getSchemaReader(schemaRegion); + } finally { + instanceNotificationExecutor.shutdown(); + } + } + @Test public void testLevelTimeSeriesCountOperator() { ExecutorService instanceNotificationExecutor = @@ -185,6 +261,43 @@ public void testLevelTimeSeriesCountOperator() { } } + @Test + public void testLevelTimeSeriesCountOperatorSkipSchemaRegion() { + ExecutorService instanceNotificationExecutor = + IoTDBThreadPoolFactory.newFixedThreadPool(1, "test-instance-notification"); + try { + QueryId queryId = new QueryId("stub_query"); + FragmentInstanceId instanceId = + new FragmentInstanceId(new PlanFragmentId(queryId, 0), "stub-instance"); + FragmentInstanceStateMachine stateMachine = + new FragmentInstanceStateMachine(instanceId, instanceNotificationExecutor); + FragmentInstanceContext fragmentInstanceContext = + createFragmentInstanceContext(instanceId, stateMachine); + DriverContext driverContext = new DriverContext(fragmentInstanceContext, 0); + PlanNodeId planNodeId = queryId.genPlanNodeId(); + OperatorContext operatorContext = + driverContext.addOperatorContext( + 1, planNodeId, CountGroupByLevelScanOperator.class.getSimpleName()); + ISchemaRegion schemaRegion = Mockito.mock(ISchemaRegion.class); + operatorContext.setDriverContext( + new SchemaDriverContext(fragmentInstanceContext, schemaRegion, 0)); + ISchemaSource schemaSource = Mockito.mock(ISchemaSource.class); + Mockito.when(schemaSource.shouldSkipSchemaRegion(schemaRegion)).thenReturn(true); + + CountGroupByLevelScanOperator timeSeriesCountOperator = + new CountGroupByLevelScanOperator<>( + planNodeId, driverContext.getOperatorContexts().get(0), 1, schemaSource); + + assertTrue(collectResult(timeSeriesCountOperator).isEmpty()); + Mockito.verify(schemaSource, Mockito.never()).getSchemaReader(schemaRegion); + } catch (Exception e) { + e.printStackTrace(); + fail(); + } finally { + instanceNotificationExecutor.shutdown(); + } + } + private List collectResult(CountGroupByLevelScanOperator operator) throws Exception { List tsBlocks = new ArrayList<>(); while (operator.hasNext()) { diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/TimeSeriesSchemaSourceTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/TimeSeriesSchemaSourceTest.java new file mode 100644 index 0000000000000..3769018ed46e2 --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/TimeSeriesSchemaSourceTest.java @@ -0,0 +1,168 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.execution.operator.schema.source; + +import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.commons.schema.SchemaConstant; +import org.apache.iotdb.commons.schema.table.Audit; +import org.apache.iotdb.db.schemaengine.rescon.ISchemaRegionStatistics; +import org.apache.iotdb.db.schemaengine.schemaregion.ISchemaRegion; +import org.apache.iotdb.db.schemaengine.schemaregion.read.resp.info.ITimeSeriesSchemaInfo; + +import org.junit.Test; +import org.mockito.Mockito; + +import java.util.Collections; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +public class TimeSeriesSchemaSourceTest { + + @Test + public void testCountSourceSkipsImplicitInternalDatabases() throws Exception { + final ISchemaSource countSource = + SchemaSourceFactory.getTimeSeriesSchemaCountSource( + new PartialPath("root.**"), + false, + null, + Collections.emptyMap(), + SchemaConstant.ALL_MATCH_SCOPE); + + assertTrue( + countSource.shouldSkipSchemaRegion(mockSchemaRegion(SchemaConstant.SYSTEM_DATABASE))); + assertTrue(countSource.shouldSkipSchemaRegion(mockSchemaRegion(SchemaConstant.AUDIT_DATABASE))); + assertTrue( + countSource.shouldSkipSchemaRegion(mockSchemaRegion(Audit.TABLE_MODEL_AUDIT_DATABASE))); + assertFalse(countSource.shouldSkipSchemaRegion(mockSchemaRegion("root.sg"))); + } + + @Test + public void testCountSourceKeepsExplicitInternalDatabaseQueries() throws Exception { + final ISchemaSource systemCountSource = + SchemaSourceFactory.getTimeSeriesSchemaCountSource( + new PartialPath("root.__system.**"), + false, + null, + Collections.emptyMap(), + SchemaConstant.ALL_MATCH_SCOPE); + assertFalse( + systemCountSource.shouldSkipSchemaRegion(mockSchemaRegion(SchemaConstant.SYSTEM_DATABASE))); + assertTrue( + systemCountSource.shouldSkipSchemaRegion(mockSchemaRegion(SchemaConstant.AUDIT_DATABASE))); + + final ISchemaSource auditCountSource = + SchemaSourceFactory.getTimeSeriesSchemaCountSource( + new PartialPath("root.__audit.**"), + false, + null, + Collections.emptyMap(), + SchemaConstant.ALL_MATCH_SCOPE); + assertFalse( + auditCountSource.shouldSkipSchemaRegion(mockSchemaRegion(SchemaConstant.AUDIT_DATABASE))); + assertTrue( + auditCountSource.shouldSkipSchemaRegion(mockSchemaRegion(SchemaConstant.SYSTEM_DATABASE))); + } + + @Test + public void testCountSourceSkipsWildcardSecondNodeForInternalDatabases() throws Exception { + final ISchemaSource countSource = + SchemaSourceFactory.getTimeSeriesSchemaCountSource( + new PartialPath("root.*.**"), + false, + null, + Collections.emptyMap(), + SchemaConstant.ALL_MATCH_SCOPE); + + assertTrue( + countSource.shouldSkipSchemaRegion(mockSchemaRegion(SchemaConstant.SYSTEM_DATABASE))); + assertTrue(countSource.shouldSkipSchemaRegion(mockSchemaRegion(SchemaConstant.AUDIT_DATABASE))); + assertFalse(countSource.shouldSkipSchemaRegion(mockSchemaRegion("root.sg"))); + } + + @Test + public void testCountSourceKeepsExactInternalDatabaseQueries() throws Exception { + final ISchemaSource systemCountSource = + SchemaSourceFactory.getTimeSeriesSchemaCountSource( + new PartialPath("root.__system"), + false, + null, + Collections.emptyMap(), + SchemaConstant.ALL_MATCH_SCOPE); + assertFalse( + systemCountSource.shouldSkipSchemaRegion(mockSchemaRegion(SchemaConstant.SYSTEM_DATABASE))); + + final ISchemaSource auditCountSource = + SchemaSourceFactory.getTimeSeriesSchemaCountSource( + new PartialPath("root.__audit"), + false, + null, + Collections.emptyMap(), + SchemaConstant.ALL_MATCH_SCOPE); + assertFalse( + auditCountSource.shouldSkipSchemaRegion(mockSchemaRegion(SchemaConstant.AUDIT_DATABASE))); + } + + @Test + public void testShowSourceDoesNotSkipInternalDatabases() throws Exception { + final ISchemaSource showSource = + SchemaSourceFactory.getTimeSeriesSchemaScanSource( + new PartialPath("root.**"), + false, + 0, + 0, + null, + Collections.emptyMap(), + SchemaConstant.ALL_MATCH_SCOPE, + null); + + assertFalse( + showSource.shouldSkipSchemaRegion(mockSchemaRegion(SchemaConstant.SYSTEM_DATABASE))); + assertFalse(showSource.shouldSkipSchemaRegion(mockSchemaRegion(SchemaConstant.AUDIT_DATABASE))); + } + + @Test + public void testCountStatisticIncludesView() throws Exception { + final ISchemaSource countSource = + SchemaSourceFactory.getTimeSeriesSchemaCountSource( + new PartialPath("root.sg.**"), + false, + null, + Collections.emptyMap(), + SchemaConstant.ALL_MATCH_SCOPE); + final ISchemaRegion schemaRegion = mockSchemaRegion("root.sg"); + final ISchemaRegionStatistics schemaRegionStatistics = + Mockito.mock(ISchemaRegionStatistics.class); + + Mockito.when(schemaRegion.getSchemaRegionStatistics()).thenReturn(schemaRegionStatistics); + Mockito.when(schemaRegionStatistics.getSeriesNumber(true)).thenReturn(5L); + + assertEquals(5L, countSource.getSchemaStatistic(schemaRegion)); + Mockito.verify(schemaRegionStatistics).getSeriesNumber(true); + Mockito.verify(schemaRegionStatistics, Mockito.never()).getSeriesNumber(false); + } + + private ISchemaRegion mockSchemaRegion(final String database) { + final ISchemaRegion schemaRegion = Mockito.mock(ISchemaRegion.class); + Mockito.when(schemaRegion.getDatabaseFullPath()).thenReturn(database); + return schemaRegion; + } +}