diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/schema/IoTDBAlterTimeSeriesTypeIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/schema/IoTDBAlterTimeSeriesTypeIT.java new file mode 100644 index 0000000000000..714b005e3c513 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/schema/IoTDBAlterTimeSeriesTypeIT.java @@ -0,0 +1,2752 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.it.schema; + +import org.apache.iotdb.commons.utils.MetadataUtils; +import org.apache.iotdb.db.utils.SchemaUtils; +import org.apache.iotdb.isession.ISession; +import org.apache.iotdb.isession.SessionDataSet; +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; + +import org.apache.tsfile.enums.ColumnCategory; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.exception.write.WriteProcessException; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.read.common.Field; +import org.apache.tsfile.read.common.Path; +import org.apache.tsfile.read.common.RowRecord; +import org.apache.tsfile.utils.Binary; +import org.apache.tsfile.write.TsFileWriter; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.IMeasurementSchema; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.apache.tsfile.write.v4.TsFileTreeWriter; +import org.apache.tsfile.write.v4.TsFileTreeWriterBuilder; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.ws.rs.NotSupportedException; + +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.sql.Connection; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.Set; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.apache.iotdb.relational.it.session.IoTDBSessionRelationalIT.genValue; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +@SuppressWarnings("ResultOfMethodCallIgnored") +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBAlterTimeSeriesTypeIT { + + private static final Logger log = LoggerFactory.getLogger(IoTDBAlterTimeSeriesTypeIT.class); + private static long timeout = -1; + private static final String database = "root.alter"; + public static final List DATA_TYPE_LIST = + Arrays.asList(TSDataType.STRING, TSDataType.TEXT, TSDataType.BOOLEAN); + public static final List UNSUPPORT_ACCUMULATOR_QUERY_DATA_TYPE_LIST = + Collections.singletonList(TSDataType.BLOB); + + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv().getConfig().getDataNodeConfig().setCompactionScheduleInterval(1000); + EnvFactory.getEnv().initClusterEnvironment(); + } + + @AfterClass + public static void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + @Test + public void testWriteAndAlter() throws Exception { + Set typesToTest = new HashSet<>(); + Collections.addAll(typesToTest, TSDataType.values()); + typesToTest.remove(TSDataType.VECTOR); + typesToTest.remove(TSDataType.UNKNOWN); + + for (TSDataType from : typesToTest) { + for (TSDataType to : typesToTest) { + if (from != to && to.isCompatible(from)) { + System.out.printf("testing %s to %s%n", from, to); + doWriteAndAlter(from, to); + + testNonAlignDeviceSequenceDataQuery(from, to); + testNonAlignDeviceUnSequenceDataQuery(from, to); + testNonAlignDeviceUnSequenceOverlappedDataQuery(from, to); + + testAlignDeviceSequenceDataQuery(from, to); + testAlignDeviceUnSequenceDataQuery(from, to); + testAlignDeviceUnSequenceOverlappedDataQuery(from, to); + } + } + } + } + + private void doWriteAndAlter(TSDataType from, TSDataType to) + throws IoTDBConnectionException, StatementExecutionException { + try (ISession session = EnvFactory.getEnv().getSessionConnection()) { + session.executeNonQueryStatement( + "SET CONFIGURATION \"enable_unseq_space_compaction\"='false'"); + if (from == TSDataType.DATE && !to.isCompatible(from)) { + throw new NotSupportedException("Not supported DATE type."); + } + + // create a time series with type of "from" + session.executeNonQueryStatement( + "CREATE TIMESERIES " + database + ".write_and_alter_column_type.s1 " + from); + + // write a sequence tsfile point of "from" + List schemaList = new ArrayList<>(); + schemaList.add(new MeasurementSchema("s1", from)); + Tablet tablet = new Tablet(database + ".write_and_alter_column_type", schemaList); + tablet.addTimestamp(0, 1); + tablet.addValue("s1", 0, genValue(from, 1)); + session.insertTablet(tablet); + tablet.reset(); + session.executeNonQueryStatement("FLUSH"); + + // write an unsequence tsfile point of "from" + tablet.addTimestamp(0, 1); + tablet.addValue("s1", 0, genValue(from, 1)); + session.insertTablet(tablet); + tablet.reset(); + session.executeNonQueryStatement("FLUSH"); + + // write a sequence memtable point of "from" + tablet.addTimestamp(0, 2); + tablet.addValue("s1", 0, genValue(from, 2)); + session.insertTablet(tablet); + tablet.reset(); + + // write an unsequence memtable point of "from" + tablet.addTimestamp(0, 1); + tablet.addValue("s1", 0, genValue(from, 1)); + session.insertTablet(tablet); + tablet.reset(); + + try (SessionDataSet dataSet1 = + session.executeQueryStatement( + "select s1 from " + database + ".write_and_alter_column_type order by time")) { + for (int i = 1; i <= 2; i++) { + RowRecord rec1 = dataSet1.next(); + long time = rec1.getTimestamp(); + Object v = rec1.getFields().get(0).getObjectValue(from); + assertEquals(time, i); + assertEquals(v, genValue(from, i)); + // System.out.println( + // "from is " + from + ", time is " + time + ", value is " + v); + } + } + + // alter the type to "to" + boolean isCompatible = MetadataUtils.canAlter(from, to); + if (isCompatible) { + session.executeNonQueryStatement( + "ALTER TIMESERIES " + database + ".write_and_alter_column_type.s1 SET DATA TYPE " + to); + } else { + try { + session.executeNonQueryStatement( + "ALTER TIMESERIES " + + database + + ".write_and_alter_column_type.s1 SET DATA TYPE " + + to); + } catch (StatementExecutionException e) { + assertEquals( + "701: New type " + to + " is not compatible with the existing one " + from, + e.getMessage()); + } + } + + // If don't execute the flush" operation, verify if result can get valid value, not be null + // when query memtable. + // session.executeNonQueryStatement("FLUSH"); + + SessionDataSet dataSet = + session.executeQueryStatement( + "select s1 from " + database + ".write_and_alter_column_type order by time"); + RowRecord rec; + TSDataType newType = isCompatible ? to : from; + for (int i = 1; i <= 2; i++) { + rec = dataSet.next(); + if (newType == TSDataType.BLOB) { + Binary v = rec.getFields().get(0).getBinaryV(); + assertEquals(genValue(newType, i), v); + } else if (newType == TSDataType.DATE) { + assertEquals(genValue(newType, i), rec.getFields().get(0).getDateV()); + } else if (newType == TSDataType.STRING || newType == TSDataType.TEXT) { + if (from == TSDataType.DATE) { + Binary v = rec.getFields().get(0).getBinaryV(); + assertEquals(new Binary(genValue(from, i).toString(), StandardCharsets.UTF_8), v); + } else { + // Object v = rec.getFields().get(0).getObjectValue(from); + log.info("from is {}", from); + assertEquals( + newType.castFromSingleValue(from, genValue(from, i)), + rec.getFields().get(0).getBinaryV()); + } + } else { + assertEquals(genValue(newType, i).toString(), rec.getFields().get(0).toString()); + } + } + + assertNull(dataSet.next()); + dataSet.close(); + + // write an altered point in sequence and unsequnce tsfile + // List newSchemaList = new ArrayList<>(); + // newSchemaList.add(new MeasurementSchema("s1", newType)); + // Arrays.asList(TSDataType.STRING, TSDataType.INT32) + // tablet = new Tablet(database + ".write_and_alter_column_type", newSchemaList); + + tablet = + new Tablet( + database + ".write_and_alter_column_type", + Collections.singletonList("s1"), + Collections.singletonList(to), + Collections.singletonList(ColumnCategory.FIELD)); + + tablet.addTimestamp(0, 3); + tablet.addValue("s1", 0, genValue(newType, 3)); + session.insertTablet(tablet); + tablet.reset(); + + tablet.addTimestamp(0, 1); + tablet.addValue("s1", 0, genValue(newType, 1)); + session.insertTablet(tablet); + tablet.reset(); + session.executeNonQueryStatement("FLUSH"); + + // write an altered point in sequence and unsequnce memtable + tablet.addTimestamp(0, 4); + tablet.addValue("s1", 0, genValue(newType, 4)); + session.insertTablet(tablet); + tablet.reset(); + + tablet.addTimestamp(0, 2); + tablet.addValue("s1", 0, genValue(newType, 2)); + session.insertTablet(tablet); + tablet.reset(); + + log.info("Write completely"); + + dataSet = + session.executeQueryStatement( + "select s1 from " + database + ".write_and_alter_column_type order by time"); + for (int i = 1; i <= 4; i++) { + rec = dataSet.next(); + if (newType == TSDataType.BLOB) { + assertEquals(genValue(newType, i), rec.getFields().get(0).getBinaryV()); + } else if (newType == TSDataType.DATE) { + assertEquals(genValue(newType, i), rec.getFields().get(0).getDateV()); + } else if (newType == TSDataType.STRING || newType == TSDataType.TEXT) { + assertEquals(genValue(to, i), rec.getFields().get(0).getBinaryV()); + } else { + assertEquals(genValue(newType, i).toString(), rec.getFields().get(0).toString()); + } + } + assertFalse(dataSet.hasNext()); + + if (DATA_TYPE_LIST.contains(from) || DATA_TYPE_LIST.contains(to)) { + dataSet = + session.executeQueryStatement( + "select first_value(s1),last_value(s1) from " + + database + + ".write_and_alter_column_type"); + rec = dataSet.next(); + int[] expectedValue = {1, 4}; + for (int i = 0; i < 2; i++) { + if (newType == TSDataType.BLOB) { + assertEquals(genValue(newType, expectedValue[i]), rec.getFields().get(i).getBinaryV()); + } else if (newType == TSDataType.DATE) { + assertEquals(genValue(newType, expectedValue[i]), rec.getFields().get(i).getDateV()); + } else { + assertEquals( + genValue(newType, expectedValue[i]).toString(), rec.getFields().get(i).toString()); + } + } + assertFalse(dataSet.hasNext()); + } else { + dataSet = + session.executeQueryStatement( + "select min_value(s1),max_value(s1),first_value(s1),last_value(s1) from " + + database + + ".write_and_alter_column_type"); + rec = dataSet.next(); + int[] expectedValue = {1, 4, 1, 4}; + for (int i = 0; i < 4; i++) { + if (newType == TSDataType.BLOB) { + assertEquals(genValue(newType, expectedValue[i]), rec.getFields().get(i).getBinaryV()); + } else if (newType == TSDataType.DATE) { + assertEquals(genValue(newType, expectedValue[i]), rec.getFields().get(i).getDateV()); + } else { + assertEquals( + genValue(newType, expectedValue[i]).toString(), rec.getFields().get(i).toString()); + } + } + assertFalse(dataSet.hasNext()); + } + + if (newType.isNumeric()) { + dataSet = + session.executeQueryStatement( + "select avg(s1),sum(s1) from " + database + ".write_and_alter_column_type"); + rec = dataSet.next(); + assertEquals(2.5, rec.getFields().get(0).getDoubleV(), 0.001); + assertEquals(10.0, rec.getFields().get(1).getDoubleV(), 0.001); + assertFalse(dataSet.hasNext()); + } + + // session.executeNonQueryStatement( + // "DROP TIMESERIES " + database + ".write_and_alter_column_type.s1"); + } finally { + try (ISession session = EnvFactory.getEnv().getSessionConnection()) { + session.executeNonQueryStatement( + "DELETE TIMESERIES " + database + ".write_and_alter_column_type.s1"); + } + } + } + + @Test + public void testAlterWithoutWrite() throws IoTDBConnectionException, StatementExecutionException { + Set typesToTest = new HashSet<>(); + Collections.addAll(typesToTest, TSDataType.values()); + typesToTest.remove(TSDataType.VECTOR); + typesToTest.remove(TSDataType.UNKNOWN); + // typesToTest.remove(TSDataType.STRING); + // typesToTest.remove(TSDataType.TEXT); + // typesToTest.remove(TSDataType.DATE); + + for (TSDataType from : typesToTest) { + for (TSDataType to : typesToTest) { + if (from != to && to.isCompatible(from)) { + System.out.printf("testing %s to %s%n", from, to); + doAlterWithoutWrite(from, to, false); + doAlterWithoutWrite(from, to, true); + } + } + } + } + + private void doAlterWithoutWrite(TSDataType from, TSDataType to, boolean flush) + throws IoTDBConnectionException, StatementExecutionException { + try (ISession session = EnvFactory.getEnv().getSessionConnection()) { + if (from == TSDataType.DATE && !to.isCompatible(from)) { + throw new NotSupportedException("Not supported DATE type."); + } + + // create a table with type of "from" + session.executeNonQueryStatement( + "CREATE TIMESERIES " + database + ".just_alter_column_type.s1 " + from); + + // alter the type to "to" + boolean isCompatible = MetadataUtils.canAlter(from, to); + if (isCompatible) { + session.executeNonQueryStatement( + "ALTER TIMESERIES " + database + ".just_alter_column_type.s1 SET DATA TYPE " + to); + } else { + try { + session.executeNonQueryStatement( + "ALTER TIMESERIES " + database + ".just_alter_column_type.s1 SET DATA TYPE " + to); + } catch (StatementExecutionException e) { + assertEquals( + "701: New type " + to + " is not compatible with the existing one " + from, + e.getMessage()); + } + } + + TSDataType newType = isCompatible ? to : from; + + // write a point + Tablet tablet = + new Tablet( + database + ".just_alter_column_type", + Collections.singletonList("s1"), + Collections.singletonList(newType), + Collections.singletonList(ColumnCategory.FIELD)); + tablet.addTimestamp(0, 1); + tablet.addValue("s1", 0, genValue(newType, 1)); + session.insertTablet(tablet); + tablet.reset(); + + if (flush) { + session.executeNonQueryStatement("FLUSH"); + } + + SessionDataSet dataSet = + session.executeQueryStatement( + "select * from " + database + ".just_alter_column_type order by time"); + RowRecord rec = dataSet.next(); + assertEquals(1, rec.getTimestamp()); + if (newType == TSDataType.BLOB) { + assertEquals(genValue(newType, 1), rec.getFields().get(0).getBinaryV()); + } else if (newType == TSDataType.DATE) { + assertEquals(genValue(newType, 1), rec.getFields().get(0).getDateV()); + } else { + assertEquals(genValue(newType, 1).toString(), rec.getFields().get(0).toString()); + } + + assertFalse(dataSet.hasNext()); + + session.executeNonQueryStatement( + "DELETE TIMESERIES " + database + ".just_alter_column_type.**"); + } + } + + @Test + public void testAlterNonExist() throws IoTDBConnectionException, StatementExecutionException { + try (ISession session = EnvFactory.getEnv().getSessionConnection()) { + try { + session.executeNonQueryStatement( + "ALTER TIMESERIES " + database + ".non_exist.s1 SET DATA TYPE INT64"); + fail("Should throw exception"); + } catch (StatementExecutionException e) { + assertEquals("508: Path [" + database + ".non_exist.s1] does not exist", e.getMessage()); + } + } + } + + @Test + public void testAlterWrongType() throws IoTDBConnectionException, StatementExecutionException { + try (ISession session = EnvFactory.getEnv().getSessionConnection()) { + session.executeNonQueryStatement("CREATE TIMESERIES " + database + ".wrong_type.s1 int32"); + + try { + session.executeNonQueryStatement( + "ALTER TIMESERIES " + database + ".wrong_type.s1 SET DATA TYPE VECTOR"); + fail("Should throw exception"); + } catch (StatementExecutionException e) { + assertEquals("701: Unsupported datatype: VECTOR", e.getMessage()); + } + } + } + + @Test + public void testContinuousAlter() throws IoTDBConnectionException, StatementExecutionException { + try (ISession session = EnvFactory.getEnv().getSessionConnection()) { + session.executeNonQueryStatement( + "CREATE TIMESERIES " + database + ".alter_and_alter.s1 int32"); + + // time=1 and time=2 are INT32 + Tablet tablet = + new Tablet( + database + ".alter_and_alter", + Collections.singletonList("s1"), + Collections.singletonList(TSDataType.INT32), + Collections.singletonList(ColumnCategory.FIELD)); + tablet.addTimestamp(0, 1); + tablet.addValue("s1", 0, genValue(TSDataType.INT32, 1)); + session.insertTablet(tablet); + tablet.reset(); + + session.executeNonQueryStatement("FLUSH"); + + tablet = + new Tablet( + database + ".alter_and_alter", + Collections.singletonList("s1"), + Collections.singletonList(TSDataType.INT32), + Collections.singletonList(ColumnCategory.FIELD)); + tablet.addTimestamp(0, 2); + tablet.addValue("s1", 0, genValue(TSDataType.INT32, 2)); + session.insertTablet(tablet); + tablet.reset(); + + // time=3 and time=4 are FLOAT + session.executeNonQueryStatement( + "ALTER TIMESERIES " + database + ".alter_and_alter.s1 SET DATA TYPE FLOAT"); + tablet = + new Tablet( + database + ".alter_and_alter", + Collections.singletonList("s1"), + Collections.singletonList(TSDataType.FLOAT), + Collections.singletonList(ColumnCategory.FIELD)); + tablet.addTimestamp(0, 3); + tablet.addValue("s1", 0, genValue(TSDataType.FLOAT, 3)); + session.insertTablet(tablet); + tablet.reset(); + + session.executeNonQueryStatement("FLUSH"); + + tablet = + new Tablet( + database + ".alter_and_alter", + Collections.singletonList("s1"), + Collections.singletonList(TSDataType.FLOAT), + Collections.singletonList(ColumnCategory.FIELD)); + tablet.addTimestamp(0, 4); + tablet.addValue("s1", 0, genValue(TSDataType.FLOAT, 4)); + session.insertTablet(tablet); + tablet.reset(); + + // time=5 and time=6 are DOUBLE + session.executeNonQueryStatement( + "ALTER TIMESERIES " + database + ".alter_and_alter.s1 SET DATA TYPE DOUBLE"); + tablet = + new Tablet( + database + ".alter_and_alter", + Collections.singletonList("s1"), + Collections.singletonList(TSDataType.DOUBLE), + Collections.singletonList(ColumnCategory.FIELD)); + tablet.addTimestamp(0, 5); + tablet.addValue("s1", 0, genValue(TSDataType.DOUBLE, 5)); + session.insertTablet(tablet); + tablet.reset(); + + session.executeNonQueryStatement("FLUSH"); + + tablet = + new Tablet( + database + ".alter_and_alter", + Collections.singletonList("s1"), + Collections.singletonList(TSDataType.DOUBLE), + Collections.singletonList(ColumnCategory.FIELD)); + tablet.addTimestamp(0, 6); + tablet.addValue("s1", 0, genValue(TSDataType.DOUBLE, 6)); + session.insertTablet(tablet); + tablet.reset(); + + SessionDataSet dataSet = + session.executeQueryStatement( + "select * from " + database + ".alter_and_alter order by time"); + RowRecord rec; + for (int i = 1; i < 7; i++) { + rec = dataSet.next(); + assertEquals(i, rec.getTimestamp()); + assertEquals(genValue(TSDataType.DOUBLE, i).toString(), rec.getFields().get(0).toString()); + } + assertFalse(dataSet.hasNext()); + } + } + + @Test + public void testConcurrentWriteAndAlter() + throws IoTDBConnectionException, StatementExecutionException, InterruptedException { + try (ISession session = EnvFactory.getEnv().getSessionConnection()) { + session.executeNonQueryStatement( + "CREATE TIMESERIES " + database + ".concurrent_write_and_alter.s1 int32"); + } + + ExecutorService threadPool = Executors.newCachedThreadPool(); + AtomicInteger writeCounter = new AtomicInteger(0); + int maxWrite = 10000; + int flushInterval = 100; + int alterStart = 5000; + threadPool.submit( + () -> { + try { + write(writeCounter, maxWrite, flushInterval); + } catch (IoTDBConnectionException | StatementExecutionException e) { + throw new RuntimeException(e); + } + }); + threadPool.submit( + () -> { + try { + alter(writeCounter, alterStart); + } catch (InterruptedException + | IoTDBConnectionException + | StatementExecutionException e) { + throw new RuntimeException(e); + } + }); + threadPool.shutdown(); + assertTrue(threadPool.awaitTermination(1, TimeUnit.MINUTES)); + + try (ISession session = EnvFactory.getEnv().getSessionConnection()) { + SessionDataSet dataSet = + session.executeQueryStatement( + "select count(s1) from " + database + ".concurrent_write_and_alter"); + RowRecord rec; + rec = dataSet.next(); + assertEquals(maxWrite, rec.getFields().get(0).getLongV()); + assertFalse(dataSet.hasNext()); + } + } + + private void write(AtomicInteger writeCounter, int maxWrite, int flushInterval) + throws IoTDBConnectionException, StatementExecutionException { + try (ISession session = EnvFactory.getEnv().getSessionConnection()) { + int writtenCnt = 0; + do { + session.executeNonQueryStatement( + String.format( + "INSERT INTO " + + database + + ".concurrent_write_and_alter (time, s1) VALUES (%d, %d)", + writtenCnt, + writtenCnt)); + if (((writtenCnt + 1) % flushInterval) == 0) { + session.executeNonQueryStatement("FLUSH"); + } + } while ((writtenCnt = writeCounter.incrementAndGet()) < maxWrite); + } + } + + private void alter(AtomicInteger writeCounter, int alterStart) + throws InterruptedException, IoTDBConnectionException, StatementExecutionException { + while (writeCounter.get() < alterStart) { + Thread.sleep(10); + } + try (ISession session = EnvFactory.getEnv().getSessionConnection()) { + session.executeNonQueryStatement( + "ALTER TABLE " + database + ".concurrent_write_and_alter.s1 SET DATA TYPE DOUBLE"); + } + } + + @Test + public void testLoadAndAlter() + throws IoTDBConnectionException, + StatementExecutionException, + IOException, + WriteProcessException { + try (ISession session = EnvFactory.getEnv().getSessionConnection()) { + session.executeNonQueryStatement("SET CONFIGURATION 'enable_unseq_space_compaction'='false'"); + } + + // file1-file4 s1=INT32 + + // file1-file3 single device small range ([1, 1]), may load without split + List filesToLoad = new ArrayList<>(); + for (int i = 1; i <= 3; i++) { + File file = new File("target", "f" + i + ".tsfile"); + try (TsFileTreeWriter tsFileWriter = new TsFileTreeWriterBuilder().file(file).build()) { + // Register time series schemas + IMeasurementSchema schema = new MeasurementSchema("s1", TSDataType.INT32); + + // Schema for device dId + tsFileWriter.registerTimeseries(database + ".load_and_alter", schema); + + Tablet tablet = + new Tablet( + database + ".load_and_alter", + Collections.singletonList("s1"), + Collections.singletonList(TSDataType.INT32), + Collections.singletonList(ColumnCategory.FIELD)); + tablet.addTimestamp(0, i); + // tablet.addValue("dId", 0, "d" + i); + tablet.addValue("s1", 0, i); + tsFileWriter.write(tablet); + } + filesToLoad.add(file); + } + // file4 multi device large range ([2, 100_000_000]), load with split + File file = new File("target", "f" + 4 + ".tsfile"); + try (TsFileTreeWriter tsFileWriter = new TsFileTreeWriterBuilder().file(file).build()) { + // Register time series schemas + IMeasurementSchema schema = new MeasurementSchema("s1", TSDataType.INT32); + + // Schema for device dId + tsFileWriter.registerTimeseries(database + ".load_and_alter", schema); + + Tablet tablet = + new Tablet( + database + ".load_and_alter", + Collections.singletonList("s1"), + Collections.singletonList(TSDataType.INT32), + Collections.singletonList(ColumnCategory.FIELD)); + int rowIndex = 0; + for (int i = 4; i <= 9; i++) { + tablet.addTimestamp(rowIndex, i); + tablet.addValue("s1", rowIndex, i); + rowIndex++; + } + tsFileWriter.write(tablet); + } + filesToLoad.add(file); + + // load file1-file4 + try (ISession session = EnvFactory.getEnv().getSessionConnection()) { + for (File f : filesToLoad) { + session.executeNonQueryStatement("LOAD '" + f.getAbsolutePath() + "'"); + } + } + // check load result + try (ISession session = EnvFactory.getEnv().getSessionConnection()) { + SessionDataSet dataSet = + session.executeQueryStatement("select count(s1) from " + database + ".load_and_alter"); + RowRecord rec; + rec = dataSet.next(); + assertEquals(9, rec.getFields().get(0).getLongV()); + assertFalse(dataSet.hasNext()); + } + + filesToLoad.forEach( + tsfile -> { + tsfile.delete(); + File resourceFile = new File(tsfile.getAbsolutePath() + ".resource"); + resourceFile.delete(); + }); + filesToLoad.clear(); + + // file5-file8 s1=DOUBLE + + // file5-file7 single device small range ([3, 3]), may load without split + for (int i = 5; i <= 7; i++) { + file = new File("target", "f" + i + ".tsfile"); + try (TsFileTreeWriter tsFileWriter = new TsFileTreeWriterBuilder().file(file).build()) { + // Register time series schemas + IMeasurementSchema schema = new MeasurementSchema("s1", TSDataType.DOUBLE); + + // Schema for device dId + tsFileWriter.registerTimeseries(database + ".load_and_alter", schema); + + Tablet tablet = + new Tablet( + database + ".load_and_alter", + Collections.singletonList("s1"), + Collections.singletonList(TSDataType.DOUBLE), + Collections.singletonList(ColumnCategory.FIELD)); + tablet.addTimestamp(0, i + 5); + tablet.addValue("s1", 0, 3.0); + tsFileWriter.write(tablet); + } + filesToLoad.add(file); + } + // file8 multi device large range ([4, 100_000_001]), load with split + file = new File("target", "f" + 8 + ".tsfile"); + try (TsFileTreeWriter tsFileWriter = new TsFileTreeWriterBuilder().file(file).build()) { + // Register time series schemas + IMeasurementSchema schema = new MeasurementSchema("s1", TSDataType.DOUBLE); + + // Schema for device dId + tsFileWriter.registerTimeseries(database + ".load_and_alter", schema); + + Tablet tablet = + new Tablet( + database + ".load_and_alter", + Collections.singletonList("s1"), + Collections.singletonList(TSDataType.DOUBLE), + Collections.singletonList(ColumnCategory.FIELD)); + int rowIndex = 0; + for (int i = 8; i <= 10; i++) { + tablet.addTimestamp(rowIndex, i + 5); + tablet.addValue("s1", rowIndex, 4.0); + rowIndex++; + } + tsFileWriter.write(tablet); + } + filesToLoad.add(file); + + // load file5-file8 + try (ISession session = EnvFactory.getEnv().getSessionConnection()) { + for (File f : filesToLoad) { + session.executeNonQueryStatement("LOAD '" + f.getAbsolutePath() + "'"); + } + } + // check load result + try (ISession session = EnvFactory.getEnv().getSessionConnection()) { + SessionDataSet dataSet = + session.executeQueryStatement("select count(s1) from " + database + ".load_and_alter"); + RowRecord rec; + rec = dataSet.next(); + // Due to the operation of load tsfile execute directly, don't access memtable or generate + // InsertNode object, so don't need to check the data type. + // When query this measurement point, will only find the data of TSDataType.INT32. So this is + // reason what cause we can't find the data of TSDataType.DOUBLE. So result is 9, is not 15. + // assertEquals(15, rec.getFields().get(0).getLongV()); + assertEquals(9, rec.getFields().get(0).getLongV()); + assertFalse(dataSet.hasNext()); + } + + // alter s1 to double + try (ISession session = EnvFactory.getEnv().getSessionConnection()) { + session.executeNonQueryStatement( + "ALTER TIMESERIES " + database + ".load_and_alter.s1 SET DATA TYPE DOUBLE"); + } + + filesToLoad.forEach( + tsfile -> { + tsfile.delete(); + File resourceFile = new File(tsfile.getAbsolutePath() + ".resource"); + resourceFile.delete(); + }); + filesToLoad.clear(); + + // file9-file12 s1=INT32 + // file9-file11 single device small range ([5, 5]), may load without split + for (int i = 9; i <= 11; i++) { + file = new File("target", "f" + i + ".tsfile"); + try (TsFileTreeWriter tsFileWriter = new TsFileTreeWriterBuilder().file(file).build()) { + // Register time series schemas + IMeasurementSchema schema = new MeasurementSchema("s1", TSDataType.INT32); + + // Schema for device dId + tsFileWriter.registerTimeseries(database + ".load_and_alter", schema); + + Tablet tablet = + new Tablet( + database + ".load_and_alter", + Collections.singletonList("s1"), + Collections.singletonList(TSDataType.INT32), + Collections.singletonList(ColumnCategory.FIELD)); + tablet.addTimestamp(0, i + 7); + tablet.addValue("s1", 0, 5); + tsFileWriter.write(tablet); + } + filesToLoad.add(file); + } + // file12 multi device large range ([6, 100_000_002]), load with split + file = new File("target", "f" + 12 + ".tsfile"); + try (TsFileTreeWriter tsFileWriter = new TsFileTreeWriterBuilder().file(file).build()) { + // Register time series schemas + IMeasurementSchema schema = new MeasurementSchema("s1", TSDataType.INT32); + + // Schema for device dId + tsFileWriter.registerTimeseries(database + ".load_and_alter", schema); + + Tablet tablet = + new Tablet( + database + ".load_and_alter", + Collections.singletonList("s1"), + Collections.singletonList(TSDataType.INT32), + Collections.singletonList(ColumnCategory.FIELD)); + int rowIndex = 0; + for (int i = 12; i <= 14; i++) { + tablet.addTimestamp(rowIndex, i + 7); + tablet.addValue("s1", rowIndex, 6); + rowIndex++; + } + tsFileWriter.write(tablet); + } + filesToLoad.add(file); + + // load file9-file12, should succeed + try (ISession session = EnvFactory.getEnv().getSessionConnection()) { + for (File f : filesToLoad) { + session.executeNonQueryStatement("LOAD '" + f.getAbsolutePath() + "'"); + } + } + // check load result + try (ISession session = EnvFactory.getEnv().getSessionConnection()) { + SessionDataSet dataSet = + session.executeQueryStatement("select count(s1) from " + database + ".load_and_alter"); + RowRecord rec; + rec = dataSet.next(); + assertEquals(21, rec.getFields().get(0).getLongV()); + assertFalse(dataSet.hasNext()); + } + + filesToLoad.forEach( + tsfile -> { + tsfile.delete(); + File resourceFile = new File(tsfile.getAbsolutePath() + ".resource"); + resourceFile.delete(); + }); + filesToLoad.clear(); + } + + @Test + public void testLoadAndAccumulator() + throws IoTDBConnectionException, + StatementExecutionException, + IOException, + WriteProcessException { + try (Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement()) { + statement.execute("set configuration \"enable_seq_space_compaction\"=\"false\""); + statement.execute("set configuration \"enable_unseq_space_compaction\"=\"false\""); + } catch (Exception e) { + fail(e.getMessage()); + } + + try (ISession session = EnvFactory.getEnv().getSessionConnection()) { + if (!session.checkTimeseriesExists("root.sg1.d1.s1")) { + session.createTimeseries( + "root.sg1.d1.s1", TSDataType.DOUBLE, TSEncoding.RLE, CompressionType.SNAPPY); + } + if (!session.checkTimeseriesExists("root.sg1.d1.s2")) { + session.createTimeseries( + "root.sg1.d1.s2", TSDataType.DOUBLE, TSEncoding.RLE, CompressionType.SNAPPY); + } + + Double firstValue = null; + Double lastValue = null; + + // file1-file3 s1=DOUBLE + + // file1-file3 + List filesToLoad = new ArrayList<>(); + for (int i = 1; i <= 3; i++) { + File file = new File("target", "f" + i + ".tsfile"); + List columnNames = Arrays.asList("root.sg1.d1.s1", "root.sg1.d1.s2"); + List columnTypes = Arrays.asList("DOUBLE", "DOUBLE"); + + if (file.exists()) { + Files.delete(file.toPath()); + } + + try (TsFileWriter tsFileWriter = new TsFileWriter(file)) { + // device -> column indices in columnNames + Map> deviceColumnIndices = new HashMap<>(); + Set alignedDevices = new HashSet<>(); + Map> deviceSchemaMap = new LinkedHashMap<>(); + // deviceSchemaMap.put("root.sg1.d1", new ArrayList<>()); + + collectSchemas( + session, + columnNames, + columnTypes, + deviceSchemaMap, + alignedDevices, + deviceColumnIndices); + + List tabletList = constructTablets(deviceSchemaMap, alignedDevices, tsFileWriter); + + if (!tabletList.isEmpty()) { + long timestamp = i; + double dataValue = new Random(10).nextDouble(); + if (firstValue == null) { + firstValue = dataValue; + } + writeWithTablets( + timestamp, + dataValue, + TSDataType.DOUBLE, + tabletList, + alignedDevices, + tsFileWriter, + deviceColumnIndices); + tsFileWriter.flush(); + } else { + fail("No tablets to write"); + } + tsFileWriter.close(); + } + + filesToLoad.add(file); + } + + // load file1-file3 into IoTDB + for (File f : filesToLoad) { + session.executeNonQueryStatement("LOAD '" + f.getAbsolutePath() + "'"); + } + + // clear data + filesToLoad.forEach( + tsfile -> { + tsfile.delete(); + File resourceFile = new File(tsfile.getAbsolutePath() + ".resource"); + resourceFile.delete(); + }); + filesToLoad.clear(); + + // check load result + SessionDataSet dataSet = session.executeQueryStatement("select count(s1) from root.sg1.d1"); + RowRecord rec; + rec = dataSet.next(); + assertEquals(3, rec.getFields().get(0).getLongV()); + assertFalse(dataSet.hasNext()); + + // file4-file6 s1=INT32 + + // file4-file6 + for (int i = 4; i <= 6; i++) { + File file = new File("target", "f" + i + ".tsfile"); + List columnNames = Arrays.asList("root.sg1.d1.s1", "root.sg1.d1.s2"); + List columnTypes = Arrays.asList("INT32", "INT32"); + + if (file.exists()) { + Files.delete(file.toPath()); + } + + try (TsFileWriter tsFileWriter = new TsFileWriter(file)) { + // device -> column indices in columnNames + Map> deviceColumnIndices = new HashMap<>(); + Set alignedDevices = new HashSet<>(); + Map> deviceSchemaMap = new LinkedHashMap<>(); + + collectSchemas( + session, + columnNames, + columnTypes, + deviceSchemaMap, + alignedDevices, + deviceColumnIndices); + + List tabletList = constructTablets(deviceSchemaMap, alignedDevices, tsFileWriter); + + if (!tabletList.isEmpty()) { + long timestamp = i; + int dataValue = new Random(10).nextInt(); + lastValue = Double.valueOf(dataValue); + writeWithTablets( + timestamp, + dataValue, + TSDataType.INT32, + tabletList, + alignedDevices, + tsFileWriter, + deviceColumnIndices); + tsFileWriter.flush(); + } else { + fail("No tablets to write"); + } + tsFileWriter.close(); + } + + filesToLoad.add(file); + } + + // load file4-file6 into IoTDB + for (File f : filesToLoad) { + session.executeNonQueryStatement("LOAD '" + f.getAbsolutePath() + "'"); + } + + // check load result + dataSet = session.executeQueryStatement("select count(s1) from root.sg1.d1"); + rec = dataSet.next(); + assertEquals(6, rec.getFields().get(0).getLongV()); + assertFalse(dataSet.hasNext()); + + dataSet = session.executeQueryStatement("select first_value(s1) from root.sg1.d1"); + rec = dataSet.next(); + assertEquals(firstValue.doubleValue(), rec.getFields().get(0).getDoubleV(), 0.001); + assertFalse(dataSet.hasNext()); + + dataSet = session.executeQueryStatement("select last_value(s1) from root.sg1.d1"); + rec = dataSet.next(); + assertEquals(lastValue.doubleValue(), rec.getFields().get(0).getDoubleV(), 0.001); + assertFalse(dataSet.hasNext()); + + // clear data + filesToLoad.forEach( + tsfile -> { + tsfile.delete(); + File resourceFile = new File(tsfile.getAbsolutePath() + ".resource"); + resourceFile.delete(); + }); + filesToLoad.clear(); + session.executeNonQueryStatement("DELETE TIMESERIES root.sg1.d1.s1"); + } + } + + public void testNonAlignDeviceSequenceDataQuery(TSDataType from, TSDataType to) throws Exception { + try (ISession session = EnvFactory.getEnv().getSessionConnection()) { + session.executeNonQueryStatement("SET CONFIGURATION 'enable_unseq_space_compaction'='false'"); + + // create a table with type of "from" + session.executeNonQueryStatement( + "CREATE TIMESERIES " + database + ".construct_and_alter_column_type.s1 " + from); + session.executeNonQueryStatement( + "CREATE TIMESERIES " + database + ".construct_and_alter_column_type.s2 " + from); + + // write a sequence tsfile point of "from" + Tablet tablet = + new Tablet( + database + ".construct_and_alter_column_type", + Arrays.asList("s1", "s2"), + Arrays.asList(from, from), + Arrays.asList(ColumnCategory.FIELD, ColumnCategory.FIELD)); + + // for (int i = 1; i <= 1024; i++) { + // int rowIndex = tablet.getRowSize(); + // tablet.addTimestamp(0, i); + // tablet.addValue("s1", rowIndex, genValue(from, i)); + // tablet.addValue("s2", rowIndex, genValue(from, i * 2)); + // session.insert(tablet); + // tablet.reset(); + // } + // + // session.executeNonQueryStatement("FLUSH"); + for (int i = 1; i <= 512; i++) { + int rowIndex = tablet.getRowSize(); + tablet.addTimestamp(0, i); + tablet.addValue("s1", rowIndex, genValue(from, i)); + tablet.addValue("s2", rowIndex, genValue(from, i * 2)); + session.insertTablet(tablet); + tablet.reset(); + } + + session.executeNonQueryStatement("FLUSH"); + + for (int i = 513; i <= 1024; i++) { + int rowIndex = tablet.getRowSize(); + tablet.addTimestamp(0, i); + tablet.addValue("s1", rowIndex, genValue(from, i)); + tablet.addValue("s2", rowIndex, genValue(from, i * 2)); + session.insertTablet(tablet); + tablet.reset(); + } + + if (DATA_TYPE_LIST.contains(from) || DATA_TYPE_LIST.contains(to)) { + SessionDataSet dataSet = + session.executeQueryStatement( + "select first_value(s1),last_value(s1) from " + + database + + ".construct_and_alter_column_type"); + RowRecord rec = dataSet.next(); + while (rec != null) { + System.out.println(rec.getFields().toString()); + rec = dataSet.next(); + } + dataSet.close(); + } else { + SessionDataSet dataSet = + session.executeQueryStatement( + "select min_value(s1),max_value(s1),first_value(s1),last_value(s1) from " + + database + + ".construct_and_alter_column_type"); + RowRecord rec = dataSet.next(); + while (rec != null) { + System.out.println(rec.getFields().toString()); + rec = dataSet.next(); + } + dataSet.close(); + } + + try { + standardSelectTest(session, from, to); + standardAccumulatorQueryTest(session, from); + } catch (Exception e) { + log.info(e.getMessage()); + } + + // alter the type to "to" + boolean isCompatible = MetadataUtils.canAlter(from, to); + if (isCompatible) { + session.executeNonQueryStatement( + "ALTER TIMESERIES " + + database + + ".construct_and_alter_column_type.s1 SET DATA TYPE " + + to); + } else { + try { + session.executeNonQueryStatement( + "ALTER TIMESERIES " + + database + + ".construct_and_alter_column_type.s1 SET DATA TYPE " + + to); + } catch (StatementExecutionException e) { + assertEquals( + "701: New type " + to + " is not compatible with the existing one " + from, + e.getMessage()); + } + } + + System.out.println( + "[testNonAlignDeviceSequenceDataQuery] AFTER ALTER TIMESERIES s1 SET DATA TYPE "); + + // If don't execute the flush" operation, verify if result can get valid value, not be null + // when query memtable. + // session.executeNonQueryStatement("FLUSH"); + + TSDataType newType = isCompatible ? to : from; + + try { + standardSelectTestAfterAlterColumnType(from, session, newType); + // Accumulator query test + standardAccumulatorQueryTest(session, from, newType); + } catch (NotSupportedException e) { + log.info(e.getMessage()); + } catch (Exception e) { + log.info(e.getMessage()); + throw new Exception(e); + } + + if (from == TSDataType.DATE) { + accumulatorQueryTestForDateType(session, to); + } + + } finally { + try (ISession session = EnvFactory.getEnv().getSessionConnection()) { + boolean isExist = + session + .executeQueryStatement( + "show timeseries " + database + ".construct_and_alter_column_type.**", timeout) + .hasNext(); + if (isExist) { + session.executeNonQueryStatement( + "DELETE TIMESERIES " + database + ".construct_and_alter_column_type.**"); + } + } + } + } + + public void testAlignDeviceSequenceDataQuery(TSDataType from, TSDataType to) throws Exception { + try (ISession session = EnvFactory.getEnv().getSessionConnection()) { + session.executeNonQueryStatement("SET CONFIGURATION 'enable_unseq_space_compaction'='false'"); + + // create a table with type of "from" + session.executeNonQueryStatement( + "CREATE ALIGNED TIMESERIES " + + database + + ".construct_and_alter_column_type(s1 " + + from + + ", s2 " + + from + + ")"); + + // write a sequence tsfile point of "from" + Tablet tablet = + new Tablet( + database + ".construct_and_alter_column_type", + Arrays.asList("s1", "s2"), + Arrays.asList(from, from), + Arrays.asList(ColumnCategory.FIELD, ColumnCategory.FIELD)); + + // for (int i = 1; i <= 1024; i++) { + // int rowIndex = tablet.getRowSize(); + // tablet.addTimestamp(0, i); + // tablet.addValue("s1", rowIndex, genValue(from, i)); + // tablet.addValue("s2", rowIndex, genValue(from, i * 2)); + // session.insert(tablet); + // tablet.reset(); + // } + // + // session.executeNonQueryStatement("FLUSH"); + for (int i = 1; i <= 512; i++) { + int rowIndex = tablet.getRowSize(); + tablet.addTimestamp(0, i); + tablet.addValue("s1", rowIndex, genValue(from, i)); + tablet.addValue("s2", rowIndex, genValue(from, i * 2)); + session.insertTablet(tablet); + tablet.reset(); + } + + session.executeNonQueryStatement("FLUSH"); + + for (int i = 513; i <= 1024; i++) { + int rowIndex = tablet.getRowSize(); + tablet.addTimestamp(0, i); + tablet.addValue("s1", rowIndex, genValue(from, i)); + tablet.addValue("s2", rowIndex, genValue(from, i * 2)); + session.insertTablet(tablet); + tablet.reset(); + } + + if (DATA_TYPE_LIST.contains(from) || DATA_TYPE_LIST.contains(to)) { + SessionDataSet dataSet = + session.executeQueryStatement( + "select first_value(s1),last_value(s1) from " + + database + + ".construct_and_alter_column_type"); + RowRecord rec = dataSet.next(); + while (rec != null) { + System.out.println(rec.getFields().toString()); + rec = dataSet.next(); + } + dataSet.close(); + } else { + SessionDataSet dataSet = + session.executeQueryStatement( + "select min_value(s1),max_value(s1),first_value(s1),last_value(s1) from " + + database + + ".construct_and_alter_column_type"); + RowRecord rec = dataSet.next(); + while (rec != null) { + System.out.println(rec.getFields().toString()); + rec = dataSet.next(); + } + dataSet.close(); + } + + try { + standardSelectTest(session, from, to); + standardAccumulatorQueryTest(session, from); + } catch (NotSupportedException e) { + log.info(e.getMessage()); + } catch (Exception e) { + log.info(e.getMessage()); + throw new Exception(e); + } + + // alter the type to "to" + boolean isCompatible = MetadataUtils.canAlter(from, to); + if (isCompatible) { + session.executeNonQueryStatement( + "ALTER TIMESERIES " + + database + + ".construct_and_alter_column_type.s1 SET DATA TYPE " + + to); + } else { + try { + session.executeNonQueryStatement( + "ALTER TIMESERIES " + + database + + ".construct_and_alter_column_type.s1 SET DATA TYPE " + + to); + } catch (StatementExecutionException e) { + assertEquals( + "701: New type " + to + " is not compatible with the existing one " + from, + e.getMessage()); + } + } + + System.out.println( + "[testAlignDeviceSequenceDataQuery] AFTER ALTER TIMESERIES s1 SET DATA TYPE "); + + // If don't execute the flush" operation, verify if result can get valid value, not be null + // when query memtable. + // session.executeNonQueryStatement("FLUSH"); + + TSDataType newType = isCompatible ? to : from; + + try { + standardSelectTestAfterAlterColumnType(from, session, newType); + // Accumulator query test + standardAccumulatorQueryTest(session, from, newType); + } catch (NotSupportedException e) { + log.info(e.getMessage()); + } catch (Exception e) { + log.info(e.getMessage()); + throw new Exception(e); + } + + if (from == TSDataType.DATE) { + accumulatorQueryTestForDateType(session, to); + } + + } finally { + try (ISession session = EnvFactory.getEnv().getSessionConnection()) { + boolean isExist = + session + .executeQueryStatement( + "show timeseries " + database + ".construct_and_alter_column_type.**", timeout) + .hasNext(); + if (isExist) { + session.executeNonQueryStatement( + "DELETE TIMESERIES " + database + ".construct_and_alter_column_type.**"); + } + } + } + } + + public void testNonAlignDeviceUnSequenceDataQuery(TSDataType from, TSDataType to) + throws Exception { + try (ISession session = EnvFactory.getEnv().getSessionConnection()) { + session.executeNonQueryStatement("SET CONFIGURATION 'enable_unseq_space_compaction'='false'"); + + // create a table with type of "from" + session.executeNonQueryStatement( + "CREATE TIMESERIES " + database + ".construct_and_alter_column_type.s1 " + from); + session.executeNonQueryStatement( + "CREATE TIMESERIES " + database + ".construct_and_alter_column_type.s2 " + from); + + // write a sequence tsfile point of "from" + Tablet tablet = + new Tablet( + database + ".construct_and_alter_column_type", + Arrays.asList("s1", "s2"), + Arrays.asList(from, from), + Arrays.asList(ColumnCategory.FIELD, ColumnCategory.FIELD)); + + System.out.println(tablet.getSchemas().toString()); + + for (int i = 513; i <= 1024; i++) { + int rowIndex = tablet.getRowSize(); + tablet.addTimestamp(0, i); + tablet.addValue("s1", rowIndex, genValue(from, i)); + tablet.addValue("s2", rowIndex, genValue(from, i * 2)); + session.insertTablet(tablet); + tablet.reset(); + } + + session.executeNonQueryStatement("FLUSH"); + + for (int i = 1; i <= 512; i++) { + int rowIndex = tablet.getRowSize(); + tablet.addTimestamp(0, i); + tablet.addValue("s1", rowIndex, genValue(from, i)); + tablet.addValue("s2", rowIndex, genValue(from, i * 2)); + session.insertTablet(tablet); + tablet.reset(); + } + // session.executeNonQueryStatement("FLUSH"); + + if (DATA_TYPE_LIST.contains(from) || DATA_TYPE_LIST.contains(to)) { + SessionDataSet dataSet = + session.executeQueryStatement( + "select first_value(s1),last_value(s1) from " + + database + + ".construct_and_alter_column_type"); + RowRecord rec = dataSet.next(); + while (rec != null) { + System.out.println(rec.getFields().toString()); + rec = dataSet.next(); + } + dataSet.close(); + } else { + SessionDataSet dataSet = + session.executeQueryStatement( + "select min_value(s1),max_value(s1),first_value(s1),last_value(s1) from " + + database + + ".construct_and_alter_column_type"); + RowRecord rec = dataSet.next(); + while (rec != null) { + System.out.println(rec.getFields().toString()); + rec = dataSet.next(); + } + dataSet.close(); + } + + try { + standardSelectTest(session, from, to); + standardAccumulatorQueryTest(session, from); + } catch (NotSupportedException e) { + log.info(e.getMessage()); + } catch (Exception e) { + log.info(e.getMessage()); + throw new Exception(e); + } + + // alter the type to "to" + boolean isCompatible = MetadataUtils.canAlter(from, to); + if (isCompatible) { + session.executeNonQueryStatement( + "ALTER TIMESERIES " + + database + + ".construct_and_alter_column_type.s1 SET DATA TYPE " + + to); + } else { + try { + session.executeNonQueryStatement( + "ALTER TIMESERIES " + + database + + ".construct_and_alter_column_type.s1 SET DATA TYPE " + + to); + } catch (StatementExecutionException e) { + assertEquals( + "701: New type " + to + " is not compatible with the existing one " + from, + e.getMessage()); + } + } + + System.out.println( + "[testNonAlignDeviceUnSequenceDataQuery] AFTER ALTER TIMESERIES s1 SET DATA TYPE "); + + // If don't execute the flush" operation, verify if result can get valid value, not be null + // when query memtable. + // session.executeNonQueryStatement("FLUSH"); + + TSDataType newType = isCompatible ? to : from; + + try { + standardSelectTestAfterAlterColumnType(from, session, newType); + // Accumulator query test + standardAccumulatorQueryTest(session, from, newType); + } catch (NotSupportedException e) { + log.info(e.getMessage()); + } catch (Exception e) { + log.info(e.getMessage()); + throw new Exception(e); + } + + if (from == TSDataType.DATE) { + accumulatorQueryTestForDateType(session, to); + } + + } finally { + try (ISession session = EnvFactory.getEnv().getSessionConnection()) { + boolean isExist = + session + .executeQueryStatement( + "show timeseries " + database + ".construct_and_alter_column_type.**", timeout) + .hasNext(); + if (isExist) { + session.executeNonQueryStatement( + "DELETE TIMESERIES " + database + ".construct_and_alter_column_type.**"); + } + } + } + } + + public void testAlignDeviceUnSequenceDataQuery(TSDataType from, TSDataType to) throws Exception { + try (ISession session = EnvFactory.getEnv().getSessionConnection()) { + session.executeNonQueryStatement("SET CONFIGURATION 'enable_unseq_space_compaction'='false'"); + + // create a table with type of "from" + session.executeNonQueryStatement( + "CREATE ALIGNED TIMESERIES " + + database + + ".construct_and_alter_column_type(s1 " + + from + + ", s2 " + + from + + ")"); + + // write a sequence tsfile point of "from" + Tablet tablet = + new Tablet( + database + ".construct_and_alter_column_type", + Arrays.asList("s1", "s2"), + Arrays.asList(from, from), + Arrays.asList(ColumnCategory.FIELD, ColumnCategory.FIELD)); + + System.out.println(tablet.getSchemas().toString()); + + for (int i = 513; i <= 1024; i++) { + int rowIndex = tablet.getRowSize(); + tablet.addTimestamp(0, i); + tablet.addValue("s1", rowIndex, genValue(from, i)); + tablet.addValue("s2", rowIndex, genValue(from, i * 2)); + session.insertTablet(tablet); + tablet.reset(); + } + + session.executeNonQueryStatement("FLUSH"); + + for (int i = 1; i <= 512; i++) { + int rowIndex = tablet.getRowSize(); + tablet.addTimestamp(0, i); + tablet.addValue("s1", rowIndex, genValue(from, i)); + tablet.addValue("s2", rowIndex, genValue(from, i * 2)); + session.insertTablet(tablet); + tablet.reset(); + } + // session.executeNonQueryStatement("FLUSH"); + + if (DATA_TYPE_LIST.contains(from) || DATA_TYPE_LIST.contains(to)) { + SessionDataSet dataSet = + session.executeQueryStatement( + "select first_value(s1),last_value(s1) from " + + database + + ".construct_and_alter_column_type"); + RowRecord rec = dataSet.next(); + while (rec != null) { + System.out.println(rec.getFields().toString()); + rec = dataSet.next(); + } + dataSet.close(); + } else { + SessionDataSet dataSet = + session.executeQueryStatement( + "select min_value(s1),max_value(s1),first_value(s1),last_value(s1) from " + + database + + ".construct_and_alter_column_type"); + RowRecord rec = dataSet.next(); + while (rec != null) { + System.out.println(rec.getFields().toString()); + rec = dataSet.next(); + } + dataSet.close(); + } + + try { + standardSelectTest(session, from, to); + standardAccumulatorQueryTest(session, from); + } catch (NotSupportedException e) { + log.info(e.getMessage()); + } catch (Exception e) { + log.info(e.getMessage()); + throw new Exception(e); + } + + // alter the type to "to" + boolean isCompatible = MetadataUtils.canAlter(from, to); + if (isCompatible) { + session.executeNonQueryStatement( + "ALTER TIMESERIES " + + database + + ".construct_and_alter_column_type.s1 SET DATA TYPE " + + to); + } else { + try { + session.executeNonQueryStatement( + "ALTER TIMESERIES " + + database + + ".construct_and_alter_column_type.s1 SET DATA TYPE " + + to); + } catch (StatementExecutionException e) { + assertEquals( + "701: New type " + to + " is not compatible with the existing one " + from, + e.getMessage()); + } + } + + System.out.println( + "[testAlignDeviceUnSequenceDataQuery] AFTER ALTER TIMESERIES s1 SET DATA TYPE "); + + // If don't execute the flush" operation, verify if result can get valid value, not be null + // when query memtable. + // session.executeNonQueryStatement("FLUSH"); + + TSDataType newType = isCompatible ? to : from; + + try { + standardSelectTestAfterAlterColumnType(from, session, newType); + // Accumulator query test + standardAccumulatorQueryTest(session, from, newType); + } catch (NotSupportedException e) { + log.info(e.getMessage()); + } catch (Exception e) { + log.info(e.getMessage()); + throw new Exception(e); + } + + if (from == TSDataType.DATE) { + accumulatorQueryTestForDateType(session, to); + } + + } finally { + try (ISession session = EnvFactory.getEnv().getSessionConnection()) { + boolean isExist = + session + .executeQueryStatement( + "show timeseries " + database + ".construct_and_alter_column_type.**", timeout) + .hasNext(); + if (isExist) { + session.executeNonQueryStatement( + "DELETE TIMESERIES " + database + ".construct_and_alter_column_type.**"); + } + } + } + } + + public void testNonAlignDeviceUnSequenceOverlappedDataQuery(TSDataType from, TSDataType to) + throws Exception { + try (ISession session = EnvFactory.getEnv().getSessionConnection()) { + session.executeNonQueryStatement("SET CONFIGURATION 'enable_unseq_space_compaction'='false'"); + + // create a table with type of "from" + session.executeNonQueryStatement( + "CREATE TIMESERIES " + database + ".construct_and_alter_column_type.s1 " + from); + session.executeNonQueryStatement( + "CREATE TIMESERIES " + database + ".construct_and_alter_column_type.s2 " + from); + + // write a sequence tsfile point of "from" + Tablet tablet = + new Tablet( + database + ".construct_and_alter_column_type", + Arrays.asList("s1", "s2"), + Arrays.asList(from, from), + Arrays.asList(ColumnCategory.FIELD, ColumnCategory.FIELD)); + + System.out.println(tablet.getSchemas().toString()); + + for (int i = 513; i <= 1024; i++) { + int rowIndex = tablet.getRowSize(); + tablet.addTimestamp(0, i); + tablet.addValue("s1", rowIndex, genValue(from, i)); + tablet.addValue("s2", rowIndex, genValue(from, i * 2)); + session.insertTablet(tablet); + tablet.reset(); + } + + session.executeNonQueryStatement("FLUSH"); + + for (int i = 1; i <= 520; i++) { + int rowIndex = tablet.getRowSize(); + tablet.addTimestamp(0, i); + tablet.addValue("s1", rowIndex, genValue(from, i)); + tablet.addValue("s2", rowIndex, genValue(from, i * 2)); + session.insertTablet(tablet); + tablet.reset(); + } + // session.executeNonQueryStatement("FLUSH"); + + if (DATA_TYPE_LIST.contains(from) || DATA_TYPE_LIST.contains(to)) { + SessionDataSet dataSet = + session.executeQueryStatement( + "select first_value(s1),last_value(s1) from " + + database + + ".construct_and_alter_column_type"); + RowRecord rec = dataSet.next(); + while (rec != null) { + System.out.println(rec.getFields().toString()); + rec = dataSet.next(); + } + dataSet.close(); + } else { + SessionDataSet dataSet = + session.executeQueryStatement( + "select min_value(s1),max_value(s1),first_value(s1),last_value(s1) from " + + database + + ".construct_and_alter_column_type"); + RowRecord rec = dataSet.next(); + while (rec != null) { + System.out.println(rec.getFields().toString()); + rec = dataSet.next(); + } + dataSet.close(); + } + + try { + standardSelectTest(session, from, to); + standardAccumulatorQueryTest(session, from); + } catch (NotSupportedException e) { + log.info(e.getMessage()); + } catch (Exception e) { + log.info(e.getMessage()); + throw new Exception(e); + } + + // alter the type to "to" + boolean isCompatible = MetadataUtils.canAlter(from, to); + if (isCompatible) { + session.executeNonQueryStatement( + "ALTER TIMESERIES " + + database + + ".construct_and_alter_column_type.s1 SET DATA TYPE " + + to); + } else { + try { + session.executeNonQueryStatement( + "ALTER TIMESERIES " + + database + + ".construct_and_alter_column_type.s1 SET DATA TYPE " + + to); + } catch (StatementExecutionException e) { + assertEquals( + "701: New type " + to + " is not compatible with the existing one " + from, + e.getMessage()); + } + } + + System.out.println( + "[testNonAlignDeviceUnSequenceOverlappedDataQuery] AFTER ALTER TIMESERIES s1 SET DATA TYPE "); + + // If don't execute the flush" operation, verify if result can get valid value, not be null + // when query memtable. + // session.executeNonQueryStatement("FLUSH"); + + TSDataType newType = isCompatible ? to : from; + + try { + standardSelectTestAfterAlterColumnType(from, session, newType); + // Accumulator query test + standardAccumulatorQueryTest(session, from, newType); + } catch (NotSupportedException e) { + log.info(e.getMessage()); + } catch (Exception e) { + log.info(e.getMessage()); + throw new Exception(e); + } + + if (from == TSDataType.DATE) { + accumulatorQueryTestForDateType(session, to); + } + + } finally { + try (ISession session = EnvFactory.getEnv().getSessionConnection()) { + boolean isExist = + session + .executeQueryStatement( + "show timeseries " + database + ".construct_and_alter_column_type.**", timeout) + .hasNext(); + if (isExist) { + session.executeNonQueryStatement( + "DELETE TIMESERIES " + database + ".construct_and_alter_column_type.**"); + } + } + } + } + + public void testAlignDeviceUnSequenceOverlappedDataQuery(TSDataType from, TSDataType to) + throws Exception { + try (ISession session = EnvFactory.getEnv().getSessionConnection()) { + session.executeNonQueryStatement("SET CONFIGURATION 'enable_unseq_space_compaction'='false'"); + + // create a table with type of "from" + session.executeNonQueryStatement( + "CREATE ALIGNED TIMESERIES " + + database + + ".construct_and_alter_column_type(s1 " + + from + + ", s2 " + + from + + ")"); + + // write a sequence tsfile point of "from" + Tablet tablet = + new Tablet( + database + ".construct_and_alter_column_type", + Arrays.asList("s1", "s2"), + Arrays.asList(from, from), + Arrays.asList(ColumnCategory.FIELD, ColumnCategory.FIELD)); + + System.out.println(tablet.getSchemas().toString()); + + for (int i = 513; i <= 1024; i++) { + int rowIndex = tablet.getRowSize(); + tablet.addTimestamp(0, i); + tablet.addValue("s1", rowIndex, genValue(from, i)); + tablet.addValue("s2", rowIndex, genValue(from, i * 2)); + session.insertTablet(tablet); + tablet.reset(); + } + + session.executeNonQueryStatement("FLUSH"); + + for (int i = 1; i <= 520; i++) { + int rowIndex = tablet.getRowSize(); + tablet.addTimestamp(0, i); + tablet.addValue("s1", rowIndex, genValue(from, i)); + tablet.addValue("s2", rowIndex, genValue(from, i * 2)); + session.insertTablet(tablet); + tablet.reset(); + } + // session.executeNonQueryStatement("FLUSH"); + + if (DATA_TYPE_LIST.contains(from) || DATA_TYPE_LIST.contains(to)) { + SessionDataSet dataSet = + session.executeQueryStatement( + "select first_value(s1),last_value(s1) from " + + database + + ".construct_and_alter_column_type"); + RowRecord rec = dataSet.next(); + while (rec != null) { + System.out.println(rec.getFields().toString()); + rec = dataSet.next(); + } + dataSet.close(); + } else { + SessionDataSet dataSet = + session.executeQueryStatement( + "select min_value(s1),max_value(s1),first_value(s1),last_value(s1) from " + + database + + ".construct_and_alter_column_type"); + RowRecord rec = dataSet.next(); + while (rec != null) { + System.out.println(rec.getFields().toString()); + rec = dataSet.next(); + } + dataSet.close(); + } + + try { + standardSelectTest(session, from, to); + standardAccumulatorQueryTest(session, from); + } catch (NotSupportedException e) { + log.info(e.getMessage()); + } catch (Exception e) { + log.info(e.getMessage()); + throw new Exception(e); + } + + // alter the type to "to" + boolean isCompatible = MetadataUtils.canAlter(from, to); + if (isCompatible) { + session.executeNonQueryStatement( + "ALTER TIMESERIES " + + database + + ".construct_and_alter_column_type.s1 SET DATA TYPE " + + to); + } else { + try { + session.executeNonQueryStatement( + "ALTER TIMESERIES " + + database + + ".construct_and_alter_column_type.s1 SET DATA TYPE " + + to); + } catch (StatementExecutionException e) { + assertEquals( + "701: New type " + to + " is not compatible with the existing one " + from, + e.getMessage()); + } + } + + System.out.println( + "[testAlignDeviceUnSequenceOverlappedDataQuery] AFTER ALTER TIMESERIES s1 SET DATA TYPE "); + + // If don't execute the flush" operation, verify if result can get valid value, not be null + // when query memtable. + // session.executeNonQueryStatement("FLUSH"); + + TSDataType newType = isCompatible ? to : from; + + try { + standardSelectTestAfterAlterColumnType(from, session, newType); + // Accumulator query test + standardAccumulatorQueryTest(session, from, newType); + } catch (Exception e) { + log.info(e.getMessage()); + } + + if (from == TSDataType.DATE) { + accumulatorQueryTestForDateType(session, to); + } + + } finally { + try (ISession session = EnvFactory.getEnv().getSessionConnection()) { + boolean isExist = + session + .executeQueryStatement( + "show timeseries " + database + ".construct_and_alter_column_type.**", timeout) + .hasNext(); + if (isExist) { + session.executeNonQueryStatement( + "DELETE TIMESERIES " + database + ".construct_and_alter_column_type.**"); + } + } + } + } + + // Don't support for non-align device unsequence data query, because non-align timeseries is not + // exist in the table model, only exist align timeseries. + // Though support for non-align timeseries in the tree model, can let tree transfer to table, but + // table is a view table, it don't allow alter column type. + // So can't support functions as below: + // testNonAlignDeviceSequenceDataQuery(); + // testNonAlignDeviceUnSequenceDataQuery(); + + private static void standardSelectTestAfterAlterColumnType( + TSDataType from, ISession session, TSDataType newType) + throws StatementExecutionException, IoTDBConnectionException { + if (from == TSDataType.DATE) { + throw new NotSupportedException("Not supported DATE type."); + } + + // Value Filter Test + // 1.satisfy the condition of value filter completely + SessionDataSet dataSet1; + if (newType == TSDataType.STRING || newType == TSDataType.TEXT) { + if (from == TSDataType.STRING || from == TSDataType.TEXT) { + dataSet1 = + session.executeQueryStatement( + "select s1 from " + + database + + ".construct_and_alter_column_type where s1 >= '1' and s2 > '2' order by time"); + } else if (from == TSDataType.BLOB) { + dataSet1 = + session.executeQueryStatement( + "select s1 from " + + database + + ".construct_and_alter_column_type where cast(s1 as TEXT) >= '1' and cast(s2 as TEXT) > '2' order by time"); + } else if (from == TSDataType.BOOLEAN) { + dataSet1 = + session.executeQueryStatement( + "select s1 from " + + database + + ".construct_and_alter_column_type where cast(s1 as TEXT) = 'true' and cast(s2 as TEXT) = 'true' order by time"); + } else { + dataSet1 = + session.executeQueryStatement( + "select s1 from " + + database + + ".construct_and_alter_column_type where s1 >= '1' and s2 > 2 order by time"); + } + } else if (newType == TSDataType.BLOB) { + // if (from == TSDataType.STRING || from == TSDataType.TEXT) { + if (from == TSDataType.STRING || from == TSDataType.TEXT || from == TSDataType.BLOB) { + dataSet1 = + session.executeQueryStatement( + "select s1 from " + + database + + ".construct_and_alter_column_type where cast(s1 as TEXT) >= '1' and cast(s2 as TEXT) > '2' order by time"); + // "select s1 from construct_and_alter_column_type where cast(s1 as + // TEXT) >= '1' and s2 > '2' order by time"); + } else { + dataSet1 = + session.executeQueryStatement( + "select s1 from " + + database + + ".construct_and_alter_column_type where cast(s1 as INT64) >= 1 and cast(s2 as INT64) > 2 order by time"); + } + } else if (newType == TSDataType.BOOLEAN) { + dataSet1 = + session.executeQueryStatement( + "select s1 from " + + database + + ".construct_and_alter_column_type where s1 = true and s2 = true order by time"); + } else { + if (from == TSDataType.STRING || from == TSDataType.TEXT) { + dataSet1 = + session.executeQueryStatement( + "select s1 from " + + database + + ".construct_and_alter_column_type where s1 >= '1' and s2 > '2' order by time"); + } else { + dataSet1 = + session.executeQueryStatement( + "select s1 from " + + database + + ".construct_and_alter_column_type where s1 >= 1 and s2 > 2 order by time"); + } + } + RowRecord rec1; + for (int i = 2; i <= 3; i++) { + rec1 = dataSet1.next(); + if (from != TSDataType.BOOLEAN) { + assertEquals(i, rec1.getTimestamp()); + if (newType == TSDataType.BLOB) { + assertEquals(genValue(newType, i), rec1.getFields().get(0).getBinaryV()); + } else if (newType == TSDataType.DATE) { + assertEquals(genValue(newType, i), rec1.getFields().get(0).getDateV()); + } else if (newType == TSDataType.STRING || newType == TSDataType.TEXT) { + if (from == TSDataType.DATE) { + assertEquals( + new Binary(genValue(from, i).toString(), StandardCharsets.UTF_8), + rec1.getFields().get(0).getBinaryV()); + } else { + assertEquals( + newType.castFromSingleValue(from, genValue(from, i)), + rec1.getFields().get(0).getBinaryV()); + } + } else { + assertEquals(genValue(newType, i).toString(), rec1.getFields().get(0).toString()); + } + } + } + dataSet1.close(); + + // 2.satisfy the condition of value filter partially + SessionDataSet dataSet2; + if (newType == TSDataType.STRING || newType == TSDataType.TEXT) { + if (from == TSDataType.STRING || from == TSDataType.TEXT) { + dataSet2 = + session.executeQueryStatement( + "select s1 from " + + database + + ".construct_and_alter_column_type where s1 >= '1' or s2 < '2' order by time"); + } else if (from == TSDataType.BLOB) { + dataSet2 = + session.executeQueryStatement( + "select s1 from " + + database + + ".construct_and_alter_column_type where cast(s1 as TEXT) >= '1' or cast(s2 as TEXT) < '2' order by time"); + } else if (from == TSDataType.BOOLEAN) { + dataSet2 = + session.executeQueryStatement( + "select s1 from " + + database + + ".construct_and_alter_column_type where cast(s1 as TEXT) = 'true' or cast(s2 as TEXT) = 'false' order by time"); + } else { + dataSet2 = + session.executeQueryStatement( + "select s1 from " + + database + + ".construct_and_alter_column_type where s1 >= '1' or s2 < 2 order by time"); + } + } else if (newType == TSDataType.BLOB) { + if (from == TSDataType.STRING || from == TSDataType.TEXT || from == TSDataType.BLOB) { + dataSet2 = + session.executeQueryStatement( + "select s1 from " + + database + + ".construct_and_alter_column_type where cast(s1 as TEXT) >= '1' or cast(s2 as TEXT) < '2' order by time"); + } else { + dataSet2 = + session.executeQueryStatement( + "select s1 from " + + database + + ".construct_and_alter_column_type where cast(s1 as INT64) >= 1 or cast(s2 as INT64) < 2 order by time"); + } + } else if (newType == TSDataType.BOOLEAN) { + dataSet2 = + session.executeQueryStatement( + "select s1 from " + + database + + ".construct_and_alter_column_type where s1 = true or s2 = false order by time"); + } else { + if (from == TSDataType.STRING || from == TSDataType.TEXT) { + dataSet2 = + session.executeQueryStatement( + "select s1 from " + + database + + ".construct_and_alter_column_type where s1 >= '1' or s2 < '2' order by time"); + } else { + dataSet2 = + session.executeQueryStatement( + "select s1 from " + + database + + ".construct_and_alter_column_type where s1 >= 1 or s2 < 2 order by time"); + } + } + + RowRecord rec2; + for (int i = 1; i <= 2; i++) { + rec2 = dataSet2.next(); + if (from != TSDataType.BOOLEAN) { + assertEquals(i, rec2.getTimestamp()); + if (newType == TSDataType.BLOB) { + assertEquals(genValue(newType, i), rec2.getFields().get(0).getBinaryV()); + } else if (newType == TSDataType.DATE) { + assertEquals(genValue(newType, i), rec2.getFields().get(0).getDateV()); + } else if (newType == TSDataType.STRING || newType == TSDataType.TEXT) { + if (from == TSDataType.DATE) { + assertEquals( + new Binary(genValue(from, i).toString(), StandardCharsets.UTF_8), + rec2.getFields().get(0).getBinaryV()); + } else { + assertEquals( + newType.castFromSingleValue(from, genValue(from, i)), + rec2.getFields().get(0).getBinaryV()); + } + } else { + assertEquals(genValue(newType, i).toString(), rec2.getFields().get(0).toString()); + } + } + } + dataSet2.close(); + + // 3.can't satisfy the condition of value filter at all + SessionDataSet dataSet3; + if (newType == TSDataType.STRING || newType == TSDataType.TEXT) { + if (from == TSDataType.BLOB) { + dataSet3 = + session.executeQueryStatement( + "select s1 from " + + database + + ".construct_and_alter_column_type where cast(s1 as TEXT) < '1' order by time"); + } else if (from == TSDataType.BOOLEAN) { + dataSet3 = + session.executeQueryStatement( + "select s1 from " + + database + + ".construct_and_alter_column_type where cast(s1 as TEXT) = 'false' order by time"); + } else { + dataSet3 = + session.executeQueryStatement( + "select s1 from " + + database + + ".construct_and_alter_column_type where s1 < '1' order by time"); + } + } else if (newType == TSDataType.BLOB || from == TSDataType.BLOB) { + dataSet3 = + session.executeQueryStatement( + "select s1 from " + + database + + ".construct_and_alter_column_type where cast(s1 as TEXT) < '1' order by time"); + } else if (newType == TSDataType.BOOLEAN) { + dataSet3 = + session.executeQueryStatement( + "select s1 from " + + database + + ".construct_and_alter_column_type where s1 = false order by time"); + } else { + if (from == TSDataType.STRING || from == TSDataType.TEXT) { + dataSet3 = + session.executeQueryStatement( + "select s1 from " + + database + + ".construct_and_alter_column_type where s1 < '1' order by time"); + } else { + dataSet3 = + session.executeQueryStatement( + "select s1 from " + + database + + ".construct_and_alter_column_type where s1 < 1 order by time"); + } + } + if (from != TSDataType.BOOLEAN) { + assertFalse(dataSet3.hasNext()); + } + dataSet3.close(); + + // Time filter + // 1.satisfy the condition of time filter + SessionDataSet dataSet4 = + session.executeQueryStatement( + "select s1 from " + + database + + ".construct_and_alter_column_type where time > 1 order by time"); + RowRecord rec4; + for (int i = 2; i <= 3; i++) { + rec4 = dataSet4.next(); + assertEquals(i, rec4.getTimestamp()); + if (newType == TSDataType.BLOB) { + assertEquals(genValue(newType, i), rec4.getFields().get(0).getBinaryV()); + } else if (newType == TSDataType.DATE) { + assertEquals(genValue(newType, i), rec4.getFields().get(0).getDateV()); + } else if (newType == TSDataType.STRING || newType == TSDataType.TEXT) { + if (from == TSDataType.DATE) { + assertEquals( + new Binary(genValue(from, i).toString(), StandardCharsets.UTF_8), + rec4.getFields().get(0).getBinaryV()); + } else { + assertEquals( + newType.castFromSingleValue(from, genValue(from, i)), + rec4.getFields().get(0).getBinaryV()); + } + } else { + assertEquals(genValue(newType, i).toString(), rec4.getFields().get(0).toString()); + } + } + dataSet4.close(); + } + + private static void standardSelectTest(ISession session, TSDataType from, TSDataType to) + throws StatementExecutionException, IoTDBConnectionException { + if (from == TSDataType.DATE) { + throw new NotSupportedException("Not supported DATE type."); + } + + // Value Filter Test + // 1.satisfy the condition of value filter completely + SessionDataSet dataSet1; + if (from == TSDataType.STRING || from == TSDataType.TEXT) { + dataSet1 = + session.executeQueryStatement( + "select s1, s2 from " + + database + + ".construct_and_alter_column_type where s1 >= '1' and s2 > '2' order by time"); + } else if (from == TSDataType.BLOB) { + dataSet1 = + session.executeQueryStatement( + "select s1, s2 from " + + database + + ".construct_and_alter_column_type where cast(s1 as TEXT) >= '1' and cast(s2 as TEXT) > '2' order by time"); + } else if (from == TSDataType.BOOLEAN) { + dataSet1 = + session.executeQueryStatement( + "select s1, s2 from " + + database + + ".construct_and_alter_column_type where s1 = true and s2 = true order by time"); + } else { + dataSet1 = + session.executeQueryStatement( + "select s1, s2 from " + + database + + ".construct_and_alter_column_type where s1 >= 1 and s2 > 2 order by time"); + } + RowRecord rec1; + for (int i = 2; i <= 3; i++) { + rec1 = dataSet1.next(); + System.out.println("rec1: " + rec1.toString()); + if (from != TSDataType.BOOLEAN) { + assertEquals(genValue(from, i), rec1.getFields().get(0).getObjectValue(from)); + assertEquals(genValue(from, i * 2), rec1.getFields().get(1).getObjectValue(from)); + } + } + dataSet1.close(); + + // 2.satisfy the condition of value filter partially + SessionDataSet dataSet2; + if (from == TSDataType.STRING || from == TSDataType.TEXT) { + dataSet2 = + session.executeQueryStatement( + "select s1, s2 from " + + database + + ".construct_and_alter_column_type where s1 >= '1' or s2 < '2' order by time"); + } else if (from == TSDataType.BLOB) { + dataSet2 = + session.executeQueryStatement( + "select s1, s2 from " + + database + + ".construct_and_alter_column_type where cast(s1 as TEXT) >= '1' or cast(s2 as TEXT) < '2' order by time"); + } else if (from == TSDataType.BOOLEAN) { + dataSet2 = + session.executeQueryStatement( + "select s1, s2 from " + + database + + ".construct_and_alter_column_type where s1 = true or s2 = false order by time"); + } else { + dataSet2 = + session.executeQueryStatement( + "select s1, s2 from " + + database + + ".construct_and_alter_column_type where s1 >= 1 or s2 < 2 order by time"); + } + RowRecord rec2; + for (int i = 1; i <= 2; i++) { + rec2 = dataSet2.next(); + if (from != TSDataType.BOOLEAN) { + assertEquals(genValue(from, i), rec2.getFields().get(0).getObjectValue(from)); + assertEquals(genValue(from, i * 2), rec2.getFields().get(1).getObjectValue(from)); + } + } + dataSet2.close(); + + // 3.can't satisfy the condition of value filter at all + SessionDataSet dataSet3; + if (from == TSDataType.STRING || from == TSDataType.TEXT) { + dataSet3 = + session.executeQueryStatement( + "select s1, s2 from " + + database + + ".construct_and_alter_column_type where s1 < '1' order by time"); + } else if (from == TSDataType.BLOB) { + dataSet3 = + session.executeQueryStatement( + "select s1, s2 from " + + database + + ".construct_and_alter_column_type where cast(s1 as TEXT) < '1' order by time"); + } else if (from == TSDataType.BOOLEAN) { + dataSet3 = + session.executeQueryStatement( + "select s1, s2 from " + + database + + ".construct_and_alter_column_type where s1 = false order by time"); + } else { + dataSet3 = + session.executeQueryStatement( + "select s1, s2 from " + + database + + ".construct_and_alter_column_type where s1 < 1 order by time"); + } + if (from != TSDataType.BOOLEAN) { + assertFalse(dataSet3.hasNext()); + } + dataSet3.close(); + + // Time filter + // 1.satisfy the condition of time filter + SessionDataSet dataSet4 = + session.executeQueryStatement( + "select s1, s2 from " + + database + + ".construct_and_alter_column_type where time > 1 order by time"); + RowRecord rec4; + for (int i = 2; i <= 3; i++) { + rec4 = dataSet4.next(); + if (from != TSDataType.BOOLEAN) { + assertEquals(genValue(from, i), rec4.getFields().get(0).getObjectValue(from)); + assertEquals(genValue(from, i * 2), rec4.getFields().get(1).getObjectValue(from)); + } + } + dataSet4.close(); + } + + private static void standardAccumulatorQueryTest(ISession session, TSDataType newType) + throws StatementExecutionException, IoTDBConnectionException { + if (newType == TSDataType.DATE) { + throw new NotSupportedException("Not supported DATE type."); + } + + SessionDataSet dataSet; + RowRecord rec; + if (!UNSUPPORT_ACCUMULATOR_QUERY_DATA_TYPE_LIST.contains(newType)) { + int[] expectedValue; + int max = 4; + if (DATA_TYPE_LIST.contains(newType)) { + dataSet = + session.executeQueryStatement( + "select first_value(s1),last_value(s1) from " + + database + + ".construct_and_alter_column_type"); + rec = dataSet.next(); + expectedValue = new int[] {1, 1024}; + if (newType == TSDataType.STRING + || newType == TSDataType.TEXT + || newType == TSDataType.BLOB) { + // expectedValue[1] = 999; + } else if (newType == TSDataType.BOOLEAN) { + expectedValue = new int[] {19700102, 19721021}; + } + max = 2; + } else { + dataSet = + session.executeQueryStatement( + "select min_value(s1),max_value(s1),first_value(s1),last_value(s1) from " + + database + + ".construct_and_alter_column_type"); + rec = dataSet.next(); + expectedValue = new int[] {1, 1024, 1, 1024}; + if (newType == TSDataType.STRING + || newType == TSDataType.TEXT + || newType == TSDataType.BLOB) { + expectedValue[1] = 999; + } else if (newType == TSDataType.BOOLEAN) { + expectedValue = new int[] {19700102, 19721021, 19700102, 19721021}; + } + } + + if (newType != TSDataType.BOOLEAN) { + for (int i = 0; i < max; i++) { + if (newType == TSDataType.BLOB) { + assertEquals(genValue(newType, expectedValue[i]), rec.getFields().get(i).getBinaryV()); + } else if (newType == TSDataType.DATE) { + assertEquals(genValue(newType, expectedValue[i]), rec.getFields().get(i).getDateV()); + } else { + log.info( + "i is {}, expected value: {}, actual value: {}", + i, + genValue(newType, expectedValue[i]).toString(), + rec.getFields().get(i).toString()); + assertEquals( + genValue(newType, expectedValue[i]).toString(), rec.getFields().get(i).toString()); + } + } + + assertFalse(dataSet.hasNext()); + + if (newType.isNumeric()) { + dataSet = + session.executeQueryStatement( + "select avg(s1),sum(s1) from " + database + ".construct_and_alter_column_type"); + rec = dataSet.next(); + assertEquals(512.5, rec.getFields().get(0).getDoubleV(), 0.001); + assertEquals(524800.0, rec.getFields().get(1).getDoubleV(), 0.001); + assertFalse(dataSet.hasNext()); + } + } + } + + // can use statistics information + dataSet = + session.executeQueryStatement( + "select count(*) from " + database + ".construct_and_alter_column_type where time > 0"); + rec = dataSet.next(); + assertEquals(1024, rec.getFields().get(0).getLongV()); + assertFalse(dataSet.hasNext()); + + // can't use statistics information + dataSet = + session.executeQueryStatement( + "select count(*) from " + + database + + ".construct_and_alter_column_type where time > 10000"); + rec = dataSet.next(); + assertEquals(0, rec.getFields().get(0).getLongV()); + assertFalse(dataSet.hasNext()); + } + + private static void standardAccumulatorQueryTest( + ISession session, TSDataType from, TSDataType newType) + throws StatementExecutionException, IoTDBConnectionException { + if (from == TSDataType.DATE) { + throw new NotSupportedException("Not supported DATE type."); + } + + if (from == TSDataType.BLOB || newType == TSDataType.BLOB) { + throw new NotSupportedException("Not supported BLOB type."); + } + // if (from == TSDataType.BOOLEAN + // && (newType == TSDataType.STRING || newType == TSDataType.TEXT)) { + if (from == TSDataType.BOOLEAN && DATA_TYPE_LIST.contains(newType)) { + SessionDataSet dataSet = + session.executeQueryStatement( + "select first_value(s1),last_value(s1) from " + + database + + ".construct_and_alter_column_type"); + RowRecord rec = dataSet.next(); + boolean[] expectedValue = {false, true}; + for (int i = 0; i < 2; i++) { + if (newType == TSDataType.STRING || newType == TSDataType.TEXT) { + assertEquals(String.valueOf(expectedValue[i]), rec.getFields().get(i).toString()); + } + } + } else { + SessionDataSet dataSet; + int[] expectedValue; + int max = 4; + if (DATA_TYPE_LIST.contains(newType)) { + dataSet = + session.executeQueryStatement( + "select first_value(s1),last_value(s1) from " + + database + + ".construct_and_alter_column_type"); + expectedValue = new int[] {1, 1024}; + max = 2; + } else { + dataSet = + session.executeQueryStatement( + "select min_value(s1),max_value(s1),first_value(s1),last_value(s1) from " + + database + + ".construct_and_alter_column_type"); + expectedValue = new int[] {1, 1024, 1, 1024}; + if (newType == TSDataType.BOOLEAN) { + expectedValue = new int[] {19700102, 19721021, 19700102, 19721021}; + } + } + RowRecord rec = dataSet.next(); + // if (newType == TSDataType.STRING + // || newType == TSDataType.TEXT + // || newType == TSDataType.BLOB) { + // expectedValue[1] = 999; + // } else if (newType == TSDataType.BOOLEAN) { + // expectedValue = new int[] {19700102, 19721021, 19700102, 19721021}; + // } + if (newType != TSDataType.BOOLEAN) { + for (int i = 0; i < max; i++) { + if (newType == TSDataType.BLOB) { + assertEquals(genValue(newType, expectedValue[i]), rec.getFields().get(i).getBinaryV()); + } else if (newType == TSDataType.DATE) { + assertEquals(genValue(newType, expectedValue[i]), rec.getFields().get(i).getDateV()); + } else if (newType == TSDataType.STRING || newType == TSDataType.TEXT) { + if (from == TSDataType.DATE) { + log.info( + "i is {}, expected value: {}, actual value: {}", + i, + new Binary(genValue(from, expectedValue[i]).toString(), StandardCharsets.UTF_8), + rec.getFields().get(i).getBinaryV()); + assertEquals( + new Binary(genValue(from, expectedValue[i]).toString(), StandardCharsets.UTF_8), + rec.getFields().get(i).getBinaryV()); + } else { + log.info( + "i is {}, expected value: {}, actual value: {}", + i, + newType.castFromSingleValue(from, genValue(from, expectedValue[i])), + rec.getFields().get(i).getBinaryV()); + assertEquals( + newType.castFromSingleValue(from, genValue(from, expectedValue[i])), + rec.getFields().get(i).getBinaryV()); + } + } else { + log.info( + "i is {}, expected value: {}, actual value: {}", + i, + genValue(newType, expectedValue[i]).toString(), + rec.getFields().get(i).toString()); + assertEquals( + genValue(newType, expectedValue[i]).toString(), rec.getFields().get(i).toString()); + } + } + + assertFalse(dataSet.hasNext()); + + if (newType.isNumeric()) { + dataSet = + session.executeQueryStatement( + "select avg(s1),sum(s1) from " + database + ".construct_and_alter_column_type"); + rec = dataSet.next(); + assertEquals(512.5, rec.getFields().get(0).getDoubleV(), 0.001); + assertEquals(524800.0, rec.getFields().get(1).getDoubleV(), 0.001); + assertFalse(dataSet.hasNext()); + } + } + + // can use statistics information + dataSet = + session.executeQueryStatement( + "select count(*) from " + + database + + ".construct_and_alter_column_type where time > 0"); + rec = dataSet.next(); + assertEquals(1024, rec.getFields().get(0).getLongV()); + assertFalse(dataSet.hasNext()); + + // can't use statistics information + dataSet = + session.executeQueryStatement( + "select count(*) from " + + database + + ".construct_and_alter_column_type where time > 10000"); + rec = dataSet.next(); + assertEquals(0, rec.getFields().get(0).getLongV()); + assertFalse(dataSet.hasNext()); + } + } + + private static void accumulatorQueryTestForDateType(ISession session, TSDataType newType) + throws StatementExecutionException, IoTDBConnectionException { + if (newType != TSDataType.STRING && newType != TSDataType.TEXT) { + return; + } + + log.info("Test the result that after transfered newType:"); + + SessionDataSet dataSet = + session.executeQueryStatement( + "select first_value(s1),last_value(s1) from " + + database + + ".construct_and_alter_column_type"); + RowRecord rec = dataSet.next(); + int[] expectedValue = {19700102, 19721021}; + if (newType != TSDataType.BOOLEAN) { + for (int i = 0; i < 2; i++) { + if (newType == TSDataType.STRING || newType == TSDataType.TEXT) { + log.info( + "i is {}, expected value: {}, actual value: {}", + i, + TSDataType.getDateStringValue(expectedValue[i]), + // rec.getFields().get(i).getBinaryV().toString()); + rec.getFields().get(i).getStringValue()); + assertEquals( + TSDataType.getDateStringValue(expectedValue[i]), + rec.getFields().get(i).getBinaryV().toString()); + } + } + } + + // can use statistics information + dataSet = + session.executeQueryStatement( + "select count(*) from " + database + ".construct_and_alter_column_type where time > 0"); + rec = dataSet.next(); + assertEquals(1024, rec.getFields().get(0).getLongV()); + assertFalse(dataSet.hasNext()); + + // can't use statistics information + dataSet = + session.executeQueryStatement( + "select count(*) from " + + database + + ".construct_and_alter_column_type where time > 10000"); + rec = dataSet.next(); + assertEquals(0, rec.getFields().get(0).getLongV()); + assertFalse(dataSet.hasNext()); + } + + private static void writeWithTablets( + long timestamp, + Object dataValue, + TSDataType tsDataType, + List tabletList, + Set alignedDevices, + TsFileWriter tsFileWriter, + Map> deviceColumnIndices) + throws IoTDBConnectionException, + StatementExecutionException, + IOException, + WriteProcessException { + RowRecord rowRecord = new RowRecord(timestamp); + rowRecord.addField(dataValue, tsDataType); + rowRecord.addField(dataValue, tsDataType); + List fields = rowRecord.getFields(); + + for (Tablet tablet : tabletList) { + String deviceId = tablet.getDeviceId(); + List columnIndices = deviceColumnIndices.get(deviceId); + int rowIndex = tablet.getRowSize(); + tablet.addTimestamp(rowIndex, rowRecord.getTimestamp()); + List schemas = tablet.getSchemas(); + + for (int i = 0, columnIndicesSize = columnIndices.size(); i < columnIndicesSize; i++) { + Integer columnIndex = columnIndices.get(i); + IMeasurementSchema measurementSchema = schemas.get(i); + // Object value = fields.get(columnIndex - + // 1).getObjectValue(measurementSchema.getType()); + Object value = fields.get(columnIndex).getObjectValue(measurementSchema.getType()); + tablet.addValue(measurementSchema.getMeasurementName(), rowIndex, value); + } + + if (tablet.getRowSize() == tablet.getMaxRowNumber()) { + writeToTsFile(alignedDevices, tsFileWriter, tablet); + tablet.reset(); + } + } + + for (Tablet tablet : tabletList) { + if (tablet.getRowSize() != 0) { + writeToTsFile(alignedDevices, tsFileWriter, tablet); + } + } + } + + private static void writeToTsFile( + Set deviceFilterSet, TsFileWriter tsFileWriter, Tablet tablet) + throws IOException, WriteProcessException { + if (deviceFilterSet.contains(tablet.getDeviceId())) { + tsFileWriter.writeAligned(tablet); + } else { + tsFileWriter.writeTree(tablet); + } + } + + private static List constructTablets( + Map> deviceSchemaMap, + Set alignedDevices, + TsFileWriter tsFileWriter) + throws WriteProcessException { + List tabletList = new ArrayList<>(deviceSchemaMap.size()); + for (Map.Entry> stringListEntry : deviceSchemaMap.entrySet()) { + String deviceId = stringListEntry.getKey(); + List schemaList = stringListEntry.getValue(); + Tablet tablet = new Tablet(deviceId, schemaList); + tablet.initBitMaps(); + Path path = new Path(tablet.getDeviceId()); + if (alignedDevices.contains(tablet.getDeviceId())) { + tsFileWriter.registerAlignedTimeseries(path, schemaList); + } else { + tsFileWriter.registerTimeseries(path, schemaList); + } + tabletList.add(tablet); + } + return tabletList; + } + + protected static TSDataType getType(String typeStr) { + try { + return TSDataType.valueOf(typeStr); + } catch (Exception e) { + return null; + } + } + + private static void collectSchemas( + ISession session, + List columnNames, + List columnTypes, + Map> deviceSchemaMap, + Set alignedDevices, + Map> deviceColumnIndices) + throws IoTDBConnectionException, StatementExecutionException { + for (int i = 0; i < columnNames.size(); i++) { + String column = columnNames.get(i); + if (!column.startsWith("root.")) { + continue; + } + TSDataType tsDataType = getType(columnTypes.get(i)); + Path path = new Path(column, true); + String deviceId = path.getDeviceString(); + // query whether the device is aligned or not + try (SessionDataSet deviceDataSet = + session.executeQueryStatement("show devices " + deviceId, timeout)) { + List deviceList = deviceDataSet.next().getFields(); + if (deviceList.size() > 1 && "true".equals(deviceList.get(1).getStringValue())) { + alignedDevices.add(deviceId); + } + } + + // query timeseries metadata + MeasurementSchema measurementSchema = + new MeasurementSchema(path.getMeasurement(), tsDataType); + List seriesList = + session.executeQueryStatement("show timeseries " + column, timeout).next().getFields(); + measurementSchema.setEncoding(TSEncoding.valueOf(seriesList.get(4).getStringValue())); + measurementSchema.setCompressionType( + CompressionType.valueOf(seriesList.get(5).getStringValue())); + + deviceSchemaMap.computeIfAbsent(deviceId, key -> new ArrayList<>()).add(measurementSchema); + deviceColumnIndices.computeIfAbsent(deviceId, key -> new ArrayList<>()).add(i); + } + } + + @Test + public void testUsingSameColumn() { + assertEquals(true, SchemaUtils.isUsingSameColumn(TSDataType.TIMESTAMP, TSDataType.INT64)); + assertEquals(true, SchemaUtils.isUsingSameColumn(TSDataType.INT64, TSDataType.TIMESTAMP)); + + assertEquals(true, SchemaUtils.isUsingSameColumn(TSDataType.DATE, TSDataType.INT32)); + assertEquals(true, SchemaUtils.isUsingSameColumn(TSDataType.INT32, TSDataType.DATE)); + + assertEquals(true, SchemaUtils.isUsingSameColumn(TSDataType.TEXT, TSDataType.BLOB)); + assertEquals(true, SchemaUtils.isUsingSameColumn(TSDataType.TEXT, TSDataType.STRING)); + assertEquals(true, SchemaUtils.isUsingSameColumn(TSDataType.BLOB, TSDataType.TEXT)); + assertEquals(true, SchemaUtils.isUsingSameColumn(TSDataType.BLOB, TSDataType.STRING)); + assertEquals(true, SchemaUtils.isUsingSameColumn(TSDataType.STRING, TSDataType.BLOB)); + assertEquals(true, SchemaUtils.isUsingSameColumn(TSDataType.STRING, TSDataType.TEXT)); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeIdempotentIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeIdempotentIT.java index ffb2fe902cf61..6c22d41822694 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeIdempotentIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeIdempotentIT.java @@ -204,6 +204,14 @@ public void testSetTableColumnCommentIdempotent() throws Exception { Collections.singletonList("create table test(a tag)"), "COMMENT ON COLUMN test.a IS 'tag'"); } + @Test + public void testAlterColumnDataTypeIdempotent() throws Exception { + testTableConfigIdempotent( + Collections.singletonList( + "CREATE TABLE t1 (time TIMESTAMP TIME,dId STRING TAG,s1 INT32 FIELD)"), + "ALTER TABLE t1 ALTER COLUMN s1 SET DATA TYPE INT64"); + } + private void testTableConfigIdempotent(final List beforeSqlList, final String testSql) throws Exception { final String database = "test"; diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeTypeConversionISessionIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeTypeConversionISessionIT.java index 8e6e13c3b69de..54f4fe2c51fe0 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeTypeConversionISessionIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeTypeConversionISessionIT.java @@ -60,6 +60,7 @@ import static org.awaitility.Awaitility.await; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; import static org.junit.Assert.fail; @RunWith(IoTDBTestRunner.class) @@ -367,7 +368,8 @@ private void prepareTypeConversionTest( expectedValues, tablet.getTimestamps()); } catch (Exception e) { - fail(); + e.printStackTrace(); + fail(e.getMessage()); } }); senderSession.close(); @@ -410,12 +412,18 @@ private void validateResultSet( int index = 0; while (dataSet.hasNext()) { RowRecord record = dataSet.next(); + System.out.println("QueryResult: " + record.toString()); + System.out.println("Expected: " + timestamps[index] + "-" + values.get(index)); List fields = record.getFields(); assertEquals(record.getTimestamp(), timestamps[index]); List rowValues = values.get(index++); for (int i = 0; i < fields.size(); i++) { Field field = fields.get(i); + if (field.getDataType() == null) { + assertNull(rowValues.get(i)); + continue; + } switch (field.getDataType()) { case INT64: case TIMESTAMP: diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBDeletionTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBDeletionTableIT.java index 3d61b64e71a64..50f9014f90224 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBDeletionTableIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBDeletionTableIT.java @@ -1651,6 +1651,27 @@ private Void randomDeviceDeletion( } // check the point count + int finalI = i; + Awaitility.await() + .atMost(5, TimeUnit.MINUTES) + .pollDelay(2, TimeUnit.SECONDS) + .pollInterval(2, TimeUnit.SECONDS) + .until( + () -> { + ResultSet set = + statement.executeQuery( + "select count(*) from table" + + testNum + + " where time <= " + + currentWrittenTime + + " AND deviceId = 'd" + + finalI + + "'"); + assertTrue(set.next()); + long expectedCnt = + currentWrittenTime + 1 - deviceDeletedPointCounters.get(finalI).get(); + return expectedCnt == set.getLong(1); + }); try (ResultSet set = statement.executeQuery( "select count(*) from table" @@ -1833,7 +1854,8 @@ public void deleteTableOfTheSameNameTest() @Test public void testConcurrentFlushAndRandomDeviceDeletion() throws InterruptedException, ExecutionException, SQLException { - int testNum = 25; + int testNum = 1; + // int testNum = 25; try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); Statement statement = connection.createStatement()) { statement.execute("drop database if exists test"); @@ -1845,9 +1867,12 @@ public void testConcurrentFlushAndRandomDeviceDeletion() } AtomicLong writtenPointCounter = new AtomicLong(-1); + // int fileNumMax = 100; + // int pointPerFile = 100; + // int deviceNum = 4; int fileNumMax = 100; int pointPerFile = 100; - int deviceNum = 4; + int deviceNum = 2; List deviceDeletedPointCounters = new ArrayList<>(deviceNum); for (int i = 0; i < deviceNum; i++) { deviceDeletedPointCounters.add(new AtomicLong(0)); diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBSetSystemStatusTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBSetSystemStatusTableIT.java index 5a3c435fb8b41..efeeb888e3fe8 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBSetSystemStatusTableIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBSetSystemStatusTableIT.java @@ -84,6 +84,7 @@ public void setSystemStatus() { .until( () -> { ResultSet resultSet = statement.executeQuery("SHOW DATANODES"); + int num = 0; try { while (resultSet.next()) { diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBAlterColumnTypeIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBAlterColumnTypeIT.java new file mode 100644 index 0000000000000..b633632908e7e --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBAlterColumnTypeIT.java @@ -0,0 +1,2283 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.relational.it.schema; + +import org.apache.iotdb.commons.utils.MetadataUtils; +import org.apache.iotdb.db.utils.SchemaUtils; +import org.apache.iotdb.isession.ISession; +import org.apache.iotdb.isession.ITableSession; +import org.apache.iotdb.isession.SessionDataSet; +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; + +import org.apache.tsfile.enums.ColumnCategory; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.exception.write.WriteProcessException; +import org.apache.tsfile.file.metadata.ColumnSchema; +import org.apache.tsfile.file.metadata.TableSchema; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.read.common.Field; +import org.apache.tsfile.read.common.Path; +import org.apache.tsfile.read.common.RowRecord; +import org.apache.tsfile.utils.Binary; +import org.apache.tsfile.write.TsFileWriter; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.IMeasurementSchema; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.apache.tsfile.write.v4.ITsFileWriter; +import org.apache.tsfile.write.v4.TsFileWriterBuilder; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.ws.rs.NotSupportedException; + +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.sql.Connection; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.Set; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.apache.iotdb.db.it.utils.TestUtils.prepareData; +import static org.apache.iotdb.db.it.utils.TestUtils.prepareTableData; +import static org.apache.iotdb.relational.it.session.IoTDBSessionRelationalIT.genValue; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +@SuppressWarnings("ResultOfMethodCallIgnored") +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBAlterColumnTypeIT { + + private static final Logger log = LoggerFactory.getLogger(IoTDBAlterColumnTypeIT.class); + private static long timeout = -1; + + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv().getConfig().getDataNodeConfig().setCompactionScheduleInterval(1000); + EnvFactory.getEnv().initClusterEnvironment(); + try (ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) { + session.executeNonQueryStatement("DROP DATABASE IF EXISTS test"); + } + try (ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) { + session.executeNonQueryStatement("CREATE DATABASE IF NOT EXISTS test"); + } + } + + @AfterClass + public static void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + @Test + public void testWriteAndAlter() + throws IoTDBConnectionException, + StatementExecutionException, + IOException, + WriteProcessException { + Set typesToTest = new HashSet<>(); + Collections.addAll(typesToTest, TSDataType.values()); + typesToTest.remove(TSDataType.VECTOR); + typesToTest.remove(TSDataType.UNKNOWN); + + for (TSDataType from : typesToTest) { + for (TSDataType to : typesToTest) { + if (from != to && to.isCompatible(from)) { + System.out.printf("testing %s to %s%n", from, to); + doWriteAndAlter(from, to); + testAlignDeviceSequenceDataQuery(from, to); + testAlignDeviceUnSequenceDataQuery(from, to); + testAlignDeviceUnSequenceOverlappedDataQuery(from, to); + } + } + } + } + + private void doWriteAndAlter(TSDataType from, TSDataType to) + throws IoTDBConnectionException, StatementExecutionException { + try (ITableSession session = EnvFactory.getEnv().getTableSessionConnectionWithDB("test")) { + session.executeNonQueryStatement("SET CONFIGURATION enable_unseq_space_compaction='false'"); + if (from == TSDataType.DATE && !to.isCompatible(from)) { + throw new NotSupportedException("Not supported DATE type."); + } + + // create a table with type of "from" + session.executeNonQueryStatement( + "CREATE TABLE IF NOT EXISTS write_and_alter_column_type (s1 " + from + ")"); + + // write a sequence tsfile point of "from" + Tablet tablet = + new Tablet( + "write_and_alter_column_type", + Collections.singletonList("s1"), + Collections.singletonList(from), + Collections.singletonList(ColumnCategory.FIELD)); + tablet.addTimestamp(0, 1); + tablet.addValue("s1", 0, genValue(from, 1)); + session.insert(tablet); + tablet.reset(); + session.executeNonQueryStatement("FLUSH"); + + // write an unsequence tsfile point of "from" + tablet.addTimestamp(0, 1); + tablet.addValue("s1", 0, genValue(from, 1)); + session.insert(tablet); + tablet.reset(); + session.executeNonQueryStatement("FLUSH"); + + // write a sequence memtable point of "from" + tablet.addTimestamp(0, 2); + tablet.addValue("s1", 0, genValue(from, 2)); + session.insert(tablet); + tablet.reset(); + + // write an unsequence memtable point of "from" + tablet.addTimestamp(0, 1); + tablet.addValue("s1", 0, genValue(from, 1)); + session.insert(tablet); + tablet.reset(); + + SessionDataSet dataSet1 = + session.executeQueryStatement("select * from write_and_alter_column_type order by time"); + RowRecord rec1; + for (int i = 1; i <= 2; i++) { + rec1 = dataSet1.next(); + assertEquals(i, rec1.getFields().get(0).getLongV()); + // System.out.println(i + " is " + rec1.getFields().get(1).toString()); + } + + // alter the type to "to" + boolean isCompatible = MetadataUtils.canAlter(from, to); + if (isCompatible) { + session.executeNonQueryStatement( + "ALTER TABLE write_and_alter_column_type ALTER COLUMN s1 SET DATA TYPE " + to); + } else { + try { + session.executeNonQueryStatement( + "ALTER TABLE write_and_alter_column_type ALTER COLUMN s1 SET DATA TYPE " + to); + } catch (StatementExecutionException e) { + assertEquals( + "701: New type " + to + " is not compatible with the existing one " + from, + e.getMessage()); + } + } + + // If don't execute the flush" operation, verify if result can get valid value, not be null + // when query memtable. + // session.executeNonQueryStatement("FLUSH"); + + SessionDataSet dataSet = + session.executeQueryStatement( + "select time, s1 from write_and_alter_column_type order by time"); + RowRecord rec; + TSDataType newType = isCompatible ? to : from; + for (int i = 1; i <= 2; i++) { + rec = dataSet.next(); + assertEquals(i, rec.getFields().get(0).getLongV()); + if (newType == TSDataType.BLOB) { + assertEquals(genValue(newType, i), rec.getFields().get(1).getBinaryV()); + } else if (newType == TSDataType.DATE) { + assertEquals(genValue(newType, i), rec.getFields().get(1).getDateV()); + } else if (newType == TSDataType.STRING || newType == TSDataType.TEXT) { + if (from == TSDataType.DATE) { + assertEquals( + new Binary(genValue(from, i).toString(), StandardCharsets.UTF_8), + rec.getFields().get(1).getBinaryV()); + } else { + assertEquals( + newType.castFromSingleValue(from, genValue(from, i)), + rec.getFields().get(1).getBinaryV()); + } + } else { + assertEquals(genValue(newType, i).toString(), rec.getFields().get(1).toString()); + } + } + assertNull(dataSet.next()); + dataSet.close(); + + // write an altered point in sequence and unsequnce tsfile + tablet = + new Tablet( + "write_and_alter_column_type", + Collections.singletonList("s1"), + Collections.singletonList(newType), + Collections.singletonList(ColumnCategory.FIELD)); + tablet.addTimestamp(0, 3); + tablet.addValue("s1", 0, genValue(newType, 3)); + session.insert(tablet); + tablet.reset(); + + tablet.addTimestamp(0, 1); + tablet.addValue("s1", 0, genValue(newType, 1)); + session.insert(tablet); + tablet.reset(); + session.executeNonQueryStatement("FLUSH"); + + // write an altered point in sequence and unsequnce memtable + tablet.addTimestamp(0, 4); + tablet.addValue("s1", 0, genValue(newType, 4)); + session.insert(tablet); + tablet.reset(); + + tablet.addTimestamp(0, 2); + tablet.addValue("s1", 0, genValue(newType, 2)); + session.insert(tablet); + tablet.reset(); + + dataSet = + session.executeQueryStatement( + "select time, s1 from write_and_alter_column_type order by time"); + for (int i = 1; i <= 4; i++) { + rec = dataSet.next(); + assertEquals(i, rec.getFields().get(0).getLongV()); + if (newType == TSDataType.BLOB) { + assertEquals(genValue(newType, i), rec.getFields().get(1).getBinaryV()); + } else if (newType == TSDataType.DATE) { + assertEquals(genValue(newType, i), rec.getFields().get(1).getDateV()); + } else if (newType == TSDataType.STRING || newType == TSDataType.TEXT) { + assertEquals(genValue(to, i), rec.getFields().get(1).getBinaryV()); + } else { + assertEquals(genValue(newType, i).toString(), rec.getFields().get(1).toString()); + } + } + assertFalse(dataSet.hasNext()); + + dataSet = + session.executeQueryStatement( + "select min(s1),max(s1),first(s1),last(s1) from write_and_alter_column_type"); + rec = dataSet.next(); + int[] expectedValue = {1, 4, 1, 4}; + for (int i = 0; i < 4; i++) { + if (newType == TSDataType.BLOB) { + assertEquals(genValue(newType, expectedValue[i]), rec.getFields().get(i).getBinaryV()); + } else if (newType == TSDataType.DATE) { + assertEquals(genValue(newType, expectedValue[i]), rec.getFields().get(i).getDateV()); + } else { + assertEquals( + genValue(newType, expectedValue[i]).toString(), rec.getFields().get(i).toString()); + } + } + assertFalse(dataSet.hasNext()); + + if (newType.isNumeric()) { + dataSet = + session.executeQueryStatement( + "select avg(s1),sum(s1) from write_and_alter_column_type"); + rec = dataSet.next(); + assertEquals(2.5, rec.getFields().get(0).getDoubleV(), 0.001); + assertEquals(10.0, rec.getFields().get(1).getDoubleV(), 0.001); + assertFalse(dataSet.hasNext()); + } + + } finally { + try (ITableSession session = EnvFactory.getEnv().getTableSessionConnectionWithDB("test")) { + session.executeNonQueryStatement("DROP TABLE write_and_alter_column_type"); + } + } + } + + @Test + public void testAlterWithoutWrite() throws IoTDBConnectionException, StatementExecutionException { + Set typesToTest = new HashSet<>(); + Collections.addAll(typesToTest, TSDataType.values()); + typesToTest.remove(TSDataType.VECTOR); + typesToTest.remove(TSDataType.UNKNOWN); + + for (TSDataType from : typesToTest) { + for (TSDataType to : typesToTest) { + if (from != to && to.isCompatible(from)) { + System.out.printf("testing %s to %s%n", from, to); + doAlterWithoutWrite(from, to, false); + doAlterWithoutWrite(from, to, true); + } + } + } + } + + private void doAlterWithoutWrite(TSDataType from, TSDataType to, boolean flush) + throws IoTDBConnectionException, StatementExecutionException { + try (ITableSession session = EnvFactory.getEnv().getTableSessionConnectionWithDB("test")) { + if (from == TSDataType.DATE && !to.isCompatible(from)) { + throw new NotSupportedException("Not supported DATE type."); + } + + // create a table with type of "from" + session.executeNonQueryStatement( + "CREATE TABLE IF NOT EXISTS just_alter_column_type (s1 " + from + ")"); + + // alter the type to "to" + boolean isCompatible = MetadataUtils.canAlter(from, to); + if (isCompatible) { + session.executeNonQueryStatement( + "ALTER TABLE just_alter_column_type ALTER COLUMN s1 SET DATA TYPE " + to); + } else { + try { + session.executeNonQueryStatement( + "ALTER TABLE just_alter_column_type ALTER COLUMN s1 SET DATA TYPE " + to); + } catch (StatementExecutionException e) { + assertEquals( + "701: New type " + to + " is not compatible with the existing one " + from, + e.getMessage()); + } + } + + TSDataType newType = isCompatible ? to : from; + + // write a point + Tablet tablet = + new Tablet( + "just_alter_column_type", + Collections.singletonList("s1"), + Collections.singletonList(newType), + Collections.singletonList(ColumnCategory.FIELD)); + tablet.addTimestamp(0, 1); + tablet.addValue("s1", 0, genValue(newType, 1)); + session.insert(tablet); + tablet.reset(); + + if (flush) { + session.executeNonQueryStatement("FLUSH"); + } + + SessionDataSet dataSet = + session.executeQueryStatement("select * from just_alter_column_type order by time"); + RowRecord rec = dataSet.next(); + assertEquals(1, rec.getFields().get(0).getLongV()); + if (newType == TSDataType.BLOB) { + assertEquals(genValue(newType, 1), rec.getFields().get(1).getBinaryV()); + } else if (newType == TSDataType.DATE) { + assertEquals(genValue(newType, 1), rec.getFields().get(1).getDateV()); + } else { + assertEquals(genValue(newType, 1).toString(), rec.getFields().get(1).toString()); + } + + assertFalse(dataSet.hasNext()); + + session.executeNonQueryStatement("DROP TABLE just_alter_column_type"); + } + } + + @Test + public void testAlterNonExist() throws IoTDBConnectionException, StatementExecutionException { + try (ITableSession session = EnvFactory.getEnv().getTableSessionConnectionWithDB("test")) { + try { + session.executeNonQueryStatement( + "ALTER TABLE non_exist ALTER COLUMN s1 SET DATA TYPE INT64"); + fail("Should throw exception"); + } catch (StatementExecutionException e) { + assertEquals("550: Table 'test.non_exist' does not exist", e.getMessage()); + } + session.executeNonQueryStatement( + "ALTER TABLE IF EXISTS non_exist ALTER COLUMN s1 SET DATA TYPE INT64"); + + session.executeNonQueryStatement("CREATE TABLE IF NOT EXISTS non_exist (s1 int32)"); + + try { + session.executeNonQueryStatement( + "ALTER TABLE non_exist ALTER COLUMN s2 SET DATA TYPE INT64"); + fail("Should throw exception"); + } catch (StatementExecutionException e) { + assertEquals("616: Column s2 in table 'test.non_exist' does not exist.", e.getMessage()); + } + session.executeNonQueryStatement( + "ALTER TABLE non_exist ALTER COLUMN IF EXISTS s2 SET DATA TYPE INT64"); + } + } + + @Test + public void testAlterWrongType() throws IoTDBConnectionException, StatementExecutionException { + try (ITableSession session = EnvFactory.getEnv().getTableSessionConnectionWithDB("test")) { + session.executeNonQueryStatement("CREATE TABLE IF NOT EXISTS wrong_type (s1 int32)"); + + try { + session.executeNonQueryStatement( + "ALTER TABLE wrong_type ALTER COLUMN s1 SET DATA TYPE VECTOR"); + fail("Should throw exception"); + } catch (StatementExecutionException e) { + assertEquals("701: Unknown type: VECTOR", e.getMessage()); + } + } + } + + @Test + public void testDropAndAlter() throws IoTDBConnectionException, StatementExecutionException { + try (ITableSession session = EnvFactory.getEnv().getTableSessionConnectionWithDB("test")) { + session.executeNonQueryStatement("SET CONFIGURATION enable_unseq_space_compaction='false'"); + session.executeNonQueryStatement("SET CONFIGURATION enable_seq_space_compaction='false'"); + session.executeNonQueryStatement("CREATE TABLE IF NOT EXISTS drop_and_alter (s1 int32)"); + + // time=1 and time=2 are INT32 and deleted by drop column + Tablet tablet = + new Tablet( + "drop_and_alter", + Collections.singletonList("s1"), + Collections.singletonList(TSDataType.INT32), + Collections.singletonList(ColumnCategory.FIELD)); + tablet.addTimestamp(0, 1); + tablet.addValue("s1", 0, genValue(TSDataType.INT32, 1)); + session.insert(tablet); + tablet.reset(); + + session.executeNonQueryStatement("FLUSH"); + + tablet = + new Tablet( + "drop_and_alter", + Collections.singletonList("s1"), + Collections.singletonList(TSDataType.INT32), + Collections.singletonList(ColumnCategory.FIELD)); + tablet.addTimestamp(0, 2); + tablet.addValue("s1", 0, genValue(TSDataType.INT32, 2)); + session.insert(tablet); + tablet.reset(); + + session.executeNonQueryStatement("ALTER TABLE drop_and_alter DROP COLUMN s1"); + + // time=3 and time=4 are STRING + tablet = + new Tablet( + "drop_and_alter", + Collections.singletonList("s1"), + Collections.singletonList(TSDataType.STRING), + Collections.singletonList(ColumnCategory.FIELD)); + tablet.addTimestamp(0, 3); + tablet.addValue("s1", 0, genValue(TSDataType.STRING, 3)); + session.insert(tablet); + tablet.reset(); + + session.executeNonQueryStatement("FLUSH"); + + tablet = + new Tablet( + "drop_and_alter", + Collections.singletonList("s1"), + Collections.singletonList(TSDataType.STRING), + Collections.singletonList(ColumnCategory.FIELD)); + tablet.addTimestamp(0, 4); + tablet.addValue("s1", 0, genValue(TSDataType.STRING, 4)); + session.insert(tablet); + tablet.reset(); + + session.executeNonQueryStatement( + "ALTER TABLE drop_and_alter ALTER COLUMN s1 SET DATA TYPE TEXT"); + + // time=5 and time=6 are TEXT + tablet = + new Tablet( + "drop_and_alter", + Collections.singletonList("s1"), + Collections.singletonList(TSDataType.TEXT), + Collections.singletonList(ColumnCategory.FIELD)); + tablet.addTimestamp(0, 5); + tablet.addValue("s1", 0, genValue(TSDataType.STRING, 5)); + session.insert(tablet); + tablet.reset(); + + session.executeNonQueryStatement("FLUSH"); + + tablet = + new Tablet( + "drop_and_alter", + Collections.singletonList("s1"), + Collections.singletonList(TSDataType.TEXT), + Collections.singletonList(ColumnCategory.FIELD)); + tablet.addTimestamp(0, 6); + tablet.addValue("s1", 0, genValue(TSDataType.STRING, 6)); + session.insert(tablet); + tablet.reset(); + + SessionDataSet dataSet = + session.executeQueryStatement("select * from drop_and_alter order by time"); + // s1 is dropped but the time should remain + RowRecord rec; + for (int i = 1; i < 3; i++) { + rec = dataSet.next(); + assertEquals(i, rec.getFields().get(0).getLongV()); + log.error( + "time is {}, value is {}, value type is {}", + rec.getFields().get(0).getLongV(), + rec.getFields().get(1), + rec.getFields().get(1).getDataType()); + // assertNull(rec.getFields().get(1).getDataType()); + } + for (int i = 3; i < 7; i++) { + rec = dataSet.next(); + assertEquals(i, rec.getFields().get(0).getLongV()); + assertEquals(genValue(TSDataType.STRING, i).toString(), rec.getFields().get(1).toString()); + } + assertFalse(dataSet.hasNext()); + } + } + + @Test + public void testContinuousAlter() throws IoTDBConnectionException, StatementExecutionException { + try (ITableSession session = EnvFactory.getEnv().getTableSessionConnectionWithDB("test")) { + session.executeNonQueryStatement("CREATE TABLE IF NOT EXISTS alter_and_alter (s1 int32)"); + + // time=1 and time=2 are INT32 + Tablet tablet = + new Tablet( + "alter_and_alter", + Collections.singletonList("s1"), + Collections.singletonList(TSDataType.INT32), + Collections.singletonList(ColumnCategory.FIELD)); + tablet.addTimestamp(0, 1); + tablet.addValue("s1", 0, genValue(TSDataType.INT32, 1)); + session.insert(tablet); + tablet.reset(); + + session.executeNonQueryStatement("FLUSH"); + + tablet = + new Tablet( + "alter_and_alter", + Collections.singletonList("s1"), + Collections.singletonList(TSDataType.INT32), + Collections.singletonList(ColumnCategory.FIELD)); + tablet.addTimestamp(0, 2); + tablet.addValue("s1", 0, genValue(TSDataType.INT32, 2)); + session.insert(tablet); + tablet.reset(); + + // time=3 and time=4 are FLOAT + session.executeNonQueryStatement( + "ALTER TABLE alter_and_alter ALTER COLUMN s1 SET DATA TYPE FLOAT"); + tablet = + new Tablet( + "alter_and_alter", + Collections.singletonList("s1"), + Collections.singletonList(TSDataType.FLOAT), + Collections.singletonList(ColumnCategory.FIELD)); + tablet.addTimestamp(0, 3); + tablet.addValue("s1", 0, genValue(TSDataType.FLOAT, 3)); + session.insert(tablet); + tablet.reset(); + + session.executeNonQueryStatement("FLUSH"); + + tablet = + new Tablet( + "alter_and_alter", + Collections.singletonList("s1"), + Collections.singletonList(TSDataType.FLOAT), + Collections.singletonList(ColumnCategory.FIELD)); + tablet.addTimestamp(0, 4); + tablet.addValue("s1", 0, genValue(TSDataType.FLOAT, 4)); + session.insert(tablet); + tablet.reset(); + + // time=5 and time=6 are DOUBLE + session.executeNonQueryStatement( + "ALTER TABLE alter_and_alter ALTER COLUMN s1 SET DATA TYPE DOUBLE"); + tablet = + new Tablet( + "alter_and_alter", + Collections.singletonList("s1"), + Collections.singletonList(TSDataType.DOUBLE), + Collections.singletonList(ColumnCategory.FIELD)); + tablet.addTimestamp(0, 5); + tablet.addValue("s1", 0, genValue(TSDataType.DOUBLE, 5)); + session.insert(tablet); + tablet.reset(); + + session.executeNonQueryStatement("FLUSH"); + + tablet = + new Tablet( + "alter_and_alter", + Collections.singletonList("s1"), + Collections.singletonList(TSDataType.DOUBLE), + Collections.singletonList(ColumnCategory.FIELD)); + tablet.addTimestamp(0, 6); + tablet.addValue("s1", 0, genValue(TSDataType.DOUBLE, 6)); + session.insert(tablet); + tablet.reset(); + + SessionDataSet dataSet = + session.executeQueryStatement("select * from alter_and_alter order by time"); + RowRecord rec; + for (int i = 1; i < 7; i++) { + rec = dataSet.next(); + assertEquals(i, rec.getFields().get(0).getLongV()); + assertEquals(genValue(TSDataType.DOUBLE, i).toString(), rec.getFields().get(1).toString()); + } + assertFalse(dataSet.hasNext()); + } + } + + @Test + public void testConcurrentWriteAndAlter() + throws IoTDBConnectionException, StatementExecutionException, InterruptedException { + try (ITableSession session = EnvFactory.getEnv().getTableSessionConnectionWithDB("test")) { + session.executeNonQueryStatement( + "CREATE TABLE IF NOT EXISTS concurrent_write_and_alter (s1 int32)"); + } + + ExecutorService threadPool = Executors.newCachedThreadPool(); + AtomicInteger writeCounter = new AtomicInteger(0); + int maxWrite = 10000; + int flushInterval = 100; + int alterStart = 5000; + threadPool.submit( + () -> { + try { + write(writeCounter, maxWrite, flushInterval); + } catch (IoTDBConnectionException | StatementExecutionException e) { + throw new RuntimeException(e); + } + }); + threadPool.submit( + () -> { + try { + alter(writeCounter, alterStart); + } catch (InterruptedException + | IoTDBConnectionException + | StatementExecutionException e) { + throw new RuntimeException(e); + } + }); + threadPool.shutdown(); + assertTrue(threadPool.awaitTermination(1, TimeUnit.MINUTES)); + + try (ITableSession session = EnvFactory.getEnv().getTableSessionConnectionWithDB("test")) { + SessionDataSet dataSet = + session.executeQueryStatement("select count(s1) from concurrent_write_and_alter"); + RowRecord rec; + rec = dataSet.next(); + assertEquals(maxWrite, rec.getFields().get(0).getLongV()); + assertFalse(dataSet.hasNext()); + } + } + + private void write(AtomicInteger writeCounter, int maxWrite, int flushInterval) + throws IoTDBConnectionException, StatementExecutionException { + try (ITableSession session = EnvFactory.getEnv().getTableSessionConnectionWithDB("test")) { + int writtenCnt = 0; + do { + session.executeNonQueryStatement( + String.format( + "INSERT INTO concurrent_write_and_alter (time, s1) VALUES (%d, %d)", + writtenCnt, writtenCnt)); + if (((writtenCnt + 1) % flushInterval) == 0) { + session.executeNonQueryStatement("FLUSH"); + } + } while ((writtenCnt = writeCounter.incrementAndGet()) < maxWrite); + } + } + + private void alter(AtomicInteger writeCounter, int alterStart) + throws InterruptedException, IoTDBConnectionException, StatementExecutionException { + while (writeCounter.get() < alterStart) { + Thread.sleep(10); + } + try (ITableSession session = EnvFactory.getEnv().getTableSessionConnectionWithDB("test")) { + session.executeNonQueryStatement( + "ALTER TABLE concurrent_write_and_alter ALTER COLUMN s1 SET DATA TYPE DOUBLE"); + } + } + + @Test + public void testLoadAndAlter() + throws IoTDBConnectionException, + StatementExecutionException, + IOException, + WriteProcessException { + try (ITableSession session = EnvFactory.getEnv().getTableSessionConnectionWithDB("test")) { + session.executeNonQueryStatement("SET CONFIGURATION enable_unseq_space_compaction='false'"); + } + + // file1-file4 s1=INT32 + TableSchema schema1 = + new TableSchema( + "load_and_alter", + Arrays.asList( + new ColumnSchema("dId", TSDataType.STRING, ColumnCategory.TAG), + new ColumnSchema("s1", TSDataType.INT32, ColumnCategory.FIELD))); + // file1-file3 single device small range ([1, 1]), may load without split + List filesToLoad = new ArrayList<>(); + for (int i = 1; i <= 3; i++) { + File file = new File("target", "f" + i + ".tsfile"); + try (ITsFileWriter tsFileWriter = + new TsFileWriterBuilder().file(file).tableSchema(schema1).build()) { + Tablet tablet = + new Tablet( + schema1.getTableName(), + Arrays.asList("dId", "s1"), + Arrays.asList(TSDataType.STRING, TSDataType.INT32), + Arrays.asList(ColumnCategory.TAG, ColumnCategory.FIELD)); + tablet.addTimestamp(0, 1); + tablet.addValue("dId", 0, "d" + i); + tablet.addValue("s1", 0, 1); + tsFileWriter.write(tablet); + } + filesToLoad.add(file); + } + // file4 multi device large range ([2, 100_000_000]), load with split + File file = new File("target", "f" + 4 + ".tsfile"); + try (ITsFileWriter tsFileWriter = + new TsFileWriterBuilder().file(file).tableSchema(schema1).build()) { + Tablet tablet = + new Tablet( + schema1.getTableName(), + Arrays.asList("dId", "s1"), + Arrays.asList(TSDataType.STRING, TSDataType.INT32), + Arrays.asList(ColumnCategory.TAG, ColumnCategory.FIELD)); + int rowIndex = 0; + for (int i = 1; i <= 3; i++) { + tablet.addTimestamp(rowIndex, 2); + tablet.addValue("dId", rowIndex, "d" + i); + tablet.addValue("s1", rowIndex, 2); + rowIndex++; + tablet.addTimestamp(rowIndex, 100_000_000); + tablet.addValue("dId", rowIndex, "d" + i); + tablet.addValue("s1", rowIndex, 100_000_000); + rowIndex++; + } + tsFileWriter.write(tablet); + } + filesToLoad.add(file); + + // load file1-file4 + try (ITableSession session = EnvFactory.getEnv().getTableSessionConnectionWithDB("test")) { + for (File f : filesToLoad) { + session.executeNonQueryStatement("LOAD '" + f.getAbsolutePath() + "'"); + } + } + // check load result + try (ITableSession session = EnvFactory.getEnv().getTableSessionConnectionWithDB("test")) { + SessionDataSet dataSet = + session.executeQueryStatement("select count(s1) from load_and_alter"); + RowRecord rec; + rec = dataSet.next(); + assertEquals(9, rec.getFields().get(0).getLongV()); + assertFalse(dataSet.hasNext()); + } + + filesToLoad.forEach( + tsfile -> { + tsfile.delete(); + File resourceFile = new File(tsfile.getAbsolutePath() + ".resource"); + resourceFile.delete(); + }); + filesToLoad.clear(); + + // file5-file8 s1=DOUBLE + TableSchema schema2 = + new TableSchema( + "load_and_alter", + Arrays.asList( + new ColumnSchema("dId", TSDataType.STRING, ColumnCategory.TAG), + new ColumnSchema("s1", TSDataType.DOUBLE, ColumnCategory.FIELD))); + // file5-file7 single device small range ([3, 3]), may load without split + for (int i = 5; i <= 7; i++) { + file = new File("target", "f" + i + ".tsfile"); + try (ITsFileWriter tsFileWriter = + new TsFileWriterBuilder().file(file).tableSchema(schema2).build()) { + Tablet tablet = + new Tablet( + schema2.getTableName(), + Arrays.asList("dId", "s1"), + Arrays.asList(TSDataType.STRING, TSDataType.DOUBLE), + Arrays.asList(ColumnCategory.TAG, ColumnCategory.FIELD)); + tablet.addTimestamp(0, 3); + tablet.addValue("dId", 0, "d" + i); + tablet.addValue("s1", 0, 3.0); + tsFileWriter.write(tablet); + } + filesToLoad.add(file); + } + // file8 multi device large range ([4, 100_000_001]), load with split + file = new File("target", "f" + 8 + ".tsfile"); + try (ITsFileWriter tsFileWriter = + new TsFileWriterBuilder().file(file).tableSchema(schema2).build()) { + Tablet tablet = + new Tablet( + schema2.getTableName(), + Arrays.asList("dId", "s1"), + Arrays.asList(TSDataType.STRING, TSDataType.DOUBLE), + Arrays.asList(ColumnCategory.TAG, ColumnCategory.FIELD)); + int rowIndex = 0; + for (int i = 1; i <= 3; i++) { + tablet.addTimestamp(rowIndex, 4); + tablet.addValue("dId", rowIndex, "d" + i); + tablet.addValue("s1", rowIndex, 4.0); + rowIndex++; + tablet.addTimestamp(rowIndex, 100_000_001); + tablet.addValue("dId", rowIndex, "d" + i); + tablet.addValue("s1", rowIndex, 100_000_001.0); + rowIndex++; + } + tsFileWriter.write(tablet); + } + filesToLoad.add(file); + + // load file5-file8 + try (ITableSession session = EnvFactory.getEnv().getTableSessionConnectionWithDB("test")) { + for (File f : filesToLoad) { + session.executeNonQueryStatement("LOAD '" + f.getAbsolutePath() + "'"); + } + } + // check load result + try (ITableSession session = EnvFactory.getEnv().getTableSessionConnectionWithDB("test")) { + SessionDataSet dataSet = + session.executeQueryStatement("select count(s1) from load_and_alter"); + RowRecord rec; + rec = dataSet.next(); + // Due to the operation of load tsfile execute directly, don't access memtable or generate + // InsertNode object, so don't need to check the data type. + // When query this measurement point, will only find the data of TSDataType.INT32. So this is + // reason what cause we can't find the data of TSDataType.DOUBLE. So result is 9, is not 18. + // assertEquals(18, rec.getFields().get(0).getLongV()); + assertEquals(9, rec.getFields().get(0).getLongV()); + assertFalse(dataSet.hasNext()); + } + + // alter s1 to double + try (ITableSession session = EnvFactory.getEnv().getTableSessionConnectionWithDB("test")) { + session.executeNonQueryStatement( + "ALTER TABLE load_and_alter ALTER COLUMN s1 SET DATA TYPE DOUBLE"); + } + + filesToLoad.forEach( + tsfile -> { + tsfile.delete(); + File resourceFile = new File(tsfile.getAbsolutePath() + ".resource"); + resourceFile.delete(); + }); + filesToLoad.clear(); + + // file9-file12 s1=INT32 + // file9-file11 single device small range ([5, 5]), may load without split + for (int i = 9; i <= 11; i++) { + file = new File("target", "f" + i + ".tsfile"); + try (ITsFileWriter tsFileWriter = + new TsFileWriterBuilder().file(file).tableSchema(schema1).build()) { + Tablet tablet = + new Tablet( + schema1.getTableName(), + Arrays.asList("dId", "s1"), + Arrays.asList(TSDataType.STRING, TSDataType.INT32), + Arrays.asList(ColumnCategory.TAG, ColumnCategory.FIELD)); + tablet.addTimestamp(0, 5); + tablet.addValue("dId", 0, "d" + i); + tablet.addValue("s1", 0, 5); + tsFileWriter.write(tablet); + } + filesToLoad.add(file); + } + // file12 multi device large range ([6, 100_000_002]), load with split + file = new File("target", "f" + 12 + ".tsfile"); + try (ITsFileWriter tsFileWriter = + new TsFileWriterBuilder().file(file).tableSchema(schema1).build()) { + Tablet tablet = + new Tablet( + schema1.getTableName(), + Arrays.asList("dId", "s1"), + Arrays.asList(TSDataType.STRING, TSDataType.INT32), + Arrays.asList(ColumnCategory.TAG, ColumnCategory.FIELD)); + int rowIndex = 0; + for (int i = 1; i <= 3; i++) { + tablet.addTimestamp(rowIndex, 6); + tablet.addValue("dId", rowIndex, "d" + i); + tablet.addValue("s1", rowIndex, 6); + rowIndex++; + tablet.addTimestamp(rowIndex, 100_000_002); + tablet.addValue("dId", rowIndex, "d" + i); + tablet.addValue("s1", rowIndex, 100_000_002); + rowIndex++; + } + tsFileWriter.write(tablet); + } + filesToLoad.add(file); + + // load file9-file12, should succeed + try (ITableSession session = EnvFactory.getEnv().getTableSessionConnectionWithDB("test")) { + for (File f : filesToLoad) { + session.executeNonQueryStatement("LOAD '" + f.getAbsolutePath() + "'"); + } + } + // check load result + try (ITableSession session = EnvFactory.getEnv().getTableSessionConnectionWithDB("test")) { + SessionDataSet dataSet = + session.executeQueryStatement("select count(s1) from load_and_alter"); + RowRecord rec; + rec = dataSet.next(); + assertEquals(27, rec.getFields().get(0).getLongV()); + assertFalse(dataSet.hasNext()); + } + + filesToLoad.forEach( + tsfile -> { + tsfile.delete(); + File resourceFile = new File(tsfile.getAbsolutePath() + ".resource"); + resourceFile.delete(); + }); + filesToLoad.clear(); + } + + @Test + public void testAlterViewType() throws IoTDBConnectionException, StatementExecutionException { + String[] createTreeDataSqls = { + "CREATE ALIGNED TIMESERIES root.db.battery.b0(voltage INT32, current FLOAT)", + "INSERT INTO root.db.battery.b0(time, voltage, current) aligned values (" + + (System.currentTimeMillis() - 100000) + + ", 1, 1)", + "CREATE ALIGNED TIMESERIES root.db.battery.b1(voltage INT32, current FLOAT)", + "INSERT INTO root.db.battery.b1(time, voltage, current) aligned values (1, 1, 1)", + "INSERT INTO root.db.battery.b1(time, voltage, current) aligned values (2, 1, 1)", + "INSERT INTO root.db.battery.b1(time, voltage, current) aligned values (3, 1, 1)", + "INSERT INTO root.db.battery.b1(time, voltage, current) aligned values (4, 1, 1)", + "INSERT INTO root.db.battery.b1(time, voltage, current) aligned values (" + + System.currentTimeMillis() + + ", 1, 1)", + "CREATE TIMESERIES root.db.battery.b2.voltage INT32", + "CREATE TIMESERIES root.db.battery.b2.current FLOAT", + "INSERT INTO root.db.battery.b2(time, voltage, current) values (1, 1, 1)", + "INSERT INTO root.db.battery.b2(time, voltage, current) values (2, 1, 1)", + "INSERT INTO root.db.battery.b2(time, voltage, current) values (3, 1, 1)", + "INSERT INTO root.db.battery.b2(time, voltage, current) values (4, 1, 1)", + "INSERT INTO root.db.battery.b2(time, voltage, current) values (" + + System.currentTimeMillis() + + ", 1, 1)", + "CREATE ALIGNED TIMESERIES root.db.battery.b3(voltage INT32, current FLOAT)", + "INSERT INTO root.db.battery.b3(time, voltage, current) aligned values (" + + (System.currentTimeMillis() - 100000) + + ", 1, 1)", + "CREATE ALIGNED TIMESERIES root.db.battery.b4(voltage INT32, current FLOAT)", + "INSERT INTO root.db.battery.b4(time, voltage, current) aligned values (" + + (System.currentTimeMillis() - 100000) + + ", 1, 1)", + "CREATE ALIGNED TIMESERIES root.db.battery.b5(voltage INT32, current FLOAT)", + "INSERT INTO root.db.battery.b5(time, voltage, current) aligned values (" + + (System.currentTimeMillis() - 100000) + + ", 1, 1)", + "CREATE ALIGNED TIMESERIES root.db.battery.b6(voltage INT32, current FLOAT)", + "INSERT INTO root.db.battery.b6(time, voltage, current) aligned values (" + + (System.currentTimeMillis() - 100000) + + ", 1, 1)", + "CREATE ALIGNED TIMESERIES root.db.battery.b7(voltage INT32, current FLOAT)", + "INSERT INTO root.db.battery.b7(time, voltage, current) aligned values (" + + (System.currentTimeMillis() - 100000) + + ", 1, 1)", + "CREATE ALIGNED TIMESERIES root.db.battery.b8(voltage INT32, current FLOAT)", + "INSERT INTO root.db.battery.b8(time, voltage, current) aligned values (" + + (System.currentTimeMillis() - 100000) + + ", 1, 1)", + "CREATE ALIGNED TIMESERIES root.db.battery.b9(voltage INT32, current FLOAT)", + "INSERT INTO root.db.battery.b9(time, voltage, current) aligned values (" + + (System.currentTimeMillis() - 100000) + + ", 1, 1)", + "flush", + "set ttl to root.db.battery.** 200000", + "set ttl to root.db.battery.b0 50000", + "set ttl to root.db.battery.b6 50000", + }; + + String[] createTableSqls = { + "CREATE DATABASE test", + "USE test", + "CREATE VIEW view1 (battery TAG, voltage INT32 FIELD, current FLOAT FIELD) as root.db.battery.**", + }; + + prepareData(createTreeDataSqls); + prepareTableData(createTableSqls); + + try (ITableSession session = EnvFactory.getEnv().getTableSessionConnectionWithDB("test")) { + SessionDataSet sessionDataSet = + session.executeQueryStatement("select count(*) from view1 where battery = 'b1'"); + assertTrue(sessionDataSet.hasNext()); + RowRecord record = sessionDataSet.next(); + assertEquals(1, record.getField(0).getLongV()); + assertFalse(sessionDataSet.hasNext()); + sessionDataSet.close(); + + sessionDataSet = session.executeQueryStatement("select * from view1"); + int count = 0; + while (sessionDataSet.hasNext()) { + sessionDataSet.next(); + count++; + } + sessionDataSet.close(); + assertEquals(8, count); + + // alter the type to "to" + TSDataType from = TSDataType.FLOAT; + TSDataType to = TSDataType.DOUBLE; + boolean isCompatible = MetadataUtils.canAlter(from, to); + if (isCompatible) { + try { + session.executeNonQueryStatement( + "ALTER TABLE view1 ALTER COLUMN current SET DATA TYPE " + to); + SessionDataSet dataSet = session.executeQueryStatement("DESC view1"); + while (dataSet.hasNext()) { + RowRecord rowRecord = dataSet.next(); + if (rowRecord.getField(0).equals("current")) { + Assert.assertEquals(to, rowRecord.getField(1)); + } + } + } catch (Exception e) { + fail(e.getMessage()); + log.error("ALTER TABLE view1 ALTER COLUMN current SET DATA TYPE {}", to, e); + } + } else { + try { + session.executeNonQueryStatement( + "ALTER TABLE view1 ALTER COLUMN current SET DATA TYPE " + to); + } catch (StatementExecutionException e) { + assertEquals( + "701: New type " + to + " is not compatible with the existing one " + from, + e.getMessage()); + } + } + + session.executeNonQueryStatement("DROP VIEW view1"); + } + + try (ISession session = EnvFactory.getEnv().getSessionConnection()) { + session.executeNonQueryStatement("DELETE TIMESERIES root.db.battery.**"); + } + } + + @Test + public void testLoadAndAccumulator() + throws IoTDBConnectionException, + StatementExecutionException, + IOException, + WriteProcessException { + try (Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement()) { + statement.execute("set configuration \"enable_seq_space_compaction\"=\"false\""); + statement.execute("set configuration \"enable_unseq_space_compaction\"=\"false\""); + } catch (Exception e) { + fail(e.getMessage()); + } + + try (ISession session = EnvFactory.getEnv().getSessionConnection()) { + if (!session.checkTimeseriesExists("root.sg1.d1.s1")) { + session.createTimeseries( + "root.sg1.d1.s1", TSDataType.DOUBLE, TSEncoding.RLE, CompressionType.SNAPPY); + } + if (!session.checkTimeseriesExists("root.sg1.d1.s2")) { + session.createTimeseries( + "root.sg1.d1.s2", TSDataType.DOUBLE, TSEncoding.RLE, CompressionType.SNAPPY); + } + + Double firstValue = null; + Double lastValue = null; + + // file1-file3 s1=DOUBLE + + // file1-file3 + List filesToLoad = new ArrayList<>(); + for (int i = 1; i <= 3; i++) { + File file = new File("target", "f" + i + ".tsfile"); + List columnNames = Arrays.asList("root.sg1.d1.s1", "root.sg1.d1.s2"); + List columnTypes = Arrays.asList("DOUBLE", "DOUBLE"); + + if (file.exists()) { + Files.delete(file.toPath()); + } + + try (TsFileWriter tsFileWriter = new TsFileWriter(file)) { + // device -> column indices in columnNames + Map> deviceColumnIndices = new HashMap<>(); + Set alignedDevices = new HashSet<>(); + Map> deviceSchemaMap = new LinkedHashMap<>(); + // deviceSchemaMap.put("root.sg1.d1", new ArrayList<>()); + + collectSchemas( + session, + columnNames, + columnTypes, + deviceSchemaMap, + alignedDevices, + deviceColumnIndices); + + List tabletList = constructTablets(deviceSchemaMap, alignedDevices, tsFileWriter); + + if (!tabletList.isEmpty()) { + long timestamp = i; + double dataValue = new Random(10).nextDouble(); + if (firstValue == null) { + firstValue = dataValue; + } + writeWithTablets( + timestamp, + dataValue, + TSDataType.DOUBLE, + tabletList, + alignedDevices, + tsFileWriter, + deviceColumnIndices); + tsFileWriter.flush(); + } else { + fail("No tablets to write"); + } + tsFileWriter.close(); + } + + filesToLoad.add(file); + } + + // load file1-file3 into IoTDB + for (File f : filesToLoad) { + session.executeNonQueryStatement("LOAD '" + f.getAbsolutePath() + "'"); + } + + // clear data + filesToLoad.forEach( + tsfile -> { + tsfile.delete(); + File resourceFile = new File(tsfile.getAbsolutePath() + ".resource"); + resourceFile.delete(); + }); + filesToLoad.clear(); + + // check load result + SessionDataSet dataSet = session.executeQueryStatement("select count(s1) from root.sg1.d1"); + RowRecord rec; + rec = dataSet.next(); + assertEquals(3, rec.getFields().get(0).getLongV()); + assertFalse(dataSet.hasNext()); + + // file4-file6 s1=INT32 + + // file4-file6 + for (int i = 4; i <= 6; i++) { + File file = new File("target", "f" + i + ".tsfile"); + List columnNames = Arrays.asList("root.sg1.d1.s1", "root.sg1.d1.s2"); + List columnTypes = Arrays.asList("INT32", "INT32"); + + if (file.exists()) { + Files.delete(file.toPath()); + } + + try (TsFileWriter tsFileWriter = new TsFileWriter(file)) { + // device -> column indices in columnNames + Map> deviceColumnIndices = new HashMap<>(); + Set alignedDevices = new HashSet<>(); + Map> deviceSchemaMap = new LinkedHashMap<>(); + + collectSchemas( + session, + columnNames, + columnTypes, + deviceSchemaMap, + alignedDevices, + deviceColumnIndices); + + List tabletList = constructTablets(deviceSchemaMap, alignedDevices, tsFileWriter); + + if (!tabletList.isEmpty()) { + long timestamp = i; + int dataValue = new Random(10).nextInt(); + lastValue = Double.valueOf(dataValue); + writeWithTablets( + timestamp, + dataValue, + TSDataType.INT32, + tabletList, + alignedDevices, + tsFileWriter, + deviceColumnIndices); + tsFileWriter.flush(); + } else { + fail("No tablets to write"); + } + tsFileWriter.close(); + } + + filesToLoad.add(file); + } + + // load file4-file6 into IoTDB + for (File f : filesToLoad) { + session.executeNonQueryStatement("LOAD '" + f.getAbsolutePath() + "'"); + } + + // check load result + dataSet = session.executeQueryStatement("select count(s1) from root.sg1.d1"); + rec = dataSet.next(); + assertEquals(6, rec.getFields().get(0).getLongV()); + assertFalse(dataSet.hasNext()); + + dataSet = session.executeQueryStatement("select first_value(s1) from root.sg1.d1"); + rec = dataSet.next(); + assertEquals(firstValue.doubleValue(), rec.getFields().get(0).getDoubleV(), 0.001); + assertFalse(dataSet.hasNext()); + + dataSet = session.executeQueryStatement("select last_value(s1) from root.sg1.d1"); + rec = dataSet.next(); + assertEquals(lastValue.doubleValue(), rec.getFields().get(0).getDoubleV(), 0.001); + assertFalse(dataSet.hasNext()); + + // clear data + filesToLoad.forEach( + tsfile -> { + tsfile.delete(); + File resourceFile = new File(tsfile.getAbsolutePath() + ".resource"); + resourceFile.delete(); + }); + filesToLoad.clear(); + session.executeNonQueryStatement("DELETE TIMESERIES root.sg1.d1.s1"); + } + } + + public void testAlignDeviceSequenceDataQuery(TSDataType from, TSDataType to) + throws IoTDBConnectionException, + StatementExecutionException, + IOException, + WriteProcessException { + try (ITableSession session = EnvFactory.getEnv().getTableSessionConnectionWithDB("test")) { + session.executeNonQueryStatement("SET CONFIGURATION enable_unseq_space_compaction='false'"); + + // create a table with type of "from" + session.executeNonQueryStatement( + "CREATE TABLE IF NOT EXISTS construct_and_alter_column_type (s1 " + + from + + ", s2 " + + from + + ")"); + + // write a sequence tsfile point of "from" + Tablet tablet = + new Tablet( + "construct_and_alter_column_type", + Arrays.asList("s1", "s2"), + Arrays.asList(from, from), + Arrays.asList(ColumnCategory.FIELD, ColumnCategory.FIELD)); + + // for (int i = 1; i <= 1024; i++) { + // int rowIndex = tablet.getRowSize(); + // tablet.addTimestamp(0, i); + // tablet.addValue("s1", rowIndex, genValue(from, i)); + // tablet.addValue("s2", rowIndex, genValue(from, i * 2)); + // session.insert(tablet); + // tablet.reset(); + // } + // + // session.executeNonQueryStatement("FLUSH"); + for (int i = 1; i <= 512; i++) { + int rowIndex = tablet.getRowSize(); + tablet.addTimestamp(0, i); + tablet.addValue("s1", rowIndex, genValue(from, i)); + tablet.addValue("s2", rowIndex, genValue(from, i * 2)); + session.insert(tablet); + tablet.reset(); + } + + session.executeNonQueryStatement("FLUSH"); + + for (int i = 513; i <= 1024; i++) { + int rowIndex = tablet.getRowSize(); + tablet.addTimestamp(0, i); + tablet.addValue("s1", rowIndex, genValue(from, i)); + tablet.addValue("s2", rowIndex, genValue(from, i * 2)); + session.insert(tablet); + tablet.reset(); + } + + SessionDataSet dataSet = + session.executeQueryStatement( + "select min(s1),max(s1),first(s1),last(s1) from construct_and_alter_column_type"); + RowRecord rec = dataSet.next(); + while (rec != null) { + System.out.println(rec.getFields().toString()); + rec = dataSet.next(); + } + dataSet.close(); + + try { + standardSelectTest(session, from, to); + standardAccumulatorQueryTest(session, from); + } catch (Exception e) { + log.error("{}", e.getStackTrace()); + log.info(e.getMessage()); + } + + // alter the type to "to" + boolean isCompatible = MetadataUtils.canAlter(from, to); + if (isCompatible) { + session.executeNonQueryStatement( + "ALTER TABLE construct_and_alter_column_type ALTER COLUMN s1 SET DATA TYPE " + to); + } else { + try { + session.executeNonQueryStatement( + "ALTER TABLE construct_and_alter_column_type ALTER COLUMN s1 SET DATA TYPE " + to); + } catch (StatementExecutionException e) { + assertEquals( + "701: New type " + to + " is not compatible with the existing one " + from, + e.getMessage()); + } + } + + System.out.println("[testAlignDeviceSequenceDataQuery] AFTER ALTER COLUMN s1 SET DATA TYPE "); + + // If don't execute the flush" operation, verify if result can get valid value, not be null + // when query memtable. + // session.executeNonQueryStatement("FLUSH"); + + TSDataType newType = isCompatible ? to : from; + + try { + standardSelectTestAfterAlterColumnType(from, session, newType); + // Accumulator query test + standardAccumulatorQueryTest(session, from, newType); + } catch (Exception e) { + log.error("{}", e.getStackTrace()); + log.info(e.getMessage()); + } + + if (from == TSDataType.DATE) { + accumulatorQueryTestForDateType(session, to); + } + + session.executeNonQueryStatement("DROP TABLE construct_and_alter_column_type"); + } + } + + public void testAlignDeviceUnSequenceDataQuery(TSDataType from, TSDataType to) + throws IoTDBConnectionException, + StatementExecutionException, + IOException, + WriteProcessException { + try (ITableSession session = EnvFactory.getEnv().getTableSessionConnectionWithDB("test")) { + session.executeNonQueryStatement("SET CONFIGURATION enable_unseq_space_compaction='false'"); + + // create a table with type of "from" + session.executeNonQueryStatement( + "CREATE TABLE IF NOT EXISTS construct_and_alter_column_type (s1 " + + from + + ", s2 " + + from + + ")"); + + // write a sequence tsfile point of "from" + Tablet tablet = + new Tablet( + "construct_and_alter_column_type", + Arrays.asList("s1", "s2"), + Arrays.asList(from, from), + Arrays.asList(ColumnCategory.FIELD, ColumnCategory.FIELD)); + + System.out.println(tablet.getSchemas().toString()); + + for (int i = 513; i <= 1024; i++) { + int rowIndex = tablet.getRowSize(); + tablet.addTimestamp(0, i); + tablet.addValue("s1", rowIndex, genValue(from, i)); + tablet.addValue("s2", rowIndex, genValue(from, i * 2)); + session.insert(tablet); + tablet.reset(); + } + + session.executeNonQueryStatement("FLUSH"); + + for (int i = 1; i <= 512; i++) { + int rowIndex = tablet.getRowSize(); + tablet.addTimestamp(0, i); + tablet.addValue("s1", rowIndex, genValue(from, i)); + tablet.addValue("s2", rowIndex, genValue(from, i * 2)); + session.insert(tablet); + tablet.reset(); + } + // session.executeNonQueryStatement("FLUSH"); + + SessionDataSet dataSet = + session.executeQueryStatement( + "select min(s1),max(s1),first(s1),last(s1) from construct_and_alter_column_type"); + RowRecord rec = dataSet.next(); + while (rec != null) { + System.out.println(rec.getFields().toString()); + rec = dataSet.next(); + } + dataSet.close(); + + try { + standardSelectTest(session, from, to); + standardAccumulatorQueryTest(session, from); + } catch (Exception e) { + log.error("{}", e.getStackTrace()); + log.info(e.getMessage()); + } + + // alter the type to "to" + boolean isCompatible = MetadataUtils.canAlter(from, to); + if (isCompatible) { + session.executeNonQueryStatement( + "ALTER TABLE construct_and_alter_column_type ALTER COLUMN s1 SET DATA TYPE " + to); + } else { + try { + session.executeNonQueryStatement( + "ALTER TABLE construct_and_alter_column_type ALTER COLUMN s1 SET DATA TYPE " + to); + } catch (StatementExecutionException e) { + assertEquals( + "701: New type " + to + " is not compatible with the existing one " + from, + e.getMessage()); + } + } + + System.out.println( + "[testAlignDeviceUnSequenceDataQuery] AFTER ALTER COLUMN s1 SET DATA TYPE "); + + // If don't execute the flush" operation, verify if result can get valid value, not be null + // when query memtable. + // session.executeNonQueryStatement("FLUSH"); + + TSDataType newType = isCompatible ? to : from; + + try { + standardSelectTestAfterAlterColumnType(from, session, newType); + // Accumulator query test + standardAccumulatorQueryTest(session, from, newType); + } catch (Exception e) { + log.error("{}", e.getStackTrace()); + log.info(e.getMessage()); + } + + if (from == TSDataType.DATE) { + accumulatorQueryTestForDateType(session, to); + } + + session.executeNonQueryStatement("DROP TABLE construct_and_alter_column_type"); + } + } + + public void testAlignDeviceUnSequenceOverlappedDataQuery(TSDataType from, TSDataType to) + throws IoTDBConnectionException, + StatementExecutionException, + IOException, + WriteProcessException { + try (ITableSession session = EnvFactory.getEnv().getTableSessionConnectionWithDB("test")) { + session.executeNonQueryStatement("SET CONFIGURATION enable_unseq_space_compaction='false'"); + + // create a table with type of "from" + session.executeNonQueryStatement( + "CREATE TABLE IF NOT EXISTS construct_and_alter_column_type (s1 " + + from + + ", s2 " + + from + + ")"); + + // write a sequence tsfile point of "from" + Tablet tablet = + new Tablet( + "construct_and_alter_column_type", + Arrays.asList("s1", "s2"), + Arrays.asList(from, from), + Arrays.asList(ColumnCategory.FIELD, ColumnCategory.FIELD)); + + System.out.println(tablet.getSchemas().toString()); + + for (int i = 513; i <= 1024; i++) { + int rowIndex = tablet.getRowSize(); + tablet.addTimestamp(0, i); + tablet.addValue("s1", rowIndex, genValue(from, i)); + tablet.addValue("s2", rowIndex, genValue(from, i * 2)); + session.insert(tablet); + tablet.reset(); + } + + session.executeNonQueryStatement("FLUSH"); + + for (int i = 1; i <= 520; i++) { + int rowIndex = tablet.getRowSize(); + tablet.addTimestamp(0, i); + tablet.addValue("s1", rowIndex, genValue(from, i)); + tablet.addValue("s2", rowIndex, genValue(from, i * 2)); + session.insert(tablet); + tablet.reset(); + } + // session.executeNonQueryStatement("FLUSH"); + + SessionDataSet dataSet = + session.executeQueryStatement( + "select min(s1),max(s1),first(s1),last(s1) from construct_and_alter_column_type"); + RowRecord rec = dataSet.next(); + while (rec != null) { + System.out.println(rec.getFields().toString()); + rec = dataSet.next(); + } + dataSet.close(); + + try { + standardSelectTest(session, from, to); + standardAccumulatorQueryTest(session, from); + } catch (Exception e) { + log.error("{}", e.getStackTrace()); + log.info(e.getMessage()); + } + + // alter the type to "to" + boolean isCompatible = MetadataUtils.canAlter(from, to); + if (isCompatible) { + session.executeNonQueryStatement( + "ALTER TABLE construct_and_alter_column_type ALTER COLUMN s1 SET DATA TYPE " + to); + } else { + try { + session.executeNonQueryStatement( + "ALTER TABLE construct_and_alter_column_type ALTER COLUMN s1 SET DATA TYPE " + to); + } catch (StatementExecutionException e) { + assertEquals( + "701: New type " + to + " is not compatible with the existing one " + from, + e.getMessage()); + } + } + + System.out.println( + "[testAlignDeviceUnSequenceDataQuery] AFTER ALTER COLUMN s1 SET DATA TYPE "); + + // If don't execute the flush" operation, verify if result can get valid value, not be null + // when query memtable. + // session.executeNonQueryStatement("FLUSH"); + + TSDataType newType = isCompatible ? to : from; + + try { + standardSelectTestAfterAlterColumnType(from, session, newType); + // Accumulator query test + standardAccumulatorQueryTest(session, from, newType); + } catch (Exception e) { + log.error("{}", e.getStackTrace()); + log.info(e.getMessage()); + } + + if (from == TSDataType.DATE) { + accumulatorQueryTestForDateType(session, to); + } + + session.executeNonQueryStatement("DROP TABLE construct_and_alter_column_type"); + } + } + + // Don't support for non-align device unsequence data query, because non-align timeseries is not + // exist in the table model, only exist align timeseries. + // Though support for non-align timeseries in the tree model, can let tree transfer to table, but + // table is a view table, it don't allow alter column type. + // So can't support functions as below: + // testNonAlignDeviceSequenceDataQuery(); + // testNonAlignDeviceUnSequenceDataQuery(); + + private static void standardSelectTestAfterAlterColumnType( + TSDataType from, ITableSession session, TSDataType newType) + throws StatementExecutionException, IoTDBConnectionException { + if (from == TSDataType.DATE) { + throw new NotSupportedException("Not supported DATE type."); + } + + // Value Filter Test + // 1.satisfy the condition of value filter completely + SessionDataSet dataSet1; + if (newType == TSDataType.STRING || newType == TSDataType.TEXT) { + if (from == TSDataType.STRING || from == TSDataType.TEXT) { + dataSet1 = + session.executeQueryStatement( + "select time, s1 from construct_and_alter_column_type where s1 >= '1' and s2 > '2' order by time"); + } else if (from == TSDataType.BLOB) { + dataSet1 = + session.executeQueryStatement( + "select time, s1 from construct_and_alter_column_type where cast(s1 as TEXT) >= '1' and cast(s2 as TEXT) > '2' order by time"); + } else if (from == TSDataType.BOOLEAN) { + dataSet1 = + session.executeQueryStatement( + "select time, s1 from construct_and_alter_column_type where cast(s1 as TEXT) = 'true' and cast(s2 as TEXT) = 'true' order by time"); + } else { + dataSet1 = + session.executeQueryStatement( + "select time, s1 from construct_and_alter_column_type where s1 >= '1' and s2 > 2 order by time"); + } + } else if (newType == TSDataType.BLOB) { + // if (from == TSDataType.STRING || from == TSDataType.TEXT) { + if (from == TSDataType.STRING || from == TSDataType.TEXT || from == TSDataType.BLOB) { + dataSet1 = + session.executeQueryStatement( + "select time, s1 from construct_and_alter_column_type where cast(s1 as TEXT) >= '1' and cast(s2 as TEXT) > '2' order by time"); + // "select time, s1 from construct_and_alter_column_type where cast(s1 as + // TEXT) >= '1' and s2 > '2' order by time"); + } else { + dataSet1 = + session.executeQueryStatement( + "select time, s1 from construct_and_alter_column_type where cast(s1 as INT64) >= 1 and cast(s2 as INT64) > 2 order by time"); + } + } else if (newType == TSDataType.BOOLEAN) { + dataSet1 = + session.executeQueryStatement( + "select time, s1 from construct_and_alter_column_type where s1 = true and s2 = true order by time"); + } else { + if (from == TSDataType.STRING || from == TSDataType.TEXT) { + dataSet1 = + session.executeQueryStatement( + "select time, s1 from construct_and_alter_column_type where s1 >= '1' and s2 > '2' order by time"); + } else { + dataSet1 = + session.executeQueryStatement( + "select time, s1 from construct_and_alter_column_type where s1 >= 1 and s2 > 2 order by time"); + } + } + RowRecord rec1; + for (int i = 2; i <= 3; i++) { + rec1 = dataSet1.next(); + if (from != TSDataType.BOOLEAN) { + assertEquals(i, rec1.getFields().get(0).getLongV()); + if (newType == TSDataType.BLOB) { + assertEquals(genValue(newType, i), rec1.getFields().get(1).getBinaryV()); + } else if (newType == TSDataType.DATE) { + assertEquals(genValue(newType, i), rec1.getFields().get(1).getDateV()); + } else if (newType == TSDataType.STRING || newType == TSDataType.TEXT) { + if (from == TSDataType.DATE) { + assertEquals( + new Binary(genValue(from, i).toString(), StandardCharsets.UTF_8), + rec1.getFields().get(1).getBinaryV()); + } else { + assertEquals( + newType.castFromSingleValue(from, genValue(from, i)), + rec1.getFields().get(1).getBinaryV()); + } + } else { + assertEquals(genValue(newType, i).toString(), rec1.getFields().get(1).toString()); + } + } + } + dataSet1.close(); + + // 2.satisfy the condition of value filter partially + SessionDataSet dataSet2; + if (newType == TSDataType.STRING || newType == TSDataType.TEXT) { + if (from == TSDataType.STRING || from == TSDataType.TEXT) { + dataSet2 = + session.executeQueryStatement( + "select time, s1 from construct_and_alter_column_type where s1 >= '1' or s2 < '2' order by time"); + } else if (from == TSDataType.BLOB) { + dataSet2 = + session.executeQueryStatement( + "select time, s1 from construct_and_alter_column_type where cast(s1 as TEXT) >= '1' or cast(s2 as TEXT) < '2' order by time"); + } else if (from == TSDataType.BOOLEAN) { + dataSet2 = + session.executeQueryStatement( + "select time, s1 from construct_and_alter_column_type where cast(s1 as TEXT) = 'true' or cast(s2 as TEXT) = 'false' order by time"); + } else { + dataSet2 = + session.executeQueryStatement( + "select time, s1 from construct_and_alter_column_type where s1 >= '1' or s2 < 2 order by time"); + } + } else if (newType == TSDataType.BLOB) { + if (from == TSDataType.STRING || from == TSDataType.TEXT || from == TSDataType.BLOB) { + dataSet2 = + session.executeQueryStatement( + "select time, s1 from construct_and_alter_column_type where cast(s1 as TEXT) >= '1' or cast(s2 as TEXT) < '2' order by time"); + } else { + dataSet2 = + session.executeQueryStatement( + "select time, s1 from construct_and_alter_column_type where cast(s1 as INT64) >= 1 or cast(s2 as INT64) < 2 order by time"); + } + } else if (newType == TSDataType.BOOLEAN) { + dataSet2 = + session.executeQueryStatement( + "select time, s1 from construct_and_alter_column_type where s1 = true or s2 = false order by time"); + } else { + if (from == TSDataType.STRING || from == TSDataType.TEXT) { + dataSet2 = + session.executeQueryStatement( + "select time, s1 from construct_and_alter_column_type where s1 >= '1' or s2 < '2' order by time"); + } else { + dataSet2 = + session.executeQueryStatement( + "select time, s1 from construct_and_alter_column_type where s1 >= 1 or s2 < 2 order by time"); + } + } + + RowRecord rec2; + for (int i = 1; i <= 2; i++) { + rec2 = dataSet2.next(); + if (from != TSDataType.BOOLEAN) { + assertEquals(i, rec2.getFields().get(0).getLongV()); + if (newType == TSDataType.BLOB) { + assertEquals(genValue(newType, i), rec2.getFields().get(1).getBinaryV()); + } else if (newType == TSDataType.DATE) { + assertEquals(genValue(newType, i), rec2.getFields().get(1).getDateV()); + } else if (newType == TSDataType.STRING || newType == TSDataType.TEXT) { + if (from == TSDataType.DATE) { + assertEquals( + new Binary(genValue(from, i).toString(), StandardCharsets.UTF_8), + rec2.getFields().get(1).getBinaryV()); + } else { + assertEquals( + newType.castFromSingleValue(from, genValue(from, i)), + rec2.getFields().get(1).getBinaryV()); + } + } else { + assertEquals(genValue(newType, i).toString(), rec2.getFields().get(1).toString()); + } + } + } + dataSet2.close(); + + // 3.can't satisfy the condition of value filter at all + SessionDataSet dataSet3; + if (newType == TSDataType.STRING || newType == TSDataType.TEXT) { + if (from == TSDataType.BLOB) { + dataSet3 = + session.executeQueryStatement( + "select time, s1 from construct_and_alter_column_type where cast(s1 as TEXT) < '1' order by time"); + } else if (from == TSDataType.BOOLEAN) { + dataSet3 = + session.executeQueryStatement( + "select time, s1 from construct_and_alter_column_type where cast(s1 as TEXT) = 'false' order by time"); + } else { + dataSet3 = + session.executeQueryStatement( + "select time, s1 from construct_and_alter_column_type where s1 < '1' order by time"); + } + } else if (newType == TSDataType.BLOB || from == TSDataType.BLOB) { + dataSet3 = + session.executeQueryStatement( + "select time, s1 from construct_and_alter_column_type where cast(s1 as TEXT) < '1' order by time"); + } else if (newType == TSDataType.BOOLEAN) { + dataSet3 = + session.executeQueryStatement( + "select time, s1 from construct_and_alter_column_type where s1 = false order by time"); + } else { + if (from == TSDataType.STRING || from == TSDataType.TEXT) { + dataSet3 = + session.executeQueryStatement( + "select time, s1 from construct_and_alter_column_type where s1 < '1' order by time"); + } else { + dataSet3 = + session.executeQueryStatement( + "select time, s1 from construct_and_alter_column_type where s1 < 1 order by time"); + } + } + if (from != TSDataType.BOOLEAN) { + assertFalse(dataSet3.hasNext()); + } + dataSet3.close(); + + // Time filter + // 1.satisfy the condition of time filter + SessionDataSet dataSet4 = + session.executeQueryStatement( + "select time, s1 from construct_and_alter_column_type where time > 1 order by time"); + RowRecord rec4; + for (int i = 2; i <= 3; i++) { + rec4 = dataSet4.next(); + assertEquals(i, rec4.getFields().get(0).getLongV()); + if (newType == TSDataType.BLOB) { + assertEquals(genValue(newType, i), rec4.getFields().get(1).getBinaryV()); + } else if (newType == TSDataType.DATE) { + assertEquals(genValue(newType, i), rec4.getFields().get(1).getDateV()); + } else if (newType == TSDataType.STRING || newType == TSDataType.TEXT) { + if (from == TSDataType.DATE) { + assertEquals( + new Binary(genValue(from, i).toString(), StandardCharsets.UTF_8), + rec4.getFields().get(1).getBinaryV()); + } else { + assertEquals( + newType.castFromSingleValue(from, genValue(from, i)), + rec4.getFields().get(1).getBinaryV()); + } + } else { + assertEquals(genValue(newType, i).toString(), rec4.getFields().get(1).toString()); + } + } + dataSet4.close(); + } + + private static void standardSelectTest(ITableSession session, TSDataType from, TSDataType to) + throws StatementExecutionException, IoTDBConnectionException { + if (from == TSDataType.DATE) { + throw new NotSupportedException("Not supported DATE type."); + } + + // Value Filter Test + // 1.satisfy the condition of value filter completely + SessionDataSet dataSet1; + if (from == TSDataType.STRING || from == TSDataType.TEXT) { + dataSet1 = + session.executeQueryStatement( + "select s1, s2 from construct_and_alter_column_type where s1 >= '1' and s2 > '2' order by time"); + } else if (from == TSDataType.BLOB) { + dataSet1 = + session.executeQueryStatement( + "select s1, s2 from construct_and_alter_column_type where cast(s1 as TEXT) >= '1' and cast(s2 as TEXT) > '2' order by time"); + } else if (from == TSDataType.BOOLEAN) { + dataSet1 = + session.executeQueryStatement( + "select s1, s2 from construct_and_alter_column_type where s1 = true and s2 = true order by time"); + } else { + dataSet1 = + session.executeQueryStatement( + "select s1, s2 from construct_and_alter_column_type where s1 >= 1 and s2 > 2 order by time"); + } + RowRecord rec1; + for (int i = 2; i <= 3; i++) { + rec1 = dataSet1.next(); + System.out.println("rec1: " + rec1.toString()); + if (from != TSDataType.BOOLEAN) { + assertEquals(genValue(from, i), rec1.getFields().get(0).getObjectValue(from)); + assertEquals(genValue(from, i * 2), rec1.getFields().get(1).getObjectValue(from)); + } + } + dataSet1.close(); + + // 2.satisfy the condition of value filter partially + SessionDataSet dataSet2; + if (from == TSDataType.STRING || from == TSDataType.TEXT) { + dataSet2 = + session.executeQueryStatement( + "select s1, s2 from construct_and_alter_column_type where s1 >= '1' or s2 < '2' order by time"); + } else if (from == TSDataType.BLOB) { + dataSet2 = + session.executeQueryStatement( + "select s1, s2 from construct_and_alter_column_type where cast(s1 as TEXT) >= '1' or cast(s2 as TEXT) < '2' order by time"); + } else if (from == TSDataType.BOOLEAN) { + dataSet2 = + session.executeQueryStatement( + "select s1, s2 from construct_and_alter_column_type where s1 = true or s2 = false order by time"); + } else { + dataSet2 = + session.executeQueryStatement( + "select s1, s2 from construct_and_alter_column_type where s1 >= 1 or s2 < 2 order by time"); + } + RowRecord rec2; + for (int i = 1; i <= 2; i++) { + rec2 = dataSet2.next(); + if (from != TSDataType.BOOLEAN) { + assertEquals(genValue(from, i), rec2.getFields().get(0).getObjectValue(from)); + assertEquals(genValue(from, i * 2), rec2.getFields().get(1).getObjectValue(from)); + } + } + dataSet2.close(); + + // 3.can't satisfy the condition of value filter at all + SessionDataSet dataSet3; + if (from == TSDataType.STRING || from == TSDataType.TEXT) { + dataSet3 = + session.executeQueryStatement( + "select s1, s2 from construct_and_alter_column_type where s1 < '1' order by time"); + } else if (from == TSDataType.BLOB) { + dataSet3 = + session.executeQueryStatement( + "select s1, s2 from construct_and_alter_column_type where cast(s1 as TEXT) < '1' order by time"); + } else if (from == TSDataType.BOOLEAN) { + dataSet3 = + session.executeQueryStatement( + "select s1, s2 from construct_and_alter_column_type where s1 = false order by time"); + } else { + dataSet3 = + session.executeQueryStatement( + "select s1, s2 from construct_and_alter_column_type where s1 < 1 order by time"); + } + if (from != TSDataType.BOOLEAN) { + assertFalse(dataSet3.hasNext()); + } + dataSet3.close(); + + // Time filter + // 1.satisfy the condition of time filter + SessionDataSet dataSet4 = + session.executeQueryStatement( + "select s1, s2 from construct_and_alter_column_type where time > 1 order by time"); + RowRecord rec4; + for (int i = 2; i <= 3; i++) { + rec4 = dataSet4.next(); + if (from != TSDataType.BOOLEAN) { + assertEquals(genValue(from, i), rec4.getFields().get(0).getObjectValue(from)); + assertEquals(genValue(from, i * 2), rec4.getFields().get(1).getObjectValue(from)); + } + } + dataSet4.close(); + } + + private static void standardAccumulatorQueryTest(ITableSession session, TSDataType newType) + throws StatementExecutionException, IoTDBConnectionException { + if (newType == TSDataType.DATE) { + throw new NotSupportedException("Not supported DATE type."); + } + SessionDataSet dataSet = + session.executeQueryStatement( + "select min(s1),max(s1),first(s1),last(s1) from construct_and_alter_column_type"); + RowRecord rec = dataSet.next(); + int[] expectedValue = {1, 1024, 1, 1024}; + if (newType == TSDataType.STRING || newType == TSDataType.TEXT || newType == TSDataType.BLOB) { + expectedValue[1] = 999; + } else if (newType == TSDataType.BOOLEAN) { + expectedValue = new int[] {19700102, 19721021, 19700102, 19721021}; + } + if (newType != TSDataType.BOOLEAN) { + for (int i = 0; i < 4; i++) { + if (newType == TSDataType.BLOB) { + assertEquals(genValue(newType, expectedValue[i]), rec.getFields().get(i).getBinaryV()); + } else if (newType == TSDataType.DATE) { + assertEquals(genValue(newType, expectedValue[i]), rec.getFields().get(i).getDateV()); + } else { + log.info( + "i is {}, expected value: {}, actual value: {}", + i, + genValue(newType, expectedValue[i]).toString(), + rec.getFields().get(i).toString()); + assertEquals( + genValue(newType, expectedValue[i]).toString(), rec.getFields().get(i).toString()); + } + } + + assertFalse(dataSet.hasNext()); + + if (newType.isNumeric()) { + dataSet = + session.executeQueryStatement( + "select avg(s1),sum(s1) from construct_and_alter_column_type"); + rec = dataSet.next(); + assertEquals(512.5, rec.getFields().get(0).getDoubleV(), 0.001); + assertEquals(524800.0, rec.getFields().get(1).getDoubleV(), 0.001); + assertFalse(dataSet.hasNext()); + } + } + + // can use statistics information + dataSet = + session.executeQueryStatement( + "select count(*) from construct_and_alter_column_type where time > 0"); + rec = dataSet.next(); + assertEquals(1024, rec.getFields().get(0).getLongV()); + assertFalse(dataSet.hasNext()); + + // can't use statistics information + dataSet = + session.executeQueryStatement( + "select count(*) from construct_and_alter_column_type where time > 10000"); + rec = dataSet.next(); + assertEquals(0, rec.getFields().get(0).getLongV()); + assertFalse(dataSet.hasNext()); + } + + private static void standardAccumulatorQueryTest( + ITableSession session, TSDataType from, TSDataType newType) + throws StatementExecutionException, IoTDBConnectionException { + if (from == TSDataType.DATE) { + throw new NotSupportedException("Not supported DATE type."); + } + if (from == TSDataType.BOOLEAN + && (newType == TSDataType.STRING || newType == TSDataType.TEXT)) { + SessionDataSet dataSet = + session.executeQueryStatement( + "select first(s1),last(s1) from construct_and_alter_column_type"); + RowRecord rec = dataSet.next(); + boolean[] expectedValue = {false, true}; + for (int i = 0; i < 2; i++) { + if (newType == TSDataType.STRING || newType == TSDataType.TEXT) { + assertEquals(String.valueOf(expectedValue[i]), rec.getFields().get(i).toString()); + } + } + } else { + SessionDataSet dataSet = + session.executeQueryStatement( + "select min(s1),max(s1),first(s1),last(s1) from construct_and_alter_column_type"); + RowRecord rec = dataSet.next(); + int[] expectedValue = {1, 1024, 1, 1024}; + if (newType == TSDataType.STRING + || newType == TSDataType.TEXT + || newType == TSDataType.BLOB) { + expectedValue[1] = 999; + } else if (newType == TSDataType.BOOLEAN) { + expectedValue = new int[] {19700102, 19721021, 19700102, 19721021}; + } + if (newType != TSDataType.BOOLEAN) { + for (int i = 0; i < 4; i++) { + if (newType == TSDataType.BLOB) { + assertEquals(genValue(newType, expectedValue[i]), rec.getFields().get(i).getBinaryV()); + } else if (newType == TSDataType.DATE) { + assertEquals(genValue(newType, expectedValue[i]), rec.getFields().get(i).getDateV()); + } else if (newType == TSDataType.STRING || newType == TSDataType.TEXT) { + if (from == TSDataType.DATE) { + log.info( + "i is {}, expected value: {}, actual value: {}", + i, + new Binary(genValue(from, expectedValue[i]).toString(), StandardCharsets.UTF_8), + rec.getFields().get(i).getBinaryV()); + assertEquals( + new Binary(genValue(from, expectedValue[i]).toString(), StandardCharsets.UTF_8), + rec.getFields().get(i).getBinaryV()); + } else { + log.info( + "i is {}, expected value: {}, actual value: {}", + i, + newType.castFromSingleValue(from, genValue(from, expectedValue[i])), + rec.getFields().get(i).getBinaryV()); + assertEquals( + newType.castFromSingleValue(from, genValue(from, expectedValue[i])), + rec.getFields().get(i).getBinaryV()); + } + } else { + log.info( + "i is {}, expected value: {}, actual value: {}", + i, + genValue(newType, expectedValue[i]).toString(), + rec.getFields().get(i).toString()); + assertEquals( + genValue(newType, expectedValue[i]).toString(), rec.getFields().get(i).toString()); + } + } + + assertFalse(dataSet.hasNext()); + + if (newType.isNumeric()) { + dataSet = + session.executeQueryStatement( + "select avg(s1),sum(s1) from construct_and_alter_column_type"); + rec = dataSet.next(); + assertEquals(512.5, rec.getFields().get(0).getDoubleV(), 0.001); + assertEquals(524800.0, rec.getFields().get(1).getDoubleV(), 0.001); + assertFalse(dataSet.hasNext()); + } + } + + // can use statistics information + dataSet = + session.executeQueryStatement( + "select count(*) from construct_and_alter_column_type where time > 0"); + rec = dataSet.next(); + assertEquals(1024, rec.getFields().get(0).getLongV()); + assertFalse(dataSet.hasNext()); + + // can't use statistics information + dataSet = + session.executeQueryStatement( + "select count(*) from construct_and_alter_column_type where time > 10000"); + rec = dataSet.next(); + assertEquals(0, rec.getFields().get(0).getLongV()); + assertFalse(dataSet.hasNext()); + } + } + + private static void accumulatorQueryTestForDateType(ITableSession session, TSDataType newType) + throws StatementExecutionException, IoTDBConnectionException { + if (newType != TSDataType.STRING && newType != TSDataType.TEXT) { + return; + } + + log.info("Test the result that after transfered newType:"); + + SessionDataSet dataSet = + session.executeQueryStatement( + "select first(s1),last(s1) from construct_and_alter_column_type"); + RowRecord rec = dataSet.next(); + int[] expectedValue = {19700102, 19721021}; + if (newType != TSDataType.BOOLEAN) { + for (int i = 0; i < 2; i++) { + if (newType == TSDataType.STRING || newType == TSDataType.TEXT) { + log.info( + "i is {}, expected value: {}, actual value: {}", + i, + TSDataType.getDateStringValue(expectedValue[i]), + // rec.getFields().get(i).getBinaryV().toString()); + rec.getFields().get(i).getStringValue()); + assertEquals( + TSDataType.getDateStringValue(expectedValue[i]), + rec.getFields().get(i).getBinaryV().toString()); + } + } + } + + // can use statistics information + dataSet = + session.executeQueryStatement( + "select count(*) from construct_and_alter_column_type where time > 0"); + rec = dataSet.next(); + assertEquals(1024, rec.getFields().get(0).getLongV()); + assertFalse(dataSet.hasNext()); + + // can't use statistics information + dataSet = + session.executeQueryStatement( + "select count(*) from construct_and_alter_column_type where time > 10000"); + rec = dataSet.next(); + assertEquals(0, rec.getFields().get(0).getLongV()); + assertFalse(dataSet.hasNext()); + } + + private static void writeWithTablets( + long timestamp, + Object dataValue, + TSDataType tsDataType, + List tabletList, + Set alignedDevices, + TsFileWriter tsFileWriter, + Map> deviceColumnIndices) + throws IoTDBConnectionException, + StatementExecutionException, + IOException, + WriteProcessException { + RowRecord rowRecord = new RowRecord(timestamp); + rowRecord.addField(dataValue, tsDataType); + rowRecord.addField(dataValue, tsDataType); + List fields = rowRecord.getFields(); + + for (Tablet tablet : tabletList) { + String deviceId = tablet.getDeviceId(); + List columnIndices = deviceColumnIndices.get(deviceId); + int rowIndex = tablet.getRowSize(); + tablet.addTimestamp(rowIndex, rowRecord.getTimestamp()); + List schemas = tablet.getSchemas(); + + for (int i = 0, columnIndicesSize = columnIndices.size(); i < columnIndicesSize; i++) { + Integer columnIndex = columnIndices.get(i); + IMeasurementSchema measurementSchema = schemas.get(i); + // Object value = fields.get(columnIndex - + // 1).getObjectValue(measurementSchema.getType()); + Object value = fields.get(columnIndex).getObjectValue(measurementSchema.getType()); + tablet.addValue(measurementSchema.getMeasurementName(), rowIndex, value); + } + + if (tablet.getRowSize() == tablet.getMaxRowNumber()) { + writeToTsFile(alignedDevices, tsFileWriter, tablet); + tablet.reset(); + } + } + + for (Tablet tablet : tabletList) { + if (tablet.getRowSize() != 0) { + writeToTsFile(alignedDevices, tsFileWriter, tablet); + } + } + } + + private static void writeToTsFile( + Set deviceFilterSet, TsFileWriter tsFileWriter, Tablet tablet) + throws IOException, WriteProcessException { + if (deviceFilterSet.contains(tablet.getDeviceId())) { + tsFileWriter.writeAligned(tablet); + } else { + tsFileWriter.writeTree(tablet); + } + } + + private static List constructTablets( + Map> deviceSchemaMap, + Set alignedDevices, + TsFileWriter tsFileWriter) + throws WriteProcessException { + List tabletList = new ArrayList<>(deviceSchemaMap.size()); + for (Map.Entry> stringListEntry : deviceSchemaMap.entrySet()) { + String deviceId = stringListEntry.getKey(); + List schemaList = stringListEntry.getValue(); + Tablet tablet = new Tablet(deviceId, schemaList); + tablet.initBitMaps(); + Path path = new Path(tablet.getDeviceId()); + if (alignedDevices.contains(tablet.getDeviceId())) { + tsFileWriter.registerAlignedTimeseries(path, schemaList); + } else { + tsFileWriter.registerTimeseries(path, schemaList); + } + tabletList.add(tablet); + } + return tabletList; + } + + protected static TSDataType getType(String typeStr) { + try { + return TSDataType.valueOf(typeStr); + } catch (Exception e) { + return null; + } + } + + private static void collectSchemas( + ISession session, + List columnNames, + List columnTypes, + Map> deviceSchemaMap, + Set alignedDevices, + Map> deviceColumnIndices) + throws IoTDBConnectionException, StatementExecutionException { + for (int i = 0; i < columnNames.size(); i++) { + String column = columnNames.get(i); + if (!column.startsWith("root.")) { + continue; + } + TSDataType tsDataType = getType(columnTypes.get(i)); + Path path = new Path(column, true); + String deviceId = path.getDeviceString(); + // query whether the device is aligned or not + try (SessionDataSet deviceDataSet = + session.executeQueryStatement("show devices " + deviceId, timeout)) { + List deviceList = deviceDataSet.next().getFields(); + if (deviceList.size() > 1 && "true".equals(deviceList.get(1).getStringValue())) { + alignedDevices.add(deviceId); + } + } + + // query timeseries metadata + MeasurementSchema measurementSchema = + new MeasurementSchema(path.getMeasurement(), tsDataType); + List seriesList = + session.executeQueryStatement("show timeseries " + column, timeout).next().getFields(); + measurementSchema.setEncoding(TSEncoding.valueOf(seriesList.get(4).getStringValue())); + measurementSchema.setCompressionType( + CompressionType.valueOf(seriesList.get(5).getStringValue())); + + deviceSchemaMap.computeIfAbsent(deviceId, key -> new ArrayList<>()).add(measurementSchema); + deviceColumnIndices.computeIfAbsent(deviceId, key -> new ArrayList<>()).add(i); + } + } + + @Test + public void testUsingSameColumn() { + assertEquals(true, SchemaUtils.isUsingSameColumn(TSDataType.TIMESTAMP, TSDataType.INT64)); + assertEquals(true, SchemaUtils.isUsingSameColumn(TSDataType.INT64, TSDataType.TIMESTAMP)); + + assertEquals(true, SchemaUtils.isUsingSameColumn(TSDataType.DATE, TSDataType.INT32)); + assertEquals(true, SchemaUtils.isUsingSameColumn(TSDataType.INT32, TSDataType.DATE)); + + assertEquals(true, SchemaUtils.isUsingSameColumn(TSDataType.TEXT, TSDataType.BLOB)); + assertEquals(true, SchemaUtils.isUsingSameColumn(TSDataType.TEXT, TSDataType.STRING)); + assertEquals(true, SchemaUtils.isUsingSameColumn(TSDataType.BLOB, TSDataType.TEXT)); + assertEquals(true, SchemaUtils.isUsingSameColumn(TSDataType.BLOB, TSDataType.STRING)); + assertEquals(true, SchemaUtils.isUsingSameColumn(TSDataType.STRING, TSDataType.BLOB)); + assertEquals(true, SchemaUtils.isUsingSameColumn(TSDataType.STRING, TSDataType.TEXT)); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/session/IoTDBSessionRelationalIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/session/IoTDBSessionRelationalIT.java index 9c02ac94208ae..fc9855a98355c 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/session/IoTDBSessionRelationalIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/session/IoTDBSessionRelationalIT.java @@ -1466,6 +1466,18 @@ private void testOneCastWithTablet( case DOUBLE: assertEquals(genValue(to, 1), rec.getFields().get(2).getDoubleV()); break; + case STRING: + case TEXT: + if (from == TSDataType.DATE) { + assertEquals( + new Binary(genValue(from, 1).toString(), StandardCharsets.UTF_8), + rec.getFields().get(2).getBinaryV()); + } else { + assertEquals( + new Binary(genValue(from, 1).toString(), StandardCharsets.UTF_8), + rec.getFields().get(2).getBinaryV()); + } + break; default: assertEquals(String.valueOf(genValue(from, 1)), rec.getFields().get(2).toString()); } @@ -1596,7 +1608,7 @@ private void testOneCastWithRow( } @SuppressWarnings("SameParameterValue") - private Object genValue(TSDataType dataType, int i) { + public static Object genValue(TSDataType dataType, int i) { switch (dataType) { case INT32: return i; diff --git a/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 b/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 index 357a004336c2f..0d86a02a1cfcb 100644 --- a/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 +++ b/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 @@ -170,6 +170,8 @@ alterTimeseries alterClause : RENAME beforeName=attributeKey TO currentName=attributeKey + // Change into new data type + | SET DATA TYPE newType=attributeValue | SET attributePair (COMMA attributePair)* | DROP attributeKey (COMMA attributeKey)* | ADD TAGS attributePair (COMMA attributePair)* diff --git a/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/SqlLexer.g4 b/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/SqlLexer.g4 index 5696d5da32a4a..85e039b1e5d86 100644 --- a/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/SqlLexer.g4 +++ b/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/SqlLexer.g4 @@ -233,6 +233,10 @@ DATA : D A T A ; +TYPE + : T Y P E + ; + DATABASE : D A T A B A S E ; diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/CnToDnAsyncRequestType.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/CnToDnAsyncRequestType.java index 0d6d439583f94..c2f8b1e9d13c7 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/CnToDnAsyncRequestType.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/CnToDnAsyncRequestType.java @@ -89,6 +89,7 @@ public enum CnToDnAsyncRequestType { INVALIDATE_MATCHED_SCHEMA_CACHE, DELETE_DATA_FOR_DELETE_SCHEMA, DELETE_TIMESERIES, + ALTER_TIMESERIES_DATATYPE, ALTER_ENCODING_COMPRESSOR, diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/CnToDnInternalServiceAsyncRequestManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/CnToDnInternalServiceAsyncRequestManager.java index cde9a174492d3..9227325596d6b 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/CnToDnInternalServiceAsyncRequestManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/CnToDnInternalServiceAsyncRequestManager.java @@ -49,6 +49,7 @@ import org.apache.iotdb.confignode.client.async.handlers.rpc.subscription.TopicPushMetaRPCHandler; import org.apache.iotdb.mpp.rpc.thrift.TActiveTriggerInstanceReq; import org.apache.iotdb.mpp.rpc.thrift.TAlterEncodingCompressorReq; +import org.apache.iotdb.mpp.rpc.thrift.TAlterTimeSeriesReq; import org.apache.iotdb.mpp.rpc.thrift.TAlterViewReq; import org.apache.iotdb.mpp.rpc.thrift.TCheckSchemaRegionUsingTemplateReq; import org.apache.iotdb.mpp.rpc.thrift.TCheckTimeSeriesExistenceReq; @@ -315,6 +316,11 @@ protected void initActionMapBuilder() { (req, client, handler) -> client.alterEncodingCompressor( (TAlterEncodingCompressorReq) req, (SchemaUpdateRPCHandler) handler)); + actionMapBuilder.put( + CnToDnAsyncRequestType.ALTER_TIMESERIES_DATATYPE, + (req, client, handler) -> + client.alterTimeSeriesDataType( + (TAlterTimeSeriesReq) req, (SchemaUpdateRPCHandler) handler)); actionMapBuilder.put( CnToDnAsyncRequestType.CONSTRUCT_SCHEMA_BLACK_LIST_WITH_TEMPLATE, (req, client, handler) -> diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/handlers/rpc/DataNodeAsyncRequestRPCHandler.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/handlers/rpc/DataNodeAsyncRequestRPCHandler.java index 301bfc71f97d8..4e3fdb09f7ff0 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/handlers/rpc/DataNodeAsyncRequestRPCHandler.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/handlers/rpc/DataNodeAsyncRequestRPCHandler.java @@ -84,6 +84,7 @@ public static DataNodeAsyncRequestRPCHandler buildHandler( case DELETE_DATA_FOR_DELETE_SCHEMA: case DELETE_TIMESERIES: case ALTER_ENCODING_COMPRESSOR: + case ALTER_TIMESERIES_DATATYPE: case CONSTRUCT_SCHEMA_BLACK_LIST_WITH_TEMPLATE: case ROLLBACK_SCHEMA_BLACK_LIST_WITH_TEMPLATE: case DEACTIVATE_TEMPLATE: diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlan.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlan.java index 65c1ee0a9fed5..e067c7e1b6ce6 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlan.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlan.java @@ -96,6 +96,7 @@ import org.apache.iotdb.confignode.consensus.request.write.sync.RecordPipeMessagePlan; import org.apache.iotdb.confignode.consensus.request.write.sync.SetPipeStatusPlanV1; import org.apache.iotdb.confignode.consensus.request.write.table.AddTableColumnPlan; +import org.apache.iotdb.confignode.consensus.request.write.table.AlterColumnDataTypePlan; import org.apache.iotdb.confignode.consensus.request.write.table.CommitCreateTablePlan; import org.apache.iotdb.confignode.consensus.request.write.table.CommitDeleteColumnPlan; import org.apache.iotdb.confignode.consensus.request.write.table.CommitDeleteTablePlan; @@ -439,6 +440,9 @@ public static ConfigPhysicalPlan create(final ByteBuffer buffer) throws IOExcept case PreDeleteViewColumn: plan = new PreDeleteViewColumnPlan(); break; + case AlterColumnDataType: + plan = new AlterColumnDataTypePlan(); + break; case CommitDeleteColumn: plan = new CommitDeleteColumnPlan(configPhysicalPlanType); break; diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanType.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanType.java index 81ef37f93daf9..3c8f6295b9e3c 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanType.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanType.java @@ -230,6 +230,8 @@ public enum ConfigPhysicalPlanType { PreDeleteViewColumn((short) 875), PreDeleteView((short) 876), RenameViewColumn((short) 877), + AlterColumnDataType((short) 878), + CommitAlterColumnDataType((short) 879), /** Deprecated types for sync, restored them for upgrade. */ @Deprecated diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanVisitor.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanVisitor.java index 1167ae96541e0..f35e63979274d 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanVisitor.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanVisitor.java @@ -32,6 +32,7 @@ import org.apache.iotdb.confignode.consensus.request.write.pipe.payload.PipeDeleteTimeSeriesPlan; import org.apache.iotdb.confignode.consensus.request.write.pipe.payload.PipeUnsetSchemaTemplatePlan; import org.apache.iotdb.confignode.consensus.request.write.table.AddTableColumnPlan; +import org.apache.iotdb.confignode.consensus.request.write.table.AlterColumnDataTypePlan; import org.apache.iotdb.confignode.consensus.request.write.table.CommitDeleteColumnPlan; import org.apache.iotdb.confignode.consensus.request.write.table.CommitDeleteTablePlan; import org.apache.iotdb.confignode.consensus.request.write.table.RenameTableColumnPlan; @@ -190,6 +191,8 @@ public R process(final ConfigPhysicalPlan plan, final C context) { return visitSetViewComment((SetViewCommentPlan) plan, context); case SetTableColumnComment: return visitSetTableColumnComment((SetTableColumnCommentPlan) plan, context); + case AlterColumnDataType: + return visitAlterColumnDataType((AlterColumnDataTypePlan) plan, context); case RenameTable: return visitRenameTable((RenameTablePlan) plan, context); case RenameView: @@ -512,4 +515,9 @@ public R visitPipeAlterEncodingCompressor( final PipeAlterEncodingCompressorPlan pipeAlterEncodingCompressorPlan, final C context) { return visitPlan(pipeAlterEncodingCompressorPlan, context); } + + public R visitAlterColumnDataType( + final AlterColumnDataTypePlan alterColumnDataTypePlan, final C context) { + return visitPlan(alterColumnDataTypePlan, context); + } } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/table/AlterColumnDataTypePlan.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/table/AlterColumnDataTypePlan.java new file mode 100644 index 0000000000000..8382e7830f02d --- /dev/null +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/table/AlterColumnDataTypePlan.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.confignode.consensus.request.write.table; + +import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlanType; + +import org.apache.tsfile.enums.TSDataType; + +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; + +public class AlterColumnDataTypePlan extends AbstractTableColumnPlan { + + private TSDataType newType; + + public AlterColumnDataTypePlan() { + super(ConfigPhysicalPlanType.AlterColumnDataType); + } + + public AlterColumnDataTypePlan( + String database, String tableName, String columnName, TSDataType newType) { + super(ConfigPhysicalPlanType.AlterColumnDataType, database, tableName, columnName); + this.newType = newType; + } + + @Override + protected void serializeImpl(DataOutputStream stream) throws IOException { + super.serializeImpl(stream); + stream.write(newType.serialize()); + } + + @Override + protected void deserializeImpl(ByteBuffer buffer) throws IOException { + super.deserializeImpl(buffer); + newType = TSDataType.deserializeFrom(buffer); + } + + public void setNewType(TSDataType newType) { + this.newType = newType; + } + + public TSDataType getNewType() { + return newType; + } +} diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/response/table/DescTableResp.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/response/table/DescTableResp.java index fe09d00d25111..7808c06889949 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/response/table/DescTableResp.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/response/table/DescTableResp.java @@ -25,6 +25,10 @@ import org.apache.iotdb.confignode.rpc.thrift.TDescTableResp; import org.apache.iotdb.consensus.common.DataSet; +import org.apache.tsfile.enums.TSDataType; + +import java.util.HashMap; +import java.util.Map; import java.util.Objects; import java.util.Set; @@ -32,12 +36,17 @@ public class DescTableResp implements DataSet { private final TSStatus status; private final TsTable table; private final Set preDeletedColumns; + private final Map preAlteredColumns; public DescTableResp( - final TSStatus status, final TsTable table, final Set preDeletedColumns) { + final TSStatus status, + final TsTable table, + final Set preDeletedColumns, + final Map preAlteredColumns) { this.status = status; this.table = table; this.preDeletedColumns = preDeletedColumns; + this.preAlteredColumns = preAlteredColumns; } public TDescTableResp convertToTDescTableResp() { @@ -47,6 +56,14 @@ public TDescTableResp convertToTDescTableResp() { Objects.nonNull(table) ? TsTableInternalRPCUtil.serializeSingleTsTable(table) : null); - return Objects.nonNull(preDeletedColumns) ? resp.setPreDeletedColumns(preDeletedColumns) : resp; + if (Objects.nonNull(preDeletedColumns)) { + resp.setPreDeletedColumns(preDeletedColumns); + } + if (Objects.nonNull(preAlteredColumns)) { + Map preAlteredColumnsMap = new HashMap<>(); + preAlteredColumns.forEach((col, type) -> preAlteredColumnsMap.put(col, type.serialize())); + resp.setPreAlteredColumns(preAlteredColumnsMap); + } + return resp; } } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java index 2cd3979861333..39639674f3907 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java @@ -58,6 +58,7 @@ import org.apache.iotdb.commons.schema.table.TsTable; import org.apache.iotdb.commons.schema.table.TsTableInternalRPCUtil; import org.apache.iotdb.commons.schema.template.Template; +import org.apache.iotdb.commons.schema.tree.AlterTimeSeriesOperationType; import org.apache.iotdb.commons.schema.ttl.TTLCache; import org.apache.iotdb.commons.service.metric.MetricService; import org.apache.iotdb.commons.utils.AuthUtils; @@ -141,6 +142,7 @@ import org.apache.iotdb.confignode.rpc.thrift.TAlterOrDropTableReq; import org.apache.iotdb.confignode.rpc.thrift.TAlterPipeReq; import org.apache.iotdb.confignode.rpc.thrift.TAlterSchemaTemplateReq; +import org.apache.iotdb.confignode.rpc.thrift.TAlterTimeSeriesReq; import org.apache.iotdb.confignode.rpc.thrift.TAuthizedPatternTreeResp; import org.apache.iotdb.confignode.rpc.thrift.TCloseConsumerReq; import org.apache.iotdb.confignode.rpc.thrift.TClusterParameters; @@ -2329,6 +2331,22 @@ public TSStatus alterLogicalView(final TAlterLogicalViewReq req) { } } + @Override + public TSStatus alterTimeSeriesDataType(final TAlterTimeSeriesReq req) { + final TSStatus status = confirmLeader(); + if (status.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + switch (AlterTimeSeriesOperationType.getType(req.operationType)) { + case ALTER_DATA_TYPE: + return procedureManager.alterTimeSeriesDataType(req); + default: + throw new IllegalArgumentException( + AlterOrDropTableOperationType.getType(req.operationType).toString()); + } + } else { + return status; + } + } + @Override public TSStatus createPipe(TCreatePipeReq req) { TSStatus status = confirmLeader(); @@ -2816,6 +2834,8 @@ public TSStatus alterOrDropTable(final TAlterOrDropTableReq req) { return procedureManager.alterTableDropColumn(req); case DROP_TABLE: return procedureManager.dropTable(req); + case ALTER_COLUMN_DATA_TYPE: + return procedureManager.alterTableColumnDataType(req); case COMMENT_TABLE: return clusterSchemaManager.setTableComment( req.getDatabase(), @@ -2833,7 +2853,8 @@ public TSStatus alterOrDropTable(final TAlterOrDropTableReq req) { case RENAME_TABLE: return procedureManager.renameTable(req); default: - throw new IllegalArgumentException(); + throw new IllegalArgumentException( + AlterOrDropTableOperationType.getType(req.operationType).toString()); } } else { return status; diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/IManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/IManager.java index dff994d70e7e3..caa189be81572 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/IManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/IManager.java @@ -67,6 +67,7 @@ import org.apache.iotdb.confignode.rpc.thrift.TAlterOrDropTableReq; import org.apache.iotdb.confignode.rpc.thrift.TAlterPipeReq; import org.apache.iotdb.confignode.rpc.thrift.TAlterSchemaTemplateReq; +import org.apache.iotdb.confignode.rpc.thrift.TAlterTimeSeriesReq; import org.apache.iotdb.confignode.rpc.thrift.TCloseConsumerReq; import org.apache.iotdb.confignode.rpc.thrift.TConfigNodeRegisterReq; import org.apache.iotdb.confignode.rpc.thrift.TConfigNodeRegisterResp; @@ -708,6 +709,9 @@ TDataPartitionTableResp getOrCreateDataPartition( TSStatus alterLogicalView(TAlterLogicalViewReq req); + /** Alter timeseries data type. */ + TSStatus alterTimeSeriesDataType(TAlterTimeSeriesReq req); + /** * Create Pipe. * diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ProcedureManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ProcedureManager.java index 25c3b391aca1b..cdb0496d7b85c 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ProcedureManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ProcedureManager.java @@ -31,6 +31,7 @@ import org.apache.iotdb.commons.conf.CommonDescriptor; import org.apache.iotdb.commons.conf.IoTDBConstant; import org.apache.iotdb.commons.exception.IoTDBException; +import org.apache.iotdb.commons.path.MeasurementPath; import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.path.PathDeserializeUtil; import org.apache.iotdb.commons.path.PathPatternTree; @@ -85,6 +86,7 @@ import org.apache.iotdb.confignode.procedure.impl.region.RemoveRegionPeerProcedure; import org.apache.iotdb.confignode.procedure.impl.schema.AlterEncodingCompressorProcedure; import org.apache.iotdb.confignode.procedure.impl.schema.AlterLogicalViewProcedure; +import org.apache.iotdb.confignode.procedure.impl.schema.AlterTimeSeriesDataTypeProcedure; import org.apache.iotdb.confignode.procedure.impl.schema.DeactivateTemplateProcedure; import org.apache.iotdb.confignode.procedure.impl.schema.DeleteDatabaseProcedure; import org.apache.iotdb.confignode.procedure.impl.schema.DeleteLogicalViewProcedure; @@ -94,6 +96,7 @@ import org.apache.iotdb.confignode.procedure.impl.schema.UnsetTemplateProcedure; import org.apache.iotdb.confignode.procedure.impl.schema.table.AbstractAlterOrDropTableProcedure; import org.apache.iotdb.confignode.procedure.impl.schema.table.AddTableColumnProcedure; +import org.apache.iotdb.confignode.procedure.impl.schema.table.AlterTableColumnDataTypeProcedure; import org.apache.iotdb.confignode.procedure.impl.schema.table.CreateTableProcedure; import org.apache.iotdb.confignode.procedure.impl.schema.table.DeleteDevicesProcedure; import org.apache.iotdb.confignode.procedure.impl.schema.table.DropTableColumnProcedure; @@ -130,6 +133,7 @@ import org.apache.iotdb.confignode.rpc.thrift.TAlterLogicalViewReq; import org.apache.iotdb.confignode.rpc.thrift.TAlterOrDropTableReq; import org.apache.iotdb.confignode.rpc.thrift.TAlterPipeReq; +import org.apache.iotdb.confignode.rpc.thrift.TAlterTimeSeriesReq; import org.apache.iotdb.confignode.rpc.thrift.TCloseConsumerReq; import org.apache.iotdb.confignode.rpc.thrift.TConfigNodeRegisterReq; import org.apache.iotdb.confignode.rpc.thrift.TCreateCQReq; @@ -154,6 +158,7 @@ import org.apache.iotdb.rpc.TSStatusCode; import org.apache.ratis.util.AutoCloseableLock; +import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.Pair; import org.apache.tsfile.utils.ReadWriteIOUtils; @@ -467,6 +472,22 @@ public TSStatus alterLogicalView(final TAlterLogicalViewReq req) { return waitingProcedureFinished(procedure); } + public TSStatus alterTimeSeriesDataType(final TAlterTimeSeriesReq req) { + AlterTimeSeriesDataTypeProcedure procedure; + synchronized (this) { + procedure = + new AlterTimeSeriesDataTypeProcedure( + req.getQueryId(), + (MeasurementPath) + PathDeserializeUtil.deserialize(ByteBuffer.wrap(req.getMeasurementPath())), + req.getOperationType(), + TSDataType.deserialize(req.updateInfo.get()), + false); + this.executor.submitProcedure(procedure); + } + return waitingProcedureFinished(procedure); + } + public TSStatus setSchemaTemplate( String queryId, String templateName, String templateSetPath, boolean isGeneratedByPipe) { SetTemplateProcedure procedure = null; @@ -2015,6 +2036,22 @@ public TSStatus alterTableDropColumn(final TAlterOrDropTableReq req) { false)); } + public TSStatus alterTableColumnDataType(TAlterOrDropTableReq req) { + return executeWithoutDuplicate( + req.database, + null, + req.tableName, + req.queryId, + ProcedureType.ALTER_TABLE_COLUMN_DATATYPE_PROCEDURE, + new AlterTableColumnDataTypeProcedure( + req.database, + req.tableName, + req.queryId, + ReadWriteIOUtils.readVarIntString(req.updateInfo), + TSDataType.deserialize(req.updateInfo.get()), + false)); + } + public TSStatus dropTable(final TAlterOrDropTableReq req) { final boolean isView = req.isSetIsView() && req.isIsView(); return executeWithoutDuplicate( @@ -2199,6 +2236,7 @@ public Pair checkDuplicateTableTask( case DROP_TABLE_PROCEDURE: case DROP_VIEW_PROCEDURE: case DELETE_DEVICES_PROCEDURE: + case ALTER_TABLE_COLUMN_DATATYPE_PROCEDURE: final AbstractAlterOrDropTableProcedure alterTableProcedure = (AbstractAlterOrDropTableProcedure) procedure; if (type == thisType && queryId.equals(alterTableProcedure.getQueryId())) { diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/receiver/protocol/IoTDBConfigNodeReceiver.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/receiver/protocol/IoTDBConfigNodeReceiver.java index daf91c51c4c48..6f114c11ebd33 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/receiver/protocol/IoTDBConfigNodeReceiver.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/receiver/protocol/IoTDBConfigNodeReceiver.java @@ -65,6 +65,7 @@ import org.apache.iotdb.confignode.consensus.request.write.pipe.payload.PipeUnsetSchemaTemplatePlan; import org.apache.iotdb.confignode.consensus.request.write.table.AbstractTablePlan; import org.apache.iotdb.confignode.consensus.request.write.table.AddTableColumnPlan; +import org.apache.iotdb.confignode.consensus.request.write.table.AlterColumnDataTypePlan; import org.apache.iotdb.confignode.consensus.request.write.table.CommitDeleteColumnPlan; import org.apache.iotdb.confignode.consensus.request.write.table.CommitDeleteTablePlan; import org.apache.iotdb.confignode.consensus.request.write.table.RenameTableColumnPlan; @@ -97,6 +98,7 @@ import org.apache.iotdb.confignode.persistence.schema.CNSnapshotFileType; import org.apache.iotdb.confignode.persistence.schema.ConfigNodeSnapshotParser; import org.apache.iotdb.confignode.procedure.impl.schema.table.AddTableColumnProcedure; +import org.apache.iotdb.confignode.procedure.impl.schema.table.AlterTableColumnDataTypeProcedure; import org.apache.iotdb.confignode.procedure.impl.schema.table.CreateTableProcedure; import org.apache.iotdb.confignode.procedure.impl.schema.table.DropTableColumnProcedure; import org.apache.iotdb.confignode.procedure.impl.schema.table.DropTableProcedure; @@ -452,6 +454,7 @@ private TSStatus checkPermission(final ConfigPhysicalPlan plan) throws IOExcepti case RenameViewColumn: case RenameTable: case RenameView: + case AlterColumnDataType: return configManager .checkUserPrivileges( username, @@ -810,6 +813,22 @@ private TSStatus executePlan(final ConfigPhysicalPlan plan) throws ConsensusExce queryId, ((CommitDeleteViewColumnPlan) plan).getColumnName(), shouldMarkAsPipeRequest.get())); + case AlterColumnDataType: + return configManager + .getProcedureManager() + .executeWithoutDuplicate( + ((AlterColumnDataTypePlan) plan).getDatabase(), + null, + ((AlterColumnDataTypePlan) plan).getTableName(), + queryId, + ProcedureType.ALTER_TABLE_COLUMN_DATATYPE_PROCEDURE, + new AlterTableColumnDataTypeProcedure( + ((AlterColumnDataTypePlan) plan).getDatabase(), + ((AlterColumnDataTypePlan) plan).getTableName(), + queryId, + ((AlterColumnDataTypePlan) plan).getColumnName(), + ((AlterColumnDataTypePlan) plan).getNewType(), + shouldMarkAsPipeRequest.get())); case RenameTableColumn: return configManager .getProcedureManager() diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/receiver/visitor/PipeConfigPhysicalPlanTSStatusVisitor.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/receiver/visitor/PipeConfigPhysicalPlanTSStatusVisitor.java index cc8b57deb8e5f..371f3a3cb19ae 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/receiver/visitor/PipeConfigPhysicalPlanTSStatusVisitor.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/receiver/visitor/PipeConfigPhysicalPlanTSStatusVisitor.java @@ -35,6 +35,7 @@ import org.apache.iotdb.confignode.consensus.request.write.pipe.payload.PipeDeleteTimeSeriesPlan; import org.apache.iotdb.confignode.consensus.request.write.pipe.payload.PipeUnsetSchemaTemplatePlan; import org.apache.iotdb.confignode.consensus.request.write.table.AddTableColumnPlan; +import org.apache.iotdb.confignode.consensus.request.write.table.AlterColumnDataTypePlan; import org.apache.iotdb.confignode.consensus.request.write.table.CommitDeleteColumnPlan; import org.apache.iotdb.confignode.consensus.request.write.table.CommitDeleteTablePlan; import org.apache.iotdb.confignode.consensus.request.write.table.RenameTableColumnPlan; @@ -536,6 +537,12 @@ public TSStatus visitRenameTableColumn( return visitCommonTablePlan(renameTableColumnPlan, context); } + @Override + public TSStatus visitAlterColumnDataType( + final AlterColumnDataTypePlan alterColumnDataTypePlan, final TSStatus context) { + return visitCommonTablePlan(alterColumnDataTypePlan, context); + } + @Override public TSStatus visitCommitDeleteTable( final CommitDeleteTablePlan commitDeleteTablePlan, final TSStatus context) { diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/source/ConfigRegionListeningFilter.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/source/ConfigRegionListeningFilter.java index a9a08611643f8..81f0212fd4d87 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/source/ConfigRegionListeningFilter.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/source/ConfigRegionListeningFilter.java @@ -125,7 +125,8 @@ public class ConfigRegionListeningFilter { ConfigPhysicalPlanType.RenameTable, ConfigPhysicalPlanType.RenameView, ConfigPhysicalPlanType.RenameTableColumn, - ConfigPhysicalPlanType.RenameViewColumn))); + ConfigPhysicalPlanType.RenameViewColumn, + ConfigPhysicalPlanType.CommitAlterColumnDataType))); OPTION_PLAN_MAP.put( new PartialPath("schema.table.drop"), Collections.unmodifiableList( diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/source/PipeConfigTablePatternParseVisitor.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/source/PipeConfigTablePatternParseVisitor.java index 6baf78f60ac0f..30a4a77a94a70 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/source/PipeConfigTablePatternParseVisitor.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/source/PipeConfigTablePatternParseVisitor.java @@ -30,6 +30,7 @@ import org.apache.iotdb.confignode.consensus.request.write.pipe.payload.PipeDeleteDevicesPlan; import org.apache.iotdb.confignode.consensus.request.write.table.AbstractTablePlan; import org.apache.iotdb.confignode.consensus.request.write.table.AddTableColumnPlan; +import org.apache.iotdb.confignode.consensus.request.write.table.AlterColumnDataTypePlan; import org.apache.iotdb.confignode.consensus.request.write.table.CommitDeleteColumnPlan; import org.apache.iotdb.confignode.consensus.request.write.table.CommitDeleteTablePlan; import org.apache.iotdb.confignode.consensus.request.write.table.RenameTableColumnPlan; @@ -111,6 +112,12 @@ public Optional visitRenameTableColumn( return visitAbstractTablePlan(renameTableColumnPlan, pattern); } + @Override + public Optional visitAlterColumnDataType( + final AlterColumnDataTypePlan alterColumnDataTypePlan, final TablePattern pattern) { + return visitAbstractTablePlan(alterColumnDataTypePlan, pattern); + } + @Override public Optional visitCommitDeleteTable( final CommitDeleteTablePlan commitDeleteTablePlan, final TablePattern pattern) { diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/source/PipeConfigTablePrivilegeParseVisitor.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/source/PipeConfigTablePrivilegeParseVisitor.java index bf0871f003232..ce7bcd80d0082 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/source/PipeConfigTablePrivilegeParseVisitor.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/source/PipeConfigTablePrivilegeParseVisitor.java @@ -30,6 +30,7 @@ import org.apache.iotdb.confignode.consensus.request.write.pipe.payload.PipeDeleteDevicesPlan; import org.apache.iotdb.confignode.consensus.request.write.table.AbstractTablePlan; import org.apache.iotdb.confignode.consensus.request.write.table.AddTableColumnPlan; +import org.apache.iotdb.confignode.consensus.request.write.table.AlterColumnDataTypePlan; import org.apache.iotdb.confignode.consensus.request.write.table.CommitDeleteColumnPlan; import org.apache.iotdb.confignode.consensus.request.write.table.CommitDeleteTablePlan; import org.apache.iotdb.confignode.consensus.request.write.table.RenameTableColumnPlan; @@ -386,4 +387,9 @@ private Optional visitUserRolePlan( ? Optional.of(plan) : Optional.empty(); } + + public Optional visitAlterColumnDataType( + final AlterColumnDataTypePlan alterColumnDataTypePlan, final String userName) { + return visitAbstractTablePlan(alterColumnDataTypePlan, userName); + } } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/schema/ClusterSchemaManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/schema/ClusterSchemaManager.java index 0648965cda617..86139b205d620 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/schema/ClusterSchemaManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/schema/ClusterSchemaManager.java @@ -1349,7 +1349,7 @@ public synchronized Pair tableColumnCheckForColumnExtension( return result.get(); } - final TsTable expandedTable = TsTable.deserialize(ByteBuffer.wrap(originalTable.serialize())); + final TsTable expandedTable = new TsTable(originalTable); final String errorMsg = String.format( @@ -1381,6 +1381,34 @@ public synchronized Pair tableColumnCheckForColumnExtension( return new Pair<>(StatusUtils.OK, expandedTable); } + public synchronized Pair tableColumnCheckForColumnAltering( + final String database, + final String tableName, + final String columnName, + final TSDataType dataType) + throws MetadataException { + final TsTable originalTable = getTableIfExists(database, tableName).orElse(null); + + if (Objects.isNull(originalTable)) { + return new Pair<>( + RpcUtils.getStatus( + TSStatusCode.TABLE_NOT_EXISTS, + String.format("Table '%s.%s' does not exist", database, tableName)), + null); + } + + TSStatus tsStatus = + clusterSchemaInfo.preAlterColumnDataType(database, tableName, columnName, dataType); + if (tsStatus.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + return new Pair<>(tsStatus, null); + } + + final TsTable alteredTable = new TsTable(originalTable); + alteredTable.getColumnSchema(columnName).setDataType(dataType); + + return new Pair<>(RpcUtils.SUCCESS_STATUS, alteredTable); + } + public synchronized Pair tableColumnCheckForColumnRenaming( final String database, final String tableName, @@ -1428,7 +1456,7 @@ public synchronized Pair tableColumnCheckForColumnRenaming( null); } - final TsTable expandedTable = TsTable.deserialize(ByteBuffer.wrap(originalTable.serialize())); + final TsTable expandedTable = new TsTable(originalTable); expandedTable.renameColumnSchema(oldName, newName); @@ -1465,7 +1493,7 @@ public synchronized Pair tableCheckForRenaming( null); } - final TsTable expandedTable = TsTable.deserialize(ByteBuffer.wrap(originalTable.serialize())); + final TsTable expandedTable = new TsTable(originalTable); expandedTable.renameTable(newName); return new Pair<>(RpcUtils.SUCCESS_STATUS, expandedTable); } @@ -1552,7 +1580,7 @@ public synchronized Pair updateTableProperties( return new Pair<>(RpcUtils.SUCCESS_STATUS, null); } - final TsTable updatedTable = TsTable.deserialize(ByteBuffer.wrap(originalTable.serialize())); + final TsTable updatedTable = new TsTable(originalTable); updatedProperties.forEach( (k, v) -> { originalProperties.put(k, originalTable.getPropValue(k).orElse(null)); diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/executor/ConfigPlanExecutor.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/executor/ConfigPlanExecutor.java index d6bad518f6f4b..1d9554084059b 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/executor/ConfigPlanExecutor.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/executor/ConfigPlanExecutor.java @@ -113,6 +113,7 @@ import org.apache.iotdb.confignode.consensus.request.write.subscription.topic.DropTopicPlan; import org.apache.iotdb.confignode.consensus.request.write.subscription.topic.runtime.TopicHandleMetaChangePlan; import org.apache.iotdb.confignode.consensus.request.write.table.AddTableColumnPlan; +import org.apache.iotdb.confignode.consensus.request.write.table.AlterColumnDataTypePlan; import org.apache.iotdb.confignode.consensus.request.write.table.CommitCreateTablePlan; import org.apache.iotdb.confignode.consensus.request.write.table.CommitDeleteColumnPlan; import org.apache.iotdb.confignode.consensus.request.write.table.CommitDeleteTablePlan; @@ -586,6 +587,9 @@ public TSStatus executeNonQueryPlan(ConfigPhysicalPlan physicalPlan) case CommitDeleteTable: case CommitDeleteView: return clusterSchemaInfo.dropTable((CommitDeleteTablePlan) physicalPlan); + case AlterColumnDataType: + return clusterSchemaInfo.commitAlterColumnDataType( + ((AlterColumnDataTypePlan) physicalPlan)); case SetTableComment: case SetViewComment: return clusterSchemaInfo.setTableComment((SetTableCommentPlan) physicalPlan); diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ClusterSchemaInfo.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ClusterSchemaInfo.java index 62fe57fb0b7ce..e3ce7d7c29db1 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ClusterSchemaInfo.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ClusterSchemaInfo.java @@ -54,6 +54,7 @@ import org.apache.iotdb.confignode.consensus.request.write.database.SetSchemaReplicationFactorPlan; import org.apache.iotdb.confignode.consensus.request.write.database.SetTimePartitionIntervalPlan; import org.apache.iotdb.confignode.consensus.request.write.table.AddTableColumnPlan; +import org.apache.iotdb.confignode.consensus.request.write.table.AlterColumnDataTypePlan; import org.apache.iotdb.confignode.consensus.request.write.table.CommitCreateTablePlan; import org.apache.iotdb.confignode.consensus.request.write.table.CommitDeleteColumnPlan; import org.apache.iotdb.confignode.consensus.request.write.table.CommitDeleteTablePlan; @@ -91,6 +92,7 @@ import org.apache.iotdb.confignode.consensus.response.template.TemplateInfoResp; import org.apache.iotdb.confignode.consensus.response.template.TemplateSetInfoResp; import org.apache.iotdb.confignode.exception.DatabaseNotExistsException; +import org.apache.iotdb.confignode.persistence.schema.ConfigMTree.TableSchemaDetails; import org.apache.iotdb.confignode.rpc.thrift.TDatabaseSchema; import org.apache.iotdb.confignode.rpc.thrift.TTableColumnInfo; import org.apache.iotdb.confignode.rpc.thrift.TTableInfo; @@ -103,6 +105,7 @@ import org.apache.iotdb.rpc.TSStatusCode; import org.apache.tsfile.annotations.TableModel; +import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.utils.Pair; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -121,6 +124,7 @@ import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Map.Entry; import java.util.Optional; import java.util.Set; import java.util.UUID; @@ -1371,16 +1375,19 @@ public DescTableResp descTable(final DescTablePlan plan) { try { final PartialPath databasePath = getQualifiedDatabasePartialPath(plan.getDatabase()); if (plan.isDetails()) { - final Pair> pair = + final TableSchemaDetails details = tableModelMTree.getTableSchemaDetails(databasePath, plan.getTableName()); - return new DescTableResp(StatusUtils.OK, pair.getLeft(), pair.getRight()); + return new DescTableResp( + StatusUtils.OK, details.table, details.preDeletedColumns, details.preAlteredColumns); } return new DescTableResp( StatusUtils.OK, tableModelMTree.getUsingTableSchema(databasePath, plan.getTableName()), + null, null); } catch (final MetadataException e) { - return new DescTableResp(RpcUtils.getStatus(e.getErrorCode(), e.getMessage()), null, null); + return new DescTableResp( + RpcUtils.getStatus(e.getErrorCode(), e.getMessage()), null, null, null); } finally { databaseReadWriteLock.readLock().unlock(); } @@ -1409,17 +1416,27 @@ public DescTable4InformationSchemaResp descTable4InformationSchema() { // Table path must exist because the "getTableSchemaDetails()" // is called in databaseReadWriteLock.readLock(). } - return new Pair>(null, null); + return new TableSchemaDetails(); }) .collect( Collectors.toMap( - pair -> pair.getLeft().getTableName(), - pair -> + tableSchemaDetails -> tableSchemaDetails.table.getTableName(), + tableSchemaDetails -> new TTableColumnInfo() .setTableInfo( TsTableInternalRPCUtil.serializeSingleTsTable( - pair.getLeft())) - .setPreDeletedColumns(pair.getRight()))); + tableSchemaDetails.table)) + .setPreDeletedColumns( + tableSchemaDetails.preDeletedColumns) + .setPreAlteredColumns( + tableSchemaDetails + .preAlteredColumns + .entrySet() + .stream() + .collect( + Collectors.toMap( + Entry::getKey, + e -> e.getValue().serialize()))))); } catch (final MetadataException ignore) { // Database path must exist because the "getAllDatabasePaths()" is called // in databaseReadWriteLock.readLock(). @@ -1530,6 +1547,41 @@ public TSStatus commitDeleteColumn(final CommitDeleteColumnPlan plan) { plan.getColumnName())); } + public TSStatus preAlterColumnDataType( + String databaseName, String tableName, String columnName, TSDataType dataType) { + databaseReadWriteLock.writeLock().lock(); + try { + final TSStatus status = new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()); + tableModelMTree.preAlterColumnDataType( + getQualifiedDatabasePartialPath(databaseName), tableName, columnName, dataType); + return status; + } catch (final MetadataException e) { + LOGGER.warn(e.getMessage(), e); + return RpcUtils.getStatus(e.getErrorCode(), e.getMessage()); + } catch (final SemanticException e) { + return RpcUtils.getStatus(TSStatusCode.SEMANTIC_ERROR.getStatusCode(), e.getMessage()); + } finally { + databaseReadWriteLock.writeLock().unlock(); + } + } + + public TSStatus commitAlterColumnDataType(AlterColumnDataTypePlan plan) { + databaseReadWriteLock.writeLock().lock(); + try { + tableModelMTree.commitAlterColumnDataType( + getQualifiedDatabasePartialPath(plan.getDatabase()), + plan.getTableName(), + plan.getColumnName(), + plan.getNewType()); + return RpcUtils.SUCCESS_STATUS; + } catch (final MetadataException e) { + LOGGER.warn(e.getMessage(), e); + return RpcUtils.getStatus(e.getErrorCode(), e.getMessage()); + } finally { + databaseReadWriteLock.writeLock().unlock(); + } + } + // endregion @TestOnly diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ConfigMTree.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ConfigMTree.java index a4231e5db83b2..06a63126d6cfa 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ConfigMTree.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ConfigMTree.java @@ -38,6 +38,7 @@ import org.apache.iotdb.commons.schema.table.column.TimeColumnSchema; import org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory; import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchema; +import org.apache.iotdb.commons.utils.MetadataUtils; import org.apache.iotdb.commons.utils.PathUtils; import org.apache.iotdb.commons.utils.ThriftConfigNodeSerDeUtils; import org.apache.iotdb.confignode.manager.schema.ClusterSchemaManager; @@ -59,6 +60,7 @@ import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.utils.Pair; +import org.apache.tsfile.utils.ReadWriteForEncodingUtils; import org.apache.tsfile.utils.ReadWriteIOUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -69,7 +71,6 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; -import java.nio.ByteBuffer; import java.util.ArrayDeque; import java.util.ArrayList; import java.util.Collections; @@ -79,6 +80,7 @@ import java.util.LinkedList; import java.util.List; import java.util.Map; +import java.util.Map.Entry; import java.util.Objects; import java.util.Optional; import java.util.Set; @@ -992,21 +994,72 @@ public void commitDeleteColumn( } } + public void preAlterColumnDataType( + PartialPath database, String tableName, String columnName, TSDataType dataType) + throws MetadataException { + final ConfigTableNode node = getTableNode(database, tableName); + final TsTableColumnSchema columnSchema = node.getTable().getColumnSchema(columnName); + + // ((FieldColumnSchema) columnSchema).getEncoding().isSupported() + if (Objects.isNull(columnSchema)) { + throw new ColumnNotExistsException( + PathUtils.unQualifyDatabaseName(database.getFullPath()), tableName, columnName); + } + if (columnSchema.getColumnCategory() != TsTableColumnCategory.FIELD) { + throw new SemanticException("Can only alter datatype of FIELD columns"); + } + if (!MetadataUtils.canAlter(columnSchema.getDataType(), dataType)) { + throw new SemanticException( + String.format( + "New type %s is not compatible with the existing one %s", + dataType, columnSchema.getDataType())); + } + + node.addPreAlteredColumn(columnName, dataType); + } + + public void commitAlterColumnDataType( + PartialPath database, String tableName, String columnName, TSDataType dataType) + throws MetadataException { + final ConfigTableNode node = getTableNode(database, tableName); + final TsTable table = getTable(database, tableName); + if (Objects.nonNull(table.getColumnSchema(columnName))) { + table.getColumnSchema(columnName).setDataType(dataType); + node.removePreAlteredColumn(columnName); + } + } + public TsTable getUsingTableSchema(final PartialPath database, final String tableName) throws MetadataException { final ConfigTableNode node = getTableNode(database, tableName); - if (node.getPreDeletedColumns().isEmpty()) { + if (node.getPreDeletedColumns().isEmpty() && node.getPreAlteredColumns().isEmpty()) { return node.getTable(); } - final TsTable newTable = TsTable.deserialize(ByteBuffer.wrap(node.getTable().serialize())); - node.getPreDeletedColumns().forEach(newTable::removeColumnSchema); + final TsTable newTable = new TsTable(node.getTable()); + if (!node.getPreDeletedColumns().isEmpty()) { + node.getPreDeletedColumns().forEach(newTable::removeColumnSchema); + } + if (!node.getPreAlteredColumns().isEmpty()) { + node.getPreAlteredColumns() + .forEach((col, type) -> newTable.getColumnSchema(col).setDataType(type)); + } return newTable; } - public Pair> getTableSchemaDetails( + public TableSchemaDetails getTableSchemaDetails( final PartialPath database, final String tableName) throws MetadataException { final ConfigTableNode node = getTableNode(database, tableName); - return new Pair<>(node.getTable(), node.getPreDeletedColumns()); + TableSchemaDetails tableSchemaDetails = new TableSchemaDetails(); + tableSchemaDetails.table = node.getTable(); + tableSchemaDetails.preDeletedColumns = node.getPreDeletedColumns(); + tableSchemaDetails.preAlteredColumns = node.getPreAlteredColumns(); + return tableSchemaDetails; + } + + public static class TableSchemaDetails { + public TsTable table; + public Set preDeletedColumns; + public Map preAlteredColumns; } private TsTable getTable(final PartialPath database, final String tableName) @@ -1088,6 +1141,11 @@ private void serializeTableNode(final ConfigTableNode tableNode, final OutputStr for (final String column : preDeletedColumns) { ReadWriteIOUtils.write(column, outputStream); } + ReadWriteForEncodingUtils.writeVarInt(tableNode.getPreAlteredColumns().size(), outputStream); + for (Entry entry : tableNode.getPreAlteredColumns().entrySet()) { + ReadWriteIOUtils.writeVar(entry.getKey(), outputStream); + ReadWriteIOUtils.write(entry.getValue(), outputStream); + } } public void deserialize(final InputStream inputStream) throws IOException { @@ -1180,10 +1238,17 @@ public static ConfigTableNode deserializeTableMNode(final InputStream inputStrea new ConfigTableNode(null, ReadWriteIOUtils.readString(inputStream)); tableNode.setTable(TsTable.deserialize(inputStream)); tableNode.setStatus(TableNodeStatus.deserialize(inputStream)); - final int size = ReadWriteIOUtils.readInt(inputStream); + int size = ReadWriteIOUtils.readInt(inputStream); for (int i = 0; i < size; ++i) { tableNode.addPreDeletedColumn(ReadWriteIOUtils.readString(inputStream)); } + + size = ReadWriteForEncodingUtils.readVarInt(inputStream); + for (int i = 0; i < size; i++) { + tableNode.addPreAlteredColumn( + ReadWriteIOUtils.readVarIntString(inputStream), + ReadWriteIOUtils.readDataType(inputStream)); + } return tableNode; } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/mnode/impl/ConfigTableNode.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/mnode/impl/ConfigTableNode.java index 0bb306e3f3f03..b95b50ff0bef5 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/mnode/impl/ConfigTableNode.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/mnode/impl/ConfigTableNode.java @@ -31,8 +31,11 @@ import org.apache.iotdb.confignode.persistence.schema.mnode.container.ConfigMNodeContainer; import org.apache.iotdb.confignode.persistence.schema.mnode.info.ConfigTableInfo; +import org.apache.tsfile.enums.TSDataType; + import java.util.ArrayList; import java.util.List; +import java.util.Map; import java.util.Set; import static org.apache.iotdb.commons.schema.SchemaConstant.NON_TEMPLATE; @@ -70,6 +73,10 @@ public Set getPreDeletedColumns() { return tableNodeInfo.getPreDeletedColumns(); } + public Map getPreAlteredColumns() { + return tableNodeInfo.getPreAlteredColumns(); + } + public void addPreDeletedColumn(final String column) { tableNodeInfo.addPreDeletedColumn(column); } @@ -78,6 +85,14 @@ public void removePreDeletedColumn(final String column) { tableNodeInfo.removePreDeletedColumn(column); } + public void addPreAlteredColumn(final String column, TSDataType dataType) { + tableNodeInfo.addPreAlteredColumn(column, dataType); + } + + public void removePreAlteredColumn(final String column) { + tableNodeInfo.removePreAlteredColumn(column); + } + @Override public String getName() { return tableNodeInfo.getName(); diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/mnode/info/ConfigTableInfo.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/mnode/info/ConfigTableInfo.java index 6a0173620a880..9af9ea6fb72ca 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/mnode/info/ConfigTableInfo.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/mnode/info/ConfigTableInfo.java @@ -23,9 +23,12 @@ import org.apache.iotdb.commons.schema.table.TsTable; import org.apache.iotdb.db.schemaengine.schemaregion.mtree.impl.mem.mnode.info.BasicMNodeInfo; +import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.utils.RamUsageEstimator; +import java.util.HashMap; import java.util.HashSet; +import java.util.Map; import java.util.Set; public class ConfigTableInfo extends BasicMNodeInfo { @@ -37,6 +40,7 @@ public class ConfigTableInfo extends BasicMNodeInfo { // This shall be only one because concurrent modifications of one table is not allowed private final Set preDeletedColumns = new HashSet<>(); + private final Map preAlteredColumns = new HashMap<>(); public ConfigTableInfo(final String name) { super(name); @@ -68,6 +72,10 @@ public Set getPreDeletedColumns() { return preDeletedColumns; } + public Map getPreAlteredColumns() { + return preAlteredColumns; + } + public void addPreDeletedColumn(final String column) { preDeletedColumns.add(column); } @@ -87,4 +95,12 @@ public int estimateSize() { .map(column -> (int) RamUsageEstimator.sizeOf(column)) .reduce(0, Integer::sum); } + + public void addPreAlteredColumn(String column, TSDataType dataType) { + preAlteredColumns.put(column, dataType); + } + + public void removePreAlteredColumn(String column) { + preAlteredColumns.remove(column); + } } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/AlterTimeSeriesDataTypeProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/AlterTimeSeriesDataTypeProcedure.java new file mode 100644 index 0000000000000..2d608623a3ae8 --- /dev/null +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/AlterTimeSeriesDataTypeProcedure.java @@ -0,0 +1,388 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.confignode.procedure.impl.schema; + +import org.apache.iotdb.common.rpc.thrift.TConsensusGroupId; +import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation; +import org.apache.iotdb.common.rpc.thrift.TRegionReplicaSet; +import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.commons.exception.MetadataException; +import org.apache.iotdb.commons.path.MeasurementPath; +import org.apache.iotdb.commons.path.PathPatternTree; +import org.apache.iotdb.confignode.client.async.CnToDnAsyncRequestType; +import org.apache.iotdb.confignode.client.async.CnToDnInternalServiceAsyncRequestManager; +import org.apache.iotdb.confignode.client.async.handlers.DataNodeAsyncRequestContext; +import org.apache.iotdb.confignode.procedure.env.ConfigNodeProcedureEnv; +import org.apache.iotdb.confignode.procedure.exception.ProcedureException; +import org.apache.iotdb.confignode.procedure.impl.StateMachineProcedure; +import org.apache.iotdb.confignode.procedure.state.schema.AlterTimeSeriesDataTypeState; +import org.apache.iotdb.confignode.procedure.store.ProcedureType; +import org.apache.iotdb.db.exception.metadata.PathNotExistException; +import org.apache.iotdb.mpp.rpc.thrift.TAlterTimeSeriesReq; +import org.apache.iotdb.mpp.rpc.thrift.TInvalidateMatchedSchemaCacheReq; +import org.apache.iotdb.rpc.TSStatusCode; + +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.utils.ReadWriteIOUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.ByteArrayOutputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.function.Consumer; + +public class AlterTimeSeriesDataTypeProcedure + extends StateMachineProcedure { + + private static final Logger LOGGER = + LoggerFactory.getLogger(AlterTimeSeriesDataTypeProcedure.class); + + private String queryId; + + private MeasurementPath measurementPath; + private transient ByteBuffer measurementPathBytes; + private byte operationType; + private TSDataType dataType; + + public AlterTimeSeriesDataTypeProcedure(final boolean isGeneratedByPipe) { + super(isGeneratedByPipe); + } + + public AlterTimeSeriesDataTypeProcedure( + final String queryId, + final MeasurementPath measurementPath, + final byte operationType, + final TSDataType dataType, + final boolean isGeneratedByPipe) { + super(isGeneratedByPipe); + this.queryId = queryId; + setMeasurementPath(measurementPath); + this.operationType = operationType; + this.dataType = dataType; + } + + @Override + protected StateMachineProcedure.Flow executeFromState( + final ConfigNodeProcedureEnv env, final AlterTimeSeriesDataTypeState state) + throws InterruptedException { + final long startTime = System.currentTimeMillis(); + try { + switch (state) { + case CHECK_AND_INVALIDATE_SERIES: + LOGGER.info( + "Check and invalidate series {} when altering time series data type", + measurementPath.getFullPath()); + checkAndPreAlterTimeSeries(); + break; + case ALTER_TIME_SERIES_DATA_TYPE: + LOGGER.info("altering time series {} data type", measurementPath.getFullPath()); + if (!alterTimeSeriesDataType(env)) { + LOGGER.error("alter time series {} data type failed", measurementPath.getFullPath()); + return Flow.NO_MORE_STATE; + } + break; + case CLEAR_CACHE: + LOGGER.info( + "clearing cache after alter time series {} data type", measurementPath.getFullPath()); + PathPatternTree patternTree = new PathPatternTree(); + patternTree.appendPathPattern(measurementPath); + patternTree.constructTree(); + invalidateCache( + env, + preparePatternTreeBytesData(patternTree), + measurementPath.getFullPath(), + this::setFailure, + true); + break; + default: + setFailure( + new ProcedureException( + "Unrecognized AlterTimeSeriesDataTypeProcedure state " + state)); + return Flow.NO_MORE_STATE; + } + return Flow.HAS_MORE_STATE; + } finally { + LOGGER.info( + "AlterTimeSeriesDataType-{}-[{}] costs {}ms", + measurementPath.getFullPath(), + state, + (System.currentTimeMillis() - startTime)); + } + } + + private void checkAndPreAlterTimeSeries() { + if (dataType != null) { + setNextState(AlterTimeSeriesDataTypeState.ALTER_TIME_SERIES_DATA_TYPE); + } else { + setFailure( + new ProcedureException( + new MetadataException("Invalid data type cannot be used as a new type"))); + } + } + + private boolean alterTimeSeriesDataType(final ConfigNodeProcedureEnv env) { + final PathPatternTree patternTree = new PathPatternTree(); + patternTree.appendFullPath(measurementPath); + patternTree.constructTree(); + + final Map relatedSchemaRegionGroup = + env.getConfigManager().getRelatedSchemaRegionGroup(patternTree, false); + + if (relatedSchemaRegionGroup.isEmpty()) { + setFailure( + new ProcedureException(new PathNotExistException(measurementPath.getFullPath(), false))); + return false; + } + + final ByteArrayOutputStream stream = new ByteArrayOutputStream(); + try { + ReadWriteIOUtils.write(dataType, stream); + } catch (final IOException ignored) { + // ByteArrayOutputStream won't throw IOException + } + + final DataNodeTSStatusTaskExecutor alterTimerSeriesTask = + new DataNodeTSStatusTaskExecutor( + env, + env.getConfigManager().getRelatedSchemaRegionGroup(patternTree, false), + false, + CnToDnAsyncRequestType.ALTER_TIMESERIES_DATATYPE, + ((dataNodeLocation, consensusGroupIdList) -> { + ByteBuffer measurementPathBuffer = null; + try (ByteArrayOutputStream baos = new ByteArrayOutputStream()) { + measurementPath.serialize(baos); + measurementPathBuffer = ByteBuffer.wrap(baos.toByteArray()); + } catch (IOException ignored) { + // ByteArrayOutputStream won't throw IOException + } + + return new TAlterTimeSeriesReq( + consensusGroupIdList, + queryId, + measurementPathBuffer, + operationType, + ByteBuffer.wrap(stream.toByteArray())); + })) { + + private final Map failureMap = new HashMap<>(); + + @Override + protected List processResponseOfOneDataNode( + final TDataNodeLocation dataNodeLocation, + final List consensusGroupIdList, + final TSStatus response) { + final List failedRegionList = new ArrayList<>(); + if (response.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + failureMap.remove(dataNodeLocation); + return failedRegionList; + } + + if (response.getCode() == TSStatusCode.MULTIPLE_ERROR.getStatusCode()) { + final List subStatus = response.getSubStatus(); + for (int i = 0; i < subStatus.size(); i++) { + if (subStatus.get(i).getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode() + && !(subStatus.get(i).getCode() + == TSStatusCode.PATH_NOT_EXIST.getStatusCode())) { + failedRegionList.add(consensusGroupIdList.get(i)); + } + } + } else if (!(response.getCode() == TSStatusCode.PATH_NOT_EXIST.getStatusCode())) { + failedRegionList.addAll(consensusGroupIdList); + } + if (!failedRegionList.isEmpty()) { + failureMap.put(dataNodeLocation, response); + } else { + failureMap.remove(dataNodeLocation); + } + return failedRegionList; + } + + @Override + protected void onAllReplicasetFailure( + final TConsensusGroupId consensusGroupId, + final Set dataNodeLocationSet) { + setFailure( + new ProcedureException( + new MetadataException( + String.format( + "Alter timeseries %s data type from %s to %s in schema regions failed. Failures: %s", + measurementPath.getFullPath(), + measurementPath.getSeriesType(), + dataType, + failureMap)))); + interruptTask(); + } + }; + alterTimerSeriesTask.execute(); + setNextState(AlterTimeSeriesDataTypeState.CLEAR_CACHE); + return true; + } + + public static void invalidateCache( + final ConfigNodeProcedureEnv env, + final ByteBuffer measurementPathBytes, + final String requestMessage, + final Consumer setFailure, + final boolean needLock) { + final Map dataNodeLocationMap = + env.getConfigManager().getNodeManager().getRegisteredDataNodeLocations(); + final DataNodeAsyncRequestContext clientHandler = + new DataNodeAsyncRequestContext<>( + CnToDnAsyncRequestType.INVALIDATE_MATCHED_SCHEMA_CACHE, + new TInvalidateMatchedSchemaCacheReq(measurementPathBytes).setNeedLock(needLock), + dataNodeLocationMap); + CnToDnInternalServiceAsyncRequestManager.getInstance().sendAsyncRequestWithRetry(clientHandler); + final Map statusMap = clientHandler.getResponseMap(); + for (final TSStatus status : statusMap.values()) { + // All dataNodes must clear the related schemaEngine cache + if (status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + LOGGER.error("Failed to invalidate schemaEngine cache of timeSeries {}", requestMessage); + setFailure.accept( + new ProcedureException(new MetadataException("Invalidate schemaEngine cache failed"))); + return; + } + } + } + + @Override + protected boolean isRollbackSupported( + final AlterTimeSeriesDataTypeState alterTimeSeriesDataTypeState) { + return false; + } + + @Override + protected void rollbackState( + final ConfigNodeProcedureEnv configNodeProcedureEnv, + final AlterTimeSeriesDataTypeState alterTimeSeriesDataTypeState) + throws IOException, InterruptedException, ProcedureException { + // Do nothing + } + + @Override + protected AlterTimeSeriesDataTypeState getState(final int stateId) { + return AlterTimeSeriesDataTypeState.values()[stateId]; + } + + @Override + protected int getStateId(final AlterTimeSeriesDataTypeState alterTimeSeriesDataTypeState) { + return alterTimeSeriesDataTypeState.ordinal(); + } + + @Override + protected AlterTimeSeriesDataTypeState getInitialState() { + return AlterTimeSeriesDataTypeState.CHECK_AND_INVALIDATE_SERIES; + } + + public String getQueryId() { + return queryId; + } + + public MeasurementPath getmeasurementPath() { + return measurementPath; + } + + public void setMeasurementPath(final MeasurementPath measurementPath) { + this.measurementPath = measurementPath; + measurementPathBytes = prepareMeasurementPathBytesData(measurementPath); + } + + public static ByteBuffer prepareMeasurementPathBytesData(final MeasurementPath measurementPath) { + final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); + final DataOutputStream dataOutputStream = new DataOutputStream(byteArrayOutputStream); + try { + measurementPath.serialize(dataOutputStream); + } catch (final IOException ignored) { + // ByteArrayOutputStream won't throw IOException + } + return ByteBuffer.wrap(byteArrayOutputStream.toByteArray()); + } + + public static ByteBuffer preparePatternTreeBytesData(final PathPatternTree patternTree) { + final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); + final DataOutputStream dataOutputStream = new DataOutputStream(byteArrayOutputStream); + try { + patternTree.serialize(dataOutputStream); + } catch (final IOException ignored) { + // ByteArrayOutputStream won't throw IOException + } + return ByteBuffer.wrap(byteArrayOutputStream.toByteArray()); + } + + @Override + public void serialize(final DataOutputStream stream) throws IOException { + stream.writeShort( + isGeneratedByPipe + ? ProcedureType.PIPE_ENRICHED_ALTER_TIMESERIES_DATATYPE_PROCEDURE.getTypeCode() + : ProcedureType.ALTER_TIMESERIES_DATATYPE_PROCEDURE.getTypeCode()); + super.serialize(stream); + ReadWriteIOUtils.write(queryId, stream); + measurementPath.serialize(stream); + ReadWriteIOUtils.write(operationType, stream); + ReadWriteIOUtils.write(dataType, stream); + } + + @Override + public void deserialize(final ByteBuffer byteBuffer) { + super.deserialize(byteBuffer); + queryId = ReadWriteIOUtils.readString(byteBuffer); + setMeasurementPath(MeasurementPath.deserialize(byteBuffer)); + if (getCurrentState() == AlterTimeSeriesDataTypeState.CLEAR_CACHE) { + LOGGER.info("Successfully restored, will set mods to the data regions anyway"); + } + if (byteBuffer.hasRemaining()) { + operationType = ReadWriteIOUtils.readByte(byteBuffer); + } + if (byteBuffer.hasRemaining()) { + dataType = ReadWriteIOUtils.readDataType(byteBuffer); + } + } + + @Override + public boolean equals(final Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + final AlterTimeSeriesDataTypeProcedure that = (AlterTimeSeriesDataTypeProcedure) o; + return this.getProcId() == that.getProcId() + && this.getCurrentState().equals(that.getCurrentState()) + && this.getCycles() == getCycles() + && this.isGeneratedByPipe == that.isGeneratedByPipe + && this.measurementPath.equals(that.measurementPath) + && this.operationType == that.operationType + && this.dataType.equals(that.dataType); + } + + @Override + public int hashCode() { + return Objects.hash( + getProcId(), getCurrentState(), getCycles(), isGeneratedByPipe, measurementPath); + } +} diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/AlterTableColumnDataTypeProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/AlterTableColumnDataTypeProcedure.java new file mode 100644 index 0000000000000..fb02c1c2e59d0 --- /dev/null +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/AlterTableColumnDataTypeProcedure.java @@ -0,0 +1,212 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.confignode.procedure.impl.schema.table; + +import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.commons.exception.IoTDBException; +import org.apache.iotdb.commons.exception.MetadataException; +import org.apache.iotdb.commons.schema.table.TsTable; +import org.apache.iotdb.confignode.consensus.request.write.table.AlterColumnDataTypePlan; +import org.apache.iotdb.confignode.procedure.env.ConfigNodeProcedureEnv; +import org.apache.iotdb.confignode.procedure.exception.ProcedureException; +import org.apache.iotdb.confignode.procedure.impl.schema.SchemaUtils; +import org.apache.iotdb.confignode.procedure.state.schema.AlterTableColumnDataTypeState; +import org.apache.iotdb.confignode.procedure.store.ProcedureType; +import org.apache.iotdb.rpc.TSStatusCode; + +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.utils.Pair; +import org.apache.tsfile.utils.ReadWriteIOUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Objects; + +public class AlterTableColumnDataTypeProcedure + extends AbstractAlterOrDropTableProcedure { + private static final Logger LOGGER = + LoggerFactory.getLogger(AlterTableColumnDataTypeProcedure.class); + + private String columnName; + private TSDataType dataType; + + public AlterTableColumnDataTypeProcedure(final boolean isGeneratedByPipe) { + super(isGeneratedByPipe); + } + + public AlterTableColumnDataTypeProcedure( + final String database, + final String tableName, + final String queryId, + final String columnName, + final TSDataType dataType, + final boolean isGeneratedByPipe) { + super(database, tableName, queryId, isGeneratedByPipe); + this.columnName = columnName; + this.dataType = dataType; + } + + @Override + protected String getActionMessage() { + return "Alter table column data type"; + } + + @Override + protected Flow executeFromState( + final ConfigNodeProcedureEnv env, final AlterTableColumnDataTypeState state) + throws InterruptedException { + final long startTime = System.currentTimeMillis(); + try { + switch (state) { + case CHECK_AND_INVALIDATE_COLUMN: + LOGGER.info( + "Check and invalidate column {} in {}.{} when altering column data type", + columnName, + database, + tableName); + checkAndPreAlterColumn(env); + break; + case PRE_RELEASE: + LOGGER.info("Pre-release info of table {}.{} when altering column", database, tableName); + preRelease(env); + break; + case ALTER_TABLE_COLUMN_DATA_TYPE: + LOGGER.info("Altering column {} in {}.{} on configNode", columnName, database, tableName); + alterColumnDataType(env); + break; + case COMMIT_RELEASE: + LOGGER.info( + "Commit release info of table {}.{} when altering column", database, tableName); + commitRelease(env); + return Flow.NO_MORE_STATE; + default: + setFailure( + new ProcedureException("Unrecognized AlterTableColumnDataTypeProcedure " + state)); + return Flow.NO_MORE_STATE; + } + return Flow.HAS_MORE_STATE; + } finally { + LOGGER.info( + "AlterTableColumnDataType-{}.{}-{} costs {}ms", + database, + tableName, + state, + (System.currentTimeMillis() - startTime)); + } + } + + @Override + protected void preRelease(ConfigNodeProcedureEnv env) { + super.preRelease(env); + setNextState(AlterTableColumnDataTypeState.ALTER_TABLE_COLUMN_DATA_TYPE); + } + + private void checkAndPreAlterColumn(final ConfigNodeProcedureEnv env) { + try { + final Pair result = + env.getConfigManager() + .getClusterSchemaManager() + .tableColumnCheckForColumnAltering(database, tableName, columnName, dataType); + final TSStatus status = result.getLeft(); + if (status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + setFailure( + new ProcedureException(new IoTDBException(status.getMessage(), status.getCode()))); + return; + } + table = result.getRight(); + setNextState(AlterTableColumnDataTypeState.PRE_RELEASE); + } catch (final MetadataException e) { + setFailure(new ProcedureException(e)); + } + } + + private void alterColumnDataType(final ConfigNodeProcedureEnv env) { + final TSStatus status = + SchemaUtils.executeInConsensusLayer( + new AlterColumnDataTypePlan(database, tableName, columnName, dataType), env, LOGGER); + if (status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + setFailure(new ProcedureException(new IoTDBException(status.getMessage(), status.getCode()))); + } + setNextState(AlterTableColumnDataTypeState.COMMIT_RELEASE); + } + + @Override + protected boolean isRollbackSupported(final AlterTableColumnDataTypeState state) { + return false; + } + + @Override + protected void rollbackState( + final ConfigNodeProcedureEnv configNodeProcedureEnv, + final AlterTableColumnDataTypeState alterTableColumnDataTypeState) + throws IOException, InterruptedException, ProcedureException { + // Do nothing + } + + @Override + protected AlterTableColumnDataTypeState getState(final int stateId) { + return AlterTableColumnDataTypeState.values()[stateId]; + } + + @Override + protected int getStateId(final AlterTableColumnDataTypeState alterTableColumnDataTypeState) { + return alterTableColumnDataTypeState.ordinal(); + } + + @Override + protected AlterTableColumnDataTypeState getInitialState() { + return AlterTableColumnDataTypeState.CHECK_AND_INVALIDATE_COLUMN; + } + + @Override + public void serialize(final DataOutputStream stream) throws IOException { + stream.writeShort( + isGeneratedByPipe + ? ProcedureType.PIPE_ENRICHED_ALTER_COLUMN_DATATYPE_PROCEDURE.getTypeCode() + : ProcedureType.ALTER_TABLE_COLUMN_DATATYPE_PROCEDURE.getTypeCode()); + super.serialize(stream); + + ReadWriteIOUtils.write(columnName, stream); + ReadWriteIOUtils.write(dataType, stream); + } + + @Override + public void deserialize(final ByteBuffer byteBuffer) { + super.deserialize(byteBuffer); + + this.columnName = ReadWriteIOUtils.readString(byteBuffer); + this.dataType = ReadWriteIOUtils.readDataType(byteBuffer); + } + + @Override + public boolean equals(final Object o) { + return super.equals(o) + && Objects.equals(columnName, ((AlterTableColumnDataTypeProcedure) o).columnName) + && Objects.equals(dataType, ((AlterTableColumnDataTypeProcedure) o).dataType); + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), columnName, dataType); + } +} diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/DropTableColumnProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/DropTableColumnProcedure.java index 65b591f0322bc..66e1789e9ba72 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/DropTableColumnProcedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/DropTableColumnProcedure.java @@ -116,7 +116,7 @@ protected Flow executeFromState( dropColumn(env); return Flow.NO_MORE_STATE; default: - setFailure(new ProcedureException("Unrecognized CreateTableState " + state)); + setFailure(new ProcedureException("Unrecognized DropTableColumnState " + state)); return Flow.NO_MORE_STATE; } return Flow.HAS_MORE_STATE; diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/state/schema/AlterTableColumnDataTypeState.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/state/schema/AlterTableColumnDataTypeState.java new file mode 100644 index 0000000000000..736e9fa4918d7 --- /dev/null +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/state/schema/AlterTableColumnDataTypeState.java @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.confignode.procedure.state.schema; + +public enum AlterTableColumnDataTypeState { + CHECK_AND_INVALIDATE_COLUMN, + PRE_RELEASE, + ALTER_TABLE_COLUMN_DATA_TYPE, + COMMIT_RELEASE +} diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/state/schema/AlterTimeSeriesDataTypeState.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/state/schema/AlterTimeSeriesDataTypeState.java new file mode 100644 index 0000000000000..15af13556fae5 --- /dev/null +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/state/schema/AlterTimeSeriesDataTypeState.java @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.confignode.procedure.state.schema; + +public enum AlterTimeSeriesDataTypeState { + CHECK_AND_INVALIDATE_SERIES, + ALTER_TIME_SERIES_DATA_TYPE, + CLEAR_CACHE +} diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/store/ProcedureFactory.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/store/ProcedureFactory.java index f20a6999d5936..dd15558608718 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/store/ProcedureFactory.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/store/ProcedureFactory.java @@ -44,6 +44,7 @@ import org.apache.iotdb.confignode.procedure.impl.region.RemoveRegionPeerProcedure; import org.apache.iotdb.confignode.procedure.impl.schema.AlterEncodingCompressorProcedure; import org.apache.iotdb.confignode.procedure.impl.schema.AlterLogicalViewProcedure; +import org.apache.iotdb.confignode.procedure.impl.schema.AlterTimeSeriesDataTypeProcedure; import org.apache.iotdb.confignode.procedure.impl.schema.DeactivateTemplateProcedure; import org.apache.iotdb.confignode.procedure.impl.schema.DeleteDatabaseProcedure; import org.apache.iotdb.confignode.procedure.impl.schema.DeleteLogicalViewProcedure; @@ -52,6 +53,7 @@ import org.apache.iotdb.confignode.procedure.impl.schema.SetTemplateProcedure; import org.apache.iotdb.confignode.procedure.impl.schema.UnsetTemplateProcedure; import org.apache.iotdb.confignode.procedure.impl.schema.table.AddTableColumnProcedure; +import org.apache.iotdb.confignode.procedure.impl.schema.table.AlterTableColumnDataTypeProcedure; import org.apache.iotdb.confignode.procedure.impl.schema.table.CreateTableProcedure; import org.apache.iotdb.confignode.procedure.impl.schema.table.DeleteDevicesProcedure; import org.apache.iotdb.confignode.procedure.impl.schema.table.DropTableColumnProcedure; @@ -145,6 +147,9 @@ public Procedure create(ByteBuffer buffer) throws IOException { case DELETE_TIMESERIES_PROCEDURE: procedure = new DeleteTimeSeriesProcedure(false); break; + case ALTER_TIMESERIES_DATATYPE_PROCEDURE: + procedure = new AlterTimeSeriesDataTypeProcedure(false); + break; case DELETE_LOGICAL_VIEW_PROCEDURE: procedure = new DeleteLogicalViewProcedure(false); break; @@ -240,6 +245,9 @@ public Procedure create(ByteBuffer buffer) throws IOException { case DROP_VIEW_COLUMN_PROCEDURE: procedure = new DropViewColumnProcedure(false); break; + case ALTER_TABLE_COLUMN_DATATYPE_PROCEDURE: + procedure = new AlterTableColumnDataTypeProcedure(false); + break; case DROP_TABLE_PROCEDURE: procedure = new DropTableProcedure(false); break; @@ -312,6 +320,12 @@ public Procedure create(ByteBuffer buffer) throws IOException { case PIPE_ENRICHED_DROP_TABLE_COLUMN_PROCEDURE: procedure = new DropTableColumnProcedure(true); break; + case PIPE_ENRICHED_ALTER_COLUMN_DATATYPE_PROCEDURE: + procedure = new AlterTableColumnDataTypeProcedure(true); + break; + case PIPE_ENRICHED_ALTER_TIMESERIES_DATATYPE_PROCEDURE: + procedure = new AlterTimeSeriesDataTypeProcedure(true); + break; case PIPE_ENRICHED_DELETE_DEVICES_PROCEDURE: procedure = new DeleteDevicesProcedure(true); break; @@ -428,6 +442,8 @@ public static ProcedureType getProcedureType(final Procedure procedure) { return ProcedureType.ALTER_ENCODING_COMPRESSOR_PROCEDURE; } else if (procedure instanceof DeleteTimeSeriesProcedure) { return ProcedureType.DELETE_TIMESERIES_PROCEDURE; + } else if (procedure instanceof AlterTimeSeriesDataTypeProcedure) { + return ProcedureType.ALTER_TIMESERIES_DATATYPE_PROCEDURE; } else if (procedure instanceof ReconstructRegionProcedure) { return ProcedureType.RECONSTRUCT_REGION_PROCEDURE; } else if (procedure instanceof NotifyRegionMigrationProcedure) { @@ -474,6 +490,8 @@ public static ProcedureType getProcedureType(final Procedure procedure) { return ProcedureType.DROP_TABLE_COLUMN_PROCEDURE; } else if (procedure instanceof DropViewProcedure) { return ProcedureType.DROP_VIEW_PROCEDURE; + } else if (procedure instanceof AlterTableColumnDataTypeProcedure) { + return ProcedureType.ALTER_TABLE_COLUMN_DATATYPE_PROCEDURE; } else if (procedure instanceof DropTableProcedure) { return ProcedureType.DROP_TABLE_PROCEDURE; } else if (procedure instanceof DeleteDevicesProcedure) { diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/store/ProcedureType.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/store/ProcedureType.java index d076a7d9d926e..820a90f7ebfb9 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/store/ProcedureType.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/store/ProcedureType.java @@ -47,6 +47,7 @@ public enum ProcedureType { /** Timeseries */ DELETE_TIMESERIES_PROCEDURE((short) 300), ALTER_ENCODING_COMPRESSOR_PROCEDURE((short) 301), + ALTER_TIMESERIES_DATATYPE_PROCEDURE((short) 302), /** Trigger */ CREATE_TRIGGER_PROCEDURE((short) 400), @@ -84,6 +85,8 @@ public enum ProcedureType { RENAME_VIEW_COLUMN_PROCEDURE((short) 763), RENAME_VIEW_PROCEDURE((short) 764), + ALTER_TABLE_COLUMN_DATATYPE_PROCEDURE((short) 765), + /** AI Model */ @Deprecated // Since 2.0.6, all models are managed by AINode CREATE_MODEL_PROCEDURE((short) 800), @@ -144,12 +147,14 @@ public enum ProcedureType { PIPE_ENRICHED_RENAME_TABLE_PROCEDURE((short) 1419), PIPE_ENRICHED_CREATE_TABLE_VIEW_PROCEDURE((short) 1420), PIPE_ENRICHED_ADD_VIEW_COLUMN_PROCEDURE((short) 1421), + PIPE_ENRICHED_ALTER_COLUMN_DATATYPE_PROCEDURE((short) 1422), PIPE_ENRICHED_DROP_VIEW_COLUMN_PROCEDURE((short) 143), PIPE_ENRICHED_DROP_VIEW_PROCEDURE((short) 144), PIPE_ENRICHED_SET_VIEW_PROPERTIES_PROCEDURE((short) 145), PIPE_ENRICHED_RENAME_VIEW_COLUMN_PROCEDURE((short) 146), PIPE_ENRICHED_RENAME_VIEW_PROCEDURE((short) 147), PIPE_ENRICHED_ALTER_ENCODING_COMPRESSOR_PROCEDURE((short) 148), + PIPE_ENRICHED_ALTER_TIMESERIES_DATATYPE_PROCEDURE((short) 149), /** Subscription */ CREATE_TOPIC_PROCEDURE((short) 1500), diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/thrift/ConfigNodeRPCServiceProcessor.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/thrift/ConfigNodeRPCServiceProcessor.java index 6582a5bfff8e1..afd399951da8d 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/thrift/ConfigNodeRPCServiceProcessor.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/thrift/ConfigNodeRPCServiceProcessor.java @@ -99,6 +99,7 @@ import org.apache.iotdb.confignode.rpc.thrift.TAlterOrDropTableReq; import org.apache.iotdb.confignode.rpc.thrift.TAlterPipeReq; import org.apache.iotdb.confignode.rpc.thrift.TAlterSchemaTemplateReq; +import org.apache.iotdb.confignode.rpc.thrift.TAlterTimeSeriesReq; import org.apache.iotdb.confignode.rpc.thrift.TAuthizedPatternTreeResp; import org.apache.iotdb.confignode.rpc.thrift.TAuthorizerRelationalReq; import org.apache.iotdb.confignode.rpc.thrift.TAuthorizerReq; @@ -1180,6 +1181,11 @@ public TSStatus alterLogicalView(TAlterLogicalViewReq req) { return configManager.alterLogicalView(req); } + @Override + public TSStatus alterTimeSeriesDataType(final TAlterTimeSeriesReq req) { + return configManager.alterTimeSeriesDataType(req); + } + @Override public TSStatus createPipe(TCreatePipeReq req) { return configManager.createPipe(req); diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanSerDeTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanSerDeTest.java index 2b5d527d75da1..14d7b0ab6d419 100644 --- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanSerDeTest.java +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanSerDeTest.java @@ -132,6 +132,7 @@ import org.apache.iotdb.confignode.consensus.request.write.sync.RecordPipeMessagePlan; import org.apache.iotdb.confignode.consensus.request.write.sync.SetPipeStatusPlanV1; import org.apache.iotdb.confignode.consensus.request.write.table.AddTableColumnPlan; +import org.apache.iotdb.confignode.consensus.request.write.table.AlterColumnDataTypePlan; import org.apache.iotdb.confignode.consensus.request.write.table.CommitCreateTablePlan; import org.apache.iotdb.confignode.consensus.request.write.table.CommitDeleteColumnPlan; import org.apache.iotdb.confignode.consensus.request.write.table.CommitDeleteTablePlan; @@ -1522,6 +1523,24 @@ public void SetTableColumnCommentPlanTest() throws IOException { setTableColumnCommentPlan.getColumnName(), setTableColumnCommentPlan1.getColumnName()); } + @Test + public void PreAlterTableColumnDataTypePlanTest() throws IOException { + final AlterColumnDataTypePlan alterColumnDataTypePlan = + new AlterColumnDataTypePlan("database1", "table1", "field", TSDataType.FLOAT); + final AlterColumnDataTypePlan alterColumnDataTypePlan1 = + (AlterColumnDataTypePlan) + ConfigPhysicalPlan.Factory.create(alterColumnDataTypePlan.serializeToByteBuffer()); + Assert.assertEquals( + alterColumnDataTypePlan.getDatabase(), alterColumnDataTypePlan1.getDatabase()); + Assert.assertEquals( + alterColumnDataTypePlan.getTableName(), alterColumnDataTypePlan1.getTableName()); + Assert.assertEquals( + alterColumnDataTypePlan.getColumnName(), alterColumnDataTypePlan1.getColumnName()); + Assert.assertEquals(alterColumnDataTypePlan.getType(), alterColumnDataTypePlan1.getType()); + Assert.assertEquals( + alterColumnDataTypePlan.getNewType(), alterColumnDataTypePlan1.getNewType()); + } + @Test public void RenameTablePlanTest() throws IOException { final RenameTablePlan renameTablePlan = diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/pipe/source/PipeConfigTablePatternParseVisitorTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/pipe/source/PipeConfigTablePatternParseVisitorTest.java index fea613bc47bb8..229c0452dcc0a 100644 --- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/pipe/source/PipeConfigTablePatternParseVisitorTest.java +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/pipe/source/PipeConfigTablePatternParseVisitorTest.java @@ -30,6 +30,7 @@ import org.apache.iotdb.confignode.consensus.request.write.pipe.payload.PipeCreateTableOrViewPlan; import org.apache.iotdb.confignode.consensus.request.write.pipe.payload.PipeDeleteDevicesPlan; import org.apache.iotdb.confignode.consensus.request.write.table.AddTableColumnPlan; +import org.apache.iotdb.confignode.consensus.request.write.table.AlterColumnDataTypePlan; import org.apache.iotdb.confignode.consensus.request.write.table.CommitDeleteColumnPlan; import org.apache.iotdb.confignode.consensus.request.write.table.CommitDeleteTablePlan; import org.apache.iotdb.confignode.consensus.request.write.table.RenameTableColumnPlan; @@ -46,6 +47,7 @@ import org.apache.iotdb.confignode.consensus.request.write.table.view.SetViewPropertiesPlan; import org.apache.iotdb.confignode.rpc.thrift.TDatabaseSchema; +import org.apache.tsfile.enums.TSDataType; import org.junit.Assert; import org.junit.Test; @@ -255,4 +257,12 @@ private void testInput( .process(falseInput2, tablePattern) .isPresent()); } + + @Test + public void testAlterTableColumnDataType() { + testInput( + new AlterColumnDataTypePlan("db1", "ab", "a", TSDataType.INT64), + new AlterColumnDataTypePlan("db1", "ac", "a", TSDataType.BLOB), + new AlterColumnDataTypePlan("da", "ac", "a", TSDataType.DATE)); + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/schemaregion/SchemaExecutionVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/schemaregion/SchemaExecutionVisitor.java index 29e914594204b..de92827457a76 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/schemaregion/SchemaExecutionVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/schemaregion/SchemaExecutionVisitor.java @@ -27,6 +27,7 @@ import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.schema.template.Template; import org.apache.iotdb.commons.schema.view.viewExpression.ViewExpression; +import org.apache.iotdb.commons.utils.MetadataUtils; import org.apache.iotdb.db.exception.metadata.MeasurementAlreadyExistException; import org.apache.iotdb.db.exception.metadata.template.TemplateIsInUseException; import org.apache.iotdb.db.pipe.agent.PipeDataNodeAgent; @@ -429,6 +430,19 @@ public TSStatus visitAlterTimeSeries( schemaRegion.upsertAliasAndTagsAndAttributes( node.getAlias(), node.getTagsMap(), node.getAttributesMap(), node.getPath()); break; + case SET_DATA_TYPE: + MeasurementPath measurementPath = schemaRegion.fetchMeasurementPath(node.getPath()); + if (!MetadataUtils.canAlter( + measurementPath.getMeasurementSchema().getType(), node.getDataType())) { + throw new MetadataException( + String.format( + "The timeseries %s used new type %s is not compatible with the existing one %s.", + node.getPath().getFullPath(), + node.getDataType(), + node.getPath().getMeasurementSchema().getType())); + } + schemaRegion.alterTimeSeriesDataType(node.getDataType(), node.getPath()); + break; } } catch (MetadataException e) { logger.error("{}: MetaData error: ", IoTDBConstant.GLOBAL_DB_NAME, e); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/exception/DataTypeInconsistentException.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/exception/DataTypeInconsistentException.java new file mode 100644 index 0000000000000..40f519abff436 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/exception/DataTypeInconsistentException.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.exception; + +import org.apache.tsfile.enums.TSDataType; + +public class DataTypeInconsistentException extends WriteProcessException { + + public DataTypeInconsistentException(TSDataType existing, TSDataType incoming) { + super( + String.format( + "Inconsistent data types, existing data type: %s, incoming: %s", existing, incoming)); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/transform/converter/ValueConverter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/transform/converter/ValueConverter.java index 911fac9f5199f..0ca6c2aa0c454 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/transform/converter/ValueConverter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/transform/converter/ValueConverter.java @@ -624,7 +624,7 @@ public static double convertDateToDouble(final int value) { } public static Binary convertDateToText(final int value) { - return parseText(Integer.toString(value)); + return parseText(DateUtils.parseIntToLocalDate(value).toString()); } public static long convertDateToTimestamp(final int value) { @@ -639,11 +639,11 @@ public static long convertDateToTimestamp(final int value) { } public static Binary convertDateToBlob(final int value) { - return parseBlob(Integer.toString(value)); + return parseBlob(DateUtils.parseIntToLocalDate(value).toString()); } public static Binary convertDateToString(final int value) { - return parseString(Integer.toString(value)); + return parseString(DateUtils.parseIntToLocalDate(value).toString()); } ///////////// BLOB ////////////// diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/visitor/PipePlanToStatementVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/visitor/PipePlanToStatementVisitor.java index fc073226043ce..4570a98e8dc4a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/visitor/PipePlanToStatementVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/visitor/PipePlanToStatementVisitor.java @@ -221,6 +221,7 @@ public AlterTimeSeriesStatement visitAlterTimeSeries( statement.setAlias(node.getAlias()); statement.setTagsMap(node.getTagsMap()); statement.setPath(node.getPath()); + statement.setDataType(node.getDataType()); return statement; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/InsertNodeMemoryEstimator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/InsertNodeMemoryEstimator.java index 7f1d7357b02e4..01fe89abdebe8 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/InsertNodeMemoryEstimator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/InsertNodeMemoryEstimator.java @@ -39,6 +39,7 @@ import org.apache.tsfile.common.constant.TsFileConstant; import org.apache.tsfile.encoding.encoder.TSEncodingBuilder; +import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.file.metadata.enums.TSEncoding; import org.apache.tsfile.utils.Binary; @@ -48,6 +49,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -232,7 +234,7 @@ private static long sizeOfInsertTabletNode(final InsertTabletNode node) { size += calculateFullInsertNodeSize(node); size += RamUsageEstimator.sizeOf(node.getTimes()); size += RamUsageEstimator.sizeOf(node.getBitMaps()); - size += sizeOfColumns(node.getColumns(), node.getMeasurementSchemas()); + size += sizeOfColumns(node.getColumns(), node.getMeasurementSchemas(), node.getDataTypes()); final List range = node.getRange(); if (range != null) { size += NUM_BYTES_OBJECT_HEADER + SIZE_OF_INT * range.size(); @@ -249,7 +251,7 @@ private static long calculateInsertTabletNodeSizeExcludingSchemas(final InsertTa size += RamUsageEstimator.sizeOf(node.getBitMaps()); - size += sizeOfColumns(node.getColumns(), node.getMeasurementSchemas()); + size += sizeOfColumns(node.getColumns(), node.getMeasurementSchemas(), node.getDataTypes()); final List range = node.getRange(); if (range != null) { @@ -261,14 +263,14 @@ private static long calculateInsertTabletNodeSizeExcludingSchemas(final InsertTa private static long sizeOfInsertRowNode(final InsertRowNode node) { long size = INSERT_ROW_NODE_SIZE; size += calculateFullInsertNodeSize(node); - size += sizeOfValues(node.getValues(), node.getMeasurementSchemas()); + size += sizeOfValues(node.getValues(), node.getMeasurementSchemas(), node.getDataTypes()); return size; } private static long calculateInsertRowNodeExcludingSchemas(final InsertRowNode node) { long size = INSERT_ROW_NODE_SIZE; size += calculateInsertNodeSizeExcludingSchemas(node); - size += sizeOfValues(node.getValues(), node.getMeasurementSchemas()); + size += sizeOfValues(node.getValues(), node.getMeasurementSchemas(), node.getDataTypes()); return size; } @@ -379,7 +381,7 @@ private static long sizeOfRelationalInsertRowsNode(final RelationalInsertRowsNod private static long sizeOfRelationalInsertRowNode(final RelationalInsertRowNode node) { long size = RELATIONAL_INSERT_ROW_NODE_SIZE; size += calculateFullInsertNodeSize(node); - size += sizeOfValues(node.getValues(), node.getMeasurementSchemas()); + size += sizeOfValues(node.getValues(), node.getMeasurementSchemas(), node.getDataTypes()); return size; } @@ -392,7 +394,7 @@ private static long sizeOfRelationalInsertTabletNode(final RelationalInsertTable size += RamUsageEstimator.sizeOf(node.getBitMaps()); - size += sizeOfColumns(node.getColumns(), node.getMeasurementSchemas()); + size += sizeOfColumns(node.getColumns(), node.getMeasurementSchemas(), node.getDataTypes()); final List range = node.getRange(); if (range != null) { @@ -556,7 +558,9 @@ private static long sizeOfBinary(final Binary binary) { } public static long sizeOfColumns( - final Object[] columns, final MeasurementSchema[] measurementSchemas) { + final Object[] columns, + final MeasurementSchema[] measurementSchemas, + TSDataType[] dataTypes) { // Directly calculate if measurementSchemas are absent if (Objects.isNull(measurementSchemas)) { return RamUsageEstimator.shallowSizeOf(columns) @@ -571,27 +575,65 @@ public static long sizeOfColumns( if (measurementSchemas[i] == null || measurementSchemas[i].getType() == null) { continue; } - switch (measurementSchemas[i].getType()) { + switch ((dataTypes != null && dataTypes[i] != null) + ? dataTypes[i] + : measurementSchemas[i].getType()) { case INT64: case TIMESTAMP: { - size += RamUsageEstimator.sizeOf((long[]) columns[i]); + if (columns[i] instanceof long[]) { + size += RamUsageEstimator.sizeOf((long[]) columns[i]); + } else { + Object[] array = (Object[]) columns[i]; + long[] targetArray = new long[array.length]; + for (int j = 0; j < array.length; j++) { + targetArray[j] = ((Number) array[j]).longValue(); + } + size += RamUsageEstimator.sizeOf(targetArray); + } break; } case DATE: case INT32: { - size += RamUsageEstimator.sizeOf((int[]) columns[i]); + if (columns[i] instanceof int[]) { + size += RamUsageEstimator.sizeOf((int[]) columns[i]); + } else { + Object[] array = (Object[]) columns[i]; + int[] targetArray = new int[array.length]; + for (int j = 0; j < array.length; j++) { + targetArray[j] = ((Number) array[j]).intValue(); + } + size += RamUsageEstimator.sizeOf(targetArray); + } break; } case DOUBLE: { - size += RamUsageEstimator.sizeOf((double[]) columns[i]); + if (columns[i] instanceof double[]) { + size += RamUsageEstimator.sizeOf((double[]) columns[i]); + } else { + Object[] array = (Object[]) columns[i]; + double[] targetArray = new double[array.length]; + for (int j = 0; j < array.length; j++) { + targetArray[j] = ((Number) array[j]).doubleValue(); + } + size += RamUsageEstimator.sizeOf(targetArray); + } break; } case FLOAT: { - size += RamUsageEstimator.sizeOf((float[]) columns[i]); + if (columns[i] instanceof float[]) { + size += RamUsageEstimator.sizeOf((float[]) columns[i]); + } else { + Object[] array = (Object[]) columns[i]; + float[] targetArray = new float[array.length]; + for (int j = 0; j < array.length; j++) { + targetArray[j] = ((Number) array[j]).floatValue(); + } + size += RamUsageEstimator.sizeOf(targetArray); + } break; } case BOOLEAN: @@ -604,6 +646,16 @@ public static long sizeOfColumns( case BLOB: case OBJECT: { + if (columns[i] instanceof Binary[]) { + size += RamUsageEstimator.sizeOf((Binary[]) columns[i]); + } else { + Object[] array = (Object[]) columns[i]; + Binary[] targetArray = new Binary[array.length]; + for (int j = 0; j < array.length; j++) { + targetArray[j] = new Binary(String.valueOf(array[j]), StandardCharsets.UTF_8); + } + size += RamUsageEstimator.sizeOf(targetArray); + } size += RamUsageEstimator.sizeOf((Binary[]) columns[i]); break; } @@ -619,7 +671,7 @@ private static long getNumBytesUnknownObject(final Object obj) { } public static long sizeOfValues( - final Object[] values, final MeasurementSchema[] measurementSchemas) { + final Object[] values, final MeasurementSchema[] measurementSchemas, TSDataType[] dataTypes) { // Directly calculate if measurementSchemas are absent if (Objects.isNull(measurementSchemas)) { return RamUsageEstimator.shallowSizeOf(values) @@ -635,7 +687,9 @@ public static long sizeOfValues( size += NUM_BYTES_OBJECT_HEADER; continue; } - switch (measurementSchemas[i].getType()) { + switch ((dataTypes != null && dataTypes[i] != null) + ? dataTypes[i] + : measurementSchemas[i].getType()) { case INT64: case TIMESTAMP: { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/ConfigNodeClient.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/ConfigNodeClient.java index df80d49b502b0..114629c0ed709 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/ConfigNodeClient.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/ConfigNodeClient.java @@ -57,6 +57,7 @@ import org.apache.iotdb.confignode.rpc.thrift.TAlterOrDropTableReq; import org.apache.iotdb.confignode.rpc.thrift.TAlterPipeReq; import org.apache.iotdb.confignode.rpc.thrift.TAlterSchemaTemplateReq; +import org.apache.iotdb.confignode.rpc.thrift.TAlterTimeSeriesReq; import org.apache.iotdb.confignode.rpc.thrift.TAuthizedPatternTreeResp; import org.apache.iotdb.confignode.rpc.thrift.TAuthorizerRelationalReq; import org.apache.iotdb.confignode.rpc.thrift.TAuthorizerReq; @@ -1124,6 +1125,12 @@ public TSStatus alterLogicalView(TAlterLogicalViewReq req) throws TException { () -> client.alterLogicalView(req), status -> !updateConfigNodeLeader(status)); } + @Override + public TSStatus alterTimeSeriesDataType(TAlterTimeSeriesReq req) throws TException { + return executeRemoteCallWithRetry( + () -> client.alterTimeSeriesDataType(req), status -> !updateConfigNodeLeader(status)); + } + @Override public TSStatus createPipe(TCreatePipeReq req) throws TException { return executeRemoteCallWithRetry( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java index 8f1b97b71964a..acdef794fa390 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java @@ -147,6 +147,7 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeType; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.load.LoadTsFilePieceNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metadata.write.AlterEncodingCompressorNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metadata.write.AlterTimeSeriesNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metadata.write.ConstructSchemaBlackListNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metadata.write.DeactivateTemplateNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metadata.write.DeleteTimeSeriesNode; @@ -187,6 +188,7 @@ import org.apache.iotdb.db.service.RegionMigrateService; import org.apache.iotdb.db.service.metrics.FileMetrics; import org.apache.iotdb.db.storageengine.StorageEngine; +import org.apache.iotdb.db.storageengine.dataregion.DataRegion; import org.apache.iotdb.db.storageengine.dataregion.compaction.repair.RepairTaskStatus; import org.apache.iotdb.db.storageengine.dataregion.compaction.schedule.CompactionScheduleTaskManager; import org.apache.iotdb.db.storageengine.dataregion.compaction.schedule.CompactionTaskManager; @@ -209,6 +211,7 @@ import org.apache.iotdb.mpp.rpc.thrift.IDataNodeRPCService; import org.apache.iotdb.mpp.rpc.thrift.TActiveTriggerInstanceReq; import org.apache.iotdb.mpp.rpc.thrift.TAlterEncodingCompressorReq; +import org.apache.iotdb.mpp.rpc.thrift.TAlterTimeSeriesReq; import org.apache.iotdb.mpp.rpc.thrift.TAlterViewReq; import org.apache.iotdb.mpp.rpc.thrift.TAttributeUpdateReq; import org.apache.iotdb.mpp.rpc.thrift.TAuditLogReq; @@ -359,6 +362,7 @@ import static org.apache.iotdb.commons.client.request.TestConnectionUtils.testConnectionsImpl; import static org.apache.iotdb.commons.conf.IoTDBConstant.MULTI_LEVEL_PATH_WILDCARD; +import static org.apache.iotdb.db.queryengine.plan.statement.metadata.AlterTimeSeriesStatement.AlterType.SET_DATA_TYPE; import static org.apache.iotdb.db.service.RegionMigrateService.REGION_MIGRATE_PROCESS; import static org.apache.iotdb.db.utils.ErrorHandlingUtils.onIoTDBException; import static org.apache.iotdb.db.utils.ErrorHandlingUtils.onQueryException; @@ -845,6 +849,32 @@ public TSStatus alterEncodingCompressor(final TAlterEncodingCompressorReq req) t }); } + @Override + public TSStatus alterTimeSeriesDataType(TAlterTimeSeriesReq req) throws TException { + final MeasurementPath measurementPath = + (MeasurementPath) + PathDeserializeUtil.deserialize(ByteBuffer.wrap(req.getMeasurementPath())); + return executeInternalSchemaTask( + req.getSchemaRegionIdList(), + consensusGroupId -> { + final String database = + schemaEngine + .getSchemaRegion(new SchemaRegionId(consensusGroupId.getId())) + .getDatabaseFullPath(); + final RegionWriteExecutor executor = new RegionWriteExecutor(); + return executor + .execute( + new SchemaRegionId(consensusGroupId.getId()), + new AlterTimeSeriesNode( + new PlanNodeId(""), + measurementPath, + SET_DATA_TYPE, + false, + TSDataType.deserialize(req.updateInfo.get()))) + .getStatus(); + }); + } + @Override public TSStatus constructSchemaBlackListWithTemplate(TConstructSchemaBlackListWithTemplateReq req) throws TException { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/ExtremeAccumulator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/ExtremeAccumulator.java index ff54a48ce5e79..13bdb9fecdd3e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/ExtremeAccumulator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/ExtremeAccumulator.java @@ -108,20 +108,20 @@ public void addStatistics(Statistics statistics) { } switch (seriesDataType) { case INT32: - updateIntResult((int) statistics.getMaxValue()); - updateIntResult((int) statistics.getMinValue()); + updateIntResult(((Number) statistics.getMaxValue()).intValue()); + updateIntResult(((Number) statistics.getMinValue()).intValue()); break; case INT64: - updateLongResult((long) statistics.getMaxValue()); - updateLongResult((long) statistics.getMinValue()); + updateLongResult(((Number) statistics.getMaxValue()).longValue()); + updateLongResult(((Number) statistics.getMinValue()).longValue()); break; case FLOAT: - updateFloatResult((float) statistics.getMaxValue()); - updateFloatResult((float) statistics.getMinValue()); + updateFloatResult(((Number) statistics.getMaxValue()).floatValue()); + updateFloatResult(((Number) statistics.getMinValue()).floatValue()); break; case DOUBLE: - updateDoubleResult((double) statistics.getMaxValue()); - updateDoubleResult((double) statistics.getMinValue()); + updateDoubleResult(((Number) statistics.getMaxValue()).doubleValue()); + updateDoubleResult(((Number) statistics.getMinValue()).doubleValue()); break; case TEXT: case STRING: diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/FirstValueAccumulator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/FirstValueAccumulator.java index 640008f0a5fda..f375ca954f344 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/FirstValueAccumulator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/FirstValueAccumulator.java @@ -22,12 +22,15 @@ import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.statistics.DateStatistics; import org.apache.tsfile.file.metadata.statistics.Statistics; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.BitMap; import org.apache.tsfile.utils.TsPrimitiveType; import org.apache.tsfile.write.UnSupportedDataTypeException; +import java.nio.charset.StandardCharsets; + import static com.google.common.base.Preconditions.checkArgument; public class FirstValueAccumulator implements Accumulator { @@ -120,23 +123,41 @@ public void addStatistics(Statistics statistics) { switch (seriesDataType) { case INT32: case DATE: - updateIntFirstValue((int) statistics.getFirstValue(), statistics.getStartTime()); + updateIntFirstValue( + ((Number) statistics.getFirstValue()).intValue(), statistics.getStartTime()); break; case INT64: case TIMESTAMP: - updateLongFirstValue((long) statistics.getFirstValue(), statistics.getStartTime()); + updateLongFirstValue( + ((Number) statistics.getFirstValue()).longValue(), statistics.getStartTime()); break; case FLOAT: - updateFloatFirstValue((float) statistics.getFirstValue(), statistics.getStartTime()); + updateFloatFirstValue( + ((Number) statistics.getFirstValue()).floatValue(), statistics.getStartTime()); break; case DOUBLE: - updateDoubleFirstValue((double) statistics.getFirstValue(), statistics.getStartTime()); + updateDoubleFirstValue( + ((Number) statistics.getFirstValue()).doubleValue(), statistics.getStartTime()); break; case TEXT: case BLOB: case OBJECT: case STRING: - updateBinaryFirstValue((Binary) statistics.getFirstValue(), statistics.getStartTime()); + if (statistics instanceof DateStatistics) { + updateBinaryFirstValue( + new Binary( + TSDataType.getDateStringValue((Integer) statistics.getFirstValue()), + StandardCharsets.UTF_8), + statistics.getStartTime()); + } else { + if (statistics.getFirstValue() instanceof Binary) { + updateBinaryFirstValue((Binary) statistics.getFirstValue(), statistics.getStartTime()); + } else { + updateBinaryFirstValue( + new Binary(String.valueOf(statistics.getFirstValue()), StandardCharsets.UTF_8), + statistics.getStartTime()); + } + } break; case BOOLEAN: updateBooleanFirstValue((boolean) statistics.getFirstValue(), statistics.getStartTime()); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/LastValueAccumulator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/LastValueAccumulator.java index 36c6ed5f88df7..ede59373461b4 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/LastValueAccumulator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/LastValueAccumulator.java @@ -22,12 +22,15 @@ import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.statistics.DateStatistics; import org.apache.tsfile.file.metadata.statistics.Statistics; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.BitMap; import org.apache.tsfile.utils.TsPrimitiveType; import org.apache.tsfile.write.UnSupportedDataTypeException; +import java.nio.charset.StandardCharsets; + import static com.google.common.base.Preconditions.checkArgument; public class LastValueAccumulator implements Accumulator { @@ -120,23 +123,41 @@ public void addStatistics(Statistics statistics) { switch (seriesDataType) { case INT32: case DATE: - updateIntLastValue((int) statistics.getLastValue(), statistics.getEndTime()); + updateIntLastValue( + ((Number) statistics.getLastValue()).intValue(), statistics.getEndTime()); break; case INT64: case TIMESTAMP: - updateLongLastValue((long) statistics.getLastValue(), statistics.getEndTime()); + updateLongLastValue( + ((Number) statistics.getLastValue()).longValue(), statistics.getEndTime()); break; case FLOAT: - updateFloatLastValue((float) statistics.getLastValue(), statistics.getEndTime()); + updateFloatLastValue( + ((Number) statistics.getLastValue()).floatValue(), statistics.getEndTime()); break; case DOUBLE: - updateDoubleLastValue((double) statistics.getLastValue(), statistics.getEndTime()); + updateDoubleLastValue( + ((Number) statistics.getLastValue()).doubleValue(), statistics.getEndTime()); break; case TEXT: case BLOB: case STRING: case OBJECT: - updateBinaryLastValue((Binary) statistics.getLastValue(), statistics.getEndTime()); + if (statistics instanceof DateStatistics) { + updateBinaryLastValue( + new Binary( + TSDataType.getDateStringValue((Integer) statistics.getLastValue()), + StandardCharsets.UTF_8), + statistics.getEndTime()); + } else { + if (statistics.getLastValue() instanceof Binary) { + updateBinaryLastValue((Binary) statistics.getLastValue(), statistics.getEndTime()); + } else { + updateBinaryLastValue( + new Binary(String.valueOf(statistics.getLastValue()), StandardCharsets.UTF_8), + statistics.getEndTime()); + } + } break; case BOOLEAN: updateBooleanLastValue((boolean) statistics.getLastValue(), statistics.getEndTime()); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/MaxValueAccumulator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/MaxValueAccumulator.java index d02678bd6225a..e507d533fa63e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/MaxValueAccumulator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/MaxValueAccumulator.java @@ -115,17 +115,17 @@ public void addStatistics(Statistics statistics) { switch (seriesDataType) { case INT32: case DATE: - updateIntResult((int) statistics.getMaxValue()); + updateIntResult(((Number) statistics.getMaxValue()).intValue()); break; case INT64: case TIMESTAMP: - updateLongResult((long) statistics.getMaxValue()); + updateLongResult(((Number) statistics.getMaxValue()).longValue()); break; case FLOAT: - updateFloatResult((float) statistics.getMaxValue()); + updateFloatResult(((Number) statistics.getMaxValue()).floatValue()); break; case DOUBLE: - updateDoubleResult((double) statistics.getMaxValue()); + updateDoubleResult(((Number) statistics.getMaxValue()).doubleValue()); break; case STRING: updateBinaryResult((Binary) statistics.getMaxValue()); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/MinValueAccumulator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/MinValueAccumulator.java index fd55ea0f1bc74..58f6da51401c1 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/MinValueAccumulator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/MinValueAccumulator.java @@ -115,17 +115,17 @@ public void addStatistics(Statistics statistics) { switch (seriesDataType) { case INT32: case DATE: - updateIntResult((int) statistics.getMinValue()); + updateIntResult(((Number) statistics.getMinValue()).intValue()); break; case INT64: case TIMESTAMP: - updateLongResult((long) statistics.getMinValue()); + updateLongResult(((Number) statistics.getMinValue()).longValue()); break; case FLOAT: - updateFloatResult((float) statistics.getMinValue()); + updateFloatResult(((Number) statistics.getMinValue()).floatValue()); break; case DOUBLE: - updateDoubleResult((double) statistics.getMinValue()); + updateDoubleResult(((Number) statistics.getMinValue()).doubleValue()); break; case STRING: updateBinaryResult((Binary) statistics.getMinValue()); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/exchange/SharedTsBlockQueue.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/exchange/SharedTsBlockQueue.java index 742d4ee58b2ac..4aea7cf22dd54 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/exchange/SharedTsBlockQueue.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/exchange/SharedTsBlockQueue.java @@ -24,6 +24,7 @@ import org.apache.iotdb.db.queryengine.execution.exchange.sink.LocalSinkChannel; import org.apache.iotdb.db.queryengine.execution.exchange.source.LocalSourceHandle; import org.apache.iotdb.db.queryengine.execution.memory.LocalMemoryManager; +import org.apache.iotdb.db.utils.CommonUtils; import org.apache.iotdb.mpp.rpc.thrift.TFragmentInstanceId; import com.google.common.util.concurrent.ListenableFuture; @@ -220,6 +221,9 @@ public TsBlock remove() { * the returned future of last invocation completes. */ public ListenableFuture add(TsBlock tsBlock) { + if (LOGGER.isDebugEnabled()) { + LOGGER.debug("[addTsBlock] TsBlock:{}", CommonUtils.toString(tsBlock)); + } if (closed) { // queue may have been closed return immediateVoidFuture(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/exchange/source/LocalSourceHandle.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/exchange/source/LocalSourceHandle.java index 4c724330c80dc..0dc6b27af04b6 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/exchange/source/LocalSourceHandle.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/exchange/source/LocalSourceHandle.java @@ -23,6 +23,7 @@ import org.apache.iotdb.db.queryengine.execution.exchange.MPPDataExchangeManager.SourceHandleListener; import org.apache.iotdb.db.queryengine.execution.exchange.SharedTsBlockQueue; import org.apache.iotdb.db.queryengine.metric.DataExchangeCostMetricSet; +import org.apache.iotdb.db.utils.CommonUtils; import org.apache.iotdb.db.utils.SetThreadName; import org.apache.iotdb.mpp.rpc.thrift.TFragmentInstanceId; import org.apache.iotdb.rpc.TSStatusCode; @@ -140,6 +141,10 @@ public TsBlock receive() { @Override public ByteBuffer getSerializedTsBlock() throws IoTDBException { TsBlock tsBlock = receive(); + if (LOGGER.isDebugEnabled()) { + LOGGER.debug("[GetSerializedTsBlock] TsBlock:{}", CommonUtils.toString(tsBlock)); + } + if (tsBlock != null) { long startTime = System.nanoTime(); try { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/executor/RegionWriteExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/executor/RegionWriteExecutor.java index 448474e1f5d1c..da394f62a07f5 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/executor/RegionWriteExecutor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/executor/RegionWriteExecutor.java @@ -29,6 +29,7 @@ import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.schema.template.Template; import org.apache.iotdb.commons.service.metric.PerformanceOverviewMetrics; +import org.apache.iotdb.commons.utils.MetadataUtils; import org.apache.iotdb.commons.utils.TestOnly; import org.apache.iotdb.consensus.ConsensusFactory; import org.apache.iotdb.consensus.IConsensus; @@ -901,6 +902,16 @@ private RegionExecutionResult executeAlterTimeSeries( throw new MetadataException( String.format("%s is not view.", measurementPath.getFullPath())); } + if (node.getDataType() != null + && !MetadataUtils.canAlter( + measurementPath.getMeasurementSchema().getType(), node.getDataType())) { + throw new MetadataException( + String.format( + "The timeseries %s used new type %s is not compatible with the existing one %s.", + measurementPath.getFullPath(), + node.getDataType(), + measurementPath.getMeasurementSchema().getType())); + } return receivedFromPipe ? super.visitPipeEnrichedWritePlanNode(new PipeEnrichedWritePlanNode(node), context) : super.visitAlterTimeSeries(node, context); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/TopKOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/TopKOperator.java index bf021e5b4f409..0df7efdd19f0b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/TopKOperator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/TopKOperator.java @@ -280,10 +280,20 @@ private void initResultTsBlock() { new boolean[positionCount]); break; case INT32: + columns[i] = + new IntColumn( + positionCount, + Optional.of(new boolean[positionCount]), + new int[positionCount], + TSDataType.INT32); + break; case DATE: columns[i] = new IntColumn( - positionCount, Optional.of(new boolean[positionCount]), new int[positionCount]); + positionCount, + Optional.of(new boolean[positionCount]), + new int[positionCount], + TSDataType.DATE); break; case INT64: case TIMESTAMP: diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java index 35d8a1705ab00..a92796dacfb78 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java @@ -36,7 +36,10 @@ import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; import org.apache.iotdb.db.storageengine.dataregion.tsfile.timeindex.ITimeIndex; import org.apache.iotdb.db.utils.ModificationUtils; +import org.apache.iotdb.db.utils.SchemaUtils; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.AbstractAlignedChunkMetadata; import org.apache.tsfile.file.metadata.AbstractAlignedTimeSeriesMetadata; import org.apache.tsfile.file.metadata.AlignedTimeSeriesMetadata; import org.apache.tsfile.file.metadata.IChunkMetadata; @@ -48,12 +51,14 @@ import org.apache.tsfile.read.filter.basic.Filter; import org.apache.tsfile.read.reader.IChunkReader; import org.apache.tsfile.read.reader.IPageReader; +import org.apache.tsfile.write.schema.IMeasurementSchema; import java.io.IOException; import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Set; +import java.util.stream.Collectors; import static com.google.common.base.Preconditions.checkArgument; @@ -107,6 +112,7 @@ public static TimeseriesMetadata loadTimeSeriesMetadata( context.isDebug(), context); if (timeSeriesMetadata != null) { + SchemaUtils.changeMetadataModified(timeSeriesMetadata, seriesPath.getSeriesType()); long t2 = System.nanoTime(); List pathModifications = context.getPathModifications( @@ -133,6 +139,7 @@ public static TimeseriesMetadata loadTimeSeriesMetadata( timeSeriesMetadata = (TimeseriesMetadata) resource.getTimeSeriesMetadata(seriesPath, globalTimeFilter); if (timeSeriesMetadata != null) { + SchemaUtils.changeMetadataModified(timeSeriesMetadata, seriesPath.getSeriesType()); timeSeriesMetadata.setChunkMetadataLoader( new MemChunkMetadataLoader(resource, seriesPath, context, globalTimeFilter)); } @@ -147,6 +154,7 @@ public static TimeseriesMetadata loadTimeSeriesMetadata( return null; } } + return timeSeriesMetadata; } finally { long costTime = System.nanoTime() - t1; @@ -190,17 +198,24 @@ public static AbstractAlignedTimeSeriesMetadata loadAlignedTimeSeriesMetadata( boolean loadFromMem = false; try { AbstractAlignedTimeSeriesMetadata alignedTimeSeriesMetadata; + List targetDataTypeList = + alignedPath.getSchemaList().stream() + .map(IMeasurementSchema::getType) + .collect(Collectors.toList()); // If the tsfile is closed, we need to load from tsfile if (resource.isClosed()) { alignedTimeSeriesMetadata = loadAlignedTimeSeriesMetadataFromDisk( resource, alignedPath, context, globalTimeFilter, ignoreAllNullRows); + SchemaUtils.changeAlignedMetadataModified(alignedTimeSeriesMetadata, targetDataTypeList); } else { // if the tsfile is unclosed, we just get it directly from TsFileResource loadFromMem = true; alignedTimeSeriesMetadata = (AbstractAlignedTimeSeriesMetadata) resource.getTimeSeriesMetadata(alignedPath, globalTimeFilter); if (alignedTimeSeriesMetadata != null) { + SchemaUtils.changeAlignedMetadataModified(alignedTimeSeriesMetadata, targetDataTypeList); + alignedTimeSeriesMetadata.setChunkMetadataLoader( new MemAlignedChunkMetadataLoader( resource, alignedPath, context, globalTimeFilter, ignoreAllNullRows)); @@ -303,6 +318,7 @@ private static AbstractAlignedTimeSeriesMetadata loadAlignedTimeSeriesMetadataFr new ArrayList<>(valueMeasurementList.size()); // if all the queried aligned sensors does not exist, we will return null boolean exist = false; + int i = 0; for (String valueMeasurement : valueMeasurementList) { TimeseriesMetadata valueColumn = cache.get( @@ -315,7 +331,12 @@ private static AbstractAlignedTimeSeriesMetadata loadAlignedTimeSeriesMetadataFr isDebug, context); exist = (exist || (valueColumn != null)); + if (valueColumn != null) { + SchemaUtils.changeAlignedMetadataModified( + valueColumn, alignedPath.getSchemaList().get(i).getType()); + } valueTimeSeriesMetadataList.add(valueColumn); + i++; } if (!ignoreAllNullRows || exist) { alignedTimeSeriesMetadata = @@ -435,12 +456,49 @@ public static List loadChunkMetadataList(ITimeSeriesMetadata tim * IOException will be thrown */ public static List loadPageReaderList( - IChunkMetadata chunkMetaData, Filter globalTimeFilter) throws IOException { + IChunkMetadata chunkMetaData, + Filter globalTimeFilter, + boolean isAligned, + List targetDataTypeList) + throws IOException { checkArgument(chunkMetaData != null, "Can't init null chunkMeta"); - IChunkLoader chunkLoader = chunkMetaData.getChunkLoader(); - IChunkReader chunkReader = chunkLoader.getChunkReader(chunkMetaData, globalTimeFilter); - return chunkReader.loadPageReaderList(); + IChunkReader chunkReader; + boolean isModified = false; + if (isAligned) { + AbstractAlignedChunkMetadata alignedChunkMetadata = + (AbstractAlignedChunkMetadata) chunkMetaData; + for (int i = 0; i < alignedChunkMetadata.getValueChunkMetadataList().size(); i++) { + if (alignedChunkMetadata.getValueChunkMetadataList().get(i) != null) { + if (!SchemaUtils.isUsingSameColumn( + alignedChunkMetadata.getValueChunkMetadataList().get(i).getDataType(), + targetDataTypeList.get(i)) + && targetDataTypeList.get(i).equals(TSDataType.STRING)) { + isModified = true; + alignedChunkMetadata.getValueChunkMetadataList().get(i).setModified(true); + } + } + } + IChunkLoader chunkLoader = alignedChunkMetadata.getChunkLoader(); + chunkReader = chunkLoader.getChunkReader(alignedChunkMetadata, globalTimeFilter); + } else { + if (!SchemaUtils.isUsingSameColumn(chunkMetaData.getDataType(), targetDataTypeList.get(0)) + && targetDataTypeList.get(0).equals(TSDataType.STRING)) { + isModified = true; + chunkMetaData.setModified(true); + } + IChunkLoader chunkLoader = chunkMetaData.getChunkLoader(); + chunkReader = chunkLoader.getChunkReader(chunkMetaData, globalTimeFilter); + } + + return isModified + ? chunkReader.loadPageReaderList().stream() + .peek( + iPageReader -> { + iPageReader.setModified(true); + }) + .collect(Collectors.toList()) + : chunkReader.loadPageReaderList(); } /** diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/SeriesScanUtil.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/SeriesScanUtil.java index 741961961045e..61b272c1b322f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/SeriesScanUtil.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/SeriesScanUtil.java @@ -38,21 +38,34 @@ import org.apache.iotdb.db.storageengine.dataregion.read.reader.common.NoDataPointReader; import org.apache.iotdb.db.storageengine.dataregion.read.reader.common.PriorityMergeReader; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; +import org.apache.iotdb.db.utils.CommonUtils; +import org.apache.iotdb.db.utils.SchemaUtils; import org.apache.iotdb.db.utils.datastructure.MemPointIterator; +import org.apache.tsfile.block.column.Column; import org.apache.tsfile.common.conf.TSFileDescriptor; import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.AbstractAlignedChunkMetadata; +import org.apache.tsfile.file.metadata.AbstractAlignedTimeSeriesMetadata; +import org.apache.tsfile.file.metadata.ChunkMetadata; import org.apache.tsfile.file.metadata.IChunkMetadata; import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.file.metadata.IMetadata; import org.apache.tsfile.file.metadata.ITimeSeriesMetadata; import org.apache.tsfile.file.metadata.StringArrayDeviceID; +import org.apache.tsfile.file.metadata.TimeseriesMetadata; import org.apache.tsfile.file.metadata.statistics.Statistics; import org.apache.tsfile.read.TimeValuePair; import org.apache.tsfile.read.common.TimeRange; import org.apache.tsfile.read.common.block.TsBlock; import org.apache.tsfile.read.common.block.TsBlockBuilder; import org.apache.tsfile.read.common.block.TsBlockUtil; +import org.apache.tsfile.read.common.block.column.BinaryColumn; +import org.apache.tsfile.read.common.block.column.BooleanColumn; +import org.apache.tsfile.read.common.block.column.DoubleColumn; +import org.apache.tsfile.read.common.block.column.FloatColumn; +import org.apache.tsfile.read.common.block.column.IntColumn; +import org.apache.tsfile.read.common.block.column.LongColumn; import org.apache.tsfile.read.controller.IChunkLoader; import org.apache.tsfile.read.filter.basic.Filter; import org.apache.tsfile.read.reader.IPageReader; @@ -61,12 +74,17 @@ import org.apache.tsfile.read.reader.page.TablePageReader; import org.apache.tsfile.read.reader.series.PaginationController; import org.apache.tsfile.utils.Accountable; +import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.RamUsageEstimator; import org.apache.tsfile.utils.TsPrimitiveType; import org.apache.tsfile.write.UnSupportedDataTypeException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.io.IOException; import java.io.Serializable; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; import java.util.Collections; import java.util.Comparator; import java.util.LinkedList; @@ -82,6 +100,7 @@ public class SeriesScanUtil implements Accountable { + private static final Logger LOGGER = LoggerFactory.getLogger(SeriesScanUtil.class); public static final StringArrayDeviceID EMPTY_DEVICE_ID = new StringArrayDeviceID(""); protected final FragmentInstanceContext context; @@ -312,6 +331,7 @@ public boolean canUseCurrentFileStatistics() { checkState(firstTimeSeriesMetadata != null, "no first file"); if (currentFileOverlapped() || firstTimeSeriesMetadata.isModified()) { + // || !firstTimeSeriesMetadata.isModified()) { return false; } return filterAllSatisfy(scanOptions.getGlobalTimeFilter(), firstTimeSeriesMetadata) @@ -395,6 +415,7 @@ private void filterFirstChunkMetadata() { } if (currentChunkOverlapped() || firstChunkMetadata.isModified()) { + // || !firstChunkMetadata.isModified()) { return; } @@ -437,6 +458,15 @@ private void initFirstChunkMetadata() throws IOException { orderUtils.getOverlapCheckTime(firstChunkMetadata.getStatistics())); unpackAllOverlappedTimeSeriesMetadataToCachedChunkMetadata( orderUtils.getOverlapCheckTime(firstChunkMetadata.getStatistics()), false); + if (isAligned) { + SchemaUtils.changeAlignedMetadataModified( + (AbstractAlignedChunkMetadata) firstChunkMetadata, + firstChunkMetadata.getDataType(), + getTsDataTypeList()); + } else { + SchemaUtils.changeMetadataModified( + firstChunkMetadata, firstChunkMetadata.getDataType(), dataType); + } if (firstChunkMetadata.equals(cachedChunkMetadata.peek())) { firstChunkMetadata = cachedChunkMetadata.poll(); break; @@ -464,13 +494,45 @@ private void unpackAllOverlappedTimeSeriesMetadataToCachedChunkMetadata( if (init && firstChunkMetadata == null && !cachedChunkMetadata.isEmpty()) { firstChunkMetadata = cachedChunkMetadata.poll(); + if (isAligned) { + SchemaUtils.changeAlignedMetadataModified( + (AbstractAlignedChunkMetadata) firstChunkMetadata, + firstChunkMetadata.getDataType(), + getTsDataTypeList()); + } else { + SchemaUtils.changeMetadataModified( + firstChunkMetadata, firstChunkMetadata.getDataType(), dataType); + } } } protected void unpackOneTimeSeriesMetadata(ITimeSeriesMetadata timeSeriesMetadata) { List chunkMetadataList = FileLoaderUtils.loadChunkMetadataList(timeSeriesMetadata); - chunkMetadataList.forEach(chunkMetadata -> chunkMetadata.setSeq(timeSeriesMetadata.isSeq())); + chunkMetadataList.forEach( + chunkMetadata -> { + if (chunkMetadata instanceof AbstractAlignedChunkMetadata) { + AbstractAlignedChunkMetadata alignedChunkMetadata = + (AbstractAlignedChunkMetadata) chunkMetadata; + for (int i = 0; i < alignedChunkMetadata.getValueChunkMetadataList().size(); i++) { + if ((alignedChunkMetadata.getValueChunkMetadataList().get(i) != null) + && !SchemaUtils.isUsingSameColumn( + alignedChunkMetadata.getValueChunkMetadataList().get(i).getDataType(), + getTsDataTypeList().get(i)) + && getTsDataTypeList().get(i).equals(TSDataType.STRING)) { + alignedChunkMetadata.getValueChunkMetadataList().get(i).setModified(true); + } + } + chunkMetadata = alignedChunkMetadata; + } else if (chunkMetadata instanceof ChunkMetadata) { + if (!SchemaUtils.isUsingSameColumn( + chunkMetadata.getDataType(), getTsDataTypeList().get(0)) + && getTsDataTypeList().get(0).equals(TSDataType.STRING)) { + chunkMetadata.setModified(true); + } + } + chunkMetadata.setSeq(timeSeriesMetadata.isSeq()); + }); cachedChunkMetadata.addAll(chunkMetadataList); } @@ -485,6 +547,7 @@ public boolean canUseCurrentChunkStatistics() { checkState(firstChunkMetadata != null, "no first chunk"); if (currentChunkOverlapped() || firstChunkMetadata.isModified()) { + // || !firstChunkMetadata.isModified()) { return false; } return filterAllSatisfy(scanOptions.getGlobalTimeFilter(), firstChunkMetadata) @@ -646,7 +709,8 @@ private void unpackOneChunkMetaData(IChunkMetadata chunkMetaData) throws IOExcep return; } List pageReaderList = - FileLoaderUtils.loadPageReaderList(chunkMetaData, scanOptions.getGlobalTimeFilter()); + FileLoaderUtils.loadPageReaderList( + chunkMetaData, scanOptions.getGlobalTimeFilter(), isAligned, getTsDataTypeList()); // init TsBlockBuilder for each page reader pageReaderList.forEach(p -> p.initTsBlockBuilder(getTsDataTypeList())); @@ -687,6 +751,12 @@ private void unpackOneChunkMetaData(IChunkMetadata chunkMetaData) throws IOExcep pageReader, false))); } + + if (LOGGER.isDebugEnabled()) { + for (IPageReader pageReader : pageReaderList) { + LOGGER.debug("[SeriesScanUtil] pageReader.isModified() is {}", pageReader.isModified()); + } + } } private void unpackOneFakeMemChunkMetaData( @@ -763,6 +833,8 @@ public boolean canUseCurrentPageStatistics() throws IOException { return false; } if (currentPageOverlapped() || firstPageReader.isModified()) { + // if (currentPageOverlapped() || firstPageReader.isModified() || + // !firstPageReader.isModified()) { return false; } return filterAllSatisfy(scanOptions.getGlobalTimeFilter(), firstPageReader.getPageReader()) @@ -824,7 +896,7 @@ public TsBlock nextPage() throws IOException { if (!lazyMemVersionPageReader.hasNextBatch()) { firstPageReader = null; } - return tsBlock; + return tsBlock == null ? null : getTransferedDataTypeTsBlock(tsBlock); } TsBlock tsBlock; @@ -839,8 +911,473 @@ public TsBlock nextPage() throws IOException { firstPageReader = null; + return getTransferedDataTypeTsBlock(tsBlock); + } + } + + private TsBlock getTransferedDataTypeTsBlock(TsBlock tsBlock) { + Column[] valueColumns = tsBlock.getValueColumns(); + int length = tsBlock.getValueColumnCount(); + boolean isTypeInconsistent = false; + if (length > 0) { + for (int i = 0; i < length; i++) { + TSDataType finalDataType = getTsDataTypeList().get(i); + if ((valueColumns[i].getDataType() != finalDataType) + && (SchemaUtils.isUsingSameColumn(valueColumns[i].getDataType(), finalDataType) + || (valueColumns[i].getDataType().equals(TSDataType.DATE) + && Arrays.asList(TSDataType.STRING, TSDataType.TEXT) + .contains(finalDataType)))) { + isTypeInconsistent = true; + break; + } + } + } + + if (!isTypeInconsistent) { return tsBlock; } + + int positionCount = tsBlock.getPositionCount(); + Column[] newValueColumns = new Column[length]; + for (int i = 0; i < length; i++) { + TSDataType sourceType = valueColumns[i].getDataType(); + TSDataType finalDataType = getTsDataTypeList().get(i); + switch (finalDataType) { + case BOOLEAN: + if (sourceType == TSDataType.BOOLEAN) { + newValueColumns[i] = valueColumns[i]; + } else { + newValueColumns[i] = + new BooleanColumn( + positionCount, + Optional.of(new boolean[positionCount]), + new boolean[positionCount]); + for (int j = 0; j < valueColumns[i].getPositionCount(); j++) { + newValueColumns[i].isNull()[j] = true; + } + } + break; + case INT32: + if (sourceType == TSDataType.INT32) { + newValueColumns[i] = valueColumns[i]; + } else { + newValueColumns[i] = + new IntColumn( + positionCount, + Optional.of(new boolean[positionCount]), + new int[positionCount], + TSDataType.INT32); + for (int j = 0; j < valueColumns[i].getPositionCount(); j++) { + newValueColumns[i].isNull()[j] = true; + } + } + break; + case INT64: + if (sourceType == TSDataType.INT64) { + newValueColumns[i] = valueColumns[i]; + } else if (sourceType == TSDataType.INT32) { + newValueColumns[i] = + new LongColumn( + positionCount, + Optional.of(new boolean[positionCount]), + new long[positionCount]); + + for (int j = 0; j < valueColumns[i].getInts().length; j++) { + newValueColumns[i].isNull()[j] = valueColumns[i].isNull()[j]; + if (!valueColumns[i].isNull()[j]) { + newValueColumns[i].getLongs()[j] = + ((Number) valueColumns[i].getInts()[j]).longValue(); + } + } + } else if (sourceType == TSDataType.TIMESTAMP) { + newValueColumns[i] = valueColumns[i]; + } else { + newValueColumns[i] = + new LongColumn( + positionCount, + Optional.of(new boolean[positionCount]), + new long[positionCount]); + for (int j = 0; j < valueColumns[i].getPositionCount(); j++) { + newValueColumns[i].isNull()[j] = true; + } + } + break; + case FLOAT: + if (sourceType == TSDataType.FLOAT) { + newValueColumns[i] = valueColumns[i]; + } else if (sourceType == TSDataType.INT32) { + newValueColumns[i] = + new FloatColumn( + positionCount, + Optional.of(new boolean[positionCount]), + new float[positionCount]); + + for (int j = 0; j < valueColumns[i].getInts().length; j++) { + newValueColumns[i].isNull()[j] = valueColumns[i].isNull()[j]; + if (!valueColumns[i].isNull()[j]) { + newValueColumns[i].getFloats()[j] = + ((Number) valueColumns[i].getInts()[j]).floatValue(); + } + } + } else { + newValueColumns[i] = + new FloatColumn( + positionCount, + Optional.of(new boolean[positionCount]), + new float[positionCount]); + for (int j = 0; j < valueColumns[i].getPositionCount(); j++) { + newValueColumns[i].isNull()[j] = true; + } + } + break; + case DOUBLE: + if (sourceType == TSDataType.DOUBLE) { + newValueColumns[i] = valueColumns[i]; + } else if (sourceType == TSDataType.INT32) { + newValueColumns[i] = + new DoubleColumn( + positionCount, + Optional.of(new boolean[positionCount]), + new double[positionCount]); + + for (int j = 0; j < valueColumns[i].getInts().length; j++) { + newValueColumns[i].isNull()[j] = valueColumns[i].isNull()[j]; + if (!valueColumns[i].isNull()[j]) { + newValueColumns[i].getDoubles()[j] = + ((Number) valueColumns[i].getInts()[j]).doubleValue(); + } + } + } else if (sourceType == TSDataType.INT64) { + newValueColumns[i] = + new DoubleColumn( + positionCount, + Optional.of(new boolean[positionCount]), + new double[positionCount]); + + for (int j = 0; j < valueColumns[i].getLongs().length; j++) { + newValueColumns[i].isNull()[j] = valueColumns[i].isNull()[j]; + if (!valueColumns[i].isNull()[j]) { + newValueColumns[i].getDoubles()[j] = + ((Number) valueColumns[i].getLongs()[j]).doubleValue(); + } + } + } else if (sourceType == TSDataType.FLOAT) { + newValueColumns[i] = + new DoubleColumn( + positionCount, + Optional.of(new boolean[positionCount]), + new double[positionCount]); + + for (int j = 0; j < valueColumns[i].getFloats().length; j++) { + newValueColumns[i].isNull()[j] = valueColumns[i].isNull()[j]; + if (!valueColumns[i].isNull()[j]) { + newValueColumns[i].getDoubles()[j] = + ((Number) valueColumns[i].getFloats()[j]).doubleValue(); + } + } + } else if (sourceType == TSDataType.TIMESTAMP) { + newValueColumns[i] = + new DoubleColumn( + positionCount, + Optional.of(new boolean[positionCount]), + new double[positionCount]); + + for (int j = 0; j < valueColumns[i].getLongs().length; j++) { + newValueColumns[i].isNull()[j] = valueColumns[i].isNull()[j]; + if (!valueColumns[i].isNull()[j]) { + newValueColumns[i].getDoubles()[j] = + ((Number) valueColumns[i].getLongs()[j]).doubleValue(); + } + } + } else { + newValueColumns[i] = + new DoubleColumn( + positionCount, + Optional.of(new boolean[positionCount]), + new double[positionCount]); + for (int j = 0; j < valueColumns[i].getPositionCount(); j++) { + newValueColumns[i].isNull()[j] = true; + } + } + break; + case TEXT: + if (SchemaUtils.isUsingSameColumn(sourceType, TSDataType.TEXT)) { + newValueColumns[i] = valueColumns[i]; + } else if (sourceType == TSDataType.INT32) { + newValueColumns[i] = + new BinaryColumn( + positionCount, + Optional.of(new boolean[positionCount]), + new Binary[positionCount]); + + for (int j = 0; j < valueColumns[i].getInts().length; j++) { + newValueColumns[i].isNull()[j] = valueColumns[i].isNull()[j]; + if (!valueColumns[i].isNull()[j]) { + newValueColumns[i].getBinaries()[j] = + new Binary( + String.valueOf(valueColumns[i].getInts()[j]), StandardCharsets.UTF_8); + } + } + } else if (sourceType == TSDataType.DATE) { + newValueColumns[i] = + new BinaryColumn( + positionCount, + Optional.of(new boolean[positionCount]), + new Binary[positionCount]); + + for (int j = 0; j < valueColumns[i].getInts().length; j++) { + newValueColumns[i].isNull()[j] = valueColumns[i].isNull()[j]; + if (!valueColumns[i].isNull()[j]) { + newValueColumns[i].getBinaries()[j] = + new Binary( + TSDataType.getDateStringValue(valueColumns[i].getInts()[j]), + StandardCharsets.UTF_8); + } + } + } else if (sourceType == TSDataType.INT64 || sourceType == TSDataType.TIMESTAMP) { + newValueColumns[i] = + new BinaryColumn( + positionCount, + Optional.of(new boolean[positionCount]), + new Binary[positionCount]); + + for (int j = 0; j < valueColumns[i].getInts().length; j++) { + newValueColumns[i].isNull()[j] = valueColumns[i].isNull()[j]; + if (!valueColumns[i].isNull()[j]) { + newValueColumns[i].getBinaries()[j] = + new Binary( + String.valueOf(valueColumns[i].getLongs()[j]), StandardCharsets.UTF_8); + } + } + } else if (sourceType == TSDataType.FLOAT) { + newValueColumns[i] = + new BinaryColumn( + positionCount, + Optional.of(new boolean[positionCount]), + new Binary[positionCount]); + + for (int j = 0; j < valueColumns[i].getInts().length; j++) { + newValueColumns[i].isNull()[j] = valueColumns[i].isNull()[j]; + if (!valueColumns[i].isNull()[j]) { + newValueColumns[i].getBinaries()[j] = + new Binary( + String.valueOf(valueColumns[i].getFloats()[j]), StandardCharsets.UTF_8); + } + } + } else if (sourceType == TSDataType.DOUBLE) { + newValueColumns[i] = + new BinaryColumn( + positionCount, + Optional.of(new boolean[positionCount]), + new Binary[positionCount]); + + for (int j = 0; j < valueColumns[i].getInts().length; j++) { + newValueColumns[i].isNull()[j] = valueColumns[i].isNull()[j]; + if (!valueColumns[i].isNull()[j]) { + newValueColumns[i].getBinaries()[j] = + new Binary( + String.valueOf(valueColumns[i].getDoubles()[j]), StandardCharsets.UTF_8); + } + } + } else if (sourceType == TSDataType.BOOLEAN) { + newValueColumns[i] = + new BinaryColumn( + positionCount, + Optional.of(new boolean[positionCount]), + new Binary[positionCount]); + + for (int j = 0; j < valueColumns[i].getInts().length; j++) { + newValueColumns[i].isNull()[j] = valueColumns[i].isNull()[j]; + if (!valueColumns[i].isNull()[j]) { + newValueColumns[i].getBinaries()[j] = + new Binary( + String.valueOf(valueColumns[i].getBooleans()[j]), StandardCharsets.UTF_8); + } + } + } else { + newValueColumns[i] = + new BinaryColumn( + positionCount, + Optional.of(new boolean[positionCount]), + new Binary[positionCount]); + for (int j = 0; j < valueColumns[i].getPositionCount(); j++) { + newValueColumns[i].isNull()[j] = true; + } + } + break; + case TIMESTAMP: + if (SchemaUtils.isUsingSameColumn(sourceType, TSDataType.TIMESTAMP)) { + newValueColumns[i] = valueColumns[i]; + } else if (sourceType == TSDataType.INT32) { + newValueColumns[i] = + new LongColumn( + positionCount, + Optional.of(new boolean[positionCount]), + new long[positionCount]); + + for (int j = 0; j < valueColumns[i].getInts().length; j++) { + newValueColumns[i].isNull()[j] = valueColumns[i].isNull()[j]; + if (!valueColumns[i].isNull()[j]) { + newValueColumns[i].getLongs()[j] = + ((Number) valueColumns[i].getInts()[j]).longValue(); + } + } + } else if (sourceType == TSDataType.INT64) { + newValueColumns[i] = valueColumns[i]; + } else { + newValueColumns[i] = + new LongColumn( + positionCount, + Optional.of(new boolean[positionCount]), + new long[positionCount]); + for (int j = 0; j < valueColumns[i].getPositionCount(); j++) { + newValueColumns[i].isNull()[j] = true; + } + } + break; + case DATE: + if (SchemaUtils.isUsingSameColumn(sourceType, TSDataType.DATE)) { + newValueColumns[i] = valueColumns[i]; + } else { + newValueColumns[i] = + new IntColumn( + positionCount, + Optional.of(new boolean[positionCount]), + new int[positionCount], + TSDataType.DATE); + for (int j = 0; j < valueColumns[i].getPositionCount(); j++) { + newValueColumns[i].isNull()[j] = true; + } + } + break; + case BLOB: + if (SchemaUtils.isUsingSameColumn(sourceType, TSDataType.BLOB)) { + newValueColumns[i] = valueColumns[i]; + } else { + newValueColumns[i] = + new BinaryColumn( + positionCount, + Optional.of(new boolean[positionCount]), + new Binary[positionCount]); + for (int j = 0; j < valueColumns[i].getPositionCount(); j++) { + newValueColumns[i].isNull()[j] = true; + } + } + break; + case STRING: + if (SchemaUtils.isUsingSameColumn(sourceType, TSDataType.STRING)) { + newValueColumns[i] = valueColumns[i]; + } else if (sourceType == TSDataType.INT32) { + newValueColumns[i] = + new BinaryColumn( + positionCount, + Optional.of(new boolean[positionCount]), + new Binary[positionCount]); + + for (int j = 0; j < valueColumns[i].getInts().length; j++) { + newValueColumns[i].isNull()[j] = valueColumns[i].isNull()[j]; + if (!valueColumns[i].isNull()[j]) { + newValueColumns[i].getBinaries()[j] = + new Binary( + String.valueOf(valueColumns[i].getInts()[j]), StandardCharsets.UTF_8); + } + } + } else if (sourceType == TSDataType.DATE) { + newValueColumns[i] = + new BinaryColumn( + positionCount, + Optional.of(new boolean[positionCount]), + new Binary[positionCount]); + + for (int j = 0; j < valueColumns[i].getInts().length; j++) { + newValueColumns[i].isNull()[j] = valueColumns[i].isNull()[j]; + if (!valueColumns[i].isNull()[j]) { + newValueColumns[i].getBinaries()[j] = + new Binary( + TSDataType.getDateStringValue(valueColumns[i].getInts()[j]), + StandardCharsets.UTF_8); + } + } + } else if (sourceType == TSDataType.INT64 || sourceType == TSDataType.TIMESTAMP) { + newValueColumns[i] = + new BinaryColumn( + positionCount, + Optional.of(new boolean[positionCount]), + new Binary[positionCount]); + + for (int j = 0; j < valueColumns[i].getInts().length; j++) { + newValueColumns[i].isNull()[j] = valueColumns[i].isNull()[j]; + if (!valueColumns[i].isNull()[j]) { + newValueColumns[i].getBinaries()[j] = + new Binary( + String.valueOf(valueColumns[i].getLongs()[j]), StandardCharsets.UTF_8); + } + } + } else if (sourceType == TSDataType.FLOAT) { + newValueColumns[i] = + new BinaryColumn( + positionCount, + Optional.of(new boolean[positionCount]), + new Binary[positionCount]); + + for (int j = 0; j < valueColumns[i].getInts().length; j++) { + newValueColumns[i].isNull()[j] = valueColumns[i].isNull()[j]; + if (!valueColumns[i].isNull()[j]) { + newValueColumns[i].getBinaries()[j] = + new Binary( + String.valueOf(valueColumns[i].getFloats()[j]), StandardCharsets.UTF_8); + } + } + } else if (sourceType == TSDataType.DOUBLE) { + newValueColumns[i] = + new BinaryColumn( + positionCount, + Optional.of(new boolean[positionCount]), + new Binary[positionCount]); + + for (int j = 0; j < valueColumns[i].getInts().length; j++) { + newValueColumns[i].isNull()[j] = valueColumns[i].isNull()[j]; + if (!valueColumns[i].isNull()[j]) { + newValueColumns[i].getBinaries()[j] = + new Binary( + String.valueOf(valueColumns[i].getDoubles()[j]), StandardCharsets.UTF_8); + } + } + } else if (sourceType == TSDataType.BOOLEAN) { + newValueColumns[i] = + new BinaryColumn( + positionCount, + Optional.of(new boolean[positionCount]), + new Binary[positionCount]); + + for (int j = 0; j < valueColumns[i].getInts().length; j++) { + newValueColumns[i].isNull()[j] = valueColumns[i].isNull()[j]; + if (!valueColumns[i].isNull()[j]) { + newValueColumns[i].getBinaries()[j] = + new Binary( + String.valueOf(valueColumns[i].getBooleans()[j]), StandardCharsets.UTF_8); + } + } + } else { + newValueColumns[i] = + new BinaryColumn( + positionCount, + Optional.of(new boolean[positionCount]), + new Binary[positionCount]); + for (int j = 0; j < valueColumns[i].getPositionCount(); j++) { + newValueColumns[i].isNull()[j] = true; + } + } + break; + case VECTOR: + case UNKNOWN: + default: + break; + } + } + + tsBlock = new TsBlock(tsBlock.getTimeColumn(), newValueColumns); + return tsBlock; } private TsBlock applyPushDownFilterAndLimitOffset( @@ -913,7 +1450,6 @@ private boolean hasNextOverlappedPage() throws IOException { ? Math.max(mergeReader.getCurrentReadStopTime(), initialEndPointTime) : Math.min(mergeReader.getCurrentReadStopTime(), initialEndPointTime); while (mergeReader.hasNextTimeValuePair()) { - /* * get current first point in mergeReader, this maybe overlapped later */ @@ -1121,7 +1657,16 @@ private void addTimeValuePairToResult(TimeValuePair timeValuePair, TsBlockBuilde case BLOB: case OBJECT: case STRING: - builder.getColumnBuilder(0).writeBinary(timeValuePair.getValue().getBinary()); + if (timeValuePair.getValue().getDataType() == TSDataType.DATE) { + builder + .getColumnBuilder(0) + .writeBinary( + new Binary( + TSDataType.getDateStringValue(timeValuePair.getValue().getInt()), + StandardCharsets.UTF_8)); + } else { + builder.getColumnBuilder(0).writeBinary(timeValuePair.getValue().getBinary()); + } break; case VECTOR: TsPrimitiveType[] values = timeValuePair.getValue().getVector(); @@ -1153,6 +1698,11 @@ private void initFirstPageReader() throws IOException { // this page after unpacking must be the first page if (firstPageReader.equals(getFirstPageReaderFromCachedReaders())) { this.firstPageReader = firstPageReader; + if (isAligned) { + + } else { + + } if (!seqPageReaders.isEmpty() && firstPageReader.equals(seqPageReaders.get(0))) { seqPageReaders.remove(0); break; @@ -1239,7 +1789,7 @@ private void putPageReaderToMergeReader(IVersionPageReader pageReader) throws IO private TsBlock nextOverlappedPage() throws IOException { if (hasCachedNextOverlappedPage || hasNextOverlappedPage()) { hasCachedNextOverlappedPage = false; - return cachedTsBlock; + return getTransferedDataTypeTsBlock(cachedTsBlock); } throw new IOException("No more batch data"); } @@ -1370,7 +1920,7 @@ private Optional unpackSeqTsFileResource() throws IOExcepti ITimeSeriesMetadata timeseriesMetadata = loadTimeSeriesMetadata(orderUtils.getNextSeqFileResource(true), true); // skip if data type is mismatched which may be caused by delete - if (timeseriesMetadata != null && timeseriesMetadata.typeMatch(getTsDataTypeList())) { + if (timeseriesMetadata != null && typeCompatible(timeseriesMetadata)) { timeseriesMetadata.setSeq(true); seqTimeSeriesMetadata.add(timeseriesMetadata); return Optional.of(timeseriesMetadata); @@ -1379,11 +1929,40 @@ private Optional unpackSeqTsFileResource() throws IOExcepti } } + private boolean typeCompatible(ITimeSeriesMetadata timeseriesMetadata) { + if (timeseriesMetadata instanceof TimeseriesMetadata) { + return getTsDataTypeList() + .get(0) + .isCompatible(((TimeseriesMetadata) timeseriesMetadata).getTsDataType()); + } else { + List valueTimeseriesMetadataList = + ((AbstractAlignedTimeSeriesMetadata) timeseriesMetadata).getValueTimeseriesMetadataList(); + if (getTsDataTypeList().isEmpty()) { + return true; + } + if (valueTimeseriesMetadataList != null) { + int incompactibleCount = 0; + for (int i = 0, size = getTsDataTypeList().size(); i < size; i++) { + TimeseriesMetadata valueTimeSeriesMetadata = valueTimeseriesMetadataList.get(i); + if (valueTimeSeriesMetadata != null + && !getTsDataTypeList() + .get(i) + .isCompatible(valueTimeSeriesMetadata.getTsDataType())) { + valueTimeseriesMetadataList.set(i, null); + incompactibleCount++; + } + } + return incompactibleCount != getTsDataTypeList().size(); + } + return true; + } + } + private Optional unpackUnseqTsFileResource() throws IOException { ITimeSeriesMetadata timeseriesMetadata = loadTimeSeriesMetadata(orderUtils.getNextUnseqFileResource(true), false); // skip if data type is mismatched which may be caused by delete - if (timeseriesMetadata != null && timeseriesMetadata.typeMatch(getTsDataTypeList())) { + if (timeseriesMetadata != null && typeCompatible(timeseriesMetadata)) { timeseriesMetadata.setSeq(false); unSeqTimeSeriesMetadata.add(timeseriesMetadata); return Optional.of(timeseriesMetadata); @@ -1499,6 +2078,9 @@ public TsBlock getAllSatisfiedPageData(boolean ascending) throws IOException { if (!ascending) { tsBlock.reverse(); } + if (LOGGER.isDebugEnabled()) { + LOGGER.debug("[getAllSatisfiedPageData] TsBlock:{}", CommonUtils.toString(tsBlock)); + } return tsBlock; } finally { long time = System.nanoTime() - startTime; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/ExtremeAccumulator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/ExtremeAccumulator.java index cc9d3e3354cb0..f2fdf2439bae3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/ExtremeAccumulator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/ExtremeAccumulator.java @@ -120,20 +120,20 @@ public void addStatistics(Statistics[] statistics) { switch (seriesDataType) { case INT32: - updateIntResult((int) statistics[0].getMaxValue()); - updateIntResult((int) statistics[0].getMinValue()); + updateIntResult(((Number) statistics[0].getMaxValue()).intValue()); + updateIntResult(((Number) statistics[0].getMinValue()).intValue()); break; case INT64: - updateLongResult((long) statistics[0].getMaxValue()); - updateLongResult((long) statistics[0].getMinValue()); + updateLongResult(((Number) statistics[0].getMaxValue()).longValue()); + updateLongResult(((Number) statistics[0].getMinValue()).longValue()); break; case FLOAT: - updateFloatResult((float) statistics[0].getMaxValue()); - updateFloatResult((float) statistics[0].getMinValue()); + updateFloatResult(((Number) statistics[0].getMaxValue()).floatValue()); + updateFloatResult(((Number) statistics[0].getMinValue()).floatValue()); break; case DOUBLE: - updateDoubleResult((double) statistics[0].getMaxValue()); - updateDoubleResult((double) statistics[0].getMinValue()); + updateDoubleResult(((Number) statistics[0].getMaxValue()).doubleValue()); + updateDoubleResult(((Number) statistics[0].getMinValue()).doubleValue()); break; case TEXT: case STRING: diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/FirstAccumulator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/FirstAccumulator.java index 2ca9b7785ccd1..7436bbf035b57 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/FirstAccumulator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/FirstAccumulator.java @@ -22,6 +22,7 @@ import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.statistics.DateStatistics; import org.apache.tsfile.file.metadata.statistics.Statistics; import org.apache.tsfile.read.common.block.column.BinaryColumn; import org.apache.tsfile.read.common.block.column.BinaryColumnBuilder; @@ -32,6 +33,8 @@ import org.apache.tsfile.utils.TsPrimitiveType; import org.apache.tsfile.write.UnSupportedDataTypeException; +import java.nio.charset.StandardCharsets; + import static com.google.common.base.Preconditions.checkArgument; import static org.apache.iotdb.db.queryengine.execution.operator.source.relational.aggregation.Utils.serializeTimeValue; @@ -212,25 +215,42 @@ public void addStatistics(Statistics[] statistics) { switch (seriesDataType) { case INT32: case DATE: - updateIntFirstValue((int) statistics[0].getFirstValue(), statistics[0].getStartTime()); + updateIntFirstValue( + ((Number) statistics[0].getFirstValue()).intValue(), statistics[0].getStartTime()); break; case INT64: case TIMESTAMP: - updateLongFirstValue((long) statistics[0].getFirstValue(), statistics[0].getStartTime()); + updateLongFirstValue( + ((Number) statistics[0].getFirstValue()).longValue(), statistics[0].getStartTime()); break; case FLOAT: - updateFloatFirstValue((float) statistics[0].getFirstValue(), statistics[0].getStartTime()); + updateFloatFirstValue( + ((Number) statistics[0].getFirstValue()).floatValue(), statistics[0].getStartTime()); break; case DOUBLE: updateDoubleFirstValue( - (double) statistics[0].getFirstValue(), statistics[0].getStartTime()); + ((Number) statistics[0].getFirstValue()).doubleValue(), statistics[0].getStartTime()); break; case TEXT: case BLOB: case STRING: case OBJECT: - updateBinaryFirstValue( - (Binary) statistics[0].getFirstValue(), statistics[0].getStartTime()); + if (statistics[0] instanceof DateStatistics) { + updateBinaryFirstValue( + new Binary( + TSDataType.getDateStringValue((Integer) statistics[0].getFirstValue()), + StandardCharsets.UTF_8), + statistics[0].getStartTime()); + } else { + if (statistics[0].getFirstValue() instanceof Binary) { + updateBinaryFirstValue( + (Binary) statistics[0].getFirstValue(), statistics[0].getStartTime()); + } else { + updateBinaryFirstValue( + new Binary(String.valueOf(statistics[0].getFirstValue()), StandardCharsets.UTF_8), + statistics[0].getStartTime()); + } + } break; case BOOLEAN: updateBooleanFirstValue( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/FirstByAccumulator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/FirstByAccumulator.java index 2ca6c201f9471..551b8f965ffbb 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/FirstByAccumulator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/FirstByAccumulator.java @@ -32,6 +32,8 @@ import org.apache.tsfile.utils.TsPrimitiveType; import org.apache.tsfile.write.UnSupportedDataTypeException; +import java.nio.charset.StandardCharsets; + import static com.google.common.base.Preconditions.checkArgument; import static org.apache.iotdb.db.queryengine.execution.operator.source.relational.aggregation.Utils.serializeTimeValue; @@ -267,23 +269,29 @@ public void addStatistics(Statistics[] statistics) { switch (xDataType) { case INT32: case DATE: - xResult.setInt((int) xStatistics.getFirstValue()); + xResult.setInt(((Number) xStatistics.getFirstValue()).intValue()); break; case INT64: case TIMESTAMP: - xResult.setLong((long) xStatistics.getFirstValue()); + xResult.setLong(((Number) xStatistics.getFirstValue()).longValue()); break; case FLOAT: - xResult.setFloat((float) statistics[0].getFirstValue()); + xResult.setFloat(((Number) statistics[0].getFirstValue()).floatValue()); break; case DOUBLE: - xResult.setDouble((double) statistics[0].getFirstValue()); + xResult.setDouble(((Number) statistics[0].getFirstValue()).doubleValue()); break; case TEXT: case BLOB: case OBJECT: case STRING: - xResult.setBinary((Binary) statistics[0].getFirstValue()); + if (statistics[0].getFirstValue() instanceof Binary) { + xResult.setBinary((Binary) statistics[0].getFirstValue()); + } else { + xResult.setBinary( + new Binary( + String.valueOf(statistics[0].getFirstValue()), StandardCharsets.UTF_8)); + } break; case BOOLEAN: xResult.setBoolean((boolean) statistics[0].getFirstValue()); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/LastAccumulator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/LastAccumulator.java index 40f8e2ef97abd..eb0d9fec5d350 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/LastAccumulator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/LastAccumulator.java @@ -22,6 +22,7 @@ import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.statistics.DateStatistics; import org.apache.tsfile.file.metadata.statistics.Statistics; import org.apache.tsfile.read.common.block.column.BinaryColumn; import org.apache.tsfile.read.common.block.column.BinaryColumnBuilder; @@ -32,6 +33,8 @@ import org.apache.tsfile.utils.TsPrimitiveType; import org.apache.tsfile.write.UnSupportedDataTypeException; +import java.nio.charset.StandardCharsets; + import static com.google.common.base.Preconditions.checkArgument; import static org.apache.iotdb.db.queryengine.execution.operator.source.relational.aggregation.Utils.serializeTimeValue; @@ -223,23 +226,42 @@ public void addStatistics(Statistics[] statistics) { switch (seriesDataType) { case INT32: case DATE: - updateIntLastValue((int) statistics[0].getLastValue(), statistics[0].getEndTime()); + updateIntLastValue( + ((Number) statistics[0].getLastValue()).intValue(), statistics[0].getEndTime()); break; case INT64: case TIMESTAMP: - updateLongLastValue((long) statistics[0].getLastValue(), statistics[0].getEndTime()); + updateLongLastValue( + ((Number) statistics[0].getLastValue()).longValue(), statistics[0].getEndTime()); break; case FLOAT: - updateFloatLastValue((float) statistics[0].getLastValue(), statistics[0].getEndTime()); + updateFloatLastValue( + ((Number) statistics[0].getLastValue()).floatValue(), statistics[0].getEndTime()); break; case DOUBLE: - updateDoubleLastValue((double) statistics[0].getLastValue(), statistics[0].getEndTime()); + updateDoubleLastValue( + ((Number) statistics[0].getLastValue()).doubleValue(), statistics[0].getEndTime()); break; case TEXT: case BLOB: case OBJECT: case STRING: - updateBinaryLastValue((Binary) statistics[0].getLastValue(), statistics[0].getEndTime()); + if (statistics[0] instanceof DateStatistics) { + updateBinaryLastValue( + new Binary( + TSDataType.getDateStringValue((Integer) statistics[0].getLastValue()), + StandardCharsets.UTF_8), + statistics[0].getEndTime()); + } else { + if (statistics[0].getLastValue() instanceof Binary) { + updateBinaryLastValue( + (Binary) statistics[0].getLastValue(), statistics[0].getEndTime()); + } else { + updateBinaryLastValue( + new Binary(String.valueOf(statistics[0].getLastValue()), StandardCharsets.UTF_8), + statistics[0].getEndTime()); + } + } break; case BOOLEAN: updateBooleanLastValue((boolean) statistics[0].getLastValue(), statistics[0].getEndTime()); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/LastByAccumulator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/LastByAccumulator.java index 2ee09380b83ab..ddfff1ed2cfee 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/LastByAccumulator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/LastByAccumulator.java @@ -275,17 +275,17 @@ public void addStatistics(Statistics[] statistics) { switch (xDataType) { case INT32: case DATE: - xResult.setInt((int) xStatistics.getLastValue()); + xResult.setInt(((Number) xStatistics.getLastValue()).intValue()); break; case INT64: case TIMESTAMP: - xResult.setLong((long) xStatistics.getLastValue()); + xResult.setLong(((Number) xStatistics.getLastValue()).longValue()); break; case FLOAT: - xResult.setFloat((float) statistics[0].getLastValue()); + xResult.setFloat(((Number) statistics[0].getLastValue()).floatValue()); break; case DOUBLE: - xResult.setDouble((double) statistics[0].getLastValue()); + xResult.setDouble(((Number) statistics[0].getLastValue()).doubleValue()); break; case TEXT: case BLOB: diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/MaxAccumulator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/MaxAccumulator.java index e1a4b3236943f..68f4dd13ae034 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/MaxAccumulator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/MaxAccumulator.java @@ -22,12 +22,15 @@ import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.statistics.BinaryStatistics; import org.apache.tsfile.file.metadata.statistics.Statistics; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.RamUsageEstimator; import org.apache.tsfile.utils.TsPrimitiveType; import org.apache.tsfile.write.UnSupportedDataTypeException; +import java.nio.charset.StandardCharsets; + import static com.google.common.base.Preconditions.checkArgument; public class MaxAccumulator implements TableAccumulator { @@ -206,22 +209,29 @@ public void addStatistics(Statistics[] statistics) { switch (seriesDataType) { case INT32: case DATE: - updateIntMaxValue((int) statistics[0].getMaxValue()); + updateIntMaxValue(((Number) statistics[0].getMaxValue()).intValue()); break; case INT64: case TIMESTAMP: - updateLongMaxValue((long) statistics[0].getMaxValue()); + updateLongMaxValue(((Number) statistics[0].getMaxValue()).longValue()); break; case FLOAT: - updateFloatMaxValue((float) statistics[0].getMaxValue()); + updateFloatMaxValue(((Number) statistics[0].getMaxValue()).floatValue()); break; case DOUBLE: - updateDoubleMaxValue((double) statistics[0].getMaxValue()); + updateDoubleMaxValue(((Number) statistics[0].getMaxValue()).doubleValue()); break; case TEXT: case BLOB: case STRING: - updateBinaryMaxValue((Binary) statistics[0].getMaxValue()); + if (!(statistics[0] instanceof BinaryStatistics)) { + if (statistics[0].getMaxValue() instanceof Binary) { + updateBinaryMaxValue((Binary) statistics[0].getMaxValue()); + } else { + updateBinaryMaxValue( + new Binary(String.valueOf(statistics[0].getMaxValue()), StandardCharsets.UTF_8)); + } + } break; case BOOLEAN: updateBooleanMaxValue((boolean) statistics[0].getMaxValue()); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/MinAccumulator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/MinAccumulator.java index 67e84e9440689..e838988c8bb64 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/MinAccumulator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/MinAccumulator.java @@ -22,12 +22,15 @@ import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.statistics.BinaryStatistics; import org.apache.tsfile.file.metadata.statistics.Statistics; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.RamUsageEstimator; import org.apache.tsfile.utils.TsPrimitiveType; import org.apache.tsfile.write.UnSupportedDataTypeException; +import java.nio.charset.StandardCharsets; + import static com.google.common.base.Preconditions.checkArgument; public class MinAccumulator implements TableAccumulator { @@ -206,22 +209,29 @@ public void addStatistics(Statistics[] statistics) { switch (seriesDataType) { case INT32: case DATE: - updateIntMinValue((int) statistics[0].getMinValue()); + updateIntMinValue(((Number) statistics[0].getMinValue()).intValue()); break; case INT64: case TIMESTAMP: - updateLongMinValue((long) statistics[0].getMinValue()); + updateLongMinValue(((Number) statistics[0].getMinValue()).longValue()); break; case FLOAT: - updateFloatMinValue((float) statistics[0].getMinValue()); + updateFloatMinValue(((Number) statistics[0].getMinValue()).floatValue()); break; case DOUBLE: - updateDoubleMinValue((double) statistics[0].getMinValue()); + updateDoubleMinValue(((Number) statistics[0].getMinValue()).doubleValue()); break; case TEXT: case BLOB: case STRING: - updateBinaryMinValue((Binary) statistics[0].getMinValue()); + if (!(statistics[0] instanceof BinaryStatistics)) { + if (statistics[0].getMinValue() instanceof Binary) { + updateBinaryMinValue((Binary) statistics[0].getMinValue()); + } else { + updateBinaryMinValue( + new Binary(String.valueOf(statistics[0].getMinValue()), StandardCharsets.UTF_8)); + } + } break; case BOOLEAN: updateBooleanMinValue((boolean) statistics[0].getMinValue()); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/Coordinator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/Coordinator.java index 3210d277d8619..29ae1ed87e7e9 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/Coordinator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/Coordinator.java @@ -65,6 +65,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.planner.optimizations.PlanOptimizer; import org.apache.iotdb.db.queryengine.plan.relational.sql.ParameterExtractor; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.AddColumn; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.AlterColumnDataType; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.AlterDB; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ClearCache; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.CreateDB; @@ -493,6 +494,7 @@ private IQueryExecution createQueryExecutionForTableModel( || statement instanceof DescribeTable || statement instanceof ShowTables || statement instanceof AddColumn + || statement instanceof AlterColumnDataType || statement instanceof SetProperties || statement instanceof DropColumn || statement instanceof DropTable diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TableConfigTaskVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TableConfigTaskVisitor.java index 1de3643cabe55..58b51b5e2da60 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TableConfigTaskVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TableConfigTaskVisitor.java @@ -74,6 +74,7 @@ import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.region.MigrateRegionTask; import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.region.ReconstructRegionTask; import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.region.RemoveRegionTask; +import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.relational.AlterColumnDataTypeTask; import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.relational.AlterDBTask; import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.relational.AlterTableAddColumnTask; import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.relational.AlterTableCommentColumnTask; @@ -136,6 +137,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.security.AccessControl; import org.apache.iotdb.db.queryengine.plan.relational.security.ITableAuthCheckerImpl; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.AddColumn; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.AlterColumnDataType; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.AlterDB; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.AlterPipe; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.AstVisitor; @@ -616,6 +618,26 @@ private Pair parseTable4CreateTableOrView( return new Pair<>(database, table); } + @Override + protected IConfigTask visitAlterColumnDataType( + AlterColumnDataType node, MPPQueryContext context) { + context.setQueryType(QueryType.WRITE); + final Pair databaseTablePair = splitQualifiedName(node.getTableName()); + final String columnName = node.getColumnName().getValue(); + final DataType dataType = node.getDataType(); + final boolean ifTableExists = node.isIfTableExists(); + final boolean ifColumnExists = node.isIfColumnExists(); + return new AlterColumnDataTypeTask( + databaseTablePair.getLeft(), + databaseTablePair.getRight(), + context.getQueryId().getId(), + ifTableExists, + ifColumnExists, + columnName, + getDataType(dataType), + node.isView()); + } + private boolean checkTimeColumnIdempotent( final TsTableColumnCategory category, final String columnName, diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TreeConfigTaskVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TreeConfigTaskVisitor.java index f3358012f3954..1e1618209a8e8 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TreeConfigTaskVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TreeConfigTaskVisitor.java @@ -31,6 +31,7 @@ import org.apache.iotdb.db.exception.sql.SemanticException; import org.apache.iotdb.db.queryengine.common.MPPQueryContext; import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.AlterEncodingCompressorTask; +import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.AlterTimeSeriesTask; import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.CountDatabaseTask; import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.CountTimeSlotListTask; import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.CreateContinuousQueryTask; @@ -126,6 +127,7 @@ import org.apache.iotdb.db.queryengine.plan.statement.StatementNode; import org.apache.iotdb.db.queryengine.plan.statement.StatementVisitor; import org.apache.iotdb.db.queryengine.plan.statement.metadata.AlterEncodingCompressorStatement; +import org.apache.iotdb.db.queryengine.plan.statement.metadata.AlterTimeSeriesStatement; import org.apache.iotdb.db.queryengine.plan.statement.metadata.CountDatabaseStatement; import org.apache.iotdb.db.queryengine.plan.statement.metadata.CountTimeSlotListStatement; import org.apache.iotdb.db.queryengine.plan.statement.metadata.CreateContinuousQueryStatement; @@ -741,6 +743,12 @@ public IConfigTask visitAlterLogicalView( return new AlterLogicalViewTask(alterLogicalViewStatement, context); } + @Override + public IConfigTask visitAlterTimeSeries( + AlterTimeSeriesStatement alterTimeSeriesStatement, MPPQueryContext context) { + return new AlterTimeSeriesTask(context.getQueryId().getId(), alterTimeSeriesStatement); + } + @Override public IConfigTask visitGetRegionId( GetRegionIdStatement getRegionIdStatement, MPPQueryContext context) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java index 658a0827d2b47..6845ee55c3556 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java @@ -78,6 +78,7 @@ import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchema; import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchemaUtil; import org.apache.iotdb.commons.schema.template.Template; +import org.apache.iotdb.commons.schema.tree.AlterTimeSeriesOperationType; import org.apache.iotdb.commons.schema.view.LogicalViewSchema; import org.apache.iotdb.commons.schema.view.viewExpression.ViewExpression; import org.apache.iotdb.commons.subscription.config.SubscriptionConfig; @@ -95,6 +96,7 @@ import org.apache.iotdb.confignode.rpc.thrift.TAlterOrDropTableReq; import org.apache.iotdb.confignode.rpc.thrift.TAlterPipeReq; import org.apache.iotdb.confignode.rpc.thrift.TAlterSchemaTemplateReq; +import org.apache.iotdb.confignode.rpc.thrift.TAlterTimeSeriesReq; import org.apache.iotdb.confignode.rpc.thrift.TCountDatabaseResp; import org.apache.iotdb.confignode.rpc.thrift.TCountTimeSlotListReq; import org.apache.iotdb.confignode.rpc.thrift.TCountTimeSlotListResp; @@ -250,6 +252,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowDB; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Use; import org.apache.iotdb.db.queryengine.plan.statement.metadata.AlterEncodingCompressorStatement; +import org.apache.iotdb.db.queryengine.plan.statement.metadata.AlterTimeSeriesStatement; import org.apache.iotdb.db.queryengine.plan.statement.metadata.CountDatabaseStatement; import org.apache.iotdb.db.queryengine.plan.statement.metadata.CountTimeSlotListStatement; import org.apache.iotdb.db.queryengine.plan.statement.metadata.CreateContinuousQueryStatement; @@ -337,6 +340,7 @@ import org.apache.thrift.transport.TTransportException; import org.apache.tsfile.common.conf.TSFileDescriptor; import org.apache.tsfile.common.constant.TsFileConstant; +import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.external.commons.codec.digest.DigestUtils; import org.apache.tsfile.utils.ReadWriteIOUtils; import org.slf4j.Logger; @@ -3176,6 +3180,72 @@ public TSStatus alterLogicalViewByPipe( return tsStatus; } + @Override + public SettableFuture alterTimeSeriesDataType( + final String queryId, final AlterTimeSeriesStatement alterTimeSeriesStatement) { + final SettableFuture future = SettableFuture.create(); + // Will only occur if no permission + if (alterTimeSeriesStatement.getPath() == null) { + future.set(new ConfigTaskResult(TSStatusCode.SUCCESS_STATUS)); + return future; + } + + ByteBuffer measurementPathBuffer = null; + try (ByteArrayOutputStream baos = new ByteArrayOutputStream()) { + alterTimeSeriesStatement.getPath().serialize(baos); + measurementPathBuffer = ByteBuffer.wrap(baos.toByteArray()); + } catch (IOException ignored) { + // ByteArrayOutputStream won't throw IOException + } + + final ByteArrayOutputStream stream = new ByteArrayOutputStream(); + try { + ReadWriteIOUtils.write(alterTimeSeriesStatement.getDataType(), stream); + } catch (final IOException ignored) { + // ByteArrayOutputStream won't throw IOException + } + + final TAlterTimeSeriesReq req = + new TAlterTimeSeriesReq( + queryId, + measurementPathBuffer, + AlterTimeSeriesOperationType.ALTER_DATA_TYPE.getTypeValue(), + ByteBuffer.wrap(stream.toByteArray())); + try (final ConfigNodeClient client = + CONFIG_NODE_CLIENT_MANAGER.borrowClient(ConfigNodeInfo.CONFIG_REGION_ID)) { + TSStatus tsStatus; + do { + try { + tsStatus = client.alterTimeSeriesDataType(req); + } catch (final TTransportException e) { + if (e.getType() == TTransportException.TIMED_OUT + || e.getCause() instanceof SocketTimeoutException) { + // time out mainly caused by slow execution, wait until + tsStatus = RpcUtils.getStatus(TSStatusCode.OVERLAP_WITH_EXISTING_TASK); + } else { + throw e; + } + } + // keep waiting until task ends + } while (TSStatusCode.OVERLAP_WITH_EXISTING_TASK.getStatusCode() == tsStatus.getCode()); + + if (TSStatusCode.SUCCESS_STATUS.getStatusCode() != tsStatus.getCode()) { + if (tsStatus.getCode() == TSStatusCode.MULTIPLE_ERROR.getStatusCode()) { + future.setException( + new BatchProcessException(tsStatus.subStatus.toArray(new TSStatus[0]))); + } else { + future.setException(new IoTDBException(tsStatus)); + } + } else { + future.set(new ConfigTaskResult(TSStatusCode.SUCCESS_STATUS)); + } + return future; + } catch (final ClientManagerException | TException e) { + future.setException(e); + return future; + } + } + @Override public SettableFuture getRegionId( final GetRegionIdStatement getRegionIdStatement) { @@ -4167,7 +4237,8 @@ public SettableFuture describeTable( } else if (Boolean.FALSE.equals(isShowCreateView)) { ShowCreateTableTask.buildTsBlock(table, future); } else if (isDetails) { - DescribeTableDetailsTask.buildTsBlock(table, resp.getPreDeletedColumns(), future); + DescribeTableDetailsTask.buildTsBlock( + table, resp.getPreDeletedColumns(), resp.preAlteredColumns, future); } else { DescribeTableTask.buildTsBlock(table, future); } @@ -4302,6 +4373,45 @@ public SettableFuture alterTableAddColumn( return future; } + @Override + public SettableFuture alterColumnDataType( + String database, + String tableName, + String columnName, + TSDataType newType, + String queryId, + boolean ifTableExists, + boolean ifColumnExists, + final boolean isView) { + final SettableFuture future = SettableFuture.create(); + try (final ConfigNodeClient client = + CLUSTER_DELETION_CONFIG_NODE_CLIENT_MANAGER.borrowClient(ConfigNodeInfo.CONFIG_REGION_ID)) { + + final TSStatus tsStatus = + sendAlterReq2ConfigNode( + database, + tableName, + queryId, + AlterOrDropTableOperationType.ALTER_COLUMN_DATA_TYPE, + TsTableColumnSchemaUtil.serialize(columnName, newType), + isView, + client); + + if (TSStatusCode.SUCCESS_STATUS.getStatusCode() == tsStatus.getCode() + || (TSStatusCode.TABLE_NOT_EXISTS.getStatusCode() == tsStatus.getCode() && ifTableExists) + || (TSStatusCode.COLUMN_NOT_EXISTS.getStatusCode() == tsStatus.getCode() + && ifColumnExists)) { + future.set(new ConfigTaskResult(TSStatusCode.SUCCESS_STATUS)); + } else { + future.setException( + new IoTDBException(getTableErrorMessage(tsStatus, database), tsStatus.getCode())); + } + } catch (final ClientManagerException | TException e) { + future.setException(e); + } + return future; + } + @Override public SettableFuture alterTableRenameColumn( final String database, diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/IConfigTaskExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/IConfigTaskExecutor.java index 2d7c0f6d1f3a6..4cb28ee6ef694 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/IConfigTaskExecutor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/IConfigTaskExecutor.java @@ -46,6 +46,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowDB; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Use; import org.apache.iotdb.db.queryengine.plan.statement.metadata.AlterEncodingCompressorStatement; +import org.apache.iotdb.db.queryengine.plan.statement.metadata.AlterTimeSeriesStatement; import org.apache.iotdb.db.queryengine.plan.statement.metadata.CountDatabaseStatement; import org.apache.iotdb.db.queryengine.plan.statement.metadata.CountTimeSlotListStatement; import org.apache.iotdb.db.queryengine.plan.statement.metadata.CreateContinuousQueryStatement; @@ -100,6 +101,7 @@ import org.apache.iotdb.service.rpc.thrift.TPipeTransferResp; import com.google.common.util.concurrent.SettableFuture; +import org.apache.tsfile.enums.TSDataType; import javax.annotation.Nullable; @@ -256,6 +258,9 @@ SettableFuture renameLogicalView( SettableFuture alterLogicalView( AlterLogicalViewStatement alterLogicalViewStatement, MPPQueryContext context); + SettableFuture alterTimeSeriesDataType( + String queryId, AlterTimeSeriesStatement alterTimeSeriesStatement); + TSStatus alterLogicalViewByPipe( AlterLogicalViewNode alterLogicalViewNode, boolean shouldMarkAsPipeRequest); @@ -358,6 +363,16 @@ SettableFuture alterTableAddColumn( final boolean columnIfExists, final boolean isView); + SettableFuture alterColumnDataType( + final String database, + final String tableName, + final String columnName, + final TSDataType newType, + final String queryId, + final boolean tableIfExists, + boolean ifColumnExists, + final boolean isView); + SettableFuture alterTableRenameColumn( final String database, final String tableName, diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/AlterTimeSeriesTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/AlterTimeSeriesTask.java new file mode 100644 index 0000000000000..65c54041d8908 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/AlterTimeSeriesTask.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.execution.config.metadata; + +import org.apache.iotdb.db.queryengine.plan.execution.config.ConfigTaskResult; +import org.apache.iotdb.db.queryengine.plan.execution.config.IConfigTask; +import org.apache.iotdb.db.queryengine.plan.execution.config.executor.IConfigTaskExecutor; +import org.apache.iotdb.db.queryengine.plan.statement.metadata.AlterTimeSeriesStatement; + +import com.google.common.util.concurrent.ListenableFuture; + +public class AlterTimeSeriesTask implements IConfigTask { + private final String queryId; + + private final AlterTimeSeriesStatement alterTimeSeriesStatement; + + public AlterTimeSeriesTask( + final String queryId, final AlterTimeSeriesStatement alterTimeSeriesStatement) { + this.queryId = queryId; + this.alterTimeSeriesStatement = alterTimeSeriesStatement; + } + + @Override + public ListenableFuture execute(final IConfigTaskExecutor configTaskExecutor) + throws InterruptedException { + if (alterTimeSeriesStatement.getDataType() != null) { + return configTaskExecutor.alterTimeSeriesDataType(queryId, alterTimeSeriesStatement); + } else { + // not support + throw new InterruptedException("AlterTimeSeriesTask is not support"); + } + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/AlterColumnDataTypeTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/AlterColumnDataTypeTask.java new file mode 100644 index 0000000000000..2f8a1983a9f9d --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/AlterColumnDataTypeTask.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.execution.config.metadata.relational; + +import org.apache.iotdb.db.queryengine.plan.execution.config.ConfigTaskResult; +import org.apache.iotdb.db.queryengine.plan.execution.config.executor.IConfigTaskExecutor; + +import com.google.common.util.concurrent.ListenableFuture; +import org.apache.tsfile.enums.TSDataType; + +public class AlterColumnDataTypeTask extends AbstractAlterOrDropTableTask { + private final String columnName; + private final TSDataType newType; + private final boolean ifColumnExists; + + public AlterColumnDataTypeTask( + String database, + String tableName, + String queryId, + boolean tableIfExists, + boolean ifColumnExists, + String columnName, + TSDataType newType, + final boolean view) { + super(database, tableName, queryId, tableIfExists, view); + this.columnName = columnName; + this.newType = newType; + this.ifColumnExists = ifColumnExists; + } + + @Override + public ListenableFuture execute(IConfigTaskExecutor configTaskExecutor) + throws InterruptedException { + return configTaskExecutor.alterColumnDataType( + database, tableName, columnName, newType, queryId, tableIfExists, ifColumnExists, view); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/DescribeTableDetailsTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/DescribeTableDetailsTask.java index be24f689b2e6e..ac2fe718850da 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/DescribeTableDetailsTask.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/DescribeTableDetailsTask.java @@ -37,6 +37,7 @@ import org.apache.tsfile.utils.Binary; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.stream.Collectors; @@ -54,6 +55,7 @@ public ListenableFuture execute(final IConfigTaskExecutor conf public static void buildTsBlock( final TsTable table, final Set preDeletedColumns, + final Map preAlteredColumns, final SettableFuture future) { final List outputDataTypes = ColumnHeaderConstant.describeTableDetailsColumnHeaders.stream() @@ -63,22 +65,24 @@ public static void buildTsBlock( final TsBlockBuilder builder = new TsBlockBuilder(outputDataTypes); for (final TsTableColumnSchema columnSchema : table.getColumnList()) { builder.getTimeColumnBuilder().writeLong(0L); + String columnStatus = "USING"; + String dataTypeName = columnSchema.getDataType().name(); + if (preDeletedColumns.contains(columnSchema.getColumnName())) { + columnStatus = "PRE_DELETE"; + } builder .getColumnBuilder(0) .writeBinary(new Binary(columnSchema.getColumnName(), TSFileConfig.STRING_CHARSET)); builder .getColumnBuilder(1) - .writeBinary(new Binary(columnSchema.getDataType().name(), TSFileConfig.STRING_CHARSET)); + .writeBinary(new Binary(dataTypeName, TSFileConfig.STRING_CHARSET)); builder .getColumnBuilder(2) .writeBinary( new Binary(columnSchema.getColumnCategory().name(), TSFileConfig.STRING_CHARSET)); builder .getColumnBuilder(3) - .writeBinary( - new Binary( - preDeletedColumns.contains(columnSchema.getColumnName()) ? "PRE_DELETE" : "USING", - TSFileConfig.STRING_CHARSET)); + .writeBinary(new Binary(columnStatus, TSFileConfig.STRING_CHARSET)); if (columnSchema.getProps().containsKey(TsTable.COMMENT_KEY)) { builder diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java index 9edffa2e9e94b..07ec27bbb48a3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java @@ -625,9 +625,18 @@ private void parseAlterClause( alterTimeSeriesStatement.setAlterType(AlterTimeSeriesStatement.AlterType.RENAME); alterMap.put(parseAttributeKey(ctx.beforeName), parseAttributeKey(ctx.currentName)); } else if (ctx.SET() != null) { - // Set - alterTimeSeriesStatement.setAlterType(AlterTimeSeriesStatement.AlterType.SET); - setMap(ctx, alterMap); + if (ctx.DATA() != null && ctx.TYPE() != null) { + // Set data type + alterTimeSeriesStatement.setAlterType(AlterTimeSeriesStatement.AlterType.SET_DATA_TYPE); + setMap(ctx, alterMap); + if (ctx.attributeValue() != null) { + alterTimeSeriesStatement.setDataType(parseDataTypeAttribute(ctx.attributeValue())); + } + } else { + // Set + alterTimeSeriesStatement.setAlterType(AlterTimeSeriesStatement.AlterType.SET); + setMap(ctx, alterMap); + } } else if (ctx.DROP() != null) { // Drop alterTimeSeriesStatement.setAlterType(AlterTimeSeriesStatement.AlterType.DROP); @@ -642,7 +651,7 @@ private void parseAlterClause( // Add attribute alterTimeSeriesStatement.setAlterType(AlterTimeSeriesStatement.AlterType.ADD_ATTRIBUTES); setMap(ctx, alterMap); - } else { + } else if (ctx.UPSERT() != null) { // Upsert alterTimeSeriesStatement.setAlterType(AlterTimeSeriesStatement.AlterType.UPSERT); if (ctx.aliasClause() != null) { @@ -665,6 +674,22 @@ public void parseAliasClause( } } + private TSDataType parseDataTypeAttribute(IoTDBSqlParser.AttributeValueContext ctx) { + TSDataType dataType = null; + if (ctx != null) { + String dataTypeString = parseAttributeValue(ctx); + try { + dataType = TSDataType.valueOf(dataTypeString); + if (TSDataType.UNKNOWN.equals(dataType) || TSDataType.VECTOR.equals(dataType)) { + throw new SemanticException(String.format("Unsupported datatype: %s", dataTypeString)); + } + } catch (Exception e) { + throw new SemanticException(String.format("Unsupported datatype: %s", dataTypeString)); + } + } + return dataType; + } + @Override public Statement visitAlterEncodingCompressor( final IoTDBSqlParser.AlterEncodingCompressorContext ctx) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LogicalPlanVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LogicalPlanVisitor.java index 2192435b4c297..1b7c3bcbfe7bf 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LogicalPlanVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LogicalPlanVisitor.java @@ -454,7 +454,8 @@ public PlanNode visitAlterTimeSeries( alterTimeSeriesStatement.getAlias(), alterTimeSeriesStatement.getTagsMap(), alterTimeSeriesStatement.getAttributesMap(), - alterTimeSeriesStatement.isAlterView()); + alterTimeSeriesStatement.isAlterView(), + alterTimeSeriesStatement.getDataType()); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java index e699ee417deb6..23c25b9ab22c5 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java @@ -2269,9 +2269,13 @@ private void semanticCheckForJoin(JoinNode node) { private BiFunction buildUpdateLastRowFunction(Type joinKeyType) { switch (joinKeyType.getTypeEnum()) { case INT32: + return (inputColumn, rowIndex) -> + new IntColumn( + 1, Optional.empty(), new int[] {inputColumn.getInt(rowIndex)}, TSDataType.INT32); case DATE: return (inputColumn, rowIndex) -> - new IntColumn(1, Optional.empty(), new int[] {inputColumn.getInt(rowIndex)}); + new IntColumn( + 1, Optional.empty(), new int[] {inputColumn.getInt(rowIndex)}, TSDataType.DATE); case INT64: case TIMESTAMP: return (inputColumn, rowIndex) -> diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanVisitor.java index 4bebb692254a1..2056cf6297c8d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanVisitor.java @@ -504,6 +504,10 @@ public R visitAlterEncodingCompressor(AlterEncodingCompressorNode node, C contex return visitPlan(node, context); } + public R visitAlterTimeSeriesDataType(AlterTimeSeriesNode node, C context) { + return visitPlan(node, context); + } + public R visitConstructSchemaBlackList(ConstructSchemaBlackListNode node, C context) { return visitPlan(node, context); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metadata/write/AlterTimeSeriesNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metadata/write/AlterTimeSeriesNode.java index 58981a8c1dfe8..02783b76f0c6b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metadata/write/AlterTimeSeriesNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metadata/write/AlterTimeSeriesNode.java @@ -32,8 +32,11 @@ import com.google.common.collect.ImmutableList; import org.apache.tsfile.common.conf.TSFileConfig; +import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.exception.NotImplementedException; import org.apache.tsfile.utils.ReadWriteIOUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.io.DataOutputStream; import java.io.IOException; @@ -44,6 +47,7 @@ import java.util.Objects; public class AlterTimeSeriesNode extends WritePlanNode { + private static final Logger LOGGER = LoggerFactory.getLogger(AlterTimeSeriesNode.class); private MeasurementPath path; private AlterType alterType; @@ -63,6 +67,8 @@ public class AlterTimeSeriesNode extends WritePlanNode { private boolean isAlterView; + private TSDataType dataType; + private TRegionReplicaSet regionReplicaSet; public AlterTimeSeriesNode( @@ -84,6 +90,40 @@ public AlterTimeSeriesNode( this.isAlterView = isAlterView; } + public AlterTimeSeriesNode( + PlanNodeId id, + MeasurementPath path, + AlterType alterType, + Map alterMap, + String alias, + Map tagsMap, + Map attributesMap, + boolean isAlterView, + TSDataType dataType) { + super(id); + this.path = path; + this.alterType = alterType; + this.alterMap = alterMap; + this.alias = alias; + this.tagsMap = tagsMap; + this.attributesMap = attributesMap; + this.isAlterView = isAlterView; + this.dataType = dataType; + } + + public AlterTimeSeriesNode( + PlanNodeId id, + MeasurementPath path, + AlterType alterType, + boolean isAlterView, + TSDataType dataType) { + super(id); + this.path = path; + this.alterType = alterType; + this.isAlterView = isAlterView; + this.dataType = dataType; + } + public MeasurementPath getPath() { return path; } @@ -140,6 +180,14 @@ public void setAlterView(boolean alterView) { isAlterView = alterView; } + public TSDataType getDataType() { + return dataType; + } + + public void setDataType(TSDataType dataType) { + this.dataType = dataType; + } + @Override public List getChildren() { return null; @@ -178,6 +226,7 @@ public static AlterTimeSeriesNode deserialize(ByteBuffer byteBuffer) { Map alterMap = null; Map tagsMap = null; Map attributesMap = null; + TSDataType dataType = null; int length = byteBuffer.getInt(); byte[] bytes = new byte[length]; @@ -226,13 +275,26 @@ public static AlterTimeSeriesNode deserialize(ByteBuffer byteBuffer) { attributesMap = ReadWriteIOUtils.readMap(byteBuffer); } + // dataType + if (byteBuffer.get() == 1) { + dataType = ReadWriteIOUtils.readDataType(byteBuffer); + } + id = ReadWriteIOUtils.readString(byteBuffer); return new AlterTimeSeriesNode( - new PlanNodeId(id), path, alterType, alterMap, alias, tagsMap, attributesMap, isAlterView); + new PlanNodeId(id), + path, + alterType, + alterMap, + alias, + tagsMap, + attributesMap, + isAlterView, + dataType); } @Override - public R accept(PlanVisitor visitor, C schemaRegion) { + public R accept(final PlanVisitor visitor, final C schemaRegion) { return visitor.visitAlterTimeSeries(this, schemaRegion); } @@ -285,6 +347,14 @@ protected void serializeAttributes(ByteBuffer byteBuffer) { byteBuffer.put((byte) 1); ReadWriteIOUtils.write(attributesMap, byteBuffer); } + + // dataType + if (dataType != null) { + byteBuffer.put((byte) 1); + ReadWriteIOUtils.write(dataType, byteBuffer); + } else { + byteBuffer.put((byte) 0); + } } @Override @@ -336,6 +406,14 @@ protected void serializeAttributes(DataOutputStream stream) throws IOException { stream.write((byte) 1); ReadWriteIOUtils.write(attributesMap, stream); } + + // dataType + if (dataType != null) { + stream.write((byte) 1); + ReadWriteIOUtils.write(dataType, stream); + } else { + stream.write((byte) 0); + } } @Override @@ -355,13 +433,14 @@ public boolean equals(Object o) { && Objects.equals(alterMap, that.alterMap) && Objects.equals(alias, that.alias) && Objects.equals(tagsMap, that.tagsMap) - && Objects.equals(attributesMap, that.attributesMap); + && Objects.equals(attributesMap, that.attributesMap) + && Objects.equals(dataType, that.dataType); } @Override public int hashCode() { return Objects.hash( - this.getPlanNodeId(), path, alias, alterType, alterMap, attributesMap, tagsMap); + this.getPlanNodeId(), path, alias, alterType, alterMap, attributesMap, tagsMap, dataType); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/pipe/PipeEnrichedInsertNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/pipe/PipeEnrichedInsertNode.java index 461822a8ab1f0..2e517700217b7 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/pipe/PipeEnrichedInsertNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/pipe/PipeEnrichedInsertNode.java @@ -23,6 +23,7 @@ import org.apache.iotdb.commons.consensus.index.ProgressIndex; import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.db.consensus.statemachine.dataregion.DataExecutionVisitor; +import org.apache.iotdb.db.exception.DataTypeInconsistentException; import org.apache.iotdb.db.queryengine.execution.executor.RegionWriteExecutor; import org.apache.iotdb.db.queryengine.plan.analyze.IAnalysis; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; @@ -32,6 +33,7 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.WritePlanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.SearchNode; +import org.apache.iotdb.db.storageengine.dataregion.memtable.AbstractMemTable; import org.apache.iotdb.db.trigger.executor.TriggerFireVisitor; import org.apache.tsfile.enums.TSDataType; @@ -297,4 +299,9 @@ public boolean equals(final Object o) { public int hashCode() { return insertNode.hashCode(); } + + @Override + public void checkDataType(AbstractMemTable memTable) throws DataTypeInconsistentException { + insertNode.checkDataType(memTable); + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertMultiTabletsNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertMultiTabletsNode.java index 5fdc7ed7dd729..b41d178b396c6 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertMultiTabletsNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertMultiTabletsNode.java @@ -22,12 +22,14 @@ import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.consensus.index.ProgressIndex; import org.apache.iotdb.commons.utils.StatusUtils; +import org.apache.iotdb.db.exception.DataTypeInconsistentException; import org.apache.iotdb.db.queryengine.plan.analyze.IAnalysis; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeType; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanVisitor; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.WritePlanNode; +import org.apache.iotdb.db.storageengine.dataregion.memtable.AbstractMemTable; import org.apache.tsfile.exception.NotImplementedException; import org.apache.tsfile.utils.ReadWriteIOUtils; @@ -293,4 +295,11 @@ public void setProgressIndex(ProgressIndex progressIndex) { this.progressIndex = progressIndex; insertTabletNodeList.forEach(node -> node.setProgressIndex(progressIndex)); } + + @Override + public void checkDataType(AbstractMemTable memTable) throws DataTypeInconsistentException { + for (InsertTabletNode node : insertTabletNodeList) { + node.checkDataType(memTable); + } + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertNode.java index f913bbd7a9762..88a6faa004745 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertNode.java @@ -27,10 +27,12 @@ import org.apache.iotdb.consensus.ConsensusFactory; import org.apache.iotdb.db.conf.IoTDBConfig; import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.exception.DataTypeInconsistentException; import org.apache.iotdb.db.pipe.resource.memory.InsertNodeMemoryEstimator; import org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.DataNodeDevicePathCache; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; +import org.apache.iotdb.db.storageengine.dataregion.memtable.AbstractMemTable; import org.apache.iotdb.db.storageengine.dataregion.memtable.DeviceIDFactory; import org.apache.iotdb.db.storageengine.dataregion.wal.buffer.IWALByteBufferView; import org.apache.iotdb.db.storageengine.dataregion.wal.utils.WALWriteUtils; @@ -448,4 +450,7 @@ public long getMemorySize() { } return memorySize; } + + public abstract void checkDataType(AbstractMemTable memTable) + throws DataTypeInconsistentException; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertRowNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertRowNode.java index 760d36f1f5cd1..a2bd6cb1a00fc 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertRowNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertRowNode.java @@ -24,6 +24,7 @@ import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.utils.TestOnly; import org.apache.iotdb.commons.utils.TimePartitionUtils; +import org.apache.iotdb.db.exception.DataTypeInconsistentException; import org.apache.iotdb.db.queryengine.plan.analyze.IAnalysis; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; @@ -31,6 +32,8 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanVisitor; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.WritePlanNode; import org.apache.iotdb.db.queryengine.plan.relational.metadata.fetcher.cache.TreeDeviceSchemaCacheManager; +import org.apache.iotdb.db.storageengine.dataregion.memtable.AbstractMemTable; +import org.apache.iotdb.db.storageengine.dataregion.memtable.IWritableMemChunkGroup; import org.apache.iotdb.db.storageengine.dataregion.wal.buffer.IWALByteBufferView; import org.apache.iotdb.db.storageengine.dataregion.wal.buffer.WALEntryValue; import org.apache.iotdb.db.storageengine.dataregion.wal.utils.WALWriteUtils; @@ -158,6 +161,10 @@ public String toString() { + time + ", values=" + Arrays.toString(values) + + ", measurements=" + + Arrays.toString(measurements) + + ", dataTypes=" + + Arrays.toString(dataTypes) + '}'; } @@ -922,4 +929,12 @@ public void updateLastCache(String databaseName) { isAligned, measurementSchemas); } + + @Override + public void checkDataType(AbstractMemTable memTable) throws DataTypeInconsistentException { + IWritableMemChunkGroup writableMemChunkGroup = memTable.getWritableMemChunkGroup(getDeviceID()); + if (writableMemChunkGroup != null) { + writableMemChunkGroup.checkDataType(this); + } + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertRowsNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertRowsNode.java index ea7732177663b..7392b7612705e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertRowsNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertRowsNode.java @@ -25,12 +25,14 @@ import org.apache.iotdb.commons.consensus.index.ProgressIndex; import org.apache.iotdb.commons.utils.StatusUtils; import org.apache.iotdb.commons.utils.TimePartitionUtils; +import org.apache.iotdb.db.exception.DataTypeInconsistentException; import org.apache.iotdb.db.queryengine.plan.analyze.IAnalysis; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeType; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanVisitor; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.WritePlanNode; +import org.apache.iotdb.db.storageengine.dataregion.memtable.AbstractMemTable; import org.apache.iotdb.db.storageengine.dataregion.wal.buffer.IWALByteBufferView; import org.apache.iotdb.db.storageengine.dataregion.wal.buffer.WALEntryValue; @@ -402,5 +404,12 @@ public InsertRowsNode emptyClone() { return new InsertRowsNode(this.getPlanNodeId()); } + @Override + public void checkDataType(AbstractMemTable memTable) throws DataTypeInconsistentException { + for (InsertRowNode node : insertRowNodeList) { + node.checkDataType(memTable); + } + } + // endregion } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertRowsOfOneDeviceNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertRowsOfOneDeviceNode.java index fc9f66221ddee..f1e28d32b104d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertRowsOfOneDeviceNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertRowsOfOneDeviceNode.java @@ -26,6 +26,7 @@ import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.utils.StatusUtils; import org.apache.iotdb.commons.utils.TimePartitionUtils; +import org.apache.iotdb.db.exception.DataTypeInconsistentException; import org.apache.iotdb.db.queryengine.plan.analyze.IAnalysis; import org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.DataNodeDevicePathCache; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; @@ -33,6 +34,7 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeType; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanVisitor; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.WritePlanNode; +import org.apache.iotdb.db.storageengine.dataregion.memtable.AbstractMemTable; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.exception.NotImplementedException; @@ -344,4 +346,11 @@ public void setProgressIndex(ProgressIndex progressIndex) { this.progressIndex = progressIndex; insertRowNodeList.forEach(insertRowNode -> insertRowNode.setProgressIndex(progressIndex)); } + + @Override + public void checkDataType(AbstractMemTable memTable) throws DataTypeInconsistentException { + for (InsertRowNode insertRowNode : insertRowNodeList) { + insertRowNode.checkDataType(memTable); + } + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertTabletNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertTabletNode.java index 3d3c25c0613a9..39683e5d9f94e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertTabletNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertTabletNode.java @@ -27,6 +27,7 @@ import org.apache.iotdb.commons.utils.CommonDateTimeUtils; import org.apache.iotdb.commons.utils.TestOnly; import org.apache.iotdb.commons.utils.TimePartitionUtils; +import org.apache.iotdb.db.exception.DataTypeInconsistentException; import org.apache.iotdb.db.exception.query.OutOfTTLException; import org.apache.iotdb.db.queryengine.plan.analyze.IAnalysis; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; @@ -35,7 +36,9 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanVisitor; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.WritePlanNode; import org.apache.iotdb.db.queryengine.plan.relational.metadata.fetcher.cache.TreeDeviceSchemaCacheManager; +import org.apache.iotdb.db.storageengine.dataregion.memtable.AbstractMemTable; import org.apache.iotdb.db.storageengine.dataregion.memtable.DeviceIDFactory; +import org.apache.iotdb.db.storageengine.dataregion.memtable.IWritableMemChunkGroup; import org.apache.iotdb.db.storageengine.dataregion.wal.buffer.IWALByteBufferView; import org.apache.iotdb.db.storageengine.dataregion.wal.buffer.WALEntryValue; import org.apache.iotdb.db.storageengine.dataregion.wal.utils.WALWriteUtils; @@ -1324,6 +1327,14 @@ public void updateLastCache(final String databaseName) { measurementSchemas); } + @Override + public void checkDataType(AbstractMemTable memTable) throws DataTypeInconsistentException { + IWritableMemChunkGroup writableMemChunkGroup = memTable.getWritableMemChunkGroup(getDeviceID()); + if (writableMemChunkGroup != null) { + writableMemChunkGroup.checkDataType(this); + } + } + @Override public String toString() { return "InsertTabletNode{" diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/RelationalInsertRowNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/RelationalInsertRowNode.java index a824448dc7873..08c783287328c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/RelationalInsertRowNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/RelationalInsertRowNode.java @@ -22,10 +22,13 @@ import org.apache.iotdb.commons.exception.IllegalPathException; import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory; +import org.apache.iotdb.db.exception.DataTypeInconsistentException; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeType; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanVisitor; import org.apache.iotdb.db.queryengine.plan.relational.metadata.fetcher.cache.TableDeviceSchemaCache; +import org.apache.iotdb.db.storageengine.dataregion.memtable.AbstractMemTable; +import org.apache.iotdb.db.storageengine.dataregion.memtable.IWritableMemChunkGroup; import org.apache.iotdb.db.storageengine.dataregion.wal.buffer.IWALByteBufferView; import org.apache.tsfile.enums.TSDataType; @@ -246,4 +249,12 @@ public void updateLastCache(String databaseName) { TableDeviceSchemaCache.getInstance() .updateLastCacheIfExists(databaseName, getDeviceID(), rawMeasurements, timeValuePairs); } + + @Override + public void checkDataType(AbstractMemTable memTable) throws DataTypeInconsistentException { + IWritableMemChunkGroup writableMemChunkGroup = memTable.getWritableMemChunkGroup(getDeviceID()); + if (writableMemChunkGroup != null) { + writableMemChunkGroup.checkDataType(this); + } + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/RelationalInsertRowsNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/RelationalInsertRowsNode.java index 6eeb3d7322fe6..cd53d65fe5a84 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/RelationalInsertRowsNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/RelationalInsertRowsNode.java @@ -22,12 +22,14 @@ import org.apache.iotdb.common.rpc.thrift.TEndPoint; import org.apache.iotdb.common.rpc.thrift.TRegionReplicaSet; import org.apache.iotdb.commons.utils.TimePartitionUtils; +import org.apache.iotdb.db.exception.DataTypeInconsistentException; import org.apache.iotdb.db.queryengine.plan.analyze.IAnalysis; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeType; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanVisitor; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.WritePlanNode; import org.apache.iotdb.db.storageengine.dataregion.IObjectPath; +import org.apache.iotdb.db.storageengine.dataregion.memtable.AbstractMemTable; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.metadata.IDeviceID; @@ -238,4 +240,11 @@ private void handleObjectValue( public RelationalInsertRowsNode emptyClone() { return new RelationalInsertRowsNode(this.getPlanNodeId()); } + + @Override + public void checkDataType(AbstractMemTable memTable) throws DataTypeInconsistentException { + for (InsertRowNode insertRowNode : getInsertRowNodeList()) { + insertRowNode.checkDataType(memTable); + } + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/RelationalInsertTabletNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/RelationalInsertTabletNode.java index 3255c11f1e963..cd2a0734c4f3d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/RelationalInsertTabletNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/RelationalInsertTabletNode.java @@ -25,6 +25,7 @@ import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory; import org.apache.iotdb.commons.utils.TestOnly; +import org.apache.iotdb.db.exception.DataTypeInconsistentException; import org.apache.iotdb.db.exception.query.OutOfTTLException; import org.apache.iotdb.db.queryengine.plan.analyze.IAnalysis; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; @@ -33,6 +34,8 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.WritePlanNode; import org.apache.iotdb.db.queryengine.plan.relational.metadata.fetcher.cache.TableDeviceSchemaCache; import org.apache.iotdb.db.storageengine.dataregion.IObjectPath; +import org.apache.iotdb.db.storageengine.dataregion.memtable.AbstractMemTable; +import org.apache.iotdb.db.storageengine.dataregion.memtable.IWritableMemChunkGroup; import org.apache.iotdb.db.storageengine.dataregion.wal.buffer.IWALByteBufferView; import org.apache.tsfile.enums.TSDataType; @@ -494,4 +497,23 @@ private void handleObjectValue( } } } + + @Override + public void checkDataType(AbstractMemTable memTable) throws DataTypeInconsistentException { + if (singleDevice) { + IWritableMemChunkGroup writableMemChunkGroup = + memTable.getWritableMemChunkGroup(getDeviceID(0)); + if (writableMemChunkGroup != null) { + writableMemChunkGroup.checkDataType(this); + } + } else { + for (int i = 0; i < rowCount; i++) { + IWritableMemChunkGroup writableMemChunkGroup = + memTable.getWritableMemChunkGroup(getDeviceID(i)); + if (writableMemChunkGroup != null) { + writableMemChunkGroup.checkDataType(this); + } + } + } + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/AlterColumnDataType.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/AlterColumnDataType.java new file mode 100644 index 0000000000000..a3e354d592e59 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/AlterColumnDataType.java @@ -0,0 +1,133 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.sql.ast; + +import org.apache.tsfile.utils.RamUsageEstimator; + +import javax.annotation.Nullable; + +import java.util.Collections; +import java.util.List; +import java.util.Objects; + +public class AlterColumnDataType extends Statement { + private static final long INSTANCE_SIZE = + RamUsageEstimator.shallowSizeOfInstance(AlterColumnDataType.class); + private final QualifiedName tableName; + private final Identifier columnName; + private final DataType dataType; + private final boolean ifTableExists; + private final boolean ifColumnExists; + private final boolean view; + + public AlterColumnDataType( + @Nullable NodeLocation location, + QualifiedName tableName, + Identifier columnName, + DataType dataType, + boolean ifTableExists, + boolean ifColumnExists, + boolean view) { + super(location); + this.tableName = tableName; + this.columnName = columnName; + this.dataType = dataType; + this.ifTableExists = ifTableExists; + this.ifColumnExists = ifColumnExists; + this.view = view; + } + + @Override + public List getChildren() { + return Collections.emptyList(); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + AlterColumnDataType that = (AlterColumnDataType) o; + return Objects.equals(tableName, that.tableName) + && Objects.equals(columnName, that.columnName) + && Objects.equals(dataType, that.dataType); + } + + @Override + public int hashCode() { + return Objects.hash(tableName, columnName, dataType); + } + + @Override + public String toString() { + return "AlterColumnDataType{" + + "tableName=" + + tableName + + ", columnName=" + + columnName + + ", dataType=" + + dataType + + ", view=" + + view + + '}'; + } + + @Override + public R accept(AstVisitor visitor, C context) { + return visitor.visitAlterColumnDataType(this, context); + } + + public QualifiedName getTableName() { + return tableName; + } + + public Identifier getColumnName() { + return columnName; + } + + public DataType getDataType() { + return dataType; + } + + public boolean isIfTableExists() { + return ifTableExists; + } + + public boolean isIfColumnExists() { + return ifColumnExists; + } + + public boolean isView() { + return view; + } + + @Override + public long ramBytesUsed() { + long size = INSTANCE_SIZE; + size += AstMemoryEstimationHelper.getEstimatedSizeOfNodeLocation(getLocationInternal()); + size += tableName == null ? 0L : tableName.ramBytesUsed(); + size += AstMemoryEstimationHelper.getEstimatedSizeOfAccountableObject(columnName); + size += AstMemoryEstimationHelper.getEstimatedSizeOfAccountableObject(dataType); + return size; + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/AstVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/AstVisitor.java index 2728750418fe2..4b32201aa8e37 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/AstVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/AstVisitor.java @@ -725,6 +725,10 @@ protected R visitKillQuery(KillQuery node, C context) { return visitStatement(node, context); } + protected R visitAlterColumnDataType(AlterColumnDataType node, C context) { + return visitStatement(node, context); + } + protected R visitRelationalAuthorPlan(RelationalAuthorStatement node, C context) { return visitStatement(node, context); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java index b041c56b2bc97..5ad2b579cb667 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java @@ -39,6 +39,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.AliasedRelation; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.AllColumns; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.AllRows; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.AlterColumnDataType; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.AlterDB; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.AlterPipe; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.AnchorPattern; @@ -560,6 +561,25 @@ public Node visitShowCreateTableStatement( getLocation(ctx), getQualifiedName(ctx.qualifiedName()), false, Boolean.FALSE); } + @Override + public Node visitAlterColumnDataType(RelationalSqlParser.AlterColumnDataTypeContext ctx) { + QualifiedName tableName = getQualifiedName(ctx.tableName); + Identifier columnName = lowerIdentifier((Identifier) visit(ctx.identifier())); + DataType dataType = (DataType) visit(ctx.new_type); + boolean ifTableExists = + ctx.EXISTS().stream() + .anyMatch( + node -> + node.getSymbol().getTokenIndex() < ctx.COLUMN().getSymbol().getTokenIndex()); + boolean ifColumnExists = + ctx.EXISTS().stream() + .anyMatch( + node -> + node.getSymbol().getTokenIndex() > ctx.COLUMN().getSymbol().getTokenIndex()); + return new AlterColumnDataType( + getLocation(ctx), tableName, columnName, dataType, ifTableExists, ifColumnExists, false); + } + @Override public Node visitCommentTable(final RelationalSqlParser.CommentTableContext ctx) { return new SetTableComment( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertRowStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertRowStatement.java index a33af21b1943d..c9b20b9051944 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertRowStatement.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertRowStatement.java @@ -579,7 +579,7 @@ public void rebuildArraysAfterExpansion( @Override protected long calculateBytesUsed() { return INSTANCE_SIZE - + InsertNodeMemoryEstimator.sizeOfValues(values, measurementSchemas) + + InsertNodeMemoryEstimator.sizeOfValues(values, measurementSchemas, dataTypes) + RamUsageEstimator.sizeOf(measurementIsAligned) + InsertNodeMemoryEstimator.sizeOfIDeviceID(deviceID); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertTabletStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertTabletStatement.java index 12369d81bfd30..50ed2f05ec7aa 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertTabletStatement.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertTabletStatement.java @@ -690,7 +690,7 @@ protected long calculateBytesUsed() { return INSTANCE_SIZE + RamUsageEstimator.sizeOf(times) + RamUsageEstimator.sizeOf(nullBitMaps) - + InsertNodeMemoryEstimator.sizeOfColumns(columns, measurementSchemas) + + InsertNodeMemoryEstimator.sizeOfColumns(columns, measurementSchemas, dataTypes) + (Objects.nonNull(deviceIDs) ? Arrays.stream(deviceIDs) .mapToLong(InsertNodeMemoryEstimator::sizeOfIDeviceID) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/AlterTimeSeriesStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/AlterTimeSeriesStatement.java index 5913fb9c2fd9f..027de6250dade 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/AlterTimeSeriesStatement.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/AlterTimeSeriesStatement.java @@ -21,10 +21,14 @@ import org.apache.iotdb.commons.path.MeasurementPath; import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.db.queryengine.plan.analyze.QueryType; +import org.apache.iotdb.db.queryengine.plan.statement.IConfigStatement; import org.apache.iotdb.db.queryengine.plan.statement.Statement; import org.apache.iotdb.db.queryengine.plan.statement.StatementType; import org.apache.iotdb.db.queryengine.plan.statement.StatementVisitor; +import org.apache.tsfile.enums.TSDataType; + import java.util.Collections; import java.util.List; import java.util.Map; @@ -37,7 +41,7 @@ * *

ALTER TIMESERIES path RENAME | SET | DROP | ADD TAGS | ADD ATTRIBUTES | UPSERT */ -public class AlterTimeSeriesStatement extends Statement { +public class AlterTimeSeriesStatement extends Statement implements IConfigStatement { private MeasurementPath path; private AlterTimeSeriesStatement.AlterType alterType; @@ -57,6 +61,8 @@ public class AlterTimeSeriesStatement extends Statement { private final boolean isAlterView; + private TSDataType dataType; + public AlterTimeSeriesStatement() { super(); isAlterView = false; @@ -130,6 +136,14 @@ public boolean isAlterView() { return isAlterView; } + public TSDataType getDataType() { + return dataType; + } + + public void setDataType(TSDataType dataType) { + this.dataType = dataType; + } + @Override public R accept(StatementVisitor visitor, C context) { return visitor.visitAlterTimeSeries(this, context); @@ -164,6 +178,12 @@ public enum AlterType { DROP, ADD_TAGS, ADD_ATTRIBUTES, - UPSERT + UPSERT, + SET_DATA_TYPE + } + + @Override + public QueryType getQueryType() { + return QueryType.WRITE; } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/util/TypeUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/util/TypeUtils.java index e8c679e2bb55f..c3d7d13b5113a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/util/TypeUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/util/TypeUtils.java @@ -36,8 +36,9 @@ public class TypeUtils { public static ColumnBuilder initColumnBuilder(TSDataType type, int count) { switch (type) { case INT32: + return new IntColumnBuilder(null, count, TSDataType.INT32); case DATE: - return new IntColumnBuilder(null, count); + return new IntColumnBuilder(null, count, TSDataType.DATE); case INT64: case TIMESTAMP: return new LongColumnBuilder(null, count); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/ISchemaRegion.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/ISchemaRegion.java index 711e5c8f98704..4a11165353cd6 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/ISchemaRegion.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/ISchemaRegion.java @@ -330,6 +330,16 @@ void setTagsOrAttributesValue(final Map alterMap, final PartialP void renameTagOrAttributeKey(final String oldKey, final String newKey, final PartialPath fullPath) throws MetadataException, IOException; + /** + * Set/change the data type of measurement + * + * @param newDataType the new data type + * @param fullPath timeseries + * @throws MetadataException write error or data type do not exist + */ + void alterTimeSeriesDataType(final TSDataType newDataType, final PartialPath fullPath) + throws MetadataException, IOException; + // endregion // region Interfaces for Template operations diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java index 6c57e0e224845..7ddf1618223ab 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java @@ -1366,6 +1366,22 @@ public void renameTagOrAttributeKey( tagManager.renameTagOrAttributeKey(oldKey, newKey, fullPath, leafMNode); } + /** + * Set/change the data type of measurement + * + * @param newDataType the new data type + * @param fullPath timeseries + * @throws MetadataException write error or data type do not exist + */ + @Override + @SuppressWarnings("squid:S3776") // Suppress high Cognitive Complexity warning + public void alterTimeSeriesDataType(final TSDataType newDataType, final PartialPath fullPath) + throws MetadataException { + final IMeasurementMNode leafMNode = mTree.getMeasurementMNode(fullPath); + mTree.alterTimeSeriesDataType( + leafMNode.getMeasurementPath(), leafMNode.getPartialPath().getMeasurement(), newDataType); + } + /** remove the node from the tag inverted index */ @SuppressWarnings("squid:S3776") // Suppress high Cognitive Complexity warning private void removeFromTagInvertedIndex(final IMeasurementMNode node) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionPBTreeImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionPBTreeImpl.java index a3f870c115827..2f4bec896ddf4 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionPBTreeImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionPBTreeImpl.java @@ -1378,6 +1378,21 @@ public void renameTagOrAttributeKey(String oldKey, String newKey, PartialPath fu } } + /** + * Set/change the data type of measurement + * + * @param newDataType the new data type + * @param fullPath timeseries + * @throws MetadataException write error or data type do not exist + */ + @Override + @SuppressWarnings("squid:S3776") // Suppress high Cognitive Complexity warning + public void alterTimeSeriesDataType(final TSDataType newDataType, final PartialPath fullPath) + throws MetadataException, IOException { + throw new UnsupportedOperationException( + "PBTree does not support altering timeseries data type."); + } + /** remove the node from the tag inverted index */ @SuppressWarnings("squid:S3776") // Suppress high Cognitive Complexity warning private void removeFromTagInvertedIndex(IMeasurementMNode node) throws IOException { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/MTreeBelowSGMemoryImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/MTreeBelowSGMemoryImpl.java index 4021b6fac01bb..9aec147ace1fe 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/MTreeBelowSGMemoryImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/MTreeBelowSGMemoryImpl.java @@ -33,6 +33,7 @@ import org.apache.iotdb.commons.schema.template.Template; import org.apache.iotdb.commons.schema.view.LogicalViewSchema; import org.apache.iotdb.commons.schema.view.viewExpression.ViewExpression; +import org.apache.iotdb.commons.utils.MetadataUtils; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.exception.metadata.AliasAlreadyExistException; import org.apache.iotdb.db.exception.metadata.MNodeTypeMismatchException; @@ -526,6 +527,50 @@ public Map checkMeasurementExistence( return failingMeasurementMap; } + public boolean alterTimeSeriesDataType( + final MeasurementPath measurementPath, final String measurement, TSDataType newDataType) + throws MetadataException { + boolean result = false; + try (final MeasurementUpdater collector = + new MeasurementUpdater( + rootNode, measurementPath, store, false, SchemaConstant.ALL_MATCH_SCOPE) { + @Override + protected void updateMeasurement(final IMeasurementMNode node) + throws MetadataException { + if (node.isLogicalView()) { + throw new MetadataException("View table is not allowed."); + } + if (node.isPreDeleted()) { + throw new MeasurementInBlackListException( + measurementPath.concatAsMeasurementPath(measurement)); + } + if (!MetadataUtils.canAlter( + measurementPath.getMeasurementSchema().getType(), newDataType)) { + throw new MetadataException( + String.format( + "The timeseries %s used new type %s is not compatible with the existing one %s", + measurementPath.getFullPath(), + newDataType, + measurementPath.getMeasurementSchema().getType())); + } + + final IMeasurementSchema schema = node.getSchema(); + node.setSchema( + new MeasurementSchema( + schema.getMeasurementName(), + newDataType, + SchemaUtils.getDataTypeCompatibleEncoding( + newDataType, schema.getEncodingType()), + schema.getCompressor(), + schema.getProps())); + } + }) { + collector.traverse(); + result = true; + } + return result; + } + public boolean changeAlias(final String alias, final PartialPath fullPath) throws MetadataException { final IMeasurementMNode measurementMNode = getMeasurementMNode(fullPath); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/mnode/info/LogicalViewInfo.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/mnode/info/LogicalViewInfo.java index a5ab55796e439..f96537aaf84ce 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/mnode/info/LogicalViewInfo.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/mnode/info/LogicalViewInfo.java @@ -45,6 +45,9 @@ public class LogicalViewInfo implements IMeasurementInfo { /** whether this measurement is pre deleted and considered in black list */ private boolean preDeleted = false; + /** whether this measurement is pre altered */ + private boolean preAltered = false; + private LogicalViewSchema schema; public LogicalViewInfo(LogicalViewSchema schema) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/mnode/info/MeasurementInfo.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/mnode/info/MeasurementInfo.java index 19d1855db0cb1..2d120698022b6 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/mnode/info/MeasurementInfo.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/mnode/info/MeasurementInfo.java @@ -38,6 +38,9 @@ public class MeasurementInfo implements IMeasurementInfo { /** whether this measurement is pre deleted and considered in black list */ private boolean preDeleted = false; + /** whether this measurement is pre altered */ + private boolean preAltered = false; + // alias length, hashCode and occupation in aliasMap, 4 + 4 + 44 = 52B private static final int ALIAS_BASE_SIZE = 52; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/utils/ResourceByPathUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/utils/ResourceByPathUtils.java index d4d84c6366013..188c4ec652984 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/utils/ResourceByPathUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/utils/ResourceByPathUtils.java @@ -37,6 +37,7 @@ import org.apache.iotdb.db.storageengine.dataregion.modification.ModEntry; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; import org.apache.iotdb.db.utils.ModificationUtils; +import org.apache.iotdb.db.utils.SchemaUtils; import org.apache.iotdb.db.utils.datastructure.TVList; import org.apache.tsfile.enums.TSDataType; @@ -259,11 +260,23 @@ public AbstractAlignedTimeSeriesMetadata generateTimeSeriesMetadata( boolean[] exist = new boolean[alignedFullPath.getSchemaList().size()]; boolean modified = false; boolean isTable = false; + int index = 0; for (IChunkMetadata chunkMetadata : chunkMetadataList) { AbstractAlignedChunkMetadata alignedChunkMetadata = (AbstractAlignedChunkMetadata) chunkMetadata; isTable = isTable || (alignedChunkMetadata instanceof TableDeviceChunkMetadata); modified = (modified || alignedChunkMetadata.isModified()); + TSDataType targetDataType = alignedFullPath.getSchemaList().get(index).getType(); + if (targetDataType.equals(TSDataType.STRING) + && (alignedChunkMetadata.getValueChunkMetadataList().stream() + .filter(iChunkMetadata -> iChunkMetadata.getDataType() != targetDataType) + .count() + > 0)) { + // create new statistics object via new data type, and merge statistics information + alignedChunkMetadata = + SchemaUtils.rewriteAlignedChunkMetadataStatistics(alignedChunkMetadata, targetDataType); + alignedChunkMetadata.setModified(true); + } if (!useFakeStatistics) { timeStatistics.mergeStatistics(alignedChunkMetadata.getTimeChunkMetadata().getStatistics()); for (int i = 0; i < valueTimeSeriesMetadataList.size(); i++) { @@ -276,10 +289,12 @@ public AbstractAlignedTimeSeriesMetadata generateTimeSeriesMetadata( alignedChunkMetadata.getValueChunkMetadataList().get(i).getStatistics()); } } + index++; continue; } startTime = Math.min(startTime, chunkMetadata.getStartTime()); endTime = Math.max(endTime, chunkMetadata.getEndTime()); + index++; } for (ReadOnlyMemChunk memChunk : readOnlyMemChunk) { @@ -575,7 +590,11 @@ public ReadOnlyMemChunk getReadOnlyMemChunkFromMemTable( IWritableMemChunk memChunk = memTableMap.get(deviceID).getMemChunkMap().get(fullPath.getMeasurement()); // check If data type matches - if (memChunk.getSchema().getType() != fullPath.getMeasurementSchema().getType()) { + if (memChunk.getSchema().getType() != fullPath.getMeasurementSchema().getType() + && !fullPath + .getMeasurementSchema() + .getType() + .isCompatible(memChunk.getSchema().getType())) { return null; } // prepare TVList for query. It should clone TVList if necessary. diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java index cc37f15d7a861..e0b9ab91cd9ba 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java @@ -53,6 +53,7 @@ import org.apache.iotdb.db.consensus.DataRegionConsensusImpl; import org.apache.iotdb.db.exception.BatchProcessException; import org.apache.iotdb.db.exception.DataRegionException; +import org.apache.iotdb.db.exception.DataTypeInconsistentException; import org.apache.iotdb.db.exception.DiskSpaceInsufficientException; import org.apache.iotdb.db.exception.TsFileProcessorException; import org.apache.iotdb.db.exception.WriteProcessException; @@ -1171,8 +1172,28 @@ public void insert(InsertRowNode insertRowNode) throws WriteProcessException { > lastFlushTimeMap.getFlushedTime(timePartitionId, insertRowNode.getDeviceID()); // insert to sequence or unSequence file - TsFileProcessor tsFileProcessor = - insertToTsFileProcessor(insertRowNode, isSequence, timePartitionId); + TsFileProcessor tsFileProcessor; + try { + tsFileProcessor = insertToTsFileProcessor(insertRowNode, isSequence, timePartitionId); + } catch (DataTypeInconsistentException e) { + // flush both MemTables so that the new type can be inserted into a new MemTable + TsFileProcessor workSequenceProcessor = workSequenceTsFileProcessors.get(timePartitionId); + if (workSequenceProcessor != null) { + fileFlushPolicy.apply(this, workSequenceProcessor, workSequenceProcessor.isSequence()); + } + TsFileProcessor workUnsequenceProcessor = + workUnsequenceTsFileProcessors.get(timePartitionId); + if (workUnsequenceProcessor != null) { + fileFlushPolicy.apply( + this, workUnsequenceProcessor, workUnsequenceProcessor.isSequence()); + } + + isSequence = + config.isEnableSeparateData() + && insertRowNode.getTime() + > lastFlushTimeMap.getFlushedTime(timePartitionId, insertRowNode.getDeviceID()); + tsFileProcessor = insertToTsFileProcessor(insertRowNode, isSequence, timePartitionId); + } // check memtable size and may asyncTryToFlush the work memtable if (tsFileProcessor != null && tsFileProcessor.shouldFlush()) { @@ -1272,7 +1293,8 @@ private boolean doInsert( InsertTabletNode insertTabletNode, Map[]> splitMap, TSStatus[] results, - long[] infoForMetrics) { + long[] infoForMetrics) + throws DataTypeInconsistentException { boolean noFailure = true; for (Entry[]> entry : splitMap.entrySet()) { long timePartitionId = entry.getKey(); @@ -1344,6 +1366,39 @@ public void insertTablet(InsertTabletNode insertTabletNode) } } + private boolean splitAndInsert( + int start, + InsertTabletNode insertTabletNode, + TSStatus[] results, + long[] infoForMetrics, + List> deviceEndOffsetPairs) { + final int initialStart = start; + try { + Map[]> splitInfo = new HashMap<>(); + for (Pair deviceEndOffsetPair : deviceEndOffsetPairs) { + int end = deviceEndOffsetPair.getRight(); + split(insertTabletNode, start, end, splitInfo); + start = end; + } + return doInsert(insertTabletNode, splitInfo, results, infoForMetrics); + } catch (DataTypeInconsistentException e) { + // the exception will trigger a flush, which requires the flush time to be recalculated + start = initialStart; + Map[]> splitInfo = new HashMap<>(); + for (Pair deviceEndOffsetPair : deviceEndOffsetPairs) { + int end = deviceEndOffsetPair.getRight(); + split(insertTabletNode, start, end, splitInfo); + start = end; + } + try { + return doInsert(insertTabletNode, splitInfo, results, infoForMetrics); + } catch (DataTypeInconsistentException ex) { + logger.error("Data inconsistent exception is not supposed to be triggered twice", ex); + return false; + } + } + } + private boolean executeInsertTablet( InsertTabletNode insertTabletNode, TSStatus[] results, long[] infoForMetrics) throws OutOfTTLException { @@ -1355,14 +1410,9 @@ private boolean executeInsertTablet( noFailure = loc == 0; List> deviceEndOffsetPairs = insertTabletNode.splitByDevice(loc, insertTabletNode.getRowCount()); - int start = loc; - Map[]> splitInfo = new HashMap<>(); - for (Pair deviceEndOffsetPair : deviceEndOffsetPairs) { - int end = deviceEndOffsetPair.getRight(); - split(insertTabletNode, start, end, splitInfo); - start = end; - } - noFailure = doInsert(insertTabletNode, splitInfo, results, infoForMetrics) && noFailure; + noFailure = + splitAndInsert(loc, insertTabletNode, results, infoForMetrics, deviceEndOffsetPairs) + && noFailure; if (CommonDescriptor.getInstance().getConfig().isLastCacheEnable() && !insertTabletNode.isGeneratedByRemoteConsensusLeader()) { @@ -1407,7 +1457,8 @@ private boolean insertTabletToTsFileProcessor( TSStatus[] results, long timePartitionId, boolean noFailure, - long[] infoForMetrics) { + long[] infoForMetrics) + throws DataTypeInconsistentException { if (insertTabletNode.allMeasurementFailed()) { if (logger.isDebugEnabled()) { logger.debug( @@ -1433,11 +1484,21 @@ private boolean insertTabletToTsFileProcessor( return false; } - // register TableSchema (and maybe more) for table insertion - registerToTsFile(insertTabletNode, tsFileProcessor); - try { + // register TableSchema (and maybe more) for table insertion + registerToTsFile(insertTabletNode, tsFileProcessor); tsFileProcessor.insertTablet(insertTabletNode, rangeList, results, noFailure, infoForMetrics); + } catch (DataTypeInconsistentException e) { + // flush both MemTables so that the new type can be inserted into a new MemTable + TsFileProcessor workSequenceProcessor = workSequenceTsFileProcessors.get(timePartitionId); + if (workSequenceProcessor != null) { + fileFlushPolicy.apply(this, workSequenceProcessor, workSequenceProcessor.isSequence()); + } + TsFileProcessor workUnsequenceProcessor = workUnsequenceTsFileProcessors.get(timePartitionId); + if (workUnsequenceProcessor != null) { + fileFlushPolicy.apply(this, workUnsequenceProcessor, workUnsequenceProcessor.isSequence()); + } + throw e; } catch (WriteProcessRejectException e) { logger.warn("insert to TsFileProcessor rejected, {}", e.getMessage()); return false; @@ -1511,6 +1572,18 @@ public int hashCode() { } } + private TsFileProcessor insertTabletWithTypeConsistencyCheck( + TsFileProcessor tsFileProcessor, + InsertTabletNode insertTabletNode, + List rangeList, + TSStatus[] results, + boolean noFailure, + long[] infoForMetrics) + throws WriteProcessException { + + return tsFileProcessor; + } + private void registerToTsFile(InsertNode node, TsFileProcessor tsFileProcessor) { final String tableName = node.getTableName(); if (tableName != null) { @@ -1643,7 +1716,8 @@ private List insertToTsFileProcessors( TsFileProcessor tsFileProcessor = entry.getKey(); InsertRowsNode subInsertRowsNode = entry.getValue(); try { - tsFileProcessor.insertRows(subInsertRowsNode, infoForMetrics); + tsFileProcessor = + insertRowsWithTypeConsistencyCheck(tsFileProcessor, subInsertRowsNode, infoForMetrics); } catch (WriteProcessException e) { insertRowsNode .getResults() @@ -1652,8 +1726,7 @@ private List insertToTsFileProcessors( RpcUtils.getStatus(e.getErrorCode(), e.getMessage())); } executedInsertRowNodeList.addAll(subInsertRowsNode.getInsertRowNodeList()); - // register TableSchema (and maybe more) for table insertion - registerToTsFile(subInsertRowsNode, tsFileProcessor); + // check memtable size and may asyncTryToFlush the work memtable if (entry.getKey().shouldFlush()) { fileFlushPolicy.apply(this, tsFileProcessor, tsFileProcessor.isSequence()); @@ -1662,6 +1735,37 @@ private List insertToTsFileProcessors( return executedInsertRowNodeList; } + private TsFileProcessor insertRowsWithTypeConsistencyCheck( + TsFileProcessor tsFileProcessor, InsertRowsNode subInsertRowsNode, long[] infoForMetrics) + throws WriteProcessException { + try { + // register TableSchema (and maybe more) for table insertion + registerToTsFile(subInsertRowsNode, tsFileProcessor); + tsFileProcessor.insertRows(subInsertRowsNode, infoForMetrics); + } catch (DataTypeInconsistentException e) { + InsertRowNode firstRow = subInsertRowsNode.getInsertRowNodeList().get(0); + long timePartitionId = TimePartitionUtils.getTimePartitionId(firstRow.getTime()); + // flush both MemTables so that the new type can be inserted into a new MemTable + TsFileProcessor workSequenceProcessor = workSequenceTsFileProcessors.get(timePartitionId); + if (workSequenceProcessor != null) { + fileFlushPolicy.apply(this, workSequenceProcessor, workSequenceProcessor.isSequence()); + } + TsFileProcessor workUnsequenceProcessor = workUnsequenceTsFileProcessors.get(timePartitionId); + if (workUnsequenceProcessor != null) { + fileFlushPolicy.apply(this, workUnsequenceProcessor, workUnsequenceProcessor.isSequence()); + } + + boolean isSequence = + config.isEnableSeparateData() + && firstRow.getTime() + > lastFlushTimeMap.getFlushedTime(timePartitionId, firstRow.getDeviceID()); + tsFileProcessor = getOrCreateTsFileProcessor(timePartitionId, isSequence); + registerToTsFile(subInsertRowsNode, tsFileProcessor); + tsFileProcessor.insertRows(subInsertRowsNode, infoForMetrics); + } + return tsFileProcessor; + } + private void tryToUpdateInsertRowsLastCache(List nodeList) { for (InsertRowNode node : nodeList) { node.updateLastCache(databaseName); @@ -3130,6 +3234,19 @@ private void deleteDataInSealedFiles(Collection sealedTsFiles, M Set involvedModificationFiles = new HashSet<>(); List deletedByMods = new ArrayList<>(); List deletedByFiles = new ArrayList<>(); + boolean isDropMeasurementExist = false; + boolean isDropTagExist = false; + + if (deletion instanceof TableDeletionEntry) { + TableDeletionEntry entry = (TableDeletionEntry) deletion; + isDropMeasurementExist = !entry.getPredicate().getMeasurementNames().isEmpty(); + } else { + TreeDeletionEntry entry = (TreeDeletionEntry) deletion; + if (entry.getPathPattern() instanceof MeasurementPath) { + Map tagMap = ((MeasurementPath) entry.getPathPattern()).getTagMap(); + isDropTagExist = (tagMap != null) && !tagMap.isEmpty(); + } + } for (TsFileResource sealedTsFile : sealedTsFiles) { if (canSkipDelete(sealedTsFile, deletion)) { continue; @@ -3226,7 +3343,7 @@ private void deleteDataInSealedFiles(Collection sealedTsFiles, M } // else do nothing } - if (!deletedByFiles.isEmpty()) { + if (!deletedByFiles.isEmpty() && !isDropMeasurementExist && !isDropTagExist) { deleteTsFileCompletely(deletedByFiles); if (logger.isDebugEnabled()) { logger.debug( @@ -4388,7 +4505,9 @@ public void insert(InsertRowsOfOneDeviceNode insertRowsOfOneDeviceNode) TsFileProcessor tsFileProcessor = entry.getKey(); InsertRowsNode subInsertRowsNode = entry.getValue(); try { - tsFileProcessor.insertRows(subInsertRowsNode, infoForMetrics); + tsFileProcessor = + insertRowsWithTypeConsistencyCheck( + tsFileProcessor, subInsertRowsNode, infoForMetrics); } catch (WriteProcessException e) { insertRowsOfOneDeviceNode .getResults() diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/impl/FastCompactionPerformer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/impl/FastCompactionPerformer.java index 3a5695a737634..a0b9f0713a8a2 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/impl/FastCompactionPerformer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/impl/FastCompactionPerformer.java @@ -33,6 +33,7 @@ import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.task.CompactionTaskSummary; import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.task.subtask.FastCompactionPerformerSubTask; import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.task.subtask.FastCompactionTaskSummary; +import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.CompactionSeriesContext; import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.CompactionTableSchemaCollector; import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.CompactionUtils; import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.MultiTsFileDeviceIterator; @@ -50,6 +51,7 @@ import org.apache.tsfile.common.conf.TSFileDescriptor; import org.apache.tsfile.encrypt.EncryptParameter; +import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.exception.StopReadTsFileByInterruptException; import org.apache.tsfile.exception.write.PageException; import org.apache.tsfile.file.metadata.IDeviceID; @@ -190,6 +192,13 @@ public void perform() throws Exception { ttlDeletion = CompactionUtils.convertTtlToDeletion( device, deviceIterator.getTimeLowerBoundForCurrentDevice()); + for (TsFileResource sourceFile : sortedSourceFiles) { + modificationCache + .computeIfAbsent( + sourceFile.getTsFile().getName(), + k -> PatternTreeMapFactory.getModsPatternTreeMap()) + .append(ttlDeletion.keyOfPatternTree(), ttlDeletion); + } } compactionWriter.setTTLDeletion(ttlDeletion); @@ -279,7 +288,18 @@ private void compactNonAlignedSeries( // offset later. Here we don't need to deserialize chunk metadata, we can deserialize them and // get their schema later. Map>> timeseriesMetadataOffsetMap = - deviceIterator.getTimeseriesMetadataOffsetOfCurrentDevice(); + new LinkedHashMap<>(); + + Map measurementDataTypeMap = new LinkedHashMap<>(); + + Map compactionSeriesContextMap = + deviceIterator.getCompactionSeriesContextOfCurrentDevice(); + + for (Map.Entry entry : compactionSeriesContextMap.entrySet()) { + timeseriesMetadataOffsetMap.put( + entry.getKey(), entry.getValue().getFileTimeseriesMetdataOffsetMap()); + measurementDataTypeMap.put(entry.getKey(), entry.getValue().getFinalType()); + } List allMeasurements = new ArrayList<>(timeseriesMetadataOffsetMap.keySet()); allMeasurements.sort((String::compareTo)); @@ -304,8 +324,8 @@ private void compactNonAlignedSeries( CompactionTaskManager.getInstance() .submitSubTask( new FastCompactionPerformerSubTask( + compactionSeriesContextMap, fastCrossCompactionWriter, - timeseriesMetadataOffsetMap, readerCacheMap, modificationCache, sortedSourceFiles, diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/impl/ReadChunkCompactionPerformer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/impl/ReadChunkCompactionPerformer.java index eaed8a7f6ebde..d406286e37f64 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/impl/ReadChunkCompactionPerformer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/impl/ReadChunkCompactionPerformer.java @@ -21,6 +21,7 @@ import org.apache.iotdb.commons.exception.IllegalPathException; import org.apache.iotdb.commons.exception.MetadataException; +import org.apache.iotdb.commons.utils.MetadataUtils; import org.apache.iotdb.commons.utils.TestOnly; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.exception.StorageEngineException; @@ -331,10 +332,14 @@ private void compactNotAlignedSeries( readerAndChunkMetadataList) { boolean dataTypeConsistent = true; for (ChunkMetadata chunkMetadata : tsFileSequenceReaderListPair.getRight()) { - if (chunkMetadata != null && chunkMetadata.getDataType() != correctDataType) { + if (chunkMetadata != null + && !MetadataUtils.canAlter(chunkMetadata.getDataType(), correctDataType)) { dataTypeConsistent = false; break; } + if (chunkMetadata != null && chunkMetadata.getDataType() != correctDataType) { + chunkMetadata.setNewType(correctDataType); + } } if (!dataTypeConsistent) { continue; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/subtask/FastCompactionPerformerSubTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/subtask/FastCompactionPerformerSubTask.java index afd82268c5dd2..873993d97df4c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/subtask/FastCompactionPerformerSubTask.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/subtask/FastCompactionPerformerSubTask.java @@ -23,6 +23,7 @@ import org.apache.iotdb.commons.path.PatternTreeMap; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.exception.WriteProcessException; +import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.CompactionSeriesContext; import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.executor.batch.BatchedFastAlignedSeriesCompactionExecutor; import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.executor.fast.FastAlignedSeriesCompactionExecutor; import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.executor.fast.FastNonAlignedSeriesCompactionExecutor; @@ -75,6 +76,8 @@ public class FastCompactionPerformerSubTask implements Callable { private List measurementSchemas; + private Map compactionSeriesContextMap; + /** Used for nonAligned timeseries. */ @SuppressWarnings("squid:S107") public FastCompactionPerformerSubTask( @@ -101,6 +104,31 @@ public FastCompactionPerformerSubTask( this.ignoreAllNullRows = true; } + public FastCompactionPerformerSubTask( + Map compactionSeriesContextMap, + AbstractCompactionWriter compactionWriter, + Map readerCacheMap, + Map> + modificationCacheMap, + List sortedSourceFiles, + List measurements, + IDeviceID deviceId, + FastCompactionTaskSummary summary, + int subTaskId) { + this.compactionWriter = compactionWriter; + this.subTaskId = subTaskId; + this.compactionSeriesContextMap = compactionSeriesContextMap; + this.timeseriesMetadataOffsetMap = null; + this.isAligned = false; + this.deviceId = deviceId; + this.readerCacheMap = readerCacheMap; + this.modificationCacheMap = modificationCacheMap; + this.sortedSourceFiles = sortedSourceFiles; + this.measurements = measurements; + this.summary = summary; + this.ignoreAllNullRows = true; + } + /** Used for aligned timeseries. */ public FastCompactionPerformerSubTask( AbstractCompactionWriter compactionWriter, @@ -140,7 +168,10 @@ public Void call() subTaskId, summary); for (String measurement : measurements) { - seriesCompactionExecutor.setNewMeasurement(timeseriesMetadataOffsetMap.get(measurement)); + seriesCompactionExecutor.setNewMeasurement( + compactionSeriesContextMap.get(measurement).getFileTimeseriesMetdataOffsetMap()); + seriesCompactionExecutor.setType( + compactionSeriesContextMap.get(measurement).getFinalType()); seriesCompactionExecutor.execute(); } } else { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/CompactionSeriesContext.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/CompactionSeriesContext.java new file mode 100644 index 0000000000000..8f6d017f2354a --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/CompactionSeriesContext.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils; + +import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; + +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.utils.Pair; + +import java.util.HashMap; +import java.util.Map; + +public class CompactionSeriesContext { + Map> fileTimeseriesMetdataOffsetMap; + TSDataType finalType; + + public CompactionSeriesContext() { + fileTimeseriesMetdataOffsetMap = new HashMap<>(); + } + + public CompactionSeriesContext( + Map> fileTimeseriesMetdataOffsetMap, TSDataType finalType) { + this.fileTimeseriesMetdataOffsetMap = fileTimeseriesMetdataOffsetMap; + this.finalType = finalType; + } + + public Map> getFileTimeseriesMetdataOffsetMap() { + return fileTimeseriesMetdataOffsetMap; + } + + public void put(TsFileResource tsFileResource, Pair pair) { + fileTimeseriesMetdataOffsetMap.put(tsFileResource, pair); + } + + public TSDataType getFinalType() { + return finalType; + } + + public void setFinalTypeIfAbsent(TSDataType finalType) { + if (this.finalType == null) { + this.finalType = finalType; + } + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/MultiTsFileDeviceIterator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/MultiTsFileDeviceIterator.java index b9b7f114e3eeb..9099586e87e5c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/MultiTsFileDeviceIterator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/MultiTsFileDeviceIterator.java @@ -25,6 +25,7 @@ import org.apache.iotdb.commons.path.MeasurementPath; import org.apache.iotdb.commons.path.PatternTreeMap; import org.apache.iotdb.commons.utils.CommonDateTimeUtils; +import org.apache.iotdb.commons.utils.MetadataUtils; import org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.DataNodeTTLCache; import org.apache.iotdb.db.storageengine.dataregion.compaction.io.CompactionTsFileReader; import org.apache.iotdb.db.storageengine.dataregion.compaction.schedule.constant.CompactionType; @@ -322,7 +323,8 @@ public Map getAllSchemasOfCurrentDevice() throws IOEx TSDataType correctDataTypeOfCurrentMeasurement = measurementDataTypeMap.putIfAbsent(measurementId, dataTypeOfCurrentTimeseriesMetadata); if (correctDataTypeOfCurrentMeasurement != null - && correctDataTypeOfCurrentMeasurement != dataTypeOfCurrentTimeseriesMetadata) { + && !MetadataUtils.canAlter( + dataTypeOfCurrentTimeseriesMetadata, correctDataTypeOfCurrentMeasurement)) { continue; } timeseriesMetadataOffsetMap.putIfAbsent(measurementId, new HashMap<>()); @@ -332,6 +334,75 @@ public Map getAllSchemasOfCurrentDevice() throws IOEx return timeseriesMetadataOffsetMap; } + /** + * Get all measurement data types of the current device from source files. Traverse all the files + * from the newest to the oldest in turn and start traversing the index tree from the + * firstMeasurementNode node to get all the measurement types under the current device. + * + * @return measurement -> data type + * @throws IOException if io errors occurred + */ + public Map getDataTypeOfCurrentDevice() throws IOException { + Map measurementNameDataTypeMap = new HashMap<>(); + for (TsFileResource resource : tsFileResourcesSortedByDesc) { + if (!deviceIteratorMap.containsKey(resource) + || !deviceIteratorMap.get(resource).current().equals(currentDevice)) { + // if this tsfile has no more device or next device is not equals to the current device, + // which means this tsfile does not contain the current device, then skip it. + continue; + } + TsFileSequenceReader reader = readerMap.get(resource); + for (Map.Entry>> entrySet : + ((CompactionTsFileReader) reader) + .getTimeseriesMetadataAndOffsetByDevice( + deviceIteratorMap.get(resource).getFirstMeasurementNodeOfCurrentDevice(), + Collections.emptySet(), + false) + .entrySet()) { + String measurementId = entrySet.getKey(); + TSDataType dataType = entrySet.getValue().left.getTsDataType(); + measurementNameDataTypeMap.putIfAbsent(measurementId, dataType); + } + } + return measurementNameDataTypeMap; + } + + public Map getCompactionSeriesContextOfCurrentDevice() + throws IOException { + Map compactionSeriesContextMap = new HashMap<>(); + for (TsFileResource resource : tsFileResourcesSortedByDesc) { + if (!deviceIteratorMap.containsKey(resource) + || !deviceIteratorMap.get(resource).current().equals(currentDevice)) { + // if this tsfile has no more device or next device is not equals to the current device, + // which means this tsfile does not contain the current device, then skip it. + continue; + } + TsFileSequenceReader reader = readerMap.get(resource); + for (Map.Entry>> entrySet : + ((CompactionTsFileReader) reader) + .getTimeseriesMetadataAndOffsetByDevice( + deviceIteratorMap.get(resource).getFirstMeasurementNodeOfCurrentDevice(), + Collections.emptySet(), + false) + .entrySet()) { + String measurementId = entrySet.getKey(); + TimeseriesMetadata timeseriesMetadata = entrySet.getValue().left; + Pair offset = entrySet.getValue().right; + TSDataType dataType = entrySet.getValue().left.getTsDataType(); + if (compactionSeriesContextMap.get(measurementId) != null + && compactionSeriesContextMap.get(measurementId).getFinalType() != null + && !MetadataUtils.canAlter( + dataType, compactionSeriesContextMap.get(measurementId).getFinalType())) { + continue; + } + compactionSeriesContextMap.putIfAbsent(measurementId, new CompactionSeriesContext()); + compactionSeriesContextMap.get(measurementId).put(resource, entrySet.getValue().right); + compactionSeriesContextMap.get(measurementId).setFinalTypeIfAbsent(dataType); + } + } + return compactionSeriesContextMap; + } + /** * Get all measurements and their schemas of the current device and the timeseries metadata offset * of each timeseries in each source file. It is used for new fast compaction to compact aligned diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/FastAlignedSeriesCompactionExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/FastAlignedSeriesCompactionExecutor.java index c2d3524697d78..121d4ca1d3a9d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/FastAlignedSeriesCompactionExecutor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/FastAlignedSeriesCompactionExecutor.java @@ -22,6 +22,7 @@ import org.apache.iotdb.commons.exception.IllegalPathException; import org.apache.iotdb.commons.path.AlignedPath; import org.apache.iotdb.commons.path.PatternTreeMap; +import org.apache.iotdb.commons.utils.MetadataUtils; import org.apache.iotdb.db.exception.WriteProcessException; import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.task.subtask.FastCompactionTaskSummary; import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.executor.ModifiedStatus; @@ -50,6 +51,7 @@ import org.apache.tsfile.file.metadata.IChunkMetadata; import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.file.metadata.TableDeviceChunkMetadata; +import org.apache.tsfile.file.metadata.statistics.Statistics; import org.apache.tsfile.read.TsFileSequenceReader; import org.apache.tsfile.read.common.Chunk; import org.apache.tsfile.utils.Pair; @@ -274,15 +276,21 @@ protected List getAlignedChunkMetadataList(TsFileR private boolean isValueChunkDataTypeMatchSchema( List chunkMetadataListOfOneValueColumn) { + boolean isMatch = false; for (IChunkMetadata chunkMetadata : chunkMetadataListOfOneValueColumn) { if (chunkMetadata == null) { continue; } String measurement = chunkMetadata.getMeasurementUid(); IMeasurementSchema schema = measurementSchemaMap.get(measurement); - return schema.getType() == chunkMetadata.getDataType(); + if (MetadataUtils.canAlter(chunkMetadata.getDataType(), schema.getType())) { + if (schema.getType() != chunkMetadata.getDataType()) { + chunkMetadata.setNewType(schema.getType()); + } + isMatch = true; + } } - return true; + return isMatch; } /** @@ -363,7 +371,21 @@ void readChunk(ChunkMetadataElement chunkMetadataElement) throws IOException { valueChunks.add(null); continue; } - valueChunks.add(readChunk(reader, (ChunkMetadata) valueChunkMetadata)); + if (valueChunkMetadata.getNewType() != null) { + Chunk chunk = + readChunk(reader, (ChunkMetadata) valueChunkMetadata) + .rewrite( + ((ChunkMetadata) valueChunkMetadata).getNewType(), chunkMetadataElement.chunk); + valueChunks.add(chunk); + + ChunkMetadata chunkMetadata = (ChunkMetadata) valueChunkMetadata; + chunkMetadata.setTsDataType(valueChunkMetadata.getNewType()); + Statistics statistics = Statistics.getStatsByType(valueChunkMetadata.getNewType()); + statistics.mergeStatistics(chunk.getChunkStatistic()); + chunkMetadata.setStatistics(statistics); + } else { + valueChunks.add(readChunk(reader, (ChunkMetadata) valueChunkMetadata)); + } } chunkMetadataElement.valueChunks = valueChunks; setForceDecoding(chunkMetadataElement); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/FastNonAlignedSeriesCompactionExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/FastNonAlignedSeriesCompactionExecutor.java index 0a77bf16ea7a7..697376c233aa8 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/FastNonAlignedSeriesCompactionExecutor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/FastNonAlignedSeriesCompactionExecutor.java @@ -36,6 +36,7 @@ import org.apache.iotdb.db.utils.datastructure.PatternTreeMapFactory; import org.apache.tsfile.encrypt.EncryptUtils; +import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.exception.write.PageException; import org.apache.tsfile.file.header.ChunkHeader; import org.apache.tsfile.file.header.PageHeader; @@ -44,6 +45,7 @@ import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.file.metadata.enums.CompressionType; import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.file.metadata.statistics.Statistics; import org.apache.tsfile.read.TsFileSequenceReader; import org.apache.tsfile.read.common.Chunk; import org.apache.tsfile.utils.Pair; @@ -67,6 +69,9 @@ public class FastNonAlignedSeriesCompactionExecutor extends SeriesCompactionExec // used to get the chunk metadatas from tsfile directly according to timeseries metadata offset. private Map> timeseriesMetadataOffsetMap; + // the data type of the current series + private TSDataType dataType; + // it is used to initialize the fileList when compacting a new series private final List sortResources; @@ -107,6 +112,10 @@ public void setNewMeasurement(Map> timeseriesMe hasStartMeasurement = false; } + public void setType(TSDataType dataType) { + this.dataType = dataType; + } + @Override protected void compactFiles() throws PageException, IOException, WriteProcessException, IllegalPathException { @@ -154,10 +163,11 @@ void deserializeFileIntoChunkMetadataQueue(List fileElements) removeFile(fileElement); } } - for (int i = 0; i < iChunkMetadataList.size(); i++) { IChunkMetadata chunkMetadata = iChunkMetadataList.get(i); - + if (dataType != null && chunkMetadata.getDataType() != dataType) { + chunkMetadata.setNewType(dataType); + } // add into queue chunkMetadataQueue.add( new ChunkMetadataElement( @@ -192,11 +202,25 @@ protected void deserializeChunkIntoPageQueue(ChunkMetadataElement chunkMetadataE @Override void readChunk(ChunkMetadataElement chunkMetadataElement) throws IOException { updateSummary(chunkMetadataElement, ChunkStatus.READ_IN); - chunkMetadataElement.chunk = - readerCacheMap - .get(chunkMetadataElement.fileElement.resource) - .readMemChunk((ChunkMetadata) chunkMetadataElement.chunkMetadata); + if (chunkMetadataElement.chunkMetadata.getNewType() != null) { + chunkMetadataElement.chunk = + readerCacheMap + .get(chunkMetadataElement.fileElement.resource) + .readMemChunk((ChunkMetadata) chunkMetadataElement.chunkMetadata) + .rewrite(((ChunkMetadata) chunkMetadataElement.chunkMetadata).getNewType()); + ChunkMetadata chunkMetadata = (ChunkMetadata) chunkMetadataElement.chunkMetadata; + chunkMetadata.setTsDataType(chunkMetadataElement.chunkMetadata.getNewType()); + Statistics statistics = Statistics.getStatsByType(chunkMetadata.getNewType()); + statistics.mergeStatistics(chunkMetadataElement.chunk.getChunkStatistic()); + chunkMetadata.setStatistics(statistics); + chunkMetadataElement.chunkMetadata = chunkMetadata; + } else { + chunkMetadataElement.chunk = + readerCacheMap + .get(chunkMetadataElement.fileElement.resource) + .readMemChunk((ChunkMetadata) chunkMetadataElement.chunkMetadata); + } if (!hasStartMeasurement) { // for nonAligned sensors, only after getting chunkMetadatas can we create metadata to // start diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/readchunk/ReadChunkAlignedSeriesCompactionExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/readchunk/ReadChunkAlignedSeriesCompactionExecutor.java index c2e0959e65234..654f8f770e74f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/readchunk/ReadChunkAlignedSeriesCompactionExecutor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/readchunk/ReadChunkAlignedSeriesCompactionExecutor.java @@ -19,6 +19,7 @@ package org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.executor.readchunk; +import org.apache.iotdb.commons.utils.MetadataUtils; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.exception.CompactionLastTimeCheckFailedException; import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.task.CompactionTaskSummary; @@ -230,12 +231,19 @@ private void compactWithAlignedChunk( for (int i = 0; i < alignedChunkMetadata.getValueChunkMetadataList().size(); i++) { IChunkMetadata chunkMetadata = alignedChunkMetadata.getValueChunkMetadataList().get(i); if (chunkMetadata == null - || !chunkMetadata.getDataType().equals(schemaList.get(i).getType())) { + || !MetadataUtils.canAlter(chunkMetadata.getDataType(), schemaList.get(i).getType())) { valueChunks.add(getChunkLoader(reader, null)); continue; } + if (chunkMetadata != null && chunkMetadata.getDataType() != schemaList.get(i).getType()) { + chunkMetadata.setNewType(schemaList.get(i).getType()); + valueChunks.add( + getRewriteValueChunkLoader( + reader, (ChunkMetadata) chunkMetadata, timeChunk.getChunk())); + } else { + valueChunks.add(getChunkLoader(reader, (ChunkMetadata) chunkMetadata)); + } pointNum += chunkMetadata.getStatistics().getCount(); - valueChunks.add(getChunkLoader(reader, (ChunkMetadata) chunkMetadata)); } summary.increaseProcessPointNum(pointNum); if (flushController.canFlushCurrentChunkWriter()) { @@ -257,6 +265,24 @@ protected ChunkLoader getChunkLoader(TsFileSequenceReader reader, ChunkMetadata return new InstantChunkLoader(reader.getFileName(), chunkMetadata, chunk); } + protected ChunkLoader getRewriteValueChunkLoader( + TsFileSequenceReader reader, ChunkMetadata chunkMetadata, Chunk timeChunk) + throws IOException { + if (chunkMetadata == null || chunkMetadata.getStatistics().getCount() == 0) { + return new InstantChunkLoader(); + } + Chunk chunk = reader.readMemChunk(chunkMetadata).rewrite(chunkMetadata.getNewType(), timeChunk); + + if (chunkMetadata.getNewType() != null) { + chunkMetadata.setTsDataType(chunkMetadata.getNewType()); + + Statistics statistics = Statistics.getStatsByType(chunkMetadata.getNewType()); + statistics.mergeStatistics(chunk.getChunkStatistic()); + chunkMetadata.setStatistics(statistics); + } + return new InstantChunkLoader(reader.getFileName(), chunkMetadata, chunk); + } + protected void flushCurrentChunkWriter() throws IOException { chunkWriter.sealCurrentPage(); writer.writeChunk(chunkWriter); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/readchunk/SingleSeriesCompactionExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/readchunk/SingleSeriesCompactionExecutor.java index cff44c7618a05..f4e9e1bc21e4d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/readchunk/SingleSeriesCompactionExecutor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/readchunk/SingleSeriesCompactionExecutor.java @@ -29,6 +29,7 @@ import org.apache.tsfile.file.header.ChunkHeader; import org.apache.tsfile.file.metadata.ChunkMetadata; import org.apache.tsfile.file.metadata.IDeviceID; +import org.apache.tsfile.file.metadata.statistics.Statistics; import org.apache.tsfile.read.TimeValuePair; import org.apache.tsfile.read.TsFileSequenceReader; import org.apache.tsfile.read.common.Chunk; @@ -129,9 +130,18 @@ public void execute() throws IOException { TsFileSequenceReader reader = readerListPair.left; List chunkMetadataList = readerListPair.right; for (ChunkMetadata chunkMetadata : chunkMetadataList) { - Chunk currentChunk = reader.readMemChunk(chunkMetadata); + Chunk currentChunk = + chunkMetadata.getNewType() != null + ? reader.readMemChunk(chunkMetadata).rewrite(chunkMetadata.getNewType()) + : reader.readMemChunk(chunkMetadata); summary.increaseProcessChunkNum(1); summary.increaseProcessPointNum(chunkMetadata.getNumOfPoints()); + if (chunkMetadata.getNewType() != null) { + chunkMetadata.setTsDataType(chunkMetadata.getNewType()); + Statistics statistics = Statistics.getStatsByType(chunkMetadata.getNewType()); + statistics.mergeStatistics(currentChunk.getChunkStatistic()); + chunkMetadata.setStatistics(statistics); + } if (this.chunkWriter == null) { constructChunkWriterFromReadChunk(currentChunk); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractMemTable.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractMemTable.java index 22606f232a8d1..af47b04a70dd1 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractMemTable.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractMemTable.java @@ -27,10 +27,12 @@ import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory; import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.exception.DataTypeInconsistentException; import org.apache.iotdb.db.exception.WriteProcessException; import org.apache.iotdb.db.exception.query.QueryProcessException; import org.apache.iotdb.db.queryengine.execution.fragment.QueryContext; import org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.DataNodeDevicePathCache; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertTabletNode; import org.apache.iotdb.db.schemaengine.schemaregion.utils.ResourceByPathUtils; @@ -60,6 +62,8 @@ import org.apache.tsfile.utils.Pair; import org.apache.tsfile.utils.ReadWriteIOUtils; import org.apache.tsfile.write.schema.IMeasurementSchema; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.io.DataInputStream; import java.io.IOException; @@ -124,6 +128,8 @@ public abstract class AbstractMemTable implements IMemTable { private String database; private String dataRegionId; + private static final Logger logger = LoggerFactory.getLogger(AbstractMemTable.class); + protected AbstractMemTable() { this.database = null; this.dataRegionId = null; @@ -377,7 +383,6 @@ public void writeTabletNode(InsertTabletNode insertTabletNode, int start, int en public void writeAlignedTablet( InsertTabletNode insertTabletNode, int start, int end, TSStatus[] results) { - List schemaList = new ArrayList<>(); for (int i = 0; i < insertTabletNode.getMeasurementSchemas().length; i++) { if (insertTabletNode.getColumns()[i] == null @@ -1089,4 +1094,13 @@ public void setDatabaseAndDataRegionId(String database, String dataRegionId) { memChunkGroup.setEncryptParameter(EncryptDBUtils.getSecondEncryptParamFromDatabase(database)); } } + + @Override + public void checkDataType(InsertNode node) throws DataTypeInconsistentException { + node.checkDataType(this); + } + + public IWritableMemChunkGroup getWritableMemChunkGroup(IDeviceID deviceID) { + return memTableMap.get(deviceID); + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunk.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunk.java index f42867d4772c1..37804cf354cea 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunk.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunk.java @@ -22,6 +22,8 @@ import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.db.conf.IoTDBConfig; import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.exception.DataTypeInconsistentException; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertNode; import org.apache.iotdb.db.storageengine.dataregion.wal.buffer.IWALByteBufferView; import org.apache.iotdb.db.storageengine.dataregion.wal.utils.WALWriteUtils; import org.apache.iotdb.db.utils.datastructure.AlignedTVList; @@ -62,7 +64,7 @@ public class AlignedWritableMemChunk extends AbstractWritableMemChunk { - private final Map measurementIndexMap; + private Map measurementIndexMap; private List dataTypes; private final List schemaList; private AlignedTVList list; @@ -949,13 +951,30 @@ public List buildColumnIndexList(List schemaList) { } IMeasurementSchema schemaInMemChunk = this.schemaList.get(measurementIndex); columnIndexList.add( - schemaInMemChunk.getType() == requiredMeasurementSchema.getType() + requiredMeasurementSchema.getType().isCompatible(schemaInMemChunk.getType()) ? measurementIndex : -1); } return columnIndexList; } + public void checkDataType(InsertNode node) throws DataTypeInconsistentException { + for (MeasurementSchema incomingSchema : node.getMeasurementSchemas()) { + if (incomingSchema == null) { + continue; + } + + Integer index = measurementIndexMap.get(incomingSchema.getMeasurementName()); + if (index != null) { + IMeasurementSchema existingSchema = schemaList.get(index); + if (existingSchema.getType() != incomingSchema.getType()) { + throw new DataTypeInconsistentException( + existingSchema.getType(), incomingSchema.getType()); + } + } + } + } + // Choose maximum avgPointSizeOfLargestColumn among working and sorted AlignedTVList as // approximate calculation public int getAvgPointSizeOfLargestColumn() { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunkGroup.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunkGroup.java index ec73e7d9c4331..a27973acdb1dd 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunkGroup.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunkGroup.java @@ -22,6 +22,8 @@ import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.path.AlignedPath; import org.apache.iotdb.commons.utils.TestOnly; +import org.apache.iotdb.db.exception.DataTypeInconsistentException; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertNode; import org.apache.iotdb.db.storageengine.dataregion.modification.ModEntry; import org.apache.iotdb.db.storageengine.dataregion.wal.buffer.IWALByteBufferView; @@ -177,6 +179,11 @@ protected static AlignedWritableMemChunkGroup deserialize( return memChunkGroup; } + @Override + public void checkDataType(InsertNode node) throws DataTypeInconsistentException { + memChunk.checkDataType(node); + } + protected static AlignedWritableMemChunkGroup deserializeSingleTVListMemChunks( DataInputStream stream, boolean isTableModel) throws IOException { AlignedWritableMemChunkGroup memChunkGroup = new AlignedWritableMemChunkGroup(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/IMemTable.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/IMemTable.java index fd9ffe90b0ae8..58a490769819b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/IMemTable.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/IMemTable.java @@ -21,9 +21,11 @@ import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.exception.MetadataException; import org.apache.iotdb.commons.path.IFullPath; +import org.apache.iotdb.db.exception.DataTypeInconsistentException; import org.apache.iotdb.db.exception.WriteProcessException; import org.apache.iotdb.db.exception.query.QueryProcessException; import org.apache.iotdb.db.queryengine.execution.fragment.QueryContext; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertTabletNode; import org.apache.iotdb.db.storageengine.dataregion.flush.FlushStatus; @@ -200,4 +202,6 @@ void queryForDeviceRegionScan( String getDataRegionId(); void setDatabaseAndDataRegionId(String database, String dataRegionId); + + void checkDataType(InsertNode node) throws DataTypeInconsistentException; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/IWritableMemChunkGroup.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/IWritableMemChunkGroup.java index 9e61d7aba9294..46a21f05840f6 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/IWritableMemChunkGroup.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/IWritableMemChunkGroup.java @@ -20,6 +20,8 @@ package org.apache.iotdb.db.storageengine.dataregion.memtable; import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.db.exception.DataTypeInconsistentException; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertNode; import org.apache.iotdb.db.storageengine.dataregion.modification.ModEntry; import org.apache.iotdb.db.storageengine.dataregion.wal.buffer.WALEntryValue; @@ -62,4 +64,6 @@ void writeTablet( long getMaxTime(); void setEncryptParameter(EncryptParameter encryptParameter); + + void checkDataType(InsertNode node) throws DataTypeInconsistentException; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/ReadOnlyMemChunk.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/ReadOnlyMemChunk.java index ac2a787349788..7304eab6fb845 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/ReadOnlyMemChunk.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/ReadOnlyMemChunk.java @@ -40,6 +40,7 @@ import org.apache.tsfile.read.common.TimeRange; import org.apache.tsfile.read.common.block.TsBlock; import org.apache.tsfile.read.common.block.TsBlockBuilder; +import org.apache.tsfile.read.common.block.column.BinaryColumnBuilder; import org.apache.tsfile.read.filter.basic.Filter; import org.apache.tsfile.read.reader.IPointReader; import org.apache.tsfile.write.UnSupportedDataTypeException; @@ -332,7 +333,12 @@ private void writeValidValuesIntoTsBlock(TsBlockBuilder builder) throws IOExcept break; case INT32: case DATE: - builder.getColumnBuilder(0).writeInt(tvPair.getValue().getInt()); + if (builder.getColumnBuilder(0) instanceof BinaryColumnBuilder) { + ((BinaryColumnBuilder) builder.getColumnBuilder(0)) + .writeDate(tvPair.getValue().getInt()); + } else { + builder.getColumnBuilder(0).writeInt(tvPair.getValue().getInt()); + } break; case INT64: case TIMESTAMP: diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java index 36cbb4e0f880d..acdac61180bf0 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java @@ -284,6 +284,8 @@ public void insert(InsertRowNode insertRowNode, long[] infoForMetrics) throws WriteProcessException { ensureMemTable(infoForMetrics); + workMemTable.checkDataType(insertRowNode); + long[] memIncrements; long memControlStartTime = System.nanoTime(); @@ -369,6 +371,7 @@ public void insertRows(InsertRowsNode insertRowsNode, long[] infoForMetrics) throws WriteProcessException { ensureMemTable(infoForMetrics); + workMemTable.checkDataType(insertRowsNode); long[] memIncrements; @@ -566,6 +569,7 @@ public void insertTablet( throws WriteProcessException { ensureMemTable(infoForMetrics); + workMemTable.checkDataType(insertTabletNode); long[] memIncrements = scheduleMemoryBlock(insertTabletNode, rangeList, results, noFailure, infoForMetrics); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunkGroup.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunkGroup.java index 07c76039ed041..89f28726c98d7 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunkGroup.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunkGroup.java @@ -20,6 +20,8 @@ package org.apache.iotdb.db.storageengine.dataregion.memtable; import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.db.exception.DataTypeInconsistentException; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertNode; import org.apache.iotdb.db.storageengine.dataregion.modification.ModEntry; import org.apache.iotdb.db.storageengine.dataregion.wal.buffer.IWALByteBufferView; import org.apache.iotdb.db.storageengine.dataregion.wal.utils.WALWriteUtils; @@ -29,6 +31,7 @@ import org.apache.tsfile.utils.BitMap; import org.apache.tsfile.utils.ReadWriteIOUtils; import org.apache.tsfile.write.schema.IMeasurementSchema; +import org.apache.tsfile.write.schema.MeasurementSchema; import java.io.DataInputStream; import java.io.IOException; @@ -211,6 +214,21 @@ public static WritableMemChunkGroup deserialize(DataInputStream stream) throws I return memChunkGroup; } + @Override + public void checkDataType(InsertNode node) throws DataTypeInconsistentException { + for (MeasurementSchema incomingSchema : node.getMeasurementSchemas()) { + if (incomingSchema == null) { + continue; + } + + IWritableMemChunk memChunk = memChunkMap.get(incomingSchema.getMeasurementName()); + if (memChunk != null && memChunk.getSchema().getType() != incomingSchema.getType()) { + throw new DataTypeInconsistentException( + memChunk.getWorkingTVList().getDataType(), incomingSchema.getType()); + } + } + } + public static WritableMemChunkGroup deserializeSingleTVListMemChunks(DataInputStream stream) throws IOException { WritableMemChunkGroup memChunkGroup = new WritableMemChunkGroup(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/MemAlignedPageReader.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/MemAlignedPageReader.java index dbe948c01b860..a299684a49cb6 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/MemAlignedPageReader.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/MemAlignedPageReader.java @@ -20,6 +20,7 @@ package org.apache.iotdb.db.storageengine.dataregion.read.reader.chunk; import org.apache.iotdb.db.storageengine.dataregion.read.reader.chunk.metadata.AlignedPageMetadata; +import org.apache.iotdb.db.utils.CommonUtils; import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnBuilder; @@ -35,6 +36,8 @@ import org.apache.tsfile.read.reader.series.PaginationController; import org.apache.tsfile.utils.TsPrimitiveType; import org.apache.tsfile.write.UnSupportedDataTypeException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.io.IOException; import java.io.Serializable; @@ -114,10 +117,14 @@ public TsBlock getAllSatisfiedData() { // build value column buildValueColumns(satisfyInfo, readEndIndex); - + if (LOGGER.isDebugEnabled()) { + LOGGER.debug("[memAlignedPageReader] TsBlock:{}", CommonUtils.toString(tsBlock)); + } return builder.build(); } + private static final Logger LOGGER = LoggerFactory.getLogger(MemAlignedPageReader.class); + private boolean[] buildSatisfyInfoArray() { if (recordFilter == null || recordFilter.allSatisfy(this)) { boolean[] satisfyInfo = new boolean[tsBlock.getPositionCount()]; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/CommonUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/CommonUtils.java index 1fa1ec44fc189..8ef32c011a1f1 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/CommonUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/CommonUtils.java @@ -48,10 +48,13 @@ import io.airlift.airline.ParseOptionConversionException; import io.airlift.airline.ParseOptionMissingException; import io.airlift.airline.ParseOptionMissingValueException; +import org.apache.tsfile.block.column.Column; import org.apache.tsfile.common.conf.TSFileConfig; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.external.commons.lang3.StringUtils; import org.apache.tsfile.file.metadata.IDeviceID; +import org.apache.tsfile.read.common.block.TsBlock; +import org.apache.tsfile.read.common.block.column.TimeColumn; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.write.UnSupportedDataTypeException; @@ -434,4 +437,20 @@ public static void addQueryLatency(StatementType statementType, long costTimeInN Tag.TYPE.toString(), statementType.name()); } + + public static String toString(TsBlock tsBlock) { + StringBuilder tsBlockBuilder = new StringBuilder(); + for (Column column : tsBlock.getAllColumns()) { + tsBlockBuilder.append("["); + for (int i = 0; i < column.getPositionCount(); i++) { + if (column instanceof TimeColumn) { + tsBlockBuilder.append(column.getLong(i)).append(","); + } else { + tsBlockBuilder.append(column.getTsPrimitiveType(i)).append(","); + } + } + tsBlockBuilder.append("] "); + } + return tsBlockBuilder.toString(); + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/SchemaUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/SchemaUtils.java index 773de36a067fd..6ca54414767dc 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/SchemaUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/SchemaUtils.java @@ -25,16 +25,48 @@ import org.apache.iotdb.db.utils.constant.SqlConstant; import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.AbstractAlignedChunkMetadata; +import org.apache.tsfile.file.metadata.AbstractAlignedTimeSeriesMetadata; +import org.apache.tsfile.file.metadata.AlignedChunkMetadata; +import org.apache.tsfile.file.metadata.ChunkMetadata; +import org.apache.tsfile.file.metadata.IChunkMetadata; +import org.apache.tsfile.file.metadata.TimeseriesMetadata; import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.file.metadata.statistics.Statistics; +import org.apache.tsfile.utils.Binary; +import org.apache.tsfile.utils.Pair; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.HashSet; import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; import static org.apache.iotdb.db.queryengine.execution.operator.AggregationUtil.addPartialSuffix; public class SchemaUtils { + private static final Set> SAME_TYPE_PAIRS; + public static final Logger logger = LoggerFactory.getLogger(SchemaUtils.class); + + static { + SAME_TYPE_PAIRS = new HashSet<>(); + + // related pair about INT + addSymmetricPairs(SAME_TYPE_PAIRS, TSDataType.DATE, TSDataType.INT32); + + // related pair about LONG + addSymmetricPairs(SAME_TYPE_PAIRS, TSDataType.TIMESTAMP, TSDataType.INT64); + + // related pair about TEXT + addSymmetricPairs(SAME_TYPE_PAIRS, TSDataType.STRING, TSDataType.BLOB, TSDataType.TEXT); + } + private SchemaUtils() {} /** @@ -256,4 +288,254 @@ public static List splitPartialBuiltinAggregation(TAggregationType aggre String.format("Invalid Aggregation function: %s", aggregationType)); } } + + private static void addSymmetricPairs( + Set> set, TSDataType... dataTypes) { + for (int i = 0; i < dataTypes.length; i++) { + for (int j = i + 1; j < dataTypes.length; j++) { + set.add(new Pair<>(dataTypes[i], dataTypes[j])); + set.add(new Pair<>(dataTypes[j], dataTypes[i])); + } + } + } + + public static boolean isUsingSameColumn(TSDataType originalDataType, TSDataType dataType) { + if (originalDataType == dataType) { + return true; + } + return SAME_TYPE_PAIRS.contains(new Pair<>(originalDataType, dataType)); + } + + public static void changeMetadataModified( + TimeseriesMetadata timeseriesMetadata, TSDataType targetDataType) { + if (timeseriesMetadata == null) { + return; + } + if (!SchemaUtils.isUsingSameColumn(timeseriesMetadata.getTsDataType(), targetDataType) + && ((targetDataType == TSDataType.STRING) || (targetDataType == TSDataType.TEXT))) { + timeseriesMetadata.setModified(true); + if (timeseriesMetadata.getChunkMetadataList() != null) { + timeseriesMetadata.setChunkMetadataList( + timeseriesMetadata.getChunkMetadataList().stream() + .map( + iChunkMetadata -> { + if (iChunkMetadata == null) return null; + iChunkMetadata.setModified(true); + return (ChunkMetadata) iChunkMetadata; + }) + .collect(Collectors.toList())); + } + } + } + + public static void changeAlignedMetadataModified( + AbstractAlignedTimeSeriesMetadata alignedTimeSeriesMetadata, + List targetDataTypeList) { + if (alignedTimeSeriesMetadata == null) { + return; + } + + int i = 0; + for (TimeseriesMetadata timeseriesMetadata : + alignedTimeSeriesMetadata.getValueTimeseriesMetadataList()) { + if ((timeseriesMetadata != null) + && !SchemaUtils.isUsingSameColumn( + timeseriesMetadata.getTsDataType(), targetDataTypeList.get(i)) + && ((targetDataTypeList.get(i) == TSDataType.STRING) + || (targetDataTypeList.get(i) == TSDataType.TEXT))) { + timeseriesMetadata.setModified(true); + alignedTimeSeriesMetadata.setModified(true); + if (timeseriesMetadata.getChunkMetadataList() != null) { + timeseriesMetadata.setChunkMetadataList( + timeseriesMetadata.getChunkMetadataList().stream() + .map( + iChunkMetadata -> { + if (iChunkMetadata == null) return null; + iChunkMetadata.setModified(true); + return (ChunkMetadata) iChunkMetadata; + }) + .collect(Collectors.toList())); + } + } + i++; + } + } + + public static void changeAlignedMetadataModified( + TimeseriesMetadata timeseriesMetadata, TSDataType targetDataType) { + if (timeseriesMetadata == null) { + return; + } + + if (!SchemaUtils.isUsingSameColumn(timeseriesMetadata.getTsDataType(), targetDataType) + && ((targetDataType == TSDataType.STRING) || (targetDataType == TSDataType.TEXT))) { + timeseriesMetadata.setModified(true); + if (timeseriesMetadata.getChunkMetadataList() != null) { + timeseriesMetadata.setChunkMetadataList( + timeseriesMetadata.getChunkMetadataList().stream() + .map( + iChunkMetadata -> { + if (iChunkMetadata == null) return null; + iChunkMetadata.setModified(true); + return (ChunkMetadata) iChunkMetadata; + }) + .collect(Collectors.toList())); + } + } + } + + public static void changeMetadataModified( + IChunkMetadata chunkMetadata, TSDataType sourceDataType, TSDataType targetDataType) { + if (chunkMetadata == null) { + return; + } + if (!SchemaUtils.isUsingSameColumn(sourceDataType, targetDataType) + && ((targetDataType == TSDataType.STRING) || (targetDataType == TSDataType.TEXT))) { + chunkMetadata.setModified(true); + } + } + + public static void changeAlignedMetadataModified( + AbstractAlignedChunkMetadata chunkMetadata, + TSDataType sourceDataType, + List targetDataTypeList) { + if (chunkMetadata == null) { + return; + } + int i = 0; + for (IChunkMetadata iChunkMetadata : chunkMetadata.getValueChunkMetadataList()) { + if ((iChunkMetadata != null) + && !SchemaUtils.isUsingSameColumn(sourceDataType, targetDataTypeList.get(i)) + && ((targetDataTypeList.get(i) == TSDataType.STRING) + || (targetDataTypeList.get(i) == TSDataType.TEXT))) { + iChunkMetadata.setModified(true); + chunkMetadata.setModified(true); + } + i++; + } + } + + public static AbstractAlignedChunkMetadata rewriteAlignedChunkMetadataStatistics( + AbstractAlignedChunkMetadata alignedChunkMetadata, TSDataType targetDataType) { + List newValueChunkMetadataList = new ArrayList<>(); + for (IChunkMetadata valueChunkMetadata : alignedChunkMetadata.getValueChunkMetadataList()) { + Statistics statistics = Statistics.getStatsByType(targetDataType); + switch (valueChunkMetadata.getDataType()) { + case INT32: + case DATE: + case INT64: + case TIMESTAMP: + case FLOAT: + case DOUBLE: + case BOOLEAN: + if (targetDataType == TSDataType.STRING) { + Binary[] binaryValues = new Binary[4]; + binaryValues[0] = + new Binary( + valueChunkMetadata.getStatistics().getFirstValue().toString(), + StandardCharsets.UTF_8); + binaryValues[1] = + new Binary( + valueChunkMetadata.getStatistics().getLastValue().toString(), + StandardCharsets.UTF_8); + if (valueChunkMetadata.getDataType() == TSDataType.BOOLEAN) { + binaryValues[2] = new Binary(Boolean.FALSE.toString(), StandardCharsets.UTF_8); + binaryValues[3] = new Binary(Boolean.TRUE.toString(), StandardCharsets.UTF_8); + } else { + binaryValues[2] = + new Binary( + valueChunkMetadata.getStatistics().getMinValue().toString(), + StandardCharsets.UTF_8); + binaryValues[3] = + new Binary( + valueChunkMetadata.getStatistics().getMaxValue().toString(), + StandardCharsets.UTF_8); + } + long[] longValues = new long[4]; + longValues[0] = valueChunkMetadata.getStatistics().getStartTime(); + longValues[1] = valueChunkMetadata.getStatistics().getEndTime(); + longValues[2] = longValues[1]; + longValues[3] = longValues[1]; + statistics.update(longValues, binaryValues, binaryValues.length); + } else if (targetDataType == TSDataType.TEXT) { + Binary[] binaryValues = new Binary[2]; + if (valueChunkMetadata.getDataType() == TSDataType.BOOLEAN) { + binaryValues[0] = new Binary(Boolean.FALSE.toString(), StandardCharsets.UTF_8); + binaryValues[1] = new Binary(Boolean.TRUE.toString(), StandardCharsets.UTF_8); + } else { + binaryValues[0] = + new Binary( + valueChunkMetadata.getStatistics().getMinValue().toString(), + StandardCharsets.UTF_8); + binaryValues[1] = + new Binary( + valueChunkMetadata.getStatistics().getMaxValue().toString(), + StandardCharsets.UTF_8); + } + long[] longValues = new long[2]; + longValues[0] = valueChunkMetadata.getStatistics().getStartTime(); + longValues[1] = valueChunkMetadata.getStatistics().getEndTime(); + statistics.update(longValues, binaryValues, binaryValues.length); + } else { + statistics = valueChunkMetadata.getStatistics(); + } + break; + case STRING: + if (targetDataType == TSDataType.TEXT) { + Binary[] binaryValues = new Binary[2]; + binaryValues[0] = + new Binary( + Arrays.asList(TSDataType.TEXT, TSDataType.BLOB) + .contains(valueChunkMetadata.getDataType()) + ? "" + : valueChunkMetadata.getStatistics().getMinValue().toString(), + StandardCharsets.UTF_8); + binaryValues[1] = + new Binary( + Arrays.asList(TSDataType.TEXT, TSDataType.BLOB) + .contains(valueChunkMetadata.getDataType()) + ? "" + : valueChunkMetadata.getStatistics().getMaxValue().toString(), + StandardCharsets.UTF_8); + long[] longValues = new long[2]; + longValues[0] = valueChunkMetadata.getStatistics().getStartTime(); + longValues[1] = valueChunkMetadata.getStatistics().getEndTime(); + statistics.update(longValues, binaryValues, binaryValues.length); + } else { + statistics = valueChunkMetadata.getStatistics(); + } + break; + case TEXT: + case BLOB: + if (targetDataType == TSDataType.STRING) { + Binary[] binaryValues = new Binary[2]; + binaryValues[0] = new Binary("", StandardCharsets.UTF_8); + binaryValues[1] = new Binary("", StandardCharsets.UTF_8); + long[] longValues = new long[2]; + longValues[0] = valueChunkMetadata.getStatistics().getStartTime(); + longValues[1] = valueChunkMetadata.getStatistics().getEndTime(); + statistics.update(longValues, binaryValues, binaryValues.length); + } else { + statistics = valueChunkMetadata.getStatistics(); + } + break; + default: + break; + } + + ChunkMetadata newChunkMetadata = (ChunkMetadata) valueChunkMetadata; + newChunkMetadata.setTsDataType(targetDataType); + newChunkMetadata.setStatistics(statistics); + newValueChunkMetadataList.add(newChunkMetadata); + } + return new AlignedChunkMetadata( + alignedChunkMetadata.getTimeChunkMetadata(), newValueChunkMetadataList); + } + + public static TSEncoding getDataTypeCompatibleEncoding(TSDataType dataType, TSEncoding encoding) { + if (!encoding.isSupported(dataType)) { + return EncodingInferenceUtils.getDefaultEncoding(dataType); + } + return encoding; + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java index d96fb2f981547..b86cf9316236d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java @@ -37,6 +37,7 @@ import org.apache.tsfile.read.common.block.TsBlock; import org.apache.tsfile.read.common.block.TsBlockBuilder; import org.apache.tsfile.read.common.block.TsBlockUtil; +import org.apache.tsfile.read.common.block.column.BinaryColumnBuilder; import org.apache.tsfile.read.common.block.column.TimeColumnBuilder; import org.apache.tsfile.read.filter.basic.Filter; import org.apache.tsfile.utils.Binary; @@ -114,25 +115,39 @@ public static AlignedTVList newAlignedList(List dataTypes) { } } + private List convertToType(TSDataType to, TSDataType from, List originalValues) { + if (!to.isCompatible(from)) { + return null; + } + return originalValues.stream().map(o -> to.castFromArray(from, o)).collect(Collectors.toList()); + } + @Override public TVList getTvListByColumnIndex( - List columnIndex, List dataTypeList, boolean ignoreAllNullRows) { + List columnIndexList, List dataTypeList, boolean ignoreAllNullRows) { List> values = new ArrayList<>(); List> bitMaps = null; - for (int i = 0; i < columnIndex.size(); i++) { + for (int i = 0; i < columnIndexList.size(); i++) { // columnIndex == -1 means querying a non-exist column, add null column here - if (columnIndex.get(i) == -1) { + Integer columnIndex = columnIndexList.get(i); + if (columnIndex == -1) { values.add(null); } else { - values.add(this.values.get(columnIndex.get(i))); - if (this.bitMaps != null && this.bitMaps.get(columnIndex.get(i)) != null) { + List column = this.values.get(columnIndex); + if (dataTypeList.get(i) == this.dataTypes.get(columnIndex)) { + values.add(column); + } else { + values.add(convertToType(dataTypeList.get(i), this.dataTypes.get(columnIndex), column)); + } + + if (this.bitMaps != null && this.bitMaps.get(columnIndex) != null) { if (bitMaps == null) { - bitMaps = new ArrayList<>(columnIndex.size()); - for (int j = 0; j < columnIndex.size(); j++) { + bitMaps = new ArrayList<>(columnIndexList.size()); + for (int j = 0; j < columnIndexList.size(); j++) { bitMaps.add(null); } } - bitMaps.set(i, this.bitMaps.get(columnIndex.get(i))); + bitMaps.set(i, this.bitMaps.get(columnIndex)); } } } @@ -417,6 +432,33 @@ public void extendColumn(TSDataType dataType) { tmpValueChunkRawSize, 0, memoryBinaryChunkSize, 0, tmpValueChunkRawSize.length); } + private Object getObjectByValueIndex(int rowIndex, int columnIndex) { + int arrayIndex = rowIndex / ARRAY_SIZE; + int elementIndex = rowIndex % ARRAY_SIZE; + List columnValues = values.get(columnIndex); + switch (dataTypes.get(columnIndex)) { + case INT32: + case DATE: + return ((int[]) columnValues.get(arrayIndex))[elementIndex]; + case INT64: + case TIMESTAMP: + return ((long[]) columnValues.get(arrayIndex))[elementIndex]; + case FLOAT: + return ((float[]) columnValues.get(arrayIndex))[elementIndex]; + case DOUBLE: + return ((double[]) columnValues.get(arrayIndex))[elementIndex]; + case BOOLEAN: + return ((boolean[]) columnValues.get(arrayIndex))[elementIndex]; + case STRING: + case BLOB: + case TEXT: + case OBJECT: + return ((Binary[]) columnValues.get(arrayIndex))[elementIndex]; + default: + throw new IllegalArgumentException(dataTypes.get(columnIndex) + " is not supported"); + } + } + /** * Get the int value at the given position in AlignedTvList. * @@ -428,7 +470,14 @@ public int getIntByValueIndex(int rowIndex, int columnIndex) { int arrayIndex = rowIndex / ARRAY_SIZE; int elementIndex = rowIndex % ARRAY_SIZE; List columnValues = values.get(columnIndex); - return ((int[]) columnValues.get(arrayIndex))[elementIndex]; + if (dataTypes.get(columnIndex) == TSDataType.INT32 + || dataTypes.get(columnIndex) == TSDataType.DATE) { + return ((int[]) columnValues.get(arrayIndex))[elementIndex]; + } else { + return (int) + TSDataType.INT32.castFromSingleValue( + dataTypes.get(columnIndex), getObjectByValueIndex(rowIndex, columnIndex)); + } } /** @@ -442,7 +491,14 @@ public long getLongByValueIndex(int rowIndex, int columnIndex) { int arrayIndex = rowIndex / ARRAY_SIZE; int elementIndex = rowIndex % ARRAY_SIZE; List columnValues = values.get(columnIndex); - return ((long[]) columnValues.get(arrayIndex))[elementIndex]; + if (dataTypes.get(columnIndex) == TSDataType.INT64 + || dataTypes.get(columnIndex) == TSDataType.TIMESTAMP) { + return ((long[]) columnValues.get(arrayIndex))[elementIndex]; + } else { + return (long) + TSDataType.INT64.castFromSingleValue( + dataTypes.get(columnIndex), getObjectByValueIndex(rowIndex, columnIndex)); + } } /** @@ -456,7 +512,13 @@ public float getFloatByValueIndex(int rowIndex, int columnIndex) { int arrayIndex = rowIndex / ARRAY_SIZE; int elementIndex = rowIndex % ARRAY_SIZE; List columnValues = values.get(columnIndex); - return ((float[]) columnValues.get(arrayIndex))[elementIndex]; + if (dataTypes.get(columnIndex) == TSDataType.FLOAT) { + return ((float[]) columnValues.get(arrayIndex))[elementIndex]; + } else { + return (float) + TSDataType.FLOAT.castFromSingleValue( + dataTypes.get(columnIndex), getObjectByValueIndex(rowIndex, columnIndex)); + } } /** @@ -470,7 +532,13 @@ public double getDoubleByValueIndex(int rowIndex, int columnIndex) { int arrayIndex = rowIndex / ARRAY_SIZE; int elementIndex = rowIndex % ARRAY_SIZE; List columnValues = values.get(columnIndex); - return ((double[]) columnValues.get(arrayIndex))[elementIndex]; + if (dataTypes.get(columnIndex) == TSDataType.DOUBLE) { + return ((double[]) columnValues.get(arrayIndex))[elementIndex]; + } else { + return (double) + TSDataType.DOUBLE.castFromSingleValue( + dataTypes.get(columnIndex), getObjectByValueIndex(rowIndex, columnIndex)); + } } /** @@ -484,7 +552,15 @@ public Binary getBinaryByValueIndex(int rowIndex, int columnIndex) { int arrayIndex = rowIndex / ARRAY_SIZE; int elementIndex = rowIndex % ARRAY_SIZE; List columnValues = values.get(columnIndex); - return ((Binary[]) columnValues.get(arrayIndex))[elementIndex]; + if (dataTypes.get(columnIndex) == TSDataType.TEXT + || dataTypes.get(columnIndex) == TSDataType.BLOB + || dataTypes.get(columnIndex) == TSDataType.STRING) { + return ((Binary[]) columnValues.get(arrayIndex))[elementIndex]; + } else { + return (Binary) + TSDataType.TEXT.castFromSingleValue( + dataTypes.get(columnIndex), getObjectByValueIndex(rowIndex, columnIndex)); + } } /** @@ -498,7 +574,13 @@ public boolean getBooleanByValueIndex(int rowIndex, int columnIndex) { int arrayIndex = rowIndex / ARRAY_SIZE; int elementIndex = rowIndex % ARRAY_SIZE; List columnValues = values.get(columnIndex); - return ((boolean[]) columnValues.get(arrayIndex))[elementIndex]; + if (dataTypes.get(columnIndex) == TSDataType.BOOLEAN) { + return ((boolean[]) columnValues.get(arrayIndex))[elementIndex]; + } else { + return (Boolean) + TSDataType.BOOLEAN.castFromSingleValue( + dataTypes.get(columnIndex), getObjectByValueIndex(rowIndex, columnIndex)); + } } /** @@ -608,8 +690,9 @@ public Pair delete(long lowerBound, long upperBound, int colum int originRowIndex = getValueIndex(i); int arrayIndex = originRowIndex / ARRAY_SIZE; int elementIndex = originRowIndex % ARRAY_SIZE; - markNullValue(columnIndex, arrayIndex, elementIndex); - deletedNumber++; + if (markNullValue(columnIndex, arrayIndex, elementIndex)) { + deletedNumber++; + } } else { deleteColumn = false; } @@ -950,9 +1033,16 @@ private void markNullValue( } } - private void markNullValue(int columnIndex, int arrayIndex, int elementIndex) { + private boolean markNullValue(int columnIndex, int arrayIndex, int elementIndex) { // mark the null value in the current bitmap - getBitMap(columnIndex, arrayIndex).mark(elementIndex); + BitMap bitMap = getBitMap(columnIndex, arrayIndex); + bitMap.mark(elementIndex); + if (bitMap.isMarked(elementIndex)) { + return false; + } else { + bitMap.mark(elementIndex); + return true; + } } @Override @@ -1155,9 +1245,16 @@ public TsBlock buildTsBlock( valueBuilder.writeBoolean(getBooleanByValueIndex(originRowIndex, columnIndex)); break; case INT32: - case DATE: valueBuilder.writeInt(getIntByValueIndex(originRowIndex, columnIndex)); break; + case DATE: + if (valueBuilder instanceof BinaryColumnBuilder) { + ((BinaryColumnBuilder) valueBuilder) + .writeDate(getIntByValueIndex(originRowIndex, columnIndex)); + } else { + valueBuilder.writeInt(getIntByValueIndex(originRowIndex, columnIndex)); + } + break; case INT64: case TIMESTAMP: valueBuilder.writeLong(getLongByValueIndex(originRowIndex, columnIndex)); @@ -1838,9 +1935,11 @@ public TsPrimitiveType getPrimitiveTypeObject(int rowIndex, int columnIndex) { return TsPrimitiveType.getByType( TSDataType.BOOLEAN, getBooleanByValueIndex(valueIndex, validColumnIndex)); case INT32: - case DATE: return TsPrimitiveType.getByType( TSDataType.INT32, getIntByValueIndex(valueIndex, validColumnIndex)); + case DATE: + return TsPrimitiveType.getByType( + TSDataType.DATE, getIntByValueIndex(valueIndex, validColumnIndex)); case INT64: case TIMESTAMP: return TsPrimitiveType.getByType( @@ -2073,9 +2172,16 @@ private void writeToColumn( valueBuilder.writeBoolean(getBooleanByValueIndex(originRowIndex, validColumnIndex)); break; case INT32: - case DATE: valueBuilder.writeInt(getIntByValueIndex(originRowIndex, validColumnIndex)); break; + case DATE: + if (valueBuilder instanceof BinaryColumnBuilder) { + ((BinaryColumnBuilder) valueBuilder) + .writeDate(getIntByValueIndex(originRowIndex, validColumnIndex)); + } else { + valueBuilder.writeInt(getIntByValueIndex(originRowIndex, validColumnIndex)); + } + break; case INT64: case TIMESTAMP: valueBuilder.writeLong(getLongByValueIndex(originRowIndex, validColumnIndex)); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/BackIntTVList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/BackIntTVList.java index 3ccc5cbc946f5..403314f581791 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/BackIntTVList.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/BackIntTVList.java @@ -19,6 +19,8 @@ package org.apache.iotdb.db.utils.datastructure; +import org.apache.tsfile.enums.TSDataType; + public class BackIntTVList extends QuickIntTVList { private final BackwardSort policy; @@ -26,6 +28,11 @@ public class BackIntTVList extends QuickIntTVList { policy = new BackwardSort(this); } + BackIntTVList(TSDataType dataType) { + policy = new BackwardSort(this); + this.dataType = dataType; + } + @Override public synchronized void sort() { if (!sorted) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/IntTVList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/IntTVList.java index 758cd64053bc5..0146ecf0e6b25 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/IntTVList.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/IntTVList.java @@ -51,20 +51,20 @@ public abstract class IntTVList extends TVList { values = new ArrayList<>(); } - public static IntTVList newList() { + public static IntTVList newList(TSDataType dataType) { switch (TVLIST_SORT_ALGORITHM) { case QUICK: - return new QuickIntTVList(); + return new QuickIntTVList(dataType); case BACKWARD: - return new BackIntTVList(); + return new BackIntTVList(dataType); default: - return new TimIntTVList(); + return new TimIntTVList(dataType); } } @Override public synchronized IntTVList clone() { - IntTVList cloneList = IntTVList.newList(); + IntTVList cloneList = IntTVList.newList(dataType); cloneAs(cloneList); cloneBitMap(cloneList); for (int[] valueArray : values) { @@ -125,9 +125,9 @@ protected void clearValue() { @Override protected void expandValues() { if (indices != null) { - indices.add((int[]) getPrimitiveArraysByType(TSDataType.INT32)); + indices.add((int[]) getPrimitiveArraysByType(dataType)); } - values.add((int[]) getPrimitiveArraysByType(TSDataType.INT32)); + values.add((int[]) getPrimitiveArraysByType(dataType)); if (bitMap != null) { bitMap.add(null); } @@ -135,14 +135,13 @@ protected void expandValues() { @Override public TimeValuePair getTimeValuePair(int index) { - return new TimeValuePair( - getTime(index), TsPrimitiveType.getByType(TSDataType.INT32, getInt(index))); + return new TimeValuePair(getTime(index), TsPrimitiveType.getByType(dataType, getInt(index))); } @Override protected TimeValuePair getTimeValuePair( int index, long time, Integer floatPrecision, TSEncoding encoding) { - return new TimeValuePair(time, TsPrimitiveType.getByType(TSDataType.INT32, getInt(index))); + return new TimeValuePair(time, TsPrimitiveType.getByType(dataType, getInt(index))); } @Override @@ -265,12 +264,14 @@ int dropNullValThenUpdateMinMaxTimeAndSorted( @Override public TSDataType getDataType() { - return TSDataType.INT32; + return dataType; } @Override public int serializedSize() { - return Byte.BYTES + Integer.BYTES + rowCount * (Long.BYTES + Integer.BYTES + Byte.BYTES); + return Byte.BYTES + + Integer.BYTES + + rowCount * (Long.BYTES + java.lang.Integer.BYTES + Byte.BYTES); } @Override @@ -284,8 +285,9 @@ public void serializeToWAL(IWALByteBufferView buffer) { } } - public static IntTVList deserialize(DataInputStream stream) throws IOException { - IntTVList tvList = IntTVList.newList(); + public static IntTVList deserialize(DataInputStream stream, TSDataType dataType) + throws IOException { + IntTVList tvList = IntTVList.newList(dataType); int rowCount = stream.readInt(); long[] times = new long[rowCount]; int[] values = new int[rowCount]; @@ -301,8 +303,9 @@ public static IntTVList deserialize(DataInputStream stream) throws IOException { return tvList; } - public static IntTVList deserializeWithoutBitMap(DataInputStream stream) throws IOException { - IntTVList tvList = IntTVList.newList(); + public static IntTVList deserializeWithoutBitMap(DataInputStream stream, TSDataType dataType) + throws IOException { + IntTVList tvList = IntTVList.newList(dataType); int rowCount = stream.readInt(); long[] times = new long[rowCount]; int[] values = new int[rowCount]; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/QuickIntTVList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/QuickIntTVList.java index e83646e32024b..035dbc4f39368 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/QuickIntTVList.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/QuickIntTVList.java @@ -18,6 +18,8 @@ */ package org.apache.iotdb.db.utils.datastructure; +import org.apache.tsfile.enums.TSDataType; + public class QuickIntTVList extends IntTVList { private final QuickSort policy; @@ -25,6 +27,11 @@ public class QuickIntTVList extends IntTVList { policy = new QuickSort(this); } + QuickIntTVList(TSDataType dataType) { + policy = new QuickSort(this); + this.dataType = dataType; + } + @Override public synchronized void sort() { if (!sorted) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TVList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TVList.java index dd5f10f37d45b..f85d5866e3a16 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TVList.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TVList.java @@ -104,6 +104,8 @@ public abstract class TVList implements WALEntryValue { protected static int defaultArrayNum = 0; protected static volatile long defaultArrayNumLastUpdatedTimeMs = 0; + protected TSDataType dataType; + protected TVList() { timestamps = new ArrayList<>(getDefaultArrayNum()); rowCount = 0; @@ -124,8 +126,9 @@ public static TVList newList(TSDataType dataType) { case FLOAT: return FloatTVList.newList(); case INT32: + return IntTVList.newList(TSDataType.INT32); case DATE: - return IntTVList.newList(); + return IntTVList.newList(TSDataType.DATE); case INT64: case TIMESTAMP: return LongTVList.newList(); @@ -700,8 +703,9 @@ public static TVList deserialize(DataInputStream stream) throws IOException { case FLOAT: return FloatTVList.deserialize(stream); case INT32: + return IntTVList.deserialize(stream, TSDataType.INT32); case DATE: - return IntTVList.deserialize(stream); + return IntTVList.deserialize(stream, TSDataType.DATE); case INT64: case TIMESTAMP: return LongTVList.deserialize(stream); @@ -726,8 +730,9 @@ public static TVList deserializeWithoutBitMap(DataInputStream stream) throws IOE case FLOAT: return FloatTVList.deserializeWithoutBitMap(stream); case INT32: + return IntTVList.deserializeWithoutBitMap(stream, TSDataType.INT32); case DATE: - return IntTVList.deserializeWithoutBitMap(stream); + return IntTVList.deserializeWithoutBitMap(stream, TSDataType.DATE); case INT64: case TIMESTAMP: return LongTVList.deserializeWithoutBitMap(stream); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TimIntTVList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TimIntTVList.java index 8216f2eb2961d..33ccb1b0e5d30 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TimIntTVList.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TimIntTVList.java @@ -18,6 +18,8 @@ */ package org.apache.iotdb.db.utils.datastructure; +import org.apache.tsfile.enums.TSDataType; + public class TimIntTVList extends IntTVList { private final TimSort policy; @@ -25,6 +27,11 @@ public class TimIntTVList extends IntTVList { policy = new TimSort(this); } + TimIntTVList(TSDataType dataType) { + policy = new TimSort(this); + this.dataType = dataType; + } + @Override public synchronized void sort() { policy.checkSortedTimestampsAndIndices(); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/CompactionDataTypeAlterTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/CompactionDataTypeAlterTest.java new file mode 100644 index 0000000000000..0d08ca7832ca4 --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/CompactionDataTypeAlterTest.java @@ -0,0 +1,322 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.storageengine.dataregion.compaction; + +import org.apache.iotdb.commons.exception.MetadataException; +import org.apache.iotdb.db.exception.StorageEngineException; +import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.performer.impl.FastCompactionPerformer; +import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.performer.impl.ReadChunkCompactionPerformer; +import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.performer.impl.ReadPointCompactionPerformer; +import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.task.InnerSpaceCompactionTask; +import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; +import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResourceStatus; +import org.apache.iotdb.db.storageengine.dataregion.utils.TsFileResourceUtils; + +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.exception.write.WriteProcessException; +import org.apache.tsfile.file.metadata.IDeviceID; +import org.apache.tsfile.read.TsFileReader; +import org.apache.tsfile.read.TsFileSequenceReader; +import org.apache.tsfile.read.common.Path; +import org.apache.tsfile.read.expression.QueryExpression; +import org.apache.tsfile.read.query.dataset.QueryDataSet; +import org.apache.tsfile.write.TsFileWriter; +import org.apache.tsfile.write.record.TSRecord; +import org.apache.tsfile.write.record.datapoint.DoubleDataPoint; +import org.apache.tsfile.write.record.datapoint.IntDataPoint; +import org.apache.tsfile.write.schema.IMeasurementSchema; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +public class CompactionDataTypeAlterTest extends AbstractCompactionTest { + private final String oldThreadName = Thread.currentThread().getName(); + private final IDeviceID device = + IDeviceID.Factory.DEFAULT_FACTORY.create(COMPACTION_TEST_SG + ".d1"); + + @Before + public void setUp() + throws IOException, WriteProcessException, MetadataException, InterruptedException { + super.setUp(); + Thread.currentThread().setName("pool-1-IoTDB-Compaction-Worker-1"); + } + + @After + public void tearDown() throws IOException, StorageEngineException { + super.tearDown(); + Thread.currentThread().setName(oldThreadName); + } + + @Test + public void testCompactNonAlignedSeriesWithReadChunkCompactionPerformer() + throws IOException, WriteProcessException { + generateDataTypeNotMatchFilesWithNonAlignedSeries(); + InnerSpaceCompactionTask task = + new InnerSpaceCompactionTask( + 0, tsFileManager, seqResources, true, new ReadChunkCompactionPerformer(), 0); + Assert.assertTrue(task.start()); + TsFileResourceUtils.validateTsFileDataCorrectness(tsFileManager.getTsFileList(true).get(0)); + Assert.assertEquals( + 1, ((long) tsFileManager.getTsFileList(true).get(0).getStartTime(device).get())); + Assert.assertEquals( + 2, ((long) tsFileManager.getTsFileList(true).get(0).getEndTime(device).get())); + TsFileResource tsFileResource = tsFileManager.getTsFileList(true).get(0); + try (TsFileSequenceReader reader = + new TsFileSequenceReader(tsFileResource.getTsFile().getAbsolutePath()); + TsFileReader readTsFile = new TsFileReader(reader)) { + ArrayList paths = new ArrayList<>(); + paths.add(new Path(device, "s1", true)); + QueryExpression queryExpression = QueryExpression.create(paths, null); + QueryDataSet queryDataSet = readTsFile.query(queryExpression); + Assert.assertTrue(queryDataSet.hasNext()); + Assert.assertEquals("1\t1.0", queryDataSet.next().toString()); + Assert.assertTrue(queryDataSet.hasNext()); + Assert.assertEquals("2\t2.0", queryDataSet.next().toString()); + Assert.assertFalse(queryDataSet.hasNext()); + } + } + + @Test + public void testCompactNonAlignedSeriesWithFastCompactionPerformer() + throws IOException, WriteProcessException { + generateDataTypeNotMatchFilesWithNonAlignedSeries(); + InnerSpaceCompactionTask task = + new InnerSpaceCompactionTask( + 0, tsFileManager, seqResources, true, new FastCompactionPerformer(false), 0); + Assert.assertTrue(task.start()); + TsFileResourceUtils.validateTsFileDataCorrectness(tsFileManager.getTsFileList(true).get(0)); + Assert.assertEquals( + 1, ((long) tsFileManager.getTsFileList(true).get(0).getStartTime(device).get())); + Assert.assertEquals( + 2, ((long) tsFileManager.getTsFileList(true).get(0).getEndTime(device).get())); + TsFileResource tsFileResource = tsFileManager.getTsFileList(true).get(0); + try (TsFileSequenceReader reader = + new TsFileSequenceReader(tsFileResource.getTsFile().getAbsolutePath()); + TsFileReader readTsFile = new TsFileReader(reader)) { + ArrayList paths = new ArrayList<>(); + paths.add(new Path(device, "s1", true)); + QueryExpression queryExpression = QueryExpression.create(paths, null); + QueryDataSet queryDataSet = readTsFile.query(queryExpression); + Assert.assertTrue(queryDataSet.hasNext()); + Assert.assertEquals("1\t1.0", queryDataSet.next().toString()); + Assert.assertTrue(queryDataSet.hasNext()); + Assert.assertEquals("2\t2.0", queryDataSet.next().toString()); + Assert.assertFalse(queryDataSet.hasNext()); + } + } + + @Test + public void testCompactNonAlignedSeriesWithReadPointCompactionPerformer() + throws IOException, WriteProcessException { + generateDataTypeNotMatchFilesWithNonAlignedSeries(); + InnerSpaceCompactionTask task = + new InnerSpaceCompactionTask( + 0, tsFileManager, seqResources, true, new ReadPointCompactionPerformer(), 0); + Assert.assertTrue(task.start()); + TsFileResourceUtils.validateTsFileDataCorrectness(tsFileManager.getTsFileList(true).get(0)); + Assert.assertEquals( + 1, ((long) tsFileManager.getTsFileList(true).get(0).getStartTime(device).get())); + Assert.assertEquals( + 2, ((long) tsFileManager.getTsFileList(true).get(0).getEndTime(device).get())); + TsFileResource tsFileResource = tsFileManager.getTsFileList(true).get(0); + try (TsFileSequenceReader reader = + new TsFileSequenceReader(tsFileResource.getTsFile().getAbsolutePath()); + TsFileReader readTsFile = new TsFileReader(reader)) { + ArrayList paths = new ArrayList<>(); + paths.add(new Path(device, "s1", true)); + QueryExpression queryExpression = QueryExpression.create(paths, null); + QueryDataSet queryDataSet = readTsFile.query(queryExpression); + Assert.assertTrue(queryDataSet.hasNext()); + Assert.assertEquals("1\t1.0", queryDataSet.next().toString()); + Assert.assertTrue(queryDataSet.hasNext()); + Assert.assertEquals("2\t2.0", queryDataSet.next().toString()); + Assert.assertFalse(queryDataSet.hasNext()); + } + } + + @Test + public void testCompactAlignedSeriesWithReadChunkCompactionPerformer() + throws IOException, WriteProcessException { + generateDataTypeNotMatchFilesWithAlignedSeries(); + InnerSpaceCompactionTask task = + new InnerSpaceCompactionTask( + 0, tsFileManager, seqResources, true, new ReadChunkCompactionPerformer(), 0); + Assert.assertTrue(task.start()); + TsFileResourceUtils.validateTsFileDataCorrectness(tsFileManager.getTsFileList(true).get(0)); + Assert.assertEquals( + 1, ((long) tsFileManager.getTsFileList(true).get(0).getStartTime(device).get())); + Assert.assertEquals( + 2, ((long) tsFileManager.getTsFileList(true).get(0).getEndTime(device).get())); + TsFileResource tsFileResource = tsFileManager.getTsFileList(true).get(0); + try (TsFileSequenceReader reader = + new TsFileSequenceReader(tsFileResource.getTsFile().getAbsolutePath()); + TsFileReader readTsFile = new TsFileReader(reader)) { + ArrayList paths = new ArrayList<>(); + paths.add(new Path(device, "s1", true)); + paths.add(new Path(device, "s2", true)); + QueryExpression queryExpression = QueryExpression.create(paths, null); + QueryDataSet queryDataSet = readTsFile.query(queryExpression); + Assert.assertTrue(queryDataSet.hasNext()); + Assert.assertEquals("1\t0.0\t1.0", queryDataSet.next().toString()); + Assert.assertTrue(queryDataSet.hasNext()); + Assert.assertEquals("2\t2.0\t3.0", queryDataSet.next().toString()); + Assert.assertFalse(queryDataSet.hasNext()); + } + } + + @Test + public void testCompactAlignedSeriesWithFastCompactionPerformer() + throws IOException, WriteProcessException { + generateDataTypeNotMatchFilesWithAlignedSeries(); + InnerSpaceCompactionTask task = + new InnerSpaceCompactionTask( + 0, tsFileManager, seqResources, true, new FastCompactionPerformer(false), 0); + Assert.assertTrue(task.start()); + TsFileResourceUtils.validateTsFileDataCorrectness(tsFileManager.getTsFileList(true).get(0)); + Assert.assertEquals( + 1, ((long) tsFileManager.getTsFileList(true).get(0).getStartTime(device).get())); + Assert.assertEquals( + 2, ((long) tsFileManager.getTsFileList(true).get(0).getEndTime(device).get())); + TsFileResource tsFileResource = tsFileManager.getTsFileList(true).get(0); + try (TsFileSequenceReader reader = + new TsFileSequenceReader(tsFileResource.getTsFile().getAbsolutePath()); + TsFileReader readTsFile = new TsFileReader(reader)) { + ArrayList paths = new ArrayList<>(); + paths.add(new Path(device, "s1", true)); + paths.add(new Path(device, "s2", true)); + QueryExpression queryExpression = QueryExpression.create(paths, null); + QueryDataSet queryDataSet = readTsFile.query(queryExpression); + Assert.assertTrue(queryDataSet.hasNext()); + Assert.assertEquals("1\t0.0\t1.0", queryDataSet.next().toString()); + Assert.assertTrue(queryDataSet.hasNext()); + Assert.assertEquals("2\t2.0\t3.0", queryDataSet.next().toString()); + Assert.assertFalse(queryDataSet.hasNext()); + } + } + + @Test + public void testCompactAlignedSeriesWithReadPointCompactionPerformer() + throws IOException, WriteProcessException { + generateDataTypeNotMatchFilesWithAlignedSeries(); + InnerSpaceCompactionTask task = + new InnerSpaceCompactionTask( + 0, tsFileManager, seqResources, true, new ReadPointCompactionPerformer(), 0); + Assert.assertTrue(task.start()); + TsFileResourceUtils.validateTsFileDataCorrectness(tsFileManager.getTsFileList(true).get(0)); + Assert.assertEquals( + 1, ((long) tsFileManager.getTsFileList(true).get(0).getStartTime(device).get())); + Assert.assertEquals( + 2, ((long) tsFileManager.getTsFileList(true).get(0).getEndTime(device).get())); + TsFileResource tsFileResource = tsFileManager.getTsFileList(true).get(0); + try (TsFileSequenceReader reader = + new TsFileSequenceReader(tsFileResource.getTsFile().getAbsolutePath()); + TsFileReader readTsFile = new TsFileReader(reader)) { + ArrayList paths = new ArrayList<>(); + paths.add(new Path(device, "s1", true)); + paths.add(new Path(device, "s2", true)); + QueryExpression queryExpression = QueryExpression.create(paths, null); + QueryDataSet queryDataSet = readTsFile.query(queryExpression); + Assert.assertTrue(queryDataSet.hasNext()); + Assert.assertEquals("1\t0.0\t1.0", queryDataSet.next().toString()); + Assert.assertTrue(queryDataSet.hasNext()); + Assert.assertEquals("2\t2.0\t3.0", queryDataSet.next().toString()); + Assert.assertFalse(queryDataSet.hasNext()); + } + } + + private void generateDataTypeNotMatchFilesWithNonAlignedSeries() + throws IOException, WriteProcessException { + MeasurementSchema measurementSchema1 = new MeasurementSchema("s1", TSDataType.INT32); + TsFileResource resource1 = createEmptyFileAndResource(true); + resource1.setStatusForTest(TsFileResourceStatus.COMPACTING); + try (TsFileWriter writer = new TsFileWriter(resource1.getTsFile())) { + writer.registerTimeseries(new Path(device), measurementSchema1); + TSRecord record = new TSRecord(device, 1); + record.addTuple(new IntDataPoint("s1", 1)); + writer.writeRecord(record); + writer.flush(); + } + resource1.updateStartTime(device, 1); + resource1.updateEndTime(device, 1); + resource1.serialize(); + seqResources.add(resource1); + + MeasurementSchema measurementSchema2 = new MeasurementSchema("s1", TSDataType.DOUBLE); + TsFileResource resource2 = createEmptyFileAndResource(true); + resource2.setStatusForTest(TsFileResourceStatus.COMPACTING); + try (TsFileWriter writer = new TsFileWriter(resource2.getTsFile())) { + writer.registerTimeseries(new Path(device), measurementSchema2); + TSRecord record = new TSRecord(device, 2); + record.addTuple(new DoubleDataPoint("s1", 2.0)); + writer.writeRecord(record); + writer.flush(); + } + resource2.updateStartTime(device, 2); + resource2.updateEndTime(device, 2); + resource2.serialize(); + seqResources.add(resource2); + } + + private void generateDataTypeNotMatchFilesWithAlignedSeries() + throws IOException, WriteProcessException { + List measurementSchemas1 = new ArrayList<>(); + measurementSchemas1.add(new MeasurementSchema("s1", TSDataType.INT32)); + measurementSchemas1.add(new MeasurementSchema("s2", TSDataType.INT32)); + + TsFileResource resource1 = createEmptyFileAndResource(true); + resource1.setStatusForTest(TsFileResourceStatus.COMPACTING); + try (TsFileWriter writer = new TsFileWriter(resource1.getTsFile())) { + writer.registerAlignedTimeseries(new Path(device), measurementSchemas1); + TSRecord record = new TSRecord(device, 1); + record.addTuple(new IntDataPoint("s1", 0)); + record.addTuple(new IntDataPoint("s2", 1)); + writer.writeRecord(record); + writer.flush(); + } + resource1.updateStartTime(device, 1); + resource1.updateEndTime(device, 1); + resource1.serialize(); + seqResources.add(resource1); + + List measurementSchemas2 = new ArrayList<>(); + measurementSchemas2.add(new MeasurementSchema("s1", TSDataType.DOUBLE)); + measurementSchemas2.add(new MeasurementSchema("s2", TSDataType.DOUBLE)); + TsFileResource resource2 = createEmptyFileAndResource(true); + resource2.setStatusForTest(TsFileResourceStatus.COMPACTING); + try (TsFileWriter writer = new TsFileWriter(resource2.getTsFile())) { + writer.registerAlignedTimeseries(new Path(device), measurementSchemas2); + TSRecord record = new TSRecord(device, 2); + record.addTuple(new DoubleDataPoint("s1", 2.0)); + record.addTuple(new DoubleDataPoint("s2", 3.0)); + writer.writeRecord(record); + writer.flush(); + } + resource2.updateStartTime(device, 2); + resource2.updateEndTime(device, 2); + resource2.serialize(); + seqResources.add(resource2); + } +} diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/CompactionDataTypeNotMatchAlterableDataTypeTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/CompactionDataTypeNotMatchAlterableDataTypeTest.java new file mode 100644 index 0000000000000..66d3c97e6c652 --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/CompactionDataTypeNotMatchAlterableDataTypeTest.java @@ -0,0 +1,221 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.storageengine.dataregion.compaction; + +import org.apache.iotdb.commons.exception.MetadataException; +import org.apache.iotdb.db.exception.StorageEngineException; +import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.performer.impl.FastCompactionPerformer; +import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.performer.impl.ReadChunkCompactionPerformer; +import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.performer.impl.ReadPointCompactionPerformer; +import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.task.InnerSpaceCompactionTask; +import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; +import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResourceStatus; +import org.apache.iotdb.db.storageengine.dataregion.utils.TsFileResourceUtils; + +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.exception.write.WriteProcessException; +import org.apache.tsfile.file.metadata.IDeviceID; +import org.apache.tsfile.read.common.Path; +import org.apache.tsfile.write.TsFileWriter; +import org.apache.tsfile.write.record.TSRecord; +import org.apache.tsfile.write.record.datapoint.DoubleDataPoint; +import org.apache.tsfile.write.record.datapoint.FloatDataPoint; +import org.apache.tsfile.write.record.datapoint.IntDataPoint; +import org.apache.tsfile.write.schema.IMeasurementSchema; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +@SuppressWarnings("OptionalGetWithoutIsPresent") +public class CompactionDataTypeNotMatchAlterableDataTypeTest extends AbstractCompactionTest { + private final String oldThreadName = Thread.currentThread().getName(); + private final IDeviceID device = + IDeviceID.Factory.DEFAULT_FACTORY.create(COMPACTION_TEST_SG + ".d1"); + + @Before + public void setUp() + throws IOException, WriteProcessException, MetadataException, InterruptedException { + super.setUp(); + Thread.currentThread().setName("pool-1-IoTDB-Compaction-Worker-1"); + } + + @After + public void tearDown() throws IOException, StorageEngineException { + super.tearDown(); + Thread.currentThread().setName(oldThreadName); + } + + @Test + public void testCompactNonAlignedSeriesWithReadChunkCompactionPerformer() + throws IOException, WriteProcessException { + generateDataTypeNotMatchFilesWithNonAlignedSeries(); + InnerSpaceCompactionTask task = + new InnerSpaceCompactionTask( + 0, tsFileManager, seqResources, true, new ReadChunkCompactionPerformer(), 0); + Assert.assertTrue(task.start()); + TsFileResourceUtils.validateTsFileDataCorrectness(tsFileManager.getTsFileList(true).get(0)); + Assert.assertEquals( + 1, ((long) tsFileManager.getTsFileList(true).get(0).getStartTime(device).get())); + } + + @Test + public void testCompactNonAlignedSeriesWithFastCompactionPerformer() + throws IOException, WriteProcessException { + generateDataTypeNotMatchFilesWithNonAlignedSeries(); + InnerSpaceCompactionTask task = + new InnerSpaceCompactionTask( + 0, tsFileManager, seqResources, true, new FastCompactionPerformer(false), 0); + Assert.assertTrue(task.start()); + TsFileResourceUtils.validateTsFileDataCorrectness(tsFileManager.getTsFileList(true).get(0)); + Assert.assertEquals( + 1, ((long) tsFileManager.getTsFileList(true).get(0).getStartTime(device).get())); + } + + @Test + public void testCompactNonAlignedSeriesWithReadPointCompactionPerformer() + throws IOException, WriteProcessException { + generateDataTypeNotMatchFilesWithNonAlignedSeries(); + InnerSpaceCompactionTask task = + new InnerSpaceCompactionTask( + 0, tsFileManager, seqResources, true, new ReadPointCompactionPerformer(), 0); + Assert.assertTrue(task.start()); + TsFileResourceUtils.validateTsFileDataCorrectness(tsFileManager.getTsFileList(true).get(0)); + Assert.assertEquals( + 1, ((long) tsFileManager.getTsFileList(true).get(0).getStartTime(device).get())); + } + + @Test + public void testCompactAlignedSeriesWithReadChunkCompactionPerformer() + throws IOException, WriteProcessException { + generateDataTypeNotMatchFilesWithAlignedSeries(); + InnerSpaceCompactionTask task = + new InnerSpaceCompactionTask( + 0, tsFileManager, seqResources, true, new ReadChunkCompactionPerformer(), 0); + Assert.assertTrue(task.start()); + TsFileResourceUtils.validateTsFileDataCorrectness(tsFileManager.getTsFileList(true).get(0)); + Assert.assertEquals( + 1, ((long) tsFileManager.getTsFileList(true).get(0).getStartTime(device).get())); + } + + @Test + public void testCompactAlignedSeriesWithFastCompactionPerformer() + throws IOException, WriteProcessException { + generateDataTypeNotMatchFilesWithAlignedSeries(); + InnerSpaceCompactionTask task = + new InnerSpaceCompactionTask( + 0, tsFileManager, seqResources, true, new FastCompactionPerformer(false), 0); + Assert.assertTrue(task.start()); + TsFileResourceUtils.validateTsFileDataCorrectness(tsFileManager.getTsFileList(true).get(0)); + Assert.assertEquals( + 1, ((long) tsFileManager.getTsFileList(true).get(0).getStartTime(device).get())); + } + + @Test + public void testCompactAlignedSeriesWithReadPointCompactionPerformer() + throws IOException, WriteProcessException { + generateDataTypeNotMatchFilesWithAlignedSeries(); + InnerSpaceCompactionTask task = + new InnerSpaceCompactionTask( + 0, tsFileManager, seqResources, true, new ReadPointCompactionPerformer(), 0); + Assert.assertTrue(task.start()); + TsFileResourceUtils.validateTsFileDataCorrectness(tsFileManager.getTsFileList(true).get(0)); + Assert.assertEquals( + 1, ((long) tsFileManager.getTsFileList(true).get(0).getStartTime(device).get())); + } + + private void generateDataTypeNotMatchFilesWithNonAlignedSeries() + throws IOException, WriteProcessException { + MeasurementSchema measurementSchema1 = new MeasurementSchema("s1", TSDataType.INT32); + TsFileResource resource1 = createEmptyFileAndResource(true); + resource1.setStatusForTest(TsFileResourceStatus.COMPACTING); + try (TsFileWriter writer = new TsFileWriter(resource1.getTsFile())) { + writer.registerTimeseries(new Path(device), measurementSchema1); + TSRecord record = new TSRecord(device, 1); + record.addTuple(new IntDataPoint("s1", 1)); + writer.writeRecord(record); + writer.flush(); + } + resource1.updateStartTime(device, 1); + resource1.updateEndTime(device, 1); + resource1.serialize(); + seqResources.add(resource1); + + MeasurementSchema measurementSchema2 = new MeasurementSchema("s1", TSDataType.FLOAT); + TsFileResource resource2 = createEmptyFileAndResource(true); + resource2.setStatusForTest(TsFileResourceStatus.COMPACTING); + try (TsFileWriter writer = new TsFileWriter(resource2.getTsFile())) { + writer.registerTimeseries(new Path(device), measurementSchema2); + TSRecord record = new TSRecord(device, 2); + record.addTuple(new FloatDataPoint("s1", 2)); + writer.writeRecord(record); + writer.flush(); + } + resource2.updateStartTime(device, 2); + resource2.updateEndTime(device, 2); + resource2.serialize(); + seqResources.add(resource2); + } + + private void generateDataTypeNotMatchFilesWithAlignedSeries() + throws IOException, WriteProcessException { + List measurementSchemas1 = new ArrayList<>(); + measurementSchemas1.add(new MeasurementSchema("s1", TSDataType.INT32)); + measurementSchemas1.add(new MeasurementSchema("s2", TSDataType.INT32)); + + TsFileResource resource1 = createEmptyFileAndResource(true); + resource1.setStatusForTest(TsFileResourceStatus.COMPACTING); + try (TsFileWriter writer = new TsFileWriter(resource1.getTsFile())) { + writer.registerAlignedTimeseries(new Path(device), measurementSchemas1); + TSRecord record = new TSRecord(device, 1); + record.addTuple(new IntDataPoint("s1", 0)); + record.addTuple(new IntDataPoint("s2", 1)); + writer.writeRecord(record); + writer.flush(); + } + resource1.updateStartTime(device, 1); + resource1.updateEndTime(device, 1); + resource1.serialize(); + seqResources.add(resource1); + + List measurementSchemas2 = new ArrayList<>(); + measurementSchemas2.add(new MeasurementSchema("s1", TSDataType.FLOAT)); + measurementSchemas2.add(new MeasurementSchema("s2", TSDataType.DOUBLE)); + TsFileResource resource2 = createEmptyFileAndResource(true); + resource2.setStatusForTest(TsFileResourceStatus.COMPACTING); + try (TsFileWriter writer = new TsFileWriter(resource2.getTsFile())) { + writer.registerAlignedTimeseries(new Path(device), measurementSchemas2); + TSRecord record = new TSRecord(device, 2); + record.addTuple(new FloatDataPoint("s1", 2)); + record.addTuple(new DoubleDataPoint("s2", 3)); + writer.writeRecord(record); + writer.flush(); + } + resource2.updateStartTime(device, 2); + resource2.updateEndTime(device, 2); + resource2.serialize(); + seqResources.add(resource2); + } +} diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/FastNonAlignedCrossCompactionTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/FastNonAlignedCrossCompactionTest.java index 8d4b5131c7a19..64d70d30cdb10 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/FastNonAlignedCrossCompactionTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/FastNonAlignedCrossCompactionTest.java @@ -66,6 +66,8 @@ public class FastNonAlignedCrossCompactionTest extends AbstractCompactionTest { + private boolean prevUseMultiType = false; + @Before public void setUp() throws IOException, WriteProcessException, MetadataException, InterruptedException { @@ -74,6 +76,8 @@ public void setUp() IoTDBDescriptor.getInstance().getConfig().setTargetChunkPointNum(100); TSFileDescriptor.getInstance().getConfig().setMaxNumberOfPointsInPage(30); TSFileDescriptor.getInstance().getConfig().setMaxDegreeOfIndexNode(3); + prevUseMultiType = org.apache.tsfile.utils.TsFileGeneratorUtils.useMultiType; + org.apache.tsfile.utils.TsFileGeneratorUtils.useMultiType = true; } @After @@ -85,6 +89,7 @@ public void tearDown() throws IOException, StorageEngineException { for (TsFileResource tsFileResource : unseqResources) { FileReaderManager.getInstance().closeFileAndRemoveReader(tsFileResource.getTsFileID()); } + org.apache.tsfile.utils.TsFileGeneratorUtils.useMultiType = prevUseMultiType; } @Test diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/ReadPointNonAlignedCrossCompactionTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/ReadPointNonAlignedCrossCompactionTest.java index 645fb0d781b27..289ff7756b8e9 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/ReadPointNonAlignedCrossCompactionTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/ReadPointNonAlignedCrossCompactionTest.java @@ -65,6 +65,7 @@ public class ReadPointNonAlignedCrossCompactionTest extends AbstractCompactionTest { private final String oldThreadName = Thread.currentThread().getName(); + private boolean prevUseMultiType = false; @Before public void setUp() @@ -75,6 +76,8 @@ public void setUp() TSFileDescriptor.getInstance().getConfig().setMaxNumberOfPointsInPage(30); TSFileDescriptor.getInstance().getConfig().setMaxDegreeOfIndexNode(3); Thread.currentThread().setName("pool-1-IoTDB-Compaction-Worker-1"); + prevUseMultiType = org.apache.tsfile.utils.TsFileGeneratorUtils.useMultiType; + org.apache.tsfile.utils.TsFileGeneratorUtils.useMultiType = true; } @After @@ -87,6 +90,7 @@ public void tearDown() throws IOException, StorageEngineException { for (TsFileResource tsFileResource : unseqResources) { FileReaderManager.getInstance().closeFileAndRemoveReader(tsFileResource.getTsFileID()); } + org.apache.tsfile.utils.TsFileGeneratorUtils.useMultiType = prevUseMultiType; } @Test diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/SchemaUtilsTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/SchemaUtilsTest.java index 85a10c7aaf32f..17dffd6978509 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/SchemaUtilsTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/SchemaUtilsTest.java @@ -22,11 +22,18 @@ import org.apache.iotdb.db.utils.constant.SqlConstant; import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.AbstractAlignedChunkMetadata; +import org.apache.tsfile.file.metadata.AlignedChunkMetadata; +import org.apache.tsfile.file.metadata.ChunkMetadata; +import org.apache.tsfile.file.metadata.IChunkMetadata; +import org.apache.tsfile.file.metadata.enums.CompressionType; import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.file.metadata.statistics.Statistics; import org.junit.Assert; import org.junit.Test; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.List; @@ -73,4 +80,36 @@ public void checkDataTypeWithEncoding() { // do nothing } } + + @Test + public void rewriteAlignedChunkMetadataStatistics() { + for (TSDataType targetDataType : Arrays.asList(TSDataType.STRING, TSDataType.TEXT)) { + for (TSDataType tsDataType : TSDataType.values()) { + if (tsDataType == TSDataType.UNKNOWN) { + continue; + } + List valueChunkMetadatas = + Collections.singletonList( + new ChunkMetadata( + "s0", + tsDataType, + TSEncoding.RLE, + CompressionType.LZ4, + 0, + Statistics.getStatsByType(tsDataType))); + AlignedChunkMetadata alignedChunkMetadata = + new AlignedChunkMetadata(new ChunkMetadata(), valueChunkMetadatas); + try { + AbstractAlignedChunkMetadata abstractAlignedChunkMetadata = + SchemaUtils.rewriteAlignedChunkMetadataStatistics( + alignedChunkMetadata, targetDataType); + Assert.assertEquals( + targetDataType, + abstractAlignedChunkMetadata.getValueChunkMetadataList().get(0).getDataType()); + } catch (ClassCastException e) { + Assert.fail(e.getMessage()); + } + } + } + } } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/datastructure/AlignedTVListTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/datastructure/AlignedTVListTest.java index 041a9b09afc5d..10d7803a83ffb 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/datastructure/AlignedTVListTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/datastructure/AlignedTVListTest.java @@ -228,24 +228,24 @@ public void testCalculateChunkSize() { tvList.deleteColumn(0); Assert.assertEquals(tvList.memoryBinaryChunkSize.length, 2); - Assert.assertEquals(tvList.memoryBinaryChunkSize[1], 720); + Assert.assertEquals(tvList.memoryBinaryChunkSize[0], 0); tvList.extendColumn(TSDataType.INT32); Assert.assertEquals(tvList.memoryBinaryChunkSize.length, 3); - Assert.assertEquals(tvList.memoryBinaryChunkSize[1], 720); + Assert.assertEquals(tvList.memoryBinaryChunkSize[0], 0); tvList.extendColumn(TSDataType.TEXT); Assert.assertEquals(tvList.memoryBinaryChunkSize.length, 4); + Assert.assertEquals(tvList.memoryBinaryChunkSize[0], 0); Assert.assertEquals(tvList.memoryBinaryChunkSize[1], 720); - Assert.assertEquals(tvList.memoryBinaryChunkSize[2], 0); tvList.delete(4, 6); Assert.assertEquals(tvList.memoryBinaryChunkSize.length, 4); + Assert.assertEquals(tvList.memoryBinaryChunkSize[0], 0); Assert.assertEquals(tvList.memoryBinaryChunkSize[1], 720); - Assert.assertEquals(tvList.memoryBinaryChunkSize[2], 0); tvList.clear(); + Assert.assertEquals(tvList.memoryBinaryChunkSize[0], 0); Assert.assertEquals(tvList.memoryBinaryChunkSize[1], 0); - Assert.assertEquals(tvList.memoryBinaryChunkSize[2], 0); } } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/datastructure/IntTVListTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/datastructure/IntTVListTest.java index 745c8e1230bb9..1235488fd911f 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/datastructure/IntTVListTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/datastructure/IntTVListTest.java @@ -18,6 +18,7 @@ */ package org.apache.iotdb.db.utils.datastructure; +import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.external.commons.lang3.ArrayUtils; import org.apache.tsfile.utils.BitMap; import org.junit.Assert; @@ -30,7 +31,7 @@ public class IntTVListTest { @Test public void testIntTVList1() { - IntTVList tvList = IntTVList.newList(); + IntTVList tvList = IntTVList.newList(TSDataType.INT32); for (int i = 0; i < 1000; i++) { tvList.putInt(i, i); } @@ -43,7 +44,7 @@ public void testIntTVList1() { @Test public void testIntTVList2() { - IntTVList tvList = IntTVList.newList(); + IntTVList tvList = IntTVList.newList(TSDataType.INT32); for (int i = 1000; i >= 0; i--) { tvList.putInt(i, i); } @@ -56,7 +57,7 @@ public void testIntTVList2() { @Test public void testPutIntsWithoutBitMap() { - IntTVList tvList = IntTVList.newList(); + IntTVList tvList = IntTVList.newList(TSDataType.INT32); List intList = new ArrayList<>(); List timeList = new ArrayList<>(); for (int i = 1000; i >= 0; i--) { @@ -77,7 +78,7 @@ public void testPutIntsWithoutBitMap() { @Test public void testPutIntsWithBitMap() { - IntTVList tvList = IntTVList.newList(); + IntTVList tvList = IntTVList.newList(TSDataType.INT32); List intList = new ArrayList<>(); List timeList = new ArrayList<>(); BitMap bitMap = new BitMap(1001); @@ -108,7 +109,7 @@ public void testPutIntsWithBitMap() { @Test public void testClone() { - IntTVList tvList = IntTVList.newList(); + IntTVList tvList = IntTVList.newList(TSDataType.INT32); List intList = new ArrayList<>(); List timeList = new ArrayList<>(); BitMap bitMap = new BitMap(1001); diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/path/MeasurementPath.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/path/MeasurementPath.java index 0718366aabfc0..56334a8aa477e 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/path/MeasurementPath.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/path/MeasurementPath.java @@ -331,6 +331,37 @@ public static MeasurementPath deserialize(ByteBuffer byteBuffer) { return measurementPath; } + public static MeasurementPath deserializeDirectly(ByteBuffer byteBuffer) { + PartialPath partialPath = PartialPath.deserialize(byteBuffer); + MeasurementPath measurementPath = new MeasurementPath(); + byte isNull = ReadWriteIOUtils.readByte(byteBuffer); + if (isNull == 1) { + byte type = ReadWriteIOUtils.readByte(byteBuffer); + if (type == MeasurementSchemaType.MEASUREMENT_SCHEMA.getMeasurementSchemaTypeInByteEnum()) { + measurementPath.measurementSchema = MeasurementSchema.deserializeFrom(byteBuffer); + } else if (type + == MeasurementSchemaType.VECTOR_MEASUREMENT_SCHEMA.getMeasurementSchemaTypeInByteEnum()) { + measurementPath.measurementSchema = VectorMeasurementSchema.deserializeFrom(byteBuffer); + } else if (type + == MeasurementSchemaType.LOGICAL_VIEW_SCHEMA.getMeasurementSchemaTypeInByteEnum()) { + measurementPath.measurementSchema = LogicalViewSchema.deserializeFrom(byteBuffer); + } else { + throw new RuntimeException( + new UnexpectedException("Type (" + type + ") of measurementSchema is unknown.")); + } + } + isNull = ReadWriteIOUtils.readByte(byteBuffer); + if (isNull == 1) { + measurementPath.tagMap = ReadWriteIOUtils.readMap(byteBuffer); + } + measurementPath.isUnderAlignedEntity = ReadWriteIOUtils.readBoolObject(byteBuffer); + measurementPath.measurementAlias = ReadWriteIOUtils.readString(byteBuffer); + measurementPath.nodes = partialPath.getNodes(); + measurementPath.device = measurementPath.getIDeviceID(); + measurementPath.fullPath = measurementPath.getFullPath(); + return measurementPath; + } + @Override public PartialPath transformToPartialPath() { return getDevicePath().concatNode(getTailNode()); diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/AlterOrDropTableOperationType.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/AlterOrDropTableOperationType.java index 0debc9ffcd00c..46a8d5ddf3e3d 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/AlterOrDropTableOperationType.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/AlterOrDropTableOperationType.java @@ -27,7 +27,8 @@ public enum AlterOrDropTableOperationType { RENAME_TABLE((byte) 4), DROP_TABLE((byte) 5), COMMENT_TABLE((byte) 6), - COMMENT_COLUMN((byte) 7); + COMMENT_COLUMN((byte) 7), + ALTER_COLUMN_DATA_TYPE((byte) 8); private final byte type; @@ -57,6 +58,8 @@ public static AlterOrDropTableOperationType getType(final byte value) { return COMMENT_TABLE; case 7: return COMMENT_COLUMN; + case 8: + return ALTER_COLUMN_DATA_TYPE; default: throw new IllegalArgumentException(); } diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/TsTable.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/TsTable.java index ae7fb8e38c930..8f5f5f44f6055 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/TsTable.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/TsTable.java @@ -39,7 +39,6 @@ import javax.annotation.concurrent.ThreadSafe; -import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; @@ -77,6 +76,7 @@ public class TsTable { private final Map columnSchemaMap = new LinkedHashMap<>(); private final Map tagColumnIndexMap = new HashMap<>(); + private final Map idColumnIndexMap = new HashMap<>(); private final ReadWriteLock readWriteLock = new ReentrantReadWriteLock(); @@ -99,6 +99,8 @@ public class TsTable { private transient long ttlValue = Long.MIN_VALUE; private transient int tagNums = 0; private transient int fieldNum = 0; + private transient int idNums = 0; + private transient int measurementNum = 0; public TsTable(final String tableName) { this.tableName = tableName; @@ -112,6 +114,16 @@ public TsTable(String tableName, ImmutableList columnSchema columnSchema -> columnSchemaMap.put(columnSchema.getColumnName(), columnSchema)); } + public TsTable(TsTable origin) { + this.tableName = origin.tableName; + origin.columnSchemaMap.forEach((col, schema) -> this.columnSchemaMap.put(col, schema.copy())); + this.idColumnIndexMap.putAll(origin.idColumnIndexMap); + this.props = origin.props == null ? null : new HashMap<>(origin.props); + this.ttlValue = origin.ttlValue; + this.idNums = origin.idNums; + this.measurementNum = origin.measurementNum; + } + public String getTableName() { return tableName; } @@ -363,16 +375,6 @@ public void removeProp(final String key) { }); } - public byte[] serialize() { - ByteArrayOutputStream stream = new ByteArrayOutputStream(); - try { - serialize(stream); - } catch (IOException ignored) { - // won't happen - } - return stream.toByteArray(); - } - public void serialize(final OutputStream stream) throws IOException { ReadWriteIOUtils.write(tableName, stream); ReadWriteIOUtils.write(columnSchemaMap.size(), stream); diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/column/AttributeColumnSchema.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/column/AttributeColumnSchema.java index 8a3f21930d2cc..513aecdf61cae 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/column/AttributeColumnSchema.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/column/AttributeColumnSchema.java @@ -25,6 +25,7 @@ import java.io.IOException; import java.io.InputStream; import java.nio.ByteBuffer; +import java.util.HashMap; import java.util.Map; public class AttributeColumnSchema extends TsTableColumnSchema { @@ -55,4 +56,10 @@ static AttributeColumnSchema deserialize(final ByteBuffer buffer) { final Map props = ReadWriteIOUtils.readMap(buffer); return new AttributeColumnSchema(columnName, dataType, props); } + + @Override + public TsTableColumnSchema copy() { + return new AttributeColumnSchema( + columnName, dataType, props == null ? null : new HashMap<>(props)); + } } diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/column/FieldColumnSchema.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/column/FieldColumnSchema.java index d6729a73b6c30..0e09f6e500826 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/column/FieldColumnSchema.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/column/FieldColumnSchema.java @@ -30,6 +30,7 @@ import java.io.InputStream; import java.io.OutputStream; import java.nio.ByteBuffer; +import java.util.HashMap; import java.util.Map; public class FieldColumnSchema extends TsTableColumnSchema { @@ -115,4 +116,10 @@ static FieldColumnSchema deserialize(final ByteBuffer buffer) { final Map props = ReadWriteIOUtils.readMap(buffer); return new FieldColumnSchema(columnName, dataType, encoding, compressor, props); } + + @Override + public TsTableColumnSchema copy() { + return new FieldColumnSchema( + columnName, dataType, encoding, compressor, props == null ? null : new HashMap<>(props)); + } } diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/column/TagColumnSchema.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/column/TagColumnSchema.java index 83c7bde36b175..8ba20a146f9a9 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/column/TagColumnSchema.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/column/TagColumnSchema.java @@ -25,6 +25,7 @@ import java.io.IOException; import java.io.InputStream; import java.nio.ByteBuffer; +import java.util.HashMap; import java.util.Map; public class TagColumnSchema extends TsTableColumnSchema { @@ -55,4 +56,9 @@ static TagColumnSchema deserialize(final ByteBuffer buffer) { Map props = ReadWriteIOUtils.readMap(buffer); return new TagColumnSchema(columnName, dataType, props); } + + @Override + public TsTableColumnSchema copy() { + return new TagColumnSchema(columnName, dataType, props == null ? null : new HashMap<>(props)); + } } diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/column/TimeColumnSchema.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/column/TimeColumnSchema.java index 4349d3e134a77..597cf51b53cf3 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/column/TimeColumnSchema.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/column/TimeColumnSchema.java @@ -25,6 +25,7 @@ import java.io.IOException; import java.io.InputStream; import java.nio.ByteBuffer; +import java.util.HashMap; import java.util.Map; public class TimeColumnSchema extends TsTableColumnSchema { @@ -55,4 +56,9 @@ static TimeColumnSchema deserialize(final ByteBuffer buffer) { final Map props = ReadWriteIOUtils.readMap(buffer); return new TimeColumnSchema(columnName, dataType, props); } + + @Override + public TsTableColumnSchema copy() { + return new TimeColumnSchema(columnName, dataType, props == null ? null : new HashMap<>(props)); + } } diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/column/TsTableColumnSchema.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/column/TsTableColumnSchema.java index 693bf930fd2e7..8f773f19be497 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/column/TsTableColumnSchema.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/column/TsTableColumnSchema.java @@ -89,6 +89,12 @@ public int hashCode() { return Objects.hash(columnName); } + public void setDataType(final TSDataType dataType) { + this.dataType = dataType; + } + + public abstract TsTableColumnSchema copy(); + @Override public String toString() { return toStringHelper(this) diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/column/TsTableColumnSchemaUtil.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/column/TsTableColumnSchemaUtil.java index 2c2f2340fdef1..ab6bbe2ac8a19 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/column/TsTableColumnSchemaUtil.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/column/TsTableColumnSchemaUtil.java @@ -19,6 +19,7 @@ package org.apache.iotdb.commons.schema.table.column; +import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.utils.ReadWriteIOUtils; import java.io.ByteArrayOutputStream; @@ -103,6 +104,17 @@ public static void serialize(List columnSchemaList, OutputS } } + public static byte[] serialize(String columnName, TSDataType dataType) { + ByteArrayOutputStream stream = new ByteArrayOutputStream(); + try { + ReadWriteIOUtils.writeVar(columnName, stream); + stream.write(dataType.serialize()); + } catch (IOException ignored) { + + } + return stream.toByteArray(); + } + public static List deserializeColumnSchemaList(ByteBuffer buffer) { int size = ReadWriteIOUtils.readInt(buffer); if (size == -1) { diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/tree/AlterTimeSeriesOperationType.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/tree/AlterTimeSeriesOperationType.java new file mode 100644 index 0000000000000..d079b5e527a3c --- /dev/null +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/tree/AlterTimeSeriesOperationType.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.commons.schema.tree; + +public enum AlterTimeSeriesOperationType { + ALTER_DATA_TYPE((byte) 0); + + private final byte type; + + AlterTimeSeriesOperationType(final byte type) { + this.type = type; + } + + public byte getTypeValue() { + return type; + } + + public static AlterTimeSeriesOperationType getType(final byte value) { + switch (value) { + case 0: + return ALTER_DATA_TYPE; + default: + throw new IllegalArgumentException(); + } + } +} diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/MetadataUtils.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/MetadataUtils.java new file mode 100644 index 0000000000000..e23e138140b06 --- /dev/null +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/MetadataUtils.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.commons.utils; + +import org.apache.tsfile.enums.TSDataType; + +public class MetadataUtils { + + private MetadataUtils() { + // util class + } + + public static boolean canAlter(TSDataType from, TSDataType to) { + return to.isCompatible(from); + } +} diff --git a/iotdb-core/relational-grammar/src/main/antlr4/org/apache/iotdb/db/relational/grammar/sql/RelationalSql.g4 b/iotdb-core/relational-grammar/src/main/antlr4/org/apache/iotdb/db/relational/grammar/sql/RelationalSql.g4 index 1690ea4c855b0..183a878a48692 100644 --- a/iotdb-core/relational-grammar/src/main/antlr4/org/apache/iotdb/db/relational/grammar/sql/RelationalSql.g4 +++ b/iotdb-core/relational-grammar/src/main/antlr4/org/apache/iotdb/db/relational/grammar/sql/RelationalSql.g4 @@ -255,6 +255,7 @@ alterTableStatement | ALTER TABLE (IF EXISTS)? tableName=qualifiedName DROP COLUMN (IF EXISTS)? column=identifier #dropColumn // set TTL can use this | ALTER TABLE (IF EXISTS)? tableName=qualifiedName SET PROPERTIES propertyAssignments #setTableProperties + | ALTER TABLE (IF EXISTS)? tableName=qualifiedName ALTER COLUMN (IF EXISTS)? column=identifier SET DATA TYPE new_type=type #alterColumnDataType ; commentStatement diff --git a/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift b/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift index 01bde7c378dc1..9680f11f138c7 100644 --- a/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift +++ b/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift @@ -944,6 +944,13 @@ struct TDeleteTimeSeriesReq { 4: optional bool mayDeleteAudit } +struct TAlterTimeSeriesReq { + 1: required string queryId + 2: required binary measurementPath + 3: required byte operationType + 4: required binary updateInfo +} + struct TDeleteLogicalViewReq { 1: required string queryId 2: required binary pathPatternTree @@ -1223,6 +1230,7 @@ struct TDescTableResp { 1: required common.TSStatus status 2: optional binary tableInfo 3: optional set preDeletedColumns + 4: optional map preAlteredColumns } struct TDescTable4InformationSchemaResp { @@ -1233,6 +1241,7 @@ struct TDescTable4InformationSchemaResp { struct TTableColumnInfo { 1: required binary tableInfo 2: optional set preDeletedColumns + 3: optional map preAlteredColumns } struct TFetchTableResp { @@ -1842,6 +1851,11 @@ service IConfigNodeRPCService { */ common.TSStatus deleteTimeSeries(TDeleteTimeSeriesReq req) + /** + * Alter timeseries measurement + **/ + common.TSStatus alterTimeSeriesDataType(TAlterTimeSeriesReq req) + common.TSStatus deleteLogicalView(TDeleteLogicalViewReq req) common.TSStatus alterLogicalView(TAlterLogicalViewReq req) diff --git a/iotdb-protocol/thrift-datanode/src/main/thrift/datanode.thrift b/iotdb-protocol/thrift-datanode/src/main/thrift/datanode.thrift index 469d2bb006aef..942a04f5f2a77 100644 --- a/iotdb-protocol/thrift-datanode/src/main/thrift/datanode.thrift +++ b/iotdb-protocol/thrift-datanode/src/main/thrift/datanode.thrift @@ -473,6 +473,14 @@ struct TAlterEncodingCompressorReq { 5: optional byte compressor } +struct TAlterTimeSeriesReq { + 1: required list schemaRegionIdList + 2: required string queryId + 3: required binary measurementPath + 4: required byte operationType + 5: required binary updateInfo +} + struct TConstructSchemaBlackListWithTemplateReq { 1: required list schemaRegionIdList 2: required map> templateSetInfo @@ -1092,6 +1100,11 @@ service IDataNodeRPCService { */ common.TSStatus alterEncodingCompressor(TAlterEncodingCompressorReq req) + /** + * Alter timeseries measurement + **/ + common.TSStatus alterTimeSeriesDataType(TAlterTimeSeriesReq req) + /** * Construct schema black list in target schemaRegion to block R/W on matched timeseries represent by template */ @@ -1217,7 +1230,6 @@ service IDataNodeRPCService { */ common.TSStatus deleteColumnData(TDeleteColumnDataReq req) - /** * Construct table device black list */ diff --git a/pom.xml b/pom.xml index 07651de14c16d..53dd7b438db1a 100644 --- a/pom.xml +++ b/pom.xml @@ -173,7 +173,7 @@ 0.14.1 1.9 1.5.6-3 - 2.2.0-251223-SNAPSHOT + 2.2.1-251230-SNAPSHOT