From efa81ce3b7c2012714e6417bd602d8e060f8f9b1 Mon Sep 17 00:00:00 2001 From: Ritvi Bhatt Date: Thu, 5 Mar 2026 08:31:23 -0800 Subject: [PATCH 01/22] add time conversion functions Signed-off-by: Ritvi Bhatt --- .../org/opensearch/sql/ast/tree/Convert.java | 1 + .../sql/calcite/CalciteRelNodeVisitor.java | 28 ++- .../function/PPLBuiltinOperators.java | 8 + .../function/udf/CTimeConvertFunction.java | 133 +++++++++++++ .../function/udf/Dur2SecConvertFunction.java | 86 ++++++++ .../function/udf/MkTimeConvertFunction.java | 183 ++++++++++++++++++ .../function/udf/MsTimeConvertFunction.java | 88 +++++++++ .../function/udf/ConversionFunctionsTest.java | 137 +++++++++++++ ppl/src/main/antlr/OpenSearchPPLParser.g4 | 2 +- .../opensearch/sql/ppl/parser/AstBuilder.java | 10 +- 10 files changed, 667 insertions(+), 9 deletions(-) create mode 100644 core/src/main/java/org/opensearch/sql/expression/function/udf/CTimeConvertFunction.java create mode 100644 core/src/main/java/org/opensearch/sql/expression/function/udf/Dur2SecConvertFunction.java create mode 100644 core/src/main/java/org/opensearch/sql/expression/function/udf/MkTimeConvertFunction.java create mode 100644 core/src/main/java/org/opensearch/sql/expression/function/udf/MsTimeConvertFunction.java diff --git a/core/src/main/java/org/opensearch/sql/ast/tree/Convert.java b/core/src/main/java/org/opensearch/sql/ast/tree/Convert.java index 74406b0daf2..259330b2dba 100644 --- a/core/src/main/java/org/opensearch/sql/ast/tree/Convert.java +++ b/core/src/main/java/org/opensearch/sql/ast/tree/Convert.java @@ -23,6 +23,7 @@ @RequiredArgsConstructor public class Convert extends UnresolvedPlan { private final List conversions; + private final String timeFormat; private UnresolvedPlan child; @Override diff --git a/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java b/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java index edfe0b22e85..c3986f3afc9 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java +++ b/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java @@ -1008,7 +1008,7 @@ public RelNode visitConvert(Convert node, CalcitePlanContext context) { ConversionState state = new ConversionState(); for (Let conversion : node.getConversions()) { - processConversion(conversion, state, context); + processConversion(conversion, node.getTimeFormat(), state, context); } return buildConversionProjection(state, context); @@ -1021,14 +1021,14 @@ private static class ConversionState { } private void processConversion( - Let conversion, ConversionState state, CalcitePlanContext context) { + Let conversion, String timeFormat, ConversionState state, CalcitePlanContext context) { String target = conversion.getVar().getField().toString(); UnresolvedExpression expression = conversion.getExpression(); if (expression instanceof Field) { processFieldCopyConversion(target, (Field) expression, state, context); } else if (expression instanceof Function) { - processFunctionConversion(target, (Function) expression, state, context); + processFunctionConversion(target, (Function) expression, timeFormat, state, context); } else { throw new SemanticCheckException("Convert command requires function call expressions"); } @@ -1051,7 +1051,7 @@ private void processFieldCopyConversion( } private void processFunctionConversion( - String target, Function function, ConversionState state, CalcitePlanContext context) { + String target, Function function, String timeFormat, ConversionState state, CalcitePlanContext context) { String functionName = function.getFuncName(); List args = function.getFuncArgs(); @@ -1068,8 +1068,7 @@ private void processFunctionConversion( state.seenFields.add(source); RexNode sourceField = context.relBuilder.field(source); - RexNode convertCall = - PPLFuncImpTable.INSTANCE.resolve(context.rexBuilder, functionName, sourceField); + RexNode convertCall = resolveConvertFunction(functionName, sourceField, timeFormat, context); if (!target.equals(source)) { state.additions.add(Pair.of(target, context.relBuilder.alias(convertCall, target))); @@ -1078,6 +1077,23 @@ private void processFunctionConversion( } } + private RexNode resolveConvertFunction( + String functionName, RexNode sourceField, String timeFormat, CalcitePlanContext context) { + + // Time functions that support timeformat parameter + Set timeFunctions = Set.of("ctime", "mktime"); + + if (timeFunctions.contains(functionName.toLowerCase()) && timeFormat != null) { + // For time functions with custom timeformat, pass the format as a second parameter + RexNode timeFormatLiteral = context.rexBuilder.makeLiteral(timeFormat); + return PPLFuncImpTable.INSTANCE.resolve( + context.rexBuilder, functionName, sourceField, timeFormatLiteral); + } else { + // Regular conversion functions or time functions without custom format + return PPLFuncImpTable.INSTANCE.resolve(context.rexBuilder, functionName, sourceField); + } + } + private RelNode buildConversionProjection(ConversionState state, CalcitePlanContext context) { List originalFields = context.relBuilder.peek().getRowType().getFieldNames(); List projectList = new ArrayList<>(); diff --git a/core/src/main/java/org/opensearch/sql/expression/function/PPLBuiltinOperators.java b/core/src/main/java/org/opensearch/sql/expression/function/PPLBuiltinOperators.java index 2aebf7efe34..6e3440acd48 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/PPLBuiltinOperators.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/PPLBuiltinOperators.java @@ -73,6 +73,10 @@ import org.opensearch.sql.expression.function.udf.RexOffsetFunction; import org.opensearch.sql.expression.function.udf.RmcommaConvertFunction; import org.opensearch.sql.expression.function.udf.RmunitConvertFunction; +import org.opensearch.sql.expression.function.udf.CTimeConvertFunction; +import org.opensearch.sql.expression.function.udf.MkTimeConvertFunction; +import org.opensearch.sql.expression.function.udf.MsTimeConvertFunction; +import org.opensearch.sql.expression.function.udf.Dur2SecConvertFunction; import org.opensearch.sql.expression.function.udf.SpanFunction; import org.opensearch.sql.expression.function.udf.ToNumberFunction; import org.opensearch.sql.expression.function.udf.ToStringFunction; @@ -431,6 +435,10 @@ public class PPLBuiltinOperators extends ReflectiveSqlOperatorTable { public static final SqlOperator RMCOMMA = new RmcommaConvertFunction().toUDF("RMCOMMA"); public static final SqlOperator RMUNIT = new RmunitConvertFunction().toUDF("RMUNIT"); public static final SqlOperator MEMK = new MemkConvertFunction().toUDF("MEMK"); + public static final SqlOperator CTIME = new CTimeConvertFunction().toUDF("CTIME"); + public static final SqlOperator MKTIME = new MkTimeConvertFunction().toUDF("MKTIME"); + public static final SqlOperator MSTIME = new MsTimeConvertFunction().toUDF("MSTIME"); + public static final SqlOperator DUR2SEC = new Dur2SecConvertFunction().toUDF("DUR2SEC"); public static final SqlOperator WIDTH_BUCKET = new org.opensearch.sql.expression.function.udf.binning.WidthBucketFunction() diff --git a/core/src/main/java/org/opensearch/sql/expression/function/udf/CTimeConvertFunction.java b/core/src/main/java/org/opensearch/sql/expression/function/udf/CTimeConvertFunction.java new file mode 100644 index 00000000000..a8dddd8dea0 --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/expression/function/udf/CTimeConvertFunction.java @@ -0,0 +1,133 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.expression.function.udf; + +import java.time.Instant; +import java.time.ZoneId; +import java.time.ZoneOffset; +import java.time.format.DateTimeFormatter; +import java.util.List; +import org.apache.calcite.adapter.enumerable.NotNullImplementor; +import org.apache.calcite.adapter.enumerable.NullPolicy; +import org.apache.calcite.adapter.enumerable.RexToLixTranslator; +import org.apache.calcite.linq4j.function.Strict; +import org.apache.calcite.linq4j.tree.Expression; +import org.apache.calcite.linq4j.tree.Expressions; +import org.apache.calcite.rex.RexCall; +import org.apache.calcite.sql.type.SqlReturnTypeInference; +import org.opensearch.sql.calcite.utils.PPLOperandTypes; +import org.opensearch.sql.calcite.utils.PPLReturnTypes; +import org.opensearch.sql.expression.function.ImplementorUDF; +import org.opensearch.sql.expression.function.UDFOperandMetadata; + +/** + * PPL ctime() conversion function. + * Converts UNIX timestamps to human-readable format. + * Format: "Mon Oct 13 20:07:13 PDT 2003" + */ +public class CTimeConvertFunction extends ImplementorUDF { + + // Format pattern matching expected output: "Mon Oct 13 20:07:13 PDT 2003" + private static final DateTimeFormatter CTIME_FORMATTER = + DateTimeFormatter.ofPattern("EEE MMM dd HH:mm:ss zzz yyyy"); + + public CTimeConvertFunction() { + super(new CTimeImplementor(), NullPolicy.ANY); + } + + @Override + public SqlReturnTypeInference getReturnTypeInference() { + return PPLReturnTypes.STRING_FORCE_NULLABLE; + } + + @Override + public UDFOperandMetadata getOperandMetadata() { + return PPLOperandTypes.OPTIONAL_ANY; + } + + public static class CTimeImplementor implements NotNullImplementor { + @Override + public Expression implement( + RexToLixTranslator translator, RexCall call, List translatedOperands) { + if (translatedOperands.isEmpty()) { + return Expressions.constant(null, String.class); + } + + Expression fieldValue = translatedOperands.get(0); + + if (translatedOperands.size() == 1) { + // Single parameter: use default ctime format + return Expressions.call(CTimeConvertFunction.class, "convert", fieldValue); + } else { + // Two parameters: field value and custom timeformat + Expression timeFormat = translatedOperands.get(1); + return Expressions.call(CTimeConvertFunction.class, "convertWithFormat", fieldValue, timeFormat); + } + } + } + + @Strict + public static String convert(Object value) { + if (value == null) { + return null; + } + + try { + double timestamp; + if (value instanceof Number) { + timestamp = ((Number) value).doubleValue(); + } else { + String str = value.toString().trim(); + if (str.isEmpty()) { + return null; + } + timestamp = Double.parseDouble(str); + } + + // Convert to Instant and format + Instant instant = Instant.ofEpochSecond((long) timestamp); + return CTIME_FORMATTER.format(instant.atZone(ZoneOffset.UTC)); + + } catch (Exception e) { + return null; + } + } + + @Strict + public static String convertWithFormat(Object value, Object timeFormatObj) { + if (value == null) { + return null; + } + + String customFormat = timeFormatObj != null ? timeFormatObj.toString().trim() : null; + if (customFormat == null || customFormat.isEmpty()) { + // Fall back to default ctime format + return convert(value); + } + + try { + double timestamp; + if (value instanceof Number) { + timestamp = ((Number) value).doubleValue(); + } else { + String str = value.toString().trim(); + if (str.isEmpty()) { + return null; + } + timestamp = Double.parseDouble(str); + } + + // Convert to Instant and format with custom formatter + Instant instant = Instant.ofEpochSecond((long) timestamp); + DateTimeFormatter customFormatter = DateTimeFormatter.ofPattern(customFormat); + return customFormatter.format(instant.atZone(ZoneOffset.UTC)); + + } catch (Exception e) { + // If custom format fails, fall back to default ctime format + return convert(value); + } + } +} \ No newline at end of file diff --git a/core/src/main/java/org/opensearch/sql/expression/function/udf/Dur2SecConvertFunction.java b/core/src/main/java/org/opensearch/sql/expression/function/udf/Dur2SecConvertFunction.java new file mode 100644 index 00000000000..a4603ecbafa --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/expression/function/udf/Dur2SecConvertFunction.java @@ -0,0 +1,86 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.expression.function.udf; + +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * PPL dur2sec() conversion function. + * Converts duration format [D+]HH:MM:SS to seconds. + * Examples: + * - "01:23:45" -> 5025 seconds (1*3600 + 23*60 + 45) + * - "2+12:30:15" -> 217815 seconds (2*24*3600 + 12*3600 + 30*60 + 15) + */ +public class Dur2SecConvertFunction extends BaseConversionUDF { + + public static final Dur2SecConvertFunction INSTANCE = new Dur2SecConvertFunction(); + + // Pattern to match [D+]HH:MM:SS format + // Optional days followed by +, then HH:MM:SS + private static final Pattern DURATION_PATTERN = + Pattern.compile("^(?:(\\d+)\\+)?(\\d{1,2}):(\\d{1,2}):(\\d{1,2})$"); + + public Dur2SecConvertFunction() { + super(Dur2SecConvertFunction.class); + } + + public static Object convert(Object value) { + return INSTANCE.convertValueImpl(value); + } + + @Override + protected Object applyConversion(String preprocessedValue) { + // First try to parse as a number (already in seconds) + Double existingSeconds = tryParseDouble(preprocessedValue); + if (existingSeconds != null) { + return existingSeconds; + } + + // Try to parse as [D+]HH:MM:SS format + Matcher matcher = DURATION_PATTERN.matcher(preprocessedValue); + if (matcher.matches()) { + try { + int days = 0; + if (matcher.group(1) != null) { + days = Integer.parseInt(matcher.group(1)); + } + + int hours = Integer.parseInt(matcher.group(2)); + int minutes = Integer.parseInt(matcher.group(3)); + int seconds = Integer.parseInt(matcher.group(4)); + + // Validate time components are in proper ranges + if (hours >= 24 || minutes >= 60 || seconds >= 60) { + return null; + } + + // Convert to total seconds + int totalSeconds = days * 24 * 3600 + hours * 3600 + minutes * 60 + seconds; + return (double) totalSeconds; + + } catch (NumberFormatException e) { + return null; + } + } + + return null; + } + + public Object convertValueImpl(Object value) { + if (value instanceof Number) { + // Already a number (seconds), return as double + return ((Number) value).doubleValue(); + } + + String str = preprocessValue(value); + if (str == null) { + return null; + } + + return applyConversion(str); + } +} \ No newline at end of file diff --git a/core/src/main/java/org/opensearch/sql/expression/function/udf/MkTimeConvertFunction.java b/core/src/main/java/org/opensearch/sql/expression/function/udf/MkTimeConvertFunction.java new file mode 100644 index 00000000000..521bdc816ad --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/expression/function/udf/MkTimeConvertFunction.java @@ -0,0 +1,183 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.expression.function.udf; + +import java.time.LocalDateTime; +import java.time.ZoneOffset; +import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeParseException; +import java.util.Arrays; +import java.util.List; +import org.apache.calcite.adapter.enumerable.NotNullImplementor; +import org.apache.calcite.adapter.enumerable.NullPolicy; +import org.apache.calcite.adapter.enumerable.RexToLixTranslator; +import org.apache.calcite.linq4j.tree.Expression; +import org.apache.calcite.linq4j.tree.Expressions; +import org.apache.calcite.rex.RexCall; +import org.apache.calcite.sql.type.ReturnTypes; +import org.apache.calcite.sql.type.SqlReturnTypeInference; +import org.apache.calcite.sql.type.SqlTypeName; +import org.opensearch.sql.calcite.utils.PPLOperandTypes; +import org.opensearch.sql.expression.function.ImplementorUDF; +import org.opensearch.sql.expression.function.UDFOperandMetadata; + +/** + * PPL mktime() conversion function. + * Converts human-readable time strings to epoch time (UNIX timestamp). + * Supports various date/time formats and optional custom timeformat parameter. + */ +public class MkTimeConvertFunction extends ImplementorUDF { + + public static final MkTimeConvertFunction INSTANCE = new MkTimeConvertFunction(); + + // Common date/time patterns to try parsing when no custom format is provided + private static final List DEFAULT_TIME_FORMATTERS = Arrays.asList( + DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"), + DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss"), + DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss'Z'"), + DateTimeFormatter.ofPattern("yyyy/MM/dd HH:mm:ss"), + DateTimeFormatter.ofPattern("MM/dd/yyyy HH:mm:ss"), + DateTimeFormatter.ofPattern("dd/MM/yyyy HH:mm:ss"), + DateTimeFormatter.ofPattern("yyyy-MM-dd"), + DateTimeFormatter.ofPattern("MM/dd/yyyy"), + DateTimeFormatter.ofPattern("dd/MM/yyyy"), + DateTimeFormatter.ofPattern("EEE MMM dd HH:mm:ss zzz yyyy") // ctime format + ); + + public MkTimeConvertFunction() { + super(new MkTimeImplementor(), NullPolicy.ANY); + } + + @Override + public SqlReturnTypeInference getReturnTypeInference() { + return ReturnTypes.explicit( + factory -> + factory.createTypeWithNullability(factory.createSqlType(SqlTypeName.DOUBLE), true)); + } + + @Override + public UDFOperandMetadata getOperandMetadata() { + return PPLOperandTypes.OPTIONAL_ANY; + } + + public static class MkTimeImplementor implements NotNullImplementor { + @Override + public Expression implement( + RexToLixTranslator translator, RexCall call, List translatedOperands) { + if (translatedOperands.isEmpty()) { + return Expressions.constant(null, Double.class); + } + + Expression fieldValue = translatedOperands.get(0); + + if (translatedOperands.size() == 1) { + // Single parameter: use default formats + return Expressions.call(MkTimeConvertFunction.class, "convert", fieldValue); + } else { + // Two parameters: field value and custom timeformat + Expression timeFormat = translatedOperands.get(1); + return Expressions.call(MkTimeConvertFunction.class, "convertWithFormat", fieldValue, timeFormat); + } + } + } + + // Method called when no custom timeformat is provided + public static Object convert(Object value) { + if (value == null) { + return null; + } + + if (value instanceof Number) { + // Already a number (timestamp), return as double + return ((Number) value).doubleValue(); + } + + String str = preprocessValue(value); + if (str == null) { + return null; + } + + return convertWithDefaultFormats(str); + } + + // Method called when custom timeformat is provided + public static Object convertWithFormat(Object value, Object timeFormatObj) { + if (value == null) { + return null; + } + + if (value instanceof Number) { + // Already a number (timestamp), return as double + return ((Number) value).doubleValue(); + } + + String str = preprocessValue(value); + if (str == null) { + return null; + } + + String timeFormat = timeFormatObj != null ? timeFormatObj.toString().trim() : null; + if (timeFormat == null || timeFormat.isEmpty()) { + return convertWithDefaultFormats(str); + } + + return convertWithCustomFormat(str, timeFormat); + } + + private static Object convertWithDefaultFormats(String preprocessedValue) { + // First try to parse as a number (already a timestamp) + Double existingTimestamp = tryParseDouble(preprocessedValue); + if (existingTimestamp != null) { + return existingTimestamp; + } + + // Try parsing with default date/time formats + for (DateTimeFormatter formatter : DEFAULT_TIME_FORMATTERS) { + try { + LocalDateTime dateTime = LocalDateTime.parse(preprocessedValue, formatter); + return (double) dateTime.toEpochSecond(ZoneOffset.UTC); + } catch (DateTimeParseException e) { + // Try next format + continue; + } + } + + return null; + } + + private static Object convertWithCustomFormat(String preprocessedValue, String customFormat) { + // First try to parse as a number (already a timestamp) + Double existingTimestamp = tryParseDouble(preprocessedValue); + if (existingTimestamp != null) { + return existingTimestamp; + } + + try { + DateTimeFormatter customFormatter = DateTimeFormatter.ofPattern(customFormat); + LocalDateTime dateTime = LocalDateTime.parse(preprocessedValue, customFormatter); + return (double) dateTime.toEpochSecond(ZoneOffset.UTC); + } catch (Exception e) { + // If custom format fails, try default formats as fallback + return convertWithDefaultFormats(preprocessedValue); + } + } + + private static String preprocessValue(Object value) { + if (value == null) { + return null; + } + String str = value instanceof String ? ((String) value).trim() : value.toString().trim(); + return str.isEmpty() ? null : str; + } + + private static Double tryParseDouble(String str) { + try { + return Double.parseDouble(str); + } catch (NumberFormatException e) { + return null; + } + } +} \ No newline at end of file diff --git a/core/src/main/java/org/opensearch/sql/expression/function/udf/MsTimeConvertFunction.java b/core/src/main/java/org/opensearch/sql/expression/function/udf/MsTimeConvertFunction.java new file mode 100644 index 00000000000..1b604e32468 --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/expression/function/udf/MsTimeConvertFunction.java @@ -0,0 +1,88 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.expression.function.udf; + +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * PPL mstime() conversion function. + * Converts MM:SS.SSS format to seconds. + * Example: "03:45.123" -> 225.123 seconds + */ +public class MsTimeConvertFunction extends BaseConversionUDF { + + public static final MsTimeConvertFunction INSTANCE = new MsTimeConvertFunction(); + + // Pattern to match MM:SS.SSS or MM:SS format + private static final Pattern MSTIME_PATTERN = + Pattern.compile("^(\\d{1,2}):(\\d{1,2})(?:\\.(\\d{1,3}))?$"); + + public MsTimeConvertFunction() { + super(MsTimeConvertFunction.class); + } + + public static Object convert(Object value) { + return INSTANCE.convertValueImpl(value); + } + + @Override + protected Object applyConversion(String preprocessedValue) { + // First try to parse as a number (already in seconds) + Double existingSeconds = tryParseDouble(preprocessedValue); + if (existingSeconds != null) { + return existingSeconds; + } + + // Try to parse as MM:SS.SSS format + Matcher matcher = MSTIME_PATTERN.matcher(preprocessedValue); + if (matcher.matches()) { + try { + int minutes = Integer.parseInt(matcher.group(1)); + int seconds = Integer.parseInt(matcher.group(2)); + + // Validate time components are in proper ranges + if (seconds >= 60) { + return null; + } + + double milliseconds = 0.0; + if (matcher.group(3) != null) { + String milliStr = matcher.group(3); + // Pad to 3 digits if necessary + while (milliStr.length() < 3) { + milliStr += "0"; + } + // Truncate to 3 digits if longer + if (milliStr.length() > 3) { + milliStr = milliStr.substring(0, 3); + } + milliseconds = Double.parseDouble(milliStr) / 1000.0; + } + + return (double) (minutes * 60 + seconds) + milliseconds; + } catch (NumberFormatException e) { + return null; + } + } + + return null; + } + + public Object convertValueImpl(Object value) { + if (value instanceof Number) { + // Already a number (seconds), return as double + return ((Number) value).doubleValue(); + } + + String str = preprocessValue(value); + if (str == null) { + return null; + } + + return applyConversion(str); + } +} \ No newline at end of file diff --git a/core/src/test/java/org/opensearch/sql/expression/function/udf/ConversionFunctionsTest.java b/core/src/test/java/org/opensearch/sql/expression/function/udf/ConversionFunctionsTest.java index 163d6508445..5fe93ddc541 100644 --- a/core/src/test/java/org/opensearch/sql/expression/function/udf/ConversionFunctionsTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/function/udf/ConversionFunctionsTest.java @@ -7,6 +7,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; import org.junit.jupiter.api.Test; @@ -336,4 +337,140 @@ public void testRmunitConvertNumericExtremes() { assertEquals(1.7e308, RmunitConvertFunction.convert("1.7e308")); assertEquals(-1.7e308, RmunitConvertFunction.convert("-1.7e308")); } + + // ctime() Function Tests + @Test + public void testCtimeConvertBasic() { + String result = (String) CTimeConvertFunction.convert(1066507633); + assertTrue(result != null && result.contains("2003")); + + result = (String) CTimeConvertFunction.convert(0); + assertTrue(result != null && result.contains("1970")); + + result = (String) CTimeConvertFunction.convert("1066507633"); + assertTrue(result != null && result.contains("2003")); + } + + @Test + public void testCtimeConvertInvalid() { + assertNull(CTimeConvertFunction.convert("invalid")); + assertNull(CTimeConvertFunction.convert(null)); + assertNull(CTimeConvertFunction.convert("")); + assertNull(CTimeConvertFunction.convert("abc123")); + } + + // mktime() Function Tests + @Test + public void testMktimeConvertBasic() { + assertEquals(1066507633.0, MkTimeConvertFunction.convert("2003-10-18 20:07:13")); + assertEquals(1066507633.0, MkTimeConvertFunction.convert("2003-10-18T20:07:13")); + assertEquals(946684800.0, MkTimeConvertFunction.convert("2000-01-01 00:00:00")); + assertEquals(1066473433.0, MkTimeConvertFunction.convert(1066473433)); + assertEquals(1066473433.0, MkTimeConvertFunction.convert("1066473433")); + } + + @Test + public void testMktimeConvertInvalid() { + assertNull(MkTimeConvertFunction.convert("invalid")); + assertNull(MkTimeConvertFunction.convert(null)); + assertNull(MkTimeConvertFunction.convert("")); + assertNull(MkTimeConvertFunction.convert("not-a-date")); + } + + // mstime() Function Tests + @Test + public void testMstimeConvertBasic() { + assertEquals(225.0, MsTimeConvertFunction.convert("03:45")); + assertEquals(225.123, MsTimeConvertFunction.convert("03:45.123")); + assertEquals(90.5, MsTimeConvertFunction.convert("01:30.5")); + assertEquals(3661.0, MsTimeConvertFunction.convert("61:01")); + + // Test already numeric + assertEquals(225.0, MsTimeConvertFunction.convert(225)); + assertEquals(225.0, MsTimeConvertFunction.convert("225")); + } + + @Test + public void testMstimeConvertEdgeCases() { + assertEquals(0.0, MsTimeConvertFunction.convert("00:00")); + assertEquals(0.001, MsTimeConvertFunction.convert("00:00.001")); + assertEquals(59.999, MsTimeConvertFunction.convert("00:59.999")); + } + + @Test + public void testMstimeConvertInvalid() { + assertNull(MsTimeConvertFunction.convert("invalid")); + assertNull(MsTimeConvertFunction.convert(null)); + assertNull(MsTimeConvertFunction.convert("")); + assertNull(MsTimeConvertFunction.convert("25:70")); + assertNull(MsTimeConvertFunction.convert("1:2:3")); + } + + // dur2sec() Function Tests + @Test + public void testDur2secConvertBasic() { + assertEquals(5025.0, Dur2SecConvertFunction.convert("01:23:45")); + assertEquals(3661.0, Dur2SecConvertFunction.convert("01:01:01")); + assertEquals(217815.0, Dur2SecConvertFunction.convert("2+12:30:15")); + assertEquals(90061.0, Dur2SecConvertFunction.convert("1+01:01:01")); + assertEquals(5025.0, Dur2SecConvertFunction.convert(5025)); + assertEquals(5025.0, Dur2SecConvertFunction.convert("5025")); + } + + @Test + public void testDur2secConvertEdgeCases() { + assertEquals(0.0, Dur2SecConvertFunction.convert("00:00:00")); + assertEquals(86400.0, Dur2SecConvertFunction.convert("1+00:00:00")); + assertEquals(3599.0, Dur2SecConvertFunction.convert("00:59:59")); + } + + @Test + public void testDur2secConvertInvalid() { + assertNull(Dur2SecConvertFunction.convert("invalid")); + assertNull(Dur2SecConvertFunction.convert(null)); + assertNull(Dur2SecConvertFunction.convert("")); + assertNull(Dur2SecConvertFunction.convert("25:70:80")); + assertNull(Dur2SecConvertFunction.convert("1:2")); + assertNull(Dur2SecConvertFunction.convert("1+2")); + } + + // timeformat tests for mktime() and ctime() + @Test + public void testMktimeWithCustomTimeformat() { + // Test mktime with custom timeformat + assertEquals(1066507633.0, MkTimeConvertFunction.convertWithFormat("18/10/2003 20:07:13", "dd/MM/yyyy HH:mm:ss")); + assertEquals(1066507633.0, MkTimeConvertFunction.convertWithFormat("2003-10-18 20:07:13", "yyyy-MM-dd HH:mm:ss")); + assertEquals(946684800.0, MkTimeConvertFunction.convertWithFormat("01/01/2000 00:00:00", "dd/MM/yyyy HH:mm:ss")); + + // Test fallback to default formats when custom format fails + assertEquals(1066507633.0, MkTimeConvertFunction.convertWithFormat("2003-10-18 20:07:13", "invalid format")); + + // Test null/empty timeformat + assertEquals(1066507633.0, MkTimeConvertFunction.convertWithFormat("2003-10-18 20:07:13", null)); + assertEquals(1066507633.0, MkTimeConvertFunction.convertWithFormat("2003-10-18 20:07:13", "")); + } + + @Test + public void testCtimeWithCustomTimeformat() { + // Test ctime with custom timeformat + String result1 = (String) CTimeConvertFunction.convertWithFormat(1066507633, "yyyy-MM-dd HH:mm:ss"); + assertTrue(result1 != null && result1.contains("2003-10-18")); + + String result2 = (String) CTimeConvertFunction.convertWithFormat(1066507633, "dd/MM/yyyy"); + assertTrue(result2 != null && result2.contains("18/10/2003")); + + String result3 = (String) CTimeConvertFunction.convertWithFormat(0, "yyyy"); + assertTrue(result3 != null && result3.contains("1970")); + + // Test fallback to default format when custom format fails + String result4 = (String) CTimeConvertFunction.convertWithFormat(1066507633, "invalid format"); + assertTrue(result4 != null && result4.contains("2003")); + + // Test null/empty timeformat + String result5 = (String) CTimeConvertFunction.convertWithFormat(1066507633, null); + assertTrue(result5 != null && result5.contains("2003")); + + String result6 = (String) CTimeConvertFunction.convertWithFormat(1066507633, ""); + assertTrue(result6 != null && result6.contains("2003")); + } } diff --git a/ppl/src/main/antlr/OpenSearchPPLParser.g4 b/ppl/src/main/antlr/OpenSearchPPLParser.g4 index 53cb4eda36c..ddd945572e0 100644 --- a/ppl/src/main/antlr/OpenSearchPPLParser.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLParser.g4 @@ -543,7 +543,7 @@ replacementPair ; convertCommand - : CONVERT convertFunction (COMMA? convertFunction)* + : CONVERT (TIMEFORMAT EQUAL timeFormat=stringLiteral)? convertFunction (COMMA? convertFunction)* ; convertFunction diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java index c422470bd39..2fe17646ed4 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java @@ -1212,12 +1212,18 @@ public UnresolvedPlan visitConvertCommand(OpenSearchPPLParser.ConvertCommandCont .map(this::buildConversion) .filter(conversion -> conversion != null) .collect(Collectors.toList()); - return new Convert(conversions); + + String timeFormat = null; + if (ctx.timeFormat != null) { + timeFormat = StringUtils.unquoteText(ctx.timeFormat.getText()); + } + + return new Convert(conversions, timeFormat); } /** Supported PPL convert function names (case-insensitive). */ private static final Set SUPPORTED_CONVERSION_FUNCTIONS = - Set.of("auto", "num", "rmcomma", "rmunit", "memk", "none"); + Set.of("auto", "num", "rmcomma", "rmunit", "memk", "none", "ctime", "mktime", "dur2sec", "mstime"); private Let buildConversion(OpenSearchPPLParser.ConvertFunctionContext funcCtx) { if (funcCtx.fieldExpression().isEmpty()) { From 13d50d006b7ce70ce7271e8552b5cd8c22881523 Mon Sep 17 00:00:00 2001 From: Ritvi Bhatt Date: Thu, 5 Mar 2026 15:30:15 -0800 Subject: [PATCH 02/22] add integ tests Signed-off-by: Ritvi Bhatt --- .../remote/CalciteConvertCommandIT.java | 121 ++++++++++++++++ .../opensearch/sql/ppl/ConvertCommandIT.java | 30 ++++ .../ppl/calcite/CalcitePPLConvertTest.java | 134 ++++++++++++++++++ 3 files changed, 285 insertions(+) diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteConvertCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteConvertCommandIT.java index 1c9b6de3454..69118300385 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteConvertCommandIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteConvertCommandIT.java @@ -5,6 +5,8 @@ package org.opensearch.sql.calcite.remote; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_BANK; import static org.opensearch.sql.util.MatcherUtils.rows; import static org.opensearch.sql.util.MatcherUtils.schema; @@ -260,4 +262,123 @@ public void testConvertAutoWithMemorySizesGigabytes() throws IOException { verifySchema(result, schema("memory", null, "double")); verifyDataRows(result, rows(2097152.0)); } + + @Test + public void testConvertMktimeWithDefaultFormat() throws IOException { + JSONObject result = + executeQuery( + String.format( + "search source=%s | eval date_str = '2003-10-18 20:07:13' | convert mktime(date_str) |" + + " fields date_str | head 1", + TEST_INDEX_BANK)); + verifySchema(result, schema("date_str", null, "double")); + verifyDataRows(result, rows(1066507633.0)); + } + + @Test + public void testConvertMktimeWithCustomTimeformat() throws IOException { + JSONObject result = + executeQuery( + String.format( + "search source=%s | eval date_str = '18/10/2003 20:07:13' | convert timeformat=\"dd/MM/yyyy HH:mm:ss\" mktime(date_str) |" + + " fields date_str | head 1", + TEST_INDEX_BANK)); + verifySchema(result, schema("date_str", null, "double")); + verifyDataRows(result, rows(1066507633.0)); + } + + @Test + public void testConvertCtimeWithDefaultFormat() throws IOException { + JSONObject result = + executeQuery( + String.format( + "search source=%s | eval timestamp = 1066507633 | convert ctime(timestamp) |" + + " fields timestamp | head 1", + TEST_INDEX_BANK)); + verifySchema(result, schema("timestamp", null, "string")); + verifyNumOfRows(result, 1); + // Verify it contains expected year - exact format may vary by timezone + String timestampValue = result.getJSONArray("datarows").getJSONArray(0).getString(0); + assertTrue("Expected timestamp to contain '2003'", timestampValue.contains("2003")); + } + + @Test + public void testConvertCtimeWithCustomTimeformat() throws IOException { + JSONObject result = + executeQuery( + String.format( + "search source=%s | eval timestamp = 1066507633 | convert timeformat=\"yyyy-MM-dd HH:mm:ss\" ctime(timestamp) |" + + " fields timestamp | head 1", + TEST_INDEX_BANK)); + verifySchema(result, schema("timestamp", null, "string")); + verifyDataRows(result, rows("2003-10-18 20:07:13")); + } + + @Test + public void testConvertDur2secFunction() throws IOException { + JSONObject result = + executeQuery( + String.format( + "search source=%s | eval duration = '01:23:45' | convert dur2sec(duration) |" + + " fields duration | head 1", + TEST_INDEX_BANK)); + verifySchema(result, schema("duration", null, "double")); + verifyDataRows(result, rows(5025.0)); + } + + @Test + public void testConvertMstimeFunction() throws IOException { + JSONObject result = + executeQuery( + String.format( + "search source=%s | eval time_str = '03:45' | convert mstime(time_str) |" + + " fields time_str | head 1", + TEST_INDEX_BANK)); + verifySchema(result, schema("time_str", null, "double")); + verifyDataRows(result, rows(225.0)); + } + + @Test + public void testConvertTimeformatWithMultipleFunctions() throws IOException { + JSONObject result = + executeQuery( + String.format( + "search source=%s | eval date_str = '18/10/2003 20:07:13', timestamp = 1066507633 |" + + " convert timeformat=\"dd/MM/yyyy HH:mm:ss\" mktime(date_str), ctime(timestamp) |" + + " fields date_str, timestamp | head 1", + TEST_INDEX_BANK)); + verifySchema(result, schema("date_str", null, "double"), schema("timestamp", null, "string")); + verifyNumOfRows(result, 1); + // Verify mktime conversion + assertEquals(1066507633.0, result.getJSONArray("datarows").getJSONArray(0).getDouble(0), 0.001); + // Verify ctime conversion contains expected year + String timestampValue = result.getJSONArray("datarows").getJSONArray(0).getString(1); + assertTrue("Expected timestamp to contain '2003'", timestampValue.contains("2003")); + } + + @Test + public void testConvertTimeformatWithWhere() throws IOException { + JSONObject result = + executeQuery( + String.format( + "search source=%s | eval date_str = '2003-10-18 20:07:13' |" + + " convert mktime(date_str) | where date_str > 1000000000 |" + + " fields date_str | head 1", + TEST_INDEX_BANK)); + verifySchema(result, schema("date_str", null, "double")); + verifyDataRows(result, rows(1066507633.0)); + } + + @Test + public void testConvertTimeformatWithStats() throws IOException { + JSONObject result = + executeQuery( + String.format( + "search source=%s | eval timestamp = 1066507633 |" + + " convert timeformat=\"yyyy\" ctime(timestamp) |" + + " stats count() by timestamp", + TEST_INDEX_BANK)); + verifySchema(result, schema("count()", null, "long"), schema("timestamp", "string")); + verifyDataRows(result, rows(1000L, "2003")); + } } diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/ConvertCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/ConvertCommandIT.java index 099992c9298..02dba43113f 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/ConvertCommandIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/ConvertCommandIT.java @@ -69,6 +69,36 @@ public void testConvertWithStats() { "source=%s | convert auto(balance) | stats avg(balance) by gender"); } + @Test + public void testConvertMktimeFunction() { + verifyQueryThrowsCalciteError( + "source=%s | eval date_str = '2003-10-18 20:07:13' | convert mktime(date_str) | fields date_str"); + } + + @Test + public void testConvertCtimeFunction() { + verifyQueryThrowsCalciteError( + "source=%s | eval timestamp = 1066507633 | convert ctime(timestamp) | fields timestamp"); + } + + @Test + public void testConvertDur2secFunction() { + verifyQueryThrowsCalciteError( + "source=%s | eval duration = '01:23:45' | convert dur2sec(duration) | fields duration"); + } + + @Test + public void testConvertMstimeFunction() { + verifyQueryThrowsCalciteError( + "source=%s | eval time_str = '03:45' | convert mstime(time_str) | fields time_str"); + } + + @Test + public void testConvertWithTimeformat() { + verifyQueryThrowsCalciteError( + "source=%s | eval date_str = '18/10/2003 20:07:13' | convert timeformat=\"dd/MM/yyyy HH:mm:ss\" mktime(date_str) | fields date_str"); + } + private void verifyQueryThrowsCalciteError(String query) { Exception e = assertThrows(Exception.class, () -> executeQuery(String.format(query, TEST_INDEX_BANK))); diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLConvertTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLConvertTest.java index 936b4212f4f..16494b72d4d 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLConvertTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLConvertTest.java @@ -269,4 +269,138 @@ public void testConvertAutoWithMemoryField() { + "FROM `scott`.`EMP`"; verifyPPLToSparkSQL(root, expectedSparkSql); } + + @Test + public void testConvertMktimeFunction() { + String ppl = "source=EMP | convert mktime(ENAME)"; + RelNode root = getRelNode(ppl); + String expectedLogical = + "LogicalProject(EMPNO=[$0], ENAME=[MKTIME($1)], JOB=[$2], MGR=[$3], HIREDATE=[$4]," + + " SAL=[$5], COMM=[$6], DEPTNO=[$7])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + + String expectedSparkSql = + "SELECT `EMPNO`, MKTIME(`ENAME`) `ENAME`, `JOB`, `MGR`, `HIREDATE`, `SAL`, `COMM`," + + " `DEPTNO`\n" + + "FROM `scott`.`EMP`"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } + + @Test + public void testConvertCtimeFunction() { + String ppl = "source=EMP | convert ctime(SAL)"; + RelNode root = getRelNode(ppl); + String expectedLogical = + "LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4], SAL=[CTIME($5)]," + + " COMM=[$6], DEPTNO=[$7])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + + String expectedSparkSql = + "SELECT `EMPNO`, `ENAME`, `JOB`, `MGR`, `HIREDATE`, CTIME(`SAL`) `SAL`, `COMM`, `DEPTNO`\n" + + "FROM `scott`.`EMP`"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } + + @Test + public void testConvertDur2secFunction() { + String ppl = "source=EMP | convert dur2sec(ENAME)"; + RelNode root = getRelNode(ppl); + String expectedLogical = + "LogicalProject(EMPNO=[$0], ENAME=[DUR2SEC($1)], JOB=[$2], MGR=[$3], HIREDATE=[$4]," + + " SAL=[$5], COMM=[$6], DEPTNO=[$7])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + + String expectedSparkSql = + "SELECT `EMPNO`, DUR2SEC(`ENAME`) `ENAME`, `JOB`, `MGR`, `HIREDATE`, `SAL`, `COMM`," + + " `DEPTNO`\n" + + "FROM `scott`.`EMP`"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } + + @Test + public void testConvertMstimeFunction() { + String ppl = "source=EMP | convert mstime(ENAME)"; + RelNode root = getRelNode(ppl); + String expectedLogical = + "LogicalProject(EMPNO=[$0], ENAME=[MSTIME($1)], JOB=[$2], MGR=[$3], HIREDATE=[$4]," + + " SAL=[$5], COMM=[$6], DEPTNO=[$7])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + + String expectedSparkSql = + "SELECT `EMPNO`, MSTIME(`ENAME`) `ENAME`, `JOB`, `MGR`, `HIREDATE`, `SAL`, `COMM`," + + " `DEPTNO`\n" + + "FROM `scott`.`EMP`"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } + + @Test + public void testConvertWithTimeformatMktime() { + String ppl = "source=EMP | convert timeformat=\"%Y-%m-%d\" mktime(ENAME)"; + RelNode root = getRelNode(ppl); + String expectedLogical = + "LogicalProject(EMPNO=[$0], ENAME=[MKTIME($1, '%Y-%m-%d')], JOB=[$2], MGR=[$3], HIREDATE=[$4]," + + " SAL=[$5], COMM=[$6], DEPTNO=[$7])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + + String expectedSparkSql = + "SELECT `EMPNO`, MKTIME(`ENAME`, '%Y-%m-%d') `ENAME`, `JOB`, `MGR`, `HIREDATE`, `SAL`, `COMM`," + + " `DEPTNO`\n" + + "FROM `scott`.`EMP`"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } + + @Test + public void testConvertWithTimeformatCtime() { + String ppl = "source=EMP | convert timeformat=\"%Y-%m-%d %H:%M:%S\" ctime(SAL)"; + RelNode root = getRelNode(ppl); + String expectedLogical = + "LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4], SAL=[CTIME($5, '%Y-%m-%d %H:%M:%S')]," + + " COMM=[$6], DEPTNO=[$7])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + + String expectedSparkSql = + "SELECT `EMPNO`, `ENAME`, `JOB`, `MGR`, `HIREDATE`, CTIME(`SAL`, '%Y-%m-%d %H:%M:%S') `SAL`, `COMM`, `DEPTNO`\n" + + "FROM `scott`.`EMP`"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } + + @Test + public void testConvertTimeformatWithMultipleFunctions() { + String ppl = "source=EMP | convert timeformat=\"%Y-%m-%d\" mktime(ENAME), ctime(SAL)"; + RelNode root = getRelNode(ppl); + String expectedLogical = + "LogicalProject(EMPNO=[$0], ENAME=[MKTIME($1, '%Y-%m-%d')], JOB=[$2], MGR=[$3], HIREDATE=[$4], SAL=[CTIME($5, '%Y-%m-%d')]," + + " COMM=[$6], DEPTNO=[$7])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + + String expectedSparkSql = + "SELECT `EMPNO`, MKTIME(`ENAME`, '%Y-%m-%d') `ENAME`, `JOB`, `MGR`, `HIREDATE`, CTIME(`SAL`, '%Y-%m-%d') `SAL`, `COMM`," + + " `DEPTNO`\n" + + "FROM `scott`.`EMP`"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } + + @Test + public void testConvertTimeformatMixedWithNonTimeFunctions() { + String ppl = "source=EMP | convert timeformat=\"%Y-%m-%d\" mktime(ENAME), auto(SAL)"; + RelNode root = getRelNode(ppl); + String expectedLogical = + "LogicalProject(EMPNO=[$0], ENAME=[MKTIME($1, '%Y-%m-%d')], JOB=[$2], MGR=[$3], HIREDATE=[$4], SAL=[AUTO($5)]," + + " COMM=[$6], DEPTNO=[$7])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + + String expectedSparkSql = + "SELECT `EMPNO`, MKTIME(`ENAME`, '%Y-%m-%d') `ENAME`, `JOB`, `MGR`, `HIREDATE`, AUTO(`SAL`) `SAL`, `COMM`," + + " `DEPTNO`\n" + + "FROM `scott`.`EMP`"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } } From eae36db14f715f0a8e5bd19833d04399a82cbfc6 Mon Sep 17 00:00:00 2001 From: Ritvi Bhatt Date: Fri, 6 Mar 2026 12:29:12 -0800 Subject: [PATCH 03/22] fix number of arguments Signed-off-by: Ritvi Bhatt --- .../sql/calcite/utils/PPLOperandTypes.java | 5 + .../datetime/StrftimeFormatterUtil.java | 60 ++++++++ .../expression/function/PPLFuncImpTable.java | 8 ++ .../function/udf/CTimeConvertFunction.java | 96 +++++-------- .../function/udf/Dur2SecConvertFunction.java | 60 +++----- .../function/udf/MkTimeConvertFunction.java | 130 ++++-------------- .../function/udf/MsTimeConvertFunction.java | 70 ++++------ .../function/udf/ConversionFunctionsTest.java | 67 ++++----- .../remote/CalciteConvertCommandIT.java | 22 ++- .../sql/ppl/utils/PPLQueryDataAnonymizer.java | 6 +- .../ppl/utils/PPLQueryDataAnonymizerTest.java | 20 +++ 11 files changed, 237 insertions(+), 307 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/calcite/utils/PPLOperandTypes.java b/core/src/main/java/org/opensearch/sql/calcite/utils/PPLOperandTypes.java index abf37e68392..a6570648981 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/utils/PPLOperandTypes.java +++ b/core/src/main/java/org/opensearch/sql/calcite/utils/PPLOperandTypes.java @@ -84,6 +84,11 @@ private PPLOperandTypes() {} UDFOperandMetadata.wrap( (CompositeOperandTypeChecker) OperandTypes.ANY.or(OperandTypes.family(SqlTypeFamily.ANY, SqlTypeFamily.INTEGER))); + public static final UDFOperandMetadata ANY_OPTIONAL_STRING = + UDFOperandMetadata.wrap( + (CompositeOperandTypeChecker) + OperandTypes.ANY.or( + OperandTypes.family(SqlTypeFamily.ANY, SqlTypeFamily.CHARACTER))); public static final UDFOperandMetadata ANY_OPTIONAL_TIMESTAMP = UDFOperandMetadata.wrap( (CompositeOperandTypeChecker) diff --git a/core/src/main/java/org/opensearch/sql/expression/datetime/StrftimeFormatterUtil.java b/core/src/main/java/org/opensearch/sql/expression/datetime/StrftimeFormatterUtil.java index f42d376f649..44b809e8598 100644 --- a/core/src/main/java/org/opensearch/sql/expression/datetime/StrftimeFormatterUtil.java +++ b/core/src/main/java/org/opensearch/sql/expression/datetime/StrftimeFormatterUtil.java @@ -249,4 +249,64 @@ private static long extractFirstNDigits(double value, int digits) { return isNegative ? -result : result; } + + /** Mapping from strftime specifiers to Java DateTimeFormatter patterns for parsing. */ + private static final Map STRFTIME_TO_JAVA_PARSE = + ImmutableMap.builder() + .put("%Y", "yyyy") + .put("%y", "yy") + .put("%m", "MM") + .put("%d", "dd") + .put("%e", "d") + .put("%H", "HH") + .put("%I", "hh") + .put("%M", "mm") + .put("%S", "ss") + .put("%p", "a") + .put("%B", "MMMM") + .put("%b", "MMM") + .put("%A", "EEEE") + .put("%a", "EEE") + .put("%j", "DDD") + .put("%Z", "zzz") + .put("%z", "xx") + .put("%T", "HH:mm:ss") + .put("%F", "yyyy-MM-dd") + .put("%x", "MM/dd/yyyy") + .put("%%", "'%'") + .build(); + + /** + * Convert a strftime format string to a Java DateTimeFormatter pattern suitable for parsing. + * + * @param strftimeFormat the strftime-style format string (e.g. {@code %Y-%m-%d %H:%M:%S}) + * @return a Java DateTimeFormatter pattern (e.g. {@code yyyy-MM-dd HH:mm:ss}) + */ + public static String toJavaPattern(String strftimeFormat) { + StringBuilder result = new StringBuilder(); + int i = 0; + while (i < strftimeFormat.length()) { + if (strftimeFormat.charAt(i) == '%' && i + 1 < strftimeFormat.length()) { + String spec = strftimeFormat.substring(i, i + 2); + String javaPattern = STRFTIME_TO_JAVA_PARSE.get(spec); + if (javaPattern != null) { + result.append(javaPattern); + } else { + // Unknown specifier — pass through as literal + result.append("'").append(spec).append("'"); + } + i += 2; + } else { + char c = strftimeFormat.charAt(i); + // Escape Java pattern letters as literals + if (Character.isLetter(c)) { + result.append("'").append(c).append("'"); + } else { + result.append(c); + } + i++; + } + } + return result.toString(); + } } diff --git a/core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java b/core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java index fe364c7b7c4..638e974ad7d 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java @@ -24,6 +24,7 @@ import static org.opensearch.sql.expression.function.BuiltinFunctionName.ATAN; import static org.opensearch.sql.expression.function.BuiltinFunctionName.ATAN2; import static org.opensearch.sql.expression.function.BuiltinFunctionName.AUTO; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.CTIME; import static org.opensearch.sql.expression.function.BuiltinFunctionName.AVG; import static org.opensearch.sql.expression.function.BuiltinFunctionName.CBRT; import static org.opensearch.sql.expression.function.BuiltinFunctionName.CEIL; @@ -138,6 +139,9 @@ import static org.opensearch.sql.expression.function.BuiltinFunctionName.MD5; import static org.opensearch.sql.expression.function.BuiltinFunctionName.MEDIAN; import static org.opensearch.sql.expression.function.BuiltinFunctionName.MEMK; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.MKTIME; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.MSTIME; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.DUR2SEC; import static org.opensearch.sql.expression.function.BuiltinFunctionName.MICROSECOND; import static org.opensearch.sql.expression.function.BuiltinFunctionName.MIN; import static org.opensearch.sql.expression.function.BuiltinFunctionName.MINSPAN_BUCKET; @@ -995,6 +999,10 @@ void populate() { registerOperator(RMCOMMA, PPLBuiltinOperators.RMCOMMA); registerOperator(RMUNIT, PPLBuiltinOperators.RMUNIT); registerOperator(MEMK, PPLBuiltinOperators.MEMK); + registerOperator(CTIME, PPLBuiltinOperators.CTIME); + registerOperator(MKTIME, PPLBuiltinOperators.MKTIME); + registerOperator(MSTIME, PPLBuiltinOperators.MSTIME); + registerOperator(DUR2SEC, PPLBuiltinOperators.DUR2SEC); register( TOSTRING, diff --git a/core/src/main/java/org/opensearch/sql/expression/function/udf/CTimeConvertFunction.java b/core/src/main/java/org/opensearch/sql/expression/function/udf/CTimeConvertFunction.java index a8dddd8dea0..e75529d5dfb 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/udf/CTimeConvertFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/udf/CTimeConvertFunction.java @@ -7,32 +7,28 @@ import java.time.Instant; import java.time.ZoneId; -import java.time.ZoneOffset; -import java.time.format.DateTimeFormatter; +import java.time.ZonedDateTime; import java.util.List; import org.apache.calcite.adapter.enumerable.NotNullImplementor; import org.apache.calcite.adapter.enumerable.NullPolicy; import org.apache.calcite.adapter.enumerable.RexToLixTranslator; -import org.apache.calcite.linq4j.function.Strict; import org.apache.calcite.linq4j.tree.Expression; import org.apache.calcite.linq4j.tree.Expressions; import org.apache.calcite.rex.RexCall; import org.apache.calcite.sql.type.SqlReturnTypeInference; import org.opensearch.sql.calcite.utils.PPLOperandTypes; import org.opensearch.sql.calcite.utils.PPLReturnTypes; +import org.opensearch.sql.expression.datetime.StrftimeFormatterUtil; import org.opensearch.sql.expression.function.ImplementorUDF; import org.opensearch.sql.expression.function.UDFOperandMetadata; /** - * PPL ctime() conversion function. - * Converts UNIX timestamps to human-readable format. - * Format: "Mon Oct 13 20:07:13 PDT 2003" + * PPL ctime() conversion function. Converts UNIX epoch timestamps to human-readable time strings + * using strftime format specifiers. Default format: {@code %m/%d/%Y %H:%M:%S} (SPL-compatible). */ public class CTimeConvertFunction extends ImplementorUDF { - // Format pattern matching expected output: "Mon Oct 13 20:07:13 PDT 2003" - private static final DateTimeFormatter CTIME_FORMATTER = - DateTimeFormatter.ofPattern("EEE MMM dd HH:mm:ss zzz yyyy"); + private static final String DEFAULT_FORMAT = "%m/%d/%Y %H:%M:%S"; public CTimeConvertFunction() { super(new CTimeImplementor(), NullPolicy.ANY); @@ -45,7 +41,7 @@ public SqlReturnTypeInference getReturnTypeInference() { @Override public UDFOperandMetadata getOperandMetadata() { - return PPLOperandTypes.OPTIONAL_ANY; + return PPLOperandTypes.ANY_OPTIONAL_STRING; } public static class CTimeImplementor implements NotNullImplementor { @@ -55,79 +51,53 @@ public Expression implement( if (translatedOperands.isEmpty()) { return Expressions.constant(null, String.class); } - Expression fieldValue = translatedOperands.get(0); - if (translatedOperands.size() == 1) { - // Single parameter: use default ctime format return Expressions.call(CTimeConvertFunction.class, "convert", fieldValue); - } else { - // Two parameters: field value and custom timeformat - Expression timeFormat = translatedOperands.get(1); - return Expressions.call(CTimeConvertFunction.class, "convertWithFormat", fieldValue, timeFormat); } + Expression timeFormat = translatedOperands.get(1); + return Expressions.call( + CTimeConvertFunction.class, "convertWithFormat", fieldValue, timeFormat); } } - @Strict public static String convert(Object value) { - if (value == null) { + return convertWithFormat(value, null); + } + + public static String convertWithFormat(Object value, Object timeFormatObj) { + Double timestamp = toEpochSeconds(value); + if (timestamp == null) { return null; } - try { - double timestamp; - if (value instanceof Number) { - timestamp = ((Number) value).doubleValue(); - } else { - String str = value.toString().trim(); - if (str.isEmpty()) { - return null; - } - timestamp = Double.parseDouble(str); - } - - // Convert to Instant and format - Instant instant = Instant.ofEpochSecond((long) timestamp); - return CTIME_FORMATTER.format(instant.atZone(ZoneOffset.UTC)); - + String format = + (timeFormatObj != null && !timeFormatObj.toString().trim().isEmpty()) + ? timeFormatObj.toString().trim() + : DEFAULT_FORMAT; + Instant instant = Instant.ofEpochSecond(timestamp.longValue()); + ZonedDateTime zdt = ZonedDateTime.ofInstant(instant, ZoneId.of("UTC")); + return StrftimeFormatterUtil.formatZonedDateTime(zdt, format).stringValue(); } catch (Exception e) { return null; } } - @Strict - public static String convertWithFormat(Object value, Object timeFormatObj) { + static Double toEpochSeconds(Object value) { if (value == null) { return null; } - - String customFormat = timeFormatObj != null ? timeFormatObj.toString().trim() : null; - if (customFormat == null || customFormat.isEmpty()) { - // Fall back to default ctime format - return convert(value); + if (value instanceof Number) { + return ((Number) value).doubleValue(); + } + String str = value.toString().trim(); + if (str.isEmpty()) { + return null; } - try { - double timestamp; - if (value instanceof Number) { - timestamp = ((Number) value).doubleValue(); - } else { - String str = value.toString().trim(); - if (str.isEmpty()) { - return null; - } - timestamp = Double.parseDouble(str); - } - - // Convert to Instant and format with custom formatter - Instant instant = Instant.ofEpochSecond((long) timestamp); - DateTimeFormatter customFormatter = DateTimeFormatter.ofPattern(customFormat); - return customFormatter.format(instant.atZone(ZoneOffset.UTC)); - - } catch (Exception e) { - // If custom format fails, fall back to default ctime format - return convert(value); + return Double.parseDouble(str); + } catch (NumberFormatException e) { + return null; } } -} \ No newline at end of file +} diff --git a/core/src/main/java/org/opensearch/sql/expression/function/udf/Dur2SecConvertFunction.java b/core/src/main/java/org/opensearch/sql/expression/function/udf/Dur2SecConvertFunction.java index a4603ecbafa..58bca365aae 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/udf/Dur2SecConvertFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/udf/Dur2SecConvertFunction.java @@ -9,18 +9,14 @@ import java.util.regex.Pattern; /** - * PPL dur2sec() conversion function. - * Converts duration format [D+]HH:MM:SS to seconds. - * Examples: - * - "01:23:45" -> 5025 seconds (1*3600 + 23*60 + 45) - * - "2+12:30:15" -> 217815 seconds (2*24*3600 + 12*3600 + 30*60 + 15) + * PPL dur2sec() conversion function. Converts duration format {@code [D+]HH:MM:SS} to seconds + * (SPL-compatible). */ public class Dur2SecConvertFunction extends BaseConversionUDF { public static final Dur2SecConvertFunction INSTANCE = new Dur2SecConvertFunction(); - // Pattern to match [D+]HH:MM:SS format - // Optional days followed by +, then HH:MM:SS + // Matches [D+]HH:MM:SS — optional days prefix with + separator private static final Pattern DURATION_PATTERN = Pattern.compile("^(?:(\\d+)\\+)?(\\d{1,2}):(\\d{1,2}):(\\d{1,2})$"); @@ -29,58 +25,34 @@ public Dur2SecConvertFunction() { } public static Object convert(Object value) { - return INSTANCE.convertValueImpl(value); + return INSTANCE.convertValue(value); } @Override protected Object applyConversion(String preprocessedValue) { - // First try to parse as a number (already in seconds) Double existingSeconds = tryParseDouble(preprocessedValue); if (existingSeconds != null) { return existingSeconds; } - // Try to parse as [D+]HH:MM:SS format Matcher matcher = DURATION_PATTERN.matcher(preprocessedValue); - if (matcher.matches()) { - try { - int days = 0; - if (matcher.group(1) != null) { - days = Integer.parseInt(matcher.group(1)); - } - - int hours = Integer.parseInt(matcher.group(2)); - int minutes = Integer.parseInt(matcher.group(3)); - int seconds = Integer.parseInt(matcher.group(4)); - - // Validate time components are in proper ranges - if (hours >= 24 || minutes >= 60 || seconds >= 60) { - return null; - } + if (!matcher.matches()) { + return null; + } - // Convert to total seconds - int totalSeconds = days * 24 * 3600 + hours * 3600 + minutes * 60 + seconds; - return (double) totalSeconds; + try { + int days = matcher.group(1) != null ? Integer.parseInt(matcher.group(1)) : 0; + int hours = Integer.parseInt(matcher.group(2)); + int minutes = Integer.parseInt(matcher.group(3)); + int seconds = Integer.parseInt(matcher.group(4)); - } catch (NumberFormatException e) { + if (hours >= 24 || minutes >= 60 || seconds >= 60) { return null; } - } - return null; - } - - public Object convertValueImpl(Object value) { - if (value instanceof Number) { - // Already a number (seconds), return as double - return ((Number) value).doubleValue(); - } - - String str = preprocessValue(value); - if (str == null) { + return (double) (days * 86400 + hours * 3600 + minutes * 60 + seconds); + } catch (NumberFormatException e) { return null; } - - return applyConversion(str); } -} \ No newline at end of file +} diff --git a/core/src/main/java/org/opensearch/sql/expression/function/udf/MkTimeConvertFunction.java b/core/src/main/java/org/opensearch/sql/expression/function/udf/MkTimeConvertFunction.java index 521bdc816ad..9fa1f7035ad 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/udf/MkTimeConvertFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/udf/MkTimeConvertFunction.java @@ -9,8 +9,8 @@ import java.time.ZoneOffset; import java.time.format.DateTimeFormatter; import java.time.format.DateTimeParseException; -import java.util.Arrays; import java.util.List; +import java.util.Locale; import org.apache.calcite.adapter.enumerable.NotNullImplementor; import org.apache.calcite.adapter.enumerable.NullPolicy; import org.apache.calcite.adapter.enumerable.RexToLixTranslator; @@ -21,31 +21,19 @@ import org.apache.calcite.sql.type.SqlReturnTypeInference; import org.apache.calcite.sql.type.SqlTypeName; import org.opensearch.sql.calcite.utils.PPLOperandTypes; +import org.opensearch.sql.expression.datetime.StrftimeFormatterUtil; import org.opensearch.sql.expression.function.ImplementorUDF; import org.opensearch.sql.expression.function.UDFOperandMetadata; /** - * PPL mktime() conversion function. - * Converts human-readable time strings to epoch time (UNIX timestamp). - * Supports various date/time formats and optional custom timeformat parameter. + * PPL mktime() conversion function. Parses a human-readable time string into UNIX epoch seconds + * using strftime format specifiers. Default format: {@code %m/%d/%Y %H:%M:%S} (SPL-compatible). */ public class MkTimeConvertFunction extends ImplementorUDF { public static final MkTimeConvertFunction INSTANCE = new MkTimeConvertFunction(); - // Common date/time patterns to try parsing when no custom format is provided - private static final List DEFAULT_TIME_FORMATTERS = Arrays.asList( - DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"), - DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss"), - DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss'Z'"), - DateTimeFormatter.ofPattern("yyyy/MM/dd HH:mm:ss"), - DateTimeFormatter.ofPattern("MM/dd/yyyy HH:mm:ss"), - DateTimeFormatter.ofPattern("dd/MM/yyyy HH:mm:ss"), - DateTimeFormatter.ofPattern("yyyy-MM-dd"), - DateTimeFormatter.ofPattern("MM/dd/yyyy"), - DateTimeFormatter.ofPattern("dd/MM/yyyy"), - DateTimeFormatter.ofPattern("EEE MMM dd HH:mm:ss zzz yyyy") // ctime format - ); + private static final String DEFAULT_FORMAT = "%m/%d/%Y %H:%M:%S"; public MkTimeConvertFunction() { super(new MkTimeImplementor(), NullPolicy.ANY); @@ -60,7 +48,7 @@ public SqlReturnTypeInference getReturnTypeInference() { @Override public UDFOperandMetadata getOperandMetadata() { - return PPLOperandTypes.OPTIONAL_ANY; + return PPLOperandTypes.ANY_OPTIONAL_STRING; } public static class MkTimeImplementor implements NotNullImplementor { @@ -70,114 +58,54 @@ public Expression implement( if (translatedOperands.isEmpty()) { return Expressions.constant(null, Double.class); } - Expression fieldValue = translatedOperands.get(0); - if (translatedOperands.size() == 1) { - // Single parameter: use default formats return Expressions.call(MkTimeConvertFunction.class, "convert", fieldValue); - } else { - // Two parameters: field value and custom timeformat - Expression timeFormat = translatedOperands.get(1); - return Expressions.call(MkTimeConvertFunction.class, "convertWithFormat", fieldValue, timeFormat); } + Expression timeFormat = translatedOperands.get(1); + return Expressions.call( + MkTimeConvertFunction.class, "convertWithFormat", fieldValue, timeFormat); } } - // Method called when no custom timeformat is provided public static Object convert(Object value) { - if (value == null) { - return null; - } - - if (value instanceof Number) { - // Already a number (timestamp), return as double - return ((Number) value).doubleValue(); - } - - String str = preprocessValue(value); - if (str == null) { - return null; - } - - return convertWithDefaultFormats(str); + return convertWithFormat(value, null); } - // Method called when custom timeformat is provided public static Object convertWithFormat(Object value, Object timeFormatObj) { if (value == null) { return null; } - if (value instanceof Number) { - // Already a number (timestamp), return as double return ((Number) value).doubleValue(); } - - String str = preprocessValue(value); - if (str == null) { + String str = value instanceof String ? ((String) value).trim() : value.toString().trim(); + if (str.isEmpty()) { return null; } - - String timeFormat = timeFormatObj != null ? timeFormatObj.toString().trim() : null; - if (timeFormat == null || timeFormat.isEmpty()) { - return convertWithDefaultFormats(str); - } - - return convertWithCustomFormat(str, timeFormat); - } - - private static Object convertWithDefaultFormats(String preprocessedValue) { - // First try to parse as a number (already a timestamp) - Double existingTimestamp = tryParseDouble(preprocessedValue); - if (existingTimestamp != null) { - return existingTimestamp; - } - - // Try parsing with default date/time formats - for (DateTimeFormatter formatter : DEFAULT_TIME_FORMATTERS) { - try { - LocalDateTime dateTime = LocalDateTime.parse(preprocessedValue, formatter); - return (double) dateTime.toEpochSecond(ZoneOffset.UTC); - } catch (DateTimeParseException e) { - // Try next format - continue; - } - } - - return null; - } - - private static Object convertWithCustomFormat(String preprocessedValue, String customFormat) { - // First try to parse as a number (already a timestamp) - Double existingTimestamp = tryParseDouble(preprocessedValue); - if (existingTimestamp != null) { - return existingTimestamp; - } - + // If already numeric, return as-is try { - DateTimeFormatter customFormatter = DateTimeFormatter.ofPattern(customFormat); - LocalDateTime dateTime = LocalDateTime.parse(preprocessedValue, customFormatter); - return (double) dateTime.toEpochSecond(ZoneOffset.UTC); - } catch (Exception e) { - // If custom format fails, try default formats as fallback - return convertWithDefaultFormats(preprocessedValue); + return Double.parseDouble(str); + } catch (NumberFormatException ignored) { + // Not a number, proceed with date parsing } - } - private static String preprocessValue(Object value) { - if (value == null) { - return null; - } - String str = value instanceof String ? ((String) value).trim() : value.toString().trim(); - return str.isEmpty() ? null : str; + String strftimeFormat = + (timeFormatObj != null && !timeFormatObj.toString().trim().isEmpty()) + ? timeFormatObj.toString().trim() + : DEFAULT_FORMAT; + return parseWithFormat(str, strftimeFormat); } - private static Double tryParseDouble(String str) { + private static Object parseWithFormat(String dateStr, String strftimeFormat) { try { - return Double.parseDouble(str); - } catch (NumberFormatException e) { + String javaPattern = StrftimeFormatterUtil.toJavaPattern(strftimeFormat); + DateTimeFormatter formatter = + DateTimeFormatter.ofPattern(javaPattern, Locale.ROOT); + LocalDateTime dateTime = LocalDateTime.parse(dateStr, formatter); + return (double) dateTime.toEpochSecond(ZoneOffset.UTC); + } catch (DateTimeParseException | IllegalArgumentException e) { return null; } } -} \ No newline at end of file +} diff --git a/core/src/main/java/org/opensearch/sql/expression/function/udf/MsTimeConvertFunction.java b/core/src/main/java/org/opensearch/sql/expression/function/udf/MsTimeConvertFunction.java index 1b604e32468..2180040c39a 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/udf/MsTimeConvertFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/udf/MsTimeConvertFunction.java @@ -9,80 +9,58 @@ import java.util.regex.Pattern; /** - * PPL mstime() conversion function. - * Converts MM:SS.SSS format to seconds. - * Example: "03:45.123" -> 225.123 seconds + * PPL mstime() conversion function. Converts {@code [MM:]SS.SSS} format to seconds + * (SPL-compatible). The minutes portion is optional. */ public class MsTimeConvertFunction extends BaseConversionUDF { public static final MsTimeConvertFunction INSTANCE = new MsTimeConvertFunction(); - // Pattern to match MM:SS.SSS or MM:SS format + // Matches optional MM: prefix, required SS, optional .SSS private static final Pattern MSTIME_PATTERN = - Pattern.compile("^(\\d{1,2}):(\\d{1,2})(?:\\.(\\d{1,3}))?$"); + Pattern.compile("^(?:(\\d{1,2}):)?(\\d{1,2})(?:\\.(\\d{1,3}))?$"); public MsTimeConvertFunction() { super(MsTimeConvertFunction.class); } public static Object convert(Object value) { - return INSTANCE.convertValueImpl(value); + return INSTANCE.convertValue(value); } @Override protected Object applyConversion(String preprocessedValue) { - // First try to parse as a number (already in seconds) Double existingSeconds = tryParseDouble(preprocessedValue); if (existingSeconds != null) { return existingSeconds; } - // Try to parse as MM:SS.SSS format Matcher matcher = MSTIME_PATTERN.matcher(preprocessedValue); - if (matcher.matches()) { - try { - int minutes = Integer.parseInt(matcher.group(1)); - int seconds = Integer.parseInt(matcher.group(2)); - - // Validate time components are in proper ranges - if (seconds >= 60) { - return null; - } + if (!matcher.matches()) { + return null; + } - double milliseconds = 0.0; - if (matcher.group(3) != null) { - String milliStr = matcher.group(3); - // Pad to 3 digits if necessary - while (milliStr.length() < 3) { - milliStr += "0"; - } - // Truncate to 3 digits if longer - if (milliStr.length() > 3) { - milliStr = milliStr.substring(0, 3); - } - milliseconds = Double.parseDouble(milliStr) / 1000.0; - } + try { + int minutes = matcher.group(1) != null ? Integer.parseInt(matcher.group(1)) : 0; + int seconds = Integer.parseInt(matcher.group(2)); - return (double) (minutes * 60 + seconds) + milliseconds; - } catch (NumberFormatException e) { + if (seconds >= 60) { return null; } - } - return null; - } - - public Object convertValueImpl(Object value) { - if (value instanceof Number) { - // Already a number (seconds), return as double - return ((Number) value).doubleValue(); - } + double millis = 0.0; + if (matcher.group(3) != null) { + String milliStr = matcher.group(3); + // Pad to 3 digits + while (milliStr.length() < 3) { + milliStr += "0"; + } + millis = Double.parseDouble(milliStr.substring(0, 3)) / 1000.0; + } - String str = preprocessValue(value); - if (str == null) { + return (double) (minutes * 60 + seconds) + millis; + } catch (NumberFormatException e) { return null; } - - return applyConversion(str); } -} \ No newline at end of file +} diff --git a/core/src/test/java/org/opensearch/sql/expression/function/udf/ConversionFunctionsTest.java b/core/src/test/java/org/opensearch/sql/expression/function/udf/ConversionFunctionsTest.java index 5fe93ddc541..4ebb445b136 100644 --- a/core/src/test/java/org/opensearch/sql/expression/function/udf/ConversionFunctionsTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/function/udf/ConversionFunctionsTest.java @@ -341,14 +341,10 @@ public void testRmunitConvertNumericExtremes() { // ctime() Function Tests @Test public void testCtimeConvertBasic() { - String result = (String) CTimeConvertFunction.convert(1066507633); - assertTrue(result != null && result.contains("2003")); - - result = (String) CTimeConvertFunction.convert(0); - assertTrue(result != null && result.contains("1970")); - - result = (String) CTimeConvertFunction.convert("1066507633"); - assertTrue(result != null && result.contains("2003")); + // Default format is %m/%d/%Y %H:%M:%S + assertEquals("10/18/2003 20:07:13", CTimeConvertFunction.convert(1066507633)); + assertEquals("01/01/1970 00:00:00", CTimeConvertFunction.convert(0)); + assertEquals("10/18/2003 20:07:13", CTimeConvertFunction.convert("1066507633")); } @Test @@ -362,9 +358,9 @@ public void testCtimeConvertInvalid() { // mktime() Function Tests @Test public void testMktimeConvertBasic() { - assertEquals(1066507633.0, MkTimeConvertFunction.convert("2003-10-18 20:07:13")); - assertEquals(1066507633.0, MkTimeConvertFunction.convert("2003-10-18T20:07:13")); - assertEquals(946684800.0, MkTimeConvertFunction.convert("2000-01-01 00:00:00")); + // Default format is %m/%d/%Y %H:%M:%S + assertEquals(1066507633.0, MkTimeConvertFunction.convert("10/18/2003 20:07:13")); + assertEquals(946684800.0, MkTimeConvertFunction.convert("01/01/2000 00:00:00")); assertEquals(1066473433.0, MkTimeConvertFunction.convert(1066473433)); assertEquals(1066473433.0, MkTimeConvertFunction.convert("1066473433")); } @@ -385,6 +381,10 @@ public void testMstimeConvertBasic() { assertEquals(90.5, MsTimeConvertFunction.convert("01:30.5")); assertEquals(3661.0, MsTimeConvertFunction.convert("61:01")); + // SS.SSS without MM: prefix + assertEquals(45.123, MsTimeConvertFunction.convert("45.123")); + assertEquals(30.0, MsTimeConvertFunction.convert("30")); + // Test already numeric assertEquals(225.0, MsTimeConvertFunction.convert(225)); assertEquals(225.0, MsTimeConvertFunction.convert("225")); @@ -437,40 +437,31 @@ public void testDur2secConvertInvalid() { // timeformat tests for mktime() and ctime() @Test public void testMktimeWithCustomTimeformat() { - // Test mktime with custom timeformat - assertEquals(1066507633.0, MkTimeConvertFunction.convertWithFormat("18/10/2003 20:07:13", "dd/MM/yyyy HH:mm:ss")); - assertEquals(1066507633.0, MkTimeConvertFunction.convertWithFormat("2003-10-18 20:07:13", "yyyy-MM-dd HH:mm:ss")); - assertEquals(946684800.0, MkTimeConvertFunction.convertWithFormat("01/01/2000 00:00:00", "dd/MM/yyyy HH:mm:ss")); + // Strftime format specifiers + assertEquals(1066507633.0, MkTimeConvertFunction.convertWithFormat("18/10/2003 20:07:13", "%d/%m/%Y %H:%M:%S")); + assertEquals(1066507633.0, MkTimeConvertFunction.convertWithFormat("2003-10-18 20:07:13", "%Y-%m-%d %H:%M:%S")); + assertEquals(946684800.0, MkTimeConvertFunction.convertWithFormat("01/01/2000 00:00:00", "%d/%m/%Y %H:%M:%S")); - // Test fallback to default formats when custom format fails - assertEquals(1066507633.0, MkTimeConvertFunction.convertWithFormat("2003-10-18 20:07:13", "invalid format")); + // Invalid format returns null + assertNull(MkTimeConvertFunction.convertWithFormat("2003-10-18 20:07:13", "invalid format")); - // Test null/empty timeformat - assertEquals(1066507633.0, MkTimeConvertFunction.convertWithFormat("2003-10-18 20:07:13", null)); - assertEquals(1066507633.0, MkTimeConvertFunction.convertWithFormat("2003-10-18 20:07:13", "")); + // Null/empty timeformat falls back to default %m/%d/%Y %H:%M:%S + assertEquals(1066507633.0, MkTimeConvertFunction.convertWithFormat("10/18/2003 20:07:13", null)); + assertEquals(1066507633.0, MkTimeConvertFunction.convertWithFormat("10/18/2003 20:07:13", "")); } @Test public void testCtimeWithCustomTimeformat() { - // Test ctime with custom timeformat - String result1 = (String) CTimeConvertFunction.convertWithFormat(1066507633, "yyyy-MM-dd HH:mm:ss"); - assertTrue(result1 != null && result1.contains("2003-10-18")); - - String result2 = (String) CTimeConvertFunction.convertWithFormat(1066507633, "dd/MM/yyyy"); - assertTrue(result2 != null && result2.contains("18/10/2003")); - - String result3 = (String) CTimeConvertFunction.convertWithFormat(0, "yyyy"); - assertTrue(result3 != null && result3.contains("1970")); - - // Test fallback to default format when custom format fails - String result4 = (String) CTimeConvertFunction.convertWithFormat(1066507633, "invalid format"); - assertTrue(result4 != null && result4.contains("2003")); + // Strftime format specifiers + assertEquals("2003-10-18 20:07:13", CTimeConvertFunction.convertWithFormat(1066507633, "%Y-%m-%d %H:%M:%S")); + assertEquals("18/10/2003", CTimeConvertFunction.convertWithFormat(1066507633, "%d/%m/%Y")); + assertEquals("1970", CTimeConvertFunction.convertWithFormat(0, "%Y")); - // Test null/empty timeformat - String result5 = (String) CTimeConvertFunction.convertWithFormat(1066507633, null); - assertTrue(result5 != null && result5.contains("2003")); + // Null/empty timeformat falls back to default + String result = CTimeConvertFunction.convertWithFormat(1066507633, null); + assertEquals("10/18/2003 20:07:13", result); - String result6 = (String) CTimeConvertFunction.convertWithFormat(1066507633, ""); - assertTrue(result6 != null && result6.contains("2003")); + result = CTimeConvertFunction.convertWithFormat(1066507633, ""); + assertEquals("10/18/2003 20:07:13", result); } } diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteConvertCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteConvertCommandIT.java index 69118300385..a6d20796899 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteConvertCommandIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteConvertCommandIT.java @@ -268,7 +268,7 @@ public void testConvertMktimeWithDefaultFormat() throws IOException { JSONObject result = executeQuery( String.format( - "search source=%s | eval date_str = '2003-10-18 20:07:13' | convert mktime(date_str) |" + "search source=%s | eval date_str = '10/18/2003 20:07:13' | convert mktime(date_str) |" + " fields date_str | head 1", TEST_INDEX_BANK)); verifySchema(result, schema("date_str", null, "double")); @@ -280,7 +280,7 @@ public void testConvertMktimeWithCustomTimeformat() throws IOException { JSONObject result = executeQuery( String.format( - "search source=%s | eval date_str = '18/10/2003 20:07:13' | convert timeformat=\"dd/MM/yyyy HH:mm:ss\" mktime(date_str) |" + "search source=%s | eval date_str = '18/10/2003 20:07:13' | convert timeformat=\"%%d/%%m/%%Y %%H:%%M:%%S\" mktime(date_str) |" + " fields date_str | head 1", TEST_INDEX_BANK)); verifySchema(result, schema("date_str", null, "double")); @@ -296,10 +296,7 @@ public void testConvertCtimeWithDefaultFormat() throws IOException { + " fields timestamp | head 1", TEST_INDEX_BANK)); verifySchema(result, schema("timestamp", null, "string")); - verifyNumOfRows(result, 1); - // Verify it contains expected year - exact format may vary by timezone - String timestampValue = result.getJSONArray("datarows").getJSONArray(0).getString(0); - assertTrue("Expected timestamp to contain '2003'", timestampValue.contains("2003")); + verifyDataRows(result, rows("10/18/2003 20:07:13")); } @Test @@ -307,7 +304,7 @@ public void testConvertCtimeWithCustomTimeformat() throws IOException { JSONObject result = executeQuery( String.format( - "search source=%s | eval timestamp = 1066507633 | convert timeformat=\"yyyy-MM-dd HH:mm:ss\" ctime(timestamp) |" + "search source=%s | eval timestamp = 1066507633 | convert timeformat=\"%%Y-%%m-%%d %%H:%%M:%%S\" ctime(timestamp) |" + " fields timestamp | head 1", TEST_INDEX_BANK)); verifySchema(result, schema("timestamp", null, "string")); @@ -344,16 +341,13 @@ public void testConvertTimeformatWithMultipleFunctions() throws IOException { executeQuery( String.format( "search source=%s | eval date_str = '18/10/2003 20:07:13', timestamp = 1066507633 |" - + " convert timeformat=\"dd/MM/yyyy HH:mm:ss\" mktime(date_str), ctime(timestamp) |" + + " convert timeformat=\"%%d/%%m/%%Y %%H:%%M:%%S\" mktime(date_str), ctime(timestamp) |" + " fields date_str, timestamp | head 1", TEST_INDEX_BANK)); verifySchema(result, schema("date_str", null, "double"), schema("timestamp", null, "string")); verifyNumOfRows(result, 1); - // Verify mktime conversion assertEquals(1066507633.0, result.getJSONArray("datarows").getJSONArray(0).getDouble(0), 0.001); - // Verify ctime conversion contains expected year - String timestampValue = result.getJSONArray("datarows").getJSONArray(0).getString(1); - assertTrue("Expected timestamp to contain '2003'", timestampValue.contains("2003")); + assertEquals("18/10/2003 20:07:13", result.getJSONArray("datarows").getJSONArray(0).getString(1)); } @Test @@ -361,7 +355,7 @@ public void testConvertTimeformatWithWhere() throws IOException { JSONObject result = executeQuery( String.format( - "search source=%s | eval date_str = '2003-10-18 20:07:13' |" + "search source=%s | eval date_str = '10/18/2003 20:07:13' |" + " convert mktime(date_str) | where date_str > 1000000000 |" + " fields date_str | head 1", TEST_INDEX_BANK)); @@ -375,7 +369,7 @@ public void testConvertTimeformatWithStats() throws IOException { executeQuery( String.format( "search source=%s | eval timestamp = 1066507633 |" - + " convert timeformat=\"yyyy\" ctime(timestamp) |" + + " convert timeformat=\"%%Y\" ctime(timestamp) |" + " stats count() by timestamp", TEST_INDEX_BANK)); verifySchema(result, schema("count()", null, "long"), schema("timestamp", "string")); diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizer.java b/ppl/src/main/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizer.java index 53c1ab71e63..611b36956ca 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizer.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizer.java @@ -528,7 +528,11 @@ public String visitConvert(Convert node, String context) { return StringUtils.format("%s(%s)%s", functionName, fields, asClause); }) .collect(Collectors.joining(",")); - return StringUtils.format("%s | convert %s", child, conversions); + String timeformatClause = + node.getTimeFormat() != null + ? StringUtils.format("timeformat=\"%s\" ", node.getTimeFormat()) + : ""; + return StringUtils.format("%s | convert %s%s", child, timeformatClause, conversions); } @Override diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizerTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizerTest.java index 7aeba3d6c98..866c45a26e0 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizerTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizerTest.java @@ -1127,6 +1127,26 @@ public void testConvertCommand() { assertEquals( "source=table | convert (identifier) AS identifier", anonymize("source=t | convert none(empno) AS empno_same")); + assertEquals( + "source=table | convert dur2sec(identifier)", + anonymize("source=t | convert dur2sec(duration)")); + assertEquals( + "source=table | convert mstime(identifier)", + anonymize("source=t | convert mstime(elapsed)")); + assertEquals( + "source=table | convert memk(identifier) AS identifier", + anonymize("source=t | convert memk(virt) AS virt_kb")); + } + + @Test + public void testConvertCommandWithTimeformat() { + assertEquals( + "source=table | convert timeformat=\"%Y-%m-%d\" mktime(identifier)", + anonymize("source=t | convert timeformat=\"%Y-%m-%d\" mktime(date_str)")); + assertEquals( + "source=table | convert timeformat=\"%m/%d/%Y %H:%M:%S\" ctime(identifier) AS identifier", + anonymize( + "source=t | convert timeformat=\"%m/%d/%Y %H:%M:%S\" ctime(ts) AS formatted_time")); } @Test From 73f4088cdbb47478672b0fccc1fbd015953bcbcd Mon Sep 17 00:00:00 2001 From: Ritvi Bhatt Date: Fri, 6 Mar 2026 14:13:47 -0800 Subject: [PATCH 04/22] fix timeformat parsing in integ tests Signed-off-by: Ritvi Bhatt --- .../function/udf/CTimeConvertFunction.java | 4 +- .../function/udf/MkTimeConvertFunction.java | 4 +- .../remote/CalciteConvertCommandIT.java | 40 +++++++++---------- 3 files changed, 24 insertions(+), 24 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/expression/function/udf/CTimeConvertFunction.java b/core/src/main/java/org/opensearch/sql/expression/function/udf/CTimeConvertFunction.java index e75529d5dfb..e4ba170d79d 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/udf/CTimeConvertFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/udf/CTimeConvertFunction.java @@ -51,11 +51,11 @@ public Expression implement( if (translatedOperands.isEmpty()) { return Expressions.constant(null, String.class); } - Expression fieldValue = translatedOperands.get(0); + Expression fieldValue = Expressions.box(translatedOperands.get(0)); if (translatedOperands.size() == 1) { return Expressions.call(CTimeConvertFunction.class, "convert", fieldValue); } - Expression timeFormat = translatedOperands.get(1); + Expression timeFormat = Expressions.box(translatedOperands.get(1)); return Expressions.call( CTimeConvertFunction.class, "convertWithFormat", fieldValue, timeFormat); } diff --git a/core/src/main/java/org/opensearch/sql/expression/function/udf/MkTimeConvertFunction.java b/core/src/main/java/org/opensearch/sql/expression/function/udf/MkTimeConvertFunction.java index 9fa1f7035ad..20c61b554d8 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/udf/MkTimeConvertFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/udf/MkTimeConvertFunction.java @@ -58,11 +58,11 @@ public Expression implement( if (translatedOperands.isEmpty()) { return Expressions.constant(null, Double.class); } - Expression fieldValue = translatedOperands.get(0); + Expression fieldValue = Expressions.box(translatedOperands.get(0)); if (translatedOperands.size() == 1) { return Expressions.call(MkTimeConvertFunction.class, "convert", fieldValue); } - Expression timeFormat = translatedOperands.get(1); + Expression timeFormat = Expressions.box(translatedOperands.get(1)); return Expressions.call( MkTimeConvertFunction.class, "convertWithFormat", fieldValue, timeFormat); } diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteConvertCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteConvertCommandIT.java index a6d20796899..05fe950ebbd 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteConvertCommandIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteConvertCommandIT.java @@ -279,10 +279,10 @@ public void testConvertMktimeWithDefaultFormat() throws IOException { public void testConvertMktimeWithCustomTimeformat() throws IOException { JSONObject result = executeQuery( - String.format( - "search source=%s | eval date_str = '18/10/2003 20:07:13' | convert timeformat=\"%%d/%%m/%%Y %%H:%%M:%%S\" mktime(date_str) |" - + " fields date_str | head 1", - TEST_INDEX_BANK)); + "search source=" + + TEST_INDEX_BANK + + " | eval date_str = '18/10/2003 20:07:13' | convert timeformat=\\\"%d/%m/%Y %H:%M:%S\\\" mktime(date_str) |" + + " fields date_str | head 1"); verifySchema(result, schema("date_str", null, "double")); verifyDataRows(result, rows(1066507633.0)); } @@ -303,10 +303,10 @@ public void testConvertCtimeWithDefaultFormat() throws IOException { public void testConvertCtimeWithCustomTimeformat() throws IOException { JSONObject result = executeQuery( - String.format( - "search source=%s | eval timestamp = 1066507633 | convert timeformat=\"%%Y-%%m-%%d %%H:%%M:%%S\" ctime(timestamp) |" - + " fields timestamp | head 1", - TEST_INDEX_BANK)); + "search source=" + + TEST_INDEX_BANK + + " | eval timestamp = 1066507633 | convert timeformat=\\\"%Y-%m-%d %H:%M:%S\\\" ctime(timestamp) |" + + " fields timestamp | head 1"); verifySchema(result, schema("timestamp", null, "string")); verifyDataRows(result, rows("2003-10-18 20:07:13")); } @@ -339,11 +339,11 @@ public void testConvertMstimeFunction() throws IOException { public void testConvertTimeformatWithMultipleFunctions() throws IOException { JSONObject result = executeQuery( - String.format( - "search source=%s | eval date_str = '18/10/2003 20:07:13', timestamp = 1066507633 |" - + " convert timeformat=\"%%d/%%m/%%Y %%H:%%M:%%S\" mktime(date_str), ctime(timestamp) |" - + " fields date_str, timestamp | head 1", - TEST_INDEX_BANK)); + "search source=" + + TEST_INDEX_BANK + + " | eval date_str = '18/10/2003 20:07:13', timestamp = 1066507633 |" + + " convert timeformat=\\\"%d/%m/%Y %H:%M:%S\\\" mktime(date_str), ctime(timestamp) |" + + " fields date_str, timestamp | head 1"); verifySchema(result, schema("date_str", null, "double"), schema("timestamp", null, "string")); verifyNumOfRows(result, 1); assertEquals(1066507633.0, result.getJSONArray("datarows").getJSONArray(0).getDouble(0), 0.001); @@ -367,12 +367,12 @@ public void testConvertTimeformatWithWhere() throws IOException { public void testConvertTimeformatWithStats() throws IOException { JSONObject result = executeQuery( - String.format( - "search source=%s | eval timestamp = 1066507633 |" - + " convert timeformat=\"%%Y\" ctime(timestamp) |" - + " stats count() by timestamp", - TEST_INDEX_BANK)); - verifySchema(result, schema("count()", null, "long"), schema("timestamp", "string")); - verifyDataRows(result, rows(1000L, "2003")); + "search source=" + + TEST_INDEX_BANK + + " | eval timestamp = 1066507633 |" + + " convert timeformat=\\\"%Y\\\" ctime(timestamp) |" + + " stats count() by timestamp"); + verifySchema(result, schema("count()", null, "bigint"), schema("timestamp", "string")); + verifyDataRows(result, rows(7, "2003")); } } From 82fd69f0a7070da1f8f5cadc556f9149c451299c Mon Sep 17 00:00:00 2001 From: Ritvi Bhatt Date: Fri, 6 Mar 2026 14:16:01 -0800 Subject: [PATCH 05/22] update comments Signed-off-by: Ritvi Bhatt --- .../sql/expression/function/udf/CTimeConvertFunction.java | 2 +- .../sql/expression/function/udf/Dur2SecConvertFunction.java | 1 - .../sql/expression/function/udf/MkTimeConvertFunction.java | 2 +- .../sql/expression/function/udf/MsTimeConvertFunction.java | 2 +- 4 files changed, 3 insertions(+), 4 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/expression/function/udf/CTimeConvertFunction.java b/core/src/main/java/org/opensearch/sql/expression/function/udf/CTimeConvertFunction.java index e4ba170d79d..5bd9f482e7e 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/udf/CTimeConvertFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/udf/CTimeConvertFunction.java @@ -24,7 +24,7 @@ /** * PPL ctime() conversion function. Converts UNIX epoch timestamps to human-readable time strings - * using strftime format specifiers. Default format: {@code %m/%d/%Y %H:%M:%S} (SPL-compatible). + * using strftime format specifiers. Default format: {@code %m/%d/%Y %H:%M:%S}. */ public class CTimeConvertFunction extends ImplementorUDF { diff --git a/core/src/main/java/org/opensearch/sql/expression/function/udf/Dur2SecConvertFunction.java b/core/src/main/java/org/opensearch/sql/expression/function/udf/Dur2SecConvertFunction.java index 58bca365aae..80365e55e49 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/udf/Dur2SecConvertFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/udf/Dur2SecConvertFunction.java @@ -10,7 +10,6 @@ /** * PPL dur2sec() conversion function. Converts duration format {@code [D+]HH:MM:SS} to seconds - * (SPL-compatible). */ public class Dur2SecConvertFunction extends BaseConversionUDF { diff --git a/core/src/main/java/org/opensearch/sql/expression/function/udf/MkTimeConvertFunction.java b/core/src/main/java/org/opensearch/sql/expression/function/udf/MkTimeConvertFunction.java index 20c61b554d8..180d890e2ed 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/udf/MkTimeConvertFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/udf/MkTimeConvertFunction.java @@ -27,7 +27,7 @@ /** * PPL mktime() conversion function. Parses a human-readable time string into UNIX epoch seconds - * using strftime format specifiers. Default format: {@code %m/%d/%Y %H:%M:%S} (SPL-compatible). + * using strftime format specifiers. Default format: {@code %m/%d/%Y %H:%M:%S}. */ public class MkTimeConvertFunction extends ImplementorUDF { diff --git a/core/src/main/java/org/opensearch/sql/expression/function/udf/MsTimeConvertFunction.java b/core/src/main/java/org/opensearch/sql/expression/function/udf/MsTimeConvertFunction.java index 2180040c39a..190aace4e1b 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/udf/MsTimeConvertFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/udf/MsTimeConvertFunction.java @@ -10,7 +10,7 @@ /** * PPL mstime() conversion function. Converts {@code [MM:]SS.SSS} format to seconds - * (SPL-compatible). The minutes portion is optional. + * The minutes portion is optional. */ public class MsTimeConvertFunction extends BaseConversionUDF { From b45ff6db595570efe570f097af1d08277da24a41 Mon Sep 17 00:00:00 2001 From: Ritvi Bhatt Date: Fri, 6 Mar 2026 14:27:07 -0800 Subject: [PATCH 06/22] apply spotless Signed-off-by: Ritvi Bhatt --- .../sql/calcite/CalciteRelNodeVisitor.java | 6 +++- .../sql/calcite/utils/PPLOperandTypes.java | 3 +- .../function/PPLBuiltinOperators.java | 8 ++--- .../expression/function/PPLFuncImpTable.java | 8 ++--- .../function/udf/Dur2SecConvertFunction.java | 4 +-- .../function/udf/MkTimeConvertFunction.java | 3 +- .../function/udf/MsTimeConvertFunction.java | 4 +-- .../function/udf/ConversionFunctionsTest.java | 20 ++++++++---- .../remote/CalciteConvertCommandIT.java | 20 ++++++------ .../opensearch/sql/ppl/ConvertCommandIT.java | 6 ++-- .../opensearch/sql/ppl/parser/AstBuilder.java | 4 ++- .../ppl/calcite/CalcitePPLConvertTest.java | 31 ++++++++++--------- 12 files changed, 65 insertions(+), 52 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java b/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java index c3986f3afc9..c7d3831a947 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java +++ b/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java @@ -1051,7 +1051,11 @@ private void processFieldCopyConversion( } private void processFunctionConversion( - String target, Function function, String timeFormat, ConversionState state, CalcitePlanContext context) { + String target, + Function function, + String timeFormat, + ConversionState state, + CalcitePlanContext context) { String functionName = function.getFuncName(); List args = function.getFuncArgs(); diff --git a/core/src/main/java/org/opensearch/sql/calcite/utils/PPLOperandTypes.java b/core/src/main/java/org/opensearch/sql/calcite/utils/PPLOperandTypes.java index a6570648981..fcd361ba229 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/utils/PPLOperandTypes.java +++ b/core/src/main/java/org/opensearch/sql/calcite/utils/PPLOperandTypes.java @@ -87,8 +87,7 @@ private PPLOperandTypes() {} public static final UDFOperandMetadata ANY_OPTIONAL_STRING = UDFOperandMetadata.wrap( (CompositeOperandTypeChecker) - OperandTypes.ANY.or( - OperandTypes.family(SqlTypeFamily.ANY, SqlTypeFamily.CHARACTER))); + OperandTypes.ANY.or(OperandTypes.family(SqlTypeFamily.ANY, SqlTypeFamily.CHARACTER))); public static final UDFOperandMetadata ANY_OPTIONAL_TIMESTAMP = UDFOperandMetadata.wrap( (CompositeOperandTypeChecker) diff --git a/core/src/main/java/org/opensearch/sql/expression/function/PPLBuiltinOperators.java b/core/src/main/java/org/opensearch/sql/expression/function/PPLBuiltinOperators.java index 6e3440acd48..0a5b0fe0e03 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/PPLBuiltinOperators.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/PPLBuiltinOperators.java @@ -63,8 +63,12 @@ import org.opensearch.sql.expression.function.jsonUDF.JsonKeysFunctionImpl; import org.opensearch.sql.expression.function.jsonUDF.JsonSetFunctionImpl; import org.opensearch.sql.expression.function.udf.AutoConvertFunction; +import org.opensearch.sql.expression.function.udf.CTimeConvertFunction; import org.opensearch.sql.expression.function.udf.CryptographicFunction; +import org.opensearch.sql.expression.function.udf.Dur2SecConvertFunction; import org.opensearch.sql.expression.function.udf.MemkConvertFunction; +import org.opensearch.sql.expression.function.udf.MkTimeConvertFunction; +import org.opensearch.sql.expression.function.udf.MsTimeConvertFunction; import org.opensearch.sql.expression.function.udf.NumConvertFunction; import org.opensearch.sql.expression.function.udf.ParseFunction; import org.opensearch.sql.expression.function.udf.RelevanceQueryFunction; @@ -73,10 +77,6 @@ import org.opensearch.sql.expression.function.udf.RexOffsetFunction; import org.opensearch.sql.expression.function.udf.RmcommaConvertFunction; import org.opensearch.sql.expression.function.udf.RmunitConvertFunction; -import org.opensearch.sql.expression.function.udf.CTimeConvertFunction; -import org.opensearch.sql.expression.function.udf.MkTimeConvertFunction; -import org.opensearch.sql.expression.function.udf.MsTimeConvertFunction; -import org.opensearch.sql.expression.function.udf.Dur2SecConvertFunction; import org.opensearch.sql.expression.function.udf.SpanFunction; import org.opensearch.sql.expression.function.udf.ToNumberFunction; import org.opensearch.sql.expression.function.udf.ToStringFunction; diff --git a/core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java b/core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java index 638e974ad7d..66713b18b80 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java @@ -24,7 +24,6 @@ import static org.opensearch.sql.expression.function.BuiltinFunctionName.ATAN; import static org.opensearch.sql.expression.function.BuiltinFunctionName.ATAN2; import static org.opensearch.sql.expression.function.BuiltinFunctionName.AUTO; -import static org.opensearch.sql.expression.function.BuiltinFunctionName.CTIME; import static org.opensearch.sql.expression.function.BuiltinFunctionName.AVG; import static org.opensearch.sql.expression.function.BuiltinFunctionName.CBRT; import static org.opensearch.sql.expression.function.BuiltinFunctionName.CEIL; @@ -40,6 +39,7 @@ import static org.opensearch.sql.expression.function.BuiltinFunctionName.COT; import static org.opensearch.sql.expression.function.BuiltinFunctionName.COUNT; import static org.opensearch.sql.expression.function.BuiltinFunctionName.CRC32; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.CTIME; import static org.opensearch.sql.expression.function.BuiltinFunctionName.CURDATE; import static org.opensearch.sql.expression.function.BuiltinFunctionName.CURRENT_DATE; import static org.opensearch.sql.expression.function.BuiltinFunctionName.CURRENT_TIME; @@ -62,6 +62,7 @@ import static org.opensearch.sql.expression.function.BuiltinFunctionName.DEGREES; import static org.opensearch.sql.expression.function.BuiltinFunctionName.DIVIDE; import static org.opensearch.sql.expression.function.BuiltinFunctionName.DIVIDEFUNCTION; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.DUR2SEC; import static org.opensearch.sql.expression.function.BuiltinFunctionName.E; import static org.opensearch.sql.expression.function.BuiltinFunctionName.EARLIEST; import static org.opensearch.sql.expression.function.BuiltinFunctionName.EQUAL; @@ -139,21 +140,20 @@ import static org.opensearch.sql.expression.function.BuiltinFunctionName.MD5; import static org.opensearch.sql.expression.function.BuiltinFunctionName.MEDIAN; import static org.opensearch.sql.expression.function.BuiltinFunctionName.MEMK; -import static org.opensearch.sql.expression.function.BuiltinFunctionName.MKTIME; -import static org.opensearch.sql.expression.function.BuiltinFunctionName.MSTIME; -import static org.opensearch.sql.expression.function.BuiltinFunctionName.DUR2SEC; import static org.opensearch.sql.expression.function.BuiltinFunctionName.MICROSECOND; import static org.opensearch.sql.expression.function.BuiltinFunctionName.MIN; import static org.opensearch.sql.expression.function.BuiltinFunctionName.MINSPAN_BUCKET; import static org.opensearch.sql.expression.function.BuiltinFunctionName.MINUTE; import static org.opensearch.sql.expression.function.BuiltinFunctionName.MINUTE_OF_DAY; import static org.opensearch.sql.expression.function.BuiltinFunctionName.MINUTE_OF_HOUR; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.MKTIME; import static org.opensearch.sql.expression.function.BuiltinFunctionName.MOD; import static org.opensearch.sql.expression.function.BuiltinFunctionName.MODULUS; import static org.opensearch.sql.expression.function.BuiltinFunctionName.MODULUSFUNCTION; import static org.opensearch.sql.expression.function.BuiltinFunctionName.MONTH; import static org.opensearch.sql.expression.function.BuiltinFunctionName.MONTHNAME; import static org.opensearch.sql.expression.function.BuiltinFunctionName.MONTH_OF_YEAR; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.MSTIME; import static org.opensearch.sql.expression.function.BuiltinFunctionName.MULTIPLY; import static org.opensearch.sql.expression.function.BuiltinFunctionName.MULTIPLYFUNCTION; import static org.opensearch.sql.expression.function.BuiltinFunctionName.MULTI_MATCH; diff --git a/core/src/main/java/org/opensearch/sql/expression/function/udf/Dur2SecConvertFunction.java b/core/src/main/java/org/opensearch/sql/expression/function/udf/Dur2SecConvertFunction.java index 80365e55e49..78facf743be 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/udf/Dur2SecConvertFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/udf/Dur2SecConvertFunction.java @@ -8,9 +8,7 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; -/** - * PPL dur2sec() conversion function. Converts duration format {@code [D+]HH:MM:SS} to seconds - */ +/** PPL dur2sec() conversion function. Converts duration format {@code [D+]HH:MM:SS} to seconds */ public class Dur2SecConvertFunction extends BaseConversionUDF { public static final Dur2SecConvertFunction INSTANCE = new Dur2SecConvertFunction(); diff --git a/core/src/main/java/org/opensearch/sql/expression/function/udf/MkTimeConvertFunction.java b/core/src/main/java/org/opensearch/sql/expression/function/udf/MkTimeConvertFunction.java index 180d890e2ed..204fafca779 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/udf/MkTimeConvertFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/udf/MkTimeConvertFunction.java @@ -100,8 +100,7 @@ public static Object convertWithFormat(Object value, Object timeFormatObj) { private static Object parseWithFormat(String dateStr, String strftimeFormat) { try { String javaPattern = StrftimeFormatterUtil.toJavaPattern(strftimeFormat); - DateTimeFormatter formatter = - DateTimeFormatter.ofPattern(javaPattern, Locale.ROOT); + DateTimeFormatter formatter = DateTimeFormatter.ofPattern(javaPattern, Locale.ROOT); LocalDateTime dateTime = LocalDateTime.parse(dateStr, formatter); return (double) dateTime.toEpochSecond(ZoneOffset.UTC); } catch (DateTimeParseException | IllegalArgumentException e) { diff --git a/core/src/main/java/org/opensearch/sql/expression/function/udf/MsTimeConvertFunction.java b/core/src/main/java/org/opensearch/sql/expression/function/udf/MsTimeConvertFunction.java index 190aace4e1b..362896b06b9 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/udf/MsTimeConvertFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/udf/MsTimeConvertFunction.java @@ -9,8 +9,8 @@ import java.util.regex.Pattern; /** - * PPL mstime() conversion function. Converts {@code [MM:]SS.SSS} format to seconds - * The minutes portion is optional. + * PPL mstime() conversion function. Converts {@code [MM:]SS.SSS} format to seconds The minutes + * portion is optional. */ public class MsTimeConvertFunction extends BaseConversionUDF { diff --git a/core/src/test/java/org/opensearch/sql/expression/function/udf/ConversionFunctionsTest.java b/core/src/test/java/org/opensearch/sql/expression/function/udf/ConversionFunctionsTest.java index 4ebb445b136..928fc61ca67 100644 --- a/core/src/test/java/org/opensearch/sql/expression/function/udf/ConversionFunctionsTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/function/udf/ConversionFunctionsTest.java @@ -7,7 +7,6 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNull; -import static org.junit.jupiter.api.Assertions.assertTrue; import org.junit.jupiter.api.Test; @@ -438,22 +437,31 @@ public void testDur2secConvertInvalid() { @Test public void testMktimeWithCustomTimeformat() { // Strftime format specifiers - assertEquals(1066507633.0, MkTimeConvertFunction.convertWithFormat("18/10/2003 20:07:13", "%d/%m/%Y %H:%M:%S")); - assertEquals(1066507633.0, MkTimeConvertFunction.convertWithFormat("2003-10-18 20:07:13", "%Y-%m-%d %H:%M:%S")); - assertEquals(946684800.0, MkTimeConvertFunction.convertWithFormat("01/01/2000 00:00:00", "%d/%m/%Y %H:%M:%S")); + assertEquals( + 1066507633.0, + MkTimeConvertFunction.convertWithFormat("18/10/2003 20:07:13", "%d/%m/%Y %H:%M:%S")); + assertEquals( + 1066507633.0, + MkTimeConvertFunction.convertWithFormat("2003-10-18 20:07:13", "%Y-%m-%d %H:%M:%S")); + assertEquals( + 946684800.0, + MkTimeConvertFunction.convertWithFormat("01/01/2000 00:00:00", "%d/%m/%Y %H:%M:%S")); // Invalid format returns null assertNull(MkTimeConvertFunction.convertWithFormat("2003-10-18 20:07:13", "invalid format")); // Null/empty timeformat falls back to default %m/%d/%Y %H:%M:%S - assertEquals(1066507633.0, MkTimeConvertFunction.convertWithFormat("10/18/2003 20:07:13", null)); + assertEquals( + 1066507633.0, MkTimeConvertFunction.convertWithFormat("10/18/2003 20:07:13", null)); assertEquals(1066507633.0, MkTimeConvertFunction.convertWithFormat("10/18/2003 20:07:13", "")); } @Test public void testCtimeWithCustomTimeformat() { // Strftime format specifiers - assertEquals("2003-10-18 20:07:13", CTimeConvertFunction.convertWithFormat(1066507633, "%Y-%m-%d %H:%M:%S")); + assertEquals( + "2003-10-18 20:07:13", + CTimeConvertFunction.convertWithFormat(1066507633, "%Y-%m-%d %H:%M:%S")); assertEquals("18/10/2003", CTimeConvertFunction.convertWithFormat(1066507633, "%d/%m/%Y")); assertEquals("1970", CTimeConvertFunction.convertWithFormat(0, "%Y")); diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteConvertCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteConvertCommandIT.java index 05fe950ebbd..7d666951b1a 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteConvertCommandIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteConvertCommandIT.java @@ -6,7 +6,6 @@ package org.opensearch.sql.calcite.remote; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_BANK; import static org.opensearch.sql.util.MatcherUtils.rows; import static org.opensearch.sql.util.MatcherUtils.schema; @@ -268,8 +267,8 @@ public void testConvertMktimeWithDefaultFormat() throws IOException { JSONObject result = executeQuery( String.format( - "search source=%s | eval date_str = '10/18/2003 20:07:13' | convert mktime(date_str) |" - + " fields date_str | head 1", + "search source=%s | eval date_str = '10/18/2003 20:07:13' | convert" + + " mktime(date_str) | fields date_str | head 1", TEST_INDEX_BANK)); verifySchema(result, schema("date_str", null, "double")); verifyDataRows(result, rows(1066507633.0)); @@ -281,8 +280,8 @@ public void testConvertMktimeWithCustomTimeformat() throws IOException { executeQuery( "search source=" + TEST_INDEX_BANK - + " | eval date_str = '18/10/2003 20:07:13' | convert timeformat=\\\"%d/%m/%Y %H:%M:%S\\\" mktime(date_str) |" - + " fields date_str | head 1"); + + " | eval date_str = '18/10/2003 20:07:13' | convert timeformat=\\\"%d/%m/%Y" + + " %H:%M:%S\\\" mktime(date_str) | fields date_str | head 1"); verifySchema(result, schema("date_str", null, "double")); verifyDataRows(result, rows(1066507633.0)); } @@ -305,8 +304,8 @@ public void testConvertCtimeWithCustomTimeformat() throws IOException { executeQuery( "search source=" + TEST_INDEX_BANK - + " | eval timestamp = 1066507633 | convert timeformat=\\\"%Y-%m-%d %H:%M:%S\\\" ctime(timestamp) |" - + " fields timestamp | head 1"); + + " | eval timestamp = 1066507633 | convert timeformat=\\\"%Y-%m-%d %H:%M:%S\\\"" + + " ctime(timestamp) | fields timestamp | head 1"); verifySchema(result, schema("timestamp", null, "string")); verifyDataRows(result, rows("2003-10-18 20:07:13")); } @@ -341,13 +340,14 @@ public void testConvertTimeformatWithMultipleFunctions() throws IOException { executeQuery( "search source=" + TEST_INDEX_BANK - + " | eval date_str = '18/10/2003 20:07:13', timestamp = 1066507633 |" - + " convert timeformat=\\\"%d/%m/%Y %H:%M:%S\\\" mktime(date_str), ctime(timestamp) |" + + " | eval date_str = '18/10/2003 20:07:13', timestamp = 1066507633 | convert" + + " timeformat=\\\"%d/%m/%Y %H:%M:%S\\\" mktime(date_str), ctime(timestamp) |" + " fields date_str, timestamp | head 1"); verifySchema(result, schema("date_str", null, "double"), schema("timestamp", null, "string")); verifyNumOfRows(result, 1); assertEquals(1066507633.0, result.getJSONArray("datarows").getJSONArray(0).getDouble(0), 0.001); - assertEquals("18/10/2003 20:07:13", result.getJSONArray("datarows").getJSONArray(0).getString(1)); + assertEquals( + "18/10/2003 20:07:13", result.getJSONArray("datarows").getJSONArray(0).getString(1)); } @Test diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/ConvertCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/ConvertCommandIT.java index 02dba43113f..0a6df8dc740 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/ConvertCommandIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/ConvertCommandIT.java @@ -72,7 +72,8 @@ public void testConvertWithStats() { @Test public void testConvertMktimeFunction() { verifyQueryThrowsCalciteError( - "source=%s | eval date_str = '2003-10-18 20:07:13' | convert mktime(date_str) | fields date_str"); + "source=%s | eval date_str = '2003-10-18 20:07:13' | convert mktime(date_str) | fields" + + " date_str"); } @Test @@ -96,7 +97,8 @@ public void testConvertMstimeFunction() { @Test public void testConvertWithTimeformat() { verifyQueryThrowsCalciteError( - "source=%s | eval date_str = '18/10/2003 20:07:13' | convert timeformat=\"dd/MM/yyyy HH:mm:ss\" mktime(date_str) | fields date_str"); + "source=%s | eval date_str = '18/10/2003 20:07:13' | convert timeformat=\"dd/MM/yyyy" + + " HH:mm:ss\" mktime(date_str) | fields date_str"); } private void verifyQueryThrowsCalciteError(String query) { diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java index 2fe17646ed4..c87cc239399 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java @@ -1223,7 +1223,9 @@ public UnresolvedPlan visitConvertCommand(OpenSearchPPLParser.ConvertCommandCont /** Supported PPL convert function names (case-insensitive). */ private static final Set SUPPORTED_CONVERSION_FUNCTIONS = - Set.of("auto", "num", "rmcomma", "rmunit", "memk", "none", "ctime", "mktime", "dur2sec", "mstime"); + Set.of( + "auto", "num", "rmcomma", "rmunit", "memk", "none", "ctime", "mktime", "dur2sec", + "mstime"); private Let buildConversion(OpenSearchPPLParser.ConvertFunctionContext funcCtx) { if (funcCtx.fieldExpression().isEmpty()) { diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLConvertTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLConvertTest.java index 16494b72d4d..f49a967aa86 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLConvertTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLConvertTest.java @@ -342,14 +342,14 @@ public void testConvertWithTimeformatMktime() { String ppl = "source=EMP | convert timeformat=\"%Y-%m-%d\" mktime(ENAME)"; RelNode root = getRelNode(ppl); String expectedLogical = - "LogicalProject(EMPNO=[$0], ENAME=[MKTIME($1, '%Y-%m-%d')], JOB=[$2], MGR=[$3], HIREDATE=[$4]," - + " SAL=[$5], COMM=[$6], DEPTNO=[$7])\n" + "LogicalProject(EMPNO=[$0], ENAME=[MKTIME($1, '%Y-%m-%d')], JOB=[$2], MGR=[$3]," + + " HIREDATE=[$4], SAL=[$5], COMM=[$6], DEPTNO=[$7])\n" + " LogicalTableScan(table=[[scott, EMP]])\n"; verifyLogical(root, expectedLogical); String expectedSparkSql = - "SELECT `EMPNO`, MKTIME(`ENAME`, '%Y-%m-%d') `ENAME`, `JOB`, `MGR`, `HIREDATE`, `SAL`, `COMM`," - + " `DEPTNO`\n" + "SELECT `EMPNO`, MKTIME(`ENAME`, '%Y-%m-%d') `ENAME`, `JOB`, `MGR`, `HIREDATE`, `SAL`," + + " `COMM`, `DEPTNO`\n" + "FROM `scott`.`EMP`"; verifyPPLToSparkSQL(root, expectedSparkSql); } @@ -359,13 +359,14 @@ public void testConvertWithTimeformatCtime() { String ppl = "source=EMP | convert timeformat=\"%Y-%m-%d %H:%M:%S\" ctime(SAL)"; RelNode root = getRelNode(ppl); String expectedLogical = - "LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4], SAL=[CTIME($5, '%Y-%m-%d %H:%M:%S')]," - + " COMM=[$6], DEPTNO=[$7])\n" + "LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4], SAL=[CTIME($5," + + " '%Y-%m-%d %H:%M:%S')], COMM=[$6], DEPTNO=[$7])\n" + " LogicalTableScan(table=[[scott, EMP]])\n"; verifyLogical(root, expectedLogical); String expectedSparkSql = - "SELECT `EMPNO`, `ENAME`, `JOB`, `MGR`, `HIREDATE`, CTIME(`SAL`, '%Y-%m-%d %H:%M:%S') `SAL`, `COMM`, `DEPTNO`\n" + "SELECT `EMPNO`, `ENAME`, `JOB`, `MGR`, `HIREDATE`, CTIME(`SAL`, '%Y-%m-%d %H:%M:%S')" + + " `SAL`, `COMM`, `DEPTNO`\n" + "FROM `scott`.`EMP`"; verifyPPLToSparkSQL(root, expectedSparkSql); } @@ -375,14 +376,14 @@ public void testConvertTimeformatWithMultipleFunctions() { String ppl = "source=EMP | convert timeformat=\"%Y-%m-%d\" mktime(ENAME), ctime(SAL)"; RelNode root = getRelNode(ppl); String expectedLogical = - "LogicalProject(EMPNO=[$0], ENAME=[MKTIME($1, '%Y-%m-%d')], JOB=[$2], MGR=[$3], HIREDATE=[$4], SAL=[CTIME($5, '%Y-%m-%d')]," - + " COMM=[$6], DEPTNO=[$7])\n" + "LogicalProject(EMPNO=[$0], ENAME=[MKTIME($1, '%Y-%m-%d')], JOB=[$2], MGR=[$3]," + + " HIREDATE=[$4], SAL=[CTIME($5, '%Y-%m-%d')], COMM=[$6], DEPTNO=[$7])\n" + " LogicalTableScan(table=[[scott, EMP]])\n"; verifyLogical(root, expectedLogical); String expectedSparkSql = - "SELECT `EMPNO`, MKTIME(`ENAME`, '%Y-%m-%d') `ENAME`, `JOB`, `MGR`, `HIREDATE`, CTIME(`SAL`, '%Y-%m-%d') `SAL`, `COMM`," - + " `DEPTNO`\n" + "SELECT `EMPNO`, MKTIME(`ENAME`, '%Y-%m-%d') `ENAME`, `JOB`, `MGR`, `HIREDATE`," + + " CTIME(`SAL`, '%Y-%m-%d') `SAL`, `COMM`, `DEPTNO`\n" + "FROM `scott`.`EMP`"; verifyPPLToSparkSQL(root, expectedSparkSql); } @@ -392,14 +393,14 @@ public void testConvertTimeformatMixedWithNonTimeFunctions() { String ppl = "source=EMP | convert timeformat=\"%Y-%m-%d\" mktime(ENAME), auto(SAL)"; RelNode root = getRelNode(ppl); String expectedLogical = - "LogicalProject(EMPNO=[$0], ENAME=[MKTIME($1, '%Y-%m-%d')], JOB=[$2], MGR=[$3], HIREDATE=[$4], SAL=[AUTO($5)]," - + " COMM=[$6], DEPTNO=[$7])\n" + "LogicalProject(EMPNO=[$0], ENAME=[MKTIME($1, '%Y-%m-%d')], JOB=[$2], MGR=[$3]," + + " HIREDATE=[$4], SAL=[AUTO($5)], COMM=[$6], DEPTNO=[$7])\n" + " LogicalTableScan(table=[[scott, EMP]])\n"; verifyLogical(root, expectedLogical); String expectedSparkSql = - "SELECT `EMPNO`, MKTIME(`ENAME`, '%Y-%m-%d') `ENAME`, `JOB`, `MGR`, `HIREDATE`, AUTO(`SAL`) `SAL`, `COMM`," - + " `DEPTNO`\n" + "SELECT `EMPNO`, MKTIME(`ENAME`, '%Y-%m-%d') `ENAME`, `JOB`, `MGR`, `HIREDATE`, AUTO(`SAL`)" + + " `SAL`, `COMM`, `DEPTNO`\n" + "FROM `scott`.`EMP`"; verifyPPLToSparkSQL(root, expectedSparkSql); } From becd73e58aad84b4478b1ce92cb2dfb56bf5a6dd Mon Sep 17 00:00:00 2001 From: Ritvi Bhatt Date: Fri, 6 Mar 2026 14:38:06 -0800 Subject: [PATCH 07/22] update convert command doc Signed-off-by: Ritvi Bhatt --- docs/user/ppl/cmd/convert.md | 105 +++++++++++++++++++++++++++++++++-- 1 file changed, 101 insertions(+), 4 deletions(-) diff --git a/docs/user/ppl/cmd/convert.md b/docs/user/ppl/cmd/convert.md index b3fbb7d3577..1c22f7598cd 100644 --- a/docs/user/ppl/cmd/convert.md +++ b/docs/user/ppl/cmd/convert.md @@ -7,7 +7,7 @@ The `convert` command uses conversion functions to transform field values into n The `convert` command has the following syntax: ```syntax -convert () [AS ] [, () [AS ]]... +convert [timeformat=] () [AS ] [, () [AS ]]... ``` ## Parameters @@ -16,20 +16,25 @@ The `convert` command supports the following parameters. | Parameter | Required/Optional | Description | | --- | --- | --- | -| `` | Required | One of the conversion functions: `auto()`, `num()`, `rmcomma()`, `rmunit()`, `memk()`, or `none()`. | +| `` | Required | One of the conversion functions: `auto()`, `ctime()`, `dur2sec()`, `memk()`, `mktime()`, `mstime()`, `none()`, `num()`, `rmcomma()`, or `rmunit()`. | | `` | Required | Single field name to convert. | | `AS ` | Optional | Create new field with converted value, preserving original field. | +| `timeformat=` | Optional | A strftime format string used by `ctime()` and `mktime()`. Default: `%m/%d/%Y %H:%M:%S`. | ## Conversion Functions | Function | Description | | --- | --- | | `auto(field)` | Automatically converts fields to numbers using intelligent conversion. Handles memory sizes (k/m/g), commas, units, and scientific notation. Returns `null` for non-convertible values. | +| `ctime(field)` | Converts a UNIX epoch timestamp to a human-readable time string. Uses the `timeformat` parameter if specified, otherwise defaults to `%m/%d/%Y %H:%M:%S`. | +| `dur2sec(field)` | Converts a duration string in `HH:MM:SS` format to total seconds. Hours must be less than 24. Returns `null` for invalid formats. | +| `memk(field)` | Converts memory size strings to kilobytes. Accepts numbers with optional k/m/g suffix (case-insensitive). Default unit is kilobytes. Returns `null` for invalid formats. | +| `mktime(field)` | Converts a human-readable time string to a UNIX epoch timestamp. Uses the `timeformat` parameter if specified, otherwise defaults to `%m/%d/%Y %H:%M:%S`. | +| `mstime(field)` | Converts a time string in `[MM:]SS.SSS` format to total seconds. The minutes portion is optional. Returns `null` for invalid formats. | +| `none(field)` | No-op function that preserves the original field value. | | `num(field)` | Extracts leading numbers from strings. For strings without letters: removes commas as thousands separators. For strings with letters: extracts leading number, stops at letters or commas. Returns `null` for non-convertible values. | | `rmcomma(field)` | Removes commas from field values and converts to a number. Returns `null` if the value contains letters. | | `rmunit(field)` | Extracts leading numeric values from strings. Stops at the first non-numeric character (including commas). Returns `null` for non-convertible values. | -| `memk(field)` | Converts memory size strings to kilobytes. Accepts numbers with optional k/m/g suffix (case-insensitive). Default unit is kilobytes. Returns `null` for invalid formats. | -| `none(field)` | No-op function that preserves the original field value. Used for excluding specific fields from wildcard conversions. | ## Example 1: Basic auto() conversion @@ -241,6 +246,98 @@ fetched rows / total rows = 3/3 **Note:** The `none()` function is particularly useful when wildcard support is implemented, allowing you to exclude specific fields from bulk conversions. +## Example 9: Convert epoch timestamp to time string with ctime() + +```ppl +source=accounts +| eval timestamp = 1066507633 +| convert ctime(timestamp) +| fields timestamp +``` + +```text +fetched rows / total rows = 1/1 ++---------------------+ +| timestamp | +|---------------------| +| 10/18/2003 20:07:13 | ++---------------------+ +``` + +## Example 10: Convert time string to epoch with mktime() + +```ppl +source=accounts +| eval date_str = '10/18/2003 20:07:13' +| convert mktime(date_str) +| fields date_str +``` + +```text +fetched rows / total rows = 1/1 ++--------------+ +| date_str | +|--------------| +| 1.066507633E9| ++--------------+ +``` + +## Example 11: Using timeformat with ctime() and mktime() + +The `timeformat` parameter specifies a strftime format string for `ctime()` and `mktime()`: + +```ppl +source=accounts +| eval timestamp = 1066507633 +| convert timeformat="%Y-%m-%d %H:%M:%S" ctime(timestamp) +| fields timestamp +``` + +```text +fetched rows / total rows = 1/1 ++---------------------+ +| timestamp | +|---------------------| +| 2003-10-18 20:07:13 | ++---------------------+ +``` + +## Example 12: Convert duration to seconds with dur2sec() + +```ppl +source=accounts +| eval duration = '01:23:45' +| convert dur2sec(duration) +| fields duration +``` + +```text +fetched rows / total rows = 1/1 ++----------+ +| duration | +|----------| +| 5025.0 | ++----------+ +``` + +## Example 13: Convert minutes and seconds with mstime() + +```ppl +source=accounts +| eval time_str = '03:45.5' +| convert mstime(time_str) +| fields time_str +``` + +```text +fetched rows / total rows = 1/1 ++----------+ +| time_str | +|----------| +| 225.5 | ++----------+ +``` + ## Notes - All conversion functions return `null` for values that cannot be converted to a number From 93f776841942a2c17b9000f9548e9250d8405d5c Mon Sep 17 00:00:00 2001 From: Ritvi Bhatt Date: Fri, 6 Mar 2026 14:45:14 -0800 Subject: [PATCH 08/22] sql cli test fix Signed-off-by: Ritvi Bhatt --- .github/workflows/sql-cli-integration-test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/sql-cli-integration-test.yml b/.github/workflows/sql-cli-integration-test.yml index 63f3e91d334..d04b1d1a34d 100644 --- a/.github/workflows/sql-cli-integration-test.yml +++ b/.github/workflows/sql-cli-integration-test.yml @@ -68,6 +68,7 @@ jobs: run: | echo "Building SQL modules from current branch..." ./gradlew publishToMavenLocal -x test -x integTest + ./gradlew clean echo "SQL modules published to Maven Local" - name: Run SQL CLI tests with local SQL modules From 80b16a9b6bba07609694b67b8fbc57ab0e969605 Mon Sep 17 00:00:00 2001 From: Ritvi Bhatt Date: Fri, 6 Mar 2026 15:26:36 -0800 Subject: [PATCH 09/22] update utils Signed-off-by: Ritvi Bhatt --- .../datetime/StrftimeFormatterUtil.java | 41 +++---------------- 1 file changed, 6 insertions(+), 35 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/expression/datetime/StrftimeFormatterUtil.java b/core/src/main/java/org/opensearch/sql/expression/datetime/StrftimeFormatterUtil.java index 44b809e8598..5e8135d2184 100644 --- a/core/src/main/java/org/opensearch/sql/expression/datetime/StrftimeFormatterUtil.java +++ b/core/src/main/java/org/opensearch/sql/expression/datetime/StrftimeFormatterUtil.java @@ -254,25 +254,13 @@ private static long extractFirstNDigits(double value, int digits) { private static final Map STRFTIME_TO_JAVA_PARSE = ImmutableMap.builder() .put("%Y", "yyyy") - .put("%y", "yy") .put("%m", "MM") .put("%d", "dd") - .put("%e", "d") .put("%H", "HH") - .put("%I", "hh") .put("%M", "mm") .put("%S", "ss") - .put("%p", "a") - .put("%B", "MMMM") - .put("%b", "MMM") - .put("%A", "EEEE") - .put("%a", "EEE") - .put("%j", "DDD") - .put("%Z", "zzz") - .put("%z", "xx") .put("%T", "HH:mm:ss") .put("%F", "yyyy-MM-dd") - .put("%x", "MM/dd/yyyy") .put("%%", "'%'") .build(); @@ -283,30 +271,13 @@ private static long extractFirstNDigits(double value, int digits) { * @return a Java DateTimeFormatter pattern (e.g. {@code yyyy-MM-dd HH:mm:ss}) */ public static String toJavaPattern(String strftimeFormat) { - StringBuilder result = new StringBuilder(); - int i = 0; - while (i < strftimeFormat.length()) { - if (strftimeFormat.charAt(i) == '%' && i + 1 < strftimeFormat.length()) { - String spec = strftimeFormat.substring(i, i + 2); - String javaPattern = STRFTIME_TO_JAVA_PARSE.get(spec); - if (javaPattern != null) { - result.append(javaPattern); - } else { - // Unknown specifier — pass through as literal - result.append("'").append(spec).append("'"); - } - i += 2; - } else { - char c = strftimeFormat.charAt(i); - // Escape Java pattern letters as literals - if (Character.isLetter(c)) { - result.append("'").append(c).append("'"); - } else { - result.append(c); - } - i++; + String result = strftimeFormat; + for (Map.Entry entry : STRFTIME_TO_JAVA_PARSE.entrySet()) { + String specifier = entry.getKey(); + if (result.contains(specifier)) { + result = result.replace(specifier, entry.getValue()); } } - return result.toString(); + return result; } } From fdc0b6793363609a836f4c1601e6e9da7bc3abf8 Mon Sep 17 00:00:00 2001 From: Ritvi Bhatt Date: Fri, 6 Mar 2026 15:40:22 -0800 Subject: [PATCH 10/22] fix integ tests Signed-off-by: Ritvi Bhatt --- .../test/java/org/opensearch/sql/ppl/ConvertCommandIT.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/ConvertCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/ConvertCommandIT.java index 0a6df8dc740..b1c794130b0 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/ConvertCommandIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/ConvertCommandIT.java @@ -97,8 +97,8 @@ public void testConvertMstimeFunction() { @Test public void testConvertWithTimeformat() { verifyQueryThrowsCalciteError( - "source=%s | eval date_str = '18/10/2003 20:07:13' | convert timeformat=\"dd/MM/yyyy" - + " HH:mm:ss\" mktime(date_str) | fields date_str"); + "source=%s | eval date_str = '18/10/2003 20:07:13' | convert" + + " timeformat=\\\"%%d/%%m/%%Y %%H:%%M:%%S\\\" mktime(date_str) | fields date_str"); } private void verifyQueryThrowsCalciteError(String query) { From da6cfeb56ef0bd438899cd7831f6a1b94370a6ac Mon Sep 17 00:00:00 2001 From: Ritvi Bhatt Date: Fri, 6 Mar 2026 16:43:04 -0800 Subject: [PATCH 11/22] add explain tests Signed-off-by: Ritvi Bhatt --- .../function/udf/CTimeConvertFunction.java | 9 +++-- .../function/udf/MkTimeConvertFunction.java | 7 ++-- .../sql/calcite/remote/CalciteExplainIT.java | 40 +++++++++++++++++++ .../calcite/explain_convert_ctime.yaml | 8 ++++ .../calcite/explain_convert_dur2sec.yaml | 8 ++++ .../calcite/explain_convert_mktime.yaml | 8 ++++ .../calcite/explain_convert_mstime.yaml | 8 ++++ .../explain_convert_ctime.yaml | 9 +++++ .../explain_convert_dur2sec.yaml | 9 +++++ .../explain_convert_mktime.yaml | 9 +++++ .../explain_convert_mstime.yaml | 9 +++++ 11 files changed, 117 insertions(+), 7 deletions(-) create mode 100644 integ-test/src/test/resources/expectedOutput/calcite/explain_convert_ctime.yaml create mode 100644 integ-test/src/test/resources/expectedOutput/calcite/explain_convert_dur2sec.yaml create mode 100644 integ-test/src/test/resources/expectedOutput/calcite/explain_convert_mktime.yaml create mode 100644 integ-test/src/test/resources/expectedOutput/calcite/explain_convert_mstime.yaml create mode 100644 integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_convert_ctime.yaml create mode 100644 integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_convert_dur2sec.yaml create mode 100644 integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_convert_mktime.yaml create mode 100644 integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_convert_mstime.yaml diff --git a/core/src/main/java/org/opensearch/sql/expression/function/udf/CTimeConvertFunction.java b/core/src/main/java/org/opensearch/sql/expression/function/udf/CTimeConvertFunction.java index 5bd9f482e7e..e04900176c8 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/udf/CTimeConvertFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/udf/CTimeConvertFunction.java @@ -70,11 +70,12 @@ public static String convertWithFormat(Object value, Object timeFormatObj) { if (timestamp == null) { return null; } + String format = + (timeFormatObj != null) ? timeFormatObj.toString().trim() : DEFAULT_FORMAT; + if (format.isEmpty()) { + return null; + } try { - String format = - (timeFormatObj != null && !timeFormatObj.toString().trim().isEmpty()) - ? timeFormatObj.toString().trim() - : DEFAULT_FORMAT; Instant instant = Instant.ofEpochSecond(timestamp.longValue()); ZonedDateTime zdt = ZonedDateTime.ofInstant(instant, ZoneId.of("UTC")); return StrftimeFormatterUtil.formatZonedDateTime(zdt, format).stringValue(); diff --git a/core/src/main/java/org/opensearch/sql/expression/function/udf/MkTimeConvertFunction.java b/core/src/main/java/org/opensearch/sql/expression/function/udf/MkTimeConvertFunction.java index 204fafca779..1175b1522ba 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/udf/MkTimeConvertFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/udf/MkTimeConvertFunction.java @@ -91,9 +91,10 @@ public static Object convertWithFormat(Object value, Object timeFormatObj) { } String strftimeFormat = - (timeFormatObj != null && !timeFormatObj.toString().trim().isEmpty()) - ? timeFormatObj.toString().trim() - : DEFAULT_FORMAT; + (timeFormatObj != null) ? timeFormatObj.toString().trim() : DEFAULT_FORMAT; + if (strftimeFormat.isEmpty()) { + return null; + } return parseWithFormat(str, strftimeFormat); } diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java index 73531a8895c..f6730e53d69 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java @@ -2420,6 +2420,46 @@ public void testConvertMultipleFunctionsExplain() throws IOException { + " balance, age")); } + @Test + public void testConvertCtimeExplain() throws IOException { + String expected = loadExpectedPlan("explain_convert_ctime.yaml"); + assertYamlEqualsIgnoreId( + expected, + explainQueryYaml( + "source=opensearch-sql_test_index_bank | eval ts=1066507633 | convert ctime(ts) |" + + " fields ts")); + } + + @Test + public void testConvertMktimeExplain() throws IOException { + String expected = loadExpectedPlan("explain_convert_mktime.yaml"); + assertYamlEqualsIgnoreId( + expected, + explainQueryYaml( + "source=opensearch-sql_test_index_bank | eval d='10/18/2003 20:07:13' | convert" + + " mktime(d) | fields d")); + } + + @Test + public void testConvertDur2secExplain() throws IOException { + String expected = loadExpectedPlan("explain_convert_dur2sec.yaml"); + assertYamlEqualsIgnoreId( + expected, + explainQueryYaml( + "source=opensearch-sql_test_index_bank | eval d='01:23:45' | convert dur2sec(d) |" + + " fields d")); + } + + @Test + public void testConvertMstimeExplain() throws IOException { + String expected = loadExpectedPlan("explain_convert_mstime.yaml"); + assertYamlEqualsIgnoreId( + expected, + explainQueryYaml( + "source=opensearch-sql_test_index_bank | eval t='03:45.5' | convert mstime(t) |" + + " fields t")); + } + @Test public void testNotBetweenPushDownExplain() throws Exception { // test for issue https://github.com/opensearch-project/sql/issues/4903 diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_convert_ctime.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_convert_ctime.yaml new file mode 100644 index 00000000000..dd3c53dc0da --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_convert_ctime.yaml @@ -0,0 +1,8 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(ts=[CTIME(1066507633)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + physical: | + EnumerableCalc(expr#0..18=[{inputs}], expr#19=[1066507633], expr#20=[CTIME($t19)], ts=[$t20]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m"}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_convert_dur2sec.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_convert_dur2sec.yaml new file mode 100644 index 00000000000..fdbe6f1e8b7 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_convert_dur2sec.yaml @@ -0,0 +1,8 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(d=[DUR2SEC('01:23:45':VARCHAR)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + physical: | + EnumerableCalc(expr#0..18=[{inputs}], expr#19=['01:23:45':VARCHAR], expr#20=[DUR2SEC($t19)], d=[$t20]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m"}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_convert_mktime.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_convert_mktime.yaml new file mode 100644 index 00000000000..a817226a708 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_convert_mktime.yaml @@ -0,0 +1,8 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(d=[MKTIME('10/18/2003 20:07:13':VARCHAR)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + physical: | + EnumerableCalc(expr#0..18=[{inputs}], expr#19=['10/18/2003 20:07:13':VARCHAR], expr#20=[MKTIME($t19)], d=[$t20]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m"}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_convert_mstime.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_convert_mstime.yaml new file mode 100644 index 00000000000..43cc390ac77 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_convert_mstime.yaml @@ -0,0 +1,8 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(t=[MSTIME('03:45.5':VARCHAR)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + physical: | + EnumerableCalc(expr#0..18=[{inputs}], expr#19=['03:45.5':VARCHAR], expr#20=[MSTIME($t19)], t=[$t20]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m"}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_convert_ctime.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_convert_ctime.yaml new file mode 100644 index 00000000000..e93685e279d --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_convert_ctime.yaml @@ -0,0 +1,9 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(ts=[CTIME(1066507633)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableCalc(expr#0..18=[{inputs}], expr#19=[1066507633], expr#20=[CTIME($t19)], ts=[$t20]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_convert_dur2sec.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_convert_dur2sec.yaml new file mode 100644 index 00000000000..3f01b02a624 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_convert_dur2sec.yaml @@ -0,0 +1,9 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(d=[DUR2SEC('01:23:45':VARCHAR)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableCalc(expr#0..18=[{inputs}], expr#19=['01:23:45':VARCHAR], expr#20=[DUR2SEC($t19)], d=[$t20]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_convert_mktime.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_convert_mktime.yaml new file mode 100644 index 00000000000..2367ea48feb --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_convert_mktime.yaml @@ -0,0 +1,9 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(d=[MKTIME('10/18/2003 20:07:13':VARCHAR)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableCalc(expr#0..18=[{inputs}], expr#19=['10/18/2003 20:07:13':VARCHAR], expr#20=[MKTIME($t19)], d=[$t20]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_convert_mstime.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_convert_mstime.yaml new file mode 100644 index 00000000000..9bd873d1b3e --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_convert_mstime.yaml @@ -0,0 +1,9 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(t=[MSTIME('03:45.5':VARCHAR)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableCalc(expr#0..18=[{inputs}], expr#19=['03:45.5':VARCHAR], expr#20=[MSTIME($t19)], t=[$t20]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) From 5ac49be4aee95b29163c2dbcf429688edde9cca4 Mon Sep 17 00:00:00 2001 From: Ritvi Bhatt Date: Fri, 6 Mar 2026 16:54:23 -0800 Subject: [PATCH 12/22] fix test Signed-off-by: Ritvi Bhatt --- .github/workflows/sql-cli-integration-test.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/sql-cli-integration-test.yml b/.github/workflows/sql-cli-integration-test.yml index d04b1d1a34d..63f3e91d334 100644 --- a/.github/workflows/sql-cli-integration-test.yml +++ b/.github/workflows/sql-cli-integration-test.yml @@ -68,7 +68,6 @@ jobs: run: | echo "Building SQL modules from current branch..." ./gradlew publishToMavenLocal -x test -x integTest - ./gradlew clean echo "SQL modules published to Maven Local" - name: Run SQL CLI tests with local SQL modules From 215b59bc4c365acc43412225be8cadd9c093709a Mon Sep 17 00:00:00 2001 From: Ritvi Bhatt Date: Fri, 6 Mar 2026 17:19:38 -0800 Subject: [PATCH 13/22] apply spotless Signed-off-by: Ritvi Bhatt --- .../sql/calcite/CalciteRelNodeVisitor.java | 52 +++++++++++++++++++ .../function/udf/CTimeConvertFunction.java | 3 +- ppl/src/main/antlr/OpenSearchPPLParser.g4 | 2 +- .../opensearch/sql/ppl/parser/AstBuilder.java | 12 +++-- 4 files changed, 61 insertions(+), 8 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java b/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java index c7d3831a947..a3720c24a72 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java +++ b/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java @@ -1005,9 +1005,61 @@ public RelNode visitConvert(Convert node, CalcitePlanContext context) { return context.relBuilder.peek(); } + // Collect none() patterns for wildcard exclusion + Set nonePatterns = new HashSet<>(); + for (Let conversion : node.getConversions()) { + UnresolvedExpression expr = conversion.getExpression(); + if (expr instanceof Function func && "none".equalsIgnoreCase(func.getFuncName())) { + nonePatterns.add(((Field) func.getFuncArgs().get(0)).getField().toString()); + } + } + + List currentFields = context.relBuilder.peek().getRowType().getFieldNames(); ConversionState state = new ConversionState(); for (Let conversion : node.getConversions()) { + UnresolvedExpression expr = conversion.getExpression(); + + // Skip none() — already collected above + if (expr instanceof Function func && "none".equalsIgnoreCase(func.getFuncName())) { + // none() with AS is a field copy + if (!conversion + .getVar() + .getField() + .toString() + .equals(((Field) func.getFuncArgs().get(0)).getField().toString())) { + processFieldCopyConversion( + conversion.getVar().getField().toString(), + (Field) func.getFuncArgs().get(0), + state, + context); + } + continue; + } + + if (expr instanceof Function func) { + String source = ((Field) func.getFuncArgs().get(0)).getField().toString(); + if (WildcardUtils.containsWildcard(source)) { + List matchingFields = + WildcardUtils.expandWildcardPattern(source, currentFields).stream() + .filter(f -> !state.seenFields.contains(f)) + .filter( + f -> + nonePatterns.stream() + .noneMatch(p -> WildcardUtils.matchesWildcardPattern(p, f))) + .toList(); + for (String field : matchingFields) { + processFunctionConversion( + field, + new Function(func.getFuncName(), List.of(AstDSL.field(field))), + node.getTimeFormat(), + state, + context); + } + continue; + } + } + processConversion(conversion, node.getTimeFormat(), state, context); } diff --git a/core/src/main/java/org/opensearch/sql/expression/function/udf/CTimeConvertFunction.java b/core/src/main/java/org/opensearch/sql/expression/function/udf/CTimeConvertFunction.java index e04900176c8..5eb78dc0c21 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/udf/CTimeConvertFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/udf/CTimeConvertFunction.java @@ -70,8 +70,7 @@ public static String convertWithFormat(Object value, Object timeFormatObj) { if (timestamp == null) { return null; } - String format = - (timeFormatObj != null) ? timeFormatObj.toString().trim() : DEFAULT_FORMAT; + String format = (timeFormatObj != null) ? timeFormatObj.toString().trim() : DEFAULT_FORMAT; if (format.isEmpty()) { return null; } diff --git a/ppl/src/main/antlr/OpenSearchPPLParser.g4 b/ppl/src/main/antlr/OpenSearchPPLParser.g4 index ddd945572e0..24122a8b209 100644 --- a/ppl/src/main/antlr/OpenSearchPPLParser.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLParser.g4 @@ -547,7 +547,7 @@ convertCommand ; convertFunction - : functionName = ident LT_PRTHS fieldExpression RT_PRTHS (AS alias = fieldExpression)? + : functionName = ident LT_PRTHS wcFieldExpression RT_PRTHS (AS alias = fieldExpression)? ; trendlineCommand diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java index c87cc239399..18032c78f6b 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java @@ -1228,7 +1228,7 @@ public UnresolvedPlan visitConvertCommand(OpenSearchPPLParser.ConvertCommandCont "mstime"); private Let buildConversion(OpenSearchPPLParser.ConvertFunctionContext funcCtx) { - if (funcCtx.fieldExpression().isEmpty()) { + if (funcCtx.wcFieldExpression() == null) { throw new IllegalArgumentException("Convert function requires a field argument"); } @@ -1242,13 +1242,15 @@ private Let buildConversion(OpenSearchPPLParser.ConvertFunctionContext funcCtx) functionName, SUPPORTED_CONVERSION_FUNCTIONS)); } - UnresolvedExpression fieldArg = internalVisitExpression(funcCtx.fieldExpression(0)); + UnresolvedExpression fieldArg = internalVisitExpression(funcCtx.wcFieldExpression()); Field targetField = determineTargetField(funcCtx, fieldArg); if ("none".equalsIgnoreCase(functionName)) { - return fieldArg.toString().equals(targetField.getField().toString()) - ? null - : new Let(targetField, fieldArg); + if (funcCtx.alias != null) { + return new Let(targetField, fieldArg); + } + // Keep none() as a function so the visitor can use it for wildcard exclusion + return new Let(targetField, AstDSL.function(functionName, fieldArg)); } return new Let(targetField, AstDSL.function(functionName, fieldArg)); From 0ef77459f79c8b912dbebf1e2023bc7aadee00c1 Mon Sep 17 00:00:00 2001 From: Ritvi Bhatt Date: Mon, 9 Mar 2026 10:18:19 -0700 Subject: [PATCH 14/22] fix null timeformat Signed-off-by: Ritvi Bhatt --- .../function/udf/ConversionFunctionsTest.java | 12 ++---------- .../sql/ppl/utils/PPLQueryDataAnonymizerTest.java | 2 +- 2 files changed, 3 insertions(+), 11 deletions(-) diff --git a/core/src/test/java/org/opensearch/sql/expression/function/udf/ConversionFunctionsTest.java b/core/src/test/java/org/opensearch/sql/expression/function/udf/ConversionFunctionsTest.java index 928fc61ca67..490f72ba346 100644 --- a/core/src/test/java/org/opensearch/sql/expression/function/udf/ConversionFunctionsTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/function/udf/ConversionFunctionsTest.java @@ -450,10 +450,7 @@ public void testMktimeWithCustomTimeformat() { // Invalid format returns null assertNull(MkTimeConvertFunction.convertWithFormat("2003-10-18 20:07:13", "invalid format")); - // Null/empty timeformat falls back to default %m/%d/%Y %H:%M:%S - assertEquals( - 1066507633.0, MkTimeConvertFunction.convertWithFormat("10/18/2003 20:07:13", null)); - assertEquals(1066507633.0, MkTimeConvertFunction.convertWithFormat("10/18/2003 20:07:13", "")); + assertNull(MkTimeConvertFunction.convertWithFormat("10/18/2003 20:07:13", "")); } @Test @@ -465,11 +462,6 @@ public void testCtimeWithCustomTimeformat() { assertEquals("18/10/2003", CTimeConvertFunction.convertWithFormat(1066507633, "%d/%m/%Y")); assertEquals("1970", CTimeConvertFunction.convertWithFormat(0, "%Y")); - // Null/empty timeformat falls back to default - String result = CTimeConvertFunction.convertWithFormat(1066507633, null); - assertEquals("10/18/2003 20:07:13", result); - - result = CTimeConvertFunction.convertWithFormat(1066507633, ""); - assertEquals("10/18/2003 20:07:13", result); + assertNull(CTimeConvertFunction.convertWithFormat(1066507633, "")); } } diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizerTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizerTest.java index 866c45a26e0..14bed2b3b6c 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizerTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizerTest.java @@ -1122,7 +1122,7 @@ public void testConvertCommand() { "source=table | convert auto(identifier),num(identifier)", anonymize("source=t | convert auto(salary), num(commission)")); assertEquals( - "source=table | convert rmcomma(identifier),rmunit(identifier),(identifier) AS identifier", + "source=table | convert rmcomma(identifier),rmunit(identifier),none(identifier)", anonymize("source=t | convert rmcomma(name), rmunit(revenue), none(id)")); assertEquals( "source=table | convert (identifier) AS identifier", From bb8ebf1150311de3a920268f6d74a64e6d6c0475 Mon Sep 17 00:00:00 2001 From: Ritvi Bhatt Date: Mon, 9 Mar 2026 10:52:30 -0700 Subject: [PATCH 15/22] fix tests Signed-off-by: Ritvi Bhatt --- .../sql/calcite/CalciteRelNodeVisitor.java | 52 ------------------- ppl/src/main/antlr/OpenSearchPPLParser.g4 | 2 +- .../opensearch/sql/ppl/parser/AstBuilder.java | 12 ++--- .../ppl/utils/PPLQueryDataAnonymizerTest.java | 2 +- 4 files changed, 7 insertions(+), 61 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java b/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java index a3720c24a72..c7d3831a947 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java +++ b/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java @@ -1005,61 +1005,9 @@ public RelNode visitConvert(Convert node, CalcitePlanContext context) { return context.relBuilder.peek(); } - // Collect none() patterns for wildcard exclusion - Set nonePatterns = new HashSet<>(); - for (Let conversion : node.getConversions()) { - UnresolvedExpression expr = conversion.getExpression(); - if (expr instanceof Function func && "none".equalsIgnoreCase(func.getFuncName())) { - nonePatterns.add(((Field) func.getFuncArgs().get(0)).getField().toString()); - } - } - - List currentFields = context.relBuilder.peek().getRowType().getFieldNames(); ConversionState state = new ConversionState(); for (Let conversion : node.getConversions()) { - UnresolvedExpression expr = conversion.getExpression(); - - // Skip none() — already collected above - if (expr instanceof Function func && "none".equalsIgnoreCase(func.getFuncName())) { - // none() with AS is a field copy - if (!conversion - .getVar() - .getField() - .toString() - .equals(((Field) func.getFuncArgs().get(0)).getField().toString())) { - processFieldCopyConversion( - conversion.getVar().getField().toString(), - (Field) func.getFuncArgs().get(0), - state, - context); - } - continue; - } - - if (expr instanceof Function func) { - String source = ((Field) func.getFuncArgs().get(0)).getField().toString(); - if (WildcardUtils.containsWildcard(source)) { - List matchingFields = - WildcardUtils.expandWildcardPattern(source, currentFields).stream() - .filter(f -> !state.seenFields.contains(f)) - .filter( - f -> - nonePatterns.stream() - .noneMatch(p -> WildcardUtils.matchesWildcardPattern(p, f))) - .toList(); - for (String field : matchingFields) { - processFunctionConversion( - field, - new Function(func.getFuncName(), List.of(AstDSL.field(field))), - node.getTimeFormat(), - state, - context); - } - continue; - } - } - processConversion(conversion, node.getTimeFormat(), state, context); } diff --git a/ppl/src/main/antlr/OpenSearchPPLParser.g4 b/ppl/src/main/antlr/OpenSearchPPLParser.g4 index 24122a8b209..ddd945572e0 100644 --- a/ppl/src/main/antlr/OpenSearchPPLParser.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLParser.g4 @@ -547,7 +547,7 @@ convertCommand ; convertFunction - : functionName = ident LT_PRTHS wcFieldExpression RT_PRTHS (AS alias = fieldExpression)? + : functionName = ident LT_PRTHS fieldExpression RT_PRTHS (AS alias = fieldExpression)? ; trendlineCommand diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java index 18032c78f6b..c87cc239399 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java @@ -1228,7 +1228,7 @@ public UnresolvedPlan visitConvertCommand(OpenSearchPPLParser.ConvertCommandCont "mstime"); private Let buildConversion(OpenSearchPPLParser.ConvertFunctionContext funcCtx) { - if (funcCtx.wcFieldExpression() == null) { + if (funcCtx.fieldExpression().isEmpty()) { throw new IllegalArgumentException("Convert function requires a field argument"); } @@ -1242,15 +1242,13 @@ private Let buildConversion(OpenSearchPPLParser.ConvertFunctionContext funcCtx) functionName, SUPPORTED_CONVERSION_FUNCTIONS)); } - UnresolvedExpression fieldArg = internalVisitExpression(funcCtx.wcFieldExpression()); + UnresolvedExpression fieldArg = internalVisitExpression(funcCtx.fieldExpression(0)); Field targetField = determineTargetField(funcCtx, fieldArg); if ("none".equalsIgnoreCase(functionName)) { - if (funcCtx.alias != null) { - return new Let(targetField, fieldArg); - } - // Keep none() as a function so the visitor can use it for wildcard exclusion - return new Let(targetField, AstDSL.function(functionName, fieldArg)); + return fieldArg.toString().equals(targetField.getField().toString()) + ? null + : new Let(targetField, fieldArg); } return new Let(targetField, AstDSL.function(functionName, fieldArg)); diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizerTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizerTest.java index 14bed2b3b6c..866c45a26e0 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizerTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizerTest.java @@ -1122,7 +1122,7 @@ public void testConvertCommand() { "source=table | convert auto(identifier),num(identifier)", anonymize("source=t | convert auto(salary), num(commission)")); assertEquals( - "source=table | convert rmcomma(identifier),rmunit(identifier),none(identifier)", + "source=table | convert rmcomma(identifier),rmunit(identifier),(identifier) AS identifier", anonymize("source=t | convert rmcomma(name), rmunit(revenue), none(id)")); assertEquals( "source=table | convert (identifier) AS identifier", From 2b10e9ac2eb06330fbce9fa1adcd1c449beac3f5 Mon Sep 17 00:00:00 2001 From: Ritvi Bhatt Date: Mon, 9 Mar 2026 16:15:54 -0700 Subject: [PATCH 16/22] empty Signed-off-by: Ritvi Bhatt From 6f6162eac8e0fa6773e50043c79117fcf624a66c Mon Sep 17 00:00:00 2001 From: Ritvi Bhatt Date: Mon, 9 Mar 2026 16:26:50 -0700 Subject: [PATCH 17/22] update docs Signed-off-by: Ritvi Bhatt --- .../sql/expression/datetime/StrftimeFormatterUtil.java | 5 +++++ docs/user/ppl/cmd/convert.md | 4 ++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/expression/datetime/StrftimeFormatterUtil.java b/core/src/main/java/org/opensearch/sql/expression/datetime/StrftimeFormatterUtil.java index 5e8135d2184..7b7830f3412 100644 --- a/core/src/main/java/org/opensearch/sql/expression/datetime/StrftimeFormatterUtil.java +++ b/core/src/main/java/org/opensearch/sql/expression/datetime/StrftimeFormatterUtil.java @@ -254,11 +254,16 @@ private static long extractFirstNDigits(double value, int digits) { private static final Map STRFTIME_TO_JAVA_PARSE = ImmutableMap.builder() .put("%Y", "yyyy") + .put("%y", "yy") .put("%m", "MM") + .put("%B", "MMMM") + .put("%b", "MMM") .put("%d", "dd") .put("%H", "HH") + .put("%I", "hh") .put("%M", "mm") .put("%S", "ss") + .put("%p", "a") .put("%T", "HH:mm:ss") .put("%F", "yyyy-MM-dd") .put("%%", "'%'") diff --git a/docs/user/ppl/cmd/convert.md b/docs/user/ppl/cmd/convert.md index 1c22f7598cd..d4d6546174c 100644 --- a/docs/user/ppl/cmd/convert.md +++ b/docs/user/ppl/cmd/convert.md @@ -26,10 +26,10 @@ The `convert` command supports the following parameters. | Function | Description | | --- | --- | | `auto(field)` | Automatically converts fields to numbers using intelligent conversion. Handles memory sizes (k/m/g), commas, units, and scientific notation. Returns `null` for non-convertible values. | -| `ctime(field)` | Converts a UNIX epoch timestamp to a human-readable time string. Uses the `timeformat` parameter if specified, otherwise defaults to `%m/%d/%Y %H:%M:%S`. | +| `ctime(field)` | Converts a UNIX epoch timestamp to a human-readable time string. Uses the `timeformat` parameter if specified, otherwise defaults to `%m/%d/%Y %H:%M:%S`. All timestamps are interpreted in UTC timezone. | | `dur2sec(field)` | Converts a duration string in `HH:MM:SS` format to total seconds. Hours must be less than 24. Returns `null` for invalid formats. | | `memk(field)` | Converts memory size strings to kilobytes. Accepts numbers with optional k/m/g suffix (case-insensitive). Default unit is kilobytes. Returns `null` for invalid formats. | -| `mktime(field)` | Converts a human-readable time string to a UNIX epoch timestamp. Uses the `timeformat` parameter if specified, otherwise defaults to `%m/%d/%Y %H:%M:%S`. | +| `mktime(field)` | Converts a human-readable time string to a UNIX epoch timestamp. Uses the `timeformat` parameter if specified, otherwise defaults to `%m/%d/%Y %H:%M:%S`. Input strings are interpreted as UTC timezone. | | `mstime(field)` | Converts a time string in `[MM:]SS.SSS` format to total seconds. The minutes portion is optional. Returns `null` for invalid formats. | | `none(field)` | No-op function that preserves the original field value. | | `num(field)` | Extracts leading numbers from strings. For strings without letters: removes commas as thousands separators. For strings with letters: extracts leading number, stops at letters or commas. Returns `null` for non-convertible values. | From afa391593ca695c23cc0d976e2a3402886c8e5d8 Mon Sep 17 00:00:00 2001 From: Ritvi Bhatt Date: Mon, 9 Mar 2026 17:08:39 -0700 Subject: [PATCH 18/22] empty Signed-off-by: Ritvi Bhatt From 4364539f3344c7ff39c65527b612124f06e6b0b8 Mon Sep 17 00:00:00 2001 From: Ritvi Bhatt Date: Tue, 10 Mar 2026 11:27:42 -0700 Subject: [PATCH 19/22] fix parsing Signed-off-by: Ritvi Bhatt --- .../expression/datetime/StrftimeFormatterUtil.java | 14 +++++++------- .../function/udf/CTimeConvertFunction.java | 2 +- .../function/udf/MkTimeConvertFunction.java | 13 ++++--------- 3 files changed, 12 insertions(+), 17 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/expression/datetime/StrftimeFormatterUtil.java b/core/src/main/java/org/opensearch/sql/expression/datetime/StrftimeFormatterUtil.java index 7b7830f3412..bd0796b05af 100644 --- a/core/src/main/java/org/opensearch/sql/expression/datetime/StrftimeFormatterUtil.java +++ b/core/src/main/java/org/opensearch/sql/expression/datetime/StrftimeFormatterUtil.java @@ -276,13 +276,13 @@ private static long extractFirstNDigits(double value, int digits) { * @return a Java DateTimeFormatter pattern (e.g. {@code yyyy-MM-dd HH:mm:ss}) */ public static String toJavaPattern(String strftimeFormat) { - String result = strftimeFormat; - for (Map.Entry entry : STRFTIME_TO_JAVA_PARSE.entrySet()) { - String specifier = entry.getKey(); - if (result.contains(specifier)) { - result = result.replace(specifier, entry.getValue()); - } + Matcher m = Pattern.compile("%[A-Za-z%]").matcher(strftimeFormat); + StringBuilder sb = new StringBuilder(); + while (m.find()) { + String replacement = STRFTIME_TO_JAVA_PARSE.getOrDefault(m.group(), m.group()); + m.appendReplacement(sb, Matcher.quoteReplacement(replacement)); } - return result; + m.appendTail(sb); + return sb.toString(); } } diff --git a/core/src/main/java/org/opensearch/sql/expression/function/udf/CTimeConvertFunction.java b/core/src/main/java/org/opensearch/sql/expression/function/udf/CTimeConvertFunction.java index 5eb78dc0c21..f83c8053ab1 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/udf/CTimeConvertFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/udf/CTimeConvertFunction.java @@ -83,7 +83,7 @@ public static String convertWithFormat(Object value, Object timeFormatObj) { } } - static Double toEpochSeconds(Object value) { + public static Double toEpochSeconds(Object value) { if (value == null) { return null; } diff --git a/core/src/main/java/org/opensearch/sql/expression/function/udf/MkTimeConvertFunction.java b/core/src/main/java/org/opensearch/sql/expression/function/udf/MkTimeConvertFunction.java index 1175b1522ba..0127d63e9cd 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/udf/MkTimeConvertFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/udf/MkTimeConvertFunction.java @@ -73,22 +73,17 @@ public static Object convert(Object value) { } public static Object convertWithFormat(Object value, Object timeFormatObj) { + Double numeric = CTimeConvertFunction.toEpochSeconds(value); + if (numeric != null) { + return numeric; + } if (value == null) { return null; } - if (value instanceof Number) { - return ((Number) value).doubleValue(); - } String str = value instanceof String ? ((String) value).trim() : value.toString().trim(); if (str.isEmpty()) { return null; } - // If already numeric, return as-is - try { - return Double.parseDouble(str); - } catch (NumberFormatException ignored) { - // Not a number, proceed with date parsing - } String strftimeFormat = (timeFormatObj != null) ? timeFormatObj.toString().trim() : DEFAULT_FORMAT; From 8d3b734c7b3b17cbb5fcd62d203e38813ba3006a Mon Sep 17 00:00:00 2001 From: Ritvi Bhatt Date: Wed, 11 Mar 2026 09:25:41 -0700 Subject: [PATCH 20/22] update docs table format Signed-off-by: Ritvi Bhatt --- docs/user/ppl/cmd/convert.md | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/docs/user/ppl/cmd/convert.md b/docs/user/ppl/cmd/convert.md index d4d6546174c..87f86b4813d 100644 --- a/docs/user/ppl/cmd/convert.md +++ b/docs/user/ppl/cmd/convert.md @@ -14,12 +14,12 @@ convert [timeformat=] () [AS ] [, ` | Required | One of the conversion functions: `auto()`, `ctime()`, `dur2sec()`, `memk()`, `mktime()`, `mstime()`, `none()`, `num()`, `rmcomma()`, or `rmunit()`. | -| `` | Required | Single field name to convert. | -| `AS ` | Optional | Create new field with converted value, preserving original field. | -| `timeformat=` | Optional | A strftime format string used by `ctime()` and `mktime()`. Default: `%m/%d/%Y %H:%M:%S`. | +| Parameter | Required/Optional | Description | Default | +| --- | --- | --- | --- | +| `` | Required | One of the conversion functions: `auto()`, `ctime()`, `dur2sec()`, `memk()`, `mktime()`, `mstime()`, `none()`, `num()`, `rmcomma()`, or `rmunit()`. | N/A | +| `` | Required | Single field name to convert. | N/A | +| `AS ` | Optional | Create new field with converted value, preserving original field. | N/A | +| `timeformat=` | Optional | A strftime format string used by `ctime()` and `mktime()`. | `%m/%d/%Y %H:%M:%S`. | ## Conversion Functions @@ -255,6 +255,8 @@ source=accounts | fields timestamp ``` +The query returns the following results: + ```text fetched rows / total rows = 1/1 +---------------------+ @@ -273,6 +275,8 @@ source=accounts | fields date_str ``` +The query returns the following results: + ```text fetched rows / total rows = 1/1 +--------------+ @@ -293,6 +297,8 @@ source=accounts | fields timestamp ``` +The query returns the following results: + ```text fetched rows / total rows = 1/1 +---------------------+ @@ -311,6 +317,8 @@ source=accounts | fields duration ``` +The query returns the following results: + ```text fetched rows / total rows = 1/1 +----------+ @@ -329,6 +337,8 @@ source=accounts | fields time_str ``` +The query returns the following results: + ```text fetched rows / total rows = 1/1 +----------+ From 39e95ecc175eb31de5cf5630b7d8b33c666e625e Mon Sep 17 00:00:00 2001 From: Ritvi Bhatt Date: Thu, 12 Mar 2026 09:14:58 -0700 Subject: [PATCH 21/22] update doc example Signed-off-by: Ritvi Bhatt --- docs/user/ppl/cmd/convert.md | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/docs/user/ppl/cmd/convert.md b/docs/user/ppl/cmd/convert.md index 87f86b4813d..457ec2563e0 100644 --- a/docs/user/ppl/cmd/convert.md +++ b/docs/user/ppl/cmd/convert.md @@ -308,6 +308,26 @@ fetched rows / total rows = 1/1 +---------------------+ ``` +Similarly, you can use `timeformat` with `mktime()` to parse dates in custom formats: + +```ppl +source=accounts +| eval date_str = '2000-01-01 00:00:00' +| convert timeformat="%Y-%m-%d %H:%M:%S" mktime(date_str) +| fields date_str +``` + +The query returns the following results: + +```text +fetched rows / total rows = 1/1 ++------------+ +| date_str | +|------------| +| 9.466848E8 | ++------------+ +``` + ## Example 12: Convert duration to seconds with dur2sec() ```ppl From 7394650bc962ad21981c0725a5b6009e775de03e Mon Sep 17 00:00:00 2001 From: Ritvi Bhatt Date: Fri, 13 Mar 2026 15:18:16 -0700 Subject: [PATCH 22/22] fix ctime fractional seconds Signed-off-by: Ritvi Bhatt --- .../sql/expression/function/udf/CTimeConvertFunction.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/org/opensearch/sql/expression/function/udf/CTimeConvertFunction.java b/core/src/main/java/org/opensearch/sql/expression/function/udf/CTimeConvertFunction.java index f83c8053ab1..6b507936348 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/udf/CTimeConvertFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/udf/CTimeConvertFunction.java @@ -75,7 +75,9 @@ public static String convertWithFormat(Object value, Object timeFormatObj) { return null; } try { - Instant instant = Instant.ofEpochSecond(timestamp.longValue()); + long seconds = timestamp.longValue(); + int nanos = (int) ((timestamp - seconds) * 1_000_000_000); + Instant instant = Instant.ofEpochSecond(seconds, nanos); ZonedDateTime zdt = ZonedDateTime.ofInstant(instant, ZoneId.of("UTC")); return StrftimeFormatterUtil.formatZonedDateTime(zdt, format).stringValue(); } catch (Exception e) {