From ea04023a4096c5c16903d4e72aa9aa0b8c9a0467 Mon Sep 17 00:00:00 2001 From: Keshav Dandeva Date: Mon, 2 Nov 2026 14:21:32 +0000 Subject: [PATCH] fix(bigquery): apply DataFormatOptions to tabledata.list requests --- .../google/cloud/bigquery/BigQueryImpl.java | 20 +++++-------- .../bigquery/spi/v2/HttpBigQueryRpc.java | 9 ++++++ .../cloud/bigquery/BigQueryImplTest.java | 28 +++++++++++++++++ .../cloud/bigquery/it/ITBigQueryTest.java | 29 ++++++++++++++---- .../bigquery/spi/v2/HttpBigQueryRpcTest.java | 30 +++++++++++++++++++ 5 files changed, 99 insertions(+), 17 deletions(-) diff --git a/java-bigquery/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryImpl.java b/java-bigquery/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryImpl.java index 9248818c9972..8bfb998d4685 100644 --- a/java-bigquery/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryImpl.java +++ b/java-bigquery/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryImpl.java @@ -2213,10 +2213,7 @@ public TableDataList call() throws IOException { new PageImpl<>( new TableDataPageFetcher(tableId, schema, serviceOptions, cursor, pageOptionMap), cursor, - transformTableData( - result.getRows(), - schema, - serviceOptions.getDataFormatOptions().useInt64Timestamp())), + transformTableData(result.getRows(), schema, serviceOptions.getDataFormatOptions())), result.getTotalRows()); } catch (BigQueryRetryHelperException e) { throw BigQueryException.translateAndThrow(e); @@ -2224,7 +2221,11 @@ public TableDataList call() throws IOException { } private static Iterable transformTableData( - Iterable tableDataPb, final Schema schema, boolean useInt64Timestamps) { + Iterable tableDataPb, final Schema schema, DataFormatOptions dataFormatOptions) { + final boolean useInt64Timestamps = + dataFormatOptions.useInt64Timestamp() + || dataFormatOptions.timestampFormatOptions() + == DataFormatOptions.TimestampFormatOptions.INT64; return ImmutableList.copyOf( Iterables.transform( tableDataPb != null ? tableDataPb : ImmutableList.of(), @@ -2490,9 +2491,7 @@ public com.google.api.services.bigquery.model.QueryResponse call() new QueryPageFetcher(jobId, schema, getOptions(), cursor, optionMap(options)), cursor, transformTableData( - results.getRows(), - schema, - getOptions().getDataFormatOptions().useInt64Timestamp()))) + results.getRows(), schema, getOptions().getDataFormatOptions()))) .setJobId(jobId) .setRowsInPage(results.getRows() != null ? (long) results.getRows().size() : 0L) .build(); @@ -2504,10 +2503,7 @@ public com.google.api.services.bigquery.model.QueryResponse call() new PageImpl<>( new TableDataPageFetcher(null, schema, getOptions(), null, optionMap(options)), null, - transformTableData( - results.getRows(), - schema, - getOptions().getDataFormatOptions().useInt64Timestamp()))) + transformTableData(results.getRows(), schema, getOptions().getDataFormatOptions()))) // Return the JobID of the successful job .setJobId( results.getJobReference() != null ? JobId.fromPb(results.getJobReference()) : null) diff --git a/java-bigquery/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/spi/v2/HttpBigQueryRpc.java b/java-bigquery/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/spi/v2/HttpBigQueryRpc.java index 41fe993b204e..526e24140408 100644 --- a/java-bigquery/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/spi/v2/HttpBigQueryRpc.java +++ b/java-bigquery/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/spi/v2/HttpBigQueryRpc.java @@ -67,6 +67,8 @@ import com.google.cloud.Tuple; import com.google.cloud.bigquery.BigQueryException; import com.google.cloud.bigquery.BigQueryOptions; +import com.google.cloud.bigquery.DataFormatOptions; +import com.google.cloud.bigquery.DataFormatOptions.TimestampFormatOptions; import com.google.cloud.bigquery.telemetry.BigQueryTelemetryTracer; import com.google.cloud.bigquery.telemetry.HttpTracingRequestInitializer; import com.google.cloud.http.HttpTransportOptions; @@ -1419,6 +1421,8 @@ public TableDataList listTableDataSkipExceptionTranslation( String projectId, String datasetId, String tableId, Map options) throws IOException { validateRPC(); + DataFormatOptions formatOptions = this.options.getDataFormatOptions(); + TimestampFormatOptions timestampFormat = formatOptions.timestampFormatOptions(); Bigquery.Tabledata.List bqListRequest = bigquery .tabledata() @@ -1429,6 +1433,11 @@ public TableDataList listTableDataSkipExceptionTranslation( .setStartIndex( Option.START_INDEX.getLong(options) != null ? BigInteger.valueOf(Option.START_INDEX.getLong(options)) + : null) + .setFormatOptionsUseInt64Timestamp(formatOptions.useInt64Timestamp() ? true : null) + .setFormatOptionsTimestampOutputFormat( + timestampFormat != TimestampFormatOptions.TIMESTAMP_OUTPUT_FORMAT_UNSPECIFIED + ? timestampFormat.toString() : null); bqListRequest diff --git a/java-bigquery/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/BigQueryImplTest.java b/java-bigquery/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/BigQueryImplTest.java index 6aab5de04b81..a84602a81815 100644 --- a/java-bigquery/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/BigQueryImplTest.java +++ b/java-bigquery/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/BigQueryImplTest.java @@ -1783,6 +1783,34 @@ void testListTableData() throws IOException { .listTableDataSkipExceptionTranslation(PROJECT, DATASET, TABLE, EMPTY_RPC_OPTIONS); } + @Test + void testListTableDataWithInt64TimestampFormat() throws IOException { + TableDataList tableDataPb = + new TableDataList() + .setTotalRows(1L) + .setRows( + ImmutableList.of( + new TableRow() + .setF(ImmutableList.of(new TableCell().setV("253402300799999999"))))); + when(bigqueryRpcMock.listTableDataSkipExceptionTranslation( + PROJECT, DATASET, TABLE, EMPTY_RPC_OPTIONS)) + .thenReturn(tableDataPb); + bigquery = + options.toBuilder() + .setDataFormatOptions( + DataFormatOptions.newBuilder() + .timestampFormatOptions(DataFormatOptions.TimestampFormatOptions.INT64) + .build()) + .build() + .getService(); + + Page page = bigquery.listTableData(DATASET, TABLE); + assertEquals( + 253402300799999999L, page.getValues().iterator().next().get(0).getTimestampValue()); + verify(bigqueryRpcMock) + .listTableDataSkipExceptionTranslation(PROJECT, DATASET, TABLE, EMPTY_RPC_OPTIONS); + } + @Test void testListTableDataFromTableId() throws IOException { when(bigqueryRpcMock.listTableDataSkipExceptionTranslation( diff --git a/java-bigquery/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java b/java-bigquery/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java index 9efb8f1ce005..54183609b685 100644 --- a/java-bigquery/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java +++ b/java-bigquery/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java @@ -1236,8 +1236,12 @@ static GoogleCredentials loadCredentials(String credentialFile) { @Test void testLosslessMaxTimestampIntegration() throws InterruptedException { - String query = "SELECT TIMESTAMP '9999-12-31 23:59:59.999999 UTC' as max_ts"; - QueryJobConfiguration config = QueryJobConfiguration.newBuilder(query).build(); + // Two rows, one per page, so the second page is read through tabledata.list. + String query = + "SELECT max_ts FROM UNNEST([TIMESTAMP '9999-12-31 23:59:59.999999 UTC'," + + " TIMESTAMP '9999-12-31 23:59:59.999999 UTC']) AS max_ts"; + QueryJobConfiguration config = + QueryJobConfiguration.newBuilder(query).setMaxResults(1L).build(); // 1. Test lossless 64-bit integer parsing (useInt64Timestamps = true) DataFormatOptions losslessOptions = @@ -1249,7 +1253,7 @@ void testLosslessMaxTimestampIntegration() throws InterruptedException { .getService(); TableResult losslessResult = losslessBigQuery.query(config); - assertEquals(1L, losslessResult.getTotalRows()); + assertEquals(2L, losslessResult.getTotalRows()); for (FieldValueList row : losslessResult.iterateAll()) { long exactMicros = row.get("max_ts").getTimestampValue(); assertEquals(253402300799999999L, exactMicros); @@ -1262,7 +1266,7 @@ void testLosslessMaxTimestampIntegration() throws InterruptedException { bigquery.getOptions().toBuilder().setDataFormatOptions(floatOptions).build().getService(); TableResult floatResult = floatBigQuery.query(config); - assertEquals(1L, floatResult.getTotalRows()); + assertEquals(2L, floatResult.getTotalRows()); for (FieldValueList row : floatResult.iterateAll()) { long roundedMicros = row.get("max_ts").getTimestampValue(); assertEquals(253402300800000000L, roundedMicros); @@ -1277,11 +1281,26 @@ void testLosslessMaxTimestampIntegration() throws InterruptedException { bigquery.getOptions().toBuilder().setDataFormatOptions(isoOptions).build().getService(); TableResult isoResult = isoBigQuery.query(config); - assertEquals(1L, isoResult.getTotalRows()); + assertEquals(2L, isoResult.getTotalRows()); for (FieldValueList row : isoResult.iterateAll()) { String isoValue = row.get("max_ts").getStringValue(); assertEquals("9999-12-31T23:59:59.999999Z", isoValue); } + + // 4. Test INT64 timestamp output format + DataFormatOptions int64Options = + DataFormatOptions.newBuilder() + .timestampFormatOptions(DataFormatOptions.TimestampFormatOptions.INT64) + .build(); + BigQuery int64BigQuery = + bigquery.getOptions().toBuilder().setDataFormatOptions(int64Options).build().getService(); + + TableResult int64Result = int64BigQuery.query(config); + assertEquals(2L, int64Result.getTotalRows()); + for (FieldValueList row : int64Result.iterateAll()) { + long exactMicros = row.get("max_ts").getTimestampValue(); + assertEquals(253402300799999999L, exactMicros); + } } @Test diff --git a/java-bigquery/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/spi/v2/HttpBigQueryRpcTest.java b/java-bigquery/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/spi/v2/HttpBigQueryRpcTest.java index 4c3bf67921e6..34f32e594203 100644 --- a/java-bigquery/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/spi/v2/HttpBigQueryRpcTest.java +++ b/java-bigquery/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/spi/v2/HttpBigQueryRpcTest.java @@ -47,6 +47,8 @@ import com.google.api.services.bigquery.model.TableReference; import com.google.cloud.NoCredentials; import com.google.cloud.bigquery.BigQueryOptions; +import com.google.cloud.bigquery.DataFormatOptions; +import com.google.cloud.bigquery.DataFormatOptions.TimestampFormatOptions; import com.google.cloud.bigquery.telemetry.BigQueryTelemetryTracer; import com.google.cloud.bigquery.telemetry.HttpTracingRequestInitializer; import io.opentelemetry.api.common.AttributeKey; @@ -1565,9 +1567,37 @@ public void testListTableDataNoTelemetry() throws Exception { verifyRequest( "GET", "/projects/" + PROJECT_ID + "/datasets/" + DATASET_ID + "/tables/" + TABLE_ID + "/data"); + assertThat(lastRequestUrl).doesNotContain("formatOptions"); verifyNoSpans(); } + @Test + public void testListTableDataWithDataFormatOptions() throws Exception { + setMockResponse("{\"kind\":\"bigquery#tableDataList\",\"rows\":[]}"); + + BigQueryOptions customOptions = + BigQueryOptions.newBuilder() + .setProjectId(PROJECT_ID) + .setCredentials(NoCredentials.getInstance()) + .setTransportOptions( + BigQueryOptions.getDefaultHttpTransportOptions().toBuilder() + .setHttpTransportFactory(() -> mockTransport) + .build()) + .setDataFormatOptions( + DataFormatOptions.newBuilder() + .useInt64Timestamp(true) + .timestampFormatOptions(TimestampFormatOptions.ISO8601_STRING) + .build()) + .build(); + HttpBigQueryRpc customRpc = new HttpBigQueryRpc(customOptions); + + customRpc.listTableDataSkipExceptionTranslation( + PROJECT_ID, DATASET_ID, TABLE_ID, new HashMap<>()); + + assertThat(lastRequestUrl).contains("formatOptions.useInt64Timestamp=true"); + assertThat(lastRequestUrl).contains("formatOptions.timestampOutputFormat=ISO8601_STRING"); + } + @Test public void testListTableDataWithRowLimitNoTelemetry() throws Exception { setMockResponse("{\"kind\":\"bigquery#tableDataList\",\"rows\":[]}");