Skip to content

GH-54957: [Python][Parquet] Translate C++ ParquetException in metadata accessors to prevent process abort #55244

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
ishaaaan17 wants to merge 1 commit into
base: main
Choose a base branch
from
+46 −9
Open

GH-54957: [Python][Parquet] Translate C++ ParquetException in metadata accessors to prevent process abort #55244

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions python/pyarrow/_parquet.pxd
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -98,7 +98,7 @@ cdef class RowGroupMetaData(_Weakrefable):
CRowGroupMetaData* metadata
FileMetaData parent

cdef inline init(self, FileMetaData parent, int index):
cdef inline init(self, FileMetaData parent, int index) except *:
if index < 0 or index >= parent.num_row_groups:
raise IndexError('{0} out of bounds'.format(index))
self.up_metadata = parent._metadata.RowGroup(index)
Expand All @@ -113,7 +113,7 @@ cdef class ColumnChunkMetaData(_Weakrefable):
CColumnChunkMetaData* metadata
RowGroupMetaData parent

cdef inline init(self, RowGroupMetaData parent, int i):
cdef inline init(self, RowGroupMetaData parent, int i) except *:
self.up_metadata = parent.metadata.ColumnChunk(i)
self.metadata = self.up_metadata.get()
self.parent = parent
Expand Down
14 changes: 7 additions & 7 deletions python/pyarrow/includes/libparquet.pxd
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -358,9 +358,9 @@ cdef extern from "parquet/api/reader.h" namespace "parquet" nogil:
int64_t num_values() const
shared_ptr[ColumnPath] path_in_schema() const
bint is_stats_set() const
shared_ptr[CStatistics] statistics() const
shared_ptr[CStatistics] statistics() const except +
c_bool is_geo_stats_set() const
shared_ptr[CParquetGeoStatistics] geo_statistics() const
shared_ptr[CParquetGeoStatistics] geo_statistics() const except +
ParquetCompression compression() const
const vector[ParquetEncoding]& encodings() const
c_bool Equals(const CColumnChunkMetaData&) const
Expand All @@ -373,7 +373,7 @@ cdef extern from "parquet/api/reader.h" namespace "parquet" nogil:
int64_t total_uncompressed_size() const
optional[int64_t] bloom_filter_offset() const
optional[int64_t] bloom_filter_length() const
unique_ptr[CColumnCryptoMetaData] crypto_metadata() const
unique_ptr[CColumnCryptoMetaData] crypto_metadata() const except +
optional[ParquetIndexLocation] GetColumnIndexLocation() const
optional[ParquetIndexLocation] GetOffsetIndexLocation() const
shared_ptr[const CKeyValueMetadata] key_value_metadata() const
Expand All @@ -389,7 +389,7 @@ cdef extern from "parquet/api/reader.h" namespace "parquet" nogil:
int64_t num_rows() const
int64_t total_byte_size() const
vector[CSortingColumn] sorting_columns() const
unique_ptr[CColumnChunkMetaData] ColumnChunk(int i) const
unique_ptr[CColumnChunkMetaData] ColumnChunk(int i) const except +

cdef cppclass CFileMetaData" parquet::FileMetaData":
c_bool Equals(const CFileMetaData&) const
Expand All @@ -404,18 +404,18 @@ cdef extern from "parquet/api/reader.h" namespace "parquet" nogil:
void set_file_path(const c_string& path)
void AppendRowGroups(const CFileMetaData& other) except +

unique_ptr[CRowGroupMetaData] RowGroup(int i)
unique_ptr[CRowGroupMetaData] RowGroup(int i) except +
const SchemaDescriptor* schema()
shared_ptr[const CKeyValueMetadata] key_value_metadata() const
void WriteTo(COutputStream* dst) const
void WriteTo(COutputStream* dst) const except +

inline c_bool is_encryption_algorithm_set() const
inline EncryptionAlgorithm encryption_algorithm() const
inline const c_string& footer_signing_key_metadata() const

cdef shared_ptr[CFileMetaData] CFileMetaData_Make \
" parquet::FileMetaData::Make"(const void* serialized_metadata,
uint32_t* metadata_len)
uint32_t* metadata_len) except +

cdef cppclass CReaderProperties" parquet::ReaderProperties":
CMemoryPool* memory_pool() const
Expand Down
37 changes: 37 additions & 0 deletions python/pyarrow/tests/parquet/test_metadata.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -893,3 +893,40 @@ def test_geospatial_types(parquet_test_datadir):
column_schema = metadata.schema.column(1)
assert column_schema.name == "geography"
assert column_schema.logical_type.type == "GEOGRAPHY"


def test_damaged_metadata_raises_exception_not_abort(tmp_path):
# GH-54957: Damaged metadata or type mismatches in Parquet footer
# should raise a Python exception rather than aborting the process.
clean_path = tmp_path / "clean.parquet"
table = pa.table({"x": pa.array([1, 2, 3], pa.int64())})
pq.write_table(table, clean_path)

data = bytearray(clean_path.read_bytes())
# Modify repetition_type in Thrift SchemaElement for "x"
# (zigzag 2 = OPTIONAL -> change to 0 = REQUIRED)
marker = b"\x25\x02\x18\x01x"
idx = data.index(marker)
data[idx + 1] = 0
crash_path = tmp_path / "corrupt_levels.parquet"
crash_path.write_bytes(data)

md = pq.ParquetFile(crash_path).metadata
with pytest.raises(OSError):
md.row_group(0).column(0)

# Test physical type mismatch in ColumnMetaData vs SchemaElement
table_float = pa.table({"x": pa.array([1.0, 2.0, 3.0], pa.float64())})
clean_float_path = tmp_path / "clean_float.parquet"
pq.write_table(table_float, clean_float_path)

data_float = bytearray(clean_float_path.read_bytes())
marker_float = b"\x15\x0a\x25\x02\x18\x01x"
idx_float = data_float.index(marker_float)
data_float[idx_float + 1] = 0 # DOUBLE -> BOOLEAN
crash_float_path = tmp_path / "corrupt_stats.parquet"
crash_float_path.write_bytes(data_float)

col = pq.ParquetFile(crash_float_path).metadata.row_group(0).column(0)
with pytest.raises(OSError):
_ = col.statistics
Loading