From 6a9bd2c1a0c3325ec210842f9e5eae002fab695f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henning=20P=C3=B6ttker?= <25299532+hpoettker@users.noreply.github.com> Date: Tue, 10 Nov 2026 17:41:26 +0100 Subject: [PATCH] Fixes for writing sas7bdat files (32-bit/row-compression) --- src/readstat.h | 3 +- src/readstat_error.c | 3 + src/sas/readstat_sas.h | 1 + src/sas/readstat_sas7bdat_write.c | 341 +++++++++++++++++++++++++++--- 4 files changed, 320 insertions(+), 28 deletions(-) diff --git a/src/readstat.h b/src/readstat.h index 10eed3fd..7290948a 100644 --- a/src/readstat.h +++ b/src/readstat.h @@ -111,7 +111,8 @@ typedef enum readstat_error_e { READSTAT_ERROR_LABEL_IS_TOO_LONG, READSTAT_ERROR_DUPLICATE_VALUE_LABEL, READSTAT_ERROR_UNUSED_STRING_REF, - READSTAT_ERROR_DUPLICATE_VARIABLE_NAME + READSTAT_ERROR_DUPLICATE_VARIABLE_NAME, + READSTAT_ERROR_TOO_MANY_ROWS } readstat_error_t; const char *readstat_error_message(readstat_error_t error_code); diff --git a/src/readstat_error.c b/src/readstat_error.c index f9c01d57..6b152c5a 100644 --- a/src/readstat_error.c +++ b/src/readstat_error.c @@ -143,5 +143,8 @@ const char *readstat_error_message(readstat_error_t error_code) { if (error_code == READSTAT_ERROR_DUPLICATE_VARIABLE_NAME) return "Two or more variables have the same name"; + if (error_code == READSTAT_ERROR_TOO_MANY_ROWS) + return "Too many rows for the file format"; + return "Unknown error"; } diff --git a/src/sas/readstat_sas.h b/src/sas/readstat_sas.h index 2ea704d9..d10e3510 100644 --- a/src/sas/readstat_sas.h +++ b/src/sas/readstat_sas.h @@ -114,6 +114,7 @@ typedef enum sas_subheader_type_e { #define SAS_PAGE_TYPE_DELETED_ROWS 0x0080 #define SAS_PAGE_TYPE_META2 0x4000 +#define SAS_PAGE_TYPE_INDEX_LEAF 0x8000 #define SAS_PAGE_TYPE_COMP 0x9000 #define SAS_SUBHEADER_POINTER_SIZE_32BIT 12 diff --git a/src/sas/readstat_sas7bdat_write.c b/src/sas/readstat_sas7bdat_write.c index d2891fa9..018f296e 100644 --- a/src/sas/readstat_sas7bdat_write.c +++ b/src/sas/readstat_sas7bdat_write.c @@ -56,6 +56,12 @@ typedef struct sas7bdat_page_s { size_t data_bytes; /* total length of subheader data on the page */ } sas7bdat_page_t; +typedef struct sas7bdat_index_record_s { + int64_t cumulative_rows; + int64_t page; + uint16_t count; +} sas7bdat_index_record_t; + typedef struct sas7bdat_write_ctx_s { sas_header_info_t *hinfo; int u64; @@ -113,6 +119,9 @@ typedef struct sas7bdat_write_ctx_s { int64_t rows_per_data_page; int64_t total_pages; + /* Index pages of compressed files (COMP root and leaves) */ + int index_pages; + /* Streaming state (uncompressed) */ char *page; int64_t physical_page; @@ -774,9 +783,9 @@ static int64_t sas7bdat_rows_on_mix_page(readstat_writer_t *writer, sas7bdat_wri } static int64_t sas7bdat_physical_page(sas7bdat_write_ctx_t *ctx, int layout_page) { - /* Compressed files carry an extra page after the last metadata page */ + /* Compressed files carry index pages after the last metadata page */ if (ctx->compressed && layout_page > ctx->last_meta_page) - return layout_page + 1; + return layout_page + ctx->index_pages; return layout_page; } @@ -821,7 +830,9 @@ static void sas7bdat_fill_row_size_subheader(readstat_writer_t *writer, sas7bdat sas7bdat_page_t *last_meta_page = &ctx->pages[ctx->last_meta_page-1]; wint(ctx, &d[1*il], ctx->compressed ? 0x131 : 0xF0); - wint(ctx, &d[2*il], total_subheaders + 2); + wint(ctx, &d[2*il], ctx->compressed + ? ctx->meta_subheader_count + writer->row_count + 2 + : total_subheaders + 2); wint(ctx, &d[4*il], ctx->compressed ? SAS7BDAT_ROW_SIZE_COMPRESSED_FLAGS : SAS7BDAT_ROW_SIZE_UNCOMPRESSED_FLAGS); wint(ctx, &d[5*il], ctx->row_length); wint(ctx, &d[6*il], writer->row_count); @@ -832,7 +843,7 @@ static void sas7bdat_fill_row_size_subheader(readstat_writer_t *writer, sas7bdat wint(ctx, &d[11*il], ctx->hash_payload_size); wint(ctx, &d[12*il], ctx->name_len_sum); wint(ctx, &d[13*il], ctx->page_size); - wint(ctx, &d[15*il], ctx->compressed ? 1 : ctx->max_rows_on_mix_page); + wint(ctx, &d[15*il], ctx->max_rows_on_mix_page); memset(&d[16*il], 0xFF, 2*il); /* The first page number */ @@ -893,7 +904,13 @@ static void sas7bdat_fill_row_size_subheader(readstat_writer_t *writer, sas7bdat w2(&d[len-56], ctx->max_label_len); w2(&d[len-42], ctx->compressed ? 0 : ctx->rows_per_data_page); wint(ctx, &d[OFF(ctx, 448, 776)], writer->row_count); - d[len-13] = OFF(ctx, 5, 1); + + /* Rows on the mix page start at the 8-byte aligned end of the subheader + * pointers (including the terminator). SAS doesn't recompute this + * alignment when reading but takes the padding from here, as 1 + the + * number of padding bytes. */ + size_t mix_ptr_end = ctx->phs + (last_meta_page->count + 1) * ctx->sps; + d[len-13] = ctx->compressed ? 1 : 1 + (align8(mix_ptr_end) - mix_ptr_end); } static void sas7bdat_fill_counts_subheader(sas7bdat_write_ctx_t *ctx) { @@ -1020,37 +1037,268 @@ static readstat_error_t sas7bdat_flush_page(readstat_writer_t *writer, sas7bdat_ return readstat_write_bytes(writer, ctx->page, ctx->page_size); } -/* The mostly empty page that SAS writes after the metadata of compressed files */ -static readstat_error_t sas7bdat_write_comp_page(readstat_writer_t *writer, sas7bdat_write_ctx_t *ctx, - int64_t physical_page) { - char *buffer = ctx->page; - sas7bdat_page_t *first_page = &ctx->pages[0]; - int64_t rows_on_first_page = 0; - int64_t i; - - /* SAS lays out the COMP page within at most 0x7FFF bytes, as the field - * at phs-2 is a signed 16-bit value, and zero-fills the rest. */ - size_t region = ctx->page_size < 0x7FFF ? ctx->page_size : 0x7FFF; +/* Index of the pages holding rows in compressed files ----------------- * + * + * SAS writes a B-tree of the pages holding rows. The root (or the only + * leaf) follows the last metadata page; SAS writes further leaves between + * the data pages as they fill up, while ReadStat writes all index pages + * right after the last metadata page. Each leaf record + * describes one page holding rows and is 2*int_len+2 bytes: + * + * int_len bytes cumulative row count up to and including the page (big-endian) + * int_len bytes 1-based physical page number (native byte order) + * 2 bytes number of subheader pointers on the page, without the terminator + * + * Records are laid out backwards from the end of the region (first page + * last). If all records fit into one page, that page is the only index page + * (type COMP). Otherwise SAS writes a root page (type COMP, level 1) + * followed by the leaf pages; all leaves but the last have type 0x8000, the + * last has type COMP. + * + * Root entries are a shortened separator key (big-endian, cut after the + * first byte where the first key of the next leaf differs from the last key + * of this leaf; all 0xFF for the last leaf) followed by the 1-based child + * page. Their 2-byte offsets follow the level field at phs+8. + * + * Page header fields of index pages: + * + * offset 16/8 next leaf (1-based page number, 0 for the last leaf) + * offset 24/12 always 1 + * phs-8 page type + * phs-6 see sas7bdat_index_page_marker() + * phs-4 number of records / root entries + * phs-2 free bytes between the header (or root offsets) and the entries + * phs level (8 bytes; 0 for leaves, 1 for the root) + * + * Pages holding compressed rows carry the 1-based number of their first + * row at offset 16/8 (0 on the first page). The data step doesn't need it, + * but the SAS data viewer numbers rows from it. + */ - for (i=first_page->first; ifirst+first_page->count; i++) { +/* Number of rows (as opposed to metadata subheaders) on a layout page */ +static int64_t sas7bdat_page_row_count(sas7bdat_write_ctx_t *ctx, int layout_page) { + sas7bdat_page_t *page = &ctx->pages[layout_page]; + int64_t rows = 0; + for (int64_t i=page->first; ifirst+page->count; i++) { if (ctx->subheaders[i]->is_row_data) - rows_on_first_page++; + rows++; } + return rows; +} + +/* Pages without rows get no index record, except the first page of a file + * without any rows, so that there is always one record. */ +static int sas7bdat_page_is_indexed(readstat_writer_t *writer, int layout_page, int64_t rows) { + return rows > 0 || (layout_page == 0 && writer->row_count == 0); +} + +/* SAS lays out index pages within at most 0x7FFF bytes, as the field + * at phs-2 is a signed 16-bit value, and zero-fills the rest. */ +static size_t sas7bdat_index_region(sas7bdat_write_ctx_t *ctx) { + return ctx->page_size < 0x7FFF ? ctx->page_size : 0x7FFF; +} + +static size_t sas7bdat_index_record_len(sas7bdat_write_ctx_t *ctx) { + return 2 * ctx->int_len + 2; +} +static size_t sas7bdat_index_records_per_leaf(sas7bdat_write_ctx_t *ctx) { + return (sas7bdat_index_region(ctx) - ctx->phs - ctx->sps) / sas7bdat_index_record_len(ctx); +} + +static int64_t sas7bdat_index_leaf_count(sas7bdat_write_ctx_t *ctx, int64_t record_count) { + int64_t per_leaf = sas7bdat_index_records_per_leaf(ctx); + return (record_count + per_leaf - 1) / per_leaf; +} + +/* Collects one record per indexed page. page_rows holds the row count of + * each layout page. Needs ctx->index_pages to be set, as the records carry + * physical page numbers. */ +static readstat_error_t sas7bdat_build_index_records(readstat_writer_t *writer, sas7bdat_write_ctx_t *ctx, + const int64_t *page_rows, sas7bdat_index_record_t **out_records, int64_t *out_count) { + sas7bdat_index_record_t *records = calloc(ctx->page_count, sizeof(sas7bdat_index_record_t)); + int64_t count = 0; + int64_t cumulative_rows = 0; + + if (records == NULL) + return READSTAT_ERROR_MALLOC; + + for (int p=0; ppage_count; p++) { + sas7bdat_page_t *page = &ctx->pages[p]; + int64_t rows = page_rows[p]; + if (!sas7bdat_page_is_indexed(writer, p, rows)) + continue; + + cumulative_rows += rows; + records[count].cumulative_rows = cumulative_rows; + records[count].page = sas7bdat_physical_page(ctx, p + 1); + records[count].count = page->count; + count++; + } + + *out_records = records; + *out_count = count; + return READSTAT_OK; +} + +static void sas7bdat_write_index_key(sas7bdat_write_ctx_t *ctx, char *key, int64_t value) { + for (size_t k=0; kint_len; k++) { + key[k] = (value >> (8 * (ctx->int_len - 1 - k))) & 0xFF; + } +} + +/* The value at phs-6 of index pages. Its meaning is unclear; these are the + * values observed in files written by SAS: + * + * single leaf (no root) total page count + * last leaf below a root total page count - 1 + * other leaves 1819 for a leaf indexing up to 0-based page 1816, + * written as page 1821 (possibly the last page + * number at the time the leaf was split off) + * root total page count - 2 + * + * SAS writes leaves where they fill up, between data pages, so the last + * case can't be reproduced; the 0-based number of the last indexed page is + * used instead, which also yields total page count - 1 for the last leaf. + * + * leaf_records is NULL for the root. */ +static int64_t sas7bdat_index_page_marker(sas7bdat_write_ctx_t *ctx, + const sas7bdat_index_record_t *leaf_records, int64_t leaf_record_count) { + if (leaf_records == NULL) + return ctx->total_pages - 2; + if (ctx->index_pages == 1 || leaf_record_count == 0) + return ctx->total_pages; + return leaf_records[leaf_record_count-1].page - 1; +} + +static void sas7bdat_init_index_page(sas7bdat_write_ctx_t *ctx, char *buffer, int64_t physical_page, + uint16_t page_type, int level, int64_t entries, size_t free_bytes, + int64_t marker, int64_t next_leaf) { memset(buffer, 0, ctx->page_size); sas7bdat_set_page_number(ctx, buffer, physical_page); + wint(ctx, &buffer[OFF(ctx, 8, 16)], next_leaf); wint(ctx, &buffer[OFF(ctx, 12, 24)], 1); - w2(&buffer[ctx->phs-8], SAS_PAGE_TYPE_COMP); - w2(&buffer[ctx->phs-6], ctx->total_pages); - w2(&buffer[ctx->phs-4], 1); - w2(&buffer[ctx->phs-2], region - ctx->phs - ctx->sps - (10 - ctx->int_len)); - buffer[region - ctx->int_len - 3] = rows_on_first_page & 0xFF; - buffer[region - ctx->int_len - 2] = 1; - w2(&buffer[region-2], first_page->count); + w2(&buffer[ctx->phs-8], page_type); + w2(&buffer[ctx->phs-6], marker); + w2(&buffer[ctx->phs-4], entries); + w2(&buffer[ctx->phs-2], free_bytes); + w8(&buffer[ctx->phs], level); +} + +static readstat_error_t sas7bdat_write_index_leaf(readstat_writer_t *writer, sas7bdat_write_ctx_t *ctx, + const sas7bdat_index_record_t *records, int64_t count, uint16_t page_type, int64_t physical_page, + int64_t next_leaf) { + char *buffer = ctx->page; + size_t region = sas7bdat_index_region(ctx); + size_t record_len = sas7bdat_index_record_len(ctx); + + sas7bdat_init_index_page(ctx, buffer, physical_page, page_type, 0, count, + region - (ctx->phs + 8) - count * record_len, + sas7bdat_index_page_marker(ctx, records, count), next_leaf); + + for (int64_t r=0; rint_len, records[r].page); + w2(record + 2 * ctx->int_len, records[r].count); + } return readstat_write_bytes(writer, buffer, ctx->page_size); } +/* Fills key with the separator key of root entry l and returns its length */ +static size_t sas7bdat_index_root_key(sas7bdat_write_ctx_t *ctx, const sas7bdat_index_record_t *records, + int64_t leaf_count, int64_t l, char key[8]) { + if (l == leaf_count - 1) { + memset(key, 0xFF, 8); + return ctx->int_len; + } + + int64_t last = (l + 1) * (int64_t)sas7bdat_index_records_per_leaf(ctx) - 1; + char last_key[8]; + sas7bdat_write_index_key(ctx, last_key, records[last].cumulative_rows); + sas7bdat_write_index_key(ctx, key, records[last + 1].cumulative_rows); + + size_t len = 0; + while (len < ctx->int_len && last_key[len] == key[len]) + len++; + return len < ctx->int_len ? len + 1 : len; +} + +/* Returns the offset of the lowest root entry, or 0 if the entries don't fit */ +static size_t sas7bdat_index_root_entry_end(sas7bdat_write_ctx_t *ctx, + const sas7bdat_index_record_t *records, int64_t leaf_count) { + size_t entry_end = sas7bdat_index_region(ctx); + size_t offsets_end = ctx->phs + 8 + 2 * leaf_count; + char key[8]; + for (int64_t l=0; lint_len; + if (entry_end < offsets_end + entry_len) + return 0; + entry_end -= entry_len; + } + return entry_end; +} + +static readstat_error_t sas7bdat_write_index_root(readstat_writer_t *writer, sas7bdat_write_ctx_t *ctx, + const sas7bdat_index_record_t *records, int64_t leaf_count, int64_t physical_page, + int64_t first_leaf_page) { + char *buffer = ctx->page; + size_t offsets_end = ctx->phs + 8 + 2 * leaf_count; + size_t entry_end = sas7bdat_index_root_entry_end(ctx, records, leaf_count); + + /* Checked up front by sas7bdat_end_compressed */ + if (entry_end == 0) + return READSTAT_ERROR_TOO_MANY_ROWS; + + sas7bdat_init_index_page(ctx, buffer, physical_page, SAS_PAGE_TYPE_COMP, 1, leaf_count, + entry_end - offsets_end, sas7bdat_index_page_marker(ctx, NULL, 0), 0); + + size_t entry_offset = sas7bdat_index_region(ctx); + char key[8]; + for (int64_t l=0; lint_len; + memcpy(&buffer[entry_offset], key, key_len); + wint(ctx, &buffer[entry_offset + key_len], first_leaf_page + l + 1); + w2(&buffer[ctx->phs + 8 + 2 * l], entry_offset); + } + + return readstat_write_bytes(writer, buffer, ctx->page_size); +} + +static readstat_error_t sas7bdat_write_index_pages(readstat_writer_t *writer, sas7bdat_write_ctx_t *ctx, + const sas7bdat_index_record_t *records, int64_t record_count, int64_t physical_page) { + readstat_error_t retval = READSTAT_OK; + int64_t per_leaf = sas7bdat_index_records_per_leaf(ctx); + int64_t leaf_count = sas7bdat_index_leaf_count(ctx, record_count); + + if (leaf_count == 1) { + return sas7bdat_write_index_leaf(writer, ctx, records, record_count, + SAS_PAGE_TYPE_COMP, physical_page, 0); + } + + int64_t first_leaf_page = physical_page + 1; + if ((retval = sas7bdat_write_index_root(writer, ctx, records, leaf_count, + physical_page, first_leaf_page)) != READSTAT_OK) + return retval; + + for (int64_t l=0; ltotal_pages = ctx->page_count + 1; + if ((page_rows = calloc(ctx->page_count, sizeof(int64_t))) == NULL) { + retval = READSTAT_ERROR_MALLOC; + goto cleanup; + } + + /* The records carry physical page numbers, which depend on the number of + * index pages, so count the indexed pages first */ + int64_t indexed_pages = 0; + for (p=0; ppage_count; p++) { + page_rows[p] = sas7bdat_page_row_count(ctx, p); + if (sas7bdat_page_is_indexed(writer, p, page_rows[p])) + indexed_pages++; + } + int64_t leaf_count = sas7bdat_index_leaf_count(ctx, indexed_pages); + ctx->index_pages = leaf_count > 1 ? 1 + leaf_count : 1; + ctx->total_pages = ctx->page_count + ctx->index_pages; + + /* Page numbers and counts in index page headers are 16-bit */ + if (ctx->total_pages > 0xFFFF) { + retval = READSTAT_ERROR_TOO_MANY_ROWS; + goto cleanup; + } + + if ((retval = sas7bdat_build_index_records(writer, ctx, page_rows, &records, &record_count)) != READSTAT_OK) + goto cleanup; + + /* Fail before writing anything if the root can't hold all leaves */ + if (leaf_count > 1 && sas7bdat_index_root_entry_end(ctx, records, leaf_count) == 0) { + retval = READSTAT_ERROR_TOO_MANY_ROWS; + goto cleanup; + } sas7bdat_fill_row_size_subheader(writer, ctx); sas7bdat_fill_counts_subheader(ctx); @@ -1285,20 +1566,26 @@ static readstat_error_t sas7bdat_end_compressed(readstat_writer_t *writer, sas7b if ((retval = sas7bdat_emit_header(writer, ctx)) != READSTAT_OK) goto cleanup; + int64_t rows_before_page = 0; for (p=0; ppage_count; p++) { int64_t physical_page = sas7bdat_physical_page(ctx, p + 1) - 1; int subheader_count = sas7bdat_render_layout_page(ctx, p, physical_page, ctx->page); sas7bdat_finish_page_header(ctx, ctx->page, SAS_PAGE_TYPE_META, subheader_count, ctx->pages[p].data_bytes, 0); + wint(ctx, &ctx->page[OFF(ctx, 8, 16)], rows_before_page ? rows_before_page + 1 : 0); + rows_before_page += page_rows[p]; if ((retval = readstat_write_bytes(writer, ctx->page, ctx->page_size)) != READSTAT_OK) goto cleanup; if (p + 1 == ctx->last_meta_page) { - if ((retval = sas7bdat_write_comp_page(writer, ctx, physical_page + 1)) != READSTAT_OK) + if ((retval = sas7bdat_write_index_pages(writer, ctx, records, record_count, + physical_page + 1)) != READSTAT_OK) goto cleanup; } } cleanup: + free(records); + free(page_rows); return retval; }