From ce387cb38b45e98c1e9e8cc591c4e8fa1021c810 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Henning=20P=C3=B6ttker?=
<25299532+hpoettker@users.noreply.github.com>
Date: Wed, 11 Nov 2026 18:24:16 +0100
Subject: [PATCH] RDC compression for sas7bdat writes
---
Makefile.am | 2 +
VS17/ReadStat.vcxproj | 2 +
VS17/ReadStat.vcxproj.filters | 6 +
src/readstat.h | 2 +-
src/sas/readstat_sas7bdat_read.c | 91 +++---------
src/sas/readstat_sas7bdat_write.c | 123 +++++++++-------
src/sas/readstat_sas_rdc.c | 230 ++++++++++++++++++++++++++++++
src/sas/readstat_sas_rdc.h | 21 +++
src/test/test_list.h | 4 +-
src/test/test_read.c | 4 +
src/test/test_readstat.h | 22 +--
src/test/test_write.c | 2 +
12 files changed, 375 insertions(+), 134 deletions(-)
create mode 100644 src/sas/readstat_sas_rdc.c
create mode 100644 src/sas/readstat_sas_rdc.h
diff --git a/Makefile.am b/Makefile.am
index 9bdcd196..6af6516c 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -21,6 +21,7 @@ libreadstat_la_SOURCES = \
src/sas/readstat_sas7bdat_read.c \
src/sas/readstat_sas7bdat_write.c \
src/sas/readstat_sas_rle.c \
+ src/sas/readstat_sas_rdc.c \
src/sas/readstat_xport.c \
src/sas/readstat_xport_read.c \
src/sas/readstat_xport_write.c \
@@ -111,6 +112,7 @@ noinst_HEADERS = \
src/sas/ieee.h \
src/sas/readstat_sas.h \
src/sas/readstat_sas_rle.h \
+ src/sas/readstat_sas_rdc.h \
src/sas/readstat_xport.h \
src/sas/readstat_xport_parse_format.h \
src/spss/readstat_por.h \
diff --git a/VS17/ReadStat.vcxproj b/VS17/ReadStat.vcxproj
index e129d8fe..bdc7f7ef 100644
--- a/VS17/ReadStat.vcxproj
+++ b/VS17/ReadStat.vcxproj
@@ -203,6 +203,7 @@
+
@@ -247,6 +248,7 @@
+
diff --git a/VS17/ReadStat.vcxproj.filters b/VS17/ReadStat.vcxproj.filters
index d9178318..3f645fa8 100644
--- a/VS17/ReadStat.vcxproj.filters
+++ b/VS17/ReadStat.vcxproj.filters
@@ -81,6 +81,9 @@
Source Files\sas
+
+ Source Files\sas
+
Source Files\sas
@@ -206,6 +209,9 @@
Header Files
+
+ Header Files
+
Header Files
diff --git a/src/readstat.h b/src/readstat.h
index 7290948a..8ba57156 100644
--- a/src/readstat.h
+++ b/src/readstat.h
@@ -616,7 +616,7 @@ readstat_error_t readstat_writer_set_file_format_is_64bit(readstat_writer_t *wri
int is_64bit); // applies only to SAS files; defaults to 1=true
readstat_error_t readstat_writer_set_compression(readstat_writer_t *writer,
readstat_compress_t compression);
- // READSTAT_COMPRESS_BINARY is supported only with SAV files (i.e. ZSAV files)
+ // READSTAT_COMPRESS_BINARY is supported only with sas7bdat and SAV files (i.e. ZSAV files)
// READSTAT_COMPRESS_ROWS is supported only with sas7bdat and SAV files
// Optional error handler
diff --git a/src/sas/readstat_sas7bdat_read.c b/src/sas/readstat_sas7bdat_read.c
index 3bf11581..9eeb5a70 100644
--- a/src/sas/readstat_sas7bdat_read.c
+++ b/src/sas/readstat_sas7bdat_read.c
@@ -8,6 +8,7 @@
#include
#include "readstat_sas.h"
#include "readstat_sas_rle.h"
+#include "readstat_sas_rdc.h"
#include "../readstat_iconv.h"
#include "../readstat_convert.h"
#include "../readstat_malloc.h"
@@ -573,88 +574,28 @@ static readstat_error_t sas7bdat_parse_rows(const char *data, size_t len,
}
static readstat_error_t sas7bdat_parse_subheader_rdc(const char *subheader, size_t len, sas7bdat_ctx_t *ctx) {
- readstat_error_t retval = READSTAT_OK;
- const unsigned char *input = (const unsigned char *)subheader;
- char *buffer = ctx->row;
- if (buffer == NULL) return READSTAT_ERROR_PARSE;
- char *output = buffer;
- while (input + 2 <= (const unsigned char *)subheader + len) {
- int i;
- unsigned short prefix = (input[0] << 8) + input[1];
- input += 2;
- for (i=0; i<16; i++) {
- if ((prefix & (1 << (15 - i))) == 0) {
- if (input + 1 > (const unsigned char *)subheader + len) {
- break;
- }
- if (output + 1 > buffer + ctx->row_length) {
- retval = READSTAT_ERROR_ROW_WIDTH_MISMATCH;
- goto cleanup;
- }
- *output++ = *input++;
- continue;
- }
+ if (ctx->row_limit == ctx->parsed_row_count)
+ return READSTAT_OK;
- if (input + 2 > (const unsigned char *)subheader + len) {
- retval = READSTAT_ERROR_PARSE;
- goto cleanup;
- }
+ readstat_error_t retval = READSTAT_OK;
+ ssize_t bytes_decompressed = 0;
- unsigned char marker_byte = *input++;
- unsigned char next_byte = *input++;
- size_t insert_len = 0, copy_len = 0;
- unsigned char insert_byte = 0x00;
- size_t back_offset = 0;
-
- if (marker_byte <= 0x0F) {
- insert_len = 3 + marker_byte;
- insert_byte = next_byte;
- } else if ((marker_byte >> 4) == 1) {
- if (input + 1 > (const unsigned char *)subheader + len) {
- retval = READSTAT_ERROR_PARSE;
- goto cleanup;
- }
- insert_len = 19 + (marker_byte & 0x0F) + next_byte * 16;
- insert_byte = *input++;
- } else if ((marker_byte >> 4) == 2) {
- if (input + 1 > (const unsigned char *)subheader + len) {
- retval = READSTAT_ERROR_PARSE;
- goto cleanup;
- }
- copy_len = 16 + (*input++);
- back_offset = 3 + (marker_byte & 0x0F) + next_byte * 16;
- } else {
- copy_len = (marker_byte >> 4);
- back_offset = 3 + (marker_byte & 0x0F) + next_byte * 16;
- }
+ if (ctx->row == NULL)
+ return READSTAT_ERROR_PARSE;
- if (insert_len) {
- if (output + insert_len > buffer + ctx->row_length) {
- retval = READSTAT_ERROR_ROW_WIDTH_MISMATCH;
- goto cleanup;
- }
- memset(output, insert_byte, insert_len);
- output += insert_len;
- } else if (copy_len) {
- if (output - buffer < back_offset || copy_len > back_offset) {
- retval = READSTAT_ERROR_PARSE;
- goto cleanup;
- }
- if (output + copy_len > buffer + ctx->row_length) {
- retval = READSTAT_ERROR_ROW_WIDTH_MISMATCH;
- goto cleanup;
- }
- memcpy(output, output - back_offset, copy_len);
- output += copy_len;
- }
- }
- }
+ bytes_decompressed = sas_rdc_decompress(ctx->row, ctx->row_length, subheader, len);
- if (output - buffer != ctx->row_length) {
+ if (bytes_decompressed != ctx->row_length) {
retval = READSTAT_ERROR_ROW_WIDTH_MISMATCH;
+ if (ctx->handle.error) {
+ snprintf(ctx->error_buf, sizeof(ctx->error_buf),
+ "ReadStat: Row #%d decompressed to %ld bytes (expected %d bytes)",
+ ctx->parsed_row_count, (long)(bytes_decompressed), ctx->row_length);
+ ctx->handle.error(ctx->error_buf, ctx->user_ctx);
+ }
goto cleanup;
}
- retval = sas7bdat_parse_single_row(buffer, ctx);
+ retval = sas7bdat_parse_single_row(ctx->row, ctx);
cleanup:
return retval;
}
diff --git a/src/sas/readstat_sas7bdat_write.c b/src/sas/readstat_sas7bdat_write.c
index 018f296e..aa359443 100644
--- a/src/sas/readstat_sas7bdat_write.c
+++ b/src/sas/readstat_sas7bdat_write.c
@@ -10,6 +10,7 @@
#include "../readstat_writer.h"
#include "readstat_sas.h"
#include "readstat_sas_rle.h"
+#include "readstat_sas_rdc.h"
#include "readstat_xport.h"
#include "readstat_xport_parse_format.h"
@@ -23,19 +24,22 @@
*
* header | META page ... | MIX page (last metadata + first rows) | DATA page ...
*
- * File layout (RLE compressed):
+ * File layout (RLE or RDC compressed):
*
* header | META page ... (metadata subheaders, then row subheaders) | COMP page | META page ...
*/
-#define SAS7BDAT_PAGE_NUMBER_MASK 0xFEDCBA98u
-#define SAS7BDAT_MAX_SUBHEADER_SIZE 32740
+#define SAS7BDAT_PAGE_NUMBER_MASK 0xFEDCBA98u
+#define SAS7BDAT_MAX_SUBHEADER_SIZE 32740
#define SAS7BDAT_MAX_ATTRS_SUBHEADER_SIZE 24588
-#define SAS7BDAT_MAX_NAMES_PER_SUBHEADER 4089
-#define SAS7BDAT_COUNTS_VECTOR_COUNT 12
+#define SAS7BDAT_MAX_NAMES_PER_SUBHEADER 4089
+#define SAS7BDAT_COUNTS_VECTOR_COUNT 12
+#define SAS7BDAT_INDEX_KEY_LEN 8
-#define SAS7BDAT_ROW_SIZE_UNCOMPRESSED_FLAGS 0x00223011
-#define SAS7BDAT_ROW_SIZE_COMPRESSED_FLAGS 0x0022703A
+#define SAS7BDAT_ROW_SIZE_UNCOMPRESSED_FLAGS 0x00223011
+#define SAS7BDAT_ROW_SIZE_COMPRESSED_FLAGS 0x0022703A
+#define SAS7BDAT_ROW_SIZE_RDC_COMPRESSED_FLAGS 0x0022303A
+#define SAS7BDAT_ROW_SIZE_FLAG_EXTEND_OBS_COUNTER 0x04000000
typedef struct sas7bdat_subheader_s {
uint32_t signature;
@@ -66,6 +70,10 @@ typedef struct sas7bdat_write_ctx_s {
sas_header_info_t *hinfo;
int u64;
int compressed;
+ /* Set if compressed: RLE (COMPRESS=CHAR) or RDC (COMPRESS=BINARY) */
+ ssize_t (*compress)(void *, size_t, const void *, size_t);
+ const char *compression_signature;
+ uint32_t compressed_row_size_flags;
size_t sig_len;
size_t footer_len;
size_t int_len;
@@ -510,7 +518,7 @@ static readstat_error_t sas7bdat_add_column_text(readstat_writer_t *writer, sas7
if ((retval = sas7bdat_text_add_bytes(ctx, first_entry, sizeof(first_entry), NULL)) != READSTAT_OK)
goto cleanup;
if (ctx->compressed) {
- if ((retval = sas7bdat_text_add(ctx, SAS_COMPRESSION_SIGNATURE_RLE, &ctx->compression_ref)) != READSTAT_OK)
+ if ((retval = sas7bdat_text_add(ctx, ctx->compression_signature, &ctx->compression_ref)) != READSTAT_OK)
goto cleanup;
}
if ((retval = sas7bdat_text_add(ctx, blank, &ctx->blank_ref)) != READSTAT_OK)
@@ -829,15 +837,17 @@ static void sas7bdat_fill_row_size_subheader(readstat_writer_t *writer, sas7bdat
sas7bdat_subheader_t *last_meta = ctx->subheaders[ctx->meta_subheader_count-1];
sas7bdat_page_t *last_meta_page = &ctx->pages[ctx->last_meta_page-1];
- wint(ctx, &d[1*il], ctx->compressed ? 0x131 : 0xF0);
+ wint(ctx, &d[1*il], !ctx->u64 ? 0x28B : ctx->compressed ? 0x131 : 0xF0);
wint(ctx, &d[2*il], ctx->compressed
? ctx->meta_subheader_count + writer->row_count + 2
: total_subheaders + 2);
- wint(ctx, &d[4*il], ctx->compressed ? SAS7BDAT_ROW_SIZE_COMPRESSED_FLAGS : SAS7BDAT_ROW_SIZE_UNCOMPRESSED_FLAGS);
+ wint(ctx, &d[4*il], (ctx->compressed ? ctx->compressed_row_size_flags
+ : SAS7BDAT_ROW_SIZE_UNCOMPRESSED_FLAGS)
+ | (ctx->u64 ? 0 : SAS7BDAT_ROW_SIZE_FLAG_EXTEND_OBS_COUNTER));
wint(ctx, &d[5*il], ctx->row_length);
wint(ctx, &d[6*il], writer->row_count);
wint(ctx, &d[7*il], 0); /* deleted rows */
- wint(ctx, &d[8*il], ctx->compressed ? 2 : 0);
+ wint(ctx, &d[8*il], ctx->compressed ? ctx->last_meta_page + 1 : 0);
wint(ctx, &d[9*il], formats_on_first_page);
wint(ctx, &d[10*il], formats_on_second_page);
wint(ctx, &d[11*il], ctx->hash_payload_size);
@@ -1043,9 +1053,10 @@ static readstat_error_t sas7bdat_flush_page(readstat_writer_t *writer, sas7bdat_
* leaf) follows the last metadata page; SAS writes further leaves between
* the data pages as they fill up, while ReadStat writes all index pages
* right after the last metadata page. Each leaf record
- * describes one page holding rows and is 2*int_len+2 bytes:
+ * describes one page holding rows and is 8+int_len+2 bytes:
*
- * int_len bytes cumulative row count up to and including the page (big-endian)
+ * 8 bytes cumulative row count up to and including the page (big-endian,
+ * 8 bytes in 32-bit files, too)
* int_len bytes 1-based physical page number (native byte order)
* 2 bytes number of subheader pointers on the page, without the terminator
*
@@ -1058,7 +1069,7 @@ static readstat_error_t sas7bdat_flush_page(readstat_writer_t *writer, sas7bdat_
* Root entries are a shortened separator key (big-endian, cut after the
* first byte where the first key of the next leaf differs from the last key
* of this leaf; all 0xFF for the last leaf) followed by the 1-based child
- * page. Their 2-byte offsets follow the level field at phs+8.
+ * page. Their 2-byte offsets follow the level field at phs+int_len.
*
* Page header fields of index pages:
*
@@ -1068,7 +1079,7 @@ static readstat_error_t sas7bdat_flush_page(readstat_writer_t *writer, sas7bdat_
* phs-6 see sas7bdat_index_page_marker()
* phs-4 number of records / root entries
* phs-2 free bytes between the header (or root offsets) and the entries
- * phs level (8 bytes; 0 for leaves, 1 for the root)
+ * phs level (int_len bytes; 0 for leaves, 1 for the root)
*
* Pages holding compressed rows carry the 1-based number of their first
* row at offset 16/8 (0 on the first page). The data step doesn't need it,
@@ -1099,11 +1110,12 @@ static size_t sas7bdat_index_region(sas7bdat_write_ctx_t *ctx) {
}
static size_t sas7bdat_index_record_len(sas7bdat_write_ctx_t *ctx) {
- return 2 * ctx->int_len + 2;
+ return SAS7BDAT_INDEX_KEY_LEN + ctx->int_len + 2;
}
static size_t sas7bdat_index_records_per_leaf(sas7bdat_write_ctx_t *ctx) {
- return (sas7bdat_index_region(ctx) - ctx->phs - ctx->sps) / sas7bdat_index_record_len(ctx);
+ size_t record_len = sas7bdat_index_record_len(ctx);
+ return (sas7bdat_index_region(ctx) - ctx->phs - ctx->int_len - record_len) / record_len;
}
static int64_t sas7bdat_index_leaf_count(sas7bdat_write_ctx_t *ctx, int64_t record_count) {
@@ -1141,21 +1153,23 @@ static readstat_error_t sas7bdat_build_index_records(readstat_writer_t *writer,
return READSTAT_OK;
}
-static void sas7bdat_write_index_key(sas7bdat_write_ctx_t *ctx, char *key, int64_t value) {
- for (size_t k=0; kint_len; k++) {
- key[k] = (value >> (8 * (ctx->int_len - 1 - k))) & 0xFF;
+static void sas7bdat_write_index_key(char *key, int64_t value) {
+ for (size_t k=0; k> (8 * (SAS7BDAT_INDEX_KEY_LEN - 1 - k))) & 0xFF;
}
}
/* The value at phs-6 of index pages. Its meaning is unclear; these are the
* values observed in files written by SAS:
*
- * single leaf (no root) total page count
+ * single leaf (no root) number of records + 1 (equal to the total page
+ * count only if no page is metadata-only; 1 for
+ * files without rows)
* last leaf below a root total page count - 1
* other leaves 1819 for a leaf indexing up to 0-based page 1816,
* written as page 1821 (possibly the last page
* number at the time the leaf was split off)
- * root total page count - 2
+ * root total page count - number of leaves
*
* SAS writes leaves where they fill up, between data pages, so the last
* case can't be reproduced; the 0-based number of the last indexed page is
@@ -1165,9 +1179,9 @@ static void sas7bdat_write_index_key(sas7bdat_write_ctx_t *ctx, char *key, int64
static int64_t sas7bdat_index_page_marker(sas7bdat_write_ctx_t *ctx,
const sas7bdat_index_record_t *leaf_records, int64_t leaf_record_count) {
if (leaf_records == NULL)
- return ctx->total_pages - 2;
- if (ctx->index_pages == 1 || leaf_record_count == 0)
- return ctx->total_pages;
+ return ctx->total_pages - (ctx->index_pages - 1);
+ if (ctx->index_pages == 1)
+ return leaf_record_count + 1;
return leaf_records[leaf_record_count-1].page - 1;
}
@@ -1182,7 +1196,7 @@ static void sas7bdat_init_index_page(sas7bdat_write_ctx_t *ctx, char *buffer, in
w2(&buffer[ctx->phs-6], marker);
w2(&buffer[ctx->phs-4], entries);
w2(&buffer[ctx->phs-2], free_bytes);
- w8(&buffer[ctx->phs], level);
+ wint(ctx, &buffer[ctx->phs], level);
}
static readstat_error_t sas7bdat_write_index_leaf(readstat_writer_t *writer, sas7bdat_write_ctx_t *ctx,
@@ -1193,14 +1207,14 @@ static readstat_error_t sas7bdat_write_index_leaf(readstat_writer_t *writer, sas
size_t record_len = sas7bdat_index_record_len(ctx);
sas7bdat_init_index_page(ctx, buffer, physical_page, page_type, 0, count,
- region - (ctx->phs + 8) - count * record_len,
+ region - (ctx->phs + ctx->int_len) - count * record_len,
sas7bdat_index_page_marker(ctx, records, count), next_leaf);
for (int64_t r=0; rint_len, records[r].page);
- w2(record + 2 * ctx->int_len, records[r].count);
+ sas7bdat_write_index_key(record, records[r].cumulative_rows);
+ wint(ctx, record + SAS7BDAT_INDEX_KEY_LEN, records[r].page);
+ w2(record + SAS7BDAT_INDEX_KEY_LEN + ctx->int_len, records[r].count);
}
return readstat_write_bytes(writer, buffer, ctx->page_size);
@@ -1208,29 +1222,29 @@ static readstat_error_t sas7bdat_write_index_leaf(readstat_writer_t *writer, sas
/* Fills key with the separator key of root entry l and returns its length */
static size_t sas7bdat_index_root_key(sas7bdat_write_ctx_t *ctx, const sas7bdat_index_record_t *records,
- int64_t leaf_count, int64_t l, char key[8]) {
+ int64_t leaf_count, int64_t l, char key[SAS7BDAT_INDEX_KEY_LEN]) {
if (l == leaf_count - 1) {
- memset(key, 0xFF, 8);
- return ctx->int_len;
+ memset(key, 0xFF, SAS7BDAT_INDEX_KEY_LEN);
+ return SAS7BDAT_INDEX_KEY_LEN;
}
int64_t last = (l + 1) * (int64_t)sas7bdat_index_records_per_leaf(ctx) - 1;
- char last_key[8];
- sas7bdat_write_index_key(ctx, last_key, records[last].cumulative_rows);
- sas7bdat_write_index_key(ctx, key, records[last + 1].cumulative_rows);
+ char last_key[SAS7BDAT_INDEX_KEY_LEN];
+ sas7bdat_write_index_key(last_key, records[last].cumulative_rows);
+ sas7bdat_write_index_key(key, records[last + 1].cumulative_rows);
size_t len = 0;
- while (len < ctx->int_len && last_key[len] == key[len])
+ while (len < SAS7BDAT_INDEX_KEY_LEN && last_key[len] == key[len])
len++;
- return len < ctx->int_len ? len + 1 : len;
+ return len < SAS7BDAT_INDEX_KEY_LEN ? len + 1 : len;
}
/* Returns the offset of the lowest root entry, or 0 if the entries don't fit */
static size_t sas7bdat_index_root_entry_end(sas7bdat_write_ctx_t *ctx,
const sas7bdat_index_record_t *records, int64_t leaf_count) {
size_t entry_end = sas7bdat_index_region(ctx);
- size_t offsets_end = ctx->phs + 8 + 2 * leaf_count;
- char key[8];
+ size_t offsets_end = ctx->phs + ctx->int_len + 2 * leaf_count;
+ char key[SAS7BDAT_INDEX_KEY_LEN];
for (int64_t l=0; lint_len;
if (entry_end < offsets_end + entry_len)
@@ -1244,7 +1258,7 @@ static readstat_error_t sas7bdat_write_index_root(readstat_writer_t *writer, sas
const sas7bdat_index_record_t *records, int64_t leaf_count, int64_t physical_page,
int64_t first_leaf_page) {
char *buffer = ctx->page;
- size_t offsets_end = ctx->phs + 8 + 2 * leaf_count;
+ size_t offsets_end = ctx->phs + ctx->int_len + 2 * leaf_count;
size_t entry_end = sas7bdat_index_root_entry_end(ctx, records, leaf_count);
/* Checked up front by sas7bdat_end_compressed */
@@ -1255,13 +1269,13 @@ static readstat_error_t sas7bdat_write_index_root(readstat_writer_t *writer, sas
entry_end - offsets_end, sas7bdat_index_page_marker(ctx, NULL, 0), 0);
size_t entry_offset = sas7bdat_index_region(ctx);
- char key[8];
+ char key[SAS7BDAT_INDEX_KEY_LEN];
for (int64_t l=0; lint_len;
memcpy(&buffer[entry_offset], key, key_len);
wint(ctx, &buffer[entry_offset + key_len], first_leaf_page + l + 1);
- w2(&buffer[ctx->phs + 8 + 2 * l], entry_offset);
+ w2(&buffer[ctx->phs + ctx->int_len + 2 * l], entry_offset);
}
return readstat_write_bytes(writer, buffer, ctx->page_size);
@@ -1379,7 +1393,17 @@ static readstat_error_t sas7bdat_write_ctx_init(readstat_writer_t *writer, sas7b
}
ctx->hinfo->page_number_mask = SAS7BDAT_PAGE_NUMBER_MASK;
ctx->u64 = ctx->hinfo->u64;
- ctx->compressed = (writer->compression == READSTAT_COMPRESS_ROWS);
+ if (writer->compression == READSTAT_COMPRESS_ROWS) {
+ ctx->compressed = 1;
+ ctx->compress = &sas_rle_compress;
+ ctx->compression_signature = SAS_COMPRESSION_SIGNATURE_RLE;
+ ctx->compressed_row_size_flags = SAS7BDAT_ROW_SIZE_COMPRESSED_FLAGS;
+ } else if (writer->compression == READSTAT_COMPRESS_BINARY) {
+ ctx->compressed = 1;
+ ctx->compress = &sas_rdc_compress;
+ ctx->compression_signature = SAS_COMPRESSION_SIGNATURE_RDC;
+ ctx->compressed_row_size_flags = SAS7BDAT_ROW_SIZE_RDC_COMPRESSED_FLAGS;
+ }
ctx->sig_len = ctx->u64 ? 8 : 4;
ctx->int_len = ctx->u64 ? 8 : 4;
ctx->footer_len = ctx->u64 ? 12 : 8;
@@ -1750,10 +1774,12 @@ static readstat_error_t sas7bdat_write_row_uncompressed(readstat_writer_t *write
static readstat_error_t sas7bdat_write_row_compressed(readstat_writer_t *writer, sas7bdat_write_ctx_t *ctx) {
readstat_error_t retval = READSTAT_OK;
size_t len = ctx->row_length;
- size_t compressed_len = sas_rle_compressed_len(ctx->phys_row, len);
+ ssize_t compressed_len = ctx->compress(NULL, 0, ctx->phys_row, len);
+ if (compressed_len < 0)
+ return READSTAT_ERROR_ROW_WIDTH_MISMATCH;
sas7bdat_subheader_t *subheader = NULL;
- if (compressed_len < len) {
+ if ((size_t)compressed_len < len) {
subheader = sas7bdat_subheader_init(0, compressed_len, 0, 1);
if (subheader == NULL) {
retval = READSTAT_ERROR_MALLOC;
@@ -1761,7 +1787,7 @@ static readstat_error_t sas7bdat_write_row_compressed(readstat_writer_t *writer,
}
subheader->is_row_data = 1;
subheader->comp = SAS_COMPRESSION_ROW;
- size_t actual_len = sas_rle_compress(subheader->data, subheader->len, ctx->phys_row, len);
+ ssize_t actual_len = ctx->compress(subheader->data, subheader->len, ctx->phys_row, len);
if (actual_len != compressed_len) {
retval = READSTAT_ERROR_ROW_WIDTH_MISMATCH;
goto cleanup;
@@ -1802,7 +1828,8 @@ static readstat_error_t sas7bdat_metadata_ok(void *writer_ctx) {
readstat_writer_t *writer = (readstat_writer_t *)writer_ctx;
if (writer->compression != READSTAT_COMPRESS_NONE &&
- writer->compression != READSTAT_COMPRESS_ROWS)
+ writer->compression != READSTAT_COMPRESS_ROWS &&
+ writer->compression != READSTAT_COMPRESS_BINARY)
return READSTAT_ERROR_UNSUPPORTED_COMPRESSION;
return sas_validate_unique_names(writer);
diff --git a/src/sas/readstat_sas_rdc.c b/src/sas/readstat_sas_rdc.c
new file mode 100644
index 00000000..e82b4bb7
--- /dev/null
+++ b/src/sas/readstat_sas_rdc.c
@@ -0,0 +1,230 @@
+
+#include
+#include
+#include
+
+#if defined(_MSC_VER)
+#include
+typedef SSIZE_T ssize_t;
+#endif
+
+#include "readstat_sas_rdc.h"
+
+/* Command layout (after a 16-bit big-endian control word; a set bit
+ * means "command", a clear bit means "literal byte"):
+ *
+ * marker 0x0N, B -> insert (3 + N) copies of B (3..18)
+ * marker 0x1N, H, B -> insert (19 + N + 16*H) copies of B (19..4114)
+ * marker 0x2N, H, L -> copy (16 + L) bytes from offset (3 + N + 16*H)
+ * marker 0xCN, H (C >= 3) -> copy C bytes from offset (3 + N + 16*H)
+ */
+
+#define RDC_MIN_RUN 3
+#define RDC_MAX_SHORT_RUN 18
+#define RDC_MAX_LONG_RUN 4114
+#define RDC_MIN_COPY 3
+#define RDC_MAX_SHORT_COPY 15
+#define RDC_MAX_LONG_COPY 271
+#define RDC_MIN_OFFSET 3
+#define RDC_MAX_OFFSET 4098
+
+#define RDC_HASH_BITS 12
+#define RDC_HASH_SIZE (1 << RDC_HASH_BITS)
+
+ssize_t sas_rdc_decompressed_len(const void *input_buf, size_t input_len) {
+ return sas_rdc_decompress(NULL, 0, input_buf, input_len);
+}
+
+ssize_t sas_rdc_decompress(void *output_buf, size_t output_len,
+ const void *input_buf, size_t input_len) {
+ const unsigned char *input = (const unsigned char *)input_buf;
+ const unsigned char *input_end = input + input_len;
+ unsigned char *buffer = (unsigned char *)output_buf;
+ size_t out = 0;
+
+ while (input + 2 <= input_end) {
+ int i;
+ unsigned short prefix = (input[0] << 8) + input[1];
+ input += 2;
+ for (i=0; i<16; i++) {
+ if ((prefix & (1 << (15 - i))) == 0) {
+ if (input + 1 > input_end)
+ break;
+ if (buffer) {
+ if (out + 1 > output_len)
+ return -1;
+ buffer[out] = *input;
+ }
+ input++;
+ out++;
+ continue;
+ }
+
+ if (input + 2 > input_end)
+ return -1;
+
+ unsigned char marker_byte = *input++;
+ unsigned char next_byte = *input++;
+ size_t insert_len = 0, copy_len = 0;
+ unsigned char insert_byte = 0x00;
+ size_t back_offset = 0;
+
+ if (marker_byte <= 0x0F) {
+ insert_len = 3 + marker_byte;
+ insert_byte = next_byte;
+ } else if ((marker_byte >> 4) == 1) {
+ if (input + 1 > input_end)
+ return -1;
+ insert_len = 19 + (marker_byte & 0x0F) + next_byte * 16;
+ insert_byte = *input++;
+ } else if ((marker_byte >> 4) == 2) {
+ if (input + 1 > input_end)
+ return -1;
+ copy_len = 16 + (*input++);
+ back_offset = 3 + (marker_byte & 0x0F) + next_byte * 16;
+ } else {
+ copy_len = (marker_byte >> 4);
+ back_offset = 3 + (marker_byte & 0x0F) + next_byte * 16;
+ }
+
+ if (insert_len) {
+ if (buffer) {
+ if (out + insert_len > output_len)
+ return -1;
+ memset(&buffer[out], insert_byte, insert_len);
+ }
+ out += insert_len;
+ } else if (copy_len) {
+ if (out < back_offset || copy_len > back_offset)
+ return -1;
+ if (buffer) {
+ if (out + copy_len > output_len)
+ return -1;
+ memcpy(&buffer[out], &buffer[out - back_offset], copy_len);
+ }
+ out += copy_len;
+ }
+ }
+ }
+
+ return out;
+}
+
+ssize_t sas_rdc_compressed_len(const void *bytes, size_t len) {
+ return sas_rdc_compress(NULL, 0, bytes, len);
+}
+
+static size_t rdc_hash(const unsigned char *p) {
+ uint32_t v = ((uint32_t)p[0] << 16) | ((uint32_t)p[1] << 8) | p[2];
+ return (v * 2654435761u) >> (32 - RDC_HASH_BITS);
+}
+
+ssize_t sas_rdc_compress(void *output_buf, size_t output_len,
+ const void *input_buf, size_t input_len) {
+ const unsigned char *input = (const unsigned char *)input_buf;
+ unsigned char *output = (unsigned char *)output_buf;
+ size_t hash_table[RDC_HASH_SIZE];
+ size_t in = 0, out = 0;
+ size_t ctrl_pos = 0;
+ int ctrl_bits = 16;
+ unsigned short ctrl = 0;
+ int i;
+
+ for (i=0; i= output_len) return -1; \
+ output[out] = (unsigned char)(b); \
+ } \
+ out++; \
+ } while (0)
+
+#define RDC_FLUSH_CTRL() do { \
+ if (output) { \
+ output[ctrl_pos] = (unsigned char)(ctrl >> 8); \
+ output[ctrl_pos+1] = (unsigned char)(ctrl & 0xFF); \
+ } \
+ } while (0)
+
+ while (in < input_len) {
+ if (ctrl_bits == 16) {
+ if (out > 0)
+ RDC_FLUSH_CTRL();
+ ctrl_pos = out;
+ ctrl = 0;
+ ctrl_bits = 0;
+ RDC_EMIT(0);
+ RDC_EMIT(0);
+ }
+
+ /* Run of identical bytes */
+ size_t run_len = 1;
+ while (in + run_len < input_len && run_len < RDC_MAX_LONG_RUN &&
+ input[in + run_len] == input[in]) {
+ run_len++;
+ }
+
+ /* Back-reference (never overlapping, as the reader requires) */
+ size_t copy_len = 0, back_offset = 0;
+ if (in + RDC_MIN_COPY <= input_len) {
+ size_t h = rdc_hash(&input[in]);
+ size_t cand = hash_table[h];
+ hash_table[h] = in;
+ if (cand != SIZE_MAX && in - cand >= RDC_MIN_OFFSET && in - cand <= RDC_MAX_OFFSET) {
+ size_t offset = in - cand;
+ size_t max_len = offset;
+ if (max_len > RDC_MAX_LONG_COPY)
+ max_len = RDC_MAX_LONG_COPY;
+ if (max_len > input_len - in)
+ max_len = input_len - in;
+ size_t len = 0;
+ while (len < max_len && input[cand + len] == input[in + len])
+ len++;
+ if (len >= RDC_MIN_COPY) {
+ copy_len = len;
+ back_offset = offset;
+ }
+ }
+ }
+
+ if (run_len >= RDC_MIN_RUN && run_len >= copy_len) {
+ ctrl |= (1 << (15 - ctrl_bits));
+ if (run_len <= RDC_MAX_SHORT_RUN) {
+ RDC_EMIT(run_len - 3);
+ RDC_EMIT(input[in]);
+ } else {
+ size_t n = run_len - 19;
+ RDC_EMIT(0x10 | (n & 0x0F));
+ RDC_EMIT(n >> 4);
+ RDC_EMIT(input[in]);
+ }
+ in += run_len;
+ } else if (copy_len) {
+ size_t n = back_offset - 3;
+ ctrl |= (1 << (15 - ctrl_bits));
+ if (copy_len <= RDC_MAX_SHORT_COPY) {
+ RDC_EMIT((copy_len << 4) | (n & 0x0F));
+ RDC_EMIT(n >> 4);
+ } else {
+ RDC_EMIT(0x20 | (n & 0x0F));
+ RDC_EMIT(n >> 4);
+ RDC_EMIT(copy_len - 16);
+ }
+ in += copy_len;
+ } else {
+ RDC_EMIT(input[in]);
+ in++;
+ }
+ ctrl_bits++;
+ }
+
+ if (out > 0)
+ RDC_FLUSH_CTRL();
+
+#undef RDC_EMIT
+#undef RDC_FLUSH_CTRL
+
+ return out;
+}
diff --git a/src/sas/readstat_sas_rdc.h b/src/sas/readstat_sas_rdc.h
new file mode 100644
index 00000000..3de19cb4
--- /dev/null
+++ b/src/sas/readstat_sas_rdc.h
@@ -0,0 +1,21 @@
+#ifndef READSTAT_SAS_RDC_H
+#define READSTAT_SAS_RDC_H
+
+/* Ross Data Compression (RDC), a.k.a. COMPRESS=BINARY in SAS.
+ *
+ * All functions return the number of bytes produced, or -1 on error.
+ * Passing a NULL output buffer computes the length without writing.
+ */
+
+#include
+
+ssize_t sas_rdc_decompress(void *output_buf, size_t output_len,
+ const void *input_buf, size_t input_len);
+ssize_t sas_rdc_compress(void *output_buf, size_t output_len,
+ const void *input_buf, size_t input_len);
+
+ssize_t sas_rdc_decompressed_len(const void *input_buf, size_t input_len);
+ssize_t sas_rdc_compressed_len(const void *bytes, size_t len);
+
+#endif
+
diff --git a/src/test/test_list.h b/src/test/test_list.h
index 24ad793e..0c466509 100644
--- a/src/test/test_list.h
+++ b/src/test/test_list.h
@@ -215,8 +215,8 @@ static rt_test_group_t _test_groups[] = {
},
{
- .label = "SAS7BDAT RLE compression",
- .test_formats = RT_FORMAT_SAS7BDAT_COMP_ROWS,
+ .label = "SAS7BDAT RLE and RDC compression",
+ .test_formats = RT_FORMAT_SAS7BDAT_COMP,
.rows = 10,
.columns = {
{
diff --git a/src/test/test_read.c b/src/test/test_read.c
index 5ccfad56..f16ffcf4 100644
--- a/src/test/test_read.c
+++ b/src/test/test_read.c
@@ -47,10 +47,14 @@ char *file_extension(long format) {
return "sas7bdat32";
if (format == RT_FORMAT_SAS7BDAT_32BIT_COMP_ROWS)
return "sas7bdat32row";
+ if (format == RT_FORMAT_SAS7BDAT_32BIT_COMP_BINARY)
+ return "sas7bdat32bin";
if (format == RT_FORMAT_SAS7BDAT_64BIT_COMP_NONE)
return "sas7bdat64";
if (format == RT_FORMAT_SAS7BDAT_64BIT_COMP_ROWS)
return "sas7bdat64row";
+ if (format == RT_FORMAT_SAS7BDAT_64BIT_COMP_BINARY)
+ return "sas7bdat64bin";
if (format == RT_FORMAT_XPORT_5)
return "xpt5";
if (format == RT_FORMAT_XPORT_8)
diff --git a/src/test/test_readstat.h b/src/test/test_readstat.h
index e985b19a..35bfa59d 100644
--- a/src/test/test_readstat.h
+++ b/src/test/test_readstat.h
@@ -39,23 +39,29 @@
#define RT_FORMAT_SAS7BDAT_32BIT_COMP_NONE 0x010000
#define RT_FORMAT_SAS7BDAT_32BIT_COMP_ROWS 0x020000
-#define RT_FORMAT_SAS7BDAT_32BIT (RT_FORMAT_SAS7BDAT_32BIT_COMP_NONE | RT_FORMAT_SAS7BDAT_32BIT_COMP_ROWS)
+#define RT_FORMAT_SAS7BDAT_32BIT_COMP_BINARY 0x040000
+#define RT_FORMAT_SAS7BDAT_32BIT (RT_FORMAT_SAS7BDAT_32BIT_COMP_NONE | RT_FORMAT_SAS7BDAT_32BIT_COMP_ROWS | \
+ RT_FORMAT_SAS7BDAT_32BIT_COMP_BINARY)
-#define RT_FORMAT_SAS7BDAT_64BIT_COMP_NONE 0x040000
-#define RT_FORMAT_SAS7BDAT_64BIT_COMP_ROWS 0x080000
-#define RT_FORMAT_SAS7BDAT_64BIT (RT_FORMAT_SAS7BDAT_64BIT_COMP_NONE | RT_FORMAT_SAS7BDAT_64BIT_COMP_ROWS)
+#define RT_FORMAT_SAS7BDAT_64BIT_COMP_NONE 0x080000
+#define RT_FORMAT_SAS7BDAT_64BIT_COMP_ROWS 0x100000
+#define RT_FORMAT_SAS7BDAT_64BIT_COMP_BINARY 0x200000
+#define RT_FORMAT_SAS7BDAT_64BIT (RT_FORMAT_SAS7BDAT_64BIT_COMP_NONE | RT_FORMAT_SAS7BDAT_64BIT_COMP_ROWS | \
+ RT_FORMAT_SAS7BDAT_64BIT_COMP_BINARY)
#define RT_FORMAT_SAS7BDAT_COMP_NONE (RT_FORMAT_SAS7BDAT_32BIT_COMP_NONE | RT_FORMAT_SAS7BDAT_64BIT_COMP_NONE)
#define RT_FORMAT_SAS7BDAT_COMP_ROWS (RT_FORMAT_SAS7BDAT_32BIT_COMP_ROWS | RT_FORMAT_SAS7BDAT_64BIT_COMP_ROWS)
+#define RT_FORMAT_SAS7BDAT_COMP_BINARY (RT_FORMAT_SAS7BDAT_32BIT_COMP_BINARY | RT_FORMAT_SAS7BDAT_64BIT_COMP_BINARY)
+#define RT_FORMAT_SAS7BDAT_COMP (RT_FORMAT_SAS7BDAT_COMP_ROWS | RT_FORMAT_SAS7BDAT_COMP_BINARY)
#define RT_FORMAT_SAS7BDAT (RT_FORMAT_SAS7BDAT_32BIT | RT_FORMAT_SAS7BDAT_64BIT)
-#define RT_FORMAT_SAS7BCAT_32BIT 0x100000
-#define RT_FORMAT_SAS7BCAT_64BIT 0x200000
+#define RT_FORMAT_SAS7BCAT_32BIT 0x400000
+#define RT_FORMAT_SAS7BCAT_64BIT 0x800000
#define RT_FORMAT_SAS7BCAT (RT_FORMAT_SAS7BCAT_32BIT | RT_FORMAT_SAS7BCAT_64BIT)
-#define RT_FORMAT_XPORT_5 0x400000
-#define RT_FORMAT_XPORT_8 0x800000
+#define RT_FORMAT_XPORT_5 0x1000000
+#define RT_FORMAT_XPORT_8 0x2000000
#define RT_FORMAT_XPORT (RT_FORMAT_XPORT_5 | RT_FORMAT_XPORT_8)
diff --git a/src/test/test_write.c b/src/test/test_write.c
index 5c909c86..fbca4400 100644
--- a/src/test/test_write.c
+++ b/src/test/test_write.c
@@ -188,6 +188,8 @@ readstat_error_t write_file_to_buffer(rt_test_file_t *file, rt_buffer_t *buffer,
} else if ((format & RT_FORMAT_SAS7BDAT)) {
if ((format & RT_FORMAT_SAS7BDAT_COMP_ROWS)) {
readstat_writer_set_compression(writer, READSTAT_COMPRESS_ROWS);
+ } else if ((format & RT_FORMAT_SAS7BDAT_COMP_BINARY)) {
+ readstat_writer_set_compression(writer, READSTAT_COMPRESS_BINARY);
}
readstat_writer_set_file_format_version(writer, sas_file_format_version(format));
readstat_writer_set_file_format_is_64bit(writer, !!(format & RT_FORMAT_SAS7BDAT_64BIT));