Skip to content

RDC compression for sas7bdat writes #433

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
evanmiller merged 1 commit into from
Nov 11, 2026
Merged

RDC compression for sas7bdat writes #433

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions Makefile.am
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ libreadstat_la_SOURCES = \
src/sas/readstat_sas7bdat_read.c \
src/sas/readstat_sas7bdat_write.c \
src/sas/readstat_sas_rle.c \
src/sas/readstat_sas_rdc.c \
src/sas/readstat_xport.c \
src/sas/readstat_xport_read.c \
src/sas/readstat_xport_write.c \
Expand Down Expand Up @@ -111,6 +112,7 @@ noinst_HEADERS = \
src/sas/ieee.h \
src/sas/readstat_sas.h \
src/sas/readstat_sas_rle.h \
src/sas/readstat_sas_rdc.h \
src/sas/readstat_xport.h \
src/sas/readstat_xport_parse_format.h \
src/spss/readstat_por.h \
Expand Down
2 changes: 2 additions & 0 deletions VS17/ReadStat.vcxproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -203,6 +203,7 @@
<ClCompile Include="..\src\sas\readstat_sas7bdat_read.c" />
<ClCompile Include="..\src\sas\readstat_sas7bdat_write.c" />
<ClCompile Include="..\src\sas\readstat_sas_rle.c" />
<ClCompile Include="..\src\sas\readstat_sas_rdc.c" />
<ClCompile Include="..\src\sas\readstat_xport.c" />
<ClCompile Include="..\src\sas\readstat_xport_read.c" />
<ClCompile Include="..\src\sas\readstat_xport_write.c" />
Expand Down Expand Up @@ -247,6 +248,7 @@
<ClInclude Include="..\src\sas\ieee.h" />
<ClInclude Include="..\src\sas\readstat_sas.h" />
<ClInclude Include="..\src\sas\readstat_sas_rle.h" />
<ClInclude Include="..\src\sas\readstat_sas_rdc.h" />
<ClInclude Include="..\src\sas\readstat_xport.h" />
<ClInclude Include="..\src\sas\readstat_xport_parse_format.h" />
<ClInclude Include="..\src\spss\readstat_por.h" />
Expand Down
6 changes: 6 additions & 0 deletions VS17/ReadStat.vcxproj.filters
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,9 @@
<ClCompile Include="..\src\sas\readstat_sas_rle.c">
<Filter>Source Files\sas</Filter>
</ClCompile>
<ClCompile Include="..\src\sas\readstat_sas_rdc.c">
<Filter>Source Files\sas</Filter>
</ClCompile>
<ClCompile Include="..\src\sas\readstat_sas7bcat_read.c">
<Filter>Source Files\sas</Filter>
</ClCompile>
Expand Down Expand Up @@ -206,6 +209,9 @@
<ClInclude Include="..\src\sas\readstat_sas_rle.h">
<Filter>Header Files</Filter>
</ClInclude>
<ClInclude Include="..\src\sas\readstat_sas_rdc.h">
<Filter>Header Files</Filter>
</ClInclude>
<ClInclude Include="..\src\sas\readstat_xport.h">
<Filter>Header Files</Filter>
</ClInclude>
Expand Down
2 changes: 1 addition & 1 deletion src/readstat.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -616,7 +616,7 @@ readstat_error_t readstat_writer_set_file_format_is_64bit(readstat_writer_t *wri
int is_64bit); // applies only to SAS files; defaults to 1=true
readstat_error_t readstat_writer_set_compression(readstat_writer_t *writer,
readstat_compress_t compression);
// READSTAT_COMPRESS_BINARY is supported only with SAV files (i.e. ZSAV files)
// READSTAT_COMPRESS_BINARY is supported only with sas7bdat and SAV files (i.e. ZSAV files)
// READSTAT_COMPRESS_ROWS is supported only with sas7bdat and SAV files

// Optional error handler
Expand Down
91 changes: 16 additions & 75 deletions src/sas/readstat_sas7bdat_read.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
#include <limits.h>
#include "readstat_sas.h"
#include "readstat_sas_rle.h"
#include "readstat_sas_rdc.h"
#include "../readstat_iconv.h"
#include "../readstat_convert.h"
#include "../readstat_malloc.h"
Expand Down Expand Up @@ -573,88 +574,28 @@ static readstat_error_t sas7bdat_parse_rows(const char *data, size_t len,
}

static readstat_error_t sas7bdat_parse_subheader_rdc(const char *subheader, size_t len, sas7bdat_ctx_t *ctx) {
readstat_error_t retval = READSTAT_OK;
const unsigned char *input = (const unsigned char *)subheader;
char *buffer = ctx->row;
if (buffer == NULL) return READSTAT_ERROR_PARSE;
char *output = buffer;
while (input + 2 <= (const unsigned char *)subheader + len) {
int i;
unsigned short prefix = (input[0] << 8) + input[1];
input += 2;
for (i=0; i<16; i++) {
if ((prefix & (1 << (15 - i))) == 0) {
if (input + 1 > (const unsigned char *)subheader + len) {
break;
}
if (output + 1 > buffer + ctx->row_length) {
retval = READSTAT_ERROR_ROW_WIDTH_MISMATCH;
goto cleanup;
}
*output++ = *input++;
continue;
}
if (ctx->row_limit == ctx->parsed_row_count)
return READSTAT_OK;

if (input + 2 > (const unsigned char *)subheader + len) {
retval = READSTAT_ERROR_PARSE;
goto cleanup;
}
readstat_error_t retval = READSTAT_OK;
ssize_t bytes_decompressed = 0;

unsigned char marker_byte = *input++;
unsigned char next_byte = *input++;
size_t insert_len = 0, copy_len = 0;
unsigned char insert_byte = 0x00;
size_t back_offset = 0;

if (marker_byte <= 0x0F) {
insert_len = 3 + marker_byte;
insert_byte = next_byte;
} else if ((marker_byte >> 4) == 1) {
if (input + 1 > (const unsigned char *)subheader + len) {
retval = READSTAT_ERROR_PARSE;
goto cleanup;
}
insert_len = 19 + (marker_byte & 0x0F) + next_byte * 16;
insert_byte = *input++;
} else if ((marker_byte >> 4) == 2) {
if (input + 1 > (const unsigned char *)subheader + len) {
retval = READSTAT_ERROR_PARSE;
goto cleanup;
}
copy_len = 16 + (*input++);
back_offset = 3 + (marker_byte & 0x0F) + next_byte * 16;
} else {
copy_len = (marker_byte >> 4);
back_offset = 3 + (marker_byte & 0x0F) + next_byte * 16;
}
if (ctx->row == NULL)
return READSTAT_ERROR_PARSE;

if (insert_len) {
if (output + insert_len > buffer + ctx->row_length) {
retval = READSTAT_ERROR_ROW_WIDTH_MISMATCH;
goto cleanup;
}
memset(output, insert_byte, insert_len);
output += insert_len;
} else if (copy_len) {
if (output - buffer < back_offset || copy_len > back_offset) {
retval = READSTAT_ERROR_PARSE;
goto cleanup;
}
if (output + copy_len > buffer + ctx->row_length) {
retval = READSTAT_ERROR_ROW_WIDTH_MISMATCH;
goto cleanup;
}
memcpy(output, output - back_offset, copy_len);
output += copy_len;
}
}
}
bytes_decompressed = sas_rdc_decompress(ctx->row, ctx->row_length, subheader, len);

if (output - buffer != ctx->row_length) {
if (bytes_decompressed != ctx->row_length) {
retval = READSTAT_ERROR_ROW_WIDTH_MISMATCH;
if (ctx->handle.error) {
snprintf(ctx->error_buf, sizeof(ctx->error_buf),
"ReadStat: Row #%d decompressed to %ld bytes (expected %d bytes)",
ctx->parsed_row_count, (long)(bytes_decompressed), ctx->row_length);
ctx->handle.error(ctx->error_buf, ctx->user_ctx);
}
goto cleanup;
}
retval = sas7bdat_parse_single_row(buffer, ctx);
retval = sas7bdat_parse_single_row(ctx->row, ctx);
cleanup:
return retval;
}
Expand Down
Loading
Loading