Skip to content

Replace sas7bdat mix page padding hack #434

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
hpoettker wants to merge 1 commit into
base: dev
Choose a base branch
from
+11 −51
Open

Replace sas7bdat mix page padding hack #434

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 2 additions & 30 deletions src/sas/readstat_sas.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -318,14 +318,8 @@ readstat_error_t sas_read_header(readstat_io_t *io, sas_header_info_t *hinfo,
retval = READSTAT_ERROR_READ;
goto cleanup;
}
char major, revision_tag, revision;
int minor;
/* The release field is not NUL-terminated in the file; copy it into a
* terminated buffer before handing it to sscanf, which calls strlen on it. */
char release[sizeof(header_end.release)+1];
memcpy(release, header_end.release, sizeof(header_end.release));
release[sizeof(header_end.release)] = '\0';
if (sscanf(release, "%c.%04d%c%c", &major, &minor, &revision_tag, &revision) != 4) {
char major = header_end.release[0];
if (header_end.release[1] != '.') {
retval = READSTAT_ERROR_PARSE;
goto cleanup;
}
Expand All @@ -340,29 +334,7 @@ readstat_error_t sas_read_header(readstat_io_t *io, sas_header_info_t *hinfo,
retval = READSTAT_ERROR_PARSE;
goto cleanup;
}
// revision_tag is usually M, but J has been observed in the wild (not created with SAS?)
// files have also been observed with TK in place of the revision details.
if ((major == '8' || major == '9') && revision_tag != 'M' && revision_tag != 'J' && revision_tag != 'T') {
retval = READSTAT_ERROR_PARSE;
goto cleanup;
}
hinfo->minor_version = minor;
if (revision >= '0' && revision <= '9') {
hinfo->revision = revision - '0';
} else if (revision_tag == 'T' && revision == 'K') {
// files have been observed with TK in place of the revision details.
hinfo->revision = 0;
} else {
retval = READSTAT_ERROR_PARSE;
goto cleanup;
}

if ((major == '8' || major == '9') && minor == 0 && hinfo->revision == 0) {
/* A bit of a hack, but most SAS installations are running a minor update */
hinfo->vendor = READSTAT_VENDOR_STAT_TRANSFER;
} else {
hinfo->vendor = READSTAT_VENDOR_SAS;
}
if (io->seek(hinfo->header_size, READSTAT_SEEK_SET, io->io_ctx) == -1) {
retval = READSTAT_ERROR_SEEK;
if (error_handler) {
Expand Down
8 changes: 0 additions & 8 deletions src/sas/readstat_sas.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,10 +36,7 @@ typedef struct sas_header_end_s {
typedef struct sas_header_info_s {
int little_endian;
int u64;
int vendor;
int major_version;
int minor_version;
int revision;
int pad1;
int64_t page_size;
int64_t page_header_size;
Expand All @@ -54,11 +51,6 @@ typedef struct sas_header_info_s {
char *encoding;
} sas_header_info_t;

enum {
READSTAT_VENDOR_STAT_TRANSFER,
READSTAT_VENDOR_SAS
};

typedef struct sas_text_ref_s {
uint16_t index;
uint16_t offset;
Expand Down
22 changes: 9 additions & 13 deletions src/sas/readstat_sas7bdat_read.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,6 @@ typedef struct sas7bdat_ctx_s {

int little_endian;
int u64;
int vendor;
void *user_ctx;
readstat_io_t *io;
int bswap;
Expand Down Expand Up @@ -70,6 +69,7 @@ typedef struct sas7bdat_ctx_s {
uint64_t page_header_size;
uint64_t subheader_signature_size;
uint64_t subheader_pointer_size;
int mix_page_padding;

int text_blob_count;
size_t *text_blob_lengths;
Expand Down Expand Up @@ -309,6 +309,8 @@ static readstat_error_t sas7bdat_parse_row_size_subheader(const char *subheader,
if (ctx->row_limit == 0 || total_row_count_after_skipping < ctx->row_limit)
ctx->row_limit = total_row_count_after_skipping;

ctx->mix_page_padding = (unsigned char)subheader[len-13] - 1;

cleanup:
return retval;
}
Expand Down Expand Up @@ -1224,18 +1226,12 @@ static readstat_error_t sas7bdat_parse_page_pass2(const char *page, size_t page_
}

if ((page_type & SAS_PAGE_TYPE_MASK) == SAS_PAGE_TYPE_MIX) {
/* HACK - this is supposed to obey 8-byte boundaries but
* some files created by Stat/Transfer don't. So verify that the
* padding is { 0, 0, 0, 0 } or { ' ', ' ', ' ', ' ' } (or that
* the file is not from Stat/Transfer) before skipping it */
if ((shp-page)%8 == 4 && shp + 4 <= page + page_size &&
(*(uint32_t *)shp == 0x00000000 ||
*(uint32_t *)shp == 0x20202020 ||
ctx->vendor != READSTAT_VENDOR_STAT_TRANSFER)) {
data = shp + 4;
} else {
data = shp;
if ((ctx->mix_page_padding != 0 && ctx->mix_page_padding != 4) ||
shp + ctx->mix_page_padding > page + page_size) {
retval = READSTAT_ERROR_PARSE;
goto cleanup;
}
data = shp + ctx->mix_page_padding;
}
}
if (data) {
Expand Down Expand Up @@ -1438,6 +1434,7 @@ readstat_error_t readstat_parse_sas7bdat(readstat_parser_t *parser, const char *
ctx->user_ctx = user_ctx;
ctx->io = parser->io;
ctx->row_limit = parser->row_limit;
ctx->mix_page_padding = -1;
ctx->moved_page_index = -1;
if (parser->row_offset > 0)
ctx->row_offset = parser->row_offset;
Expand Down Expand Up @@ -1471,7 +1468,6 @@ readstat_error_t readstat_parse_sas7bdat(readstat_parser_t *parser, const char *

ctx->u64 = hinfo->u64;
ctx->little_endian = hinfo->little_endian;
ctx->vendor = hinfo->vendor;
ctx->bswap = machine_is_little_endian() ^ hinfo->little_endian;
ctx->header_size = hinfo->header_size;
ctx->page_count = hinfo->page_count;
Expand Down
Loading