From 80df8eb80a54bcae94b99f94b956c59af692a854 Mon Sep 17 00:00:00 2001 From: "Allen, Timothy" Date: Mon, 2 Nov 2026 13:58:11 -0500 Subject: [PATCH] Support converting CSV with JSON metadata to sas7bdat Add a read_csv module for sas7bdat output, alongside the DTA and SAV ones. Numeric columns are written as doubles: DATE as days since 1960-01-01 (DATE9.), DATE_TIME as seconds since 1960 without leap seconds (DATETIME19.), TIME as seconds since midnight (TIME8.), and other numbers unscaled with Fw.d or BEST12. Values listed under "missing" become SAS special missing values .A, .B, ... Value labels are not written, since a sas7bdat file keeps them in a separate SAS catalog; the CLI warns instead. For SAS output, read_csv passes each variable's format as its value-label name, as the SAS readers do, rather than the column name, which the writer would otherwise take as the column's format. readstat_parse_csv no longer crashes when the output format is not supported: it called csv_free() on a parser csv_init() had not set up. Document the SAS metadata in variablemetadata_schema.json and the README, and add a round-trip test that runs when the CSV reader is built. --- .gitignore | 3 + Makefile.am | 18 +- README.md | 4 +- VS17/ReadStat_App/ReadStat_App.vcxproj | 2 + .../ReadStat_App/ReadStat_App.vcxproj.filters | 6 + resources/csv_to_sas7bdat.csv | 5 + resources/csv_to_sas7bdat.expected | 40 +++ resources/csv_to_sas7bdat.json | 11 + src/bin/read_csv/mod_sas7bdat.c | 272 ++++++++++++++++++ src/bin/read_csv/mod_sas7bdat.h | 1 + src/bin/read_csv/read_csv.c | 14 +- src/test/dump_sas7bdat.c | 50 ++++ src/test/test_csv_to_sas7bdat.sh | 24 ++ variablemetadata_schema.json | 189 ++++++++++++ 14 files changed, 633 insertions(+), 6 deletions(-) create mode 100644 resources/csv_to_sas7bdat.csv create mode 100644 resources/csv_to_sas7bdat.expected create mode 100644 resources/csv_to_sas7bdat.json create mode 100644 src/bin/read_csv/mod_sas7bdat.c create mode 100644 src/bin/read_csv/mod_sas7bdat.h create mode 100644 src/test/dump_sas7bdat.c create mode 100755 src/test/test_csv_to_sas7bdat.sh diff --git a/.gitignore b/.gitignore index f4ca3daf..958759f8 100644 --- a/.gitignore +++ b/.gitignore @@ -45,9 +45,12 @@ gen.json extract_metadata test_sav_date test_double_decimals +dump_sas7bdat generate_corpus *.csv *.json +!resources/csv_to_sas7bdat.csv +!resources/csv_to_sas7bdat.json *.py *.tar.gz *.zip diff --git a/Makefile.am b/Makefile.am index 9bdcd196..3613cbe5 100644 --- a/Makefile.am +++ b/Makefile.am @@ -95,7 +95,11 @@ EXTRA_DIST = \ resources/compression_type_0x0d.sas7bdat \ resources/format_with_default_32bit.sas7bcat \ resources/format_with_default_64bit.sas7bcat \ - resources/sav_missing_continuations.sav + resources/sav_missing_continuations.sav \ + resources/csv_to_sas7bdat.csv \ + resources/csv_to_sas7bdat.json \ + resources/csv_to_sas7bdat.expected \ + src/test/test_csv_to_sas7bdat.sh include_HEADERS = src/readstat.h @@ -137,6 +141,7 @@ noinst_HEADERS = \ src/bin/read_csv/mod_csv.h \ src/bin/read_csv/mod_dta.h \ src/bin/read_csv/mod_sav.h \ + src/bin/read_csv/mod_sas7bdat.h \ src/bin/read_csv/read_csv.h \ src/bin/read_csv/read_module.h \ src/bin/read_csv/value.h \ @@ -177,6 +182,7 @@ readstat_SOURCES = \ src/bin/read_csv/mod_csv.c \ src/bin/read_csv/mod_dta.c \ src/bin/read_csv/mod_sav.c \ + src/bin/read_csv/mod_sas7bdat.c \ src/bin/read_csv/value.c \ src/bin/write/double_decimals.c \ src/bin/write/mod_csv.c \ @@ -260,6 +266,16 @@ test_double_decimals_CFLAGS = -g -Wall @EXTRA_WARNINGS@ -Werror -pedantic-errors TESTS = test_readstat test_dta_days test_sav_date test_double_decimals +if HAVE_CSVREADER +check_PROGRAMS += dump_sas7bdat + +dump_sas7bdat_SOURCES = src/test/dump_sas7bdat.c +dump_sas7bdat_LDADD = libreadstat.la +dump_sas7bdat_CFLAGS = -Wall @EXTRA_WARNINGS@ -Werror -pedantic-errors -std=c99 + +TESTS += src/test/test_csv_to_sas7bdat.sh +endif + EXTRA_PROGRAMS = \ generate_corpus diff --git a/README.md b/README.md index 2471dd69..49a44518 100644 --- a/README.md +++ b/README.md @@ -138,7 +138,9 @@ CSV input is supported together with a metadata file describing the data: readstat -The `` should end with `.dta`, `.sav`, or `.csv`. +The `` should end with `.dta`, `.sav`, `.sas7bdat`, or `.csv`. +A `.sas7bdat` file cannot hold value labels, which SAS keeps in a separate +catalog, so any `categories` in the metadata are not written to it. The `` is a regular CSV file. diff --git a/VS17/ReadStat_App/ReadStat_App.vcxproj b/VS17/ReadStat_App/ReadStat_App.vcxproj index c8ea81cc..39076b86 100644 --- a/VS17/ReadStat_App/ReadStat_App.vcxproj +++ b/VS17/ReadStat_App/ReadStat_App.vcxproj @@ -25,6 +25,7 @@ + @@ -43,6 +44,7 @@ + diff --git a/VS17/ReadStat_App/ReadStat_App.vcxproj.filters b/VS17/ReadStat_App/ReadStat_App.vcxproj.filters index 155fe62f..9ed86167 100644 --- a/VS17/ReadStat_App/ReadStat_App.vcxproj.filters +++ b/VS17/ReadStat_App/ReadStat_App.vcxproj.filters @@ -45,6 +45,9 @@ Source Files + + Source Files + Source Files @@ -95,6 +98,9 @@ Header Files + + Header Files + Header Files diff --git a/resources/csv_to_sas7bdat.csv b/resources/csv_to_sas7bdat.csv new file mode 100644 index 00000000..1702404d --- /dev/null +++ b/resources/csv_to_sas7bdat.csv @@ -0,0 +1,5 @@ +id,amount,pct,score,born,stamp,clock,name +1,12.5,25,3,2020-02-29,2020-02-29 13:45:30.250,08:15:00,Zoë +2,,50,-9,1959-12-31,1960-01-01T00:00:01,23:59:59.5, +3,-0.75,,999,,2016-12-31 23:59:59,,"a, b" +4,1e3,100,7,1960-01-01,,00:00:00,plain diff --git a/resources/csv_to_sas7bdat.expected b/resources/csv_to_sas7bdat.expected new file mode 100644 index 00000000..5c82a046 --- /dev/null +++ b/resources/csv_to_sas7bdat.expected @@ -0,0 +1,40 @@ +variable id format=BEST12. label=Identifier +variable amount format=F12.2 label= +variable pct format=F11.1 label= +variable score format=BEST12. label= +variable born format=DATE9. label= +variable stamp format=DATETIME19. label= +variable clock format=TIME8. label= +variable name format= label=Name +0 id 1 +0 amount 12.5 +0 pct 25 +0 score 3 +0 born 21974 +0 stamp 1898603130.25 +0 clock 29700 +0 name "Zoë" +1 id 2 +1 amount . +1 pct 50 +1 score .A +1 born .A +1 stamp 1 +1 clock 86399.5 +1 name "" +2 id 3 +2 amount -0.75 +2 pct . +2 score .B +2 born . +2 stamp 1798847999 +2 clock . +2 name "a, b" +3 id 4 +3 amount 1000 +3 pct 100 +3 score 7 +3 born 0 +3 stamp . +3 clock 0 +3 name "plain" diff --git a/resources/csv_to_sas7bdat.json b/resources/csv_to_sas7bdat.json new file mode 100644 index 00000000..78559613 --- /dev/null +++ b/resources/csv_to_sas7bdat.json @@ -0,0 +1,11 @@ +{"type": "SAS", "variables": [ + {"type": "NUMERIC", "name": "id", "label": "Identifier"}, + {"type": "NUMERIC", "name": "amount", "format": "CURRENCY", "decimals": 2}, + {"type": "NUMERIC", "name": "pct", "format": "PERCENT", "decimals": 1}, + {"type": "NUMERIC", "name": "score", "missing": {"type": "DISCRETE", "values": [-9, 999]}, + "categories": [{"code": 3, "label": "Three"}]}, + {"type": "NUMERIC", "name": "born", "format": "DATE", "missing": {"type": "RANGE", "low": "1959-01-01", "high": "1959-12-31"}}, + {"type": "NUMERIC", "name": "stamp", "format": "DATE_TIME"}, + {"type": "NUMERIC", "name": "clock", "format": "TIME"}, + {"type": "STRING", "name": "name", "label": "Name"} +]} diff --git a/src/bin/read_csv/mod_sas7bdat.c b/src/bin/read_csv/mod_sas7bdat.c new file mode 100644 index 00000000..fa4b1438 --- /dev/null +++ b/src/bin/read_csv/mod_sas7bdat.c @@ -0,0 +1,272 @@ +#include +#include +#include + +#include "../../readstat.h" +#include "json_metadata.h" +#include "read_module.h" +#include "csv_metadata.h" +#include "value.h" +#include "../util/file_format.h" +#include "../util/readstat_dta_days.h" + +/* SAS dates count days, and SAS datetimes seconds, from 1960-01-01, the + * same epoch as Stata. Unlike Stata's %tC, SAS datetimes do not count leap + * seconds. A sas7bdat file cannot hold value labels (SAS keeps them in a + * separate format catalog), so categories are not written. */ + +/* Each special missing value takes a lo/hi pair in missing_ranges */ +#define SAS_MAX_MISSING_TAGS \ + (sizeof(((readstat_missingness_t *)0)->missing_ranges) / sizeof(readstat_value_t) / 2) + +void produce_column_header_sas7bdat(void *csv_metadata, const char *column, readstat_variable_t* var); +void produce_missingness_sas7bdat(void *csv_metadata, const char* column); +void produce_value_label_sas7bdat(void *csv_metadata, const char* column); +void produce_csv_value_sas7bdat(void *csv_metadata, const char *s, size_t len); + +rs_read_module_t rs_read_mod_sas7bdat = { + .format = RS_FORMAT_SAS_DATA, + .header = &produce_column_header_sas7bdat, + .missingness = &produce_missingness_sas7bdat, + .value_label = &produce_value_label_sas7bdat, + .csv_value = &produce_csv_value_sas7bdat }; + +static int is_time_sas7bdat(const readstat_variable_t *var) { + return strncmp(var->format, "TIME", 4) == 0; +} + +static double sas_days(const char *s) { + char *dest; + int days = readstat_dta_num_days(s, &dest); + if (dest == s) { + fprintf(stderr, "%s:%d not a valid date: %s (expected format: yyyy-mm-dd)\n", __FILE__, __LINE__, s); + exit(EXIT_FAILURE); + } + return days; +} + +static double sas_seconds_since_midnight(const char *s) { + int hour, minute; + double second = 0.0; + int matched = sscanf(s, "%d:%d:%lf", &hour, &minute, &second); + if (matched < 2) { + fprintf(stderr, "%s:%d not a valid time: %s (expected format: hh:MM:SS with optional fractional seconds)\n", __FILE__, __LINE__, s); + exit(EXIT_FAILURE); + } + return hour * 3600.0 + minute * 60.0 + second; +} + +static double sas_seconds_since_epoch(const char *s) { + int year, month, day, hour, minute; + double second = 0.0; + int matched = sscanf(s, "%d-%d-%d%*[ T]%d:%d:%lf", &year, &month, &day, &hour, &minute, &second); + if (matched < 5) { + fprintf(stderr, "%s:%d not a valid date-time: %s (expected format: yyyy-mm-dd hh:MM:SS with optional fractional seconds)\n", __FILE__, __LINE__, s); + exit(EXIT_FAILURE); + } + char date[32]; + snprintf(date, sizeof(date), "%04d-%02d-%02d", year, month, day); + return 86400.0 * sas_days(date) + hour * 3600.0 + minute * 60.0 + second; +} + +/* Converts a CSV value or a JSON missing value to what SAS stores */ +static double sas_double(const char *s, const readstat_variable_t *var, int is_date, int is_date_time) { + if (is_date) + return sas_days(s); + if (is_date_time) + return sas_seconds_since_epoch(s); + if (is_time_sas7bdat(var)) + return sas_seconds_since_midnight(s); + + char *dest; + double val = strtod(s, &dest); + if (dest == s) { + fprintf(stderr, "%s:%d not a number: %s\n", __FILE__, __LINE__, s); + exit(EXIT_FAILURE); + } + return val; +} + +static double sas_double_from_token(struct csv_metadata *c, jsmntok_t* token) { + char buf[255]; + int len = token->end - token->start; + snprintf(buf, sizeof(buf), "%.*s", len, c->json_md->js + token->start); + return sas_double(buf, &c->variables[c->columns], + c->is_date[c->columns], c->is_date_time[c->columns]); +} + +/* Values from lo to hi are written as the next special missing value: .A, .B, ... */ +static void sas_add_missing_range(readstat_variable_t* var, const char *column, double lo, double hi) { + int idx = var->missingness.missing_ranges_count; + if (idx >= (int)SAS_MAX_MISSING_TAGS) { + fprintf(stderr, "%s:%d column %s has more than %d missing value definitions\n", + __FILE__, __LINE__, column, (int)SAS_MAX_MISSING_TAGS); + exit(EXIT_FAILURE); + } + readstat_value_t value = { + .type = READSTAT_TYPE_DOUBLE, + .is_tagged_missing = 1, + .tag = 'A' + idx + }; + value.v.double_value = lo; + var->missingness.missing_ranges[(idx*2)] = value; + value.v.double_value = hi; + var->missingness.missing_ranges[(idx*2)+1] = value; + var->missingness.missing_ranges_count++; +} + +static void produce_missingness_discrete_sas7bdat(struct csv_metadata *c, jsmntok_t* missing, const char* column) { + readstat_variable_t* var = &c->variables[c->columns]; + const char *js = c->json_md->js; + + jsmntok_t* values = find_object_property(js, missing, "values"); + if (!values) { + fprintf(stderr, "%s:%d Expected to find missing 'values' property\n", __FILE__, __LINE__); + exit(EXIT_FAILURE); + } + + int j = 1; + for (int i=0; isize; i++) { + jsmntok_t* missing_value_token = values + j; + if (var->type == READSTAT_TYPE_DOUBLE) { + double v = sas_double_from_token(c, missing_value_token); + sas_add_missing_range(var, column, v, v); + } + j += slurp_object(missing_value_token); + } +} + +static void produce_missingness_range_sas7bdat(struct csv_metadata *c, jsmntok_t* missing, const char* column) { + readstat_variable_t* var = &c->variables[c->columns]; + const char *js = c->json_md->js; + + jsmntok_t* low = find_object_property(js, missing, "low"); + jsmntok_t* high = find_object_property(js, missing, "high"); + jsmntok_t* discrete = find_object_property(js, missing, "discrete-value"); + + if (low && !high) { + fprintf(stderr, "%s:%d missing.low specified for column %s, but missing.high not specified\n", __FILE__, __LINE__, column); + exit(EXIT_FAILURE); + } + if (high && !low) { + fprintf(stderr, "%s:%d missing.high specified for column %s, but missing.low not specified\n", __FILE__, __LINE__, column); + exit(EXIT_FAILURE); + } + if (low && high) { + sas_add_missing_range(var, column, sas_double_from_token(c, low), sas_double_from_token(c, high)); + } + if (discrete) { + double v = sas_double_from_token(c, discrete); + sas_add_missing_range(var, column, v, v); + } +} + +void produce_missingness_sas7bdat(void *csv_metadata, const char* column) { + struct csv_metadata *c = (struct csv_metadata *)csv_metadata; + const char *js = c->json_md->js; + readstat_variable_t* var = &c->variables[c->columns]; + var->missingness.missing_ranges_count = 0; + + jsmntok_t* missing = find_variable_property(js, c->json_md->tok, column, "missing"); + if (!missing) { + return; + } + + jsmntok_t* missing_type = find_object_property(js, missing, "type"); + if (!missing_type) { + fprintf(stderr, "%s:%d expected to find missing.type for column %s\n", __FILE__, __LINE__, column); + exit(EXIT_FAILURE); + } + + if (match_token(js, missing_type, "DISCRETE")) { + produce_missingness_discrete_sas7bdat(c, missing, column); + } else if (match_token(js, missing_type, "RANGE")) { + produce_missingness_range_sas7bdat(c, missing, column); + } else { + fprintf(stderr, "%s:%d unknown missing type %.*s\n", __FILE__, __LINE__, missing_type->end - missing_type->start, js+missing_type->start); + exit(EXIT_FAILURE); + } +} + +void produce_column_header_sas7bdat(void *csv_metadata, const char *column, readstat_variable_t* var) { + struct csv_metadata *c = (struct csv_metadata *)csv_metadata; + extract_metadata_type_t coltype = column_type(c->json_md, column, c->output_format); + if (coltype == EXTRACT_METADATA_TYPE_NUMERIC) { + /* SAS stores every number as a double */ + var->type = READSTAT_TYPE_DOUBLE; + extract_metadata_format_t colformat = column_format(c->json_md, column); + switch (colformat) { + case EXTRACT_METADATA_FORMAT_DATE: + snprintf(var->format, sizeof(var->format), "%s", "DATE9."); + break; + case EXTRACT_METADATA_FORMAT_TIME: + snprintf(var->format, sizeof(var->format), "%s", "TIME8."); + break; + case EXTRACT_METADATA_FORMAT_DATE_TIME: + snprintf(var->format, sizeof(var->format), "%s", "DATETIME19."); + break; + default:; + /* PERCENT and CURRENCY values are written as plain numbers: + * SAS's PERCENTw.d would multiply them by 100 for display */ + int decimals = get_decimals(c->json_md, column); + if (decimals > 0) { + snprintf(var->format, sizeof(var->format), "F%d.%d", 10 + decimals, decimals); + } else { + snprintf(var->format, sizeof(var->format), "%s", "BEST12."); + } + } + } else if (coltype == EXTRACT_METADATA_TYPE_STRING) { + var->type = READSTAT_TYPE_STRING; + } +} + +void produce_value_label_sas7bdat(void *csv_metadata, const char* column) { + struct csv_metadata *c = (struct csv_metadata *)csv_metadata; + jsmntok_t* categories = find_variable_property(c->json_md->js, c->json_md->tok, column, "categories"); + if (categories) { + fprintf(stderr, "Warning: value labels for column %s are not written; " + "a sas7bdat file keeps value labels in a separate SAS catalog\n", column); + } +} + +static readstat_value_t value_double_sas7bdat(const char *s, size_t len, struct csv_metadata *c) { + readstat_variable_t *var = &c->variables[c->columns]; + double val = sas_double(s, var, c->is_date[c->columns], c->is_date_time[c->columns]); + + readstat_value_t value = { + .type = READSTAT_TYPE_DOUBLE, + .v = { .double_value = val } + }; + int missing_ranges_count = readstat_variable_get_missing_ranges_count(var); + for (int i=0; i= lo_val.v.double_value && val <= hi_val.v.double_value) { + value.is_tagged_missing = 1; + value.tag = lo_val.tag; + break; + } + } + return value; +} + +void produce_csv_value_sas7bdat(void *csv_metadata, const char *s, size_t len) { + struct csv_metadata *c = (struct csv_metadata *)csv_metadata; + readstat_variable_t *var = &c->variables[c->columns]; + int obs_index = c->rows - 1; + readstat_value_t value; + + if (len == 0) { + value = value_sysmiss(s, len, c); + } else if (var->type == READSTAT_TYPE_DOUBLE) { + value = value_double_sas7bdat(s, len, c); + } else if (var->type == READSTAT_TYPE_STRING) { + value = value_string(s, len, c); + } else { + fprintf(stderr, "%s:%d unsupported variable type %d\n", __FILE__, __LINE__, var->type); + exit(EXIT_FAILURE); + } + + c->handle.value(obs_index, var, value, c->user_ctx); +} diff --git a/src/bin/read_csv/mod_sas7bdat.h b/src/bin/read_csv/mod_sas7bdat.h new file mode 100644 index 00000000..624265d7 --- /dev/null +++ b/src/bin/read_csv/mod_sas7bdat.h @@ -0,0 +1 @@ +extern rs_read_module_t rs_read_mod_sas7bdat; diff --git a/src/bin/read_csv/read_csv.c b/src/bin/read_csv/read_csv.c index 65fb343a..0e2fe1fa 100644 --- a/src/bin/read_csv/read_csv.c +++ b/src/bin/read_csv/read_csv.c @@ -7,6 +7,7 @@ #include "../../readstat.h" +#include "../util/file_format.h" #include "../util/readstat_dta_days.h" #include "json_metadata.h" #include "read_module.h" @@ -15,6 +16,7 @@ #include "mod_csv.h" #include "mod_dta.h" #include "mod_sav.h" +#include "mod_sas7bdat.h" #define UNUSED(x) (void)(x) @@ -66,7 +68,10 @@ static void produce_column_header(struct csv_metadata *c, void *s, size_t len) { } if (c->handle.variable) { - c->handle.variable(c->columns, var, column, c->user_ctx); + /* SAS names a variable's value labels by its format, as the SAS + * readers report them */ + const char *val_labels = c->output_format == RS_FORMAT_SAS_DATA ? var->format : column; + c->handle.variable(c->columns, var, val_labels, c->user_ctx); } } @@ -111,7 +116,8 @@ readstat_error_t readstat_parse_csv(readstat_parser_t *parser, readstat_io_t *io = parser->io; size_t file_size = 0; size_t bytes_read; - struct csv_parser csvparser; + /* Zeroed so that cleanup can call csv_free() before csv_init() has run */ + struct csv_parser csvparser = { 0 }; struct csv_parser *p = &csvparser; char buf[BUFSIZ]; size_t* column_width = md->column_width; @@ -123,8 +129,8 @@ readstat_error_t readstat_parse_csv(readstat_parser_t *parser, md->user_ctx = user_ctx; md->handle = parser->handlers; - rs_read_module_t modules[3] = { rs_read_mod_csv, rs_read_mod_dta, rs_read_mod_sav }; - if ((md->output_module = rs_read_module_for_filename(modules, 3, md->output_format)) == NULL) { + rs_read_module_t modules[4] = { rs_read_mod_csv, rs_read_mod_dta, rs_read_mod_sav, rs_read_mod_sas7bdat }; + if ((md->output_module = rs_read_module_for_filename(modules, 4, md->output_format)) == NULL) { fprintf(stderr, "Unsupported file format\n"); retval = READSTAT_ERROR_WRITE; goto cleanup; diff --git a/src/test/dump_sas7bdat.c b/src/test/dump_sas7bdat.c new file mode 100644 index 00000000..c264d61a --- /dev/null +++ b/src/test/dump_sas7bdat.c @@ -0,0 +1,50 @@ +/* Prints a sas7bdat file's variables (name, format, label) and values, with + * special missing values as .A, .B, ..., for comparison in tests. */ + +#include + +#include "../readstat.h" + +static int handle_variable(int index, readstat_variable_t *variable, + const char *val_labels, void *ctx) { + const char *format = readstat_variable_get_format(variable); + const char *label = readstat_variable_get_label(variable); + printf("variable %s format=%s label=%s\n", readstat_variable_get_name(variable), + format ? format : "", label ? label : ""); + return READSTAT_HANDLER_OK; +} + +static int handle_value(int obs_index, readstat_variable_t *variable, + readstat_value_t value, void *ctx) { + printf("%d %s ", obs_index, readstat_variable_get_name(variable)); + if (readstat_value_is_tagged_missing(value)) { + printf(".%c\n", readstat_value_tag(value)); + } else if (readstat_value_is_system_missing(value)) { + printf(".\n"); + } else if (readstat_value_type(value) == READSTAT_TYPE_STRING) { + printf("\"%s\"\n", readstat_string_value(value)); + } else { + printf("%.15g\n", readstat_double_value(value)); + } + return READSTAT_HANDLER_OK; +} + +int main(int argc, char *argv[]) { + if (argc != 2) { + fprintf(stderr, "Usage: %s file.sas7bdat\n", argv[0]); + return 1; + } + + readstat_parser_t *parser = readstat_parser_init(); + readstat_set_variable_handler(parser, &handle_variable); + readstat_set_value_handler(parser, &handle_value); + + readstat_error_t error = readstat_parse_sas7bdat(parser, argv[1], NULL); + readstat_parser_free(parser); + + if (error != READSTAT_OK) { + fprintf(stderr, "Error reading %s: %s\n", argv[1], readstat_error_message(error)); + return 1; + } + return 0; +} diff --git a/src/test/test_csv_to_sas7bdat.sh b/src/test/test_csv_to_sas7bdat.sh new file mode 100755 index 00000000..a87bf775 --- /dev/null +++ b/src/test/test_csv_to_sas7bdat.sh @@ -0,0 +1,24 @@ +#!/bin/sh +# Converts a CSV file with JSON metadata to sas7bdat, then checks the +# variables' formats and labels, the values (dates, datetimes, times and +# special missing values) and that an unsupported output format fails cleanly. + +srcdir=${srcdir:-.} +tmpdir=$(mktemp -d) || exit 1 +trap 'rm -rf "$tmpdir"' EXIT + +./readstat "$srcdir/resources/csv_to_sas7bdat.csv" "$srcdir/resources/csv_to_sas7bdat.json" \ + "$tmpdir/out.sas7bdat" || exit 1 + +./dump_sas7bdat "$tmpdir/out.sas7bdat" > "$tmpdir/out.txt" || exit 1 +diff -u "$srcdir/resources/csv_to_sas7bdat.expected" "$tmpdir/out.txt" || exit 1 + +if ./readstat "$srcdir/resources/csv_to_sas7bdat.csv" "$srcdir/resources/csv_to_sas7bdat.json" \ + "$tmpdir/out.xpt" 2>/dev/null; then + echo "Converting CSV to an unsupported format should fail" + exit 1 +fi +if [ -e "$tmpdir/out.xpt" ]; then + echo "A failed conversion should not leave its output file behind" + exit 1 +fi diff --git a/variablemetadata_schema.json b/variablemetadata_schema.json index f4a43ddc..a97e2048 100644 --- a/variablemetadata_schema.json +++ b/variablemetadata_schema.json @@ -6,6 +6,9 @@ }, { "$ref": "#/definitions/STATA" + }, + { + "$ref": "#/definitions/SAS" } ], "definitions": { @@ -475,6 +478,192 @@ "values" ], "additionalProperties": false + }, + "SAS": { + "type": "object", + "properties": { + "type": { + "enum": [ + "SAS" + ] + }, + "separator": { + "enum": [ + ",", + ";", + "\t" + ] + }, + "variables": { + "type": "array", + "items": { + "type": "object", + "oneOf": [ + { + "$ref": "#/definitions/SAS-NUMERIC" + }, + { + "$ref": "#/definitions/SAS-STRING" + } + ] + } + } + }, + "additionalProperties": false + }, + "SAS-NUMERIC": { + "properties": { + "type": { + "enum": [ + "NUMERIC" + ] + }, + "name": { + "type": "string", + "minLength": 1, + "maxLength": 32, + "pattern": "^[A-Za-z_][A-Za-z0-9_]*$" + }, + "label": { + "type": "string", + "minLength": 0, + "maxLength": 256 + }, + "format": { + "description": "DATE is written as days since 1960-01-01 with format DATE9., DATE_TIME as seconds since 1960-01-01 00:00:00 (no leap seconds) with DATETIME19., and TIME as seconds since midnight with TIME8. NUMBER, PERCENT and CURRENCY are written unscaled with format Fw.d, or BEST12. without decimals.", + "enum": [ + "NUMBER", + "PERCENT", + "CURRENCY", + "DATE", + "TIME", + "DATE_TIME", + "UNSPECIFIED" + ] + }, + "pattern": { + "type": "string", + "minLength": 1 + }, + "decimals": { + "type": "integer", + "minimum": 0, + "maximum": 16 + }, + "categories": { + "description": "Accepted so that one metadata file can serve every format, but not written: a sas7bdat file keeps value labels in a separate SAS catalog.", + "type": "array", + "items": { + "type": "object", + "properties": { + "code": { + "type": "number" + }, + "label": { + "type": "string" + } + }, + "required": [ + "code", + "label" + ], + "additionalProperties": false + } + }, + "missing": { + "description": "Matching values are written as SAS special missing values, assigned .A, .B, ... in order: one per DISCRETE value, or one for the RANGE and one for its discrete-value.", + "type": "object", + "oneOf": [ + { + "$ref": "#/definitions/SAS-NUMERIC-DISCRETE" + }, + { + "$ref": "#/definitions/NUMERIC-RANGE" + }, + { + "$ref": "#/definitions/DATE-RANGE" + } + ] + } + }, + "required": [ + "type", + "name" + ], + "additionalProperties": false + }, + "SAS-NUMERIC-DISCRETE": { + "type": "object", + "properties": { + "type": { + "enum": [ + "DISCRETE" + ] + }, + "values": { + "description": "Numbers, or yyyy-mm-dd strings for DATE columns", + "type": "array", + "items": { + "type": [ + "number", + "string" + ] + }, + "minItems": 1, + "maxItems": 16, + "uniqueItems": true + } + }, + "required": [ + "type", + "values" + ], + "additionalProperties": false + }, + "SAS-STRING": { + "properties": { + "type": { + "enum": [ + "STRING" + ] + }, + "name": { + "type": "string", + "minLength": 1, + "maxLength": 32, + "pattern": "^[A-Za-z_][A-Za-z0-9_]*$" + }, + "label": { + "type": "string", + "minLength": 0, + "maxLength": 256 + }, + "categories": { + "description": "Accepted so that one metadata file can serve every format, but not written: a sas7bdat file keeps value labels in a separate SAS catalog.", + "type": "array", + "items": { + "type": "object", + "properties": { + "code": { + "type": "string" + }, + "label": { + "type": "string" + } + }, + "required": [ + "code", + "label" + ], + "additionalProperties": false + } + } + }, + "required": [ + "type", + "name" + ], + "additionalProperties": false } } } \ No newline at end of file