fix: quote load paths, imports, and transpile inputs

mimic-iii/buildmimic/duckdb/import_duckdb.sh

@@ -85,9 +85,11 @@ make_table_name () {
 # load data into database
 find "$MIMIC_DIR" -type f -regex '.*\.csv\(.gz\)*' | while IFS= read -r FILE; do
     make_table_name "$FILE"
-    echo "Loading $FILE .. \c"
+    # Escape single quotes for SQL string literal
+    FILE_SQL=$(printf '%s' "$FILE" | sed "s/'/''/g")
+    printf "Loading %s .. " "$FILE"
     try duckdb "$OUTFILE" <<-EOSQL
-		COPY $TABLE_NAME FROM '$FILE' (HEADER, DELIM ',', QUOTE '"', ESCAPE '"');
+		COPY $TABLE_NAME FROM '$FILE_SQL' (HEADER, DELIM ',', QUOTE '"', ESCAPE '"');
 EOSQL
     echo "done!"
 done && echo "Successfully finished loading data into $OUTFILE."

mimic-iii/buildmimic/postgres/Makefile

@@ -104,7 +104,7 @@ mimic-build-gz:
 	@echo '------------------'
 	@echo ''
 	@sleep 2
-	psql "$(DBSTRING)" -v ON_ERROR_STOP=1 -f postgres_load_data_gz.sql -v mimic_data_dir=${datadir}
+	psql "$(DBSTRING)" -v ON_ERROR_STOP=1 -f postgres_load_data_gz.sql -v "mimic_data_dir=$(DATADIR)"
 	@echo ''
 	@echo '--------------------'
 	@echo '-- Adding indexes --'
@@ -151,7 +151,7 @@ mimic-build:
 	@echo '------------------'
 	@echo ''
 	@sleep 2
-	psql "$(DBSTRING)" -v ON_ERROR_STOP=1 -f postgres_load_data.sql -v mimic_data_dir=${DATADIR}
+	psql "$(DBSTRING)" -v ON_ERROR_STOP=1 -f postgres_load_data.sql -v "mimic_data_dir=$(DATADIR)"
 	@echo ''
 	@echo '--------------------'
 	@echo '-- Adding indexes --'
@@ -184,15 +184,15 @@ ifeq ("$(physionetuser)","")
 	@echo 'Call the makefile again with physionetuser=<USERNAME>'
 	@echo ' e.g. make eicu-download datadir=/path/to/data physionetuser=hello@physionet.org'
 else
-	wget --user $(physionetuser) --ask-password -P $(DATADIR) -A csv.gz -m -p -E -k -K -np -nd "$(PHYSIONETURL)"
+	wget --user $(physionetuser) --ask-password -P "$(DATADIR)" -A csv.gz -m -p -E -k -K -np -nd "$(PHYSIONETURL)"
 endif
 
 mimic-demo-download:
 	@echo '------------------------------------------'
 	@echo '-- Downloading MIMIC-III from PhysioNet --'
 	@echo '------------------------------------------'
 	@echo ''
-	wget --user $(physionetuser) --ask-password -P $(DATADIR) -A csv.gz -m -p -E -k -K -np -nd "$(PHYSIONETDEMOURL)"
+	wget --user $(physionetuser) --ask-password -P "$(DATADIR)" -A csv.gz -m -p -E -k -K -np -nd "$(PHYSIONETDEMOURL)"
 
 #This is fairly inelegant and could be tidied with a for loop and an if to check for gzip,
 #but need to maintain compatibility with Windows, which baffling lacks these things

mimic-iii/buildmimic/postgres/create_mimic_user.sh

@@ -20,6 +20,13 @@ else
   echo "User is set to '$MIMIC_USER'";
 fi
 
+# escape ' in password for SQL string literal
+MIMIC_PASSWORD_SQL=$(printf '%s' "$MIMIC_PASSWORD" | sed "s/'/''/g")
+# quote SQL identifiers (double any embedded ")
+sql_ident () { printf '%s' "$1" | sed 's/"/""/g; s/^/"/; s/$/"/'; }
+MIMIC_USER_SQL=$(sql_ident "$MIMIC_USER")
+MIMIC_DB_SQL=$(sql_ident "$MIMIC_DB")
+
 PSQL='psql'
 
 # add in the host/port, if they were specified (not null, -n)
@@ -58,11 +65,11 @@ fi
 if [ "$MIMIC_USER" != "postgres" ]; then
     # we need to create this user via postgres
     # use SUDO to login as postgres
-    $PSQL -U postgres -d postgres -c "DROP USER IF EXISTS $MIMIC_USER; CREATE USER $MIMIC_USER WITH PASSWORD '$MIMIC_PASSWORD';"
+    $PSQL -U postgres -d postgres -c "DROP USER IF EXISTS $MIMIC_USER_SQL; CREATE USER $MIMIC_USER_SQL WITH PASSWORD '$MIMIC_PASSWORD_SQL';"
 fi
 
 if [ "$MIMIC_DB" != "postgres" ]; then
   # drop and recreate the database
-  $PSQL -U postgres -d postgres -c "DROP DATABASE IF EXISTS $MIMIC_DB;"
-  $PSQL -U postgres -d postgres -c "CREATE DATABASE $MIMIC_DB OWNER $MIMIC_USER;"
-fi
+  $PSQL -U postgres -d postgres -c "DROP DATABASE IF EXISTS $MIMIC_DB_SQL;"
+  $PSQL -U postgres -d postgres -c "CREATE DATABASE $MIMIC_DB_SQL OWNER $MIMIC_USER_SQL;"
+fi

mimic-iv-ed/buildmimic/duckdb/import_duckdb.sh

@@ -94,18 +94,29 @@ make_table_name () {
 
 
 # load data into database
-find "$MIMIC_DIR" -type f -name '*.csv???' | sort | while IFS= read -r FILE; do
+# Match both .csv and .csv.gz ( '*.csv???' only matched .csv.gz ).
+LOAD_COUNT_FILE=$(mktemp) || die "mktemp failed"
+trap 'rm -f "$LOAD_COUNT_FILE"' EXIT
+: > "$LOAD_COUNT_FILE"
+find "$MIMIC_DIR" -type f \( -name '*.csv' -o -name '*.csv.gz' \) | sort | while IFS= read -r FILE; do
     make_table_name "$FILE"
 
     # skip directories which we do not expect in mimic-iv-ed
     # avoids syntax errors if mimic-iv in the same dir
-    case $DIRNAME in
+    case "$DIRNAME" in
       (ed) ;; # OK
       (*) continue;
     esac
+    FILE_SQL=$(printf '%s' "$FILE" | sed "s/'/''/g")
     echo "Loading $FILE .. "
     try duckdb "$OUTFILE" <<-EOSQL
-		COPY $TABLE_NAME FROM '$FILE' (HEADER, DELIM ',', QUOTE '"', ESCAPE '"');
+		COPY $TABLE_NAME FROM '$FILE_SQL' (HEADER, DELIM ',', QUOTE '"', ESCAPE '"');
 EOSQL
+    echo x >> "$LOAD_COUNT_FILE"
     echo "done!"
-done && echo "Successfully finished loading data into $OUTFILE."
+done || exit $?
+
+if [ ! -s "$LOAD_COUNT_FILE" ]; then
+    die "No .csv / .csv.gz files loaded from $MIMIC_DIR (expected ed/)."
+fi
+echo "Successfully finished loading data into $OUTFILE."

mimic-iv-note/buildmimic/duckdb/import_duckdb.sh

@@ -97,18 +97,23 @@ make_table_name () {
 
 
 # load data into database
-find "$MIMIC_DIR" -type f -name '*.csv???' | sort | while IFS= read -r FILE; do
+# Match both .csv and .csv.gz ( '*.csv???' only matched .csv.gz ).
+LOAD_COUNT_FILE=$(mktemp) || die "mktemp failed"
+trap 'rm -f "$LOAD_COUNT_FILE"' EXIT
+: > "$LOAD_COUNT_FILE"
+find "$MIMIC_DIR" -type f \( -name '*.csv' -o -name '*.csv.gz' \) | sort | while IFS= read -r FILE; do
     make_table_name "$FILE"
 
     # skip directories which we do not expect in mimic-iv-note
     # avoids syntax errors if mimic-iv in the same dir
-    case $DIRNAME in
+    case "$DIRNAME" in
       (note) ;; # OK
       (*) continue;
     esac
+    FILE_SQL=$(printf '%s' "$FILE" | sed "s/'/''/g")
     echo "Loading $FILE .."
     OUTPUT=$(duckdb "$OUTFILE" 2>&1 <<-EOSQL
-		COPY $TABLE_NAME FROM '$FILE' (HEADER, DELIM ',', QUOTE '"', ESCAPE '"');
+		COPY $TABLE_NAME FROM '$FILE_SQL' (HEADER, DELIM ',', QUOTE '"', ESCAPE '"');
 EOSQL
     )
     # If the table is missing in the DB, we emit a warning and continue.
@@ -123,5 +128,11 @@ EOSQL
         yell "$OUTPUT"
         die "Exiting due to load error."
     fi
+    echo x >> "$LOAD_COUNT_FILE"
     echo "done!"
-done && echo "Successfully finished loading data into $OUTFILE."
+done || exit $?
+
+if [ ! -s "$LOAD_COUNT_FILE" ]; then
+    die "No .csv / .csv.gz files loaded from $MIMIC_DIR (expected note/)."
+fi
+echo "Successfully finished loading data into $OUTFILE."