From 74a78e0577fda6b79a76f2455a47ebfa736693f5 Mon Sep 17 00:00:00 2001 From: Mustafa Senoglu Date: Wed, 8 Jul 2026 11:03:00 +0300 Subject: [PATCH 1/2] fix: add delimiter parameter to load_labels_from_file and load_edges_from_file Closes #2449 Both load_labels_from_file and load_edges_from_file relied on PostgreSQL COPY's default comma delimiter, causing silent data corruption on labels and segfault on edges when using non-comma-delimited files. This adds an optional 'delimiter' TEXT parameter (default ',') to both functions, allowing users to specify the field separator: SELECT load_labels_from_file('graph', 'Label', 'file.csv', true, false, '|'); SELECT load_edges_from_file('graph', 'Edge', 'file.csv', false, '|'); Changes: - create_copy_options() now accepts and passes the delimiter to COPY - SQL function signatures updated with new delimiter parameter - Regression tests added for pipe-delimited vertex and edge files --- regress/age_load/data/pipe_edges.csv | 3 ++ regress/age_load/data/pipe_vertices.csv | 4 +++ regress/sql/age_load.sql | 19 ++++++++++++ sql/age_main.sql | 6 ++-- src/backend/utils/load/ag_load_edges.c | 18 +++++++++-- src/backend/utils/load/ag_load_labels.c | 18 +++++++++-- src/backend/utils/load/age_load.c | 41 +++++++++++++++++++++++-- src/include/utils/load/ag_load_edges.h | 2 +- src/include/utils/load/ag_load_labels.h | 3 +- 9 files changed, 102 insertions(+), 12 deletions(-) create mode 100644 regress/age_load/data/pipe_edges.csv create mode 100644 regress/age_load/data/pipe_vertices.csv diff --git a/regress/age_load/data/pipe_edges.csv b/regress/age_load/data/pipe_edges.csv new file mode 100644 index 000000000..776fdf728 --- /dev/null +++ b/regress/age_load/data/pipe_edges.csv @@ -0,0 +1,3 @@ +start_id|start_vertex_type|end_id|end_vertex_type|distance +1|City|2|City|2500 +2|City|3|City|1050 diff --git a/regress/age_load/data/pipe_vertices.csv b/regress/age_load/data/pipe_vertices.csv new file mode 100644 index 000000000..48e6d1e26 --- /dev/null +++ b/regress/age_load/data/pipe_vertices.csv @@ -0,0 +1,4 @@ +id|name|country +1|Moscow|Russia +2|Berlin|Germany +3|Paris|France diff --git a/regress/sql/age_load.sql b/regress/sql/age_load.sql index 196b09806..78bdaf25e 100644 --- a/regress/sql/age_load.sql +++ b/regress/sql/age_load.sql @@ -223,6 +223,25 @@ SELECT load_labels_from_file('agload_delim', 'V', 'age_load/labels_long_row.csv' SELECT drop_graph('agload_delim', true); +-- +-- Test delimiter parameter for pipe-delimited files +-- +SELECT create_graph('agload_pipe'); +SELECT create_vlabel('agload_pipe', 'City'); +SELECT create_elabel('agload_pipe', 'Connected'); + +-- pipe-delimited vertex file with delimiter parameter +SELECT load_labels_from_file('agload_pipe', 'City', 'age_load/pipe_vertices.csv', true, false, '|'); + +-- pipe-delimited edge file with delimiter parameter +SELECT load_edges_from_file('agload_pipe', 'Connected', 'age_load/pipe_edges.csv', false, '|'); + +-- verify data loaded correctly +MATCH (n:City) RETURN n.name, n.country ORDER BY n.name; +MATCH (a:City)-[e:Connected]->(b:City) RETURN a.name, b.name, e.distance ORDER BY a.name; + +SELECT drop_graph('agload_pipe', true); + -- -- Test security and permissions -- diff --git a/sql/age_main.sql b/sql/age_main.sql index 233d0d23f..5b1044d5a 100644 --- a/sql/age_main.sql +++ b/sql/age_main.sql @@ -167,7 +167,8 @@ CREATE FUNCTION ag_catalog.load_labels_from_file(graph_name name, label_name name, file_path text, id_field_exists bool default true, - load_as_agtype bool default false) + load_as_agtype bool default false, + delimiter text default ',') RETURNS void LANGUAGE c AS 'MODULE_PATHNAME'; @@ -175,7 +176,8 @@ CREATE FUNCTION ag_catalog.load_labels_from_file(graph_name name, CREATE FUNCTION ag_catalog.load_edges_from_file(graph_name name, label_name name, file_path text, - load_as_agtype bool default false) + load_as_agtype bool default false, + delimiter text default ',') RETURNS void LANGUAGE c AS 'MODULE_PATHNAME'; diff --git a/src/backend/utils/load/ag_load_edges.c b/src/backend/utils/load/ag_load_edges.c index 01585bab0..912547c01 100644 --- a/src/backend/utils/load/ag_load_edges.c +++ b/src/backend/utils/load/ag_load_edges.c @@ -134,7 +134,7 @@ static void process_edge_row(char **fields, int nfields, * Create COPY options for CSV parsing. * Returns a List of DefElem nodes. */ -static List *create_copy_options(void) +static List *create_copy_options(char delimiter) { List *options = NIL; @@ -150,6 +150,17 @@ static List *create_copy_options(void) (Node *) makeBoolean(false), -1)); + /* DELIMITER */ + { + char delimiter_str[2]; + delimiter_str[0] = delimiter; + delimiter_str[1] = '\0'; + options = lappend(options, + makeDefElem("delimiter", + (Node *) makeString(pstrdup(delimiter_str)), + -1)); + } + return options; } @@ -161,7 +172,8 @@ int create_edges_from_csv_file(char *file_path, Oid graph_oid, char *label_name, int label_id, - bool load_as_agtype) + bool load_as_agtype, + char delimiter) { Relation label_rel; Oid label_relid; @@ -196,7 +208,7 @@ int create_edges_from_csv_file(char *file_path, init_batch_insert(&batch_state, label_name, graph_oid); /* Create COPY options for CSV parsing */ - copy_options = create_copy_options(); + copy_options = create_copy_options(delimiter); /* Create a minimal ParseState for BeginCopyFrom */ pstate = make_parsestate(NULL); diff --git a/src/backend/utils/load/ag_load_labels.c b/src/backend/utils/load/ag_load_labels.c index 236d47a1d..927c9008e 100644 --- a/src/backend/utils/load/ag_load_labels.c +++ b/src/backend/utils/load/ag_load_labels.c @@ -120,7 +120,7 @@ static void process_vertex_row(char **fields, int nfields, * Create COPY options for csv parsing. * Returns a List of DefElem nodes. */ -static List *create_copy_options(void) +static List *create_copy_options(char delimiter) { List *options = NIL; @@ -136,6 +136,17 @@ static List *create_copy_options(void) (Node *) makeBoolean(false), -1)); + /* DELIMITER */ + { + char delimiter_str[2]; + delimiter_str[0] = delimiter; + delimiter_str[1] = '\0'; + options = lappend(options, + makeDefElem("delimiter", + (Node *) makeString(pstrdup(delimiter_str)), + -1)); + } + return options; } @@ -148,7 +159,8 @@ int create_labels_from_csv_file(char *file_path, char *label_name, int label_id, bool id_field_exists, - bool load_as_agtype) + bool load_as_agtype, + char delimiter) { Relation label_rel; Oid label_relid; @@ -193,7 +205,7 @@ int create_labels_from_csv_file(char *file_path, init_batch_insert(&batch_state, label_name, graph_oid); /* Create COPY options for CSV parsing */ - copy_options = create_copy_options(); + copy_options = create_copy_options(delimiter); /* Create a minimal ParseState for BeginCopyFrom */ pstate = make_parsestate(NULL); diff --git a/src/backend/utils/load/age_load.c b/src/backend/utils/load/age_load.c index e4f10d7e4..d655ef2eb 100644 --- a/src/backend/utils/load/age_load.c +++ b/src/backend/utils/load/age_load.c @@ -576,6 +576,7 @@ Datum load_labels_from_file(PG_FUNCTION_ARGS) int32 label_id; bool id_field_exists; bool load_as_agtype; + char delimiter; if (PG_ARGISNULL(0)) { @@ -604,6 +605,23 @@ Datum load_labels_from_file(PG_FUNCTION_ARGS) id_field_exists = PG_GETARG_BOOL(3); load_as_agtype = PG_GETARG_BOOL(4); + if (PG_NARGS() > 5 && !PG_ARGISNULL(5)) + { + text *delim_text = PG_GETARG_TEXT_P(5); + char *delim_str = text_to_cstring(delim_text); + if (strlen(delim_str) != 1) + { + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("delimiter must be a single character"))); + } + delimiter = delim_str[0]; + } + else + { + delimiter = ','; + } + graph_name_str = NameStr(*graph_name); label_name_str = NameStr(*label_name); @@ -625,7 +643,7 @@ Datum load_labels_from_file(PG_FUNCTION_ARGS) create_labels_from_csv_file(file_path_str, graph_name_str, graph_oid, label_name_str, label_id, id_field_exists, - load_as_agtype); + load_as_agtype, delimiter); free(file_path_str); @@ -645,6 +663,7 @@ Datum load_edges_from_file(PG_FUNCTION_ARGS) Oid label_relid; int32 label_id; bool load_as_agtype; + char delimiter; if (PG_ARGISNULL(0)) { @@ -672,6 +691,23 @@ Datum load_edges_from_file(PG_FUNCTION_ARGS) file_name = PG_GETARG_TEXT_P(2); load_as_agtype = PG_GETARG_BOOL(3); + if (PG_NARGS() > 4 && !PG_ARGISNULL(4)) + { + text *delim_text = PG_GETARG_TEXT_P(4); + char *delim_str = text_to_cstring(delim_text); + if (strlen(delim_str) != 1) + { + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("delimiter must be a single character"))); + } + delimiter = delim_str[0]; + } + else + { + delimiter = ','; + } + graph_name_str = NameStr(*graph_name); label_name_str = NameStr(*label_name); @@ -692,7 +728,8 @@ Datum load_edges_from_file(PG_FUNCTION_ARGS) check_rls_for_load(label_relid); create_edges_from_csv_file(file_path_str, graph_name_str, graph_oid, - label_name_str, label_id, load_as_agtype); + label_name_str, label_id, load_as_agtype, + delimiter); free(file_path_str); diff --git a/src/include/utils/load/ag_load_edges.h b/src/include/utils/load/ag_load_edges.h index 4db00d93a..b2f336489 100644 --- a/src/include/utils/load/ag_load_edges.h +++ b/src/include/utils/load/ag_load_edges.h @@ -39,6 +39,6 @@ */ int create_edges_from_csv_file(char *file_path, char *graph_name, Oid graph_oid, char *label_name, int label_id, - bool load_as_agtype); + bool load_as_agtype, char delimiter); #endif /* AG_LOAD_EDGES_H */ diff --git a/src/include/utils/load/ag_load_labels.h b/src/include/utils/load/ag_load_labels.h index c3d517f30..5a9c0246b 100644 --- a/src/include/utils/load/ag_load_labels.h +++ b/src/include/utils/load/ag_load_labels.h @@ -39,6 +39,7 @@ */ int create_labels_from_csv_file(char *file_path, char *graph_name, Oid graph_oid, char *label_name, int label_id, - bool id_field_exists, bool load_as_agtype); + bool id_field_exists, bool load_as_agtype, + char delimiter); #endif /* AG_LOAD_LABELS_H */ From c3d1f5bbeeb30a1410064f8c55be59f34326a08e Mon Sep 17 00:00:00 2001 From: Mustafa Senoglu Date: Tue, 14 Jul 2026 23:46:01 +0300 Subject: [PATCH 2/2] fix: update expected regression output and fix bare Cypher in pipe delimiter tests - Fix bare Cypher queries to use proper cypher() wrapper - Add missing pipe delimiter test section to age_load.out expected output The CI Build/Regression was failing because: 1. regress/sql/age_load.sql had bare Cypher queries without cypher() wrapper 2. regress/expected/age_load.out was not updated with pipe delimiter test output --- regress/expected/age_load.out | 66 +++++++++++++++++++++++++++++++++++ regress/sql/age_load.sql | 4 +-- 2 files changed, 68 insertions(+), 2 deletions(-) diff --git a/regress/expected/age_load.out b/regress/expected/age_load.out index 17c5ecc27..b0c21e9fe 100644 --- a/regress/expected/age_load.out +++ b/regress/expected/age_load.out @@ -511,6 +511,72 @@ NOTICE: graph "agload_delim" has been dropped (1 row) +-- +-- Test delimiter parameter for pipe-delimited files +-- +SELECT create_graph('agload_pipe'); +NOTICE: graph "agload_pipe" has been created + create_graph +-------------- + +(1 row) + +SELECT create_vlabel('agload_pipe', 'City'); +NOTICE: VLabel "City" has been created + create_vlabel +--------------- + +(1 row) + +SELECT create_elabel('agload_pipe', 'Connected'); +NOTICE: ELabel "Connected" has been created + create_elabel +--------------- + +(1 row) + +-- pipe-delimited vertex file with delimiter parameter +SELECT load_labels_from_file('agload_pipe', 'City', 'age_load/pipe_vertices.csv', true, false, '|'); + load_labels_from_file +----------------------- + +(1 row) + +-- pipe-delimited edge file with delimiter parameter +SELECT load_edges_from_file('agload_pipe', 'Connected', 'age_load/pipe_edges.csv', false, '|'); + load_edges_from_file +---------------------- + +(1 row) + +-- verify data loaded correctly +SELECT * FROM cypher('agload_pipe', $$ MATCH (n:City) RETURN n.name, n.country ORDER BY n.name $$) AS (name agtype, country agtype); + name | country +-----------+---------- + "Berlin" | "Germany" + "Moscow" | "Russia" + "Paris" | "France" +(3 rows) + +SELECT * FROM cypher('agload_pipe', $$ MATCH (a:City)-[e:Connected]->(b:City) RETURN a.name, b.name, e.distance ORDER BY a.name $$) AS (a_name agtype, b_name agtype, distance agtype); + a_name | b_name | distance +-----------+----------+---------- + "Berlin" | "Paris" | "1050" + "Moscow" | "Berlin" | "2500" +(2 rows) + +SELECT drop_graph('agload_pipe', true); +NOTICE: drop cascades to 4 other objects +DETAIL: drop cascades to table agload_pipe._ag_label_vertex +drop cascades to table agload_pipe._ag_label_edge +drop cascades to table agload_pipe."City" +drop cascades to table agload_pipe."Connected" +NOTICE: graph "agload_pipe" has been dropped + drop_graph +------------ + +(1 row) + -- -- Test security and permissions -- diff --git a/regress/sql/age_load.sql b/regress/sql/age_load.sql index 78bdaf25e..3492b57c9 100644 --- a/regress/sql/age_load.sql +++ b/regress/sql/age_load.sql @@ -237,8 +237,8 @@ SELECT load_labels_from_file('agload_pipe', 'City', 'age_load/pipe_vertices.csv' SELECT load_edges_from_file('agload_pipe', 'Connected', 'age_load/pipe_edges.csv', false, '|'); -- verify data loaded correctly -MATCH (n:City) RETURN n.name, n.country ORDER BY n.name; -MATCH (a:City)-[e:Connected]->(b:City) RETURN a.name, b.name, e.distance ORDER BY a.name; +SELECT * FROM cypher('agload_pipe', $$ MATCH (n:City) RETURN n.name, n.country ORDER BY n.name $$) AS (name agtype, country agtype); +SELECT * FROM cypher('agload_pipe', $$ MATCH (a:City)-[e:Connected]->(b:City) RETURN a.name, b.name, e.distance ORDER BY a.name $$) AS (a_name agtype, b_name agtype, distance agtype); SELECT drop_graph('agload_pipe', true);