diff --git a/pages/advanced-algorithms/available-algorithms/_meta.ts b/pages/advanced-algorithms/available-algorithms/_meta.ts index 6795c2595..885416252 100644 --- a/pages/advanced-algorithms/available-algorithms/_meta.ts +++ b/pages/advanced-algorithms/available-algorithms/_meta.ts @@ -13,6 +13,7 @@ export default { "convert_c": "convert_c", "do": "do", "create": "create", + "cross_database": "cross_database", "cugraph": "cugraph", "cycles": "cycles", "date": "date", diff --git a/pages/advanced-algorithms/available-algorithms/cross_database.mdx b/pages/advanced-algorithms/available-algorithms/cross_database.mdx new file mode 100644 index 000000000..dd60e8a42 --- /dev/null +++ b/pages/advanced-algorithms/available-algorithms/cross_database.mdx @@ -0,0 +1,462 @@ +--- +title: cross_database +description: Query other databases (Memgraph, Neo4j, PostgreSQL, MySQL, Oracle, SQL Server, S3, Arrow Flight, DuckDB, ServiceNow) directly from Memgraph and stream their rows into your graph. +--- + +import { Cards } from 'nextra/components' +import GitHub from '/components/icons/GitHub' +import { Callout } from 'nextra/components'; + +# cross_database + +The `cross_database` module lets you reach into another database from a running Cypher query and stream +its rows into Memgraph. Use it to migrate data, build hybrid OLTP/graph pipelines, or join graph data +with rows fetched on-demand from a relational/Bolt/object-store source. + + +**The `migrate` module is deprecated as of Memgraph 3.11** and has been replaced by `cross_database`. +Existing `migrate.*` calls keep working via the [aliases](#backwards-compatibility-with-migrate) shipped +with Memgraph, but new code should call `cross_database.*` directly. The previous `migrate.memgraph()` +procedure has been replaced by the more general [`cross_database.bolt()`](#bolt). + + + + } + title="Source code" + href="https://github.com/memgraph/memgraph/blob/master/mage/python/cross_database.py" + /> + + +| Trait | Value | +| ------------------- | ---------- | +| **Module type** | util | +| **Implementation** | Python | +| **Parallelism** | sequential | + + +When running multiple cross-database calls against the same source, avoid repeating the `config` map in every call. +Use [server-side parameters](/database-management/server-side-parameters) to store the connection config once +and reference it as `$config` across all your queries: + +```cypher +SET GLOBAL PARAMETER pg_config = {user: 'memgraph', password: 'password', host: 'localhost', database: 'demo_db'}; + +CALL cross_database.postgresql('users', $pg_config) YIELD row CREATE (u:User {id: row.id}); +CALL cross_database.postgresql('orders', $pg_config) YIELD row CREATE (o:Order {id: row.id}); +``` + + +## Backwards compatibility with `migrate` + +Every procedure listed below has a `migrate.*` alias preserved from earlier versions, so existing queries +keep working unchanged. The aliases are wired up through Memgraph's callable-mapping mechanism +(`/etc/memgraph/apoc_compatibility_mappings.json`, enabled by default), so no configuration is required. + +| Pre-3.11 name | New name | +| -------------------------- | --------------------------------- | +| `migrate.memgraph` | `cross_database.bolt` | +| `migrate.neo4j` | `cross_database.neo4j` | +| `migrate.mysql` | `cross_database.mysql` | +| `migrate.postgresql` | `cross_database.postgresql` | +| `migrate.sql_server` | `cross_database.sql_server` | +| `migrate.oracle_db` | `cross_database.oracle_db` | +| `migrate.s3` | `cross_database.s3` | +| `migrate.arrow_flight` | `cross_database.arrow_flight` | +| `migrate.duckdb` | `cross_database.duckdb` | +| `migrate.servicenow` | `cross_database.servicenow` | + +If you mix old and new names within the same transaction, treat them as the same procedure — calling +`migrate.postgresql` and `cross_database.postgresql` with identical arguments will trigger the +[same-parameters guard](#same-parameters-guard). + +You can inspect the active mapping at runtime with: + +```cypher +SHOW QUERY CALLABLE MAPPINGS; +``` + +## Type conversion + +For Bolt-based sources (`bolt`, `neo4j`), primitives (`Boolean`, `Integer`, `Float`, `String`, `Null`), +lists, maps, the calendar-aware temporal types (`Date`, `LocalTime`, `LocalDateTime`, `DateTime`), and +spatial points (`Point2d`, `Point3d`, with their `srid` preserved) pass through cleanly. The cases worth +knowing about: + +| Source type | Result | Notes | +| ---------------------------- | ------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `Duration` with `months > 0` | flattened | Months are coerced to 30 days each. **This differs from native Neo4j behavior**, where months are preserved and `date + P1M` stays calendar-aware. See the warning below. | +| `Time` (zoned) | *unsupported* | Memgraph has no zoned-time type. Convert to `LocalTime` on the source side, or carry the offset as a separate column. | +| Enum | *unsupported* | Memgraph enums can't cross the Bolt boundary — the call raises an error. | +| Node / Relationship / Path | *unsupported* | Streaming raw graph objects is not supported. Return their `properties()` / `labels()` instead. | + + +The 30-days-per-month flattening means a duration like `P1M` becomes `30 days` on the Memgraph side, and +`P1Y` becomes `360 days`. Native Neo4j keeps months separate from days, so `date + duration('P1M')` stays +calendar-aware there; after crossing the Bolt boundary into Memgraph that calendar information is gone. +If you need exact calendar arithmetic, fetch the months and days components as separate columns and +reconstruct them on the Memgraph side. + + +## Same-parameters guard + +`cross_database` deliberately rejects two concurrent calls with the same `(query, config, params)` inside +a single transaction. Doing so would race on the underlying connection and yield duplicate rows. The error +message is: + +``` +Cross database module with these parameters is already running. +Please wait for it to finish before starting a new one. +``` + +If you intentionally need two reads from the same source in one transaction, vary the query (for example +add a different `LIMIT`, alias, or comment) so the cache key differs. + +--- + +## Procedures + +### `bolt()` + +`cross_database.bolt()` queries any Bolt-compatible database — Memgraph itself, another Memgraph instance, +or Neo4j — and streams the rows back. This replaces the old `migrate.memgraph()` procedure and is the +recommended way to read from any Bolt source. + +{

Input:

} + +- `label_or_rel_or_query: str` ➡ A label name (`(:Label)`), a relationship type (`[:REL_TYPE]`), or a plain + Cypher query. When a label/relationship shorthand is used, `cross_database.bolt()` synthesises the + matching `MATCH … RETURN labels/properties …` query for you. +- `config: mgp.Map` ➡ Connection parameters. Notable keys: `host` (default `localhost`), `port` + (default `7687`), `username`, `password`, `database`, `uri_scheme` (default `bolt`). +- `config_path: str` (optional) ➡ Path to a JSON file containing connection parameters; values in the + file override values in `config`. +- `params: mgp.Nullable[mgp.Any]` (optional, default `None`) ➡ A `Map` of Cypher parameters passed to + the remote query (e.g. `{val: 42}` for a query using `$val`). + +{

Output:

} + +- `row: mgp.Map` ➡ The result table as a stream of rows. + - When fetching with the `(:Label)` syntax, each row has `labels` and `properties`. + - When fetching with the `[:REL_TYPE]` syntax, each row has `from_labels`, `to_labels`, + `from_properties`, `to_properties`, and `edge_properties`. + - When passing a plain Cypher query, row keys match the columns the remote query returns. + +{

Usage:

} + +#### Retrieve nodes of a certain label and recreate them locally +```cypher +CALL cross_database.bolt('(:Person)', {host: 'localhost', port: 7687}) +YIELD row +WITH row.labels AS labels, row.properties AS props +CREATE (n:labels) SET n += props; +``` + +#### Pass query parameters +```cypher +CALL cross_database.bolt( + 'MATCH (u:User) WHERE u.id = $id RETURN u.name AS name', + {host: 'localhost', port: 7687}, + '', + {id: 42} +) +YIELD row +RETURN row.name AS name; +``` + +#### Connect to Neo4j with explicit credentials +```cypher +CALL cross_database.bolt( + 'MATCH (n) RETURN count(n) AS cnt', + {host: 'neo4j-host', port: 7687, username: 'neo4j', password: 'secret'} +) +YIELD row +RETURN row.cnt AS cnt; +``` + +--- + +### `neo4j()` + +`cross_database.neo4j()` is a thin convenience wrapper around [`bolt()`](#bolt) that defaults the +credentials to Neo4j's stock `neo4j` / `password`. Use it when you only need to override the host/port. + +{

Input:

} + +Same as [`bolt()`](#bolt). If `username` / `password` are missing from `config`, they default to +`neo4j` / `password`. + +{

Output:

} + +Same as [`bolt()`](#bolt). + +{

Usage:

} + +```cypher +CALL cross_database.neo4j('(:Person)', {host: 'neo4j-host', port: 7687}) +YIELD row +WITH row.labels AS labels, row.properties AS props +CREATE (n:labels) SET n += props; +``` + +--- + +### `mysql()` + +Query MySQL and stream rows back. The result table is converted into a stream that can be used to create +graph structures. + +{

Input:

} + +- `table_or_sql: str` ➡ A table name (a single word — automatically expanded to `SELECT * FROM `) + or a full SQL query. +- `config: mgp.Map` ➡ Connection parameters (as in `mysql.connector.connect`). +- `config_path: str` (optional) ➡ Path to a JSON file with connection parameters. +- `params: mgp.Nullable[mgp.Any]` (optional, default `None`) ➡ Query parameters. Accepts a `List` for + `%s`-style placeholders or a `Map` for `%(name)s`-style placeholders (both supported by + `mysql.connector`). + +{

Output:

} + +- `row: mgp.Map` ➡ The result table as a stream of rows. + +{

Usage:

} + +#### Retrieve and inspect data +```cypher +CALL cross_database.mysql('example_table', {user: 'memgraph', + password: 'password', + host: 'localhost', + database: 'demo_db'}) +YIELD row +RETURN row +LIMIT 5000; +``` + +#### Create nodes from migrated data +```cypher +CALL cross_database.mysql('SELECT id, name, age FROM users', {user: 'memgraph', + password: 'password', + host: 'localhost', + database: 'demo_db'}) +YIELD row +CREATE (u:User {id: row.id, name: row.name, age: row.age}); +``` + +--- + +### `postgresql()` + +Query PostgreSQL and stream rows back. + +{

Input:

} + +- `table_or_sql: str` ➡ Table name or SQL query. +- `config: mgp.Map` ➡ Connection parameters (as in `psycopg2.connect`). +- `config_path: str` (optional) ➡ Path to a JSON file with connection parameters. +- `params: mgp.Nullable[mgp.Any]` (optional, default `None`) ➡ Query parameters as a `List` + (psycopg2 uses positional `%s` placeholders). Passing a `Map` raises a `TypeError`. + +{

Output:

} + +- `row: mgp.Map` ➡ The result table as a stream of rows. + +{

Usage:

} + +#### Retrieve and inspect data +```cypher +CALL cross_database.postgresql('example_table', {user: 'memgraph', + password: 'password', + host: 'localhost', + database: 'demo_db'}) +YIELD row +RETURN row +LIMIT 5000; +``` + +#### Establish relationships between orders and customers +```cypher +CALL cross_database.postgresql('SELECT order_id, customer_id FROM orders', {user: 'memgraph', + password: 'password', + host: 'localhost', + database: 'retail_db'}) +YIELD row +MATCH (o:Order {id: row.order_id}), (c:Customer {id: row.customer_id}) +CREATE (c)-[:PLACED]->(o); +``` + +--- + +### `sql_server()` + +Query SQL Server and stream rows back. + +{

Input:

} + +- `table_or_sql: str` ➡ Table name or SQL query. +- `config: mgp.Map` ➡ Connection parameters (as in `pyodbc.connect`). +- `config_path: str` (optional) ➡ Path to a JSON file with connection parameters. +- `params: mgp.Nullable[mgp.Any]` (optional, default `None`) ➡ Query parameters as a `List`. + +{

Output:

} + +- `row: mgp.Map` ➡ The result table as a stream of rows. + +{

Usage:

} + +```cypher +CALL cross_database.sql_server('SELECT id, name, role FROM employees', {user: 'memgraph', + password: 'password', + host: 'localhost', + database: 'company_db'}) +YIELD row +CREATE (e:Employee {id: row.id, name: row.name, role: row.role}); +``` + +--- + +### `oracle_db()` + +Query Oracle DB and stream rows back. + +{

Input:

} + +- `table_or_sql: str` ➡ Table name or SQL query. +- `config: mgp.Map` ➡ Connection parameters (as in `oracledb.connect`). +- `config_path: str` (optional) ➡ Path to a JSON file with connection parameters. +- `params: mgp.Nullable[mgp.Any]` (optional, default `None`) ➡ Query parameters. Accepts a `List` for + positional placeholders or a `Map` for `:name`-style named placeholders (both supported by `oracledb`). + +{

Output:

} + +- `row: mgp.Map` ➡ The result table as a stream of rows. + +{

Usage:

} + +```cypher +CALL cross_database.oracle_db('SELECT id, name FROM companies', {user: 'memgraph', + password: 'password', + host: 'localhost', + database: 'business_db'}) +YIELD row +MERGE (c:Company {id: row.id}) +SET c.name = row.name; +``` + +--- + +### `s3()` + +Read a CSV file directly from AWS S3 and stream its rows into Memgraph. + +{

Input:

} + +- `file_path: str` ➡ S3 path in the form `s3://bucket-name/path/to/file.csv`. +- `config: mgp.Map` ➡ AWS credentials; all keys optional. Missing keys fall back to the corresponding + environment variables: + - `aws_access_key_id` (env: `AWS_ACCESS_KEY_ID`) + - `aws_secret_access_key` (env: `AWS_SECRET_ACCESS_KEY`) + - `region_name` (env: `AWS_REGION`) + - `aws_session_token` (env: `AWS_SESSION_TOKEN`) +- `config_path: str` (optional) ➡ Path to a JSON file with AWS credentials. + +{

Output:

} + +- `row: mgp.Map` ➡ Each CSV row as a `{column_name: value}` map. + +{

Usage:

} + +```cypher +CALL cross_database.s3('s3://my-bucket/employees.csv', {aws_access_key_id: 'your-key', + aws_secret_access_key: 'your-secret', + region_name: 'eu-central-1'}) +YIELD row +CREATE (e:Employee {id: row.id, name: row.name, position: row.position}); +``` + +--- + +### `arrow_flight()` + +Connect to any data source that speaks the [Arrow Flight RPC protocol](https://arrow.apache.org/docs/format/Flight.html) +(for example, [Dremio](https://www.dremio.com/)) and stream rows in. + +{

Input:

} + +- `query: str` ➡ Query against the data source. +- `config: mgp.Map` ➡ Connection parameters (as in `pyarrow.flight.connect`). Notable keys: `host`, + `port`, `username`, `password`. +- `config_path: str` (optional) ➡ Path to a JSON file with connection parameters. + +{

Output:

} + +- `row: mgp.Map` ➡ The result table as a stream of rows. + +{

Usage:

} + +```cypher +CALL cross_database.arrow_flight('SELECT id, name, age FROM users', {username: 'memgraph', + password: 'password', + host: 'localhost', + port: '12345'}) +YIELD row +CREATE (u:User {id: row.id, name: row.name, age: row.age}); +``` + +--- + +### `duckdb()` + +Connect to DuckDB and use it as a proxy to query the [data sources DuckDB +supports](https://duckdb.org/docs/stable/data/data_sources.html). DuckDB is run in-memory with no +persistence — it's only used to proxy to underlying sources. + +{

Input:

} + +- `query: str` ➡ Table name or SQL query. +- `setup_queries: mgp.Nullable[List[str]]` (optional) ➡ Queries executed before `query`, used to attach + or configure the source DuckDB will proxy to (e.g. `INSTALL httpfs; LOAD httpfs;`). + +{

Output:

} + +- `row: mgp.Map` ➡ The result table as a stream of rows. + +{

Usage:

} + +```cypher +CALL cross_database.duckdb( + 'SELECT * FROM read_csv_auto(''s3://my-bucket/users.csv'')', + ['INSTALL httpfs;', 'LOAD httpfs;'] +) +YIELD row +CREATE (u:User {id: row.id, name: row.name}); +``` + +--- + +### `servicenow()` + +Pull data from the [ServiceNow REST API](https://developer.servicenow.com/dev.do#!/reference/api/xanadu/rest/). +The endpoint must return JSON of the form `{"results": [...]}`. + +{

Input:

} + +- `endpoint: str` ➡ The full ServiceNow URL, including any query parameters. +- `config: mgp.Map` ➡ Connection parameters. Notable keys: `username`, `password` (passed through to + `requests.get`). +- `config_path: str` (optional) ➡ Path to a JSON file with connection parameters. +- `params: mgp.Nullable[mgp.Any]` (optional, default `None`) ➡ Additional URL query parameters, + forwarded to `requests.get(endpoint, params=...)`. Unlike the SQL backends, these are HTTP query + string parameters, not SQL placeholders. + +{

Output:

} + +- `row: mgp.Map` ➡ Each element of `results` as a structured dictionary. + +{

Usage:

} + +```cypher +CALL cross_database.servicenow('http://my_endpoint/api/data', {}) +YIELD row +CREATE (e:Employee {id: row.id, name: row.name, position: row.position}); +``` diff --git a/pages/advanced-algorithms/available-algorithms/migrate.mdx b/pages/advanced-algorithms/available-algorithms/migrate.mdx index 884b57e6d..6cf31632f 100644 --- a/pages/advanced-algorithms/available-algorithms/migrate.mdx +++ b/pages/advanced-algorithms/available-algorithms/migrate.mdx @@ -1,568 +1,19 @@ --- title: migrate -description: Discover the migration capabilities of Memgraph for efficient transfer of graph data between instances. Access tutorials and comprehensive documentation for improved experience throughout the migration. +description: The migrate module was renamed to cross_database in Memgraph 3.11. The migrate.* procedure names continue to work as aliases. --- -import { Cards } from 'nextra/components' -import GitHub from '/components/icons/GitHub' -import { Steps } from 'nextra/components' import { Callout } from 'nextra/components'; # migrate -The `migrate` module provides an efficient way to transfer graph data from various relational databases -into Memgraph. This module allows you to retrieve data from various source systems, -transforming tabular data into graph structures. - -With Cypher, you can shape the migrated data dynamically, making it easy to create nodes, -establish relationships, and enrich your graph. Below are examples showing how to retrieve, -filter, and convert relational data into a graph format. - - - } - title="Source code" - href="https://github.com/memgraph/memgraph/blob/master/mage/python/migrate.py" - /> - - -| Trait | Value | -| ------------------- | ---------- | -| **Module type** | util | -| **Implementation** | Python | -| **Parallelism** | sequential | - - -When running multiple migrations against the same source, avoid repeating the `config` map in every call. -Use [server-side parameters](/database-management/server-side-parameters) to store the connection config once -and reference it as `$config` across all your queries: - -```cypher -SET GLOBAL PARAMETER pg_config = {user: 'memgraph', password: 'password', host: 'localhost', database: 'demo_db'}; - -CALL migrate.postgresql('users', $pg_config) YIELD row CREATE (u:User {id: row.id}); -CALL migrate.postgresql('orders', $pg_config) YIELD row CREATE (o:Order {id: row.id}); -``` + +**The `migrate` module was renamed to [`cross_database`](./cross_database) in Memgraph 3.11.** +The `migrate.*` procedure names continue to work via aliases shipped with Memgraph, but new code +should call `cross_database.*` directly. See the [`cross_database`](./cross_database) page for the +full reference, including the new [`bolt()`](./cross_database#bolt) procedure (which supersedes +`migrate.memgraph()`) and the type-conversion rules that apply when reading from Bolt sources. ---- - -## Procedures - -### `arrow_flight()` - -With the `arrow_flight()` procedure, users can access data sources which support the [Arrow Flight RPC protocol](https://arrow.apache.org/docs/format/Flight.html) for transfer -of large data records to achieve high performance. Underlying implementation is using the `pyarrow` Python library to stream rows to -Memgraph. [Dremio](https://www.dremio.com/) is a confirmed data source that works with the `arrow_flight()` procedure. Other sources may also be compatible, but Dremio is based on previous experience. - -{

Input:

} - -- `query: str` ➡ Query used to query the data source. -- `config: mgp.Map` ➡ Connection parameters (as in `pyarrow.flight.connect`). Useful parameters for connecting are `host`, `port`, `username` and `password`. -- `config_path` ➡ Path to a JSON file containing configuration parameters. - -{

Output:

} - -- `row: mgp.Map` ➡ The result table as a stream of rows. - -#### Retrieve and inspect data -```cypher -CALL migrate.arrow_flight('SELECT * FROM users', {username: 'memgraph', - password: 'password', - host: 'localhost', - port: '12345'} ) -YIELD row -RETURN row -LIMIT 5000; -``` - -#### Filter specific data -```cypher -CALL migrate.arrow_flight('SELECT * FROM users', {username: 'memgraph', - password: 'password', - host: 'localhost', - port: '12345'} ) -YIELD row -WHERE row.age >= 30 -RETURN row; -``` - -#### Create nodes from migrated data -```cypher -CALL migrate.arrow_flight('SELECT id, name, age FROM users', {username: 'memgraph', - password: 'password', - host: 'localhost', - port: '12345'} ) -YIELD row -CREATE (u:User {id: row.id, name: row.name, age: row.age}); -``` - -#### Create relationships between users -```cypher -CALL migrate.arrow_flight('SELECT user1_id, user2_id FROM friendships', {username: 'memgraph', - password: 'password', - host: 'localhost', - port: '12345'} ) -YIELD row -MATCH (u1:User {id: row.user1_id}), (u2:User {id: row.user2_id}) -CREATE (u1)-[:FRIENDS_WITH]->(u2); -``` - -### `duckdb()` -With the `migrate.duckdb()` procedure, users can connect to the ** DuckDB** database and query various data sources. -List of data sources that are supported by DuckDB can be found on their [official documentation page](https://duckdb.org/docs/stable/data/data_sources.html). -The underlying implementation streams results from DuckDB to Memgraph using the `duckdb` Python Library. DuckDB is started with the in-memory mode, without any -persistence and is used just to proxy to the underlying data sources. - -{

Input:

} - -- `query: str` ➡ Table name or an SQL query. -- `setup_queries: mgp.Nullable[List[str]]` ➡ List of queries that will be executed prior to the query provided as the initial argument. -Used for setting up the connection to additional data sources. - -{

Output:

} - -- `row: mgp.Map` ➡ The result table as a stream of rows. - -{

Usage:

} - -#### Retrieve and inspect data -```cypher -CALL migrate.duckdb("SELECT * FROM 'test.parquet';") -YIELD row -RETURN row -LIMIT 5000; -``` - -#### Filter specific data -```cypher -CALL migrate.duckdb("SELECT * FROM 'test.parquet';") -YIELD row -WHERE row.age >= 30 -RETURN row; -``` - -#### Create nodes from migrated data -```cypher -CALL migrate.duckdb("SELECT * FROM 'test.parquet';") -YIELD row -CREATE (u:User {id: row.id, name: row.name, age: row.age}); -``` - -#### Create relationships between users -```cypher -CALL migrate.duckdb("SELECT * FROM 'test.parquet';") -YIELD row -MATCH (u1:User {id: row.user1_id}), (u2:User {id: row.user2_id}) -CREATE (u1)-[:FRIENDS_WITH]->(u2); -``` - -#### Setup connection to query additional data sources -```cypher -CALL migrate.duckdb("SELECT * FROM 's3://your_bucket/your_file.parquet';", ["CREATE SECRET secret1 (TYPE s3, KEY_ID 'key', SECRET 'secret', REGION 'region');"]) -YIELD row -MATCH (u1:User {id: row.user1_id}), (u2:User {id: row.user2_id}) -CREATE (u1)-[:FRIENDS_WITH]->(u2); -``` - ---- - -### `memgraph()` - -With the `migrate.memgraph()` procedure, you can access another Memgraph instance and migrate your data to a new Memgraph instance. -The resulting nodes and edges are converted into a stream of rows which can include labels, properties, and primitives. - - -Streaming of raw node and relationship objects is not supported and users are advised to migrate all the necessary identifiers in order to recreate the same graph in Memgraph. - - -{

Input:

} - -- `label_or_rel_or_query: str` ➡ Label name (written in format `(:Label)`), relationship name (written in format `[:rel_type]`) or a plain cypher query. -- `config: mgp.Map` ➡ Connection parameters (as in `gqlalchemy.Memgraph`). Notable parameters are `host[String]`, and `port[Integer]` -- `config_path` ➡ Path to a JSON file containing configuration parameters. -- `params: mgp.Nullable[mgp.Any] (default=None)` ➡ Query parameters (if applicable). - -{

Output:

} - -- `row: mgp.Map` ➡ The result table as a stream of rows. - - when retrieving nodes using the `(:Label)` syntax, row will have the following keys: `labels`, and `properties` - - when retrieving relationships using the `[:REL_TYPE]` syntax, row will have the following keys: `from_labels`, `to_labels`, `from_properties`, `to_properties`, and `edge_properties` - - when retrieving results using a plain Cypher query, row will have keys identical to the returned column names from the Cypher query - -{

Usage:

} - -#### Retrieve nodes of certain label and create them in a new Memgraph instance -```cypher -CALL migrate.memgraph('(:Person)', {host: 'localhost', port: 7687}) -YIELD row -WITH row.labels AS labels, row.properties as props -CREATE (n:labels) SET n += row.props -``` - -#### Retrieve relationships of certain type and create them in a new Memgraph instance -```cypher -CALL migrate.memgraph('[:KNOWS]', {host: 'localhost', port: 7687}) -YIELD row -WITH row.from_labels AS from_labels, - row.to_labels AS to_labels, - row.from_properties AS from_properties, - row.to_properties AS to_properties, - row.edge_properties AS edge_properties -MATCH (p1:Person {id: row.from_properties.id}) -MATCH (p2:Person {id: row.to_properties.id}) -CREATE (p1)-[r:KNOWS]->(p2) -SET r += edge_properties; -``` - -#### Retrieve information from Memgraph using an arbitrary Cypher query -```cypher -CALL migrate.memgraph('MATCH (n) RETURN count(n) as cnt', {host: 'localhost', port: 7687}) -YIELD row -RETURN row.cnt as cnt; -``` - ---- - -### `mysql()` - -With the `migrate.mysql()` procedure, you can access MySQL and migrate your data to Memgraph. -The result table is converted into a stream, and the returned rows can be used to create graph structures. - -{

Input:

} - -- `table_or_sql: str` ➡ Table name or an SQL query. -- `config: mgp.Map` ➡ Connection parameters (as in `mysql.connector.connect`). -- `config_path` ➡ Path to a JSON file containing configuration parameters. -- `params: mgp.Nullable[mgp.Any] (default=None)` ➡ Query parameters (if applicable). - -{

Output:

} - -- `row: mgp.Map` ➡ The result table as a stream of rows. - -{

Usage:

} - -#### Retrieve and inspect data -```cypher -CALL migrate.mysql('example_table', {user: 'memgraph', - password: 'password', - host: 'localhost', - database: 'demo_db'} ) -YIELD row -RETURN row -LIMIT 5000; -``` - -#### Filter specific data -```cypher -CALL migrate.mysql('SELECT * FROM users', {user: 'memgraph', - password: 'password', - host: 'localhost', - database: 'demo_db'} ) -YIELD row -WHERE row.age >= 30 -RETURN row; -``` - -#### Create nodes from migrated data -```cypher -CALL migrate.mysql('SELECT id, name, age FROM users', {user: 'memgraph', - password: 'password', - host: 'localhost', - database: 'demo_db'} ) -YIELD row -CREATE (u:User {id: row.id, name: row.name, age: row.age}); -``` - -#### Create relationships between users -```cypher -CALL migrate.mysql('SELECT user1_id, user2_id FROM friendships', {user: 'memgraph', - password: 'password', - host: 'localhost', - database: 'demo_db'} ) -YIELD row -MATCH (u1:User {id: row.user1_id}), (u2:User {id: row.user2_id}) -CREATE (u1)-[:FRIENDS_WITH]->(u2); -``` - ---- - -### `neo4j()` - -With the `migrate.neo4j()` procedure, you can access Neo4j and migrate your data to Memgraph. -The resulting nodes and edges are converted into a stream of rows which can include labels, properties, and primitives. -**Streaming of raw node and relationship objects is not supported**, and users are advised to migrate all the necessary identifiers -in order to recreate the same graph in Memgraph. - -{

Input:

} - -- `label_or_rel_or_query: str` ➡ Label name (written in format `(:Label)`), relationship name (written in format `[:rel_type]`) or a plain cypher query. -- `config: mgp.Map` ➡ Connection parameters (as in `gqlalchemy.Neo4j`). Notable parameters are `host[String]` and `port[Integer]`. -- `config_path` ➡ Path to a JSON file containing configuration parameters. -- `params: mgp.Nullable[mgp.Any] (default=None)` ➡ Query parameters (if applicable). - -{

Output:

} - -- `row: mgp.Map` ➡ The result table as a stream of rows. - - When retrieving nodes using the `(:Label)` syntax, row will have the following keys: `labels` and `properties`. - - When retrieving relationships using the `[:REL_TYPE]` syntax, row will have the following keys: `from_labels`, `to_labels`, `from_properties`, `to_properties` and `edge_properties`. - - When retrieving results using a plain Cypher query, row will have keys identical to the returned column names from the Cypher query. - -{

Usage:

} - -#### Retrieve nodes of certain label and create them in Memgraph -```cypher -CALL migrate.neo4j('(:Person)', {host: 'localhost', port: 7687}) -YIELD row -WITH row.labels AS labels, row.properties as props -CREATE (n:labels) SET n += row.props -``` - -#### Retrieve relationships of certain type and create them in Memgraph -```cypher -CALL migrate.neo4j('[:KNOWS]', {host: 'localhost', port: 7687}) -YIELD row -WITH row.from_labels AS from_labels, - row.to_labels AS to_labels, - row.from_properties AS from_properties, - row.to_properties AS to_properties, - row.edge_properties AS edge_properties -MATCH (p1:Person {id: row.from_properties.id}) -MATCH (p2:Person {id: row.to_properties.id}) -CREATE (p1)-[r:KNOWS]->(p2) -SET r += edge_properties; -``` - -#### Retrieve information from Neo4j using an arbitrary Cypher query -```cypher -CALL migrate.neo4j('MATCH (n) RETURN count(n) as cnt', {host: 'localhost', port: 7687}) -YIELD row -RETURN row.cnt as cnt; -``` - ---- - -### `oracle_db()` - -With the `migrate.oracle_db()` procedure, you can access Oracle DB and migrate your data to Memgraph. - -{

Input:

} - -- `table_or_sql: str` ➡ Table name or an SQL query. -- `config: mgp.Map` ➡ Connection parameters (as in `mysql.connector.connect`). -- `config_path` ➡ Path to a JSON file containing configuration parameters. -- `params: mgp.Nullable[mgp.Any] (default=None)` ➡ Query parameters (if applicable). - -{

Output:

} - -- `row: mgp.Map` ➡ The result table as a stream of rows. - -{

Usage:

} - -#### Retrieve and inspect data -```cypher -CALL migrate.oracle_db('example_table', {user: 'memgraph', - password: 'password', - host: 'localhost', - database: 'demo_db'} ) -YIELD row -RETURN row -LIMIT 5000; -``` - -#### Merge nodes to avoid duplicates -```cypher -CALL migrate.oracle_db('SELECT id, name FROM companies', {user: 'memgraph', - password: 'password', - host: 'localhost', - database: 'business_db'} ) -YIELD row -MERGE (c:Company {id: row.id}) -SET c.name = row.name; -``` - ---- - -### `postgresql()` - -With the `migrate.postgresql()` procedure, you can access PostgreSQL and migrate your data to Memgraph. - -{

Input:

} - -- `table_or_sql: str` ➡ Table name or an SQL query. -- `config: mgp.Map` ➡ Connection parameters (as in `mysql.connector.connect`). -- `config_path` ➡ Path to a JSON file containing configuration parameters. -- `params: mgp.Nullable[mgp.Any] (default=None)` ➡ Query parameters (if applicable). - -{

Output:

} - -- `row: mgp.Map` ➡ The result table as a stream of rows. - -{

Usage:

} - -#### Retrieve and inspect data -```cypher -CALL migrate.postgresql('example_table', {user: 'memgraph', - password: 'password', - host: 'localhost', - database: 'demo_db'} ) -YIELD row -RETURN row -LIMIT 5000; -``` - -#### Create nodes for products -```cypher -CALL migrate.postgresql('SELECT product_id, name, price FROM products', {user: 'memgraph', - password: 'password', - host: 'localhost', - database: 'retail_db'} ) -YIELD row -CREATE (p:Product {id: row.product_id, name: row.name, price: row.price}); -``` - -#### Establish relationships between orders and customers -```cypher -CALL migrate.postgresql('SELECT order_id, customer_id FROM orders', {user: 'memgraph', - password: 'password', - host: 'localhost', - database: 'retail_db'} ) -YIELD row -MATCH (o:Order {id: row.order_id}), (c:Customer {id: row.customer_id}) -CREATE (c)-[:PLACED]->(o); -``` - ---- - -### `sql_server()` - -With the `migrate.sql_server()` procedure, you can access SQL Server and migrate your data to Memgraph. - -{

Input:

} - -- `table_or_sql: str` ➡ Table name or an SQL query. -- `config: mgp.Map` ➡ Connection parameters (as in `mysql.connector.connect`). -- `config_path` ➡ Path to a JSON file containing configuration parameters. -- `params: mgp.Nullable[mgp.Any] (default=None)` ➡ Query parameters (if applicable). - -{

Output:

} - -- `row: mgp.Map` ➡ The result table as a stream of rows. - -{

Usage:

} - -#### Retrieve and inspect data -```cypher -CALL migrate.sql_server('example_table', {user: 'memgraph', - password: 'password', - host: 'localhost', - database: 'demo_db'} ) -YIELD row -RETURN row -LIMIT 5000; -``` - -#### Convert SQL table rows into graph nodes -```cypher -CALL migrate.sql_server('SELECT id, name, role FROM employees', {user: 'memgraph', - password: 'password', - host: 'localhost', - database: 'company_db'} ) -YIELD row -CREATE (e:Employee {id: row.id, name: row.name, role: row.role}); -``` - ---- - -### `s3()` - -With the `migrate.s3()` procedure, you can **access a CSV file in AWS S3**, stream the data into Memgraph, -and transform it into a **graph representation** using Cypher. The migration is using the Python `boto3` client. - -{

Input:

} - -- `file_path: str` ➡ S3 file path in the format `'s3://bucket-name/path/to/file.csv'`. -- `config: mgp.Map` ➡ AWS connection parameters. All of them are optional. - - `aws_access_key_id` - if not provided, environment variable `AWS_ACCESS_KEY_ID` will be used - - `aws_secret_access_key` - if not provided, environment variable `AWS_SECRET_ACCESS_KEY` will be used - - `region_name` - if not provided, environment variable `AWS_REGION` will be used - - `aws_session_token` - if not provided, environment variable `AWS_SESSION_TOKEN` will be used -- `config_path: str` (optional) ➡ Path to a JSON file containing AWS credentials. - -{

Output:

} - -- `row: mgp.Map` ➡ Each row from the CSV file as a structured dictionary. - -{

Usage:

} - -#### Retrieve and inspect CSV data from S3 -```cypher -CALL migrate.s3('s3://my-bucket/data.csv', {aws_access_key_id: 'your-key', - aws_secret_access_key: 'your-secret', - region_name: 'us-east-1'} ) -YIELD row -RETURN row -LIMIT 100; -``` - -#### Filter specific rows from the CSV -```cypher -CALL migrate.s3('s3://my-bucket/customers.csv', {aws_access_key_id: 'your-key', - aws_secret_access_key: 'your-secret', - region_name: 'us-west-2'} ) -YIELD row -WHERE row.age >= 30 -RETURN row; -``` - -#### Create nodes dynamically from CSV data -```cypher -CALL migrate.s3('s3://my-bucket/employees.csv', {aws_access_key_id: 'your-key', - aws_secret_access_key: 'your-secret', - region_name: 'eu-central-1'} ) -YIELD row -CREATE (e:Employee {id: row.id, name: row.name, position: row.position}); -``` - ---- - -### `servicenow()` - -With the `migrate.servicenow()` procedure, you can access [ServiceNow REST API](https://developer.servicenow.com/dev.do#!/reference/api/xanadu/rest/) and transfer your data to Memgraph. -The underlying implementation is using the [`requests` Python library] to migrate results to Memgraph. The REST API from -ServiceNow must provide results in the format `{results: []}` in order for Memgraph to stream it into result rows. - -{

Input:

} - -- `endpoint: str` ➡ ServiceNow endpoint. Users can optionally include their own query parameters to filter results. -- `config: mgp.Map` ➡ Connection parameters. Notable connection parameters are `username` and `password`, per `requests.get()` method. -- `config_path: str` ➡ Path to a JSON file containing configuration parameters. - -{

Output:

} - -- `row: mgp.Map` ➡ Each row from the CSV file as a structured dictionary. - -{

Usage:

} - -#### Retrieve and inspect CSV data from ServiceNow -```cypher -CALL migrate.servicenow('http://my_endpoint/api/data', {}) -YIELD row -RETURN row -LIMIT 100; -``` - -#### Filter specific rows from the CSV -```cypher -CALL migrate.servicenow('http://my_endpoint/api/data', {}) -YIELD row -WHERE row.age >= 30 -RETURN row; -``` - -#### Create nodes dynamically from CSV data -```cypher -CALL migrate.servicenow('http://my_endpoint/api/data', {}) -YIELD row -CREATE (e:Employee {id: row.id, name: row.name, position: row.position}); -``` +This page is kept for backwards-compatible inbound links. All content has moved to +[`cross_database`](./cross_database).