Map XSD types sharing a local name to distinct tables

CLAUDE.md

@@ -57,6 +57,25 @@ cd tests/sample_models && python models.py
 
 then commit the updated snapshot files alongside the code change.
 
+## Fixing bugs
+
+Aim at the smallest change that fixes the bug. Keep the fix local to the code that is wrong, do
+not refactor around it, and do not widen the scope to related cases that are not reported.
+
+Cover the fix by extending what already exists rather than adding new files:
+
+- Add the failing construct as a sub-case of an existing sample model, typically
+  `tests/sample_models/orders/orders.xsd` with matching data in `tests/sample_models/orders/xml/`,
+  and comment in the XSD what the construct tests. New sample models are for new kinds of schema,
+  not for single edge cases.
+- Add assertions to the existing test module covering that area instead of creating a test file
+  for one case. The parametrized tests pick up sample models and XML files automatically, so a
+  construct added to a sample model is exercised by the parsing, round-trip and insertion tests
+  without new test code.
+- Regenerate the snapshots when the data model changes, as described above.
+
+Check that the bug reproduces before the fix and is gone after it.
+
 ## Writing style
 
 - After any code change, check whether docstrings, inline docs, or `docs/` pages need updating and update them as part of the same task.

docs/getting_started.md

@@ -51,6 +51,12 @@ xml2db render schema.xsd --format source-tree
 xml2db render schema.xsd --format ddl --db-type postgresql
 ```
 
+!!! note "Documents mixing several schemas"
+
+    `DataModel` reads a single XSD file, but that file can pull in others with `xs:import` (for
+    another namespace) or `xs:include` (for the same one), so documents mixing several schemas work
+    as long as one schema describes the whole document.
+
 See [Configuring your data model](configuring.md) for a full description of the available config options.
 
 ## Importing XML files

docs/how_it_works.md

@@ -124,6 +124,12 @@ the process much more complex. Whenever a field which would introduce a dependen
 discarded with a warning, which means that the corresponding data in XML files will not be imported. The rest of the
 data should be processed correctly.
 
+#### Wildcards
+
+Elements declared as `xs:any` are discarded with a warning, because a wildcard does not tell which elements may
+appear, and therefore which tables to create. When the content of such an element follows a known schema, declaring it
+explicitly in a wrapper schema makes it importable.
+
 #### Mixed content elements
 
 XML elements with mixed content can contain both text and children elements (tags). `xml2db` offers partial support for

src/xml2db/model.py

@@ -117,6 +117,7 @@ def __init__(
         self.temp_prefix = str(uuid4())[:8] if temp_prefix is None else temp_prefix
 
         self.tables = {}
+        self.types_keys = {}
         self.names_types_map = {}
         self.root_table = None
 
@@ -274,6 +275,40 @@ def _build_model(self):
         for tb in self.fk_ordered_tables:
             tb.build_sqlalchemy_tables()
 
+    def _get_type_key(self, node) -> str:
+        """Get the key identifying the XSD type of a node among the data model tables.
+
+        Types are keyed by their local name, which is not unique when a schema imports other schemas: two namespaces
+        may define different types sharing the same local name. A numeric suffix is appended in this case, so that
+        each XSD type gets its own table.
+
+        Args:
+            node: the XSD node whose type key is needed
+
+        Returns:
+            The key of the type in `self.tables`.
+        """
+        xsd_type = getattr(node, "type", None)
+        if xsd_type is None:
+            return self.data_flow_name
+        if xsd_type in self.types_keys:
+            return self.types_keys[xsd_type]
+        key = xsd_type.local_name
+        if key is None:
+            key = node.local_name
+        reserved_keys = set(self.types_keys.values())
+        if key in reserved_keys:
+            i = 1
+            while f"{key}_{i}" in reserved_keys:
+                i += 1
+            key = f"{key}_{i}"
+            logger.warning(
+                f"two different XSD types share the local name '{xsd_type.local_name or node.local_name}', "
+                f"the second one is mapped to the table key '{key}'"
+            )
+        self.types_keys[xsd_type] = key
+        return key
+
     def _parse_tree(self, parent_node: xmlschema.XsdElement, nodes_path: list = None):
         """Parse a node of an XML schema recursively and create a target data model without any simplification
 
@@ -292,13 +327,7 @@ def _parse_tree(self, parent_node: xmlschema.XsdElement, nodes_path: list = None
         """
 
         # find current node type and name and returns corresponding table if it already exists
-        parent_type = (
-            parent_node.type.local_name
-            if hasattr(parent_node, "type")
-            else self.data_flow_name
-        )
-        if parent_type is None:
-            parent_type = parent_node.local_name
+        parent_type = self._get_type_key(parent_node)
 
         nodes_path = (nodes_path if nodes_path else []) + [parent_type]
 
@@ -511,7 +540,7 @@ def get_occurs(particle):
 
                 elif ct.is_complex():
                     # ignoring recursive definitions by skipping these fields
-                    if child.type.local_name in nodes_path:
+                    if self._get_type_key(child) in nodes_path:
                         logger.warning(
                             f"type '{child.type.local_name}' contains a recursive definition"
                         )

tests/sample_models/orders/orders.xsd

@@ -92,6 +92,17 @@
             <xs:element name="price" type="bt:dectype"/>
             <xs:element name="currency" type="bt:currencytype"/>
             <xs:element name="delivery" type="deliveryType" minOccurs="0" maxOccurs="1"/>
+            <!-- 'detail' is declared twice with a different anonymous complex type, here and in
+                 shipordertype: both types share the local name 'detail' and each one needs its
+                 own table -->
+            <xs:element name="detail" minOccurs="0" maxOccurs="unbounded">
+                <xs:complexType>
+                    <xs:sequence>
+                        <xs:element name="weight" type="bt:dectype"/>
+                        <xs:element name="unit" type="bt:stringtype"/>
+                    </xs:sequence>
+                </xs:complexType>
+            </xs:element>
         </xs:sequence>
     </xs:complexType>
 
@@ -100,6 +111,14 @@
             <xs:element name="orderperson" type="contacttype" minOccurs="1" maxOccurs="1"/>
             <xs:element name="shipto" type="contacttype" minOccurs="0" maxOccurs="1"/>
             <xs:element name="item" maxOccurs="unbounded" type="itemtype"/>
+            <xs:element name="detail" minOccurs="0" maxOccurs="unbounded">
+                <xs:complexType>
+                    <xs:sequence>
+                        <xs:element name="reference" type="bt:stringtype"/>
+                        <xs:element name="carrier" type="bt:stringtype"/>
+                    </xs:sequence>
+                </xs:complexType>
+            </xs:element>
         </xs:sequence>
         <xs:attribute name="orderid" type="bt:stringtype" use="required" />
         <xs:attribute name="processed_at" type="xs:dateTime" />

tests/sample_models/orders/orders_ddl_mssql_version0.sql

@@ -20,6 +20,26 @@ CREATE TABLE orderperson (
 )
 
 
+CREATE TABLE detail_1 (
+	pk_detail_1 INTEGER NOT NULL IDENTITY, 
+	reference VARCHAR(1000) NULL, 
+	carrier VARCHAR(1000) NULL, 
+	record_hash BINARY(20) NULL, 
+	CONSTRAINT cx_pk_detail_1 PRIMARY KEY CLUSTERED (pk_detail_1), 
+	CONSTRAINT detail_1_xml2db_record_hash UNIQUE (record_hash)
+)
+
+
+CREATE TABLE detail (
+	pk_detail INTEGER NOT NULL IDENTITY, 
+	weight DOUBLE PRECISION NULL, 
+	unit VARCHAR(1000) NULL, 
+	record_hash BINARY(20) NULL, 
+	CONSTRAINT cx_pk_detail PRIMARY KEY CLUSTERED (pk_detail), 
+	CONSTRAINT detail_xml2db_record_hash UNIQUE (record_hash)
+)
+
+
 CREATE TABLE intfeature_with_peculiarly_long_suffix_which_overflow_max_length (
 	pk_intfeature_with_peculiarly_long_suffix_which_overflow_max_length INTEGER NOT NULL IDENTITY, 
 	id VARCHAR(1000) NULL, 
@@ -58,6 +78,14 @@ CREATE TABLE item (
 )
 
 
+CREATE TABLE item_detail (
+	fk_item INTEGER NOT NULL, 
+	fk_detail INTEGER NOT NULL, 
+	FOREIGN KEY(fk_item) REFERENCES item (pk_item), 
+	FOREIGN KEY(fk_detail) REFERENCES detail (pk_detail)
+)
+
+
 CREATE TABLE item_product_features_intfeature_with_peculiarly_long_suffix_which_overflow_max_length (
 	fk_item INTEGER NOT NULL, 
 	fk_intfeature_with_peculiarly_long_suffix_which_overflow_max_length INTEGER NOT NULL, 
@@ -96,6 +124,14 @@ CREATE TABLE shiporder_item (
 )
 
 
+CREATE TABLE shiporder_detail_detail_1 (
+	fk_shiporder INTEGER NOT NULL, 
+	fk_detail_1 INTEGER NOT NULL, 
+	FOREIGN KEY(fk_shiporder) REFERENCES shiporder (pk_shiporder), 
+	FOREIGN KEY(fk_detail_1) REFERENCES detail_1 (pk_detail_1)
+)
+
+
 CREATE TABLE orders (
 	pk_orders INTEGER NOT NULL IDENTITY, 
 	batch_id VARCHAR(1000) NULL, 
@@ -114,6 +150,10 @@ CREATE TABLE orders_shiporder (
 	FOREIGN KEY(fk_shiporder) REFERENCES shiporder (pk_shiporder)
 )
 
+CREATE CLUSTERED INDEX ix_fk_item_detail ON item_detail (fk_item, fk_detail)
+
+CREATE INDEX ix_item_detail_fk_detail ON item_detail (fk_detail)
+
 CREATE CLUSTERED INDEX ix_fk_item_product_features_intfeature_with_peculiarly_long_suffix_which_overflow_max_length ON item_product_features_intfeature_with_peculiarly_long_suffix_which_overflow_max_length (fk_item, fk_intfeature_with_peculiarly_long_suffix_which_overflow_max_length)
 
 CREATE INDEX ix_item_product_features_intfeature_with_peculiarly_long_suffix_which_overflow_max_length_fk_intfeature_with_peculiarly__7aff ON item_product_features_intfeature_with_peculiarly_long_suffix_which_overflow_max_length (fk_intfeature_with_peculiarly_long_suffix_which_overflow_max_length)
@@ -126,6 +166,10 @@ CREATE CLUSTERED INDEX ix_fk_shiporder_item ON shiporder_item (fk_shiporder, fk_
 
 CREATE INDEX ix_shiporder_item_fk_item ON shiporder_item (fk_item)
 
+CREATE CLUSTERED INDEX ix_fk_shiporder_detail_detail_1 ON shiporder_detail_detail_1 (fk_shiporder, fk_detail_1)
+
+CREATE INDEX ix_shiporder_detail_detail_1_fk_detail_1 ON shiporder_detail_detail_1 (fk_detail_1)
+
 CREATE CLUSTERED INDEX ix_fk_orders_shiporder ON orders_shiporder (fk_orders, fk_shiporder)
 
 CREATE INDEX ix_orders_shiporder_fk_shiporder ON orders_shiporder (fk_shiporder)

tests/sample_models/orders/orders_ddl_mssql_version1.sql

@@ -21,6 +21,26 @@ CREATE TABLE orderperson (
 )
 
 
+CREATE TABLE detail_1 (
+	pk_detail_1 INTEGER NOT NULL IDENTITY, 
+	reference VARCHAR(1000) NULL, 
+	carrier VARCHAR(1000) NULL, 
+	record_hash BINARY(16) NULL, 
+	CONSTRAINT cx_pk_detail_1 PRIMARY KEY CLUSTERED (pk_detail_1), 
+	CONSTRAINT detail_1_xml2db_record_hash UNIQUE (record_hash)
+)
+
+
+CREATE TABLE detail (
+	pk_detail INTEGER NOT NULL IDENTITY, 
+	weight DOUBLE PRECISION NULL, 
+	unit VARCHAR(1000) NULL, 
+	record_hash BINARY(16) NULL, 
+	CONSTRAINT cx_pk_detail PRIMARY KEY CLUSTERED (pk_detail), 
+	CONSTRAINT detail_xml2db_record_hash UNIQUE (record_hash)
+)
+
+
 CREATE TABLE intfeature_with_peculiarly_long_suffix_which_overflow_max_length (
 	pk_intfeature_with_peculiarly_long_suffix_which_overflow_max_length INTEGER NOT NULL IDENTITY, 
 	id VARCHAR(1000) NULL, 
@@ -55,6 +75,15 @@ CREATE TABLE shiporder (
 )
 
 
+CREATE TABLE shiporder_detail_detail_1 (
+	fk_shiporder INTEGER NOT NULL, 
+	fk_detail_1 INTEGER NOT NULL, 
+	xml2db_row_number INTEGER NOT NULL, 
+	FOREIGN KEY(fk_shiporder) REFERENCES shiporder (pk_shiporder), 
+	FOREIGN KEY(fk_detail_1) REFERENCES detail_1 (pk_detail_1)
+)
+
+
 CREATE TABLE orders (
 	pk_orders INTEGER NOT NULL IDENTITY, 
 	batch_id VARCHAR(1000) NULL, 
@@ -96,6 +125,15 @@ CREATE TABLE item (
 )
 
 
+CREATE TABLE item_detail (
+	fk_item INTEGER NOT NULL, 
+	fk_detail INTEGER NOT NULL, 
+	xml2db_row_number INTEGER NOT NULL, 
+	FOREIGN KEY(fk_item) REFERENCES item (pk_item), 
+	FOREIGN KEY(fk_detail) REFERENCES detail (pk_detail)
+)
+
+
 CREATE TABLE item_product_features_intfeature_with_peculiarly_long_suffix_which_overflow_max_length (
 	fk_item INTEGER NOT NULL, 
 	fk_intfeature_with_peculiarly_long_suffix_which_overflow_max_length INTEGER NOT NULL, 
@@ -113,10 +151,18 @@ CREATE TABLE item_product_features_stringfeature (
 	FOREIGN KEY(fk_stringfeature) REFERENCES stringfeature (pk_stringfeature)
 )
 
+CREATE CLUSTERED INDEX ix_fk_shiporder_detail_detail_1 ON shiporder_detail_detail_1 (fk_shiporder, fk_detail_1)
+
+CREATE INDEX ix_shiporder_detail_detail_1_fk_detail_1 ON shiporder_detail_detail_1 (fk_detail_1)
+
 CREATE CLUSTERED INDEX ix_fk_orders_shiporder ON orders_shiporder (fk_orders, fk_shiporder)
 
 CREATE INDEX ix_orders_shiporder_fk_shiporder ON orders_shiporder (fk_shiporder)
 
+CREATE CLUSTERED INDEX ix_fk_item_detail ON item_detail (fk_item, fk_detail)
+
+CREATE INDEX ix_item_detail_fk_detail ON item_detail (fk_detail)
+
 CREATE CLUSTERED INDEX ix_fk_item_product_features_intfeature_with_peculiarly_long_suffix_which_overflow_max_length ON item_product_features_intfeature_with_peculiarly_long_suffix_which_overflow_max_length (fk_item, fk_intfeature_with_peculiarly_long_suffix_which_overflow_max_length)
 
 CREATE INDEX ix_item_product_features_intfeature_with_peculiarly_long_suffix_which_overflow_max_length_fk_intfeature_with_peculiarly__7aff ON item_product_features_intfeature_with_peculiarly_long_suffix_which_overflow_max_length (fk_intfeature_with_peculiarly_long_suffix_which_overflow_max_length)

tests/sample_models/orders/orders_ddl_mssql_version2.sql

@@ -31,6 +31,26 @@ CREATE TABLE orderperson (
 )
 
 
+CREATE TABLE detail_1 (
+	pk_detail_1 INTEGER NOT NULL IDENTITY, 
+	reference VARCHAR(1000) NULL, 
+	carrier VARCHAR(1000) NULL, 
+	xml2db_record_hash BINARY(20) NULL, 
+	CONSTRAINT cx_pk_detail_1 PRIMARY KEY CLUSTERED (pk_detail_1), 
+	CONSTRAINT detail_1_xml2db_record_hash UNIQUE (xml2db_record_hash)
+)
+
+
+CREATE TABLE detail (
+	pk_detail INTEGER NOT NULL IDENTITY, 
+	weight DOUBLE PRECISION NULL, 
+	unit VARCHAR(1000) NULL, 
+	xml2db_record_hash BINARY(20) NULL, 
+	CONSTRAINT cx_pk_detail PRIMARY KEY CLUSTERED (pk_detail), 
+	CONSTRAINT detail_xml2db_record_hash UNIQUE (xml2db_record_hash)
+)
+
+
 CREATE TABLE intfeature_with_peculiarly_long_suffix_which_overflow_max_length (
 	pk_intfeature_with_peculiarly_long_suffix_which_overflow_max_length INTEGER NOT NULL IDENTITY, 
 	id VARCHAR(1000) NULL, 
@@ -95,6 +115,14 @@ CREATE TABLE item (
 )
 
 
+CREATE TABLE item_detail (
+	fk_item INTEGER NOT NULL, 
+	fk_detail INTEGER NOT NULL, 
+	FOREIGN KEY(fk_item) REFERENCES item (pk_item), 
+	FOREIGN KEY(fk_detail) REFERENCES detail (pk_detail)
+)
+
+
 CREATE TABLE shiporder (
 	pk_shiporder INTEGER NOT NULL IDENTITY, 
 	temp_pk_shiporder INTEGER NULL, 
@@ -128,6 +156,14 @@ CREATE TABLE shiporder_item (
 	FOREIGN KEY(fk_item) REFERENCES item (pk_item)
 )
 
+
+CREATE TABLE shiporder_detail_detail_1 (
+	fk_shiporder INTEGER NOT NULL, 
+	fk_detail_1 INTEGER NOT NULL, 
+	FOREIGN KEY(fk_shiporder) REFERENCES shiporder (pk_shiporder), 
+	FOREIGN KEY(fk_detail_1) REFERENCES detail_1 (pk_detail_1)
+)
+
 CREATE CLUSTERED INDEX ix_fk_product_features_intfeature_with_peculiarly_long_suffix_which_overflow_max_length ON product_features_intfeature_with_peculiarly_long_suffix_which_overflow_max_length (fk_product, fk_intfeature_with_peculiarly_long_suffix_which_overflow_max_length)
 
 CREATE INDEX ix_product_features_intfeature_with_peculiarly_long_suffix_which_overflow_max_length_fk_intfeature_with_peculiarly_long__3ab3 ON product_features_intfeature_with_peculiarly_long_suffix_which_overflow_max_length (fk_intfeature_with_peculiarly_long_suffix_which_overflow_max_length)
@@ -136,7 +172,15 @@ CREATE CLUSTERED INDEX ix_fk_product_features_stringfeature ON product_features_
 
 CREATE INDEX ix_product_features_stringfeature_fk_stringfeature ON product_features_stringfeature (fk_stringfeature)
 
+CREATE CLUSTERED INDEX ix_fk_item_detail ON item_detail (fk_item, fk_detail)
+
+CREATE INDEX ix_item_detail_fk_detail ON item_detail (fk_detail)
+
 CREATE CLUSTERED INDEX ix_fk_shiporder_item ON shiporder_item (fk_shiporder, fk_item)
 
 CREATE INDEX ix_shiporder_item_fk_item ON shiporder_item (fk_item)
 
+CREATE CLUSTERED INDEX ix_fk_shiporder_detail_detail_1 ON shiporder_detail_detail_1 (fk_shiporder, fk_detail_1)
+
+CREATE INDEX ix_shiporder_detail_detail_1_fk_detail_1 ON shiporder_detail_detail_1 (fk_detail_1)
+