1414from .errors import DataJointError
1515from .settings import config
1616
17- # Core DataJoint type aliases - scientist-friendly names mapped to native SQL types
18- # These types can be used without angle brackets in table definitions
19- CORE_TYPE_ALIASES = {
20- # Numeric types
21- "FLOAT32" : "float" ,
22- "FLOAT64" : "double" ,
23- "INT64" : "bigint" ,
24- "UINT64" : "bigint unsigned" ,
25- "INT32" : "int" ,
26- "UINT32" : "int unsigned" ,
27- "INT16" : "smallint" ,
28- "UINT16" : "smallint unsigned" ,
29- "INT8" : "tinyint" ,
30- "UINT8" : "tinyint unsigned" ,
31- "BOOL" : "tinyint" ,
32- # UUID type
33- "UUID" : "binary(16)" ,
17+ # Core DataJoint types - scientist-friendly names that are fully supported
18+ # These are recorded in field comments using :type: syntax for reconstruction
19+ # Format: pattern_name -> (regex_pattern, mysql_type or None if same as matched)
20+ CORE_TYPES = {
21+ # Numeric types (aliased to native SQL)
22+ "float32" : (r"float32$" , "float" ),
23+ "float64" : (r"float64$" , "double" ),
24+ "int64" : (r"int64$" , "bigint" ),
25+ "uint64" : (r"uint64$" , "bigint unsigned" ),
26+ "int32" : (r"int32$" , "int" ),
27+ "uint32" : (r"uint32$" , "int unsigned" ),
28+ "int16" : (r"int16$" , "smallint" ),
29+ "uint16" : (r"uint16$" , "smallint unsigned" ),
30+ "int8" : (r"int8$" , "tinyint" ),
31+ "uint8" : (r"uint8$" , "tinyint unsigned" ),
32+ "bool" : (r"bool$" , "tinyint" ),
33+ # UUID (stored as binary)
34+ "uuid" : (r"uuid$" , "binary(16)" ),
35+ # JSON
36+ "json" : (r"json$" , None ), # json passes through as-is
37+ # Binary (blob maps to longblob)
38+ "blob" : (r"blob$" , "longblob" ),
39+ # Temporal
40+ "date" : (r"date$" , None ),
41+ "datetime" : (r"datetime$" , None ),
42+ # String types (with parameters)
43+ "char" : (r"char\s*\(\d+\)$" , None ),
44+ "varchar" : (r"varchar\s*\(\d+\)$" , None ),
45+ # Enumeration
46+ "enum" : (r"enum\s*\(.+\)$" , None ),
3447}
3548
49+ # Compile core type patterns
50+ CORE_TYPE_PATTERNS = {name : re .compile (pattern , re .I ) for name , (pattern , _ ) in CORE_TYPES .items ()}
51+
52+ # Get SQL mapping for core types
53+ CORE_TYPE_SQL = {name : sql_type for name , (_ , sql_type ) in CORE_TYPES .items ()}
54+
3655MAX_TABLE_NAME_LENGTH = 64
3756CONSTANT_LITERALS = {
3857 "CURRENT_TIMESTAMP" ,
3958 "NULL" ,
4059} # SQL literals to be used without quotes (case insensitive)
4160
4261# Type patterns for declaration parsing
43- # Two categories: core type aliases and native passthrough types
4462TYPE_PATTERN = {
4563 k : re .compile (v , re .I )
4664 for k , v in dict (
47- # Core DataJoint type aliases (scientist-friendly names)
48- FLOAT32 = r"float32$" ,
49- FLOAT64 = r"float64$" ,
50- INT64 = r"int64$" ,
51- UINT64 = r"uint64$" ,
52- INT32 = r"int32$" ,
53- UINT32 = r"uint32$" ,
54- INT16 = r"int16$" ,
55- UINT16 = r"uint16$" ,
56- INT8 = r"int8$" ,
57- UINT8 = r"uint8$" ,
58- BOOL = r"bool$" ,
59- UUID = r"uuid$" ,
60- # Native SQL types (passthrough)
65+ # Core DataJoint types
66+ ** {name .upper (): pattern for name , (pattern , _ ) in CORE_TYPES .items ()},
67+ # Native SQL types (passthrough with warning for non-standard use)
6168 INTEGER = r"((tiny|small|medium|big|)int|integer)(\s*\(.+\))?(\s+unsigned)?(\s+auto_increment)?|serial$" ,
6269 DECIMAL = r"(decimal|numeric)(\s*\(.+\))?(\s+unsigned)?$" ,
6370 FLOAT = r"(double|float|real)(\s*\(.+\))?(\s+unsigned)?$" ,
64- STRING = r"(var)?char\s*\(.+\)$" ,
65- JSON = r"json$" ,
66- ENUM = r"enum\s*\(.+\)$" ,
67- TEMPORAL = r"(date|datetime|time|timestamp|year)(\s*\(.+\))?$" ,
68- BLOB = r"(tiny|small|medium|long|)blob$" ,
71+ STRING = r"(var)?char\s*\(.+\)$" , # Catches char/varchar not matched by core types
72+ TEMPORAL = r"(time|timestamp|year)(\s*\(.+\))?$" , # time, timestamp, year (not date/datetime)
73+ NATIVE_BLOB = r"(tiny|small|medium|long)blob$" , # Specific blob variants
74+ TEXT = r"(tiny|small|medium|long)?text$" , # Text types
6975 # AttributeTypes use angle brackets
7076 ADAPTED = r"<.+>$" ,
7177 ).items ()
7278}
7379
74- # Types that require special handling (stored in attribute comment for reconstruction)
75- SPECIAL_TYPES = {"ADAPTED" } | set (CORE_TYPE_ALIASES )
80+ # Core types are stored in attribute comment for reconstruction
81+ CORE_TYPE_NAMES = {name .upper () for name in CORE_TYPES }
82+
83+ # Special types that need comment storage (core types + adapted)
84+ SPECIAL_TYPES = CORE_TYPE_NAMES | {"ADAPTED" }
7685
77- # Native SQL types that pass through without modification
86+ # Native SQL types that pass through (with optional warning)
7887NATIVE_TYPES = set (TYPE_PATTERN ) - SPECIAL_TYPES
7988
8089assert SPECIAL_TYPES <= set (TYPE_PATTERN )
8190
8291
8392def match_type (attribute_type ):
93+ """Match an attribute type string to a category."""
8494 try :
8595 return next (category for category , pattern in TYPE_PATTERN .items () if pattern .match (attribute_type ))
8696 except StopIteration :
@@ -444,7 +454,7 @@ def substitute_special_type(match, category, foreign_key_sql, context):
444454 Substitute special types with their native SQL equivalents.
445455
446456 Special types are:
447- - Core type aliases (float32 → float, uuid → binary(16), etc.)
457+ - Core DataJoint types (float32 → float, uuid → binary(16), blob → longblob , etc.)
448458 - ADAPTED types (AttributeTypes in angle brackets)
449459
450460 :param match: dict containing with keys "type" and "comment" -- will be modified in place
@@ -462,9 +472,13 @@ def substitute_special_type(match, category, foreign_key_sql, context):
462472 category = match_type (match ["type" ])
463473 if category in SPECIAL_TYPES :
464474 substitute_special_type (match , category , foreign_key_sql , context )
465- elif category in CORE_TYPE_ALIASES :
466- # Core type alias - substitute with native SQL type
467- match ["type" ] = CORE_TYPE_ALIASES [category ]
475+ elif category in CORE_TYPE_NAMES :
476+ # Core DataJoint type - substitute with native SQL type if mapping exists
477+ core_name = category .lower ()
478+ sql_type = CORE_TYPE_SQL .get (core_name )
479+ if sql_type is not None :
480+ match ["type" ] = sql_type
481+ # else: type passes through as-is (json, date, datetime, char, varchar, enum)
468482 else :
469483 assert False , f"Unknown special type: { category }
470484
@@ -510,13 +524,22 @@ def compile_attribute(line, in_key, foreign_key_sql, context):
510524 raise DataJointError ('An attribute comment must not start with a colon in comment "{comment}"' .format (** match ))
511525
512526 category = match_type (match ["type" ])
527+
513528 if category in SPECIAL_TYPES :
514- match ["comment" ] = ":{type}:{comment}" .format (** match ) # insert custom type into comment
529+ # Core types and AttributeTypes are recorded in comment for reconstruction
530+ match ["comment" ] = ":{type}:{comment}" .format (** match )
515531 substitute_special_type (match , category , foreign_key_sql , context )
532+ elif category in NATIVE_TYPES :
533+ # Non-standard native type - warn user
534+ logger .warning (
535+ f"Non-standard native type '{ match ['type' ]} { match ['name' ]}
536+ "Consider using a core DataJoint type for better portability."
537+ )
516538
517539 # Check for invalid default values on blob types (after type substitution)
518- final_category = match_type (match ["type" ])
519- if final_category == "BLOB" and match ["default" ] not in "DEFAULT NULL" , "NOT NULL" }:
540+ # Note: blob → longblob, so check for NATIVE_BLOB or longblob result
541+ final_type = match ["type" ].lower ()
542+ if ("blob" in final_type ) and match ["default" ] not in "DEFAULT NULL" , "NOT NULL" }:
520543 raise DataJointError ("The default value for blob attributes can only be NULL in:\n {line}" .format (line = line ))
521544
522545 sql = ("`{name}` {type} {default}" + (' COMMENT "{comment}"' if match ["comment" ] else "" )).format (** match )
0 commit comments