From d03ecb09f1ea530f9b49be7bc030045d593cd9f4 Mon Sep 17 00:00:00 2001 From: Tobias Marstaller Date: Mon, 27 Apr 2026 15:33:50 +0200 Subject: [PATCH 1/4] Add SQL exporter --- ctpkLib/SQLConverter.cs | 151 ++++++++++++++++++++++++++++++++++++++++ ctpkLib/ctpkLib.csproj | 1 + 2 files changed, 152 insertions(+) create mode 100644 ctpkLib/SQLConverter.cs diff --git a/ctpkLib/SQLConverter.cs b/ctpkLib/SQLConverter.cs new file mode 100644 index 0000000..600027b --- /dev/null +++ b/ctpkLib/SQLConverter.cs @@ -0,0 +1,151 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Reflection; + +namespace ctpkLib +{ + public static class SQLConverter + { + public static void Main(string[] args) + { + if (args.Length < 1) + { + Console.Error.WriteLine("Usage: ctpkTools [output.sql]"); + Environment.Exit(1); + } + + CTPKLib lib = new CTPKLib(new FileStream(args[0], FileMode.Open)); + + if (args.Length >= 2) + { + using (var writer = new StreamWriter(args[1])) + WriteSQL(lib, writer); + } + else + { + WriteSQL(lib, Console.Out); + } + } + + public static void WriteSQL(CTPKLib lib, TextWriter writer) + { + WriteSchemaSQL(writer); + WriteDataSQL(lib, writer); + } + + private static void WriteSchemaSQL(TextWriter writer) + { + foreach (var objType in GetTypedSectionTypes().OrderBy(GetTableName)) + { + var mapType = GetMapType(objType); + if (mapType == null) continue; + + var fields = mapType.GetFields(BindingFlags.Public | BindingFlags.Instance); + + writer.WriteLine($"CREATE TABLE IF NOT EXISTS [{GetTableName(objType)}] ("); + writer.Write(" [id] INTEGER PRIMARY KEY"); + + foreach (var field in fields) + { + writer.WriteLine(","); + writer.Write($" [{field.Name}] {GetSqlType(field)}"); + } + + writer.WriteLine(); + writer.WriteLine(");"); + writer.WriteLine(); + } + } + + private static void WriteDataSQL(CTPKLib lib, TextWriter writer) + { + var sectionTypeMap = GetTypedSectionTypes() + .ToDictionary(t => t.GetCustomAttribute
().Id); + + foreach (var kvp in lib.Objects.ObjectMap) + { + if (!sectionTypeMap.TryGetValue(kvp.Key, out var objType)) continue; + + var mapType = GetMapType(objType); + if (mapType == null) continue; + + var fields = mapType.GetFields(BindingFlags.Public | BindingFlags.Instance); + if (fields.Length == 0) continue; + + string tableName = GetTableName(objType); + string colList = "[id], " + string.Join(", ", fields.Select(f => $"[{f.Name}]")); + + foreach (var obj in kvp.Value) + { + if (obj.Map.GetType() == typeof(ObjMap)) continue; + + var vals = new List { obj.Id.ToString() }; + foreach (var field in fields) + vals.Add(GetSqlValue(lib, obj, field)); + + writer.WriteLine($"INSERT INTO [{tableName}] ({colList}) VALUES ({string.Join(", ", vals)});"); + } + } + } + + private static string GetSqlValue(CTPKLib lib, CatalogueObject obj, FieldInfo field) + { + object raw = field.GetValue(obj.Map); + + if (Attribute.IsDefined(field, typeof(MappedString))) + { + uint hash = (uint)raw; + if (hash == 0) return "NULL"; + return lib.Strings.StringMap.TryGetValue(hash, out var str) + ? "'" + str.Replace("'", "''") + "'" + : "NULL"; + } + + if (Attribute.IsDefined(field, typeof(MappedObject))) + { + uint id = (uint)raw; + return id == 0 ? "NULL" : id.ToString(); + } + + if (field.FieldType == typeof(bool)) + return (bool)raw ? "1" : "0"; + + if (field.FieldType == typeof(float)) + return ((float)raw).ToString("R", CultureInfo.InvariantCulture); + + return raw.ToString(); + } + + private static string GetSqlType(FieldInfo field) + { + if (Attribute.IsDefined(field, typeof(MappedString))) + return "VARCHAR"; + if (field.FieldType == typeof(float)) + return "REAL"; + return "INTEGER"; + } + + private static string GetTableName(Type objType) + { + var name = objType.Name; + return name.EndsWith("_obj") ? name.Substring(0, name.Length - 4) : name; + } + + private static Type GetMapType(Type objType) + { + return Assembly.GetExecutingAssembly().GetType(objType.FullName + "_map"); + } + + private static IEnumerable GetTypedSectionTypes() + { + return Assembly.GetExecutingAssembly() + .GetTypes() + .Where(t => t.IsSubclassOf(typeof(CatalogueObject)) && + t.GetCustomAttribute
() != null); + } + + } +} diff --git a/ctpkLib/ctpkLib.csproj b/ctpkLib/ctpkLib.csproj index 0dca139..dcfc1dc 100644 --- a/ctpkLib/ctpkLib.csproj +++ b/ctpkLib/ctpkLib.csproj @@ -409,6 +409,7 @@ + From 754af4af10e4323954bd9ca08c1d550f09a0530e Mon Sep 17 00:00:00 2001 From: Tobias Marstaller Date: Mon, 27 Apr 2026 16:01:14 +0200 Subject: [PATCH 2/4] sqlconverter: add comments with proto field ids and fks --- ctpkLib/SQLConverter.cs | 58 +++++++++++++++++++++++++++++++++-------- 1 file changed, 47 insertions(+), 11 deletions(-) diff --git a/ctpkLib/SQLConverter.cs b/ctpkLib/SQLConverter.cs index 600027b..d73abb4 100644 --- a/ctpkLib/SQLConverter.cs +++ b/ctpkLib/SQLConverter.cs @@ -4,6 +4,7 @@ using System.IO; using System.Linq; using System.Reflection; +using ProtoBuf; namespace ctpkLib { @@ -38,23 +39,44 @@ public static void WriteSQL(CTPKLib lib, TextWriter writer) private static void WriteSchemaSQL(TextWriter writer) { + var sectionTableMap = GetTypedSectionTypes() + .ToDictionary(t => t.GetCustomAttribute
().Id, GetTableName); + foreach (var objType in GetTypedSectionTypes().OrderBy(GetTableName)) { var mapType = GetMapType(objType); if (mapType == null) continue; - var fields = mapType.GetFields(BindingFlags.Public | BindingFlags.Instance); + var fields = GetDataFields(mapType); + var sectionId = objType.GetCustomAttribute
().Id; + var idField = mapType.GetField("field_1", BindingFlags.Public | BindingFlags.Instance); + + var fks = fields + .Select(f => new { field = f, attr = f.GetCustomAttribute() }) + .Where(x => x.attr != null && sectionTableMap.ContainsKey(x.attr.SectionId)) + .Select(x => new { x.field, refTable = sectionTableMap[x.attr.SectionId] }) + .ToList(); - writer.WriteLine($"CREATE TABLE IF NOT EXISTS [{GetTableName(objType)}] ("); - writer.Write(" [id] INTEGER PRIMARY KEY"); + writer.WriteLine($"CREATE TABLE IF NOT EXISTS [{GetTableName(objType)}] ( -- 0x{sectionId:X8}"); - foreach (var field in fields) + string idProto = idField != null ? $" -- proto {GetProtoTag(idField)}" : ""; + writer.WriteLine($" [id] INTEGER PRIMARY KEY{(fields.Length > 0 || fks.Count > 0 ? "," : "")}{idProto}"); + + for (int i = 0; i < fields.Length; i++) { - writer.WriteLine(","); - writer.Write($" [{field.Name}] {GetSqlType(field)}"); + var field = fields[i]; + bool isLast = i == fields.Length - 1 && fks.Count == 0; + int tag = GetProtoTag(field); + string proto = tag >= 0 ? $" -- proto {tag}" : ""; + writer.WriteLine($" [{field.Name}] {GetSqlType(field)}{(isLast ? "" : ",")}{proto}"); + } + + for (int i = 0; i < fks.Count; i++) + { + bool isLast = i == fks.Count - 1; + writer.WriteLine($" FOREIGN KEY ([{fks[i].field.Name}]) REFERENCES [{fks[i].refTable}]([id]){(isLast ? "" : ",")}"); } - writer.WriteLine(); writer.WriteLine(");"); writer.WriteLine(); } @@ -72,17 +94,19 @@ private static void WriteDataSQL(CTPKLib lib, TextWriter writer) var mapType = GetMapType(objType); if (mapType == null) continue; - var fields = mapType.GetFields(BindingFlags.Public | BindingFlags.Instance); - if (fields.Length == 0) continue; + var idField = mapType.GetField("field_1", BindingFlags.Public | BindingFlags.Instance); + var fields = GetDataFields(mapType); + if (idField == null && fields.Length == 0) continue; string tableName = GetTableName(objType); - string colList = "[id], " + string.Join(", ", fields.Select(f => $"[{f.Name}]")); + string colList = "[id]" + (fields.Length > 0 ? ", " + string.Join(", ", fields.Select(f => $"[{f.Name}]")) : ""); foreach (var obj in kvp.Value) { if (obj.Map.GetType() == typeof(ObjMap)) continue; - var vals = new List { obj.Id.ToString() }; + string idVal = idField != null ? idField.GetValue(obj.Map).ToString() : obj.Id.ToString(); + var vals = new List { idVal }; foreach (var field in fields) vals.Add(GetSqlValue(lib, obj, field)); @@ -91,6 +115,18 @@ private static void WriteDataSQL(CTPKLib lib, TextWriter writer) } } + private static FieldInfo[] GetDataFields(Type mapType) + { + return mapType.GetFields(BindingFlags.Public | BindingFlags.Instance) + .Where(f => f.Name != "field_1") + .ToArray(); + } + + private static int GetProtoTag(FieldInfo field) + { + return field.GetCustomAttribute()?.Tag ?? -1; + } + private static string GetSqlValue(CTPKLib lib, CatalogueObject obj, FieldInfo field) { object raw = field.GetValue(obj.Map); From efea6f0512c56052856b1832d882c1ec8d44313a Mon Sep 17 00:00:00 2001 From: Tobias Marstaller Date: Mon, 27 Apr 2026 16:01:24 +0200 Subject: [PATCH 3/4] make sqlconverter callable from main entry point --- ctpkParser/Program.cs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/ctpkParser/Program.cs b/ctpkParser/Program.cs index fa5d3e4..d5871dd 100644 --- a/ctpkParser/Program.cs +++ b/ctpkParser/Program.cs @@ -3,6 +3,7 @@ using System.Linq; using System.Threading.Tasks; using System.Windows.Forms; +using ctpkLib; namespace ctpkParser { @@ -12,8 +13,14 @@ static class Program /// The main entry point for the application. /// [STAThread] - static void Main() + static void Main(string[] args) { + if (args.Length > 0) + { + SQLConverter.Main(args); + return; + } + Application.EnableVisualStyles(); Application.SetCompatibleTextRenderingDefault(false); Application.Run(new Form1()); From 98b5791eb3ac32c672a3034944ed1220bd54304f Mon Sep 17 00:00:00 2001 From: Tobias Marstaller Date: Mon, 27 Apr 2026 16:12:32 +0200 Subject: [PATCH 4/4] sqlconverter: auto name columns based on foreign key / reference --- ctpkLib/SQLConverter.cs | 75 +++++++++++++++++++++++++++++++++-------- 1 file changed, 61 insertions(+), 14 deletions(-) diff --git a/ctpkLib/SQLConverter.cs b/ctpkLib/SQLConverter.cs index d73abb4..8dc6d65 100644 --- a/ctpkLib/SQLConverter.cs +++ b/ctpkLib/SQLConverter.cs @@ -32,29 +32,30 @@ public static void Main(string[] args) } public static void WriteSQL(CTPKLib lib, TextWriter writer) - { - WriteSchemaSQL(writer); - WriteDataSQL(lib, writer); - } - - private static void WriteSchemaSQL(TextWriter writer) { var sectionTableMap = GetTypedSectionTypes() .ToDictionary(t => t.GetCustomAttribute
().Id, GetTableName); + WriteSchemaSQL(writer, sectionTableMap); + WriteDataSQL(lib, writer, sectionTableMap); + } + private static void WriteSchemaSQL(TextWriter writer, Dictionary sectionTableMap) + { foreach (var objType in GetTypedSectionTypes().OrderBy(GetTableName)) { var mapType = GetMapType(objType); if (mapType == null) continue; var fields = GetDataFields(mapType); + var colNames = ComputeColumnNames(fields, sectionTableMap); var sectionId = objType.GetCustomAttribute
().Id; var idField = mapType.GetField("field_1", BindingFlags.Public | BindingFlags.Instance); var fks = fields - .Select(f => new { field = f, attr = f.GetCustomAttribute() }) + .Zip(colNames, (f, n) => new { field = f, colName = n }) + .Select(x => new { x.colName, attr = x.field.GetCustomAttribute() }) .Where(x => x.attr != null && sectionTableMap.ContainsKey(x.attr.SectionId)) - .Select(x => new { x.field, refTable = sectionTableMap[x.attr.SectionId] }) + .Select(x => new { x.colName, refTable = sectionTableMap[x.attr.SectionId] }) .ToList(); writer.WriteLine($"CREATE TABLE IF NOT EXISTS [{GetTableName(objType)}] ( -- 0x{sectionId:X8}"); @@ -64,17 +65,16 @@ private static void WriteSchemaSQL(TextWriter writer) for (int i = 0; i < fields.Length; i++) { - var field = fields[i]; bool isLast = i == fields.Length - 1 && fks.Count == 0; - int tag = GetProtoTag(field); + int tag = GetProtoTag(fields[i]); string proto = tag >= 0 ? $" -- proto {tag}" : ""; - writer.WriteLine($" [{field.Name}] {GetSqlType(field)}{(isLast ? "" : ",")}{proto}"); + writer.WriteLine($" [{colNames[i]}] {GetSqlType(fields[i])}{(isLast ? "" : ",")}{proto}"); } for (int i = 0; i < fks.Count; i++) { bool isLast = i == fks.Count - 1; - writer.WriteLine($" FOREIGN KEY ([{fks[i].field.Name}]) REFERENCES [{fks[i].refTable}]([id]){(isLast ? "" : ",")}"); + writer.WriteLine($" FOREIGN KEY ([{fks[i].colName}]) REFERENCES [{fks[i].refTable}]([id]){(isLast ? "" : ",")}"); } writer.WriteLine(");"); @@ -82,7 +82,7 @@ private static void WriteSchemaSQL(TextWriter writer) } } - private static void WriteDataSQL(CTPKLib lib, TextWriter writer) + private static void WriteDataSQL(CTPKLib lib, TextWriter writer, Dictionary sectionTableMap) { var sectionTypeMap = GetTypedSectionTypes() .ToDictionary(t => t.GetCustomAttribute
().Id); @@ -98,8 +98,10 @@ private static void WriteDataSQL(CTPKLib lib, TextWriter writer) var fields = GetDataFields(mapType); if (idField == null && fields.Length == 0) continue; + var colNames = ComputeColumnNames(fields, sectionTableMap); + string tableName = GetTableName(objType); - string colList = "[id]" + (fields.Length > 0 ? ", " + string.Join(", ", fields.Select(f => $"[{f.Name}]")) : ""); + string colList = "[id]" + (fields.Length > 0 ? ", " + string.Join(", ", colNames.Select(n => $"[{n}]")) : ""); foreach (var obj in kvp.Value) { @@ -122,6 +124,51 @@ private static FieldInfo[] GetDataFields(Type mapType) .ToArray(); } + private static string[] ComputeColumnNames(FieldInfo[] fields, Dictionary sectionTableMap) + { + var baseCounts = new Dictionary(); + foreach (var field in fields) + { + var attr = field.GetCustomAttribute(); + if (attr != null && IsHexFieldName(field.Name) && sectionTableMap.TryGetValue(attr.SectionId, out var refTable)) + { + string key = refTable + "_id"; + baseCounts[key] = baseCounts.TryGetValue(key, out int c) ? c + 1 : 1; + } + } + + var names = new string[fields.Length]; + for (int i = 0; i < fields.Length; i++) + { + var field = fields[i]; + var attr = field.GetCustomAttribute(); + string name; + + if (attr != null && IsHexFieldName(field.Name) && sectionTableMap.TryGetValue(attr.SectionId, out var refTable)) + { + string key = refTable + "_id"; + name = baseCounts[key] > 1 ? $"{refTable}_id_{GetProtoTag(field)}" : key; + } + else + { + name = field.Name; + } + + names[i] = name; + } + + return names; + } + + private static bool IsHexFieldName(string name) + { + if (!name.StartsWith("field_") || name.Length <= 6) return false; + foreach (char c in name.Substring(6)) + if (!((c >= '0' && c <= '9') || (c >= 'a' && c <= 'f'))) + return false; + return true; + } + private static int GetProtoTag(FieldInfo field) { return field.GetCustomAttribute()?.Tag ?? -1;