diff --git a/README.md b/README.md index 6541a3f..10330e7 100644 --- a/README.md +++ b/README.md @@ -8,13 +8,16 @@ LLT is designed to make prompt engineering and content generation as powerful an ## ✨ Key Features -- **Razor-inspired DSL** with `@if`, `@foreach`, expressions, metadata, and inline variables -- **Message-oriented syntax** for LLM role structures (system, user, assistant) -- **Powerful metadata filtering** — select the best template by language, model, or custom qualifiers -- **Composable templates** — reuse and render nested templates inside others -- **Deterministic formatting** — preserves whitespace and indentation, ensuring predictable output -- **Expression evaluator** — supports arithmetic, logic, method access, array indexing, and ternary operators -- **Library-driven workflow** — import templates from assemblies, files, or strings +- **Razor-inspired DSL** with `@if`, `@foreach`, expressions, metadata, and inline variables +- **Full expression evaluator** — arithmetic, logic, ternary, null-coalescing (`??`), property checks (`?:`), safe navigation (`?.`), array/object literals, and method calls +- **Message-oriented syntax** for LLM role structures (system, user, assistant, tool) +- **Powerful metadata filtering** — select the best template by language, model, or custom qualifiers +- **Metadata fallback schemes** — hierarchical language fallback (`en-US` → `en` → sibling → default) +- **Composable templates** — reuse and render nested templates inside others, with optional context override (`@render 'name' with ctx`) +- **Extensible function system** — built-in functions plus custom ones registered from C# +- **Deterministic formatting** — indentation is normalized during parsing, inner spacing is preserved, output is predictable +- **Library-driven workflow** — import templates from assemblies, files, or strings +- **Body parsing** — parse template bodies without the `@template` wrapper via `ParseTextTemplate` / `ParseMessagesTemplate` ## 📦 Installation @@ -106,6 +109,14 @@ Console.WriteLine(template.Render()); // Hello GPT-4! ``` Templates are resolved by **metadata specificity**, similar to CSS selector priority. +If no template matches exactly, use fallback retrieval: + +```csharp +// en-US → en → any sibling of the same language root → default language +lib.SetLanguageFallbackScheme(new HierarchicalLanguageFallbackScheme("en")); + +var fallback = lib.RetrieveWithFallback("greeting", new LanguageMetadata("en-US")); +``` ## 💬 Message Templates for LLM Chats @@ -134,59 +145,247 @@ LLT supports special `@messages` syntax for structured chat prompts: } ``` -**Rendered result:** a sequence of chat messages with proper roles (`system`, `user`, `assistant`). - -## 🧠 Supported Syntax - -- Inline expressions: `@(a + b * c)` -- Conditionals: - ```llt - @if cond { - ... - } else if other { - ... - } else { - ... - } - ``` -- Loops: - ```llt - @foreach item in items { - @item - } - ``` -- Variables: - ```llt - @let x = 5 - @x = 10 - @x - ``` -- Nested or external templates: - ```llt - @render 'other_template' - ``` -- Comments: - ```llt - @// line comment - @* - block comment - *@ - ``` +**Rendered result:** a sequence of chat messages with proper roles (`system`, `user`, `assistant`, `tool`). +Roles can be declared statically (`@system message`, `@user message`, `@assistant message`, `@tool message`) +or dynamically via `@message { @role ... }`. + +## 🧠 Language Reference + +### Expressions + +#### Literals + +| Literal | Examples | +|---|---| +| Numbers | `42`, `3.14`, `-7` | +| Strings | `'single'`, `"double"` | +| Booleans | `true`, `false` | +| Null | `null` | +| Arrays | `[1, 2, 3]`, `['a', ctx?.value]`, `[]` (trailing commas allowed) | +| Objects | `{ name: 'Fish', qty: 5 }`, `{ 'key with spaces': 1, [expr]: 2 }` | + +#### Access + +| Syntax | Meaning | +|---|---| +| `ctx` | The root of the rendering context | +| `name` | Property of the root context (same as `ctx.name`) | +| `?name` | **Safe** property access — returns `null` instead of throwing when missing | +| `a.b` / `a.b.c` | Property access (chained) | +| `a?.b` | **Safe navigation** — `null` if `a` is `null` or lacks `b` | +| `a[expr]` | Index access (arrays, dictionaries, strings) | +| `a?[expr]` | **Safe** index access | +| `a.method(args)` | Method call (any public .NET method) | +| `a?.method(args)` | **Safe** method call | +| `func(args)` | Global template function (see [Functions](#-functions)) | + +#### Unary operators + +| Operator | Meaning | Example | +|---|---|---| +| `-` | Negation | `@(-x)` | +| `!` | Logical NOT | `@(!flag)` | +| `#` | Length (string / array / dictionary) | `@(#name)`, `@if #items > 0` | +| `+` | No-op (kept for symmetry) | `@(+x)` | + +#### Binary operators (by precedence, high → low) + +| Precedence | Operators | Meaning | +|---|---|---| +| 1 | `*` `/` `%` | Multiplication, division, modulus | +| 2 | `+` `-` | Addition, subtraction, string concatenation | +| 3 | `<` `<=` `>` `>=` | Relational | +| 4 | `?:` | **Has** operator — checks whether the left operand has the right property | +| 5 | `==` `!=` | Equality | +| 6 | `&&` | Logical AND | +| 7 | `\|\|` | Logical OR | +| 8 | `??` | **Null-coalescing** — right operand when the left is `null` or missing | +| 9 | `? :` | Ternary conditional | + +#### Null-coalescing, Has and Safe navigation + +```llt +@template t { + 1: @(?value ?? 'No value') + 2: @(ctx ?: 'value' ? value ?? 'Null' : 'No value') +} +``` + +| Render context | Output | +|---|---| +| `new { value = "Hello" }` | `1: Hello` / `2: Hello` | +| `new { value = (string?)null }` | `1: No value` / `2: Null` — the property *exists* but is `null` | +| `new { }` | `1: No value` / `2: No value` — the property is *missing* | + +- `??` treats **missing** and `null` the same way. +- `?:` (has) distinguishes them: it only checks **existence**, so it works great as the condition + when you need different fallbacks for `null` vs. missing: + +```llt +@(user ?: 'name' ? user.name : 'Anonymous') +@(user?.name ?? 'Anonymous') @/ Safe navigation + coalescing +``` + +> [!TIP] +> `@`-statements in plain text accept *simple* expressions (unary + member access). +> Wrap binary expressions in parentheses: `@(a + b)` — inside `@if`, `@while` etc. full expressions are allowed. + +#### Formatting + +An expression can be followed by `:` and a format string: + +```llt +Price: @price:'0.00' +``` + +### Statements + +| Statement | Syntax | Description | +|---|---|---| +| If | `@if cond { ... }` | Conditional block | +| Else | `else { ... }`, `else if cond { ... }` or `@else { ... }` | Optional `@` before `else` is allowed | +| Foreach | `@foreach item in items { ... }` | Iteration; loop variable is scoped to the block | +| While | `@while cond { ... }` | Conditional loop | +| Let | `@let x = expr` | Declares a new variable (lexically scoped) | +| Assign | `@x = expr` | Assigns to an existing variable | +| Render | `@render 'name'`, `@render 'name' with expr` | Renders another template, optionally with a new context | +| Output | `@expr`, `@(expr)`, `@expr:'format'` | Prints an expression value | +| Comment | `@/ line comment` | C#-style line comment — to the end of the line | +| Comment | `@* block comment *@` | Razor-style block comment | + +Comments are skipped by the parser and can be placed wherever whitespace is allowed, including inside template bodies. + +Escapes: `@@` renders a literal `@`, `{{` renders `{`, `}}` renders `}`. + +Multi-line raw text can be wrapped in five backticks to avoid escaping: + +```llt +@template code_sample +{ + ````` + @if isNotParsed { this is raw text, not a statement } + ````` +} +``` + +### Metadata + +Attach metadata to a template with the `@metadata` block (only constant values are allowed): + +```llt +@template greeting +{ + @metadata { lang: 'en', model: 'gpt-4', version: 2 } + Hello! +} +``` + +Built-in metadata keys (parsed by the corresponding factories): + +| Key | Metadata type | +|---|---| +| `lang` | `LanguageMetadata` | +| `model` | `TargetModelMetadata` | +| `model_family` | `TargetModelFamilyMetadata` | +| `version` | `VersionMetadata` | + +Pass factories to `Parse` so the keys are recognized; unknown keys become `AdditionalMetadata`: + +```csharp +var parser = new LLTParser(); +var templates = parser.Parse(src, new MetadataFactory[] +{ + new LanguageMetadataFactory(), + new TargetModelMetadataFactory(), + new VersionMetadataFactory(), +}); +``` + +#### Custom metadata + +```csharp +public class MyMetadata : IMetadata +{ + public MyMetadata(string value) => Value = value; + public string Value { get; } +} + +public class MyMetadataFactory : MetadataFactory +{ + public override bool TryCreateMetadata(string key, TemplateDataAccessor value, out IMetadata metadata) + { + if (key == "my_key") + { + metadata = new MyMetadata(value.ToString()); + return true; + } + metadata = null; + return false; + } +} +``` + +#### Fallback schemes + +- `lib.SetLanguageFallbackScheme(new HierarchicalLanguageFallbackScheme("en"))` — walks up the language + hierarchy (`en-US` → `en`), then tries siblings, then the default language. +- `lib.SetLanguageFallbackScheme(new MajorLanguageFallbackScheme())` — falls back to the major language group. +- `lib.SetFallbackScheme(typeof(LanguageMetadata), scheme)` — register a custom `MetadataFallbackScheme` for any metadata type. + +Retrieval API: `Retrieve`, `TryRetrieve`, `TryRetrieveBest`, `RetrieveWithFallback`, `RetrieveAll` — +each with optional `identifier` and metadata arguments. + +## 🔧 Functions + +Built-in functions (`TemplateFunctions.All`): + +| Function | Description | Example | +|---|---|---| +| `type(x)` | Type name of the value (`string`, `number`, `boolean`, `object`, `array`, `null`, ...) | `@type(item)` | +| `length(x)` | Length of a string, array or dictionary | `@length(items)` | +| `strcat(a, b, ...)` | Concatenates values into a string | `@strcat('Hi, ', name)` | +| `substr(s, start, len)` | Substring | `@substr(name, 0, 3)` | + +Register custom functions and pass them to `Render`: + +```csharp +var functions = new TemplateFunctionSet(includeDefault: true, + new TemplateFunction("shout", (self, args) => args[0].ToString().ToUpperInvariant())); + +var rendered = template.Render(new { name = "Andrew" }, functions); +``` + +```llt +Hello, @shout(name)! @/ → Hello, ANDREW! +``` ## 🧾 Formatting & Scoping Rules -- Whitespace and indentation in source templates are **fully preserved** +- Leading indentation is **normalized during parsing**: every line loses up to `depth × TabSize` leading + whitespace, where `depth` is the block nesting level. You can write templates with comfortable + indentation — the common base indent is stripped and the output stays clean +- Inner whitespace, line breaks and blank lines are preserved as written +- Leading/trailing blank lines of blocks are trimmed; indentation before complex statements + (`@if`, `@foreach`, `@while`) is removed +- Lines containing only non-rendering constructs — `@/` and `@* *@` comments, `@let` declarations, + variable assignments — are removed entirely: the surrounding line breaks are trimmed so they + leave no blank lines in the output +- `TabSize` (default `4`) on `LLTParser` controls how many columns one indent level takes during refinement - `@let` variables are **lexically scoped** - Loop variables do **not leak outside** their block - Nested `@if` and `@foreach` blocks behave predictably, matching C#‑like logical semantics ## 🏗️ Architecture Overview -- **LLTParser** – parses source text into ASTs (`TemplateNode`, `TemplateExpressionNode`, etc.) -- **Template** – runtime object capable of rendering with dynamic context -- **TemplateLibrary** – registry and loader for collections of templates with filtering +- **LLTParser** – parses source text into ASTs (`TemplateNode`, `TemplateExpressionNode`, etc.); + supports full-file parsing (`Parse`) and body-only parsing (`ParseTextTemplate`, `ParseMessagesTemplate`) +- **Template** – runtime object capable of rendering with dynamic context (`TextTemplate`, `MessagesTemplate`) +- **TemplateLibrary** – registry and loader for collections of templates with filtering and fallbacks - **Metadata System** – extensible mechanism for attaching attributes such as language, model, or custom version -- **ChatMessage Model** – unified representation for structured message templates (`system`, `user`, `assistant`) +- **Data Accessors** – reflection / dictionary / array wrappers (`DataAccessorFactory`) with + `PropertiesToLowerCase`, `KeysToLowerCase` and `Snapshot` options +- **Function System** – `TemplateFunctionSet` / `TemplateFunction` for extending template logic +- **ChatMessage Model** – unified representation for structured message templates (`system`, `user`, `assistant`, `tool`) ## 🔖 License diff --git a/src/LLTSharp/LLTParser.cs b/src/LLTSharp/LLTParser.cs index c2471f3..73897e6 100644 --- a/src/LLTSharp/LLTParser.cs +++ b/src/LLTSharp/LLTParser.cs @@ -17,10 +17,12 @@ namespace LLTSharp /// public class LLTParser : ITemplateParser { - protected class LLTParsingContext + public class LLTParsingContext { public TemplateLibrary LocalLibrary { get; set; } + public int TabSize { get; set; } = 4; public IEnumerable MetadataFactories { get; set; } + public IMetadataCollection? DefaultMetadataCollection { get; set; } } private static void DeclareValues(ParserBuilder builder) @@ -428,7 +430,7 @@ private static void DeclareMessagesTemplates(ParserBuilder builder) .Literal('{') .Optional(b => // 5 b.Rule("metadata_block")) - .Rule("message_statements") // 6 + .Rule("messages_statements") // 6 .Literal('}') .Transform(v => @@ -444,10 +446,11 @@ private static void DeclareMessagesTemplates(ParserBuilder builder) if (v.TryGetValue(5) is MetadataCollection collection) metadata = metadata.Concat(collection); + var context = v.GetParsingParameter(); var node = v.GetValue(6); - node.Refine(depth: 1); + node.Refine(depth: 1, context.TabSize); - var library = v.GetParsingParameter().LocalLibrary; + var library = context.LocalLibrary; var template = new MessagesTemplate(node, new MetadataCollection(metadata), library); library.Add(template); return template; @@ -455,11 +458,11 @@ private static void DeclareMessagesTemplates(ParserBuilder builder) builder.CreateRule("messages_template_block") .Literal('{') - .Rule("message_statements") + .Rule("messages_statements") .Literal('}') .Transform(v => v.GetValue(1)); - builder.CreateRule("message_statements") + builder.CreateRule("messages_statements") .ZeroOrMore(b => b .Literal('@') .Choice( @@ -513,11 +516,12 @@ private static void DeclareMessagesTemplates(ParserBuilder builder) .Rule("messages_template_block") .Optional( b => b + .Optional(b => b.Literal('@')) .Keyword("else") .Choice( b => b.Rule("messages_if"), b => b.Rule("messages_template_block")) - .Transform(v => v.GetValue(1))) + .Transform(v => v.GetValue(2))) .Transform(v => { var condition = v.GetValue(1); @@ -612,10 +616,11 @@ private static void DeclareTextTemplates(ParserBuilder builder) if (v.TryGetValue(4) is MetadataCollection collection) metadata = metadata.Concat(collection); + var context = v.GetParsingParameter(); var node = v.GetValue(5); - node.Refine(depth: 1); + node.Refine(depth: 1, context.TabSize); - var library = v.GetParsingParameter().LocalLibrary; + var library = context.LocalLibrary; var template = new TextTemplate(node, new MetadataCollection(metadata), library); library.Add(template); return template; @@ -695,11 +700,12 @@ private static void DeclareTextTemplates(ParserBuilder builder) .Rule("text_template_block") .Optional( b => b + .Optional(b => b.Literal('@')) .Keyword("else") .Choice( b => b.Rule("text_if"), b => b.Rule("text_template_block")) - .Transform(v => v.GetValue(1))) + .Transform(v => v.GetValue(2))) .Transform(v => { var condition = v.GetValue(1); @@ -814,6 +820,40 @@ private static void DeclareMainRules(ParserBuilder builder) .ZeroOrMore(b => b.Rule("template")) .EOF() .Transform(v => v.Children[0].SelectArray()); + + builder.CreateRule("text_template_content") + .Rule("text_statements") + .EOF() + .Transform(v => + { + var context = v.GetParsingParameter(); + var node = v.Children[0].GetValue(); + node.Refine(depth: 0, context.TabSize); + + var metadata = context.DefaultMetadataCollection ?? MetadataCollection.Empty; + var library = context.LocalLibrary; + var template = new TextTemplate(node, metadata, library); + library.Add(template); + + return template; + }); + + builder.CreateRule("messages_template_content") + .Rule("messages_statements") + .EOF() + .Transform(v => + { + var context = v.GetParsingParameter(); + var node = v.Children[0].GetValue(); + node.Refine(depth: 0, context.TabSize); + + var metadata = context.DefaultMetadataCollection ?? MetadataCollection.Empty; + var library = context.LocalLibrary; + var template = new MessagesTemplate(node, metadata, library); + library.Add(template); + + return template; + }); } /// @@ -821,6 +861,11 @@ private static void DeclareMainRules(ParserBuilder builder) /// public Parser Parser { get; } + /// + /// The tab size used for indentation. + /// + public int TabSize { get; set; } = 4; + public LLTParser() { var builder = new ParserBuilder(); @@ -863,14 +908,60 @@ protected virtual void ModifyParser(ParserBuilder builder) { } + /// + /// Parses the input string into a collection of templates. + /// + /// The string that contains template declarations + /// + /// A collection of metadata factories that can be used to create metadata for the templates. + /// If null, all metadata will be created as . + /// + /// A collection of templates that were parsed from the input string. public virtual IEnumerable Parse(string templateString, IEnumerable? metadataFactories = null) { var ctx = new LLTParsingContext { LocalLibrary = new TemplateLibrary(), + TabSize = TabSize, MetadataFactories = metadataFactories?.ToList() ?? Enumerable.Empty() }; return Parser.Parse(templateString, ctx).GetValue>(); } + + /// + /// Parses a text template from a body string. + /// + /// The body of the template. + /// The metadata to put into the template. + /// The parsed text template. + public virtual ITextTemplate ParseTextTemplate(string templateBody, IMetadataCollection? metadata = null) + { + var ctx = new LLTParsingContext + { + LocalLibrary = new TemplateLibrary(), + TabSize = TabSize, + MetadataFactories = Enumerable.Empty(), + DefaultMetadataCollection = metadata + }; + return Parser.ParseRule("text_template_content", templateBody, ctx).GetValue(); + } + + /// + /// Parses a messages template from a body string. + /// + /// The body of the template. + /// The metadata to put into the template. + /// The parsed messages template. + public virtual IMessagesTemplate ParseMessagesTemplate(string templateBody, IMetadataCollection? metadata = null) + { + var ctx = new LLTParsingContext + { + LocalLibrary = new TemplateLibrary(), + TabSize = TabSize, + MetadataFactories = Enumerable.Empty(), + DefaultMetadataCollection = metadata + }; + return Parser.ParseRule("messages_template_content", templateBody, ctx).GetValue(); + } } } \ No newline at end of file diff --git a/src/LLTSharp/LLTSharp.csproj b/src/LLTSharp/LLTSharp.csproj index 466d1d9..efccdac 100644 --- a/src/LLTSharp/LLTSharp.csproj +++ b/src/LLTSharp/LLTSharp.csproj @@ -8,7 +8,7 @@ LLTSharp - 1.5.0 + 1.6.0 Roman K. RomeCore LLTSharp @@ -25,7 +25,7 @@ - + \ No newline at end of file diff --git a/src/LLTSharp/Locale/HierarchicalLanguageFallbackScheme.cs b/src/LLTSharp/Locale/HierarchicalLanguageFallbackScheme.cs index bdf8e79..02b680d 100644 --- a/src/LLTSharp/Locale/HierarchicalLanguageFallbackScheme.cs +++ b/src/LLTSharp/Locale/HierarchicalLanguageFallbackScheme.cs @@ -32,13 +32,13 @@ public LanguageCode GetFallbackLanguage(LanguageCode targetLanguage, IEnumerable throw new ArgumentNullException(nameof(availableLanguages)); // Materialise to a list to avoid multiple enumeration. - List availableList = availableLanguages as List ?? availableLanguages.ToList(); + HashSet availableSet = availableLanguages as HashSet ?? new HashSet(availableLanguages); - if (availableList.Count == 0) + if (availableSet.Count == 0) throw new ArgumentException("Available languages collection is empty.", nameof(availableLanguages)); // 1. Exact match is already the best choice. - if (availableList.Any(l => l == targetLanguage)) + if (availableSet.Contains(targetLanguage)) return targetLanguage; // 2. Walk up the parent chain: e.g., zh-Hans-CN → zh-Hans → zh @@ -49,23 +49,23 @@ public LanguageCode GetFallbackLanguage(LanguageCode targetLanguage, IEnumerable if (parent == current) break; - if (availableList.Any(l => l == parent)) + if (availableSet.Contains(parent)) return parent; current = parent; } // 3. Look for any sibling belonging to the same root language (e.g., fr-FR when fr-CA was requested) - var sibling = availableList.FirstOrDefault(l => l.IsSubLanguageOf(current)); + var sibling = availableSet.FirstOrDefault(l => l.IsSubLanguageOf(current)); if (sibling.FullCode != null) return sibling; // 4. Fall back to the explicitly configured default language (if available). - if (_defaultLanguage.HasValue && availableList.Any(l => l == _defaultLanguage.Value)) + if (_defaultLanguage.HasValue && availableSet.Contains(_defaultLanguage.Value)) return _defaultLanguage.Value; // 5. Ultimate fallback – return the first language from the available set. - return availableList[0]; + return availableSet.First(); } } } \ No newline at end of file diff --git a/src/LLTSharp/MessagesTemplateNode.cs b/src/LLTSharp/MessagesTemplateNode.cs index 325cf3e..762e1d3 100644 --- a/src/LLTSharp/MessagesTemplateNode.cs +++ b/src/LLTSharp/MessagesTemplateNode.cs @@ -19,7 +19,8 @@ public abstract class MessagesTemplateNode /// Refines the template after parsing an AST to remove indents and unnecessary leading/trailing whitespaces. /// /// The current depth of refinement. Used for indentation purposes. - public virtual void Refine(int depth) + /// The size of a tab character. Used for indentation purposes. + public virtual void Refine(int depth, int tabSize) { } } diff --git a/src/LLTSharp/TemplateNodes/MessagesTemplateEntryNode.cs b/src/LLTSharp/TemplateNodes/MessagesTemplateEntryNode.cs index 496f4ce..a83f000 100644 --- a/src/LLTSharp/TemplateNodes/MessagesTemplateEntryNode.cs +++ b/src/LLTSharp/TemplateNodes/MessagesTemplateEntryNode.cs @@ -48,9 +48,9 @@ public override IEnumerable Render(TemplateContextAccessor context) return new Message[] { message }; } - public override void Refine(int depth) + public override void Refine(int depth, int tabSize) { - Child.Refine(depth + 1); + Child.Refine(depth + 1, tabSize); } public override string ToString() diff --git a/src/LLTSharp/TemplateNodes/MessagesTemplateForeachNode.cs b/src/LLTSharp/TemplateNodes/MessagesTemplateForeachNode.cs index 909acee..91ddd1a 100644 --- a/src/LLTSharp/TemplateNodes/MessagesTemplateForeachNode.cs +++ b/src/LLTSharp/TemplateNodes/MessagesTemplateForeachNode.cs @@ -62,9 +62,9 @@ public override IEnumerable Render(TemplateContextAccessor context) return messages; } - public override void Refine(int depth) + public override void Refine(int depth, int tabSize) { - Child.Refine(depth + 1); + Child.Refine(depth + 1, tabSize); } public override string ToString() diff --git a/src/LLTSharp/TemplateNodes/MessagesTemplateIfElseNode.cs b/src/LLTSharp/TemplateNodes/MessagesTemplateIfElseNode.cs index 8802936..6941520 100644 --- a/src/LLTSharp/TemplateNodes/MessagesTemplateIfElseNode.cs +++ b/src/LLTSharp/TemplateNodes/MessagesTemplateIfElseNode.cs @@ -56,17 +56,17 @@ public override IEnumerable Render(TemplateContextAccessor context) return result ?? Enumerable.Empty(); } - public override void Refine(int depth) + public override void Refine(int depth, int tabSize) { - IfBranch.Refine(depth + 1); + IfBranch.Refine(depth + 1, tabSize); if (ElseBranch == null) return; if (ElseBranch is MessagesTemplateIfElseNode) - ElseBranch?.Refine(depth); // Same depth for nested if-else + ElseBranch?.Refine(depth, tabSize); // Same depth for nested if-else else - ElseBranch?.Refine(depth + 1); + ElseBranch?.Refine(depth + 1, tabSize); } public override string ToString() diff --git a/src/LLTSharp/TemplateNodes/MessagesTemplateSequentialNode.cs b/src/LLTSharp/TemplateNodes/MessagesTemplateSequentialNode.cs index f730e00..c6b0d78 100644 --- a/src/LLTSharp/TemplateNodes/MessagesTemplateSequentialNode.cs +++ b/src/LLTSharp/TemplateNodes/MessagesTemplateSequentialNode.cs @@ -35,10 +35,10 @@ public override IEnumerable Render(TemplateContextAccessor context) return messages; } - public override void Refine(int depth) + public override void Refine(int depth, int tabSize) { foreach (var child in Children) - child.Refine(depth + 1); + child.Refine(depth + 1, tabSize); } public override string ToString() @@ -46,7 +46,7 @@ public override string ToString() StringBuilder sb = new StringBuilder(); foreach (var child in Children) - sb.Append(child); + sb.AppendLine(child.ToString()); return sb.ToString(); } diff --git a/src/LLTSharp/TemplateNodes/MessagesTemplateWhileNode.cs b/src/LLTSharp/TemplateNodes/MessagesTemplateWhileNode.cs index 263e9d1..997792a 100644 --- a/src/LLTSharp/TemplateNodes/MessagesTemplateWhileNode.cs +++ b/src/LLTSharp/TemplateNodes/MessagesTemplateWhileNode.cs @@ -54,9 +54,9 @@ private bool EvaluateCondition(TemplateContextAccessor context) return value.AsBoolean(); } - public override void Refine(int depth) + public override void Refine(int depth, int tabSize) { - Child.Refine(depth + 1); + Child.Refine(depth + 1, tabSize); } public override string ToString() diff --git a/src/LLTSharp/TemplateNodes/TextTemplateForeachNode.cs b/src/LLTSharp/TemplateNodes/TextTemplateForeachNode.cs index 35c97c4..2965f1d 100644 --- a/src/LLTSharp/TemplateNodes/TextTemplateForeachNode.cs +++ b/src/LLTSharp/TemplateNodes/TextTemplateForeachNode.cs @@ -10,6 +10,8 @@ namespace LLTSharp.TemplateNodes /// public class TextTemplateForeachNode : TextTemplateNode { + public override bool IsComplexStatement => true; + /// /// Gets the source expression that provides the data for iteration. /// @@ -67,14 +69,14 @@ public override string Render(TemplateContextAccessor context) return result.ToString(); } - public override void Refine(int depth) + public override void Refine(int depth, int tabSize) { - Child.Refine(depth + 1); + Child.Refine(depth + 1, tabSize); } public override string ToString() { - return $"@foreach {IterableName} in {Source} {{\n{Child}\n}}"; + return $"@foreach {IterableName} in {Source}\n{{\n{Child}\n}}"; } } } \ No newline at end of file diff --git a/src/LLTSharp/TemplateNodes/TextTemplateIfElseNode.cs b/src/LLTSharp/TemplateNodes/TextTemplateIfElseNode.cs index ad40b5a..caad6f4 100644 --- a/src/LLTSharp/TemplateNodes/TextTemplateIfElseNode.cs +++ b/src/LLTSharp/TemplateNodes/TextTemplateIfElseNode.cs @@ -9,6 +9,8 @@ namespace LLTSharp.TemplateNodes /// public class TextTemplateIfElseNode : TextTemplateNode { + public override bool IsComplexStatement => true; + /// /// The condition to evaluate. /// @@ -56,24 +58,24 @@ public override string Render(TemplateContextAccessor context) return result; } - public override void Refine(int depth) + public override void Refine(int depth, int tabSize) { - IfBranch.Refine(depth + 1); + IfBranch.Refine(depth + 1, tabSize); if (ElseBranch == null) return; if (ElseBranch is TextTemplateIfElseNode) - ElseBranch?.Refine(depth); // Same depth for nested if-else + ElseBranch?.Refine(depth, tabSize); // Same depth for nested if-else else - ElseBranch?.Refine(depth + 1); + ElseBranch?.Refine(depth + 1, tabSize); } public override string ToString() { if (ElseBranch == null) - return $"@if {Condition} \n {{\n{IfBranch}\n}} \n"; - return $"@if {Condition} \n {{\n{IfBranch}\n}} \n else \n {{\n{ElseBranch}\n}}"; + return $"@if {Condition}\n{{\n{IfBranch}\n}} \n"; + return $"@if {Condition}\n{{\n{IfBranch}\n}}\nelse\n{{\n{ElseBranch}\n}}"; } } } \ No newline at end of file diff --git a/src/LLTSharp/TemplateNodes/TextTemplatePlainTextNode.cs b/src/LLTSharp/TemplateNodes/TextTemplatePlainTextNode.cs index 5d7aa78..01362df 100644 --- a/src/LLTSharp/TemplateNodes/TextTemplatePlainTextNode.cs +++ b/src/LLTSharp/TemplateNodes/TextTemplatePlainTextNode.cs @@ -25,7 +25,7 @@ public TextTemplatePlainTextNode(string text) // This will be called only if it is a single node in the parent's node. // So we can remove indentation and some leading/trailing whitespaces. - public override void Refine(int depth) + public override void Refine(int depth, int tabSize) { var sb = new StringBuilder(); @@ -34,7 +34,7 @@ public override void Refine(int depth) int startLine = lines.Length > 1 && string.IsNullOrWhiteSpace(lines[0]) ? 1 : 0; int endLine = lines.Length > 1 && string.IsNullOrWhiteSpace(lines[lines.Length - 1]) ? lines.Length - 1 : lines.Length; - int maxIndent = depth * 4; + int maxIndent = depth * tabSize; for (int li = startLine; li < endLine; li++) { var line = lines[li]; @@ -44,7 +44,7 @@ public override void Refine(int depth) while (startIndex < line.Length && indent < maxIndent) { if (line[startIndex] == '\t') - indent += 4; + indent += tabSize - indent % tabSize; else if (line[startIndex] == ' ') indent++; else @@ -52,10 +52,9 @@ public override void Refine(int depth) startIndex++; } - if (li == endLine - 1) - sb.Append(line.Substring(startIndex)); - else - sb.AppendLine(line.Substring(startIndex)); + if (li != startLine) + sb.AppendLine(); + sb.Append(line, startIndex, line.Length - startIndex); } Text = sb.ToString(); diff --git a/src/LLTSharp/TemplateNodes/TextTemplateSequentialNode.cs b/src/LLTSharp/TemplateNodes/TextTemplateSequentialNode.cs index ef09328..14dfa43 100644 --- a/src/LLTSharp/TemplateNodes/TextTemplateSequentialNode.cs +++ b/src/LLTSharp/TemplateNodes/TextTemplateSequentialNode.cs @@ -63,7 +63,7 @@ public override string Render(TemplateContextAccessor context) return result.ToString(); } - public override void Refine(int depth) + public override void Refine(int depth, int tabSize) { // 1. Remove indents and unnecessary start and end lines. for (int i = 0; i < Children.Count; i++) @@ -80,7 +80,7 @@ public override void Refine(int depth) : lines.Length; var sb = new StringBuilder(); - int maxIndent = depth * 4; + int maxIndent = depth * tabSize; for (int li = startLine; li < endLine; li++) { @@ -92,7 +92,7 @@ public override void Refine(int depth) while (startIndex < line.Length && indent < maxIndent) { if (line[startIndex] == '\t') - indent += 4; + indent += tabSize - indent % tabSize; else if (line[startIndex] == ' ') indent++; else @@ -100,17 +100,16 @@ public override void Refine(int depth) startIndex++; } - if (li == endLine - 1) - sb.Append(line.Substring(startIndex)); - else - sb.AppendLine(line.Substring(startIndex)); + if (li != startLine) + sb.AppendLine(); + sb.Append(line, startIndex, line.Length - startIndex); } plainTextChild.Text = sb.ToString(); } else { - child.Refine(depth); + child.Refine(depth, tabSize); } } @@ -198,7 +197,7 @@ public override void Refine(int depth) // 3. Combine and remove plaintext nodes. var newChildren = new List(); - StringBuilder childrenSb = new StringBuilder(); + var childrenSb = new StringBuilder(); foreach (var child in Children) { @@ -210,6 +209,33 @@ public override void Refine(int depth) { if (childrenSb.Length > 0) { + // Remove trailing indentation if the next element is complex statement. + if (child.IsComplexStatement) + { + int i = childrenSb.Length - 1; + int cutoff = i; + int indent = 0; + int maxIndent = depth * tabSize; + bool hitNewLine = false; + while (i >= 0 && childrenSb[i] is ' ' or '\t' or '\n' or '\r') + { + if (childrenSb[i] == '\t') + indent += tabSize - indent % tabSize; + else if (childrenSb[i] == ' ') + indent++; + else + { + hitNewLine = true; + break; + } + i--; + if (indent < maxIndent) + cutoff = i; + } + if (hitNewLine) + childrenSb.Length = cutoff + 1; + } + newChildren.Add(new TextTemplatePlainTextNode(childrenSb.ToString())); childrenSb.Clear(); } diff --git a/src/LLTSharp/TemplateNodes/TextTemplateWhileNode.cs b/src/LLTSharp/TemplateNodes/TextTemplateWhileNode.cs index d3bc743..0bdadb2 100644 --- a/src/LLTSharp/TemplateNodes/TextTemplateWhileNode.cs +++ b/src/LLTSharp/TemplateNodes/TextTemplateWhileNode.cs @@ -8,6 +8,8 @@ namespace LLTSharp.TemplateNodes /// public class TextTemplateWhileNode : TextTemplateNode { + public override bool IsComplexStatement => true; + /// /// Gets the condition expression controlling the loop execution. /// @@ -58,14 +60,14 @@ private bool EvaluateCondition(TemplateContextAccessor context) return value.AsBoolean(); } - public override void Refine(int depth) + public override void Refine(int depth, int tabSize) { - Child.Refine(depth + 1); + Child.Refine(depth + 1, tabSize); } public override string ToString() { - return $"@while {Condition} {{\n{Child}\n}}"; + return $"@while {Condition}\n{{\n{Child}\n}}"; } } diff --git a/src/LLTSharp/TextTemplateNode.cs b/src/LLTSharp/TextTemplateNode.cs index 06f9f2e..e64e6d5 100644 --- a/src/LLTSharp/TextTemplateNode.cs +++ b/src/LLTSharp/TextTemplateNode.cs @@ -1,8 +1,4 @@ -using System; -using System.Collections.Generic; -using System.Text; - -namespace LLTSharp +namespace LLTSharp { /// /// Represents a node in the prompt template hierarchy. @@ -14,6 +10,11 @@ public abstract class TextTemplateNode /// public virtual bool Renderable => true; + /// + /// Gets the value indicating whether this node represents a complex statement. + /// + public virtual bool IsComplexStatement => false; + /// /// Renders the prompt template node using the provided data accessor. /// @@ -25,7 +26,8 @@ public abstract class TextTemplateNode /// Refines the template after parsing an AST to remove indents and unnecessary leading/trailing whitespaces. /// /// The current depth of refinement. Used for indentation purposes. - public virtual void Refine(int depth) + /// The size of a tab character. Used for indentation purposes. + public virtual void Refine(int depth, int tabSize) { } } diff --git a/tests/LLTSharp.Tests/TemplateFormattingTests.cs b/tests/LLTSharp.Tests/TemplateFormattingTests.cs index 6b9c218..f6d1910 100644 --- a/tests/LLTSharp.Tests/TemplateFormattingTests.cs +++ b/tests/LLTSharp.Tests/TemplateFormattingTests.cs @@ -1,15 +1,14 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; +using Microsoft.VisualStudio.TestPlatform.Utilities; +using Xunit.Abstractions; namespace LLTSharp.Tests { /// /// The tests that verify the formatting of templates, including newlines and indentation. /// - public class TemplateFormattingTests +#pragma warning disable CS9113 + public class TemplateFormattingTests(ITestOutputHelper output) +#pragma warning restore CS9113 { [Fact] public void IfElseTemplateFormatting() @@ -313,6 +312,88 @@ No positive values! Assert.Equal(expected, rendered); } + [Fact] + public void ComplexIndentedTemplateFormatting() + { + var parser = new LLTParser(); + + var templateStr = + """ + @template indented_format + { + You are helpful assistant. + @if skills + { + + You are prived with skills, please load with + `skill-load` tool when needed. + @foreach skill in skills + { + + @skill.name + @skill.description + @if skill.body + { + + @skill.body + + } + + } + + } + Be helpful. + } + """; + + var template = parser.Parse(templateStr).First(); + + var rendered = template.Render(new + { + skills = new[] + { + new + { + name = "caveman", + description = "Skill that uses 75% less tokens", + body = "" + }, + new + { + name = "find-skills", + description = "A guide how to find skills", + body = "Just use `web-search` tool!" + }, + } + }).ToString(); + + var expected = + """ + You are helpful assistant. + + You are prived with skills, please load with + `skill-load` tool when needed. + + caveman + Skill that uses 75% less tokens + + + find-skills + A guide how to find skills + + Just use `web-search` tool! + + + + Be helpful. + """; + + // output.WriteLine(((TextTemplate)template).MainNode.ToString()); + // output.WriteLine(rendered); + + Assert.Equal(expected, rendered); + } + [Fact] public void WhileTemplateFormatting() { diff --git a/tests/LLTSharp.Tests/TemplateRenderingTests.cs b/tests/LLTSharp.Tests/TemplateRenderingTests.cs index 8e6f67e..95c681e 100644 --- a/tests/LLTSharp.Tests/TemplateRenderingTests.cs +++ b/tests/LLTSharp.Tests/TemplateRenderingTests.cs @@ -1,9 +1,4 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; -using LLTSharp.Metadata; +using LLTSharp.Metadata; namespace LLTSharp.Tests { @@ -38,7 +33,7 @@ @if age > 18 { You are an adult. } - else if age + else { You are too young! }