Fix #12796: handle arrays of user-defined types in custom attributes

Empty arrays of non-encodable types (classes, records, structs) in custom
attributes now compile successfully by encoding them as System.Object[] per
ECMA-335. Non-empty arrays of such types produce FS3885 instead of the
internal error FS0192.

Changes:
- IlxGen.fs: detect unencodable element types in array attribute args
- il.fs: add 0x51 (SERIALIZATION_TYPE_TAGGED_OBJECT) decoder
- FSComp.txt: new diagnostic FS3885
- Tests: 7 new AttributeCheckingTests (compile + ILVerify + run)

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

Author: T-Gro

docs/release-notes/.FSharp.Compiler.Service/11.0.100.md

@@ -27,6 +27,7 @@
 * Fix box instruction for literal upcasts. (Issue [#18319](https://github.com/dotnet/fsharp/issues/18319), [PR #19338](https://github.com/dotnet/fsharp/pull/19338))
 * Fix Decimal Literal causes InvalidProgramException in Debug builds. (Issue [#18956](https://github.com/dotnet/fsharp/issues/18956), [PR #19338](https://github.com/dotnet/fsharp/pull/19338))
 * Fix `AttributeUsage.AllowMultiple` not being inherited for attributes subclassed in C#. ([Issue #17107](https://github.com/dotnet/fsharp/issues/17107), [PR #19315](https://github.com/dotnet/fsharp/pull/19315))
+* Fix internal error `FS0192: encodeCustomAttrElemType` when using arrays of user-defined types as custom attribute arguments. Empty arrays (e.g. `[<DefaultValue([||] : A[])>]`) now compile successfully; non-empty arrays of unencodable types report a proper diagnostic (FS3885) instead of an internal error. ([Issue #12796](https://github.com/dotnet/fsharp/issues/12796), [PR #19472](https://github.com/dotnet/fsharp/pull/19472))
 
 ### Added
 

src/Compiler/AbstractIL/il.fs

@@ -4978,6 +4978,7 @@ let rec decodeCustomAttrElemType bytes sigptr x =
         let elemTy, sigptr = decodeCustomAttrElemType bytes sigptr et
         mkILArr1DTy elemTy, sigptr
     | x when x = 0x50uy -> PrimaryAssemblyILGlobals.typ_Type, sigptr
+    | x when x = 0x51uy -> PrimaryAssemblyILGlobals.typ_Object, sigptr // SERIALIZATION_TYPE_TAGGED_OBJECT (ECMA-335 II.23.3)
     | _ -> failwithf "decodeCustomAttrElemType ilg: unrecognized custom element type: %A" x
 
 /// Given a custom attribute element, encode it to a binary representation according to the rules in Ecma 335 Partition II.

src/Compiler/CodeGen/IlxGen.fs

@@ -10294,7 +10294,7 @@ and EmitRestoreStack cgbuf (savedStack, savedStackLocals) =
 //GenAttr: custom attribute generation
 //-------------------------------------------------------------------------
 
-and GenAttribArg amap g eenv x (ilArgTy: ILType) =
+and GenAttribArg amap (g: TcGlobals) eenv x (ilArgTy: ILType) =
 
     match stripDebugPoints x, ilArgTy with
     // Detect 'null' used for an array argument
@@ -10334,7 +10334,40 @@ and GenAttribArg amap g eenv x (ilArgTy: ILType) =
     // Detect '[| ... |]' nodes
     | Expr.Op(TOp.Array, [ elemTy ], args, m), _ ->
         let ilElemTy = GenType amap m eenv.tyenv elemTy
-        ILAttribElem.Array(ilElemTy, List.map (fun arg -> GenAttribArg amap g eenv arg ilElemTy) args)
+
+        // Check if element type can be encoded in custom attribute metadata (ECMA-335 II.23.3).
+        // Valid element types: primitives, enums, string, System.Type, System.Object.
+        let isEncodableElemType =
+            match ilElemTy with
+            | ILType.Value tspec ->
+                match tspec.Name with
+                | "System.SByte"
+                | "System.Byte"
+                | "System.Int16"
+                | "System.UInt16"
+                | "System.Int32"
+                | "System.UInt32"
+                | "System.Int64"
+                | "System.UInt64"
+                | "System.Double"
+                | "System.Single"
+                | "System.Char"
+                | "System.Boolean" -> true
+                | _ -> isEnumTy g elemTy
+            | ILType.Boxed tspec ->
+                tspec.Name = "System.String"
+                || tspec.Name = "System.Object"
+                || tspec.Name = "System.Type"
+            | _ -> false
+
+        if not isEncodableElemType then
+            if args.IsEmpty then
+                // Empty arrays: substitute System.Object as element type since no elements need encoding.
+                ILAttribElem.Array(g.ilg.typ_Object, [])
+            else
+                error (Error(FSComp.SR.ilCustomAttrInvalidArrayElemType (ilElemTy.TypeRef.Name), m))
+        else
+            ILAttribElem.Array(ilElemTy, List.map (fun arg -> GenAttribArg amap g eenv arg ilElemTy) args)
 
     // Detect 'typeof<ty>' calls
     | TypeOfExpr g ty, _ -> ILAttribElem.Type(Some(GenType amap x.Range eenv.tyenv ty))

src/Compiler/FSComp.txt

@@ -1814,3 +1814,4 @@ featurePreprocessorElif,"#elif preprocessor directive"
 3882,lexHashElifMustBeFirst,"#elif directive must appear as the first non-whitespace character on a line"
 3883,lexHashElifMustHaveIdent,"#elif directive should be immediately followed by an identifier"
 3884,tcFunctionValueUsedAsInterpolatedStringArg,"This expression is a function value. When used in an interpolated string it will be formatted using its 'ToString' method, which is likely not the intended behavior. Consider applying the function to its arguments."
+3885,ilCustomAttrInvalidArrayElemType,"The type '%s' is not a valid custom attribute argument type. Custom attribute arrays must have elements of primitive types, enums, string, System.Type, or System.Object."