Fix scanning issues related to Unicode escapes in RegExp group names and refactor identifier scanning

tsc/internal/scanner/regexp.go

@@ -500,8 +500,7 @@ func (p *regExpParser) scanCharacterEscape(atomEscape bool) string {
 func (p *regExpParser) scanGroupName(isReference bool) {
 	debug.Assert(p.pos() > 0 && p.text()[p.pos()-1] == '<')
 	p.scanner.tokenStart = p.pos()
-	p.scanner.scanIdentifier(0)
-	if p.pos() == p.scanner.tokenStart {
+	if !p.scanner.scanIdentifier(0, identifierVariantRegExpGroupName) {
 		p.error(diagnostics.Expected_a_capturing_group_name, p.pos(), 0)
 	} else if isReference {
 		p.groupNameReferences = append(p.groupNameReferences, groupNameReference{pos: p.scanner.tokenStart, end: p.pos(), name: p.scanner.tokenValue})

tsc/internal/scanner/scanner.go

@@ -18,6 +18,14 @@ import (
 	"github.com/microsoft/TypeScript/tsc/internal/stringutil"
 )
 
+type identifierVariant int32
+
+const (
+	identifierVariantStandard identifierVariant = iota
+	identifierVariantJSX
+	identifierVariantRegExpGroupName
+)
+
 type EscapeSequenceScanningFlags int32
 
 const (
@@ -887,9 +895,7 @@ func (s *Scanner) Scan() ast.Kind {
 			s.pos++
 			s.token = ast.KindAtToken
 		case '\\':
-			cp := s.peekUnicodeEscape()
-			if cp >= 0 && IsIdentifierStart(cp) {
-				s.tokenValue = string(s.scanUnicodeEscape(true)) + s.scanIdentifierParts()
+			if s.scanIdentifier(0, identifierVariantStandard) {
 				s.token = GetIdentifierToken(s.tokenValue)
 			} else {
 				s.scanInvalidCharacter()
@@ -904,21 +910,11 @@ func (s *Scanner) Scan() ast.Kind {
 					continue
 				}
 				s.errorAt(diagnostics.X_can_only_be_used_at_the_start_of_a_file, s.pos, 2)
-				s.pos++
+				s.pos += 2
 				s.token = ast.KindUnknown
 				break
 			}
-			if s.charAt(1) == '\\' {
-				s.pos++
-				cp := s.peekUnicodeEscape()
-				if cp >= 0 && IsIdentifierStart(cp) {
-					s.tokenValue = "#" + string(s.scanUnicodeEscape(true)) + s.scanIdentifierParts()
-					s.token = ast.KindPrivateIdentifier
-					break
-				}
-				s.pos--
-			}
-			if !s.scanIdentifier(1) {
+			if !s.scanIdentifier(1, identifierVariantStandard) {
 				s.errorAt(diagnostics.Invalid_character, s.pos-1, 1)
 				s.tokenValue = "#"
 			}
@@ -928,7 +924,7 @@ func (s *Scanner) Scan() ast.Kind {
 				s.token = ast.KindEndOfFile
 				break
 			}
-			if s.scanIdentifier(0) {
+			if s.scanIdentifier(0, identifierVariantStandard) {
 				s.token = GetIdentifierToken(s.tokenValue)
 				break
 			}
@@ -1324,22 +1320,8 @@ func (s *Scanner) ScanJsxIdentifier() ast.Kind {
 		// everything after it to the token
 		// Do note that this means that `scanJsxIdentifier` effectively _mutates_ the visible token without advancing to a new token
 		// Any caller should be expecting this behavior and should only read the pos or token value after calling it.
-		for {
-			ch := s.char()
-			if ch < 0 {
-				break
-			}
-			if ch == '-' {
-				s.tokenValue += "-"
-				s.pos++
-				continue
-			}
-			oldPos := s.pos
-			s.tokenValue += s.scanIdentifierParts() // reuse `scanIdentifierParts` so unicode escapes are handled
-			if s.pos == oldPos {
-				break
-			}
-		}
+		// Here scanIdentifierParts is reused to ensure Unicode escapes are handled.
+		s.tokenValue += s.scanIdentifierParts(identifierVariantJSX)
 		s.token = GetIdentifierToken(s.tokenValue)
 	}
 	return s.token
@@ -1487,49 +1469,25 @@ func (s *Scanner) ScanJSDocToken() ast.Kind {
 	case '#':
 		s.token = ast.KindHashToken
 		return s.token
-	case '\\':
-		s.pos--
-		cp := s.peekUnicodeEscape()
-		if cp >= 0 && IsIdentifierStart(cp) {
-			s.tokenValue = string(s.scanUnicodeEscape(true)) + s.scanIdentifierParts()
-			s.token = GetIdentifierToken(s.tokenValue)
-		} else {
-			s.pos++
-			s.token = ast.KindUnknown
-		}
-		return s.token
 	}
 
-	if IsIdentifierStart(ch) {
-		char := ch
-		for {
-			if s.pos >= len(s.text) {
-				break
-			}
-			char, size = s.charAndSize()
-			if !IsIdentifierPart(char) && char != '-' {
-				break
-			}
-			s.pos += size
-		}
-		s.tokenValue = s.text[s.tokenStart:s.pos]
-		if char == '\\' {
-			s.tokenValue += s.scanIdentifierParts()
-		}
+	s.pos = s.tokenStart
+	if s.scanIdentifier(0, identifierVariantJSX) {
 		s.token = GetIdentifierToken(s.tokenValue)
 		return s.token
-	} else {
-		s.token = ast.KindUnknown
-		return s.token
 	}
+	s.pos = s.tokenStart + size
+	s.token = ast.KindUnknown
+	return s.token
 }
 
-func (s *Scanner) scanIdentifier(prefixLength int) bool {
+func (s *Scanner) scanIdentifier(prefixLength int, variant identifierVariant) bool {
 	start := s.pos
 	s.pos += prefixLength
+	identifierStart := s.pos
 	ch := s.char()
 	// Fast path for simple ASCII identifiers
-	if stringutil.IsASCIILetter(ch) || ch == '_' || ch == '$' {
+	if variant != identifierVariantJSX && (stringutil.IsASCIILetter(ch) || ch == '_' || ch == '$') {
 		s.pos++
 		s.scanASCIIWhile(func(b byte) bool {
 			return (b >= 'a' && b <= 'z') || (b >= 'A' && b <= 'Z') || (b >= '0' && b <= '9') || b == '_' || b == '$'
@@ -1539,40 +1497,57 @@ func (s *Scanner) scanIdentifier(prefixLength int) bool {
 			s.tokenValue = s.text[start:s.pos]
 			return true
 		}
-		s.pos = start + prefixLength
+		s.pos = identifierStart
 	}
-	ch, size := s.charAndSize()
-	if IsIdentifierStart(ch) {
-		for {
-			s.pos += size
-			ch, size = s.charAndSize()
-			if !IsIdentifierPart(ch) {
-				break
-			}
-		}
-		s.tokenValue = s.text[start:s.pos]
-		if ch == '\\' {
-			s.tokenValue += s.scanIdentifierParts()
+	identifier := s.scanIdentifierStart(variant)
+	if identifier != "" {
+		// Preserve the original source slice when decoding did not change the identifier;
+		// otherwise combine any prefix (such as "#") with the decoded identifier start.
+		if s.text[identifierStart:s.pos] == identifier {
+			s.tokenValue = s.text[start:s.pos]
+		} else {
+			s.tokenValue = s.text[start:identifierStart] + identifier
 		}
+		s.tokenValue += s.scanIdentifierParts(variant)
 		return true
 	}
 	return false
 }
 
-func (s *Scanner) scanIdentifierParts() string {
+func (s *Scanner) scanIdentifierStart(variant identifierVariant) string {
+	ch, size := s.charAndSize()
+	if IsIdentifierStart(ch) {
+		s.pos += size
+		return string(ch)
+	}
+	if ch == '\\' {
+		if escaped, ok := s.scanIdentifierEscape(IsIdentifierStart, variant == identifierVariantRegExpGroupName); ok {
+			return string(escaped)
+		}
+	}
+	return ""
+}
+
+func (s *Scanner) scanIdentifierParts(variant identifierVariant) string {
 	var sb strings.Builder
 	start := s.pos
+	languageVariant := core.LanguageVariantStandard
+	if variant == identifierVariantJSX {
+		languageVariant = core.LanguageVariantJSX
+	}
 	for {
 		ch, size := s.charAndSize()
-		if IsIdentifierPart(ch) {
+		if IsIdentifierPartEx(ch, languageVariant) {
 			s.pos += size
 			continue
 		}
 		if ch == '\\' {
-			escaped := s.peekUnicodeEscape()
-			if escaped >= 0 && IsIdentifierPart(escaped) {
-				sb.WriteString(s.text[start:s.pos])
-				sb.WriteRune(s.scanUnicodeEscape(true))
+			escapeStart := s.pos
+			if escaped, ok := s.scanIdentifierEscape(func(ch rune) bool {
+				return IsIdentifierPartEx(ch, languageVariant)
+			}, variant == identifierVariantRegExpGroupName); ok {
+				sb.WriteString(s.text[start:escapeStart])
+				sb.WriteRune(escaped)
 				start = s.pos
 				continue
 			}
@@ -1583,6 +1558,31 @@ func (s *Scanner) scanIdentifierParts() string {
 	return sb.String()
 }
 
+func (s *Scanner) scanIdentifierEscape(isValid func(rune) bool, allowSurrogatePairEscape bool) (rune, bool) {
+	escaped := s.peekUnicodeEscape()
+	if escaped >= 0 && isValid(escaped) {
+		return s.scanUnicodeEscape(true), true
+	}
+	if allowSurrogatePairEscape && s.charAt(2) != '{' && stringutil.IsHighSurrogate(escaped) {
+		// Unlike normal identifiers, group names in regular expressions, whether in Unicode mode or not,
+		// accept \u HexLeadSurrogate \u HexTrailSurrogate as part of RegExpIdentifierName.
+		// See https://github.com/tc39/ecma262/pull/1869 for the change.
+		savedPos := s.pos
+		savedTokenFlags := s.tokenFlags
+		s.scanUnicodeEscape(false)
+		// scanLowSurrogateEscape also accepts the braced form used in string literals,
+		// but RegExpIdentifierName does not allow it.
+		if s.charAt(2) != '{' {
+			if codePoint, ok := s.scanLowSurrogateEscape(escaped); ok && isValid(codePoint) {
+				return codePoint, true
+			}
+		}
+		s.pos = savedPos
+		s.tokenFlags = savedTokenFlags
+	}
+	return 0, false
+}
+
 func (s *Scanner) scanString(jsxAttributeString bool) string {
 	quote := s.char()
 	if quote == '\'' {
@@ -2024,7 +2024,7 @@ func (s *Scanner) scanNumber() ast.Kind {
 	ch, _ := s.charAndSize()
 	if IsIdentifierStart(ch) {
 		idStart := s.pos
-		id := s.scanIdentifierParts()
+		id := s.scanIdentifierParts(identifierVariantStandard)
 		if result != ast.KindBigIntLiteral && len(id) == 1 && s.text[idStart] == 'n' {
 			if s.tokenFlags&ast.TokenFlagsScientific != 0 {
 				s.errorAt(diagnostics.A_bigint_literal_cannot_use_exponential_notation, start, s.pos-start)
@@ -2249,7 +2249,7 @@ func IsIdentifierPart(ch rune) bool {
 func IsIdentifierPartEx(ch rune, languageVariant core.LanguageVariant) bool {
 	return isWordCharacter(ch) || ch == '$' ||
 		ch >= utf8.RuneSelf && stringutil.IsUnicodeIdentifierPart(ch) ||
-		languageVariant == core.LanguageVariantJSX && (ch == '-' || ch == ':') // "-" and ":" are valid in JSX Identifiers
+		languageVariant == core.LanguageVariantJSX && ch == '-' // ":" is part of JSXNamespacedName, but not JSXIdentifier.
 }
 
 var tokenToText = func() [ast.KindCount]string {

tsc/testdata/baselines/reference/compiler/extendedUnicodeEscapeSequenceIdentifiers.js

@@ -3,12 +3,14 @@
 //// [extendedUnicodeEscapeSequenceIdentifiers.ts]
 const \u{0061} = 12;
 const a\u{0061} = 12;
+const a\u{62}c\u{64}e = 12;
 
-console.log(a + aa);
+console.log(a + aa + abcde);
 
 
 //// [extendedUnicodeEscapeSequenceIdentifiers.js]
 "use strict";
 const \u{0061} = 12;
 const a\u{0061} = 12;
-console.log(a + aa);
+const a\u{62}c\u{64}e = 12;
+console.log(a + aa + abcde);

tsc/testdata/baselines/reference/compiler/extendedUnicodeEscapeSequenceIdentifiers.symbols

@@ -7,10 +7,14 @@ const \u{0061} = 12;
 const a\u{0061} = 12;
 >a\u{0061} : Symbol(a\u{0061}, Decl(extendedUnicodeEscapeSequenceIdentifiers.ts, 1, 5))
 
-console.log(a + aa);
+const a\u{62}c\u{64}e = 12;
+>a\u{62}c\u{64}e : Symbol(a\u{62}c\u{64}e, Decl(extendedUnicodeEscapeSequenceIdentifiers.ts, 2, 5))
+
+console.log(a + aa + abcde);
 >console.log : Symbol(Console.log, Decl(lib.dom.d.ts, --, --))
 >console : Symbol(console, Decl(lib.dom.d.ts, --, --))
 >log : Symbol(Console.log, Decl(lib.dom.d.ts, --, --))
 >a : Symbol(\u{0061}, Decl(extendedUnicodeEscapeSequenceIdentifiers.ts, 0, 5))
 >aa : Symbol(a\u{0061}, Decl(extendedUnicodeEscapeSequenceIdentifiers.ts, 1, 5))
+>abcde : Symbol(a\u{62}c\u{64}e, Decl(extendedUnicodeEscapeSequenceIdentifiers.ts, 2, 5))
 

tsc/testdata/baselines/reference/compiler/extendedUnicodeEscapeSequenceIdentifiers.types

@@ -9,12 +9,18 @@ const a\u{0061} = 12;
 >a\u{0061} : 12
 >12 : 12
 
-console.log(a + aa);
->console.log(a + aa) : void
+const a\u{62}c\u{64}e = 12;
+>a\u{62}c\u{64}e : 12
+>12 : 12
+
+console.log(a + aa + abcde);
+>console.log(a + aa + abcde) : void
 >console.log : (...data: any[]) => void
 >console : Console
 >log : (...data: any[]) => void
+>a + aa + abcde : number
 >a + aa : number
 >a : 12
 >aa : 12
+>abcde : 12
 

tsc/testdata/baselines/reference/compiler/manyCompilerErrorsInTheTwoFiles.errors.txt

@@ -3,25 +3,20 @@
 1 const a =!@#!@$
              ~~
 
-a.ts:1:13 - error TS1134: Variable declaration expected.
+a.ts:1:14 - error TS1134: Variable declaration expected.
 
 1 const a =!@#!@$
-              ~
-
-a.ts:1:16 - error TS1109: Expression expected.
-
-1 const a =!@#!@$
-                 ~
+               ~
 
 a.ts:2:13 - error TS18026: '#!' can only be used at the start of a file.
 
 2 const b = !@#!@#!@#!
               ~~
 
-a.ts:2:14 - error TS1134: Variable declaration expected.
+a.ts:2:15 - error TS1134: Variable declaration expected.
 
 2 const b = !@#!@#!@#!
-               ~
+                ~
 
 a.ts:2:16 - error TS18026: '#!' can only be used at the start of a file.
 
@@ -94,18 +89,16 @@
   ~~~~~
 
 
-==== a.ts (16 errors) ====
+==== a.ts (15 errors) ====
     const a =!@#!@$
                ~~
 !!! error TS18026: '#!' can only be used at the start of a file.
-                ~
+                 ~
 !!! error TS1134: Variable declaration expected.
-
-!!! error TS1109: Expression expected.
     const b = !@#!@#!@#!
                 ~~
 !!! error TS18026: '#!' can only be used at the start of a file.
-                 ~
+                  ~
 !!! error TS1134: Variable declaration expected.
                    ~~
 !!! error TS18026: '#!' can only be used at the start of a file.
@@ -143,9 +136,9 @@
     limit
     ~~~~~
 !!! error TS2304: Cannot find name 'limit'.
-Found 19 errors in 2 files.
+Found 18 errors in 2 files.
 
 Errors  Files
-    16  a.ts:1
+    15  a.ts:1
      3  b.ts:1