Make bundled/lib the canonical lib file source

tsc/internal/bundled/source/README.md → tsc/internal/bundled/README.md

@@ -1,8 +1,8 @@
 # Read this!
 
-The files within this directory are copied and deployed with TypeScript as the set of APIs  available as a part of the JavaScript language.
+The files in `libs` are copied and deployed with TypeScript as the set of APIs available as a part of the JavaScript language.
 
-There are three main domains of APIs in `src/lib`:
+There are three main domains of APIs:
 
  - **ECMAScript language features** - e.g. JavaScript APIs like functions on Array etc which are documented in [ECMA-262](https://tc39.es/ecma262/)
  - **DOM APIs** - e.g. APIs which are available in web browsers
@@ -21,6 +21,6 @@ For the DOM APIs, which are a bit more free-form, we have asked that APIs are av
 
 ## Generated files
 
-The DOM files ending in `.generated.d.ts` aren't meant to be edited by hand.
+The DOM and web worker files aren't meant to be edited by hand.
 
 If you need to make changes to such files, make a change to the input files for [**our library generator**](https://github.com/microsoft/TypeScript-DOM-lib-generator).

tsc/internal/bundled/generate.go

@@ -4,72 +4,28 @@ package main
 
 import (
 	"bytes"
-	"encoding/json"
 	"fmt"
 	"go/format"
 	"log"
 	"os"
 	"path/filepath"
-	"regexp"
-	"slices"
 	"strings"
 )
 
-var (
-	libInputDir     = "source"
-	copyrightNotice = filepath.Join("source", "CopyrightNotice.txt")
+const (
+	libsDir         = "libs"
+	copyrightNotice = "CopyrightNotice.txt"
 )
 
 func main() {
 	log.SetFlags(log.LstdFlags | log.Lshortfile)
 
 	libs := readLibs()
-	generateLibs(libs)
 	generateLibList(libs)
 	generateEmbedded(libs)
 }
 
-type lib struct {
-	target  string   // target relative to libs dir
-	sources []string // sources relative to src/lib dir
-}
-
-func generateLibs(libs []lib) {
-	const outputDir = "libs"
-
-	copyright := readCopyright()
-
-	if err := os.RemoveAll(outputDir); err != nil {
-		log.Fatalf("failed to remove libs directory: %v", err)
-	}
-
-	if err := os.MkdirAll(outputDir, 0o755); err != nil {
-		log.Fatalf("failed to create libs directory: %v", err)
-	}
-
-	for _, lib := range libs {
-		var output bytes.Buffer
-		output.Write(copyright)
-
-		for _, source := range lib.sources {
-			sourcePath := filepath.Join(libInputDir, source)
-			b, err := os.ReadFile(sourcePath)
-			if err != nil {
-				log.Fatalf("failed to read %s: %v", sourcePath, err)
-			}
-
-			output.WriteByte('\n')
-			output.Write(removeCRLF(b))
-		}
-
-		outputPath := filepath.Join(outputDir, lib.target)
-		if err := os.WriteFile(outputPath, output.Bytes(), 0o644); err != nil {
-			log.Fatalf("failed to write %s: %v", outputPath, err)
-		}
-	}
-}
-
-func generateLibList(libs []lib) {
+func generateLibList(libs []string) {
 	var code bytes.Buffer
 	code.WriteString("// Code generated by generate.go; DO NOT EDIT.\n\n")
 	code.WriteString("package bundled\n\n")
@@ -78,17 +34,17 @@ func generateLibList(libs []lib) {
 	code.WriteString("// For the list of libs sorted by load order, use [tsoptions.Libs].\n")
 	code.WriteString("var LibNames = []string{\n")
 	for _, lib := range libs {
-		code.WriteString("\t\"" + lib.target + "\",\n")
+		code.WriteString("\t\"" + lib + "\",\n")
 	}
 	code.WriteString("}\n")
 
 	writeCode("libs_generated.go", code.Bytes())
 }
 
-func generateEmbedded(libs []lib) {
+func generateEmbedded(libs []string) {
 	libVarNames := make([]string, len(libs))
 	for i, lib := range libs {
-		libVarNames[i] = "libs_" + strings.ReplaceAll(lib.target, ".", "_")
+		libVarNames[i] = "libs_" + strings.ReplaceAll(lib, ".", "_")
 	}
 
 	var code bytes.Buffer
@@ -103,104 +59,68 @@ func generateEmbedded(libs []lib) {
 	code.WriteString("var (\n")
 	for i, lib := range libs {
 		varName := libVarNames[i]
-		code.WriteString("//go:embed libs/" + lib.target + "\n")
+		code.WriteString("//go:embed libs/" + lib + "\n")
 		code.WriteString("" + varName + " string\n")
 	}
 	code.WriteString(")\n\n")
 
 	code.WriteString("var embeddedContents = map[string]string{\n")
 	for i, lib := range libs {
 		varName := libVarNames[i]
-		code.WriteString("\t\"libs/" + lib.target + "\": " + varName + ",\n")
+		code.WriteString("\t\"libs/" + lib + "\": " + varName + ",\n")
 	}
 	code.WriteString("}\n\n")
 
 	code.WriteString("var libsEntries = []fs.DirEntry{\n")
 	for i, lib := range libs {
 		varName := libVarNames[i]
-		fmt.Fprintf(&code, "\t&fileInfo{name: %q, size: int64(len(%s))},\n", lib.target, varName)
+		fmt.Fprintf(&code, "\t&fileInfo{name: %q, size: int64(len(%s))},\n", lib, varName)
 	}
 	code.WriteString("}\n")
 
 	writeCode("embed_generated.go", code.Bytes())
 }
 
-var (
-	// Match escaped characters, double-quoted strings, single-line comments, and multi-line comments.
-	reJSONComments = regexp.MustCompile(`\\.|"(?:\\.|[^"])*"|//.*|/\*[\s\S]*?\*/`)
-	// Match double-quoted strings (to skip) or trailing commas before ] or }.
-	reTrailingComma = regexp.MustCompile(`"(?:\\.|[^"])*"|,\s*([}\]])`)
-)
-
-// stripJSONC replaces comments and trailing commas with spaces in JSONC content,
-// producing valid JSON. The input slice is mutated in place.
-func stripJSONC(b []byte) {
-	for _, loc := range reJSONComments.FindAllIndex(b, -1) {
-		if b[loc[0]] == '/' {
-			for i := loc[0]; i < loc[1]; i++ {
-				if b[i] != '\n' {
-					b[i] = ' '
-				}
-			}
-		}
-	}
-	for _, loc := range reTrailingComma.FindAllSubmatchIndex(b, -1) {
-		// loc[2]:loc[3] is the capture group; -1 means this matched a string, not a comma.
-		if loc[2] < 0 {
-			continue
-		}
-		// Blank the comma (at loc[0]), keep whitespace and closing bracket.
-		b[loc[0]] = ' '
-	}
-}
-
-func readLibs() []lib {
-	libsFile := filepath.Join(libInputDir, "libs.json")
-
-	b, err := os.ReadFile(libsFile)
+func readLibs() []string {
+	entries, err := os.ReadDir(libsDir)
 	if err != nil {
-		log.Fatalf("failed to open libs.json: %v", err)
+		log.Fatalf("failed to read %s: %v", libsDir, err)
 	}
-	stripJSONC(b)
 
-	var meta struct {
-		Libs  []string          `json:"libs"`
-		Paths map[string]string `json:"paths"`
+	copyright, err := os.ReadFile(copyrightNotice)
+	if err != nil {
+		log.Fatalf("failed to read %s: %v", copyrightNotice, err)
 	}
-
-	if err := json.Unmarshal(b, &meta); err != nil {
-		log.Fatalf("failed to parse libs.json: %v", err)
+	if bytes.ContainsRune(copyright, '\r') || !bytes.HasSuffix(copyright, []byte("\n\n")) {
+		log.Fatalf("%s must use LF line endings and end with a blank line", copyrightNotice)
 	}
+	header := append(copyright, '\n')
 
-	var libs []lib
-	for _, libName := range meta.Libs {
-		sources := []string{libName + ".d.ts"}
-		var target string
-		if path, ok := meta.Paths[libName]; ok {
-			target = path
-		} else {
-			target = "lib." + libName + ".d.ts"
+	libs := make([]string, 0, len(entries))
+	for _, entry := range entries {
+		name := entry.Name()
+		if entry.IsDir() || !strings.HasPrefix(name, "lib.") || !strings.HasSuffix(name, ".d.ts") {
+			log.Fatalf("unexpected entry in %s: %s", libsDir, name)
 		}
-		libs = append(libs, lib{target: target, sources: sources})
-	}
 
-	slices.SortFunc(libs, func(a lib, b lib) int {
-		return strings.Compare(a.target, b.target)
-	})
-
-	return libs
-}
+		path := filepath.Join(libsDir, name)
+		content, err := os.ReadFile(path)
+		if err != nil {
+			log.Fatalf("failed to read %s: %v", path, err)
+		}
+		if bytes.ContainsRune(content, '\r') {
+			log.Fatalf("%s must use LF line endings", path)
+		}
+		if !bytes.HasPrefix(content, header) {
+			log.Fatalf("%s must start with %s followed by a blank line", path, copyrightNotice)
+		}
+		if !bytes.HasSuffix(content, []byte("\n")) {
+			log.Fatalf("%s must end with a newline", path)
+		}
 
-func readCopyright() []byte {
-	b, err := os.ReadFile(copyrightNotice)
-	if err != nil {
-		log.Fatalf("failed to read copyright notice: %v", err)
+		libs = append(libs, name)
 	}
-	return removeCRLF(b)
-}
-
-func removeCRLF(b []byte) []byte {
-	return bytes.ReplaceAll(b, []byte("\r\n"), []byte("\n"))
+	return libs
 }
 
 func writeCode(filename string, code []byte) {