Implement native clang launcher

.circleci/config.yml

@@ -54,6 +54,7 @@ commands:
     description: "bootstrap"
     steps:
       - run: "$EMSDK_PYTHON ./bootstrap.py"
+      - build-emcc-native
   pip-install:
     description: "pip install"
     parameters:
@@ -131,6 +132,21 @@ commands:
             export PATH="`pwd`/node-v15.14.0-linux-x64/bin:${PATH}"
             npm install jsvu -g
             jsvu --os=default --engines=v8
+  build-emcc-native:
+    description: "Build emcc_native"
+    steps:
+      - run:
+          name: Install dependencies (Linux)
+          command: |
+            if command -v apt-get >/dev/null 2>&1; then
+              apt-get install -q -y cmake ninja-build
+            fi
+      - run:
+          name: Build emcc_native
+          command: |
+            cmake -B out/build_emcc_native -S tools/emcc_native -DCMAKE_BUILD_TYPE=Release
+            cmake --build out/build_emcc_native --config Release
+            cmake --install out/build_emcc_native --config Release
   install-emsdk:
     description: "Install emsdk"
     steps:
@@ -147,6 +163,8 @@ commands:
             cd ~/emsdk
             ./emsdk install ${EMSDK_VERSION}
             ./emsdk activate ${EMSDK_VERSION}
+            # Hack: Replace emsdk_path expressions with $CFGDIR so emcc_native can parse without fallback
+            python -c 's = open(".emscripten").read(); open(".emscripten", "w").write(s.replace("emsdk_path + " + chr(39), chr(39) + "$CFGDIR").replace("emsdk_path + " + chr(34), chr(34) + "$CFGDIR"))' 2>/dev/null || python3 -c 's = open(".emscripten").read(); open(".emscripten", "w").write(s.replace("emsdk_path + " + chr(39), chr(39) + "$CFGDIR").replace("emsdk_path + " + chr(34), chr(34) + "$CFGDIR"))'
             # Write the version of clang into a file for use in the ccache key
             ./upstream/bin/clang --version > clang_version.txt
             echo "clang version:"
@@ -1188,6 +1206,10 @@ jobs:
       shell: bash.exe -eo pipefail
     steps:
       - checkout
+      - run:
+          name: Install packages
+          command: |
+            choco install -y cmake.portable ninja
       - run:
           name: "build pylauncher"
           shell: cmd.exe

.github/workflows/ci.yml

@@ -66,6 +66,9 @@ jobs:
             echo "Be sure that you have installed the current emsdk version.  See test/emsdk_version.txt ($(cat test/emsdk_version.txt))."
             exit 1
           fi
+      - name: Check emcc_native generated settings
+        run: |
+          ./tools/emcc_native/gen_settings.py --check
 
   clang-format-diff:
     # This job is disabled until we can make it more precise

.gitignore

@@ -30,6 +30,9 @@ coverage.xml
 # Test output
 /out/
 
+# Native emcc launcher lives here.
+/bin/
+
 # When updating the website we check it out here.
 /site/emscripten-site/
 

Makefile

@@ -13,10 +13,15 @@ install:
 	./tools/install.py $(DESTDIR)
 	npm install --omit=dev --prefix $(DESTDIR)
 
+emcc_native:
+	cmake -B out/build_emcc_native -S tools/emcc_native -G Ninja
+	cmake --build out/build_emcc_native
+	cmake --install out/build_emcc_native
+
 # Create an distributable archive of emscripten suitable for use
 # by end users. This archive excludes node_modules as it can include native
 # modules which can't be safely pre-packaged.
 $(DISTFILE): install
 	tar cf $@ --exclude=node_modules -C `dirname $(DESTDIR)` `basename $(DESTDIR)`
 
-.PHONY: dist install
+.PHONY: dist install emcc_native

docs/design/03-native-clang-frontend.md

@@ -1,6 +1,6 @@
 # Design Doc: Native Launcher / Clang Frontend
 
-- **Status**: Draft
+- **Status**: Phase 1 Completed
 - **Bug**: https://github.com/emscripten-core/emscripten/issues/26453
 
 ## Context

emcc.py

@@ -427,7 +427,7 @@ def phase_setup(state):
             'unused-command-line-argument',
             "linker setting ignored during compilation: '%s'" % key)
     for arg in state.orig_args:
-      if arg in LINK_ONLY_FLAGS:
+      if arg.split('=')[0] in LINK_ONLY_FLAGS:
         diagnostics.warning(
             'unused-command-line-argument',
             "linker flag ignored during compilation: '%s'" % arg)

test/common.py

@@ -899,7 +899,7 @@ def get_cflags(self, main_file=False, compile_only=False, asm_only=False):
     def is_ldflag(f):
       return f.startswith(('-l', '-sEXPORT_ES6', '-sGL_TESTING', '-sPROXY_TO_PTHREAD',
                            '-sENVIRONMENT=', '--pre-js=', '--post-js=', '-sPTHREAD_POOL_SIZE=',
-                           '--profiling-funcs'))
+                           '--profiling-funcs', '--closure'))
 
     args = self.serialize_settings(compile_only or asm_only) + self.cflags
     if asm_only:

test/test_core.py

@@ -6940,7 +6940,7 @@ def test_zlib(self, use_cmake):
     zlib = self.get_zlib_library(use_cmake)
 
     # example.c uses K&R style function declarations
-    self.cflags += ['-Wno-deprecated-non-prototype']
+    self.cflags += ['-Wno-deprecated-non-prototype', '-Wno-unused-command-line-argument']
     self.do_core_test('test_zlib.c', libraries=zlib, includes=[test_file('third_party/zlib')])
 
   @needs_make('make')

test/test_other.py

@@ -12277,9 +12277,9 @@ def test_xclang_flag(self):
     self.run_process([EMCC, '-c', '-o', 'out.o', '-Xclang', '-include', '-Xclang', 'foo.h', test_file('hello_world.c')])
 
   def test_emcc_size_parsing(self):
-    create_file('foo.h', ' ')
-    self.assert_fail([EMCC, '-sTOTAL_MEMORY=X', 'foo.h'], 'error: invalid byte size `X`.  Valid suffixes are: kb, mb, gb, tb')
-    self.assert_fail([EMCC, '-sTOTAL_MEMORY=11PB', 'foo.h'], 'error: invalid byte size `11PB`.  Valid suffixes are: kb, mb, gb, tb')
+    create_file('foo.c', ' ')
+    self.assert_fail([EMCC, '-sTOTAL_MEMORY=X', 'foo.c'], 'error: invalid byte size `X`.  Valid suffixes are: kb, mb, gb, tb')
+    self.assert_fail([EMCC, '-sTOTAL_MEMORY=11PB', 'foo.c'], 'error: invalid byte size `11PB`.  Valid suffixes are: kb, mb, gb, tb')
 
   def test_native_call_before_init(self):
     self.set_setting('ASSERTIONS')
@@ -13106,6 +13106,10 @@ def test_link_only_flag_warning(self):
     err = self.run_process([EMCC, '--embed-file', 'file', '-c', test_file('hello_world.c')], stderr=PIPE).stderr
     self.assertContained("warning: linker flag ignored during compilation: '--embed-file' [-Wunused-command-line-argument]", err)
 
+    # Also test for the format that includes an =arg suffix
+    err = self.run_process([EMCC, '--embed-file=file', '-c', test_file('hello_world.c')], stderr=PIPE).stderr
+    self.assertContained("warning: linker flag ignored during compilation: '--embed-file=file' [-Wunused-command-line-argument]", err)
+
   def test_no_deprecated(self):
     # Test that -Wno-deprecated is passed on to clang driver
     create_file('test.c', '''\

tools/compile.py

@@ -16,6 +16,9 @@
 
 get_cflags(): In addition to compiler flags this function also returns pre-processor
 flags. For example, include paths and macro definitions.
+
+NOTE: Default compiler flag construction logic here is also implemented natively
+in tools/emcc_native/driver.cpp. Keep changes in sync between both places!
 """
 
 import os

tools/emcc_native/CMakeLists.txt

@@ -0,0 +1,53 @@
+cmake_minimum_required(VERSION 3.20)
+project(emcc_native CXX)
+
+if(CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT)
+  set(CMAKE_INSTALL_PREFIX "${CMAKE_CURRENT_SOURCE_DIR}/../.." CACHE PATH "Default install prefix to Emscripten root" FORCE)
+endif()
+
+set(CMAKE_CXX_STANDARD 20)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+set(CMAKE_CXX_EXTENSIONS OFF)
+
+if(MSVC)
+  add_compile_options(/W4 /WX)
+  add_compile_definitions(_CRT_SECURE_NO_WARNINGS)
+else()
+  add_compile_options(-Wall -Wextra -Werror)
+endif()
+
+enable_testing()
+
+add_library(native_launcher_lib OBJECT
+    config.cpp
+    driver.cpp
+    exec.cpp
+)
+
+include_directories(${CMAKE_CURRENT_SOURCE_DIR})
+
+# Build `emcc` native launcher
+add_executable(emcc main.cpp)
+target_link_libraries(emcc PRIVATE native_launcher_lib)
+
+# Create `em++` executable (symlink on Unix, copy on Windows)
+if(WIN32)
+  set(CREATE_EMXX_COMMAND ${CMAKE_COMMAND} -E copy_if_different $<TARGET_FILE:emcc> $<TARGET_FILE_DIR:emcc>/em++$<TARGET_FILE_SUFFIX:emcc>)
+else()
+  set(CREATE_EMXX_COMMAND ${CMAKE_COMMAND} -E create_symlink $<TARGET_FILE_NAME:emcc> $<TARGET_FILE_DIR:emcc>/em++$<TARGET_FILE_SUFFIX:emcc>)
+endif()
+
+add_custom_command(
+    TARGET emcc POST_BUILD
+    COMMAND ${CREATE_EMXX_COMMAND}
+    COMMENT "Creating em++ launcher executable"
+)
+
+# Build unit tests
+add_executable(native_tests tests/test_native.cpp)
+target_link_libraries(native_tests PRIVATE native_launcher_lib)
+
+add_test(NAME native_tests COMMAND native_tests)
+
+install(TARGETS emcc DESTINATION bin)
+install(PROGRAMS $<TARGET_FILE_DIR:emcc>/em++$<TARGET_FILE_SUFFIX:emcc> DESTINATION bin)

tools/emcc_native/README.md

@@ -0,0 +1,96 @@
+# Native Clang Frontend Launcher (`emcc_native`)
+
+`emcc_native` is a high-performance C++ launcher for Emscripten's compiler
+drivers (`emcc` and `em++`). It drastically reduces compiler startup latency for
+compile-only invocations (`-c`, `-S`, `-E`, `-M`, `-MM`) by bypassing Python and directly
+executing `clang` / `clang++`.
+
+## Overview & Architecture
+
+When running large CMake or Ninja builds, `emcc` is launched hundreds or
+thousands of times. Executing the Python interpreter for each single compile
+unit adds overhead (especially on Windows).
+
+`emcc_native` provides native executables (`bin/emcc`, `bin/em++`) that:
+1. **Directly invoke Clang** for pure compilation steps (`-c`, `-S`, `-E`, `-M`, `-MM`), injecting:
+   - Target triple (`-target wasm32-unknown-emscripten` or `wasm64-unknown-emscripten`)
+   - Frontend exceptions flag (`-fignore-exceptions`)
+   - Default LLVM backend flags (e.g. `-mllvm -enable-emscripten-sjlj`)
+   - Emscripten sysroot (`--sysroot=<cache>/sysroot`)
+   - Clang sysroot include paths (e.g. `-Xclang -iwithsysroot/include/compat`)
+   - SIMD/SSE/NEON preprocessor macros (`-D__SSE__=1`, `-D__SSE2__=1`,
+     `-D__ARM_NEON__=1`, etc.) when architecture flags are specified
+   - Visibility flag (`-fvisibility=default` when `-fPIC` is passed without
+     `-fvisibility`)
+2. **Ignore compile-unused linker flags**: Link-only flags (`--js-library`,
+   `--embed-file`, etc.) and linker settings (`-sEXPORTED_FUNCTIONS`, etc.) are
+   ignored during compilation (with diagnostic warnings matching `emcc.py`),
+   allowing compile steps with link flags to run natively.
+3. **Fall back to Python** (`emcc.py` / `em++.py`) when link-phase invocations
+   are run, or when compile-time `-s` settings or system flags (`--clear-cache`,
+   `--build`, `--tracing`, etc.) are present.
+
+## Building
+
+Building requires CMake 3.20+ and a C++20 compiler.
+
+```bash
+cmake -B out/build_emcc_native -S tools/emcc_native
+cmake --build out/build_emcc_native
+cmake --install out/build_emcc_native
+```
+
+The output executables (`emcc`, `em++`) will be installed in `./bin`.
+
+## Running Tests
+
+To run the unit and integration tests:
+
+```bash
+ctest --test-dir out/build_emcc_native --output-on-failure
+```
+
+## Code Generation
+
+Compile-time settings, link-only flags, and Emscripten warning options are
+generated in `generated_settings.h`. To update this header from Python
+definitions, run:
+
+```bash
+./tools/emcc_native/gen_settings.py
+```
+
+To verify whether `generated_settings.h` is up to date:
+
+```bash
+./tools/emcc_native/gen_settings.py --check
+```
+
+## Configuration & Environment Variables
+
+- `EMCC_NATIVE`:
+  - Set to `0` to disable the native driver and unconditionally fall back to `emcc.py`.
+  - Set to `1` to force strict native mode; if an invocation requires falling back to Python, `emcc_native` will print the fallback reason and exit with an error (useful for debugging).
+- `EMCC_DEBUG`: When set (e.g. `EMCC_DEBUG=1`), logs launcher decision details (whether direct Clang execution or Python fallback was selected, reason, target binary, and command arguments).
+- `EMSDK_PYTHON`: Path to the Python executable (defaults to `python3` or `python.exe` on Windows).
+- `EM_CACHE`: Path to Emscripten cache directory (defaults to `<emscripten_root>/cache`).
+- `EM_CONFIG`: Path to `.emscripten` configuration file (reads `LLVM_ROOT` and `CACHE`).
+- `EM_LLVM_ROOT`: Environment variable override for the directory containing LLVM binaries (`clang`, `clang++`).
+
+## CI Benchmark Results
+
+Compile-time performance is automatically benchmarked on CI across Linux,
+macOS, and Windows (`embuilder build libc --force` compiling 1,075 files
+sequentially with `EMCC_CORES=1`, `EMCC_USE_NINJA=0`, and
+`EMCC_BATCH_BUILD=0`).
+
+| Platform    | Before (Python Baseline) | After (Native Launcher) | Improvement                |  Speedup  |
+| :---------: | :----------------------: | :---------------------: | :------------------------: | :-------: |
+| **Linux**   | 181.96 s (169.3 ms/file) |  64.82 s (60.3 ms/file) | -117.14 s (-109.0 ms/file) | **2.81x** |
+| **Windows** | 343.90 s (319.9 ms/file) | 105.12 s (97.8 ms/file) | -238.78 s (-222.1 ms/file) | **3.27x** |
+| **macOS**   | 162.08 s (150.6 ms/file) |  64.73 s (60.1 ms/file) |  -97.35 s (-90.5 ms/file)  | **2.50x** |
+
+As expected, because process creation and `python.exe` startup carry
+significantly higher overhead on Windows than on POSIX systems, the speedup on
+Windows CI (**3.27x**, saving over 222 ms per invocation) is even larger than on
+Linux and macOS.