diff --git a/.github/workflows/archetype-smoke.yml b/.github/workflows/archetype-smoke.yml index 42bea2c4405..47ddf971c70 100644 --- a/.github/workflows/archetype-smoke.yml +++ b/.github/workflows/archetype-smoke.yml @@ -27,6 +27,10 @@ on: - 'maven/integration-tests/cn1app-archetype-test.sh' - 'maven/integration-tests/cn1app-staged-jar-test.sh' - 'maven/integration-tests/cn1app-minimal-layout-test.sh' + - 'scripts/check-first-build-footprint.py' + # Any module's dependencies can land on the class path a first build + # downloads; the footprint gate has to see the change that adds one. + - 'maven/*/pom.xml' - 'maven/integration-tests/backend-cn1lib-test.sh' - 'scripts/initializr/common/src/main/resources/common-hosted-platform-profiles.xml' - 'scripts/initializr/common/src/main/resources/backend-only-pom.xml' @@ -55,6 +59,10 @@ on: - 'maven/integration-tests/cn1app-archetype-test.sh' - 'maven/integration-tests/cn1app-staged-jar-test.sh' - 'maven/integration-tests/cn1app-minimal-layout-test.sh' + - 'scripts/check-first-build-footprint.py' + # Any module's dependencies can land on the class path a first build + # downloads; the footprint gate has to see the change that adds one. + - 'maven/*/pom.xml' - 'maven/integration-tests/backend-cn1lib-test.sh' - 'scripts/initializr/common/src/main/resources/common-hosted-platform-profiles.xml' - 'scripts/initializr/common/src/main/resources/backend-only-pom.xml' @@ -135,6 +143,19 @@ jobs: cd maven mvn archetype:update-local-catalog -Plocal-dev-javase mvn archetype:crawl -Plocal-dev-javase + # What `./run.sh` downloads on a machine that has never built anything. + # The plugin's class path is resolved whole before its first goal, so one + # dependency added to it (or to build-engine) is megabytes on every new + # developer's first run and changes no test anywhere. The script builds a + # generated project into an empty local repository, with the repository + # the step above filled as its remote, and fails on a port, on an ffmpeg + # native for another platform, or on the total passing its budget. + - name: Check what a new project's first build downloads + # What a developer runs the simulator on. JAVA_HOME is the JDK 8 the + # framework builds with, which the plugin refuses for a simulator build. + env: + JAVA_HOME: ${{ env.CN1_BACKEND_PACKAGE_JDK }} + run: python3 scripts/check-first-build-footprint.py --repo "$HOME/.m2/repository" - name: Run archetype simulator smoke (auto-bundle pref forced on) env: # The packaging check must not be skipped on a runner that has clang. diff --git a/.github/workflows/starter-launchers.yml b/.github/workflows/starter-launchers.yml index c45120edfe6..00cdd0d825c 100644 --- a/.github/workflows/starter-launchers.yml +++ b/.github/workflows/starter-launchers.yml @@ -7,6 +7,17 @@ on: paths: - 'scripts/test-starter-launchers.py' - 'scripts/sync-initializr-launchers.py' + # The launchers this workflow compares, and their copies: a change to one + # without the other is the drift it exists to report. + - 'maven/cn1app-archetype/src/main/resources/archetype-resources/build.*' + - 'maven/cn1app-archetype/src/main/resources/archetype-resources/run.*' + - 'maven/cn1app-archetype/src/main/resources/archetype-resources/mvnw*' + - 'maven/cn1app-archetype/src/main/resources/archetype-resources/mvnconfig.toml' + - 'maven/cn1app-archetype/src/main/resources/archetype-resources/.vscode/**' + - 'maven/cn1app-archetype/src/main/resources/archetype-resources/tools/**' + - 'maven/cli/src/main/batsh/build.batsh' + - 'scripts/initializr/common/src/main/resources/*.zip' + - 'maven/codenameone-maven-plugin/src/main/resources/com/codename1/maven/intellij/workspace.xml' - '.github/workflows/starter-launchers.yml' workflow_dispatch: schedule: @@ -28,8 +39,10 @@ jobs: os: [ubuntu-24.04, windows-latest, macos-latest] runs-on: ${{ matrix.os }} # macOS runners take ~37 s per generated download, and there are past - # thirty of them now: twenty minutes cancelled a run that was passing. - timeout-minutes: 40 + # thirty of them now: twenty minutes cancelled a run that was passing, and + # then forty did -- passing runs finish in 35 to 39 on macOS, and the nightly + # was cancelled at the limit with nothing wrong. + timeout-minutes: 60 steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 diff --git a/CLAUDE.md b/CLAUDE.md index b57c5bf0da7..6d1f4a68465 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -305,6 +305,63 @@ scripts/check-build-hint-catalog.sh # every hint the code reads is de means a hint went in undeclared. Do not re-run `tools/build-hint-bootstrap/`; it seeded the catalog once and would overwrite hand edits. +### A new project's first build is a download, and it is gated + +The first thing a developer does is run a project from the initializr on a +machine that has never built one. Maven resolves a plugin's **whole** class path +before its first goal, so every runtime dependency of `codenameone-maven-plugin` +and `codenameone-build-engine` is downloaded by a Hello World in the simulator. +That was 346 MB: 55 MB of ports nothing in a simulator run opens, and 157 MB of +ffmpeg for seven platforms. No test fails when it grows. + +- **The ports are not dependencies of the build tools.** `codenameone-ios`, + `-android`, `-mac`, `-windows`, `-linux` and `-parparvm` are `provided` and + `optional` in both poms (reactor order and the test class path keep them). + `AppBuilder.LocalBuild` names the ports each local target reads, + `ProjectHost.getPortJar` resolves one at the build tool's own version when + that target is built, and `Executor.getResourceAsStream` reads the class path + first and those jars second. A builder must read a port **through the + Executor**, never `SomeClass.class.getResourceAsStream`: that sees the class + path alone and finds nothing in a customer build while passing every test + here, where the ports are on the test class path. +- **A new local target, or a new port, is a row in `LocalBuild`** and in + `LocalBuildPortsTest`, and a new port also in `PORT_ARTIFACTS` in the script + below. +- **Ports are resolved at the plugin's own version**, by both build tools, and a + port the *project* depends on is never used: a builder paired with an older + port writes a project that builds and does not start. A port under the plugin + declaration's own `` still wins, which is how one is pinned. +- **ffmpeg is the host's alone.** `maven/cn1-binaries/pom.xml` selects the + natives with OS-activated profiles, which Maven evaluates on the machine that + resolves the pom -- so that pom must never be flattened. Gradle does not + evaluate them; `PluginInfo.hostFfmpegPlatforms` is the same table there. +- **The desktop jar is the exception, because it runs somewhere else.** + `javase-executable-jar` resolves the other desktop platforms' natives itself + (`missingDesktopNatives`). A project with a `javase` module packages straight + from its class path, so its generated launch commands pass + `-Dcn1.ffmpeg.allPlatforms=true` (Maven only), and + `generate-desktop-app-wrapper` warns when an older project's do not. Those + commands exist in several copies nothing regenerates: the archetype's + scripts and IDE launchers, the Initializr's `common.zip` and per-IDE zips, + the plugin's IntelliJ `workspace.xml`, and `cn1:buildExecutableJar`, which + sends the property to its child build. `scripts/sync-initializr-launchers.py` + (`starter-launchers.yml`) fails on a copy that builds the jar without it. +- **`codename1.stageOnly` resolves the target's ports** although it never runs + the builder, so the staging tests cover resolution for every target they stage. + +`scripts/check-first-build-footprint.py` (`archetype-smoke.yml`) generates a +project, builds what `run.sh` builds into an **empty** local repository with the +workspace's repository as a `file://` remote, and fails on a port, on an +`org.bytedeco` native for another platform, on a host with no ffmpeg at all, or +on the total passing `BUDGET_MB`. CI runs it on Linux only, so it first holds +the profile of *every* host in that pom to a table (`HOST_PROFILES`). It +measures a floor: the Maven wrapper's distribution and what the simulator +fetches once running are not in a local repository. + +```bash +scripts/check-first-build-footprint.py --repo "$PWD/.m2-repo" +``` + ### The CSS compiler is headless, and has to stay that way Compiling `css/theme.css` to `theme.res` opens no window, boots no JavaSE port diff --git a/Ports/JavaSE/src/com/codename1/impl/javase/ffmpeg/FFMPEGMedia.java b/Ports/JavaSE/src/com/codename1/impl/javase/ffmpeg/FFMPEGMedia.java index d1f517a4f8b..dd41735b1fc 100644 --- a/Ports/JavaSE/src/com/codename1/impl/javase/ffmpeg/FFMPEGMedia.java +++ b/Ports/JavaSE/src/com/codename1/impl/javase/ffmpeg/FFMPEGMedia.java @@ -133,7 +133,7 @@ static File resolveExecutable(String name) { /** * Resolves an ffmpeg/ffprobe executable bundled by the - * {@code org.bytedeco:ffmpeg-platform} dependency, so simulator media + * {@code org.bytedeco:ffmpeg} dependency, so simulator media * playback works with no externally installed ffmpeg. The bytedeco binaries * are full builds whose native decoders cover H.264/AAC/MP3 (those require * no GPL/x264 encoder), which is what the codec-free upstream CEF can no diff --git a/Ports/JavaSE/src/com/codename1/impl/javase/ffmpeg/FFMPEGVideoIO.java b/Ports/JavaSE/src/com/codename1/impl/javase/ffmpeg/FFMPEGVideoIO.java index 2ca01856207..690bbda6d1d 100644 --- a/Ports/JavaSE/src/com/codename1/impl/javase/ffmpeg/FFMPEGVideoIO.java +++ b/Ports/JavaSE/src/com/codename1/impl/javase/ffmpeg/FFMPEGVideoIO.java @@ -40,7 +40,7 @@ /// `com.codename1.media.VideoIO` implementation for the JavaSE simulator backed by the /// `ffmpeg`/`ffprobe` executables (resolved by `FFMPEGMedia`, which bundles a binary via -/// the optional `org.bytedeco:ffmpeg-platform` dependency). This is a fully functional +/// the optional `org.bytedeco:ffmpeg` dependency). This is a fully functional /// reference implementation: it encodes application frames + audio, decodes clips to /// frame accurate RGBA frames + PCM, and enumerates the codecs the local ffmpeg build /// exposes. diff --git a/docs/developer-guide/Maven-Appendix-Archetypes.adoc b/docs/developer-guide/Maven-Appendix-Archetypes.adoc index bcbc3636cea..a67da588d7a 100644 --- a/docs/developer-guide/Maven-Appendix-Archetypes.adoc +++ b/docs/developer-guide/Maven-Appendix-Archetypes.adoc @@ -23,6 +23,8 @@ Native interface implementations live in the same directories either way. `mvn c A project without a `javase` module keeps the desktop app's native theme in `common/src/desktop/resources`. `mvn package -Pexecutable-jar -Dcodename1.platform=javase` writes the desktop jar to `common/target`. +The desktop jar is one archive for Windows, macOS and Linux, so it carries the media libraries of all three. This layout adds them without being asked. A project with a `javase` module packages the libraries of the machine that builds it unless the command passes `-Dcn1.ffmpeg.allPlatforms=true`, which the generated `build.sh`, `build.bat` and IDE launchers do. Add the property to a launcher written before it existed; the build prints a warning when it's missing. + [#maven-backend-only] ==== Backend-only projects diff --git a/maven/build-engine/pom.xml b/maven/build-engine/pom.xml index 16b28a86c59..2a85d5c53a0 100644 --- a/maven/build-engine/pom.xml +++ b/maven/build-engine/pom.xml @@ -247,22 +247,36 @@ cn1-builder-resources-android runtime + com.codenameone codenameone-android - runtime + provided + true com.codenameone codenameone-ios bundle - runtime + provided + true com.codenameone codenameone-parparvm bundle - runtime + provided + true @@ -271,21 +285,24 @@ codenameone-windows bundle ${project.version} - runtime + provided + true com.codenameone codenameone-mac bundle ${project.version} - runtime + provided + true com.codenameone codenameone-linux bundle ${project.version} - runtime + provided + true org.bytedeco - ffmpeg-platform + ffmpeg + ${ffmpeg.version} + + + org.bytedeco + javacpp + ${cn1.javacpp.version} + + + + ffmpeg-macosx + + + mac + + + + + org.bytedeco + ffmpeg + ${ffmpeg.version} + macosx-arm64 + + + org.bytedeco + javacpp + ${cn1.javacpp.version} + macosx-arm64 + + + org.bytedeco + ffmpeg + ${ffmpeg.version} + macosx-x86_64 + + + org.bytedeco + javacpp + ${cn1.javacpp.version} + macosx-x86_64 + + + + + ffmpeg-windows + + + windows + + + + + org.bytedeco + ffmpeg + ${ffmpeg.version} + windows-x86_64 + + + org.bytedeco + javacpp + ${cn1.javacpp.version} + windows-x86_64 + + + + + ffmpeg-linux-x86_64 + + + Linux + amd64 + + + + + org.bytedeco + ffmpeg + ${ffmpeg.version} + linux-x86_64 + + + org.bytedeco + javacpp + ${cn1.javacpp.version} + linux-x86_64 + + + + + ffmpeg-linux-arm64 + + + Linux + aarch64 + + + + + org.bytedeco + ffmpeg + ${ffmpeg.version} + linux-arm64 + + + org.bytedeco + javacpp + ${cn1.javacpp.version} + linux-arm64 + + + + + ffmpeg-all-platforms + + + cn1.ffmpeg.allPlatforms + true + + + + + org.bytedeco + ffmpeg + ${ffmpeg.version} + linux-arm64 + + + org.bytedeco + javacpp + ${cn1.javacpp.version} + linux-arm64 + + + org.bytedeco + ffmpeg + ${ffmpeg.version} + linux-x86_64 + + + org.bytedeco + javacpp + ${cn1.javacpp.version} + linux-x86_64 + + + org.bytedeco + ffmpeg + ${ffmpeg.version} + macosx-arm64 + + + org.bytedeco + javacpp + ${cn1.javacpp.version} + macosx-arm64 + + + org.bytedeco + ffmpeg + ${ffmpeg.version} + macosx-x86_64 + + + org.bytedeco + javacpp + ${cn1.javacpp.version} + macosx-x86_64 + + + org.bytedeco + ffmpeg + ${ffmpeg.version} + windows-x86_64 + + + org.bytedeco + javacpp + ${cn1.javacpp.version} + windows-x86_64 + + + + diff --git a/maven/cn1app-archetype/src/main/resources/archetype-resources/.vscode/settings.json b/maven/cn1app-archetype/src/main/resources/archetype-resources/.vscode/settings.json index a75d12eb1bf..ee136de99c8 100644 --- a/maven/cn1app-archetype/src/main/resources/archetype-resources/.vscode/settings.json +++ b/maven/cn1app-archetype/src/main/resources/archetype-resources/.vscode/settings.json @@ -85,6 +85,6 @@ }, { "alias": "Local > Cross-platform (JavaSE) Desktop App", - "command": "\"-Pexecutable-jar\" \"package\" \"-Dcodename1.platform=javase\" \"-DskipTests\" \"-U\" \"-e\"" + "command": "\"-Pexecutable-jar\" \"package\" \"-Dcodename1.platform=javase\" \"-Dcn1.ffmpeg.allPlatforms=true\" \"-DskipTests\" \"-U\" \"-e\"" } ]} diff --git a/maven/cn1app-archetype/src/main/resources/archetype-resources/build.bat b/maven/cn1app-archetype/src/main/resources/archetype-resources/build.bat index 466edb65be6..df93a76f4e7 100755 --- a/maven/cn1app-archetype/src/main/resources/archetype-resources/build.bat +++ b/maven/cn1app-archetype/src/main/resources/archetype-resources/build.bat @@ -87,7 +87,7 @@ call "%MVNW%" package -DskipTests -Dcodename1.platform^=ios -Dcodename1.buildTar goto :finish :jar -call "%MVNW%" -Pexecutable-jar package -Dcodename1.platform^=javase -DskipTests -U -e +call "%MVNW%" -Pexecutable-jar package -Dcodename1.platform^=javase -Dcn1.ffmpeg.allPlatforms^=true -DskipTests -U -e goto :finish :help diff --git a/maven/cn1app-archetype/src/main/resources/archetype-resources/build.sh b/maven/cn1app-archetype/src/main/resources/archetype-resources/build.sh index 3d56b48f493..c709b7a0be5 100755 --- a/maven/cn1app-archetype/src/main/resources/archetype-resources/build.sh +++ b/maven/cn1app-archetype/src/main/resources/archetype-resources/build.sh @@ -72,7 +72,7 @@ function ios_release { } function jar { - "$MVNW" "-Pexecutable-jar" "package" "-Dcodename1.platform=javase" "-DskipTests" "-U" "-e" + "$MVNW" "-Pexecutable-jar" "package" "-Dcodename1.platform=javase" "-Dcn1.ffmpeg.allPlatforms=true" "-DskipTests" "-U" "-e" } function help { "echo" "-e" "build.sh [COMMAND]" diff --git a/maven/cn1app-archetype/src/main/resources/archetype-resources/javase/pom.xml b/maven/cn1app-archetype/src/main/resources/archetype-resources/javase/pom.xml index a4e08617d4b..34763bd445c 100644 --- a/maven/cn1app-archetype/src/main/resources/archetype-resources/javase/pom.xml +++ b/maven/cn1app-archetype/src/main/resources/archetype-resources/javase/pom.xml @@ -98,8 +98,9 @@ JCEF Maven transitively from codenameone-javase; JCEF Maven downloads and installs the matching native runtime on first use. - Runtime scope, never compile: this is ~300 MB of ffmpeg natives for - every OS and CPU. The simulator and desktop run classpaths are runtime + Runtime scope, never compile: these are ffmpeg natives, for the machine + that builds or (with -Dcn1.ffmpeg.allPlatforms, which the desktop jar + commands pass) for every desktop platform. The simulator and desktop run classpaths are runtime classpaths, so they still get it; the jar that cn1:build uploads is assembled from the compile classpath, so it does not. At compile scope it went into every desktop upload and the build client refused the jar. diff --git a/maven/cn1app-archetype/src/main/resources/archetype-resources/mvnconfig.toml b/maven/cn1app-archetype/src/main/resources/archetype-resources/mvnconfig.toml index 1fe97129ad8..82c5a0fc762 100644 --- a/maven/cn1app-archetype/src/main/resources/archetype-resources/mvnconfig.toml +++ b/maven/cn1app-archetype/src/main/resources/archetype-resources/mvnconfig.toml @@ -86,7 +86,7 @@ label="Gradle Android Project" group="Local" [jar] -command=["$MVNW", "-Pexecutable-jar", "package", "-Dcodename1.platform=javase", "-DskipTests", "-U", "-e"] +command=["$MVNW", "-Pexecutable-jar", "package", "-Dcodename1.platform=javase", "-Dcn1.ffmpeg.allPlatforms=true", "-DskipTests", "-U", "-e"] label="Cross-platform (JavaSE) Desktop App" group="Local" build=true diff --git a/maven/cn1app-archetype/src/main/resources/archetype-resources/tools/eclipse/__mainName__ - Build JavaSE Desktop App.launch b/maven/cn1app-archetype/src/main/resources/archetype-resources/tools/eclipse/__mainName__ - Build JavaSE Desktop App.launch index 86c7b84a926..0d4147a5eab 100644 --- a/maven/cn1app-archetype/src/main/resources/archetype-resources/tools/eclipse/__mainName__ - Build JavaSE Desktop App.launch +++ b/maven/cn1app-archetype/src/main/resources/archetype-resources/tools/eclipse/__mainName__ - Build JavaSE Desktop App.launch @@ -11,6 +11,7 @@ + diff --git a/maven/cn1app-archetype/src/main/resources/archetype-resources/tools/netbeans/nbactions-JavaSE Desktop App.xml b/maven/cn1app-archetype/src/main/resources/archetype-resources/tools/netbeans/nbactions-JavaSE Desktop App.xml index f1c86c5acb0..00c6da4b35e 100644 --- a/maven/cn1app-archetype/src/main/resources/archetype-resources/tools/netbeans/nbactions-JavaSE Desktop App.xml +++ b/maven/cn1app-archetype/src/main/resources/archetype-resources/tools/netbeans/nbactions-JavaSE Desktop App.xml @@ -22,6 +22,7 @@ true + true executable-jar @@ -38,6 +39,7 @@ true + true executable-jar @@ -77,6 +79,9 @@ package + + true + executable-jar diff --git a/maven/cn1app-archetype/src/main/resources/archetype-resources/tools/netbeans/nbactions-Simulator.xml b/maven/cn1app-archetype/src/main/resources/archetype-resources/tools/netbeans/nbactions-Simulator.xml index b9707637fcc..dffb81d0da0 100644 --- a/maven/cn1app-archetype/src/main/resources/archetype-resources/tools/netbeans/nbactions-Simulator.xml +++ b/maven/cn1app-archetype/src/main/resources/archetype-resources/tools/netbeans/nbactions-Simulator.xml @@ -38,6 +38,7 @@ true + true executable-jar @@ -55,6 +56,7 @@ true + true executable-jar @@ -72,6 +74,7 @@ true + true executable-jar diff --git a/maven/codenameone-gradle-plugin/src/main/java/com/codename1/gradle/AppSupport.java b/maven/codenameone-gradle-plugin/src/main/java/com/codename1/gradle/AppSupport.java index 8fc3dfcfb3f..9293ac97529 100644 --- a/maven/codenameone-gradle-plugin/src/main/java/com/codename1/gradle/AppSupport.java +++ b/maven/codenameone-gradle-plugin/src/main/java/com/codename1/gradle/AppSupport.java @@ -131,6 +131,26 @@ static void apply(final Project project, final ProjectLayout layout, final Coden addFramework(project, simulator.getName(), version, "codenameone-core", "codenameone-javase"); project.getDependencies().addProvider(simulator.getName(), version.map(v -> PluginInfo.GROUP + ":cn1-binaries-javase:" + v)); + // cn1-binaries-javase brings ffmpeg's Java half; its natives are chosen by + // Maven profiles keyed on the host OS, which Gradle does not evaluate. So + // the host's natives are named here, at the versions that pom was built + // with -- one platform, where the aggregator used to bring all seven. + String ffmpegVersion = PluginInfo.builtWith("ffmpeg.version"); + String javacppVersion = PluginInfo.builtWith("javacpp.version"); + if (ffmpegVersion == null || javacppVersion == null) { + // Skipping the natives here would be a simulator without media and + // without a word about why. The versions are written into the plugin + // when it is built, so this is a plugin that was built wrong. + throw new org.gradle.api.GradleException("The Codename One plugin does not know the ffmpeg" + + " and javacpp versions it was built with (plugin.properties). Rebuild the plugin."); + } + for (String platform : PluginInfo.hostFfmpegPlatforms( + System.getProperty("os.name"), System.getProperty("os.arch"))) { + project.getDependencies().add(simulator.getName(), + "org.bytedeco:ffmpeg:" + ffmpegVersion + ":" + platform); + project.getDependencies().add(simulator.getName(), + "org.bytedeco:javacpp:" + javacppVersion + ":" + platform); + } Configuration cssCompiler = resolvable(project, "cn1CssCompiler", "The Codename One CSS compiler"); addFramework(project, cssCompiler.getName(), version, "codenameone-css-cli"); @@ -741,6 +761,31 @@ private static void registerBuild(final Project project, String name, final Stri t.getOverwrite().set(false); }); + // The ports this task's target reads, and no others: the build tool does not + // depend on them, so that a project which only ever runs the simulator or + // builds in the cloud never downloads one. Nothing resolves this until the + // task runs. Not transitive -- a port is read as a file, and its pom names the + // framework jars the project already has. + final Configuration ports = resolvable(project, "cn1Ports" + + name.substring(0, 1).toUpperCase(java.util.Locale.ROOT) + name.substring(1), + "The port jars a local build of " + name + " reads"); + ports.setTransitive(false); + // At the plugin's version, never the project's: codenameone.version is the + // framework the application compiles against and may be held behind the + // plugin, and a builder paired with an older port writes a project that + // builds and does not start. The Maven plugin resolves them the same way. + final Provider portVersion = project.provider(PluginInfo::version); + ports.getDependencies().addAllLater(targetProvider.flatMap(target -> portVersion.map(v -> { + List out = new ArrayList(); + for (String port : com.codename1.maven.AppBuilder.portArtifacts(target)) { + int colon = port.indexOf(':'); + out.add(project.getDependencies().create(PluginInfo.GROUP + ":" + + (colon < 0 ? port : port.substring(0, colon)) + ":" + v + + (colon < 0 ? "" : ":" + port.substring(colon + 1)))); + } + return out; + })).orElse(Collections.emptyList())); + project.getTasks().register(name, Cn1BuildTask.class, t -> { common(t, project, layout, ext, userProperties); t.setGroup(BUILD_GROUP); @@ -759,6 +804,7 @@ private static void registerBuild(final Project project, String name, final Stri .map(Boolean::parseBoolean).orElse(Boolean.FALSE)); t.getOpen().set(ext.getOpenGeneratedProjects()); t.getFrameworkJars().from(framework); + t.getPortJars().from(ports); // Resolve after build-script configuration, including Kotlin and added/generated roots. t.getSourceRoots().set(project.provider(() -> sourceRoots(main, layout))); t.getFinalName().set(project.getName()); diff --git a/maven/codenameone-gradle-plugin/src/main/java/com/codename1/gradle/DelegatingProjectHost.java b/maven/codenameone-gradle-plugin/src/main/java/com/codename1/gradle/DelegatingProjectHost.java index cd252fae258..b2acf8a24e2 100644 --- a/maven/codenameone-gradle-plugin/src/main/java/com/codename1/gradle/DelegatingProjectHost.java +++ b/maven/codenameone-gradle-plugin/src/main/java/com/codename1/gradle/DelegatingProjectHost.java @@ -125,6 +125,11 @@ public File getJar(BuildArtifact artifact) { return delegate.getJar(artifact); } + @Override + public File getPortJar(String artifactId, String classifier) { + return delegate.getPortJar(artifactId, classifier); + } + @Override public File getJar(String groupId, String artifactId, String classifier) { return delegate.getJar(groupId, artifactId, classifier); diff --git a/maven/codenameone-gradle-plugin/src/main/java/com/codename1/gradle/GradleHostFactory.java b/maven/codenameone-gradle-plugin/src/main/java/com/codename1/gradle/GradleHostFactory.java index c8fdb0b29a1..fa67a46e505 100644 --- a/maven/codenameone-gradle-plugin/src/main/java/com/codename1/gradle/GradleHostFactory.java +++ b/maven/codenameone-gradle-plugin/src/main/java/com/codename1/gradle/GradleHostFactory.java @@ -83,4 +83,61 @@ public static ProjectHost create(Task task, Log log, ProjectLayout layout, Strin return new GradleProjectHost(log, layout, finalName, groupId, projectProperties, userProperties, elements, sourceRoots, artifacts, framework, codenameOneVersion); } + + /// A host for a task that may build a target locally, which reads port jars + /// (`AppBuilder.portArtifacts`). + /// + /// @param ports what the task's build target reads, as `artifactId[:classifier]` + /// @param portJars the files of exactly those, from a configuration of their own + /// that nothing resolves until this task runs + public static ProjectHost create(Task task, Log log, ProjectLayout layout, String finalName, String groupId, + Map projectProperties, Map userProperties, + FileCollection classpath, List sourceRoots, List artifacts, + FileCollection frameworkJars, String codenameOneVersion, + List ports, FileCollection portJars) { + GradleProjectHost host = (GradleProjectHost) create(task, log, layout, finalName, groupId, + projectProperties, userProperties, classpath, sourceRoots, artifacts, frameworkJars, + codenameOneVersion); + if (ports != null && portJars != null && !ports.isEmpty()) { + Map resolved = new LinkedHashMap(); + for (File f : portJars) { + String port = portOf(f.getName(), ports); + if (port != null) { + resolved.put(port, f); + } + } + host.portJars(resolved); + } + return host; + } + + /// Which of `ports` the file `name` is. The most specific artifact id wins, so + /// `codenameone-android-compat-1.jar` is never taken for `codenameone-android`. + static String portOf(String name, List ports) { + String best = null; + int bestLength = -1; + for (String port : ports) { + int colon = port.indexOf(':'); + String artifactId = colon < 0 ? port : port.substring(0, colon); + String classifier = colon < 0 ? null : port.substring(colon + 1); + if (!name.startsWith(artifactId + "-") || !name.endsWith(".jar")) { + continue; + } + // What follows the artifact id is the version, then the classifier. + // A version starts with a digit; "compat-1.0" does not. + String rest = name.substring(artifactId.length() + 1, name.length() - ".jar".length()); + if (rest.length() == 0 || !Character.isDigit(rest.charAt(0))) { + continue; + } + boolean classified = classifier != null && rest.endsWith("-" + classifier); + if (classifier != null ? !classified : rest.endsWith("-bundle")) { + continue; + } + if (artifactId.length() > bestLength) { + best = port; + bestLength = artifactId.length(); + } + } + return best; + } } diff --git a/maven/codenameone-gradle-plugin/src/main/java/com/codename1/gradle/GradleProjectHost.java b/maven/codenameone-gradle-plugin/src/main/java/com/codename1/gradle/GradleProjectHost.java index efde7ace619..162786caacd 100644 --- a/maven/codenameone-gradle-plugin/src/main/java/com/codename1/gradle/GradleProjectHost.java +++ b/maven/codenameone-gradle-plugin/src/main/java/com/codename1/gradle/GradleProjectHost.java @@ -58,6 +58,7 @@ final class GradleProjectHost implements ProjectHost { private final List artifacts; private final Map frameworkJars; private final String codenameOneVersion; + private Map portJars = Collections.emptyMap(); GradleProjectHost(Log log, ProjectLayout layout, String finalName, String groupId, Map projectProperties, Map userProperties, @@ -234,6 +235,17 @@ public File getJar(BuildArtifact artifact) { return artifact.getFile(); } + /// The ports the task's own configuration resolved for its build target. + void portJars(Map resolved) { + this.portJars = resolved; + } + + @Override + public File getPortJar(String artifactId, String classifier) { + File f = portJars.get(classifier == null ? artifactId : artifactId + ":" + classifier); + return f != null ? f : getJar("com.codenameone", artifactId, classifier); + } + @Override public File getJar(String groupId, String artifactId, String classifier) { File f = frameworkJars.get(groupId + ":" + artifactId + (classifier == null ? "" : ":" + classifier)); diff --git a/maven/codenameone-gradle-plugin/src/main/java/com/codename1/gradle/PluginInfo.java b/maven/codenameone-gradle-plugin/src/main/java/com/codename1/gradle/PluginInfo.java index 55cfdb94e9f..cc7e818da18 100644 --- a/maven/codenameone-gradle-plugin/src/main/java/com/codename1/gradle/PluginInfo.java +++ b/maven/codenameone-gradle-plugin/src/main/java/com/codename1/gradle/PluginInfo.java @@ -38,10 +38,10 @@ final class PluginInfo { private PluginInfo() { } - /// The plugin's version, which is also the Codename One version a project - /// builds against unless it sets `codenameone { version = ... }`. - static synchronized String version() { - if (version == null) { + private static Properties properties; + + private static synchronized Properties properties() { + if (properties == null) { Properties p = new Properties(); InputStream in = PluginInfo.class.getResourceAsStream("plugin.properties"); try { @@ -49,7 +49,7 @@ static synchronized String version() { p.load(in); } } catch (IOException ignored) { - // Falls through to the unknown version below. + // Falls through to the defaults of each reader below. } finally { if (in != null) { try { @@ -59,9 +59,52 @@ static synchronized String version() { } } } - String v = p.getProperty("version"); + properties = p; + } + return properties; + } + + /// The plugin's version, which is also the Codename One version a project + /// builds against unless it sets `codenameone { version = ... }`. + static synchronized String version() { + if (version == null) { + String v = properties().getProperty("version"); version = v == null || v.startsWith("${") ? "8.0-SNAPSHOT" : v.trim(); } return version; } + + /// A version the plugin was built with (`ffmpeg.version`, `javacpp.version`), + /// or null when the plugin runs from classes nothing filtered. + static String builtWith(String key) { + String v = properties().getProperty(key); + return v == null || v.trim().length() == 0 || v.startsWith("${") ? null : v.trim(); + } + + /// The bytedeco platform names of the native jars the simulator can load on + /// this machine -- the same table as the profiles of `cn1-binaries-javase`, + /// which Maven evaluates and Gradle does not. A Mac gets both architectures: + /// the JVM that configures the build and the one that runs the simulator are + /// routinely different there (an x64 JDK under Rosetta beside an arm64 one). + /// Empty where bytedeco has no build; the simulator then looks for ffmpeg on + /// `PATH`. + static java.util.List hostFfmpegPlatforms(String osName, String osArch) { + String os = osName == null ? "" : osName.toLowerCase(java.util.Locale.ROOT); + String arch = osArch == null ? "" : osArch.toLowerCase(java.util.Locale.ROOT); + if (os.contains("mac") || os.contains("darwin")) { + return java.util.Arrays.asList("macosx-arm64", "macosx-x86_64"); + } + if (os.contains("win")) { + return java.util.Collections.singletonList("windows-x86_64"); + } + if (os.contains("linux")) { + if ("amd64".equals(arch) || "x86_64".equals(arch)) { + return java.util.Collections.singletonList("linux-x86_64"); + } + if ("aarch64".equals(arch) || "arm64".equals(arch)) { + return java.util.Collections.singletonList("linux-arm64"); + } + } + return java.util.Collections.emptyList(); + } } diff --git a/maven/codenameone-gradle-plugin/src/main/java/com/codename1/gradle/tasks/Cn1BuildTask.java b/maven/codenameone-gradle-plugin/src/main/java/com/codename1/gradle/tasks/Cn1BuildTask.java index 18cd13de1d4..a85ca080b1b 100644 --- a/maven/codenameone-gradle-plugin/src/main/java/com/codename1/gradle/tasks/Cn1BuildTask.java +++ b/maven/codenameone-gradle-plugin/src/main/java/com/codename1/gradle/tasks/Cn1BuildTask.java @@ -88,6 +88,15 @@ public abstract class Cn1BuildTask extends Cn1Task { @PathSensitive(PathSensitivity.NAME_ONLY) public abstract ConfigurableFileCollection getFrameworkJars(); + /// The ports a local build of this task's target reads -- the iOS port and the + /// translator for an Xcode project, the Android port for a Gradle one -- and + /// nothing for a cloud build. Their configuration holds only what + /// `AppBuilder.portArtifacts` names for the target, so a build never downloads a + /// port it does not open. + @InputFiles + @PathSensitive(PathSensitivity.NAME_ONLY) + public abstract ConfigurableFileCollection getPortJars(); + /// The source roots the project compiles. @Input public abstract ListProperty getSourceRoots(); @@ -112,7 +121,8 @@ public void build() { AppBuilder builder = new TaskAppBuilder(GradleHostFactory.create(this, log(), layout(), getFinalName().get(), getGroupId().getOrElse(""), getProjectProperties().get(), getUserProperties().get(), getUploadClasspath(), getSourceRoots().get(), getArtifacts().get(), - getFrameworkJars(), getCodenameOneVersion().get()), work); + getFrameworkJars(), getCodenameOneVersion().get(), + AppBuilder.portArtifacts(getBuildTarget().get()), getPortJars()), work); builder.platform(getPlatform().get()) .buildTarget(getBuildTarget().get()) .automated(getAutomated().get()) diff --git a/maven/codenameone-gradle-plugin/src/main/resources/com/codename1/gradle/plugin.properties b/maven/codenameone-gradle-plugin/src/main/resources/com/codename1/gradle/plugin.properties index 15eb22f560e..b9dbbee79e5 100644 --- a/maven/codenameone-gradle-plugin/src/main/resources/com/codename1/gradle/plugin.properties +++ b/maven/codenameone-gradle-plugin/src/main/resources/com/codename1/gradle/plugin.properties @@ -1,2 +1,5 @@ # The plugin version, which is also the default Codename One version a project builds against. version=${project.version} +# The bundled desktop ffmpeg, whose host natives the plugin adds to the simulator. +ffmpeg.version=${ffmpeg.version} +javacpp.version=${cn1.javacpp.version} diff --git a/maven/codenameone-gradle-plugin/src/test/java/com/codename1/gradle/PortJarSelectionTest.java b/maven/codenameone-gradle-plugin/src/test/java/com/codename1/gradle/PortJarSelectionTest.java new file mode 100644 index 00000000000..aead5877be0 --- /dev/null +++ b/maven/codenameone-gradle-plugin/src/test/java/com/codename1/gradle/PortJarSelectionTest.java @@ -0,0 +1,90 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.gradle; + +import org.junit.jupiter.api.Test; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; + +/// How a resolved file is matched to the port it is, and which ffmpeg natives a +/// host is given. Both are decided from names alone, so both are tables. +class PortJarSelectionTest { + private static final List IOS = Arrays.asList("codenameone-ios:bundle", "codenameone-parparvm:bundle"); + private static final List ANDROID = Collections.singletonList("codenameone-android"); + + @Test + void aBundleIsMatchedByArtifactAndClassifier() { + assertEquals("codenameone-ios:bundle", GradleHostFactory.portOf("codenameone-ios-8.0-SNAPSHOT-bundle.jar", IOS)); + assertEquals("codenameone-parparvm:bundle", + GradleHostFactory.portOf("codenameone-parparvm-7.0.281-bundle.jar", IOS)); + // A timestamped snapshot, as a remote repository serves it. + assertEquals("codenameone-ios:bundle", + GradleHostFactory.portOf("codenameone-ios-8.0-20261111.101500-3-bundle.jar", IOS)); + } + + @Test + void theMainJarOfABundledArtifactIsNotTheBundle() { + assertNull(GradleHostFactory.portOf("codenameone-ios-8.0-SNAPSHOT.jar", IOS)); + assertNull(GradleHostFactory.portOf("codenameone-ios-8.0-SNAPSHOT-sources.jar", IOS)); + } + + @Test + void anUnclassifiedPortIsItsMainJar() { + assertEquals("codenameone-android", GradleHostFactory.portOf("codenameone-android-8.0-SNAPSHOT.jar", ANDROID)); + assertNull(GradleHostFactory.portOf("codenameone-android-8.0-SNAPSHOT-bundle.jar", ANDROID)); + } + + @Test + void aLongerArtifactIdIsNeverTakenForAShorterOne() { + assertNull(GradleHostFactory.portOf("codenameone-android-compat-8.0-SNAPSHOT.jar", ANDROID)); + assertNull(GradleHostFactory.portOf("codenameone-android-res-compiler-8.0-SNAPSHOT.jar", ANDROID)); + assertNull(GradleHostFactory.portOf("codenameone-core-8.0-SNAPSHOT.jar", IOS)); + assertNull(GradleHostFactory.portOf("codenameone-ios-8.0-SNAPSHOT-bundle.zip", IOS)); + } + + @Test + void aHostGetsItsOwnFfmpegAndNoOther() { + // A Mac takes both: an Intel JDK under Rosetta loads the x86_64 natives. + assertEquals(Arrays.asList("macosx-arm64", "macosx-x86_64"), + PluginInfo.hostFfmpegPlatforms("Mac OS X", "aarch64")); + assertEquals(Arrays.asList("macosx-arm64", "macosx-x86_64"), + PluginInfo.hostFfmpegPlatforms("Mac OS X", "x86_64")); + assertEquals(Collections.singletonList("windows-x86_64"), + PluginInfo.hostFfmpegPlatforms("Windows 11", "amd64")); + assertEquals(Collections.singletonList("linux-x86_64"), PluginInfo.hostFfmpegPlatforms("Linux", "amd64")); + assertEquals(Collections.singletonList("linux-arm64"), PluginInfo.hostFfmpegPlatforms("Linux", "aarch64")); + } + + @Test + void anUnknownHostGetsNone() { + // The simulator then finds ffmpeg on the PATH, or reports that it has none. + assertEquals(Collections.emptyList(), PluginInfo.hostFfmpegPlatforms("Linux", "riscv64")); + assertEquals(Collections.emptyList(), PluginInfo.hostFfmpegPlatforms("FreeBSD", "amd64")); + assertEquals(Collections.emptyList(), PluginInfo.hostFfmpegPlatforms(null, null)); + } +} diff --git a/maven/codenameone-maven-plugin/pom.xml b/maven/codenameone-maven-plugin/pom.xml index 3f97e47610f..8ed80e8a47f 100644 --- a/maven/codenameone-maven-plugin/pom.xml +++ b/maven/codenameone-maven-plugin/pom.xml @@ -274,10 +274,22 @@ cn1-builder-resources-android runtime + com.codenameone codenameone-android - runtime + provided + true + 1.5.11 1.7.11 3.8.0 ${maven.multiModuleProjectDirectory}/target/cn1-binaries @@ -187,11 +191,6 @@ jcefmaven ${jcefmaven.version} - - org.bytedeco - ffmpeg-platform - ${ffmpeg.version} - com.codenameone codenameone-core diff --git a/scripts/check-first-build-footprint.py b/scripts/check-first-build-footprint.py new file mode 100755 index 00000000000..98a9c2610ea --- /dev/null +++ b/scripts/check-first-build-footprint.py @@ -0,0 +1,398 @@ +#!/usr/bin/env python3 +"""Hold the size of a new project's first build. + +A developer's first contact with Codename One is a project from the +initializr and `./run.sh`. Maven then downloads everything that build +resolves, and for a long time that was 346 MB in 869 files of which the +simulator opened a fraction: + + * 157 MB was org.bytedeco's ffmpeg for seven platforms, two of them + Android, on a machine that can run one. + * 55 MB was the iOS, Android, macOS, Windows and Linux ports and the + ParparVM translator. They were dependencies of the Maven plugin, and + Maven resolves a plugin's whole class path before its first goal, so + a Hello World started in the simulator fetched all of them. + +Nothing failed, so nothing noticed: a dependency added to the plugin is +invisible in every test, and shows only as a slower first run on a +machine nobody on the project has -- an empty one. + +This script is that machine. It generates an application from the +archetype this tree builds, runs the build `run.sh` runs (stopping +before the simulator window) against an EMPTY local repository, and +reads what arrived. It is a floor, not the whole bill: the Maven wrapper's +own distribution, and whatever the simulator fetches once it is up (the +browser component's natives), are not in a local repository and are not +counted. + + 1. No port artifact. They are resolved when a target that reads one is + built (AppBuilder.portArtifacts); a port in a simulator build means + something depends on it again. + 2. No org.bytedeco native for a platform other than the host's. + 3. The total stays under a budget. + +Before any of that it reads maven/cn1-binaries/pom.xml and holds the +profile of EVERY host to the table below. CI runs this on one operating +system; a profile for another that named the wrong native, or lost its +activation, would otherwise reach the developers on that system as a +simulator with no ffmpeg and no error. + +The empty repository is filled from the workspace's own, offered to Maven +as a `file://` remote. That is what makes the run cheap and exact: the +snapshot artifacts this tree just built are the ones measured, and +nothing third party is fetched twice. Anything the workspace repository +does not have still comes from the network, as in any build. + + scripts/check-first-build-footprint.py --repo "$PWD/.m2-repo" + scripts/check-first-build-footprint.py --repo ... --platform-modules all + +Needs the workspace built and installed into --repo first: core, javase, +the Maven plugin and the app archetype. +""" + +import argparse +import os +import platform +import re +import shutil +import subprocess +import sys +import tempfile +import xml.etree.ElementTree as ET +from pathlib import Path + +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + +# The ports a local build reads, and only a local build. Kept as a list of +# its own rather than derived from AppBuilder: this is the independent +# check on that table, and a port added there has to be added here by hand. +PORT_ARTIFACTS = ( + "codenameone-android", + "codenameone-ios", + "codenameone-mac", + "codenameone-windows", + "codenameone-linux", + "codenameone-parparvm", +) + +# Artifacts the build tools use at build time only, in the same position the +# ports were in: provided and optional in codenameone-maven-plugin or +# codenameone-build-engine, where a change of scope sends them to every +# developer. None is read by a simulator run of a plain project. +BUILD_ONLY_ARTIFACTS = ( + "codenameone-android-compat", + "codenameone-swing-compat", + "codenameone-javafx-compat", + "cn1-hardening", +) + +# The profiles of maven/cn1-binaries/pom.xml: id -> (what activates it, the +# natives it adds). Each native is added for ffmpeg and for javacpp. +HOST_PROFILES = { + "ffmpeg-macosx": ({"family": "mac"}, {"macosx-arm64", "macosx-x86_64"}), + "ffmpeg-windows": ({"family": "windows"}, {"windows-x86_64"}), + "ffmpeg-linux-x86_64": ({"name": "Linux", "arch": "amd64"}, {"linux-x86_64"}), + "ffmpeg-linux-arm64": ({"name": "Linux", "arch": "aarch64"}, {"linux-arm64"}), +} + +# The profile a desktop jar is built with, because that jar runs on machines +# other than the one that built it. +ALL_PLATFORMS_PROFILE = "ffmpeg-all-platforms" +ALL_PLATFORMS_PROPERTY = "cn1.ffmpeg.allPlatforms" + +# What a first build may download, in MB, counting every file in the local +# repository. Measured at 174 MB on a Mac when this gate was written, against +# 346 MB before it; a Mac takes ffmpeg for both of its architectures (see +# maven/cn1-binaries/pom.xml), 43 MB where a Linux or Windows host takes 25, +# which is the difference between the two rows. The margin is room for +# ordinary dependency updates, not for a new port or a new native: a change +# that needs this raised should say what the megabytes are and why a +# simulator run has to carry them. +BUDGET_MB = {"mac": 195, "other": 180} + +# Fewer artifacts than this and the build did not really run: the report +# would be a clean bill for an empty directory. +MIN_ARTIFACTS = 150 + + +def host_ffmpeg_platforms(): + """The bytedeco platforms this machine may be handed. + + The same table as the profiles in maven/cn1-binaries/pom.xml and + PluginInfo.hostFfmpegPlatforms in the Gradle plugin, written out a + third time on purpose: it is what those two are checked against. + """ + system = platform.system().lower() + machine = platform.machine().lower() + if system == "darwin": + return {"macosx-arm64", "macosx-x86_64"} + if system == "windows": + return {"windows-x86_64"} + if system == "linux": + if machine in ("x86_64", "amd64"): + return {"linux-x86_64"} + if machine in ("aarch64", "arm64"): + return {"linux-arm64"} + return set() + + +def check_host_profiles(): + """Holds every host's profile in cn1-binaries-javase to HOST_PROFILES.""" + path = os.path.join(ROOT, "maven", "cn1-binaries", "pom.xml") + ns = {"m": "http://maven.apache.org/POM/4.0.0"} + root = ET.parse(path).getroot() + failures = [] + unconditional = {(d.findtext("m:artifactId", "", ns), d.findtext("m:classifier", "", ns)) + for d in root.findall("m:dependencies/m:dependency", ns) + if d.findtext("m:groupId", "", ns) == "org.bytedeco"} + for artifact in ("ffmpeg", "javacpp"): + if (artifact, "") not in unconditional: + failures.append("maven/cn1-binaries/pom.xml no longer depends on org.bytedeco:%s " + "unconditionally; the natives are useless without it" % artifact) + for artifact, classifier in sorted(unconditional): + if classifier or artifact.endswith("-platform"): + failures.append("maven/cn1-binaries/pom.xml depends on org.bytedeco:%s%s on every host; " + "natives belong in the profile of the host that runs them" + % (artifact, ":" + classifier if classifier else "")) + profiles = {p.findtext("m:id", "", ns): p for p in root.findall("m:profiles/m:profile", ns)} + for name, (activation, natives) in sorted(HOST_PROFILES.items()): + profile = profiles.get(name) + if profile is None: + failures.append("maven/cn1-binaries/pom.xml has no profile %s" % name) + continue + os_el = profile.find("m:activation/m:os", ns) + actual = {} if os_el is None else {c.tag.split("}")[-1]: (c.text or "").strip() for c in os_el} + if actual != activation: + failures.append("profile %s is activated by %s, expected %s" % (name, actual or "nothing", activation)) + wanted = {(a, n) for a in ("ffmpeg", "javacpp") for n in natives} + got = {(d.findtext("m:artifactId", "", ns), d.findtext("m:classifier", "", ns)) + for d in profile.findall("m:dependencies/m:dependency", ns)} + if got != wanted: + failures.append("profile %s adds %s, expected %s" % (name, sorted(got), sorted(wanted))) + # The desktop jar's profile: every command that builds one passes the + # property, and what it adds is every host's natives and no others. + everything = profiles.get(ALL_PLATFORMS_PROFILE) + if everything is None: + failures.append("maven/cn1-binaries/pom.xml has no profile %s" % ALL_PLATFORMS_PROFILE) + else: + name = everything.findtext("m:activation/m:property/m:name", "", ns).strip() + if name != ALL_PLATFORMS_PROPERTY: + failures.append("profile %s is activated by property %s, expected %s; the launchers pass " + "-D%s=true" % (ALL_PLATFORMS_PROFILE, name or "nothing", + ALL_PLATFORMS_PROPERTY, ALL_PLATFORMS_PROPERTY)) + if everything.findtext("m:activation/m:property/m:value", "", ns).strip() != "true": + failures.append("profile %s must be activated by %s=true, which is what the launchers pass " + "and what the plugin's own check reads" + % (ALL_PLATFORMS_PROFILE, ALL_PLATFORMS_PROPERTY)) + wanted = {(a, n) for a in ("ffmpeg", "javacpp") + for _, natives in HOST_PROFILES.values() for n in natives} + got = {(d.findtext("m:artifactId", "", ns), d.findtext("m:classifier", "", ns)) + for d in everything.findall("m:dependencies/m:dependency", ns)} + if got != wanted: + failures.append("profile %s adds %s, expected every host's natives: %s" + % (ALL_PLATFORMS_PROFILE, sorted(got), sorted(wanted))) + return failures + + +def workspace_version(): + with open(os.path.join(ROOT, "maven", "pom.xml"), encoding="utf-8") as f: + text = f.read() + # The project's own : the first one that is not inside . + text = re.sub(r".*?", "", text, flags=re.S) + m = re.search(r"([^<]+)", text) + if not m: + sys.exit("could not read the version from maven/pom.xml") + return m.group(1).strip() + + +def run(cmd, cwd, log): + with open(log, "ab") as out: + out.write(("\n$ " + " ".join(cmd) + "\n").encode()) + out.flush() + code = subprocess.call(cmd, cwd=cwd, stdout=out, stderr=subprocess.STDOUT, + stdin=subprocess.DEVNULL) + if code != 0: + with open(log, "rb") as f: + tail = f.read().decode("utf-8", "replace").splitlines()[-40:] + print("\n".join(tail)) + sys.exit("FAIL: `%s` exited %d (log: %s)" % (" ".join(cmd[:3]), code, log)) + + +def write_settings(path, repo): + url = Path(repo).as_uri() + block = """ + <{tag}> + workspace + {url} + trueignore + trueignore + """ + with open(path, "w", encoding="utf-8") as f: + f.write(""" + + + workspace + %s + + %s + + + + workspace + +""" % (block.format(tag="repository", url=url), block.format(tag="pluginRepository", url=url))) + + +def artifacts(repo): + """Every artifact file in a local repository as (group, artifact, version, classifier, bytes).""" + found = [] + total = 0 + for base, _dirs, files in os.walk(repo): + for name in files: + size = os.path.getsize(os.path.join(base, name)) + total += size + rel = os.path.relpath(base, repo).split(os.sep) + if len(rel) < 3: + continue + version, artifact, group = rel[-1], rel[-2], ".".join(rel[:-2]) + stem, ext = os.path.splitext(name) + if ext not in (".jar", ".zip", ".aar", ".so", ".dll", ".dylib"): + continue + prefix = artifact + "-" + version + if not stem.startswith(prefix): + continue + classifier = stem[len(prefix):].lstrip("-") + found.append((group, artifact, version, classifier, size)) + return found, total + + +def main(): + ap = argparse.ArgumentParser(description=__doc__.split("\n")[0]) + ap.add_argument("--repo", required=True, + help="the workspace's local Maven repository, with this tree installed") + ap.add_argument("--version", help="the Codename One version to build against (default: this tree's)") + ap.add_argument("--platform-modules", default=None, + help="the archetype's platformModules (default: the archetype's own)") + ap.add_argument("--budget-mb", type=float, help="override the budget") + ap.add_argument("--work", help="a directory to work in, kept afterwards (default: a temporary one)") + args = ap.parse_args() + + profile_failures = check_host_profiles() + if profile_failures: + for f in profile_failures: + print("FAIL: " + f) + return 1 + + repo = os.path.abspath(args.repo) + version = args.version or workspace_version() + marker = os.path.join(repo, "com", "codenameone", "codenameone-maven-plugin", version) + if not os.path.isdir(marker): + sys.exit("FAIL: %s has no codenameone-maven-plugin %s; build and install the workspace first" + % (repo, version)) + + work = os.path.abspath(args.work) if args.work else tempfile.mkdtemp(prefix="cn1-first-build-") + os.makedirs(work, exist_ok=True) + log = os.path.join(work, "first-build.log") + fresh = os.path.join(work, "empty-local-repo") + if os.path.exists(fresh) and os.listdir(fresh): + sys.exit("FAIL: %s is not empty; the measurement needs a repository nothing has filled" % fresh) + mvn = shutil.which("mvn") or shutil.which("mvn.cmd") + if not mvn: + sys.exit("FAIL: mvn is not on PATH") + + settings = os.path.join(work, "settings.xml") + write_settings(settings, repo) + # Generated from a repository of its own. Not from --repo, which is the + # remote of this run and must stay as the workspace build left it; and not + # from the empty one, which would count the archetype plugin as a download + # of the first build. + generate = [mvn, "-B", "-ntp", "-s", settings, + "-Dmaven.repo.local=" + os.path.join(work, "generate-local-repo"), "archetype:generate", + "-DarchetypeGroupId=com.codenameone", "-DarchetypeArtifactId=cn1app-archetype", + "-DarchetypeVersion=" + version, "-DgroupId=com.example.firstbuild", + "-DartifactId=firstbuild", "-Dversion=1.0-SNAPSHOT", "-DmainName=FirstBuild", + "-Dcn1Version=" + version, "-Dcn1PluginVersion=" + version, "-DinteractiveMode=false"] + if args.platform_modules: + generate.append("-DplatformModules=" + args.platform_modules) + project = os.path.join(work, "firstbuild") + if os.path.exists(project): + sys.exit("FAIL: %s already exists" % project) + run(generate, work, log) + + # What run.sh runs, less the last phase: `verify` is where the simulator + # is started. The application's whole runtime class path, which is what + # the simulator runs from, is resolved by `package`. + run([mvn, "-B", "-ntp", "-s", settings, "-Dmaven.repo.local=" + fresh, "package", "-Psimulator", + "-DskipTests", "-Dcodename1.platform=javase"], project, log) + + found, total = artifacts(fresh) + failures = [] + + if len(found) < MIN_ARTIFACTS: + failures.append("only %d artifacts arrived; the build did not resolve what a first build resolves" + % len(found)) + present = {(g, a) for g, a, _v, _c, _s in found} + for needed in ("codenameone-core", "codenameone-javase", "codenameone-maven-plugin"): + if ("com.codenameone", needed) not in present: + failures.append("%s was not resolved, so this run measured something other than a " + "simulator build" % needed) + + for g, a, v, c, size in found: + if g == "com.codenameone" and a in PORT_ARTIFACTS: + failures.append("%s:%s:%s%s (%.1f MB) was downloaded by a simulator build. A port is " + "resolved by the first build of a target that reads it " + "(AppBuilder.portArtifacts); something depends on this one again -- " + "look at the dependencies of codenameone-maven-plugin and " + "codenameone-build-engine, where it must be provided and optional." + % (g, a, v, ":" + c if c else "", size / 1e6)) + + if g == "com.codenameone" and a in BUILD_ONLY_ARTIFACTS: + failures.append("%s:%s:%s (%.1f MB) was downloaded by a simulator build. It is used at " + "build time by the projects that need it and is provided and optional " + "in the build tools' poms; something made it a dependency of every project." + % (g, a, v, size / 1e6)) + + allowed = host_ffmpeg_platforms() + natives = [(a, c, size) for g, a, _v, c, size in found if g == "org.bytedeco" and c] + for a, c, size in natives: + if c not in allowed: + failures.append("org.bytedeco:%s:%s (%.1f MB) is a native for another platform. " + "cn1-binaries-javase selects the host's alone (maven/cn1-binaries/pom.xml); " + "something reaches org.bytedeco's -platform aggregator again." + % (a, c, size / 1e6)) + if allowed and not any(a == "ffmpeg" for a, _c, _s in natives): + failures.append("no ffmpeg native arrived for this host (%s). The simulator would have no " + "bundled ffmpeg: the host profile in maven/cn1-binaries/pom.xml did not " + "activate." % ", ".join(sorted(allowed))) + + budget = args.budget_mb if args.budget_mb is not None else \ + BUDGET_MB["mac" if platform.system() == "Darwin" else "other"] + mb = total / 1e6 + if mb > budget: + failures.append("the first build downloaded %.0f MB, over the budget of %.0f MB. See BUDGET_MB " + "in this script before raising it." % (mb, budget)) + + by_group = {} + for g, a, _v, _c, size in found: + key = ".".join(g.split(".")[:2]) + by_group[key] = by_group.get(key, 0) + size + print("First build of a new project: %.0f MB in %d artifacts (budget %.0f MB)" % (mb, len(found), budget)) + for key, size in sorted(by_group.items(), key=lambda kv: -kv[1])[:8]: + print(" %7.1f MB %s" % (size / 1e6, key)) + for g, a, _v, c, size in sorted(found, key=lambda t: -t[4])[:8]: + print(" %7.1f MB %s:%s%s" % (size / 1e6, g, a, ":" + c if c else "")) + + if failures: + print() + for f in failures: + print("FAIL: " + f) + print("\nlog: %s\nrepository: %s" % (log, fresh)) + return 1 + if not args.work: + shutil.rmtree(work, ignore_errors=True) + print("OK") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/initializr/common/src/main/resources/common.zip b/scripts/initializr/common/src/main/resources/common.zip index 1a36322b76a..2478707f48d 100644 Binary files a/scripts/initializr/common/src/main/resources/common.zip and b/scripts/initializr/common/src/main/resources/common.zip differ diff --git a/scripts/initializr/common/src/main/resources/eclipse.zip b/scripts/initializr/common/src/main/resources/eclipse.zip index 8c52dc4e630..40a57a2c931 100644 Binary files a/scripts/initializr/common/src/main/resources/eclipse.zip and b/scripts/initializr/common/src/main/resources/eclipse.zip differ diff --git a/scripts/initializr/common/src/main/resources/idea.zip b/scripts/initializr/common/src/main/resources/idea.zip index 10a6803943a..49cb581c781 100644 Binary files a/scripts/initializr/common/src/main/resources/idea.zip and b/scripts/initializr/common/src/main/resources/idea.zip differ diff --git a/scripts/initializr/common/src/main/resources/netbeans.zip b/scripts/initializr/common/src/main/resources/netbeans.zip index 9fd5336c844..d0f5c20ceee 100644 Binary files a/scripts/initializr/common/src/main/resources/netbeans.zip and b/scripts/initializr/common/src/main/resources/netbeans.zip differ diff --git a/scripts/initializr/common/src/main/resources/vscode.zip b/scripts/initializr/common/src/main/resources/vscode.zip index 78c79f0010b..d8bc6b66076 100644 Binary files a/scripts/initializr/common/src/main/resources/vscode.zip and b/scripts/initializr/common/src/main/resources/vscode.zip differ diff --git a/scripts/sync-initializr-launchers.py b/scripts/sync-initializr-launchers.py index 7172074ebc0..bb4908e6e03 100644 --- a/scripts/sync-initializr-launchers.py +++ b/scripts/sync-initializr-launchers.py @@ -33,4 +33,50 @@ elif stale: raise SystemExit('Initializr launcher drift: ' + ', '.join(stale) + '. Run python scripts/sync-initializr-launchers.py --write') +# The desktop jar runs on machines other than the one that built it, so every +# command that builds one asks for every desktop platform's media natives. The +# IDE launchers are separate copies of the archetype's and nothing regenerates +# them, so a copy that builds the jar without the flag is reported here. +FLAG = b'cn1.ffmpeg.allPlatforms' +resources = archive.parent +host_only = [] + + +def builds_without_flag(data): + """True when a command in this launcher builds the desktop jar without the flag.""" + if b'' in data: + # NetBeans: one file, several actions, each a command of its own. + # An action that only cleans resolves no media and needs no flag. + return any(b'executable-jar' in action and FLAG not in action + and action.count(b'') != action.count(b'clean') + for action in data.split(b'')) + lines = [line for line in data.splitlines() if b'-Pexecutable-jar' in line] + if lines: + return any(FLAG not in line for line in lines) + return b'executable-jar' in data and FLAG not in data + + +for ide in ('eclipse.zip', 'idea.zip', 'netbeans.zip', 'vscode.zip', 'common.zip'): + with zipfile.ZipFile(resources / ide) as source: + for entry in source.infolist(): + if entry.is_dir() or entry.filename.endswith('pom.xml'): + continue + data = source.read(entry.filename) + if builds_without_flag(data): + host_only.append(ide + '!' + entry.filename) +# The archetype's own copies, which no archive above is built from. +for name in ['.vscode/settings.json', 'mvnconfig.toml', 'tools/eclipse', 'tools/netbeans']: + path = archetype / name + for file in ([path] if path.is_file() else sorted(p for p in path.rglob('*') if p.is_file())): + if builds_without_flag(file.read_bytes()): + host_only.append(str(file.relative_to(root))) +if builds_without_flag((root / 'maven/cli/src/main/batsh/build.batsh').read_bytes()): + host_only.append('maven/cli/src/main/batsh/build.batsh') +run_profiles = root / 'maven/codenameone-maven-plugin/src/main/resources/com/codename1/maven/intellij/workspace.xml' +if FLAG not in run_profiles.read_bytes(): + host_only.append(str(run_profiles.relative_to(root))) +if host_only: + raise SystemExit('These build the desktop jar without -D' + FLAG.decode() + + '=true, so it would carry media natives for the build machine only: ' + + ', '.join(host_only)) print('Initializr launcher parity OK') diff --git a/scripts/test-starter-launchers.py b/scripts/test-starter-launchers.py index 05c256109af..66c1df9badc 100644 --- a/scripts/test-starter-launchers.py +++ b/scripts/test-starter-launchers.py @@ -143,6 +143,8 @@ def run(launcher, target, expected, code=0): ('ios_source', '-Dcodename1.buildTarget=ios-source')]: run('build', target, argument) run('build', '', '-Pexecutable-jar') + # The jar runs on other machines, so it takes every desktop platform's media natives. + run('build', '', '-Dcn1.ffmpeg.allPlatforms') run('run', '', '-Psimulator') run('build', 'javascript_cloud', '-Dcodename1.buildTarget=javascript', 37) run('run', 'simulator', '-Psimulator', 37)