Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 14 additions & 1 deletion automation/run-e2e/docker/mxbuild.Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -26,15 +26,28 @@ echo "Downloading mxbuild ${MENDIX_VERSION} and docker building for ${BUILDPLATF
\
rm -rf /var/lib/apt/lists/* && \
apt-get update --allow-insecure-repositories -qqy && \
apt-get install -qqy --allow-unauthenticated libicu70 && \
apt-get install -qqy --allow-unauthenticated libicu70 libfontconfig1 libfreetype6 libharfbuzz0b && \
apt-get -qqy remove --auto-remove wget && \
apt-get clean && \
rm -rf /var/lib/apt/lists/* && \
\
echo "#!/bin/bash -x" >/bin/mxbuild && \
echo "source /bin/mxlibs.sh" >>/bin/mxbuild && \
echo "/tmp/mxbuild/modeler/mxbuild --java-home=/opt/java/openjdk --java-exe-path=/opt/java/openjdk/bin/java \$@" >>/bin/mxbuild && \
chmod +x /bin/mxbuild && \
\
echo "#!/bin/bash -x" >/bin/mx && \
echo "source /bin/mxlibs.sh" >>/bin/mx && \
echo "/tmp/mxbuild/modeler/mx \$@" >>/bin/mx && \
chmod +x /bin/mx

# libSkiaSharp.so shipped with mxbuild (Mendix 11) does not link libuuid/libfreetype
# itself, so their symbols must be preloaded. Paths differ per architecture.
RUN cat >/bin/mxlibs.sh <<'SH'
libs=""
for lib in libuuid.so.1 libfreetype.so.6; do
path=$(ls /lib/*/$lib /usr/lib/*/$lib 2>/dev/null | head -1)
[ -n "$path" ] && libs="$libs $path"
done
export LD_PRELOAD="${LD_PRELOAD:+$LD_PRELOAD }${libs# }"
SH
168 changes: 168 additions & 0 deletions automation/run-e2e/lib/atlas.mjs
Original file line number Diff line number Diff line change
@@ -0,0 +1,168 @@
import crossZip from "cross-zip";
import fetch from "node-fetch";
import { createWriteStream } from "node:fs";
import { execFileSync } from "node:child_process";
import { mkdtemp } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { pipeline } from "node:stream";
import { promisify } from "node:util";
import sh from "shelljs";

const { cp, rm, mkdir, test, exec } = sh;
const streamPipe = promisify(pipeline);

const releasesUrl = "https://api.github.com/repos/mendix/atlas/releases";

const themesourceDirsToRemove = [
"themesource/atlas_ui_resources",
"themesource/atlas_core",
"themesource/atlas_nativemobile_content",
"themesource/atlas_web_content",
"themesource/datawidgets"
];

// Mendix 11 test projects ship Atlas 4, which renamed design properties and moved
// JavaScript actions into the Atlas Core package. Mendix 10 and below stay on Atlas 3.
const atlas3 = {
themeTag: "atlasui-theme-files-2024-01-25",
coreTag: "atlas-core-v3.17.0",
dirsToRemove: themesourceDirsToRemove,
renameDesignProperties: false
};

const atlas4 = {
themeTag: "atlasui-theme-files-2025-10-08",
coreTag: "atlas-core-v4.4.0",
dirsToRemove: [...themesourceDirsToRemove, "javascriptsource/atlas_core"],
renameDesignProperties: true
};

/** Atlas release matching the given Mendix version (e.g. "11.12.0" or "10.24.0.73019"). */
export function getAtlasConfig(mendixVersion) {
const major = Number.parseInt(mendixVersion, 10);
return Number.isFinite(major) && major >= 11 ? atlas4 : atlas3;
}

/** True when the model needs `mx rename-design-properties` after the Atlas update. */
export function needsDesignPropertyRename(mendixVersion) {
return getAtlasConfig(mendixVersion).renameDesignProperties;
}

/**
* Mendix version the test project was last saved with, read from the .mpr file.
* Returns undefined when sqlite3 is unavailable or the file can't be read.
*/
export function detectProjectMendixVersion(mprFile) {
try {
return execFileSync("sqlite3", [mprFile, "select _ProductVersion from _MetaData;"], {
encoding: "utf-8",
stdio: ["pipe", "pipe", "pipe"]
}).trim();
} catch {
return undefined;
}
}

async function usetmp() {
return mkdtemp(join(tmpdir(), "atlas_files_"));
}

async function getReleaseByTag(tag) {
const token = process.env.GITHUB_TOKEN;
const headers = {
Accept: "application/vnd.github+json",
"X-GitHub-Api-Version": "2022-11-28",
// Anonymous requests work for these public releases, a token only raises the rate limit.
...(token ? { Authorization: `Bearer ${token}` } : {})
};

const response = await fetch(`${releasesUrl}/tags/${tag}`, { headers });
if (!response.ok) {
throw new Error(`Can't fetch release for tag: ${tag} (HTTP ${response.status})`);
}
return response.json();
}

async function downloadAndExtract(url, downloadPath, extractPath) {
try {
const response = await fetch(url);
if (!response.ok) {
throw new Error(`HTTP ${response.status} while downloading ${url}`);
}
await streamPipe(response.body, createWriteStream(downloadPath));
crossZip.unzipSync(downloadPath, extractPath);
} catch (e) {
throw new Error(`Unable to download and extract from ${url}`, { cause: e });
} finally {
rm("-f", downloadPath);
}
}

async function updateAtlasTheme(projectDir, atlas) {
console.log(`Copying Atlas theme files from ${atlas.themeTag}`);

const release = await getReleaseByTag(atlas.themeTag);
const asset = release.assets?.find(a => a.name.endsWith(".zip"));
if (!asset) {
throw new Error(`No .zip asset found for release tag: ${atlas.themeTag}`);
}

const outPath = await usetmp();
await downloadAndExtract(asset.browser_download_url, join(await usetmp(), "AtlasTheme.zip"), outPath);

const themePath = join(outPath, "theme");
mkdir("-p", themePath);
for (const dir of ["web", "native"]) {
const src = join(outPath, dir);
if (test("-d", src)) {
cp("-r", src, themePath);
}
}

rm("-rf", join(projectDir, "theme"));
cp("-r", themePath, projectDir);
}

async function updateAtlasThemesource(projectDir, atlas) {
console.log(`Copying Atlas themesource files from ${atlas.coreTag}`);

const release = await getReleaseByTag(atlas.coreTag);
const asset = release.assets?.find(a => a.name.endsWith(".mpk"));
if (!asset) {
throw new Error(`No .mpk asset found for release tag: ${atlas.coreTag}`);
}

const outPath = await usetmp();
await downloadAndExtract(asset.browser_download_url, join(await usetmp(), "AtlasCore.zip"), outPath);

rm(
"-rf",
atlas.dirsToRemove.map(dir => join(projectDir, dir))
);

// The Atlas files are copied with read-only permissions, but mxbuild writes to
// some generated files during the build.
for (const dir of ["themesource", "javascriptsource"]) {
const src = join(outPath, dir);
if (!test("-d", src)) {
continue;
}
cp("-r", src, projectDir);
exec(`chmod -R +w "${join(projectDir, dir)}"`, { silent: true });
}
}

/**
* Replaces the Atlas theme and themesource of a test project with the release that
* matches the given Mendix version. Falls back to the version stored in the .mpr file.
*/
export async function updateAtlas(projectDir, mendixVersion, mprFile) {
const version = mendixVersion || (mprFile && detectProjectMendixVersion(mprFile));
const atlas = getAtlasConfig(version);

console.log(`Updating Atlas for Mendix ${version ?? "unknown"} (${atlas.coreTag}, ${atlas.themeTag})`);

await updateAtlasTheme(projectDir, atlas);
await updateAtlasThemesource(projectDir, atlas);
}
4 changes: 2 additions & 2 deletions automation/run-e2e/lib/ci.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@ export async function ci() {
}

if (options.updateProject) {
await updateTestProject();
await updateTestProject(mendixVersion);
}

if (options.useCompose) {
Expand Down Expand Up @@ -127,7 +127,7 @@ async function runWithDockerRaw({ mendixVersion, ip, freePort }) {
const mxruntimeImage = await prepareImage("mxruntime", mendixVersion);

const projectFile = ls(config.mprFileGlob).toString();
createDeploymentBundle(mxbuildImage, projectFile);
createDeploymentBundle(mxbuildImage, projectFile, mendixVersion);

runtimeContainerId = await startRuntime(mxruntimeImage, mendixVersion, ip, freePort);
startPlaywright(ip, freePort);
Expand Down
10 changes: 0 additions & 10 deletions automation/run-e2e/lib/config.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -3,16 +3,6 @@ export const testProjectDir = "tests/testProject";
export const postUnzipProjectDirGlob = "tests/testProjects-*";
export const mprFileGlob = "tests/testProject/*.mpr";
export const nameForDownloadedArchive = "testProject.zip";
export const nameForDownloadedAtlasCore = "AtlasCore.zip";
export const nameForDownloadedAtlasTheme = "AtlasTheme.zip";
export const atlasCoreReleaseUrl = "https://api.github.com/repos/mendix/atlas/releases";
export const mxVersionMapUrl =
"https://raw.githubusercontent.com/mendix/web-widgets/main/automation/run-e2e/mendix-versions.json";
export const tmpDirPrefix = "run_e2e_files_";
export const atlasDirsToRemove = [
"tests/testProject/themesource/atlas_ui_resources",
"tests/testProject/themesource/atlas_core",
"tests/testProject/themesource/atlas_nativemobile_content",
"tests/testProject/themesource/atlas_web_content",
"tests/testProject/themesource/datawidgets"
];
4 changes: 3 additions & 1 deletion automation/run-e2e/lib/dev.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,9 @@ export async function dev() {
}
if (options.withPreps || options.updateProject) {
// Run update project hook
await updateTestProject();
// No explicit version in dev mode — updateTestProject falls back to the
// version the .mpr was saved with to pick the matching Atlas release.
await updateTestProject(process.env.MENDIX_VERSION);

console.log(
c.yellow(
Expand Down
9 changes: 7 additions & 2 deletions automation/run-e2e/lib/docker-utils.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
import fetch from "node-fetch";
import c from "ansi-colors";
import sh from "shelljs";
import { needsDesignPropertyRename } from "./atlas.mjs";

const { cat } = sh;

Expand Down Expand Up @@ -46,7 +47,7 @@
return image;
}

export function createDeploymentBundle(mxbuildImage, projectFile) {
export function createDeploymentBundle(mxbuildImage, projectFile, mendixVersion) {
console.log(`Start building deployment bundle.`);

const mprPath = `/source/${projectFile}`;
Expand All @@ -55,7 +56,11 @@
const subCommands = [
// 1. Update widgets in project.
`mx update-widgets --loose-version-check ${mprPath}`,
// 2. Build project to:
// 2. Sync the model with the Atlas version copied into the test project.
// Atlas 4 (Mendix 11) renamed design properties, without this the build
// fails on renamed design properties and Atlas layouts.
...(needsDesignPropertyRename(mendixVersion) ? [`mx rename-design-properties ${mprPath}`] : []),
// 3. Build project to:
// a. Check errors.
// b. Prepare `deployment` dir for mxruntime.
// Output file is not used, so put it to tmp.
Expand Down Expand Up @@ -120,7 +125,7 @@
if (response.ok) {
attempts = 0;
}
} catch (e) {

Check warning on line 128 in automation/run-e2e/lib/docker-utils.mjs

View workflow job for this annotation

GitHub Actions / Run code quality check

'e' is defined but never used
console.log(`Could not reach http://${ip}:${freePort}, trying again...`);
}
await new Promise(resolve => setTimeout(resolve, 3000));
Expand Down
90 changes: 5 additions & 85 deletions automation/run-e2e/lib/update-test-project.mjs
Original file line number Diff line number Diff line change
@@ -1,90 +1,12 @@
import crossZip from "cross-zip";
import assert from "node:assert/strict";
import { spawnSync } from "node:child_process";
import { createWriteStream } from "node:fs";
import { join, resolve } from "node:path";
import sh from "shelljs";
import * as config from "./config.mjs";
import { fetchGithubRestAPI, fetchWithReport, packageMeta, streamPipe, usetmp } from "./utils.mjs";
import { atlasCoreReleaseUrl } from "./config.mjs";
import { packageMeta } from "./utils.mjs";
import { updateAtlas } from "./atlas.mjs";

const { cp, rm, mkdir, test } = sh;

async function getReleaseByTag(tag) {
const url = `${atlasCoreReleaseUrl}/tags/${tag}`;
const response = await fetchGithubRestAPI(url);
if (!response.ok) {
throw new Error(`Can't fetch release for tag: ${tag}`);
}
return await response.json();
}

async function downloadAndExtract(url, downloadPath, extractPath) {
try {
await streamPipe((await fetchWithReport(url)).body, createWriteStream(downloadPath));
crossZip.unzipSync(downloadPath, extractPath);
} catch (e) {
throw new Error(`Unable to download and extract from ${url}`, { cause: e });
} finally {
rm("-f", downloadPath);
}
}

async function updateAtlasThemeSource() {
console.log("Copying Atlas themesource files from latest Atlas Core release");

rm("-rf", config.atlasDirsToRemove);

const release = await getReleaseByTag("atlas-core-v3.17.0");
const { browser_download_url } = release.assets[0];
const downloadedPath = join(await usetmp(), config.nameForDownloadedAtlasCore);
const outPath = await usetmp();

await downloadAndExtract(browser_download_url, downloadedPath, outPath);

const themeSourcePath = join(outPath, "themesource");
if (!test("-d", themeSourcePath)) {
console.log(`Directory not found: ${themeSourcePath}. Creating it.`);
mkdir("-p", themeSourcePath);
}

cp("-r", themeSourcePath, config.testProjectDir);

// Fix file permissions to ensure Docker can write to theme files
// The Atlas theme files are copied with read-only permissions
// but mxbuild needs to write to some generated files during build
sh.exec(`chmod -R +w "${config.testProjectDir}/themesource"`, { silent: true });
}

async function updateAtlasTheme() {
console.log("Copying Atlas theme files from latest Atlas UI theme release");

rm("-rf", "tests/testProject/theme");

// Fetch the specific release by tag from GitHub API
const tag = "atlasui-theme-files-2024-01-25";
const release = await getReleaseByTag(tag);
if (!release.assets || release.assets.length === 0) {
throw new Error(`No assets found for release tag: ${tag}`);
}
const [{ browser_download_url }] = release.assets;
const downloadedPath = join(await usetmp(), config.nameForDownloadedAtlasTheme);
const outPath = await usetmp();

await downloadAndExtract(browser_download_url, downloadedPath, outPath);

const themePath = join(outPath, "theme");
if (!test("-d", themePath)) {
console.log(`Directory not found: ${themePath}. Creating it.`);
mkdir("-p", themePath);
}
const webPath = join(outPath, "web");
const nativePath = join(outPath, "native");
cp("-r", webPath, themePath);
cp("-r", nativePath, themePath);

cp("-r", themePath, config.testProjectDir);
}
const { cp, ls, mkdir } = sh;

async function runReleaseScript() {
const { name: packageName, version } = packageMeta;
Expand Down Expand Up @@ -121,12 +43,10 @@ async function runUpdateProjectScript() {
spawnSync(command, args, { stdio: "inherit", shell: true });
}

export async function updateTestProject() {
export async function updateTestProject(mendixVersion) {
console.log("Updating test project files (widgets, themesource, atlas, etc.)");

await updateAtlasTheme();

await updateAtlasThemeSource();
await updateAtlas(config.testProjectDir, mendixVersion, ls(config.mprFileGlob)[0]);

process.env.MX_PROJECT_PATH = resolve(process.cwd(), config.testProjectDir);

Expand Down
Loading
Loading