diff --git a/jest.config.js b/jest.config.js index 3b89ea2c4e..c221c66a00 100644 --- a/jest.config.js +++ b/jest.config.js @@ -18,7 +18,7 @@ const browserEnvConfig = { // Transform ESM modules to CommonJS for Jest // These packages ship as pure ESM and need to be transformed by Babel transformIgnorePatterns: [ - '/node_modules/(?!(query-string|decode-uri-component|iongraph-web|split-on-first|filter-obj|fetch-mock|devtools-reps)/)', + '/node_modules/(?!(query-string|decode-uri-component|iongraph-web|split-on-first|filter-obj|fetch-mock|devtools-reps|json-slabs)/)', ], // Mock static assets (images, CSS, etc.) diff --git a/package.json b/package.json index 1c082839f2..0668526259 100644 --- a/package.json +++ b/package.json @@ -91,6 +91,7 @@ "gecko-profiler-demangle": "^0.4.0", "idb": "^8.0.3", "iongraph-web": "0.2.1", + "json-slabs": "^0.3.0", "jszip": "^3.10.1", "long": "^5.3.2", "memoize-immutable": "^3.0.0", diff --git a/src/actions/publish.ts b/src/actions/publish.ts index 3cbef3751b..5b427b929f 100644 --- a/src/actions/publish.ts +++ b/src/actions/publish.ts @@ -60,7 +60,7 @@ import type { ProfileIndexTranslationMaps, } from 'firefox-profiler/types'; import { compress } from 'firefox-profiler/utils/gz'; -import { serializeProfile } from 'firefox-profiler/profile-logic/process-profile'; +import { serializeProfileToJsonString } from 'firefox-profiler/profile-logic/process-profile'; export function updateSharingOption( slug: keyof CheckedSharingOptions, @@ -322,7 +322,9 @@ export function encodeSanitizedProfile( const encodingPromise: Promise = (async function () { try { dispatch(sanitizedProfileEncodingStarted(sanitizedProfile)); - const gzipData = await compress(serializeProfile(sanitizedProfile)); + const gzipData = await compress( + serializeProfileToJsonString(sanitizedProfile) + ); const blob = new Blob([gzipData], { type: 'application/octet-binary' }); dispatch(sanitizedProfileEncodingCompleted(sanitizedProfile, blob)); return { type: 'SUCCESS', profileData: blob }; diff --git a/src/node-tools/profiler-edit.ts b/src/node-tools/profiler-edit.ts index 49c11a55f8..1de294e4dd 100644 --- a/src/node-tools/profiler-edit.ts +++ b/src/node-tools/profiler-edit.ts @@ -4,7 +4,11 @@ import fs from 'fs'; import { Command, CommanderError, Option } from 'commander'; -import { unserializeProfileOfArbitraryFormat } from 'firefox-profiler/profile-logic/process-profile'; +import { + serializeProfileToJsonSlabsFile, + serializeProfileToJsonString, + unserializeProfileOfArbitraryFormat, +} from 'firefox-profiler/profile-logic/process-profile'; import { computeCompactedProfile } from 'firefox-profiler/profile-logic/profile-compacting'; import { GOOGLE_STORAGE_BUCKET } from 'firefox-profiler/app-logic/constants'; import { compress } from 'firefox-profiler/utils/gz'; @@ -128,6 +132,24 @@ async function loadProfile(source: ProfileSource): Promise { } } +async function encodeProfileWithFilename( + profile: Profile, + filename: string +): Promise { + if (filename.endsWith('.jslb') || filename.endsWith('.jslb.gz')) { + const bytes = serializeProfileToJsonSlabsFile(profile); + if (filename.endsWith('.jslb.gz')) { + return compress(bytes); + } + return bytes; + } + const s = serializeProfileToJsonString(profile); + if (filename.endsWith('.gz')) { + return compress(s); + } + return new TextEncoder().encode(s); +} + export async function run(options: CliOptions) { const profile = await loadProfile(options.input); @@ -185,15 +207,13 @@ export async function run(options: CliOptions) { const { profile: compactedProfile } = computeCompactedProfile(profile); - console.log(`Saving profile to ${options.output}`); - if (options.output.endsWith('.gz')) { - fs.writeFileSync( - options.output, - await compress(JSON.stringify(compactedProfile)) - ); - } else { - fs.writeFileSync(options.output, JSON.stringify(compactedProfile)); - } + const outputFilename = options.output; + console.log(`Saving profile to ${outputFilename}`); + const bytes = await encodeProfileWithFilename( + compactedProfile, + outputFilename + ); + fs.writeFileSync(outputFilename, bytes); console.log('Finished.'); } diff --git a/src/profile-logic/process-profile.ts b/src/profile-logic/process-profile.ts index d1f44222dc..24846ec17b 100644 --- a/src/profile-logic/process-profile.ts +++ b/src/profile-logic/process-profile.ts @@ -1,6 +1,12 @@ /* This Source Code Form is subject to the terms of the Mozilla Public * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ +import { + isJsonSlabsFile, + decode as decodeJsonSlabs, + encode as encodeJsonSlabs, +} from 'json-slabs'; + import { attemptToConvertChromeProfile } from './import/chrome'; import { attemptToConvertDhat } from './import/dhat'; import { GlobalDataCollector } from './global-data-collector'; @@ -1922,10 +1928,42 @@ export function processGeckoProfile(geckoProfile: GeckoProfile): Profile { /** * Take a processed profile and convert it to a string. */ -export function serializeProfile(profile: Profile): string { +export function serializeProfileToJsonString(profile: Profile): string { return JSON.stringify(profile); } +/** + * Take a profile and convert it to a Uint8Array in the JsonSlabs format. + * + * This is more efficient than JSON if the profile contains large typed arrays. + */ +export function serializeProfileToJsonSlabsFile( + profile: Profile +): Uint8Array { + // Encode the profile object with the binary JsonSlabs container format. + return encodeJsonSlabs(profile, [ + // "Split-out" slabs: + // + // This second argument to the encode function is an array of objects which + // should be pulled out into their own dedicated slabs. This is totally + // optional and doesn't change what the decoded object will look like. + // We use it to "split out" some large tables as long as we haven't converted + // them to use typed arrays. This already gives us a benefit: It means that + // decoding will use several JSON.parse calls rather than just one single + // JSON.parse call, and each individual JSON.parse will act on a smaller + // string, which means it's less likely to hit any string size limits. + // + // As we convert more and more tables / columns to typed arrays, the "skeleton + // JSON" for these tables will become much smaller and we won't need to split + // out those tables anymore. + profile.threads, + profile.shared.stackTable, + profile.shared.frameTable, + profile.shared.funcTable, + profile.shared.stringArray, + ]); +} + // If applicable, this function will try to "fix" a processed profile that was // copied from the console on an old version of the UI, where such a profile // would have a `stringTable` property rather than a `stringArray` property on @@ -2062,7 +2100,9 @@ export async function unserializeProfileOfArbitraryFormat( profileBytes = await decompress(profileBytes); } - if (isArtTraceFormat(profileBytes)) { + if (isJsonSlabsFile(profileBytes)) { + arbitraryFormat = decodeJsonSlabs(profileBytes); + } else if (isArtTraceFormat(profileBytes)) { arbitraryFormat = convertArtTraceProfile(profileBytes); } else if (verifyMagic(SIMPLEPERF_MAGIC, profileBytes)) { const { convertSimpleperfTraceProfile } = diff --git a/src/test/components/DragAndDrop.test.tsx b/src/test/components/DragAndDrop.test.tsx index 96e26a3bf2..c3618112ff 100644 --- a/src/test/components/DragAndDrop.test.tsx +++ b/src/test/components/DragAndDrop.test.tsx @@ -12,7 +12,7 @@ import { DragAndDropOverlay, } from '../../components/app/DragAndDrop'; import { getProfileFromTextSamples } from '../fixtures/profiles/processed-profile'; -import { serializeProfile } from '../../profile-logic/process-profile'; +import { serializeProfileToJsonString } from '../../profile-logic/process-profile'; import { getView } from 'firefox-profiler/selectors'; import { updateBrowserConnectionStatus } from 'firefox-profiler/actions/app'; import { mockWebChannel } from '../fixtures/mocks/web-channel'; @@ -100,9 +100,13 @@ describe('app/DragAndDrop', () => { const [dragAndDrop, overlay] = container.children; const { profile } = getProfileFromTextSamples('A'); - const file = new File([serializeProfile(profile)], 'profile.json', { - type: 'application/json', - }); + const file = new File( + [serializeProfileToJsonString(profile)], + 'profile.json', + { + type: 'application/json', + } + ); const files = [file]; fireEvent.dragEnter(dragAndDrop); diff --git a/src/test/fixtures/profiles/zip-file.ts b/src/test/fixtures/profiles/zip-file.ts index 07d19e37d7..a0f4c527ce 100644 --- a/src/test/fixtures/profiles/zip-file.ts +++ b/src/test/fixtures/profiles/zip-file.ts @@ -2,7 +2,7 @@ * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ import { getProfileFromTextSamples } from '../../fixtures/profiles/processed-profile'; -import { serializeProfile } from '../../../profile-logic/process-profile'; +import { serializeProfileToJsonString } from '../../../profile-logic/process-profile'; import { receiveZipFile } from '../../../actions/receive-profile'; import { setDataSource } from '../../../actions/profile-view'; import type { @@ -17,7 +17,7 @@ import JSZip from 'jszip'; */ export function getZippedProfiles(files: string[] = []): JSZip { const { profile } = getProfileFromTextSamples('A'); - const profileText = serializeProfile(profile); + const profileText = serializeProfileToJsonString(profile); const zip = new JSZip(); files.forEach((fileName) => { diff --git a/src/test/store/receive-profile.test.ts b/src/test/store/receive-profile.test.ts index 08fa5b3601..f480f467c9 100644 --- a/src/test/store/receive-profile.test.ts +++ b/src/test/store/receive-profile.test.ts @@ -36,7 +36,7 @@ import { createGeckoProfile } from '../fixtures/profiles/gecko-profile'; import { blankStore, storeWithProfile } from '../fixtures/stores'; import { processGeckoProfile, - serializeProfile, + serializeProfileToJsonString, } from '../../profile-logic/process-profile'; import { getProfileFromTextSamples, @@ -1044,7 +1044,7 @@ describe('actions/receive-profile', function () { const expectedUrl = 'https://profiles.club/shared.json'; window.fetchMock.get( expectedUrl, - compress(serializeProfile(_getSimpleProfile())) + compress(serializeProfileToJsonString(_getSimpleProfile())) ); const store = blankStore(); await store.dispatch(retrieveProfileOrZipFromUrl(expectedUrl)); @@ -1174,7 +1174,7 @@ describe('actions/receive-profile', function () { const { getState, view } = await setupTestWithFile({ type: 'application/json', - payload: serializeProfile(profile), + payload: serializeProfileToJsonString(profile), }); expect(view.phase).toBe('DATA_LOADED'); expect(ProfileViewSelectors.getProfile(getState()).meta.product).toEqual( @@ -1194,7 +1194,7 @@ describe('actions/receive-profile', function () { const { getState, view } = await setupTestWithFile({ type: 'application/json', - payload: JSON.stringify(profile), // Note: No serializeProfile call! + payload: JSON.stringify(profile), // Note: No serializeProfileToJsonString call! }); expect(view.phase).toBe('DATA_LOADED'); @@ -1243,7 +1243,7 @@ describe('actions/receive-profile', function () { const { getState, view } = await setupTestWithFile({ type: '', - payload: serializeProfile(profile), + payload: serializeProfileToJsonString(profile), }); expect(view.phase).toBe('DATA_LOADED'); expect(ProfileViewSelectors.getProfile(getState()).meta.product).toEqual( @@ -1257,7 +1257,9 @@ describe('actions/receive-profile', function () { const { getState, view } = await setupTestWithFile({ type: '', - payload: extractArrayBuffer(await compress(serializeProfile(profile))), + payload: extractArrayBuffer( + await compress(serializeProfileToJsonString(profile)) + ), }); expect(view.phase).toBe('DATA_LOADED'); expect(ProfileViewSelectors.getProfile(getState()).meta.product).toEqual( @@ -1289,7 +1291,9 @@ describe('actions/receive-profile', function () { const { getState, view } = await setupTestWithFile({ type: 'application/gzip', - payload: extractArrayBuffer(await compress(serializeProfile(profile))), + payload: extractArrayBuffer( + await compress(serializeProfileToJsonString(profile)) + ), }); expect(view.phase).toBe('DATA_LOADED'); expect(ProfileViewSelectors.getProfile(getState()).meta.product).toEqual( @@ -1303,7 +1307,9 @@ describe('actions/receive-profile', function () { const { getState, view } = await setupTestWithFile({ type: 'application/json', - payload: extractArrayBuffer(await compress(serializeProfile(profile))), + payload: extractArrayBuffer( + await compress(serializeProfileToJsonString(profile)) + ), }); expect(view.phase).toBe('DATA_LOADED'); expect(ProfileViewSelectors.getProfile(getState()).meta.product).toEqual( @@ -1345,7 +1351,7 @@ describe('actions/receive-profile', function () { it('can load a zipped profile', async function () { const { getState, view } = await setupZipTestWithProfile( 'profile.json', - serializeProfile(_getSimpleProfile()) + serializeProfileToJsonString(_getSimpleProfile()) ); expect(view.phase).toBe('DATA_LOADED'); const zipInStore = ZippedProfilesSelectors.getZipFile(getState()); @@ -1358,7 +1364,7 @@ describe('actions/receive-profile', function () { it('will load and view a simple profile with no errors', async function () { const { getState, dispatch } = await setupZipTestWithProfile( 'profile.json', - serializeProfile(_getSimpleProfile()) + serializeProfileToJsonString(_getSimpleProfile()) ); expect(ZippedProfilesSelectors.getZipFileState(getState()).phase).toEqual( @@ -1376,7 +1382,7 @@ describe('actions/receive-profile', function () { it('will be an error to view a profile with no threads', async function () { const { getState, dispatch } = await setupZipTestWithProfile( 'profile.json', - serializeProfile(getEmptyProfile()) + serializeProfileToJsonString(getEmptyProfile()) ); expect(ZippedProfilesSelectors.getZipFileState(getState()).phase).toEqual( diff --git a/src/test/store/zipped-profiles.test.ts b/src/test/store/zipped-profiles.test.ts index 19db955c36..66a40e40ce 100644 --- a/src/test/store/zipped-profiles.test.ts +++ b/src/test/store/zipped-profiles.test.ts @@ -16,7 +16,7 @@ import * as ZippedProfilesActions from '../../actions/zipped-profiles'; import * as ReceiveProfileActions from '../../actions/receive-profile'; import * as ProfileViewActions from '../../actions/profile-view'; import { getProfileFromTextSamples } from '../fixtures/profiles/processed-profile'; -import { serializeProfile } from '../../profile-logic/process-profile'; +import { serializeProfileToJsonString } from '../../profile-logic/process-profile'; import { compress } from '../../utils/gz'; import type { PreviewSelection } from 'firefox-profiler/types'; @@ -112,7 +112,7 @@ describe('reducer zipFileState', function () { // returns a typed array whose `instanceof Uint8Array` check fails against // the main realm's global, so JSZip wouldn't recognize it directly. const gzippedProfile = new Uint8Array( - await compress(serializeProfile(profile)) + await compress(serializeProfileToJsonString(profile)) ); const zip = new JSZip(); diff --git a/src/test/unit/process-profile.test.ts b/src/test/unit/process-profile.test.ts index 27cc48a646..f446068b0f 100644 --- a/src/test/unit/process-profile.test.ts +++ b/src/test/unit/process-profile.test.ts @@ -1,10 +1,13 @@ /* This Source Code Form is subject to the terms of the Mozilla Public * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ +import { isJsonSlabsFile } from 'json-slabs'; + import { extractFuncsAndResourcesFromFrameLocations, processGeckoProfile, - serializeProfile, + serializeProfileToJsonSlabsFile, + serializeProfileToJsonString, unserializeProfileOfArbitraryFormat, } from '../../profile-logic/process-profile'; import { GlobalDataCollector } from 'firefox-profiler/profile-logic/global-data-collector'; @@ -382,25 +385,47 @@ describe('gecko profilerOverhead processing', function () { describe('serializeProfile', function () { it('should produce a parsable profile string', async function () { const profile = processGeckoProfile(createGeckoProfile()); - const serialized = serializeProfile(profile); + const serialized = serializeProfileToJsonString(profile); expect(JSON.parse.bind(null, serialized)).not.toThrow(); }); it('should produce the same profile in a roundtrip', async function () { const profile = processGeckoProfile(createGeckoProfile()); - const serialized = serializeProfile(profile); + const serialized = serializeProfileToJsonString(profile); const roundtrip = await unserializeProfileOfArbitraryFormat(serialized); // FIXME: Uncomment this line after resolving `undefined` serialization issue // See: https://github.com/firefox-devtools/profiler/issues/1599 // expect(profile).toEqual(roundtrip); - const secondSerialized = serializeProfile(roundtrip); + const secondSerialized = serializeProfileToJsonString(roundtrip); const secondRountrip = await unserializeProfileOfArbitraryFormat(secondSerialized); expect(roundtrip).toEqual(secondRountrip); }); }); +describe('serializeProfileToJsonSlabsFile', function () { + it('should produce bytes recognized as a JsonSlabs file', function () { + const profile = processGeckoProfile(createGeckoProfile()); + const bytes = serializeProfileToJsonSlabsFile(profile); + expect(isJsonSlabsFile(bytes)).toBe(true); + }); + + it('should produce the same profile in a roundtrip', async function () { + const profile = processGeckoProfile(createGeckoProfile()); + const bytes = serializeProfileToJsonSlabsFile(profile); + const roundtrip = await unserializeProfileOfArbitraryFormat(bytes); + + // Two roundtrips should be stable, mirroring the JSON serializer test + // above (see issue #1599 for why we can't compare against the original + // profile directly). + const secondBytes = serializeProfileToJsonSlabsFile(roundtrip); + const secondRoundtrip = + await unserializeProfileOfArbitraryFormat(secondBytes); + expect(roundtrip).toEqual(secondRoundtrip); + }); +}); + describe('js allocation processing', function () { function getAllocationMarkerHelper(geckoThread: GeckoThread) { let time = 0; diff --git a/src/test/unit/profile-fetch.test.ts b/src/test/unit/profile-fetch.test.ts index f971b291ad..8484c9aee1 100644 --- a/src/test/unit/profile-fetch.test.ts +++ b/src/test/unit/profile-fetch.test.ts @@ -4,7 +4,7 @@ import JSZip from 'jszip'; import { fetchProfile } from '../../utils/profile-fetch'; -import { serializeProfile } from '../../profile-logic/process-profile'; +import { serializeProfileToJsonString } from '../../profile-logic/process-profile'; import { getProfileFromTextSamples } from '../fixtures/profiles/processed-profile'; import { extractArrayBuffer } from '../fixtures/utils'; @@ -37,7 +37,7 @@ describe('fetchProfile', function () { content: 'generated-zip' | 'generated-json' | Uint8Array; }) { const { url, contentType, content } = obj; - const stringProfile = serializeProfile(_getSimpleProfile()); + const stringProfile = serializeProfileToJsonString(_getSimpleProfile()); const profile = JSON.parse(stringProfile); let arrayBuffer; diff --git a/src/test/unit/profile-upgrading.test.ts b/src/test/unit/profile-upgrading.test.ts index f510cec935..6345faf12b 100644 --- a/src/test/unit/profile-upgrading.test.ts +++ b/src/test/unit/profile-upgrading.test.ts @@ -4,7 +4,7 @@ import { unserializeProfileOfArbitraryFormat, - serializeProfile, + serializeProfileToJsonString, } from '../../profile-logic/process-profile'; import { upgradeGeckoProfileToCurrentVersion } from '../../profile-logic/gecko-profile-versioning'; import { @@ -79,7 +79,9 @@ describe('upgrading processed profiles', function () { expect(upgradedProfile.meta.preprocessedProfileVersion).toEqual( PROCESSED_PROFILE_VERSION ); - expect(JSON.parse(serializeProfile(upgradedProfile))).toMatchSnapshot(); + expect( + JSON.parse(serializeProfileToJsonString(upgradedProfile)) + ).toMatchSnapshot(); } it('should upgrade processed-1.json all the way to the current version', async function () { diff --git a/yarn.lock b/yarn.lock index d55c597ecf..24efca10c9 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6918,6 +6918,11 @@ json-schema@^0.4.0: resolved "https://registry.yarnpkg.com/json-schema/-/json-schema-0.4.0.tgz#f7de4cf6efab838ebaeb3236474cbba5a1930ab5" integrity sha512-es94M3nTIfsEPisRafak+HDLfHXnKBhV3vU5eqPcS3flIWqcxJWgXHXiey3YrpaNsanY5ei1VoYEbOzijuq9BA== +json-slabs@^0.3.0: + version "0.3.0" + resolved "https://registry.yarnpkg.com/json-slabs/-/json-slabs-0.3.0.tgz#e1306d100c3b9946f2d897f18d8889ff24839a6c" + integrity sha512-C1EPAoAweC1R8YIjxK7Wu22ms/gVQF+fYQlYlUDjQHPJ3uj1XLPVPxkSG3Zx55Z6g5r4Bte1I79n7qu03FRC6Q== + json-stable-stringify-without-jsonify@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/json-stable-stringify-without-jsonify/-/json-stable-stringify-without-jsonify-1.0.1.tgz#9db7b59496ad3f3cfef30a75142d2d930ad72651"