diff --git a/packages/cli-upload/test/upload.test.js b/packages/cli-upload/test/upload.test.js index 503796e7b..e4aaa8afe 100644 --- a/packages/cli-upload/test/upload.test.js +++ b/packages/cli-upload/test/upload.test.js @@ -108,7 +108,9 @@ describe('percy upload', () => { 'resource-url': 'http://local/test-1', mimetype: 'text/html', 'for-widths': null, - 'is-root': true + 'is-root': true, + 'content-md5': jasmine.any(String), + 'content-length': jasmine.any(Number) } }, { type: 'resources', @@ -117,7 +119,9 @@ describe('percy upload', () => { 'resource-url': 'http://local/test-1.png', mimetype: 'image/png', 'for-widths': null, - 'is-root': null + 'is-root': null, + 'content-md5': jasmine.any(String), + 'content-length': jasmine.any(Number) } }]) } diff --git a/packages/client/src/client.js b/packages/client/src/client.js index 452c60e93..3b11e7701 100644 --- a/packages/client/src/client.js +++ b/packages/client/src/client.js @@ -9,6 +9,7 @@ import { request, formatBytes, sha256hash, + md5base64, base64encode, getPackageJSON, waitForTimeout, @@ -21,6 +22,15 @@ import { // Default client API URL can be set with an env var for API development const { PERCY_CLIENT_API_URL = 'https://percy.io/api/v1' } = process.env; let pkg = getPackageJSON(import.meta.url); + +// Strict boolean env var parser. Only "1" or "true" (case-insensitive) count as +// enabled. Avoids the JS-truthy trap where "0" or "false" would silently enable +// a flag because both are non-empty strings. +function envBool(value) { + if (value == null) return false; + let v = String(value).toLowerCase(); + return v === '1' || v === 'true'; +} // minimum polling interval milliseconds const MIN_POLLING_INTERVAL = 1_000; const INVALID_TOKEN_ERROR_MESSAGE = 'Unable to retrieve snapshot details with write access token. Kindly use a full access token for retrieving snapshot details with Synchronous CLI.'; @@ -310,6 +320,15 @@ export class PercyClient { }); } + // Returns capability headers advertised on snapshot creation. `direct-upload-v1` + // tells percy-api to mint Content-MD5/Content-Length-bound GCS signed URLs in + // the missing-resources response. PERCY_DISABLE_DIRECT_UPLOAD is the + // customer-side escape hatch that forces fallback to POST /resources. + directUploadCapabilityHeaders() { + if (envBool(process.env.PERCY_DISABLE_DIRECT_UPLOAD)) return {}; + return { 'X-Percy-Capabilities': 'direct-upload-v1' }; + } + // Creates a build with optional build resources. Only one build can be // created at a time per instance so snapshots and build finalization can be // done more seamlessly without manually tracking build ids @@ -562,6 +581,66 @@ export class PercyClient { }, this, uploadConcurrency); } + // PUTs a single resource directly to its percy-api-issued GCS signed URL. + // Returns the resource when the upload hits a transport-class failure + // (caller falls back to legacy POST /resources). Throws on correctness-class + // failures (400 BadDigest, 412 if-generation-match) — those signal a real + // bug in the declared md5/length or a leaked URL replay, and must not be + // silently masked by the fallback path. + async uploadResourceDirect(resource, meta = {}) { + let { signedUploadUrl, content, md5, contentLength, url, mimetype } = resource; + this.log.debug(`Direct-uploading ${formatBytes(contentLength)} resource: ${url}`, meta); + this.mayBeLogUploadSize(contentLength, meta); + + let headers = { + 'Content-MD5': md5, + 'Content-Length': contentLength + }; + if (mimetype) headers['Content-Type'] = mimetype; + + try { + await request(signedUploadUrl, { + method: 'PUT', + rawBody: true, + body: content, + headers, + meta + }); + return null; + } catch (err) { + let status = err.response?.statusCode; + if (status === 400 || status === 412) throw err; + this.log.debug(`Direct upload failed for ${url}, falling back to legacy: ${err.message}`, meta); + return resource; + } + } + + // Uploads resources directly to GCS signed URLs concurrently. Returns the + // list of resources that hit transport-class failures so the caller can + // route them through the legacy upload path. Concurrency is shared with + // the legacy uploader via PERCY_RESOURCE_UPLOAD_CONCURRENCY. + async uploadResourcesDirect(resources, meta = {}) { + this.log.debug(`Direct-uploading ${resources.length} resource(s)...`, meta); + + const uploadConcurrency = parseInt(process.env.PERCY_RESOURCE_UPLOAD_CONCURRENCY) || 2; + let fallback = []; + + await pool(function*() { + for (let resource of resources) { + let resourceMeta = { + url: resource.url, + sha: resource.sha, + ...meta + }; + yield this.uploadResourceDirect(resource, resourceMeta).then(failed => { + if (failed) fallback.push(failed); + }); + } + }, this, uploadConcurrency); + + return fallback; + } + // Creates a snapshot for the active build using the provided attributes. async createSnapshot(buildId, { name, @@ -630,13 +709,15 @@ export class PercyClient { 'resource-url': r.url || null, 'is-root': r.root || null, 'for-widths': r.widths || null, - mimetype: r.mimetype || null + mimetype: r.mimetype || null, + 'content-md5': r.md5 ?? (r.content != null ? md5base64(r.content) : null), + 'content-length': r.contentLength ?? (r.content != null ? Buffer.byteLength(r.content, 'utf-8') : null) } })) } } } - }, { identifier: 'snapshot.post', ...meta }); + }, { identifier: 'snapshot.post', ...meta }, this.directUploadCapabilityHeaders()); } // Finalizes a snapshot. @@ -657,7 +738,29 @@ export class PercyClient { this.log.debug(`${missing?.length || 0} Missing resources: ${options.name}...`, meta); if (missing?.length) { let resources = options.resources.reduce((acc, r) => Object.assign(acc, { [r.sha]: r }), {}); - await this.uploadResources(buildId, missing.map(({ id }) => resources[id]), meta); + let disabled = envBool(process.env.PERCY_DISABLE_DIRECT_UPLOAD); + let direct = []; + let legacy = []; + + for (let entry of missing) { + let resource = resources[entry.id]; + if (!resource) continue; + let signedUploadUrl = entry.attributes?.['signed-upload-url']; + if (!disabled && signedUploadUrl) { + direct.push({ ...resource, signedUploadUrl }); + } else { + legacy.push(resource); + } + } + + if (direct.length) { + let failed = await this.uploadResourcesDirect(direct, meta); + legacy.push(...failed); + } + + if (legacy.length) { + await this.uploadResources(buildId, legacy, meta); + } } this.log.debug(`Resources uploaded: ${options.name}...`, meta); diff --git a/packages/client/src/utils.js b/packages/client/src/utils.js index 323d4ca9b..344dfef54 100644 --- a/packages/client/src/utils.js +++ b/packages/client/src/utils.js @@ -25,6 +25,15 @@ export function sha256hash(content) { .digest('hex'); } +// Returns a base64-encoded MD5 of a string or buffer. Used as Content-MD5 for +// GCS direct-bucket-upload integrity (RFC 1864), declared alongside the SHA-256. +export function md5base64(content) { + return crypto + .createHash('md5') + .update(content, 'utf-8') + .digest('base64'); +} + // Returns a base64 encoding of a string or buffer. export function base64encode(content) { return Buffer @@ -132,7 +141,7 @@ export async function request(url, options = {}, callback) { // gather request options let { body, headers, retries, retryNotFound, - interval, noProxy, buffer, meta = {}, ...requestOptions + interval, noProxy, buffer, rawBody, meta = {}, ...requestOptions } = options; let { protocol, hostname, port, pathname, search, hash } = new URL(url); @@ -142,8 +151,9 @@ export async function request(url, options = {}, callback) { let { default: http } = protocol === 'https:' ? await import('https') : await import('http'); let { proxyAgentFor } = await import('./proxy.js'); - // automatically stringify body content - if (body !== undefined && typeof body !== 'string') { + // rawBody: send body and headers verbatim (e.g. binary PUT to a GCS signed URL). + // Otherwise stringify non-string bodies as JSON. + if (!rawBody && body !== undefined && typeof body !== 'string') { headers = { 'Content-Type': 'application/json', ...headers }; body = JSON.stringify(body); } diff --git a/packages/client/test/client.test.js b/packages/client/test/client.test.js index 6227423ae..b9185333c 100644 --- a/packages/client/test/client.test.js +++ b/packages/client/test/client.test.js @@ -1,7 +1,7 @@ import fs from 'fs'; import logger from '@percy/logger/test/helpers'; import { mockgit } from '@percy/env/test/helpers'; -import { sha256hash, base64encode } from '@percy/client/utils'; +import { md5base64, sha256hash, base64encode } from '@percy/client/utils'; import PercyClient from '@percy/client'; import api, { mockRequests } from './helpers.js'; import * as CoreConfig from '@percy/core/config'; @@ -1142,6 +1142,129 @@ describe('PercyClient', () => { }); }); + describe('#uploadResourceDirect() / #uploadResourcesDirect()', () => { + const GCS_BASE = 'https://storage.googleapis.com'; + let gcsReply; + + function makeResource(overrides = {}) { + const content = overrides.content ?? 'foo'; + return { + url: '/foo.css', + sha: sha256hash(content), + mimetype: 'text/css', + content, + md5: md5base64(content), + contentLength: Buffer.byteLength(content, 'utf-8'), + signedUploadUrl: `${GCS_BASE}/percy-resources/ns_gid_${sha256hash(content)}?sig=fake`, + ...overrides + }; + } + + beforeEach(async () => { + gcsReply = await mockRequests(GCS_BASE); + }); + + it('PUTs raw bytes with Content-MD5, Content-Length, Content-Type headers and resolves to null on 200', async () => { + gcsReply.and.callFake(() => [200]); + + const resource = makeResource({ content: 'p { color: purple; }' }); + await expectAsync(client.uploadResourceDirect(resource)).toBeResolvedTo(null); + + const req = gcsReply.calls.mostRecent().args[0]; + expect(req.method).toBe('PUT'); + expect(req.headers['Content-MD5']).toBe(resource.md5); + expect(req.headers['Content-Length']).toBe(resource.contentLength); + expect(req.headers['Content-Type']).toBe('text/css'); + }); + + it('returns the resource for fallback on 403 (signed URL expired)', async () => { + gcsReply.and.callFake(() => [403, 'expired']); + + const resource = makeResource(); + await expectAsync(client.uploadResourceDirect(resource)).toBeResolvedTo(resource); + }); + + it('returns the resource for fallback on 429 (rate limited)', async () => { + gcsReply.and.callFake(() => [429]); + + const resource = makeResource(); + await expectAsync(client.uploadResourceDirect(resource)).toBeResolvedTo(resource); + }); + + it('returns the resource for fallback on 5xx after retries', async () => { + gcsReply.and.callFake(() => [503]); + + const resource = makeResource(); + await expectAsync(client.uploadResourceDirect(resource)).toBeResolvedTo(resource); + }); + + it('throws on 400 BadDigest (correctness failure must surface, not be masked)', async () => { + gcsReply.and.callFake(() => [400, 'BadDigest']); + + const resource = makeResource(); + await expectAsync(client.uploadResourceDirect(resource)).toBeRejected(); + }); + + it('throws on 412 Precondition Failed (object already exists — replay or bug)', async () => { + gcsReply.and.callFake(() => [412, 'PreconditionFailed']); + + const resource = makeResource(); + await expectAsync(client.uploadResourceDirect(resource)).toBeRejected(); + }); + + it('omits Content-Type when resource has no mimetype', async () => { + gcsReply.and.callFake(() => [200]); + + const resource = makeResource(); + delete resource.mimetype; + await client.uploadResourceDirect(resource); + + const req = gcsReply.calls.mostRecent().args[0]; + expect(req.headers['Content-Type']).toBeUndefined(); + }); + + describe('#uploadResourcesDirect()', () => { + it('resolves to empty fallback list when every resource succeeds', async () => { + gcsReply.and.callFake(() => [200]); + + await expectAsync(client.uploadResourcesDirect([ + makeResource({ content: 'a' }), + makeResource({ content: 'b' }) + ])).toBeResolvedTo([]); + }); + + it('collects transport-failed resources for fallback while letting successes pass', async () => { + const resources = [ + makeResource({ content: 'a' }), + makeResource({ content: 'b' }), + makeResource({ content: 'c' }) + ]; + + // succeed for 'a' and 'c', 5xx for 'b' + gcsReply.and.callFake(({ path }) => { + if (path.includes(resources[1].sha)) return [503]; + return [200]; + }); + + const fallback = await client.uploadResourcesDirect(resources); + expect(fallback.length).toBe(1); + expect(fallback[0].sha).toBe(resources[1].sha); + }); + + it('propagates correctness failures from any resource', async () => { + gcsReply.and.callFake(() => [400, 'BadDigest']); + + await expectAsync(client.uploadResourcesDirect([ + makeResource() + ])).toBeRejected(); + }); + + it('does nothing for an empty resource list', async () => { + await expectAsync(client.uploadResourcesDirect([])).toBeResolvedTo([]); + }); + }); + }); + describe('#createSnapshot()', () => { it('throws when missing a build id', async () => { await expectAsync(client.createSnapshot()) @@ -1222,7 +1345,9 @@ describe('PercyClient', () => { 'resource-url': '/foo', mimetype: 'text/html', 'for-widths': [1000], - 'is-root': true + 'is-root': true, + 'content-md5': md5base64('foo'), + 'content-length': Buffer.byteLength('foo', 'utf-8') } }, { type: 'resources', @@ -1231,7 +1356,9 @@ describe('PercyClient', () => { 'resource-url': '/bar', mimetype: 'image/png', 'for-widths': null, - 'is-root': null + 'is-root': null, + 'content-md5': md5base64('bar'), + 'content-length': Buffer.byteLength('bar', 'utf-8') } }] } @@ -1314,7 +1441,9 @@ describe('PercyClient', () => { 'resource-url': '/foo', mimetype: 'text/html', 'for-widths': [1000], - 'is-root': true + 'is-root': true, + 'content-md5': md5base64('foo'), + 'content-length': Buffer.byteLength('foo', 'utf-8') } }, { type: 'resources', @@ -1323,7 +1452,9 @@ describe('PercyClient', () => { 'resource-url': '/bar', mimetype: 'image/png', 'for-widths': null, - 'is-root': null + 'is-root': null, + 'content-md5': md5base64('bar'), + 'content-length': Buffer.byteLength('bar', 'utf-8') } }] } @@ -1365,7 +1496,9 @@ describe('PercyClient', () => { 'resource-url': null, 'for-widths': null, 'is-root': null, - mimetype: null + mimetype: null, + 'content-md5': null, + 'content-length': null } }] } @@ -1373,6 +1506,77 @@ describe('PercyClient', () => { } }); }); + + it('advertises direct-upload-v1 capability header by default', async () => { + await expectAsync( + client.createSnapshot(123, { name: 'cap-test', resources: [{ sha: 'sha' }] }) + ).toBeResolved(); + + expect(api.requests['/builds/123/snapshots'][0].headers).toEqual( + jasmine.objectContaining({ 'X-Percy-Capabilities': 'direct-upload-v1' }) + ); + }); + + it('suppresses capability header when PERCY_DISABLE_DIRECT_UPLOAD is set', async () => { + process.env.PERCY_DISABLE_DIRECT_UPLOAD = '1'; + try { + await expectAsync( + client.createSnapshot(123, { name: 'cap-test', resources: [{ sha: 'sha' }] }) + ).toBeResolved(); + + expect(api.requests['/builds/123/snapshots'][0].headers['X-Percy-Capabilities']).toBeUndefined(); + } finally { + delete process.env.PERCY_DISABLE_DIRECT_UPLOAD; + } + }); + + it('PERCY_DISABLE_DIRECT_UPLOAD uses strict parsing — "0" / "false" / "anything-else" do NOT disable', async () => { + // regression guard: an earlier !!process.env.X would treat any non-empty + // string as truthy, so PERCY_DISABLE_DIRECT_UPLOAD=0 would silently + // disable direct upload despite reading like "off" + for (const v of ['0', 'false', 'no', 'off', 'garbage']) { + process.env.PERCY_DISABLE_DIRECT_UPLOAD = v; + try { + await client.createSnapshot(123, { name: `cap-${v}`, resources: [{ sha: 'sha' }] }); + const last = api.requests['/builds/123/snapshots'].slice(-1)[0]; + expect(last.headers['X-Percy-Capabilities']).toBe('direct-upload-v1'); + } finally { + delete process.env.PERCY_DISABLE_DIRECT_UPLOAD; + } + } + }); + + it('PERCY_DISABLE_DIRECT_UPLOAD strict-true values (1/true, case-insensitive) all disable', async () => { + for (const v of ['1', 'true', 'True', 'TRUE']) { + process.env.PERCY_DISABLE_DIRECT_UPLOAD = v; + try { + await client.createSnapshot(123, { name: `cap-${v}`, resources: [{ sha: 'sha' }] }); + const last = api.requests['/builds/123/snapshots'].slice(-1)[0]; + expect(last.headers['X-Percy-Capabilities']).toBeUndefined(); + } finally { + delete process.env.PERCY_DISABLE_DIRECT_UPLOAD; + } + } + }); + + it('prefers caller-supplied md5/contentLength over recomputing from content', async () => { + await expectAsync( + client.createSnapshot(123, { + name: 'precomputed', + resources: [{ + url: '/x', + sha: 'precomputed-sha', + content: 'foo', + md5: 'caller-md5-base64', + contentLength: 999 + }] + }) + ).toBeResolved(); + + const attrs = api.requests['/builds/123/snapshots'][0].body.data.relationships.resources.data[0].attributes; + expect(attrs['content-md5']).toBe('caller-md5-base64'); + expect(attrs['content-length']).toBe(999); + }); }); describe('#finalizeSnapshot()', () => { @@ -1438,7 +1642,9 @@ describe('PercyClient', () => { mimetype: 'text/html', 'resource-url': null, 'for-widths': [1000], - 'is-root': true + 'is-root': true, + 'content-md5': md5base64(testDOM), + 'content-length': Buffer.byteLength(testDOM, 'utf-8') } }] } @@ -1475,6 +1681,193 @@ describe('PercyClient', () => { await expectAsync(client.sendSnapshot(123, { name: 'test snapshot name' })).toBeResolved(); expect(api.requests['/snapshots/4567/finalize']).toBeDefined(); }); + + describe('direct bucket upload', () => { + const GCS_BASE = 'https://storage.googleapis.com'; + let gcsReply; + + function withSignedUrls(reqBody, includeFor = () => true) { + return { + data: { + id: '4567', + attributes: reqBody.attributes, + relationships: { + 'missing-resources': { + data: reqBody.data.relationships.resources.data.map(({ id }) => ( + includeFor(id) + ? { id, attributes: { 'signed-upload-url': `${GCS_BASE}/percy-resources/ns_gid_${id}?sig=fake` } } + : { id } + )) + } + } + } + }; + } + + beforeEach(async () => { + gcsReply = await mockRequests(GCS_BASE); + }); + + it('PUTs directly to GCS when the server returns signed-upload-url', async () => { + gcsReply.and.callFake(() => [200]); + api.reply('/builds/123/snapshots', ({ body }) => [201, withSignedUrls(body)]); + + let content = 'foo-content'; + await expectAsync( + client.sendSnapshot(123, { + name: 'direct-test', + resources: [{ + url: '/foo.css', + sha: sha256hash(content), + mimetype: 'text/css', + content, + md5: md5base64(content), + contentLength: Buffer.byteLength(content, 'utf-8'), + root: true + }] + }) + ).toBeResolved(); + + expect(gcsReply).toHaveBeenCalledTimes(1); + expect(gcsReply.calls.mostRecent().args[0].method).toBe('PUT'); + expect(gcsReply.calls.mostRecent().args[0].headers['Content-MD5']).toBe(md5base64(content)); + // legacy upload path NOT used when direct succeeds + expect(api.requests['/builds/123/resources']).toBeUndefined(); + expect(api.requests['/snapshots/4567/finalize']).toBeDefined(); + }); + + it('partitions per-resource: signed URL → direct, no URL → legacy', async () => { + gcsReply.and.callFake(() => [200]); + let directContent = 'direct-bytes'; + let legacyContent = 'legacy-bytes'; + let directSha = sha256hash(directContent); + let legacySha = sha256hash(legacyContent); + + api.reply('/builds/123/snapshots', ({ body }) => [201, withSignedUrls(body, id => id === directSha)]); + + await expectAsync( + client.sendSnapshot(123, { + name: 'mixed', + resources: [ + { url: '/d', sha: directSha, content: directContent, mimetype: 'text/plain', md5: md5base64(directContent), contentLength: Buffer.byteLength(directContent, 'utf-8') }, + { url: '/l', sha: legacySha, content: legacyContent, mimetype: 'text/plain', md5: md5base64(legacyContent), contentLength: Buffer.byteLength(legacyContent, 'utf-8') } + ] + }) + ).toBeResolved(); + + expect(gcsReply).toHaveBeenCalledTimes(1); + expect(api.requests['/builds/123/resources']).toBeDefined(); + expect(api.requests['/builds/123/resources'].length).toBe(1); + expect(api.requests['/builds/123/resources'][0].body.data.id).toBe(legacySha); + }); + + it('PERCY_DISABLE_DIRECT_UPLOAD forces every resource through legacy even when URLs are returned', async () => { + gcsReply.and.callFake(() => [200]); + api.reply('/builds/123/snapshots', ({ body }) => [201, withSignedUrls(body)]); + + let content = 'forced-legacy'; + process.env.PERCY_DISABLE_DIRECT_UPLOAD = '1'; + try { + await expectAsync( + client.sendSnapshot(123, { + name: 'forced-legacy', + resources: [{ + url: '/x', + sha: sha256hash(content), + content, + mimetype: 'text/plain', + md5: md5base64(content), + contentLength: Buffer.byteLength(content, 'utf-8') + }] + }) + ).toBeResolved(); + + expect(gcsReply).not.toHaveBeenCalled(); + expect(api.requests['/builds/123/resources'].length).toBe(1); + } finally { + delete process.env.PERCY_DISABLE_DIRECT_UPLOAD; + } + }); + + it('falls back to legacy when the direct PUT hits a transport error (e.g., 403)', async () => { + gcsReply.and.callFake(() => [403, 'expired']); + api.reply('/builds/123/snapshots', ({ body }) => [201, withSignedUrls(body)]); + + let content = 'falls-back'; + await expectAsync( + client.sendSnapshot(123, { + name: 'fallback', + resources: [{ + url: '/x', + sha: sha256hash(content), + content, + mimetype: 'text/plain', + md5: md5base64(content), + contentLength: Buffer.byteLength(content, 'utf-8'), + root: true + }] + }) + ).toBeResolved(); + + expect(gcsReply).toHaveBeenCalled(); + expect(api.requests['/builds/123/resources']).toBeDefined(); + expect(api.requests['/builds/123/resources'][0].body.data.id).toBe(sha256hash(content)); + expect(api.requests['/snapshots/4567/finalize']).toBeDefined(); + }); + + it('propagates correctness failures (400 BadDigest) and does not fall back to legacy', async () => { + gcsReply.and.callFake(() => [400, 'BadDigest']); + api.reply('/builds/123/snapshots', ({ body }) => [201, withSignedUrls(body)]); + + let content = 'bad-digest'; + await expectAsync( + client.sendSnapshot(123, { + name: 'fail-loud', + resources: [{ + url: '/x', + sha: sha256hash(content), + content, + mimetype: 'text/plain', + md5: md5base64(content), + contentLength: Buffer.byteLength(content, 'utf-8') + }] + }) + ).toBeRejected(); + + expect(api.requests['/builds/123/resources']).toBeUndefined(); + }); + + it('skips missing-resource entries whose SHA was not sent in the request', async () => { + api.reply('/builds/123/snapshots', () => [201, { + data: { + id: '4567', + relationships: { + 'missing-resources': { + data: [{ id: 'orphan-sha-not-in-request' }] + } + } + } + }]); + + let content = 'orphan'; + await expectAsync( + client.sendSnapshot(123, { + name: 'orphan', + resources: [{ + url: '/x', + sha: sha256hash(content), + content, + mimetype: 'text/plain', + md5: md5base64(content), + contentLength: Buffer.byteLength(content, 'utf-8') + }] + }) + ).toBeResolved(); + + expect(gcsReply).not.toHaveBeenCalled(); + expect(api.requests['/builds/123/resources']).toBeUndefined(); + }); + }); }); describe('#createComparison()', () => { diff --git a/packages/client/test/unit/request.test.js b/packages/client/test/unit/request.test.js index 05fe441e7..6cafe0d64 100644 --- a/packages/client/test/unit/request.test.js +++ b/packages/client/test/unit/request.test.js @@ -210,6 +210,51 @@ describe('Unit / Request', () => { .toBeRejectedWithError('403 \nSTOP'); }); + describe('rawBody', () => { + it('JSON-stringifies non-string bodies and forces application/json by default', async () => { + await server.request('/', { + method: 'POST', + body: { hello: 'world' } + }); + + const req = server.received[0]; + expect(req.headers['content-type']).toBe('application/json'); + expect(req.body).toBe('{"hello":"world"}'); + }); + + it('sends Buffer body verbatim with caller-supplied headers when rawBody is true', async () => { + const payload = Buffer.from('raw-bytes-here', 'utf-8'); + + await server.request('/', { + method: 'PUT', + rawBody: true, + body: payload, + headers: { + 'Content-Type': 'image/png', + 'Content-MD5': 'fakebase64md5==', + 'Content-Length': payload.length + } + }); + + const req = server.received[0]; + expect(req.headers['content-type']).toBe('image/png'); + expect(req.headers['content-md5']).toBe('fakebase64md5=='); + expect(req.headers['content-length']).toBe(String(payload.length)); + expect(Buffer.from(req.body).toString('utf-8')).toBe('raw-bytes-here'); + }); + + it('does not inject Content-Type when rawBody is true and caller omits one', async () => { + await server.request('/', { + method: 'PUT', + rawBody: true, + body: Buffer.from('x') + }); + + const req = server.received[0]; + expect(req.headers['content-type']).toBeUndefined(); + }); + }); + describe('retries', () => { it('automatically retries server 500 errors', async () => { let responses = [[502], [503], [520], [200]]; diff --git a/packages/client/test/unit/utils.test.js b/packages/client/test/unit/utils.test.js index ca34c2605..422a0813a 100644 --- a/packages/client/test/unit/utils.test.js +++ b/packages/client/test/unit/utils.test.js @@ -1,6 +1,28 @@ -import { normalizeBrowsers } from '@percy/client/utils'; +import { md5base64, normalizeBrowsers, sha256hash } from '@percy/client/utils'; +import crypto from 'crypto'; describe('utils', () => { + describe('md5base64', () => { + it('returns the base64-encoded MD5 of a string', () => { + // RFC 1864 test vector: empty string MD5 base64 + expect(md5base64('')).toEqual('1B2M2Y8AsgTpgAmY7PhCfg=='); + }); + + it('returns the base64-encoded MD5 of a buffer', () => { + const buf = Buffer.from('hello world', 'utf-8'); + const expected = crypto.createHash('md5').update(buf).digest('base64'); + expect(md5base64(buf)).toEqual(expected); + }); + + it('treats strings as UTF-8 (same encoding as sha256hash)', () => { + const utf8 = 'café — résumé'; + const expectedMd5 = crypto.createHash('md5').update(utf8, 'utf-8').digest('base64'); + const expectedSha = crypto.createHash('sha256').update(utf8, 'utf-8').digest('hex'); + expect(md5base64(utf8)).toEqual(expectedMd5); + expect(sha256hash(utf8)).toEqual(expectedSha); + }); + }); + describe('normalizeBrowsers', () => { describe('when browser values are in kebabcase', () => { it('returns snakecase values', () => { diff --git a/packages/core/src/discovery.js b/packages/core/src/discovery.js index f5e7c015d..f4edf8cfc 100644 --- a/packages/core/src/discovery.js +++ b/packages/core/src/discovery.js @@ -18,7 +18,8 @@ import { } from './utils.js'; import { ByteLRU, entrySize, DiskSpillStore, createSpillDir } from './cache/byte-lru.js'; import { - sha256hash + sha256hash, + md5base64 } from '@percy/client/utils'; import Pako from 'pako'; @@ -228,6 +229,8 @@ function processSnapshotResources({ domSnapshot, resources, ...snapshot }) { if (!alreadyZipped) { resources[index].content = Pako.gzip(resources[index].content); resources[index].sha = sha256hash(resources[index].content); + resources[index].md5 = md5base64(resources[index].content); + resources[index].contentLength = Buffer.byteLength(resources[index].content, 'utf-8'); } } } diff --git a/packages/core/src/utils.js b/packages/core/src/utils.js index dee1dea6f..a334f243f 100644 --- a/packages/core/src/utils.js +++ b/packages/core/src/utils.js @@ -1,5 +1,5 @@ import EventEmitter from 'events'; -import { sha256hash, request } from '@percy/client/utils'; +import { sha256hash, md5base64, request } from '@percy/client/utils'; import { camelcase, merge } from '@percy/config/utils'; import YAML from 'yaml'; import path from 'path'; @@ -301,9 +301,19 @@ function processSendEventData(input, cliVersion) { } // Creates a local resource object containing the resource URL, mimetype, content, sha, and any -// other additional resources attributes. +// other additional resources attributes. md5 + contentLength are declared on snapshot creation +// so percy-api can mint Content-MD5/Content-Length-bound signed URLs for direct bucket upload; +// they must stay in lockstep with the bytes in `content` (see discovery.js PERCY_GZIP path). export function createResource(url, content, mimetype, attrs) { - return { ...attrs, sha: sha256hash(content), mimetype, content, url }; + return { + ...attrs, + sha: sha256hash(content), + md5: md5base64(content), + contentLength: Buffer.byteLength(content, 'utf-8'), + mimetype, + content, + url + }; } // Creates a root resource object with an additional `root: true` property. The URL is normalized diff --git a/packages/core/test/snapshot.test.js b/packages/core/test/snapshot.test.js index b8dbebad2..50ac5a406 100644 --- a/packages/core/test/snapshot.test.js +++ b/packages/core/test/snapshot.test.js @@ -1,5 +1,5 @@ import { sha256hash, base64encode } from '@percy/client/utils'; -import { logger, api, setupTest, createTestServer, dedent } from './helpers/index.js'; +import { logger, api, mockRequests, setupTest, createTestServer, dedent } from './helpers/index.js'; import { waitFor } from '@percy/core/utils'; import Percy from '@percy/core'; import { handleSyncJob } from '../src/snapshot.js'; @@ -2069,6 +2069,139 @@ describe('Snapshot', () => { }); }); }); + + describe('direct bucket upload (end-to-end)', () => { + const GCS_BASE = 'https://storage.googleapis.com'; + let gcsReply; + + function withSignedUrls(reqBody) { + return [201, { + data: { + id: '4567', + attributes: reqBody.attributes, + relationships: { + 'missing-resources': { + data: reqBody.data.relationships.resources.data.map(({ id }) => ({ + id, + attributes: { + 'signed-upload-url': `${GCS_BASE}/percy-resources/ns_gid_${id}?sig=fake` + } + })) + } + } + } + }]; + } + + beforeEach(async () => { + gcsReply = await mockRequests(GCS_BASE); + }); + + it('PUTs DOM + asset bytes direct to GCS when the API hands back signed URLs, and finalizes the snapshot', async () => { + gcsReply.and.callFake(() => [200]); + api.reply('/builds/123/snapshots', ({ body }) => withSignedUrls(body)); + + await percy.snapshot({ + name: 'direct-upload-e2e', + url: 'http://localhost:8000', + domSnapshot: testDOM + }); + await percy.idle(); + + // capability header reached percy-api on snapshot creation + expect(api.requests['/builds/123/snapshots'][0].headers['X-Percy-Capabilities']) + .toBe('direct-upload-v1'); + + // every declared resource carried content-md5 + content-length + const declared = api.requests['/builds/123/snapshots'][0].body.data.relationships.resources.data; + for (const entry of declared) { + expect(entry.attributes['content-md5']).toEqual(jasmine.any(String)); + expect(entry.attributes['content-length']).toEqual(jasmine.any(Number)); + } + + // every missing resource PUT to GCS (one call per resource) + expect(gcsReply).toHaveBeenCalledTimes(declared.length); + + // each PUT carried Content-MD5 and Content-Length + const puts = gcsReply.calls.allArgs().map(([req]) => req); + for (const put of puts) { + expect(put.method).toBe('PUT'); + expect(put.headers['Content-MD5']).toEqual(jasmine.any(String)); + expect(put.headers['Content-Length']).toEqual(jasmine.any(Number)); + } + + // when direct succeeds, legacy POST /resources path is NOT used + expect(api.requests['/builds/123/resources']).toBeUndefined(); + + // snapshot still finalizes + expect(api.requests['/snapshots/4567/finalize']).toBeDefined(); + }); + + it('falls back to legacy POST /resources for any resource that fails the direct PUT (transport-class)', async () => { + api.reply('/builds/123/snapshots', ({ body }) => withSignedUrls(body)); + gcsReply.and.callFake(() => [403, 'expired']); + + await percy.snapshot({ + name: 'fallback-e2e', + url: 'http://localhost:8000', + domSnapshot: testDOM + }); + await percy.idle(); + + expect(gcsReply).toHaveBeenCalled(); + + const declared = api.requests['/builds/123/snapshots'][0].body.data.relationships.resources.data; + expect(api.requests['/builds/123/resources']).toBeDefined(); + expect(api.requests['/builds/123/resources'].length).toBe(declared.length); + + expect(api.requests['/snapshots/4567/finalize']).toBeDefined(); + }); + + it('PERCY_DISABLE_DIRECT_UPLOAD forces every resource through legacy end-to-end', async () => { + process.env.PERCY_DISABLE_DIRECT_UPLOAD = '1'; + api.reply('/builds/123/snapshots', ({ body }) => withSignedUrls(body)); + + try { + await percy.snapshot({ + name: 'forced-legacy-e2e', + url: 'http://localhost:8000', + domSnapshot: testDOM + }); + await percy.idle(); + + expect(api.requests['/builds/123/snapshots'][0].headers['X-Percy-Capabilities']).toBeUndefined(); + expect(gcsReply).not.toHaveBeenCalled(); + + const declared = api.requests['/builds/123/snapshots'][0].body.data.relationships.resources.data; + expect(api.requests['/builds/123/resources'].length).toBe(declared.length); + } finally { + delete process.env.PERCY_DISABLE_DIRECT_UPLOAD; + } + }); + + it('Content-MD5 sent to GCS matches the declared md5 from snapshot creation (integrity end-to-end)', async () => { + gcsReply.and.callFake(() => [200]); + api.reply('/builds/123/snapshots', ({ body }) => withSignedUrls(body)); + + await percy.snapshot({ + name: 'integrity-e2e', + url: 'http://localhost:8000', + domSnapshot: testDOM + }); + await percy.idle(); + + // declared md5 keyed by sha (which is the resource id and also embedded + // in the signed-upload URL path) + const declared = api.requests['/builds/123/snapshots'][0].body.data.relationships.resources.data; + const md5BySha = Object.fromEntries(declared.map(e => [e.id, e.attributes['content-md5']])); + + for (const [req] of gcsReply.calls.allArgs()) { + const m = req.path.match(/_(?[a-f0-9]{64})/); + expect(m).withContext(`PUT path missing sha: ${req.path}`).not.toBeNull(); + expect(req.headers['Content-MD5']).toBe(md5BySha[m.groups.sha]); + } + }); + }); }); // ── runDoctorOnFailure ──────────────────────────────────────────────────────── diff --git a/packages/core/test/utils.test.js b/packages/core/test/utils.test.js index 09374b6e2..0ca19771e 100644 --- a/packages/core/test/utils.test.js +++ b/packages/core/test/utils.test.js @@ -1,4 +1,5 @@ -import { decodeAndEncodeURLWithLogging, waitForSelectorInsideBrowser, compareObjectTypes, isGzipped, checkSDKVersion, percyAutomateRequestHandler, detectFontMimeType, handleIncorrectFontMimeType, computeResponsiveWidths, appendUrlSearchParam, processCorsIframesInDomSnapshot, processCorsIframes } from '../src/utils.js'; +import { createLogResource, createPercyCSSResource, createResource, createRootResource, decodeAndEncodeURLWithLogging, waitForSelectorInsideBrowser, compareObjectTypes, isGzipped, checkSDKVersion, percyAutomateRequestHandler, detectFontMimeType, handleIncorrectFontMimeType, computeResponsiveWidths, appendUrlSearchParam, processCorsIframesInDomSnapshot, processCorsIframes } from '../src/utils.js'; +import { md5base64, sha256hash } from '@percy/client/utils'; import { logger, setupTest, mockRequests } from './helpers/index.js'; import percyLogger from '@percy/logger'; import Percy from '@percy/core'; @@ -14,6 +15,59 @@ describe('utils', () => { await logger.mock({ level: 'debug' }); }); + describe('createResource', () => { + it('returns sha, md5, contentLength, mimetype, content, and url', () => { + const content = 'p { color: purple; }'; + const r = createResource('https://example.com/a.css', content, 'text/css'); + expect(r).toEqual({ + url: 'https://example.com/a.css', + mimetype: 'text/css', + content, + sha: sha256hash(content), + md5: md5base64(content), + contentLength: Buffer.byteLength(content, 'utf-8') + }); + }); + + it('preserves extra attrs and computes byte length over UTF-8', () => { + const content = 'café — résumé'; + const r = createResource('https://example.com/x', content, 'text/plain', { root: true }); + expect(r.root).toBeTrue(); + expect(r.contentLength).toEqual(Buffer.byteLength(content, 'utf-8')); + expect(r.contentLength).not.toEqual(content.length); // sanity: multi-byte chars + }); + + it('handles Buffer content', () => { + const buf = Buffer.from([0x00, 0x01, 0x02, 0xff]); + const r = createResource('https://example.com/blob', buf, 'application/octet-stream'); + expect(r.contentLength).toEqual(buf.length); + expect(r.md5).toEqual(md5base64(buf)); + expect(r.sha).toEqual(sha256hash(buf)); + }); + + it('propagates new fields through createRootResource', () => { + const html = 'hi'; + const r = createRootResource('https://example.com/', html); + expect(r.root).toBeTrue(); + expect(r.md5).toEqual(md5base64(html)); + expect(r.contentLength).toEqual(Buffer.byteLength(html, 'utf-8')); + }); + + it('propagates new fields through createPercyCSSResource', () => { + const css = 'body { color: red; }'; + const r = createPercyCSSResource('https://example.com/page', css); + expect(r.md5).toEqual(md5base64(css)); + expect(r.contentLength).toEqual(Buffer.byteLength(css, 'utf-8')); + }); + + it('propagates new fields through createLogResource', () => { + const r = createLogResource([{ level: 'info', message: 'hi' }]); + expect(r.log).toBeTrue(); + expect(r.md5).toEqual(md5base64(r.content)); + expect(r.contentLength).toEqual(Buffer.byteLength(r.content, 'utf-8')); + }); + }); + describe('percyAutomateRequestHandler', () => { it('maps client/environment info, camelCases options, merges config, concatenates percyCSS and attaches buildInfo', () => { const req = {