diff --git a/.github/workflows/extraction.yml b/.github/workflows/extraction.yml new file mode 100644 index 00000000..29a55217 --- /dev/null +++ b/.github/workflows/extraction.yml @@ -0,0 +1,25 @@ +name: Extraction Service +on: + pull_request: + paths: ['packages/extraction/**', '.github/workflows/extraction.yml'] + push: + branches: [main] + paths: ['packages/extraction/**', '.github/workflows/extraction.yml'] +permissions: + contents: read +jobs: + extraction: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 + - uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 + with: + node-version: 24.16.0 + cache: npm + cache-dependency-path: packages/extraction/package-lock.json + - run: npm ci --ignore-scripts + working-directory: packages/extraction + - run: npm run format:check && npm run typecheck && npm test + working-directory: packages/extraction + - run: npm run test:container + working-directory: packages/extraction diff --git a/packages/extraction/.gitattributes b/packages/extraction/.gitattributes new file mode 100644 index 00000000..67da8786 --- /dev/null +++ b/packages/extraction/.gitattributes @@ -0,0 +1,2 @@ +tests/fixtures/*.pdf binary +tests/fixtures/*.docx binary diff --git a/packages/extraction/.gitignore b/packages/extraction/.gitignore new file mode 100644 index 00000000..2e3a563c --- /dev/null +++ b/packages/extraction/.gitignore @@ -0,0 +1 @@ +native/guard diff --git a/packages/extraction/Dockerfile b/packages/extraction/Dockerfile new file mode 100644 index 00000000..4acf5d9c --- /dev/null +++ b/packages/extraction/Dockerfile @@ -0,0 +1,32 @@ +FROM node:24.16.0-bookworm-slim AS build +WORKDIR /build +COPY packages/extraction/package*.json ./ +RUN npm ci --ignore-scripts --no-audit --no-fund +COPY packages/extraction/tsconfig.json ./ +COPY packages/extraction/src ./src +RUN npm run build +RUN apt-get update && apt-get install -y --no-install-recommends gcc libc6-dev linux-libc-dev && rm -rf /var/lib/apt/lists/* +COPY packages/extraction/native/guard.c ./guard.c +RUN gcc -O2 -Wall -Wextra -Werror -o guard guard.c +COPY packages/extraction/tests/block-landlock.c ./block-landlock.c +RUN gcc -O2 -Wall -Wextra -Werror -o block-landlock block-landlock.c + +FROM python:3.13-slim-bookworm@sha256:5024f48ba9441d4b13a95d3945abc6365538e3a31109833367a1923523c6efed AS runtime +RUN apt-get update && apt-get install -y --no-install-recommends libstdc++6 libatomic1 && rm -rf /var/lib/apt/lists/* +COPY packages/extraction/worker/requirements.txt /opt/extraction/requirements.txt +RUN pip install --no-cache-dir -r /opt/extraction/requirements.txt +COPY --from=build /usr/local/bin/node /usr/local/bin/node +COPY --from=build --chmod=0555 /build/guard /opt/extraction/guard +COPY --from=build --chmod=0555 /build/dist /opt/extraction/dist +COPY --chmod=0444 packages/extraction/package.json /opt/extraction/package.json +COPY --chmod=0444 packages/extraction/worker/extract.py /opt/extraction/extract.py +RUN groupadd -g 10001 extraction && useradd -u 10001 -g extraction -M extraction && \ + mkdir /socket /jobs && chown extraction:extraction /socket /jobs && chmod 0700 /jobs && chmod 0770 /socket +USER 10001:10001 +WORKDIR /opt/extraction +HEALTHCHECK --interval=10s --timeout=2s --start-period=10s --retries=3 CMD ["node", "dist/health.js"] +CMD ["node", "dist/main.js"] + +FROM runtime AS qualification +COPY --chown=10001:10001 packages/extraction/tests /tests +COPY --from=build --chmod=0555 /build/block-landlock /tests/block-landlock diff --git a/packages/extraction/compose.yaml b/packages/extraction/compose.yaml new file mode 100644 index 00000000..f6bbf9f4 --- /dev/null +++ b/packages/extraction/compose.yaml @@ -0,0 +1,22 @@ +services: + extraction: + image: ${EXTRACTION_IMAGE:-librechat-extraction:local} + build: + context: ../.. + dockerfile: packages/extraction/Dockerfile + target: runtime + init: true + user: '10001:10001' + read_only: true + cap_drop: [ALL] + security_opt: [no-new-privileges:true] + network_mode: none + mem_limit: 768m + cpus: 2 + pids_limit: 32 + tmpfs: + - /jobs:rw,noexec,nosuid,nodev,size=96m,nr_inodes=1024,uid=10001,gid=10001,mode=0700 + volumes: + - extraction-socket:/socket +volumes: + extraction-socket: diff --git a/packages/extraction/native/guard.c b/packages/extraction/native/guard.c new file mode 100644 index 00000000..ee1d2cbc --- /dev/null +++ b/packages/extraction/native/guard.c @@ -0,0 +1,156 @@ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* Stable UAPI values missing from Debian 12's older build headers. */ +#ifndef LANDLOCK_ACCESS_FS_TRUNCATE +#define LANDLOCK_ACCESS_FS_TRUNCATE (1ULL << 14) +#endif +#ifndef __NR_fchmodat2 +#define __NR_fchmodat2 452 +#endif + +#if defined(__x86_64__) +#define ARCH AUDIT_ARCH_X86_64 +#define LOADER "/lib64/ld-linux-x86-64.so.2" +#elif defined(__aarch64__) +#define ARCH AUDIT_ARCH_AARCH64 +#define LOADER "/lib/ld-linux-aarch64.so.1" +#else +#error Unsupported architecture +#endif + +static void fail_at(int line) { fprintf(stderr, "ISOLATION_UNAVAILABLE:%d:%d\n", line, errno); exit(125); } +#define fail() fail_at(__LINE__) +static void limit(int resource, rlim_t value) { + struct rlimit r = {value, value}; + if (setrlimit(resource, &r)) fail(); +} +static void allow(int rules, const char *path, __u64 access) { + int fd = open(path, O_PATH | O_CLOEXEC); + if (fd < 0) fail(); + struct landlock_path_beneath_attr rule = {.allowed_access = access, .parent_fd = fd}; + if (syscall(SYS_landlock_add_rule, rules, LANDLOCK_RULE_PATH_BENEATH, &rule, 0)) fail(); + close(fd); +} +#define DENY(nr) BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, nr, 0, 1), BPF_STMT(BPF_RET | BPF_K, SECCOMP_RET_ERRNO | EPERM) +static void restrict_syscalls(void) { + struct sock_filter filter[] = { + BPF_STMT(BPF_LD | BPF_W | BPF_ABS, offsetof(struct seccomp_data, arch)), + BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, ARCH, 1, 0), + BPF_STMT(BPF_RET | BPF_K, SECCOMP_RET_KILL_PROCESS), + BPF_STMT(BPF_LD | BPF_W | BPF_ABS, offsetof(struct seccomp_data, nr)), +#ifdef __x86_64__ + BPF_JUMP(BPF_JMP | BPF_JGE | BPF_K, 0x40000000, 0, 1), + BPF_STMT(BPF_RET | BPF_K, SECCOMP_RET_KILL_PROCESS), +#endif + DENY(__NR_socket), DENY(__NR_socketpair), DENY(__NR_connect), + DENY(__NR_bind), DENY(__NR_listen), DENY(__NR_accept), DENY(__NR_accept4), + DENY(__NR_clone), DENY(__NR_clone3), +#ifdef __NR_fork + DENY(__NR_fork), DENY(__NR_vfork), +#endif + DENY(__NR_ptrace), DENY(__NR_process_vm_readv), DENY(__NR_process_vm_writev), + DENY(__NR_kill), DENY(__NR_tkill), DENY(__NR_tgkill), DENY(__NR_pidfd_send_signal), + DENY(__NR_rt_sigqueueinfo), DENY(__NR_rt_tgsigqueueinfo), + /* CPython marks opened files close-on-exec through FIOCLEX. */ + BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, __NR_ioctl, 0, 3), + BPF_STMT(BPF_LD | BPF_W | BPF_ABS, offsetof(struct seccomp_data, args[1])), + BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, FIOCLEX, 1, 0), + BPF_STMT(BPF_RET | BPF_K, SECCOMP_RET_ERRNO | EPERM), + BPF_STMT(BPF_LD | BPF_W | BPF_ABS, offsetof(struct seccomp_data, nr)), + /* Async descriptor ownership can cause kernel-delivered signals. */ + BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, __NR_fcntl, 0, 7), + BPF_STMT(BPF_LD | BPF_W | BPF_ABS, offsetof(struct seccomp_data, args[1])), + DENY(F_SETOWN), DENY(F_SETOWN_EX), DENY(F_SETSIG), + BPF_STMT(BPF_LD | BPF_W | BPF_ABS, offsetof(struct seccomp_data, nr)), + DENY(__NR_mq_open), DENY(__NR_mq_unlink), DENY(__NR_mq_timedsend), + DENY(__NR_mq_timedreceive), DENY(__NR_mq_notify), DENY(__NR_mq_getsetattr), + DENY(__NR_mount), DENY(__NR_umount2), DENY(__NR_pivot_root), DENY(__NR_chroot), + DENY(__NR_unshare), DENY(__NR_setns), DENY(__NR_bpf), DENY(__NR_userfaultfd), + DENY(__NR_io_uring_setup), DENY(__NR_perf_event_open), + DENY(__NR_setpriority), DENY(__NR_sched_setparam), DENY(__NR_sched_setscheduler), + DENY(__NR_sched_setaffinity), DENY(__NR_sched_setattr), + DENY(__NR_pidfd_open), DENY(__NR_pidfd_getfd), + DENY(__NR_keyctl), DENY(__NR_add_key), DENY(__NR_request_key), + DENY(__NR_shmget), DENY(__NR_shmat), DENY(__NR_shmctl), + DENY(__NR_semget), DENY(__NR_semop), DENY(__NR_semctl), + DENY(__NR_msgget), DENY(__NR_msgsnd), DENY(__NR_msgrcv), DENY(__NR_msgctl), + BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, __NR_prlimit64, 0, 3), + BPF_STMT(BPF_LD | BPF_W | BPF_ABS, offsetof(struct seccomp_data, args[0])), + BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, 0, 1, 0), + BPF_STMT(BPF_RET | BPF_K, SECCOMP_RET_ERRNO | EPERM), + BPF_STMT(BPF_LD | BPF_W | BPF_ABS, offsetof(struct seccomp_data, nr)), + DENY(__NR_utimensat), +#ifdef __NR_utime + DENY(__NR_utime), DENY(__NR_utimes), DENY(__NR_futimesat), +#endif +#ifdef __NR_fchmodat2 + DENY(__NR_fchmodat2), +#endif + DENY(__NR_open_by_handle_at), DENY(__NR_name_to_handle_at), + DENY(__NR_fchmod), DENY(__NR_fchmodat), DENY(__NR_fchown), DENY(__NR_fchownat), +#ifdef __NR_chmod + DENY(__NR_chmod), DENY(__NR_chown), DENY(__NR_lchown), +#endif + DENY(__NR_mknodat), DENY(__NR_setuid), DENY(__NR_setgid), + DENY(__NR_setreuid), DENY(__NR_setregid), DENY(__NR_setresuid), DENY(__NR_setresgid), + DENY(__NR_setgroups), DENY(__NR_setxattr), DENY(__NR_lsetxattr), DENY(__NR_fsetxattr), + DENY(__NR_removexattr), DENY(__NR_lremovexattr), DENY(__NR_fremovexattr), + BPF_STMT(BPF_RET | BPF_K, SECCOMP_RET_ALLOW) + }; + struct sock_fprog program = {.len = sizeof(filter) / sizeof(filter[0]), .filter = filter}; + if (prctl(PR_SET_SECCOMP, SECCOMP_MODE_FILTER, &program)) fail(); +} +int main(int argc, char **argv) { + if (argc != 5 || geteuid() == 0) fail(); + /* Paths are supervisor-owned, never supplied in the HTTP protocol. */ + if (chdir(argv[1])) fail(); + pid_t parent = getppid(); + if (parent == 1 || prctl(PR_SET_PDEATHSIG, SIGKILL) || getppid() != parent) fail(); + limit(RLIMIT_AS, 512ULL * 1024 * 1024); + limit(RLIMIT_CPU, 8); + limit(RLIMIT_FSIZE, 4ULL * 1024 * 1024); + limit(RLIMIT_NOFILE, 64); + limit(RLIMIT_CORE, 0); + if (prctl(PR_SET_DUMPABLE, 0) || prctl(PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0)) fail(); + if (syscall(SYS_close_range, 3, ~0U, 0)) fail(); + int abi = syscall(SYS_landlock_create_ruleset, NULL, 0, LANDLOCK_CREATE_RULESET_VERSION); + if (abi < 3) fail(); + __u64 read = LANDLOCK_ACCESS_FS_READ_FILE | LANDLOCK_ACCESS_FS_READ_DIR; + __u64 write = LANDLOCK_ACCESS_FS_WRITE_FILE | LANDLOCK_ACCESS_FS_TRUNCATE | LANDLOCK_ACCESS_FS_MAKE_REG; + struct landlock_ruleset_attr attr = { + .handled_access_fs = (1ULL << 15) - 1 /* ABI 3, including REFER and TRUNCATE */ + }; + int rules = syscall(SYS_landlock_create_ruleset, &attr, sizeof(attr), 0); + if (rules < 0) fail(); + allow(rules, "/usr", read); + allow(rules, "/lib", read); + if (access("/lib64", F_OK) == 0) allow(rules, "/lib64", read); + allow(rules, argv[2], LANDLOCK_ACCESS_FS_READ_FILE | LANDLOCK_ACCESS_FS_EXECUTE); + allow(rules, LOADER, LANDLOCK_ACCESS_FS_READ_FILE | LANDLOCK_ACCESS_FS_EXECUTE); + allow(rules, argv[3], LANDLOCK_ACCESS_FS_READ_FILE); + allow(rules, argv[1], read | write); + allow(rules, "/dev/null", LANDLOCK_ACCESS_FS_READ_FILE | LANDLOCK_ACCESS_FS_WRITE_FILE); + allow(rules, "/dev/urandom", LANDLOCK_ACCESS_FS_READ_FILE); + if (syscall(SYS_landlock_restrict_self, rules, 0)) fail(); + close(rules); + restrict_syscalls(); + if (clearenv() || setenv("LANG", "C.UTF-8", 1)) fail(); + execl(argv[2], argv[2], "-I", "-B", argv[3], argv[4], (char *)NULL); + fail(); +} diff --git a/packages/extraction/package-lock.json b/packages/extraction/package-lock.json new file mode 100644 index 00000000..86e21c9f --- /dev/null +++ b/packages/extraction/package-lock.json @@ -0,0 +1,64 @@ +{ + "name": "@librechat/extraction-service", + "version": "0.1.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "@librechat/extraction-service", + "version": "0.1.0", + "devDependencies": { + "@types/node": "24.10.1", + "prettier": "2.4.1", + "typescript": "5.9.3" + }, + "engines": { + "node": ">=24 <25" + } + }, + "node_modules/@types/node": { + "version": "24.10.1", + "resolved": "https://registry.npmjs.org/@types/node/-/node-24.10.1.tgz", + "integrity": "sha512-GNWcUTRBgIRJD5zj+Tq0fKOJ5XZajIiBroOF0yvj2bSU1WvNdYS/dn9UxwsujGW4JX06dnHyjV2y9rRaybH0iQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": "~7.16.0" + } + }, + "node_modules/prettier": { + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/prettier/-/prettier-2.4.1.tgz", + "integrity": "sha512-9fbDAXSBcc6Bs1mZrDYb3XKzDLm4EXXL9sC1LqKP5rZkT6KRr/rf9amVUcODVXgguK/isJz0d0hP72WeaKWsvA==", + "dev": true, + "license": "MIT", + "bin": { + "prettier": "bin-prettier.js" + }, + "engines": { + "node": ">=10.13.0" + } + }, + "node_modules/typescript": { + "version": "5.9.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", + "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "node_modules/undici-types": { + "version": "7.16.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.16.0.tgz", + "integrity": "sha512-Zz+aZWSj8LE6zoxD+xrjh4VfkIG8Ya6LvYkZqtUQGJPZjYl53ypCaUwWqo7eI0x66KBGeRo+mlBEkMSeSZ38Nw==", + "dev": true, + "license": "MIT" + } + } +} diff --git a/packages/extraction/package.json b/packages/extraction/package.json new file mode 100644 index 00000000..5934c126 --- /dev/null +++ b/packages/extraction/package.json @@ -0,0 +1,22 @@ +{ + "name": "@librechat/extraction-service", + "version": "0.1.0", + "private": true, + "type": "module", + "engines": { + "node": ">=24 <25" + }, + "scripts": { + "build": "tsc", + "typecheck": "tsc --noEmit", + "test": "npm run build && node --test dist/*.test.js", + "start": "node dist/main.js", + "test:container": "bash tests/container.sh", + "format:check": "prettier --check src tests/*.mjs *.json compose.yaml ../../.github/workflows/extraction.yml" + }, + "devDependencies": { + "@types/node": "24.10.1", + "prettier": "2.4.1", + "typescript": "5.9.3" + } +} diff --git a/packages/extraction/src/backend.ts b/packages/extraction/src/backend.ts new file mode 100644 index 00000000..a3831c65 --- /dev/null +++ b/packages/extraction/src/backend.ts @@ -0,0 +1,166 @@ +import { spawn } from 'node:child_process'; +import { constants } from 'node:fs'; +import { mkdtemp, open, rm, writeFile } from 'node:fs/promises'; +import { join } from 'node:path'; +import { ExtractionError, policy, validateResult } from './protocol.js'; +import type { Format } from './protocol.js'; + +export interface Backend { + healthy: boolean; + create(): Promise; + run( + job: string, + format: Format, + digest: string, + signal: AbortSignal + ): Promise; + cleanup(job: string): Promise; +} +export interface Runtime { + jobs: string; + guard: string; + python: string; + worker: string; +} +const parserErrors = new Set([ + 'INVALID_DOCUMENT', + 'EMPTY_OUTPUT', + 'OUTPUT_LIMIT', + 'DECOMPRESSION_LIMIT', + 'STRUCTURE_LIMIT', + 'ENCRYPTED_DOCUMENT', + 'UNSUPPORTED_ENCODING', + 'UNSUPPORTED_CONTENT', + 'RESOURCE_LIMIT', +]); +export class RestrictedBackend implements Backend { + healthy = true; + constructor(private readonly runtime: Runtime) {} + async create(): Promise { + if (!this.healthy) throw new ExtractionError('UNAVAILABLE'); + return mkdtemp(join(this.runtime.jobs, 'job-')); + } + private async execute( + job: string, + mode: string, + signal: AbortSignal + ): Promise { + signal.throwIfAborted(); + const child = spawn( + this.runtime.guard, + [job, this.runtime.python, this.runtime.worker, mode], + { + detached: true, + stdio: ['ignore', 'pipe', 'pipe'], + env: {}, + } + ); + let output = ''; + let count = 0; + let overflow = false; + const kill = () => { + if ( + child.pid && + child.exitCode === null && + child.signalCode === null + ) { + try { + process.kill(-child.pid, 'SIGKILL'); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'ESRCH') + this.healthy = false; + } + } + }; + signal.addEventListener('abort', kill, { once: true }); + if (signal.aborted) kill(); + let spawnError = false; + child.on('error', () => { + spawnError = true; + }); + child.stdout.on('data', (chunk: Buffer) => { + count += chunk.length; + if (count > 1024) { + overflow = true; + kill(); + } else output += chunk.toString('ascii'); + }); + child.stderr.on('data', (chunk: Buffer) => { + count += chunk.length; + if (count > 2048) { + overflow = true; + kill(); + } + }); + const code = await new Promise(resolve => + child.once('close', resolve) + ); + signal.removeEventListener('abort', kill); + signal.throwIfAborted(); + if (spawnError || overflow) throw new ExtractionError('UNAVAILABLE'); + if (code !== 0) { + const value = output.trim(); + if (parserErrors.has(value)) + throw new ExtractionError(value as 'INVALID_DOCUMENT'); + throw new ExtractionError( + code === null ? 'RESOURCE_LIMIT' : 'UNAVAILABLE' + ); + } + return output; + } + async probe(): Promise { + const job = await this.create(); + try { + const output = await this.execute( + job, + 'probe', + AbortSignal.timeout(policy.deadlineMs) + ); + if (output.trim() !== 'READY') + throw new ExtractionError('UNAVAILABLE'); + } finally { + await this.cleanup(job); + } + } + async run( + job: string, + format: Format, + digest: string, + signal: AbortSignal + ): Promise { + await this.execute(job, format, signal); + const file = await open( + join(job, 'result'), + constants.O_RDONLY | constants.O_NOFOLLOW + ); + try { + const stat = await file.stat(); + if ( + !stat.isFile() || + stat.nlink !== 1 || + stat.size > policy.resultBytes + ) { + throw new ExtractionError('OUTPUT_LIMIT'); + } + const bytes = await file.readFile(); + signal.throwIfAborted(); + validateResult(bytes, format, digest); + return bytes; + } finally { + await file.close(); + } + } + async cleanup(job: string): Promise { + try { + await rm(job, { recursive: true, force: true, maxRetries: 0 }); + } catch { + this.healthy = false; + throw new ExtractionError('UNAVAILABLE'); + } + } +} + +/** Used by focused worker tests, not by HTTP callers. */ +export async function writeInput(job: string, input: Buffer): Promise { + await writeFile(join(job, 'input'), input, { flag: 'wx', mode: 0o600 }); +} diff --git a/packages/extraction/src/boundary.test.ts b/packages/extraction/src/boundary.test.ts new file mode 100644 index 00000000..da54749e --- /dev/null +++ b/packages/extraction/src/boundary.test.ts @@ -0,0 +1,22 @@ +import assert from 'node:assert/strict'; +import { readFile } from 'node:fs/promises'; +import { test } from 'node:test'; + +test('supervisor has no parser or application stack dependencies', async () => { + const manifest = JSON.parse( + await readFile(new URL('../package.json', import.meta.url), 'utf8') + ); + assert.equal(manifest.dependencies, undefined); + for (const name of ['server.ts', 'backend.ts', 'protocol.ts', 'main.ts']) { + const source = await readFile( + new URL(`../src/${name}`, import.meta.url), + 'utf8' + ); + for (const match of source.matchAll(/from ['"]([^'"]+)['"]/g)) { + assert.ok( + match[1]?.startsWith('node:') || match[1]?.startsWith('./'), + match[1] + ); + } + } +}); diff --git a/packages/extraction/src/health.ts b/packages/extraction/src/health.ts new file mode 100644 index 00000000..641575c4 --- /dev/null +++ b/packages/extraction/src/health.ts @@ -0,0 +1,15 @@ +import { request } from 'node:http'; +const req = request( + { + socketPath: '/socket/extraction.sock', + path: '/v1/capabilities', + timeout: 1000, + }, + res => { + res.resume(); + res.once('end', () => process.exit(res.statusCode === 200 ? 0 : 1)); + } +); +req.on('timeout', () => req.destroy()); +req.on('error', () => process.exit(1)); +req.end(); diff --git a/packages/extraction/src/main.ts b/packages/extraction/src/main.ts new file mode 100644 index 00000000..424a6ed6 --- /dev/null +++ b/packages/extraction/src/main.ts @@ -0,0 +1,64 @@ +import { chmod, lstat, readdir, unlink } from 'node:fs/promises'; +import { connect } from 'node:net'; +import { RestrictedBackend } from './backend.js'; +import { extractionServer } from './server.js'; + +// Deployment-owned locations, not caller-selected paths or runtimes. +const backend = new RestrictedBackend({ + jobs: '/jobs', + guard: '/opt/extraction/guard', + python: '/usr/local/bin/python3.13', + worker: '/opt/extraction/extract.py', +}); +try { + if ((await readdir('/jobs')).length) throw new Error('DIRTY_SCRATCH'); + try { + const stat = await lstat('/socket/extraction.sock'); + if (!stat.isSocket()) throw new Error('INVALID_SOCKET'); + await new Promise((resolve, reject) => { + const client = connect('/socket/extraction.sock'); + client.setTimeout(1000, () => { + client.destroy(); + reject(new Error('SOCKET_EXISTS')); + }); + client.once('connect', () => { + client.destroy(); + reject(new Error('SOCKET_EXISTS')); + }); + client.once('error', error => { + if ((error as NodeJS.ErrnoException).code === 'ECONNREFUSED') + resolve(); + else reject(error); + }); + }); + await unlink('/socket/extraction.sock'); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error; + } + await backend.probe(); + const server = extractionServer(backend); + server.on('error', () => { + console.error('EXTRACTION_UNAVAILABLE'); + process.exit(1); + }); + process.umask(0o007); + server.listen('/socket/extraction.sock', async () => { + try { + await chmod('/socket/extraction.sock', 0o660); + } catch { + console.error('EXTRACTION_UNAVAILABLE'); + process.exit(1); + } + console.log('Extraction service ready'); + }); + const shutdown = () => { + server.close(); + server.closeAllConnections(); + setTimeout(() => process.exit(0), 12_000).unref(); + }; + process.once('SIGTERM', shutdown); + process.once('SIGINT', shutdown); +} catch { + console.error('EXTRACTION_UNAVAILABLE'); + process.exit(1); +} diff --git a/packages/extraction/src/protocol.test.ts b/packages/extraction/src/protocol.test.ts new file mode 100644 index 00000000..6688fa49 --- /dev/null +++ b/packages/extraction/src/protocol.test.ts @@ -0,0 +1,68 @@ +import assert from 'node:assert/strict'; +import { test } from 'node:test'; +import { policy, validateResult } from './protocol.js'; +const digest = 'a'.repeat(64); +const value = (text = 'hello') => ({ + version: 1, + operation: 'document.extract-text', + format: 'pdf', + inputSha256: digest, + textBytes: Buffer.byteLength(text), + segments: [{ kind: 'page', index: 1, text }], +}); +const encode = (result: unknown) => Buffer.from(JSON.stringify(result)); +test('preserves ordered page segments and validates digest and byte count', () => { + const result = value(); + assert.deepEqual(validateResult(encode(result), 'pdf', digest), result); + for (const changed of [ + { ...result, inputSha256: 'b'.repeat(64) }, + { ...result, textBytes: 4 }, + { ...result, extra: true }, + { ...result, format: 'docx' }, + { ...result, segments: [{ kind: 'page', index: 0, text: 'hello' }] }, + { ...result, segments: [{ kind: 'sheet', index: 1, text: 'hello' }] }, + ]) + assert.throws(() => validateResult(encode(changed), 'pdf', digest)); +}); +test('rejects malformed UTF-8, lone surrogates, emptiness and invalid JSON', () => { + for (const bytes of [ + Buffer.from([0xff]), + Buffer.from('{'), + encode(value('\ud800')), + encode(value(' ')), + encode(null), + ]) { + assert.throws(() => validateResult(bytes, 'pdf', digest)); + } +}); +test('accepts exact text limit and rejects one byte more or oversized wire result', () => { + assert.equal( + validateResult( + encode(value('x'.repeat(policy.textBytes))), + 'pdf', + digest + ).textBytes, + policy.textBytes + ); + assert.throws(() => + validateResult( + encode(value('x'.repeat(policy.textBytes + 1))), + 'pdf', + digest + ) + ); + assert.throws(() => + validateResult(Buffer.alloc(policy.resultBytes + 1), 'pdf', digest) + ); +}); +test('rejects excess pages and does not accept PDF pages as DOCX structure', () => { + const result = value(); + result.segments = Array.from( + { length: policy.maxSegments + 1 }, + (_, index) => ({ kind: 'page', index: index + 1, text: 'hello' }) + ); + assert.throws(() => validateResult(encode(result), 'pdf', digest)); + assert.throws(() => + validateResult(encode({ ...value(), format: 'docx' }), 'docx', digest) + ); +}); diff --git a/packages/extraction/src/protocol.ts b/packages/extraction/src/protocol.ts new file mode 100644 index 00000000..935c902c --- /dev/null +++ b/packages/extraction/src/protocol.ts @@ -0,0 +1,107 @@ +import { createHash } from 'node:crypto'; + +export const policy = Object.freeze({ + inputBytes: 10 * 1024 * 1024, + textBytes: 1024 * 1024, + resultBytes: 3 * 1024 * 1024, + deadlineMs: 10_000, + maxSegments: 128, +}); +export type Format = 'pdf' | 'docx'; +export type ErrorCode = + | 'INVALID_REQUEST' + | 'INPUT_LIMIT' + | 'BUSY' + | 'DEADLINE' + | 'CANCELLED' + | 'UNAVAILABLE' + | 'INVALID_DOCUMENT' + | 'EMPTY_OUTPUT' + | 'OUTPUT_LIMIT' + | 'DECOMPRESSION_LIMIT' + | 'STRUCTURE_LIMIT' + | 'ENCRYPTED_DOCUMENT' + | 'UNSUPPORTED_ENCODING' + | 'UNSUPPORTED_CONTENT' + | 'RESOURCE_LIMIT'; +export class ExtractionError extends Error { + constructor(readonly code: ErrorCode) { + super(code); + } +} +export interface Segment { + kind: 'page' | 'document'; + index: number; + text: string; +} +export interface Result { + version: 1; + operation: 'document.extract-text'; + format: Format; + inputSha256: string; + textBytes: number; + segments: Segment[]; +} +const keys = (value: object, expected: string[]) => + Object.keys(value).length === expected.length && + expected.every(key => Object.hasOwn(value, key)); +export function validateResult( + bytes: Buffer, + format: Format, + digest: string +): Result { + if (bytes.length > policy.resultBytes) + throw new ExtractionError('OUTPUT_LIMIT'); + let result: Result; + try { + result = JSON.parse( + new TextDecoder('utf-8', { fatal: true }).decode(bytes) + ) as Result; + } catch { + throw new ExtractionError('INVALID_DOCUMENT'); + } + if ( + !result || + typeof result !== 'object' || + !keys(result, [ + 'version', + 'operation', + 'format', + 'inputSha256', + 'textBytes', + 'segments', + ]) || + result.version !== 1 || + result.operation !== 'document.extract-text' || + result.format !== format || + result.inputSha256 !== digest || + !Array.isArray(result.segments) || + !result.segments.length || + result.segments.length > (format === 'pdf' ? policy.maxSegments : 1) + ) { + throw new ExtractionError('INVALID_DOCUMENT'); + } + let size = 0; + let nonempty = false; + for (const [index, segment] of result.segments.entries()) { + if ( + !segment || + typeof segment !== 'object' || + !keys(segment, ['kind', 'index', 'text']) || + segment.kind !== (format === 'pdf' ? 'page' : 'document') || + segment.index !== index + 1 || + typeof segment.text !== 'string' || + !segment.text.isWellFormed() + ) + throw new ExtractionError('INVALID_DOCUMENT'); + size += Buffer.byteLength(segment.text); + nonempty ||= Boolean(segment.text.trim()); + } + if (size > policy.textBytes) throw new ExtractionError('OUTPUT_LIMIT'); + if (size !== result.textBytes || !nonempty) + throw new ExtractionError('INVALID_DOCUMENT'); + return result; +} +export function sha256(bytes: Buffer): string { + return createHash('sha256').update(bytes).digest('hex'); +} diff --git a/packages/extraction/src/server.test.ts b/packages/extraction/src/server.test.ts new file mode 100644 index 00000000..60f9bc63 --- /dev/null +++ b/packages/extraction/src/server.test.ts @@ -0,0 +1,267 @@ +import assert from 'node:assert/strict'; +import { mkdtemp, rm, readdir } from 'node:fs/promises'; +import { request } from 'node:http'; +import { join } from 'node:path'; +import { tmpdir } from 'node:os'; +import { once } from 'node:events'; +import { test } from 'node:test'; +import { extractionServer } from './server.js'; +import { ExtractionError, policy } from './protocol.js'; +import type { Backend } from './backend.js'; + +async function fixture(deadline = 500) { + const root = await mkdtemp(join(tmpdir(), 'extraction-test-')); + const events: string[] = []; + const backend: Backend = { + healthy: true, + create: async () => { + events.push('create'); + return mkdtemp(join(root, 'job-')); + }, + run: async () => { + events.push('run'); + return Buffer.from('{"segments":[]}'); + }, + cleanup: async job => { + events.push('cleanup'); + await rm(job, { recursive: true }); + }, + }; + const server = extractionServer(backend, deadline); + const socketPath = join(root, 'service.sock'); + server.listen(socketPath); + await once(server, 'listening'); + return { + backend, + events, + root, + socketPath, + server, + close: async () => { + server.closeAllConnections(); + await new Promise(resolve => server.close(() => resolve())); + await rm(root, { recursive: true, force: true }); + }, + }; +} +const headers = { + 'Content-Type': 'application/octet-stream', + 'Content-Length': '4', + 'X-Extraction-Version': '1', + 'X-Extraction-Format': 'pdf', +}; +async function send( + socketPath: string, + overrides = {}, + path = '/v1/extract-text' +) { + return new Promise<{ status: number; body: string }>((resolve, reject) => { + const req = request( + { + socketPath, + path, + method: 'POST', + headers: { ...headers, ...overrides }, + }, + res => { + let body = ''; + res.on('data', bytes => { + body += bytes; + }); + res.on('end', () => resolve({ status: res.statusCode!, body })); + } + ); + req.on('error', reject); + req.end('test'); + }); +} +test('binary upload executes once and cleanup completes before success', async () => { + const f = await fixture(); + try { + assert.equal((await send(f.socketPath)).status, 200); + assert.deepEqual(f.events, ['create', 'run', 'cleanup']); + assert.deepEqual(await readdir(f.root), ['service.sock']); + } finally { + await f.close(); + } +}); +test('closed parameters and declared input limit are rejected before admission', async () => { + const f = await fixture(); + try { + for (const changes of [ + { 'X-Extraction-Format': 'xls' }, + { 'X-Extraction-Version': '2' }, + { 'X-Extraction-Url': 'https://example.com' }, + { 'Content-Type': 'text/plain' }, + ]) { + assert.equal((await send(f.socketPath, changes)).status, 400); + } + assert.equal( + (await send(f.socketPath, {}, '/v1/extract-text?path=x')).status, + 404 + ); + assert.equal( + ( + await send(f.socketPath, { + 'Content-Length': String(policy.inputBytes + 1), + }) + ).status, + 413 + ); + assert.equal(f.events.length, 0); + } finally { + await f.close(); + } +}); +test('cleanup failure suppresses success and quarantines backend', async () => { + const f = await fixture(); + f.backend.cleanup = async () => { + f.backend.healthy = false; + throw new ExtractionError('UNAVAILABLE'); + }; + try { + assert.equal((await send(f.socketPath)).status, 503); + assert.equal((await send(f.socketPath)).status, 503); + assert.equal(f.events.filter(e => e === 'run').length, 1); + } finally { + await f.close(); + } +}); +test('deadline aborts execution and waits for its close before cleanup', async () => { + const f = await fixture(50); + f.backend.run = async (_job, _format, _digest, signal) => { + await new Promise(resolve => + signal.addEventListener( + 'abort', + () => { + setTimeout(() => { + f.events.push('closed'); + resolve(); + }, 15); + }, + { once: true } + ) + ); + signal.throwIfAborted(); + return Buffer.alloc(0); + }; + try { + assert.equal((await send(f.socketPath)).status, 504); + assert.deepEqual(f.events, ['create', 'closed', 'cleanup']); + } finally { + await f.close(); + } +}); +test('upload disconnect and stalled upload both release scratch and admission', async () => { + const f = await fixture(60); + try { + for (const disconnect of [true, false]) { + const req = request({ + socketPath: f.socketPath, + path: '/v1/extract-text', + method: 'POST', + headers, + }); + req.on('error', () => {}); + req.flushHeaders(); + req.write('t'); + await new Promise(resolve => setTimeout(resolve, 20)); + if (disconnect) req.destroy(); + await new Promise(resolve => setTimeout(resolve, 80)); + assert.deepEqual(await readdir(f.root), ['service.sock']); + } + assert.equal(f.events.includes('run'), false); + assert.equal((await send(f.socketPath)).status, 200); + } finally { + await f.close(); + } +}); +test('two active jobs bound concurrency without a queue', async () => { + const f = await fixture(200); + let active = 0; + f.backend.run = async (_job, _format, _digest, signal) => { + active++; + await new Promise(resolve => + signal.addEventListener('abort', () => resolve(), { once: true }) + ); + signal.throwIfAborted(); + return Buffer.alloc(0); + }; + try { + const a = send(f.socketPath), + b = send(f.socketPath); + while (active !== 2) + await new Promise(resolve => setTimeout(resolve, 5)); + assert.equal((await send(f.socketPath)).status, 429); + assert.equal((await a).status, 504); + assert.equal((await b).status, 504); + } finally { + await f.close(); + } +}); +test('deadline closes stalled output readers while keeping their slots admitted', async () => { + const f = await fixture(250); + f.backend.run = async () => Buffer.alloc(policy.resultBytes, 'a'); + const clients = [0, 1].map(() => { + const req = request( + { + socketPath: f.socketPath, + path: '/v1/extract-text', + method: 'POST', + headers, + }, + res => { + res.pause(); + res.on('error', () => {}); + } + ); + req.on('error', () => {}); + req.end('test'); + return req; + }); + try { + while (f.events.filter(e => e === 'cleanup').length !== 2) + await new Promise(resolve => setTimeout(resolve, 5)); + assert.equal((await send(f.socketPath)).status, 429); + await new Promise(resolve => setTimeout(resolve, 300)); + f.backend.run = async () => Buffer.from('{}'); + assert.equal((await send(f.socketPath)).status, 200); + } finally { + clients.forEach(client => client.destroy()); + await f.close(); + } +}); +test('duplicate extraction control headers are rejected before worker creation', async () => { + const f = await fixture(); + try { + assert.equal( + ( + await send(f.socketPath, { + 'X-Extraction-Format': ['pdf', 'pdf'], + }) + ).status, + 400 + ); + assert.equal(f.events.length, 0); + } finally { + await f.close(); + } +}); + +test('unsupported document content returns a safe failure after cleanup', async () => { + const f = await fixture(); + f.backend.run = async () => { + throw new ExtractionError('UNSUPPORTED_CONTENT'); + }; + try { + const response = await send(f.socketPath); + assert.equal(response.status, 422); + assert.deepEqual(JSON.parse(response.body), { + error: { code: 'UNSUPPORTED_CONTENT' }, + }); + assert.deepEqual(f.events, ['create', 'cleanup']); + assert.deepEqual(await readdir(f.root), ['service.sock']); + } finally { + await f.close(); + } +}); diff --git a/packages/extraction/src/server.ts b/packages/extraction/src/server.ts new file mode 100644 index 00000000..b3bd37c8 --- /dev/null +++ b/packages/extraction/src/server.ts @@ -0,0 +1,260 @@ +import { createHash } from 'node:crypto'; +import { createServer } from 'node:http'; +import { open } from 'node:fs/promises'; +import { join } from 'node:path'; +import { ExtractionError, policy } from './protocol.js'; +import type { IncomingMessage, ServerResponse } from 'node:http'; +import type { Backend } from './backend.js'; +import type { Format } from './protocol.js'; + +export const capabilities = Object.freeze({ + version: 1, + operation: 'document.extract-text', + formats: ['pdf', 'docx'], + structure: { + pdf: 'ordered pages, one-based index', + docx: 'document body, paragraphs and tables', + }, + limitations: { + pdf: 'Text only. Simple fonts need interpretable standard/Unicode mappings. Type3/composite fonts need complete Unicode maps. Unmapped glyphs, codec recovery and graphics-state font selection are rejected.', + docx: 'Body paragraphs/tables and direct runs/hyperlinks only. Other content structures, fields, images and wrappers are rejected; formatting is ignored.', + }, + limits: { + ...policy, + concurrency: 2, + expandedBytes: 32 * 1024 * 1024, + archiveEntries: 512, + tableColumns: 512, + }, +}); +const status = (code: string): number => { + if (code === 'BUSY') return 429; + if (code === 'UNAVAILABLE') return 503; + if (code === 'DEADLINE') return 504; + if ( + code === 'INPUT_LIMIT' || + code === 'OUTPUT_LIMIT' || + code === 'DECOMPRESSION_LIMIT' || + code === 'STRUCTURE_LIMIT' + ) + return 413; + if (code === 'INVALID_REQUEST') return 400; + return 422; +}; +function reply(res: ServerResponse, code: number, bytes: Buffer): void { + res.writeHead(code, { + 'Content-Type': 'application/json; charset=utf-8', + 'Content-Length': bytes.length, + Connection: 'close', + 'Cache-Control': 'no-store', + 'X-Content-Type-Options': 'nosniff', + }); + res.end(bytes); +} +function requestFormat(req: IncomingMessage): Format { + const seen = new Set(); + for (let i = 0; i < req.rawHeaders.length; i += 2) { + const name = req.rawHeaders[i]!.toLowerCase(); + if ( + name.startsWith('x-extraction-') && + !['x-extraction-version', 'x-extraction-format'].includes(name) + ) { + throw new ExtractionError('INVALID_REQUEST'); + } + if ( + [ + 'content-length', + 'content-type', + 'x-extraction-version', + 'x-extraction-format', + ].includes(name) + ) { + if (seen.has(name)) throw new ExtractionError('INVALID_REQUEST'); + seen.add(name); + } + } + if ( + req.headers['content-type'] !== 'application/octet-stream' || + req.headers['x-extraction-version'] !== '1' || + req.headers['transfer-encoding'] + ) { + throw new ExtractionError('INVALID_REQUEST'); + } + const format = req.headers['x-extraction-format']; + if (format !== 'pdf' && format !== 'docx') + throw new ExtractionError('INVALID_REQUEST'); + const length = req.headers['content-length']; + if (typeof length !== 'string' || !/^[1-9]\d*$/.test(length)) + throw new ExtractionError('INVALID_REQUEST'); + if ( + !Number.isSafeInteger(Number(length)) || + Number(length) > policy.inputBytes + ) + throw new ExtractionError('INPUT_LIMIT'); + return format; +} +export function extractionServer( + backend: Backend, + deadlineMs: number = policy.deadlineMs +) { + if ( + !Number.isInteger(deadlineMs) || + deadlineMs <= 0 || + deadlineMs > policy.deadlineMs + ) + throw new Error('INVALID_DEADLINE'); + let admitted = 0; + const server = createServer( + { + maxHeaderSize: 4096, + requireHostHeader: true, + connectionsCheckingInterval: 1000, + }, + (req, res) => { + void handle(req, res); + } + ); + server.maxConnections = 16; + server.maxRequestsPerSocket = 1; + server.headersTimeout = 5000; + server.requestTimeout = policy.deadlineMs; + server.setTimeout(policy.deadlineMs, socket => socket.destroy()); + server.on('checkContinue', (req, res) => { + reply(res, 400, Buffer.from('{"error":{"code":"INVALID_REQUEST"}}')); + }); + async function handle( + req: IncomingMessage, + res: ServerResponse + ): Promise { + if (req.method === 'GET' && req.url === '/v1/capabilities') { + reply( + res, + backend.healthy ? 200 : 503, + Buffer.from(JSON.stringify(capabilities)) + ); + return; + } + if (req.method !== 'POST' || req.url !== '/v1/extract-text') { + reply( + res, + 404, + Buffer.from('{"error":{"code":"INVALID_REQUEST"}}') + ); + return; + } + let format: Format; + try { + format = requestFormat(req); + } catch (error) { + const code = + error instanceof ExtractionError + ? error.code + : 'INVALID_REQUEST'; + reply( + res, + status(code), + Buffer.from(JSON.stringify({ error: { code } })) + ); + return; + } + if (!backend.healthy || admitted >= 2) { + const code = backend.healthy ? 'BUSY' : 'UNAVAILABLE'; + reply( + res, + status(code), + Buffer.from(JSON.stringify({ error: { code } })) + ); + return; + } + admitted++; + const controller = new AbortController(); + const timeout = setTimeout( + () => controller.abort(new ExtractionError('DEADLINE')), + deadlineMs + ); + const aborted = () => + controller.abort(new ExtractionError('CANCELLED')); + controller.signal.addEventListener( + 'abort', + () => { + if (!req.complete) req.destroy(); + if (res.headersSent) res.destroy(); + }, + { once: true } + ); + req.once('aborted', aborted); + res.once('close', () => { + if (!res.writableFinished) aborted(); + }); + const delivered = new Promise(resolve => { + res.once('finish', resolve); + res.once('close', resolve); + }); + let job: string | undefined; + let bytes: Buffer | undefined; + let failure: unknown; + try { + controller.signal.throwIfAborted(); + job = await backend.create(); + const file = await open(join(job, 'input'), 'wx', 0o600); + const digest = createHash('sha256'); + let size = 0; + try { + for await (const chunk of req) { + controller.signal.throwIfAborted(); + size += chunk.length; + if (size > policy.inputBytes) + throw new ExtractionError('INPUT_LIMIT'); + digest.update(chunk); + await file.writeFile(chunk); + } + } finally { + await file.close(); + } + if (size !== Number(req.headers['content-length'])) + throw new ExtractionError('INVALID_REQUEST'); + controller.signal.throwIfAborted(); + bytes = await backend.run( + job, + format, + digest.digest('hex'), + controller.signal + ); + } catch (error) { + failure = error; + } finally { + if (job) { + try { + await backend.cleanup(job); + } catch (error) { + failure = error; + } + } + } + try { + if (controller.signal.aborted && !failure) + failure = controller.signal.reason; + if (controller.signal.reason instanceof ExtractionError) + failure = controller.signal.reason; + if (!res.destroyed) { + if (failure || !bytes) { + const code = + failure instanceof ExtractionError + ? failure.code + : 'UNAVAILABLE'; + reply( + res, + status(code), + Buffer.from(JSON.stringify({ error: { code } })) + ); + } else reply(res, 200, bytes); + } + await delivered; + } finally { + clearTimeout(timeout); + req.removeListener('aborted', aborted); + admitted--; + } + } + return server; +} diff --git a/packages/extraction/tests/adversarial.py b/packages/extraction/tests/adversarial.py new file mode 100644 index 00000000..710c3a11 --- /dev/null +++ b/packages/extraction/tests/adversarial.py @@ -0,0 +1,488 @@ +"""Create bounded adversarial test documents inside the disposable test container.""" +from pathlib import Path +import struct +import zipfile +from docx import Document +from pypdf import PdfReader, PdfWriter + +root = Path('/tests/fixtures') +(root / 'bad.docx').write_bytes(b'PK\x03\x04not a zip') +with zipfile.ZipFile(root / 'entries.docx', 'w') as file: + for index in range(513): + file.writestr(str(index), 'x') +with zipfile.ZipFile(root / 'duplicate.docx', 'w') as file: + file.writestr('same', 'a') + file.writestr('same', 'b') +with zipfile.ZipFile(root / 'bomb.docx', 'w', compression=zipfile.ZIP_DEFLATED) as file: + file.writestr('content.xml', b'x' * (8 * 1024 * 1024 + 1)) +with zipfile.ZipFile(root / 'aggregate.docx', 'w', compression=zipfile.ZIP_DEFLATED) as file: + for index in range(5): + file.writestr(str(index), b'x' * (7 * 1024 * 1024)) +for name, value in [('utf8.docx', b'\xff'), ('entities.docx', b']>')]: + with zipfile.ZipFile(root / name, 'w') as file: + file.writestr('word/document.xml', value) +with zipfile.ZipFile(root / 'forged.docx', 'w', compression=zipfile.ZIP_DEFLATED) as file: + file.writestr('content.xml', 'forged-content' * 20) +value = bytearray((root / 'forged.docx').read_bytes()) +central = value.index(b'PK\x01\x02') +struct.pack_into('') + 2 + value = value[:declaration_end] + b']>' + value[declaration_end:] + value = value.replace(b'First paragraph', b'&controlled;') + elif payload == 'utf8': + value = value.replace(b'First paragraph', b'\xff') + elif member == 'word/_rels/document.xml.rels': + member = 'word/_rels/document.bin.rels' + target.writestr(member, value) +print('DOCX completeness and content-type fixtures generated') + +# The supported structure is closed, not an expanding list of forbidden tags. +for tag in ['dir', 'bdo', 'smartTag', 'unknownWrapper']: + for placement in ['paragraph', 'cell', 'hyperlink']: + document = Document() + document.add_paragraph('Ordinary text must not hide wrapped content') + paragraph = (document.add_table(rows=1, cols=1).cell(0, 0).add_paragraph() + if placement == 'cell' else document.add_paragraph()) + run = paragraph.add_run('Wrapped run text')._r + parent = run.getparent() + if placement == 'hyperlink': + link = OxmlElement('w:hyperlink') + link.set(qn('w:anchor'), 'test') + parent.append(link) + parent = link + wrapper = OxmlElement('w:' + tag) + if tag in ('dir', 'bdo'): + wrapper.set(qn('w:val'), 'rtl') + wrapper.append(run) + parent.append(wrapper) + document.save(root / f'wrapper-{tag}-{placement}.docx') + +# A malformed unknown block and run child cannot bypass the same invariant. +document = Document() +document.add_paragraph('Ordinary text') +paragraph = document.add_paragraph('Unknown body wrapper text') +parent = paragraph._p.getparent() +wrapper = OxmlElement('w:unknownWrapper') +wrapper.append(paragraph._p) +parent.insert(0, wrapper) +document.save(root / 'wrapper-block.docx') +document = Document() +document.add_paragraph('Ordinary text') +run = document.add_paragraph().add_run('Run wrapper text')._r +text = run[0] +wrapper = OxmlElement('w:unknownWrapper') +wrapper.append(text) +run.append(wrapper) +document.save(root / 'wrapper-run.docx') + +# Accepted formatting, hyperlinks, bookmarks and run text primitives retain order. +document = Document() +paragraph = document.add_paragraph() +bookmark = OxmlElement('w:bookmarkStart') +bookmark.set(qn('w:id'), '1') +bookmark.set(qn('w:name'), 'test') +paragraph._p.append(bookmark) +paragraph.add_run('Bold ').bold = True +link = OxmlElement('w:hyperlink') +link.set(qn('w:anchor'), 'test') +run = OxmlElement('w:r') +text = OxmlElement('w:t') +text.text = 'Link' +run.append(text) +link.append(run) +paragraph._p.append(link) +paragraph.add_run('\tTabbed\nNext') +run = paragraph.add_run()._r +run.append(OxmlElement('w:noBreakHyphen')) +run.append(OxmlElement('w:ptab')) +run.append(OxmlElement('w:cr')) +text = OxmlElement('w:t') +text.text = 'End' +run.append(text) +run.append(OxmlElement('w:lastRenderedPageBreak')) +bookmark = OxmlElement('w:bookmarkEnd') +bookmark.set(qn('w:id'), '1') +paragraph._p.append(bookmark) +document.save(root / 'supported-runs.docx') +print('DOCX closed-structure fixtures generated') + +# Physical cells are unique. A merged origin's text must never be repeated. +for name, end in [('horizontal.docx', (0, 1)), ('vertical.docx', (1, 0)), + ('rectangle.docx', (1, 1))]: + document = Document() + table = document.add_table(rows=2, cols=3) + for row in range(2): + for column in range(3): + table.cell(row, column).text = f'R{row}C{column}' + table.cell(0, 0).merge(table.cell(*end)).text = 'Merged text' + document.save(root / name) + +document = Document() +table = document.add_table(rows=2, cols=2) +origin = table.cell(0, 0).merge(table.cell(1, 1)) +origin.text = 'Before merged nested table' +origin.add_table(rows=1, cols=1).cell(0, 0).text = 'Unique nested content' +origin.add_paragraph('After merged nested table') +document.save(root / 'merged-nested.docx') + +for name, end in [('merged-large.docx', (0, 1)), ('merged-large-vertical.docx', (1, 0))]: + document = Document() + table = document.add_table(rows=2, cols=2) + table.cell(0, 0).merge(table.cell(*end)).text = 'x' * (600 * 1024) + document.save(root / name) + +document = Document() +table = document.add_table(rows=1, cols=3) +table.cell(0, 1).text = 'Middle cell' +row = table.rows[0]._tr +row.remove(row.tc_lst[-1]) +row.remove(row.tc_lst[0]) +properties = row.get_or_add_trPr() +for tag in ('gridBefore', 'gridAfter'): + value = OxmlElement('w:' + tag) + value.set(qn('w:val'), '1') + properties.append(value) +document.save(root / 'omitted-cells.docx') + +for name, mode in [('merge-content.docx', 'content'), ('merge-orphan.docx', 'orphan'), + ('merge-span.docx', 'span'), ('merge-legacy.docx', 'legacy'), + ('wide-table.docx', 'width')]: + document = Document() + table = document.add_table(rows=2, cols=2) + if mode == 'legacy': + value = OxmlElement('w:hMerge') + value.set(qn('w:val'), 'restart') + table.cell(0, 0)._tc.get_or_add_tcPr().append(value) + elif mode == 'width': + for _ in range(511): + table._tbl.tblGrid.append(OxmlElement('w:gridCol')) + else: + table.cell(0, 0).merge(table.cell(1, 0)).text = 'Origin text' + continuation = table.rows[1]._tr.tc_lst[0] + if mode == 'content': + continuation.p_lst[0].append(OxmlElement('w:r')) + text = OxmlElement('w:t') + text.text = 'Unexpected continuation text' + continuation.p_lst[0][-1].append(text) + elif mode == 'orphan': + table.rows[0]._tr.tc_lst[0].get_or_add_tcPr().remove( + table.rows[0]._tr.tc_lst[0].tcPr.vMerge) + elif mode == 'span': + continuation.get_or_add_tcPr().get_or_add_gridSpan().val = 2 + document.save(root / name) +print('DOCX merged and omitted-cell fixtures generated') + +from pypdf.generic import FloatObject, TextStringObject + +def font_pdf(name, subtype, raw=b'A', mapping=None, glyph='/unmappedGlyph', encoding=None, form=False): + writer = PdfWriter() + writer.append(root / 'pages.pdf') + page = writer.pages[0] + font = DictionaryObject({NameObject('/Type'): NameObject('/Font'), + NameObject('/Subtype'): NameObject('/' + subtype), + NameObject('/BaseFont'): NameObject('/Helvetica')}) + if subtype == 'Type3': + proc = DecodedStreamObject() + proc.set_data(b'600 0 0 0 500 700 d1 0 0 500 700 re f') + font[NameObject('/CharProcs')] = DictionaryObject({NameObject(glyph): writer._add_object(proc)}) + font[NameObject('/FontBBox')] = ArrayObject([NumberObject(v) for v in [0, 0, 500, 700]]) + font[NameObject('/FontMatrix')] = ArrayObject([FloatObject(v) for v in [0.001, 0, 0, 0.001, 0, 0]]) + font[NameObject('/Resources')] = DictionaryObject() + font[NameObject('/FirstChar')] = NumberObject(65) + font[NameObject('/LastChar')] = NumberObject(66) + font[NameObject('/Widths')] = ArrayObject([NumberObject(600), NumberObject(600)]) + if subtype == 'Type0': + descendant = DictionaryObject({ + NameObject('/Type'): NameObject('/Font'), NameObject('/Subtype'): NameObject('/CIDFontType2'), + NameObject('/BaseFont'): NameObject('/TestComposite'), + NameObject('/CIDSystemInfo'): DictionaryObject({ + NameObject('/Registry'): TextStringObject('Adobe'), + NameObject('/Ordering'): TextStringObject('Identity'), NameObject('/Supplement'): NumberObject(0), + }), + NameObject('/FontDescriptor'): DictionaryObject({ + NameObject('/Type'): NameObject('/FontDescriptor'), NameObject('/FontName'): NameObject('/TestComposite'), + NameObject('/Flags'): NumberObject(4), NameObject('/Ascent'): NumberObject(800), + NameObject('/Descent'): NumberObject(-200), NameObject('/CapHeight'): NumberObject(700), + NameObject('/ItalicAngle'): NumberObject(0), NameObject('/StemV'): NumberObject(80), + NameObject('/FontBBox'): ArrayObject([NumberObject(v) for v in [0, -200, 1000, 800]]), + }), NameObject('/CIDToGIDMap'): NameObject('/Identity'), + }) + font[NameObject('/DescendantFonts')] = ArrayObject([writer._add_object(descendant)]) + font[NameObject('/Encoding')] = NameObject('/Identity-H') + else: + font[NameObject('/Encoding')] = DictionaryObject({ + NameObject('/BaseEncoding'): NameObject('/WinAnsiEncoding'), + NameObject('/Differences'): ArrayObject([NumberObject(65), NameObject(glyph)]), + }) + if encoding: + font[NameObject('/Encoding')] = NameObject(encoding) + if mapping is not None: + cmap = DecodedStreamObject() + size = 2 if subtype == 'Type0' else 1 + start, end = ('0000', 'FFFF') if size == 2 else ('00', 'FF') + pairs = '\n'.join(f'<{code:0{size*2}X}> <{value.encode("utf-16-be", errors="surrogatepass").hex()}>' + for code, value in mapping.items()) + cmap.set_data((f'/CIDInit /ProcSet findresource begin\n12 dict begin\nbegincmap\n' + f'/CMapName /TestUnicode def\n/CMapType 2 def\n' + f'1 begincodespacerange\n<{start}> <{end}>\nendcodespacerange\n' + f'{len(mapping)} beginbfchar\n{pairs}\nendbfchar\nendcmap\n' + f'CMapName currentdict /CMap defineresource pop\nend\nend').encode('ascii')) + font[NameObject('/ToUnicode')] = writer._add_object(cmap.flate_encode()) + resources = DictionaryObject({NameObject('/Font'): DictionaryObject({NameObject('/F2'): writer._add_object(font)})}) + stream = DecodedStreamObject() + stream.set_data(b'BT /F2 12 Tf 72 700 Td <' + raw.hex().encode() + b'> Tj ET') + if form: + stream[NameObject('/Type')] = NameObject('/XObject') + stream[NameObject('/Subtype')] = NameObject('/Form') + stream[NameObject('/BBox')] = ArrayObject([NumberObject(v) for v in [0, 0, 612, 792]]) + stream[NameObject('/Resources')] = resources + page['/Resources'][NameObject('/XObject')] = DictionaryObject({NameObject('/FontForm'): writer._add_object(stream)}) + content = page['/Contents'].get_object() + content.set_data(content.get_data() + b'\n/FontForm Do') + else: + page[NameObject('/Resources')] = resources + page[NameObject('/Contents')] = writer._add_object(stream) + writer.write(root / name) + +font_pdf('font-type3-unmapped.pdf', 'Type3') +font_pdf('font-simple-unmapped.pdf', 'Type1') +font_pdf('font-form-unmapped.pdf', 'Type3', form=True) +font_pdf('font-partial-map.pdf', 'Type3', raw=b'AB', mapping={65: 'A'}) +font_pdf('font-empty-map.pdf', 'Type3', mapping={}) +font_pdf('font-bad-unicode.pdf', 'Type3', mapping={65: '\ud800'}) +font_pdf('font-composite-unmapped.pdf', 'Type0', raw=b'\x00A') +font_pdf('font-codec-fallback.pdf', 'Type0', raw=b'\x01', mapping={1: 'A'}) +font_pdf('font-composite-partial.pdf', 'Type0', raw=b'\x00\x01\x00\x02', mapping={1: 'A'}) +font_pdf('font-type3-unicode.pdf', 'Type3', mapping={65: '\u03a9'}) +font_pdf('font-simple-unicode.pdf', 'Type1', mapping={65: '\u03a9'}) +font_pdf('font-known-glyph.pdf', 'Type1', glyph='/Aacute') +font_pdf('font-ligature.pdf', 'Type3', mapping={65: 'fi'}) +font_pdf('font-emoji.pdf', 'Type3', mapping={65: '\U0001f600'}) +font_pdf('font-composite-unicode.pdf', 'Type0', raw=b'\x00\x01\x00\x02', mapping={1: 'A', 2: 'fi'}) +font_pdf('font-form-unicode.pdf', 'Type3', mapping={65: '\u03a9'}, form=True) +writer = PdfWriter() +writer.append(root / 'pages.pdf') +content = writer.pages[0]['/Contents'].get_object() +content.set_data(content.get_data().replace(b'/F1', b'/MissingFont')) +writer.write(root / 'font-missing.pdf') +print('PDF font mapping fixtures generated') + +font_pdf('font-replacement.pdf', 'Type3', mapping={65: '\ufffd'}) +font_pdf('font-partial-simple.pdf', 'Type1', raw=b'AB', mapping={65: 'A'}) +writer = PdfWriter() +writer.append(root / 'pages.pdf') +del writer.pages[0]['/Resources'] +writer.write(root / 'font-no-resources.pdf') +writer = PdfWriter() +writer.append(root / 'pages.pdf') +del writer.pages[0]['/Resources'] +writer.pages[0]['/Contents'].get_object().set_data(b'q 0 0 10 10 re f Q') +writer.write(root / 'no-resources-graphics.pdf') +writer = PdfWriter() +writer.append(root / 'pages.pdf') +page = writer.pages[0] +page['/Resources'][NameObject('/ExtGState')] = DictionaryObject({NameObject('/GS1'): DictionaryObject({ + NameObject('/Font'): ArrayObject([page['/Resources']['/Font'].raw_get('/F1'), NumberObject(12)]), +})}) +stream = page['/Contents'].get_object() +stream.set_data(b'/GS1 gs\n' + stream.get_data()) +writer.write(root / 'font-graphics-state.pdf') + +from pypdf.generic import NullObject + +# Null optional dictionary entries mean absent, including indirect nulls. +for subtype in ['Type1', 'TrueType']: + for mode in ['absent', 'null', 'indirect-null', 'empty-dict', 'null-base', 'invalid-number']: + writer = PdfWriter() + writer.append(root / 'pages.pdf') + font = writer.pages[0]['/Resources']['/Font']['/F1'] + font[NameObject('/Subtype')] = NameObject('/' + subtype) + font[NameObject('/BaseFont')] = NameObject('/UnknownCustomFont') + if mode == 'null': + font[NameObject('/Encoding')] = NullObject() + elif mode == 'indirect-null': + font[NameObject('/Encoding')] = writer._add_object(NullObject()) + elif mode == 'empty-dict': + font[NameObject('/Encoding')] = DictionaryObject() + elif mode == 'null-base': + font[NameObject('/Encoding')] = DictionaryObject({NameObject('/BaseEncoding'): NullObject()}) + elif mode == 'invalid-number': + font[NameObject('/Encoding')] = NumberObject(1) + writer.write(root / f'font-{subtype.lower()}-{mode}.pdf') + +for mode in ['null', 'indirect-null', 'null-unicode', 'indirect-null-unicode', 'null-base']: + writer = PdfWriter() + writer.append(root / 'pages.pdf') + font = writer.pages[0]['/Resources']['/Font']['/F1'] + if mode == 'null-base': + font[NameObject('/Encoding')] = DictionaryObject({NameObject('/BaseEncoding'): NullObject()}) + else: + key = '/ToUnicode' if 'unicode' in mode else '/Encoding' + font[NameObject(key)] = writer._add_object(NullObject()) if mode.startswith('indirect') else NullObject() + writer.write(root / f'font-core-{mode}.pdf') + +writer = PdfWriter() +writer.append(root / 'font-simple-unicode.pdf') +font = writer.pages[0]['/Resources']['/Font']['/F2'] +font[NameObject('/BaseFont')] = NameObject('/UnknownCustomFont') +font[NameObject('/Encoding')] = NullObject() +writer.write(root / 'font-null-with-unicode.pdf') +writer = PdfWriter() +writer.append(root / 'font-known-glyph.pdf') +writer.pages[0]['/Resources']['/Font']['/F2'][NameObject('/BaseFont')] = NameObject('/UnknownCustomFont') +writer.write(root / 'font-custom-standard.pdf') +print('PDF null/absent encoding fixtures generated') diff --git a/packages/extraction/tests/block-landlock.c b/packages/extraction/tests/block-landlock.c new file mode 100644 index 00000000..1185ec95 --- /dev/null +++ b/packages/extraction/tests/block-landlock.c @@ -0,0 +1,21 @@ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include + +/* Negative startup probe: add a restriction to Docker's existing filter. */ +int main(void) { + struct sock_filter filter[] = { + BPF_STMT(BPF_LD | BPF_W | BPF_ABS, offsetof(struct seccomp_data, nr)), + BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, SYS_landlock_create_ruleset, 0, 1), + BPF_STMT(BPF_RET | BPF_K, SECCOMP_RET_ERRNO | 38), + BPF_STMT(BPF_RET | BPF_K, SECCOMP_RET_ALLOW) + }; + struct sock_fprog program = {.len = sizeof(filter) / sizeof(filter[0]), .filter = filter}; + if (prctl(PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0) || prctl(PR_SET_SECCOMP, SECCOMP_MODE_FILTER, &program)) return 125; + execl("/usr/local/bin/node", "node", "/opt/extraction/dist/main.js", (char *)NULL); + return 125; +} diff --git a/packages/extraction/tests/container-client.mjs b/packages/extraction/tests/container-client.mjs new file mode 100644 index 00000000..14e70f74 --- /dev/null +++ b/packages/extraction/tests/container-client.mjs @@ -0,0 +1,303 @@ +import assert from 'node:assert/strict'; +import { readFileSync } from 'node:fs'; +import { request } from 'node:http'; +import { createHash } from 'node:crypto'; + +const socketPath = '/socket/extraction.sock'; +function send(format, bytes, path = '/v1/extract-text', extra = {}) { + return new Promise((resolve, reject) => { + const req = request( + { + socketPath, + path, + method: 'POST', + headers: { + 'Content-Type': 'application/octet-stream', + 'Content-Length': bytes.length, + 'X-Extraction-Version': '1', + 'X-Extraction-Format': format, + ...extra, + }, + }, + res => { + const chunks = []; + res.on('data', chunk => chunks.push(chunk)); + res.on('end', () => + resolve({ + status: res.statusCode, + value: JSON.parse( + Buffer.concat(chunks).toString('utf8') + ), + }) + ); + res.on('error', reject); + } + ); + req.on('error', reject); + req.end(bytes); + }); +} +const fixtures = '/tests/fixtures'; +const pages = readFileSync(`${fixtures}/pages.pdf`); +const result = await send('pdf', pages); +assert.equal(result.status, 200); +assert.deepEqual( + result.value.segments.map(s => [s.kind, s.index, s.text]), + [ + ['page', 1, 'First page text'], + ['page', 2, 'Second page text'], + ] +); +assert.equal( + result.value.inputSha256, + createHash('sha256').update(pages).digest('hex') +); +assert.equal( + (await send('pdf', readFileSync(`${fixtures}/compressed.pdf`))).value + .segments[0].text, + 'Compressed page text' +); +const docx = await send('docx', readFileSync(`${fixtures}/body.docx`)); +assert.equal(docx.status, 200); +assert.deepEqual(docx.value.segments, [ + { + kind: 'document', + index: 1, + text: 'First paragraph\nLeft cell\tRight cell\nLast paragraph\n', + }, +]); +const form = await send('pdf', readFileSync(`${fixtures}/form-valid.pdf`)); +assert.equal(form.status, 200); +assert.match(form.value.segments[0].text, /Form content/); +const nested = await send('docx', readFileSync(`${fixtures}/nested.docx`)); +assert.equal(nested.status, 200); +assert.match( + nested.value.segments[0].text, + /Before nested table\nNested left\tNested right[\n]+After nested table/ +); +const nestedOnly = await send( + 'docx', + readFileSync(`${fixtures}/nested-only.docx`) +); +assert.equal(nestedOnly.status, 200); +assert.match(nestedOnly.value.segments[0].text, /Only nested text/); +for (const name of [ + 'renamed-valid.docx', + 'renamed-default.docx', + 'renamed-case.docx', +]) { + const response = await send('docx', readFileSync(`${fixtures}/${name}`)); + assert.equal(response.status, 200, name); + assert.deepEqual(response.value.segments, docx.value.segments, name); +} +const primitives = await send( + 'docx', + readFileSync(`${fixtures}/supported-runs.docx`) +); +assert.equal(primitives.status, 200); +assert.equal( + primitives.value.segments[0].text, + 'Bold Link\tTabbed\nNext-\t\nEnd\n' +); +for (const [name, expected] of [ + ['horizontal.docx', 'Merged text\t\tR0C2\nR1C0\tR1C1\tR1C2\n'], + ['vertical.docx', 'Merged text\tR0C1\tR0C2\n\tR1C1\tR1C2\n'], + ['rectangle.docx', 'Merged text\t\tR0C2\n\t\tR1C2\n'], + ['omitted-cells.docx', '\tMiddle cell\t\n'], +]) { + const response = await send('docx', readFileSync(`${fixtures}/${name}`)); + assert.equal(response.status, 200, name); + assert.equal(response.value.segments[0].text, expected, name); +} +const mergedNested = await send( + 'docx', + readFileSync(`${fixtures}/merged-nested.docx`) +); +assert.equal(mergedNested.status, 200); +assert.equal( + mergedNested.value.segments[0].text.match(/Unique nested content/g).length, + 1 +); +assert.match( + mergedNested.value.segments[0].text, + /Before merged nested table[\n]+Unique nested content[\n]+After merged nested table/ +); +for (const name of ['merged-large.docx', 'merged-large-vertical.docx']) { + const response = await send('docx', readFileSync(`${fixtures}/${name}`)); + assert.equal(response.status, 200, name); + assert.equal( + response.value.segments[0].text.match(/x/g).length, + 600 * 1024, + name + ); +} +for (const [name, expected] of [ + ['font-type3-unicode.pdf', '\u03a9'], + ['font-simple-unicode.pdf', '\u03a9'], + ['font-known-glyph.pdf', '\u00c1'], + ['font-ligature.pdf', 'fi'], + ['font-emoji.pdf', '\ud83d\ude00'], + ['font-composite-unicode.pdf', 'Afi'], +]) { + const response = await send('pdf', readFileSync(`${fixtures}/${name}`)); + assert.equal(response.status, 200, name); + assert.equal(response.value.segments[0].text, expected, name); +} +const fontForm = await send( + 'pdf', + readFileSync(`${fixtures}/font-form-unicode.pdf`) +); +assert.equal(fontForm.status, 200); +assert.match(fontForm.value.segments[0].text, /First page text[\n]+\u03a9/); +const noResources = await send( + 'pdf', + readFileSync(`${fixtures}/no-resources-graphics.pdf`) +); +assert.equal(noResources.status, 200); +assert.deepEqual( + noResources.value.segments.map(segment => segment.text), + ['', 'Second page text'] +); +for (const mode of [ + 'null', + 'indirect-null', + 'null-unicode', + 'indirect-null-unicode', + 'null-base', +]) { + const response = await send( + 'pdf', + readFileSync(`${fixtures}/font-core-${mode}.pdf`) + ); + assert.equal(response.status, 200, mode); + assert.deepEqual(response.value.segments, result.value.segments, mode); +} +for (const [name, expected] of [ + ['font-null-with-unicode.pdf', '\u03a9'], + ['font-custom-standard.pdf', '\u00c1'], +]) { + const response = await send('pdf', readFileSync(`${fixtures}/${name}`)); + assert.equal(response.status, 200, name); + assert.equal(response.value.segments[0].text, expected, name); +} +const rejectedFixtures = [ + ['font-replacement.pdf', 'pdf', 'UNSUPPORTED_ENCODING'], + ['font-partial-simple.pdf', 'pdf', 'UNSUPPORTED_ENCODING'], + ['font-no-resources.pdf', 'pdf', 'UNSUPPORTED_ENCODING'], + ['font-graphics-state.pdf', 'pdf', 'UNSUPPORTED_ENCODING'], + ['font-type3-unmapped.pdf', 'pdf', 'UNSUPPORTED_ENCODING'], + ['font-simple-unmapped.pdf', 'pdf', 'UNSUPPORTED_ENCODING'], + ['font-form-unmapped.pdf', 'pdf', 'UNSUPPORTED_ENCODING'], + ['font-partial-map.pdf', 'pdf', 'UNSUPPORTED_ENCODING'], + ['font-empty-map.pdf', 'pdf', 'UNSUPPORTED_ENCODING'], + ['font-bad-unicode.pdf', 'pdf', 'UNSUPPORTED_ENCODING'], + ['font-composite-unmapped.pdf', 'pdf', 'UNSUPPORTED_ENCODING'], + ['font-codec-fallback.pdf', 'pdf', 'UNSUPPORTED_ENCODING'], + ['font-composite-partial.pdf', 'pdf', 'UNSUPPORTED_ENCODING'], + ['font-missing.pdf', 'pdf', 'UNSUPPORTED_ENCODING'], + ['merge-content.docx', 'docx', 'UNSUPPORTED_CONTENT'], + ['merge-orphan.docx', 'docx', 'INVALID_DOCUMENT'], + ['merge-span.docx', 'docx', 'INVALID_DOCUMENT'], + ['merge-legacy.docx', 'docx', 'UNSUPPORTED_CONTENT'], + ['wide-table.docx', 'docx', 'STRUCTURE_LIMIT'], + ['control.docx', 'docx', 'UNSUPPORTED_CONTENT'], + ['control-cell.docx', 'docx', 'UNSUPPORTED_CONTENT'], + ['control-inline.docx', 'docx', 'UNSUPPORTED_CONTENT'], + ['control-only.docx', 'docx', 'UNSUPPORTED_CONTENT'], + ['custom-wrapper.docx', 'docx', 'UNSUPPORTED_CONTENT'], + ['revision.docx', 'docx', 'UNSUPPORTED_CONTENT'], + ['simple-field.docx', 'docx', 'UNSUPPORTED_CONTENT'], + ['renamed-entities.docx', 'docx', 'INVALID_DOCUMENT'], + ['renamed-default-entities.docx', 'docx', 'INVALID_DOCUMENT'], + ['renamed-case-entities.docx', 'docx', 'INVALID_DOCUMENT'], + ['renamed-utf8.docx', 'docx', 'INVALID_DOCUMENT'], + ['empty.pdf', 'pdf', 'EMPTY_OUTPUT'], + ['empty.docx', 'docx', 'EMPTY_OUTPUT'], + ['inflate.pdf', 'pdf', 'DECOMPRESSION_LIMIT'], + ['aggregate.pdf', 'pdf', 'DECOMPRESSION_LIMIT'], + ['raw-limit.pdf', 'pdf', 'DECOMPRESSION_LIMIT'], + ['form-limit.pdf', 'pdf', 'DECOMPRESSION_LIMIT'], + ['form-filter.pdf', 'pdf', 'UNSUPPORTED_ENCODING'], + ['aggregate.docx', 'docx', 'DECOMPRESSION_LIMIT'], + ['bad.docx', 'docx', 'INVALID_DOCUMENT'], + ['entries.docx', 'docx', 'STRUCTURE_LIMIT'], + ['duplicate.docx', 'docx', 'INVALID_DOCUMENT'], + ['forged.docx', 'docx', 'INVALID_DOCUMENT'], + ['forged-bomb.docx', 'docx', 'DECOMPRESSION_LIMIT'], + ['utf8.docx', 'docx', 'INVALID_DOCUMENT'], + ['entities.docx', 'docx', 'INVALID_DOCUMENT'], + ['bomb.docx', 'docx', 'DECOMPRESSION_LIMIT'], + ['output.docx', 'docx', 'OUTPUT_LIMIT'], + ['encrypted.pdf', 'pdf', 'ENCRYPTED_DOCUMENT'], + ['too-many-pages.pdf', 'pdf', 'STRUCTURE_LIMIT'], +]; +for (const subtype of ['type1', 'truetype']) { + for (const mode of [ + 'absent', + 'null', + 'indirect-null', + 'empty-dict', + 'null-base', + 'invalid-number', + ]) { + rejectedFixtures.push([ + `font-${subtype}-${mode}.pdf`, + 'pdf', + 'UNSUPPORTED_ENCODING', + ]); + } +} +for (const tag of ['dir', 'bdo', 'smartTag', 'unknownWrapper']) { + for (const placement of ['paragraph', 'cell', 'hyperlink']) { + rejectedFixtures.push([ + `wrapper-${tag}-${placement}.docx`, + 'docx', + 'UNSUPPORTED_CONTENT', + ]); + } +} +for (const name of ['wrapper-block.docx', 'wrapper-run.docx']) { + rejectedFixtures.push([name, 'docx', 'UNSUPPORTED_CONTENT']); +} +for (const [name, format, code] of rejectedFixtures) { + const response = await send(format, readFileSync(`${fixtures}/${name}`)); + assert.equal( + response.value.error?.code, + code, + `${name}: ${JSON.stringify(response)}` + ); +} +assert.equal( + (await send('pdf', readFileSync(`${fixtures}/raw-exact.pdf`))).status, + 200 +); +const exact = await send('docx', readFileSync(`${fixtures}/exact.docx`)); +assert.equal(exact.status, 200); +assert.equal(exact.value.textBytes, 1024 * 1024); +assert.equal( + (await send('pdf', Buffer.from('not a pdf'))).value.error.code, + 'INVALID_DOCUMENT' +); +assert.equal((await send('docx', pages)).value.error.code, 'INVALID_DOCUMENT'); +assert.equal((await send('xls', pages)).status, 400); +assert.equal((await send('pdf', pages, '/exec')).status, 404); +assert.equal( + (await send('pdf', pages, '/v1/extract-text?script=x')).status, + 404 +); +assert.equal( + ( + await send('pdf', pages, '/v1/extract-text', { + 'X-Extraction-Url': 'http://example.com', + }) + ).status, + 400 +); +// Exact input cap is accepted; one byte more is rejected before parsing. +const padded = Buffer.alloc(10 * 1024 * 1024, 32); +pages.copy(padded); +assert.equal((await send('pdf', padded)).status, 200); +assert.equal((await send('pdf', Buffer.alloc(padded.length + 1))).status, 413); +console.log( + `Container endpoint: PDF/DOCX success, structure, input limits and ${rejectedFixtures.length} adversarial fixtures passed` +); diff --git a/packages/extraction/tests/container.sh b/packages/extraction/tests/container.sh new file mode 100755 index 00000000..839670c5 --- /dev/null +++ b/packages/extraction/tests/container.sh @@ -0,0 +1,61 @@ +#!/usr/bin/env bash +set -euo pipefail +cd "$(dirname "$0")/../../.." +run_id=$(node -e 'console.log(require("node:crypto").randomUUID())') +project="extraction-test-$run_id" +image="librechat-extraction-test:$run_id" +export EXTRACTION_IMAGE="librechat-extraction:$run_id" +compose=(docker compose -p "$project" -f packages/extraction/compose.yaml) +cleanup() { + "${compose[@]}" down -v --remove-orphans >/dev/null + docker image rm "$image" "$EXTRACTION_IMAGE" >/dev/null 2>&1 || true +} +trap cleanup EXIT +"${compose[@]}" build extraction +docker build --target qualification -t "$image" -f packages/extraction/Dockerfile . +"${compose[@]}" up -d --wait --wait-timeout 30 extraction +id=$("${compose[@]}" ps -q extraction) +volume="${project}_extraction-socket" +# Inspect exactly the shipped recipe, not a relaxed test variant. +docker inspect "$id" | python3 -c ' +import json,sys +c=json.load(sys.stdin)[0]; h=c["HostConfig"] +assert h["NetworkMode"] == "none" and h["ReadonlyRootfs"] and not h["Privileged"] +assert h["CapDrop"] == ["ALL"] and h["PidsLimit"] == 32 +assert h["Memory"] == 768*1024*1024 and h["NanoCpus"] == 2*10**9 +assert "no-new-privileges:true" in h["SecurityOpt"] +assert c["Config"]["User"] == "10001:10001" +assert len(c["Mounts"]) == 1 and c["Mounts"][0]["Destination"] == "/socket" +assert set(h["Tmpfs"]) == {"/jobs"} +for flag in ("noexec", "nosuid", "nodev", "size=96m", "nr_inodes=1024", "uid=10001", "gid=10001", "mode=0700"): + assert flag in h["Tmpfs"]["/jobs"] +assert not c["Config"]["ExposedPorts"] if "ExposedPorts" in c["Config"] else True +print("Shipped Compose mounts, resource limits and security options verified")' +docker run --rm --init --network none --cap-drop ALL --security-opt no-new-privileges \ + --mount "type=volume,src=$volume,dst=/socket,readonly" "$image" \ + sh -c 'python /tests/adversarial.py && node /tests/container-client.mjs' +docker exec "$id" node -e 'require("node:fs").readdir("/jobs",(e,v)=>{if(e||v.length)process.exit(1); console.log("Endpoint scratch cleanup verified")})' +# Same outer boundary, with a test-only worker that attempts forbidden operations. +docker run --rm --init --network none --cap-drop ALL --security-opt no-new-privileges \ + --read-only --memory 768m --cpus 2 --pids-limit 32 \ + --tmpfs /jobs:rw,noexec,nosuid,nodev,size=96m,nr_inodes=1024,uid=10001,gid=10001,mode=0700 \ + --tmpfs /socket:rw,noexec,nosuid,nodev,size=1m,uid=10001,gid=10001,mode=0700 \ + "$image" node /tests/isolation.mjs +# Denying the mandatory isolation syscall must fail closed, never start unsandboxed. +set +e +docker run --rm --init --network none --cap-drop ALL --security-opt no-new-privileges \ + --read-only --memory 768m --cpus 2 --pids-limit 32 \ + --tmpfs /jobs:rw,noexec,nosuid,nodev,size=96m,uid=10001,gid=10001,mode=0700 \ + --tmpfs /socket:rw,noexec,nosuid,nodev,size=1m,uid=10001,gid=10001,mode=0700 \ + "$image" /tests/block-landlock > /dev/null 2>&1 +blocked=$? +set -e +if [ "$blocked" -ne 1 ]; then + printf 'FAIL: missing isolation did not reach fail-closed startup (exit %s)\n' "$blocked" >&2 + exit 1 +fi +printf 'Unavailable isolation fails closed\n' +# A stale socket left by SIGKILL must not prevent a safe restart. +docker kill -s KILL "$id" >/dev/null +"${compose[@]}" up -d --wait --wait-timeout 30 extraction +printf 'Hardened container restart passed\n' diff --git a/packages/extraction/tests/fixtures/aggregate.pdf b/packages/extraction/tests/fixtures/aggregate.pdf new file mode 100644 index 00000000..70f476e3 Binary files /dev/null and b/packages/extraction/tests/fixtures/aggregate.pdf differ diff --git a/packages/extraction/tests/fixtures/body.docx b/packages/extraction/tests/fixtures/body.docx new file mode 100644 index 00000000..85a4932c Binary files /dev/null and b/packages/extraction/tests/fixtures/body.docx differ diff --git a/packages/extraction/tests/fixtures/compressed.pdf b/packages/extraction/tests/fixtures/compressed.pdf new file mode 100644 index 00000000..90bbbf92 Binary files /dev/null and b/packages/extraction/tests/fixtures/compressed.pdf differ diff --git a/packages/extraction/tests/fixtures/empty.docx b/packages/extraction/tests/fixtures/empty.docx new file mode 100644 index 00000000..38ea79ce Binary files /dev/null and b/packages/extraction/tests/fixtures/empty.docx differ diff --git a/packages/extraction/tests/fixtures/empty.pdf b/packages/extraction/tests/fixtures/empty.pdf new file mode 100644 index 00000000..fc99c829 Binary files /dev/null and b/packages/extraction/tests/fixtures/empty.pdf differ diff --git a/packages/extraction/tests/fixtures/generate.py b/packages/extraction/tests/fixtures/generate.py new file mode 100644 index 00000000..70f01625 --- /dev/null +++ b/packages/extraction/tests/fixtures/generate.py @@ -0,0 +1,37 @@ +"""Regenerate synthetic, redistributable fixtures with the pinned worker libraries.""" +from pathlib import Path +from pypdf import PdfWriter +from pypdf.generic import DecodedStreamObject, DictionaryObject, NameObject +from docx import Document + +root = Path(__file__).parent + +def pdf(name, pages, compressed=False, padding=0): + writer = PdfWriter() + font = writer._add_object(DictionaryObject({ + NameObject('/Type'): NameObject('/Font'), NameObject('/Subtype'): NameObject('/Type1'), + NameObject('/BaseFont'): NameObject('/Helvetica'), + })) + for text in pages: + page = writer.add_blank_page(width=612, height=792) + page[NameObject('/Resources')] = DictionaryObject({NameObject('/Font'): DictionaryObject({NameObject('/F1'): font})}) + stream = DecodedStreamObject() + stream.set_data(b' ' * padding + b'BT /F1 12 Tf 72 720 Td (' + text.encode() + b') Tj ET') + page[NameObject('/Contents')] = writer._add_object(stream.flate_encode() if compressed else stream) + with (root / name).open('wb') as file: + writer.write(file) + +pdf('pages.pdf', ['First page text', 'Second page text']) +pdf('compressed.pdf', ['Compressed page text'], True) +pdf('empty.pdf', ['']) +pdf('inflate.pdf', [' ' * (8 * 1024 * 1024 + 1)], True) +pdf('aggregate.pdf', ['small'] * 5, True, padding=7 * 1024 * 1024) +document = Document() +document.add_paragraph('First paragraph') +table = document.add_table(rows=1, cols=2) +table.cell(0, 0).text = 'Left cell' +table.cell(0, 1).text = 'Right cell' +document.add_paragraph('Last paragraph') +document.save(root / 'body.docx') +Document().save(root / 'empty.docx') +print('Generated PDF and DOCX fixtures') diff --git a/packages/extraction/tests/fixtures/inflate.pdf b/packages/extraction/tests/fixtures/inflate.pdf new file mode 100644 index 00000000..2fe8291e Binary files /dev/null and b/packages/extraction/tests/fixtures/inflate.pdf differ diff --git a/packages/extraction/tests/fixtures/pages.pdf b/packages/extraction/tests/fixtures/pages.pdf new file mode 100644 index 00000000..1e7acc24 Binary files /dev/null and b/packages/extraction/tests/fixtures/pages.pdf differ diff --git a/packages/extraction/tests/isolation.mjs b/packages/extraction/tests/isolation.mjs new file mode 100644 index 00000000..d44ef800 --- /dev/null +++ b/packages/extraction/tests/isolation.mjs @@ -0,0 +1,64 @@ +import assert from 'node:assert/strict'; +import { spawn } from 'node:child_process'; +import { mkdir, mkdtemp, writeFile, rm, readdir } from 'node:fs/promises'; +import { once } from 'node:events'; +import { RestrictedBackend } from '/opt/extraction/dist/backend.js'; + +await mkdir('/jobs/other'); +await writeFile('/jobs/other/secret', 'cross-job-canary'); +await writeFile('/socket/secret', 'transport-canary'); +const job = await mkdtemp('/jobs/job-'); +const child = spawn( + '/opt/extraction/guard', + [job, '/usr/local/bin/python3.13', '/tests/isolation.py', 'probe'], + { + env: { EXTRACTION_SECRET_CANARY: 'private-canary' }, + stdio: ['ignore', 'pipe', 'pipe'], + } +); +let output = '', + errors = ''; +child.stdout.on('data', bytes => { + output += bytes; +}); +child.stderr.on('data', bytes => { + errors += bytes; +}); +const [code] = await once(child, 'close'); +assert.equal(code, 0, errors); +assert.equal(output.trim(), 'ISOLATED'); +await rm(job, { recursive: true }); +const runtime = { + jobs: '/jobs', + guard: '/opt/extraction/guard', + python: '/usr/local/bin/python3.13', + worker: '/tests/isolation.py', +}; +const backend = new RestrictedBackend(runtime); +for (const abort of ['immediate', 'running']) { + const job = await backend.create(); + const signal = new AbortController(); + const pending = backend.run(job, 'hang', 'a'.repeat(64), signal.signal); + if (abort === 'running') + await new Promise(resolve => setTimeout(resolve, 100)); + signal.abort(new Error('CANCELLED')); + await assert.rejects(pending); + await backend.cleanup(job); +} +const job2 = await backend.create(); +await assert.rejects( + backend.run(job2, 'hang', 'a'.repeat(64), AbortSignal.timeout(100)) +); +await backend.cleanup(job2); +await rm('/jobs/other', { recursive: true }); +await rm('/socket/secret'); +assert.deepEqual(await readdir('/jobs'), []); +const missing = new RestrictedBackend({ ...runtime, guard: '/nonexistent' }); +const job3 = await missing.create(); +await assert.rejects( + missing.run(job3, 'pdf', 'a'.repeat(64), AbortSignal.timeout(1000)) +); +await missing.cleanup(job3); +console.log( + 'Container isolation: cross-job/credential paths, sockets, fork, signals, limits, startup/running cancellation, deadline and missing launcher passed' +); diff --git a/packages/extraction/tests/isolation.py b/packages/extraction/tests/isolation.py new file mode 100644 index 00000000..a50d0084 --- /dev/null +++ b/packages/extraction/tests/isolation.py @@ -0,0 +1,73 @@ +import ctypes +import errno +import fcntl +import os +import resource +import signal +import socket +import struct +import sys + +if sys.argv[1] == 'hang': + # Exercise supervisor cancellation/deadline termination, not a cooperative parser. + while True: + pass + +for path, mode in [('/jobs/other/secret', 'rb'), ('/jobs/other/secret', 'wb'), + ('/socket/secret', 'rb'), ('/proc/self/environ', 'rb'), ('/etc/passwd', 'rb')]: + try: + with open(path, mode) as file: + if mode == 'rb': + file.read() + except PermissionError: + pass + else: + raise RuntimeError('Cross-boundary file access') +for operation in [lambda: socket.socket(), lambda: socket.socket(socket.AF_UNIX), + lambda: os.fork(), lambda: os.kill(os.getppid(), 0), + lambda: os.chmod('/jobs/other/secret', 0o777), + lambda: fcntl.fcntl(1, fcntl.F_SETOWN, os.getppid()), + lambda: fcntl.fcntl(1, 15, struct.pack("ii", 1, os.getppid())), + lambda: fcntl.fcntl(1, fcntl.F_SETSIG, 0), + lambda: fcntl.ioctl(1, 0x8901, struct.pack("i", os.getppid()))]: + try: + operation() + except PermissionError: + pass + else: + raise RuntimeError('Forbidden syscall') +assert 'EXTRACTION_SECRET_CANARY' not in os.environ +assert resource.getrlimit(resource.RLIMIT_AS) == (512 * 1024 * 1024,) * 2 +assert resource.getrlimit(resource.RLIMIT_CPU) == (8, 8) +assert resource.getrlimit(resource.RLIMIT_FSIZE) == (4 * 1024 * 1024,) * 2 +assert resource.getrlimit(resource.RLIMIT_NOFILE) == (64, 64) + + +# Harmless signal-zero probes distinguish filtering from ordinary permissions. +class Sigval(ctypes.Union): + _fields_ = [("integer", ctypes.c_int), ("pointer", ctypes.c_void_p)] +libc = ctypes.CDLL(None, use_errno=True) +libc.sigqueue.argtypes = [ctypes.c_int, ctypes.c_int, Sigval] +assert libc.sigqueue(os.getppid(), 0, Sigval(integer=0)) == -1 +assert ctypes.get_errno() == errno.EPERM +number = {"x86_64": 297, "aarch64": 240}[os.uname().machine] +info = (ctypes.c_int * 32)() +info[2] = -1 # SI_QUEUE, with signal zero: never delivers a signal. +assert libc.syscall(number, os.getppid(), os.getppid(), 0, ctypes.byref(info)) == -1 +assert ctypes.get_errno() == errno.EPERM + + + +try: + fcntl.ioctl(1, 0, 0) +except PermissionError: + pass +else: + raise RuntimeError('Forbidden ioctl') + + + +libc.mq_open.argtypes = [ctypes.c_char_p, ctypes.c_int] +assert libc.mq_open(b"/extraction-negative-probe", os.O_RDONLY) == -1 +assert ctypes.get_errno() == errno.EPERM +print("ISOLATED", flush=True) diff --git a/packages/extraction/tsconfig.json b/packages/extraction/tsconfig.json new file mode 100644 index 00000000..aff02060 --- /dev/null +++ b/packages/extraction/tsconfig.json @@ -0,0 +1,15 @@ +{ + "compilerOptions": { + "target": "ES2023", + "lib": ["ES2024"], + "module": "NodeNext", + "moduleResolution": "NodeNext", + "strict": true, + "noUncheckedIndexedAccess": true, + "esModuleInterop": true, + "skipLibCheck": true, + "outDir": "dist", + "rootDir": "src" + }, + "include": ["src/**/*.ts"] +} diff --git a/packages/extraction/worker/extract.py b/packages/extraction/worker/extract.py new file mode 100644 index 00000000..20f0360f --- /dev/null +++ b/packages/extraction/worker/extract.py @@ -0,0 +1,503 @@ +"""Request-local parser. Only the supervisor can choose its mode and files.""" +import hashlib +import io +import json +import logging +import os +from pathlib import PurePosixPath +import socket +import struct +import sys +import xml.etree.ElementTree as ET +import zipfile +import zlib + +INPUT_BYTES = 10 * 1024 * 1024 +TEXT_BYTES = 1024 * 1024 +RESULT_BYTES = 3 * 1024 * 1024 +EXPANDED_BYTES = 32 * 1024 * 1024 +ENTRY_BYTES = 8 * 1024 * 1024 +SEGMENTS = 128 + + +class Rejected(Exception): + def __init__(self, code): + self.code = code + + +class Text: + def __init__(self): + self.size = 0 + self.parts = [] + + def add(self, value): + size = len(value.encode("utf-8", errors="strict")) + if self.size + size > TEXT_BYTES: + raise Rejected("OUTPUT_LIMIT") + self.size += size + self.parts.append(value) + + def value(self): + return "".join(self.parts) + + +def pdf_segments(data): + import pypdf + from pypdf import filters, _page + from pypdf.generic import DecodedStreamObject, DictionaryObject, NameObject, NullObject, StreamObject + from pypdf._cmap import charset_encoding + from pypdf._codecs.core_font_metrics import CORE_FONT_METRICS + from pypdf._text_extraction._text_extractor import TextExtraction + + used = 0 + failure = None + credited = {} + shown_bytes = 0 + + def reject(code): + nonlocal failure + failure = failure or code + raise Rejected(failure) + + def check_failure(): + if failure is not None: + raise Rejected(failure) + + class Warnings(logging.Handler): + def emit(self, record): + nonlocal failure + failure = failure or "INVALID_DOCUMENT" + + logger = logging.getLogger("pypdf") + logger.addHandler(Warnings()) + logger.setLevel(logging.WARNING) + logger.propagate = False + original = filters.decode_stream_data + original_data = DecodedStreamObject.get_data + original_show = TextExtraction._handle_tj_operation + original_resources = _page._get_page_resources + + def inflate(value): + nonlocal used + # Strict bounded decoding, without pypdf's recovery/fallback paths. + cap = min(ENTRY_BYTES, EXPANDED_BYTES - used) + decoder = zlib.decompressobj() + try: + result = decoder.decompress(value, cap + 1) + except zlib.error: + reject("INVALID_DOCUMENT") + if len(result) > cap or decoder.unconsumed_tail: + reject("DECOMPRESSION_LIMIT") + if not decoder.eof or decoder.unused_data: + reject("INVALID_DOCUMENT") + used += len(result) + credited[id(result)] = result + return result + + def decoded_data(stream): + nonlocal used + value = original_data(stream) + if not getattr(stream, "_extraction_counted", False): + if credited.get(id(value)) is not value: + used += len(value) + if len(value) > ENTRY_BYTES or used > EXPANDED_BYTES: + reject("DECOMPRESSION_LIMIT") + stream._extraction_counted = True + return value + + def decode(stream): + names = stream.get("/Filter", []) + if not isinstance(names, list): + names = [names] + if len(names) > 1 or any(str(name) not in { + "/FlateDecode", "/Fl", "/ASCIIHexDecode", "/AHx", "/ASCII85Decode", "/A85" + } for name in names): + reject("UNSUPPORTED_ENCODING") + try: + return original(stream) + except Rejected: + raise + except MemoryError: + reject("RESOURCE_LIMIT") + except Exception: + reject("INVALID_DOCUMENT") + + def resources_for_text(obj): + resources = original_resources(obj) + if resources: + fonts = resources.get("/Font") + if fonts is not None and not isinstance(fonts.get_object(), NullObject): + for ref in fonts.get_object().values(): + resource = ref.get_object() + for key in ("/Encoding", "/ToUnicode"): + if key in resource and isinstance(resource[key], NullObject): + del resource[key] + encoding = resource.get("/Encoding") + if encoding is not None: + encoding = encoding.get_object() + if not isinstance(encoding, (NameObject, DictionaryObject)): + reject("UNSUPPORTED_ENCODING") + if isinstance(encoding, DictionaryObject): + base = encoding.get("/BaseEncoding") + if base is not None and isinstance(base.get_object(), NullObject): + del encoding["/BaseEncoding"] + unicode_map = resource.get("/ToUnicode") + if unicode_map is not None and not isinstance(unicode_map.get_object(), StreamObject): + reject("UNSUPPORTED_ENCODING") + for state in resources.get("/ExtGState", DictionaryObject()).get_object().values(): + if "/Font" in state.get_object(): + reject("UNSUPPORTED_ENCODING") + return resources + # Do not let the parser skip text operators solely because resources + # are absent. A used missing font must reach the rejection below. + return DictionaryObject({NameObject("/Font"): DictionaryObject()}) + + def show_text(extractor, operands): + nonlocal shown_bytes + if operands and operands[0]: + font, resource = extractor.font, extractor.font_resource + if resource is None or not font.interpretable: + reject("UNSUPPORTED_ENCODING") + if font.sub_type not in {"Type1", "MMType1", "TrueType", "Type3", "Type0"}: + reject("UNSUPPORTED_ENCODING") + encoding = resource.get("/Encoding") + if encoding is not None: + encoding = encoding.get_object() + if isinstance(encoding, DictionaryObject): + encoding = encoding.get("/BaseEncoding") + if encoding is not None: + encoding = encoding.get_object() + standard_encoding = isinstance(encoding, NameObject) and encoding in charset_encoding + require_map = ("/ToUnicode" in resource or font.sub_type in {"Type3", "Type0"} + or isinstance(font.encoding, str) + or (not standard_encoding and font.name not in CORE_FONT_METRICS)) + value = operands[0] + if isinstance(value, bytes): + try: + if isinstance(font.encoding, str): + # Never take pypdf's charmap recovery on codec failure. + characters = value.decode(font.encoding, errors="surrogatepass") + else: + characters = (font.encoding[code] for code in value) + for character in characters: + if require_map and character not in font.character_map: + reject("UNSUPPORTED_ENCODING") + mapped = font.character_map.get(character, character) + # Unknown Adobe glyph names are preserved as NameObject. + if (not isinstance(mapped, str) or isinstance(mapped, NameObject) + or not mapped or "\ufffd" in mapped + or any(ord(char) < 32 and char not in "\t\n\r" for char in mapped)): + reject("UNSUPPORTED_ENCODING") + shown_bytes += len(mapped.encode("utf-8", errors="strict")) + if shown_bytes > TEXT_BYTES: + reject("OUTPUT_LIMIT") + except (LookupError, UnicodeError, TypeError): + reject("UNSUPPORTED_ENCODING") + elif not isinstance(value, str): + reject("UNSUPPORTED_ENCODING") + return original_show(extractor, operands) + + _page._get_page_resources = resources_for_text + TextExtraction._handle_tj_operation = show_text + DecodedStreamObject.get_data = decoded_data + filters.decompress = inflate + filters.decode_stream_data = decode + reader = pypdf.PdfReader(io.BytesIO(data), strict=True) + check_failure() + if reader.is_encrypted: + raise Rejected("ENCRYPTED_DOCUMENT") + if len(reader.pages) > SEGMENTS: + raise Rejected("STRUCTURE_LIMIT") + result = [] + text = Text() + for index, page in enumerate(reader.pages): + value = page.extract_text() or "" + check_failure() + text.add(value) + result.append({"kind": "page", "index": index + 1, "text": value}) + check_failure() + return result, text.size + + +def validate_xml(value): + xml = value.decode("utf-8", errors="strict") + if "\x00" in xml or " 512: + raise Rejected("STRUCTURE_LIMIT") + for entry in archive.infolist(): + name = entry.filename + if entry.orig_filename != name: + raise Rejected("INVALID_DOCUMENT") + path = PurePosixPath(name) + if (name in names or path.is_absolute() or ".." in path.parts or "\\" in name + or "\x00" in name or (entry.external_attr >> 16) & 0o170000 == 0o120000 + or entry.flag_bits & 1 or entry.compress_type not in (0, 8)): + raise Rejected("INVALID_DOCUMENT") + names.add(name) + if entry.file_size > ENTRY_BYTES: + raise Rejected("DECOMPRESSION_LIMIT") + # Let ZipFile validate local names, flags and entry overlap, but do + # not use ZipExtFile.read(): it trims data to untrusted file_size. + with archive.open(entry): + offset = entry.header_offset + if offset < 0 or offset + 30 > len(data): + raise Rejected("INVALID_DOCUMENT") + method = struct.unpack_from(" len(data): + raise Rejected("INVALID_DOCUMENT") + decoder = zlib.decompressobj(-zlib.MAX_WBITS) if entry.compress_type == 8 else None + chunks = [] + size = 0 + crc = 0 + for position in range(start, end, 64 * 1024): + chunk = data[position:min(position + 64 * 1024, end)] + if decoder: + cap = min(ENTRY_BYTES - size, EXPANDED_BYTES - total) + chunk = decoder.decompress(chunk, cap + 1) + size += len(chunk) + total += len(chunk) + if size > ENTRY_BYTES or total > EXPANDED_BYTES: + raise Rejected("DECOMPRESSION_LIMIT") + crc = zlib.crc32(chunk, crc) + chunks.append(chunk) + if decoder and (not decoder.eof or decoder.unused_data or decoder.unconsumed_tail): + raise Rejected("INVALID_DOCUMENT") + if size != entry.file_size or crc != entry.CRC: + raise Rejected("INVALID_DOCUMENT") + value = b"".join(chunks) + parts[name] = value + xml_parts(parts) + output = io.BytesIO() + with zipfile.ZipFile(output, "w") as clean: + for name, value in parts.items(): + # The document parser sees only measured, validated parts. + clean.writestr(name, value, compress_type=zipfile.ZIP_STORED) + return output.getvalue() + + +def validate_docx_body(body): + from lxml.etree import _Element + + literal = _Element.text.__get__ + word = "{http://schemas.openxmlformats.org/wordprocessingml/2006/main}" + # These are the exact content paths read by pinned python-docx. Unknown + # wrappers fail closed rather than hiding text from its direct-child XPath. + grammar = { + "body": {"p", "tbl", "sectPr"}, + "tbl": {"tblPr", "tblGrid", "tr"}, + "tr": {"trPr", "tc"}, + "tc": {"tcPr", "p", "tbl"}, + "p": {"pPr", "r", "hyperlink"}, + "hyperlink": {"r"}, + "r": {"rPr", "t", "tab", "br", "cr", "noBreakHyphen", "ptab", "lastRenderedPageBreak"}, + } + properties = {"sectPr", "tblPr", "tblGrid", "trPr", "tcPr", "pPr", "rPr"} + markers = {"bookmarkStart", "bookmarkEnd", "commentRangeStart", "commentRangeEnd", + "permStart", "permEnd", "proofErr"} + stack = [(body, "body")] + while stack: + parent, kind = stack.pop() + if literal(parent) and literal(parent).strip(): + raise Rejected("UNSUPPORTED_CONTENT") + for child in parent: + tag = child.tag.removeprefix(word) if isinstance(child.tag, str) else "" + marker = kind in {"body", "p", "hyperlink", "tc"} and tag in markers + if child.tag != word + tag or (tag not in grammar[kind] and not marker): + raise Rejected("UNSUPPORTED_CONTENT") + if child.tail and child.tail.strip(): + raise Rejected("UNSUPPORTED_CONTENT") + if tag in properties: + # Formatting is ignored, but cannot conceal textual content. + if any(node.tag in {word + "t", word + "instrText", word + "delText"} + or (literal(node) and literal(node).strip()) + or (node.tail and node.tail.strip()) for node in child.iter()): + raise Rejected("UNSUPPORTED_CONTENT") + elif tag in grammar: + stack.append((child, tag)) + elif len(child) or (tag != "t" and literal(child) and literal(child).strip()): + raise Rejected("UNSUPPORTED_CONTENT") + + +def docx_segments(data): + from docx import Document + from docx.table import Table, _Cell + + document = Document(io.BytesIO(safe_docx(data))) + validate_docx_body(document.element.body) + text = Text() + def blocks(container, depth=0, cell=False): + if depth > 32: + raise Rejected("STRUCTURE_LIMIT") + first = True + for block in container.iter_inner_content(): + if cell and not first: + text.add("\n") + first = False + if isinstance(block, Table): + width = len(block._tbl.tblGrid.gridCol_lst) + if width < 1 or width > 512: + raise Rejected("STRUCTURE_LIMIT") + previous = set() + for row_index, row in enumerate(block.rows): + if row_index: + text.add("\n") + before, after = row.grid_cols_before, row.grid_cols_after + if before < 0 or after < 0 or before + after >= width: + raise Rejected("INVALID_DOCUMENT") + column = before + text.add("\t" * before) + current = set() + for index, tc in enumerate(row._tr.tc_lst): + span, merge = tc.grid_span, tc.vMerge + if span < 1 or column + span > width - after: + raise Rejected("INVALID_DOCUMENT") + if tc.tcPr is not None and tc.tcPr.find( + "{http://schemas.openxmlformats.org/wordprocessingml/2006/main}hMerge") is not None: + raise Rejected("UNSUPPORTED_CONTENT") + if index: + text.add("\t") + position = (column, span) + if merge == "continue": + if position not in previous: + raise Rejected("INVALID_DOCUMENT") + # Continuations occupy columns, not another copy of + # the originating cell. Hidden content is rejected. + if tc.xpath(".//w:t | .//w:tbl | .//w:tab | .//w:br | .//w:cr | .//w:noBreakHyphen | .//w:ptab"): + raise Rejected("UNSUPPORTED_CONTENT") + else: + blocks(_Cell(tc, block), depth + 1, cell=True) + if merge is not None: + if merge not in {"restart", "continue"}: + raise Rejected("INVALID_DOCUMENT") + current.add(position) + text.add("\t" * (span - 1)) + column += span + if column + after != width: + raise Rejected("INVALID_DOCUMENT") + text.add("\t" * after) + previous = current + if not cell: + text.add("\n") + else: + text.add(block.text) + if not cell: + text.add("\n") + + blocks(document) + return [{"kind": "document", "index": 1, "text": text.value()}], text.size + + +def probe(): + import pypdf + import docx + import lxml.etree + + for path in ("/socket", "/jobs", "/proc/self/environ", "/etc/passwd"): + try: + if path in ("/socket", "/jobs"): + os.listdir(path) + else: + open(path, "rb").read(1) + except PermissionError: + continue + raise Rejected("ISOLATION_UNAVAILABLE") + try: + socket.socket(socket.AF_INET, socket.SOCK_STREAM) + except PermissionError: + pass + else: + raise Rejected("ISOLATION_UNAVAILABLE") + try: + os.fork() + except PermissionError: + pass + else: + raise Rejected("ISOLATION_UNAVAILABLE") + if "EXTRACTION_SECRET_CANARY" in os.environ: + raise Rejected("ISOLATION_UNAVAILABLE") + print("READY", flush=True) + + +def main(): + if sys.argv[1] == "probe": + probe() + return + data = open("input", "rb").read(INPUT_BYTES + 1) + if not data or len(data) > INPUT_BYTES: + raise Rejected("INPUT_LIMIT") + mode = sys.argv[1] + if mode == "pdf": + if not data.startswith(b"%PDF-"): + raise Rejected("INVALID_DOCUMENT") + segments, size = pdf_segments(data) + elif mode == "docx": + if not data.startswith(b"PK\x03\x04"): + raise Rejected("INVALID_DOCUMENT") + segments, size = docx_segments(data) + else: + raise Rejected("UNSUPPORTED_FORMAT") + if not any(item["text"].strip() for item in segments): + raise Rejected("EMPTY_OUTPUT") + result = json.dumps({"version": 1, "operation": "document.extract-text", "format": mode, + "inputSha256": hashlib.sha256(data).hexdigest(), "textBytes": size, + "segments": segments}, ensure_ascii=False, separators=(",", ":")).encode("utf-8") + if len(result) > RESULT_BYTES: + raise Rejected("OUTPUT_LIMIT") + with open("result", "xb") as target: + target.write(result) + + +if __name__ == "__main__": + try: + main() + except Rejected as error: + print(error.code, flush=True) + sys.exit(2) + except MemoryError: + print("RESOURCE_LIMIT", flush=True) + sys.exit(2) + except Exception: + print("INVALID_DOCUMENT", flush=True) + sys.exit(2) diff --git a/packages/extraction/worker/requirements.txt b/packages/extraction/worker/requirements.txt new file mode 100644 index 00000000..25ad0ede --- /dev/null +++ b/packages/extraction/worker/requirements.txt @@ -0,0 +1,4 @@ +pypdf==6.19.0 +python-docx==1.2.0 +lxml==6.0.2 +typing_extensions==4.15.0