diff --git a/docs/CLOUD-SYNC.md b/docs/CLOUD-SYNC.md new file mode 100644 index 000000000..a1588ae8b --- /dev/null +++ b/docs/CLOUD-SYNC.md @@ -0,0 +1,374 @@ +# 云同步实现说明 + +本文是 ScriptCat 云同步的维护文档,描述当前分支上的实际实现。它面向需要修改或 review 同步逻辑的维护者,重点解释同步操作、状态文件、provider 差异、错误分类和生产数据兼容要求。 + +相关代码入口: + +- [`src/app/service/service_worker/synchronize.ts`](../src/app/service/service_worker/synchronize.ts):同步服务、队列、状态合并、digest 更新。 +- [`packages/filesystem/filesystem.ts`](../packages/filesystem/filesystem.ts):统一文件系统接口、条件操作参数、provider capability。 +- [`packages/filesystem/error.ts`](../packages/filesystem/error.ts):统一 typed provider error。 +- [`packages/filesystem/*`](../packages/filesystem/):各云盘 provider 实现。 + +## 维护目标 + +云同步的第一目标不是强事务,而是在浏览器扩展和多 provider 限制下做到“尽量正确且不破坏旧数据”。 + +必须保持的不变量: + +1. 单个脚本失败不能阻塞其他脚本同步。 +2. 成功脚本可以推进自己的 `file_digest`,失败脚本必须保留旧 digest。 +3. `scriptcat-sync.json` 写回前要合并远端较新状态,避免覆盖其他设备状态。 +4. provider 没有声明原子能力时,同步层不能传条件写删参数,也不能把 best-effort 行为描述成 CAS。 +5. 旧 `.user.js`、旧 `.meta.json`、旧 `file_digest` string map、缺字段 `scriptcat-sync.json` 必须继续可读。 +6. filesystem 包只负责暴露能力、执行文件操作、抛 typed error;同步冲突策略属于 `SynchronizeService`。 + +## 同步目录和文件 + +同步目录由云同步配置决定,业务上使用 `ScriptCat/sync` 作为脚本同步目录。同步目录中主要有四类文件。 + +### `.user.js` + +脚本源码文件。同步层用文件名中的 `uuid` 和 `FileInfo.digest` 识别脚本及远端内容状态。 + +### `.meta.json` + +脚本元信息文件。当前读取时只要求兼容以下字段: + +```ts +type SyncMeta = { + uuid: string; + origin?: string; + downloadUrl?: string; + checkUpdateUrl?: string; + isDeleted?: boolean; +}; +``` + +`origin`、`downloadUrl`、`checkUpdateUrl` 是安装或更新时的辅助信息。新增字段必须保持 optional,读取旧文件时不能要求存在。 + +### tombstone 删除标记 + +当用户启用同步删除时,删除云端脚本不会简单移除所有文件,而是删除 `.user.js` 并写入 `.meta.json`: + +```json +{ + "uuid": "", + "isDeleted": true +} +``` + +其他设备看到“只有 `.meta.json` 且 `isDeleted: true`”时,会删除本地脚本。当前没有单独的 `tombstone_digest`,也没有 tombstone GC 机制;不要在没有生命周期设计前新增这类状态。 + +### `scriptcat-sync.json` + +保存脚本启用状态、排序和更新时间。当前结构: + +```ts +type ScriptcatSync = { + version: string; + status: { + scripts: { + [uuid: string]: { + enable: boolean; + sort: number; + updatetime: number; + } | undefined; + }; + }; +}; +``` + +兼容要求: + +- 文件可能不存在。 +- 文件可能缺少 `status` 或 `status.scripts`。 +- 文件可能损坏或无法 JSON parse。 +- 写回时必须尽量保留远端较新状态,尤其是本轮失败脚本和 orphan 脚本的状态。 + +## 本地 `file_digest` + +`file_digest` 存在 `ChromeStorage("sync")` 中,用于记录上一次确认同步成功的云端文件 digest。 + +当前格式: + +```ts +type FileDigestMap = { + [filename: string]: string; +}; +``` + +注意事项: + +- digest 是 provider 返回的 opaque token,不一定是 md5。 +- WebDAV、S3、OneDrive 使用 ETag 风格 digest。 +- Dropbox 使用 `content_hash`。 +- Google Drive、Baidu 接近 md5。 +- Zip 可能为空。 +- 不能用本地 md5 覆盖 provider 已返回的原生 digest。 +- 文件操作失败时,对应文件名必须保留旧 digest,不能写入“看起来成功”的新值。 + +`updateFileDigest()` 会重新 `fs.list()` 构造新 map;对于刚 push 但 provider list 暂时不可见的文件,才使用 `pushScript()` 返回的本地 md5 作为兜底。 + +## 同步入口和队列 + +`SynchronizeService` 使用 `SYNC_SERVICE_TASK_KEY = "cloud_sync_queue"` 串行化同步任务。以下入口都会进入同一队列: + +- 配置启用后触发的 `syncOnce()`。 +- 定时同步,Chrome alarm 名称为 `cloudSync`,周期为 60 分钟。 +- 非 sync 来源安装脚本后的 `scriptInstall()`。 +- 非 sync 来源删除脚本后的 `scriptsDelete()`。 + +串行队列很重要:安装、删除和定时同步都可能写同一批云端文件,如果并发执行,会扩大覆盖和 digest 污染风险。 + +## `syncOnceInternal()` 流程 + +`syncOnceInternal(syncConfig, fs)` 是主同步流程: + +1. 调用 `fs.list()` 获取云端目录。 +2. 按文件名把 `.user.js` 和 `.meta.json` 组装成 `uuidMap`。 +3. 读取本地脚本列表,生成 `scriptMap`。 +4. 尝试读取 `scriptcat-sync.json`,失败时允许脚本同步继续,但本轮跳过 status 写回。 +5. 对每个云端 uuid 和本地脚本做决策。 +6. 用 `Promise.allSettled()` 等待所有文件任务,保持 per-file best-effort。 +7. 收集成功任务返回的 digest patch。 +8. 对失败任务记录 `failedSyncUuids` 和 `preserveDigestFiles`。 +9. 如果启用 `syncStatus`,合并本地状态、初始云端状态、写回前重新读取的最新云端状态。 +10. 调用 `updateFileDigest()`,成功文件推进 digest,失败文件保留旧 digest。 + +### 决策规则 + +本地脚本存在、云端脚本也存在: + +- 如果 `.user.js` digest 与 `file_digest` 一致,跳过。 +- 如果本地脚本更新时间更晚,push 本地脚本。 +- 如果云端更新时间更晚,pull 云端脚本。 +- 如果云端缺 `.meta.json`,push 本地脚本补齐 meta。 + +本地脚本存在、云端只有 `.meta.json`: + +- 如果 meta 是 tombstone,本地删除脚本。 +- 如果 meta 不是 tombstone,删除无效 meta 并重新 push 本地脚本。 + +本地脚本不存在、云端 `.user.js` 和 `.meta.json` 都存在: + +- pull 并安装云端脚本。 + +本地脚本不存在、云端只有 `.user.js`: + +- 视为 orphan cloud script,跳过。 +- 不删除、不覆盖、不清空对应远端 status。 + +遍历结束后,剩余只存在于本地的脚本会 push 到云端。 + +## push / pull / delete + +### `pushScript()` + +`pushScript()` 写两个文件: + +- `.user.js` +- `.meta.json` + +`modifiedDate` 使用 `script.updatetime || script.createtime || Date.now()`。 + +写入参数由 `buildPushCreateOptions()` 决定: + +- 云端文件不存在且 provider 支持 `supportsCreateOnly`:传 `createOnly: true`。 +- 云端文件存在且 provider 支持 `supportsAtomicCompareAndSwap`,并且 `file_digest` 中有旧 digest:传 `expectedDigest`。 +- provider 未声明能力时,只传 `modifiedDate`,保持旧覆盖语义。 + +`pushScript()` 成功后返回本地计算的 md5 patch,仅用于 provider list 暂时看不到刚上传文件时兜底。它不能覆盖 provider 已返回的原生 digest。 + +### `pullScript()` + +`pullScript()` 会读取源码和 meta: + +1. `fs.open(file.script).read("string")`。 +2. `fs.open(file.meta).read("string")`。 +3. `JSON.parse(meta)`。 +4. `prepareScriptByCode()` 解析脚本。 +5. 根据 `scriptcat-sync.json` status 调整 enable/sort。 +6. `script.installScript({ upsertBy: "sync" })` 写入本地。 + +真实失败会向上抛出,由 `syncOnceInternal()` 作为单文件失败处理。不要在 `pullScript()` 内吞掉错误,否则会重新引入 digest 污染。 + +### `deleteCloudScript()` + +删除云端脚本时: + +- 先删除 `.user.js`。 +- 如果 `syncDelete` 为 true,写 tombstone meta。 +- 如果 `syncDelete` 为 false,删除 `.meta.json`。 + +删除参数由 `buildDeleteOptions()` 决定: + +- provider 支持 `supportsConditionalDelete` 且 `file_digest` 有旧 digest 时,传 `expectedDigest`。 +- 否则走普通 delete。 + +失败会向上抛出。`scriptsDelete()` 必须逐条 catch,保证批量删除中一个 uuid 失败不影响后续 uuid。 + +## status 合并 + +`scriptcat-sync.json` 是 best-effort 状态同步,不是强事务。合并时遵守以下规则: + +1. 本轮文件同步失败的 uuid 保留云端原 status。 +2. 本轮刚 pull 的脚本保留云端 status,避免刚按云端更新后又写回本地旧状态。 +3. 本地状态更新时间更新时,候选写回本地 status。 +4. 云端状态更新时,应用云端 enable/sort 到本地。 +5. orphan uuid 的云端 status 保留。 +6. 写回前重新读取最新 `scriptcat-sync.json`,再用 `mergeScriptcatSyncStatus()` 合并,减少覆盖其他设备更新的概率。 + +如果初始读取 `scriptcat-sync.json` 失败,本轮不会写回 status 文件。 + +## 错误分类 + +provider 应尽量抛 `FileSystemError`。同步层用 `classifySyncError()` 映射: + +| 条件 | `SyncErrorKind` | 语义 | +| --- | --- | --- | +| `FileSystemError.conflict` | `conflict` | 条件写删失败或 provider 冲突 | +| `FileSystemError.rateLimit` 或 `retryable` | `transient` | 429、5xx、可重试错误 | +| `FileSystemError.notFound` | `stale_snapshot` | list 到操作之间远端消失或缓存过期 | +| `FileSystemError.auth` 或 `WarpTokenError` | `fatal` | 授权失败 | +| error message 包含 `unsupported` | `unsupported` | provider 不支持 | +| 其他 | `fatal` | 未分类错误 | + +错误分类主要用于日志、保留 digest、后续 retry 策略和 review 判断。它不是用户可见错误协议。 + +## retry 策略 + +`LimiterFileSystem` 对不同操作使用不同重试策略: + +- 会重试:`verify`、`open`、`read`、`openDir`、`list`、`getDirUrl`。 +- 会重试:带 `expectedDigest` 或 `createOnly` 的 `writer.write()`。 +- 会重试:带 `expectedDigest` 的 `delete()`。 +- 不重试:普通 `create`、`createDir`、普通 `writer.write()`、普通 `delete()`。 + +原因:没有条件保护的写入和删除不是安全幂等操作。重复执行可能创建重复文件、覆盖并发更新或误删。 + +## provider 差异 + +| Provider | digest 来源 | atomic CAS | create-only | conditional delete | 关键实现 | +| --- | --- | --- | --- | --- | --- | +| WebDAV | `etag` | 支持 | 支持 | 支持 | 写入用 `If-Match`;create-only 用 `overwrite: false`;删除用 `If-Match` | +| S3 | `ETag` 去引号 | 支持 | 支持 | 支持 | 写入用 `if-match`;create-only 用 `if-none-match: *`;删除用 `if-match` | +| OneDrive | `eTag` | 支持 | 支持 | 支持 | simple upload / upload session 带 `If-Match` 或 `If-None-Match`;删除带 `If-Match` | +| Google Drive | `md5Checksum` | 不支持 | 不支持 | 不支持 | 先按路径查 fileId,再 PATCH 或 POST;path cache 可能 stale | +| Dropbox | `content_hash` | 不支持 | 不支持 | 不支持 | 先 `exists()`,存在 overwrite,不存在 add;未暴露 rev CAS | +| Baidu | `md5` | 不支持 | 不支持 | 不支持 | precreate/upload/create,`rtype=3` 覆盖;HTTP 429/5xx typed | +| Zip | 空或 zip metadata | 不支持 | 不支持 | 不支持 | 备份用途,不参与云端 CAS | + +### WebDAV + +WebDAV 是原生条件能力 provider。维护时注意: + +- `capabilities` 三项均为 true。 +- ETag 写入和删除时需要保持引号,`quoteETag()` 会补齐。 +- create-only 依赖 `putFileContents(..., { overwrite: false })`。 +- `putFileContents()` 返回 false 时,在 create-only 场景转 typed conflict。 +- 删除 404 视为幂等成功。 + +### S3 + +S3 是原生条件能力 provider。维护时注意: + +- `capabilities` 三项均为 true。 +- list 返回的 `ETag` 会去掉引号作为 digest。 +- 写入 `expectedDigest` 时发送 `if-match`。 +- create-only 发送 `if-none-match: *`。 +- 删除 `expectedDigest` 时发送 `if-match`。 +- `PreconditionFailed` / 412 转 typed conflict。 +- `NoSuchKey` 删除视为成功。 + +### OneDrive + +OneDrive 是原生条件能力 provider。维护时注意: + +- `capabilities` 三项均为 true。 +- list 使用 `eTag` 作为 digest。 +- 空内容走 simple PUT,非空内容走 upload session。 +- 条件写会把 `If-Match` / `If-None-Match` 放到 simple PUT 或 upload session 创建请求。 +- upload session URL 不带 bearer token,request 层保留这个特殊路径。 +- read/delete 使用 raw `Response` 路径,需要手动转 typed error。 + +### Google Drive + +Google Drive 当前不声明 atomic 能力。维护时注意: + +- digest 来自 `md5Checksum`。 +- 目录和文件通过 appDataFolder + path cache 查 fileId。 +- 写入是“先查同名文件,再 PATCH 或 POST”,不是 atomic CAS。 +- 删除是“先查 fileId,再 DELETE”,不是 conditional delete。 +- reader path lookup miss 已转 typed notFound。 +- path cache stale 时 writer/list 会清缓存并重试一次。 +- 不要把 `fileId` 或 `fileId:version` 包装成通用 version 语义。 + +### Dropbox + +Dropbox 当前不声明 atomic 能力。维护时注意: + +- digest 来自 `content_hash`,必须当作 opaque provider digest。 +- 写入是 `exists()` 后 overwrite 或 add,存在 TOCTOU。 +- 当前没有使用 Dropbox rev CAS。 +- request 层已解析 `error_summary` 和 structured `path_lookup` / `path`。 +- raw download 429 会转 typed rateLimit。 +- 删除 not_found 视为幂等成功。 + +### Baidu + +Baidu 当前不声明 atomic 能力。维护时注意: + +- digest 来自 `md5`。 +- 写入流程是 precreate、upload、create,`rtype=3` 覆盖。 +- 只把明确 file-exists errno 判为 conflict。 +- HTTP 429 转 typed rateLimit。 +- HTTP 5xx 转 typed retryable。 +- `filemetas` errno 或空列表转 typed notFound。 +- request 显式 `credentials: "omit"`,不要重新依赖全局 DNR 规则。 +- Baidu 没有被声明为 create-only 或 CAS provider。 + +### Zip + +ZipFileSystem 主要服务备份/导出,不应接入云同步 CAS 语义。它不声明 capabilities。 + +## 生产兼容要求 + +改同步逻辑前必须检查: + +1. 旧云端目录只有 `.user.js` 和 `.meta.json` 时能否继续同步。 +2. 旧 `file_digest` 只有 string digest 时能否继续比较。 +3. 旧 `scriptcat-sync.json` 缺字段时是否会崩溃。 +4. 损坏 `scriptcat-sync.json` 是否会被本轮覆盖。 +5. orphan `.user.js` 是否仍被跳过而不是删除。 +6. 单文件失败是否只保留该文件旧 digest。 +7. provider 原生 digest 是否被本地 md5 覆盖。 +8. 非 atomic provider 是否被错误声明 capability。 + +## 不要做的事 + +- 不要把整个 sync round 改成 all-or-nothing。 +- 不要在 `pullScript()` 或 `deleteCloudScript()` 内 catch 后吞掉真实失败。 +- 不要让失败文件推进 digest。 +- 不要在无法读取远端 `scriptcat-sync.json` 时覆盖写回。 +- 不要把 Google Drive / Baidu 的 preflight 当 atomic CAS。 +- 不要把 Dropbox `content_hash` 当 rev。 +- 不要新增 `tombstone_digest`,除非同时定义 GC 和兼容策略。 +- 不要对普通无条件写入开启 transient retry。 + +## 测试重点 + +同步层测试在 [`src/app/service/service_worker/synchronize.test.ts`](../src/app/service/service_worker/synchronize.test.ts)。provider 测试在各自 `packages/filesystem/*/*.test.ts`。 + +修改同步逻辑时至少考虑以下场景: + +1. 多个文件中一个 push/pull/delete 失败,其他文件继续同步。 +2. 失败文件 digest 保留,成功文件 digest 推进。 +3. `scriptcat-sync.json` 写回失败不污染文件 digest。 +4. 损坏或旧格式 `scriptcat-sync.json` 不阻塞脚本同步。 +5. orphan `.user.js` 跳过并保留 status。 +6. provider conflict/transient/notFound 能映射到正确 `SyncErrorKind`。 +7. 条件写删只在 provider 声明能力时使用。 +8. Google Drive / Dropbox / Baidu 明确保持非 atomic capability。 + +真实 provider 验证仍需要账号和夹具。不能把 unit test 或 mock response 结果宣称为真实云端验证。 + diff --git a/docs/DOC-MAINTENANCE.md b/docs/DOC-MAINTENANCE.md index c1f434e15..0ba399b9d 100644 --- a/docs/DOC-MAINTENANCE.md +++ b/docs/DOC-MAINTENANCE.md @@ -29,6 +29,7 @@ Aspirational / feature-branch content belongs in that branch's docs, or is clear | [`DEVELOP.md`](./DEVELOP.md) | The concrete "how": commands, structure, style, testing, i18n, commit/PR. | | [`VERIFICATION.md`](./VERIFICATION.md) | Lightweight end-to-end functional verification — throwaway scratch scripts driving the real built extension. | | [`ARCHITECTURE.md`](./ARCHITECTURE.md) | Deep internals: process model, message passing, service/data layers, GM API, execution, build. | +| [`CLOUD-SYNC.md`](./CLOUD-SYNC.md) | Cloud sync internals: sync files, digest/status semantics, provider differences, error classification, retry policy. | | [`translation/README.md`](./translation/README.md) | Translation / localization single source of truth. | | [`DOC-MAINTENANCE.md`](./DOC-MAINTENANCE.md) | This guide: doc-set organization rules + fact-check / anti-drift discipline. | | [`README.md`](./README.md) | The index that points to all of the above. | @@ -97,7 +98,7 @@ git ls-files eslint-rules/; git grep -l "require-last-error-check" -- eslint.con Link integrity — confirm every relative markdown link in the core docs resolves: ```bash -for doc in AGENTS.md docs/README.md docs/DEVELOP.md docs/VERIFICATION.md docs/ARCHITECTURE.md docs/DOC-MAINTENANCE.md docs/translation/README.md; do +for doc in AGENTS.md docs/README.md docs/DEVELOP.md docs/VERIFICATION.md docs/ARCHITECTURE.md docs/CLOUD-SYNC.md docs/DOC-MAINTENANCE.md docs/translation/README.md; do grep -oE '\]\(([^)]+)\)' "$doc" | sed -E 's/^\]\(|\)$//g' | grep -vE '^https?:|^#' | while read -r link; do target="$(dirname "$doc")/${link%%#*}" [ -e "$target" ] && echo "ok $doc → $link" || echo "BROKEN $doc → $link" diff --git a/docs/README.md b/docs/README.md index 653ea9e25..02d25d449 100644 --- a/docs/README.md +++ b/docs/README.md @@ -10,6 +10,7 @@ | [`DEVELOP.md`](./DEVELOP.md) | 开发规范:命令、目录结构、编码风格、UI/主题、测试机制、i18n、提交/PR 流程。**写代码前先读。** | | [`VERIFICATION.md`](./VERIFICATION.md) | 功能验证指南:用一次性 scratch 脚本驱动真实扩展做端到端验证(不跑全量 E2E、不加永久用例)。**验证改动是否真正跑通时读。** | | [`ARCHITECTURE.md`](./ARCHITECTURE.md) | 内部原理深入:多进程模型、消息传递、服务/数据层、GM API、脚本执行、构建管线。 | +| [`CLOUD-SYNC.md`](./CLOUD-SYNC.md) | 云同步实现说明:同步文件语义、主流程、状态合并、provider 差异、错误分类、retry 策略和维护注意事项。 | | [`DOC-MAINTENANCE.md`](./DOC-MAINTENANCE.md) | 文档维护与事实核对指南:组织规则、逐条核对清单、一键校验脚本。**改/审文档前先读。** | ## 翻译 / Translation diff --git a/packages/filesystem/baidu/baidu.test.ts b/packages/filesystem/baidu/baidu.test.ts index 033aec0fa..af36b4cf9 100644 --- a/packages/filesystem/baidu/baidu.test.ts +++ b/packages/filesystem/baidu/baidu.test.ts @@ -1,21 +1,42 @@ import { describe, expect, it, vi, afterEach } from "vitest"; +import { initTestEnv } from "@Tests/utils"; +import { isNotFoundError, isRateLimitError } from "../error"; +import { getFileSystemCapabilities } from "../filesystem"; import BaiduFileSystem from "./baidu"; +initTestEnv(); + describe("BaiduFileSystem", () => { afterEach(() => { vi.unstubAllGlobals(); vi.restoreAllMocks(); }); + it("不应声明原子条件写入能力", () => { + const fs = new BaiduFileSystem("/apps", "token"); + + expect(getFileSystemCapabilities(fs)).toEqual({ + supportsAtomicCompareAndSwap: false, + supportsCreateOnly: false, + supportsConditionalDelete: false, + }); + }); + it("request should omit credentials without using global DNR rules", async () => { const fetchMock = vi.fn().mockResolvedValue({ + ok: true, + status: 200, json: async () => ({ errno: 0 }), }); vi.stubGlobal("fetch", fetchMock); // 监视 updateDynamicRules,确保不再依赖全局 DNR 规则 const updateDynamicRulesMock = vi.fn(); - (chrome as any).declarativeNetRequest.updateDynamicRules = updateDynamicRulesMock; + vi.stubGlobal("chrome", { + declarativeNetRequest: { + updateDynamicRules: updateDynamicRulesMock, + }, + }); const fs = new BaiduFileSystem("/apps", "token"); @@ -33,6 +54,38 @@ describe("BaiduFileSystem", () => { expect(updateDynamicRulesMock).not.toHaveBeenCalled(); }); + it("request 遇到 HTTP 429 时应抛出 typed rate-limit 错误", async () => { + const fetchMock = vi.fn().mockResolvedValue({ + ok: false, + status: 429, + statusText: "Too Many Requests", + json: async () => ({ errno: 0, errmsg: "rate limited" }), + }); + vi.stubGlobal("fetch", fetchMock); + const fs = new BaiduFileSystem("/apps", "token"); + + await expect(fs.request("https://pan.baidu.com/rest/2.0/xpan/file?method=list")).rejects.toSatisfy( + isRateLimitError + ); + }); + + it("request 遇到 HTTP 5xx 时应抛出 typed retryable 错误", async () => { + const fetchMock = vi.fn().mockResolvedValue({ + ok: false, + status: 503, + statusText: "Service Unavailable", + json: async () => ({ errno: 0, errmsg: "service unavailable" }), + }); + vi.stubGlobal("fetch", fetchMock); + const fs = new BaiduFileSystem("/apps", "token"); + + await expect(fs.request("https://pan.baidu.com/rest/2.0/xpan/file?method=list")).rejects.toMatchObject({ + provider: "baidu", + status: 503, + retryable: true, + }); + }); + it("create should normalize double slashes in paths", async () => { const fs = new BaiduFileSystem("/apps//ScriptCat", "token"); @@ -52,4 +105,54 @@ describe("BaiduFileSystem", () => { `async=0&filelist=${encodeURIComponent(JSON.stringify(["/apps/ScriptCat/dir/file.user.js"]))}` ); }); + + it("创建目录遇到明确已存在 errno 时才标记为冲突", async () => { + const fs = new BaiduFileSystem("/apps", "token"); + vi.spyOn(fs, "request").mockResolvedValue({ errno: 31061, errmsg: "file already exists" }); + + await expect(fs.createDir("ScriptCat")).rejects.toMatchObject({ + provider: "baidu", + code: "31061", + conflict: true, + }); + }); + + it("创建目录遇到普通 errno 时不能误标记为冲突", async () => { + const fs = new BaiduFileSystem("/apps", "token"); + vi.spyOn(fs, "request").mockResolvedValue({ errno: 2, errmsg: "access denied" }); + + await expect(fs.createDir("ScriptCat")).rejects.toMatchObject({ + provider: "baidu", + code: "2", + conflict: false, + }); + }); + + it("写入预创建失败时保留普通 errno 的非冲突语义", async () => { + const fs = new BaiduFileSystem("/apps", "token"); + vi.spyOn(fs, "request").mockResolvedValue({ errno: 2, errmsg: "access denied" }); + const writer = await fs.create("dir/file.user.js"); + + await expect(writer.write("code")).rejects.toMatchObject({ + provider: "baidu", + code: "2", + conflict: false, + }); + }); + + it("读取文件元数据缺失时应抛出 typed not found 错误", async () => { + const fs = new BaiduFileSystem("/apps", "token"); + vi.spyOn(fs, "request").mockResolvedValue({ errno: -9, errmsg: "file not found" }); + const reader = await fs.open({ + fsid: 123, + name: "missing.user.js", + path: "/apps", + size: 0, + digest: "", + createtime: 0, + updatetime: 0, + }); + + await expect(reader.read("string")).rejects.toSatisfy(isNotFoundError); + }); }); diff --git a/packages/filesystem/baidu/baidu.ts b/packages/filesystem/baidu/baidu.ts index 9184de2f0..4c3ddd8be 100644 --- a/packages/filesystem/baidu/baidu.ts +++ b/packages/filesystem/baidu/baidu.ts @@ -2,6 +2,7 @@ import { AuthVerify } from "../auth"; import type FileSystem from "../filesystem"; import type { FileInfo, FileCreateOptions, FileReader, FileWriter } from "../filesystem"; import { joinPath } from "../utils"; +import { createBaiduFileSystemError } from "./error"; import { BaiduFileReader, BaiduFileWriter } from "./rw"; export default class BaiduFileSystem implements FileSystem { @@ -52,7 +53,7 @@ export default class BaiduFileSystem implements FileSystem { } ); if (data.errno) { - throw new Error(JSON.stringify(data)); + throw createBaiduFileSystemError(data); } } @@ -62,18 +63,27 @@ export default class BaiduFileSystem implements FileSystem { config.headers = headers; // 对百度网盘请求显式禁用 cookie,避免依赖全局 DNR 规则造成并发竞态 config.credentials = "omit"; + const parseResponse = async (response: Response) => { + const data = await response.json().catch(() => ({ + errmsg: response.statusText || `HTTP ${response.status}`, + })); + if (!response.ok) { + throw createBaiduFileSystemError({ ...data, httpStatus: response.status }); + } + return data; + }; return fetch(url, config) - .then((data) => data.json()) + .then(parseResponse) .then(async (data) => { if (data.errno === 111 || data.errno === -6) { const token = await AuthVerify("baidu", true); this.accessToken = token; url = url.replace(/access_token=[^&]+/, `access_token=${token}`); return fetch(url, config) - .then((data2) => data2.json()) + .then(parseResponse) .then((data2) => { if (data2.errno === 111 || data2.errno === -6) { - throw new Error(JSON.stringify(data2)); + throw createBaiduFileSystemError(data2); } return data2; }); @@ -95,7 +105,7 @@ export default class BaiduFileSystem implements FileSystem { } ).then((data) => { if (data.errno) { - throw new Error(JSON.stringify(data)); + throw createBaiduFileSystemError(data); } return data; }); @@ -126,7 +136,7 @@ export default class BaiduFileSystem implements FileSystem { if (data.errno === -9) { break; } - throw new Error(JSON.stringify(data)); + throw createBaiduFileSystemError(data); } if (!data.list || data.list.length === 0) { diff --git a/packages/filesystem/baidu/error.ts b/packages/filesystem/baidu/error.ts new file mode 100644 index 000000000..d5cb49bc4 --- /dev/null +++ b/packages/filesystem/baidu/error.ts @@ -0,0 +1,35 @@ +import { FileSystemError } from "../error"; + +export type BaiduErrorResponse = { + errno?: number; + httpStatus?: number; + errmsg?: string; + error_msg?: string; + [key: string]: unknown; +}; + +const BAIDU_FILE_EXISTS_ERRNOS = new Set([31061]); + +export function createBaiduFileSystemError(data: BaiduErrorResponse): FileSystemError { + const code = typeof data.errno === "number" ? String(data.errno) : undefined; + const status = data.httpStatus; + const message = + data.errmsg || data.error_msg || (code ? `Baidu request failed with errno ${code}` : "Baidu request failed"); + const conflict = typeof data.errno === "number" && BAIDU_FILE_EXISTS_ERRNOS.has(data.errno); + const auth = data.errno === 111 || data.errno === -6 || status === 401; + const notFound = data.errno === -9 || status === 404; + const rateLimit = status === 429; + + return new FileSystemError({ + provider: "baidu", + message, + status, + code, + conflict, + auth, + notFound, + rateLimit, + retryable: rateLimit || (status !== undefined && status >= 500), + raw: data, + }); +} diff --git a/packages/filesystem/baidu/rw.ts b/packages/filesystem/baidu/rw.ts index df2e54519..6b3284982 100644 --- a/packages/filesystem/baidu/rw.ts +++ b/packages/filesystem/baidu/rw.ts @@ -1,6 +1,7 @@ import type { FileInfo, FileReader, FileWriter } from "../filesystem"; import { calculateMd5, md5OfText } from "@App/pkg/utils/crypto"; import type BaiduFileSystem from "./baidu"; +import { createBaiduFileSystemError } from "./error"; export class BaiduFileReader implements FileReader { file: FileInfo; @@ -19,8 +20,11 @@ export class BaiduFileReader implements FileReader { this.fs.accessToken }&fsids=[${this.file.fsid!}]&dlink=1` ); - if (!data.list.length) { - throw new Error("file not found"); + if (data.errno) { + throw createBaiduFileSystemError(data); + } + if (!data.list?.length) { + throw createBaiduFileSystemError({ errno: -9, errmsg: "file not found" }); } const resp = await fetch(`${data.list[0].dlink}&access_token=${this.fs.accessToken}`); switch (type) { @@ -80,7 +84,7 @@ export class BaiduFileWriter implements FileWriter { } ); if (data.errno) { - throw new Error(JSON.stringify(data)); + throw createBaiduFileSystemError(data); } const uploadid = data.uploadid; const body = new FormData(); @@ -102,7 +106,7 @@ export class BaiduFileWriter implements FileWriter { } ); if (data.errno) { - throw new Error(JSON.stringify(data)); + throw createBaiduFileSystemError(data); } // 创建文件 urlencoded = new URLSearchParams(); @@ -121,7 +125,7 @@ export class BaiduFileWriter implements FileWriter { } ); if (data.errno) { - throw new Error(JSON.stringify(data)); + throw createBaiduFileSystemError(data); } } } diff --git a/packages/filesystem/dropbox/dropbox.test.ts b/packages/filesystem/dropbox/dropbox.test.ts index 6bf549d41..a0fd65325 100644 --- a/packages/filesystem/dropbox/dropbox.test.ts +++ b/packages/filesystem/dropbox/dropbox.test.ts @@ -1,4 +1,6 @@ -import { beforeEach, describe, expect, it, vi } from "vitest"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { FileSystemError } from "../error"; +import { getFileSystemCapabilities } from "../filesystem"; import DropboxFileSystem from "./dropbox"; describe("DropboxFileSystem", () => { @@ -6,10 +8,166 @@ describe("DropboxFileSystem", () => { vi.clearAllMocks(); }); + afterEach(() => { + vi.unstubAllGlobals(); + }); + + it("不应声明原子条件写入能力", () => { + const fs = new DropboxFileSystem("/", "token"); + + expect(getFileSystemCapabilities(fs)).toEqual({ + supportsAtomicCompareAndSwap: false, + supportsCreateOnly: false, + supportsConditionalDelete: false, + }); + }); + + it("request should throw typed not found error", async () => { + const fs = new DropboxFileSystem("/", "token"); + vi.stubGlobal( + "fetch", + vi.fn().mockResolvedValue({ + ok: false, + status: 409, + text: async () => + JSON.stringify({ + error_summary: "path_lookup/not_found/...", + error: { ".tag": "path_lookup", path_lookup: { ".tag": "not_found" } }, + }), + }) + ); + + await expect(fs.request("https://api.dropboxapi.com/2/files/get_metadata")).rejects.toMatchObject({ + provider: "dropbox", + status: 409, + code: "path_lookup/not_found/...", + notFound: true, + conflict: false, + }); + }); + + it("request should classify structured path_lookup not_found without error_summary", async () => { + const fs = new DropboxFileSystem("/", "token"); + vi.stubGlobal( + "fetch", + vi.fn().mockResolvedValue({ + ok: false, + status: 409, + text: async () => + JSON.stringify({ + error: { ".tag": "path_lookup", path_lookup: { ".tag": "not_found" } }, + }), + }) + ); + + await expect(fs.request("https://api.dropboxapi.com/2/files/get_metadata")).rejects.toMatchObject({ + provider: "dropbox", + status: 409, + notFound: true, + conflict: false, + }); + }); + + it("request should throw typed conflict error", async () => { + const fs = new DropboxFileSystem("/", "token"); + vi.stubGlobal( + "fetch", + vi.fn().mockResolvedValue({ + ok: false, + status: 409, + text: async () => + JSON.stringify({ + error_summary: "path/conflict/folder/...", + error: { ".tag": "path", path: { ".tag": "conflict" } }, + }), + }) + ); + + await expect(fs.request("https://api.dropboxapi.com/2/files/create_folder_v2")).rejects.toMatchObject({ + provider: "dropbox", + status: 409, + code: "path/conflict/folder/...", + conflict: true, + notFound: false, + }); + }); + + it("request should classify structured path conflict without error_summary", async () => { + const fs = new DropboxFileSystem("/", "token"); + vi.stubGlobal( + "fetch", + vi.fn().mockResolvedValue({ + ok: false, + status: 409, + text: async () => + JSON.stringify({ + error: { ".tag": "path", path: { ".tag": "conflict" } }, + }), + }) + ); + + await expect(fs.request("https://api.dropboxapi.com/2/files/create_folder_v2")).rejects.toMatchObject({ + provider: "dropbox", + status: 409, + conflict: true, + notFound: false, + }); + }); + + it("request should throw typed rate-limit error", async () => { + const fs = new DropboxFileSystem("/", "token"); + vi.stubGlobal( + "fetch", + vi.fn().mockResolvedValue({ + ok: false, + status: 429, + text: async () => JSON.stringify({ error_summary: "too_many_requests/..." }), + }) + ); + + await expect(fs.request("https://api.dropboxapi.com/2/files/list_folder")).rejects.toMatchObject({ + provider: "dropbox", + status: 429, + code: "too_many_requests/...", + rateLimit: true, + retryable: true, + }); + }); + + it("读取文件遇到 raw 429 响应时抛出 typed 限流错误", async () => { + const fs = new DropboxFileSystem("/", "token"); + vi.spyOn(fs, "request").mockResolvedValue({ + status: 429, + text: vi.fn().mockResolvedValue(JSON.stringify({ error_summary: "too_many_requests/..." })), + } as unknown as Response); + const reader = await fs.open({ + name: "limited.user.js", + path: "/", + size: 1, + digest: "digest", + createtime: 1, + updatetime: 1, + }); + + await expect(reader.read("string")).rejects.toMatchObject({ + provider: "dropbox", + status: 429, + code: "too_many_requests/...", + rateLimit: true, + retryable: true, + }); + }); + it("delete should be idempotent on path not found", async () => { const fs = new DropboxFileSystem("/", "token"); vi.spyOn(fs, "request").mockRejectedValue( - new Error('Dropbox API Error: 409 - {"error_summary":"path_lookup/not_found/..."}') + new FileSystemError({ + provider: "dropbox", + message: "not found", + status: 409, + code: "path_lookup/not_found/...", + notFound: true, + }) ); await expect(fs.delete("missing.txt")).resolves.toBeUndefined(); @@ -18,7 +176,13 @@ describe("DropboxFileSystem", () => { it("exists should return false on path not found", async () => { const fs = new DropboxFileSystem("/", "token"); vi.spyOn(fs, "request").mockRejectedValue( - new Error('Dropbox API Error: 409 - {"error_summary":"path/not_found/..."}') + new FileSystemError({ + provider: "dropbox", + message: "not found", + status: 409, + code: "path/not_found/...", + notFound: true, + }) ); await expect(fs.exists("/missing.txt")).resolves.toBe(false); @@ -31,6 +195,31 @@ describe("DropboxFileSystem", () => { await expect(fs.exists("/test.txt")).rejects.toThrow("invalid_access_token"); }); + it("list should preserve Dropbox content_hash as opaque digest", async () => { + const fs = new DropboxFileSystem("/ScriptCat/sync", "token"); + vi.spyOn(fs, "request").mockResolvedValue({ + entries: [ + { + ".tag": "file", + name: "script.user.js", + size: 10, + content_hash: "dropbox-content-hash", + client_modified: "2026-01-01T00:00:00Z", + server_modified: "2026-01-01T00:00:01Z", + }, + ], + has_more: false, + }); + + await expect(fs.list()).resolves.toMatchObject([ + { + name: "script.user.js", + path: "/sync", + digest: "dropbox-content-hash", + }, + ]); + }); + it("create should normalize double slashes after the Dropbox app root", async () => { const fs = new DropboxFileSystem("/ScriptCat//sync", "token"); diff --git a/packages/filesystem/dropbox/dropbox.ts b/packages/filesystem/dropbox/dropbox.ts index 46c264322..538d07899 100644 --- a/packages/filesystem/dropbox/dropbox.ts +++ b/packages/filesystem/dropbox/dropbox.ts @@ -1,12 +1,71 @@ import { AuthVerify } from "../auth"; +import { FileSystemError, isConflictError, isNotFoundError } from "../error"; import type FileSystem from "../filesystem"; import type { FileInfo, FileCreateOptions, FileReader, FileWriter } from "../filesystem"; import { joinPath } from "../utils"; import { DropboxFileReader, DropboxFileWriter } from "./rw"; -function isDropboxPathNotFound(error: unknown): boolean { - const message = error instanceof Error ? error.message : String(error); - return message.includes("path_lookup/not_found") || message.includes("path/not_found"); +type DropboxErrorBody = { + error_summary?: string; + error?: { + ".tag"?: string; + path_lookup?: { + ".tag"?: string; + }; + path?: { + ".tag"?: string; + }; + }; +}; + +function parseDropboxError(raw: unknown): { summary?: string; raw: unknown } { + let parsed = raw; + if (typeof raw === "string") { + try { + parsed = JSON.parse(raw); + } catch { + return { summary: raw, raw }; + } + } + + if (parsed && typeof parsed === "object") { + const body = parsed as DropboxErrorBody; + if (typeof body.error_summary === "string") { + return { summary: body.error_summary, raw: parsed }; + } + if (typeof body.error?.path_lookup?.[".tag"] === "string") { + return { summary: `path_lookup/${body.error.path_lookup[".tag"]}`, raw: parsed }; + } + if (typeof body.error?.path?.[".tag"] === "string") { + return { summary: `path/${body.error.path[".tag"]}`, raw: parsed }; + } + if (typeof body.error?.[".tag"] === "string") { + return { summary: body.error[".tag"], raw: parsed }; + } + } + + return { raw: parsed }; +} + +function toDropboxFileSystemError(status: number, raw: unknown): FileSystemError { + const { summary, raw: parsed } = parseDropboxError(raw); + const code = summary; + const notFound = summary?.includes("path_lookup/not_found") === true || summary?.includes("path/not_found") === true; + const conflict = !notFound && (status === 409 || summary?.includes("path/conflict") === true); + const rateLimit = status === 429; + const auth = status === 401 || summary?.includes("invalid_access_token") === true; + return new FileSystemError({ + provider: "dropbox", + message: `Dropbox API Error${status ? `: ${status}` : ""}${summary ? ` - ${summary}` : ""}`, + status, + code, + notFound, + conflict, + rateLimit, + auth, + retryable: rateLimit || status >= 500, + raw: parsed, + }); } export default class DropboxFileSystem implements FileSystem { @@ -69,7 +128,7 @@ export default class DropboxFileSystem implements FileSystem { this.pathCache.add(fullPath); } catch (error: any) { // 如果目录已存在,Dropbox 会返回错误,但这是正常情况 - if (error.message && error.message.includes("path/conflict")) { + if (isConflictError(error)) { // 目录已存在,不需要报错 this.pathCache.add(fullPath); return; @@ -107,7 +166,7 @@ export default class DropboxFileSystem implements FileSystem { } if (!response.ok) { const errorText = await response.text(); - throw new Error(`Dropbox API Error: ${response.status} - ${errorText}`); + throw toDropboxFileSystemError(response.status, errorText); } return response.json(); }) @@ -122,23 +181,27 @@ export default class DropboxFileSystem implements FileSystem { .then(async (retryResponse) => { if (!retryResponse.ok) { const errorText = await retryResponse.text(); - throw new Error(`Dropbox API Error: ${retryResponse.status} - ${errorText}`); + throw toDropboxFileSystemError(retryResponse.status, errorText); } return retryResponse.json(); }) .then((retryData) => { if (retryData.error) { - throw new Error(JSON.stringify(retryData)); + throw toDropboxFileSystemError(200, retryData); } return retryData; }); } - throw new Error(JSON.stringify(data)); + throw toDropboxFileSystemError(200, data); } return data; }); } + async createResponseError(resp: Response): Promise { + return toDropboxFileSystemError(resp.status, await resp.text()); + } + async delete(path: string): Promise { const fullPath = joinPath(this.path, path); @@ -154,7 +217,7 @@ export default class DropboxFileSystem implements FileSystem { }), }); } catch (e: any) { - if (isDropboxPathNotFound(e)) { + if (isNotFoundError(e)) { return; } throw e; @@ -182,7 +245,7 @@ export default class DropboxFileSystem implements FileSystem { path: folderPath, }), }).catch((e) => { - if (e.message.includes("path/not_found")) { + if (isNotFoundError(e)) { return { entries: [], has_more: false }; // 返回空数组以避免后续错误 } throw e; @@ -247,7 +310,7 @@ export default class DropboxFileSystem implements FileSystem { }); return true; } catch (e) { - if (isDropboxPathNotFound(e)) { + if (isNotFoundError(e)) { return false; } throw e; diff --git a/packages/filesystem/dropbox/rw.ts b/packages/filesystem/dropbox/rw.ts index f63bfc2ea..076fc9e02 100644 --- a/packages/filesystem/dropbox/rw.ts +++ b/packages/filesystem/dropbox/rw.ts @@ -35,7 +35,7 @@ export class DropboxFileReader implements FileReader { ); if (data.status !== 200) { - return Promise.reject(await data.text()); + throw await this.fs.createResponseError(data); } switch (type) { diff --git a/packages/filesystem/filesystem.test.ts b/packages/filesystem/filesystem.test.ts new file mode 100644 index 000000000..7bef88c5e --- /dev/null +++ b/packages/filesystem/filesystem.test.ts @@ -0,0 +1,60 @@ +import { describe, expect, it, vi } from "vitest"; +import type FileSystem from "./filesystem"; +import { getFileSystemCapabilities } from "./filesystem"; +import LimiterFileSystem from "./limiter"; + +function createFs(overrides: Partial = {}): FileSystem { + return { + verify: vi.fn(async () => {}), + open: vi.fn(async () => ({ read: vi.fn(async () => "") })), + openDir: vi.fn(async () => createFs()), + create: vi.fn(async () => ({ write: vi.fn(async () => {}) })), + createDir: vi.fn(async () => {}), + delete: vi.fn(async () => {}), + list: vi.fn(async () => []), + getDirUrl: vi.fn(async () => ""), + ...overrides, + }; +} + +describe("FileSystem capabilities", () => { + it("未声明能力时应当默认不支持原子同步能力", () => { + const fs = createFs(); + + expect(getFileSystemCapabilities(fs)).toEqual({ + supportsAtomicCompareAndSwap: false, + supportsCreateOnly: false, + supportsConditionalDelete: false, + }); + }); + + it("应当合并 provider 显式声明的能力", () => { + const fs = createFs({ + capabilities: { + supportsCreateOnly: true, + }, + }); + + expect(getFileSystemCapabilities(fs)).toEqual({ + supportsAtomicCompareAndSwap: false, + supportsCreateOnly: true, + supportsConditionalDelete: false, + }); + }); + + it("LimiterFileSystem 应当透传底层 provider 能力", () => { + const fs = createFs({ + capabilities: { + supportsAtomicCompareAndSwap: true, + supportsConditionalDelete: true, + }, + }); + const limiter = new LimiterFileSystem(fs); + + expect(getFileSystemCapabilities(limiter)).toEqual({ + supportsAtomicCompareAndSwap: true, + supportsCreateOnly: false, + supportsConditionalDelete: true, + }); + }); +}); diff --git a/packages/filesystem/filesystem.ts b/packages/filesystem/filesystem.ts index be40308f0..5fa2c6259 100644 --- a/packages/filesystem/filesystem.ts +++ b/packages/filesystem/filesystem.ts @@ -29,10 +29,36 @@ export type FileReadWriter = FileReader & FileWriter; export type FileCreateOptions = { modifiedDate?: number; + expectedDigest?: string; + createOnly?: boolean; }; +export type FileDeleteOptions = { + expectedDigest?: string; +}; + +export type FileSystemCapabilities = { + supportsAtomicCompareAndSwap: boolean; + supportsCreateOnly: boolean; + supportsConditionalDelete: boolean; +}; + +export const DEFAULT_FILE_SYSTEM_CAPABILITIES: FileSystemCapabilities = { + supportsAtomicCompareAndSwap: false, + supportsCreateOnly: false, + supportsConditionalDelete: false, +}; + +export function getFileSystemCapabilities(fs: FileSystem): FileSystemCapabilities { + return { + ...DEFAULT_FILE_SYSTEM_CAPABILITIES, + ...fs.capabilities, + }; +} + // 文件读取 export default interface FileSystem { + readonly capabilities?: Partial; // 授权验证 verify(): Promise; // 打开文件 @@ -44,7 +70,7 @@ export default interface FileSystem { // 创建目录 createDir(dir: string, opts?: FileCreateOptions): Promise; // 删除文件 - delete(path: string): Promise; + delete(path: string, opts?: FileDeleteOptions): Promise; // 文件列表 list(): Promise; // getDirUrl 获取目录的url diff --git a/packages/filesystem/googledrive/googledrive.test.ts b/packages/filesystem/googledrive/googledrive.test.ts index 320f2cb55..428c5f473 100644 --- a/packages/filesystem/googledrive/googledrive.test.ts +++ b/packages/filesystem/googledrive/googledrive.test.ts @@ -1,6 +1,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { LocalStorageDAO } from "@App/app/repo/localStorage"; import { FileSystemError, isAuthError, isConflictError, isNotFoundError, isRateLimitError } from "../error"; +import { getFileSystemCapabilities } from "../filesystem"; import { joinPath } from "../utils"; import GoogleDriveFileSystem from "./googledrive"; @@ -29,6 +30,16 @@ describe("GoogleDriveFileSystem", () => { vi.stubGlobal("fetch", originalFetch); }); + it("不应声明原子条件写入能力", () => { + const fs = new GoogleDriveFileSystem("/", "token"); + + expect(getFileSystemCapabilities(fs)).toEqual({ + supportsAtomicCompareAndSwap: false, + supportsCreateOnly: false, + supportsConditionalDelete: false, + }); + }); + it("delete should be idempotent when file id is missing", async () => { const fs = new GoogleDriveFileSystem("/", "token"); vi.spyOn(fs, "getFileId").mockResolvedValue(null); @@ -49,6 +60,67 @@ describe("GoogleDriveFileSystem", () => { await expect(fs.delete("missing.txt")).resolves.toBeUndefined(); }); + it("删除文件遇到 raw 429 响应时抛出 typed 限流错误", async () => { + const fs = new GoogleDriveFileSystem("/", "token"); + vi.spyOn(fs, "getFileId").mockResolvedValue("file-1"); + vi.spyOn(fs, "request").mockResolvedValue( + createMockResponse({ + ok: false, + status: 429, + text: JSON.stringify({ + error: { + code: 429, + message: "Quota exceeded", + status: "RESOURCE_EXHAUSTED", + }, + }), + }) + ); + + await expect(fs.delete("limited.txt")).rejects.toMatchObject({ + provider: "googledrive", + status: 429, + rateLimit: true, + retryable: true, + }); + }); + + it("读取文件遇到 raw 429 响应时抛出 typed 限流错误", async () => { + const fs = new GoogleDriveFileSystem("/", "token"); + vi.spyOn(fs, "getFileId").mockResolvedValue("file-1"); + vi.spyOn(fs, "request").mockResolvedValue( + createMockResponse({ + ok: false, + status: 429, + text: JSON.stringify({ + error: { + code: 429, + message: "Quota exceeded", + status: "RESOURCE_EXHAUSTED", + }, + }), + }) + ); + const reader = await fs.open({ name: "limited.txt", path: "/", size: 0, digest: "", createtime: 0, updatetime: 0 }); + + await expect(reader.read("string")).rejects.toMatchObject({ + provider: "googledrive", + status: 429, + rateLimit: true, + retryable: true, + }); + }); + + it("读取文件路径查找失败时应抛出 typed not found 错误", async () => { + const fs = new GoogleDriveFileSystem("/", "token"); + vi.spyOn(fs, "getFileId").mockResolvedValue(null); + const requestSpy = vi.spyOn(fs, "request"); + const reader = await fs.open({ name: "missing.txt", path: "/", size: 0, digest: "", createtime: 0, updatetime: 0 }); + + await expect(reader.read("string")).rejects.toSatisfy(isNotFoundError); + expect(requestSpy).not.toHaveBeenCalled(); + }); + it("ensureDirExists should create missing nested directories and return final id", async () => { const fs = new GoogleDriveFileSystem("/", "token"); const findSpy = vi.spyOn(fs, "findFolderByName").mockResolvedValue(null); diff --git a/packages/filesystem/googledrive/googledrive.ts b/packages/filesystem/googledrive/googledrive.ts index b5934f20e..b0fb91921 100644 --- a/packages/filesystem/googledrive/googledrive.ts +++ b/packages/filesystem/googledrive/googledrive.ts @@ -147,7 +147,7 @@ export default class GoogleDriveFileSystem implements FileSystem { }); } - private async createResponseError(resp: Response): Promise { + async createResponseError(resp: Response): Promise { const text = await resp.text(); let raw; try { @@ -232,7 +232,7 @@ export default class GoogleDriveFileSystem implements FileSystem { return; } if (resp.status !== 204 && resp.status !== 200) { - throw new Error(await resp.text()); + throw await this.createResponseError(resp); } }); diff --git a/packages/filesystem/googledrive/rw.ts b/packages/filesystem/googledrive/rw.ts index a19a6828d..ad69fef72 100644 --- a/packages/filesystem/googledrive/rw.ts +++ b/packages/filesystem/googledrive/rw.ts @@ -1,4 +1,4 @@ -import { isNotFoundError } from "../error"; +import { FileSystemError, isNotFoundError } from "../error"; import type { FileInfo, FileReader, FileWriter } from "../filesystem"; import { joinPath } from "../utils"; import type GoogleDriveFileSystem from "./googledrive"; @@ -17,7 +17,12 @@ export class GoogleDriveFileReader implements FileReader { // 首先获取文件ID const fileId = await this.fs.getFileId(joinPath(this.file.path, this.file.name)); if (!fileId) { - return Promise.reject(new Error(`File not found: ${this.file.name}`)); + throw new FileSystemError({ + provider: "googledrive", + message: `File not found: ${this.file.name}`, + status: 404, + notFound: true, + }); } // 获取文件内容 @@ -28,7 +33,7 @@ export class GoogleDriveFileReader implements FileReader { ); if (data.status !== 200) { - return Promise.reject(await data.text()); + throw await this.fs.createResponseError(data); } switch (type) { diff --git a/packages/filesystem/limiter.test.ts b/packages/filesystem/limiter.test.ts index f76ed56f4..04c8e0fbe 100644 --- a/packages/filesystem/limiter.test.ts +++ b/packages/filesystem/limiter.test.ts @@ -1,4 +1,5 @@ import { afterEach, describe, expect, it, vi } from "vitest"; +import { FileSystemError } from "./error"; import type FileSystem from "./filesystem"; import type { FileInfo, FileReader, FileWriter } from "./filesystem"; import LimiterFileSystem from "./limiter"; @@ -54,6 +55,66 @@ describe("LimiterFileSystem", () => { expect(fs.list).toHaveBeenCalledTimes(2); }); + it("应当根据 typed rateLimit 错误重试 list", async () => { + vi.useFakeTimers(); + const fs = createFs(); + vi.mocked(fs.list) + .mockRejectedValueOnce( + new FileSystemError({ + provider: "dropbox", + message: "Quota exhausted", + status: 429, + rateLimit: true, + retryable: true, + }) + ) + .mockResolvedValueOnce([]); + const limiter = new LimiterFileSystem(fs); + + const promise = limiter.list(); + await vi.runOnlyPendingTimersAsync(); + + await expect(promise).resolves.toEqual([]); + expect(fs.list).toHaveBeenCalledTimes(2); + }); + + it("应当根据 typed retryable 5xx 错误重试 list", async () => { + vi.useFakeTimers(); + const fs = createFs(); + vi.mocked(fs.list) + .mockRejectedValueOnce( + new FileSystemError({ + provider: "googledrive", + message: "Service unavailable", + status: 503, + retryable: true, + }) + ) + .mockResolvedValueOnce([]); + const limiter = new LimiterFileSystem(fs); + + const promise = limiter.list(); + await vi.runOnlyPendingTimersAsync(); + + await expect(promise).resolves.toEqual([]); + expect(fs.list).toHaveBeenCalledTimes(2); + }); + + it("遇到 typed conflict 错误时不能重试 list", async () => { + const fs = createFs(); + const conflictError = new FileSystemError({ + provider: "onedrive", + message: "Conflict", + status: 409, + conflict: true, + }); + vi.mocked(fs.list).mockRejectedValueOnce(conflictError); + const limiter = new LimiterFileSystem(fs); + + await expect(limiter.list()).rejects.toBe(conflictError); + expect(fs.list).toHaveBeenCalledTimes(1); + }); + it("should retry verify on 429", async () => { vi.useFakeTimers(); const fs = createFs(); @@ -163,6 +224,93 @@ describe("LimiterFileSystem", () => { expect(write).toHaveBeenCalledTimes(1); }); + it("writer.write 遇到 typed retryable 错误时仍不能重试", async () => { + const fs = createFs(); + const write = vi.fn(async () => {}); + const retryableError = new FileSystemError({ + provider: "googledrive", + message: "Service unavailable", + status: 503, + retryable: true, + }); + vi.mocked(fs.create).mockResolvedValueOnce({ + write, + }); + write.mockRejectedValueOnce(retryableError); + const limiter = new LimiterFileSystem(fs); + const writer = await limiter.create(file.path); + + await expect(writer.write("content")).rejects.toBe(retryableError); + expect(write).toHaveBeenCalledTimes(1); + }); + + it("受 expectedDigest 保护的 writer.write 遇到 typed retryable 错误时应当重试", async () => { + vi.useFakeTimers(); + const fs = createFs(); + const write = vi.fn(async () => {}); + const retryableError = new FileSystemError({ + provider: "s3", + message: "Service unavailable", + status: 503, + retryable: true, + }); + vi.mocked(fs.create).mockResolvedValueOnce({ + write, + }); + write.mockRejectedValueOnce(retryableError).mockResolvedValueOnce(undefined); + const limiter = new LimiterFileSystem(fs); + const writer = await limiter.create(file.path, { expectedDigest: "digest" }); + + const promise = writer.write("content"); + await vi.runOnlyPendingTimersAsync(); + + await expect(promise).resolves.toBeUndefined(); + expect(write).toHaveBeenCalledTimes(2); + }); + + it("受 createOnly 保护的 writer.write 遇到 typed retryable 错误时应当重试", async () => { + vi.useFakeTimers(); + const fs = createFs(); + const write = vi.fn(async () => {}); + const retryableError = new FileSystemError({ + provider: "webdav", + message: "Service unavailable", + status: 503, + retryable: true, + }); + vi.mocked(fs.create).mockResolvedValueOnce({ + write, + }); + write.mockRejectedValueOnce(retryableError).mockResolvedValueOnce(undefined); + const limiter = new LimiterFileSystem(fs); + const writer = await limiter.create(file.path, { createOnly: true }); + + const promise = writer.write("content"); + await vi.runOnlyPendingTimersAsync(); + + await expect(promise).resolves.toBeUndefined(); + expect(write).toHaveBeenCalledTimes(2); + }); + + it("受 expectedDigest 保护的 delete 遇到 typed retryable 错误时应当重试", async () => { + vi.useFakeTimers(); + const fs = createFs(); + const retryableError = new FileSystemError({ + provider: "onedrive", + message: "Service unavailable", + status: 503, + retryable: true, + }); + vi.mocked(fs.delete).mockRejectedValueOnce(retryableError).mockResolvedValueOnce(undefined); + const limiter = new LimiterFileSystem(fs); + + const promise = limiter.delete(file.path, { expectedDigest: "digest" }); + await vi.runOnlyPendingTimersAsync(); + + await expect(promise).resolves.toBeUndefined(); + expect(fs.delete).toHaveBeenCalledTimes(2); + }); + it("should retry reader.read on 429", async () => { vi.useFakeTimers(); const fs = createFs(); diff --git a/packages/filesystem/limiter.ts b/packages/filesystem/limiter.ts index 5dfb54b81..84847915f 100644 --- a/packages/filesystem/limiter.ts +++ b/packages/filesystem/limiter.ts @@ -1,7 +1,18 @@ import type FileSystem from "./filesystem"; -import type { FileCreateOptions, FileInfo, FileReader, FileWriter } from "./filesystem"; - -const RETRYABLE_429_OPS = new Set(["verify", "open", "read", "openDir", "list", "getDirUrl"]); +import type { FileCreateOptions, FileDeleteOptions, FileInfo, FileReader, FileWriter } from "./filesystem"; +import { getFileSystemCapabilities, type FileSystemCapabilities } from "./filesystem"; +import { FileSystemError } from "./error"; + +const RETRYABLE_TRANSIENT_OPS = new Set([ + "verify", + "open", + "read", + "openDir", + "list", + "getDirUrl", + "conditionalWrite", + "conditionalDelete", +]); /** * 速率限制器 @@ -21,7 +32,7 @@ export class RateLimiter { /** * 执行限速操作 * @param fn 要执行的操作函数 - * @param op 操作类型,用于在遇到 429 时判断是否允许自动重试。默认值 "unknown" 不在白名单内,会被视为不可重试 + * @param op 操作类型,用于判断 transient 错误是否允许自动重试。默认值 "unknown" 不在白名单内,会被视为不可重试 * @returns 操作结果 */ async execute(fn: () => Promise, op = "unknown"): Promise { @@ -46,9 +57,9 @@ export class RateLimiter { } /** - * 执行操作并处理 429 错误重试 + * 执行操作并处理 transient 错误重试 * @param fn 要执行的操作函数 - * @param op 操作类型,用于判定该操作在遇到 429 时是否进入指数退避重试 + * @param op 操作类型,用于判定该操作在遇到 transient 错误时是否进入指数退避重试 * @returns 操作结果 */ private async executeWithRetry(fn: () => Promise, op: string): Promise { @@ -57,10 +68,8 @@ export class RateLimiter { try { return await fn(); } catch (error) { - // 检查错误字符串中是否包含 429 - const errorStr = String(error).toLowerCase(); - if (this.shouldRetry429(op, ` ${errorStr} `) && i < 10) { - // 遇到 429 错误且未达到重试上限,采用指数退避策略延迟后继续重试 + if (this.shouldRetryTransient(op, error) && i < 10) { + // 遇到 transient 错误且未达到重试上限,采用指数退避策略延迟后继续重试 const delay = Math.min(2000 * Math.pow(2, i), 60000); await new Promise((resolve) => setTimeout(resolve, delay)); // 继续下一次循环重试 @@ -74,11 +83,15 @@ export class RateLimiter { throw new Error(`Max retries exceeded (op=${op})`); } - private shouldRetry429(op: string, errorStr: string): boolean { - return ( - ((errorStr.includes("429") && /[^a-z\d]429[^a-z\d]/.test(errorStr)) || errorStr.includes("too many requests")) && - RETRYABLE_429_OPS.has(op) - ); + private shouldRetryTransient(op: string, error: unknown): boolean { + if (!RETRYABLE_TRANSIENT_OPS.has(op)) { + return false; + } + if (error instanceof FileSystemError) { + return error.rateLimit || error.retryable; + } + const errorStr = ` ${String(error).toLowerCase()} `; + return (errorStr.includes("429") && /[^a-z\d]429[^a-z\d]/.test(errorStr)) || errorStr.includes("too many requests"); } } @@ -94,6 +107,10 @@ export default class LimiterFileSystem implements FileSystem { this.limiter = limiter || new RateLimiter(); } + get capabilities(): FileSystemCapabilities { + return getFileSystemCapabilities(this.fs); + } + verify(): Promise { return this.limiter.execute(() => this.fs.verify(), "verify"); } @@ -117,8 +134,9 @@ export default class LimiterFileSystem implements FileSystem { async create(path: string, opts?: FileCreateOptions): Promise { return this.limiter.execute(async () => { const writer = await this.fs.create(path, opts); + const writeOp = opts?.expectedDigest || opts?.createOnly ? "conditionalWrite" : "write"; return { - write: (content) => this.limiter.execute(() => writer.write(content), "write"), + write: (content) => this.limiter.execute(() => writer.write(content), writeOp), }; }, "create"); } @@ -127,8 +145,9 @@ export default class LimiterFileSystem implements FileSystem { return this.limiter.execute(() => this.fs.createDir(dir, opts), "createDir"); } - delete(path: string): Promise { - return this.limiter.execute(() => this.fs.delete(path), "delete"); + delete(path: string, opts?: FileDeleteOptions): Promise { + const op = opts?.expectedDigest ? "conditionalDelete" : "delete"; + return this.limiter.execute(() => this.fs.delete(path, opts), op); } list(): Promise { diff --git a/packages/filesystem/onedrive/onedrive.test.ts b/packages/filesystem/onedrive/onedrive.test.ts index a7400cad1..c5f4a06fa 100644 --- a/packages/filesystem/onedrive/onedrive.test.ts +++ b/packages/filesystem/onedrive/onedrive.test.ts @@ -28,6 +28,16 @@ describe("OneDriveFileSystem", () => { vi.stubGlobal("fetch", originalFetch); }); + it("应当声明支持原子条件写入和条件删除能力", () => { + const fs = new OneDriveFileSystem("/", "token"); + + expect((fs as any).capabilities).toMatchObject({ + supportsAtomicCompareAndSwap: true, + supportsCreateOnly: true, + supportsConditionalDelete: true, + }); + }); + it("request should return retry result after token refresh", async () => { await localStorageDAO.saveValue("netdisk:token:onedrive", { accessToken: "expired-token", @@ -95,6 +105,53 @@ describe("OneDriveFileSystem", () => { await expect(fs.delete("missing.txt")).resolves.toBeUndefined(); }); + it("删除文件遇到 raw 429 响应时抛出 typed 限流错误", async () => { + const fs = new OneDriveFileSystem("/", "token"); + vi.spyOn(fs, "request").mockResolvedValue( + createMockResponse({ + ok: false, + status: 429, + text: JSON.stringify({ + error: { + code: "TooManyRequests", + message: "Too many requests", + }, + }), + }) + ); + + await expect(fs.delete("limited.txt")).rejects.toMatchObject({ + provider: "onedrive", + status: 429, + rateLimit: true, + retryable: true, + }); + }); + + it("读取文件遇到 raw 429 响应时抛出 typed 限流错误", async () => { + const fs = new OneDriveFileSystem("/", "token"); + vi.spyOn(fs, "request").mockResolvedValue( + createMockResponse({ + ok: false, + status: 429, + text: JSON.stringify({ + error: { + code: "TooManyRequests", + message: "Too many requests", + }, + }), + }) + ); + const reader = await fs.open({ name: "limited.txt", path: "/", size: 0, digest: "", createtime: 0, updatetime: 0 }); + + await expect(reader.read("string")).rejects.toMatchObject({ + provider: "onedrive", + status: 429, + rateLimit: true, + retryable: true, + }); + }); + it("create should normalize double slashes in paths", async () => { const fs = new OneDriveFileSystem("/ScriptCat//sync", "token"); @@ -116,6 +173,24 @@ describe("OneDriveFileSystem", () => { ); }); + it("条件删除应当将 expectedDigest 转成 If-Match", async () => { + const fs = new OneDriveFileSystem("/", "token"); + const request = vi.spyOn(fs, "request").mockResolvedValue({ status: 204 }); + + await (fs as any).delete("test.txt", { expectedDigest: "abc123" }); + + expect(request).toHaveBeenCalledWith( + "https://graph.microsoft.com/v1.0/me/drive/special/approot:/test.txt", + { + method: "DELETE", + headers: expect.objectContaining({ + "If-Match": "abc123", + }), + }, + true + ); + }); + it("createDir should create nested directories from root", async () => { const fs = new OneDriveFileSystem("/", "token"); const requestSpy = vi.spyOn(fs, "request").mockResolvedValue({}); @@ -352,4 +427,34 @@ describe("OneDriveFileSystem", () => { const headers = (requestSpy.mock.calls[1][1] as RequestInit).headers as Headers; expect(headers.get("Content-Range")).toBe("bytes 0-2/3"); }); + + it("条件写入应当将 expectedDigest 传给 upload session 的 If-Match", async () => { + const fs = new OneDriveFileSystem("/", "token"); + const requestSpy = vi + .spyOn(fs, "request") + .mockResolvedValueOnce({ uploadUrl: "https://upload.example/session" }) + .mockResolvedValueOnce({}); + + const writer = await (fs as any).create("not-empty.txt", { expectedDigest: "abc123" }); + await writer.write("abc"); + + const headers = (requestSpy.mock.calls[0][1] as RequestInit).headers as Headers; + expect(headers.get("If-Match")).toBe("abc123"); + }); + + it("createOnly 写入应当将 If-None-Match 传给 upload session 并使用 fail 语义", async () => { + const fs = new OneDriveFileSystem("/", "token"); + const requestSpy = vi + .spyOn(fs, "request") + .mockResolvedValueOnce({ uploadUrl: "https://upload.example/session" }) + .mockResolvedValueOnce({}); + + const writer = await (fs as any).create("not-empty.txt", { createOnly: true }); + await writer.write("abc"); + + const headers = (requestSpy.mock.calls[0][1] as RequestInit).headers as Headers; + const body = JSON.parse((requestSpy.mock.calls[0][1] as RequestInit).body as string); + expect(headers.get("If-None-Match")).toBe("*"); + expect(body.item["@microsoft.graph.conflictBehavior"]).toBe("fail"); + }); }); diff --git a/packages/filesystem/onedrive/onedrive.ts b/packages/filesystem/onedrive/onedrive.ts index 3a2bef2b0..e7529e1fc 100644 --- a/packages/filesystem/onedrive/onedrive.ts +++ b/packages/filesystem/onedrive/onedrive.ts @@ -1,11 +1,17 @@ import { AuthVerify } from "../auth"; import { FileSystemError } from "../error"; -import type { FileInfo, FileCreateOptions, FileReader, FileWriter } from "../filesystem"; +import type { FileInfo, FileCreateOptions, FileDeleteOptions, FileReader, FileWriter } from "../filesystem"; import type FileSystem from "../filesystem"; import { joinPath } from "../utils"; import { OneDriveFileReader, OneDriveFileWriter } from "./rw"; export default class OneDriveFileSystem implements FileSystem { + readonly capabilities = { + supportsAtomicCompareAndSwap: true, + supportsCreateOnly: true, + supportsConditionalDelete: true, + }; + accessToken?: string; path: string; @@ -32,8 +38,8 @@ export default class OneDriveFileSystem implements FileSystem { return new OneDriveFileSystem(joinPath(this.path, path), this.accessToken); } - async create(path: string, _opts?: FileCreateOptions): Promise { - return new OneDriveFileWriter(this, joinPath(this.path, path)); + async create(path: string, opts?: FileCreateOptions): Promise { + return new OneDriveFileWriter(this, joinPath(this.path, path), opts); } async createDir(dir: string, _opts?: FileCreateOptions): Promise { @@ -114,7 +120,7 @@ export default class OneDriveFileSystem implements FileSystem { }); } - private async createResponseError(resp: Response): Promise { + async createResponseError(resp: Response): Promise { const text = await resp.text(); let raw; try { @@ -127,7 +133,7 @@ export default class OneDriveFileSystem implements FileSystem { request(url: string, config?: RequestInit, nothen?: boolean): Promise { config = config || {}; - const headers = config.headers || new Headers(); + const headers = new Headers(config.headers); if (!url.includes("uploadSession")) { headers.set(`Authorization`, `Bearer ${this.accessToken}`); } @@ -182,19 +188,25 @@ export default class OneDriveFileSystem implements FileSystem { }); } - async delete(path: string): Promise { + async delete(path: string, opts?: FileDeleteOptions): Promise { + const config: RequestInit = { + method: "DELETE", + }; + if (opts?.expectedDigest) { + config.headers = { + "If-Match": opts.expectedDigest, + }; + } const resp = await this.request( `https://graph.microsoft.com/v1.0/me/drive/special/approot:${joinPath(this.path, path)}`, - { - method: "DELETE", - }, + config, true ); if (resp.status === 404) { return; } if (resp.status !== 204) { - throw new Error(await resp.text()); + throw await this.createResponseError(resp); } } diff --git a/packages/filesystem/onedrive/rw.ts b/packages/filesystem/onedrive/rw.ts index b0d33d55a..7c6158b38 100644 --- a/packages/filesystem/onedrive/rw.ts +++ b/packages/filesystem/onedrive/rw.ts @@ -1,5 +1,5 @@ import { calculateMd5, md5OfText } from "@App/pkg/utils/crypto"; -import type { FileInfo, FileReader, FileWriter } from "../filesystem"; +import type { FileCreateOptions, FileInfo, FileReader, FileWriter } from "../filesystem"; import { joinPath } from "../utils"; import type OneDriveFileSystem from "./onedrive"; @@ -20,7 +20,7 @@ export class OneDriveFileReader implements FileReader { true ); if (data.status !== 200) { - throw new Error(await data.text()); + throw await this.fs.createResponseError(data); } switch (type) { case "string": @@ -37,9 +37,12 @@ export class OneDriveFileWriter implements FileWriter { fs: OneDriveFileSystem; - constructor(fs: OneDriveFileSystem, path: string) { + opts?: FileCreateOptions; + + constructor(fs: OneDriveFileSystem, path: string, opts?: FileCreateOptions) { this.fs = fs; this.path = path; + this.opts = opts; } size(content: string | Blob) { @@ -60,21 +63,30 @@ export class OneDriveFileWriter implements FileWriter { // 预上传获取id const size = this.size(content); if (size === 0) { - return this.fs.request(`https://graph.microsoft.com/v1.0/me/drive/special/approot:${this.path}:/content`, { + const config: RequestInit = { method: "PUT", body: content, - }); + }; + const writeHeaders = this.buildConditionalHeaders(); + if (writeHeaders) { + config.headers = writeHeaders; + } + return this.fs.request(`https://graph.microsoft.com/v1.0/me/drive/special/approot:${this.path}:/content`, config); } let myHeaders = new Headers(); myHeaders.append("Content-Type", "application/json"); + const conditionHeaders = this.buildConditionalHeaders(); + if (conditionHeaders) { + conditionHeaders.forEach((value, key) => myHeaders.set(key, value)); + } const uploadUrl = await this.fs .request(`https://graph.microsoft.com/v1.0/me/drive/special/approot:${this.path}:/createUploadSession`, { method: "POST", headers: myHeaders, body: JSON.stringify({ item: { - "@microsoft.graph.conflictBehavior": "replace", + "@microsoft.graph.conflictBehavior": this.opts?.createOnly ? "fail" : "replace", // description: "description", // fileSystemInfo: { // "@odata.type": "microsoft.graph.fileSystemInfo", @@ -97,4 +109,18 @@ export class OneDriveFileWriter implements FileWriter { headers: myHeaders, }); } + + private buildConditionalHeaders(): Headers | undefined { + if (this.opts?.expectedDigest) { + const headers = new Headers(); + headers.set("If-Match", this.opts.expectedDigest); + return headers; + } + if (this.opts?.createOnly) { + const headers = new Headers(); + headers.set("If-None-Match", "*"); + return headers; + } + return undefined; + } } diff --git a/packages/filesystem/s3/error.ts b/packages/filesystem/s3/error.ts new file mode 100644 index 000000000..78cbbb67b --- /dev/null +++ b/packages/filesystem/s3/error.ts @@ -0,0 +1,23 @@ +import { FileSystemError } from "../error"; +import { S3Error } from "./client"; + +export function createS3FileSystemError(error: unknown): unknown { + if (!(error instanceof S3Error)) { + return error; + } + + const rateLimit = error.statusCode === 429 || error.code === "SlowDown"; + + return new FileSystemError({ + provider: "s3", + message: error.message, + status: error.statusCode, + code: error.code, + auth: error.statusCode === 401 || error.statusCode === 403, + notFound: error.statusCode === 404 || error.code === "NoSuchKey" || error.code === "NoSuchBucket", + conflict: error.statusCode === 409 || error.statusCode === 412 || error.code === "PreconditionFailed", + rateLimit, + retryable: rateLimit || error.statusCode >= 500, + raw: error, + }); +} diff --git a/packages/filesystem/s3/rw.ts b/packages/filesystem/s3/rw.ts index 378fb18ad..87b11749d 100644 --- a/packages/filesystem/s3/rw.ts +++ b/packages/filesystem/s3/rw.ts @@ -1,5 +1,10 @@ import type { S3Client } from "./client"; -import type { FileReader, FileWriter } from "../filesystem"; +import type { FileCreateOptions, FileReader, FileWriter } from "../filesystem"; +import { createS3FileSystemError } from "./error"; + +function quoteETag(digest: string): string { + return digest.startsWith('"') && digest.endsWith('"') ? digest : `"${digest}"`; +} /** * S3 文件读取器 @@ -25,11 +30,15 @@ export class S3FileReader implements FileReader { * @throws {S3Error} 文件不存在或读取失败 */ async read(type: "string" | "blob" = "blob"): Promise { - const response = await this.client.request("GET", this.bucket, this.key); - if (type === "string") { - return response.text(); + try { + const response = await this.client.request("GET", this.bucket, this.key); + if (type === "string") { + return response.text(); + } + return response.blob(); + } catch (error) { + throw createS3FileSystemError(error); } - return response.blob(); } } @@ -46,11 +55,17 @@ export class S3FileWriter implements FileWriter { modifiedDate?: number; - constructor(client: S3Client, bucket: string, key: string, modifiedDate?: number) { + expectedDigest?: string; + + createOnly?: boolean; + + constructor(client: S3Client, bucket: string, key: string, opts?: FileCreateOptions) { this.client = client; this.bucket = bucket; this.key = key; - this.modifiedDate = modifiedDate; + this.modifiedDate = opts?.modifiedDate; + this.expectedDigest = opts?.expectedDigest; + this.createOnly = opts?.createOnly; } /** @@ -68,10 +83,19 @@ export class S3FileWriter implements FileWriter { // 历史兼容:S3 侧使用 createtime 元数据保存文件时间,实际来源是 FileCreateOptions.modifiedDate。 headers["x-amz-meta-createtime"] = new Date(this.modifiedDate).toISOString(); } + if (this.expectedDigest) { + headers["if-match"] = quoteETag(this.expectedDigest); + } else if (this.createOnly) { + headers["if-none-match"] = "*"; + } - await this.client.request("PUT", this.bucket, this.key, { - body: typeof body === "string" ? body : body, - headers, - }); + try { + await this.client.request("PUT", this.bucket, this.key, { + body: typeof body === "string" ? body : body, + headers, + }); + } catch (error) { + throw createS3FileSystemError(error); + } } } diff --git a/packages/filesystem/s3/s3.test.ts b/packages/filesystem/s3/s3.test.ts index 8e52a51c7..8244bdbc1 100644 --- a/packages/filesystem/s3/s3.test.ts +++ b/packages/filesystem/s3/s3.test.ts @@ -1,4 +1,5 @@ import { describe, it, expect, vi, beforeEach } from "vitest"; +import { isConflictError } from "../error"; import S3FileSystem from "./s3"; import { S3Client, S3Error } from "./client"; import type { FileInfo } from "../filesystem"; @@ -47,6 +48,14 @@ describe("S3FileSystem", () => { fs = new S3FileSystem("test-bucket", mockClient); }); + it("应当声明支持原子条件写入和条件删除能力", () => { + expect((fs as any).capabilities).toMatchObject({ + supportsAtomicCompareAndSwap: true, + supportsCreateOnly: true, + supportsConditionalDelete: true, + }); + }); + // ---- verify ---- describe("verify", () => { it("应当成功验证 bucket", async () => { @@ -155,6 +164,29 @@ describe("S3FileSystem", () => { expect(mockClient.request).toHaveBeenCalledWith("GET", "test-bucket", "data/hello.txt"); expect(content).toBe("file content"); }); + + it("读取文件遇到 503 时应当抛出 typed 可重试错误", async () => { + const fileInfo: FileInfo = { + name: "busy.user.js", + path: "/", + size: 50, + digest: "xyz", + createtime: 1000, + updatetime: 2000, + }; + (mockClient.request as ReturnType).mockRejectedValue( + new S3Error("ServiceUnavailable", "Please retry later", 503) + ); + + const reader = await fs.open(fileInfo); + + await expect(reader.read("string")).rejects.toMatchObject({ + provider: "s3", + status: 503, + code: "ServiceUnavailable", + retryable: true, + }); + }); }); // ---- create ---- @@ -203,6 +235,52 @@ describe("S3FileSystem", () => { ); }); + it("条件写入应当将 expectedDigest 转成 If-Match", async () => { + (mockClient.request as ReturnType).mockResolvedValue(createMockResponse({ ok: true })); + + const writer = await (fs as any).create("output.txt", { expectedDigest: "abc123" }); + await writer.write("hello world"); + + expect(mockClient.request).toHaveBeenCalledWith( + "PUT", + "test-bucket", + "output.txt", + expect.objectContaining({ + headers: expect.objectContaining({ + "if-match": '"abc123"', + }), + }) + ); + }); + + it("createOnly 写入应当使用 If-None-Match", async () => { + (mockClient.request as ReturnType).mockResolvedValue(createMockResponse({ ok: true })); + + const writer = await (fs as any).create("new.txt", { createOnly: true }); + await writer.write("hello world"); + + expect(mockClient.request).toHaveBeenCalledWith( + "PUT", + "test-bucket", + "new.txt", + expect.objectContaining({ + headers: expect.objectContaining({ + "if-none-match": "*", + }), + }) + ); + }); + + it("createOnly 写入遇到 412 时应当抛出 typed conflict 错误", async () => { + (mockClient.request as ReturnType).mockRejectedValue( + new S3Error("PreconditionFailed", "At least one precondition failed", 412) + ); + + const writer = await (fs as any).create("new.txt", { createOnly: true }); + + await expect(writer.write("hello world")).rejects.toSatisfy(isConflictError); + }); + it("normalizes double slashes in object keys", async () => { const subFs = new S3FileSystem("test-bucket", mockClient, "/ScriptCat//sync"); @@ -252,6 +330,31 @@ describe("S3FileSystem", () => { expect(mockClient.request).toHaveBeenCalledWith("DELETE", "test-bucket", "ScriptCat/sync/dir/file.user.js"); }); + + it("条件删除应当将 expectedDigest 转成 If-Match", async () => { + (mockClient.request as ReturnType).mockResolvedValue(createMockResponse({ ok: true, status: 204 })); + + await (fs as any).delete("test.txt", { expectedDigest: "abc123" }); + + expect(mockClient.request).toHaveBeenCalledWith( + "DELETE", + "test-bucket", + "test.txt", + expect.objectContaining({ + headers: { + "if-match": '"abc123"', + }, + }) + ); + }); + + it("条件删除遇到 412 时应当抛出 typed conflict 错误", async () => { + (mockClient.request as ReturnType).mockRejectedValue( + new S3Error("PreconditionFailed", "At least one precondition failed", 412) + ); + + await expect((fs as any).delete("test.txt", { expectedDigest: "abc123" })).rejects.toSatisfy(isConflictError); + }); }); // ---- list ---- @@ -400,6 +503,20 @@ describe("S3FileSystem", () => { await expect(fs.list()).rejects.toThrow("Permission denied"); }); + + it("列目录遇到 429 时应当抛出 typed 限流错误", async () => { + (mockClient.request as ReturnType).mockRejectedValue( + new S3Error("SlowDown", "Please reduce your request rate", 429) + ); + + await expect(fs.list()).rejects.toMatchObject({ + provider: "s3", + status: 429, + code: "SlowDown", + rateLimit: true, + retryable: true, + }); + }); }); // ---- openDir ---- diff --git a/packages/filesystem/s3/s3.ts b/packages/filesystem/s3/s3.ts index 41ce89e6a..fe9f76459 100644 --- a/packages/filesystem/s3/s3.ts +++ b/packages/filesystem/s3/s3.ts @@ -2,10 +2,11 @@ import { XMLParser } from "fast-xml-parser"; import { S3Client, S3Error } from "./client"; import type { S3ClientConfig } from "./client"; import type FileSystem from "../filesystem"; -import type { FileInfo, FileCreateOptions, FileReader, FileWriter } from "../filesystem"; +import type { FileInfo, FileCreateOptions, FileDeleteOptions, FileReader, FileWriter } from "../filesystem"; import { joinPath } from "../utils"; import { S3FileReader, S3FileWriter } from "./rw"; import { WarpTokenError } from "../error"; +import { createS3FileSystemError } from "./error"; // ---- ListObjectsV2 XML 解析 ---- @@ -25,6 +26,10 @@ const xmlParser = new XMLParser({ isArray: (name) => name === "Contents", }); +function quoteETag(digest: string): string { + return digest.startsWith('"') && digest.endsWith('"') ? digest : `"${digest}"`; +} + /** 从 ListObjectsV2 XML 响应中解析对象列表 */ function parseListObjectsV2(xml: string): ListObjectsV2Result { const parsed = xmlParser.parse(xml); @@ -52,6 +57,12 @@ function parseListObjectsV2(xml: string): ListObjectsV2Result { * 使用原生 fetch + AWS Signature V4 签名,不依赖 @aws-sdk/client-s3 */ export default class S3FileSystem implements FileSystem { + readonly capabilities = { + supportsAtomicCompareAndSwap: true, + supportsCreateOnly: true, + supportsConditionalDelete: true, + }; + client: S3Client; bucket: string; @@ -167,7 +178,7 @@ export default class S3FileSystem implements FileSystem { * @returns 文件写入器 */ async create(path: string, opts?: FileCreateOptions): Promise { - return new S3FileWriter(this.client, this.bucket, joinPath(this.basePath, path).substring(1), opts?.modifiedDate); + return new S3FileWriter(this.client, this.bucket, joinPath(this.basePath, path).substring(1), opts); } /** @@ -182,15 +193,22 @@ export default class S3FileSystem implements FileSystem { * 此操作幂等——删除不存在的文件也会成功 * @param path 相对于当前 basePath 的文件路径 */ - async delete(path: string): Promise { + async delete(path: string, opts?: FileDeleteOptions): Promise { try { - await this.client.request("DELETE", this.bucket, joinPath(this.basePath, path).substring(1)); + const key = joinPath(this.basePath, path).substring(1); + if (opts?.expectedDigest) { + await this.client.request("DELETE", this.bucket, key, { + headers: { "if-match": quoteETag(opts.expectedDigest) }, + }); + return; + } + await this.client.request("DELETE", this.bucket, key); } catch (error: any) { // S3 delete 是幂等的,key 不存在时也视为成功 if (error instanceof S3Error && error.code === "NoSuchKey") { return; } - throw error; + throw createS3FileSystemError(error); } } @@ -254,7 +272,7 @@ export default class S3FileSystem implements FileSystem { if (error instanceof S3Error && error.code === "AccessDenied") { throw new Error(`Permission denied. Check your IAM permissions for bucket: ${this.bucket}`); } - throw error; + throw createS3FileSystemError(error); } } diff --git a/packages/filesystem/webdav/error.ts b/packages/filesystem/webdav/error.ts new file mode 100644 index 000000000..4c7a9a86c --- /dev/null +++ b/packages/filesystem/webdav/error.ts @@ -0,0 +1,30 @@ +import { FileSystemError } from "../error"; + +type WebDAVLikeError = { + message?: string; + response?: { + status?: number; + }; +}; + +export function createWebDAVFileSystemError(error: unknown): unknown { + const webdavError = error as WebDAVLikeError; + const status = webdavError?.response?.status; + if (typeof status !== "number") { + return error; + } + + const rateLimit = status === 429; + + return new FileSystemError({ + provider: "webdav", + message: webdavError.message || `WebDAV request failed with status ${status}`, + status, + auth: status === 401 || status === 403, + notFound: status === 404, + conflict: status === 409 || status === 412, + rateLimit, + retryable: rateLimit || status >= 500, + raw: error, + }); +} diff --git a/packages/filesystem/webdav/rw.ts b/packages/filesystem/webdav/rw.ts index c13afe2f4..c3c2d43f9 100644 --- a/packages/filesystem/webdav/rw.ts +++ b/packages/filesystem/webdav/rw.ts @@ -1,5 +1,9 @@ import type { WebDAVClient } from "webdav"; -import type { FileReader, FileWriter } from "../filesystem"; +import type { FileCreateOptions, FileReader, FileWriter } from "../filesystem"; +import { FileSystemError } from "../error"; +import { createWebDAVFileSystemError } from "./error"; + +const quoteETag = (digest: string) => (digest.startsWith('"') && digest.endsWith('"') ? digest : `"${digest}"`); export class WebDAVFileReader implements FileReader { client: WebDAVClient; @@ -12,17 +16,21 @@ export class WebDAVFileReader implements FileReader { } async read(type?: "string" | "blob"): Promise { - switch (type) { - case "string": - return await (this.client.getFileContents(this.path, { - format: "text", - }) as Promise); - default: { - const resp = (await this.client.getFileContents(this.path, { - format: "binary", - })) as ArrayBuffer; - return new Blob([resp]); + try { + switch (type) { + case "string": + return await (this.client.getFileContents(this.path, { + format: "text", + }) as Promise); + default: { + const resp = (await this.client.getFileContents(this.path, { + format: "binary", + })) as ArrayBuffer; + return new Blob([resp]); + } } + } catch (error) { + throw createWebDAVFileSystemError(error); } } } @@ -32,16 +40,53 @@ export class WebDAVFileWriter implements FileWriter { path: string; - constructor(client: WebDAVClient, path: string) { + opts?: FileCreateOptions; + + constructor(client: WebDAVClient, path: string, opts?: FileCreateOptions) { this.client = client; this.path = path; + this.opts = opts; } async write(content: string | Blob): Promise { const data = content instanceof Blob ? await content.arrayBuffer() : content; - const resp = await this.client.putFileContents(this.path, data); + let resp: boolean; + try { + const opts = this.buildWriteOptions(); + if (opts) { + resp = await this.client.putFileContents(this.path, data, opts); + } else { + resp = await this.client.putFileContents(this.path, data); + } + } catch (error) { + throw createWebDAVFileSystemError(error); + } if (!resp) { + if (this.opts?.createOnly) { + throw new FileSystemError({ + provider: "webdav", + message: "WebDAV create-only write conflict", + status: 412, + conflict: true, + }); + } throw new Error("write error"); } } + + private buildWriteOptions() { + if (this.opts?.expectedDigest) { + return { + headers: { + "If-Match": quoteETag(this.opts.expectedDigest), + }, + }; + } + if (this.opts?.createOnly) { + return { + overwrite: false, + }; + } + return undefined; + } } diff --git a/packages/filesystem/webdav/webdav.test.ts b/packages/filesystem/webdav/webdav.test.ts index 4d91a5edb..5ac6fadd5 100644 --- a/packages/filesystem/webdav/webdav.test.ts +++ b/packages/filesystem/webdav/webdav.test.ts @@ -2,7 +2,7 @@ import { describe, it, expect, vi, beforeEach } from "vitest"; import type { WebDAVClient } from "webdav"; import { getPatcher } from "webdav"; import WebDAVFileSystem from "./webdav"; -import { WarpTokenError } from "../error"; +import { isConflictError, WarpTokenError } from "../error"; /** 创建 mock WebDAVClient */ function createMockClient(overrides?: Partial): WebDAVClient { @@ -32,6 +32,16 @@ describe("WebDAVFileSystem", () => { mockClient = createMockClient(); }); + it("应当声明支持原子条件写入和条件删除能力", () => { + const fs = createTestFS(mockClient); + + expect((fs as any).capabilities).toMatchObject({ + supportsAtomicCompareAndSwap: true, + supportsCreateOnly: true, + supportsConditionalDelete: true, + }); + }); + describe("initWebDAVPatch", () => { it("应当通过 getPatcher 注册 fetch patch,设置 credentials 为 omit", () => { // fromCredentials 内部调用 initWebDAVPatch,验证 patcher 已注册 fetch @@ -206,6 +216,18 @@ describe("WebDAVFileSystem", () => { await expect(fs.delete("missing.txt")).resolves.toBeUndefined(); }); + + it("条件删除应当将 expectedDigest 转成 If-Match", async () => { + const fs = createTestFS(mockClient); + + await (fs as any).delete("test.txt", { expectedDigest: '"abc123"' }); + + expect(mockClient.deleteFile).toHaveBeenCalledWith("/test.txt", { + headers: { + "If-Match": '"abc123"', + }, + }); + }); }); describe("create", () => { @@ -219,6 +241,69 @@ describe("WebDAVFileSystem", () => { expect((writer as any).path).toBe("/ScriptCat/sync/dir/file.user.js"); }); + + it("条件写入应当将 expectedDigest 转成 If-Match", async () => { + const fs = createTestFS(mockClient); + const writer = await (fs as any).create("test.txt", { expectedDigest: '"abc123"' }); + + await writer.write("content"); + + expect(mockClient.putFileContents).toHaveBeenCalledWith( + "/test.txt", + "content", + expect.objectContaining({ + headers: { + "If-Match": '"abc123"', + }, + }) + ); + }); + + it("createOnly 写入应当使用 WebDAV overwrite=false", async () => { + const fs = createTestFS(mockClient); + const writer = await (fs as any).create("new.txt", { createOnly: true }); + + await writer.write("content"); + + expect(mockClient.putFileContents).toHaveBeenCalledWith( + "/new.txt", + "content", + expect.objectContaining({ + overwrite: false, + }) + ); + }); + + it("createOnly 冲突返回 false 时应当抛出 typed conflict 错误", async () => { + (mockClient.putFileContents as ReturnType).mockResolvedValue(false); + const fs = createTestFS(mockClient); + const writer = await (fs as any).create("exists.txt", { createOnly: true }); + + await expect(writer.write("content")).rejects.toSatisfy(isConflictError); + }); + }); + + describe("open", () => { + it("读取文件遇到 503 时应当抛出 typed 可重试错误", async () => { + const err = new Error("Service Unavailable"); + (err as any).response = { status: 503 }; + (mockClient.getFileContents as ReturnType).mockRejectedValue(err); + const fs = createTestFS(mockClient); + const reader = await fs.open({ + name: "busy.user.js", + path: "/", + size: 1, + digest: "digest", + createtime: 1, + updatetime: 1, + }); + + await expect(reader.read("string")).rejects.toMatchObject({ + provider: "webdav", + status: 503, + retryable: true, + }); + }); }); describe("list", () => { @@ -270,6 +355,20 @@ describe("WebDAVFileSystem", () => { await expect(fs.list()).rejects.toThrow("Server Error"); }); + + it("列目录遇到 429 时应当抛出 typed 限流错误", async () => { + const err = new Error("Too Many Requests"); + (err as any).response = { status: 429 }; + (mockClient.getDirectoryContents as ReturnType).mockRejectedValue(err); + const fs = createTestFS(mockClient); + + await expect(fs.list()).rejects.toMatchObject({ + provider: "webdav", + status: 429, + rateLimit: true, + retryable: true, + }); + }); }); describe("getDirUrl", () => { diff --git a/packages/filesystem/webdav/webdav.ts b/packages/filesystem/webdav/webdav.ts index 871097537..db8440954 100644 --- a/packages/filesystem/webdav/webdav.ts +++ b/packages/filesystem/webdav/webdav.ts @@ -5,6 +5,8 @@ import type { FileInfo, FileCreateOptions, FileReader, FileWriter } from "../fil import { joinPath } from "../utils"; import { WebDAVFileReader, WebDAVFileWriter } from "./rw"; import { WarpTokenError } from "../error"; +import { createWebDAVFileSystemError } from "./error"; +import type { FileDeleteOptions } from "../filesystem"; // 禁止 WebDAV 请求携带浏览器 cookies,只通过账号密码认证 (#1297) // 全局单次注册 @@ -23,7 +25,15 @@ const initWebDAVPatch = () => { }); }; +const quoteETag = (digest: string) => (digest.startsWith('"') && digest.endsWith('"') ? digest : `"${digest}"`); + export default class WebDAVFileSystem implements FileSystem { + readonly capabilities = { + supportsAtomicCompareAndSwap: true, + supportsCreateOnly: true, + supportsConditionalDelete: true, + }; + client: WebDAVClient; url: string; @@ -75,8 +85,8 @@ export default class WebDAVFileSystem implements FileSystem { return WebDAVFileSystem.fromSameClient(this, joinPath(this.basePath, path)); } - async create(path: string, _opts?: FileCreateOptions): Promise { - return new WebDAVFileWriter(this.client, joinPath(this.basePath, path)); + async create(path: string, opts?: FileCreateOptions): Promise { + return new WebDAVFileWriter(this.client, joinPath(this.basePath, path), opts); } async createDir(path: string, _opts?: FileCreateOptions): Promise { @@ -87,18 +97,26 @@ export default class WebDAVFileSystem implements FileSystem { if (e.response?.status === 405 || e.message?.includes("405")) { return; } - throw e; + throw createWebDAVFileSystemError(e); } } - async delete(path: string): Promise { + async delete(path: string, opts?: FileDeleteOptions): Promise { try { + if (opts?.expectedDigest) { + await this.client.deleteFile(joinPath(this.basePath, path), { + headers: { + "If-Match": quoteETag(opts.expectedDigest), + }, + }); + return; + } await this.client.deleteFile(joinPath(this.basePath, path)); } catch (e: any) { if (e.response?.status === 404 || e.message?.includes("404")) { return; } - throw e; + throw createWebDAVFileSystemError(e); } } @@ -108,7 +126,7 @@ export default class WebDAVFileSystem implements FileSystem { dir = (await this.client.getDirectoryContents(this.basePath)) as FileStat[]; } catch (e: any) { if (e.response?.status === 404) return [] as FileInfo[]; // 目录不存在视为空 - throw e; + throw createWebDAVFileSystemError(e); } const ret: FileInfo[] = []; for (const item of dir) { diff --git a/src/app/service/service_worker/synchronize.test.ts b/src/app/service/service_worker/synchronize.test.ts index 4fc2ba742..3fb1a7e1c 100644 --- a/src/app/service/service_worker/synchronize.test.ts +++ b/src/app/service/service_worker/synchronize.test.ts @@ -2,6 +2,7 @@ import { describe, it, expect, vi, beforeEach } from "vitest"; import { SynchronizeService } from "./synchronize"; import { initTestEnv } from "@Tests/utils"; import type FileSystem from "@Packages/filesystem/filesystem"; +import { FileSystemError } from "@Packages/filesystem/error"; import type { CloudSyncConfig } from "@App/pkg/config/config"; import { stackAsyncTask } from "@App/pkg/utils/async_queue"; import { md5OfText } from "@App/pkg/utils/crypto"; @@ -187,31 +188,1380 @@ describe("SynchronizeService", () => { expect(written.status.scripts.orphan).toEqual(orphanStatus); }); - it("waits for installScript during pullScript", async () => { - let releaseInstall!: () => void; - const installGate = new Promise((resolve) => { - releaseInstall = resolve; + it("兼容缺少 status.scripts 的旧版 scriptcat-sync.json", async () => { + const writeMock = vi.fn().mockResolvedValue(undefined); + const fs = createFs({ + list: vi.fn().mockResolvedValue([ + { + name: "scriptcat-sync.json", + path: "scriptcat-sync.json", + size: 1, + digest: "sync-digest", + createtime: 1, + updatetime: 1, + }, + ]), + open: vi.fn().mockResolvedValue({ + read: vi.fn().mockResolvedValue(JSON.stringify({ version: "1.0.0" })), + }), + create: vi.fn().mockResolvedValue({ write: writeMock }), + }); + const service = new SynchronizeService( + {} as any, + {} as any, + {} as any, + {} as any, + {} as any, + {} as any, + {} as any, + { + scriptCodeDAO: {}, + all: vi.fn().mockResolvedValue([]), + } as any + ); + + await service.syncOnce(syncConfig, fs); + + expect(writeMock).toHaveBeenCalledTimes(1); + const written = JSON.parse(writeMock.mock.calls[0][0] as string); + expect(written.status.scripts).toEqual({}); + }); + + it("scriptcat-sync.json 损坏时不阻塞脚本同步且不覆盖状态文件", async () => { + const createMock = vi.fn().mockResolvedValue({ + write: vi.fn().mockResolvedValue(undefined), }); - const installScript = vi.fn().mockImplementation(() => installGate); const fs = createFs({ - open: vi.fn().mockImplementation(async (file) => ({ + list: vi + .fn() + .mockResolvedValueOnce([ + { + name: "scriptcat-sync.json", + path: "scriptcat-sync.json", + size: 1, + digest: "sync-digest", + createtime: 1, + updatetime: 1, + }, + ]) + .mockResolvedValueOnce([ + { + name: "ok.user.js", + path: "ok.user.js", + size: 1, + digest: "cloud-ok-user", + createtime: 1, + updatetime: 1, + }, + { + name: "ok.meta.json", + path: "ok.meta.json", + size: 1, + digest: "cloud-ok-meta", + createtime: 1, + updatetime: 1, + }, + ]), + open: vi.fn().mockResolvedValue({ + read: vi.fn().mockResolvedValue("{"), + }), + create: createMock, + }); + const service = new SynchronizeService( + {} as any, + {} as any, + {} as any, + {} as any, + {} as any, + {} as any, + {} as any, + { + scriptCodeDAO: {}, + all: vi.fn().mockResolvedValue([ + { + uuid: "ok", + name: "ok", + updatetime: 1, + createtime: 1, + status: 1, + sort: 0, + metadata: {}, + }, + ]), + } as any + ); + vi.spyOn(service, "pushScript").mockResolvedValue({ + "ok.user.js": "pushed-ok-user", + "ok.meta.json": "pushed-ok-meta", + }); + + await service.syncOnce(syncConfig, fs); + + expect(service.pushScript).toHaveBeenCalledTimes(1); + expect(createMock).not.toHaveBeenCalled(); + await expect((service as any).storage.get("file_digest")).resolves.toEqual({ + "ok.user.js": "cloud-ok-user", + "ok.meta.json": "cloud-ok-meta", + }); + }); + + it("写回 scriptcat-sync.json 前重新读取远端状态,避免覆盖其他设备更新", async () => { + const initialStatus = { enable: false, sort: 7, updatetime: 200 }; + const latestStatus = { enable: true, sort: 9, updatetime: 300 }; + const writeMock = vi.fn().mockResolvedValue(undefined); + const syncFile = { + name: "scriptcat-sync.json", + path: "scriptcat-sync.json", + size: 1, + digest: "sync-digest", + createtime: 1, + updatetime: 1, + }; + const fs = createFs({ + list: vi.fn().mockResolvedValue([ + { + name: "status-uuid.user.js", + path: "status-uuid.user.js", + size: 1, + digest: "script-digest", + createtime: 1, + updatetime: 1, + }, + { + name: "status-uuid.meta.json", + path: "status-uuid.meta.json", + size: 1, + digest: "meta-digest", + createtime: 1, + updatetime: 1, + }, + syncFile, + ]), + open: vi + .fn() + .mockResolvedValueOnce({ + read: vi.fn().mockResolvedValue( + JSON.stringify({ + version: "1.0.0", + status: { scripts: { "status-uuid": initialStatus } }, + }) + ), + }) + .mockResolvedValueOnce({ + read: vi.fn().mockResolvedValue( + JSON.stringify({ + version: "1.0.0", + status: { scripts: { "status-uuid": latestStatus } }, + }) + ), + }), + create: vi.fn().mockResolvedValue({ write: writeMock }), + }); + const scriptDAO = { + scriptCodeDAO: {}, + all: vi.fn().mockResolvedValue([ + { + uuid: "status-uuid", + name: "status", + updatetime: 100, + createtime: 1, + status: 1, + sort: 1, + metadata: {}, + }, + ]), + update: vi.fn().mockResolvedValue(undefined), + }; + const service = new SynchronizeService( + {} as any, + {} as any, + { + enableScript: vi.fn().mockResolvedValue(undefined), + } as any, + {} as any, + {} as any, + {} as any, + {} as any, + scriptDAO as any + ); + await (service as any).storage.set("file_digest", { + "status-uuid.user.js": "script-digest", + }); + + await service.syncOnce(syncConfig, fs); + + expect(fs.open).toHaveBeenCalledTimes(2); + const written = JSON.parse(writeMock.mock.calls[0][0] as string); + expect(written.status.scripts["status-uuid"]).toEqual(latestStatus); + }); + + it("写回 scriptcat-sync.json 时本地较新的状态仍覆盖远端旧状态", async () => { + const initialStatus = { enable: false, sort: 7, updatetime: 100 }; + const latestStatus = { enable: false, sort: 8, updatetime: 150 }; + const localStatus = { enable: true, sort: 1, updatetime: 200 }; + const writeMock = vi.fn().mockResolvedValue(undefined); + const fs = createFs({ + list: vi.fn().mockResolvedValue([ + { + name: "status-uuid.user.js", + path: "status-uuid.user.js", + size: 1, + digest: "script-digest", + createtime: 1, + updatetime: 1, + }, + { + name: "status-uuid.meta.json", + path: "status-uuid.meta.json", + size: 1, + digest: "meta-digest", + createtime: 1, + updatetime: 1, + }, + { + name: "scriptcat-sync.json", + path: "scriptcat-sync.json", + size: 1, + digest: "sync-digest", + createtime: 1, + updatetime: 1, + }, + ]), + open: vi + .fn() + .mockResolvedValueOnce({ + read: vi.fn().mockResolvedValue( + JSON.stringify({ + version: "1.0.0", + status: { scripts: { "status-uuid": initialStatus } }, + }) + ), + }) + .mockResolvedValueOnce({ + read: vi.fn().mockResolvedValue( + JSON.stringify({ + version: "1.0.0", + status: { scripts: { "status-uuid": latestStatus } }, + }) + ), + }), + create: vi.fn().mockResolvedValue({ write: writeMock }), + }); + const service = new SynchronizeService( + {} as any, + {} as any, + { + enableScript: vi.fn().mockResolvedValue(undefined), + } as any, + {} as any, + {} as any, + {} as any, + {} as any, + { + scriptCodeDAO: {}, + all: vi.fn().mockResolvedValue([ + { + uuid: "status-uuid", + name: "status", + updatetime: localStatus.updatetime, + createtime: 1, + status: 1, + sort: localStatus.sort, + metadata: {}, + }, + ]), + update: vi.fn().mockResolvedValue(undefined), + } as any + ); + await (service as any).storage.set("file_digest", { + "status-uuid.user.js": "script-digest", + }); + + await service.syncOnce(syncConfig, fs); + + expect(fs.open).toHaveBeenCalledTimes(2); + const written = JSON.parse(writeMock.mock.calls[0][0] as string); + expect(written.status.scripts["status-uuid"]).toEqual(localStatus); + }); + + it("syncStatus 单个脚本排序更新失败时不阻塞整轮同步", async () => { + const cloudStatus = { enable: true, sort: 9, updatetime: 200 }; + const writeMock = vi.fn().mockResolvedValue(undefined); + const fs = createFs({ + list: vi.fn().mockResolvedValue([ + { + name: "status-uuid.user.js", + path: "status-uuid.user.js", + size: 1, + digest: "script-digest", + createtime: 1, + updatetime: 1, + }, + { + name: "status-uuid.meta.json", + path: "status-uuid.meta.json", + size: 1, + digest: "meta-digest", + createtime: 1, + updatetime: 1, + }, + { + name: "scriptcat-sync.json", + path: "scriptcat-sync.json", + size: 1, + digest: "sync-digest", + createtime: 1, + updatetime: 1, + }, + ]), + open: vi.fn().mockResolvedValue({ read: vi.fn().mockResolvedValue( - file.name.endsWith(".user.js") - ? `// ==UserScript== -// @name Pull Test -// @namespace sync-test -// @match https://example.com/* -// ==/UserScript== -console.log("ok");` - : JSON.stringify({ uuid: "pull-uuid" }) + JSON.stringify({ + version: "1.0.0", + status: { scripts: { "status-uuid": cloudStatus } }, + }) ), - })), + }), + create: vi.fn().mockResolvedValue({ write: writeMock }), + }); + const scriptDAO = { + scriptCodeDAO: {}, + all: vi.fn().mockResolvedValue([ + { + uuid: "status-uuid", + name: "status", + updatetime: 100, + createtime: 1, + status: 1, + sort: 1, + metadata: {}, + }, + ]), + update: vi.fn().mockRejectedValue(new Error("sort update failed")), + }; + const service = new SynchronizeService( + {} as any, + {} as any, + { + enableScript: vi.fn().mockResolvedValue(undefined), + } as any, + {} as any, + {} as any, + {} as any, + {} as any, + scriptDAO as any + ); + await (service as any).storage.set("file_digest", { + "status-uuid.user.js": "script-digest", + "status-uuid.meta.json": "meta-digest", + "scriptcat-sync.json": "sync-digest", + }); + + await service.syncOnce(syncConfig, fs); + + expect(scriptDAO.update).toHaveBeenCalledWith("status-uuid", { sort: cloudStatus.sort }); + expect(writeMock).toHaveBeenCalledTimes(1); + const written = JSON.parse(writeMock.mock.calls[0][0] as string); + expect(written.status.scripts["status-uuid"]).toEqual(cloudStatus); + await expect((service as any).storage.get("file_digest")).resolves.toMatchObject({ + "status-uuid.user.js": "script-digest", + "status-uuid.meta.json": "meta-digest", + "scriptcat-sync.json": "sync-digest", + }); + }); + + it("脚本文件同步失败时仍可回写其它脚本的 syncStatus 并保留失败脚本云端状态", async () => { + const failedCloudStatus = { enable: false, sort: 3, updatetime: 300 }; + const okCloudStatus = { enable: false, sort: 1, updatetime: 100 }; + const okLocalStatus = { enable: true, sort: 9, updatetime: 200 }; + const writeMock = vi.fn().mockResolvedValue(undefined); + const fs = createFs({ + list: vi.fn().mockResolvedValue([ + { + name: "failed.user.js", + path: "failed.user.js", + size: 1, + digest: "failed-new-digest", + createtime: 1, + updatetime: 300, + }, + { + name: "failed.meta.json", + path: "failed.meta.json", + size: 1, + digest: "failed-meta-digest", + createtime: 1, + updatetime: 300, + }, + { + name: "ok.user.js", + path: "ok.user.js", + size: 1, + digest: "ok-digest", + createtime: 1, + updatetime: 1, + }, + { + name: "ok.meta.json", + path: "ok.meta.json", + size: 1, + digest: "ok-meta-digest", + createtime: 1, + updatetime: 1, + }, + { + name: "scriptcat-sync.json", + path: "scriptcat-sync.json", + size: 1, + digest: "sync-digest", + createtime: 1, + updatetime: 1, + }, + ]), + open: vi.fn().mockImplementation((file) => { + if (file.name === "scriptcat-sync.json") { + return Promise.resolve({ + read: vi.fn().mockResolvedValue( + JSON.stringify({ + version: "1.0.0", + status: { + scripts: { + failed: failedCloudStatus, + ok: okCloudStatus, + }, + }, + }) + ), + }); + } + return Promise.reject(new Error("read failed")); + }), + create: vi.fn().mockResolvedValue({ write: writeMock }), + }); + const service = new SynchronizeService( + {} as any, + {} as any, + {} as any, + {} as any, + {} as any, + {} as any, + {} as any, + { + scriptCodeDAO: {}, + all: vi.fn().mockResolvedValue([ + { + uuid: "failed", + name: "failed", + updatetime: 1, + createtime: 1, + status: 1, + sort: 1, + metadata: {}, + }, + { + uuid: "ok", + name: "ok", + updatetime: okLocalStatus.updatetime, + createtime: 1, + status: 1, + sort: okLocalStatus.sort, + metadata: {}, + }, + ]), + update: vi.fn().mockResolvedValue(undefined), + } as any + ); + await (service as any).storage.set("file_digest", { + "failed.user.js": "failed-old-digest", + "failed.meta.json": "failed-meta-digest", + "ok.user.js": "ok-digest", + "ok.meta.json": "ok-meta-digest", + }); + + await service.syncOnce(syncConfig, fs); + + expect(writeMock).toHaveBeenCalledTimes(1); + const written = JSON.parse(writeMock.mock.calls[0][0] as string); + expect(written.status.scripts.failed).toEqual(failedCloudStatus); + expect(written.status.scripts.ok).toEqual(okLocalStatus); + }); + + it("waits for installScript during pullScript", async () => { + let releaseInstall!: () => void; + const installGate = new Promise((resolve) => { + releaseInstall = resolve; + }); + const installScript = vi.fn().mockImplementation(() => installGate); + const fs = createFs({ + open: vi.fn().mockImplementation(async (file) => ({ + read: vi.fn().mockResolvedValue( + file.name.endsWith(".user.js") + ? `// ==UserScript== +// @name Pull Test +// @namespace sync-test +// @match https://example.com/* +// ==/UserScript== +console.log("ok");` + : JSON.stringify({ uuid: "pull-uuid" }) + ), + })), + }); + const service = new SynchronizeService( + {} as any, + {} as any, + { + installScript, + } as any, + {} as any, + {} as any, + {} as any, + {} as any, + { + scriptCodeDAO: {}, + } as any + ); + + let settled = false; + const pullPromise = service + .pullScript( + fs, + { + script: { + name: "pull-uuid.user.js", + path: "pull-uuid.user.js", + size: 1, + digest: "d1", + createtime: 1, + updatetime: 1, + }, + meta: { + name: "pull-uuid.meta.json", + path: "pull-uuid.meta.json", + size: 1, + digest: "d2", + createtime: 1, + updatetime: 1, + }, + }, + undefined + ) + .then(() => { + settled = true; + }); + + await Promise.resolve(); + expect(settled).toBe(false); + + releaseInstall(); + await pullPromise; + + expect(installScript).toHaveBeenCalledTimes(1); + expect(settled).toBe(true); + }); + + it("waits for deleteScript before updating file digest", async () => { + let releaseDelete!: () => void; + const deleteGate = new Promise((resolve) => { + releaseDelete = resolve; + }); + const order: string[] = []; + const deleteScript = vi.fn().mockImplementation(async () => { + order.push("delete:start"); + await deleteGate; + order.push("delete:end"); + }); + // fs.list 第二次调用对应 updateFileDigest,这是 syncOnce 的最后一步 + const fsList = vi + .fn() + .mockImplementationOnce(async () => [ + { + name: "del-uuid.meta.json", + path: "del-uuid.meta.json", + size: 1, + digest: "d1", + createtime: 1, + updatetime: 1, + }, + ]) + .mockImplementationOnce(async () => { + order.push("digest:list"); + return []; + }); + const fs = createFs({ + list: fsList, + open: vi.fn().mockResolvedValue({ + read: vi.fn().mockResolvedValue(JSON.stringify({ uuid: "del-uuid", isDeleted: true })), + }), + }); + const service = new SynchronizeService( + {} as any, + {} as any, + { deleteScript } as any, + {} as any, + {} as any, + {} as any, + {} as any, + { + scriptCodeDAO: {}, + all: vi.fn().mockResolvedValue([ + { + uuid: "del-uuid", + name: "del", + updatetime: 1, + createtime: 1, + status: 1, + sort: 0, + metadata: {}, + }, + ]), + } as any + ); + + const promise = service.syncOnce(syncConfig, fs); + await flushMicrotasks(); + + // delete 已经开始但没结束,updateFileDigest 还没被调用 + expect(order).toEqual(["delete:start"]); + + releaseDelete(); + await promise; + + // delete 必须在 updateFileDigest 之前完成 + expect(order).toEqual(["delete:start", "delete:end", "digest:list"]); + }); + + it("同一轮同步多个远端删除标记只发送一条删除通知", async () => { + const fs = createFs({ + list: vi + .fn() + .mockResolvedValueOnce([ + { + name: "first.meta.json", + path: "first.meta.json", + size: 1, + digest: "d1", + createtime: 1, + updatetime: 1, + }, + { + name: "second.meta.json", + path: "second.meta.json", + size: 1, + digest: "d2", + createtime: 1, + updatetime: 1, + }, + ]) + .mockResolvedValueOnce([]), + open: vi.fn().mockImplementation(async (file) => ({ + read: vi.fn().mockResolvedValue(JSON.stringify({ uuid: file.name.replace(".meta.json", ""), isDeleted: true })), + })), + }); + const deleteScript = vi.fn().mockResolvedValue(undefined); + const notificationSpy = vi.spyOn(chrome.notifications, "create"); + const service = new SynchronizeService( + {} as any, + {} as any, + { deleteScript } as any, + {} as any, + {} as any, + {} as any, + {} as any, + { + scriptCodeDAO: {}, + all: vi.fn().mockResolvedValue([ + { + uuid: "first", + name: "first", + updatetime: 1, + createtime: 1, + status: 1, + sort: 0, + metadata: {}, + }, + { + uuid: "second", + name: "second", + updatetime: 1, + createtime: 1, + status: 1, + sort: 1, + metadata: {}, + }, + ]), + } as any + ); + + await service.syncOnce({ ...syncConfig, syncStatus: false }, fs); + + expect(deleteScript).toHaveBeenCalledTimes(2); + expect(notificationSpy).toHaveBeenCalledTimes(1); + }); + + it("waits for pushScript before updating file digest", async () => { + let releasePush!: () => void; + const pushGate = new Promise((resolve) => { + releasePush = resolve; + }); + const order: string[] = []; + // pushScript 内部第一步是 fs.create(uuid.user.js),gate 在这里就能拦住整个 push + const fsCreate = vi.fn().mockImplementation(async (filename: string) => { + if (filename === "push-uuid.user.js") { + order.push("push:start"); + await pushGate; + order.push("push:end"); + } + return { write: vi.fn().mockResolvedValue(undefined) }; + }); + const fsList = vi + .fn() + .mockImplementationOnce(async () => []) + .mockImplementationOnce(async () => { + order.push("digest:list"); + return []; + }); + const fs = createFs({ + list: fsList, + create: fsCreate, + }); + const service = new SynchronizeService( + {} as any, + {} as any, + {} as any, + {} as any, + {} as any, + {} as any, + {} as any, + { + scriptCodeDAO: { + get: vi.fn().mockResolvedValue({ code: "// code" }), + }, + all: vi.fn().mockResolvedValue([ + { + uuid: "push-uuid", + name: "push", + updatetime: 1, + createtime: 1, + status: 1, + sort: 0, + metadata: {}, + }, + ]), + } as any + ); + + const promise = service.syncOnce(syncConfig, fs); + await flushMicrotasks(); + + // push 已经开始但没结束,updateFileDigest 还没被调用 + expect(order).toEqual(["push:start"]); + + releasePush(); + await promise; + + // push 必须在 updateFileDigest 之前完成 + expect(order).toContain("push:end"); + expect(order).toContain("digest:list"); + expect(order.indexOf("push:end")).toBeLessThan(order.indexOf("digest:list")); + }); + + it("keeps pushed script digest when cloud list is stale after upload", async () => { + const scriptCode = "// code"; + const script = { + uuid: "push-uuid", + name: "push", + origin: "origin", + downloadUrl: "download-url", + checkUpdateUrl: "check-update-url", + updatetime: 1, + createtime: 1, + status: 1, + sort: 0, + metadata: {}, + }; + const fs = createFs({ + list: vi.fn().mockResolvedValueOnce([]).mockResolvedValueOnce([]), + }); + const service = new SynchronizeService( + {} as any, + {} as any, + {} as any, + {} as any, + {} as any, + {} as any, + {} as any, + { + scriptCodeDAO: { + get: vi.fn().mockResolvedValue({ code: scriptCode }), + }, + all: vi.fn().mockResolvedValue([script]), + } as any + ); + + await service.syncOnce({ ...syncConfig, syncStatus: false }, fs); + + const metaJson = JSON.stringify({ + uuid: script.uuid, + origin: script.origin, + downloadUrl: script.downloadUrl, + checkUpdateUrl: script.checkUpdateUrl, + }); + await expect((service as any).storage.get("file_digest")).resolves.toEqual({ + "push-uuid.user.js": md5OfText(scriptCode), + "push-uuid.meta.json": md5OfText(metaJson), + }); + }); + + it("push 已有云端文件时应当用旧 digest 作为 expectedDigest", async () => { + const script = { + uuid: "push-uuid", + name: "push", + origin: "origin", + downloadUrl: "download-url", + checkUpdateUrl: "check-update-url", + updatetime: 10, + createtime: 1, + status: 1, + sort: 0, + metadata: {}, + }; + const fs = createFs({ + capabilities: { + supportsAtomicCompareAndSwap: true, + }, + list: vi + .fn() + .mockResolvedValueOnce([ + { + name: "push-uuid.user.js", + path: "push-uuid.user.js", + size: 1, + digest: "cloud-user-new", + createtime: 1, + updatetime: 1, + }, + { + name: "push-uuid.meta.json", + path: "push-uuid.meta.json", + size: 1, + digest: "cloud-meta-new", + createtime: 1, + updatetime: 1, + }, + ]) + .mockResolvedValueOnce([]), + }); + const service = new SynchronizeService( + {} as any, + {} as any, + {} as any, + {} as any, + {} as any, + {} as any, + {} as any, + { + scriptCodeDAO: { + get: vi.fn().mockResolvedValue({ code: "// code" }), + }, + all: vi.fn().mockResolvedValue([script]), + } as any + ); + await (service as any).storage.set("file_digest", { + "push-uuid.user.js": "old-user-digest", + "push-uuid.meta.json": "old-meta-digest", + }); + + await service.syncOnce({ ...syncConfig, syncStatus: false }, fs); + + expect(fs.create).toHaveBeenCalledWith( + "push-uuid.user.js", + expect.objectContaining({ expectedDigest: "old-user-digest" }) + ); + expect(fs.create).toHaveBeenCalledWith( + "push-uuid.meta.json", + expect.objectContaining({ expectedDigest: "old-meta-digest" }) + ); + }); + + it("push 云端缺失文件时应当使用 createOnly,避免覆盖并发新增", async () => { + const script = { + uuid: "new-uuid", + name: "new", + origin: "origin", + downloadUrl: "download-url", + checkUpdateUrl: "check-update-url", + updatetime: 10, + createtime: 1, + status: 1, + sort: 0, + metadata: {}, + }; + const fs = createFs({ + capabilities: { + supportsCreateOnly: true, + }, + list: vi.fn().mockResolvedValueOnce([]).mockResolvedValueOnce([]), + }); + const service = new SynchronizeService( + {} as any, + {} as any, + {} as any, + {} as any, + {} as any, + {} as any, + {} as any, + { + scriptCodeDAO: { + get: vi.fn().mockResolvedValue({ code: "// code" }), + }, + all: vi.fn().mockResolvedValue([script]), + } as any + ); + + await service.syncOnce({ ...syncConfig, syncStatus: false }, fs); + + expect(fs.create).toHaveBeenCalledWith("new-uuid.user.js", expect.objectContaining({ createOnly: true })); + expect(fs.create).toHaveBeenCalledWith("new-uuid.meta.json", expect.objectContaining({ createOnly: true })); + }); + + it("没有能力声明时 push 不应传条件写入参数", async () => { + const script = { + uuid: "push-uuid", + name: "push", + origin: "origin", + downloadUrl: "download-url", + checkUpdateUrl: "check-update-url", + updatetime: 10, + createtime: 1, + status: 1, + sort: 0, + metadata: {}, + }; + const fs = createFs({ + list: vi + .fn() + .mockResolvedValueOnce([ + { + name: "push-uuid.user.js", + path: "push-uuid.user.js", + size: 1, + digest: "cloud-user-new", + createtime: 1, + updatetime: 1, + }, + ]) + .mockResolvedValueOnce([]), + }); + const service = new SynchronizeService( + {} as any, + {} as any, + {} as any, + {} as any, + {} as any, + {} as any, + {} as any, + { + scriptCodeDAO: { + get: vi.fn().mockResolvedValue({ code: "// code" }), + }, + all: vi.fn().mockResolvedValue([script]), + } as any + ); + await (service as any).storage.set("file_digest", { + "push-uuid.user.js": "old-user-digest", + }); + + await service.syncOnce({ ...syncConfig, syncStatus: false }, fs); + + expect(fs.create).toHaveBeenCalledWith( + "push-uuid.user.js", + expect.not.objectContaining({ + expectedDigest: expect.anything(), + createOnly: expect.anything(), + }) + ); + }); + + it("部分 push 失败时只推进成功文件 digest 并保留失败文件旧 digest", async () => { + const fs = createFs({ + list: vi + .fn() + .mockResolvedValueOnce([]) + .mockResolvedValueOnce([ + { + name: "ok.user.js", + path: "ok.user.js", + size: 1, + digest: "cloud-ok-new", + createtime: 1, + updatetime: 1, + }, + { + name: "bad.user.js", + path: "bad.user.js", + size: 1, + digest: "cloud-bad-new", + createtime: 1, + updatetime: 1, + }, + { + name: "bad.meta.json", + path: "bad.meta.json", + size: 1, + digest: "cloud-bad-meta-new", + createtime: 1, + updatetime: 1, + }, + ]), + }); + const service = new SynchronizeService( + {} as any, + {} as any, + {} as any, + {} as any, + {} as any, + {} as any, + {} as any, + { + scriptCodeDAO: {}, + all: vi.fn().mockResolvedValue([ + { + uuid: "ok", + name: "ok", + updatetime: 1, + createtime: 1, + status: 1, + sort: 0, + metadata: {}, + }, + { + uuid: "bad", + name: "bad", + updatetime: 1, + createtime: 1, + status: 1, + sort: 1, + metadata: {}, + }, + ]), + } as any + ); + vi.spyOn(service, "pushScript").mockImplementation(async (_fs, script: any) => { + if (script.uuid === "bad") { + throw new Error("push failed"); + } + return { + "ok.user.js": "ok-user-new", + "ok.meta.json": "ok-meta-new", + }; + }); + await (service as any).storage.set("file_digest", { + "bad.user.js": "bad-user-old", + "bad.meta.json": "bad-meta-old", + }); + + await service.syncOnce({ ...syncConfig, syncStatus: false }, fs); + + await expect((service as any).storage.get("file_digest")).resolves.toEqual({ + "ok.user.js": "cloud-ok-new", + "bad.user.js": "bad-user-old", + "bad.meta.json": "bad-meta-old", + "ok.meta.json": "ok-meta-new", + }); + }); + + it("单文件同步遇到 typed conflict 时应在日志中标记 conflict 分类", async () => { + const fs = createFs({ + list: vi + .fn() + .mockResolvedValueOnce([]) + .mockResolvedValueOnce([ + { + name: "ok.user.js", + path: "ok.user.js", + size: 1, + digest: "cloud-ok-new", + createtime: 1, + updatetime: 1, + }, + { + name: "bad.user.js", + path: "bad.user.js", + size: 1, + digest: "cloud-bad-new", + createtime: 1, + updatetime: 1, + }, + ]), + }); + const service = new SynchronizeService( + {} as any, + {} as any, + {} as any, + {} as any, + {} as any, + {} as any, + {} as any, + { + scriptCodeDAO: {}, + all: vi.fn().mockResolvedValue([ + { + uuid: "ok", + name: "ok", + updatetime: 1, + createtime: 1, + status: 1, + sort: 0, + metadata: {}, + }, + { + uuid: "bad", + name: "bad", + updatetime: 1, + createtime: 1, + status: 1, + sort: 1, + metadata: {}, + }, + ]), + } as any + ); + const warnSpy = vi.spyOn(service.logger, "warn"); + vi.spyOn(service, "pushScript").mockImplementation(async (_fs, script: any) => { + if (script.uuid === "bad") { + throw new FileSystemError({ + provider: "s3", + message: "Precondition failed", + status: 412, + conflict: true, + }); + } + return { "ok.user.js": "ok-user-new" }; + }); + await (service as any).storage.set("file_digest", { + "bad.user.js": "bad-user-old", + }); + + await service.syncOnce({ ...syncConfig, syncStatus: false }, fs); + + expect(warnSpy).toHaveBeenCalledWith( + "sync task failed", + expect.objectContaining({ error: "Precondition failed" }), + expect.objectContaining({ + errorKind: "conflict", + files: ["bad.user.js", "bad.meta.json"], + }) + ); + await expect((service as any).storage.get("file_digest")).resolves.toMatchObject({ + "ok.user.js": "cloud-ok-new", + "bad.user.js": "bad-user-old", + }); + }); + + it.each([ + { + title: "typed transient", + error: new FileSystemError({ + provider: "webdav", + message: "Service unavailable", + status: 503, + retryable: true, + }), + expectedKind: "transient", + expectedMessage: "Service unavailable", + }, + { + title: "typed stale snapshot", + error: new FileSystemError({ + provider: "onedrive", + message: "File moved", + status: 404, + notFound: true, + }), + expectedKind: "stale_snapshot", + expectedMessage: "File moved", + }, + { + title: "unsupported", + error: new Error("unsupported conditional write"), + expectedKind: "unsupported", + expectedMessage: "unsupported conditional write", + }, + ])( + "单文件同步遇到 $title 错误时应在日志中标记 $expectedKind 分类", + async ({ error, expectedKind, expectedMessage }) => { + const fs = createFs({ + list: vi + .fn() + .mockResolvedValueOnce([]) + .mockResolvedValueOnce([ + { + name: "bad.user.js", + path: "bad.user.js", + size: 1, + digest: "cloud-bad-new", + createtime: 1, + updatetime: 1, + }, + ]), + }); + const service = new SynchronizeService( + {} as any, + {} as any, + {} as any, + {} as any, + {} as any, + {} as any, + {} as any, + { + scriptCodeDAO: {}, + all: vi.fn().mockResolvedValue([ + { + uuid: "bad", + name: "bad", + updatetime: 1, + createtime: 1, + status: 1, + sort: 0, + metadata: {}, + }, + ]), + } as any + ); + const warnSpy = vi.spyOn(service.logger, "warn"); + vi.spyOn(service, "pushScript").mockRejectedValue(error); + await (service as any).storage.set("file_digest", { + "bad.user.js": "bad-user-old", + }); + + await service.syncOnce({ ...syncConfig, syncStatus: false }, fs); + + expect(warnSpy).toHaveBeenCalledWith( + "sync task failed", + expect.objectContaining({ error: expectedMessage }), + expect.objectContaining({ + errorKind: expectedKind, + files: ["bad.user.js", "bad.meta.json"], + }) + ); + await expect((service as any).storage.get("file_digest")).resolves.toMatchObject({ + "bad.user.js": "bad-user-old", + }); + } + ); + + it("scriptcat-sync.json 写回失败时仍推进已成功脚本 digest", async () => { + const fs = createFs({ + list: vi + .fn() + .mockResolvedValueOnce([]) + .mockResolvedValueOnce([ + { + name: "ok.user.js", + path: "ok.user.js", + size: 1, + digest: "cloud-ok-user", + createtime: 1, + updatetime: 1, + }, + { + name: "ok.meta.json", + path: "ok.meta.json", + size: 1, + digest: "cloud-ok-meta", + createtime: 1, + updatetime: 1, + }, + ]), + create: vi.fn().mockResolvedValue({ + write: vi.fn().mockRejectedValue(new Error("status write failed")), + }), + }); + const service = new SynchronizeService( + {} as any, + {} as any, + {} as any, + {} as any, + {} as any, + {} as any, + {} as any, + { + scriptCodeDAO: {}, + all: vi.fn().mockResolvedValue([ + { + uuid: "ok", + name: "ok", + updatetime: 1, + createtime: 1, + status: 1, + sort: 0, + metadata: {}, + }, + ]), + } as any + ); + vi.spyOn(service, "pushScript").mockResolvedValue({ + "ok.user.js": "pushed-ok-user", + "ok.meta.json": "pushed-ok-meta", + }); + + await service.syncOnce(syncConfig, fs); + + await expect((service as any).storage.get("file_digest")).resolves.toEqual({ + "ok.user.js": "cloud-ok-user", + "ok.meta.json": "cloud-ok-meta", + }); + }); + + it("pullScript 失败时不推进对应云端 digest", async () => { + const fs = createFs({ + list: vi + .fn() + .mockResolvedValueOnce([ + { + name: "pull-uuid.user.js", + path: "pull-uuid.user.js", + size: 1, + digest: "cloud-user-new", + createtime: 1, + updatetime: 10, + }, + { + name: "pull-uuid.meta.json", + path: "pull-uuid.meta.json", + size: 1, + digest: "cloud-meta-new", + createtime: 1, + updatetime: 10, + }, + ]) + .mockResolvedValueOnce([ + { + name: "pull-uuid.user.js", + path: "pull-uuid.user.js", + size: 1, + digest: "cloud-user-new", + createtime: 1, + updatetime: 10, + }, + { + name: "pull-uuid.meta.json", + path: "pull-uuid.meta.json", + size: 1, + digest: "cloud-meta-new", + createtime: 1, + updatetime: 10, + }, + ]), + open: vi.fn().mockResolvedValue({ + read: vi.fn().mockRejectedValue(new Error("read failed")), + }), }); const service = new SynchronizeService( {} as any, {} as any, { - installScript, + installScript: vi.fn().mockResolvedValue(undefined), } as any, {} as any, {} as any, @@ -219,143 +1569,183 @@ console.log("ok");` {} as any, { scriptCodeDAO: {}, + all: vi.fn().mockResolvedValue([]), } as any ); + await (service as any).storage.set("file_digest", { + "pull-uuid.user.js": "user-old", + "pull-uuid.meta.json": "meta-old", + }); - let settled = false; - const pullPromise = service - .pullScript( - fs, - { - script: { - name: "pull-uuid.user.js", - path: "pull-uuid.user.js", - size: 1, - digest: "d1", - createtime: 1, - updatetime: 1, - }, - meta: { - name: "pull-uuid.meta.json", - path: "pull-uuid.meta.json", - size: 1, - digest: "d2", - createtime: 1, - updatetime: 1, - }, - }, - undefined - ) - .then(() => { - settled = true; - }); - - await Promise.resolve(); - expect(settled).toBe(false); - - releaseInstall(); - await pullPromise; + await service.syncOnce({ ...syncConfig, syncStatus: false }, fs); - expect(installScript).toHaveBeenCalledTimes(1); - expect(settled).toBe(true); + await expect((service as any).storage.get("file_digest")).resolves.toEqual({ + "pull-uuid.user.js": "user-old", + "pull-uuid.meta.json": "meta-old", + }); }); - it("waits for deleteScript before updating file digest", async () => { - let releaseDelete!: () => void; - const deleteGate = new Promise((resolve) => { - releaseDelete = resolve; - }); - const order: string[] = []; - const deleteScript = vi.fn().mockImplementation(async () => { - order.push("delete:start"); - await deleteGate; - order.push("delete:end"); - }); - // fs.list 第二次调用对应 updateFileDigest,这是 syncOnce 的最后一步 - const fsList = vi - .fn() - .mockImplementationOnce(async () => [ + it("批量删除单条云端删除失败时继续处理后续脚本并保留失败 digest", async () => { + const deleteCalls: string[] = []; + const fs = createFs({ + delete: vi.fn().mockImplementation(async (path: string) => { + deleteCalls.push(path); + if (path === "fail.user.js") { + throw new Error("delete failed"); + } + }), + list: vi.fn().mockResolvedValue([ { - name: "del-uuid.meta.json", - path: "del-uuid.meta.json", + name: "fail.user.js", + path: "fail.user.js", size: 1, - digest: "d1", + digest: "fail-user-new", createtime: 1, updatetime: 1, }, - ]) - .mockImplementationOnce(async () => { - order.push("digest:list"); - return []; - }); - const fs = createFs({ - list: fsList, - open: vi.fn().mockResolvedValue({ - read: vi.fn().mockResolvedValue(JSON.stringify({ uuid: "del-uuid", isDeleted: true })), - }), + { + name: "fail.meta.json", + path: "fail.meta.json", + size: 1, + digest: "fail-meta-new", + createtime: 1, + updatetime: 1, + }, + ]), }); const service = new SynchronizeService( {} as any, {} as any, - { deleteScript } as any, {} as any, {} as any, {} as any, {} as any, + { + getCloudSync: vi.fn().mockResolvedValue({ ...syncConfig, enable: true, syncDelete: false }), + } as any, { scriptCodeDAO: {}, - all: vi.fn().mockResolvedValue([ - { - uuid: "del-uuid", - name: "del", - updatetime: 1, - createtime: 1, - status: 1, - sort: 0, - metadata: {}, - }, - ]), + all: vi.fn().mockResolvedValue([]), } as any ); + vi.spyOn(service as any, "buildFileSystem").mockResolvedValue(fs); + const warnSpy = vi.spyOn(service.logger, "warn"); + await (service as any).storage.set("file_digest", { + "fail.user.js": "fail-user-old", + "fail.meta.json": "fail-meta-old", + }); - const promise = service.syncOnce(syncConfig, fs); - await flushMicrotasks(); - - // delete 已经开始但没结束,updateFileDigest 还没被调用 - expect(order).toEqual(["delete:start"]); - - releaseDelete(); - await promise; + await service.scriptsDelete([{ uuid: "fail", deleteBy: "user" } as any, { uuid: "ok", deleteBy: "user" } as any]); + await stackAsyncTask("cloud_sync_queue", async () => "barrier"); - // delete 必须在 updateFileDigest 之前完成 - expect(order).toEqual(["delete:start", "delete:end", "digest:list"]); + expect(deleteCalls).toEqual(["fail.user.js", "ok.user.js", "ok.meta.json"]); + expect(warnSpy).toHaveBeenCalledWith( + "delete cloud script item failed", + expect.objectContaining({ error: "delete failed" }), + expect.objectContaining({ uuid: "fail", errorKind: "fatal" }) + ); + await expect((service as any).storage.get("file_digest")).resolves.toEqual({ + "fail.user.js": "fail-user-old", + "fail.meta.json": "fail-meta-old", + }); }); - it("waits for pushScript before updating file digest", async () => { - let releasePush!: () => void; - const pushGate = new Promise((resolve) => { - releasePush = resolve; - }); - const order: string[] = []; - // pushScript 内部第一步是 fs.create(uuid.user.js),gate 在这里就能拦住整个 push - const fsCreate = vi.fn().mockImplementation(async (filename: string) => { - if (filename === "push-uuid.user.js") { - order.push("push:start"); - await pushGate; - order.push("push:end"); - } - return { write: vi.fn().mockResolvedValue(undefined) }; - }); - const fsList = vi - .fn() - .mockImplementationOnce(async () => []) - .mockImplementationOnce(async () => { - order.push("digest:list"); - return []; + it.each([ + { + title: "transient", + error: new FileSystemError({ + provider: "webdav", + message: "Service unavailable", + status: 503, + retryable: true, + }), + expectedKind: "transient", + expectedMessage: "Service unavailable", + }, + { + title: "conflict", + error: new FileSystemError({ + provider: "s3", + message: "Precondition failed", + status: 412, + conflict: true, + }), + expectedKind: "conflict", + expectedMessage: "Precondition failed", + }, + ])( + "批量删除遇到 typed $title 失败时只阻塞对应脚本并标记 $expectedKind", + async ({ error, expectedKind, expectedMessage }) => { + const deleteCalls: string[] = []; + const fs = createFs({ + delete: vi.fn().mockImplementation(async (path: string) => { + deleteCalls.push(path); + if (path === "fail.user.js") { + throw error; + } + }), + list: vi.fn().mockResolvedValue([ + { + name: "fail.user.js", + path: "fail.user.js", + size: 1, + digest: "fail-user-new", + createtime: 1, + updatetime: 1, + }, + { + name: "fail.meta.json", + path: "fail.meta.json", + size: 1, + digest: "fail-meta-new", + createtime: 1, + updatetime: 1, + }, + ]), }); + const service = new SynchronizeService( + {} as any, + {} as any, + {} as any, + {} as any, + {} as any, + {} as any, + { + getCloudSync: vi.fn().mockResolvedValue({ ...syncConfig, enable: true, syncDelete: false }), + } as any, + { + scriptCodeDAO: {}, + all: vi.fn().mockResolvedValue([]), + } as any + ); + vi.spyOn(service as any, "buildFileSystem").mockResolvedValue(fs); + const warnSpy = vi.spyOn(service.logger, "warn"); + await (service as any).storage.set("file_digest", { + "fail.user.js": "fail-user-old", + "fail.meta.json": "fail-meta-old", + }); + + await service.scriptsDelete([{ uuid: "fail", deleteBy: "user" } as any, { uuid: "ok", deleteBy: "user" } as any]); + await stackAsyncTask("cloud_sync_queue", async () => "barrier"); + + expect(deleteCalls).toEqual(["fail.user.js", "ok.user.js", "ok.meta.json"]); + expect(warnSpy).toHaveBeenCalledWith( + "delete cloud script item failed", + expect.objectContaining({ error: expectedMessage }), + expect.objectContaining({ uuid: "fail", errorKind: expectedKind }) + ); + await expect((service as any).storage.get("file_digest")).resolves.toEqual({ + "fail.user.js": "fail-user-old", + "fail.meta.json": "fail-meta-old", + }); + } + ); + + it("deleteCloudScript 支持条件删除时应当传 expectedDigest", async () => { const fs = createFs({ - list: fsList, - create: fsCreate, + capabilities: { + supportsConditionalDelete: true, + }, }); const service = new SynchronizeService( {} as any, @@ -366,55 +1756,29 @@ console.log("ok");` {} as any, {} as any, { - scriptCodeDAO: { - get: vi.fn().mockResolvedValue({ code: "// code" }), - }, - all: vi.fn().mockResolvedValue([ - { - uuid: "push-uuid", - name: "push", - updatetime: 1, - createtime: 1, - status: 1, - sort: 0, - metadata: {}, - }, - ]), + scriptCodeDAO: {}, + all: vi.fn().mockResolvedValue([]), } as any ); + await (service as any).storage.set("file_digest", { + "delete-uuid.user.js": "old-user-digest", + "delete-uuid.meta.json": "old-meta-digest", + }); - const promise = service.syncOnce(syncConfig, fs); - await flushMicrotasks(); - - // push 已经开始但没结束,updateFileDigest 还没被调用 - expect(order).toEqual(["push:start"]); - - releasePush(); - await promise; + await service.deleteCloudScript(fs, "delete-uuid", false); - // push 必须在 updateFileDigest 之前完成 - expect(order).toContain("push:end"); - expect(order).toContain("digest:list"); - expect(order.indexOf("push:end")).toBeLessThan(order.indexOf("digest:list")); + expect(fs.delete).toHaveBeenCalledWith( + "delete-uuid.user.js", + expect.objectContaining({ expectedDigest: "old-user-digest" }) + ); + expect(fs.delete).toHaveBeenCalledWith( + "delete-uuid.meta.json", + expect.objectContaining({ expectedDigest: "old-meta-digest" }) + ); }); - it("keeps pushed script digest when cloud list is stale after upload", async () => { - const scriptCode = "// code"; - const script = { - uuid: "push-uuid", - name: "push", - origin: "origin", - downloadUrl: "download-url", - checkUpdateUrl: "check-update-url", - updatetime: 1, - createtime: 1, - status: 1, - sort: 0, - metadata: {}, - }; - const fs = createFs({ - list: vi.fn().mockResolvedValueOnce([]).mockResolvedValueOnce([]), - }); + it("deleteCloudScript 无条件删除能力时不应传 expectedDigest", async () => { + const fs = createFs(); const service = new SynchronizeService( {} as any, {} as any, @@ -424,25 +1788,19 @@ console.log("ok");` {} as any, {} as any, { - scriptCodeDAO: { - get: vi.fn().mockResolvedValue({ code: scriptCode }), - }, - all: vi.fn().mockResolvedValue([script]), + scriptCodeDAO: {}, + all: vi.fn().mockResolvedValue([]), } as any ); + await (service as any).storage.set("file_digest", { + "delete-uuid.user.js": "old-user-digest", + "delete-uuid.meta.json": "old-meta-digest", + }); - await service.syncOnce({ ...syncConfig, syncStatus: false }, fs); + await service.deleteCloudScript(fs, "delete-uuid", false); - const metaJson = JSON.stringify({ - uuid: script.uuid, - origin: script.origin, - downloadUrl: script.downloadUrl, - checkUpdateUrl: script.checkUpdateUrl, - }); - await expect((service as any).storage.get("file_digest")).resolves.toEqual({ - "push-uuid.user.js": md5OfText(scriptCode), - "push-uuid.meta.json": md5OfText(metaJson), - }); + expect((fs.delete as any).mock.calls[0]).toEqual(["delete-uuid.user.js"]); + expect((fs.delete as any).mock.calls[1]).toEqual(["delete-uuid.meta.json"]); }); it("passes script modifiedDate when pushing script and meta files", async () => { @@ -674,6 +2032,56 @@ console.log("ok");` expect(order).toEqual(["sync:list", "sync:digest", "install:push", "install:digest"]); }); + it("scriptInstall 触发 push transient 失败时不污染 file_digest", async () => { + const fs = createFs(); + const service = new SynchronizeService( + {} as any, + {} as any, + {} as any, + {} as any, + {} as any, + {} as any, + { + getCloudSync: vi.fn().mockResolvedValue({ ...syncConfig, enable: true }), + } as any, + { + scriptCodeDAO: {}, + all: vi.fn().mockResolvedValue([]), + } as any + ); + const transientError = new FileSystemError({ + provider: "webdav", + message: "Service unavailable", + status: 503, + retryable: true, + }); + vi.spyOn(service as any, "buildFileSystem").mockResolvedValue(fs); + vi.spyOn(service, "pushScript").mockRejectedValue(transientError); + const updateSpy = vi.spyOn(service, "updateFileDigest"); + const errorSpy = vi.spyOn(service.logger, "error").mockImplementation(() => undefined as any); + await (service as any).storage.set("file_digest", { + "install-uuid.user.js": "old-user-digest", + "install-uuid.meta.json": "old-meta-digest", + }); + + await service.scriptInstall({ + script: { uuid: "install-uuid", name: "install" } as any, + upsertBy: "user", + } as any); + await stackAsyncTask("cloud_sync_queue", async () => "barrier"); + + expect(updateSpy).not.toHaveBeenCalled(); + expect(errorSpy).toHaveBeenCalledWith( + "push script on install error", + expect.objectContaining({ error: "Service unavailable" }), + expect.objectContaining({ errorKind: "transient" }) + ); + await expect((service as any).storage.get("file_digest")).resolves.toEqual({ + "install-uuid.user.js": "old-user-digest", + "install-uuid.meta.json": "old-meta-digest", + }); + }); + it("scriptsDelete enters cloud_sync queue and updates digest after deleting", async () => { let releaseSync!: () => void; const syncGate = new Promise((resolve) => { diff --git a/src/app/service/service_worker/synchronize.ts b/src/app/service/service_worker/synchronize.ts index 371f19178..c44d9aaee 100644 --- a/src/app/service/service_worker/synchronize.ts +++ b/src/app/service/service_worker/synchronize.ts @@ -10,11 +10,12 @@ import { } from "@App/app/repo/scripts"; import BackupExport from "@App/pkg/backup/export"; import type { BackupData, ResourceBackup, ScriptBackupData, ScriptOptions, ValueStorage } from "@App/pkg/backup/struct"; -import type { FileInfo } from "@Packages/filesystem/filesystem"; +import type { FileCreateOptions, FileInfo } from "@Packages/filesystem/filesystem"; import type FileSystem from "@Packages/filesystem/filesystem"; +import { getFileSystemCapabilities } from "@Packages/filesystem/filesystem"; import ZipFileSystem from "@Packages/filesystem/zip/zip"; import FileSystemFactory, { type FileSystemType } from "@Packages/filesystem/factory"; -import { isWarpTokenError } from "@Packages/filesystem/error"; +import { FileSystemError, isWarpTokenError } from "@Packages/filesystem/error"; import type { Group } from "@Packages/message/server"; import type { MessageSend } from "@Packages/message/types"; import { type IMessageQueue } from "@Packages/message/message_queue"; @@ -73,6 +74,20 @@ type FileDigestMap = { [key: string]: string; }; +type PushScriptOptions = { + fileDigestMap?: FileDigestMap; + scriptFile?: FileInfo; + metaFile?: FileInfo; +}; + +type SyncTask = { + uuid: string; + promise: Promise; + preserveDigestFiles: string[]; +}; + +type SyncErrorKind = "conflict" | "stale_snapshot" | "transient" | "unsupported" | "fatal"; + const SYNC_SERVICE_TASK_KEY = "cloud_sync_queue"; function getScriptModifiedDate(script: PushScriptParam): number { @@ -397,27 +412,44 @@ export class SynchronizeService { }, } as ScriptcatSync; let cloudStatus: ScriptcatSync["status"]["scripts"] = {}; + let canWriteScriptcatSync = true; if (file) { - // 如果有,则读取文件内容 - const cloudScriptCatSync = JSON.parse(await fs.open(file).then((f) => f.read("string"))) as ScriptcatSync; - cloudStatus = cloudScriptCatSync.status.scripts; + try { + // 如果有,则读取文件内容 + const cloudScriptCatSync = JSON.parse( + await fs.open(file).then((f) => f.read("string")) + ) as Partial; + cloudStatus = cloudScriptCatSync.status?.scripts || {}; + } catch (e) { + canWriteScriptcatSync = false; + this.logger.warn("read scriptcat-sync.json file failed", Logger.E(e)); + } } // 对比脚本列表和文件列表,进行同步 - const result: Promise[] = []; + const result: SyncTask[] = []; const updateScript: Map = new Map(); // 记录被跳过的孤儿云端脚本(仅 .user.js 无 .meta.json) // 避免本机回写 scriptcat-sync.json 时丢失对应 uuid 的云端 status const skippedOrphanUuids = new Set(); + let hasNotifiedSyncDelete = false; // 需要是同步操作,后续上传剩下的脚本 // 最后使用 Promise.allSettled 进行等待 + const addSyncTask = (uuid: string, promise: Promise, files?: string[]) => { + result.push({ + uuid, + promise, + preserveDigestFiles: files || [`${uuid}.user.js`, `${uuid}.meta.json`], + }); + }; uuidMap.forEach((file, uuid) => { const script = scriptMap.get(uuid); if (script) { scriptMap.delete(uuid); // 脚本存在但是文件不存在,则读取.meta.json内容判断是否需要删除脚本 if (!file.script) { - result.push( + addSyncTask( + uuid, (async () => { // 读取meta文件 const meta = await fs.open(file.meta!); @@ -426,18 +458,22 @@ export class SynchronizeService { if (metaObj.isDeleted) { // 删除脚本 await this.script.deleteScript(script.uuid, "sync"); - InfoNotification( - i18n.t("notification.script_sync_delete"), - i18n.t("notification.script_sync_delete_desc", { - scriptName: i18nName(script), - }) - ); + if (!hasNotifiedSyncDelete) { + hasNotifiedSyncDelete = true; + InfoNotification( + i18n.t("notification.script_sync_delete"), + i18n.t("notification.script_sync_delete_desc", { + scriptName: i18nName(script), + }) + ); + } } else { // 否则认为是一个无效的.meta文件,进行删除,并进行同步 await fs.delete(file.meta!.name); - return await this.pushScript(fs, script); + return await this.pushScript(fs, script, { fileDigestMap }); } - })() + })(), + [file.meta!.name, `${uuid}.user.js`] ); return; } @@ -450,11 +486,14 @@ export class SynchronizeService { if (updatetime > file.script!.updatetime || !file.meta) { // 如果脚本更新时间大于文件更新时间 // 或者不存在.meta文件,则上传文件 - result.push(this.pushScript(fs, script)); + addSyncTask( + uuid, + this.pushScript(fs, script, { fileDigestMap, scriptFile: file.script, metaFile: file.meta }) + ); } else { // 如果脚本更新时间小于文件更新时间,则更新脚本 updateScript.set(uuid, true); - result.push(this.pullScript(fs, file as SyncFiles, cloudStatus[uuid], script)); + addSyncTask(uuid, this.pullScript(fs, file as SyncFiles, cloudStatus[uuid], script)); } return; } @@ -470,104 +509,206 @@ export class SynchronizeService { return; } updateScript.set(uuid, true); - result.push(this.pullScript(fs, file as SyncFiles, cloudStatus[uuid])); + addSyncTask(uuid, this.pullScript(fs, file as SyncFiles, cloudStatus[uuid])); } }); // 上传剩下的脚本 scriptMap.forEach((script) => { - result.push(this.pushScript(fs, script)); + addSyncTask(script.uuid, this.pushScript(fs, script, { fileDigestMap })); }); // 忽略错误 - const syncResults = await Promise.allSettled(result); + const syncResults = await Promise.allSettled(result.map((item) => item.promise)); const pushedFileDigestMap: FileDigestMap = {}; - syncResults.forEach((ret) => { + const preserveDigestFiles = new Set(); + const failedSyncUuids = new Set(); + syncResults.forEach((ret, index) => { if (ret.status === "fulfilled" && ret.value) { Object.assign(pushedFileDigestMap, ret.value); + } else if (ret.status === "rejected") { + failedSyncUuids.add(result[index].uuid); + result[index].preserveDigestFiles.forEach((name) => preserveDigestFiles.add(name)); + this.logger.warn("sync task failed", Logger.E(ret.reason), { + errorKind: this.classifySyncError(ret.reason), + files: result[index].preserveDigestFiles, + }); } }); // 同步状态 - if (syncConfig.syncStatus) { - const scriptlist = await this.scriptDAO.all(); - await Promise.allSettled( - scriptlist.map(async (script) => { - // 判断云端状态是否与本地状态一致 - const status = cloudStatus[script.uuid]; - const updatetime = script.updatetime || script.createtime; - if (!status) { - scriptcatSync.status.scripts[script.uuid] = { - enable: script.status === SCRIPT_STATUS_ENABLE, - sort: script.sort, - updatetime: updatetime, - }; - } else { - if (updateScript.has(script.uuid)) { - // 脚本已经更新过了,跳过状态同步 - scriptcatSync.status.scripts[script.uuid] = status; + if (syncConfig.syncStatus && canWriteScriptcatSync) { + try { + const scriptlist = await this.scriptDAO.all(); + await Promise.allSettled( + scriptlist.map(async (script) => { + if (failedSyncUuids.has(script.uuid)) { + scriptcatSync.status.scripts[script.uuid] = cloudStatus[script.uuid]; return; } - // 判断时间 - // 如果云端状态的更新时间小于本地状态的更新时间,则更新云端状态 - if (status.updatetime < updatetime) { + // 判断云端状态是否与本地状态一致 + const status = cloudStatus[script.uuid]; + const updatetime = script.updatetime || script.createtime; + if (!status) { scriptcatSync.status.scripts[script.uuid] = { enable: script.status === SCRIPT_STATUS_ENABLE, sort: script.sort, updatetime: updatetime, }; - return; - } - // 否则采用云端状态 - scriptcatSync.status.scripts[script.uuid] = status; - // 脚本顺序 - if (status.sort !== script.sort) { - await this.scriptDAO.update(script.uuid, { - sort: status.sort, - }); - } - // 脚本状态 - if (status.enable !== (script.status === SCRIPT_STATUS_ENABLE)) { - // 开启脚本 - await this.script.enableScript({ - uuid: script.uuid, - enable: status.enable, - }); + } else { + if (updateScript.has(script.uuid)) { + // 脚本已经更新过了,跳过状态同步 + scriptcatSync.status.scripts[script.uuid] = status; + return; + } + // 判断时间 + // 如果云端状态的更新时间小于本地状态的更新时间,则更新云端状态 + if (status.updatetime < updatetime) { + scriptcatSync.status.scripts[script.uuid] = { + enable: script.status === SCRIPT_STATUS_ENABLE, + sort: script.sort, + updatetime: updatetime, + }; + return; + } + // 否则采用云端状态 + scriptcatSync.status.scripts[script.uuid] = status; + // 脚本顺序 + if (status.sort !== script.sort) { + await this.scriptDAO.update(script.uuid, { + sort: status.sort, + }); + } + // 脚本状态 + if (status.enable !== (script.status === SCRIPT_STATUS_ENABLE)) { + // 开启脚本 + await this.script.enableScript({ + uuid: script.uuid, + enable: status.enable, + }); + } } + }) + ); + // 保留被跳过的 orphan uuid 的云端 status,避免覆盖另一台设备半上传的状态 + skippedOrphanUuids.forEach((uuid) => { + const status = cloudStatus[uuid]; + if (status) { + scriptcatSync.status.scripts[uuid] = status; } - }) - ); - // 保留被跳过的 orphan uuid 的云端 status,避免覆盖另一台设备半上传的状态 - skippedOrphanUuids.forEach((uuid) => { - const status = cloudStatus[uuid]; - if (status) { - scriptcatSync.status.scripts[uuid] = status; + }); + if (file) { + const latestCloudStatus = await this.readScriptcatSyncStatus(fs, file); + scriptcatSync.status.scripts = this.mergeScriptcatSyncStatus( + cloudStatus, + latestCloudStatus, + scriptcatSync.status.scripts + ); } - }); - // 保存脚本猫同步状态 - const modifiedDate = Date.now(); - const syncFile = await fs.create("scriptcat-sync.json", { modifiedDate }); - await syncFile.write(JSON.stringify(scriptcatSync, null, 2)); - this.logger.info("sync scriptcat-sync.json file success"); + // 保存脚本猫同步状态 + const modifiedDate = Date.now(); + const syncFile = await fs.create("scriptcat-sync.json", { modifiedDate }); + await syncFile.write(JSON.stringify(scriptcatSync, null, 2)); + this.logger.info("sync scriptcat-sync.json file success"); + } catch (e) { + this.logger.warn("sync scriptcat-sync.json file failed", Logger.E(e)); + } + } else if (syncConfig.syncStatus && !canWriteScriptcatSync) { + this.logger.warn("skip scriptcat-sync.json write because cloud status could not be read"); } // 重新获取文件列表,保存文件摘要 this.logger.info("update file digest"); - await this.updateFileDigest(fs, pushedFileDigestMap); + await this.updateFileDigest(fs, pushedFileDigestMap, preserveDigestFiles); this.logger.info("sync complete"); return; } - async updateFileDigest(fs: FileSystem, knownFileDigestMap: FileDigestMap = {}) { + private classifySyncError(error: unknown): SyncErrorKind { + if (error instanceof FileSystemError) { + if (error.conflict) { + return "conflict"; + } + if (error.rateLimit || error.retryable) { + return "transient"; + } + if (error.notFound) { + return "stale_snapshot"; + } + if (error.auth) { + return "fatal"; + } + return "fatal"; + } + if (isWarpTokenError(error)) { + return "fatal"; + } + if (error instanceof Error && /\bunsupported\b/i.test(error.message)) { + return "unsupported"; + } + return "fatal"; + } + + private async readScriptcatSyncStatus(fs: FileSystem, file: FileInfo): Promise { + const cloudScriptCatSync = JSON.parse(await fs.open(file).then((f) => f.read("string"))) as Partial; + return cloudScriptCatSync.status?.scripts || {}; + } + + private mergeScriptcatSyncStatus( + initialStatus: ScriptcatSync["status"]["scripts"], + latestStatus: ScriptcatSync["status"]["scripts"], + candidateStatus: ScriptcatSync["status"]["scripts"] + ): ScriptcatSync["status"]["scripts"] { + const merged: ScriptcatSync["status"]["scripts"] = { ...latestStatus }; + for (const uuid of Object.keys(candidateStatus)) { + const candidate = candidateStatus[uuid]; + if (!candidate) { + continue; + } + const initial = initialStatus[uuid]; + const latest = latestStatus[uuid]; + const candidateOnlyPreservedInitial = + initial && + candidate.enable === initial.enable && + candidate.sort === initial.sort && + candidate.updatetime === initial.updatetime; + if (candidateOnlyPreservedInitial) { + merged[uuid] = latest || candidate; + continue; + } + if (!latest || candidate.updatetime >= latest.updatetime) { + merged[uuid] = candidate; + } + } + return merged; + } + + async updateFileDigest( + fs: FileSystem, + knownFileDigestMap: FileDigestMap = {}, + preserveFileNames = new Set() + ) { + const oldFileDigestMap = ((await this.storage.get("file_digest")) as FileDigestMap) || {}; const newList = await fs.list(); const newFileDigestMap: FileDigestMap = {}; for (const file of newList) { + if (preserveFileNames.has(file.name)) { + if (oldFileDigestMap[file.name] !== undefined) { + newFileDigestMap[file.name] = oldFileDigestMap[file.name]; + } + continue; + } newFileDigestMap[file.name] = file.digest; } // 各后端 digest 格式不一(WebDAV/OneDrive/S3 是 etag、Dropbox 是 content_hash、Zip 为空, // 仅 GoogleDrive/Baidu 是 md5),只在云端列表暂时漏掉刚上传的文件时用本地 md5 兜底, // 不能覆盖 fs.list 已返回的原生 digest,否则下次同步比对会因格式不一致而误判 for (const name in knownFileDigestMap) { - if (!(name in newFileDigestMap)) { + if (!(name in newFileDigestMap) && !preserveFileNames.has(name)) { newFileDigestMap[name] = knownFileDigestMap[name]; } } + preserveFileNames.forEach((name) => { + if (!(name in newFileDigestMap) && oldFileDigestMap[name] !== undefined) { + newFileDigestMap[name] = oldFileDigestMap[name]; + } + }); await this.storage.set("file_digest", newFileDigestMap); return; } @@ -575,16 +716,18 @@ export class SynchronizeService { // 删除云端脚本数据 async deleteCloudScript(fs: FileSystem, uuid: string, syncDelete: boolean) { const filename = `${uuid}.user.js`; + const metaFilename = `${uuid}.meta.json`; + const fileDigestMap = ((await this.storage.get("file_digest")) as FileDigestMap) || {}; const logger = this.logger.with({ uuid: uuid, file: filename, }); try { - await fs.delete(filename); + await this.deleteCloudFile(fs, filename, fileDigestMap); if (syncDelete) { // 留下一个.meta.json删除标记 const modifiedDate = Date.now(); - const meta = await fs.create(`${uuid}.meta.json`, { modifiedDate }); + const meta = await fs.create(metaFilename, { modifiedDate }); await meta.write( JSON.stringify({ uuid: uuid, @@ -596,17 +739,38 @@ export class SynchronizeService { ); } else { // 直接删除所有相关文件 - await fs.delete(`${uuid}.meta.json`); + await this.deleteCloudFile(fs, metaFilename, fileDigestMap); } logger.info("delete success"); } catch (e) { logger.error("delete file error", Logger.E(e)); + throw e; } return; } + private buildDeleteOptions(fs: FileSystem, filename: string, fileDigestMap: FileDigestMap) { + if (!getFileSystemCapabilities(fs).supportsConditionalDelete) { + return undefined; + } + const expectedDigest = fileDigestMap[filename]; + if (!expectedDigest) { + return undefined; + } + return { expectedDigest }; + } + + private async deleteCloudFile(fs: FileSystem, filename: string, fileDigestMap: FileDigestMap) { + const opts = this.buildDeleteOptions(fs, filename, fileDigestMap); + if (opts) { + await fs.delete(filename, opts); + return; + } + await fs.delete(filename); + } + // 上传脚本 - async pushScript(fs: FileSystem, script: PushScriptParam): Promise { + async pushScript(fs: FileSystem, script: PushScriptParam, opts: PushScriptOptions = {}): Promise { const filename = `${script.uuid}.user.js`; const metaFilename = `${script.uuid}.meta.json`; const logger = this.logger.with({ @@ -616,12 +780,18 @@ export class SynchronizeService { }); try { const modifiedDate = getScriptModifiedDate(script); - const w = await fs.create(filename, { modifiedDate }); + const w = await fs.create( + filename, + this.buildPushCreateOptions(fs, filename, modifiedDate, opts.scriptFile, opts.fileDigestMap) + ); // 获取脚本代码 const code = await this.scriptCodeDAO.get(script.uuid); const scriptCode = code!.code; await w.write(scriptCode); - const meta = await fs.create(metaFilename, { modifiedDate }); + const meta = await fs.create( + metaFilename, + this.buildPushCreateOptions(fs, metaFilename, modifiedDate, opts.metaFile, opts.fileDigestMap) + ); const metaJson = JSON.stringify({ uuid: script.uuid, origin: script.origin, @@ -640,6 +810,28 @@ export class SynchronizeService { } } + private buildPushCreateOptions( + fs: FileSystem, + filename: string, + modifiedDate: number, + existingFile?: FileInfo, + fileDigestMap?: FileDigestMap + ): FileCreateOptions { + const capabilities = getFileSystemCapabilities(fs); + const createOptions: FileCreateOptions = { modifiedDate }; + if (!existingFile) { + if (capabilities.supportsCreateOnly) { + createOptions.createOnly = true; + } + return createOptions; + } + const expectedDigest = fileDigestMap?.[filename]; + if (expectedDigest && capabilities.supportsAtomicCompareAndSwap) { + createOptions.expectedDigest = expectedDigest; + } + return createOptions; + } + async pullScript(fs: FileSystem, file: SyncFiles, status: ScriptcatSyncStatus | undefined, existingScript?: Script) { const logger = this.logger.with({ uuid: existingScript?.uuid || "", @@ -681,6 +873,7 @@ export class SynchronizeService { logger.info("pull script success"); } catch (e) { logger.error("pull script error", Logger.E(e)); + throw e; } } @@ -736,7 +929,9 @@ export class SynchronizeService { const pushedFileDigestMap = await this.pushScript(fs, params.script); await this.updateFileDigest(fs, pushedFileDigestMap); }).catch((e) => { - this.logger.error("push script on install error", Logger.E(e)); + this.logger.error("push script on install error", Logger.E(e), { + errorKind: this.classifySyncError(e), + }); }); } } @@ -753,10 +948,20 @@ export class SynchronizeService { if (config.enable) { stackAsyncTask(SYNC_SERVICE_TASK_KEY, async () => { const fs = await this.buildFileSystem(config); + const preserveDigestFiles = new Set(); for (const { uuid } of items) { - await this.deleteCloudScript(fs, uuid, config.syncDelete); + try { + await this.deleteCloudScript(fs, uuid, config.syncDelete); + } catch (e) { + preserveDigestFiles.add(`${uuid}.user.js`); + preserveDigestFiles.add(`${uuid}.meta.json`); + this.logger.warn("delete cloud script item failed", Logger.E(e), { + uuid, + errorKind: this.classifySyncError(e), + }); + } } - await this.updateFileDigest(fs); + await this.updateFileDigest(fs, {}, preserveDigestFiles); }).catch((e) => { this.logger.error("delete cloud script error", Logger.E(e)); });