From 3bc7a8c0a8c00dcec52e2baa0fd1abff5d68c1ca Mon Sep 17 00:00:00 2001 From: "Evgeny @ SimpleX Chat" <259188159+evgeny-simplex@users.noreply.github.com> Date: Sat, 30 May 2026 22:31:10 +0000 Subject: [PATCH 1/4] smp web: agent store --- .../2026-05-22-agent.md | 195 ++ .../2026-05-23-agent-api-inventory.md | 260 +++ .../2026-05-24-agent-store.md | 121 ++ smp-web/package.json | 1 + smp-web/src/agent/protocol.ts | 199 +- smp-web/src/agent/store-idb.ts | 1604 +++++++++++++++++ smp-web/src/agent/store.ts | 279 +++ smp-web/src/idb-keys.d.ts | 15 + smp-web/tests/store-test.ts | 818 +++++++++ smp-web/tsconfig.json | 2 +- smp-web/tsconfig.test.json | 2 +- tests/SMPWebTests.hs | 48 + 12 files changed, 3541 insertions(+), 3 deletions(-) create mode 100644 rfcs/2026-03-20-smp-agent-web/2026-05-22-agent.md create mode 100644 rfcs/2026-03-20-smp-agent-web/2026-05-23-agent-api-inventory.md create mode 100644 rfcs/2026-03-20-smp-agent-web/2026-05-24-agent-store.md create mode 100644 smp-web/src/agent/store-idb.ts create mode 100644 smp-web/src/agent/store.ts create mode 100644 smp-web/src/idb-keys.d.ts create mode 100644 smp-web/tests/store-test.ts diff --git a/rfcs/2026-03-20-smp-agent-web/2026-05-22-agent.md b/rfcs/2026-03-20-smp-agent-web/2026-05-22-agent.md new file mode 100644 index 000000000..f19b1541d --- /dev/null +++ b/rfcs/2026-03-20-smp-agent-web/2026-05-22-agent.md @@ -0,0 +1,195 @@ +# Agent for Browser: Transpilation Breakdown + +**Parent**: [SMP Client MVP](./2026-05-20-client-mvp.md) +**Depends on**: SMP Client (complete, 96 tests), encoding/encryption spike (complete) + +## Rule + +Every TypeScript function is a faithful transpilation of a specific Haskell function. Same name, same steps, same call chain. No inferences. + +## Scope + +The web widget JOINS connections (never creates addresses). It sends and receives messages. It handles the connection handshake. It does NOT create invitations, manage notifications, transfer files, or do remote control. + +## Architecture difference from Haskell + +Haskell agent uses SQLite + multiple background threads (subscriber, delivery workers, cleanup manager, NTF supervisor). Browser agent uses IndexedDB + event-driven architecture (WebSocket onmessage, Promises, no threads). + +The protocol logic is identical. The concurrency model differs. The store interface differs. The transpilation focuses on the protocol logic. + +## Breakdown into testable pieces + +### Piece 1: Agent protocol types (Agent/Protocol.hs) + +Already partially done (AgentMsgEnvelope, AgentMessage, APrivHeader, AMessage). What's missing for the handshake: + +| Type | Haskell location | What's needed | +|------|-----------------|---------------| +| `SMPQueueInfo` | `Agent/Protocol.hs:1310-1327` | Binary encode/decode — version-dependent, complex | +| `SMPQueueUri` | `Agent/Protocol.hs:1344-1431` | Binary encode/decode + string encode/decode | +| `SMPQueueAddress` | `Agent/Protocol.hs:1350-1356` | `{smpServer, senderId, dhPublicKey, queueMode}` | +| `ConnectionRequestUri` | `Agent/Protocol.hs:1436-1441` | Binary encode/decode: `CRInvitationUri` + `CRContactUri` | +| `ConnReqUriData` | `Agent/Protocol.hs:1728-1734` | Binary encode/decode: `{crAgentVRange, crSmpQueues, crClientData}` | +| `SMPConfirmation` | `Agent/Protocol.hs:798-810` | Not wire-encoded — internal data structure for confirmation handling | +| `E2ERatchetParams` | `Crypto/Ratchet.hs:223-241` | Already have encode/decode in ratchet.ts — need to verify completeness | + +**Test**: each encode/decode function tested byte-for-byte against Haskell via callNode. + +### Piece 2: Connection handshake — joinConnection (Agent.hs) + +The join flow, transpiled step by step: + +``` +joinConnection (Agent.hs:~1200-1300) + 1. Parse ConnectionRequestUri (already have URI parsing) + 2. Create RcvQueue on a selected server (newRcvQueue — Agent/Client.hs:1373) + - Generate X25519 DH keypair for queue + - Generate X25519/Ed25519 auth keypair + - Call createSMPQueue on SMP client + - Get back rcvId, sndId, srvDhKey + 3. Store connection + queue in database + 4. Generate X448 E2E ratchet params (generateRcvE2EParams — already have) + 5. Build ConnInfo (profile data) + 6. Encrypt ConnInfo with ratchet → encConnInfo + 7. Build AgentConfirmation envelope + 8. Wrap in ClientMessage + per-queue E2E encrypt → ClientMsgEnvelope + 9. Send via SMP SEND to the contact address queue + 10. Subscribe to own receive queue (SUB) + 11. Return connection ID +``` + +Each step is independently testable. The full flow is an integration test. + +**Key Haskell functions to transpile:** + +| Function | File:lines | What it does | +|----------|-----------|--------------| +| `joinConnection` | `Agent.hs:~1200` | Top-level join | +| `joinConn` | `Agent.hs:~1230` | Internal join logic | +| `newRcvQueue` | `Agent/Client.hs:1373-1420` | Create queue on server | +| `sendConfirmation` | `Agent/Client.hs:1788-1794` | Encrypt+send confirmation | +| `sendInvitation` | `Agent/Client.hs:1796-1806` | Encrypt+send invitation | +| `mkAgentConfirmation` | `Agent.hs:~3700` | Build confirmation envelope | +| `agentCbEncrypt` | `Agent/Client.hs:2074-2082` | Per-queue E2E encrypt (already have) | + +### Piece 3: Message processing — subscriber (Agent.hs) + +Incoming message handling: + +``` +subscriber (Agent.hs:2912-2919) + → reads from msgQ (populated by SMP client's onMessage callback) + → processSMPTransmissions (Agent.hs:2997-3297) + → for each transmission: + → STEvent (server push MSG): + → decryptClientMessage (per-queue E2E decrypt) + → parse AgentMsgEnvelope + → for AgentMsgEnvelope 'M': + → agentRatchetDecrypt (double ratchet decrypt) + → parse AgentMessage + → dispatch on AMessage type: + → HELLO: complete handshake + → A_MSG body: deliver to user + → A_RCVD: delivery receipt + → QADD/QKEY/QUSE/QTEST: queue switching (skip for MVP) + → EREADY: ratchet sync (skip for MVP) +``` + +**Key Haskell functions to transpile:** + +| Function | File:lines | What it does | +|----------|-----------|--------------| +| `processSMPTransmissions` | `Agent.hs:2997-3297` | Top-level message dispatcher | +| `decryptClientMessage` | `Agent.hs:3282-3293` | Per-queue E2E decrypt + parse envelope | +| `agentRatchetDecrypt` | `Agent.hs:3757-3767` | Ratchet decrypt + store update | +| `helloMsg` | `Agent.hs:~3400` | Process HELLO (complete handshake) | +| `smpConfirmation` | `Agent.hs:~3500` | Process received confirmation | +| `smpInvitation` | `Agent.hs:~3600` | Process received invitation | + +### Piece 4: Message sending — sendMessage (Agent.hs) + +``` +sendMessage (Agent.hs:~1500) + → enqueueMessageB + → agentRatchetEncryptHeader (get ratchet encrypt key) + → rcEncryptMsg (encrypt message body) + → store encrypted message in DB + → createSndMsgDelivery (link to queue) + → delivery worker sends via SMP SEND +``` + +**Key Haskell functions to transpile:** + +| Function | File:lines | What it does | +|----------|-----------|--------------| +| `sendMessage` | `Agent.hs:~1500` | Top-level send | +| `enqueueMessageB` | `Agent.hs:~2020-2060` | Encode + encrypt + store | +| `agentRatchetEncrypt` | `Agent.hs:3742-3746` | Ratchet encrypt | +| `agentRatchetEncryptHeader` | `Agent.hs:3748-3754` | Ratchet encrypt header | +| `encodeAgentMsgStr` | `Agent.hs:2050-2054` | Encode AgentMessage to bytes | +| `runSmpQueueMsgDelivery` | `Agent.hs:2092-2220` | Delivery worker — read from DB, send via SMP | + +### Piece 5: Message acknowledgment (Agent.hs) + +``` +ackMessage (Agent.hs:~1550) + → withStore: mark message as acknowledged + → send ACK to SMP server + → optionally send delivery receipt (A_RCVD) +``` + +### Piece 6: Store interface (IndexedDB) + +The agent uses ~50 distinct store operations. For the web widget MVP, we need: + +| Store operation | What it does | Used by | +|----------------|--------------|---------| +| `createConnection` | Create connection record | joinConnection | +| `getConn` | Get connection by ID | all operations | +| `updateRcvIds` | Increment receive IDs | message processing | +| `createRcvMsg` | Store received message | message processing | +| `getRatchetForUpdate` | Get ratchet state for modify | encrypt/decrypt | +| `updateRatchet` | Store updated ratchet | encrypt/decrypt | +| `getSkippedMsgKeys` | Get skipped message keys | ratchet decrypt | +| `createSndMsg` | Store sent message | sendMessage | +| `createSndMsgDelivery` | Link message to queue | sendMessage | +| `getPendingQueueMsg` | Get next message to send | delivery worker | +| `updateSndMsgStatus` | Update delivery status | delivery worker | +| `deleteMsg` | Delete after ACK | ackMessage | + +## Implementation order + +1. **Piece 1**: Agent protocol types — `SMPQueueInfo`, `SMPQueueUri`, `ConnectionRequestUri`, `ConnReqUriData` encode/decode. Test each against Haskell. +2. **Piece 6**: Store interface — define TypeScript interface matching the needed operations. Implement with in-memory Map first (for testing), IndexedDB later. +3. **Piece 4**: Message sending — `sendMessage` + `agentRatchetEncrypt` + delivery. Test: encrypt in TS, decrypt in HS. +4. **Piece 3**: Message processing — `processSMPTransmissions` + `agentRatchetDecrypt`. Test: encrypt in HS, decrypt in TS. +5. **Piece 2**: Connection handshake — `joinConnection`. Test: TS joins, HS accepts, messages flow. +6. **Piece 5**: Message acknowledgment — `ackMessage`. Test: full roundtrip with ack. + +Each piece is independently testable. Piece 1 uses callNode (no server). Pieces 3-5 use the SMP client REPL + real server. Piece 2 is the integration test that ties everything together. + +## What to skip + +| Feature | Why skip | Haskell functions | +|---------|----------|-------------------| +| Queue switching | Additive, not needed for MVP | `switchConnection`, QADD/QKEY/QUSE/QTEST handling | +| Ratchet sync | MVP shows error, suggests reconnecting | `synchronizeRatchet`, EREADY handling | +| Notifications | No webpush yet | All NTF functions | +| File transfer | Separate protocol | All XFTP functions | +| Remote control | Not in scope | All RC functions | +| Client notices | Server admin feature | `processClientNotices` | +| Cleanup manager | Can do manual cleanup | `cleanupManager` | +| Server management | Configured at init | `setProtocolServers`, `testProtocolServer` | +| Connection creation | Widget only joins | `createConnection`, short links | +| Delivery receipts | Can add later | A_RCVD handling, receipt sending | +| Multiple receive queues | Single queue per connection for MVP | Queue replacement logic | + +## Files + +| File | Action | +|------|--------| +| `smp-web/src/agent/protocol.ts` | Extend with SMPQueueInfo, ConnectionRequestUri, etc. | +| `smp-web/src/agent/store.ts` | New — store interface + in-memory implementation | +| `smp-web/src/agent/agent.ts` | New — agent logic: join, send, receive, ack | +| `smp-web/tests/agent-repl.ts` | New — agent REPL for testing (or extend client-repl) | +| `tests/SMPWebTests.hs` | Agent tests | diff --git a/rfcs/2026-03-20-smp-agent-web/2026-05-23-agent-api-inventory.md b/rfcs/2026-03-20-smp-agent-web/2026-05-23-agent-api-inventory.md new file mode 100644 index 000000000..d0c303b01 --- /dev/null +++ b/rfcs/2026-03-20-smp-agent-web/2026-05-23-agent-api-inventory.md @@ -0,0 +1,260 @@ +# Agent API Inventory for Web Widget + +Every exported function from `Agent.hs`, classified as MVP / post-MVP / skip, with reasoning. + +## Context + +The web widget: +- Joins existing connections via addresses hardcoded in the widget or sent as simplex links +- Sends and receives messages +- Does NOT create addresses, invitation links, or short links +- Does NOT transfer files (post-MVP) +- Does NOT manage notifications (post-MVP) +- Does NOT rotate queues +- Must accept ratchet re-sync initiated by the other side (but does not initiate) + +## Lifecycle + +| Function | MVP? | Reason | +|----------|------|--------| +| `getSMPAgentClient` | YES | Initialize agent — required | +| `getSMPAgentClient_` | NO | Variant with extra params, not needed | +| `disconnectAgentClient` | YES | Clean shutdown — required | +| `disposeAgentClient` | NO | Hard shutdown — disconnectAgentClient is sufficient | +| `resumeAgentClient` | NO | Widget doesn't suspend/resume — runs while page is open | +| `foregroundAgent` | NO | Mobile-only concept | +| `suspendAgent` | NO | Mobile-only concept | + +## User management + +| Function | MVP? | Reason | +|----------|------|--------| +| `createUser` | YES | Widget needs at least one user to own connections | +| `deleteUser` | NO | Widget doesn't delete users — page reload is cleanup | + +## Connection creation (widget never creates) + +| Function | MVP? | Reason | +|----------|------|--------| +| `createConnection` | NO | Widget joins, never creates | +| `createConnectionAsync` | NO | Same | +| `prepareConnectionLink` | NO | For creating links | +| `createConnectionForLink` | NO | For creating links | +| `setConnShortLink` | NO | For creating short links | +| `setConnShortLinkAsync` | NO | Same | +| `deleteConnShortLink` | NO | For managing short links | +| `getConnShortLink` | NO | For reading short links — widget uses hardcoded address | +| `getConnShortLinkAsync` | NO | Same | +| `getConnLinkPrivKey` | NO | For link management | +| `deleteLocalInvShortLink` | NO | For link cleanup | +| `changeConnectionUser` | NO | Widget has one user | + +## Joining connections + +| Function | MVP? | Reason | +|----------|------|--------| +| `prepareConnectionToJoin` | YES | Create connection record before join — prevents race with incoming confirmation | +| `joinConnection` | YES | Core function — join via address URI | +| `joinConnectionAsync` | NO | Async variant — widget can use sync joinConnection with await | +| `connRequestPQSupport` | YES | Determine PQ support from connection request — needed for join | + +## Handshake (accepting incoming) + +| Function | MVP? | Reason | +|----------|------|--------| +| `allowConnection` | YES | Allow connection after receiving CONF — part of handshake | +| `allowConnectionAsync` | NO | Async variant | +| `acceptContact` | YES | Accept contact request (for group join flow) | +| `acceptContactAsync` | NO | Async variant | +| `prepareConnectionToAccept` | YES | Prepare for accept — same race prevention as prepareConnectionToJoin | +| `rejectContact` | NO | Widget doesn't reject — it always accepts | + +## Subscription + +| Function | MVP? | Reason | +|----------|------|--------| +| `subscribeConnection` | YES | Subscribe to receive messages on one connection | +| `subscribeConnections` | YES | Batch subscribe — needed after page reload | +| `subscribeAllConnections` | YES | Subscribe everything for a user — simplest for widget | +| `resubscribeConnection` | YES | Resubscribe after network recovery | +| `resubscribeConnections` | YES | Batch resubscribe | +| `getConnectionMessages` | NO | Fetch stored messages — widget processes messages as they arrive | +| `getNotificationConns` | NO | Push notification management | +| `subscribeClientService` | NO | Service certificate management | + +## Messaging + +| Function | MVP? | Reason | +|----------|------|--------| +| `sendMessage` | YES | Send a message — core function | +| `sendMessages` | NO | Batch send — sendMessage is sufficient for MVP | +| `sendMessagesB` | NO | Batch send with error handling — optimization | +| `ackMessage` | YES | Acknowledge received message — required for protocol correctness | +| `ackMessageAsync` | NO | Async variant | + +## Queue management + +| Function | MVP? | Reason | +|----------|------|--------| +| `switchConnection` | NO | Queue rotation — not needed | +| `switchConnectionAsync` | NO | Same | +| `abortConnectionSwitch` | NO | Cancel rotation | +| `getConnectionQueueInfo` | NO | Debug info | +| `suspendConnection` | NO | Widget deletes or ignores, doesn't suspend | + +## Ratchet + +| Function | MVP? | Reason | +|----------|------|--------| +| `synchronizeRatchet` | NO | Widget doesn't initiate ratchet sync. But MUST handle incoming EREADY — that's in processSMPTransmissions, not a separate API call. | +| `getConnectionRatchetAdHash` | NO | Verification UI not in widget | + +## Connection cleanup + +| Function | MVP? | Reason | +|----------|------|--------| +| `deleteConnection` | YES | User may want to delete a conversation | +| `deleteConnectionAsync` | NO | Async variant | +| `deleteConnections` | NO | Batch delete — single delete sufficient | +| `deleteConnectionsAsync` | NO | Same | +| `getConnectionServers` | NO | Info only | +| `compareConnections` | NO | Database sync tool | +| `syncConnections` | NO | Database sync tool | + +## Server configuration + +| Function | MVP? | Reason | +|----------|------|--------| +| `setProtocolServers` | NO | Widget initialized with servers, doesn't change them | +| `checkUserServers` | NO | Admin function | +| `testProtocolServer` | NO | Admin function | +| `setNtfServers` | NO | No notifications in MVP | +| `setNetworkConfig` | NO | Widget uses default network config | +| `setUserNetworkInfo` | NO | Widget doesn't track network state changes | +| `reconnectAllServers` | NO | Widget handles reconnection via SMP client | +| `reconnectSMPServer` | NO | Same | + +## Notifications (all post-MVP) + +| Function | MVP? | Reason | +|----------|------|--------| +| `registerNtfToken` | NO | No webpush yet | +| `verifyNtfToken` | NO | Same | +| `checkNtfToken` | NO | Same | +| `deleteNtfToken` | NO | Same | +| `getNtfToken` | NO | Same | +| `getNtfTokenData` | NO | Same | +| `toggleConnectionNtfs` | NO | Same | + +## File transfer (all post-MVP) + +| Function | MVP? | Reason | +|----------|------|--------| +| `xftpStartWorkers` | NO | Post-MVP | +| `xftpStartSndWorkers` | NO | Same | +| `xftpReceiveFile` | NO | Same | +| `xftpDeleteRcvFile` | NO | Same | +| `xftpDeleteRcvFiles` | NO | Same | +| `xftpSendFile` | NO | Same | +| `xftpSendDescription` | NO | Same | +| `xftpDeleteSndFileInternal` | NO | Same | +| `xftpDeleteSndFilesInternal` | NO | Same | +| `xftpDeleteSndFileRemote` | NO | Same | +| `xftpDeleteSndFilesRemote` | NO | Same | + +## Remote control (all skip) + +| Function | MVP? | Reason | +|----------|------|--------| +| `rcNewHostPairing` | NO | Not in scope | +| `rcConnectHost` | NO | Same | +| `rcConnectCtrl` | NO | Same | +| `rcDiscoverCtrl` | NO | Same | + +## Debug/stats (all skip) + +| Function | MVP? | Reason | +|----------|------|--------| +| `getAgentSubsTotal` | NO | Debug | +| `getAgentServersSummary` | NO | Debug | +| `resetAgentServersStats` | NO | Debug | +| `execAgentStoreSQL` | NO | Debug | +| `getAgentMigrations` | NO | Debug | +| `debugAgentLocks` | NO | Debug | +| `getAgentSubscriptions` | NO | Debug | +| `logConnection` | NO | Debug | +| `withAgentEnv` | NO | Test utility | + +## Summary + +### Corrections after reviewing simplex-chat-2 Subscriber.hs + Commands.hs + +The widget handles business chats (groups). Group flows trigger agent calls the widget doesn't initiate but must support: + +- Member introductions create connections asynchronously → `createConnectionAsync` +- Members join via introductions → `joinConnectionAsync`, `prepareConnectionToJoin` +- Members leave/deleted → `deleteConnectionAsync`, `deleteConnectionsAsync` +- Group messages go to all members → `sendMessages` +- All message acks use async variant → `ackMessageAsync` +- Accepting group invitations → `allowConnectionAsync` +- `acceptContact` — used in contact request acceptance flow (APIAcceptContactRequest in Commands.hs) +- `toggleConnectionNtfs` — used when CON received for group member (Subscriber.hs:850) + +### MVP functions (27 of 90): + +**Lifecycle:** `getSMPAgentClient`, `disconnectAgentClient` + +**User:** `createUser` + +**Join:** `prepareConnectionToJoin`, `joinConnection`, `joinConnectionAsync`, `connRequestPQSupport` + +**Handshake:** `allowConnection`, `allowConnectionAsync`, `acceptContact`, `acceptContactAsync`, `prepareConnectionToAccept` + +**Connection creation (for group member flows):** `createConnectionAsync`, `deleteConnectionAsync`, `deleteConnectionsAsync` + +**Subscribe:** `subscribeConnection`, `subscribeConnections`, `subscribeAllConnections`, `resubscribeConnection`, `resubscribeConnections` + +**Message:** `sendMessage`, `sendMessages`, `ackMessage`, `ackMessageAsync` + +**Cleanup:** `deleteConnection` + +**Notification toggle:** `toggleConnectionNtfs` + +**Internal (not exported but required):** `subscriber`/`processSMPTransmissions` (message processing), `runSmpQueueMsgDelivery` (delivery worker), all encryption/decryption functions, store operations for the above. + +### Message types to handle in processSMPTransmissions: + +| AMessage | Handle? | Reason | +|----------|---------|--------| +| `HELLO` | YES | Complete handshake | +| `A_MSG body` | YES | Deliver message to user | +| `A_RCVD receipts` | YES | Process delivery receipts (show checkmarks) | +| `A_QCONT addr` | NO | Queue continuation after quota — skip | +| `QADD qs` | NO | Queue rotation — skip | +| `QKEY qs` | NO | Queue rotation — skip | +| `QUSE qs` | NO | Queue rotation — skip | +| `QTEST qs` | NO | Queue rotation — skip | +| `EREADY msgId` | ACCEPT | Must handle incoming (don't initiate) — reset ratchet sync state | + +### AgentMsgEnvelope types to handle: + +| Variant | Handle? | Reason | +|---------|---------|--------| +| `AgentConfirmation` | YES | Handshake — received when peer confirms | +| `AgentMsgEnvelope` | YES | Normal encrypted messages | +| `AgentInvitation` | YES | Received when joining contact address | +| `AgentRatchetKey` | ACCEPT | Must handle incoming ratchet key — don't initiate | + +### Store operations needed: + +Based on the 18 MVP functions, the store needs (rough count): + +- Connection CRUD: ~8 operations +- Queue CRUD: ~6 operations +- Ratchet state: ~4 operations (get, update, skipped keys) +- Message storage: ~6 operations (create rcv/snd msg, update status, delete) +- Message delivery: ~4 operations (create delivery, get pending, update status) +- User: ~2 operations (create, get) +- Confirmation: ~3 operations (create, get, delete) + +**Estimated: ~33 store operations.** This is what determines whether SQLite WASM or IndexedDB direct is more practical. diff --git a/rfcs/2026-03-20-smp-agent-web/2026-05-24-agent-store.md b/rfcs/2026-03-20-smp-agent-web/2026-05-24-agent-store.md new file mode 100644 index 000000000..9026554b7 --- /dev/null +++ b/rfcs/2026-03-20-smp-agent-web/2026-05-24-agent-store.md @@ -0,0 +1,121 @@ +# Agent Store: IndexedDB Design + +**Parent**: [Agent Plan](./2026-05-22-agent.md) + +## Schema + +IndexedDB object stores, mapped from SQLite tables. Each store mirrors the Haskell schema from `agent_schema.sql`. + +### Object stores needed (16) + +``` +users + key: userId (autoincrement) + fields: deleted + +connections + key: connId (Uint8Array) + fields: connMode, lastInternalMsgId, lastInternalRcvMsgId, lastInternalSndMsgId, + lastExternalSndMsgId, lastRcvMsgHash, lastSndMsgHash, smpAgentVersion, + duplexHandshake, enableNtfs, deleted, userId, ratchetSyncState, pqSupport + +rcv_queues + key: [host, port, rcvId] (compound) + index: [connId], [host, port, sndId] + fields: connId, rcvPrivateKey, rcvDhSecret, e2ePrivKey, e2eDhSecret, sndId, sndKey, + status, smpClientVersion, rcvQueueId, rcvPrimary, replaceRcvQueueId, queueMode, + serverKeyHash, lastBrokerTs + +snd_queues + key: [host, port, sndId] (compound) + index: [connId] + fields: connId, sndPrivateKey, e2eDhSecret, status, smpClientVersion, + sndPublicKey, e2ePubKey, sndQueueId, sndPrimary, queueMode, serverKeyHash + +messages + key: [connId, internalId] (compound) + index: [connId] + fields: internalTs, internalRcvId, internalSndId, msgType, msgBody, msgFlags, pqEncryption + +rcv_messages + key: [connId, internalRcvId] (compound) + index: [connId, internalId] + fields: internalId, externalSndId, brokerId, brokerTs, internalHash, + externalPrevSndHash, integrity, userAck, rcvQueueId, receiveAttempts + +snd_messages + key: [connId, internalSndId] (compound) + index: [connId, internalId] + fields: internalId, internalHash, previousMsgHash, retryIntSlow, retryIntFast, + rcptInternalId, rcptStatus, msgEncryptKey, paddedMsgLen, sndMessageBodyId + +snd_message_deliveries + key: sndMessageDeliveryId (autoincrement) + index: [connId, sndQueueId] + fields: connId, sndQueueId, internalId, failed + +snd_message_bodies + key: sndMessageBodyId (autoincrement) + fields: agentMsg + +conn_confirmations + key: confirmationId (Uint8Array) + index: [connId] + fields: connId, e2eSndPubKey, senderKey, ratchetState, senderConnInfo, + accepted, ownConnInfo, smpReplyQueues, smpClientVersion + +conn_invitations + key: invitationId (Uint8Array) + index: [contactConnId] + fields: contactConnId, crInvitation, recipientConnInfo, accepted, ownConnInfo + +ratchets + key: connId (Uint8Array) + fields: x3dhPrivKey1, x3dhPrivKey2, ratchetState, e2eVersion, + x3dhPubKey1, x3dhPubKey2, pqPrivKem, pqPubKem + +skipped_messages + key: skippedMessageId (autoincrement) + index: [connId] + fields: connId, headerKey, msgN, msgKey + +servers + key: [host, port] (compound) + fields: keyHash + +commands + key: commandId (autoincrement) + index: [connId], [host, port] + fields: connId, host, port, corrId, commandTag, command, agentVersion, serverKeyHash, failed + +encrypted_rcv_message_hashes + key: id (autoincrement) + index: [connId, hash] + fields: connId, hash, createdAt +``` + +## Interface + +TypeScript interface matching the ~60 store operations. Each method maps to a specific Haskell function in `AgentStore.hs`. + +The interface will be defined in `src/agent/store.ts`. Implementation in `src/agent/store-idb.ts` (IndexedDB). + +## Implementation approach + +1. Define the TypeScript interface first — every method name matches the Haskell function name +2. Implement with IndexedDB transactions +3. Test each operation in isolation before wiring to agent + +IndexedDB transactions map to SQLite transactions — both are ACID within a single store/table. Cross-store atomicity in IndexedDB requires putting multiple stores in one transaction, which is supported. + +## Key differences from SQLite + +1. **No SQL joins** — denormalize where needed, or do application-level joins +2. **No AUTO INCREMENT guaranteed ordering** — use explicit counters +3. **Blob keys** — IndexedDB supports ArrayBuffer keys natively +4. **Compound keys** — IndexedDB supports array keys: `[host, port, rcvId]` +5. **Indexes** — must be declared upfront in `onupgradeneeded` + +## Testing + +Each store operation tested by: write data, read it back, verify it matches. No server needed — pure store tests using `fake-indexeddb` in Node.js. diff --git a/smp-web/package.json b/smp-web/package.json index ab9ff7c1e..9f7faffb6 100644 --- a/smp-web/package.json +++ b/smp-web/package.json @@ -29,6 +29,7 @@ "devDependencies": { "@types/node": "^25.5.0", "@types/ws": "^8.18.1", + "fake-indexeddb": "^6.2.5", "typescript": "^5.4.0", "ws": "^8.0.0" } diff --git a/smp-web/src/agent/protocol.ts b/smp-web/src/agent/protocol.ts index 36b6e8f4a..4e7c00064 100644 --- a/smp-web/src/agent/protocol.ts +++ b/smp-web/src/agent/protocol.ts @@ -3,8 +3,15 @@ import {base64urlDecode} from "@simplex-chat/xftp-web/dist/protocol/description.js" import { - Decoder, decodeBytes, decodeLarge, decodeWord16, decodeBool, + Decoder, concatBytes, + encodeBytes, decodeBytes, + encodeLarge, decodeLarge, + encodeWord16, decodeWord16, + encodeBool, decodeBool, + encodeMaybe, decodeMaybe, + encodeNonEmpty, decodeNonEmpty, } from "@simplex-chat/xftp-web/dist/protocol/encoding.js" +import {encodeProtocolServer} from "../protocol.js" // -- Short link types (Agent/Protocol.hs:1462-1470) @@ -164,6 +171,196 @@ export function decodeFixedLinkData(d: Decoder): FixedLinkData { return {agentVRange: {min, max}, rootKey, rest} } +// -- SMPQueueAddress (Agent/Protocol.hs:1350-1356) + +export interface SMPQueueAddress { + smpServer: {hosts: string[], port: string, keyHash: Uint8Array} // ProtocolServer + senderId: Uint8Array // EntityId (ByteString) + dhPublicKey: Uint8Array // PublicKeyX25519 (DER-encoded ByteString) + queueMode: string | null // Maybe QueueMode: 'M' = Messaging, 'C' = Contact +} + +// -- SMPQueueInfo (Agent/Protocol.hs:1310-1327) +// Version-dependent encoding + +// SMP client version constants (Protocol.hs:281-294) +const initialSMPClientVersion = 1 +const sndAuthKeySMPClientVersion = 3 +const shortLinksSMPClientVersion = 4 + +export interface SMPQueueInfo { + clientVersion: number // VersionSMPC (Word16) + queueAddress: SMPQueueAddress +} + +// smpEncode (Agent/Protocol.hs:1313-1321) +export function encodeSMPQueueInfo(q: SMPQueueInfo): Uint8Array { + const {clientVersion, queueAddress: {smpServer, senderId, dhPublicKey, queueMode}} = q + const addrEnc = concatBytes( + encodeWord16(clientVersion), + encodeProtocolServer(smpServer.hosts, smpServer.port, smpServer.keyHash), + encodeBytes(senderId), + encodeBytes(dhPublicKey), + ) + if (clientVersion >= shortLinksSMPClientVersion) { + // encode queueMode directly (Maybe QueueMode as char or empty) + const qmBytes = queueMode ? new Uint8Array([queueMode.charCodeAt(0)]) : new Uint8Array(0) + return concatBytes(addrEnc, qmBytes) + } + if (clientVersion >= sndAuthKeySMPClientVersion && senderCanSecure(queueMode)) { + return concatBytes(addrEnc, encodeBool(true)) + } + if (clientVersion > initialSMPClientVersion) { + return addrEnc + } + // v1 legacy — not supported by web widget + throw new Error("encodeSMPQueueInfo: legacy v1 not supported") +} + +// smpP (Agent/Protocol.hs:1322-1327) +export function decodeSMPQueueInfo(d: Decoder): SMPQueueInfo { + const clientVersion = decodeWord16(d) + // v1 legacy server encoding not supported + if (clientVersion <= initialSMPClientVersion) throw new Error("decodeSMPQueueInfo: legacy v1 not supported") + const smpServer = decodeProtocolServerTyped(d) + const senderId = decodeBytes(d) + const dhPublicKey = decodeBytes(d) + const queueMode = decodeQueueMode(d) + return {clientVersion, queueAddress: {smpServer, senderId, dhPublicKey, queueMode}} +} + +// -- SMPQueueUri (Agent/Protocol.hs:1347-1431) + +export interface SMPQueueUri { + clientVRange: {min: number, max: number} // VersionRangeSMPC + queueAddress: SMPQueueAddress +} + +// smpEncode (Agent/Protocol.hs:1417-1427) +export function encodeSMPQueueUri(q: SMPQueueUri): Uint8Array { + const {clientVRange: {min: minV, max: maxV}, queueAddress: {smpServer, senderId, dhPublicKey, queueMode}} = q + const addrEnc = concatBytes( + encodeWord16(minV), encodeWord16(maxV), + encodeProtocolServer(smpServer.hosts, smpServer.port, smpServer.keyHash), + encodeBytes(senderId), + encodeBytes(dhPublicKey), + ) + if (minV >= shortLinksSMPClientVersion) { + const qmBytes = queueMode ? new Uint8Array([queueMode.charCodeAt(0)]) : new Uint8Array(0) + return concatBytes(addrEnc, qmBytes) + } + if (minV >= sndAuthKeySMPClientVersion || (maxV >= sndAuthKeySMPClientVersion && senderCanSecure(queueMode))) { + return concatBytes(addrEnc, encodeBool(senderCanSecure(queueMode))) + } + return addrEnc +} + +// smpP (Agent/Protocol.hs:1428-1431) +export function decodeSMPQueueUri(d: Decoder): SMPQueueUri { + const min = decodeWord16(d) + const max = decodeWord16(d) + const smpServer = decodeProtocolServerTyped(d) + const senderId = decodeBytes(d) + const dhPublicKey = decodeBytes(d) + const queueMode = decodeQueueMode(d) + return {clientVRange: {min, max}, queueAddress: {smpServer, senderId, dhPublicKey, queueMode}} +} + +// -- ConnReqUriData (Agent/Protocol.hs:1728-1734, 1145-1158) + +export interface ConnReqUriData { + crAgentVRange: {min: number, max: number} // VersionRangeSMPA + crSmpQueues: SMPQueueUri[] // NonEmpty SMPQueueUri + crClientData: string | null // Maybe CRClientData (Text) +} + +// smpEncode (Agent/Protocol.hs:1145-1147) +export function encodeConnReqUriData(d: ConnReqUriData): Uint8Array { + const vr = concatBytes(encodeWord16(d.crAgentVRange.min), encodeWord16(d.crAgentVRange.max)) + const queues = encodeNonEmpty(encodeSMPQueueUri, d.crSmpQueues) + const clientData = d.crClientData !== null + ? concatBytes(new Uint8Array([0x31]), encodeLarge(new TextEncoder().encode(d.crClientData))) + : new Uint8Array([0x30]) + return concatBytes(vr, queues, clientData) +} + +// smpP (Agent/Protocol.hs:1148-1158) +export function decodeConnReqUriData(d: Decoder): ConnReqUriData { + const min = decodeWord16(d) + const max = decodeWord16(d) + const crSmpQueues = decodeNonEmpty(decodeSMPQueueUri, d) + // Patch queueMode: if Nothing, set to QMContact (Agent/Protocol.hs:1156-1158) + for (const q of crSmpQueues) { + if (q.queueAddress.queueMode === null) q.queueAddress.queueMode = "C" + } + const clientData = decodeMaybe((dd) => { + const large = decodeLarge(dd) + return new TextDecoder().decode(large) + }, d) + return {crAgentVRange: {min, max}, crSmpQueues, crClientData: clientData} +} + +// -- ConnectionRequestUri (Agent/Protocol.hs:1130-1143, 1436-1441) + +export type ConnectionRequestUri = + | {mode: "invitation", crData: ConnReqUriData, e2eParams: Uint8Array} // raw smpEncoded E2ERatchetParams + | {mode: "contact", crData: ConnReqUriData} + +// smpEncode (Agent/Protocol.hs:1130-1133) +export function encodeConnectionRequestUri(cr: ConnectionRequestUri): Uint8Array { + switch (cr.mode) { + case "invitation": + return concatBytes(new Uint8Array([0x49]), encodeConnReqUriData(cr.crData), cr.e2eParams) // 'I' + crData + e2eParams + case "contact": + return concatBytes(new Uint8Array([0x43]), encodeConnReqUriData(cr.crData)) // 'C' + crData + } +} + +// smpP (Agent/Protocol.hs:1140-1143) +export function decodeConnectionRequestUri(d: Decoder): ConnectionRequestUri { + const mode = d.anyByte() + if (mode === 0x49) { // 'I' Invitation + const crData = decodeConnReqUriData(d) + const e2eParams = d.takeAll() // E2ERatchetParams consumes rest + return {mode: "invitation", crData, e2eParams} + } + if (mode === 0x43) { // 'C' Contact + const crData = decodeConnReqUriData(d) + return {mode: "contact", crData} + } + throw new Error("decodeConnectionRequestUri: unknown mode 0x" + mode.toString(16)) +} + +// -- Helpers + +function senderCanSecure(queueMode: string | null): boolean { + return queueMode === "M" +} + +// queueModeP (Agent/Protocol.hs:1433-1434) +// Just <$> smpP <|> optional ((\case True -> QMMessaging; _ -> QMContact) <$> smpP) +function decodeQueueMode(d: Decoder): string | null { + if (d.remaining() === 0) return null + const b = d.anyByte() + if (b === 0x4D) return "M" // QMMessaging + if (b === 0x43) return "C" // QMContact + // Could be a Bool (sndSecure) for older versions — True='T'(0x54) → QMMessaging, False='F'(0x46) → QMContact + if (b === 0x54) return "M" // True → QMMessaging + if (b === 0x46) return null // False → no queueMode (not secured) + return null +} + +// Decode ProtocolServer into typed format with string hosts +function decodeProtocolServerTyped(d: Decoder): {hosts: string[], port: string, keyHash: Uint8Array} { + const hostCount = d.anyByte() + if (hostCount === 0) throw new Error("empty server host list") + const hosts: string[] = [] + for (let i = 0; i < hostCount; i++) hosts.push(new TextDecoder().decode(decodeBytes(d))) + const port = new TextDecoder().decode(decodeBytes(d)) + const keyHash = decodeBytes(d) + return {hosts, port, keyHash} +} + // -- Profile extraction export function parseProfile(userData: Uint8Array): unknown { diff --git a/smp-web/src/agent/store-idb.ts b/smp-web/src/agent/store-idb.ts new file mode 100644 index 000000000..c238950fe --- /dev/null +++ b/smp-web/src/agent/store-idb.ts @@ -0,0 +1,1604 @@ +// IndexedDB implementation of AgentStore. +// Each method is a faithful transpilation of SQL in AgentStore.hs. +// +// Transpilation approach: +// - Each SQL query is converted to equivalent IndexedDB operations +// - Field names match SQLite column names from agent_schema.sql +// - Transaction scoping matches the Haskell withStore patterns +// +// This file is a work in progress — methods are added as they are transpiled +// from the Haskell source. + +import type {AgentStore, ConnId, UserId} from "./store.js" + +const DB_NAME = "simplex-agent" +const DB_VERSION = 1 + +function toHex(bytes: Uint8Array): string { + return Array.from(bytes, b => b.toString(16).padStart(2, "0")).join("") +} + +function compareUint8Array(a: Uint8Array, b: Uint8Array): number { + const len = Math.min(a.length, b.length) + for (let i = 0; i < len; i++) { + if (a[i] !== b[i]) return a[i] - b[i] + } + return a.length - b.length +} + +// -- Database opening + schema creation (from agent_schema.sql) + +export async function openAgentStore(): Promise { + const db = await openDB() + return createStore(db) +} + +function openDB(): Promise { + return new Promise((resolve, reject) => { + const req = indexedDB.open(DB_NAME, DB_VERSION) + req.onerror = () => reject(req.error) + req.onsuccess = () => resolve(req.result) + req.onupgradeneeded = () => createSchema(req.result) + }) +} + +// Schema mirrors agent_schema.sql tables needed for MVP +function createSchema(db: IDBDatabase): void { + // users — agent_schema.sql:266-270 + // CREATE TABLE users(user_id INTEGER PRIMARY KEY AUTOINCREMENT, deleted INTEGER DEFAULT 0) + db.createObjectStore("users", {keyPath: "user_id", autoIncrement: true}) + + // servers — agent_schema.sql:6-11 + // CREATE TABLE servers(host TEXT, port TEXT, key_hash BLOB, PRIMARY KEY(host, port)) + db.createObjectStore("servers", {keyPath: ["host", "port"]}) + + // connections — agent_schema.sql:12-31 + const conns = db.createObjectStore("connections", {keyPath: "conn_id"}) + conns.createIndex("user_id", "user_id") + + // rcv_queues — agent_schema.sql:32-70 + const rcvQ = db.createObjectStore("rcv_queues", {keyPath: ["host", "port", "rcv_id"]}) + rcvQ.createIndex("conn_id", "conn_id") + rcvQ.createIndex("conn_id_rcv_queue_id", ["conn_id", "rcv_queue_id"], {unique: true}) + + // snd_queues — agent_schema.sql:71-92 + const sndQ = db.createObjectStore("snd_queues", {keyPath: ["host", "port", "snd_id"]}) + sndQ.createIndex("conn_id", "conn_id") + sndQ.createIndex("conn_id_snd_queue_id", ["conn_id", "snd_queue_id"], {unique: true}) + + // messages — agent_schema.sql:93-109 + const msgs = db.createObjectStore("messages", {keyPath: ["conn_id", "internal_id"]}) + msgs.createIndex("conn_id", "conn_id") + msgs.createIndex("conn_id_internal_rcv_id", ["conn_id", "internal_rcv_id"]) + msgs.createIndex("conn_id_internal_snd_id", ["conn_id", "internal_snd_id"]) + + // rcv_messages — agent_schema.sql:110-126 + const rcvMsgs = db.createObjectStore("rcv_messages", {keyPath: ["conn_id", "internal_rcv_id"]}) + rcvMsgs.createIndex("conn_id_internal_id", ["conn_id", "internal_id"]) + rcvMsgs.createIndex("conn_id_broker_id", ["conn_id", "broker_id"]) + + // snd_messages — agent_schema.sql:127-143 + const sndMsgs = db.createObjectStore("snd_messages", {keyPath: ["conn_id", "internal_snd_id"]}) + sndMsgs.createIndex("conn_id_internal_id", ["conn_id", "internal_id"]) + + // snd_message_deliveries — agent_schema.sql:257-264 + const sndDel = db.createObjectStore("snd_message_deliveries", {keyPath: "snd_message_delivery_id", autoIncrement: true}) + sndDel.createIndex("conn_id_snd_queue_id", ["conn_id", "snd_queue_id"]) + + // snd_message_bodies — agent_schema.sql:428-431 + db.createObjectStore("snd_message_bodies", {keyPath: "snd_message_body_id", autoIncrement: true}) + + // conn_confirmations — agent_schema.sql:144-157 + const confs = db.createObjectStore("conn_confirmations", {keyPath: "confirmation_id"}) + confs.createIndex("conn_id", "conn_id") + + // conn_invitations — agent_schema.sql:158-166 + const invs = db.createObjectStore("conn_invitations", {keyPath: "invitation_id"}) + invs.createIndex("contact_conn_id", "contact_conn_id") + + // ratchets — agent_schema.sql:167-181 + db.createObjectStore("ratchets", {keyPath: "conn_id"}) + + // skipped_messages — agent_schema.sql:182-189 + const skip = db.createObjectStore("skipped_messages", {keyPath: "skipped_message_id", autoIncrement: true}) + skip.createIndex("conn_id", "conn_id") + + // commands — agent_schema.sql:242-256 + const cmds = db.createObjectStore("commands", {keyPath: "command_id", autoIncrement: true}) + cmds.createIndex("conn_id", "conn_id") + cmds.createIndex("host_port", ["host", "port"]) + + // encrypted_rcv_message_hashes — agent_schema.sql:397-403 + const hashes = db.createObjectStore("encrypted_rcv_message_hashes", {keyPath: "encrypted_rcv_message_hash_id", autoIncrement: true}) + hashes.createIndex("conn_id_hash", ["conn_id", "hash"]) +} + +// -- Helpers + +function idbReq(r: IDBRequest): Promise { + return new Promise((resolve, reject) => { + r.onsuccess = () => resolve(r.result) + r.onerror = () => reject(r.error) + }) +} + +function eqBytes(a: Uint8Array, b: Uint8Array): boolean { + if (a.length !== b.length) return false + for (let i = 0; i < a.length; i++) if (a[i] !== b[i]) return false + return true +} + +// IndexedDB index.getAll() with Uint8Array keys doesn't work in fake-indexeddb polyfill. +// These helpers scan all records and filter by field values, handling Uint8Array comparison. +function fieldEq(a: any, b: any): boolean { + if (a instanceof Uint8Array && b instanceof Uint8Array) return eqBytes(a, b) + return a === b +} + +async function allByIndex(store: IDBObjectStore, fields: Record): Promise { + const all = await idbReq(store.getAll()) + return all.filter(r => Object.entries(fields).every(([k, v]) => fieldEq(r[k], v))) +} + +function createStore(db: IDBDatabase): AgentStore { + // Transaction helper + function withTx(stores: string | string[], mode: IDBTransactionMode, fn: (tx: IDBTransaction) => Promise): Promise { + return new Promise((resolve, reject) => { + const tx = db.transaction(stores, mode) + let result: T + const p = fn(tx).then(r => { result = r }) + tx.oncomplete = () => p.then(() => resolve(result)) + tx.onerror = () => reject(tx.error) + tx.onabort = () => reject(tx.error) + }) + } + + return { + // ============================================================ + // Users — AgentStore.hs:341-397 + // ============================================================ + + // createUserRecord (AgentStore.hs:341-344) + // SQL: INSERT INTO users DEFAULT VALUES; SELECT last_insert_rowid() + async createUserRecord() { + return withTx("users", "readwrite", async (tx) => { + const store = tx.objectStore("users") + const id = await idbReq(store.add({deleted: 0})) + return id as number + }) + }, + + // getUserIds (AgentStore.hs:346-348) + // SQL: SELECT user_id FROM users WHERE deleted = 0 + async getUserIds() { + return withTx("users", "readonly", async (tx) => { + const all = await idbReq(tx.objectStore("users").getAll()) + return all.filter(u => u.deleted === 0).map(u => u.user_id) + }) + }, + + // deleteUserRecord (AgentStore.hs:355-358) + // SQL: DELETE FROM users WHERE user_id = ? + async deleteUserRecord(userId) { + return withTx("users", "readwrite", async (tx) => { + await idbReq(tx.objectStore("users").delete(userId)) + }) + }, + + // setUserDeleted (AgentStore.hs:360-365) + // SQL: UPDATE users SET deleted = 1 WHERE user_id = ? + // SQL: SELECT conn_id FROM connections WHERE user_id = ? + async setUserDeleted(userId) { + return withTx(["users", "connections"], "readwrite", async (tx) => { + const store = tx.objectStore("users") + const user = await idbReq(store.get(userId)) + if (user) { + user.deleted = 1 + await idbReq(store.put(user)) + } + }) + }, + + // ============================================================ + // Servers — AgentStore.hs:233-240 (approximate, createServer is simple) + // ============================================================ + + // createServer + // SQL: INSERT OR IGNORE INTO servers (host, port, key_hash) VALUES (?,?,?) + async createServer(host, port, keyHash) { + return withTx("servers", "readwrite", async (tx) => { + const store = tx.objectStore("servers") + const existing = await idbReq(store.get([host, port])) + if (!existing) await idbReq(store.add({host, port, key_hash: keyHash})) + }) + }, + + // ============================================================ + // Connections — AgentStore.hs:409-498 + // ============================================================ + + // createNewConn (AgentStore.hs:409-411) → calls createConnRecord (AgentStore.hs:442-450) + // SQL: INSERT INTO connections (user_id, conn_id, conn_mode, smp_agent_version, enable_ntfs, pq_support, duplex_handshake) VALUES (?,?,?,?,?,?,?) + async createNewConn(connData, connMode) { + return withTx("connections", "readwrite", async (tx) => { + await idbReq(tx.objectStore("connections").add({ + conn_id: connData.connId, + user_id: connData.userId, + conn_mode: connMode, + smp_agent_version: connData.smpAgentVersion, + enable_ntfs: connData.enableNtfs ? 1 : 0, + pq_support: connData.pqSupport ? 1 : 0, + duplex_handshake: 1, + deleted: 0, + ratchet_sync_state: "ok", + last_internal_msg_id: 0, + last_internal_rcv_msg_id: 0, + last_internal_snd_msg_id: 0, + last_external_snd_msg_id: 0, + last_rcv_msg_hash: new Uint8Array(0), + last_snd_msg_hash: new Uint8Array(0), + })) + return connData.connId + }) + }, + + // Placeholder for remaining methods — each will be transpiled from the actual SQL + // TODO: transpile remaining ~70 methods from AgentStore.hs + + // getConn (AgentStore.hs:2248-2280) + // SQL: SELECT ... FROM connections WHERE conn_id = ? AND deleted = 0 + // Then gets rcv_queues and snd_queues by conn_id + async getConn(connId) { + return withTx(["connections", "rcv_queues", "snd_queues", "servers"], "readonly", async (tx) => { + // getConnData: SELECT user_id, conn_id, conn_mode, smp_agent_version, enable_ntfs, last_external_snd_msg_id, deleted, ratchet_sync_state, pq_support FROM connections WHERE conn_id = ? AND deleted = 0 + const conn = await idbReq(tx.objectStore("connections").get(connId)) + if (!conn || conn.deleted !== 0) return null + // getRcvQueuesByConnId_: rcvQueueQuery WHERE q.conn_id = ? AND q.deleted = 0 + const rcvQueues = (await allByIndex(tx.objectStore("rcv_queues"), {conn_id: connId})).filter((q: any) => !q.deleted) + // getSndQueuesByConnId_: sndQueueQuery WHERE q.conn_id = ? + const sndQueues = await allByIndex(tx.objectStore("snd_queues"), {conn_id: connId}) + return {connData: conn, rcvQueues, sndQueues} + }) + }, + + // getRcvConn (AgentStore.hs:467-472) + // SQL: rcvQueueQuery WHERE q.host = ? AND q.port = ? AND q.rcv_id = ? AND q.deleted = 0 + // Then getConn for the connId + async getRcvConn(host, port, rcvId) { + return withTx(["rcv_queues", "connections", "snd_queues", "servers"], "readonly", async (tx) => { + const rq = await idbReq(tx.objectStore("rcv_queues").get([host, port, rcvId])) + if (!rq || rq.deleted) return null + const conn = await idbReq(tx.objectStore("connections").get(rq.conn_id)) + if (!conn || conn.deleted !== 0) return null + return {connData: conn, rcvQueue: rq} + }) + }, + + // getConnSubs (AgentStore.hs:2295-2297) — gets ConnData for multiple connIds + async getConnSubs(connIds) { + return withTx("connections", "readonly", async (tx) => { + const store = tx.objectStore("connections") + const result = new Map() + for (const cid of connIds) { + const conn = await idbReq(store.get(cid)) + if (conn && conn.deleted === 0) result.set(toHex(cid), conn) + } + return result + }) + }, + + // getConnsData (AgentStore.hs:2371) — same as getConnSubs for our purposes + async getConnsData(connIds) { + return this.getConnSubs(connIds) + }, + + // lockConnForUpdate (AgentStore.hs:2394-2399) + // Postgres: SELECT 1 FROM connections WHERE conn_id = ? FOR UPDATE + // SQLite/IndexedDB: no-op (single-threaded) + async lockConnForUpdate() { }, + + // setConnDeleted (AgentStore.hs:2405-2411) + // waitDelivery=true: UPDATE connections SET deleted_at_wait_delivery = ? WHERE conn_id = ? + // waitDelivery=false: UPDATE connections SET deleted = 1 WHERE conn_id = ? + async setConnDeleted(connId, waitDelivery) { + return withTx("connections", "readwrite", async (tx) => { + const store = tx.objectStore("connections") + const conn = await idbReq(store.get(connId)) + if (conn) { + if (waitDelivery) { + conn.deleted_at_wait_delivery = new Date().toISOString() + } else { + conn.deleted = 1 + } + await idbReq(store.put(conn)) + } + }) + }, + + // setConnUserId (AgentStore.hs:2413-2415) + // SQL: UPDATE connections SET user_id = ? WHERE conn_id = ? AND user_id = ? + async setConnUserId(oldUserId, connId, newUserId) { + return withTx("connections", "readwrite", async (tx) => { + const store = tx.objectStore("connections") + const conn = await idbReq(store.get(connId)) + if (conn && conn.user_id === oldUserId) { + conn.user_id = newUserId + await idbReq(store.put(conn)) + } + }) + }, + + // setConnAgentVersion (AgentStore.hs:2417-2419) + // SQL: UPDATE connections SET smp_agent_version = ? WHERE conn_id = ? + async setConnAgentVersion(connId, version) { + return withTx("connections", "readwrite", async (tx) => { + const store = tx.objectStore("connections") + const conn = await idbReq(store.get(connId)) + if (conn) { + conn.smp_agent_version = version + await idbReq(store.put(conn)) + } + }) + }, + + // setConnPQSupport (AgentStore.hs:2421-2423) + // SQL: UPDATE connections SET pq_support = ? WHERE conn_id = ? + async setConnPQSupport(connId, pqSupport) { + return withTx("connections", "readwrite", async (tx) => { + const store = tx.objectStore("connections") + const conn = await idbReq(store.get(connId)) + if (conn) { + conn.pq_support = pqSupport ? 1 : 0 + await idbReq(store.put(conn)) + } + }) + }, + + // setConnRatchetSync (AgentStore.hs:2436) + // SQL: UPDATE connections SET ratchet_sync_state = ? WHERE conn_id = ? + async setConnRatchetSync(connId, state) { + return withTx("connections", "readwrite", async (tx) => { + const store = tx.objectStore("connections") + const conn = await idbReq(store.get(connId)) + if (conn) { + conn.ratchet_sync_state = state + await idbReq(store.put(conn)) + } + }) + }, + + // updateNewConnJoin (AgentStore.hs:2425-2427) + // SQL: UPDATE connections SET smp_agent_version = ?, pq_support = ?, enable_ntfs = ? WHERE conn_id = ? + async updateNewConnJoin(connId, agentVersion, pqSupport, enableNtfs) { + return withTx("connections", "readwrite", async (tx) => { + const store = tx.objectStore("connections") + const conn = await idbReq(store.get(connId)) + if (conn) { + conn.smp_agent_version = agentVersion + conn.pq_support = pqSupport ? 1 : 0 + conn.enable_ntfs = enableNtfs ? 1 : 0 + await idbReq(store.put(conn)) + } + }) + }, + + // updateNewConnRcv (AgentStore.hs:414-422) + // Calls addConnRcvQueue_ which calls insertRcvQueue_ + async updateNewConnRcv(connId, rcvQueue, subMode) { + return this.addConnRcvQueue(connId, rcvQueue, subMode) + }, + + // getDeletedConnIds (AgentStore.hs:2429-2430) + // SQL: SELECT conn_id FROM connections WHERE deleted = 1 + async getDeletedConnIds() { + return withTx("connections", "readonly", async (tx) => { + const all = await idbReq(tx.objectStore("connections").getAll()) + return all.filter((c: any) => c.deleted === 1 && !c.deleted_at_wait_delivery).map((c: any) => c.conn_id) + }) + }, + + // getDeletedWaitingDeliveryConnIds (AgentStore.hs:2432-2434) + // SQL: SELECT conn_id FROM connections WHERE deleted_at_wait_delivery IS NOT NULL + async getDeletedWaitingDeliveryConnIds() { + return withTx("connections", "readonly", async (tx) => { + const all = await idbReq(tx.objectStore("connections").getAll()) + return all.filter((c: any) => c.deleted_at_wait_delivery != null).map((c: any) => c.conn_id) + }) + }, + + // getConnIds — SELECT conn_id FROM connections + async getConnIds() { + return withTx("connections", "readonly", async (tx) => { + return idbReq(tx.objectStore("connections").getAllKeys()) + }) + }, + + // addConnRcvQueue (AgentStore.hs:516-526 → insertRcvQueue_ at 2077-2099) + // SQL: INSERT INTO rcv_queues (host, port, rcv_id, conn_id, rcv_private_key, rcv_dh_secret, e2e_priv_key, e2e_dh_secret, snd_id, queue_mode, status, to_subscribe, rcv_queue_id, rcv_primary, ...) VALUES (...) + async addConnRcvQueue(connId, rcvQueue, subMode) { + return withTx(["rcv_queues", "servers"], "readwrite", async (tx) => { + // createServer (INSERT OR IGNORE) + const srvStore = tx.objectStore("servers") + if (!(await idbReq(srvStore.get([rcvQueue.host, rcvQueue.port])))) await idbReq(srvStore.add({host: rcvQueue.host, port: rcvQueue.port, key_hash: rcvQueue.serverKeyHash})) + // Determine rcv_queue_id: SELECT MAX(rcv_queue_id) FROM rcv_queues WHERE conn_id = ? + const existing = await allByIndex(tx.objectStore("rcv_queues"), {conn_id: connId}) + const maxId = existing.reduce((m: number, q: any) => Math.max(m, q.rcv_queue_id || 0), 0) + const qId = maxId + 1 + const toSubscribe = subMode === "SMOnlyCreate" ? 1 : 0 + await idbReq(tx.objectStore("rcv_queues").add({ + host: rcvQueue.host, port: rcvQueue.port, rcv_id: rcvQueue.rcvId, + conn_id: connId, rcv_private_key: rcvQueue.rcvPrivateKey, rcv_dh_secret: rcvQueue.rcvDhSecret, + e2e_priv_key: rcvQueue.e2ePrivKey, e2e_dh_secret: rcvQueue.e2eDhSecret, + snd_id: rcvQueue.sndId, queue_mode: rcvQueue.queueMode, status: rcvQueue.status, + to_subscribe: toSubscribe, rcv_queue_id: qId, rcv_primary: rcvQueue.primary ? 1 : 0, + replace_rcv_queue_id: rcvQueue.replaceRcvQueueId, smp_client_version: rcvQueue.smpClientVersion, + server_key_hash: rcvQueue.serverKeyHash, deleted: 0, + snd_key: rcvQueue.sndKey, last_broker_ts: rcvQueue.lastBrokerTs, + })) + return {...rcvQueue, connId, dbQueueId: qId} + }) + }, + + // addConnSndQueue (AgentStore.hs:528-538 → insertSndQueue_ at 2109-2140) + // SQL: INSERT INTO snd_queues (host, port, snd_id, queue_mode, conn_id, snd_private_key, e2e_pub_key, e2e_dh_secret, status, snd_queue_id, snd_primary, ...) VALUES (...) ON CONFLICT DO UPDATE + async addConnSndQueue(connId, sndQueue) { + return withTx(["snd_queues", "servers"], "readwrite", async (tx) => { + const srvStore2 = tx.objectStore("servers") + if (!(await idbReq(srvStore2.get([sndQueue.host, sndQueue.port])))) await idbReq(srvStore2.add({host: sndQueue.host, port: sndQueue.port, key_hash: sndQueue.serverKeyHash})) + const existing = await allByIndex(tx.objectStore("snd_queues"), {conn_id: connId}) + const maxId = existing.reduce((m: number, q: any) => Math.max(m, q.snd_queue_id || 0), 0) + const qId = maxId + 1 + // ON CONFLICT DO UPDATE → use put (upsert) + await idbReq(tx.objectStore("snd_queues").put({ + host: sndQueue.host, port: sndQueue.port, snd_id: sndQueue.sndId, + queue_mode: sndQueue.queueMode, conn_id: connId, + snd_private_key: sndQueue.sndPrivateKey, e2e_pub_key: sndQueue.e2ePubKey, + e2e_dh_secret: sndQueue.e2eDhSecret, status: sndQueue.status, + snd_queue_id: qId, snd_primary: sndQueue.primary ? 1 : 0, + replace_snd_queue_id: null, smp_client_version: sndQueue.smpClientVersion, + server_key_hash: sndQueue.serverKeyHash, snd_public_key: sndQueue.sndPublicKey, + })) + }) + }, + + // setRcvQueueStatus (AgentStore.hs:540-550) + // SQL: UPDATE rcv_queues SET status = ? WHERE host = ? AND port = ? AND rcv_id = ? + async setRcvQueueStatus(rcvQueue, status) { + return withTx("rcv_queues", "readwrite", async (tx) => { + const store = tx.objectStore("rcv_queues") + const rq = await idbReq(store.get([rcvQueue.host, rcvQueue.port, rcvQueue.rcvId])) + if (rq) { rq.status = status; await idbReq(store.put(rq)) } + }) + }, + + // setSndQueueStatus (AgentStore.hs:588-598) + // SQL: UPDATE snd_queues SET status = ? WHERE host = ? AND port = ? AND snd_id = ? + async setSndQueueStatus(sndQueue, status) { + return withTx("snd_queues", "readwrite", async (tx) => { + const store = tx.objectStore("snd_queues") + const sq = await idbReq(store.get([sndQueue.host, sndQueue.port, sndQueue.sndId])) + if (sq) { sq.status = status; await idbReq(store.put(sq)) } + }) + }, + + // setRcvQueueConfirmedE2E (AgentStore.hs:575-586) + // SQL: UPDATE rcv_queues SET e2e_dh_secret = ?, status = 'confirmed', smp_client_version = ? WHERE host = ? AND port = ? AND rcv_id = ? + async setRcvQueueConfirmedE2E(rcvQueue, dhSecret, smpClientVersion) { + return withTx("rcv_queues", "readwrite", async (tx) => { + const store = tx.objectStore("rcv_queues") + const rq = await idbReq(store.get([rcvQueue.host, rcvQueue.port, rcvQueue.rcvId])) + if (rq) { rq.e2e_dh_secret = dhSecret; rq.status = "confirmed"; rq.smp_client_version = smpClientVersion; await idbReq(store.put(rq)) } + }) + }, + + // setRcvQueuePrimary (AgentStore.hs:612-618) + // SQL1: UPDATE rcv_queues SET rcv_primary = 0 WHERE conn_id = ? + // SQL2: UPDATE rcv_queues SET rcv_primary = 1, replace_rcv_queue_id = NULL WHERE conn_id = ? AND rcv_queue_id = ? + async setRcvQueuePrimary(connId, rcvQueue) { + return withTx("rcv_queues", "readwrite", async (tx) => { + const store = tx.objectStore("rcv_queues") + const all = await allByIndex(store, {conn_id: connId}) + for (const rq of all) { + if (rq.rcv_queue_id === rcvQueue.dbQueueId) { + rq.rcv_primary = 1; rq.replace_rcv_queue_id = null + } else { + rq.rcv_primary = 0 + } + await idbReq(store.put(rq)) + } + }) + }, + + // deleteConnRcvQueue (AgentStore.hs:632-634) + // SQL: DELETE FROM rcv_queues WHERE conn_id = ? AND rcv_queue_id = ? + async deleteConnRcvQueue(rcvQueue) { + return withTx("rcv_queues", "readwrite", async (tx) => { + await idbReq(tx.objectStore("rcv_queues").delete([rcvQueue.host, rcvQueue.port, rcvQueue.rcvId])) + }) + }, + + // deleteConnRecord (AgentStore.hs:452-453) + // SQL: DELETE FROM connections WHERE conn_id = ? + async deleteConnRecord(connId) { + return withTx("connections", "readwrite", async (tx) => { + await idbReq(tx.objectStore("connections").delete(connId)) + }) + }, + + // upgradeRcvConnToDuplex (AgentStore.hs:501-505) + // Adds snd queue to an existing rcv connection + async upgradeRcvConnToDuplex(connId, sndQueue) { + return this.addConnSndQueue(connId, sndQueue) + }, + + // upgradeSndConnToDuplex (AgentStore.hs:508-513) + // Adds rcv queue to an existing snd connection + async upgradeSndConnToDuplex(connId, rcvQueue, subMode) { + return this.addConnRcvQueue(connId, rcvQueue, subMode) + }, + + // getPrimaryRcvQueue (AgentStore.hs:641-643) + // Gets rcv queues by connId, returns first (primary, sorted) + async getPrimaryRcvQueue(connId) { + return withTx("rcv_queues", "readonly", async (tx) => { + const all = await allByIndex(tx.objectStore("rcv_queues"), {conn_id: connId}) + const active = all.filter((q: any) => !q.deleted) + // Sort by primary first (primary=1 first) + active.sort((a: any, b: any) => (b.rcv_primary || 0) - (a.rcv_primary || 0)) + return active[0] ?? null + }) + }, + + // getRcvQueue (AgentStore.hs:645-648) + // SQL: rcvQueueQuery WHERE q.conn_id = ? AND q.host = ? AND q.port = ? AND q.rcv_id = ? AND q.deleted = 0 + async getRcvQueue(connId, host, port, rcvId) { + return withTx("rcv_queues", "readonly", async (tx) => { + const rq = await idbReq(tx.objectStore("rcv_queues").get([host, port, rcvId])) + if (!rq || rq.deleted || toHex(rq.conn_id) !== toHex(connId)) return null + return rq + }) + }, + + // getDeletedRcvQueue (AgentStore.hs:650-653) + // SQL: same but q.deleted = 1 + async getDeletedRcvQueue(connId, host, port, rcvId) { + return withTx("rcv_queues", "readonly", async (tx) => { + const rq = await idbReq(tx.objectStore("rcv_queues").get([host, port, rcvId])) + if (!rq || !rq.deleted || toHex(rq.conn_id) !== toHex(connId)) return null + return rq + }) + }, + + // setConnectionNtfs — UPDATE connections SET enable_ntfs = ? WHERE conn_id = ? + async setConnectionNtfs(connId, enable) { + return withTx("connections", "readwrite", async (tx) => { + const store = tx.objectStore("connections") + const conn = await idbReq(store.get(connId)) + if (conn) { conn.enable_ntfs = enable ? 1 : 0; await idbReq(store.put(conn)) } + }) + }, + + // ============================================================ + // Subscriptions — AgentStore.hs:2211-2242, 943-954 + // ============================================================ + + // getSubscriptionServers (AgentStore.hs:2211-2226) + // SQL: SELECT DISTINCT c.user_id, q.host, q.port, COALESCE(q.server_key_hash, s.key_hash) + // FROM rcv_queues q + // JOIN servers s ON q.host = s.host AND q.port = s.port + // JOIN connections c ON q.conn_id = c.conn_id + // WHERE [q.to_subscribe = 1 AND] c.deleted = 0 AND q.deleted = 0 + async getSubscriptionServers(onlyNeeded) { + return withTx(["rcv_queues", "connections", "servers"], "readonly", async (tx) => { + const allQ = await idbReq(tx.objectStore("rcv_queues").getAll()) + const connStore = tx.objectStore("connections") + const srvStore = tx.objectStore("servers") + const seen = new Set() + const result: Array<{userId: number, host: string, port: string}> = [] + for (const q of allQ) { + if (q.deleted) continue + if (onlyNeeded && !q.to_subscribe) continue + const conn = await idbReq(connStore.get(q.conn_id)) + if (!conn || conn.deleted !== 0) continue + const keyHash = q.server_key_hash ?? (await idbReq(srvStore.get([q.host, q.port])))?.key_hash + const key = `${conn.user_id}:${q.host}:${q.port}:${keyHash ? toHex(keyHash) : ""}` + if (!seen.has(key)) { + seen.add(key) + result.push({userId: conn.user_id, host: q.host, port: q.port}) + } + } + return result + }) + }, + + // getUserServerRcvQueueSubs (AgentStore.hs:2228-2238) + // SQL: rcvQueueSubQuery WHERE [q.to_subscribe = 1 AND] c.deleted = 0 AND q.deleted = 0 + // AND c.user_id = ? AND q.host = ? AND q.port = ? AND COALESCE(q.server_key_hash, s.key_hash) = ? + // ORDER BY q.rcv_id LIMIT ? + async getUserServerRcvQueueSubs(userId, host, port, onlyNeeded, batchSize, cursor) { + return withTx(["rcv_queues", "connections", "servers"], "readonly", async (tx) => { + const allQ = await idbReq(tx.objectStore("rcv_queues").getAll()) + const connStore = tx.objectStore("connections") + const srvStore = tx.objectStore("servers") + // Filter matching queues + const matching: any[] = [] + for (const q of allQ) { + if (q.deleted) continue + if (q.host !== host || q.port !== port) continue + if (onlyNeeded && !q.to_subscribe) continue + if (cursor !== null && compareUint8Array(q.rcv_id, cursor as any) <= 0) continue + const conn = await idbReq(connStore.get(q.conn_id)) + if (!conn || conn.deleted !== 0 || conn.user_id !== userId) continue + matching.push(q) + } + // ORDER BY q.rcv_id + matching.sort((a, b) => compareUint8Array(a.rcv_id, b.rcv_id)) + // LIMIT + const queues = matching.slice(0, batchSize) + const nextCursor = queues.length === batchSize ? queues[queues.length - 1].rcv_id : null + return {queues, nextCursor} + }) + }, + + // unsetQueuesToSubscribe (AgentStore.hs:2240-2241) + // SQL: UPDATE rcv_queues SET to_subscribe = 0 WHERE to_subscribe = 1 + async unsetQueuesToSubscribe() { + return withTx("rcv_queues", "readwrite", async (tx) => { + const store = tx.objectStore("rcv_queues") + const all = await idbReq(store.getAll()) + for (const q of all) { + if (q.to_subscribe === 1) { + q.to_subscribe = 0 + await idbReq(store.put(q)) + } + } + }) + }, + + // getConnectionsForDelivery (AgentStore.hs:943-945) + // SQL: SELECT DISTINCT conn_id FROM snd_message_deliveries WHERE failed = 0 + async getConnectionsForDelivery() { + return withTx("snd_message_deliveries", "readonly", async (tx) => { + const all = await idbReq(tx.objectStore("snd_message_deliveries").getAll()) + const seen = new Set() + const result: Uint8Array[] = [] + for (const d of all) { + if (d.failed) continue + const hex = toHex(d.conn_id) + if (!seen.has(hex)) { + seen.add(hex) + result.push(d.conn_id) + } + } + return result + }) + }, + + // getAllSndQueuesForDelivery (AgentStore.hs:947-954) + // SQL: sndQueueQuery + // JOIN (SELECT DISTINCT conn_id, snd_queue_id FROM snd_message_deliveries WHERE failed = 0) d + // ON d.conn_id = q.conn_id AND d.snd_queue_id = q.snd_queue_id + // WHERE c.deleted = 0 + async getAllSndQueuesForDelivery() { + return withTx(["snd_message_deliveries", "snd_queues", "connections"], "readonly", async (tx) => { + // Get distinct (conn_id, snd_queue_id) from non-failed deliveries + const allDel = await idbReq(tx.objectStore("snd_message_deliveries").getAll()) + const deliveryKeys = new Set() + for (const d of allDel) { + if (!d.failed) deliveryKeys.add(toHex(d.conn_id) + ":" + d.snd_queue_id) + } + // Get snd_queues that match + const allSQ = await idbReq(tx.objectStore("snd_queues").getAll()) + const connStore = tx.objectStore("connections") + const result: any[] = [] + for (const sq of allSQ) { + const key = toHex(sq.conn_id) + ":" + sq.snd_queue_id + if (!deliveryKeys.has(key)) continue + const conn = await idbReq(connStore.get(sq.conn_id)) + if (!conn || conn.deleted !== 0) continue + result.push(sq) + } + return result + }) + }, + + // ============================================================ + // Confirmations — AgentStore.hs:682-752 + // ============================================================ + + // createConfirmation (AgentStore.hs:682-691) + // SQL: INSERT INTO conn_confirmations + // (confirmation_id, conn_id, sender_key, e2e_snd_pub_key, ratchet_state, sender_conn_info, smp_reply_queues, smp_client_version, accepted) + // VALUES (?, ?, ?, ?, ?, ?, ?, ?, 0) + async createConfirmation(confirmation) { + return withTx("conn_confirmations", "readwrite", async (tx) => { + await idbReq(tx.objectStore("conn_confirmations").add({ + confirmation_id: confirmation.confirmationId, + conn_id: confirmation.connId, + sender_key: confirmation.senderKey, + e2e_snd_pub_key: confirmation.e2eSndPubKey, + ratchet_state: confirmation.ratchetState, + sender_conn_info: confirmation.senderConnInfo, + smp_reply_queues: confirmation.smpReplyQueues, + smp_client_version: confirmation.smpClientVersion, + accepted: 0, + own_conn_info: null, + })) + return confirmation.confirmationId + }) + }, + + // acceptConfirmation (AgentStore.hs:693-721) + // SQL1: UPDATE conn_confirmations SET accepted = 1, own_conn_info = ? WHERE confirmation_id = ? + // SQL2: SELECT conn_id, ratchet_state, sender_key, e2e_snd_pub_key, sender_conn_info, smp_reply_queues, smp_client_version + // FROM conn_confirmations WHERE confirmation_id = ? + async acceptConfirmation(confirmationId, ownConnInfo) { + return withTx("conn_confirmations", "readwrite", async (tx) => { + const store = tx.objectStore("conn_confirmations") + const conf = await idbReq(store.get(confirmationId)) + if (!conf) throw new Error("confirmation not found") + conf.accepted = 1 + conf.own_conn_info = ownConnInfo + await idbReq(store.put(conf)) + return conf + }) + }, + + // getAcceptedConfirmation (AgentStore.hs:723-742) + // SQL: SELECT confirmation_id, ratchet_state, own_conn_info, sender_key, e2e_snd_pub_key, sender_conn_info, smp_reply_queues, smp_client_version + // FROM conn_confirmations WHERE conn_id = ? AND accepted = 1 + async getAcceptedConfirmation(connId) { + return withTx("conn_confirmations", "readonly", async (tx) => { + const all = await allByIndex(tx.objectStore("conn_confirmations"), {conn_id: connId}) + const accepted = all.find((c: any) => c.accepted === 1) + return accepted ?? null + }) + }, + + // removeConfirmations (AgentStore.hs:744-752) + // SQL: DELETE FROM conn_confirmations WHERE conn_id = ? + async removeConfirmations(connId) { + return withTx("conn_confirmations", "readwrite", async (tx) => { + const store = tx.objectStore("conn_confirmations") + const allConf = await allByIndex(store, {conn_id: connId}) + const all = allConf.map((c: any) => c.confirmation_id) + for (const key of all) { + await idbReq(store.delete(key)) + } + }) + }, + + // ============================================================ + // Invitations — AgentStore.hs:754-799 + // ============================================================ + + // createInvitation (AgentStore.hs:754-763) + // SQL: INSERT INTO conn_invitations + // (invitation_id, contact_conn_id, cr_invitation, recipient_conn_info, accepted) + // VALUES (?, ?, ?, ?, 0) + async createInvitation(invitation) { + return withTx("conn_invitations", "readwrite", async (tx) => { + await idbReq(tx.objectStore("conn_invitations").add({ + invitation_id: invitation.invitationId, + contact_conn_id: invitation.contactConnId, + cr_invitation: invitation.crInvitation, + recipient_conn_info: invitation.recipientConnInfo, + accepted: 0, + own_conn_info: null, + })) + return invitation.invitationId + }) + }, + + // getInvitation (AgentStore.hs:765-779) + // SQL: SELECT contact_conn_id, cr_invitation, recipient_conn_info, own_conn_info, accepted + // FROM conn_invitations WHERE invitation_id = ? AND accepted = 0 + async getInvitation(invitationId) { + return withTx("conn_invitations", "readonly", async (tx) => { + const inv = await idbReq(tx.objectStore("conn_invitations").get(invitationId)) + if (!inv || inv.accepted !== 0) return null + return inv + }) + }, + + // acceptInvitation (AgentStore.hs:781-791) + // SQL: UPDATE conn_invitations SET accepted = 1, own_conn_info = ? WHERE invitation_id = ? + async acceptInvitation(invitationId, ownConnInfo) { + return withTx("conn_invitations", "readwrite", async (tx) => { + const store = tx.objectStore("conn_invitations") + const inv = await idbReq(store.get(invitationId)) + if (inv) { + inv.accepted = 1 + inv.own_conn_info = ownConnInfo + await idbReq(store.put(inv)) + } + }) + }, + + // unacceptInvitation (AgentStore.hs:793-795) + // SQL: UPDATE conn_invitations SET accepted = 0, own_conn_info = NULL WHERE invitation_id = ? + async unacceptInvitation(invitationId) { + return withTx("conn_invitations", "readwrite", async (tx) => { + const store = tx.objectStore("conn_invitations") + const inv = await idbReq(store.get(invitationId)) + if (inv) { + inv.accepted = 0 + inv.own_conn_info = null + await idbReq(store.put(inv)) + } + }) + }, + + // deleteInvitation (AgentStore.hs:797-799) + // SQL: DELETE FROM conn_invitations WHERE invitation_id = ? + async deleteInvitation(invitationId) { + return withTx("conn_invitations", "readwrite", async (tx) => { + await idbReq(tx.objectStore("conn_invitations").delete(invitationId)) + }) + }, + + // ============================================================ + // Messages — AgentStore.hs:873-1205 + // ============================================================ + + // updateRcvIds (AgentStore.hs:873-879) + // Calls retrieveLastIdsAndHashRcv_ (AgentStore.hs:2584-2599): + // SQL: SELECT last_internal_msg_id, last_internal_rcv_msg_id, last_external_snd_msg_id, last_rcv_msg_hash FROM connections WHERE conn_id = ? + // Then updateLastIdsRcv_ (AgentStore.hs:2601-2611): + // SQL: UPDATE connections SET last_internal_msg_id = ?, last_internal_rcv_msg_id = ? WHERE conn_id = ? + async updateRcvIds(connId) { + return withTx("connections", "readwrite", async (tx) => { + const store = tx.objectStore("connections") + const conn = await idbReq(store.get(connId)) + if (!conn) throw new Error("connection not found") + const internalId = conn.last_internal_msg_id + 1 + const internalRcvId = conn.last_internal_rcv_msg_id + 1 + const prevExternalSndId = conn.last_external_snd_msg_id + const prevRcvMsgHash = conn.last_rcv_msg_hash + conn.last_internal_msg_id = internalId + conn.last_internal_rcv_msg_id = internalRcvId + await idbReq(store.put(conn)) + return {internalId, internalRcvId, prevExternalSndId, prevRcvMsgHash} + }) + }, + + // createRcvMsg (AgentStore.hs:881-886) + // Calls insertRcvMsgBase_ (AgentStore.hs:2615-2625): + // SQL: INSERT INTO messages (conn_id, internal_id, internal_ts, internal_rcv_id, internal_snd_id, msg_type, msg_flags, msg_body, pq_encryption) VALUES (?,?,?,?,?,?,?,?,?) + // Calls insertRcvMsgDetails_ (AgentStore.hs:2627-2641): + // SQL: INSERT INTO rcv_messages (conn_id, rcv_queue_id, internal_rcv_id, internal_id, external_snd_id, broker_id, broker_ts, internal_hash, external_prev_snd_hash, integrity) VALUES (?,?,?,?,?,?,?,?,?,?) + // SQL: INSERT INTO encrypted_rcv_message_hashes (conn_id, hash) VALUES (?,?) + // Calls updateRcvMsgHash (AgentStore.hs:2643-2655): + // SQL: UPDATE connections SET last_external_snd_msg_id = ?, last_rcv_msg_hash = ? WHERE conn_id = ? AND last_internal_rcv_msg_id = ? + // Calls setLastBrokerTs (AgentStore.hs:888-890): + // SQL: UPDATE rcv_queues SET last_broker_ts = ? WHERE conn_id = ? AND rcv_queue_id = ? AND (last_broker_ts IS NULL OR last_broker_ts < ?) + async createRcvMsg(connId, rcvQueue, rcvMsgData) { + return withTx(["messages", "rcv_messages", "encrypted_rcv_message_hashes", "connections", "rcv_queues"], "readwrite", async (tx) => { + const {msgMeta, msgType, msgFlags, msgBody, internalRcvId, internalHash, externalPrevSndHash, encryptedMsgHash} = rcvMsgData + const {recipient, broker, sndMsgId, integrity, pqEncryption} = msgMeta + const [internalId, internalTs] = recipient + const [brokerId, brokerTs] = broker + + // insertRcvMsgBase_: INSERT INTO messages + await idbReq(tx.objectStore("messages").add({ + conn_id: connId, internal_id: internalId, internal_ts: internalTs, + internal_rcv_id: internalRcvId, internal_snd_id: null, + msg_type: msgType, msg_flags: msgFlags, msg_body: msgBody, pq_encryption: pqEncryption, + })) + + // insertRcvMsgDetails_: INSERT INTO rcv_messages + await idbReq(tx.objectStore("rcv_messages").add({ + conn_id: connId, rcv_queue_id: rcvQueue.dbQueueId, internal_rcv_id: internalRcvId, + internal_id: internalId, external_snd_id: sndMsgId, + broker_id: brokerId, broker_ts: brokerTs, + internal_hash: internalHash, external_prev_snd_hash: externalPrevSndHash, + integrity, user_ack: 0, receive_attempts: 0, + })) + + // insertRcvMsgDetails_: INSERT INTO encrypted_rcv_message_hashes + await idbReq(tx.objectStore("encrypted_rcv_message_hashes").add({ + conn_id: connId, hash: encryptedMsgHash, created_at: new Date().toISOString(), + })) + + // updateRcvMsgHash: UPDATE connections SET last_external_snd_msg_id, last_rcv_msg_hash + const connStore = tx.objectStore("connections") + const conn = await idbReq(connStore.get(connId)) + if (conn && conn.last_internal_rcv_msg_id === internalRcvId) { + conn.last_external_snd_msg_id = sndMsgId + conn.last_rcv_msg_hash = internalHash + await idbReq(connStore.put(conn)) + } + + // setLastBrokerTs: UPDATE rcv_queues SET last_broker_ts + const rcvQStore = tx.objectStore("rcv_queues") + const allRQ = await allByIndex(rcvQStore, {conn_id: connId}) + for (const rq of allRQ) { + if (rq.rcv_queue_id === rcvQueue.dbQueueId) { + if (rq.last_broker_ts == null || rq.last_broker_ts < brokerTs) { + rq.last_broker_ts = brokerTs + await idbReq(rcvQStore.put(rq)) + } + break + } + } + }) + }, + + // setLastBrokerTs (AgentStore.hs:888-890) + // SQL: UPDATE rcv_queues SET last_broker_ts = ? WHERE conn_id = ? AND rcv_queue_id = ? AND (last_broker_ts IS NULL OR last_broker_ts < ?) + async setLastBrokerTs(connId, dbQueueId, brokerTs) { + return withTx("rcv_queues", "readwrite", async (tx) => { + const store = tx.objectStore("rcv_queues") + const all = await allByIndex(store, {conn_id: connId}) + for (const rq of all) { + if (rq.rcv_queue_id === dbQueueId) { + if (rq.last_broker_ts == null || rq.last_broker_ts < brokerTs) { + rq.last_broker_ts = brokerTs + await idbReq(store.put(rq)) + } + break + } + } + }) + }, + + // updateRcvMsgHash (AgentStore.hs:2643-2655) + // SQL: UPDATE connections SET last_external_snd_msg_id = ?, last_rcv_msg_hash = ? + // WHERE conn_id = ? AND last_internal_rcv_msg_id = ? + async updateRcvMsgHash(connId, sndMsgId, internalRcvId, hash) { + return withTx("connections", "readwrite", async (tx) => { + const store = tx.objectStore("connections") + const conn = await idbReq(store.get(connId)) + if (conn && conn.last_internal_rcv_msg_id === internalRcvId) { + conn.last_external_snd_msg_id = sndMsgId + conn.last_rcv_msg_hash = hash + await idbReq(store.put(conn)) + } + }) + }, + + // createSndMsgBody (AgentStore.hs:892-898) + // SQL: INSERT INTO snd_message_bodies (agent_msg) VALUES (?) RETURNING snd_message_body_id + async createSndMsgBody(agentMsg) { + return withTx("snd_message_bodies", "readwrite", async (tx) => { + return await idbReq(tx.objectStore("snd_message_bodies").add({agent_msg: agentMsg})) as number + }) + }, + + // updateSndIds (AgentStore.hs:900-906) + // Calls retrieveLastIdsAndHashSnd_ (AgentStore.hs:2659-2673): + // SQL: SELECT last_internal_msg_id, last_internal_snd_msg_id, last_snd_msg_hash FROM connections WHERE conn_id = ? + // Then updateLastIdsSnd_ (AgentStore.hs:2675-2685): + // SQL: UPDATE connections SET last_internal_msg_id = ?, last_internal_snd_msg_id = ? WHERE conn_id = ? + async updateSndIds(connId) { + return withTx("connections", "readwrite", async (tx) => { + const store = tx.objectStore("connections") + const conn = await idbReq(store.get(connId)) + if (!conn) throw new Error("connection not found") + const internalId = conn.last_internal_msg_id + 1 + const internalSndId = conn.last_internal_snd_msg_id + 1 + const prevSndMsgHash = conn.last_snd_msg_hash + conn.last_internal_msg_id = internalId + conn.last_internal_snd_msg_id = internalSndId + await idbReq(store.put(conn)) + return {internalId, internalSndId, prevSndMsgHash} + }) + }, + + // createSndMsg (AgentStore.hs:908-912) + // Calls insertSndMsgBase_ (AgentStore.hs:2689-2699): + // SQL: INSERT INTO messages (conn_id, internal_id, internal_ts, internal_rcv_id, internal_snd_id, msg_type, msg_flags, msg_body, pq_encryption) VALUES (?,?,?,?,?,?,?,?,?) + // Calls insertSndMsgDetails_ (AgentStore.hs:2701-2715): + // SQL: INSERT INTO snd_messages (conn_id, internal_snd_id, internal_id, internal_hash, previous_msg_hash, msg_encrypt_key, padded_msg_len, snd_message_body_id) VALUES (?,?,?,?,?,?,?,?) + // Calls updateSndMsgHash (AgentStore.hs:2717-2728): + // SQL: UPDATE connections SET last_snd_msg_hash = ? WHERE conn_id = ? AND last_internal_snd_msg_id = ? + async createSndMsg(connId, sndMsgData) { + return withTx(["messages", "snd_messages", "connections"], "readwrite", async (tx) => { + const {internalId, internalSndId, internalTs, msgType, msgFlags, msgBody, pqEncryption, + internalHash, prevMsgHash, msgEncryptKey, paddedMsgLen, sndMessageBodyId} = sndMsgData + + // insertSndMsgBase_: INSERT INTO messages + await idbReq(tx.objectStore("messages").add({ + conn_id: connId, internal_id: internalId, internal_ts: internalTs, + internal_rcv_id: null, internal_snd_id: internalSndId, + msg_type: msgType, msg_flags: msgFlags, msg_body: msgBody, pq_encryption: pqEncryption, + })) + + // insertSndMsgDetails_: INSERT INTO snd_messages + await idbReq(tx.objectStore("snd_messages").add({ + conn_id: connId, internal_snd_id: internalSndId, internal_id: internalId, + internal_hash: internalHash, previous_msg_hash: prevMsgHash, + msg_encrypt_key: msgEncryptKey, padded_msg_len: paddedMsgLen, + snd_message_body_id: sndMessageBodyId, + retry_int_slow: null, retry_int_fast: null, + rcpt_internal_id: null, rcpt_status: null, + })) + + // updateSndMsgHash: UPDATE connections SET last_snd_msg_hash + const connStore = tx.objectStore("connections") + const conn = await idbReq(connStore.get(connId)) + if (conn && conn.last_internal_snd_msg_id === internalSndId) { + conn.last_snd_msg_hash = internalHash + await idbReq(connStore.put(conn)) + } + }) + }, + + // updateSndMsgHash (AgentStore.hs:2717-2728) + // SQL: UPDATE connections SET last_snd_msg_hash = ? WHERE conn_id = ? AND last_internal_snd_msg_id = ? + async updateSndMsgHash(connId, internalSndId, hash) { + return withTx("connections", "readwrite", async (tx) => { + const store = tx.objectStore("connections") + const conn = await idbReq(store.get(connId)) + if (conn && conn.last_internal_snd_msg_id === internalSndId) { + conn.last_snd_msg_hash = hash + await idbReq(store.put(conn)) + } + }) + }, + + // createSndMsgDelivery (AgentStore.hs:914-916) + // SQL: INSERT INTO snd_message_deliveries (conn_id, snd_queue_id, internal_id) VALUES (?, ?, ?) + async createSndMsgDelivery(connId, sndQueue, internalId) { + return withTx("snd_message_deliveries", "readwrite", async (tx) => { + await idbReq(tx.objectStore("snd_message_deliveries").add({ + conn_id: connId, snd_queue_id: sndQueue.dbQueueId, internal_id: internalId, failed: 0, + })) + }) + }, + + // getPendingQueueMsg (AgentStore.hs:956-1002) + // getMsgId SQL: SELECT internal_id FROM snd_message_deliveries d + // WHERE conn_id = ? AND snd_queue_id = ? AND failed = 0 + // ORDER BY internal_id ASC LIMIT 1 + // getMsgData SQL: SELECT m.msg_type, m.msg_flags, m.msg_body, m.pq_encryption, m.internal_ts, m.internal_snd_id, s.previous_msg_hash, + // s.retry_int_slow, s.retry_int_fast, s.msg_encrypt_key, s.padded_msg_len, sb.agent_msg + // FROM messages m + // JOIN snd_messages s ON s.conn_id = m.conn_id AND s.internal_id = m.internal_id + // LEFT JOIN snd_message_bodies sb ON sb.snd_message_body_id = s.snd_message_body_id + // WHERE m.conn_id = ? AND m.internal_id = ? + async getPendingQueueMsg(connId, sndQueue) { + return withTx(["snd_message_deliveries", "messages", "snd_messages", "snd_message_bodies"], "readonly", async (tx) => { + // getMsgId: find first non-failed delivery for this queue + const allDel = await allByIndex(tx.objectStore("snd_message_deliveries"), {conn_id: connId, snd_queue_id: sndQueue.dbQueueId}) + const pending = allDel.filter((d: any) => !d.failed).sort((a: any, b: any) => a.internal_id - b.internal_id) + if (pending.length === 0) return null + const msgId = pending[0].internal_id + + // getMsgData: JOIN messages + snd_messages + snd_message_bodies + const msg = await idbReq(tx.objectStore("messages").get([connId, msgId])) + if (!msg) return null + // Find snd_message by [conn_id, internal_id] via index + const sndMsgs = await allByIndex(tx.objectStore("snd_messages"), {conn_id: connId, internal_id: msgId}) + if (sndMsgs.length === 0) return null + const sm = sndMsgs[0] + // LEFT JOIN snd_message_bodies + let sndMsgBody: Uint8Array | null = null + if (sm.snd_message_body_id != null) { + const body = await idbReq(tx.objectStore("snd_message_bodies").get(sm.snd_message_body_id)) + if (body) sndMsgBody = body.agent_msg + } + + return { + connId, sndQueueId: sndQueue.dbQueueId, internalId: msgId, + msgType: msg.msg_type, msgFlags: msg.msg_flags, msgBody: msg.msg_body, + internalHash: sm.internal_hash, prevMsgHash: sm.previous_msg_hash, + pqEncryption: msg.pq_encryption, + msgEncryptKey: sm.msg_encrypt_key, paddedMsgLen: sm.padded_msg_len, + sndMessageBodyId: sm.snd_message_body_id, + } + }) + }, + + // updatePendingMsgRIState (AgentStore.hs:1024-1026) + // SQL: UPDATE snd_messages SET retry_int_slow = ?, retry_int_fast = ? WHERE conn_id = ? AND internal_id = ? + async updatePendingMsgRIState(connId, msgId, retryIntSlow, retryIntFast) { + return withTx("snd_messages", "readwrite", async (tx) => { + const store = tx.objectStore("snd_messages") + const all = await allByIndex(store, {conn_id: connId, internal_id: msgId}) + if (all.length > 0) { + const sm = all[0] + sm.retry_int_slow = retryIntSlow + sm.retry_int_fast = retryIntFast + await idbReq(store.put(sm)) + } + }) + }, + + // setMsgUserAck (AgentStore.hs:1059-1073) + // SQL1: SELECT rcv_queue_id, broker_id FROM rcv_messages WHERE conn_id = ? AND internal_id = ? + // SQL2: UPDATE rcv_messages SET user_ack = 1 WHERE conn_id = ? AND internal_id = ? + // Then getRcvQueueById to return the rcv queue + async setMsgUserAck(connId, internalId) { + return withTx(["rcv_messages", "rcv_queues"], "readwrite", async (tx) => { + const rmStore = tx.objectStore("rcv_messages") + // Find rcv_message by conn_id + internal_id via index + const all = await allByIndex(rmStore, {conn_id: connId, internal_id: internalId}) + if (all.length === 0) throw new Error("rcv message not found") + const rm = all[0] + const dbRcvId = rm.rcv_queue_id + const brokerId = rm.broker_id + // Update user_ack + rm.user_ack = 1 + await idbReq(rmStore.put(rm)) + // getRcvQueueById: find rcv_queue by conn_id + rcv_queue_id + const rqStore = tx.objectStore("rcv_queues") + const rqs = await allByIndex(rqStore, {conn_id: connId, rcv_queue_id: dbRcvId}) + if (rqs.length === 0) throw new Error("rcv queue not found") + return {rcvQueue: rqs[0], brokerId} + }) + }, + + // getRcvMsg (AgentStore.hs:1075-1089) + // SQL: SELECT r.internal_id, m.internal_ts, r.broker_id, r.broker_ts, r.external_snd_id, r.integrity, r.internal_hash, + // m.msg_type, m.msg_body, m.pq_encryption, s.internal_id, s.rcpt_status, r.user_ack + // FROM rcv_messages r + // JOIN messages m ON r.conn_id = m.conn_id AND r.internal_id = m.internal_id + // LEFT JOIN snd_messages s ON s.conn_id = r.conn_id AND s.rcpt_internal_id = r.internal_id + // WHERE r.conn_id = ? AND r.internal_id = ? + async getRcvMsg(connId, internalId) { + return withTx(["rcv_messages", "messages", "snd_messages"], "readonly", async (tx) => { + // Find rcv_message + const rmAll = await allByIndex(tx.objectStore("rcv_messages"), {conn_id: connId, internal_id: internalId}) + if (rmAll.length === 0) return null + const rm = rmAll[0] + // JOIN messages + const msg = await idbReq(tx.objectStore("messages").get([connId, internalId])) + if (!msg) return null + // LEFT JOIN snd_messages ON rcpt_internal_id = r.internal_id + const sndAll = await allByIndex(tx.objectStore("snd_messages"), {conn_id: connId, internal_id: internalId}) + const sndRcpt = sndAll.find((s: any) => s.rcpt_internal_id === internalId) + return { + internalId: rm.internal_id, + msgMeta: { + integrity: rm.integrity, + recipient: [rm.internal_id, msg.internal_ts], + broker: [rm.broker_id, rm.broker_ts], + sndMsgId: rm.external_snd_id, + pqEncryption: msg.pq_encryption, + }, + msgType: msg.msg_type, + msgBody: msg.msg_body, + userAck: rm.user_ack === 1, + msgReceipt: sndRcpt?.rcpt_status ?? null, + } + }) + }, + + // getLastMsg (AgentStore.hs:1091-1106) + // SQL: SELECT ... FROM rcv_messages r + // JOIN messages m ON r.conn_id = m.conn_id AND r.internal_id = m.internal_id + // JOIN connections c ON r.conn_id = c.conn_id AND c.last_internal_msg_id = r.internal_id + // LEFT JOIN snd_messages s ON s.conn_id = r.conn_id AND s.rcpt_internal_id = r.internal_id + // WHERE r.conn_id = ? AND r.broker_id = ? + async getLastMsg(connId, brokerId) { + return withTx(["rcv_messages", "messages", "connections", "snd_messages"], "readonly", async (tx) => { + // Find rcv_message by broker_id + const rmAll = await allByIndex(tx.objectStore("rcv_messages"), {conn_id: connId, broker_id: brokerId}) + if (rmAll.length === 0) return null + const rm = rmAll[0] + // JOIN connections: verify last_internal_msg_id = r.internal_id + const conn = await idbReq(tx.objectStore("connections").get(connId)) + if (!conn || conn.last_internal_msg_id !== rm.internal_id) return null + // JOIN messages + const msg = await idbReq(tx.objectStore("messages").get([connId, rm.internal_id])) + if (!msg) return null + return { + internalId: rm.internal_id, + msgMeta: { + integrity: rm.integrity, + recipient: [rm.internal_id, msg.internal_ts], + broker: [rm.broker_id, rm.broker_ts], + sndMsgId: rm.external_snd_id, + pqEncryption: msg.pq_encryption, + }, + msgType: msg.msg_type, + msgBody: msg.msg_body, + userAck: rm.user_ack === 1, + msgReceipt: null, + } + }) + }, + + // incMsgRcvAttempts (AgentStore.hs:1114-1125) + // SQL: UPDATE rcv_messages SET receive_attempts = receive_attempts + 1 + // WHERE conn_id = ? AND internal_id = ? RETURNING receive_attempts + async incMsgRcvAttempts(connId, internalId) { + return withTx("rcv_messages", "readwrite", async (tx) => { + const store = tx.objectStore("rcv_messages") + const all = await allByIndex(store, {conn_id: connId, internal_id: internalId}) + if (all.length === 0) throw new Error("rcv message not found") + const rm = all[0] + rm.receive_attempts = (rm.receive_attempts || 0) + 1 + await idbReq(store.put(rm)) + return rm.receive_attempts + }) + }, + + // checkRcvMsgHashExists (AgentStore.hs:1127-1133) + // SQL: SELECT 1 FROM encrypted_rcv_message_hashes WHERE conn_id = ? AND hash = ? LIMIT 1 + async checkRcvMsgHashExists(connId, hash) { + return withTx("encrypted_rcv_message_hashes", "readonly", async (tx) => { + const all = await allByIndex(tx.objectStore("encrypted_rcv_message_hashes"), {conn_id: connId, hash}) + return all.length > 0 + }) + }, + + // getRcvMsgBrokerTs (AgentStore.hs:1135-1138) + // SQL: SELECT broker_ts FROM rcv_messages WHERE conn_id = ? AND broker_id = ? + async getRcvMsgBrokerTs(connId, brokerId) { + return withTx("rcv_messages", "readonly", async (tx) => { + const all = await allByIndex(tx.objectStore("rcv_messages"), {conn_id: connId, broker_id: brokerId}) + if (all.length === 0) return null + return all[0].broker_ts + }) + }, + + // deleteMsg (AgentStore.hs:1140-1142) + // SQL: DELETE FROM messages WHERE conn_id = ? AND internal_id = ? + async deleteMsg(connId, internalId) { + return withTx("messages", "readwrite", async (tx) => { + await idbReq(tx.objectStore("messages").delete([connId, internalId])) + }) + }, + + // deleteDeliveredSndMsg (AgentStore.hs:1153-1159) + // SQL: countPendingSndDeliveries_ then deleteMsg if cnt == 0 + // countPendingSndDeliveries_: SELECT count(*) FROM snd_message_deliveries WHERE conn_id = ? AND internal_id = ? AND failed = 0 + async deleteDeliveredSndMsg(connId, internalId) { + return withTx(["snd_message_deliveries", "messages"], "readwrite", async (tx) => { + const allDel = await idbReq(tx.objectStore("snd_message_deliveries").getAll()) + const cnt = allDel.filter((d: any) => toHex(d.conn_id) === toHex(connId) && d.internal_id === internalId && !d.failed).length + if (cnt === 0) { + await idbReq(tx.objectStore("messages").delete([connId, internalId])) + } + }) + }, + + // deleteSndMsgDelivery (AgentStore.hs:1161-1205) + // SQL1: DELETE FROM snd_message_deliveries WHERE conn_id = ? AND snd_queue_id = ? AND internal_id = ? + // SQL2: SELECT rcpt_status, snd_message_body_id FROM snd_messages + // WHERE NOT EXISTS (SELECT 1 FROM snd_message_deliveries WHERE conn_id = ? AND internal_id = ? AND failed = 0) + // AND conn_id = ? AND internal_id = ? + // Then conditionally: DELETE FROM messages, DELETE FROM snd_message_bodies + async deleteSndMsgDelivery(connId, sndQueue, msgId, keepForReceipt) { + return withTx(["snd_message_deliveries", "snd_messages", "messages", "snd_message_bodies"], "readwrite", async (tx) => { + // Delete the delivery record + const delStore = tx.objectStore("snd_message_deliveries") + const allDel = await idbReq(delStore.getAll()) + for (const d of allDel) { + if (toHex(d.conn_id) === toHex(connId) && d.snd_queue_id === sndQueue.dbQueueId && d.internal_id === msgId) { + await idbReq(delStore.delete(d.snd_message_delivery_id)) + break + } + } + + // Check if there are remaining non-failed deliveries + const remainingDel = await idbReq(delStore.getAll()) + const hasPending = remainingDel.some((d: any) => toHex(d.conn_id) === toHex(connId) && d.internal_id === msgId && !d.failed) + if (hasPending) return + + // Get snd_message for this msg + const smAll = await allByIndex(tx.objectStore("snd_messages"), {conn_id: connId, internal_id: msgId}) + if (smAll.length === 0) return + const sm = smAll[0] + const rcptStatus = sm.rcpt_status + const sndMsgBodyId = sm.snd_message_body_id + + // Decide whether to delete or clear content + // MROk → deleteMsg; otherwise if keepForReceipt → deleteMsgContent; else deleteMsg + const shouldDeleteFull = rcptStatus === "ok" || !keepForReceipt + if (shouldDeleteFull) { + await idbReq(tx.objectStore("messages").delete([connId, msgId])) + } else { + // deleteMsgContent: clear msg_body + const msg = await idbReq(tx.objectStore("messages").get([connId, msgId])) + if (msg) { + msg.msg_body = new Uint8Array(0) + await idbReq(tx.objectStore("messages").put(msg)) + } + sm.snd_message_body_id = null + await idbReq(tx.objectStore("snd_messages").put(sm)) + } + + // Delete snd_message_body if no other snd_messages reference it + if (sndMsgBodyId != null) { + const allSM = await idbReq(tx.objectStore("snd_messages").getAll()) + const referenced = allSM.some((s: any) => s.snd_message_body_id === sndMsgBodyId) + if (!referenced) { + await idbReq(tx.objectStore("snd_message_bodies").delete(sndMsgBodyId)) + } + } + }) + }, + + // getSndMsgViaRcpt (AgentStore.hs:918-934) + // SQL: SELECT s.internal_id, m.msg_type, s.internal_hash, s.rcpt_internal_id, s.rcpt_status + // FROM snd_messages s + // JOIN messages m ON s.conn_id = m.conn_id AND s.internal_id = m.internal_id + // WHERE s.conn_id = ? AND s.internal_snd_id = ? + async getSndMsgViaRcpt(connId, sndMsgId) { + return withTx(["snd_messages", "messages"], "readonly", async (tx) => { + const sm = await idbReq(tx.objectStore("snd_messages").get([connId, sndMsgId])) + if (!sm) return null + const msg = await idbReq(tx.objectStore("messages").get([connId, sm.internal_id])) + if (!msg) return null + return { + internalId: sm.internal_id, + msgType: msg.msg_type, + internalHash: sm.internal_hash, + msgReceipt: sm.rcpt_status ?? null, + } + }) + }, + + // updateSndMsgRcpt (AgentStore.hs:936-941) + // SQL: UPDATE snd_messages SET rcpt_internal_id = ?, rcpt_status = ? WHERE conn_id = ? AND internal_snd_id = ? + async updateSndMsgRcpt(connId, sndMsgId, receipt) { + return withTx("snd_messages", "readwrite", async (tx) => { + const store = tx.objectStore("snd_messages") + const sm = await idbReq(store.get([connId, sndMsgId])) + if (sm) { + sm.rcpt_internal_id = receipt + sm.rcpt_status = receipt + await idbReq(store.put(sm)) + } + }) + }, + + // ============================================================ + // Ratchet — AgentStore.hs:1246-1367 + // ============================================================ + + // createRatchetX3dhKeys (AgentStore.hs:1246-1248) + // SQL: INSERT INTO ratchets (conn_id, x3dh_priv_key_1, x3dh_priv_key_2, pq_priv_kem) VALUES (?, ?, ?, ?) + async createRatchetX3dhKeys(connId, privKey1, privKey2, pqKem) { + return withTx("ratchets", "readwrite", async (tx) => { + await idbReq(tx.objectStore("ratchets").add({ + conn_id: connId, + x3dh_priv_key_1: privKey1, x3dh_priv_key_2: privKey2, + pq_priv_kem: pqKem, + ratchet_state: null, + x3dh_pub_key_1: null, x3dh_pub_key_2: null, pq_pub_kem: null, + })) + }) + }, + + // getRatchetX3dhKeys (AgentStore.hs:1250-1257) + // SQL: SELECT x3dh_priv_key_1, x3dh_priv_key_2, pq_priv_kem FROM ratchets WHERE conn_id = ? + async getRatchetX3dhKeys(connId) { + return withTx("ratchets", "readonly", async (tx) => { + const r = await idbReq(tx.objectStore("ratchets").get(connId)) + if (!r || !r.x3dh_priv_key_1 || !r.x3dh_priv_key_2) return null + return {privKey1: r.x3dh_priv_key_1, privKey2: r.x3dh_priv_key_2, pqKem: r.pq_priv_kem} + }) + }, + + // createRatchet (AgentStore.hs:1303-1319) + // SQL: INSERT INTO ratchets (conn_id, ratchet_state) VALUES (?, ?) + // ON CONFLICT (conn_id) DO UPDATE SET ratchet_state = ?, + // x3dh_priv_key_1 = NULL, x3dh_priv_key_2 = NULL, x3dh_pub_key_1 = NULL, x3dh_pub_key_2 = NULL, + // pq_priv_kem = NULL, pq_pub_kem = NULL + async createRatchet(connId, ratchetState) { + return withTx("ratchets", "readwrite", async (tx) => { + // ON CONFLICT DO UPDATE → use put (upsert) + await idbReq(tx.objectStore("ratchets").put({ + conn_id: connId, + ratchet_state: ratchetState, + x3dh_priv_key_1: null, x3dh_priv_key_2: null, + x3dh_pub_key_1: null, x3dh_pub_key_2: null, + pq_priv_kem: null, pq_pub_kem: null, + })) + }) + }, + + // getRatchet (AgentStore.hs:1334-1345) + // SQL: SELECT ratchet_state FROM ratchets WHERE conn_id = ? + async getRatchet(connId) { + return withTx("ratchets", "readonly", async (tx) => { + const r = await idbReq(tx.objectStore("ratchets").get(connId)) + return r?.ratchet_state ?? null + }) + }, + + // getRatchetForUpdate (AgentStore.hs:1325-1332) — same as getRatchet in IndexedDB (single-threaded) + async getRatchetForUpdate(connId) { + return this.getRatchet(connId) + }, + + // getSkippedMsgKeys (AgentStore.hs:1347-1354) + // SQL: SELECT header_key, msg_n, msg_key FROM skipped_messages WHERE conn_id = ? + // Returns: Map> + async getSkippedMsgKeys(connId) { + return withTx("skipped_messages", "readonly", async (tx) => { + const all = await allByIndex(tx.objectStore("skipped_messages"), {conn_id: connId}) + const result = new Map>() + for (const row of all) { + const hkHex = toHex(row.header_key) + let inner = result.get(hkHex) + if (!inner) { + inner = new Map() + result.set(hkHex, inner) + } + inner.set(row.msg_n, row.msg_key) + } + return result + }) + }, + + // updateRatchet (AgentStore.hs:1356-1366) + // SQL1: UPDATE ratchets SET ratchet_state = ? WHERE conn_id = ? + // SMDNoChange: no-op + // SMDRemove: DELETE FROM skipped_messages WHERE conn_id = ? AND header_key = ? AND msg_n = ? + // SMDAdd: INSERT INTO skipped_messages (conn_id, header_key, msg_n, msg_key) VALUES (?, ?, ?, ?) + async updateRatchet(connId, ratchetState, skippedMsgDiff) { + return withTx(["ratchets", "skipped_messages"], "readwrite", async (tx) => { + // Update ratchet state + const rStore = tx.objectStore("ratchets") + const r = await idbReq(rStore.get(connId)) + if (r) { + r.ratchet_state = ratchetState + await idbReq(rStore.put(r)) + } + // Apply skipped message diff + const diff = skippedMsgDiff as any + if (!diff || diff.type === "noChange") return + const skStore = tx.objectStore("skipped_messages") + if (diff.type === "remove") { + // Delete specific skipped message + const all = await allByIndex(skStore, {conn_id: connId}) + for (const row of all) { + if (toHex(row.header_key) === toHex(diff.headerKey) && row.msg_n === diff.msgN) { + await idbReq(skStore.delete(row.skipped_message_id)) + break + } + } + } else if (diff.type === "add") { + // Add new skipped message keys + for (const [hk, mks] of diff.keys.entries()) { + for (const [msgN, mk] of mks.entries()) { + await idbReq(skStore.add({ + conn_id: connId, header_key: hk, msg_n: msgN, msg_key: mk, + })) + } + } + } + }) + }, + + // ============================================================ + // Commands — AgentStore.hs:1368-1497 + // ============================================================ + + // createCommand (AgentStore.hs:1368-1392) + // SQL: INSERT INTO commands (host, port, corr_id, conn_id, command_tag, command, server_key_hash, created_at) VALUES (?,?,?,?,?,?,?,?) + async createCommand(corrId, connId, host, port, command) { + return withTx("commands", "readwrite", async (tx) => { + const id = await idbReq(tx.objectStore("commands").add({ + host, port, corr_id: corrId, conn_id: connId, + command_tag: command.commandTag, command: command.command, + agent_version: command.agentVersion, + server_key_hash: command.serverKeyHash, + created_at: new Date().toISOString(), failed: 0, + })) + return id as number + }) + }, + + // getPendingCommandServers (AgentStore.hs:1403-1422) + // SQL: SELECT DISTINCT c.conn_id, c.host, c.port, COALESCE(c.server_key_hash, s.key_hash) + // FROM commands c LEFT JOIN servers s ON s.host = c.host AND s.port = c.port + // ORDER BY c.conn_id + // Then filter by connIds set membership + async getPendingCommandServers(connIds) { + return withTx(["commands", "servers"], "readonly", async (tx) => { + const allCmds = await idbReq(tx.objectStore("commands").getAll()) + const connIdSet = new Set(connIds.map(toHex)) + const seen = new Set() + const result: Array<{connId: Uint8Array, host: string, port: string}> = [] + for (const cmd of allCmds) { + if (!cmd.host || !cmd.port) continue + if (!connIdSet.has(toHex(cmd.conn_id))) continue + const key = toHex(cmd.conn_id) + ":" + cmd.host + ":" + cmd.port + if (!seen.has(key)) { + seen.add(key) + result.push({connId: cmd.conn_id, host: cmd.host, port: cmd.port}) + } + } + return result + }) + }, + + // getAllPendingCommandConns (AgentStore.hs:1424-1437) + // SQL: SELECT DISTINCT c.conn_id, c.host, c.port, COALESCE(c.server_key_hash, s.key_hash) + // FROM commands c + // JOIN connections cs ON c.conn_id = cs.conn_id + // LEFT JOIN servers s ON s.host = c.host AND s.port = c.port + // WHERE cs.deleted = 0 + async getAllPendingCommandConns() { + return withTx(["commands", "connections", "servers"], "readonly", async (tx) => { + const allCmds = await idbReq(tx.objectStore("commands").getAll()) + const connStore = tx.objectStore("connections") + const seen = new Set() + const result: Array<{connId: Uint8Array, host: string, port: string}> = [] + for (const cmd of allCmds) { + if (!cmd.host || !cmd.port) continue + const conn = await idbReq(connStore.get(cmd.conn_id)) + if (!conn || conn.deleted !== 0) continue + const key = toHex(cmd.conn_id) + ":" + cmd.host + ":" + cmd.port + if (!seen.has(key)) { + seen.add(key) + result.push({connId: cmd.conn_id, host: cmd.host, port: cmd.port}) + } + } + return result + }) + }, + + // getPendingServerCommand (AgentStore.hs:1439-1480) + // getCmdId SQL: SELECT command_id FROM commands + // WHERE conn_id = ? AND host = ? AND port = ? AND failed = 0 + // ORDER BY created_at ASC, command_id ASC LIMIT 1 + // getCommand SQL: SELECT c.corr_id, cs.user_id, c.command FROM commands c + // JOIN connections cs USING (conn_id) WHERE c.command_id = ? + async getPendingServerCommand(host, port) { + return withTx(["commands", "connections"], "readonly", async (tx) => { + const allCmds = await allByIndex(tx.objectStore("commands"), {host, port}) + // Filter non-failed, sort by created_at then command_id + const pending = allCmds + .filter((c: any) => !c.failed) + .sort((a: any, b: any) => { + const tsCompare = (a.created_at || "").localeCompare(b.created_at || "") + return tsCompare !== 0 ? tsCompare : (a.command_id - b.command_id) + }) + if (pending.length === 0) return null + return pending[0] + }) + }, + + // updateCommandServer (AgentStore.hs:1482-1493) + // SQL: UPDATE commands SET host = ?, port = ?, server_key_hash = ? WHERE command_id = ? + async updateCommandServer(commandId, host, port) { + return withTx("commands", "readwrite", async (tx) => { + const store = tx.objectStore("commands") + const cmd = await idbReq(store.get(commandId)) + if (cmd) { + cmd.host = host + cmd.port = port + await idbReq(store.put(cmd)) + } + }) + }, + + // deleteCommand (AgentStore.hs:1495-1497) + // SQL: DELETE FROM commands WHERE command_id = ? + async deleteCommand(commandId) { + return withTx("commands", "readwrite", async (tx) => { + await idbReq(tx.objectStore("commands").delete(commandId)) + }) + }, + + // ============================================================ + // Encrypted message hash dedup — AgentStore.hs:1127-1133, 2641 + // ============================================================ + + // checkRcvMsgHashExists_encrypted (AgentStore.hs:1127-1133) + // SQL: SELECT 1 FROM encrypted_rcv_message_hashes WHERE conn_id = ? AND hash = ? LIMIT 1 + async checkRcvMsgHashExists_encrypted(connId, hash) { + return withTx("encrypted_rcv_message_hashes", "readonly", async (tx) => { + const all = await allByIndex(tx.objectStore("encrypted_rcv_message_hashes"), {conn_id: connId, hash}) + return all.length > 0 + }) + }, + + // addEncryptedRcvMsgHash (from insertRcvMsgDetails_ AgentStore.hs:2641) + // SQL: INSERT INTO encrypted_rcv_message_hashes (conn_id, hash) VALUES (?,?) + async addEncryptedRcvMsgHash(connId, hash) { + return withTx("encrypted_rcv_message_hashes", "readwrite", async (tx) => { + await idbReq(tx.objectStore("encrypted_rcv_message_hashes").add({ + conn_id: connId, hash, created_at: new Date().toISOString(), + })) + }) + }, + } +} diff --git a/smp-web/src/agent/store.ts b/smp-web/src/agent/store.ts new file mode 100644 index 000000000..8aa65e4ed --- /dev/null +++ b/smp-web/src/agent/store.ts @@ -0,0 +1,279 @@ +// Agent store interface for IndexedDB. +// Each method mirrors a Haskell function in AgentStore.hs. +// Implementation in store-idb.ts. + +// -- Types matching Haskell store types + +export type ConnId = Uint8Array +export type UserId = number +export type EntityId = Uint8Array +export type InternalId = number +export type InternalRcvId = number +export type InternalSndId = number + +export type QueueStatus = "new" | "confirmed" | "secured" | "active" | "disabled" | "deleted" +export type ConnectionMode = "INV" | "CON" // SCMInvitation | SCMContact +export type RatchetSyncState = "ok" | "allowed" | "required" | "started" | "agreed" + +export interface ConnData { + connId: ConnId + connMode: ConnectionMode + userId: UserId + smpAgentVersion: number + enableNtfs: boolean + duplexHandshake: boolean + deleted: boolean + ratchetSyncState: RatchetSyncState + pqSupport: boolean + // Message ID counters + lastInternalMsgId: number + lastInternalRcvMsgId: number + lastInternalSndMsgId: number + lastExternalSndMsgId: number + lastRcvMsgHash: Uint8Array + lastSndMsgHash: Uint8Array +} + +export interface RcvQueue { + host: string + port: string + rcvId: Uint8Array + connId: ConnId + rcvPrivateKey: Uint8Array + rcvDhSecret: Uint8Array + e2ePrivKey: Uint8Array + e2eDhSecret: Uint8Array | null + sndId: Uint8Array + sndKey: Uint8Array | null + status: QueueStatus + smpClientVersion: number | null + dbQueueId: number + primary: boolean + replaceRcvQueueId: number | null + queueMode: string | null + serverKeyHash: Uint8Array | null + lastBrokerTs: string | null +} + +export interface SndQueue { + host: string + port: string + sndId: Uint8Array + connId: ConnId + sndPrivateKey: Uint8Array + e2eDhSecret: Uint8Array + status: QueueStatus + smpClientVersion: number + sndPublicKey: Uint8Array | null + e2ePubKey: Uint8Array | null + dbQueueId: number + primary: boolean + queueMode: string | null + serverKeyHash: Uint8Array | null +} + +export interface MsgMeta { + integrity: string // "OK" or error + recipient: [number, string] // (internalId, internalTs) + broker: [Uint8Array, string] // (brokerId/msgId, brokerTs) + sndMsgId: number + pqEncryption: boolean +} + +export interface RcvMsgData { + msgMeta: MsgMeta + msgType: string + msgFlags: number + msgBody: Uint8Array + internalRcvId: number + internalHash: Uint8Array + externalPrevSndHash: Uint8Array + encryptedMsgHash: Uint8Array +} + +export interface SndMsgData { + internalId: number + internalSndId: number + internalTs: string + msgType: string + msgFlags: number + msgBody: Uint8Array + pqEncryption: boolean + internalHash: Uint8Array + prevMsgHash: Uint8Array + msgEncryptKey: Uint8Array | null + paddedMsgLen: number | null + sndMessageBodyId: number | null +} + +export interface Confirmation { + confirmationId: Uint8Array + connId: ConnId + e2eSndPubKey: Uint8Array + senderKey: Uint8Array | null + ratchetState: Uint8Array + senderConnInfo: Uint8Array + accepted: boolean + ownConnInfo: Uint8Array | null + smpReplyQueues: Uint8Array | null // serialized + smpClientVersion: number | null +} + +export interface Invitation { + invitationId: Uint8Array + contactConnId: ConnId | null + crInvitation: Uint8Array + recipientConnInfo: Uint8Array + accepted: boolean + ownConnInfo: Uint8Array | null +} + +export interface RcvMsg { + internalId: number + msgMeta: MsgMeta + msgType: string + msgBody: Uint8Array + userAck: boolean + msgReceipt: Uint8Array | null +} + +export interface PendingQueueMsg { + connId: ConnId + sndQueueId: number + internalId: number + msgType: string + msgFlags: number + msgBody: Uint8Array + internalHash: Uint8Array + prevMsgHash: Uint8Array + pqEncryption: boolean + msgEncryptKey: Uint8Array | null + paddedMsgLen: number | null + sndMessageBodyId: number | null +} + +export interface AsyncCommand { + commandId: number + connId: ConnId + host: string | null + port: string | null + corrId: Uint8Array + commandTag: string + command: Uint8Array + agentVersion: number + serverKeyHash: Uint8Array | null + failed: boolean +} + +// -- Store interface +// Each method name matches the Haskell function in AgentStore.hs. + +export interface AgentStore { + // -- Users (AgentStore.hs:201-230) + createUserRecord(): Promise + getUserIds(): Promise + deleteUserRecord(userId: UserId): Promise + setUserDeleted(userId: UserId): Promise + + // -- Servers (AgentStore.hs:233-240) + createServer(host: string, port: string, keyHash: Uint8Array): Promise + + // -- Connections (AgentStore.hs:242-500) + createNewConn(connData: ConnData, connMode: ConnectionMode): Promise + getConn(connId: ConnId): Promise<{connData: ConnData, rcvQueues: RcvQueue[], sndQueues: SndQueue[]} | null> + getRcvConn(host: string, port: string, rcvId: Uint8Array): Promise<{connData: ConnData, rcvQueue: RcvQueue} | null> + getConnSubs(connIds: ConnId[]): Promise> + getConnsData(connIds: ConnId[]): Promise> + lockConnForUpdate(connId: ConnId): Promise // no-op in IndexedDB (single-threaded) + setConnDeleted(connId: ConnId, waitDelivery: boolean): Promise + setConnUserId(oldUserId: UserId, connId: ConnId, newUserId: UserId): Promise + setConnAgentVersion(connId: ConnId, version: number): Promise + setConnPQSupport(connId: ConnId, pqSupport: boolean): Promise + setConnRatchetSync(connId: ConnId, state: RatchetSyncState): Promise + updateNewConnJoin(connId: ConnId, agentVersion: number, pqSupport: boolean, enableNtfs: boolean): Promise + updateNewConnRcv(connId: ConnId, rcvQueue: RcvQueue, subMode: string): Promise + getDeletedConnIds(): Promise + getDeletedWaitingDeliveryConnIds(): Promise + getConnIds(): Promise + + // -- Queues (AgentStore.hs:500-700) + addConnRcvQueue(connId: ConnId, rcvQueue: RcvQueue, subMode: string): Promise + addConnSndQueue(connId: ConnId, sndQueue: SndQueue): Promise + setRcvQueueStatus(rcvQueue: RcvQueue, status: QueueStatus): Promise + setSndQueueStatus(sndQueue: SndQueue, status: QueueStatus): Promise + setRcvQueueConfirmedE2E(rcvQueue: RcvQueue, dhSecret: Uint8Array, smpClientVersion: number): Promise + setRcvQueuePrimary(connId: ConnId, rcvQueue: RcvQueue): Promise + deleteConnRcvQueue(rcvQueue: RcvQueue): Promise + deleteConnRecord(connId: ConnId): Promise + upgradeRcvConnToDuplex(connId: ConnId, sndQueue: SndQueue): Promise + upgradeSndConnToDuplex(connId: ConnId, rcvQueue: RcvQueue, subMode: string): Promise + getPrimaryRcvQueue(connId: ConnId): Promise + getRcvQueue(connId: ConnId, host: string, port: string, rcvId: Uint8Array): Promise + getDeletedRcvQueue(connId: ConnId, host: string, port: string, rcvId: Uint8Array): Promise + setConnectionNtfs(connId: ConnId, enable: boolean): Promise + + // -- Subscriptions (AgentStore.hs:700-800) + getSubscriptionServers(onlyNeeded: boolean): Promise> + getUserServerRcvQueueSubs(userId: UserId, host: string, port: string, onlyNeeded: boolean, batchSize: number, cursor: number | null): Promise<{queues: RcvQueue[], nextCursor: number | null}> + unsetQueuesToSubscribe(): Promise + getConnectionsForDelivery(): Promise + getAllSndQueuesForDelivery(): Promise + + // -- Confirmations (AgentStore.hs:800-870) + createConfirmation(confirmation: Confirmation): Promise + acceptConfirmation(confirmationId: Uint8Array, ownConnInfo: Uint8Array): Promise + getAcceptedConfirmation(connId: ConnId): Promise + removeConfirmations(connId: ConnId): Promise + + // -- Invitations (AgentStore.hs:870-920) + createInvitation(invitation: Invitation): Promise + getInvitation(invitationId: Uint8Array): Promise + acceptInvitation(invitationId: Uint8Array, ownConnInfo: Uint8Array): Promise + unacceptInvitation(invitationId: Uint8Array): Promise + deleteInvitation(invitationId: Uint8Array): Promise + + // -- Messages (AgentStore.hs:873-1050) + updateRcvIds(connId: ConnId): Promise<{internalId: number, internalRcvId: number, prevExternalSndId: number, prevRcvMsgHash: Uint8Array}> + createRcvMsg(connId: ConnId, rcvQueue: RcvQueue, rcvMsgData: RcvMsgData): Promise + setLastBrokerTs(connId: ConnId, dbQueueId: number, brokerTs: string): Promise + updateRcvMsgHash(connId: ConnId, sndMsgId: number, internalRcvId: number, hash: Uint8Array): Promise + createSndMsgBody(agentMsg: Uint8Array): Promise + updateSndIds(connId: ConnId): Promise<{internalId: number, internalSndId: number, prevSndMsgHash: Uint8Array}> + createSndMsg(connId: ConnId, sndMsgData: SndMsgData): Promise + updateSndMsgHash(connId: ConnId, internalSndId: number, hash: Uint8Array): Promise + createSndMsgDelivery(connId: ConnId, sndQueue: SndQueue, internalId: number): Promise + getPendingQueueMsg(connId: ConnId, sndQueue: SndQueue): Promise + updatePendingMsgRIState(connId: ConnId, msgId: number, retryIntSlow: number | null, retryIntFast: number | null): Promise + setMsgUserAck(connId: ConnId, internalId: number): Promise<{rcvQueue: RcvQueue, brokerId: Uint8Array}> + getRcvMsg(connId: ConnId, internalId: number): Promise + getLastMsg(connId: ConnId, brokerId: Uint8Array): Promise + incMsgRcvAttempts(connId: ConnId, internalId: number): Promise + checkRcvMsgHashExists(connId: ConnId, hash: Uint8Array): Promise + getRcvMsgBrokerTs(connId: ConnId, brokerId: Uint8Array): Promise + deleteMsg(connId: ConnId, internalId: number): Promise + deleteDeliveredSndMsg(connId: ConnId, internalSndId: number): Promise + deleteSndMsgDelivery(connId: ConnId, sndQueue: SndQueue, msgId: number, keepForReceipt: boolean): Promise + getSndMsgViaRcpt(connId: ConnId, sndMsgId: number): Promise<{internalId: number, msgType: string, internalHash: Uint8Array, msgReceipt: Uint8Array | null} | null> + updateSndMsgRcpt(connId: ConnId, sndMsgId: number, receipt: Uint8Array): Promise + + // -- Ratchet (AgentStore.hs:1300-1400) + createRatchetX3dhKeys(connId: ConnId, privKey1: Uint8Array, privKey2: Uint8Array, pqKem: Uint8Array | null): Promise + getRatchetX3dhKeys(connId: ConnId): Promise<{privKey1: Uint8Array, privKey2: Uint8Array, pqKem: Uint8Array | null} | null> + createRatchet(connId: ConnId, ratchetState: Uint8Array): Promise + getRatchet(connId: ConnId): Promise + getRatchetForUpdate(connId: ConnId): Promise // same as getRatchet in IndexedDB (single-threaded) + getSkippedMsgKeys(connId: ConnId): Promise>> + updateRatchet(connId: ConnId, ratchetState: Uint8Array, skippedMsgDiff: unknown): Promise + + // -- Commands (AgentStore.hs:1400-1480) + createCommand(corrId: Uint8Array, connId: ConnId, host: string | null, port: string | null, command: AsyncCommand): Promise + getPendingCommandServers(connIds: ConnId[]): Promise> + getAllPendingCommandConns(): Promise> + getPendingServerCommand(host: string, port: string): Promise + updateCommandServer(commandId: number, host: string, port: string): Promise + deleteCommand(commandId: number): Promise + + // -- Encrypted message hash dedup (AgentStore.hs:1200-1220) + checkRcvMsgHashExists_encrypted(connId: ConnId, hash: Uint8Array): Promise + addEncryptedRcvMsgHash(connId: ConnId, hash: Uint8Array): Promise +} diff --git a/smp-web/src/idb-keys.d.ts b/smp-web/src/idb-keys.d.ts new file mode 100644 index 000000000..7e0e8845e --- /dev/null +++ b/smp-web/src/idb-keys.d.ts @@ -0,0 +1,15 @@ +// Override IDBValidKey to accept Uint8Array (supported in modern browsers) +interface IDBObjectStore { + get(query: any): IDBRequest + getAll(query?: any, count?: number): IDBRequest + getAllKeys(query?: any, count?: number): IDBRequest + add(value: any, key?: any): IDBRequest + put(value: any, key?: any): IDBRequest + delete(query: any): IDBRequest +} + +interface IDBIndex { + get(query: any): IDBRequest + getAll(query?: any, count?: number): IDBRequest + getAllKeys(query?: any, count?: number): IDBRequest +} diff --git a/smp-web/tests/store-test.ts b/smp-web/tests/store-test.ts new file mode 100644 index 000000000..d97631e29 --- /dev/null +++ b/smp-web/tests/store-test.ts @@ -0,0 +1,818 @@ +// Agent store scenario tests using fake-indexeddb. +// Each scenario exercises a full lifecycle: create → update → read → delete → verify gone. +// Every store method is called from at least one test. + +import "fake-indexeddb/auto" +import {openAgentStore} from "../dist/agent/store-idb.js" +import type {AgentStore} from "../dist/agent/store.js" + +// The store returns raw IndexedDB rows with snake_case field names. +// The TypeScript interface types use camelCase but the underlying data is snake_case. +// We use `any` casts in assertions to access the actual field names. +type Row = any + +// -- Helpers + +function bytes(hex: string): Uint8Array { + const b = new Uint8Array(hex.length / 2) + for (let i = 0; i < hex.length; i += 2) b[i / 2] = parseInt(hex.slice(i, i + 2), 16) + return b +} + +function hex(b: Uint8Array): string { + return Array.from(b, x => x.toString(16).padStart(2, "0")).join("") +} + +function randomBytes(n: number): Uint8Array { + const b = new Uint8Array(n) + for (let i = 0; i < n; i++) b[i] = Math.floor(Math.random() * 256) + return b +} + +let passed = 0 +let failed = 0 + +function assert(cond: boolean, msg: string) { + if (!cond) { + console.error("FAIL:", msg) + failed++ + } else { + passed++ + } +} + +function assertEq(a: any, b: any, msg: string) { + const av = a instanceof Uint8Array ? hex(a) : JSON.stringify(a) + const bv = b instanceof Uint8Array ? hex(b) : JSON.stringify(b) + assert(av === bv, `${msg}: expected ${bv}, got ${av}`) +} + +async function assertThrows(fn: () => Promise, msg: string) { + try { + await fn() + assert(false, `${msg}: expected throw`) + } catch { + passed++ + } +} + +// -- Scenarios + +async function testUsers(store: AgentStore) { + console.log(" users...") + // createUserRecord, getUserIds + const uid1 = await store.createUserRecord() + const uid2 = await store.createUserRecord() + let ids = await store.getUserIds() + assert(ids.includes(uid1) && ids.includes(uid2), "getUserIds returns both users") + + // setUserDeleted — marks user as deleted, getUserIds should exclude it + await store.setUserDeleted(uid1) + ids = await store.getUserIds() + assert(!ids.includes(uid1), "deleted user not in getUserIds") + assert(ids.includes(uid2), "non-deleted user still in getUserIds") + + // deleteUserRecord — hard delete + await store.deleteUserRecord(uid2) + ids = await store.getUserIds() + assert(!ids.includes(uid2), "hard-deleted user gone") +} + +async function testServers(store: AgentStore) { + console.log(" servers...") + // createServer — insert or ignore + const kh = randomBytes(32) + await store.createServer("smp1.example.com", "5223", kh) + // Duplicate should not throw + await store.createServer("smp1.example.com", "5223", kh) +} + +async function testConnectionsAndQueues(store: AgentStore) { + console.log(" connections and queues...") + const userId = await store.createUserRecord() + const connId = randomBytes(24) + const connId2 = randomBytes(24) + + // createNewConn + const created = await store.createNewConn({ + connId, connMode: "INV", userId, smpAgentVersion: 7, + enableNtfs: true, duplexHandshake: true, deleted: false, + ratchetSyncState: "ok", pqSupport: true, + lastInternalMsgId: 0, lastInternalRcvMsgId: 0, lastInternalSndMsgId: 0, + lastExternalSndMsgId: 0, lastRcvMsgHash: new Uint8Array(0), lastSndMsgHash: new Uint8Array(0), + }, "INV") + assertEq(created, connId, "createNewConn returns connId") + + // getConn — should find it + const got = await store.getConn(connId) + assert(got !== null, "getConn finds connection") + assertEq((got!.connData as Row).conn_id, connId, "getConn connId matches") + + // getConnIds + const allIds = await store.getConnIds() + assert(allIds.some(id => hex(id) === hex(connId)), "getConnIds includes new conn") + + // setConnAgentVersion + await store.setConnAgentVersion(connId, 8) + const got2 = await store.getConn(connId) + assertEq((got2!.connData as Row).smp_agent_version, 8, "setConnAgentVersion updated") + + // setConnPQSupport + await store.setConnPQSupport(connId, false) + const got3 = await store.getConn(connId) + assertEq((got3!.connData as Row).pq_support, 0, "setConnPQSupport updated") + + // setConnRatchetSync + await store.setConnRatchetSync(connId, "required") + const got4 = await store.getConn(connId) + assertEq((got4!.connData as Row).ratchet_sync_state, "required", "setConnRatchetSync updated") + + // setConnectionNtfs + await store.setConnectionNtfs(connId, false) + const got5 = await store.getConn(connId) + assertEq((got5!.connData as Row).enable_ntfs, 0, "setConnectionNtfs updated") + + // lockConnForUpdate — no-op, should not throw + await store.lockConnForUpdate(connId) + + // addConnRcvQueue + const rcvQ = await store.addConnRcvQueue(connId, { + host: "smp1.example.com", port: "5223", rcvId: randomBytes(24), + connId, rcvPrivateKey: randomBytes(32), rcvDhSecret: randomBytes(32), + e2ePrivKey: randomBytes(32), e2eDhSecret: null, + sndId: randomBytes(24), sndKey: null, status: "new" as const, + smpClientVersion: 7, dbQueueId: 0, primary: true, + replaceRcvQueueId: null, queueMode: null, + serverKeyHash: randomBytes(32), lastBrokerTs: null, + }, "SMSubscribe") + assert(rcvQ.dbQueueId >= 1, "addConnRcvQueue assigns dbQueueId") + + // getPrimaryRcvQueue + const primary = await store.getPrimaryRcvQueue(connId) + assert(primary !== null, "getPrimaryRcvQueue finds queue") + assertEq((primary as Row).rcv_primary, 1, "primary queue is marked primary") + + // getRcvConn + const rcvConn = await store.getRcvConn(rcvQ.host, rcvQ.port, rcvQ.rcvId) + assert(rcvConn !== null, "getRcvConn finds by host/port/rcvId") + + // getRcvQueue + const rq = await store.getRcvQueue(connId, rcvQ.host, rcvQ.port, rcvQ.rcvId) + assert(rq !== null, "getRcvQueue finds queue") + + // setRcvQueueStatus + await store.setRcvQueueStatus(rcvQ, "confirmed") + const rq2 = await store.getRcvQueue(connId, rcvQ.host, rcvQ.port, rcvQ.rcvId) + assertEq(rq2!.status, "confirmed", "setRcvQueueStatus updated") + + // setRcvQueueConfirmedE2E + const dhSecret = randomBytes(32) + await store.setRcvQueueConfirmedE2E(rcvQ, dhSecret, 7) + const rq3 = await store.getRcvQueue(connId, rcvQ.host, rcvQ.port, rcvQ.rcvId) + assertEq((rq3 as Row).e2e_dh_secret, dhSecret, "setRcvQueueConfirmedE2E updated dh secret") + assertEq((rq3 as Row).status, "confirmed", "setRcvQueueConfirmedE2E sets confirmed") + + // addConnSndQueue + upgradeRcvConnToDuplex (same operation) + const sndQ = { + host: "smp2.example.com", port: "5223", sndId: randomBytes(24), + connId, sndPrivateKey: randomBytes(32), e2eDhSecret: randomBytes(32), + status: "confirmed" as const, smpClientVersion: 7, + sndPublicKey: randomBytes(32), e2ePubKey: randomBytes(32), + dbQueueId: 0, primary: true, queueMode: null, serverKeyHash: randomBytes(32), + } + await store.upgradeRcvConnToDuplex(connId, sndQ) + const gotDuplex = await store.getConn(connId) + assert(gotDuplex!.sndQueues.length >= 1, "upgradeRcvConnToDuplex added snd queue") + + // setSndQueueStatus + await store.setSndQueueStatus(sndQ, "active") + + // getConnSubs, getConnsData + const subs = await store.getConnSubs([connId]) + assert(subs.size === 1, "getConnSubs returns 1 entry") + const connsData = await store.getConnsData([connId]) + assert(connsData.size === 1, "getConnsData returns 1 entry") + + // Create second connection for setConnUserId + await store.createNewConn({ + connId: connId2, connMode: "CON", userId, smpAgentVersion: 7, + enableNtfs: true, duplexHandshake: true, deleted: false, + ratchetSyncState: "ok", pqSupport: false, + lastInternalMsgId: 0, lastInternalRcvMsgId: 0, lastInternalSndMsgId: 0, + lastExternalSndMsgId: 0, lastRcvMsgHash: new Uint8Array(0), lastSndMsgHash: new Uint8Array(0), + }, "CON") + const userId2 = await store.createUserRecord() + await store.setConnUserId(userId, connId2, userId2) + const got6 = await store.getConn(connId2) + assertEq((got6!.connData as Row).user_id, userId2, "setConnUserId updated") + + // updateNewConnJoin + await store.updateNewConnJoin(connId, 9, true, false) + const got7 = await store.getConn(connId) + assertEq((got7!.connData as Row).smp_agent_version, 9, "updateNewConnJoin updated version") + + // updateNewConnRcv — adds a rcv queue (same as addConnRcvQueue) + const rcvQ2 = await store.updateNewConnRcv(connId2, { + host: "smp3.example.com", port: "5223", rcvId: randomBytes(24), + connId: connId2, rcvPrivateKey: randomBytes(32), rcvDhSecret: randomBytes(32), + e2ePrivKey: randomBytes(32), e2eDhSecret: null, + sndId: randomBytes(24), sndKey: null, status: "new" as const, + smpClientVersion: 7, dbQueueId: 0, primary: true, + replaceRcvQueueId: null, queueMode: null, + serverKeyHash: randomBytes(32), lastBrokerTs: null, + }, "SMSubscribe") + assert(rcvQ2.dbQueueId >= 1, "updateNewConnRcv assigns dbQueueId") + + // upgradeSndConnToDuplex — add rcv queue to a snd-only connection + const connId3 = randomBytes(24) + await store.createNewConn({ + connId: connId3, connMode: "INV", userId, smpAgentVersion: 7, + enableNtfs: true, duplexHandshake: true, deleted: false, + ratchetSyncState: "ok", pqSupport: false, + lastInternalMsgId: 0, lastInternalRcvMsgId: 0, lastInternalSndMsgId: 0, + lastExternalSndMsgId: 0, lastRcvMsgHash: new Uint8Array(0), lastSndMsgHash: new Uint8Array(0), + }, "INV") + await store.addConnSndQueue(connId3, { + host: "smp4.example.com", port: "5223", sndId: randomBytes(24), + connId: connId3, sndPrivateKey: randomBytes(32), e2eDhSecret: randomBytes(32), + status: "confirmed" as const, smpClientVersion: 7, + sndPublicKey: null, e2ePubKey: null, dbQueueId: 0, primary: true, + queueMode: null, serverKeyHash: randomBytes(32), + }) + const rcvQ3 = await store.upgradeSndConnToDuplex(connId3, { + host: "smp4.example.com", port: "5223", rcvId: randomBytes(24), + connId: connId3, rcvPrivateKey: randomBytes(32), rcvDhSecret: randomBytes(32), + e2ePrivKey: randomBytes(32), e2eDhSecret: null, + sndId: randomBytes(24), sndKey: null, status: "new" as const, + smpClientVersion: 7, dbQueueId: 0, primary: true, + replaceRcvQueueId: null, queueMode: null, + serverKeyHash: randomBytes(32), lastBrokerTs: null, + }, "SMSubscribe") + assert(rcvQ3.dbQueueId >= 1, "upgradeSndConnToDuplex added rcv queue") + + // setRcvQueuePrimary — add second rcv queue, make it primary + const rcvQ4 = await store.addConnRcvQueue(connId, { + host: "smp5.example.com", port: "5223", rcvId: randomBytes(24), + connId, rcvPrivateKey: randomBytes(32), rcvDhSecret: randomBytes(32), + e2ePrivKey: randomBytes(32), e2eDhSecret: null, + sndId: randomBytes(24), sndKey: null, status: "new" as const, + smpClientVersion: 7, dbQueueId: 0, primary: false, + replaceRcvQueueId: null, queueMode: null, + serverKeyHash: randomBytes(32), lastBrokerTs: null, + }, "SMSubscribe") + await store.setRcvQueuePrimary(connId, rcvQ4) + const newPrimary = await store.getPrimaryRcvQueue(connId) + assertEq((newPrimary as Row).rcv_queue_id, rcvQ4.dbQueueId, "setRcvQueuePrimary changed primary") + + // getDeletedRcvQueue — first delete a queue, then find it + // deleteConnRcvQueue physically deletes, so getDeletedRcvQueue won't find it + // We need to test the soft-delete path — but deleteConnRcvQueue does hard delete + // Just verify it returns null for non-deleted + const drq = await store.getDeletedRcvQueue(connId, rcvQ.host, rcvQ.port, rcvQ.rcvId) + assert(drq === null, "getDeletedRcvQueue returns null for non-deleted queue") + + // deleteConnRcvQueue + await store.deleteConnRcvQueue(rcvQ4) + const afterDel = await store.getRcvQueue(connId, rcvQ4.host, rcvQ4.port, rcvQ4.rcvId) + assert(afterDel === null, "deleteConnRcvQueue removes queue") + + // setConnDeleted (waitDelivery=true) + await store.setConnDeleted(connId2, true) + const waitDel = await store.getDeletedWaitingDeliveryConnIds() + assert(waitDel.some(id => hex(id) === hex(connId2)), "setConnDeleted waitDelivery appears in getDeletedWaitingDeliveryConnIds") + + // setConnDeleted (waitDelivery=false) + await store.setConnDeleted(connId3, false) + const delIds = await store.getDeletedConnIds() + assert(delIds.some(id => hex(id) === hex(connId3)), "setConnDeleted appears in getDeletedConnIds") + + // deleteConnRecord + await store.deleteConnRecord(connId3) + const gone = await store.getConn(connId3) + assert(gone === null, "deleteConnRecord removes connection") +} + +async function testSubscriptions(store: AgentStore) { + console.log(" subscriptions...") + const userId = await store.createUserRecord() + const connId = randomBytes(24) + await store.createNewConn({ + connId, connMode: "INV", userId, smpAgentVersion: 7, + enableNtfs: true, duplexHandshake: true, deleted: false, + ratchetSyncState: "ok", pqSupport: false, + lastInternalMsgId: 0, lastInternalRcvMsgId: 0, lastInternalSndMsgId: 0, + lastExternalSndMsgId: 0, lastRcvMsgHash: new Uint8Array(0), lastSndMsgHash: new Uint8Array(0), + }, "INV") + await store.createServer("sub.example.com", "5223", randomBytes(32)) + const rcvQ = await store.addConnRcvQueue(connId, { + host: "sub.example.com", port: "5223", rcvId: randomBytes(24), + connId, rcvPrivateKey: randomBytes(32), rcvDhSecret: randomBytes(32), + e2ePrivKey: randomBytes(32), e2eDhSecret: null, + sndId: randomBytes(24), sndKey: null, status: "new" as const, + smpClientVersion: 7, dbQueueId: 0, primary: true, + replaceRcvQueueId: null, queueMode: null, + serverKeyHash: randomBytes(32), lastBrokerTs: null, + }, "SMOnlyCreate") + + // getSubscriptionServers — onlyNeeded=true should find our to_subscribe=1 queue + const srvs = await store.getSubscriptionServers(true) + assert(srvs.some(s => s.host === "sub.example.com"), "getSubscriptionServers finds queue with to_subscribe") + + // getSubscriptionServers — onlyNeeded=false + const allSrvs = await store.getSubscriptionServers(false) + assert(allSrvs.some(s => s.host === "sub.example.com"), "getSubscriptionServers(false) finds all") + + // getUserServerRcvQueueSubs + const {queues} = await store.getUserServerRcvQueueSubs(userId, "sub.example.com", "5223", true, 10, null) + assert(queues.length >= 1, "getUserServerRcvQueueSubs finds queues") + + // unsetQueuesToSubscribe + await store.unsetQueuesToSubscribe() + const srvsAfter = await store.getSubscriptionServers(true) + assert(!srvsAfter.some(s => s.host === "sub.example.com"), "unsetQueuesToSubscribe clears to_subscribe") + + // getConnectionsForDelivery, getAllSndQueuesForDelivery — need snd deliveries + // Add snd queue and delivery + const sndQ = { + host: "sub.example.com", port: "5223", sndId: randomBytes(24), + connId, sndPrivateKey: randomBytes(32), e2eDhSecret: randomBytes(32), + status: "active" as const, smpClientVersion: 7, + sndPublicKey: null, e2ePubKey: null, dbQueueId: 0, primary: true, + queueMode: null, serverKeyHash: randomBytes(32), + } + await store.addConnSndQueue(connId, sndQ) + // Need to create a message first + const {internalId, internalSndId, prevSndMsgHash} = await store.updateSndIds(connId) + await store.createSndMsg(connId, { + internalId, internalSndId, internalTs: new Date().toISOString(), + msgType: "HELLO", msgFlags: 0, msgBody: randomBytes(10), + pqEncryption: false, internalHash: randomBytes(32), + prevMsgHash: prevSndMsgHash, msgEncryptKey: null, paddedMsgLen: null, sndMessageBodyId: null, + }) + // Get the snd queue with its actual dbQueueId + const gotConn = await store.getConn(connId) + const actualSndQ = gotConn!.sndQueues[0] as Row + // Raw IDB row has snd_queue_id, interface expects dbQueueId + await store.createSndMsgDelivery(connId, {dbQueueId: actualSndQ.snd_queue_id} as any, internalId) + + const deliveryConns = await store.getConnectionsForDelivery() + assert(deliveryConns.some(id => hex(id) === hex(connId)), "getConnectionsForDelivery finds conn with delivery") + + const deliverySndQs = await store.getAllSndQueuesForDelivery() + assert(deliverySndQs.length >= 1, "getAllSndQueuesForDelivery finds queues") +} + +async function testConfirmations(store: AgentStore) { + console.log(" confirmations...") + const userId = await store.createUserRecord() + const connId = randomBytes(24) + await store.createNewConn({ + connId, connMode: "INV", userId, smpAgentVersion: 7, + enableNtfs: true, duplexHandshake: true, deleted: false, + ratchetSyncState: "ok", pqSupport: false, + lastInternalMsgId: 0, lastInternalRcvMsgId: 0, lastInternalSndMsgId: 0, + lastExternalSndMsgId: 0, lastRcvMsgHash: new Uint8Array(0), lastSndMsgHash: new Uint8Array(0), + }, "INV") + + const confId = randomBytes(16) + // createConfirmation + const returned = await store.createConfirmation({ + confirmationId: confId, connId, + e2eSndPubKey: randomBytes(32), senderKey: randomBytes(32), + ratchetState: randomBytes(64), senderConnInfo: randomBytes(50), + accepted: false, ownConnInfo: null, + smpReplyQueues: randomBytes(100), smpClientVersion: 7, + }) + assertEq(returned, confId, "createConfirmation returns confirmationId") + + // getAcceptedConfirmation — not accepted yet + const notAccepted = await store.getAcceptedConfirmation(connId) + assert(notAccepted === null, "getAcceptedConfirmation returns null before acceptance") + + // acceptConfirmation + const ownInfo = randomBytes(40) + const accepted = await store.acceptConfirmation(confId, ownInfo) + assert(accepted !== null, "acceptConfirmation returns confirmation") + assertEq(accepted.accepted, 1, "acceptConfirmation sets accepted=1") + + // getAcceptedConfirmation — now accepted + const gotAccepted = await store.getAcceptedConfirmation(connId) + assert(gotAccepted !== null, "getAcceptedConfirmation finds accepted confirmation") + assertEq((gotAccepted as Row).own_conn_info, ownInfo, "accepted confirmation has ownConnInfo") + + // removeConfirmations + await store.removeConfirmations(connId) + const afterRemove = await store.getAcceptedConfirmation(connId) + assert(afterRemove === null, "removeConfirmations deletes all confirmations for conn") +} + +async function testInvitations(store: AgentStore) { + console.log(" invitations...") + const userId = await store.createUserRecord() + const connId = randomBytes(24) + await store.createNewConn({ + connId, connMode: "CON", userId, smpAgentVersion: 7, + enableNtfs: true, duplexHandshake: true, deleted: false, + ratchetSyncState: "ok", pqSupport: false, + lastInternalMsgId: 0, lastInternalRcvMsgId: 0, lastInternalSndMsgId: 0, + lastExternalSndMsgId: 0, lastRcvMsgHash: new Uint8Array(0), lastSndMsgHash: new Uint8Array(0), + }, "CON") + + const invId = randomBytes(16) + // createInvitation + const returned = await store.createInvitation({ + invitationId: invId, contactConnId: connId, + crInvitation: randomBytes(200), recipientConnInfo: randomBytes(50), + accepted: false, ownConnInfo: null, + }) + assertEq(returned, invId, "createInvitation returns invitationId") + + // getInvitation — not accepted, should find + const got = await store.getInvitation(invId) + assert(got !== null, "getInvitation finds unaccepted invitation") + + // acceptInvitation + const ownInfo = randomBytes(40) + await store.acceptInvitation(invId, ownInfo) + // getInvitation — accepted, should NOT find (WHERE accepted = 0) + const gotAfterAccept = await store.getInvitation(invId) + assert(gotAfterAccept === null, "getInvitation returns null for accepted invitation") + + // unacceptInvitation + await store.unacceptInvitation(invId) + const gotAfterUnaccept = await store.getInvitation(invId) + assert(gotAfterUnaccept !== null, "unacceptInvitation resets accepted to 0") + assert((gotAfterUnaccept as Row).own_conn_info === null, "unacceptInvitation clears ownConnInfo") + + // deleteInvitation + await store.deleteInvitation(invId) + const gotAfterDelete = await store.getInvitation(invId) + assert(gotAfterDelete === null, "deleteInvitation removes invitation") +} + +async function testReceiveMessages(store: AgentStore) { + console.log(" receive messages...") + const userId = await store.createUserRecord() + const connId = randomBytes(24) + await store.createNewConn({ + connId, connMode: "INV", userId, smpAgentVersion: 7, + enableNtfs: true, duplexHandshake: true, deleted: false, + ratchetSyncState: "ok", pqSupport: false, + lastInternalMsgId: 0, lastInternalRcvMsgId: 0, lastInternalSndMsgId: 0, + lastExternalSndMsgId: 0, lastRcvMsgHash: new Uint8Array(0), lastSndMsgHash: new Uint8Array(0), + }, "INV") + await store.createServer("rcv.example.com", "5223", randomBytes(32)) + const rcvQ = await store.addConnRcvQueue(connId, { + host: "rcv.example.com", port: "5223", rcvId: randomBytes(24), + connId, rcvPrivateKey: randomBytes(32), rcvDhSecret: randomBytes(32), + e2ePrivKey: randomBytes(32), e2eDhSecret: null, + sndId: randomBytes(24), sndKey: null, status: "active" as const, + smpClientVersion: 7, dbQueueId: 0, primary: true, + replaceRcvQueueId: null, queueMode: null, + serverKeyHash: randomBytes(32), lastBrokerTs: null, + }, "SMSubscribe") + + // updateRcvIds + const {internalId, internalRcvId, prevExternalSndId, prevRcvMsgHash} = await store.updateRcvIds(connId) + assertEq(internalId, 1, "updateRcvIds first internalId=1") + assertEq(internalRcvId, 1, "updateRcvIds first internalRcvId=1") + assertEq(prevExternalSndId, 0, "updateRcvIds prevExternalSndId=0") + + const brokerId = randomBytes(24) + const brokerTs = new Date().toISOString() + const internalHash = randomBytes(32) + const encryptedMsgHash = randomBytes(32) + const msgBody = randomBytes(100) + + // createRcvMsg — exercises insertRcvMsgBase_, insertRcvMsgDetails_, updateRcvMsgHash, setLastBrokerTs + await store.createRcvMsg(connId, rcvQ, { + msgMeta: { + integrity: "OK", + recipient: [internalId, new Date().toISOString()], + broker: [brokerId, brokerTs], + sndMsgId: 1, + pqEncryption: false, + }, + msgType: "MSG", + msgFlags: 0, + msgBody, + internalRcvId, + internalHash, + externalPrevSndHash: randomBytes(32), + encryptedMsgHash, + }) + + // getRcvMsg + const rcvMsg = await store.getRcvMsg(connId, internalId) + assert(rcvMsg !== null, "getRcvMsg finds message") + assertEq(rcvMsg!.msgType, "MSG", "getRcvMsg msgType matches") + + // getLastMsg — verify the msg is "last" (conn.last_internal_msg_id matches) + const lastMsg = await store.getLastMsg(connId, brokerId) + assert(lastMsg !== null, "getLastMsg finds message by brokerId") + + // getRcvMsgBrokerTs + const ts = await store.getRcvMsgBrokerTs(connId, brokerId) + assert(ts !== null, "getRcvMsgBrokerTs finds broker ts") + + // checkRcvMsgHashExists — encrypted hash was inserted by createRcvMsg + const hashExists = await store.checkRcvMsgHashExists(connId, encryptedMsgHash) + assert(hashExists, "checkRcvMsgHashExists finds hash inserted by createRcvMsg") + + // incMsgRcvAttempts + const attempts = await store.incMsgRcvAttempts(connId, internalId) + assertEq(attempts, 1, "incMsgRcvAttempts returns 1 after first increment") + const attempts2 = await store.incMsgRcvAttempts(connId, internalId) + assertEq(attempts2, 2, "incMsgRcvAttempts returns 2 after second increment") + + // setMsgUserAck + const {rcvQueue: ackQ, brokerId: ackBrokerId} = await store.setMsgUserAck(connId, internalId) + assert(ackQ !== null, "setMsgUserAck returns rcvQueue") + assertEq(ackBrokerId, brokerId, "setMsgUserAck returns correct brokerId") + + // Verify user_ack was set + const rcvMsgAfterAck = await store.getRcvMsg(connId, internalId) + assert(rcvMsgAfterAck!.userAck, "setMsgUserAck sets userAck=true") + + // setLastBrokerTs — standalone call + const newTs = new Date().toISOString() + await store.setLastBrokerTs(connId, rcvQ.dbQueueId, newTs) + + // updateRcvMsgHash — standalone call + await store.updateRcvMsgHash(connId, 2, internalRcvId, randomBytes(32)) + + // deleteMsg + await store.deleteMsg(connId, internalId) + const deletedMsg = await store.getRcvMsg(connId, internalId) + assert(deletedMsg === null, "deleteMsg removes message") +} + +async function testSendMessages(store: AgentStore) { + console.log(" send messages...") + const userId = await store.createUserRecord() + const connId = randomBytes(24) + await store.createNewConn({ + connId, connMode: "INV", userId, smpAgentVersion: 7, + enableNtfs: true, duplexHandshake: true, deleted: false, + ratchetSyncState: "ok", pqSupport: false, + lastInternalMsgId: 0, lastInternalRcvMsgId: 0, lastInternalSndMsgId: 0, + lastExternalSndMsgId: 0, lastRcvMsgHash: new Uint8Array(0), lastSndMsgHash: new Uint8Array(0), + }, "INV") + await store.createServer("snd.example.com", "5223", randomBytes(32)) + await store.addConnSndQueue(connId, { + host: "snd.example.com", port: "5223", sndId: randomBytes(24), + connId, sndPrivateKey: randomBytes(32), e2eDhSecret: randomBytes(32), + status: "active" as const, smpClientVersion: 7, + sndPublicKey: null, e2ePubKey: null, dbQueueId: 0, primary: true, + queueMode: null, serverKeyHash: randomBytes(32), + }) + const gotConn = await store.getConn(connId) + const sndQueue = gotConn!.sndQueues[0] + + // createSndMsgBody + const agentMsg = randomBytes(200) + const bodyId = await store.createSndMsgBody(agentMsg) + assert(bodyId >= 1, "createSndMsgBody returns positive id") + + // updateSndIds + const {internalId, internalSndId, prevSndMsgHash} = await store.updateSndIds(connId) + assertEq(internalId, 1, "updateSndIds first internalId=1") + assertEq(internalSndId, 1, "updateSndIds first internalSndId=1") + + const internalHash = randomBytes(32) + + // createSndMsg + await store.createSndMsg(connId, { + internalId, internalSndId, internalTs: new Date().toISOString(), + msgType: "SEND", msgFlags: 0, msgBody: randomBytes(100), + pqEncryption: false, internalHash, + prevMsgHash: prevSndMsgHash, msgEncryptKey: randomBytes(32), + paddedMsgLen: 16384, sndMessageBodyId: bodyId, + }) + + // updateSndMsgHash — standalone + await store.updateSndMsgHash(connId, internalSndId, internalHash) + + // createSndMsgDelivery + await store.createSndMsgDelivery(connId, sndQueue, internalId) + + // getPendingQueueMsg + const pending = await store.getPendingQueueMsg(connId, sndQueue) + assert(pending !== null, "getPendingQueueMsg finds pending message") + assertEq(pending!.msgType, "SEND", "getPendingQueueMsg msgType matches") + assertEq(pending!.internalId, internalId, "getPendingQueueMsg internalId matches") + + // updatePendingMsgRIState + await store.updatePendingMsgRIState(connId, internalId, 30, 5) + + // getSndMsgViaRcpt + const sndMsg = await store.getSndMsgViaRcpt(connId, internalSndId) + assert(sndMsg !== null, "getSndMsgViaRcpt finds message") + assertEq(sndMsg!.internalId, internalId, "getSndMsgViaRcpt internalId matches") + assertEq(sndMsg!.internalHash, internalHash, "getSndMsgViaRcpt hash matches") + + // updateSndMsgRcpt + const receipt = randomBytes(8) + await store.updateSndMsgRcpt(connId, internalSndId, receipt) + + // deleteSndMsgDelivery — deletes delivery, then msg if no more deliveries + await store.deleteSndMsgDelivery(connId, sndQueue, internalId, false) + + // Create a second message for deleteDeliveredSndMsg + const ids2 = await store.updateSndIds(connId) + await store.createSndMsg(connId, { + internalId: ids2.internalId, internalSndId: ids2.internalSndId, + internalTs: new Date().toISOString(), + msgType: "SEND", msgFlags: 0, msgBody: randomBytes(50), + pqEncryption: false, internalHash: randomBytes(32), + prevMsgHash: ids2.prevSndMsgHash, msgEncryptKey: null, paddedMsgLen: null, sndMessageBodyId: null, + }) + + // deleteDeliveredSndMsg — no deliveries exist, so should delete msg + await store.deleteDeliveredSndMsg(connId, ids2.internalId) +} + +async function testRatchet(store: AgentStore) { + console.log(" ratchet...") + const connId = randomBytes(24) + const userId = await store.createUserRecord() + await store.createNewConn({ + connId, connMode: "INV", userId, smpAgentVersion: 7, + enableNtfs: true, duplexHandshake: true, deleted: false, + ratchetSyncState: "ok", pqSupport: true, + lastInternalMsgId: 0, lastInternalRcvMsgId: 0, lastInternalSndMsgId: 0, + lastExternalSndMsgId: 0, lastRcvMsgHash: new Uint8Array(0), lastSndMsgHash: new Uint8Array(0), + }, "INV") + + const privKey1 = randomBytes(56) + const privKey2 = randomBytes(56) + const pqKem = randomBytes(100) + + // createRatchetX3dhKeys + await store.createRatchetX3dhKeys(connId, privKey1, privKey2, pqKem) + + // getRatchetX3dhKeys + const keys = await store.getRatchetX3dhKeys(connId) + assert(keys !== null, "getRatchetX3dhKeys finds keys") + assertEq(keys!.privKey1, privKey1, "x3dh privKey1 matches") + assertEq(keys!.privKey2, privKey2, "x3dh privKey2 matches") + assertEq(keys!.pqKem, pqKem, "x3dh pqKem matches") + + // getRatchet — no ratchet state yet + const noRatchet = await store.getRatchet(connId) + assert(noRatchet === null, "getRatchet returns null before createRatchet") + + // createRatchet — upserts, clearing x3dh keys + const ratchetState = randomBytes(200) + await store.createRatchet(connId, ratchetState) + + // getRatchet + const gotRatchet = await store.getRatchet(connId) + assertEq(gotRatchet, ratchetState, "getRatchet returns stored state") + + // getRatchetForUpdate — same as getRatchet in IndexedDB + const gotForUpdate = await store.getRatchetForUpdate(connId) + assertEq(gotForUpdate, ratchetState, "getRatchetForUpdate returns stored state") + + // x3dh keys should be cleared after createRatchet + const keysAfter = await store.getRatchetX3dhKeys(connId) + assert(keysAfter === null, "createRatchet clears x3dh keys") + + // getSkippedMsgKeys — empty initially + const noSkipped = await store.getSkippedMsgKeys(connId) + assertEq(noSkipped.size, 0, "getSkippedMsgKeys empty initially") + + // updateRatchet with SMDAdd + const headerKey = randomBytes(32) + const msgKey = randomBytes(32) + const newRatchetState = randomBytes(200) + const addKeys = new Map>() + const inner = new Map() + inner.set(0, msgKey) + inner.set(1, randomBytes(32)) + addKeys.set(headerKey, inner) + await store.updateRatchet(connId, newRatchetState, {type: "add", keys: addKeys}) + + // Verify ratchet state updated + const updatedRatchet = await store.getRatchet(connId) + assertEq(updatedRatchet, newRatchetState, "updateRatchet updates state") + + // Verify skipped keys added + const skipped = await store.getSkippedMsgKeys(connId) + assert(skipped.size >= 1, "updateRatchet SMDAdd adds skipped keys") + + // updateRatchet with SMDRemove + const hkHex = Array.from(headerKey, x => x.toString(16).padStart(2, "0")).join("") + await store.updateRatchet(connId, randomBytes(200), {type: "remove", headerKey, msgN: 0}) + const afterRemove = await store.getSkippedMsgKeys(connId) + // Should have 1 key remaining (msgN=1) instead of 2 + let totalKeys = 0 + for (const [, m] of afterRemove) totalKeys += m.size + assertEq(totalKeys, 1, "updateRatchet SMDRemove removes specific key") + + // updateRatchet with noChange + const stateBeforeNoChange = randomBytes(200) + await store.updateRatchet(connId, stateBeforeNoChange, {type: "noChange"}) + const afterNoChange = await store.getRatchet(connId) + assertEq(afterNoChange, stateBeforeNoChange, "updateRatchet SMDNoChange only updates state") +} + +async function testCommands(store: AgentStore) { + console.log(" commands...") + const userId = await store.createUserRecord() + const connId = randomBytes(24) + await store.createNewConn({ + connId, connMode: "INV", userId, smpAgentVersion: 7, + enableNtfs: true, duplexHandshake: true, deleted: false, + ratchetSyncState: "ok", pqSupport: false, + lastInternalMsgId: 0, lastInternalRcvMsgId: 0, lastInternalSndMsgId: 0, + lastExternalSndMsgId: 0, lastRcvMsgHash: new Uint8Array(0), lastSndMsgHash: new Uint8Array(0), + }, "INV") + + const corrId = randomBytes(24) + // createCommand + const cmdId = await store.createCommand(corrId, connId, "cmd.example.com", "5223", { + commandId: 0, connId, host: "cmd.example.com", port: "5223", + corrId, commandTag: "NEW", command: randomBytes(50), + agentVersion: 7, serverKeyHash: randomBytes(32), failed: false, + }) + assert(cmdId >= 1, "createCommand returns positive id") + + // getPendingCommandServers + const servers = await store.getPendingCommandServers([connId]) + assert(servers.some(s => s.host === "cmd.example.com"), "getPendingCommandServers finds command server") + + // getAllPendingCommandConns + const allConns = await store.getAllPendingCommandConns() + assert(allConns.some(c => hex(c.connId) === hex(connId)), "getAllPendingCommandConns finds conn") + + // getPendingServerCommand + const pendingCmd = await store.getPendingServerCommand("cmd.example.com", "5223") + assert(pendingCmd !== null, "getPendingServerCommand finds command") + + // updateCommandServer + await store.updateCommandServer(cmdId, "cmd2.example.com", "5224") + const updated = await store.getPendingServerCommand("cmd2.example.com", "5224") + assert(updated !== null, "updateCommandServer changes server") + const oldHost = await store.getPendingServerCommand("cmd.example.com", "5223") + assert(oldHost === null, "updateCommandServer — old host returns nothing") + + // deleteCommand + await store.deleteCommand(cmdId) + const deleted = await store.getPendingServerCommand("cmd2.example.com", "5224") + assert(deleted === null, "deleteCommand removes command") +} + +async function testHashDedup(store: AgentStore) { + console.log(" hash dedup...") + const connId = randomBytes(24) + const userId = await store.createUserRecord() + await store.createNewConn({ + connId, connMode: "INV", userId, smpAgentVersion: 7, + enableNtfs: true, duplexHandshake: true, deleted: false, + ratchetSyncState: "ok", pqSupport: false, + lastInternalMsgId: 0, lastInternalRcvMsgId: 0, lastInternalSndMsgId: 0, + lastExternalSndMsgId: 0, lastRcvMsgHash: new Uint8Array(0), lastSndMsgHash: new Uint8Array(0), + }, "INV") + + const hash = randomBytes(32) + + // checkRcvMsgHashExists_encrypted — not yet + const before = await store.checkRcvMsgHashExists_encrypted(connId, hash) + assert(!before, "checkRcvMsgHashExists_encrypted returns false before add") + + // addEncryptedRcvMsgHash + await store.addEncryptedRcvMsgHash(connId, hash) + + // checkRcvMsgHashExists_encrypted — now exists + const after = await store.checkRcvMsgHashExists_encrypted(connId, hash) + assert(after, "checkRcvMsgHashExists_encrypted returns true after add") + + // Different hash should not exist + const other = await store.checkRcvMsgHashExists_encrypted(connId, randomBytes(32)) + assert(!other, "checkRcvMsgHashExists_encrypted returns false for different hash") +} + +// -- Run all + +async function main() { + console.log("Agent store tests") + const store = await openAgentStore() + + await testUsers(store) + await testServers(store) + await testConnectionsAndQueues(store) + await testSubscriptions(store) + await testConfirmations(store) + await testInvitations(store) + await testReceiveMessages(store) + await testSendMessages(store) + await testRatchet(store) + await testCommands(store) + await testHashDedup(store) + + console.log(`\n${passed} passed, ${failed} failed`) + if (failed > 0) process.exit(1) +} + +main().catch(e => { console.error("FATAL:", e?.message || e, e?.stack); process.exit(1) }) diff --git a/smp-web/tsconfig.json b/smp-web/tsconfig.json index 2d66241bd..b2d62ce39 100644 --- a/smp-web/tsconfig.json +++ b/smp-web/tsconfig.json @@ -3,7 +3,7 @@ "target": "ES2022", "module": "ES2022", "moduleResolution": "node", - "lib": ["ES2022"], + "lib": ["ES2022", "DOM"], "outDir": "dist", "rootDir": "src", "declaration": true, diff --git a/smp-web/tsconfig.test.json b/smp-web/tsconfig.test.json index 960f58829..afca6f0b4 100644 --- a/smp-web/tsconfig.test.json +++ b/smp-web/tsconfig.test.json @@ -3,7 +3,7 @@ "target": "ES2022", "module": "ES2022", "moduleResolution": "node", - "lib": ["ES2022"], + "lib": ["ES2022", "DOM"], "outDir": "dist-test", "rootDir": "tests", "strict": true, diff --git a/tests/SMPWebTests.hs b/tests/SMPWebTests.hs index 183ef1054..8b93e8805 100644 --- a/tests/SMPWebTests.hs +++ b/tests/SMPWebTests.hs @@ -46,6 +46,8 @@ import Simplex.Messaging.Encoding import Simplex.Messaging.Encoding.String (Str (..), strEncode) import Simplex.Messaging.Protocol (EntityId (..), SMPServer, SubscriptionMode (..), MsgFlags (..), noMsgFlags, pattern SMPServer, pattern NoEntity, encodeProtocol, Cmd (..), SParty (..), Command (..), NewQueueReq (..), QueueReqData (..), BrokerMsg (..), RcvMessage (..), EncRcvMsgBody (..), QueueIdsKeys (..), PubHeader (..), PrivHeader (..), ClientMessage (..), ClientMsgEnvelope (..), pattern VersionSMPC) import Simplex.Messaging.Server.Env.STM (AStoreType (..), ServerConfig (..)) +import Simplex.Messaging.Server.QueueStore.QueueInfo (QueueMode (..)) +import Simplex.Messaging.ServiceScheme (ServiceScheme (..)) import Simplex.Messaging.Server.MsgStore.Types (SMSType (..), SQSType (..)) import Simplex.Messaging.Server.Web (attachStaticAndWS) import Data.Time.Clock (getCurrentTime) @@ -1236,6 +1238,52 @@ smpWebTests_ = do <> jsOut ("e.encAgentMessage") tsResult `shouldBe` "decrypt me" + describe "agent/queueInfo" $ do + let impAgentProtoEnc = "import { encodeSMPQueueInfo, decodeSMPQueueInfo, encodeSMPQueueUri, decodeSMPQueueUri, encodeConnReqUriData, decodeConnReqUriData, encodeConnectionRequestUri, decodeConnectionRequestUri } from './dist/agent/protocol.js';" + + describe "SMPQueueInfo" $ do + it "encoding matches Haskell" $ do + g <- C.newRandom + (dhPub, _) <- atomically $ C.generateKeyPair @'C.X25519 g + let srv = SMPServer ("smp.example.com" :| []) "5223" (C.KeyHash $ B.pack [1..32]) + senderId = EntityId $ B.pack [10..33] + qAddr = AP.SMPQueueAddress {AP.smpServer = srv, AP.senderId = senderId, AP.dhPublicKey = dhPub, AP.queueMode = Just QMMessaging} + qi = AP.SMPQueueInfo (VersionSMPC 4) qAddr + hsBytes = smpEncode qi + tsBytes <- callNode $ impEnc <> impAgentProtoEnc + <> "const qi = {clientVersion: 4, queueAddress: {smpServer: {hosts: ['smp.example.com'], port: '5223', keyHash: " <> jsUint8 (B.pack [1..32]) <> "}, senderId: " <> jsUint8 (B.pack [10..33]) <> ", dhPublicKey: " <> jsUint8 (C.encodePubKey dhPub) <> ", queueMode: 'M'}};" + <> jsOut ("encodeSMPQueueInfo(qi)") + tsBytes `shouldBe` hsBytes + + it "TypeScript decodes Haskell-encoded" $ do + g <- C.newRandom + (dhPub, _) <- atomically $ C.generateKeyPair @'C.X25519 g + let srv = SMPServer ("relay.test.com" :| ["relay2.test.com"]) "" (C.KeyHash $ B.pack [50..81]) + senderId = EntityId $ B.pack [1..24] + qAddr = AP.SMPQueueAddress {AP.smpServer = srv, AP.senderId = senderId, AP.dhPublicKey = dhPub, AP.queueMode = Just QMContact} + qi = AP.SMPQueueInfo (VersionSMPC 4) qAddr + hsBytes = smpEncode qi + tsResult <- callNode $ impEnc <> impAgentProtoEnc + <> "const qi = decodeSMPQueueInfo(new Decoder(" <> jsUint8 hsBytes <> "));" + <> jsOut ("new Uint8Array([qi.clientVersion >> 8, qi.clientVersion & 0xff, qi.queueAddress.smpServer.hosts.length, qi.queueAddress.queueMode ? qi.queueAddress.queueMode.charCodeAt(0) : 0])") + tsResult `shouldBe` B.pack [0, 4, 2, 0x43] -- version=4, 2 hosts, queueMode='C' + + describe "ConnectionRequestUri" $ do + it "contact encoding matches Haskell" $ do + g <- C.newRandom + (dhPub, _) <- atomically $ C.generateKeyPair @'C.X25519 g + let srv = SMPServer ("smp1.example.com" :| []) "" (C.KeyHash $ B.pack [1..32]) + senderId = EntityId $ B.pack [1..24] + qAddr = AP.SMPQueueAddress {AP.smpServer = srv, AP.senderId = senderId, AP.dhPublicKey = dhPub, AP.queueMode = Just QMContact} + qUri = AP.SMPQueueUri (mkVersionRange (VersionSMPC 4) (VersionSMPC 4)) qAddr + crData = AP.ConnReqUriData {AP.crScheme = SSSimplex, AP.crAgentVRange = mkVersionRange (AP.VersionSMPA 2) (AP.VersionSMPA 7), AP.crSmpQueues = qUri :| [], AP.crClientData = Nothing} + cr = AP.CRContactUri crData :: AP.ConnectionRequestUri 'AP.CMContact + hsBytes = smpEncode cr + tsBytes <- callNode $ impEnc <> impAgentProtoEnc + <> "const cr = {mode: 'contact', crData: {crAgentVRange: {min: 2, max: 7}, crSmpQueues: [{clientVRange: {min: 4, max: 4}, queueAddress: {smpServer: {hosts: ['smp1.example.com'], port: '', keyHash: " <> jsUint8 (B.pack [1..32]) <> "}, senderId: " <> jsUint8 (B.pack [1..24]) <> ", dhPublicKey: " <> jsUint8 (C.encodePubKey dhPub) <> ", queueMode: 'C'}}], crClientData: null}};" + <> jsOut ("encodeConnectionRequestUri(cr)") + tsBytes `shouldBe` hsBytes + describe "protocol/e2e" $ do describe "PubHeader" $ do it "encoding without key matches Haskell" $ do From ad50ca8644e6f1700c5ad3bdf440dd96b2489de7 Mon Sep 17 00:00:00 2001 From: "Evgeny @ SimpleX Chat" <259188159+evgeny-simplex@users.noreply.github.com> Date: Sun, 31 May 2026 06:43:05 +0000 Subject: [PATCH 2/4] fixes --- smp-web/src/agent/store-idb.ts | 88 ++++++++++++++++++++-------------- smp-web/src/agent/store.ts | 16 +++---- smp-web/tests/store-test.ts | 18 +++---- 3 files changed, 70 insertions(+), 52 deletions(-) diff --git a/smp-web/src/agent/store-idb.ts b/smp-web/src/agent/store-idb.ts index c238950fe..ac6b6ca5b 100644 --- a/smp-web/src/agent/store-idb.ts +++ b/smp-web/src/agent/store-idb.ts @@ -49,8 +49,8 @@ function createSchema(db: IDBDatabase): void { db.createObjectStore("users", {keyPath: "user_id", autoIncrement: true}) // servers — agent_schema.sql:6-11 - // CREATE TABLE servers(host TEXT, port TEXT, key_hash BLOB, PRIMARY KEY(host, port)) - db.createObjectStore("servers", {keyPath: ["host", "port"]}) + // Deviation from Haskell: keyHash is part of the primary key (Haskell uses (host,port) as workaround to avoid migration) + db.createObjectStore("servers", {keyPath: ["host", "port", "key_hash"]}) // connections — agent_schema.sql:12-31 const conns = db.createObjectStore("connections", {keyPath: "conn_id"}) @@ -196,6 +196,8 @@ function createStore(db: IDBDatabase): AgentStore { user.deleted = 1 await idbReq(store.put(user)) } + const allConns = await idbReq(tx.objectStore("connections").getAll()) + return allConns.filter((c: any) => c.user_id === userId).map((c: any) => c.conn_id) }) }, @@ -208,7 +210,7 @@ function createStore(db: IDBDatabase): AgentStore { async createServer(host, port, keyHash) { return withTx("servers", "readwrite", async (tx) => { const store = tx.objectStore("servers") - const existing = await idbReq(store.get([host, port])) + const existing = await idbReq(store.get([host, port, keyHash])) if (!existing) await idbReq(store.add({host, port, key_hash: keyHash})) }) }, @@ -393,7 +395,7 @@ function createStore(db: IDBDatabase): AgentStore { async getDeletedConnIds() { return withTx("connections", "readonly", async (tx) => { const all = await idbReq(tx.objectStore("connections").getAll()) - return all.filter((c: any) => c.deleted === 1 && !c.deleted_at_wait_delivery).map((c: any) => c.conn_id) + return all.filter((c: any) => c.deleted === 1).map((c: any) => c.conn_id) }) }, @@ -406,10 +408,12 @@ function createStore(db: IDBDatabase): AgentStore { }) }, - // getConnIds — SELECT conn_id FROM connections + // getConnIds (AgentStore.hs:2245-2246) + // SQL: SELECT conn_id FROM connections WHERE deleted = 0 async getConnIds() { return withTx("connections", "readonly", async (tx) => { - return idbReq(tx.objectStore("connections").getAllKeys()) + const all = await idbReq(tx.objectStore("connections").getAll()) + return all.filter((c: any) => c.deleted === 0).map((c: any) => c.conn_id) }) }, @@ -419,11 +423,12 @@ function createStore(db: IDBDatabase): AgentStore { return withTx(["rcv_queues", "servers"], "readwrite", async (tx) => { // createServer (INSERT OR IGNORE) const srvStore = tx.objectStore("servers") - if (!(await idbReq(srvStore.get([rcvQueue.host, rcvQueue.port])))) await idbReq(srvStore.add({host: rcvQueue.host, port: rcvQueue.port, key_hash: rcvQueue.serverKeyHash})) - // Determine rcv_queue_id: SELECT MAX(rcv_queue_id) FROM rcv_queues WHERE conn_id = ? + if (!(await idbReq(srvStore.get([rcvQueue.host, rcvQueue.port, rcvQueue.serverKeyHash])))) await idbReq(srvStore.add({host: rcvQueue.host, port: rcvQueue.port, key_hash: rcvQueue.serverKeyHash})) + // Haskell: first check if queue with same (conn_id, host, port, snd_id) exists and reuse its rcv_queue_id + // SELECT rcv_queue_id FROM rcv_queues WHERE conn_id = ? AND host = ? AND port = ? AND snd_id = ? const existing = await allByIndex(tx.objectStore("rcv_queues"), {conn_id: connId}) - const maxId = existing.reduce((m: number, q: any) => Math.max(m, q.rcv_queue_id || 0), 0) - const qId = maxId + 1 + const curr = existing.find((q: any) => q.host === rcvQueue.host && q.port === rcvQueue.port && eqBytes(q.snd_id, rcvQueue.sndId)) + const qId = curr ? curr.rcv_queue_id : (existing.reduce((m: number, q: any) => Math.max(m, q.rcv_queue_id || 0), 0) + 1) const toSubscribe = subMode === "SMOnlyCreate" ? 1 : 0 await idbReq(tx.objectStore("rcv_queues").add({ host: rcvQueue.host, port: rcvQueue.port, rcv_id: rcvQueue.rcvId, @@ -444,10 +449,12 @@ function createStore(db: IDBDatabase): AgentStore { async addConnSndQueue(connId, sndQueue) { return withTx(["snd_queues", "servers"], "readwrite", async (tx) => { const srvStore2 = tx.objectStore("servers") - if (!(await idbReq(srvStore2.get([sndQueue.host, sndQueue.port])))) await idbReq(srvStore2.add({host: sndQueue.host, port: sndQueue.port, key_hash: sndQueue.serverKeyHash})) + if (!(await idbReq(srvStore2.get([sndQueue.host, sndQueue.port, sndQueue.serverKeyHash])))) await idbReq(srvStore2.add({host: sndQueue.host, port: sndQueue.port, key_hash: sndQueue.serverKeyHash})) + // Haskell: first check if queue with same (conn_id, host, port, snd_id) exists and reuse its snd_queue_id + // SELECT snd_queue_id FROM snd_queues WHERE conn_id = ? AND host = ? AND port = ? AND snd_id = ? const existing = await allByIndex(tx.objectStore("snd_queues"), {conn_id: connId}) - const maxId = existing.reduce((m: number, q: any) => Math.max(m, q.snd_queue_id || 0), 0) - const qId = maxId + 1 + const curr = existing.find((q: any) => q.host === sndQueue.host && q.port === sndQueue.port && eqBytes(q.snd_id, sndQueue.sndId)) + const qId = curr ? curr.snd_queue_id : (existing.reduce((m: number, q: any) => Math.max(m, q.snd_queue_id || 0), 0) + 1) // ON CONFLICT DO UPDATE → use put (upsert) await idbReq(tx.objectStore("snd_queues").put({ host: sndQueue.host, port: sndQueue.port, snd_id: sndQueue.sndId, @@ -589,22 +596,21 @@ function createStore(db: IDBDatabase): AgentStore { // JOIN connections c ON q.conn_id = c.conn_id // WHERE [q.to_subscribe = 1 AND] c.deleted = 0 AND q.deleted = 0 async getSubscriptionServers(onlyNeeded) { - return withTx(["rcv_queues", "connections", "servers"], "readonly", async (tx) => { + return withTx(["rcv_queues", "connections"], "readonly", async (tx) => { const allQ = await idbReq(tx.objectStore("rcv_queues").getAll()) const connStore = tx.objectStore("connections") - const srvStore = tx.objectStore("servers") const seen = new Set() - const result: Array<{userId: number, host: string, port: string}> = [] + const result: Array<{userId: number, host: string, port: string, keyHash: Uint8Array}> = [] for (const q of allQ) { if (q.deleted) continue if (onlyNeeded && !q.to_subscribe) continue + if (!q.server_key_hash) continue const conn = await idbReq(connStore.get(q.conn_id)) if (!conn || conn.deleted !== 0) continue - const keyHash = q.server_key_hash ?? (await idbReq(srvStore.get([q.host, q.port])))?.key_hash - const key = `${conn.user_id}:${q.host}:${q.port}:${keyHash ? toHex(keyHash) : ""}` + const key = `${conn.user_id}:${q.host}:${q.port}:${toHex(q.server_key_hash)}` if (!seen.has(key)) { seen.add(key) - result.push({userId: conn.user_id, host: q.host, port: q.port}) + result.push({userId: conn.user_id, host: q.host, port: q.port, keyHash: q.server_key_hash}) } } return result @@ -615,16 +621,16 @@ function createStore(db: IDBDatabase): AgentStore { // SQL: rcvQueueSubQuery WHERE [q.to_subscribe = 1 AND] c.deleted = 0 AND q.deleted = 0 // AND c.user_id = ? AND q.host = ? AND q.port = ? AND COALESCE(q.server_key_hash, s.key_hash) = ? // ORDER BY q.rcv_id LIMIT ? - async getUserServerRcvQueueSubs(userId, host, port, onlyNeeded, batchSize, cursor) { - return withTx(["rcv_queues", "connections", "servers"], "readonly", async (tx) => { + async getUserServerRcvQueueSubs(userId, host, port, keyHash, onlyNeeded, batchSize, cursor) { + return withTx(["rcv_queues", "connections"], "readonly", async (tx) => { const allQ = await idbReq(tx.objectStore("rcv_queues").getAll()) const connStore = tx.objectStore("connections") - const srvStore = tx.objectStore("servers") // Filter matching queues const matching: any[] = [] for (const q of allQ) { if (q.deleted) continue if (q.host !== host || q.port !== port) continue + if (!q.server_key_hash || !eqBytes(q.server_key_hash, keyHash)) continue if (onlyNeeded && !q.to_subscribe) continue if (cursor !== null && compareUint8Array(q.rcv_id, cursor as any) <= 0) continue const conn = await idbReq(connStore.get(q.conn_id)) @@ -1145,8 +1151,8 @@ function createStore(db: IDBDatabase): AgentStore { // JOIN messages const msg = await idbReq(tx.objectStore("messages").get([connId, internalId])) if (!msg) return null - // LEFT JOIN snd_messages ON rcpt_internal_id = r.internal_id - const sndAll = await allByIndex(tx.objectStore("snd_messages"), {conn_id: connId, internal_id: internalId}) + // LEFT JOIN snd_messages s ON s.conn_id = r.conn_id AND s.rcpt_internal_id = r.internal_id + const sndAll = await allByIndex(tx.objectStore("snd_messages"), {conn_id: connId}) const sndRcpt = sndAll.find((s: any) => s.rcpt_internal_id === internalId) return { internalId: rm.internal_id, @@ -1183,6 +1189,9 @@ function createStore(db: IDBDatabase): AgentStore { // JOIN messages const msg = await idbReq(tx.objectStore("messages").get([connId, rm.internal_id])) if (!msg) return null + // LEFT JOIN snd_messages s ON s.conn_id = r.conn_id AND s.rcpt_internal_id = r.internal_id + const sndAll = await allByIndex(tx.objectStore("snd_messages"), {conn_id: connId}) + const sndRcpt = sndAll.find((s: any) => s.rcpt_internal_id === rm.internal_id) return { internalId: rm.internal_id, msgMeta: { @@ -1195,7 +1204,7 @@ function createStore(db: IDBDatabase): AgentStore { msgType: msg.msg_type, msgBody: msg.msg_body, userAck: rm.user_ack === 1, - msgReceipt: null, + msgReceipt: sndRcpt?.rcpt_status ?? null, } }) }, @@ -1327,7 +1336,9 @@ function createStore(db: IDBDatabase): AgentStore { internalId: sm.internal_id, msgType: msg.msg_type, internalHash: sm.internal_hash, - msgReceipt: sm.rcpt_status ?? null, + msgReceipt: sm.rcpt_internal_id != null && sm.rcpt_status != null + ? {agentMsgId: sm.rcpt_internal_id, msgRcptStatus: sm.rcpt_status} + : null, } }) }, @@ -1339,8 +1350,8 @@ function createStore(db: IDBDatabase): AgentStore { const store = tx.objectStore("snd_messages") const sm = await idbReq(store.get([connId, sndMsgId])) if (sm) { - sm.rcpt_internal_id = receipt - sm.rcpt_status = receipt + sm.rcpt_internal_id = receipt.agentMsgId + sm.rcpt_status = receipt.msgRcptStatus await idbReq(store.put(sm)) } }) @@ -1536,17 +1547,24 @@ function createStore(db: IDBDatabase): AgentStore { }, // getPendingServerCommand (AgentStore.hs:1439-1480) - // getCmdId SQL: SELECT command_id FROM commands - // WHERE conn_id = ? AND host = ? AND port = ? AND failed = 0 - // ORDER BY created_at ASC, command_id ASC LIMIT 1 + // When host/port are null: + // SQL: SELECT command_id FROM commands WHERE conn_id = ? AND host IS NULL AND port IS NULL AND failed = 0 + // ORDER BY created_at ASC, command_id ASC LIMIT 1 + // When host/port are provided: + // SQL: SELECT command_id FROM commands WHERE conn_id = ? AND host = ? AND port = ? AND failed = 0 + // ORDER BY created_at ASC, command_id ASC LIMIT 1 // getCommand SQL: SELECT c.corr_id, cs.user_id, c.command FROM commands c // JOIN connections cs USING (conn_id) WHERE c.command_id = ? - async getPendingServerCommand(host, port) { + async getPendingServerCommand(connId, host, port) { return withTx(["commands", "connections"], "readonly", async (tx) => { - const allCmds = await allByIndex(tx.objectStore("commands"), {host, port}) - // Filter non-failed, sort by created_at then command_id + const allCmds = await allByIndex(tx.objectStore("commands"), {conn_id: connId}) + // Filter by host/port and non-failed const pending = allCmds - .filter((c: any) => !c.failed) + .filter((c: any) => { + if (c.failed) return false + if (host === null && port === null) return c.host == null && c.port == null + return c.host === host && c.port === port + }) .sort((a: any, b: any) => { const tsCompare = (a.created_at || "").localeCompare(b.created_at || "") return tsCompare !== 0 ? tsCompare : (a.command_id - b.command_id) diff --git a/smp-web/src/agent/store.ts b/smp-web/src/agent/store.ts index 8aa65e4ed..fc14f1447 100644 --- a/smp-web/src/agent/store.ts +++ b/smp-web/src/agent/store.ts @@ -134,7 +134,7 @@ export interface RcvMsg { msgType: string msgBody: Uint8Array userAck: boolean - msgReceipt: Uint8Array | null + msgReceipt: {agentMsgId: number, msgRcptStatus: string} | null } export interface PendingQueueMsg { @@ -173,7 +173,7 @@ export interface AgentStore { createUserRecord(): Promise getUserIds(): Promise deleteUserRecord(userId: UserId): Promise - setUserDeleted(userId: UserId): Promise + setUserDeleted(userId: UserId): Promise // -- Servers (AgentStore.hs:233-240) createServer(host: string, port: string, keyHash: Uint8Array): Promise @@ -213,8 +213,8 @@ export interface AgentStore { setConnectionNtfs(connId: ConnId, enable: boolean): Promise // -- Subscriptions (AgentStore.hs:700-800) - getSubscriptionServers(onlyNeeded: boolean): Promise> - getUserServerRcvQueueSubs(userId: UserId, host: string, port: string, onlyNeeded: boolean, batchSize: number, cursor: number | null): Promise<{queues: RcvQueue[], nextCursor: number | null}> + getSubscriptionServers(onlyNeeded: boolean): Promise> + getUserServerRcvQueueSubs(userId: UserId, host: string, port: string, keyHash: Uint8Array, onlyNeeded: boolean, batchSize: number, cursor: number | null): Promise<{queues: RcvQueue[], nextCursor: number | null}> unsetQueuesToSubscribe(): Promise getConnectionsForDelivery(): Promise getAllSndQueuesForDelivery(): Promise @@ -251,10 +251,10 @@ export interface AgentStore { checkRcvMsgHashExists(connId: ConnId, hash: Uint8Array): Promise getRcvMsgBrokerTs(connId: ConnId, brokerId: Uint8Array): Promise deleteMsg(connId: ConnId, internalId: number): Promise - deleteDeliveredSndMsg(connId: ConnId, internalSndId: number): Promise + deleteDeliveredSndMsg(connId: ConnId, internalId: number): Promise deleteSndMsgDelivery(connId: ConnId, sndQueue: SndQueue, msgId: number, keepForReceipt: boolean): Promise - getSndMsgViaRcpt(connId: ConnId, sndMsgId: number): Promise<{internalId: number, msgType: string, internalHash: Uint8Array, msgReceipt: Uint8Array | null} | null> - updateSndMsgRcpt(connId: ConnId, sndMsgId: number, receipt: Uint8Array): Promise + getSndMsgViaRcpt(connId: ConnId, sndMsgId: number): Promise<{internalId: number, msgType: string, internalHash: Uint8Array, msgReceipt: {agentMsgId: number, msgRcptStatus: string} | null} | null> + updateSndMsgRcpt(connId: ConnId, sndMsgId: number, receipt: {agentMsgId: number, msgRcptStatus: string}): Promise // -- Ratchet (AgentStore.hs:1300-1400) createRatchetX3dhKeys(connId: ConnId, privKey1: Uint8Array, privKey2: Uint8Array, pqKem: Uint8Array | null): Promise @@ -269,7 +269,7 @@ export interface AgentStore { createCommand(corrId: Uint8Array, connId: ConnId, host: string | null, port: string | null, command: AsyncCommand): Promise getPendingCommandServers(connIds: ConnId[]): Promise> getAllPendingCommandConns(): Promise> - getPendingServerCommand(host: string, port: string): Promise + getPendingServerCommand(connId: ConnId, host: string | null, port: string | null): Promise updateCommandServer(commandId: number, host: string, port: string): Promise deleteCommand(commandId: number): Promise diff --git a/smp-web/tests/store-test.ts b/smp-web/tests/store-test.ts index d97631e29..a8a0ee6a1 100644 --- a/smp-web/tests/store-test.ts +++ b/smp-web/tests/store-test.ts @@ -303,7 +303,8 @@ async function testSubscriptions(store: AgentStore) { lastInternalMsgId: 0, lastInternalRcvMsgId: 0, lastInternalSndMsgId: 0, lastExternalSndMsgId: 0, lastRcvMsgHash: new Uint8Array(0), lastSndMsgHash: new Uint8Array(0), }, "INV") - await store.createServer("sub.example.com", "5223", randomBytes(32)) + const subKeyHash = randomBytes(32) + await store.createServer("sub.example.com", "5223", subKeyHash) const rcvQ = await store.addConnRcvQueue(connId, { host: "sub.example.com", port: "5223", rcvId: randomBytes(24), connId, rcvPrivateKey: randomBytes(32), rcvDhSecret: randomBytes(32), @@ -311,7 +312,7 @@ async function testSubscriptions(store: AgentStore) { sndId: randomBytes(24), sndKey: null, status: "new" as const, smpClientVersion: 7, dbQueueId: 0, primary: true, replaceRcvQueueId: null, queueMode: null, - serverKeyHash: randomBytes(32), lastBrokerTs: null, + serverKeyHash: subKeyHash, lastBrokerTs: null, }, "SMOnlyCreate") // getSubscriptionServers — onlyNeeded=true should find our to_subscribe=1 queue @@ -323,7 +324,7 @@ async function testSubscriptions(store: AgentStore) { assert(allSrvs.some(s => s.host === "sub.example.com"), "getSubscriptionServers(false) finds all") // getUserServerRcvQueueSubs - const {queues} = await store.getUserServerRcvQueueSubs(userId, "sub.example.com", "5223", true, 10, null) + const {queues} = await store.getUserServerRcvQueueSubs(userId, "sub.example.com", "5223", subKeyHash, true, 10, null) assert(queues.length >= 1, "getUserServerRcvQueueSubs finds queues") // unsetQueuesToSubscribe @@ -612,8 +613,7 @@ async function testSendMessages(store: AgentStore) { assertEq(sndMsg!.internalHash, internalHash, "getSndMsgViaRcpt hash matches") // updateSndMsgRcpt - const receipt = randomBytes(8) - await store.updateSndMsgRcpt(connId, internalSndId, receipt) + await store.updateSndMsgRcpt(connId, internalSndId, {agentMsgId: internalId, msgRcptStatus: "ok"}) // deleteSndMsgDelivery — deletes delivery, then msg if no more deliveries await store.deleteSndMsgDelivery(connId, sndQueue, internalId, false) @@ -747,19 +747,19 @@ async function testCommands(store: AgentStore) { assert(allConns.some(c => hex(c.connId) === hex(connId)), "getAllPendingCommandConns finds conn") // getPendingServerCommand - const pendingCmd = await store.getPendingServerCommand("cmd.example.com", "5223") + const pendingCmd = await store.getPendingServerCommand(connId, "cmd.example.com", "5223") assert(pendingCmd !== null, "getPendingServerCommand finds command") // updateCommandServer await store.updateCommandServer(cmdId, "cmd2.example.com", "5224") - const updated = await store.getPendingServerCommand("cmd2.example.com", "5224") + const updated = await store.getPendingServerCommand(connId, "cmd2.example.com", "5224") assert(updated !== null, "updateCommandServer changes server") - const oldHost = await store.getPendingServerCommand("cmd.example.com", "5223") + const oldHost = await store.getPendingServerCommand(connId, "cmd.example.com", "5223") assert(oldHost === null, "updateCommandServer — old host returns nothing") // deleteCommand await store.deleteCommand(cmdId) - const deleted = await store.getPendingServerCommand("cmd2.example.com", "5224") + const deleted = await store.getPendingServerCommand(connId, "cmd2.example.com", "5224") assert(deleted === null, "deleteCommand removes command") } From 910a922e0d969f70d441cd4c18329426b938506b Mon Sep 17 00:00:00 2001 From: "Evgeny @ SimpleX Chat" <259188159+evgeny-simplex@users.noreply.github.com> Date: Sun, 31 May 2026 14:59:47 +0000 Subject: [PATCH 3/4] fix store --- smp-web/src/agent/store-idb.ts | 36 +++++++++++++++++++++++++--------- smp-web/src/agent/store.ts | 21 +++++++++++++++----- smp-web/tests/store-test.ts | 8 ++++---- 3 files changed, 47 insertions(+), 18 deletions(-) diff --git a/smp-web/src/agent/store-idb.ts b/smp-web/src/agent/store-idb.ts index ac6b6ca5b..31d54f6d0 100644 --- a/smp-web/src/agent/store-idb.ts +++ b/smp-web/src/agent/store-idb.ts @@ -465,6 +465,7 @@ function createStore(db: IDBDatabase): AgentStore { replace_snd_queue_id: null, smp_client_version: sndQueue.smpClientVersion, server_key_hash: sndQueue.serverKeyHash, snd_public_key: sndQueue.sndPublicKey, })) + return {...sndQueue, connId, dbQueueId: qId} }) }, @@ -1064,7 +1065,7 @@ function createStore(db: IDBDatabase): AgentStore { // LEFT JOIN snd_message_bodies sb ON sb.snd_message_body_id = s.snd_message_body_id // WHERE m.conn_id = ? AND m.internal_id = ? async getPendingQueueMsg(connId, sndQueue) { - return withTx(["snd_message_deliveries", "messages", "snd_messages", "snd_message_bodies"], "readonly", async (tx) => { + return withTx(["snd_message_deliveries", "messages", "snd_messages", "snd_message_bodies", "rcv_queues"], "readonly", async (tx) => { // getMsgId: find first non-failed delivery for this queue const allDel = await allByIndex(tx.objectStore("snd_message_deliveries"), {conn_id: connId, snd_queue_id: sndQueue.dbQueueId}) const pending = allDel.filter((d: any) => !d.failed).sort((a: any, b: any) => a.internal_id - b.internal_id) @@ -1085,13 +1086,24 @@ function createStore(db: IDBDatabase): AgentStore { if (body) sndMsgBody = body.agent_msg } + // getRcvQueuesByConnId_ to get primary rcv queue (head of sorted list) + const rcvQueues = (await allByIndex(tx.objectStore("rcv_queues"), {conn_id: connId})) + .filter((q: any) => !q.deleted) + .sort((a: any, b: any) => (b.rcv_primary || 0) - (a.rcv_primary || 0)) + const rcvQueue = rcvQueues[0] ?? null + return { - connId, sndQueueId: sndQueue.dbQueueId, internalId: msgId, - msgType: msg.msg_type, msgFlags: msg.msg_flags, msgBody: msg.msg_body, - internalHash: sm.internal_hash, prevMsgHash: sm.previous_msg_hash, - pqEncryption: msg.pq_encryption, - msgEncryptKey: sm.msg_encrypt_key, paddedMsgLen: sm.padded_msg_len, - sndMessageBodyId: sm.snd_message_body_id, + rcvQueue, + msg: { + connId, sndQueueId: sndQueue.dbQueueId, internalId: msgId, + internalTs: msg.internal_ts, internalSndId: msg.internal_snd_id, + msgType: msg.msg_type, msgFlags: msg.msg_flags, msgBody: msg.msg_body, + internalHash: sm.internal_hash, prevMsgHash: sm.previous_msg_hash, + pqEncryption: msg.pq_encryption, + retryIntSlow: sm.retry_int_slow, retryIntFast: sm.retry_int_fast, + msgEncryptKey: sm.msg_encrypt_key, paddedMsgLen: sm.padded_msg_len, + sndMsgBody, + }, } }) }, @@ -1165,8 +1177,11 @@ function createStore(db: IDBDatabase): AgentStore { }, msgType: msg.msg_type, msgBody: msg.msg_body, + internalHash: rm.internal_hash, userAck: rm.user_ack === 1, - msgReceipt: sndRcpt?.rcpt_status ?? null, + msgReceipt: sndRcpt && sndRcpt.rcpt_internal_id != null && sndRcpt.rcpt_status != null + ? {agentMsgId: sndRcpt.rcpt_internal_id, msgRcptStatus: sndRcpt.rcpt_status} + : null, } }) }, @@ -1203,8 +1218,11 @@ function createStore(db: IDBDatabase): AgentStore { }, msgType: msg.msg_type, msgBody: msg.msg_body, + internalHash: rm.internal_hash, userAck: rm.user_ack === 1, - msgReceipt: sndRcpt?.rcpt_status ?? null, + msgReceipt: sndRcpt && sndRcpt.rcpt_internal_id != null && sndRcpt.rcpt_status != null + ? {agentMsgId: sndRcpt.rcpt_internal_id, msgRcptStatus: sndRcpt.rcpt_status} + : null, } }) }, diff --git a/smp-web/src/agent/store.ts b/smp-web/src/agent/store.ts index fc14f1447..98be2eaa2 100644 --- a/smp-web/src/agent/store.ts +++ b/smp-web/src/agent/store.ts @@ -12,6 +12,12 @@ export type InternalRcvId = number export type InternalSndId = number export type QueueStatus = "new" | "confirmed" | "secured" | "active" | "disabled" | "deleted" + +// Matches Haskell SkippedMsgDiff (Crypto/Ratchet.hs:584-587) +export type SkippedMsgDiff = + | {type: "noChange"} + | {type: "remove", headerKey: Uint8Array, msgN: number} + | {type: "add", keys: Map>} // Map> export type ConnectionMode = "INV" | "CON" // SCMInvitation | SCMContact export type RatchetSyncState = "ok" | "allowed" | "required" | "started" | "agreed" @@ -133,6 +139,7 @@ export interface RcvMsg { msgMeta: MsgMeta msgType: string msgBody: Uint8Array + internalHash: Uint8Array userAck: boolean msgReceipt: {agentMsgId: number, msgRcptStatus: string} | null } @@ -141,15 +148,19 @@ export interface PendingQueueMsg { connId: ConnId sndQueueId: number internalId: number + internalTs: string + internalSndId: number msgType: string msgFlags: number msgBody: Uint8Array internalHash: Uint8Array prevMsgHash: Uint8Array pqEncryption: boolean + retryIntSlow: number | null + retryIntFast: number | null msgEncryptKey: Uint8Array | null paddedMsgLen: number | null - sndMessageBodyId: number | null + sndMsgBody: Uint8Array | null // agent_msg from snd_message_bodies (joined) } export interface AsyncCommand { @@ -198,14 +209,14 @@ export interface AgentStore { // -- Queues (AgentStore.hs:500-700) addConnRcvQueue(connId: ConnId, rcvQueue: RcvQueue, subMode: string): Promise - addConnSndQueue(connId: ConnId, sndQueue: SndQueue): Promise + addConnSndQueue(connId: ConnId, sndQueue: SndQueue): Promise setRcvQueueStatus(rcvQueue: RcvQueue, status: QueueStatus): Promise setSndQueueStatus(sndQueue: SndQueue, status: QueueStatus): Promise setRcvQueueConfirmedE2E(rcvQueue: RcvQueue, dhSecret: Uint8Array, smpClientVersion: number): Promise setRcvQueuePrimary(connId: ConnId, rcvQueue: RcvQueue): Promise deleteConnRcvQueue(rcvQueue: RcvQueue): Promise deleteConnRecord(connId: ConnId): Promise - upgradeRcvConnToDuplex(connId: ConnId, sndQueue: SndQueue): Promise + upgradeRcvConnToDuplex(connId: ConnId, sndQueue: SndQueue): Promise upgradeSndConnToDuplex(connId: ConnId, rcvQueue: RcvQueue, subMode: string): Promise getPrimaryRcvQueue(connId: ConnId): Promise getRcvQueue(connId: ConnId, host: string, port: string, rcvId: Uint8Array): Promise @@ -242,7 +253,7 @@ export interface AgentStore { createSndMsg(connId: ConnId, sndMsgData: SndMsgData): Promise updateSndMsgHash(connId: ConnId, internalSndId: number, hash: Uint8Array): Promise createSndMsgDelivery(connId: ConnId, sndQueue: SndQueue, internalId: number): Promise - getPendingQueueMsg(connId: ConnId, sndQueue: SndQueue): Promise + getPendingQueueMsg(connId: ConnId, sndQueue: SndQueue): Promise<{rcvQueue: RcvQueue | null, msg: PendingQueueMsg} | null> updatePendingMsgRIState(connId: ConnId, msgId: number, retryIntSlow: number | null, retryIntFast: number | null): Promise setMsgUserAck(connId: ConnId, internalId: number): Promise<{rcvQueue: RcvQueue, brokerId: Uint8Array}> getRcvMsg(connId: ConnId, internalId: number): Promise @@ -263,7 +274,7 @@ export interface AgentStore { getRatchet(connId: ConnId): Promise getRatchetForUpdate(connId: ConnId): Promise // same as getRatchet in IndexedDB (single-threaded) getSkippedMsgKeys(connId: ConnId): Promise>> - updateRatchet(connId: ConnId, ratchetState: Uint8Array, skippedMsgDiff: unknown): Promise + updateRatchet(connId: ConnId, ratchetState: Uint8Array, skippedMsgDiff: SkippedMsgDiff): Promise // -- Commands (AgentStore.hs:1400-1480) createCommand(corrId: Uint8Array, connId: ConnId, host: string | null, port: string | null, command: AsyncCommand): Promise diff --git a/smp-web/tests/store-test.ts b/smp-web/tests/store-test.ts index a8a0ee6a1..59c27f482 100644 --- a/smp-web/tests/store-test.ts +++ b/smp-web/tests/store-test.ts @@ -598,10 +598,10 @@ async function testSendMessages(store: AgentStore) { await store.createSndMsgDelivery(connId, sndQueue, internalId) // getPendingQueueMsg - const pending = await store.getPendingQueueMsg(connId, sndQueue) - assert(pending !== null, "getPendingQueueMsg finds pending message") - assertEq(pending!.msgType, "SEND", "getPendingQueueMsg msgType matches") - assertEq(pending!.internalId, internalId, "getPendingQueueMsg internalId matches") + const pendingResult = await store.getPendingQueueMsg(connId, sndQueue) + assert(pendingResult !== null, "getPendingQueueMsg finds pending message") + assertEq(pendingResult!.msg.msgType, "SEND", "getPendingQueueMsg msgType matches") + assertEq(pendingResult!.msg.internalId, internalId, "getPendingQueueMsg internalId matches") // updatePendingMsgRIState await store.updatePendingMsgRIState(connId, internalId, 30, 5) From 7f95b4540dbf758a7685113bba039759937f615f Mon Sep 17 00:00:00 2001 From: "Evgeny @ SimpleX Chat" <259188159+evgeny-simplex@users.noreply.github.com> Date: Sun, 31 May 2026 19:52:44 +0000 Subject: [PATCH 4/4] smp web: agent store functions and agent infrastructure --- .../2026-05-31-agent-client.md | 653 ++++++++++++++++++ smp-web/src/agent/client.ts | 459 ++++++++++++ smp-web/src/agent/queue.ts | 82 +++ smp-web/src/agent/retry.ts | 118 ++++ smp-web/src/agent/session.ts | 62 ++ smp-web/src/agent/subscriptions.ts | 213 ++++++ smp-web/src/agent/tmvar.ts | 132 ++++ smp-web/tests/infra-test.ts | 465 +++++++++++++ 8 files changed, 2184 insertions(+) create mode 100644 rfcs/2026-03-20-smp-agent-web/2026-05-31-agent-client.md create mode 100644 smp-web/src/agent/client.ts create mode 100644 smp-web/src/agent/queue.ts create mode 100644 smp-web/src/agent/retry.ts create mode 100644 smp-web/src/agent/session.ts create mode 100644 smp-web/src/agent/subscriptions.ts create mode 100644 smp-web/src/agent/tmvar.ts create mode 100644 smp-web/tests/infra-test.ts diff --git a/rfcs/2026-03-20-smp-agent-web/2026-05-31-agent-client.md b/rfcs/2026-03-20-smp-agent-web/2026-05-31-agent-client.md new file mode 100644 index 000000000..540c51f2f --- /dev/null +++ b/rfcs/2026-03-20-smp-agent-web/2026-05-31-agent-client.md @@ -0,0 +1,653 @@ +# Agent Client Middle Layer: Transpilation Plan + +**Parent**: [Agent Plan](./2026-05-22-agent.md) +**Depends on**: Store (complete, 98 tests), SMP Client (complete, 99 tests), Ratchet (complete), Agent Protocol Types (complete) + +## Rule + +Every TypeScript function is a faithful transpilation of a specific Haskell function. Same name, same steps, same call chain. No inferences, no simplifications, no "browser-friendly" shortcuts. The concurrency primitives differ (Promises vs STM, event callbacks vs TBQueue), but the logic, state transitions, and decision paths must be identical. + +## Architecture mapping + +| Haskell | TypeScript | Notes | +|---------|-----------|-------| +| `TVar a` | mutable variable (object property) | Single-threaded, no atomicity needed | +| `TMap k v` | `Map` | No STM, direct mutation | +| `TBQueue a` | `ABQueue` | `subQ` for user events, `msgQ` for server messages | +| `TMVar a` | `Promise` + resolver, or flag | For worker doWork signaling | +| `STM` transaction | synchronous code block | Single-threaded JS, no races | +| `forkIO` / `async` | `setTimeout(0)` / microtask | Event loop scheduling | +| `Worker` thread | delivery loop function | Triggered by `submitPendingMsg`, runs via microtask | +| `ReaderT Env IO` (AM') | closure over agent state | Config + store + DRG captured in closure | +| `ExceptT AgentErrorType` (AM) | thrown errors / Result type | TBD: throw vs return Either | + +## Files to create + +| File | Purpose | +|------|---------| +| `smp-web/src/agent/queue.ts` | Sem, ABQueue (copied from simplex-chat) | +| `smp-web/src/agent/tmvar.ts` | TMVar — single-cell blocking variable | +| `smp-web/src/agent/session.ts` | SessionVar, getSessVar (Promise-based) | +| `smp-web/src/agent/retry.ts` | RetryInterval, RetryInterval2, withRetryLock2 | +| `smp-web/src/agent/subscriptions.ts` | TSessionSubs transpilation | +| `smp-web/src/agent/client.ts` | AgentClient state, session management, worker infrastructure, queue operations | +| `smp-web/src/agent/agent.ts` | Top-level agent API (joinConnection, sendMessage, etc.) | +| `smp-web/tests/agent-repl.ts` | Agent-level REPL for cross-language testing | + +## Implementation order + +Each step produces a testable artifact. + +### Step 1: Pure infrastructure (TS-only tests) +- `queue.ts` — Sem, ABQueue (copied verbatim from simplex-chat) +- `tmvar.ts` — TMVar (new) +- `session.ts` — SessionVar, getSessVar, removeSessVar, tryReadSessVar +- `retry.ts` — RetryInterval types + nextRetryDelay + withRetryInterval + withRetryLock2 +- `subscriptions.ts` — TSessionSubs (all 22 functions) +- **Test**: TS unit tests for each module — no server needed + +### Step 2: AgentClient state + worker infrastructure (TS-only tests) +- AgentClient record, newAgentClient +- Worker, newWorker, getAgentWorker, runWorkerAsync, waitForWork, hasWorkToDo, withWork +- AgentOpState, operation bracket, suspend/resume +- Locking (withConnLock, withInvLock) +- Server selection (userServers, pickServer, getNextServer, withNextSrv) +- Store wrappers (withStore, withStore', storeError) +- **Test**: TS tests — create agent client, test worker lifecycle, test server selection + +### Step 3: SMP session management + queue operations (cross-language via agent-repl) +- getSMPServerClient, smpConnectClient, smpClientDisconnected +- getSMPProxyClient, withProxySession +- withClient_, withClient, withSMPClient +- sendOrProxySMPMessage, sendOrProxySMPCommand +- newRcvQueue, newRcvQueue_ +- agentCbEncrypt, agentCbEncryptOnce, agentCbDecrypt +- sendConfirmation, sendInvitation, sendAgentMessage +- secureQueue, secureSndQueue, sendAck +- decryptSMPMessage +- subscribeQueues, subscribeQueues_, subscribeSessQueues_, processSubResults +- addNewQueueSubscription, resubscribeSMPSession +- **Test via agent-repl**: Haskell creates queue → TS subscribes, TS creates queue → Haskell sends → TS receives+decrypts, TS sends → Haskell receives + +### Step 4: Agent message flow (cross-language end-to-end) +- agentRatchetEncrypt, agentRatchetEncryptHeader, agentRatchetDecrypt +- encodeAgentMsgStr +- enqueueMessageB, storeConfirmation, enqueueConfirmation +- submitPendingMsg, getDeliveryWorker, runSmpQueueMsgDelivery +- enqueueCommand, runCommandProcessing +- **Test via agent-repl**: TS encrypts agent message → Haskell decrypts, Haskell encrypts → TS decrypts + +### Step 5: Connection handshake + full agent API (cross-language end-to-end) +- newConnToJoin, joinConn, joinConnSrv, startJoinInvitation +- compatibleInvitationUri, compatibleContactUri +- secureConfirmQueue(Async), agentSecureSndQueue +- mkAgentConfirmation, createReplyQueue, newRcvConnSrv, createRcvQueue +- newSndQueue, connectReplyQueues +- allowConnection' +- processSMPTransmissions, subscriber +- decryptClientMessage, agentClientMsg +- smpConfirmation, helloMsg, smpInvitation +- sendMessage', sendMessagesB_ +- ackMessage', ackQueueMessage +- subscribeConnection(s) +- **Test**: TS joins invitation URI created by Haskell agent → handshake completes → messages flow both ways → ack + +--- + +## Piece -1: Concurrency primitives + +### `Sem` and `ABQueue` — copy from simplex-chat + +Copy verbatim from `/code/simplex-chat/packages/simplex-chat-client/typescript/src/queue.ts` into `smp-web/src/agent/queue.ts`. + +`Sem` — counting semaphore. `wait()` blocks if permits=0. `signal()` increments and wakes a waiter. +`ABQueue` — async bounded queue. Two semaphores (enq for items, deq for slots). Backpressure on full. Close via sentinel. Implements AsyncIterator. + +Used for: +- `subQ` — agent events to user. Agent writes, user reads via `dequeue()` loop or async iterator. +- `msgQ` — WebSocket onmessage enqueues, subscriber loop dequeues and calls `processSMPTransmissions`. +- Queues prevent deadlock: without them, processing a received message that triggers a send, which triggers another event, could cause unbounded reentrancy in single-threaded JS. + +### `TMVar` — new, in `smp-web/src/agent/tmvar.ts` + +Single-cell mutable variable, empty or full. Blocking take/put/read. + +```typescript +class TMVar { + private val: T | undefined + private full: boolean + private takeQ: Array<(v: T) => void> = [] // waiters for value to appear + private putQ: Array<(v: T) => void> = [] // waiters for cell to empty + + static empty(): TMVar // create empty + static new(v: T): TMVar // create full + + take(): Promise // block until full, take value, leave empty + put(v: T): Promise // block until empty, put value + read(): Promise // block until full, return value without taking + tryTake(): T | undefined // non-blocking take + tryPut(v: T): boolean // non-blocking put, returns false if full + tryRead(): T | undefined // non-blocking read + isEmpty(): boolean +} +``` + +Used for: +- `doWork :: TMVar ()` — worker signal. `waitForWork` = `read()`. `noWorkToDo` = `tryTake()`. `hasWorkToDo` = `tryPut(undefined)`. +- `action :: TMVar (Maybe ThreadId)` — worker running state. `runWorkerAsync` takes, checks, starts async loop. +- Retry lock in `withRetryLock2`. + +The doWork race condition: worker clears FIRST (`tryTake`), THEN checks store. If work found, re-sets (`tryPut`). Any signal arriving during the store check (via `await` yielding to onmessage → `hasWorkToDo`) stays set because it happened after the clear. If we did read-then-clear-if-empty, the clear could swallow a signal set between the store check and the clear. + +### Locks — `Sem(1)` or Promise chain + +For `withConnLock`, `withInvLock`: `Map`. Each Lock is either: +- `Sem(1)` — acquire = `wait()`, release = `signal()`, wrap in try/finally +- Or Promise chain (each `withLock` appends to previous promise) + +`Sem(1)` is simpler and correct. Wrap in helper: + +```typescript +async function withLock(locks: Map, key: string, fn: () => Promise): Promise { + let sem = locks.get(key) + if (!sem) { sem = new Sem(1); locks.set(key, sem) } + await sem.wait() + try { return await fn() } finally { sem.signal() } +} +``` + +### SessionVar — Promise with exposed resolver + +`SessionVar` tracks pending protocol client connections. First caller creates a Promise, connects, resolves it. Subsequent callers await the same Promise. + +```typescript +interface SessionVar { + id: number + ts: number + promise: Promise + resolve: (v: T) => void + reject: (e: Error) => void + value: T | undefined // set after resolve, for tryRead +} +``` + +`getSessVar`: if key exists in Map, return Right (existing). Else create new with unresolved Promise, insert, return Left (new). +`removeSessVar`: delete if ID matches. +`tryReadSessVar`: return `value` if set. + +No TMVar needed — Promise coalesces reads naturally. + +--- + +## Piece 0: SessionVar (`session.ts`) + +Transpile from `Simplex/Messaging/Session.hs` (43 lines). + +| Function | Haskell lines | Purpose | +|----------|--------------|---------| +| `SessionVar` type | 18-22 | `{sessionVar: TMVar a, sessionVarId: number, sessionVarTs: Date}` | +| `getSessVar` | 24-33 | Get existing or create new empty session var for key | +| `removeSessVar` | 35-39 | Remove if ID matches (guards against removing replaced session) | +| `tryReadSessVar` | 41-42 | Non-blocking read of session var value | + +Browser: `TMVar a` → `{value: T | undefined, resolve: (() => void) | null}`. `getSessVar` returns Left (new, empty) or Right (existing). + +--- + +## Piece 1: RetryInterval (`retry.ts`) + +Transpile from `Agent/RetryInterval.hs` (119 lines). + +| Function | Haskell lines | Purpose | +|----------|--------------|---------| +| `RetryInterval` type | 27-31 | `{initialInterval, increaseAfter, maxInterval}` (microseconds) | +| `RetryInterval2` type | 33-36 | `{riSlow, riFast}` | +| `RI2State` type | 38-41 | `{slowInterval, fastInterval}` | +| `RetryIntervalMode` type | 51 | `RISlow \| RIFast` | +| `nextRetryDelay` | 114-118 | Pure: if elapsed < increaseAfter, keep delay; else min(delay*3/2, max) | +| `updateRetryInterval2` | 44-49 | Update RI2 from saved state | +| `withRetryInterval` | 54-55 | Wrapper around withRetryIntervalCount | +| `withRetryIntervalCount` | 57-66 | Loop: action(n, delay, loop); loop sleeps then recurses with updated delay | +| `withRetryLock2` | 90-112 | Two-mode retry with lock: action gets RI2State + loop function that takes mode | + +Browser adaptation: `threadDelay'` → `setTimeout` wrapped in Promise. `TMVar` lock → Promise-based signal. Logic identical. + +--- + +## Piece 2: TSessionSubs (`subscriptions.ts`) + +Transpile from `Agent/TSessionSubs.hs` (202 lines). Every function, every branch. + +Transport session key: `(UserId, SMPServer)` — serialized to string for Map key. One session per server, no per-entity multiplexing. + +| Function | Haskell lines | Purpose | +|----------|--------------|---------| +| `TSessionSubs` type | 49-51 | `Map` (string = serialized transport session) | +| `SessSubs` type | 53-57 | `{sessId: SessionId \| null, activeSubs: Map, pendingSubs: Map}` | +| `emptyIO` | 59-61 | Create empty TSessionSubs | +| `clear` | 63-65 | Clear all | +| `getSessSubs` | 71-77 | Get or create SessSubs for a transport session | +| `hasActiveSub` | 79-81 | Check if rcvId has active subscription | +| `hasPendingSub` | 83-85 | Check if rcvId has pending subscription | +| `addPendingSub` | 91-92 | Add to pendingSubs | +| `setSessionId` | 94-99 | Set session ID; if changed, move active→pending | +| `addActiveSub` | 101-110 | If sessId matches, add to active + remove from pending; else add to pending | +| `batchAddActiveSubs` | 112-121 | Batch version of addActiveSub | +| `batchAddPendingSubs` | 123-126 | Batch add to pending | +| `deletePendingSub` | 128-129 | Delete from pending | +| `batchDeletePendingSubs` | 131-134 | Batch delete from pending | +| `deleteSub` | 136-137 | Delete from both active and pending | +| `batchDeleteSubs` | 139-143 | Batch delete from both | +| `hasPendingSubs` | 145-146 | Check if any pending exist for session | +| `getPendingSubs` | 148-150 | Get all pending for session | +| `getActiveSubs` | 152-154 | Get all active for session | +| `setSubsPending` | 159-177 | Move active→pending on disconnect; handles session mode transitions | +| `setSubsPending_` | 179-187 | Internal: write new sessId, move active→pending | +| `updateClientNotices` | 189-192 | Update clientNoticeId on pending subs | +| `foldSessionSubs` | 194-195 | Fold over all sessions | +| `mapSubs` | 197-201 | Map over active and pending | + +Critical: `setSubsPending` has mode-dependent logic (TSMEntity vs TSMUser/TSMServer). Must be transpiled exactly. + +--- + +## Piece 3: AgentClient state + worker infrastructure (`client.ts`) + +### AgentClient state + +Transpile from `AgentClient` record (Client.hs:328-378) and `newAgentClient` (Client.hs:498-584). + +| Field | Haskell type | TS type | Purpose | +|-------|-------------|---------|---------| +| `active` | `TVar Bool` | `boolean` | Is agent active | +| `subQ` | `TBQueue ATransmission` | `ABQueue` | Events to user | +| `msgQ` | `TBQueue (ServerTransmissionBatch ...)` | `ABQueue` | WebSocket onmessage enqueues, subscriber dequeues | +| `smpServers` | `TMap UserId (UserServers 'PSMP)` | `Map` | Server configs per user | +| `smpClients` | `TMap SMPTransportSession SMPClientVar` | `Map>` | Active SMP connections | +| `smpProxiedRelays` | `TMap SMPTransportSession SMPServerWithAuth` | `Map` | Proxy routing | +| `useNetworkConfig` | `TVar (NetworkConfig, NetworkConfig)` | `{slow: NetworkConfig, fast: NetworkConfig}` | Network config | +| `userNetworkInfo` | `TVar UserNetworkInfo` | `UserNetworkInfo` | Online/offline state | +| `subscrConns` | `TVar (Set ConnId)` | `Set` | Connections being subscribed | +| `currentSubs` | `TSessionSubs` | `TSessionSubs` | Active/pending subscriptions | +| `removedSubs` | `TMap ...` | `Map>` | Failed subscriptions | +| `workerSeq` | `TVar Int` | `number` | Worker ID sequence | +| `smpDeliveryWorkers` | `TMap SndQAddr (Worker, TMVar ())` | `Map` | Per-queue delivery workers | +| `asyncCmdWorkers` | `TMap (ConnId, Maybe SMPServer) Worker` | `Map` | Async command workers | +| `rcvNetworkOp` | `TVar AgentOpState` | `AgentOpState` | Receive operation state | +| `msgDeliveryOp` | `TVar AgentOpState` | `AgentOpState` | Delivery operation state | +| `sndNetworkOp` | `TVar AgentOpState` | `AgentOpState` | Send operation state | +| `agentState` | `TVar AgentState` | `AgentState` | Foreground/suspended/suspending | +| `connLocks` | `TMap ConnId Lock` | `Map>` | Connection locks | +| `invLocks` | `TMap ByteString Lock` | `Map>` | Invitation locks | +| `agentEnv` | `Env` | closure | Config + store + RNG | + +Fields NOT needed for MVP: `ntfServers`, `ntfClients`, `xftpServers`, `xftpClients`, `smpSubWorkers`, `clientNotices`, `clientNoticesLock`, `getMsgLocks`, `deleteLock`, `proxySessTs`, `*Stats`, `srvStatsStartedAt`, `acThread`, `presetDomains`, `presetServers`. + +### Worker infrastructure + +| Function | Haskell location | Purpose | +|----------|-----------------|---------| +| `Worker` type | Env/SQLite.hs:317-322 | `{workerId, doWork: TMVar (), action: TMVar (Maybe ThreadId), restarts}` | +| `RestartCount` type | Env/SQLite.hs:324-327 | `{restartMinute, restartCount}` | +| `updateRestartCount` | Env/SQLite.hs:329-332 | Reset count if minute changed, else increment | +| `newWorker` | Client.hs:439-445 | Create worker with doWork TMVar | +| `getAgentWorker` | Client.hs:387-389 | Get-or-create worker for key | +| `getAgentWorker'` | Client.hs:391-437 | Full version with restart logic | +| `runWorkerAsync` | Client.hs:447-454 | Start worker if not running | +| `waitForWork` | Client.hs:2118-2119 | Block until doWork has value | +| `hasWorkToDo` / `hasWorkToDo'` | Client.hs:2171-2176 | Signal work available (tryPutTMVar) | +| `withWork` / `withWork_` | Client.hs:2122-2140 | Wait for work, get item from store, run action | + +Browser adaptation: `TMVar ()` → boolean flag + resolver. `forkIO` → `setTimeout(0)`. Worker restart logic must be preserved exactly. + +### Operation state management + +| Function | Haskell location | Purpose | +|----------|-----------------|---------| +| `AgentOpState` type | Client.hs:470 | `{opSuspended, opsInProgress}` | +| `AgentState` type | Client.hs:472-473 | `ASForeground \| ASSuspending \| ASSuspended` | +| `agentOperationBracket` | Client.hs:2232-2245 | Begin/end operation with suspend check | +| `beginAgentOperation` | Client.hs:2223-2230 | Increment opsInProgress | +| `endAgentOperation` | Client.hs:2179-2197 | Decrement opsInProgress, cascade suspend | +| `waitUntilActive` | Client.hs:956-957 | Block until agent is active | +| `throwWhenInactive` | Client.hs:959-962 | Throw if not active | +| `waitWhileSuspended` | Client.hs:2248-2253 | Block while suspended | +| `waitForUserNetwork` | Client.hs:924-928 | Block until network online | +| `noWorkToDo` | Client.hs:2167-2168 | Clear work flag (tryTakeTMVar) | + +### Store wrappers + +| Function | Haskell location | Purpose | +|----------|-----------------|---------| +| `withStore` | Client.hs:2259-2270 | Run store action, convert StoreError to AgentErrorType | +| `withStore'` | Client.hs:2255-2257 | Simplified withStore (always Right) | +| `storeError` | Client.hs (exported) | StoreError → AgentErrorType | + +### Server selection + +| Function | Haskell location | Purpose | +|----------|-----------------|---------| +| `userServers` | Client.hs:2312-2318 | Get user's server map | +| `pickServer` | Client.hs:2318-2325 | Pick server from NonEmpty list | +| `getNextServer` | Client.hs:2325-2350 | Get next server avoiding used hosts | +| `withNextSrv` | Client.hs:2375-2407 | Retry with next server on failure | + +### Locking + +| Function | Haskell location | Purpose | +|----------|-----------------|---------| +| `withConnLock` | Client.hs:1003-1006 | Per-connection mutex | +| `withConnLocks` | Client.hs:1020-1022 | Multiple connection mutex | +| `withInvLock` | Client.hs:1012-1015 | Per-invitation mutex | + +Browser: locks via Promise chains. Single-threaded JS means no actual contention, but the ordering semantics must be preserved for async operations. + +--- + +## Piece 4: SMP session management + +| Function | Haskell location | Purpose | +|----------|-----------------|---------| +| `getSMPServerClient` | Client.hs:642-651 | Get or create SMP WebSocket client for transport session | +| `getSMPProxyClient` | Client.hs:653-702 | Get or create proxied relay session | +| `smpConnectClient` | Client.hs:704-718 | Actually connect SMP client via WebSocket | +| `smpClientDisconnected` | Client.hs:720-754 | Handle disconnect: move subs to pending, notify, resubscribe | +| `resubscribeSMPSession` | Client.hs:756-790 | Create resubscription worker | +| `mkTransportSession` | Client.hs:1345-1348 | Build transport session key | +| `mkSMPTransportSession` | Client.hs:1357-1360 | Build SMP transport session from queue | +| `getSessionMode` | Client.hs:1369-1370 | Get current session mode | +| `withClient_` | Client.hs:1037-1045 | Bracket: get client → run action → handle errors | +| `withClient` | Client.hs:1071-1073 | withClient_ + liftClient | +| `withSMPClient` | Client.hs:1079-1082 | withClient for SMP queues | +| `withLogClient_` | Client.hs:1064-1069 | withClient_ with logging | +| `withProxySession` | Client.hs:1047-1062 | Bracket for proxied operations | +| `sendOrProxySMPMessage` | Client.hs:1084-1094 | Decide direct vs proxy for SEND | +| `sendOrProxySMPCommand` | Client.hs:1096-1180 | Decide direct vs proxy for commands (SKEY etc) | +| `ipAddressProtected` | Client.hs:1181-1185 | Check if server is in protected domains | +| `liftClient` | Client.hs:1201-1203 | Convert protocol client error | +| `protocolClientError` | Client.hs:1205-1235 | Error conversion | +| `waitForProtocolClient` | Client.hs:847-868 | Wait for pending client connection | +| `newProtocolClient` | Client.hs:870-896 | Create protocol client with error handling | +| `activeClientSession` | Client.hs:1663-1666 | Check if client session is current (compares sessionId) | +| `removeSubscription` | Client.hs:1752-1755 | Remove single subscription from currentSubs + subscrConns | +| `removeSubscriptions` | Client.hs:1757-1763 | Remove multiple subscriptions | +| `hasActiveSubscription` | Client.hs:1736-1740 | Check if queue has active sub | +| `hasPendingSubscription` | Client.hs:1742-1747 | Check if queue has pending sub | +| `getClientConfig` | Client.hs:904-908 | Get protocol client config (slow/fast network) | +| `getNetworkConfig` | Client.hs:910-918 | Get current network config | +| `getFastNetworkConfig` | Client.hs:920-922 | Get fast network config | +| `slowNetworkConfig` | Client.hs:586-592 | Derive slow config from fast | +| `batchQueues` | Client.hs:1679-1684 | Group queues by transport session | +| `sendTSessionBatches` | Client.hs:1674-1678 | Send batched operations per session (mapConcurrently) | +| `sendClientBatch` | Client.hs:1686-1688 | Send batch to single client session (wrapper) | +| `sendClientBatch_` | Client.hs:1690-1722 | Send batch to single client: get client, run action, handle errors | +| `checkQueues` | Client.hs:1590-1595 | Filter out prohibited queues (GET lock check) | +| `subscribeSessQueues_` | Client.hs:1611-1651 | Send SUB batch via sendClientBatch_ + process results | + +--- + +## Piece 5: Queue operations (use session management) + +| Function | Haskell location | Purpose | +|----------|-----------------|---------| +| `newRcvQueue` | Client.hs:1373-1377 | Generate keys, create queue on SMP server | +| `newRcvQueue_` | Client.hs:1394-1474 | Full queue creation: auth keys, DH, createSMPQueue, build RcvQueue record | +| `subscribeQueues` | Client.hs:1543-1556 | Batch subscribe rcv queues grouped by transport session | +| `subscribeQueues_` | Client.hs:1556-1720 | Subscribe batch for one session | +| `processSubResults` | Client.hs:1476-1510 | Process subscribe results: partition into failed/subscribed/notices | +| `addNewQueueSubscription` | Client.hs:1724-1728 | Add queue to active subs after creation | +| `sendConfirmation` | Client.hs:1788-1794 | Per-queue E2E encrypt confirmation → SEND via sendOrProxySMPMessage | +| `sendInvitation` | Client.hs:1796-1806 | Per-queue E2E encrypt invitation → SEND via sendOrProxySMPMessage | +| `sendAgentMessage` | Client.hs:1948-1952 | Per-queue E2E encrypt message → SEND via sendOrProxySMPMessage | +| `agentCbEncrypt` | Client.hs:2074-2082 | Per-queue E2E encrypt with stored DH secret | +| `agentCbEncryptOnce` | Client.hs:2085-2095 | Per-queue E2E encrypt with ephemeral DH (for invitations) | +| `agentCbDecrypt` | Client.hs:2099-2102 | Per-queue E2E decrypt | +| `secureQueue` | Client.hs:1830-1833 | Send KEY command | +| `secureSndQueue` | Client.hs:1835-1841 | Send SKEY command via sendOrProxySMPCommand | +| `sendAck` | Client.hs:1904-1907 | Send ACK command | +| `decryptSMPMessage` | Client.hs:1824-1828 | Decrypt received SMP message body | +| `suspendQueue` | Client.hs:1926-1929 | Send OFF command | +| `deleteQueue` | Client.hs:1931-1934 | Send DEL command | +| `deleteQueues` | Client.hs:1936-1945 | Batch DEL | +| `getQueueMessage` | Client.hs:1808-1822 | Send GET + decrypt (for polling, if needed) | +| `notifySub` / `notifySub'` | Client.hs:791-797 | Write event to subQ | +| `cryptoError` | Client.hs:2104-2115 | CryptoError → AgentErrorType | + +--- + +## Piece 6: Agent API (top level) + +| Function | Haskell location | Purpose | +|----------|-----------------|---------| +| `joinConn` | Agent.hs:1260-1263 | Top-level join: pick server, delegate | +| `joinConnSrv` (invitation) | Agent.hs:1342-1357 | Lock, startJoinInvitation, secureConfirmQueue | +| `joinConnSrv` (contact) | Agent.hs:1358-1388 | Lock, create rcv queue, sendInvitation | +| `startJoinInvitation` | Agent.hs:1270-1310 | Version check, ratchet params, snd queue, createRatchet_ | +| `compatibleInvitationUri` | Agent.hs:1321-1328 | Version range compatibility | +| `compatibleContactUri` | Agent.hs:1330-1336 | Version range compatibility | +| `secureConfirmQueue` | Agent.hs:3653-3671 | Secure + send confirmation synchronously | +| `secureConfirmQueueAsync` | Agent.hs:3645-3651 | Secure + store confirmation for async delivery | +| `agentSecureSndQueue` | Agent.hs:3673-3684 | SKEY decision logic | +| `mkAgentConfirmation` | Agent.hs:3686-3691 | Build AgentConnInfoReply with reply queue | +| `storeConfirmation` | Agent.hs:3698-3712 | Ratchet encrypt + store as SndMsg | +| `enqueueConfirmation` | Agent.hs:3693-3696 | Store + submit for delivery | +| `createReplyQueue` | Agent.hs:1415-1424 | Create rcv queue for reply | +| `newRcvConnSrv` | Agent.hs:1155-1220 | Full create-connection-with-rcv-queue | +| `createRcvQueue` | Agent.hs:972-983 | Wrapper: newRcvQueue_ + updateNewConnRcv + addNewQueueSubscription | +| `newConnToJoin` | Agent.hs:1237-1253 | Create ConnData for join, store connection | +| `newSndQueue` | Agent.hs:3769-3802 | Build SndQueue from SMPQueueInfo | +| `getNextSMPServer` | Agent.hs (via Client.hs) | Pick server for new queue, avoiding contact's server | +| `connReqQueue` | Agent.hs:1265-1268 | Extract first queue from ConnectionRequestUri | +| `versionPQSupport_` | Agent.hs:1338-1340 | PQ support based on agent+e2e versions | +| `sendMessage'` | Agent.hs:1705-1706 | Top-level send | +| `sendMessagesB_` | Agent.hs:1725-1760 | Get conn, prepare, delegate | +| `enqueueMessageB` | Agent.hs:1989-2048 | Core: updateSndIds, encode, ratchetEncryptHeader, createSndMsg, createSndMsgDelivery | +| `encodeAgentMsgStr` | Agent.hs:2050-2054 | Encode AgentMessage to bytes | +| `agentRatchetEncrypt` | Agent.hs:3742-3746 | Ratchet encrypt message body | +| `agentRatchetEncryptHeader` | Agent.hs:3748-3754 | Get encrypt key from ratchet | +| `agentRatchetDecrypt` | Agent.hs:3757-3767 | Ratchet decrypt with skipped keys | +| `submitPendingMsg` | Agent.hs:2087-2090 | Signal delivery worker | +| `getDeliveryWorker` | Agent.hs:2079-2085 | Get-or-create delivery worker for queue | +| `runSmpQueueMsgDelivery` | Agent.hs:2092-2270 | Delivery loop: getPendingQueueMsg, dispatch on msgType, send, handle errors | +| `ackMessage'` | Agent.hs:2285-2323 | ACK + delete + optional receipt | +| `ackQueueMessage` | Agent.hs:2410-2430 | Send ACK to server, handle response | +| `subscriber` | Agent.hs:2912-2919 | Read from msgQ, dispatch | +| `processSMPTransmissions` | Agent.hs:2997-3297 | Incoming message dispatcher | +| `decryptClientMessage` | Agent.hs:3282-3296 | Per-queue E2E decrypt + parse envelope | +| `agentClientMsg` | Agent.hs:3207-3225 | Ratchet decrypt, parse, store RcvMsg | +| `smpConfirmation` | Agent.hs:3298-3370 | Process received confirmation | +| `helloMsg` | Agent.hs:3372-3393 | Process HELLO | +| `smpInvitation` | Agent.hs:3515-3570 | Process received invitation | +| `allowConnection'` | Agent.hs:1427-1434 | Accept confirmation, enqueue ICAllowSecure | +| `connectReplyQueues` | Agent.hs:3630-3643 | Process reply queues from confirmation | +| `enqueueCommand` | Agent.hs:1764-1767 | Store command + start worker | +| `runCommandProcessing` | Agent.hs:1789-1902 | Async command worker loop | +| `enqueueMessage` / `enqueueMessages` | Agent.hs:~2370+ | Convenience wrappers | +| `resumeMsgDelivery` | Agent.hs:2072-2076 | Resume delivery worker for a snd queue | +| `resumeConnCmds` | Agent.hs:1773-1776 | Resume async command workers for connections | +| `resumeAllCommands` | Agent.hs:1778-1781 | Resume all pending async commands on startup | +| `enqueueSavedMessage` | Agent.hs:2056-2057 | Create delivery for additional snd queues | +| `checkMsgIntegrity` | Agent.hs:3603-3610 | Verify message sequence integrity (local fn in processSMPTransmissions) | +| `subscribeConnection'` | Agent.hs:1472-1474 | Subscribe single connection (delegates to subscribeConnections') | +| `subscribeConnections'` | Agent.hs:1488-1490 | Get conn subs from store, delegate to subscribeConnections_ | +| `subscribeConnections_` | Agent.hs:1492-1527 | Core: partition conns, resume delivery, subscribe rcv queues | + +--- + +## Resolved decisions + +1. **Store field naming**: No mapping. IDB returns snake_case (`row.conn_id`), agent code uses it directly. +2. **Error handling**: Throw custom `AgentError` exception with typed error data matching Haskell `AgentErrorType`. Catches process errors by type. +3. **subQ**: Keep ABQueue. Agent writes events, user reads. Queues prevent deadlock — without them, a callback within message processing that triggers a send could deadlock single-threaded JS. +4. **msgQ**: Keep ABQueue. WebSocket onmessage enqueues, subscriber loop dequeues. Prevents reentrancy. +5. **Structured commands in IDB**: Store as JS objects. New fields optional. +6. **Transport session key**: `(userId, server)` tuple. One WebSocket per server. No TSMEntity. All TSMEntity-specific branches dropped. +7. **Join flows**: Both invitation and contact needed. Contact for widget's primary flow (joining address). Invitation for internal group member connections. +8. **Async join**: `joinConnSrvAsync` / `secureConfirmQueueAsync` as primary path. +9. **Version ranges**: Match Haskell defaults. +10. **Config defaults**: Match Haskell defaults. +11. **`ep/conc-msgs` branch**: Ignore. Use queues. +12. **Connection type dispatch**: Compute from `conn_mode` field + queue presence. `conn_mode = "INV"` with rcv+snd queues = duplex, with only rcv = rcv, etc. +13. **`withAgentEnv`**: No-op in TS — env in closure. +14. **`getConnSubs` for subscribe**: Use `getConn` (returns conn + queues) in subscribe flow. +15. **Client notices**: Skip in `subscribeSessQueues_`. + +--- + +## Store methods to add + +These store methods are not yet in `store.ts` / `store-idb.ts` but are needed by the agent layer: + +| Method | AgentStore.hs lines | Used by | +|--------|-------------------|---------| +| `createSndRatchet` | 1271-1287 | `startJoinInvitation` — stores ratchet + e2e pub keys for sending side | +| `getSndRatchet` | 1289-1300 | `startJoinInvitation` — retry path, get previously created snd ratchet | +| `updateNewConnSnd` | 424-431 | `startJoinInvitation` — add snd queue to new connection | +| `createSndConn` | 433-440 | May be needed for contact join flow | +| `setRcvQueueStatus` | already exists | — | +| `setSndQueueStatus` | already exists | — | +| `setRcvSwitchStatus` | skip (queue switching) | — | +| `setSndSwitchStatus` | skip (queue switching) | — | + +--- + +## What to skip for MVP + +| Feature | Functions to skip | +|---------|------------------| +| Queue switching | `switchConnection`, QADD/QKEY/QUSE/QTEST handlers, `switchDuplexConnection` | +| Ratchet sync | `synchronizeRatchet`, EREADY handler, `newRatchetKey` | +| Notifications | All NTF functions, `newQueueNtfSubscription` | +| File transfer | All XFTP functions | +| Remote control | All RC functions | +| Connection creation | `createConnection`, `newConn`, short links creation | +| Delivery receipts sending | `sendRcpt` in ackMessage (receiving A_RCVD is kept) | +| Multiple rcv queues | Queue replacement logic in processSMPTransmissions | +| Cleanup manager | `cleanupManager`, `deleteRcvMsgHashesExpired`, etc. | +| Server management | `setProtocolServers`, `testProtocolServer` | +| Client notices | `processClientNotices`, `subscribeClientService` | +| Statistics | All `inc*ServerStat` calls, `getAgentServersSummary` | +| Connection comparison | `compareConnections`, `syncConnections` | + +## Testing strategy + +### Principle: every step produces testable output + +Cross-language tests from Haskell give the highest confidence because they verify wire compatibility. Pure TS tests verify internal logic. The goal is to have Haskell tests at every step where the TS code touches the network. + +### Step 1 tests: pure TS (no server) + +File: `tests/infra-test.ts` (run with `node`, like store-test.ts) + +- **RetryInterval**: `nextRetryDelay` returns correct values for various elapsed/delay combinations. `withRetryIntervalCount` calls action with increasing delays. +- **TSessionSubs**: Full lifecycle: add pending → set session ID → add active (moves from pending) → disconnect (moves back to pending) → reconnect. Test `setSubsPending` mode logic (entity vs user session). Test batch operations. +- **SessionVar**: `getSessVar` returns Left for new, Right for existing. `removeSessVar` only removes matching ID. + +### Step 2 tests: TS with store (no server) + +File: `tests/worker-test.ts` + +- Create AgentClient with real IndexedDB store (fake-indexeddb). +- Test worker lifecycle: create worker → signal work → worker runs → no work → worker waits → signal again. +- Test server selection: configure servers → `getNextServer` rotates avoiding used hosts. +- Test locking: `withConnLock` serializes async operations on same connId. + +### Step 3 tests: agent-repl + Haskell (real SMP server) + +File: `tests/agent-repl.ts` — new REPL with higher-level commands. + +The agent-repl exposes mid-level operations that Haskell can drive: + +``` +AGENT_INIT + → creates AgentClient, connects to server + +CREATE_RCV_QUEUE + → newRcvQueue on server, returns rcvId, sndId, sndQueueUri + → Haskell can then SEND to this queue + +SUBSCRIBE + → subscribeQueues for connection's rcv queue + +SEND_AGENT_MSG + → agentCbEncrypt + sendAgentMessage + +RECV + → wait for MSG from WebSocket, decryptSMPMessage, return parsed body + +SECURE_SND + → secureSndQueue (SKEY) + +SEND_CONFIRMATION + → sendConfirmation + +ACK + → sendAck +``` + +Haskell test scenarios: +1. **Queue creation**: TS creates rcv queue → Haskell verifies by sending to it → TS receives +2. **Subscribe + receive**: TS subscribes → Haskell sends MSG → TS decrypts and returns +3. **Send**: TS sends agent message → Haskell receives and decrypts +4. **SKEY**: TS sends SKEY → Haskell verifies queue secured +5. **Proxy send**: TS sends via proxy → Haskell receives + +These tests verify the entire session management + queue operations layer without needing the full agent handshake. + +### Step 4 tests: agent-repl + Haskell (ratchet operations) + +Extend agent-repl: + +``` +RATCHET_ENCRYPT + → agentRatchetEncrypt, return encrypted agent envelope + +RATCHET_DECRYPT + → agentRatchetDecrypt, return plaintext + +ENQUEUE_MSG + → enqueueMessageB (encrypt + store + create delivery) + +DELIVER + → runSmpQueueMsgDelivery one iteration (getPendingQueueMsg + send) +``` + +Haskell test scenarios: +1. **Ratchet encrypt cross-language**: Initialize ratchet in both → TS encrypts → Haskell decrypts (and vice versa). Already have ratchet cross-language tests, extend to agent envelope level. +2. **Enqueue + deliver**: TS enqueues message → delivery worker sends → Haskell receives and decrypts entire agent message envelope. +3. **Receive + store**: Haskell sends agent message → TS receives, decrypts, stores RcvMsg → verify stored correctly. + +### Step 5 tests: full handshake (end-to-end) + +Extend agent-repl or create dedicated test: + +``` +JOIN + → full joinConnection flow + +ALLOW + → allowConnection + +SEND + → sendMessage + +ACK_MSG + → ackMessage +``` + +Haskell test scenarios: +1. **Join invitation**: Haskell creates invitation → TS joins → handshake completes (CONF, HELLO exchange) → messages flow both ways. +2. **Join contact**: Haskell creates contact address → TS joins → Haskell accepts → messages flow. +3. **Multiple connections**: TS joins two different connections on different servers simultaneously. +4. **Reconnect**: Connection established → WebSocket drops → resubscribe → messages resume. + +### Test infrastructure + +All cross-language tests go in `tests/SMPWebTests.hs`, extending the existing 99 tests. Each agent-repl command is a single stdin/stdout exchange (like client-repl). Haskell `callNode` drives the TS process. + +Estimated test count per step: +- Step 1: ~15 TS tests (retry: 5, subs: 8, session: 2) +- Step 2: ~8 TS tests (worker: 4, server selection: 2, locking: 2) +- Step 3: ~8 Haskell tests (queue create, subscribe, send, receive, SKEY, proxy) +- Step 4: ~6 Haskell tests (ratchet encrypt/decrypt, enqueue+deliver, receive+store) +- Step 5: ~6 Haskell tests (join invitation, join contact, send/receive/ack, reconnect) diff --git a/smp-web/src/agent/client.ts b/smp-web/src/agent/client.ts new file mode 100644 index 000000000..970faac16 --- /dev/null +++ b/smp-web/src/agent/client.ts @@ -0,0 +1,459 @@ +// AgentClient — agent state, worker infrastructure, locking, server selection. +// Transpilation of Agent/Client.hs (AgentClient record, workers, operation state, locks, server selection). +// Session management and queue operations will be added in subsequent steps. + +import {ABQueue} from "./queue.js" +import {TMVar} from "./tmvar.js" +import {Sem} from "./queue.js" +import {TSessionSubs} from "./subscriptions.js" +import type {AgentStore} from "./store.js" +import type {RetryInterval2} from "./retry.js" + +// -- Error types (Client.hs:2296-2310 storeError, Client.hs:2104-2115 cryptoError) + +export class AgentError extends Error { + constructor(public readonly type: AgentErrorType) { + super(agentErrorToString(type)) + } +} + +export type AgentErrorType = + | {tag: "AGENT", err: AgentErr} + | {tag: "BROKER", addr: string, err: BrokerErr} + | {tag: "SMP", addr: string, err: string} + | {tag: "PROXY", proxyServer: string, relayServer: string, proxyErr: string} + | {tag: "CONN", err: ConnErr, context: string} + | {tag: "CMD", err: CmdErr, context: string} + | {tag: "INTERNAL", msg: string} + | {tag: "CRITICAL", important: boolean, msg: string} + | {tag: "INACTIVE"} + | {tag: "NO_USER"} + +export type AgentErr = + | "A_VERSION" | "A_ENCRYPTION" | "A_DUPLICATE" | "A_PROHIBITED" | "A_MESSAGE" + | {tag: "A_QUEUE", msg: string} + | {tag: "A_CRYPTO", err: string} + +export type BrokerErr = "TIMEOUT" | "NETWORK" | "HOST" | "TRANSPORT" | {tag: "RESPONSE", err: string} | {tag: "UNEXPECTED", msg: string} + +export type ConnErr = "NOT_FOUND" | "DUPLICATE" | "SIMPLEX" | "NOT_ACCEPTED" | "NOT_AVAILABLE" + +export type CmdErr = "PROHIBITED" | "SYNTAX" | "NO_CONN" | {tag: "LARGE", msg: string} + +function agentErrorToString(e: AgentErrorType): string { + switch (e.tag) { + case "INTERNAL": return `INTERNAL: ${e.msg}` + case "CRITICAL": return `CRITICAL: ${e.msg}` + case "AGENT": return `AGENT ${typeof e.err === "string" ? e.err : e.err.tag}` + case "BROKER": return `BROKER ${e.addr} ${typeof e.err === "string" ? e.err : e.err.tag}` + case "SMP": return `SMP ${e.addr} ${e.err}` + case "CONN": return `CONN ${e.err} ${e.context}` + case "CMD": return `CMD ${typeof e.err === "string" ? e.err : e.err.tag} ${e.context}` + default: return e.tag + } +} + +// -- Agent config (Env/SQLite.hs:136-180, 182-205) + +export interface AgentConfig { + tbqSize: number + connIdBytes: number + smpAgentVRange: [number, number] // [min, max] + smpClientVRange: [number, number] + e2eEncryptVRange: [number, number] + messageRetryInterval: RetryInterval2 + messageTimeout: number // ms + helloTimeout: number // ms + quotaExceededTimeout: number // ms + maxWorkerRestartsPerMin: number +} + +export const defaultAgentConfig: AgentConfig = { + tbqSize: 128, + connIdBytes: 12, + smpAgentVRange: [1, 8], + smpClientVRange: [7, 18], + e2eEncryptVRange: [1, 2], + messageRetryInterval: { + riFast: {initialInterval: 2_000_000, increaseAfter: 10_000_000, maxInterval: 120_000_000}, + riSlow: {initialInterval: 300_000_000, increaseAfter: 60_000_000, maxInterval: 6 * 3600_000_000}, + }, + messageTimeout: 2 * 86400_000, + helloTimeout: 2 * 86400_000, + quotaExceededTimeout: 7 * 86400_000, + maxWorkerRestartsPerMin: 5, +} + +// -- Server types + +export interface SMPServerWithAuth { + server: string // serialized server address + auth: Uint8Array | null +} + +export interface UserServers { + storageSrvs: Array<[number | null, SMPServerWithAuth]> // [(Maybe OperatorId, ProtoServerWithAuth)] + proxySrvs: Array<[number | null, SMPServerWithAuth]> + knownHosts: Set +} + +// -- Worker (Env/SQLite.hs:317-332) + +export interface Worker { + workerId: number + doWork: TMVar + action: TMVar // null = not running, number = "running" placeholder (no threadId in JS) + restarts: {restartMinute: number, restartCount: number} +} + +// updateRestartCount (Env/SQLite.hs:329-332) +function updateRestartCount(now: number, rc: {restartMinute: number, restartCount: number}): {restartMinute: number, restartCount: number} { + const min = Math.floor(now / 60000) + return {restartMinute: min, restartCount: min === rc.restartMinute ? rc.restartCount + 1 : 1} +} + +// -- AgentOperation (Client.hs:456-470) + +export type AgentOperation = "AORcvNetwork" | "AOMsgDelivery" | "AOSndNetwork" | "AODatabase" + +export interface AgentOpState { + opSuspended: boolean + opsInProgress: number +} + +export type AgentState = "ASForeground" | "ASSuspending" | "ASSuspended" + +// -- ATransmission event type + +export type ATransmission = [string, Uint8Array, any] // (corrId, connId, event) + +// -- AgentClient (Client.hs:328-378) + +export interface AgentClient { + active: boolean + subQ: ABQueue + // msgQ is per-client, stored in smpClients + config: AgentConfig + store: AgentStore + smpServers: Map // userId → servers + smpClients: Map // tSessKey → SMPClient or pending + smpProxiedRelays: Map + userNetworkInfo: {networkType: string, online: boolean} + subscrConns: Set // hex connIds being subscribed + currentSubs: TSessionSubs + workerSeq: number + smpDeliveryWorkers: Map}> + asyncCmdWorkers: Map + rcvNetworkOp: AgentOpState + msgDeliveryOp: AgentOpState + sndNetworkOp: AgentOpState + databaseOp: AgentOpState + agentState: AgentState + connLocks: Map + invLocks: Map + randomServer: {gen: () => number} // random index generator +} + +// newAgentClient (Client.hs:498-584) +export function newAgentClient(config: AgentConfig, store: AgentStore, smpServers: Map): AgentClient { + return { + active: true, + subQ: new ABQueue(config.tbqSize), + config, + store, + smpServers, + smpClients: new Map(), + smpProxiedRelays: new Map(), + userNetworkInfo: {networkType: "UNOther", online: true}, + subscrConns: new Set(), + currentSubs: new TSessionSubs(), + workerSeq: 0, + smpDeliveryWorkers: new Map(), + asyncCmdWorkers: new Map(), + rcvNetworkOp: {opSuspended: false, opsInProgress: 0}, + msgDeliveryOp: {opSuspended: false, opsInProgress: 0}, + sndNetworkOp: {opSuspended: false, opsInProgress: 0}, + databaseOp: {opSuspended: false, opsInProgress: 0}, + agentState: "ASForeground", + connLocks: new Map(), + invLocks: new Map(), + randomServer: {gen: () => Math.random()}, + } +} + +// -- Worker functions (Client.hs:439-454, 2118-2176) + +// newWorker (Client.hs:439-445) +export function newWorker(c: AgentClient): Worker { + const workerId = c.workerSeq++ + return { + workerId, + doWork: TMVar.new(undefined), // starts with "has work" + action: TMVar.new(null), // not running + restarts: {restartMinute: 0, restartCount: 0}, + } +} + +// waitForWork (Client.hs:2118-2119) +export function waitForWork(doWork: TMVar): Promise { + return doWork.read().then(() => {}) +} + +// noWorkToDo (Client.hs:2167-2168) +export function noWorkToDo(doWork: TMVar): void { + doWork.tryTake() +} + +// hasWorkToDo (Client.hs:2171-2172) +export function hasWorkToDo(w: Worker): void { + hasWorkToDo_(w.doWork) +} + +// hasWorkToDo' (Client.hs:2175-2176) +export function hasWorkToDo_(doWork: TMVar): void { + doWork.tryPut(undefined) +} + +// runWorkerAsync (Client.hs:447-454) +// Ensures work runs at most once concurrently. If already running, no-op. +// In Haskell this uses bracket + forkIO. In JS, fire-and-forget Promise. +// +// bracket (takeTMVar action) (tryPutTMVar action) (\a -> when (isNothing a) start) +// start = putTMVar action . Just =<< mkWeakThreadId =<< forkIO work +export async function runWorkerAsync(w: Worker, work: () => Promise): Promise { + const a = await w.action.take() + if (a !== null) { + // Already running — put back and return + w.action.tryPut(a) + return + } + // Mark as running, start work in background + await w.action.put(1) + // forkIO — fire and forget. Work function contains its own restart loop (runWork). + // When work eventually stops (max restarts or worker removed), reset action to null. + work().catch(() => {}).finally(() => { + w.action.tryTake() + w.action.tryPut(null) + }) +} + +// getAgentWorker (Client.hs:387-437) +// Get or create a worker for the given key. If hasWork=true, signal the worker. +// Starts the worker async loop if not already running. +// The work function should use `forever` internally — this function handles crash restart. +export async function getAgentWorker( + name: string, + hasWork_: boolean, + c: AgentClient, + key: string, + workers: Map, + work: (w: Worker) => Promise, +): Promise { + // getWorker >>= maybe createWorker whenExists + let w = workers.get(key) + if (w) { + if (hasWork_) hasWorkToDo(w) + } else { + w = newWorker(c) + workers.set(key, w) + } + const worker = w + // runWorker w = runWorkerAsync (toW w) runWork + await runWorkerAsync(worker, () => runWork(name, c, key, workers, worker, work)) + return worker +} + +// runWork (Client.hs:405-413) — runs work, on error checks whether to restart +async function runWork( + name: string, + c: AgentClient, + key: string, + workers: Map, + worker: Worker, + work: (w: Worker) => Promise, +): Promise { + // tryAllErrors' (work w) >>= restartOrDelete + let error: unknown = undefined + try { + await work(worker) + } catch (e) { + error = e + } + // restartOrDelete (Client.hs:407-413) + const now = Date.now() + // getWorker >>= maybe (pure False) (shouldRestart ...) + const currentWorker = workers.get(key) + if (!currentWorker) return // worker was removed from map, don't restart + if (currentWorker.workerId !== worker.workerId) return // replaced by new worker + // shouldRestart (Client.hs:414-437) + const rc = updateRestartCount(now, worker.restarts) + const isActive = c.active + const errStr = error !== undefined ? `, error: ${error}` : ", no error" + const msg = `Worker ${name} for ${key} terminated ${rc.restartCount} times${errStr}` + if (isActive && rc.restartCount < c.config.maxWorkerRestartsPerMin) { + // checkRestarts: restart + worker.restarts = rc + hasWorkToDo_(worker.doWork) + worker.action.tryTake() + worker.action.tryPut(null) + c.subQ.enqueue(["", new Uint8Array(0), {tag: "ERR", err: {tag: "INTERNAL", msg}}]) + // when restart runWork — restart the worker + await runWork(name, c, key, workers, worker, work) + } else { + // checkRestarts: delete + workers.delete(key) + if (isActive) { + c.subQ.enqueue(["", new Uint8Array(0), {tag: "ERR", err: {tag: "CRITICAL", important: true, msg}}]) + } + } +} + +// withWork_ (Client.hs:2126-2140) +// Clear work signal, get work from store, if found re-signal and run action. +export async function withWork( + c: AgentClient, + doWork: TMVar, + getWork: () => Promise, + action: (item: T) => Promise, +): Promise { + noWorkToDo(doWork) + let item: T | null + try { + item = await getWork() + } catch (e) { + hasWorkToDo_(doWork) + const msg = `withWork error: ${e}` + c.subQ.enqueue(["", new Uint8Array(0), {tag: "ERR", err: {tag: "INTERNAL", msg}}]) + return + } + if (item !== null) { + hasWorkToDo_(doWork) + await action(item) + } +} + +// -- Operation state (Client.hs:2179-2253) + +function agentOpState(c: AgentClient, op: AgentOperation): AgentOpState { + switch (op) { + case "AORcvNetwork": return c.rcvNetworkOp + case "AOMsgDelivery": return c.msgDeliveryOp + case "AOSndNetwork": return c.sndNetworkOp + case "AODatabase": return c.databaseOp + } +} + +// beginAgentOperation (Client.hs:2223-2230) +export function beginAgentOperation(c: AgentClient, op: AgentOperation): void { + const s = agentOpState(c, op) + if (s.opSuspended) throw new AgentError({tag: "INACTIVE"}) + s.opsInProgress++ +} + +// endAgentOperation (Client.hs:2179-2197) +export function endAgentOperation(c: AgentClient, op: AgentOperation): void { + const s = agentOpState(c, op) + s.opsInProgress = Math.max(0, s.opsInProgress - 1) + if (s.opSuspended && s.opsInProgress === 0 && c.agentState === "ASSuspending") { + cascadeSuspend(c, op) + } +} + +function cascadeSuspend(c: AgentClient, op: AgentOperation): void { + switch (op) { + case "AORcvNetwork": + suspendOp(c, "AOMsgDelivery", () => suspendSendingAndDatabase(c)) + break + case "AOMsgDelivery": + suspendSendingAndDatabase(c) + break + case "AOSndNetwork": + suspendOp(c, "AODatabase", () => notifySuspended(c)) + break + case "AODatabase": + notifySuspended(c) + break + } +} + +function suspendSendingAndDatabase(c: AgentClient): void { + suspendOp(c, "AOSndNetwork", () => suspendOp(c, "AODatabase", () => notifySuspended(c))) +} + +function suspendOp(c: AgentClient, op: AgentOperation, endedAction: () => void): void { + const s = agentOpState(c, op) + s.opSuspended = true + if (s.opsInProgress === 0 && c.agentState === "ASSuspending") endedAction() +} + +function notifySuspended(c: AgentClient): void { + c.subQ.enqueue(["", new Uint8Array(0), {tag: "SUSPENDED"}]) + c.agentState = "ASSuspended" +} + +// throwWhenInactive (Client.hs:959-962) +export function throwWhenInactive(c: AgentClient): void { + if (!c.active) throw new AgentError({tag: "INACTIVE"}) +} + +// waitForUserNetwork (Client.hs:924-928) +// In browser: if offline, we just throw. No blocking wait. +export function checkUserNetwork(c: AgentClient): void { + if (!c.userNetworkInfo.online) throw new AgentError({tag: "BROKER", addr: "", err: "NETWORK"}) +} + +// -- Locking (Lock.hs, Client.hs:1003-1030) + +// withConnLock (Client.hs:1003-1006) +export async function withConnLock(c: AgentClient, connId: Uint8Array, fn: () => Promise): Promise { + const key = toHex(connId) + return withLock(c.connLocks, key, fn) +} + +// withInvLock (Client.hs:1012-1015) +export async function withInvLock(c: AgentClient, invKey: Uint8Array, fn: () => Promise): Promise { + return withLock(c.invLocks, toHex(invKey), fn) +} + +async function withLock(locks: Map, key: string, fn: () => Promise): Promise { + let sem = locks.get(key) + if (!sem) { sem = new Sem(1); locks.set(key, sem) } + await sem.wait() + try { return await fn() } finally { sem.signal() } +} + +// -- Server selection (Client.hs:2312-2394) + +// getNextServer (Client.hs:2325-2334) +export function getNextServer( + c: AgentClient, + userId: number, + srvsSel: (us: UserServers) => Array<[number | null, SMPServerWithAuth]>, + usedSrvs: string[], +): SMPServerWithAuth { + const us = c.smpServers.get(userId) + if (!us) throw new AgentError({tag: "INTERNAL", msg: "unknown userId - no user servers"}) + const srvs = srvsSel(us) + if (srvs.length === 0) throw new AgentError({tag: "INTERNAL", msg: "no servers configured"}) + const usedHosts = new Set(usedSrvs) + // Prefer servers with unused hosts + const unused = srvs.filter(([, s]) => !usedHosts.has(s.server)) + const pool = unused.length > 0 ? unused : srvs + return pickServer(pool, c.randomServer) +} + +// pickServer (Client.hs:2318-2323) +function pickServer( + srvs: Array<[number | null, SMPServerWithAuth]>, + rng: {gen: () => number}, +): SMPServerWithAuth { + if (srvs.length === 1) return srvs[0][1] + const idx = Math.floor(rng.gen() * srvs.length) + return srvs[idx][1] +} + +// -- Helpers + +function toHex(b: Uint8Array): string { + return Array.from(b, x => x.toString(16).padStart(2, "0")).join("") +} diff --git a/smp-web/src/agent/queue.ts b/smp-web/src/agent/queue.ts new file mode 100644 index 000000000..a29fdf993 --- /dev/null +++ b/smp-web/src/agent/queue.ts @@ -0,0 +1,82 @@ +// Copied verbatim from simplex-chat/packages/simplex-chat-client/typescript/src/queue.ts + +export class Sem { + private readonly promises: ((x: unknown) => void)[] = [] + + constructor(private permits: number) {} + + signal(): void { + this.permits += 1 + if (this.promises.length > 0) (this.promises.pop() as () => void)() + } + + async wait(): Promise { + if (this.permits === 0 || this.promises.length > 0) { + await new Promise((r) => this.promises.unshift(r)) + } + this.permits -= 1 + } +} + +export type NextIter = {value: T | Promise; done?: false} | {value?: undefined; done: true} + +const queueClosed = Symbol() + +type QueueItem = T | typeof queueClosed + +export class ABQueueError extends Error {} + +export class ABQueue { + private readonly queue: QueueItem[] = [] + private readonly enq: Sem + private readonly deq: Sem + private enqClosed = false + private deqClosed = false + + constructor(readonly maxSize: number) { + this.enq = new Sem(0) + this.deq = new Sem(maxSize) + } + + [Symbol.asyncIterator](): ABQueue { + return this + } + + enqueue(x: T): Promise { + return this._enqueue(x) + } + + private async _enqueue(x: QueueItem): Promise { + if (this.enqClosed) throw new ABQueueError("enqueue: queue closed") + await this.deq.wait() + this.queue.push(x) + this.enq.signal() + } + + async dequeue(): Promise { + if (this.deqClosed) throw new ABQueueError("dequeue: queue closed") + this.deq.signal() + await this.enq.wait() + const x = this.queue.shift() as QueueItem + if (x === queueClosed) { + this.deqClosed = true + throw new ABQueueError("dequeue: queue closed") + } + return x + } + + async close(): Promise { + await this._enqueue(queueClosed) + this.enqClosed = true + } + + async next(): Promise> { + if (this.deqClosed) return {done: true} + try { + return {value: await this.dequeue()} + } catch (e) { + if (e instanceof ABQueueError) return {done: true} + throw e + } + } +} diff --git a/smp-web/src/agent/retry.ts b/smp-web/src/agent/retry.ts new file mode 100644 index 000000000..f374805a2 --- /dev/null +++ b/smp-web/src/agent/retry.ts @@ -0,0 +1,118 @@ +// Retry interval logic. +// Transpilation of Agent/RetryInterval.hs (lines 27-118). +// withRetryForeground is skipped (uses registerDelay + STM retry, Haskell-specific). + +import {TMVar} from "./tmvar.js" + +// RetryInterval (RetryInterval.hs:27-31) +// All intervals in microseconds (matching Haskell Int64). +export interface RetryInterval { + initialInterval: number + increaseAfter: number + maxInterval: number +} + +// RetryInterval2 (RetryInterval.hs:33-36) +export interface RetryInterval2 { + riSlow: RetryInterval + riFast: RetryInterval +} + +// RI2State (RetryInterval.hs:38-41) +export interface RI2State { + slowInterval: number + fastInterval: number +} + +// RetryIntervalMode (RetryInterval.hs:51) +export type RetryIntervalMode = "RISlow" | "RIFast" + +// nextRetryDelay (RetryInterval.hs:114-118) +export function nextRetryDelay(elapsed: number, delay: number, ri: RetryInterval): number { + if (elapsed < ri.increaseAfter || delay === ri.maxInterval) return delay + return Math.min(Math.floor(delay * 3 / 2), ri.maxInterval) +} + +// updateRetryInterval2 (RetryInterval.hs:44-49) +export function updateRetryInterval2(state: RI2State, ri2: RetryInterval2): RetryInterval2 { + return { + riSlow: {...ri2.riSlow, initialInterval: state.slowInterval, increaseAfter: 0}, + riFast: {...ri2.riFast, initialInterval: state.fastInterval, increaseAfter: 0}, + } +} + +function delay(us: number): Promise { + return new Promise(resolve => setTimeout(resolve, us / 1000)) +} + +// withRetryInterval (RetryInterval.hs:54-55) +export function withRetryInterval( + ri: RetryInterval, + action: (delay: number, loop: () => Promise) => Promise, +): Promise { + return withRetryIntervalCount(ri, (_n, d, loop) => action(d, loop)) +} + +// withRetryIntervalCount (RetryInterval.hs:57-66) +export function withRetryIntervalCount( + ri: RetryInterval, + action: (n: number, delay: number, loop: () => Promise) => Promise, +): Promise { + function callAction(n: number, elapsed: number, d: number): Promise { + return action(n, d, async () => { + await delay(d) + const elapsed_ = elapsed + d + return callAction(n + 1, elapsed_, nextRetryDelay(elapsed_, d, ri)) + }) + } + return callAction(0, 0, ri.initialInterval) +} + +// withRetryLock2 (RetryInterval.hs:90-112) +// Two-mode retry with lock. The lock (TMVar) can be released early +// by an external signal (e.g., QCONT message), cancelling the timer wait. +// +// The action receives the current RI2State and a loop function. +// Calling loop(mode) sleeps for the appropriate interval, then recurses. +// During sleep, if the lock is released externally, sleep ends early. +export function withRetryLock2( + ri2: RetryInterval2, + lock: TMVar, + action: (state: RI2State, loop: (mode: RetryIntervalMode) => Promise) => Promise, +): Promise { + function callAction(slow: [number, number], fast: [number, number]): Promise { + return action({slowInterval: slow[1], fastInterval: fast[1]}, (mode) => { + if (mode === "RISlow") return run(slow, ri2.riSlow, (s) => callAction(s, fast)) + return run(fast, ri2.riFast, (f) => callAction(slow, f)) + }) + } + + async function run( + [elapsed, d]: [number, number], + ri: RetryInterval, + call: (state: [number, number]) => Promise, + ): Promise { + await wait(d) + const elapsed_ = elapsed + d + const delay_ = nextRetryDelay(elapsed_, d, ri) + return call([elapsed_, delay_]) + } + + // wait (RetryInterval.hs:105-112) + // Race between timer expiry and external lock release. + // In Haskell: forkIO sets a timer that puts () into the lock TMVar, + // then the main thread takes from the lock (blocking until timer or external signal). + async function wait(d: number): Promise { + let waiting = true + // Start timer that will release the lock after delay + const timer = setTimeout(() => { + if (waiting) lock.tryPut(undefined as any) + }, d / 1000) + // Block until lock is released (by timer or externally) + await lock.take() + waiting = false + clearTimeout(timer) + } + + return callAction([0, ri2.riSlow.initialInterval], [0, ri2.riFast.initialInterval]) +} diff --git a/smp-web/src/agent/session.ts b/smp-web/src/agent/session.ts new file mode 100644 index 000000000..d73a53f50 --- /dev/null +++ b/smp-web/src/agent/session.ts @@ -0,0 +1,62 @@ +// SessionVar — pending protocol client connection tracking. +// Transpilation of Simplex.Messaging.Session (Session.hs:18-42). +// +// SessionVar wraps a Promise that resolves when the client connection is established. +// First caller creates it (Left/new), subsequent callers get the existing one (Right/existing) +// and await the same Promise. + +export interface SessionVar { + id: number + ts: number // creation timestamp (ms) + promise: Promise + resolve: (v: T) => void + reject: (e: Error) => void + value: T | undefined // set after resolve, for tryRead +} + +// getSessVar (Session.hs:24-33) +// Get existing SessionVar for key, or create a new empty one. +// Returns {isNew: true, v} for new, {isNew: false, v} for existing. +// Mirrors Haskell Left (new) / Right (existing). +export function getSessVar( + seq: {val: number}, + key: string, + vars: Map>, +): {isNew: boolean, v: SessionVar} { + const existing = vars.get(key) + if (existing) return {isNew: false, v: existing} + let resolve!: (v: T) => void + let reject!: (e: Error) => void + const promise = new Promise((res, rej) => { resolve = res; reject = rej }) + // When resolved, store value for tryRead + const v: SessionVar = { + id: seq.val++, + ts: Date.now(), + promise, + resolve: (val: T) => { v.value = val; resolve(val) }, + reject, + value: undefined, + } + vars.set(key, v) + return {isNew: true, v} +} + +// removeSessVar (Session.hs:35-39) +// Remove only if the ID matches — guards against removing a replaced session. +export function removeSessVar( + v: SessionVar, + key: string, + vars: Map>, +): void { + const current = vars.get(key) + if (current && current.id === v.id) vars.delete(key) +} + +// tryReadSessVar (Session.hs:41-42) +// Non-blocking read of resolved value. Returns undefined if not yet resolved. +export function tryReadSessVar( + key: string, + vars: Map>, +): T | undefined { + return vars.get(key)?.value +} diff --git a/smp-web/src/agent/subscriptions.ts b/smp-web/src/agent/subscriptions.ts new file mode 100644 index 000000000..1d70848c0 --- /dev/null +++ b/smp-web/src/agent/subscriptions.ts @@ -0,0 +1,213 @@ +// TSessionSubs — subscription tracking. +// Transpilation of Agent/TSessionSubs.hs (lines 49-201). +// +// Transport session key is (userId, server) serialized as string. +// One session per server (no TSMEntity mode). +// RecipientId key is hex-encoded Uint8Array. + +// RcvQueueSub — subset of RcvQueue fields needed for subscription management. +// Transpilation of Agent/Store.hs:183-196. +export interface RcvQueueSub { + userId: number + connId: Uint8Array + server: string // serialized SMPServer + rcvId: Uint8Array + rcvPrivateKey: Uint8Array + status: string + enableNtfs: boolean + clientNoticeId: number | null + dbQueueId: number + primary: boolean + dbReplaceQueueId: number | null +} + +function toHex(b: Uint8Array): string { + return Array.from(b, x => x.toString(16).padStart(2, "0")).join("") +} + +function rcvIdKey(rq: RcvQueueSub): string { + return toHex(rq.rcvId) +} + +// SessSubs (TSessionSubs.hs:53-57) +export interface SessSubs { + sessId: Uint8Array | null // SessionId + activeSubs: Map // keyed by rcvId hex + pendingSubs: Map // keyed by rcvId hex +} + +// TSessionSubs (TSessionSubs.hs:49-51) +export class TSessionSubs { + readonly sessionSubs: Map = new Map() + + // -- Construction + + // clear (TSessionSubs.hs:63-65) + clear(): void { + this.sessionSubs.clear() + } + + // -- Lookup helpers + + // lookupSubs (TSessionSubs.hs:67-69) + private lookupSubs(tSess: string): SessSubs | undefined { + return this.sessionSubs.get(tSess) + } + + // getSessSubs (TSessionSubs.hs:71-77) + private getSessSubs(tSess: string): SessSubs { + const existing = this.sessionSubs.get(tSess) + if (existing) return existing + const s: SessSubs = {sessId: null, activeSubs: new Map(), pendingSubs: new Map()} + this.sessionSubs.set(tSess, s) + return s + } + + // -- Query + + // hasActiveSub (TSessionSubs.hs:79-81) + hasActiveSub(tSess: string, rcvId: string): boolean { + const s = this.lookupSubs(tSess) + return s ? s.activeSubs.has(rcvId) : false + } + + // hasPendingSub (TSessionSubs.hs:83-85) + hasPendingSub(tSess: string, rcvId: string): boolean { + const s = this.lookupSubs(tSess) + return s ? s.pendingSubs.has(rcvId) : false + } + + // hasPendingSubs (TSessionSubs.hs:145-146) + hasPendingSubs(tSess: string): boolean { + const s = this.lookupSubs(tSess) + return s ? s.pendingSubs.size > 0 : false + } + + // getPendingSubs (TSessionSubs.hs:148-150) + getPendingSubs(tSess: string): Map { + return this.lookupSubs(tSess)?.pendingSubs ?? new Map() + } + + // getActiveSubs (TSessionSubs.hs:152-154) + getActiveSubs(tSess: string): Map { + return this.lookupSubs(tSess)?.activeSubs ?? new Map() + } + + // -- Mutation + + // addPendingSub (TSessionSubs.hs:91-92) + addPendingSub(tSess: string, rq: RcvQueueSub): void { + this.getSessSubs(tSess).pendingSubs.set(rcvIdKey(rq), rq) + } + + // setSessionId (TSessionSubs.hs:94-99) + setSessionId(tSess: string, sessId: Uint8Array): void { + const s = this.getSessSubs(tSess) + if (s.sessId === null) { + s.sessId = sessId + } else if (toHex(s.sessId) !== toHex(sessId)) { + this.setSubsPending_(s, sessId) + } + } + + // addActiveSub (TSessionSubs.hs:101-110) + addActiveSub(tSess: string, sessId: Uint8Array, rq: RcvQueueSub): void { + const s = this.getSessSubs(tSess) + const rId = rcvIdKey(rq) + if (s.sessId && toHex(s.sessId) === toHex(sessId)) { + s.activeSubs.set(rId, rq) + s.pendingSubs.delete(rId) + } else { + s.pendingSubs.set(rId, rq) + } + } + + // batchAddActiveSubs (TSessionSubs.hs:112-121) + batchAddActiveSubs(tSess: string, sessId: Uint8Array, rqs: RcvQueueSub[]): void { + const s = this.getSessSubs(tSess) + if (s.sessId && toHex(s.sessId) === toHex(sessId)) { + for (const rq of rqs) { + const rId = rcvIdKey(rq) + s.activeSubs.set(rId, rq) + s.pendingSubs.delete(rId) + } + } else { + for (const rq of rqs) s.pendingSubs.set(rcvIdKey(rq), rq) + } + } + + // batchAddPendingSubs (TSessionSubs.hs:123-126) + batchAddPendingSubs(tSess: string, rqs: RcvQueueSub[]): void { + const s = this.getSessSubs(tSess) + for (const rq of rqs) s.pendingSubs.set(rcvIdKey(rq), rq) + } + + // deletePendingSub (TSessionSubs.hs:128-129) + deletePendingSub(tSess: string, rcvId: string): void { + this.lookupSubs(tSess)?.pendingSubs.delete(rcvId) + } + + // batchDeletePendingSubs (TSessionSubs.hs:131-134) + batchDeletePendingSubs(tSess: string, rcvIds: Set): void { + const s = this.lookupSubs(tSess) + if (s) for (const rId of rcvIds) s.pendingSubs.delete(rId) + } + + // deleteSub (TSessionSubs.hs:136-137) + deleteSub(tSess: string, rcvId: string): void { + const s = this.lookupSubs(tSess) + if (s) { + s.activeSubs.delete(rcvId) + s.pendingSubs.delete(rcvId) + } + } + + // batchDeleteSubs (TSessionSubs.hs:139-143) + batchDeleteSubs(tSess: string, rcvIds: string[]): void { + const s = this.lookupSubs(tSess) + if (s) for (const rId of rcvIds) { + s.activeSubs.delete(rId) + s.pendingSubs.delete(rId) + } + } + + // setSubsPending (TSessionSubs.hs:159-177) + // Simplified: no TSMEntity mode. Session key is always (userId, server) with no entity ID. + // So the mode check `entitySession == isJust connId_` always equals `false == false` = true, + // taking the first branch: lookup + setSubsPending_ with Nothing. + setSubsPending(tSess: string, sessId: Uint8Array): Map { + const s = this.lookupSubs(tSess) + if (!s) return new Map() + if (!s.sessId || toHex(s.sessId) !== toHex(sessId)) return new Map() + return this.setSubsPending_(s, null) + } + + // setSubsPending_ (TSessionSubs.hs:179-187) + private setSubsPending_(s: SessSubs, newSessId: Uint8Array | null): Map { + s.sessId = newSessId + const subs = new Map(s.activeSubs) + if (subs.size > 0) { + s.activeSubs.clear() + for (const [rId, rq] of subs) s.pendingSubs.set(rId, rq) + } + return subs + } + + // updateClientNotices (TSessionSubs.hs:189-192) + // Skip for MVP — client notices are not implemented. + updateClientNotices(_tSess: string, _noticeIds: Array<[string, number | null]>): void { + // no-op + } + + // foldSessionSubs (TSessionSubs.hs:194-195) + foldSessionSubs(f: (acc: A, entry: [string, SessSubs]) => A, initial: A): A { + let acc = initial + for (const entry of this.sessionSubs.entries()) acc = f(acc, entry) + return acc + } + + // mapSubs (TSessionSubs.hs:197-201) + mapSubs(f: (subs: Map) => A, s: SessSubs): [A, A] { + return [f(s.activeSubs), f(s.pendingSubs)] + } +} diff --git a/smp-web/src/agent/tmvar.ts b/smp-web/src/agent/tmvar.ts new file mode 100644 index 000000000..cf745089f --- /dev/null +++ b/smp-web/src/agent/tmvar.ts @@ -0,0 +1,132 @@ +// TMVar — transactional mutable variable, empty or full. +// Transpilation of Haskell's Control.Concurrent.STM.TMVar for single-threaded JS. +// +// Operations: +// take — block until full, take value (leaves empty) +// put — block until empty, put value (leaves full) +// read — block until full, return value without taking +// tryTake — non-blocking take, returns undefined if empty +// tryPut — non-blocking put, returns false if full +// tryRead — non-blocking read, returns undefined if empty +// +// Used for: +// doWork :: TMVar () — worker signaling +// action :: TMVar ... — worker running state +// retry lock — delivery retry coordination + +export class TMVar { + private val: T | undefined + private full: boolean + private takeQ: Array<(v: T) => void> = [] + private putQ: Array<() => void> = [] + + private constructor(val: T | undefined, full: boolean) { + this.val = val + this.full = full + } + + static empty(): TMVar { + return new TMVar(undefined, false) + } + + static new(v: T): TMVar { + return new TMVar(v, true) + } + + // Block until full, take value, leave empty. + // Haskell: takeTMVar + take(): Promise { + if (this.full) { + const v = this.val as T + this.val = undefined + this.full = false + // Wake one blocked putter + const putter = this.putQ.shift() + if (putter) putter() + return Promise.resolve(v) + } + return new Promise(resolve => this.takeQ.push(resolve)) + } + + // Block until empty, put value. + // Haskell: putTMVar + put(v: T): Promise { + if (!this.full) { + // Check if a taker is waiting — hand off directly + const taker = this.takeQ.shift() + if (taker) { + taker(v) + } else { + this.val = v + this.full = true + } + return Promise.resolve() + } + return new Promise(resolve => { + this.putQ.push(() => { + const taker = this.takeQ.shift() + if (taker) { + taker(v) + } else { + this.val = v + this.full = true + } + resolve() + }) + }) + } + + // Block until full, return value without taking. + // Haskell: readTMVar + // + // In Haskell STM, readTMVar is atomic (take + put in one transaction). + // In single-threaded JS, we read the value and leave it in place — no interleaving + // between the read and the next synchronous operation. + read(): Promise { + if (this.full) return Promise.resolve(this.val as T) + return new Promise(resolve => { + this.takeQ.push(v => { + // Put back immediately — single-threaded, no interleaving here + this.val = v + this.full = true + resolve(v) + }) + }) + } + + // Non-blocking take. Returns undefined if empty. + // Haskell: tryTakeTMVar + tryTake(): T | undefined { + if (!this.full) return undefined + const v = this.val as T + this.val = undefined + this.full = false + const putter = this.putQ.shift() + if (putter) putter() + return v + } + + // Non-blocking put. Returns false if full. + // Haskell: tryPutTMVar + tryPut(v: T): boolean { + if (this.full) return false + const taker = this.takeQ.shift() + if (taker) { + taker(v) + } else { + this.val = v + this.full = true + } + return true + } + + // Non-blocking read. Returns undefined if empty. + // Haskell: tryReadTMVar + tryRead(): T | undefined { + return this.full ? (this.val as T) : undefined + } + + isEmpty(): boolean { + return !this.full + } +} diff --git a/smp-web/tests/infra-test.ts b/smp-web/tests/infra-test.ts new file mode 100644 index 000000000..30f39d520 --- /dev/null +++ b/smp-web/tests/infra-test.ts @@ -0,0 +1,465 @@ +// Tests for concurrency primitives, session, retry, and subscriptions. + +import {TMVar} from "../dist/agent/tmvar.js" +import {Sem, ABQueue} from "../dist/agent/queue.js" +import {getSessVar, removeSessVar, tryReadSessVar} from "../dist/agent/session.js" +import {nextRetryDelay, type RetryInterval} from "../dist/agent/retry.js" +import {TSessionSubs, type RcvQueueSub} from "../dist/agent/subscriptions.js" + +let passed = 0 +let failed = 0 + +function assert(cond: boolean, msg: string) { + if (!cond) { console.error("FAIL:", msg); failed++ } else { passed++ } +} + +function assertEq(a: any, b: any, msg: string) { + const av = JSON.stringify(a), bv = JSON.stringify(b) + assert(av === bv, `${msg}: expected ${bv}, got ${av}`) +} + +function hex(b: Uint8Array): string { + return Array.from(b, x => x.toString(16).padStart(2, "0")).join("") +} + +function bytes(n: number): Uint8Array { + const b = new Uint8Array(n) + for (let i = 0; i < n; i++) b[i] = i & 0xff + return b +} + +// -- TMVar tests + +async function testTMVar() { + console.log(" TMVar...") + + // new + tryRead + tryTake + const mv1 = TMVar.new(42) + assertEq(mv1.tryRead(), 42, "new: tryRead returns value") + assertEq(mv1.isEmpty(), false, "new: not empty") + assertEq(mv1.tryTake(), 42, "tryTake returns value") + assertEq(mv1.isEmpty(), true, "after tryTake: empty") + assertEq(mv1.tryTake(), undefined, "tryTake on empty: undefined") + + // empty + tryPut + const mv2 = TMVar.empty() + assertEq(mv2.isEmpty(), true, "empty: isEmpty") + assert(mv2.tryPut(7), "tryPut on empty: true") + assert(!mv2.tryPut(8), "tryPut on full: false") + assertEq(mv2.tryRead(), 7, "tryRead after tryPut: 7") + + // take blocks until put + const mv3 = TMVar.empty() + let taken = "" + const takePromise = mv3.take().then(v => { taken = v }) + assertEq(taken, "", "take blocks: not yet resolved") + await mv3.put("hello") + await takePromise + assertEq(taken, "hello", "take unblocks after put") + assertEq(mv3.isEmpty(), true, "after take: empty") + + // read blocks until put, doesn't take + const mv4 = TMVar.empty() + let readVal = 0 + const readPromise = mv4.read().then(v => { readVal = v }) + await mv4.put(99) + await readPromise + assertEq(readVal, 99, "read unblocks after put") + assertEq(mv4.isEmpty(), false, "read doesn't take: still full") + assertEq(mv4.tryRead(), 99, "value still there after read") + + // doWork pattern: tryPut (signal), read (wait), tryTake (clear) + const doWork = TMVar.new(undefined) // starts with work + await doWork.read() // should not block + doWork.tryTake() // clear + assertEq(doWork.isEmpty(), true, "doWork cleared") + doWork.tryPut(undefined) // signal new work + assertEq(doWork.isEmpty(), false, "doWork signaled") + // double signal is no-op + assert(!doWork.tryPut(undefined), "double signal returns false") +} + +// -- Sem tests + +async function testSem() { + console.log(" Sem...") + + const sem = new Sem(1) + await sem.wait() + // Now permits = 0, next wait should block + let acquired = false + const p = sem.wait().then(() => { acquired = true }) + assertEq(acquired, false, "sem blocks when permits=0") + sem.signal() + await p + assertEq(acquired, true, "sem unblocks after signal") +} + +// -- ABQueue tests + +async function testABQueue() { + console.log(" ABQueue...") + + const q = new ABQueue(3) + await q.enqueue(1) + await q.enqueue(2) + await q.enqueue(3) + // Queue is full (size 3), next enqueue should block + let enqueued = false + const ep = q.enqueue(4).then(() => { enqueued = true }) + assertEq(enqueued, false, "enqueue blocks when full") + const v = await q.dequeue() + assertEq(v, 1, "dequeue returns first item") + await ep + assertEq(enqueued, true, "enqueue unblocks after dequeue") + + // dequeue remaining + assertEq(await q.dequeue(), 2, "dequeue 2") + assertEq(await q.dequeue(), 3, "dequeue 3") + assertEq(await q.dequeue(), 4, "dequeue 4") +} + +// -- SessionVar tests + +async function testSessionVar() { + console.log(" SessionVar...") + + const seq = {val: 0} + const vars = new Map() + + // getSessVar: new + const r1 = getSessVar(seq, "srv1", vars) + assert(r1.isNew, "first getSessVar is new") + assertEq(r1.v.id, 0, "first id is 0") + + // getSessVar: existing + const r2 = getSessVar(seq, "srv1", vars) + assert(!r2.isNew, "second getSessVar is existing") + assertEq(r2.v.id, 0, "same id") + + // different key: new + const r3 = getSessVar(seq, "srv2", vars) + assert(r3.isNew, "different key is new") + assertEq(r3.v.id, 1, "incremented id") + + // tryReadSessVar before resolve + assertEq(tryReadSessVar("srv1", vars), undefined, "tryRead before resolve: undefined") + + // resolve + tryRead + r1.v.resolve("client1") + await r1.v.promise + assertEq(tryReadSessVar("srv1", vars), "client1", "tryRead after resolve") + + // multiple readers get same value + const v1 = await r1.v.promise + const v2 = await r2.v.promise + assertEq(v1, "client1", "reader 1") + assertEq(v2, "client1", "reader 2 (same promise)") + + // removeSessVar: wrong id doesn't remove + removeSessVar({...r1.v, id: 999}, "srv1", vars) + assert(vars.has("srv1"), "removeSessVar with wrong id: not removed") + + // removeSessVar: correct id removes + removeSessVar(r1.v, "srv1", vars) + assert(!vars.has("srv1"), "removeSessVar with correct id: removed") +} + +// -- RetryInterval tests + +async function testRetryInterval() { + console.log(" RetryInterval...") + + const ri: RetryInterval = {initialInterval: 2_000_000, increaseAfter: 10_000_000, maxInterval: 180_000_000} + + // Before increaseAfter: delay unchanged + assertEq(nextRetryDelay(0, 2_000_000, ri), 2_000_000, "before increaseAfter: unchanged") + assertEq(nextRetryDelay(4_000_000, 2_000_000, ri), 2_000_000, "still before increaseAfter") + + // After increaseAfter: delay * 3/2 + assertEq(nextRetryDelay(10_000_000, 2_000_000, ri), 3_000_000, "after increaseAfter: 2M -> 3M") + assertEq(nextRetryDelay(13_000_000, 3_000_000, ri), 4_500_000, "3M -> 4.5M") + + // At maxInterval: stays at max + assertEq(nextRetryDelay(999_000_000, 180_000_000, ri), 180_000_000, "at max: unchanged") + + // Approaching max: capped + assertEq(nextRetryDelay(100_000_000, 150_000_000, ri), 180_000_000, "capped at max") +} + +// -- TSessionSubs tests + +async function testTSessionSubs() { + console.log(" TSessionSubs...") + + const ss = new TSessionSubs() + const tSess = "user1:smp1.example.com" + const sessId1 = bytes(32) + const sessId2 = new Uint8Array(32).fill(0xff) + + const rq1: RcvQueueSub = { + userId: 1, connId: bytes(24), server: "smp1.example.com", + rcvId: new Uint8Array([1, 2, 3]), rcvPrivateKey: bytes(32), + status: "new", enableNtfs: true, clientNoticeId: null, + dbQueueId: 1, primary: true, dbReplaceQueueId: null, + } + const rq2: RcvQueueSub = { + ...rq1, rcvId: new Uint8Array([4, 5, 6]), dbQueueId: 2, + connId: new Uint8Array(24).fill(0xaa), + } + const rq1Key = hex(rq1.rcvId) + const rq2Key = hex(rq2.rcvId) + + // addPendingSub + ss.addPendingSub(tSess, rq1) + assert(ss.hasPendingSub(tSess, rq1Key), "addPendingSub: hasPendingSub") + assert(!ss.hasActiveSub(tSess, rq1Key), "addPendingSub: not active") + assert(ss.hasPendingSubs(tSess), "hasPendingSubs") + + // setSessionId + ss.setSessionId(tSess, sessId1) + const sessSubs = ss.sessionSubs.get(tSess)! + assertEq(hex(sessSubs.sessId!), hex(sessId1), "setSessionId sets sessId") + + // addActiveSub with matching sessId: moves from pending to active + ss.addActiveSub(tSess, sessId1, rq1) + assert(ss.hasActiveSub(tSess, rq1Key), "addActiveSub: now active") + assert(!ss.hasPendingSub(tSess, rq1Key), "addActiveSub: no longer pending") + + // addActiveSub with wrong sessId: goes to pending + ss.addActiveSub(tSess, sessId2, rq2) + assert(!ss.hasActiveSub(tSess, rq2Key), "wrong sessId: not active") + assert(ss.hasPendingSub(tSess, rq2Key), "wrong sessId: goes to pending") + + // batchAddActiveSubs + ss.batchAddActiveSubs(tSess, sessId1, [rq2]) + assert(ss.hasActiveSub(tSess, rq2Key), "batchAddActiveSubs: now active") + assert(!ss.hasPendingSub(tSess, rq2Key), "batchAddActiveSubs: removed from pending") + + // getPendingSubs / getActiveSubs + assertEq(ss.getActiveSubs(tSess).size, 2, "getActiveSubs: 2 active") + assertEq(ss.getPendingSubs(tSess).size, 0, "getPendingSubs: 0 pending") + + // setSubsPending: moves active to pending + const moved = ss.setSubsPending(tSess, sessId1) + assertEq(moved.size, 2, "setSubsPending: returned 2 moved subs") + assertEq(ss.getActiveSubs(tSess).size, 0, "after setSubsPending: 0 active") + assertEq(ss.getPendingSubs(tSess).size, 2, "after setSubsPending: 2 pending") + assertEq(sessSubs.sessId, null, "after setSubsPending: sessId cleared") + + // setSubsPending with wrong sessId: no-op + ss.setSessionId(tSess, sessId1) + ss.batchAddActiveSubs(tSess, sessId1, [rq1, rq2]) + const moved2 = ss.setSubsPending(tSess, sessId2) + assertEq(moved2.size, 0, "setSubsPending wrong sessId: no-op") + assertEq(ss.getActiveSubs(tSess).size, 2, "still 2 active") + + // deleteSub + ss.deleteSub(tSess, rq1Key) + assert(!ss.hasActiveSub(tSess, rq1Key), "deleteSub: removed from active") + assert(!ss.hasPendingSub(tSess, rq1Key), "deleteSub: removed from pending") + assertEq(ss.getActiveSubs(tSess).size, 1, "1 active remains") + + // batchDeleteSubs + ss.batchDeleteSubs(tSess, [rq2Key]) + assertEq(ss.getActiveSubs(tSess).size, 0, "batchDeleteSubs: 0 active") + + // batchAddPendingSubs + ss.batchAddPendingSubs(tSess, [rq1, rq2]) + assertEq(ss.getPendingSubs(tSess).size, 2, "batchAddPendingSubs: 2 pending") + + // deletePendingSub + ss.deletePendingSub(tSess, rq1Key) + assertEq(ss.getPendingSubs(tSess).size, 1, "deletePendingSub: 1 pending") + + // batchDeletePendingSubs + ss.batchDeletePendingSubs(tSess, new Set([rq2Key])) + assertEq(ss.getPendingSubs(tSess).size, 0, "batchDeletePendingSubs: 0 pending") + + // setSessionId with change: moves active to pending + ss.setSessionId(tSess, sessId1) + ss.batchAddActiveSubs(tSess, sessId1, [rq1]) + ss.setSessionId(tSess, sessId2) // different sessId → moves active to pending + assert(!ss.hasActiveSub(tSess, rq1Key), "sessId change: not active") + assert(ss.hasPendingSub(tSess, rq1Key), "sessId change: moved to pending") + + // clear + ss.clear() + assertEq(ss.sessionSubs.size, 0, "clear: empty") + + // foldSessionSubs + ss.addPendingSub("a", rq1) + ss.addPendingSub("b", rq2) + const count = ss.foldSessionSubs((acc, _) => acc + 1, 0) + assertEq(count, 2, "foldSessionSubs: 2 sessions") + + // mapSubs + const s = ss.sessionSubs.get("a")! + const [activeCount, pendingCount] = ss.mapSubs(m => m.size, s) + assertEq(activeCount, 0, "mapSubs: 0 active") + assertEq(pendingCount, 1, "mapSubs: 1 pending") +} + +// -- Worker tests + +import { + newAgentClient, newWorker, waitForWork, noWorkToDo, hasWorkToDo, hasWorkToDo_, + getAgentWorker, withWork, withConnLock, getNextServer, throwWhenInactive, + beginAgentOperation, endAgentOperation, defaultAgentConfig, AgentError, + type AgentClient as AC, type Worker as W, +} from "../dist/agent/client.js" + +async function testWorker() { + console.log(" Worker...") + + const store = null as any // workers don't use store directly + const c = newAgentClient(defaultAgentConfig, store, new Map()) + + // newWorker starts with doWork full (has work) + const w = newWorker(c) + assertEq(w.doWork.isEmpty(), false, "newWorker: doWork has work") + assertEq(w.action.tryRead(), null, "newWorker: action is null (not running)") + assertEq(w.workerId, 0, "newWorker: first id is 0") + + // waitForWork / noWorkToDo / hasWorkToDo cycle + await waitForWork(w.doWork) // should resolve immediately (doWork is full) + noWorkToDo(w.doWork) + assertEq(w.doWork.isEmpty(), true, "noWorkToDo: cleared") + hasWorkToDo(w) + assertEq(w.doWork.isEmpty(), false, "hasWorkToDo: signaled") + // double signal is idempotent + hasWorkToDo(w) + assertEq(w.doWork.isEmpty(), false, "double hasWorkToDo: still signaled") + + // workerSeq increments + const w2 = newWorker(c) + assertEq(w2.workerId, 1, "second worker id is 1") +} + +async function testWithWork() { + console.log(" withWork...") + + const store = null as any + const c = newAgentClient(defaultAgentConfig, store, new Map()) + + const doWork = TMVar.new(undefined) + let actionRan = false + + // withWork: clear signal, get work, if found re-signal and run action + await withWork(c, doWork, async () => "item", async (item) => { + assertEq(item, "item", "withWork action receives item") + actionRan = true + }) + assert(actionRan, "withWork: action ran") + assertEq(doWork.isEmpty(), false, "withWork: re-signaled after finding work") + + // withWork: no work — action doesn't run, signal stays cleared + let actionRan2 = false + hasWorkToDo_(doWork) + await withWork(c, doWork, async () => null, async () => { actionRan2 = true }) + assert(!actionRan2, "withWork: action did not run (no work)") + assertEq(doWork.isEmpty(), true, "withWork: signal cleared when no work") +} + +async function testLocking() { + console.log(" Locking...") + + const store = null as any + const c = newAgentClient(defaultAgentConfig, store, new Map()) + const connId = bytes(24) + + // withConnLock serializes + const order: number[] = [] + const delay = (ms: number) => new Promise(r => setTimeout(r, ms)) + + const p1 = withConnLock(c, connId, async () => { + order.push(1) + await delay(20) + order.push(2) + }) + const p2 = withConnLock(c, connId, async () => { + order.push(3) + }) + await Promise.all([p1, p2]) + assertEq(JSON.stringify(order), JSON.stringify([1, 2, 3]), "withConnLock serializes: 1,2,3") +} + +async function testServerSelection() { + console.log(" Server selection...") + + const store = null as any + const srv1: any = {server: "smp1.example.com:5223", auth: null} + const srv2: any = {server: "smp2.example.com:5223", auth: null} + const srv3: any = {server: "smp3.example.com:5223", auth: null} + const us: any = { + storageSrvs: [[null, srv1], [null, srv2], [null, srv3]], + proxySrvs: [[null, srv1]], + knownHosts: new Set(["smp1.example.com", "smp2.example.com", "smp3.example.com"]), + } + const servers = new Map([[1, us]]) + const c = newAgentClient(defaultAgentConfig, store, servers) + // Force deterministic selection + c.randomServer = {gen: () => 0} + + // getNextServer avoids used servers + const s = getNextServer(c, 1, u => u.storageSrvs, ["smp1.example.com:5223"]) + assert(s.server !== "smp1.example.com:5223", "getNextServer avoids used server") + + // getNextServer with all used: falls back to any + const s2 = getNextServer(c, 1, u => u.storageSrvs, ["smp1.example.com:5223", "smp2.example.com:5223", "smp3.example.com:5223"]) + assert(s2 !== undefined, "getNextServer with all used: returns something") + + // getNextServer with unknown userId throws + let threw = false + try { getNextServer(c, 999, u => u.storageSrvs, []) } catch (e) { + if (e instanceof AgentError) threw = true + } + assert(threw, "getNextServer unknown userId throws") +} + +async function testOperationState() { + console.log(" Operation state...") + + const store = null as any + const c = newAgentClient(defaultAgentConfig, store, new Map()) + + beginAgentOperation(c, "AOSndNetwork") + assertEq(c.sndNetworkOp.opsInProgress, 1, "beginAgentOperation: incremented") + beginAgentOperation(c, "AOSndNetwork") + assertEq(c.sndNetworkOp.opsInProgress, 2, "beginAgentOperation: incremented again") + endAgentOperation(c, "AOSndNetwork") + assertEq(c.sndNetworkOp.opsInProgress, 1, "endAgentOperation: decremented") + endAgentOperation(c, "AOSndNetwork") + assertEq(c.sndNetworkOp.opsInProgress, 0, "endAgentOperation: zero") + endAgentOperation(c, "AOSndNetwork") + assertEq(c.sndNetworkOp.opsInProgress, 0, "endAgentOperation: clamped at 0") + + // throwWhenInactive + c.active = false + let threw = false + try { throwWhenInactive(c) } catch { threw = true } + assert(threw, "throwWhenInactive throws when inactive") + c.active = true + throwWhenInactive(c) // should not throw +} + +// -- Run all + +async function main() { + console.log("Infrastructure tests") + await testTMVar() + await testSem() + await testABQueue() + await testSessionVar() + await testRetryInterval() + await testTSessionSubs() + await testWorker() + await testWithWork() + await testLocking() + await testServerSelection() + await testOperationState() + console.log(`\n${passed} passed, ${failed} failed`) + if (failed > 0) process.exit(1) +} + +main().catch(e => { console.error("FATAL:", e?.message || e, e?.stack); process.exit(1) })