Sync iaser-ai/ansari develop (c3f32fc1) into MachineWisdomAI develop

.github/workflows/ci.yml

@@ -177,6 +177,33 @@ jobs:
       - name: Build
         run: pnpm exec turbo run build --filter ansari-frontend...
 
+  # Python WhatsApp service (spec 321). Not part of the pnpm/turbo graph, so it
+  # gets its own job. conftest.py supplies mock-mode defaults, so no env or
+  # repository secrets are needed.
+  whatsapp:
+    name: whatsapp (lint, format, test)
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    defaults:
+      run:
+        working-directory: legacy/whatsapp
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+
+      - uses: astral-sh/setup-uv@07a047e608db84b3d8c06cd9a7c7dcc9c355c18e # v10.3.2
+
+      - name: Install
+        run: uv sync --frozen --extra dev
+
+      - name: Lint
+        run: uv run ruff check .
+
+      - name: Format check
+        run: uv run ruff format --check .
+
+      - name: Test
+        run: uv run pytest tests/ -v
+
   gitleaks:
     name: gitleaks (secret scan)
     runs-on: ubuntu-latest

.gitleaks.toml

@@ -19,4 +19,12 @@ regexes = [
   # check while the test drives sign-up/sign-in against pglite. Literal string,
   # not a credential; trips generic-api-key purely on length + the "secret" key.
   '''test-secret-at-least-32-characters-long-000''',
+  # Spec 329 probe scripts, in branch history only: packages/auth/scripts/probes/probe-db.ts
+  # (commit 90aff8c) and packages/auth/scripts/probes/exchange-lock-probe.ts (commit 427f8fd).
+  # Synthetic BETTER_AUTH_SECRET and RESEND_API_KEY values, set on process.env so createAuth()
+  # passes its zod checks for probes run against a throwaway local Postgres. Literal strings,
+  # not credentials; adjudicated by the security architect, spec 329. Exact full literals,
+  # never a prefix or a pattern. The tip uses setAuthEnv's placeholders instead.
+  '''probe-secret-at-least-32-characters-long-00''',
+  '''re_probe_key_at_least_20_chars''',
 ]

CONTRIBUTING.md

@@ -128,9 +128,10 @@ You can still target one package directly — `pnpm api lint`, `pnpm frontend we
 - **Branch from `develop` and target `develop`** — not `main`. `main` is
   production and is only ever fast-forwarded to `develop` during a release (see
   [RELEASE.md](RELEASE.md)), so PRs against it can't be merged.
-- CI must be green before a PR can merge. `develop` requires all four checks:
+- CI must be green before a PR can merge. `develop` requires all five checks:
   `api (lint, typecheck, test, build)`, `auth (lint, typecheck, test, build)`,
-  `frontend (lint, typecheck)`, and `gitleaks (secret scan)`. The
+  `frontend (lint, typecheck)`, `whatsapp (lint, format, test)`, and
+  `gitleaks (secret scan)`. The
   [Checks](#checks) above are the same tasks, so passing them locally is a good
   predictor. If the secret scan fails, see [Secret scanning](#secret-scanning).
 - Stage files explicitly (`git add path/to/file`). Please don't use `git add -A`

RELEASE.md

@@ -29,15 +29,16 @@ migration runbook were both proven in real releases on 2026-08-02.
   > either half on its own.
 > ### Railway service configuration (spec 48 — monorepo layout)
 >
-> Both services build with the **repo root** as their Root Directory: the pnpm
-> workspace needs the root `pnpm-lock.yaml`, and both Dockerfiles now also
+> The services build with the **repo root** as their Root Directory: the pnpm
+> workspace needs the root `pnpm-lock.yaml`, and the TypeScript Dockerfiles now also
 > `COPY packages packages`. Do **not** set Root Directory to the app folder.
 >
 > | Service | Dockerfile path |
 > |---|---|
 > | backend | `apps/api/Dockerfile` |
 > | frontend (web) | `apps/frontend/Dockerfile.web` |
 > | auth | `apps/auth/Dockerfile` |
+> | whatsapp | `legacy/whatsapp/Dockerfile` |
 >
 > **Watch paths** — set these per service, or Railway rebuilds *both* on every push
 > (any commit touches the root lockfile):
@@ -46,17 +47,33 @@ migration runbook were both proven in real releases on 2026-08-02.
 > backend:   apps/api/**       packages/**  package.json  pnpm-lock.yaml  pnpm-workspace.yaml
 > frontend:  apps/frontend/**  packages/**  package.json  pnpm-lock.yaml  pnpm-workspace.yaml
 > auth:      apps/auth/**      packages/**  package.json  pnpm-lock.yaml  pnpm-workspace.yaml
+> whatsapp:  legacy/whatsapp/**
 > ```
 >
-> `packages/**` is **not optional**. The shared config packages are copied into both
+> For backend, frontend and auth, `packages/**` is **not optional**. The shared config packages are copied into those
 > images, so a packages-only change alters the built artifact. Omit the pattern and
 > that change ships nothing — no rebuild, no deploy, and no error. The image simply
-> drifts from the repo.
+> drifts from the repo. **whatsapp is the exception:** it is a Python service outside the pnpm workspace, so its image contains nothing from
+> `packages/`, the root lockfile or `package.json`; watching them would only rebuild it for changes it does not ship.
 >
 > **Deploy settings** — backend healthcheck `/api/health` (timeout 300s), frontend
-> healthcheck `/`, auth healthcheck `/health` (200 only when the auth database
+> healthcheck `/`, whatsapp healthcheck `/`, auth healthcheck `/health` (200 only when the auth database
 > answers; timeout 300s); restart ON_FAILURE, max 3 retries.
 >
+>
+> **whatsapp service (spec 321, interim Python import; lives in `legacy/` because `apps/*` is reserved for the
+> TypeScript port)** — one listener on `PORT` (8001 by default; the public domain's target port must match).
+> Build: repo-root context, `legacy/whatsapp/Dockerfile`. Health check `/`. The Meta callback URL is
+> `https://<domain>/whatsapp/v2`, one Meta app per environment. Environment surface (full list in
+> `legacy/whatsapp/.env.example`):
+> `DEPLOYMENT_TYPE` (required, `staging` or `production`), `BACKEND_SERVER_URL` (apps/api),
+> `AUTH_SERVER_URL` and `AUTH_WHATSAPP_API_KEY` (apps/auth, issue #329),
+> `META_BUSINESS_PHONE_NUMBER_ID`, `META_ACCESS_TOKEN_FROM_SYS_USER`, `META_ANSARI_APP_SECRET` and
+> `META_WEBHOOK_VERIFY_TOKEN` (all four of the same Meta app, all required),
+> `SENTRY_DSN`, `WA_MAINTENANCE_MODE`, `WHATSAPP_CHAT_RETENTION_HOURS`, `LOGGING_LEVEL` (unset = `INFO`).
+> The service refuses to start if a secret above is missing, or if any `MOCK_*` switch is set outside
+> `DEPLOYMENT_TYPE=local`.
+>
 > **auth service (spec 300)** — one listener: the public domain
 > (`auth.askansari.ai`, `auth-staging.askansari.ai`) targets `PORT` (3100); set it as
 > a service variable. `backend` reaches it at that domain too (`AUTH_URL`), like any
@@ -283,7 +300,7 @@ Order proven for migration `0003` on 2026-08-02:
      steps: `grant-admin.ts <email>` shows the account (id, email, created_at,
      email_verified, role) and grants nothing; `grant-admin.ts <email> --confirm
      <id>` grants exactly that account. It never creates an account or touches a
-     password, and it refuses the system domain. A new admin signs up first; a
+     password, and it never grants to a system- or noreply-domain address (`--revoke` demotes any account). A new admin signs up first; a
      warning on a recent or unverified account means: confirm the person out of
      band before `--confirm`. There is no `ADMIN_EMAILS` on `apps/auth` (spec
      H-7): the role in the auth database is the only admin truth.

apps/api/lib/sentry-scrub.ts

@@ -0,0 +1,89 @@
+import type { Event, NodeOptions } from "@sentry/nextjs";
+
+// The client IP travels in x-real-ip (Railway's edge) and x-forwarded-for;
+// `sendDefaultPii: false` means Sentry should not receive it. The SDK drops
+// these itself only while `userInfo` is off, which Sentry v11 no longer defaults.
+const DROPPED_HEADERS = new Set(["authorization", "cookie", "x-real-ip", "x-forwarded-for"]);
+// Span attributes that carry the same data. @sentry/nextjs copies headers onto
+// the root span (`http.request.header.cookie.<name>`, dashes as underscores),
+// masking only sensitive-looking names; Next sets `http.target` to the request
+// URL with its query; an outgoing fetch's span carries its query string (the
+// search tools put the user's question there); requestdata writes the URL,
+// query and body as attributes when spans stream; the client IP has an
+// attribute of its own.
+const DROPPED_HEADER_ATTRIBUTE =
+  /^http\.(request\.header\.(authorization|cookie|x_real_ip|x_forwarded_for)|response\.header\.set_cookie)(\.|$)/;
+const DROPPED_ATTRIBUTES = new Set([
+  "url.full",
+  "url.query",
+  "http.query",
+  "http.url",
+  "http.target",
+  "http.request.body.data",
+  "http.client_ip",
+  "client.address",
+]);
+
+function dropAttributes(data: Record<string, unknown> | undefined): void {
+  if (!data) return;
+  for (const key of Object.keys(data)) {
+    if (DROPPED_ATTRIBUTES.has(key) || DROPPED_HEADER_ATTRIBUTE.test(key)) delete data[key];
+  }
+}
+
+/**
+ * Sentry's request-data integration attaches the in-flight request to every
+ * event raised while it is handled. Strip what can carry user content or a
+ * credential: the body (user messages), the query string (`/v2/mcp-complete?q=`
+ * is the user's question), the Authorization header (the user's JWT),
+ * cookies and the client IP (its headers, its attribute and
+ * `user.ip_address`), from the request and from span attributes (issue #333).
+ * Node lowercases incoming header names, but a header object built elsewhere
+ * may not, so names are matched case-blind.
+ */
+export function scrubRequestEvent<E extends Event>(event: E): E {
+  const request = event.request;
+  if (request) {
+    delete request.data;
+    delete request.cookies;
+    delete request.query_string;
+    if (request.url) request.url = request.url.split("?")[0];
+    if (request.headers) {
+      for (const name of Object.keys(request.headers)) {
+        if (DROPPED_HEADERS.has(name.toLowerCase())) delete request.headers[name];
+      }
+    }
+  }
+  // captureRequestError records the request path with its query.
+  const nextjs = event.contexts?.nextjs;
+  if (typeof nextjs?.request_path === "string") nextjs.request_path = nextjs.request_path.split("?")[0];
+  if (event.user) delete event.user.ip_address;
+  dropAttributes(event.contexts?.trace?.data);
+  for (const span of event.spans ?? []) dropAttributes(span.data);
+  return event;
+}
+
+/** The same attribute scrub for each span sent on its own. */
+export function scrubSpan<S extends { data?: Record<string, unknown> }>(span: S): S {
+  dropAttributes(span.data);
+  return span;
+}
+
+/**
+ * The server's request-privacy options; tests spread exactly these. Transactions
+ * and spans carry the request too. Breadcrumbs are off: an HTTP breadcrumb
+ * records an outgoing request's query string, and the search tools put the
+ * user's question there. `beforeSendSpan` covers the static span lifecycle
+ * only; under `traceLifecycle: 'stream'` it needs `withStreamedSpan` and the
+ * streamed `attributes` format (a test pins the lifecycle). Never set
+ * `dataCollection` partially here: any value switches every category it omits
+ * to collect-all (user info, bodies, GenAI inputs and outputs, database query
+ * data).
+ */
+export const SENTRY_REQUEST_PRIVACY = {
+  maxBreadcrumbs: 0,
+  beforeBreadcrumb: () => null,
+  beforeSend: scrubRequestEvent,
+  beforeSendTransaction: scrubRequestEvent,
+  beforeSendSpan: scrubSpan,
+} satisfies NodeOptions;

apps/api/sentry.server.config.ts

@@ -1,16 +1,11 @@
 import * as Sentry from "@sentry/nextjs";
 
+import { SENTRY_REQUEST_PRIVACY } from "./lib/sentry-scrub";
+
 Sentry.init({
   dsn: process.env.SENTRY_DSN,
   enabled: process.env.NODE_ENV !== "test" && !process.env.VITEST,
   environment: process.env.RAILWAY_ENVIRONMENT || process.env.NODE_ENV,
   tracesSampleRate: process.env.NODE_ENV === "production" ? 0.1 : 1.0,
-
-  beforeSend(event) {
-    // Strip request body data to prevent user messages from leaking to Sentry
-    if (event.request) {
-      delete event.request.data;
-    }
-    return event;
-  },
+  ...SENTRY_REQUEST_PRIVACY,
 });

apps/api/tests/auth/resolve-user.test.ts

@@ -243,6 +243,22 @@ describe('scenario 9f: an email the app database already holds', () => {
     }
   });
 
+  // Spec 329: the first-use refusal covers the system domain only. A WhatsApp
+  // user's profile carries apps/auth's other reserved domain, so a tightening
+  // that refused every reserved address would break WhatsApp here first.
+  it('a whatsapp-<id>@noreply.askansari.ai profile is accepted, and its row created', async () => {
+    const { id, jwt } = await signUpDirect(nextEmail('whatsapp'));
+    const email = `whatsapp-${id}@noreply.askansari.ai`;
+    await service.client.query(
+      "update auth.users set email = $1, registered_via = 'whatsapp' where id = $2",
+      [email, id]
+    );
+
+    expect((await resolveUser(id, jwt)).kind).toBe('user');
+    expect(await appUser(id)).toMatchObject({ id, email, registered_via: 'whatsapp' });
+    expect(h.sentry).toEqual([]);
+  });
+
   it('the email is compared normalized', async () => {
     const email = nextEmail('case');
     const legacy = await insertLegacyUserRow(app, email);