From dd6ca24a874969023f4d839725e8e21054fcff08 Mon Sep 17 00:00:00 2001 From: badnikhil Date: Sun, 9 Aug 2026 18:17:45 +0530 Subject: [PATCH 1/4] add local mqtt test broker and test-topic publisher --- mqtt/Dockerfile | 19 +++ mqtt/docker-compose.yml | 45 +++++++ mqtt/mosquitto/mosquitto.conf | 56 +++++++++ mqtt/publisher.py | 214 ++++++++++++++++++++++++++++++++++ mqtt/requirements.txt | 4 + 5 files changed, 338 insertions(+) create mode 100644 mqtt/Dockerfile create mode 100644 mqtt/docker-compose.yml create mode 100644 mqtt/mosquitto/mosquitto.conf create mode 100644 mqtt/publisher.py create mode 100644 mqtt/requirements.txt diff --git a/mqtt/Dockerfile b/mqtt/Dockerfile new file mode 100644 index 0000000..49df666 --- /dev/null +++ b/mqtt/Dockerfile @@ -0,0 +1,19 @@ +# Publisher image: a tiny paho-mqtt client that generates deterministic MQTT +# test traffic against the broker. +# +# Build context is the mqtt/ directory (see docker-compose.yml). +FROM python:3.12-slim + +WORKDIR /app + +# Install the MQTT client library first (better layer caching). +COPY requirements.txt ./ +RUN pip install --no-cache-dir -r requirements.txt + +# The publisher script. +COPY publisher.py ./ + +# Unbuffered stdout so `docker compose logs publisher` streams in real time. +ENV PYTHONUNBUFFERED=1 + +CMD ["python", "publisher.py"] diff --git a/mqtt/docker-compose.yml b/mqtt/docker-compose.yml new file mode 100644 index 0000000..66df44c --- /dev/null +++ b/mqtt/docker-compose.yml @@ -0,0 +1,45 @@ +# Local MQTT test rig for API Dash's MQTT client. +# +# docker compose -f mqtt/docker-compose.yml up +# +# Brings up: +# - broker : Eclipse Mosquitto 2.x (MQTT v3.1 / v3.1.1 / v5) +# mqtt://localhost:1883 and ws://localhost:9001 +# - publisher : a small paho-mqtt client that produces deterministic, +# subscribable test traffic (ticker / retained / echo / LWT). +# +# Nothing here touches the FastAPI app -- MQTT is not an HTTP route. +# +# Paths below are relative to this file's directory (mqtt/), so the command +# above works from the repository root. + +services: + broker: + image: eclipse-mosquitto:2 + container_name: apidash-mqtt-broker + ports: + - "1883:1883" # MQTT over TCP + - "9001:9001" # MQTT over WebSocket + volumes: + - ./mosquitto/mosquitto.conf:/mosquitto/config/mosquitto.conf:ro + - mosquitto-data:/mosquitto/data + - mosquitto-log:/mosquitto/log + restart: unless-stopped + + publisher: + build: + context: . + dockerfile: Dockerfile + container_name: apidash-mqtt-publisher + depends_on: + - broker + environment: + # The publisher reaches the broker over the compose network by service + # name. (From your host, the same broker is at localhost:1883.) + MQTT_HOST: broker + MQTT_PORT: "1883" + restart: unless-stopped + +volumes: + mosquitto-data: + mosquitto-log: diff --git a/mqtt/mosquitto/mosquitto.conf b/mqtt/mosquitto/mosquitto.conf new file mode 100644 index 0000000..50f033f --- /dev/null +++ b/mqtt/mosquitto/mosquitto.conf @@ -0,0 +1,56 @@ +# Mosquitto configuration for API Dash local MQTT testing. +# +# This config is intentionally frictionless for LOCAL testing: +# - anonymous access is allowed (no username/password needed) +# - persistence is on, so retained messages + persistent sessions +# survive a broker restart +# +# NOTE: Mosquitto 2.x speaks MQTT v3.1, v3.1.1 AND v5 out of the box -- +# no extra configuration is required to enable MQTT v5. + +# --------------------------------------------------------------------------- +# Listeners +# --------------------------------------------------------------------------- + +# Plain MQTT over TCP. Point API Dash's MQTT client at mqtt://localhost:1883 +listener 1883 +protocol mqtt + +# MQTT over WebSocket. Browser / web clients connect at ws://localhost:9001 +listener 9001 +protocol websockets + +# --------------------------------------------------------------------------- +# Authentication +# --------------------------------------------------------------------------- + +# Allow clients to connect without a username/password. This keeps local +# testing frictionless. Set this to `false` (and configure `password_file` +# below) to exercise API Dash's username/password auth. +allow_anonymous true + +# --- To test username/password auth: --------------------------------------- +# 1. Create a password file (run inside the broker container, or in a +# mounted directory the broker can read): +# mosquitto_passwd -c -b /mosquitto/config/passwd testuser testpass +# 2. Uncomment the two lines below. +# 3. Set `allow_anonymous false` (comment out / change the line above). +# 4. Recreate the broker: +# docker compose -f mqtt/docker-compose.yml up -d --force-recreate broker +# +# password_file /mosquitto/config/passwd +# allow_anonymous false + +# --------------------------------------------------------------------------- +# Persistence (retained messages + persistent sessions survive a restart) +# --------------------------------------------------------------------------- +persistence true +persistence_location /mosquitto/data/ + +# --------------------------------------------------------------------------- +# Logging (to stdout so `docker compose logs broker` shows connect/pub/sub) +# --------------------------------------------------------------------------- +log_dest stdout +log_type all +connection_messages true +log_timestamp true diff --git a/mqtt/publisher.py b/mqtt/publisher.py new file mode 100644 index 0000000..a067025 --- /dev/null +++ b/mqtt/publisher.py @@ -0,0 +1,214 @@ +#!/usr/bin/env python3 +""" +Deterministic MQTT test publisher for API Dash's MQTT client. + +This is the MQTT analogue of the WebSocket echo / ticker / heartbeat test +routes (see docs/ws/*). It connects to a real Mosquitto broker and produces +predictable, subscribable traffic so an MQTT client (such as API Dash) can be +exercised end to end -- QoS, retained messages, wildcards, Last Will and +MQTT v5 request/response properties. + +Scenarios (every topic lives under the `apidash/test/` prefix): + + ticker -> publishes JSON {"seq": n, "ts": } to + apidash/test/ticker every 2s (QoS 0). + retained -> publishes a RETAINED message to apidash/test/retained at + startup, so a client that subscribes LATER still receives it + immediately. + echo -> subscribes apidash/test/echo/request; each message received is + republished to apidash/test/echo/response. If the incoming + MQTT v5 PUBLISH carries a `response_topic` property, the reply + is sent there instead, echoing back any `correlation_data`. + lwt -> registers a Last Will on apidash/test/status = "offline" + (retained), and publishes "online" (retained) on connect. + Kill this process ungracefully (e.g. + `docker compose -f mqtt/docker-compose.yml kill publisher`) + to watch the broker deliver the Will. + +Configuration (environment variables): + MQTT_HOST broker hostname (default: localhost) + MQTT_PORT broker TCP port (default: 1883) + MQTT_USER username, optional + MQTT_PASS password, optional + MQTT_CLIENT_ID client id (default: apidash-test-publisher) + +Everything here is intentionally simple and heavily commented -- it is a +testing fixture, not production code. +""" + +import json +import os +import signal +import threading +import time +from datetime import datetime, timezone + +import paho.mqtt.client as mqtt +from paho.mqtt.packettypes import PacketTypes +from paho.mqtt.properties import Properties + +# --- Topic names (single source of truth; mirrored in docs/mqtt/*.md) -------- +TOPIC_PREFIX = "apidash/test" +TICKER_TOPIC = f"{TOPIC_PREFIX}/ticker" +RETAINED_TOPIC = f"{TOPIC_PREFIX}/retained" +ECHO_REQUEST_TOPIC = f"{TOPIC_PREFIX}/echo/request" +ECHO_RESPONSE_TOPIC = f"{TOPIC_PREFIX}/echo/response" +STATUS_TOPIC = f"{TOPIC_PREFIX}/status" + +TICKER_INTERVAL_SECONDS = 2 + +# --- Configuration from the environment ------------------------------------- +MQTT_HOST = os.environ.get("MQTT_HOST", "localhost") +MQTT_PORT = int(os.environ.get("MQTT_PORT", "1883")) +MQTT_USER = os.environ.get("MQTT_USER") +MQTT_PASS = os.environ.get("MQTT_PASS") +CLIENT_ID = os.environ.get("MQTT_CLIENT_ID", "apidash-test-publisher") + + +def log(msg): + """Timestamped stdout logging (see `docker compose logs publisher`).""" + print(f"[{datetime.now(timezone.utc).isoformat()}] {msg}", flush=True) + + +def now_iso(): + return datetime.now(timezone.utc).isoformat() + + +def build_client(): + """Create an MQTT v5 client using the modern (v2) callback API when + available, falling back to the paho-mqtt 1.x constructor otherwise.""" + try: + return mqtt.Client( + callback_api_version=mqtt.CallbackAPIVersion.VERSION2, + client_id=CLIENT_ID, + protocol=mqtt.MQTTv5, + ) + except AttributeError: + # paho-mqtt 1.x has no CallbackAPIVersion enum. + return mqtt.Client(client_id=CLIENT_ID, protocol=mqtt.MQTTv5) + + +# --- Callbacks -------------------------------------------------------------- +def on_connect(client, userdata, flags, reason_code, properties=None): + """Runs on every (re)connect -- so it is safe to re-publish + re-subscribe.""" + log(f"connected to {MQTT_HOST}:{MQTT_PORT} (reason_code={reason_code})") + + # LWT counterpart: announce we are online (retained). A late subscriber to + # apidash/test/status sees "online" until we disconnect ungracefully. + client.publish(STATUS_TOPIC, payload="online", qos=1, retain=True) + log(f"published retained '{STATUS_TOPIC}' = 'online'") + + # Retained scenario: any FUTURE subscriber gets this immediately. + retained_payload = json.dumps( + { + "note": ( + "This is a retained message. A client that subscribes AFTER " + "it was published still receives it immediately on subscribe." + ), + "ts": now_iso(), + } + ) + client.publish(RETAINED_TOPIC, payload=retained_payload, qos=1, retain=True) + log(f"published retained '{RETAINED_TOPIC}'") + + # Echo scenario: listen for requests (QoS 1 so the broker can redeliver). + client.subscribe(ECHO_REQUEST_TOPIC, qos=1) + log(f"subscribed '{ECHO_REQUEST_TOPIC}' (qos=1)") + + +def on_disconnect(client, userdata, *args): + # Signature differs across paho versions, so accept whatever is passed. + log(f"disconnected {args}; paho will auto-reconnect") + + +def on_message(client, userdata, msg): + """Echo handler: mirror each request onto the response topic.""" + payload = msg.payload + log(f"echo request on '{msg.topic}' ({len(payload)} bytes)") + + # MQTT v5 request/response: honour `response_topic` + `correlation_data`. + response_topic = ECHO_RESPONSE_TOPIC + reply_props = None + incoming = getattr(msg, "properties", None) + if incoming is not None: + response_topic = getattr(incoming, "ResponseTopic", None) or ECHO_RESPONSE_TOPIC + correlation = getattr(incoming, "CorrelationData", None) + if correlation is not None: + reply_props = Properties(PacketTypes.PUBLISH) + reply_props.CorrelationData = correlation + + client.publish(response_topic, payload=payload, qos=1, properties=reply_props) + log(f"echo reply -> '{response_topic}'") + + +# --- Ticker thread ---------------------------------------------------------- +def ticker_loop(client, stop_event): + seq = 0 + while not stop_event.is_set(): + seq += 1 + payload = json.dumps({"seq": seq, "ts": now_iso()}) + # QoS 0 -- fire and forget, like the WebSocket ticker route. + client.publish(TICKER_TOPIC, payload=payload, qos=0) + log(f"ticker -> '{TICKER_TOPIC}' seq={seq}") + stop_event.wait(TICKER_INTERVAL_SECONDS) + + +def main(): + client = build_client() + + if MQTT_USER: + client.username_pw_set(MQTT_USER, MQTT_PASS) + + # Last Will and Testament: if we drop off ungracefully, the broker + # publishes this retained message on our behalf. + client.will_set(STATUS_TOPIC, payload="offline", qos=1, retain=True) + + client.on_connect = on_connect + client.on_disconnect = on_disconnect + client.on_message = on_message + + # Robust reconnect: paho retries with exponential backoff between these + # bounds after any unexpected disconnect. + client.reconnect_delay_set(min_delay=1, max_delay=30) + + # The broker may not be ready yet (docker compose starts it in parallel), + # so retry the INITIAL connect until it succeeds. + while True: + try: + log(f"connecting to {MQTT_HOST}:{MQTT_PORT} ...") + client.connect(MQTT_HOST, MQTT_PORT, keepalive=30) + break + except Exception as exc: # noqa: BLE001 - test fixture: log and retry + log(f"connect failed: {exc!r}; retrying in 2s") + time.sleep(2) + + stop_event = threading.Event() + + def handle_signal(signum, frame): + # NOTE: we deliberately do NOT send a clean DISCONNECT here. Exiting + # without a DISCONNECT packet makes the broker fire the Last Will, so + # stopping/killing this container demonstrates LWT on apidash/test/status. + log(f"signal {signum} received; stopping (Will fires on ungraceful close)") + stop_event.set() + + signal.signal(signal.SIGTERM, handle_signal) + signal.signal(signal.SIGINT, handle_signal) + + # Network loop runs in the background; the ticker runs on its own thread. + stop_event_thread = threading.Thread( + target=ticker_loop, args=(client, stop_event), daemon=True + ) + client.loop_start() + stop_event_thread.start() + + # Block the main thread until we are told to stop. + while not stop_event.is_set(): + time.sleep(0.5) + + stop_event_thread.join(timeout=5) + client.loop_stop() # stop the network thread WITHOUT sending DISCONNECT + log("stopped") + + +if __name__ == "__main__": + main() diff --git a/mqtt/requirements.txt b/mqtt/requirements.txt new file mode 100644 index 0000000..f99d0de --- /dev/null +++ b/mqtt/requirements.txt @@ -0,0 +1,4 @@ +# Dependencies for the MQTT test publisher image (mqtt/Dockerfile). +# Kept separate from the project's requirements.txt -- the FastAPI app does +# not depend on MQTT. +paho-mqtt>=2.0.0 From 27804d852e460cb82a7c655596f76bd1a6b1acac Mon Sep 17 00:00:00 2001 From: badnikhil Date: Sun, 9 Aug 2026 18:17:45 +0530 Subject: [PATCH 2/4] add mqtt local-testing docs --- docs/mqtt/README.md | 80 ++++++++++++++++++++++++++++++++++++++ docs/mqtt/auth.md | 54 +++++++++++++++++++++++++ docs/mqtt/echo.md | 47 ++++++++++++++++++++++ docs/mqtt/lwt.md | 57 +++++++++++++++++++++++++++ docs/mqtt/qos.md | 47 ++++++++++++++++++++++ docs/mqtt/retained.md | 46 ++++++++++++++++++++++ docs/mqtt/ticker.md | 44 +++++++++++++++++++++ docs/mqtt/v5_properties.md | 55 ++++++++++++++++++++++++++ docs/mqtt/websocket.md | 46 ++++++++++++++++++++++ docs/mqtt/wildcards.md | 44 +++++++++++++++++++++ 10 files changed, 520 insertions(+) create mode 100644 docs/mqtt/README.md create mode 100644 docs/mqtt/auth.md create mode 100644 docs/mqtt/echo.md create mode 100644 docs/mqtt/lwt.md create mode 100644 docs/mqtt/qos.md create mode 100644 docs/mqtt/retained.md create mode 100644 docs/mqtt/ticker.md create mode 100644 docs/mqtt/v5_properties.md create mode 100644 docs/mqtt/websocket.md create mode 100644 docs/mqtt/wildcards.md diff --git a/docs/mqtt/README.md b/docs/mqtt/README.md new file mode 100644 index 0000000..21531ff --- /dev/null +++ b/docs/mqtt/README.md @@ -0,0 +1,80 @@ +--- +protocol: mqtt +title: MQTT Test Broker +desc: A one-command local Mosquitto broker plus a deterministic test publisher for exercising API Dash's MQTT client (QoS, retained, wildcards, LWT, MQTT v5). +path: mqtt +--- + +This is a **local** MQTT test rig for API Dash's MQTT client. MQTT is not an +HTTP route, so unlike the WebSocket endpoints it cannot be hosted inside the +FastAPI app. Instead we ship a Dockerized [Eclipse Mosquitto](https://mosquitto.org/) +2.x broker plus a small `paho-mqtt` publisher that generates deterministic, +subscribable traffic -- the MQTT analogue of the WebSocket echo / ticker / +heartbeat routes. + +Running a real, spec-compliant broker is the only way to genuinely test QoS 1/2 +handshakes, retained messages, persistent sessions, Last Will and MQTT v5 +properties. + +## Run it + +From the repository root: + +``` +docker compose -f mqtt/docker-compose.yml up +``` + +This starts two services: + +| Service | What it is | +| ----------- | ----------- | +| `broker` | Eclipse Mosquitto 2.x (MQTT v3.1 / v3.1.1 / v5) | +| `publisher` | A `paho-mqtt` client that emits the test scenarios below | + +Stop it with `Ctrl-C`, or run detached with `-d` and stop via +`docker compose -f mqtt/docker-compose.yml down`. + +## Endpoints + +| Transport | URL | +| ----------- | ----------- | +| MQTT over TCP | `mqtt://localhost:1883` | +| MQTT over WebSocket | `ws://localhost:9001` | + +Anonymous access is allowed by default (no username/password needed). See +[Auth](auth.md) to test username/password. + +## Point API Dash at it + +In API Dash's MQTT client, create a connection to: + +- **Host:** `localhost` +- **Port:** `1883` (TCP) or `9001` (WebSocket) +- **Protocol:** MQTT v3.1.1 or v5 (both work) +- **Auth:** none (anonymous), unless you enabled it + +Then subscribe to `apidash/test/#` to see every test topic at once. + +## Test topics + +| Topic | Scenario | Doc | +| ----------- | ----------- | ----------- | +| `apidash/test/ticker` | JSON message every 2s (QoS 0) | [ticker](ticker.md) | +| `apidash/test/retained` | Retained message, delivered on subscribe | [retained](retained.md) | +| `apidash/test/echo/request` -> `apidash/test/echo/response` | Request/response echo | [echo](echo.md) | +| `apidash/test/status` | `online` / `offline` via Last Will (retained) | [lwt](lwt.md) | + +Further reference pages: [wildcards](wildcards.md), [qos](qos.md), +[auth](auth.md), [MQTT v5 properties](v5_properties.md), +[WebSocket transport](websocket.md). + +## Tests + +`tests/mqtt/test_mqtt.py` exercises the broker with `paho-mqtt`. The tests skip +gracefully if no broker is reachable, so CI without a broker still passes: + +``` +docker compose -f mqtt/docker-compose.yml up -d +pip install -r requirements-dev.txt +pytest tests/mqtt/test_mqtt.py +``` diff --git a/docs/mqtt/auth.md b/docs/mqtt/auth.md new file mode 100644 index 0000000..78c4c62 --- /dev/null +++ b/docs/mqtt/auth.md @@ -0,0 +1,54 @@ +--- +protocol: mqtt +title: MQTT Username/Password Auth +desc: Anonymous by default for frictionless testing; optionally enable a password file to test username/password authentication. +path: mqtt/auth +--- + +By default the broker allows **anonymous** connections so local testing is +frictionless (`allow_anonymous true` in `mqtt/mosquitto/mosquitto.conf`). You +can switch on username/password auth to test API Dash's credential handling. + +## Enable auth + +1. Create a password file inside the broker container: + + ``` + docker compose -f mqtt/docker-compose.yml exec broker \ + mosquitto_passwd -c -b /mosquitto/config/passwd testuser testpass + ``` + +2. In `mqtt/mosquitto/mosquitto.conf`, uncomment the `password_file` line and + set `allow_anonymous false` (a commented-out template block is already + there). + +3. Recreate the broker: + + ``` + docker compose -f mqtt/docker-compose.yml up -d --force-recreate broker + ``` + +The publisher itself reads optional `MQTT_USER` / `MQTT_PASS` env vars, so set +those in `docker-compose.yml` if you enable auth and want the publisher to keep +working. + +## Behavior + +| Credentials | Result | +| ----------- | ----------- | +| `allow_anonymous true` (default) | Any client may connect without credentials | +| `allow_anonymous false` + valid `testuser` / `testpass` | Connection accepted | +| `allow_anonymous false` + missing/wrong credentials | Broker refuses the connection (CONNACK "not authorized") | + +## Sample Usage + +### Python (`paho-mqtt`) + +```python +import paho.mqtt.client as mqtt + +c = mqtt.Client(mqtt.CallbackAPIVersion.VERSION2, protocol=mqtt.MQTTv5) +c.username_pw_set("testuser", "testpass") +c.connect("localhost", 1883) +c.loop_forever() +``` diff --git a/docs/mqtt/echo.md b/docs/mqtt/echo.md new file mode 100644 index 0000000..d42115e --- /dev/null +++ b/docs/mqtt/echo.md @@ -0,0 +1,47 @@ +--- +protocol: mqtt +title: MQTT Echo +desc: Publish to apidash/test/echo/request and the publisher mirrors the payload back on apidash/test/echo/response (honouring the MQTT v5 response topic if present). +path: mqtt/echo +--- + +The publisher subscribes to `apidash/test/echo/request`. Every message it +receives is republished unchanged to `apidash/test/echo/response`. This is the +MQTT analogue of the WebSocket echo route and is useful for testing a full +publish -> subscribe round-trip. + +If the incoming PUBLISH carries an **MQTT v5 `response_topic` property**, the +reply is sent there instead, and any `correlation_data` is echoed back. See +[MQTT v5 properties](v5_properties.md). + +## Topics + +| Topic | Direction | +| ----------- | ----------- | +| `apidash/test/echo/request` | You publish here | +| `apidash/test/echo/response` | Publisher replies here (default) | + +## Behavior + +| Event | Result | +| ----------- | ----------- | +| Publish to `.../echo/request` | Payload is republished to `.../echo/response`, unchanged | +| Request carries v5 `response_topic` | Reply is sent to that topic instead | +| Request carries v5 `correlation_data` | The same `correlation_data` is set on the reply | + +## Sample Usage + +### Python (`paho-mqtt`) + +```python +import paho.mqtt.client as mqtt + +c = mqtt.Client(mqtt.CallbackAPIVersion.VERSION2, protocol=mqtt.MQTTv5) +c.on_message = lambda cl, ud, msg: print("reply:", msg.payload.decode()) +c.connect("localhost", 1883) +c.subscribe("apidash/test/echo/response", qos=1) +c.loop_start() + +c.publish("apidash/test/echo/request", payload="ping", qos=1) +# reply: ping +``` diff --git a/docs/mqtt/lwt.md b/docs/mqtt/lwt.md new file mode 100644 index 0000000..7f20e73 --- /dev/null +++ b/docs/mqtt/lwt.md @@ -0,0 +1,57 @@ +--- +protocol: mqtt +title: MQTT Last Will (LWT) +desc: The publisher registers a Last Will on apidash/test/status; killing it makes the broker publish "offline" automatically. +path: mqtt/lwt +--- + +A **Last Will and Testament (LWT)** is a message the client registers at connect +time; the broker publishes it automatically if the client disconnects +**ungracefully** (crash, network drop, kill). It is how MQTT signals presence. + +The publisher registers a Will on `apidash/test/status` with payload `offline` +(retained), and publishes `online` (retained) once connected. + +## Topic + +``` +apidash/test/status +``` + +## Behavior + +| Event | Value on `apidash/test/status` | +| ----------- | ----------- | +| Publisher connected | `online` (retained) | +| Publisher killed / crashes / loses connection | `offline` (retained, published by the broker as the Will) | + +Because the value is retained, a client that subscribes at any time +immediately sees the current status. + +## Try it + +1. Subscribe to `apidash/test/status` -- you see `online`. +2. Ungracefully stop the publisher so it cannot send a clean disconnect: + + ``` + docker compose -f mqtt/docker-compose.yml kill publisher + ``` + +3. The broker publishes the Will -- you now see `offline`. + +## Sample Usage + +### Python (`paho-mqtt`) + +```python +import paho.mqtt.client as mqtt + +c = mqtt.Client(mqtt.CallbackAPIVersion.VERSION2, protocol=mqtt.MQTTv5) +c.on_message = lambda cl, ud, msg: print("status:", msg.payload.decode()) +c.connect("localhost", 1883) +c.subscribe("apidash/test/status", qos=1) +c.loop_forever() +# status: online +# (after `docker compose -f mqtt/docker-compose.yml kill publisher`) +# status: offline +``` diff --git a/docs/mqtt/qos.md b/docs/mqtt/qos.md new file mode 100644 index 0000000..0024e1c --- /dev/null +++ b/docs/mqtt/qos.md @@ -0,0 +1,47 @@ +--- +protocol: mqtt +title: MQTT Quality of Service (QoS) +desc: Test QoS 0, 1 and 2 delivery guarantees against the broker; the effective QoS is the minimum of the publish and subscribe QoS. +path: mqtt/qos +--- + +MQTT defines three Quality of Service (QoS) levels for message delivery. A real +broker is required to exercise the QoS 1 and QoS 2 handshakes, which is one of +the main reasons this rig ships a Dockerized Mosquitto. + +## Levels + +| QoS | Guarantee | Handshake | +| ----------- | ----------- | ----------- | +| 0 | At most once ("fire and forget") | none | +| 1 | At least once (may duplicate) | PUBLISH / PUBACK | +| 2 | Exactly once | PUBLISH / PUBREC / PUBREL / PUBCOMP | + +The QoS a subscriber actually receives is the **minimum** of the publisher's +QoS and the subscription's QoS. For example, a QoS 2 publish delivered to a QoS +1 subscription is received at QoS 1. + +## Try it with the test topics + +- The [ticker](ticker.md) publishes at **QoS 0** -- subscribe at any QoS. +- The [echo](echo.md) and [retained](retained.md) topics use **QoS 1**. +- To exercise **QoS 2**, publish to your own topic (e.g. + `apidash/test/scratch`) at QoS 2 and subscribe at QoS 2. + +## Sample Usage + +### Python (`paho-mqtt`) + +```python +import paho.mqtt.client as mqtt + +c = mqtt.Client(mqtt.CallbackAPIVersion.VERSION2, protocol=mqtt.MQTTv5) +c.on_message = lambda cl, ud, msg: print("qos", msg.qos, msg.payload.decode()) +c.connect("localhost", 1883) +c.subscribe("apidash/test/scratch", qos=2) +c.loop_start() + +info = c.publish("apidash/test/scratch", payload="exactly once", qos=2) +info.wait_for_publish(5) # completes only after the full QoS 2 handshake +# qos 2 exactly once +``` diff --git a/docs/mqtt/retained.md b/docs/mqtt/retained.md new file mode 100644 index 0000000..1082c45 --- /dev/null +++ b/docs/mqtt/retained.md @@ -0,0 +1,46 @@ +--- +protocol: mqtt +title: MQTT Retained Message +desc: The publisher stores a retained message on apidash/test/retained so any client that subscribes later receives it immediately. +path: mqtt/retained +--- + +At startup (and on every reconnect) the publisher stores a **retained** message +on `apidash/test/retained`. The broker keeps the last retained message for a +topic and delivers it immediately to any client that subscribes **later** -- a +core MQTT feature that has no WebSocket equivalent. + +## Topic + +``` +apidash/test/retained +``` + +## Behavior + +| Event | Result | +| ----------- | ----------- | +| Subscribe (any time) | You immediately receive the retained JSON payload, with the retain flag set | +| Publisher restarts | The retained message is refreshed with a new timestamp | + +The payload is a JSON object with a `note` and a `ts` (ISO-8601) field. + +To clear a retained message on any topic, publish an **empty** payload to it +with the retain flag set. + +## Sample Usage + +### Python (`paho-mqtt`) + +```python +import paho.mqtt.client as mqtt + +c = mqtt.Client(mqtt.CallbackAPIVersion.VERSION2, protocol=mqtt.MQTTv5) +c.on_message = lambda cl, ud, msg: print("retain=%s" % msg.retain, msg.payload.decode()) +c.connect("localhost", 1883) +# Even though this runs long after the publisher started, the message arrives +# immediately on subscribe. +c.subscribe("apidash/test/retained", qos=1) +c.loop_forever() +# retain=True {"note": "This is a retained message. ...", "ts": "..."} +``` diff --git a/docs/mqtt/ticker.md b/docs/mqtt/ticker.md new file mode 100644 index 0000000..fa8c69d --- /dev/null +++ b/docs/mqtt/ticker.md @@ -0,0 +1,44 @@ +--- +protocol: mqtt +title: MQTT Ticker +desc: The publisher emits a JSON ticker message to apidash/test/ticker every 2 seconds at QoS 0. +path: mqtt/ticker +--- + +The test publisher emits a JSON message to `apidash/test/ticker` every **2 +seconds** at **QoS 0**. It is the MQTT analogue of the WebSocket ticker route +and is intended for testing server-push / streaming handling in an MQTT client +such as API Dash. + +## Topic + +``` +apidash/test/ticker +``` + +## Behavior + +| Event | Result | +| ----------- | ----------- | +| Every 2s | Publisher sends `{"seq": , "ts": ""}` at QoS 0 | +| Subscribe | You start receiving ticks from the next interval onward | + +`seq` starts at `1` and increments for the lifetime of the publisher process. +Because this topic is **not** retained, a new subscriber only sees ticks +published after it subscribes. + +## Sample Usage + +### Python (`paho-mqtt`) + +```python +import paho.mqtt.client as mqtt + +c = mqtt.Client(mqtt.CallbackAPIVersion.VERSION2, protocol=mqtt.MQTTv5) +c.on_message = lambda cl, ud, msg: print(msg.topic, msg.payload.decode()) +c.connect("localhost", 1883) +c.subscribe("apidash/test/ticker", qos=0) +c.loop_forever() +# apidash/test/ticker {"seq": 1, "ts": "..."} +# apidash/test/ticker {"seq": 2, "ts": "..."} +``` diff --git a/docs/mqtt/v5_properties.md b/docs/mqtt/v5_properties.md new file mode 100644 index 0000000..f688f58 --- /dev/null +++ b/docs/mqtt/v5_properties.md @@ -0,0 +1,55 @@ +--- +protocol: mqtt +title: MQTT v5 Properties +desc: The broker and echo scenario support MQTT v5 features such as response topic and correlation data for request/response messaging. +path: mqtt/v5_properties +--- + +Mosquitto 2.x supports **MQTT v5** with no extra configuration, so you can test +API Dash's v5 features against this rig. Connect with the v5 protocol +(`protocol=mqtt.MQTTv5` in paho). + +## Request/response (echo) + +The [echo](echo.md) scenario demonstrates the v5 request/response pattern: + +| Property | Behavior | +| ----------- | ----------- | +| `response_topic` | If a request to `apidash/test/echo/request` sets a response topic, the publisher sends the reply there instead of `apidash/test/echo/response`. | +| `correlation_data` | If present on the request, the same bytes are set on the reply, so you can match responses to requests. | + +## Other v5 features you can test against the broker + +- **User properties** -- arbitrary key/value metadata on a PUBLISH. +- **Message expiry interval** -- retained/queued messages that auto-expire. +- **Session expiry interval** -- how long the broker keeps session state after + disconnect. +- **Content type / payload format indicator**. + +These are broker-level features; publish/subscribe with the properties set and +observe them on the receiving side. + +## Sample Usage + +### Python (`paho-mqtt`), request/response + +```python +import paho.mqtt.client as mqtt +from paho.mqtt.packettypes import PacketTypes +from paho.mqtt.properties import Properties + +c = mqtt.Client(mqtt.CallbackAPIVersion.VERSION2, protocol=mqtt.MQTTv5) +c.on_message = lambda cl, ud, msg: print( + "reply:", msg.payload.decode(), + "correlation:", getattr(msg.properties, "CorrelationData", None), +) +c.connect("localhost", 1883) +c.subscribe("apidash/test/reply", qos=1) +c.loop_start() + +props = Properties(PacketTypes.PUBLISH) +props.ResponseTopic = "apidash/test/reply" +props.CorrelationData = b"req-42" +c.publish("apidash/test/echo/request", payload="ping", qos=1, properties=props) +# reply: ping correlation: b'req-42' +``` diff --git a/docs/mqtt/websocket.md b/docs/mqtt/websocket.md new file mode 100644 index 0000000..19bec03 --- /dev/null +++ b/docs/mqtt/websocket.md @@ -0,0 +1,46 @@ +--- +protocol: mqtt +title: MQTT over WebSocket +desc: The broker also accepts MQTT over WebSocket at ws://localhost:9001, for browser-based and web MQTT clients. +path: mqtt/websocket +--- + +In addition to plain MQTT over TCP on port `1883`, the broker exposes **MQTT +over WebSocket** on port `9001`. This is what browser-based MQTT clients (and +API Dash's web build) use, since browsers cannot open raw TCP sockets. + +Both listeners talk to the same broker, so a message published over TCP is +received over WebSocket and vice versa. + +## Endpoint + +``` +ws://localhost:9001 +``` + +Use `mqtt://localhost:1883` for the TCP transport (see the +[overview](README.md)). + +## Behavior + +| Transport | URL | Notes | +| ----------- | ----------- | ----------- | +| TCP | `mqtt://localhost:1883` | Native MQTT clients | +| WebSocket | `ws://localhost:9001` | Browser / web clients; same broker, same topics | + +All scenarios ([ticker](ticker.md), [retained](retained.md), [echo](echo.md), +[LWT](lwt.md)) and all QoS levels work identically over WebSocket. + +## Sample Usage + +### JavaScript (browser, MQTT.js) + +```javascript +// MQTT.js in a browser connects over WebSocket. +const client = mqtt.connect("ws://localhost:9001"); + +client.on("connect", () => client.subscribe("apidash/test/#")); +client.on("message", (topic, payload) => + console.log(topic, payload.toString()) +); +``` diff --git a/docs/mqtt/wildcards.md b/docs/mqtt/wildcards.md new file mode 100644 index 0000000..6a1ae92 --- /dev/null +++ b/docs/mqtt/wildcards.md @@ -0,0 +1,44 @@ +--- +protocol: mqtt +title: MQTT Topic Wildcards +desc: Use the + (single level) and # (multi level) wildcards to subscribe across the apidash/test/ topic tree. +path: mqtt/wildcards +--- + +MQTT topics are hierarchical, using `/` as a separator (e.g. +`apidash/test/echo/response`). Subscriptions can use two wildcards. All test +topics live under the `apidash/test/` prefix, so wildcards are easy to try. + +## Wildcards + +| Wildcard | Meaning | Example subscription | Matches | +| ----------- | ----------- | ----------- | ----------- | +| `+` | Exactly one level | `apidash/test/echo/+` | `apidash/test/echo/request`, `apidash/test/echo/response` | +| `#` | Zero or more trailing levels | `apidash/test/#` | every topic under `apidash/test/` | + +`+` matches a single level only: `apidash/test/+` matches `apidash/test/ticker` +but **not** `apidash/test/echo/response`. `#` must be the last character in the +filter. + +## Try it + +Subscribe to `apidash/test/#` to receive every test topic at once (ticker, +retained, echo responses, status). This is the quickest way to confirm the rig +is alive. + +## Sample Usage + +### Python (`paho-mqtt`) + +```python +import paho.mqtt.client as mqtt + +c = mqtt.Client(mqtt.CallbackAPIVersion.VERSION2, protocol=mqtt.MQTTv5) +c.on_message = lambda cl, ud, msg: print(msg.topic, "->", msg.payload[:60]) +c.connect("localhost", 1883) +c.subscribe("apidash/test/#", qos=0) +c.loop_forever() +# apidash/test/status -> b'online' +# apidash/test/retained -> b'{"note": ...}' +# apidash/test/ticker -> b'{"seq": 1, ...}' +``` From c24272a535c0872a923190c9b7680643f86b5c7b Mon Sep 17 00:00:00 2001 From: badnikhil Date: Sun, 9 Aug 2026 18:17:45 +0530 Subject: [PATCH 3/4] add mqtt broker round-trip tests --- requirements-dev.txt | 1 + tests/mqtt/test_mqtt.py | 250 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 251 insertions(+) create mode 100644 tests/mqtt/test_mqtt.py diff --git a/requirements-dev.txt b/requirements-dev.txt index 65d0867..0b0ad87 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,3 +1,4 @@ pytest httpx<0.28.0 pytest-asyncio +paho-mqtt>=2.0.0 # tests/mqtt: local MQTT broker round-trip tests (not needed by the prod app) diff --git a/tests/mqtt/test_mqtt.py b/tests/mqtt/test_mqtt.py new file mode 100644 index 0000000..6c6fbb1 --- /dev/null +++ b/tests/mqtt/test_mqtt.py @@ -0,0 +1,250 @@ +""" +Integration tests for the local MQTT test rig (mqtt/docker-compose.yml). + +These talk to a REAL broker over TCP (paho-mqtt against localhost:1883), which +is the only way to genuinely exercise QoS handshakes, retained messages, +wildcards and MQTT v5 -- the same reason we ship a Dockerized Mosquitto rather +than a hand-rolled FastAPI route. + +The whole module SKIPS gracefully when: + - paho-mqtt is not installed, or + - no broker is reachable at MQTT_HOST:MQTT_PORT (default localhost:1883), +so CI without a broker still passes. To run them for real: + + docker compose -f mqtt/docker-compose.yml up -d + pytest tests/mqtt/test_mqtt.py + +The echo request->response test additionally needs the `publisher` service +running; it skips (rather than fails) if no echo reply arrives. +""" + +import os +import socket +import threading +import time +import uuid +from queue import Empty, Queue + +import pytest + +# Skip cleanly if paho-mqtt is not installed. +mqtt = pytest.importorskip("paho.mqtt.client", reason="paho-mqtt not installed") + +MQTT_HOST = os.environ.get("MQTT_HOST", "localhost") +MQTT_PORT = int(os.environ.get("MQTT_PORT", "1883")) + + +def _broker_reachable(host, port, timeout=1.0): + try: + with socket.create_connection((host, port), timeout=timeout): + return True + except OSError: + return False + + +# Skip the entire module if there is no broker to talk to. This keeps CI green +# when no MQTT broker is running. +if not _broker_reachable(MQTT_HOST, MQTT_PORT): + pytest.skip( + f"No MQTT broker reachable at {MQTT_HOST}:{MQTT_PORT} " + "(start one with `docker compose -f mqtt/docker-compose.yml up`)", + allow_module_level=True, + ) + + +def _make_raw_client(): + cid = f"pytest-{uuid.uuid4().hex[:8]}" + try: + return mqtt.Client( + callback_api_version=mqtt.CallbackAPIVersion.VERSION2, + client_id=cid, + protocol=mqtt.MQTTv5, + ) + except AttributeError: + # paho-mqtt 1.x fallback. + return mqtt.Client(client_id=cid, protocol=mqtt.MQTTv5) + + +def _topic(): + """A unique topic per test, so tests never interfere with one another.""" + return f"apidash/test/pytest/{uuid.uuid4().hex}" + + +@pytest.fixture +def client_factory(): + """Returns a factory that builds connected clients; all are cleaned up.""" + created = [] + + def _factory(): + client = _make_raw_client() + messages = Queue() + subacks = Queue() + connected = threading.Event() + + client.on_message = lambda cl, ud, msg: messages.put(msg) + client.on_subscribe = lambda *args: subacks.put(True) + client.on_connect = lambda *args: connected.set() + + client.connect(MQTT_HOST, MQTT_PORT, keepalive=30) + client.loop_start() + if not connected.wait(timeout=5): + pytest.fail(f"Timed out connecting to {MQTT_HOST}:{MQTT_PORT}") + + # Stash the queues on the client for the helpers below. + client._messages = messages + client._subacks = subacks + created.append(client) + return client + + yield _factory + + for client in created: + try: + client.loop_stop() + client.disconnect() + except Exception: + pass + + +def _subscribe(client, topic, qos=0, timeout=5): + """Subscribe and block until the SUBACK arrives (avoids lost messages).""" + client.subscribe(topic, qos=qos) + try: + client._subacks.get(timeout=timeout) + except Empty: + pytest.fail(f"No SUBACK for '{topic}'") + + +def _next_message(client, timeout=5): + try: + return client._messages.get(timeout=max(0.01, timeout)) + except Empty: + return None + + +def _wait_for_payload(client, payload, timeout=5): + """Drain messages until one matches `payload` (ignores unrelated traffic).""" + deadline = time.time() + timeout + while time.time() < deadline: + msg = _next_message(client, timeout=deadline - time.time()) + if msg is None: + return None + if msg.payload == payload: + return msg + return None + + +# --------------------------------------------------------------------------- # +# Tests +# --------------------------------------------------------------------------- # +def test_connect_publish_subscribe_roundtrip(client_factory): + topic = _topic() + sub = client_factory() + _subscribe(sub, topic, qos=0) + + pub = client_factory() + pub.publish(topic, payload="hello-mqtt", qos=0) + + msg = _next_message(sub) + assert msg is not None, "did not receive published message" + assert msg.topic == topic + assert msg.payload == b"hello-mqtt" + + +def test_retained_delivered_on_subscribe(client_factory): + topic = _topic() + + # Publish a retained message BEFORE anyone is subscribed. + pub = client_factory() + pub.publish(topic, payload="i-am-retained", qos=1, retain=True).wait_for_publish(5) + + # A brand-new subscriber must receive it immediately on subscribe. + sub = client_factory() + _subscribe(sub, topic, qos=1) + msg = _next_message(sub) + assert msg is not None, "retained message not delivered on subscribe" + assert msg.payload == b"i-am-retained" + assert msg.retain is True + + # Cleanup: clear the retained message (empty payload + retain=True). + pub.publish(topic, payload=b"", qos=1, retain=True).wait_for_publish(5) + + +def test_wildcard_multi_level_hash(client_factory): + base = _topic() + sub = client_factory() + _subscribe(sub, f"{base}/#", qos=0) + + pub = client_factory() + pub.publish(f"{base}/a/b/c", payload="deep", qos=0) + + msg = _next_message(sub) + assert msg is not None, "'#' wildcard did not match nested topic" + assert msg.topic == f"{base}/a/b/c" + assert msg.payload == b"deep" + + +def test_wildcard_single_level_plus(client_factory): + base = _topic() + sub = client_factory() + _subscribe(sub, f"{base}/+/leaf", qos=0) + + pub = client_factory() + pub.publish(f"{base}/x/leaf", payload="match", qos=0) + msg = _next_message(sub) + assert msg is not None, "'+' wildcard did not match single level" + assert msg.topic == f"{base}/x/leaf" + + # A topic with an EXTRA level must NOT match `+/leaf`. + pub.publish(f"{base}/x/y/leaf", payload="nomatch", qos=0) + extra = _next_message(sub, timeout=1) + assert extra is None, "'+' wildcard wrongly matched an extra level" + + +def test_qos1_delivery(client_factory): + topic = _topic() + sub = client_factory() + _subscribe(sub, topic, qos=1) + + pub = client_factory() + pub.publish(topic, payload="qos1", qos=1).wait_for_publish(5) + + msg = _next_message(sub) + assert msg is not None, "QoS 1 message not delivered" + assert msg.payload == b"qos1" + assert msg.qos == 1 + + +def test_qos2_delivery(client_factory): + topic = _topic() + sub = client_factory() + _subscribe(sub, topic, qos=2) + + pub = client_factory() + pub.publish(topic, payload="qos2", qos=2).wait_for_publish(5) + + msg = _next_message(sub) + assert msg is not None, "QoS 2 message not delivered" + assert msg.payload == b"qos2" + assert msg.qos == 2 + + +def test_echo_request_response(client_factory): + """Exercises the publisher service (mqtt/publisher.py). + + Skips (rather than fails) when the publisher is not running. + """ + sub = client_factory() + _subscribe(sub, "apidash/test/echo/response", qos=1) + + pub = client_factory() + payload = f"echo-{uuid.uuid4().hex}".encode() + pub.publish("apidash/test/echo/request", payload=payload, qos=1) + + msg = _wait_for_payload(sub, payload, timeout=5) + if msg is None: + pytest.skip( + "No echo response -- the `publisher` service is not running " + "(start it with `docker compose -f mqtt/docker-compose.yml up`)" + ) + assert msg.payload == payload From 55ef77caf5d3a5754e01376f92a7ca862d1364fd Mon Sep 17 00:00:00 2001 From: badnikhil Date: Sun, 9 Aug 2026 18:38:36 +0530 Subject: [PATCH 4/4] docs: explain why mqtt testing needs docker (vs http/ws endpoints) --- docs/mqtt/README.md | 45 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/docs/mqtt/README.md b/docs/mqtt/README.md index 21531ff..3b29c6d 100644 --- a/docs/mqtt/README.md +++ b/docs/mqtt/README.md @@ -16,6 +16,51 @@ Running a real, spec-compliant broker is the only way to genuinely test QoS 1/2 handshakes, retained messages, persistent sessions, Last Will and MQTT v5 properties. +## Why this is different from the other test endpoints (and why Docker) + +**Before — HTTP / WebSocket / SSE:** every test endpoint is a FastAPI route +inside this app, served over its single HTTP(S) port. That works because HTTP, +WebSocket (an HTTP upgrade) and SSE all ride on HTTP — and Azure App Service +exposes exactly one HTTP/HTTPS port, which is all those need. No extra process, +no extra infra. + +**MQTT is fundamentally different.** It's a stateful, connection-oriented +**pub/sub** protocol over its **own TCP ports** (1883 plaintext, 8883 TLS, +8083/8084 for MQTT-over-WebSocket), and it needs a **real broker** to hold +sessions, subscriptions, retained messages and QoS state. It is **not** HTTP, so: + +- It **can't be a FastAPI route** like `/ws/echo` — there is no HTTP request to + answer; the client keeps a long-lived MQTT connection open to a broker. +- Azure App Service **can't host a broker** either — it forwards only one + HTTP/HTTPS port and cannot open raw TCP 1883/8883. (The WS endpoints work there + *only* because WS is an HTTP upgrade over that same port.) +- Faking broker semantics (QoS 1/2 handshakes, retained, sessions, LWT, v5 + properties) as a FastAPI endpoint would mean re-implementing a broker, with + poor fidelity — not worth it. + +**So to test locally we run a real broker (Eclipse Mosquitto) + a publisher via +Docker** — one command, full fidelity, no cloud needed. That is the "extra +hassle": it buys you a spec-compliant broker to point the client at, instead of a +fake HTTP shim. + +**What about production?** A shared, always-on hosted MQTT test endpoint (the +equivalent of `api.apidash.dev/ws/echo`) is a **separate, maintainer-owned +decision** — it needs a broker somewhere *outside* App Service (a managed broker +like HiveMQ/EMQX Cloud, or a self-hosted VM). **This change intentionally covers +local testing only;** the Azure/production infra is wired up separately. + +## What this adds + +- `mqtt/docker-compose.yml`, `mqtt/Dockerfile`, `mqtt/mosquitto/mosquitto.conf` — + the Dockerized broker + publisher service. +- `mqtt/publisher.py` — the deterministic test-topic publisher (ticker, retained, + echo request/response incl. v5 `response_topic`, Last Will). +- `docs/mqtt/` — this README plus per-scenario pages. +- `tests/mqtt/test_mqtt.py` — broker round-trip tests that skip when no broker is + running (CI-safe). +- `paho-mqtt` added to `requirements-dev.txt` only — the production app + (`requirements.txt`) is unchanged and pulls in no MQTT dependency. + ## Run it From the repository root: