feat: Enhance MQTT response parsing and logging

- Updated the response parser to support additional delimiters (`;` and `=`) for better command matching.
- Implemented strict prefix matching to prevent incorrect command matches.
- Introduced a new logging mechanism to clean payloads by removing preambles before logging.
- Added functionality to skip publishing messages with empty or invalid data.
- Ensured JSON output for MQTT messages is compact to maintain compatibility with brokers.
- Added tests to verify the new parsing logic and ensure robustness against invalid payloads.
- Refactored MQTT publisher to handle raw lines and improved error handling for empty payloads.

Author: sidey79

docs/architecture/decisions/ADR-004-mqtt-response-parsing.adoc

@@ -24,7 +24,11 @@ Dies stellt sicher, dass alle erfolgreichen `GET` Anfragen über MQTT eine struk
 
 === Detaillierte Logik-Anpassungen
 
-1.  **`get_config` (CG):**
+1.  **Erweitertes Parsing (Update 2026-01-13):**
+    *   Der Response-Parser unterstützt nun `;` und `=` als Trennzeichen zusätzlich zu Leerzeichen (z.B. für `SR;R=...`).
+    *   Es wird ein striktes Prefix-Matching erzwungen, um zu verhindern, dass Befehls-Prefixe längere Nachrichtentypen matchen (z.B. darf Befehl `M` nicht `MN;...` matchen).
+
+2.  **`get_config` (CG):**
     *   Wird eine private Hilfsfunktion `_parse_decoder_config(response: str) -> Dict[str, int]` in [`signalduino/commands.py`](signalduino/commands.py) implementiert.
     *   Diese Funktion parst den `key=value;` String in ein Dictionary (z.B. `{'MS': 1, 'MU': 1, 'MC': 1, 'Mred': 1}`).
     *   Der Rückgabetyp von `get_config` wird von `str` auf `Dict[str, int]` geändert.

docs/architecture/decisions/ADR-006-json-output-schema.adoc

@@ -17,6 +17,9 @@ Wir werden das JSON-Output-Schema von `DecodedMessage` wie folgt anpassen:
 
 Die Umbenennung des Nutzdatenfeldes von `payload` zu `data` sowie die Einführung des Feldes `raw` (für die ursprüngliche Nachricht) sind in link:ADR-007-data-and-raw-fields.adoc[ADR-007] dokumentiert, das dieses Schema ergänzt und präzisiert. Dieses ADR dient als Grundlage für die Einführung des `protocol`-Feldes und die Bereinigung des Nutzdateninhalts.
 
+=== Serialisierungsformat
+Der JSON-Output für MQTT-Nutzdaten muss **kompakt** (ohne Zeilenumbrüche und Einrückungen) serialisiert werden, um die Kompatibilität mit MQTT-Brokern und Downstream-Systemen zu gewährleisten, die multiline-Nutzdaten falsch interpretieren könnten. Dies wird durch das Weglassen des `indent`-Parameters beim `json.dumps`-Aufruf in `MqttPublisher` sichergestellt.
+
 === Details zur neuen Struktur (Präzisiert durch ADR-007)
 
 [cols="1,1,4"]

main.py

@@ -14,6 +14,13 @@
 from signalduino.transport import SerialTransport, TCPTransport
 from signalduino.types import DecodedMessage, RawFrame # NEU: RawFrame
 
+# NEU: Importiere Protokoll-Handler für Log-Bereinigung
+try:
+    from sd_protocols.loader import _protocol_handler
+except ImportError:
+    _protocol_handler = None
+
+
 # Konfiguration des Loggings
 def initialize_logging(log_level_str: str):
     """Initialisiert das Logging basierend auf dem übergebenen String."""
@@ -39,10 +46,28 @@ def initialize_logging(log_level_str: str):
 async def message_callback(message: DecodedMessage):
     """Callback-Funktion, die aufgerufen wird, wenn eine Nachricht dekodiert wurde."""
     model = message.metadata.get("model", "Unknown")
+    
+    # NEU: Bereinige die Payload für die Log-Ausgabe, da der Parser die Preamble möglicherweise nicht entfernt hat
+    # (oder der Preamble-Eintrag im Protokoll-Handler fehlt).
+    log_payload = message.data
+    preamble = ""
+    
+    if _protocol_handler and message.protocol.get('id'):
+        try:
+            protocol_id = message.protocol['id']
+            # Abrufen der Preamble, falls vorhanden
+            preamble = _protocol_handler.check_property(protocol_id, 'preamble', '')
+        except Exception:
+            logger.debug("Konnte Preamble nicht abrufen für Protokoll %s", message.protocol.get('id'))
+    
+    if preamble and log_payload.upper().startswith(preamble.upper()):
+        # Entferne die Preamble aus der Payload für das Logging
+        log_payload = log_payload[len(preamble):]
+        
     logger.info(
         f"Decoded message received: protocol={message.protocol_id}, "
         f"model={model}, "
-        f"payload={message.payload}"
+        f"payload={log_payload}"
     )
     logger.debug(f"Full Metadata: {message.metadata}")
     # NEU: Überprüfe, ob RawFrame vorhanden ist und das Attribut 'line' hat

signalduino/constants.py

@@ -17,3 +17,7 @@
 SDUINO_MC_DISPATCH_LOG_ID = "12.1"
 SDUINO_PARSE_DEFAULT_LENGHT_MIN = 8
 SDUINO_GET_CONFIGQUERY_DELAY = 0.75
+
+# Protocol Framing
+ASCII_STX = "\x02"
+ASCII_ETX = "\x03"

signalduino/controller.py

@@ -5,6 +5,7 @@
 import logging
 import asyncio
 from datetime import datetime, timedelta, timezone
+from dataclasses import asdict
 from typing import Any, Awaitable, Callable, List, Optional, Dict, Tuple, Pattern
 
 from .commands import SignalduinoCommands
@@ -250,13 +251,42 @@ async def _parser_task(self) -> None:
                     # Führe die rechenintensive Parsing-Logik in einem separaten Thread aus.
                     # Dadurch wird die asyncio-Event-Schleife nicht blockiert.
                     decoded = await asyncio.to_thread(self.parser.parse_line, line)
-                    if decoded and self.message_callback:
-                        await self.message_callback(decoded[0])
-                    if self.mqtt_publisher and decoded:
-                        # Verwende die neue MqttPublisher.publish(message: DecodedMessage) Signatur
-                        await self.mqtt_publisher.publish(decoded[0])
-                    await self._handle_as_command_response(line)
-                
+                    if decoded:
+                        for message in decoded:
+                            if isinstance(message, DecodedMessage):
+                               # Überspringe die Veröffentlichung, wenn DecodedMessage keine Daten enthält.
+                               # Das Feld 'data' kann leer sein, wenn die Decodierung fehlschlägt (z.B. Checksumme)
+                               # oder ungültige Werte wie '[]' (leere JSON-Liste als String) enthält,
+                               # was auf eine nicht parsbare Nachricht hinweist, die aber als DecodedMessage
+                               # zurückgegeben wurde.
+                               if not message.data or message.data.strip() == "[]":
+                                   self.logger.info("Skipping decoded message with empty/invalid data for protocol %s: %s",
+                                                   message.protocol.get('id', 'N/A'), message)
+                                   continue
+
+                               if self.message_callback:
+                                   try:
+                                       await self.message_callback(message)
+                                   except Exception as exc:
+                                       self.logger.error("Error in message callback: %s", exc)
+                                       
+                               if self.mqtt_publisher:
+                                   try:
+                                       # message is a DecodedMessage dataclass, pass directly to publish
+                                       # The MqttPublisher handles serialization.
+                                       await self.mqtt_publisher.publish(message)
+                                   except Exception as exc:
+                                       self.logger.error("Error publishing message to MQTT: %s", exc)
+                    matched_cmd = False
+                    if not decoded:
+                        # Nur die Zeile als Befehlsantwort verarbeiten, wenn sie NICHT erfolgreich als Sensordaten geparst wurde.
+                        matched_cmd = await self._handle_as_command_response(line)
+                    
+                    # If line was not parsed as a decoded message and was not a command response,
+                    # publish it as a raw line via MQTT (Problem 2).
+                    if not decoded and not matched_cmd and self.mqtt_publisher and line.strip():
+                        await self.mqtt_publisher.publish_raw_line(line)
+                        
                 # Ensure a minimal yield time for other tasks when the queue is rapidly processed.
                 await asyncio.sleep(0.01)
             except Exception as e:
@@ -357,34 +387,69 @@ async def _send_and_wait(self, command: str, timeout: float, response_pattern: O
                 raise SignalduinoConnectionError(str(e))
             raise
 
-    async def _handle_as_command_response(self, line: str) -> None:
-        """Check if the received line matches any pending command response."""
+    async def _handle_as_command_response(self, line: str) -> bool:
+        """Check if the received line matches any pending command response.
+        
+        Returns:
+            True if a response was matched and processed, False otherwise.
+        """
         self.logger.debug("Hardware response received: %s", line)
+        matched = False
+        
         async with self._pending_responses_lock:
             self.logger.debug(f"Current pending responses: {len(self._pending_responses)}")
-            for pending in self._pending_responses:
+            
+            # Use a copy of the list for safe iteration/removal if performance allows.
+            # Since the number of pending responses is usually small, this is safe.
+            for pending in list(self._pending_responses):
                 try:
-                    self.logger.debug(f"Checking pending response for command: {pending.command.payload}. Line: {line.strip()}")
+                    cmd_payload = pending.command.payload
+                    self.logger.debug(f"Checking pending response for command: {cmd_payload}. Line: {line.strip()}")
 
                     pattern = pending.command.response_pattern
                     if pattern:
                         self.logger.debug(f"Testing pattern: {pattern.pattern}")
                         if pattern.match(line):
-                            self.logger.debug(f"Matched response pattern for command: {pending.command.payload}")
+                            self.logger.debug(f"Matched response pattern for command: {cmd_payload}")
                             pending.future.set_result(line)
                             self._pending_responses.remove(pending)
-                            return
+                            matched = True
+                            break # Exit for-loop
 
-                    self.logger.debug(f"Testing direct match for: {pending.command.payload}")
-                    if line.startswith(pending.command.payload):
-                        self.logger.debug(f"Matched direct response for command: {pending.command.payload}")
+                    self.logger.debug(f"Testing direct match for: {cmd_payload}")
+                    
+                    # Robust check for command match:
+                    # 1. Startswith command payload (e.g., 'V', 'SR')
+                    # 2. Response must be either exact match, or followed by a valid separator
+                    # Separators: space (V command), ';' (SR command), '=' (Config)
+                    # This ensures "V" matches "V 4.0.0", "SR" matches "SR;R=1", but "M" does NOT match "MN..."
+                    if line.startswith(cmd_payload):
+                        if len(line) == len(cmd_payload):
+                            is_direct_match = True
+                        else:
+                            separator = line[len(cmd_payload)]
+                            is_direct_match = separator.isspace() or separator in (';', '=')
+                    else:
+                        is_direct_match = False
+
+                    if is_direct_match:
+                        self.logger.debug(f"Matched direct response for command: {cmd_payload}")
                         pending.future.set_result(line)
                         self._pending_responses.remove(pending)
-                        return
+                        matched = True
+                        break # Exit for-loop
+                        
                 except Exception as e:
                     self.logger.error(f"Error processing pending response: {e}")
+                    # Remove the potentially failing pending response to prevent further errors
+                    if pending in self._pending_responses:
+                        self._pending_responses.remove(pending)
                     continue
+
+        if not matched:
             self.logger.debug("No matching pending response found")
+            
+        return matched
 
     async def _init_task_start_loop(self) -> None:
         """Main initialization task that handles version check and XQ command."""

signalduino/mqtt.py

@@ -1,8 +1,8 @@
 import json
 import logging
 import os
-from dataclasses import asdict
-from typing import Optional, Any, Callable, Awaitable # NEU: Awaitable für async callbacks
+from dataclasses import asdict, is_dataclass
+from typing import Optional, Any, Callable, Awaitable, Union # NEU: Awaitable für async callbacks
 
 from .commands import MqttCommandDispatcher, CommandValidationError, SignalduinoCommandTimeout # NEU: Import Dispatcher
 import aiomqtt as mqtt
@@ -231,8 +231,28 @@ async def _handle_command(self, command_name: str, payload: str) -> None:
             )
 
 
-    def _message_to_json(self, message: DecodedMessage) -> str:
-        """Serializes a DecodedMessage to a JSON string."""
+    def _message_to_json(self, message: Union[DecodedMessage, Any]) -> Optional[str]:
+        """Serializes a DecodedMessage or other payload to a JSON string."""
+        
+        # Check if message is a dataclass instance
+        if not is_dataclass(message):
+            # If not a dataclass, try to serialize it directly or wrap it
+            if isinstance(message, dict):
+                return json.dumps(message)
+            elif isinstance(message, str):
+                try:
+                    # Check if it's already valid JSON
+                    json.loads(message)
+                    return message
+                except json.JSONDecodeError:
+                    # Wrap string in a simple object
+                    return json.dumps({"data": message})
+            else:
+                 # Fallback for other types
+                 try:
+                     return json.dumps(message)
+                 except (TypeError, ValueError):
+                     return json.dumps({"data": str(message)})
 
         # DecodedMessage uses dataclasses, but RawFrame inside it also uses a dataclass.
         # We need a custom serializer to handle nested dataclasses like RawFrame.
@@ -249,11 +269,13 @@ def _raw_frame_to_dict(raw_frame: RawFrame) -> dict:
         # Note: 'raw' is now a string (ADR-007) and should be published.
         # The pop operation (line 249 in original) is removed to include it.
 
-        # Append preamble to data for FHEM compatibility (PreambleProtocolID#HexData)
         preamble = ""
         if self._protocol_handler:
             try:
-                protocol_id = message.protocol.get('id')
+                # Use .get only if message has a protocol attribute (it should if it's DecodedMessage)
+                protocol = getattr(message, 'protocol', {})
+                protocol_id = protocol.get('id') if isinstance(protocol, dict) else None
+                
                 if protocol_id:
                     # check_property returns the value or default
                     preamble = self._protocol_handler.check_property(protocol_id, 'preamble', '')
@@ -264,9 +286,32 @@ def _raw_frame_to_dict(raw_frame: RawFrame) -> dict:
         message_dict["preamble"] = preamble
 
         # Ensure data (formerly payload) is uppercase
-        message_dict["data"] = message.data.upper()
+        # Use getattr to be safe even if dataclass structure changed
+        message_data = getattr(message, 'data', '')
+        if isinstance(message_data, str):
+            message_data = message_data.upper()
+        else:
+            message_data = str(message_data).upper()
+
+        # REMOVE PREAMBLE FROM DATA FIELD IF PRESENT
+        # This ensures the 'data' field only contains the payload, consistent with ADR-007.
+        if preamble and message_data.startswith(preamble.upper()):
+            message_data = message_data[len(preamble):]
+        
+        # NEU: Fehlerbehebung für ungültige Parser-Rückgaben (dmsg=[])
+        # Wenn der Parser eine leere Liste als String-Literal zurückgibt, wird dies als ungültiger
+        # Datenwert interpretiert. Setze auf leeren String.
+        if message_data.strip() == "[]":
+            message_data = ""
+            self.logger.warning("Invalid data '[]' received from parser, setting to empty string.")
 
-        return json.dumps(message_dict, indent=4)
+        # Check for empty payload to prevent sending empty messages
+        if not message_data:
+            return None
+
+        message_dict["data"] = message_data
+
+        return json.dumps(message_dict)
 
     async def publish_simple(self, subtopic: str, payload: str, retain: bool = False) -> None:
         """Publishes a simple string payload to a subtopic of the main topic."""
@@ -276,11 +321,19 @@ async def publish_simple(self, subtopic: str, payload: str, retain: bool = False
 
         try:
             topic = f"{self.base_topic}/{subtopic}"
-            await self.client.publish(topic, payload, retain=retain)
+            await self.client.publish(topic, payload.encode("utf-8"), retain=retain)
             self.logger.debug("Published simple message to %s: %s", topic, payload)
         except Exception:
             self.logger.error("Failed to publish simple message to %s", subtopic, exc_info=True)
 
+    async def publish_raw_line(self, line: str) -> None:
+        """Publishes a raw, non-decoded line from the transport (e.g., non-command status messages)."""
+        await self.publish_simple(
+            subtopic="state/raw_lines",
+            payload=json.dumps({"line": line.strip()}),
+            retain=False
+        )
+
     async def publish(self, message: DecodedMessage) -> None:
         """Publishes a DecodedMessage."""
         if not self.client:
@@ -290,8 +343,14 @@ async def publish(self, message: DecodedMessage) -> None:
         try:
             topic = f"{self.base_topic}/state/messages"
             payload = self._message_to_json(message)
-            await self.client.publish(topic, payload)
+            
+            if payload is None:
+                protocol_id = message.protocol.get('id', 'N/A')
+                self.logger.debug("Skipping MQTT publish due to empty payload for protocol %s", protocol_id)
+                return
+
+            await self.client.publish(topic, payload.encode("utf-8"))
             protocol_id = message.protocol.get('id', 'N/A')
-            self.logger.debug("Published message for protocol %s to %s", protocol_id, topic)
+            self.logger.debug("Published message for protocol %s to %s (Payload length: %s)", protocol_id, topic, len(payload))
         except Exception:
             self.logger.error("Failed to publish message", exc_info=True)

signalduino/parser/base.py

@@ -6,8 +6,9 @@
 from typing import Optional, List, Tuple
 
 from ..exceptions import SignalduinoParserError
+from ..constants import ASCII_STX, ASCII_ETX
 
-_STX_ETX = re.compile(r"^\x02(M[sSuUcCNOo];.*;)\x03$")
+_STX_ETX = re.compile(f"^{ASCII_STX}(M[sSuUcCNOo];.*;){ASCII_ETX}$")
 
 
 def decompress_payload(compressed_payload: str) -> str:

signalduino/parser/mc.py

@@ -100,9 +100,14 @@ def parse(self, frame: RawFrame) -> Iterable[DecodedMessage]:
                 "preamble": protocol_data.get("preamble", ""),
             }
 
-            # 1. Entferne die Preamble aus der Payload
-            preamble_len = len(protocol_meta["preamble"])
-            payload = raw_payload[preamble_len:]
+            # 1. Entferne die Preamble aus der Payload, falls vorhanden.
+            # Normalisiere Preamble und Payload zur korrekten Erkennung der Groß-/Kleinschreibung.
+            preamble = protocol_meta["preamble"]
+            if preamble and raw_payload.upper().startswith(preamble.upper()):
+                preamble_len = len(preamble)
+                payload = raw_payload[preamble_len:]
+            else:
+                payload = raw_payload
 
             yield DecodedMessage(
                 data=payload,

signalduino/types.py

@@ -30,6 +30,11 @@ class DecodedMessage:
     metadata: dict = field(default_factory=dict)
     protocol: dict = field(default_factory=dict)
 
+    @property
+    def protocol_id(self) -> Optional[str]:
+        """Provides backward compatibility for message.protocol_id."""
+        return self.protocol.get('id')
+
 
 @dataclass(slots=True)
 class QueuedCommand: