From 06496a5cd692651aa5508b66c659422c86e8d23c Mon Sep 17 00:00:00 2001 From: Blake Azuela Date: Mon, 16 Nov 2026 10:46:07 -0500 Subject: [PATCH 01/10] fix(firmware): detect and recover from I2C bus faults A robot on v0.2.0-72-g946c02b froze with a dim display, dead servos and a stuttering bell while the ESP32 kept running. Its serial log showed i2c_master_transmit failing with ESP_ERR_INVALID_STATE every 60 ms, for every write to the PCA9685 and the OLED. The Adafruit drivers ignore failed writes and the firmware set both chips up only at boot, so it never noticed and each failed write blocked the loop for its timeout. Check both chips every 500 ms: the OLED must answer, and the PCA9685 must answer and still hold the prescale it was given, which also catches a chip that lost power and came back with its defaults. Two failed checks in a row count as a fault. While down, servo and display writes are skipped so audio and HTTP keep running. Recovery restarts Wire, clocks out a device left holding SDA, sets both chips up again and rewrites every servo channel from the firmware's state; a failed attempt is retried after 1 s, doubling up to 30 s. A PCA9685 that still answers is left alone while only the OLED is retried. Boot clears the bus the same way before Wire starts, for a reset that lands in the middle of a transfer. Each fault and recovery is logged with the bus line levels, and GET /health reports i2c.ok, i2c.pca9685, i2c.faults, i2c.recoveries and i2c.last_fault_ms. `oled` now goes false while the display is unreachable. Not yet run on a robot: host tests cover the fault and retry timing, and both firmware environments build. --- docs/api.md | 16 +- platformio.ini | 1 + src/display/oled.h | 2 + src/display/oled_init.cpp | 36 +++-- src/hardware/i2c_fault.cpp | 58 +++++++ src/hardware/i2c_fault.h | 50 ++++++ src/hardware/i2c_health.cpp | 205 +++++++++++++++++++++++++ src/hardware/i2c_health.h | 30 ++++ src/hardware/servo_wrapper.cpp | 78 ++++++++-- src/hardware/servo_wrapper.h | 6 + src/http/health_handlers.cpp | 9 ++ src/main.cpp | 8 +- src/robot_tick.cpp | 2 + test/test_i2c_fault/test_i2c_fault.cpp | 140 +++++++++++++++++ 14 files changed, 610 insertions(+), 31 deletions(-) create mode 100644 src/hardware/i2c_fault.cpp create mode 100644 src/hardware/i2c_fault.h create mode 100644 src/hardware/i2c_health.cpp create mode 100644 src/hardware/i2c_health.h create mode 100644 test/test_i2c_fault/test_i2c_fault.cpp diff --git a/docs/api.md b/docs/api.md index 343a52cf..a423a31f 100644 --- a/docs/api.md +++ b/docs/api.md @@ -120,7 +120,14 @@ curl -H "Authorization: Bearer YOUR_TOKEN" http://tiny-engineer.local/health "provisioning": false, "setup_ap_ssid": "", "setup_ap_ip": "", - "oled": true + "oled": true, + "i2c": { + "ok": true, + "pca9685": true, + "faults": 0, + "recoveries": 0, + "last_fault_ms": 0 + } } ``` @@ -140,7 +147,12 @@ curl -H "Authorization: Bearer YOUR_TOKEN" http://tiny-engineer.local/health | `provisioning` | `true` when setup AP mode is active | | `setup_ap_ssid` | Setup AP SSID when `provisioning` is true, else `""` | | `setup_ap_ip` | Setup AP IP when `provisioning` is true (usually `192.168.4.1`), else `""` | -| `oled` | SSD1306 probed and initialized | +| `oled` | SSD1306 probed and initialized. Goes `false` while an I2C fault keeps the display unreachable | +| `i2c.ok` | `false` while the I2C bus shared by the PCA9685 and the OLED is down. The firmware checks both every 500 ms and, on a fault, restarts the bus and sets the chips up again, retrying with a growing delay (1 s up to 30 s) | +| `i2c.pca9685` | Servo driver reachable. Servo writes are skipped while `false` | +| `i2c.faults` | I2C faults since boot | +| `i2c.recoveries` | Faults the firmware recovered from since boot | +| `i2c.last_fault_ms` | `uptime_ms` at the latest fault; `0` when there has been none | ### `GET /settings` diff --git a/platformio.ini b/platformio.ini index d2316dc6..728671bc 100644 --- a/platformio.ini +++ b/platformio.ini @@ -89,6 +89,7 @@ build_src_filter = + + + + + build_flags = -std=c++17 -Iinclude -Isrc ; Opt-in OLED-only application. Building does not upload or change robot settings. diff --git a/src/display/oled.h b/src/display/oled.h index 3e930873..f1556f1f 100644 --- a/src/display/oled.h +++ b/src/display/oled.h @@ -34,6 +34,8 @@ void showTextThenIdle(const char* line1, const char* line2 = nullptr); void showIdleScreen(); void initOled(); +// Sets the OLED up again after an I2C fault. Leaves a blank, lit panel. +bool restartOled(); void runOledTest(); void showServoProgress( diff --git a/src/display/oled_init.cpp b/src/display/oled_init.cpp index 3901a4bb..2e1311a3 100644 --- a/src/display/oled_init.cpp +++ b/src/display/oled_init.cpp @@ -13,6 +13,27 @@ bool i2cDeviceConnected(uint8_t address) { return Wire.endTransmission() == 0; } +namespace { + +// Sends the SSD1306 its full setup. Safe to repeat: a panel that lost power +// comes back dark until it gets this again. +bool configureOled() { + if (!display.begin( + SSD1306_SWITCHCAPVCC, + OLED_ADDRESS + )) { + return false; + } + + display.setRotation(settingsOledRotate180() ? 2 : 0); + + display.clearDisplay(); + display.display(); + return true; +} + +} // namespace + void initOled() { serialLogPrintln(); serialLogPrintln("Checking OLED at 0x3C..."); @@ -25,21 +46,18 @@ void initOled() { serialLogPrintln("OLED found"); - if (!display.begin( - SSD1306_SWITCHCAPVCC, - OLED_ADDRESS - )) { + if (!configureOled()) { serialLogPrintln("ERROR: OLED initialization failed"); oledAvailable = false; return; } - display.setRotation(settingsOledRotate180() ? 2 : 0); - - display.clearDisplay(); - display.display(); - oledAvailable = true; serialLogPrintln("OLED initialized"); } + +bool restartOled() { + oledAvailable = i2cDeviceConnected(OLED_ADDRESS) && configureOled(); + return oledAvailable; +} diff --git a/src/hardware/i2c_fault.cpp b/src/hardware/i2c_fault.cpp new file mode 100644 index 00000000..7febc106 --- /dev/null +++ b/src/hardware/i2c_fault.cpp @@ -0,0 +1,58 @@ +#include "hardware/i2c_fault.h" + +I2cFaultAction I2cFaultTracker::due(uint32_t now) const { + // Unsigned subtraction keeps the intervals right across the millis() wrap. + if (down_) { + return now - lastAttemptMs_ >= retryDelayMs_ + ? I2cFaultAction::Recover + : I2cFaultAction::None; + } + + return !checked_ || now - lastCheckMs_ >= kI2cCheckIntervalMs + ? I2cFaultAction::Check + : I2cFaultAction::None; +} + +bool I2cFaultTracker::noteCheck(bool ok, uint32_t now) { + checked_ = true; + lastCheckMs_ = now; + + if (ok) { + failures_ = 0; + return false; + } + + if (++failures_ < kI2cFailuresToFault) { + return false; + } + + down_ = true; + faults_++; + lastFaultMs_ = now; + attempts_ = 0; + // The first recovery attempt runs right away. + retryDelayMs_ = 0; + lastAttemptMs_ = now; + return true; +} + +void I2cFaultTracker::noteRecovery(bool ok, uint32_t now) { + attempts_++; + lastAttemptMs_ = now; + + if (ok) { + down_ = false; + failures_ = 0; + recoveries_++; + lastCheckMs_ = now; + return; + } + + if (retryDelayMs_ == 0) { + retryDelayMs_ = kI2cFirstRetryMs; + } else if (retryDelayMs_ >= kI2cMaxRetryMs / 2) { + retryDelayMs_ = kI2cMaxRetryMs; + } else { + retryDelayMs_ *= 2; + } +} diff --git a/src/hardware/i2c_fault.h b/src/hardware/i2c_fault.h new file mode 100644 index 00000000..8bd46c99 --- /dev/null +++ b/src/hardware/i2c_fault.h @@ -0,0 +1,50 @@ +#pragma once + +#include + +constexpr uint32_t kI2cCheckIntervalMs = 500; +// One failed check can be a glitch; two in a row take the bus down. +constexpr uint8_t kI2cFailuresToFault = 2; +constexpr uint32_t kI2cFirstRetryMs = 1000; +constexpr uint32_t kI2cMaxRetryMs = 30000; + +enum class I2cFaultAction : uint8_t { + None, + Check, // bus is up: probe the devices + Recover // bus is down: try to bring it back +}; + +// Decides when the I2C bus counts as down and when to try bringing it back. +// Pure logic: the caller does the bus work and reports each result. +class I2cFaultTracker { +public: + I2cFaultAction due(uint32_t now) const; + + // Result of a health check while up. True when this check took the bus down. + bool noteCheck(bool ok, uint32_t now); + + // Result of a recovery attempt while down. A failed attempt doubles the wait + // before the next one, up to kI2cMaxRetryMs. + void noteRecovery(bool ok, uint32_t now); + + bool down() const { return down_; } + uint32_t faults() const { return faults_; } + uint32_t recoveries() const { return recoveries_; } + // millis() of the latest fault; 0 when there has been none. + uint32_t lastFaultMs() const { return lastFaultMs_; } + // Recovery attempts since the current fault began. + uint32_t attempts() const { return attempts_; } + uint32_t retryDelayMs() const { return retryDelayMs_; } + +private: + bool down_ = false; + bool checked_ = false; + uint8_t failures_ = 0; + uint32_t lastCheckMs_ = 0; + uint32_t lastAttemptMs_ = 0; + uint32_t retryDelayMs_ = 0; + uint32_t attempts_ = 0; + uint32_t faults_ = 0; + uint32_t recoveries_ = 0; + uint32_t lastFaultMs_ = 0; +}; diff --git a/src/hardware/i2c_health.cpp b/src/hardware/i2c_health.cpp new file mode 100644 index 00000000..67b36612 --- /dev/null +++ b/src/hardware/i2c_health.cpp @@ -0,0 +1,205 @@ +#include +#include + +#include "display/oled.h" +#include "hardware/i2c_fault.h" +#include "hardware/i2c_health.h" +#include "hardware/servo_wrapper.h" +#include "pins.h" +#include "serial_log.h" +#include "sleep.h" + +namespace { + +constexpr uint8_t kBusClearPulses = 9; +constexpr uint32_t kBusClearHalfPeriodUs = 5; + +I2cFaultTracker g_tracker; +bool g_started = false; +// The robot runs without an OLED; only watch one that was there at boot. +bool g_oledExpected = false; +bool g_servoDriverUp = true; + +struct BusLines { + bool sdaHigh; + bool sclHigh; +}; + +// A device interrupted mid-byte keeps SDA low until it has been clocked +// through the rest of that byte. Clock it out, send a STOP, and report where +// the lines settled: a line still low means something is holding the bus. +BusLines clearBus() { + pinMode(I2C_SDA, INPUT_PULLUP); + pinMode(I2C_SCL, OUTPUT_OPEN_DRAIN); + digitalWrite(I2C_SCL, HIGH); + delayMicroseconds(kBusClearHalfPeriodUs); + + for (uint8_t i = 0; i < kBusClearPulses && digitalRead(I2C_SDA) == LOW; i++) { + digitalWrite(I2C_SCL, LOW); + delayMicroseconds(kBusClearHalfPeriodUs); + digitalWrite(I2C_SCL, HIGH); + delayMicroseconds(kBusClearHalfPeriodUs); + } + + // STOP: SDA rises while SCL is high. + digitalWrite(I2C_SCL, LOW); + delayMicroseconds(kBusClearHalfPeriodUs); + pinMode(I2C_SDA, OUTPUT_OPEN_DRAIN); + digitalWrite(I2C_SDA, LOW); + delayMicroseconds(kBusClearHalfPeriodUs); + digitalWrite(I2C_SCL, HIGH); + delayMicroseconds(kBusClearHalfPeriodUs); + digitalWrite(I2C_SDA, HIGH); + delayMicroseconds(kBusClearHalfPeriodUs); + + pinMode(I2C_SDA, INPUT_PULLUP); + pinMode(I2C_SCL, INPUT_PULLUP); + delayMicroseconds(kBusClearHalfPeriodUs); + + return {digitalRead(I2C_SDA) == HIGH, digitalRead(I2C_SCL) == HIGH}; +} + +void logLines(const BusLines& lines) { + serialLogPrint(" sda="); + serialLogPrint(lines.sdaHigh ? "high" : "LOW"); + serialLogPrint(" scl="); + serialLogPrintln(lines.sclHigh ? "high" : "LOW"); +} + +void logDevices(bool servoDriver, bool oled) { + serialLogPrint(" pca9685="); + serialLogPrint(servoDriver ? "ok" : "fail"); + serialLogPrint(" oled="); + serialLogPrint(!g_oledExpected ? "absent" : oled ? "ok" : "fail"); +} + +// Restarts whatever is down. Returns true when every expected device is up. +bool bringUp() { + bool busUsable = true; + + // A PCA9685 that still answers proves the bus works: leave it alone and + // only retry the OLED. + if (!(g_servoDriverUp && servoPwmDriverResponds())) { + g_servoDriverUp = false; + oledAvailable = false; + + Wire.end(); + const BusLines lines = clearBus(); + Wire.begin(I2C_SDA, I2C_SCL); + + serialLogPrint("[i2c] bus restarted"); + logLines(lines); + + // A held line fails every transfer; skip the probes and their timeouts. + busUsable = lines.sdaHigh && lines.sclHigh; + + if (busUsable && i2cDeviceConnected(PCA9685_ADDRESS)) { + restartServoPwmDriver(); + g_servoDriverUp = servoPwmDriverResponds(); + } + } + + if (busUsable && g_oledExpected && !oledAvailable && restartOled()) { + // The restart turned the panel on; a sleeping robot keeps it dark. Awake, + // the next eye frame or setup screen redraws it. + if (isSleeping()) { + blankOled(); + sleepOled(); + } + } + + return g_servoDriverUp && (!g_oledExpected || oledAvailable); +} + +void check(uint32_t now) { + const bool servoDriver = servoPwmDriverResponds(); + const bool oled = !g_oledExpected || i2cDeviceConnected(OLED_ADDRESS); + + if (!g_tracker.noteCheck(servoDriver && oled, now)) { + return; + } + + // Stop writing to both: they share the bus, and each failed write blocks + // the loop for its timeout. + g_servoDriverUp = false; + oledAvailable = false; + + serialLogPrint("[i2c] FAULT #"); + serialLogPrint(g_tracker.faults()); + serialLogPrint(" at "); + serialLogPrint(now); + serialLogPrint(" ms:"); + logDevices(servoDriver, oled); + serialLogPrintln(); +} + +void recover() { + const bool ok = bringUp(); + g_tracker.noteRecovery(ok, millis()); + + if (ok) { + serialLogPrint("[i2c] recovered, attempt "); + serialLogPrint(g_tracker.attempts()); + serialLogPrint(", down "); + serialLogPrint(millis() - g_tracker.lastFaultMs()); + serialLogPrintln(" ms"); + return; + } + + serialLogPrint("[i2c] still down, attempt "); + serialLogPrint(g_tracker.attempts()); + serialLogPrint(":"); + logDevices(g_servoDriverUp, oledAvailable); + serialLogPrint(" retry in "); + serialLogPrint(g_tracker.retryDelayMs()); + serialLogPrintln(" ms"); +} + +} // namespace + +void initI2cBus() { + const BusLines lines = clearBus(); + + if (!lines.sdaHigh || !lines.sclHigh) { + serialLogPrint("[i2c] bus held at boot:"); + logLines(lines); + } + + Wire.begin(I2C_SDA, I2C_SCL); +} + +void beginI2cHealth() { + g_oledExpected = oledAvailable; + g_started = true; +} + +void updateI2cHealth(uint32_t now) { + if (!g_started) { + return; + } + + switch (g_tracker.due(now)) { + case I2cFaultAction::Check: + check(now); + break; + case I2cFaultAction::Recover: + recover(); + break; + case I2cFaultAction::None: + break; + } +} + +bool i2cServoDriverUp() { + return g_servoDriverUp; +} + +I2cHealth i2cHealth() { + return { + !g_tracker.down(), + g_servoDriverUp, + g_tracker.faults(), + g_tracker.recoveries(), + g_tracker.lastFaultMs() + }; +} diff --git a/src/hardware/i2c_health.h b/src/hardware/i2c_health.h new file mode 100644 index 00000000..39e88a98 --- /dev/null +++ b/src/hardware/i2c_health.h @@ -0,0 +1,30 @@ +#pragma once + +#include + +// Watches the I2C bus the PCA9685 and the OLED share. The Adafruit drivers +// ignore failed writes, so without this a dead bus leaves the robot frozen +// while every write waits out its timeout and starves audio and HTTP. + +// Frees a bus a device still holds from before a reset, then starts Wire. +void initI2cBus(); + +// Starts the checks. Call once the PCA9685 and the OLED are set up. +void beginI2cHealth(); + +// Probes the devices while the bus is up; restarts the bus and sets the +// devices up again while it is down. Cheap enough for every tick. +void updateI2cHealth(uint32_t now); + +// False while the PCA9685 cannot be reached: skip servo writes. +bool i2cServoDriverUp(); + +struct I2cHealth { + bool ok; + bool servoDriver; + uint32_t faults; + uint32_t recoveries; + uint32_t lastFaultMs; +}; + +I2cHealth i2cHealth(); diff --git a/src/hardware/servo_wrapper.cpp b/src/hardware/servo_wrapper.cpp index ebe5679e..29078d14 100644 --- a/src/hardware/servo_wrapper.cpp +++ b/src/hardware/servo_wrapper.cpp @@ -5,8 +5,10 @@ #include #include "animation/util.h" +#include "display/oled.h" #include "pins.h" #include "servos.h" +#include "hardware/i2c_health.h" #include "hardware/servo_wrapper.h" #include "serial_log.h" #include "settings/settings.h" @@ -19,6 +21,12 @@ constexpr const char* kPoseNs = "te"; constexpr const char* kPoseKey = "spose"; constexpr uint32_t kPoseSaveDebounceMs = 500; +constexpr uint16_t kServoPwmHz = 50; + +// Prescale the PCA9685 reported after setup. A chip that lost power comes +// back with its default, which is how a silent reset shows up. +static uint8_t g_pwmPrescale = 0; + static bool g_allOutputsReleased = false; static bool g_parkHoldUntilReboot = false; static bool g_poseDirty = false; @@ -238,11 +246,48 @@ void setServoParkHoldUntilReboot(bool hold) { g_parkHoldUntilReboot = hold; } +namespace { + +// Skipped while the PCA9685 is unreachable, so a dead bus does not stall the +// loop. restartServoPwmDriver() rewrites every channel once it is back. +void writeChannel(uint8_t channel, uint16_t off) { + if (!i2cServoDriverUp()) { + return; + } + + pwm.setPWM(channel, 0, off); +} + +} // namespace + +bool servoPwmDriverResponds() { + return i2cDeviceConnected(PCA9685_ADDRESS) && + pwm.readPrescale() == g_pwmPrescale; +} + +void restartServoPwmDriver() { + disableServoOutputs(); + + pwm.begin(); + pwm.setPWMFreq(kServoPwmHz); + g_pwmPrescale = pwm.readPrescale(); + + // The servos kept animating in software while the chip was unreachable; + // put every channel where the firmware thinks it is. + for (int i = 0; i < SERVO_COUNT; i++) { + g_servos[i].resyncOutput(); + } + + delay(10); + enableServoOutputs(); +} + void initServoPwmDriver() { disableServoOutputs(); pwm.begin(); - pwm.setPWMFreq(50); + pwm.setPWMFreq(kServoPwmHz); + g_pwmPrescale = pwm.readPrescale(); delay(100); uint8_t pose[SERVO_COUNT]; @@ -403,11 +448,7 @@ void ServoWrapper::ensureServoOutput() { lastPulse_ = UINT16_MAX; const uint16_t pulse = angleToPulse(angle_); - pwm.setPWM( - SERVO_SPECS[index_].channel, - 0, - pulse - ); + writeChannel(SERVO_SPECS[index_].channel, pulse); lastPulse_ = pulse; serialLogPrint("[servo] pwm on "); @@ -421,15 +462,24 @@ void ServoWrapper::releaseServoOutput() { return; } - pwm.setPWM( - SERVO_SPECS[index_].channel, - 0, - kPca9685FullOff - ); + writeChannel(SERVO_SPECS[index_].channel, kPca9685FullOff); lastPulse_ = UINT16_MAX; released_ = true; } +void ServoWrapper::resyncOutput() { + // Written straight to the chip: this runs while it still counts as down. + if (released_) { + pwm.setPWM(SERVO_SPECS[index_].channel, 0, kPca9685FullOff); + lastPulse_ = UINT16_MAX; + return; + } + + const uint16_t pulse = angleToPulse(angle_); + pwm.setPWM(SERVO_SPECS[index_].channel, 0, pulse); + lastPulse_ = pulse; +} + void ServoWrapper::writeAngle(float angle, bool log, bool electrical) { ensureServoOutput(); angle = clampAngle(index_, angle, electrical); @@ -446,11 +496,7 @@ void ServoWrapper::writeAngle(float angle, bool log, bool electrical) { } if (pulse != lastPulse_) { - pwm.setPWM( - SERVO_SPECS[index_].channel, - 0, - pulse - ); + writeChannel(SERVO_SPECS[index_].channel, pulse); lastPulse_ = pulse; } diff --git a/src/hardware/servo_wrapper.h b/src/hardware/servo_wrapper.h index 3f3f5dd0..b3a95e73 100644 --- a/src/hardware/servo_wrapper.h +++ b/src/hardware/servo_wrapper.h @@ -29,6 +29,8 @@ class ServoWrapper { bool isMoving() const; void ensureServoOutput(); void releaseServoOutput(); + // Rewrites this channel from the firmware's state after a driver restart. + void resyncOutput(); friend void servoMoveAllSmoothTo( const float targets[SERVO_COUNT], @@ -56,6 +58,10 @@ void initServoOutputPin(); void disableServoOutputs(); void enableServoOutputs(); void initServoPwmDriver(); +// True when the PCA9685 answers and still holds the setup it was given. +bool servoPwmDriverResponds(); +// Sets the PCA9685 up again and rewrites every channel; keeps the current pose. +void restartServoPwmDriver(); void servoMoveAllSmoothTo( const float targets[SERVO_COUNT], float speedDegS = SERVO_BOOT_SPEED_DEG_S diff --git a/src/http/health_handlers.cpp b/src/http/health_handlers.cpp index 583e942b..71348b30 100644 --- a/src/http/health_handlers.cpp +++ b/src/http/health_handlers.cpp @@ -4,6 +4,7 @@ #include "display/oled.h" #include "firmware_version.h" #include "hardware/chip_temp.h" +#include "hardware/i2c_health.h" #include "http/handlers.h" #include "http/response.h" #include "network/wifi_connect.h" @@ -51,5 +52,13 @@ void handleHealth(WebServer& server) { body["setup_ap_ssid"] = provisioning ? wifiApSsid() : ""; body["setup_ap_ip"] = provisioning ? wifiApIpText() : ""; body["oled"] = oledAvailable; + + const I2cHealth bus = i2cHealth(); + JsonObject i2c = body["i2c"].to(); + i2c["ok"] = bus.ok; + i2c["pca9685"] = bus.servoDriver; + i2c["faults"] = bus.faults; + i2c["recoveries"] = bus.recoveries; + i2c["last_fault_ms"] = bus.lastFaultMs; sendJson(server, 200, body); } diff --git a/src/main.cpp b/src/main.cpp index 6f844498..3e188bca 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -6,6 +6,7 @@ #include "firmware_version.h" #include "hardware/rgb.h" #include "hardware/chip_temp.h" +#include "hardware/i2c_health.h" #include "display/oled.h" #include "audio/audio.h" #include "animation.h" @@ -60,10 +61,7 @@ void setup() { serialLogPrintln("SDA = GP0"); serialLogPrintln("SCL = GP1"); - Wire.begin( - I2C_SDA, - I2C_SCL - ); + initI2cBus(); delay(100); @@ -71,6 +69,8 @@ void setup() { initOled(); + beginI2cHealth(); + const bool sleepServos = bootSleepInertiaUsesServos(); if (bootLoadingIsProgress()) { diff --git a/src/robot_tick.cpp b/src/robot_tick.cpp index c8c68021..9a61a207 100644 --- a/src/robot_tick.cpp +++ b/src/robot_tick.cpp @@ -5,6 +5,7 @@ #include "animation.h" #include "audio/wav_stream.h" #include "display/oled.h" +#include "hardware/i2c_health.h" #include "hardware/rgb.h" #include "hardware/servo_wrapper.h" #include "network/wifi_connect.h" @@ -12,6 +13,7 @@ void tickRobot() { const uint32_t now = millis(); + updateI2cHealth(now); pollWifi(); updateProvisioningOled(now); updateAnimation(); diff --git a/test/test_i2c_fault/test_i2c_fault.cpp b/test/test_i2c_fault/test_i2c_fault.cpp new file mode 100644 index 00000000..3cbd6f18 --- /dev/null +++ b/test/test_i2c_fault/test_i2c_fault.cpp @@ -0,0 +1,140 @@ +#include + +#include "hardware/i2c_fault.h" + +namespace { + +void expectAction(I2cFaultAction want, I2cFaultAction got) { + TEST_ASSERT_EQUAL(static_cast(want), static_cast(got)); +} + +// Fails enough checks in a row to take the bus down; returns the time of the last. +uint32_t takeDown(I2cFaultTracker& tracker, uint32_t start) { + uint32_t now = start; + for (uint8_t i = 0; i < kI2cFailuresToFault; i++) { + now = start + i * kI2cCheckIntervalMs; + tracker.noteCheck(false, now); + } + return now; +} + +} // namespace + +void test_checks_on_first_call_then_on_interval() { + I2cFaultTracker tracker; + expectAction(I2cFaultAction::Check, tracker.due(0)); + + tracker.noteCheck(true, 100); + expectAction(I2cFaultAction::None, tracker.due(100 + kI2cCheckIntervalMs - 1)); + expectAction(I2cFaultAction::Check, tracker.due(100 + kI2cCheckIntervalMs)); +} + +void test_one_failed_check_is_not_a_fault() { + I2cFaultTracker tracker; + TEST_ASSERT_FALSE(tracker.noteCheck(false, 1000)); + TEST_ASSERT_FALSE(tracker.down()); + TEST_ASSERT_EQUAL_UINT32(0, tracker.faults()); + + // A good check in between resets the count. + tracker.noteCheck(true, 1500); + TEST_ASSERT_FALSE(tracker.noteCheck(false, 2000)); + TEST_ASSERT_FALSE(tracker.down()); +} + +void test_consecutive_failures_take_the_bus_down() { + I2cFaultTracker tracker; + TEST_ASSERT_FALSE(tracker.noteCheck(false, 1000)); + TEST_ASSERT_TRUE(tracker.noteCheck(false, 1500)); + TEST_ASSERT_TRUE(tracker.down()); + TEST_ASSERT_EQUAL_UINT32(1, tracker.faults()); + TEST_ASSERT_EQUAL_UINT32(1500, tracker.lastFaultMs()); +} + +void test_first_recovery_attempt_is_immediate() { + I2cFaultTracker tracker; + const uint32_t at = takeDown(tracker, 1000); + expectAction(I2cFaultAction::Recover, tracker.due(at)); +} + +void test_failed_recovery_backs_off_to_the_cap() { + I2cFaultTracker tracker; + uint32_t now = takeDown(tracker, 1000); + + tracker.noteRecovery(false, now); + TEST_ASSERT_EQUAL_UINT32(kI2cFirstRetryMs, tracker.retryDelayMs()); + expectAction(I2cFaultAction::None, tracker.due(now + kI2cFirstRetryMs - 1)); + expectAction(I2cFaultAction::Recover, tracker.due(now + kI2cFirstRetryMs)); + + uint32_t previous = tracker.retryDelayMs(); + for (int i = 0; i < 12; i++) { + now += tracker.retryDelayMs(); + tracker.noteRecovery(false, now); + TEST_ASSERT_TRUE(tracker.retryDelayMs() >= previous); + TEST_ASSERT_TRUE(tracker.retryDelayMs() <= kI2cMaxRetryMs); + previous = tracker.retryDelayMs(); + } + + TEST_ASSERT_EQUAL_UINT32(kI2cMaxRetryMs, tracker.retryDelayMs()); + TEST_ASSERT_TRUE(tracker.down()); + TEST_ASSERT_EQUAL_UINT32(13, tracker.attempts()); + TEST_ASSERT_EQUAL_UINT32(0, tracker.recoveries()); +} + +void test_recovery_brings_the_bus_up_and_resumes_checks() { + I2cFaultTracker tracker; + const uint32_t at = takeDown(tracker, 1000); + + tracker.noteRecovery(false, at); + tracker.noteRecovery(true, at + kI2cFirstRetryMs); + + TEST_ASSERT_FALSE(tracker.down()); + TEST_ASSERT_EQUAL_UINT32(1, tracker.recoveries()); + TEST_ASSERT_EQUAL_UINT32(2, tracker.attempts()); + expectAction(I2cFaultAction::None, tracker.due(at + kI2cFirstRetryMs)); + expectAction( + I2cFaultAction::Check, + tracker.due(at + kI2cFirstRetryMs + kI2cCheckIntervalMs) + ); +} + +void test_second_fault_starts_over() { + I2cFaultTracker tracker; + uint32_t now = takeDown(tracker, 1000); + tracker.noteRecovery(false, now); + tracker.noteRecovery(false, now + kI2cFirstRetryMs); + tracker.noteRecovery(true, now + 3 * kI2cFirstRetryMs); + + now = takeDown(tracker, 60000); + TEST_ASSERT_TRUE(tracker.down()); + TEST_ASSERT_EQUAL_UINT32(2, tracker.faults()); + TEST_ASSERT_EQUAL_UINT32(0, tracker.attempts()); + // Back to an immediate first attempt, not the old backoff. + expectAction(I2cFaultAction::Recover, tracker.due(now)); +} + +void test_intervals_survive_millis_wrap() { + I2cFaultTracker tracker; + const uint32_t beforeWrap = UINT32_MAX - 100; + tracker.noteCheck(true, beforeWrap); + expectAction(I2cFaultAction::None, tracker.due(beforeWrap + 200)); + expectAction(I2cFaultAction::Check, tracker.due(beforeWrap + kI2cCheckIntervalMs)); + + takeDown(tracker, beforeWrap + kI2cCheckIntervalMs); + const uint32_t failedAt = beforeWrap + 2 * kI2cCheckIntervalMs; + tracker.noteRecovery(false, failedAt); + expectAction(I2cFaultAction::None, tracker.due(failedAt + kI2cFirstRetryMs - 1)); + expectAction(I2cFaultAction::Recover, tracker.due(failedAt + kI2cFirstRetryMs)); +} + +int main(int, char**) { + UNITY_BEGIN(); + RUN_TEST(test_checks_on_first_call_then_on_interval); + RUN_TEST(test_one_failed_check_is_not_a_fault); + RUN_TEST(test_consecutive_failures_take_the_bus_down); + RUN_TEST(test_first_recovery_attempt_is_immediate); + RUN_TEST(test_failed_recovery_backs_off_to_the_cap); + RUN_TEST(test_recovery_brings_the_bus_up_and_resumes_checks); + RUN_TEST(test_second_fault_starts_over); + RUN_TEST(test_intervals_survive_millis_wrap); + return UNITY_END(); +} From b84376a3ea37981777e9a97c91d08826fa21fa66 Mon Sep 17 00:00:00 2001 From: Blake Azuela Date: Mon, 16 Nov 2026 12:26:00 -0500 Subject: [PATCH 02/10] fix(firmware): restore and watch the PCA9685 output mode Bench testing the I2C recovery left a robot whose eyes animated while its servos stayed still. The firmware saw nothing wrong: the PCA9685 answered, held its prescale and accepted every write. A register dump taken before setup showed MODE2 at 0x10 (inverted, open-drain) where a servo signal needs 0x04, so no usable pulse reached the servos. The Adafruit driver never writes MODE2, and the chip keeps its registers across an ESP32 reset, so the value survived every reboot and every recovery. It was most likely written by a transfer corrupted during fault injection, which forced SDA low mid-write; a real bus glitch can do the same. Write MODE2 as part of the PCA9685 setup, at boot and after a recovery. Include MODE1's sleep bit and MODE2 in the health check, so a chip that answers but lost its setup counts as a fault and is set up again. The fault log line now carries the three register values. On the robot: setup brought MODE2 back to 0x04 and the servos moved again. Writing 0x10 by hand was caught after 0.9 s with "pca9685=fail oled=ok mode1=0x20 mode2=0x10 prescale=0x79" and repaired in 58 ms. --- docs/api.md | 2 +- src/hardware/i2c_health.cpp | 14 ++++++- src/hardware/servo_wrapper.cpp | 71 +++++++++++++++++++++++++++++----- src/hardware/servo_wrapper.h | 11 +++++- 4 files changed, 86 insertions(+), 12 deletions(-) diff --git a/docs/api.md b/docs/api.md index a423a31f..97cfe619 100644 --- a/docs/api.md +++ b/docs/api.md @@ -149,7 +149,7 @@ curl -H "Authorization: Bearer YOUR_TOKEN" http://tiny-engineer.local/health | `setup_ap_ip` | Setup AP IP when `provisioning` is true (usually `192.168.4.1`), else `""` | | `oled` | SSD1306 probed and initialized. Goes `false` while an I2C fault keeps the display unreachable | | `i2c.ok` | `false` while the I2C bus shared by the PCA9685 and the OLED is down. The firmware checks both every 500 ms and, on a fault, restarts the bus and sets the chips up again, retrying with a growing delay (1 s up to 30 s) | -| `i2c.pca9685` | Servo driver reachable. Servo writes are skipped while `false` | +| `i2c.pca9685` | Servo driver reachable and still holding its setup (awake, output mode, prescale). Servo writes are skipped while `false` | | `i2c.faults` | I2C faults since boot | | `i2c.recoveries` | Faults the firmware recovered from since boot | | `i2c.last_fault_ms` | `uptime_ms` at the latest fault; `0` when there has been none | diff --git a/src/hardware/i2c_health.cpp b/src/hardware/i2c_health.cpp index 67b36612..c9d85541 100644 --- a/src/hardware/i2c_health.cpp +++ b/src/hardware/i2c_health.cpp @@ -112,7 +112,8 @@ bool bringUp() { } void check(uint32_t now) { - const bool servoDriver = servoPwmDriverResponds(); + ServoPwmDriverState pca; + const bool servoDriver = servoPwmDriverState(pca); const bool oled = !g_oledExpected || i2cDeviceConnected(OLED_ADDRESS); if (!g_tracker.noteCheck(servoDriver && oled, now)) { @@ -130,6 +131,17 @@ void check(uint32_t now) { serialLogPrint(now); serialLogPrint(" ms:"); logDevices(servoDriver, oled); + + // It answered but lost its setup: the registers say what changed. + if (!servoDriver && pca.answered) { + serialLogPrint(" mode1=0x"); + serialLogPrint(pca.mode1, HEX); + serialLogPrint(" mode2=0x"); + serialLogPrint(pca.mode2, HEX); + serialLogPrint(" prescale=0x"); + serialLogPrint(pca.prescale, HEX); + } + serialLogPrintln(); } diff --git a/src/hardware/servo_wrapper.cpp b/src/hardware/servo_wrapper.cpp index 29078d14..64b2eed2 100644 --- a/src/hardware/servo_wrapper.cpp +++ b/src/hardware/servo_wrapper.cpp @@ -1,11 +1,11 @@ #include +#include #include #include #include #include #include "animation/util.h" -#include "display/oled.h" #include "pins.h" #include "servos.h" #include "hardware/i2c_health.h" @@ -23,6 +23,14 @@ constexpr uint32_t kPoseSaveDebounceMs = 500; constexpr uint16_t kServoPwmHz = 50; +constexpr uint8_t kPcaMode1Reg = 0x00; +constexpr uint8_t kPcaMode2Reg = 0x01; +constexpr uint8_t kPcaMode1Sleep = 0x10; +// Totem-pole outputs, not inverted: what a servo signal line needs. Any other +// value leaves the servos without a usable pulse while every write still +// succeeds. +constexpr uint8_t kPcaMode2Servo = 0x04; + // Prescale the PCA9685 reported after setup. A chip that lost power comes // back with its default, which is how a silent reset shows up. static uint8_t g_pwmPrescale = 0; @@ -258,19 +266,66 @@ void writeChannel(uint8_t channel, uint16_t off) { pwm.setPWM(channel, 0, off); } +bool readPcaRegister(uint8_t reg, uint8_t& value) { + Wire.beginTransmission(PCA9685_ADDRESS); + Wire.write(reg); + + if (Wire.endTransmission(false) != 0) { + return false; + } + + if (Wire.requestFrom(PCA9685_ADDRESS, static_cast(1)) != 1) { + return false; + } + + value = static_cast(Wire.read()); + return true; +} + +// Full PCA9685 setup, shared by boot and recovery. +void configureServoPwmDriver() { + pwm.begin(); + + // The Adafruit driver never writes MODE2, so a value corrupted by a bus + // glitch would outlive every restart. The chip keeps its registers across + // an ESP32 reset: only its own power cycle clears them. + Wire.beginTransmission(PCA9685_ADDRESS); + Wire.write(kPcaMode2Reg); + Wire.write(kPcaMode2Servo); + Wire.endTransmission(); + + pwm.setPWMFreq(kServoPwmHz); + g_pwmPrescale = pwm.readPrescale(); +} + } // namespace +bool servoPwmDriverState(ServoPwmDriverState& state) { + state = {}; + state.answered = + readPcaRegister(kPcaMode1Reg, state.mode1) && + readPcaRegister(kPcaMode2Reg, state.mode2); + + if (!state.answered) { + return false; + } + + state.prescale = pwm.readPrescale(); + + return (state.mode1 & kPcaMode1Sleep) == 0 && + state.mode2 == kPcaMode2Servo && + state.prescale == g_pwmPrescale; +} + bool servoPwmDriverResponds() { - return i2cDeviceConnected(PCA9685_ADDRESS) && - pwm.readPrescale() == g_pwmPrescale; + ServoPwmDriverState state; + return servoPwmDriverState(state); } void restartServoPwmDriver() { disableServoOutputs(); - pwm.begin(); - pwm.setPWMFreq(kServoPwmHz); - g_pwmPrescale = pwm.readPrescale(); + configureServoPwmDriver(); // The servos kept animating in software while the chip was unreachable; // put every channel where the firmware thinks it is. @@ -285,9 +340,7 @@ void restartServoPwmDriver() { void initServoPwmDriver() { disableServoOutputs(); - pwm.begin(); - pwm.setPWMFreq(kServoPwmHz); - g_pwmPrescale = pwm.readPrescale(); + configureServoPwmDriver(); delay(100); uint8_t pose[SERVO_COUNT]; diff --git a/src/hardware/servo_wrapper.h b/src/hardware/servo_wrapper.h index b3a95e73..cee37966 100644 --- a/src/hardware/servo_wrapper.h +++ b/src/hardware/servo_wrapper.h @@ -58,7 +58,16 @@ void initServoOutputPin(); void disableServoOutputs(); void enableServoOutputs(); void initServoPwmDriver(); -// True when the PCA9685 answers and still holds the setup it was given. +struct ServoPwmDriverState { + bool answered; + uint8_t mode1; + uint8_t mode2; + uint8_t prescale; +}; + +// Reads the PCA9685's setup registers. True when it answers, is awake, and +// still holds the output mode and prescale it was given. +bool servoPwmDriverState(ServoPwmDriverState& state); bool servoPwmDriverResponds(); // Sets the PCA9685 up again and rewrites every channel; keeps the current pose. void restartServoPwmDriver(); From 8e841f670f34ac50d7fd5271224ac7480031b953 Mon Sep 17 00:00:00 2001 From: Blake Azuela Date: Mon, 16 Nov 2026 13:21:21 -0500 Subject: [PATCH 03/10] fix(firmware): track each I2C chip on its own and guard the bus restart An independent review of the I2C recovery found defects that the single bus-wide tracker caused, and one hang. - One tracker meant no checks while it was down. A chip that never came back left the other unchecked for up to 30 s, so a later bus hang could stall the loop again. - An OLED fault always reset the PCA9685: outputs off, chip reset, servos resynced. - A failed Wire restart could hang the loop for good. Wire.end() drops its buffers even when it fails, the next transfer then leaks Wire's lock, and the following Wire.end() waits on that lock forever. - The PCA9685 prescale reference was read back from the chip, so a lost write at setup became the baseline, and a lost auto-increment bit went unnoticed. - Servo outputs were re-enabled before the setup was verified. Give each chip its own fault tracker. The PCA9685's recovery redoes only its setup when the chip still answers, and restarts the bus otherwise; it then sets the OLED up again, since both share a supply. The OLED's recovery never restarts the bus. Nothing uses Wire after a failed restart until a later one succeeds. The PCA9685 check compares against a computed prescale and requires auto-increment, and a restart leaves the outputs disabled unless the setup reads back correctly. GET /health now reports ok, faults, recoveries and last_fault_ms per chip under i2c.pca9685 and i2c.oled. Serial lines name the chip. Document the runtime checks in hardware/interfaces.md and add the [i2c] lines to the troubleshooting table in hardware/testing.md. On the robot, with a temporary injector: a held SDA line took both chips down and both recovered after release; a corrupted MODE2 was repaired in 55 ms with no bus restart; a display-only fault recovered without touching the PCA9685. A healthy check of both chips takes about 1.7 ms every 500 ms. --- docs/api.md | 17 +- ...8-tools-debugging-and-embedded-workflow.md | 2 +- docs/hardware/interfaces.md | 17 +- docs/hardware/testing.md | 10 +- src/hardware/i2c_health.cpp | 301 ++++++++++++------ src/hardware/i2c_health.h | 15 +- src/hardware/servo_wrapper.cpp | 43 +-- src/hardware/servo_wrapper.h | 8 +- src/http/health_handlers.cpp | 17 +- 9 files changed, 290 insertions(+), 140 deletions(-) diff --git a/docs/api.md b/docs/api.md index 97cfe619..edfb03c9 100644 --- a/docs/api.md +++ b/docs/api.md @@ -123,10 +123,8 @@ curl -H "Authorization: Bearer YOUR_TOKEN" http://tiny-engineer.local/health "oled": true, "i2c": { "ok": true, - "pca9685": true, - "faults": 0, - "recoveries": 0, - "last_fault_ms": 0 + "pca9685": { "ok": true, "faults": 0, "recoveries": 0, "last_fault_ms": 0 }, + "oled": { "ok": true, "faults": 0, "recoveries": 0, "last_fault_ms": 0 } } } ``` @@ -148,11 +146,12 @@ curl -H "Authorization: Bearer YOUR_TOKEN" http://tiny-engineer.local/health | `setup_ap_ssid` | Setup AP SSID when `provisioning` is true, else `""` | | `setup_ap_ip` | Setup AP IP when `provisioning` is true (usually `192.168.4.1`), else `""` | | `oled` | SSD1306 probed and initialized. Goes `false` while an I2C fault keeps the display unreachable | -| `i2c.ok` | `false` while the I2C bus shared by the PCA9685 and the OLED is down. The firmware checks both every 500 ms and, on a fault, restarts the bus and sets the chips up again, retrying with a growing delay (1 s up to 30 s) | -| `i2c.pca9685` | Servo driver reachable and still holding its setup (awake, output mode, prescale). Servo writes are skipped while `false` | -| `i2c.faults` | I2C faults since boot | -| `i2c.recoveries` | Faults the firmware recovered from since boot | -| `i2c.last_fault_ms` | `uptime_ms` at the latest fault; `0` when there has been none | +| `i2c.ok` | `false` while either chip on the shared I2C bus is in a fault. The firmware checks each every 500 ms and sets a faulted chip up again, retrying with a growing delay (1 s up to 30 s) | +| `i2c.pca9685.ok` | Servo driver reachable and still holding its setup (awake, auto-increment, output mode, prescale). Servo writes are skipped while `false` | +| `i2c.oled.ok` | Same value as `oled`. Stays `false`, with zero counts, on a robot that booted without a display | +| `i2c..faults` | Faults on that chip since boot. A bus fault that takes both chips down counts once on each | +| `i2c..recoveries` | Faults on that chip the firmware recovered from since boot | +| `i2c..last_fault_ms` | `uptime_ms` at that chip's latest fault; `0` when there has been none | ### `GET /settings` diff --git a/docs/hardware-for-software-engineers/08-tools-debugging-and-embedded-workflow.md b/docs/hardware-for-software-engineers/08-tools-debugging-and-embedded-workflow.md index 35082317..dbf6391b 100644 --- a/docs/hardware-for-software-engineers/08-tools-debugging-and-embedded-workflow.md +++ b/docs/hardware-for-software-engineers/08-tools-debugging-and-embedded-workflow.md @@ -86,7 +86,7 @@ Failure table and expected boot sequence: [testing.md](../hardware/testing.md). Once on Wi-Fi, the robot exposes **integration-test-style endpoints**: -- `/health` — alive, CPU temp +- `/health` — alive, CPU temp, I2C fault and recovery counts - `/test/servo` — single joint - Web UI — hardware tests without memorizing curl diff --git a/docs/hardware/interfaces.md b/docs/hardware/interfaces.md index 7737c056..3889b776 100644 --- a/docs/hardware/interfaces.md +++ b/docs/hardware/interfaces.md @@ -11,12 +11,25 @@ Shared bus. | SDA | GPIO0 | GP0 → PCA9685 SDA, OLED SDA | | SCL | GPIO1 | GP1 → PCA9685 SCL, Waveshare OLED **SCL** | -| Device | Address | Firmware behaviour if missing | +| Device | Address | Firmware behaviour if missing at boot | | --- | --- | --- | | PCA9685 | **`0x40`** | Hard fail: hang, 1 red blink (see [blink codes](testing.md#boot-failure-blink-codes)) | | SSD1306 OLED | **`0x3C`** | Soft fail: tests continue without display | -Bus start: `Wire.begin(I2C_SDA, I2C_SCL)` with no explicit frequency → Arduino-ESP32 default (typically **100 kHz**). Not set in this repo. +That table is the boot check. After boot the firmware keeps watching both devices (`src/hardware/i2c_health.cpp`): + +- **Check:** every 500 ms the OLED must answer, and the PCA9685 must answer, be awake with auto-increment on, and still hold its output mode (`MODE2 = 0x04`) and prescale. Two failed checks in a row are a fault. Each chip is tracked separately, so one that stays down does not stop the checks on the other. A healthy check costs about 1.7 ms per 500 ms. +- **While down:** writes to the faulted chip are skipped, so failed transfers do not stall audio and HTTP. +- **PCA9685 recovery:** if the chip still answers, only its setup is redone. Otherwise restart `Wire` and clock out a device left holding SDA first. Then rewrite every servo channel, and set the OLED up again too, since both share a supply. Servos jump to where the firmware thinks they are. Outputs stay disabled if the setup does not read back correctly. +- **OLED recovery:** set the panel up again. Never restarts the bus, so an OLED fault alone does not disturb the servos. +- **Retry:** a failed attempt is retried after 1 s, doubling up to 30 s. +- **Report:** `[i2c]` lines on the serial log (with SDA / SCL levels and, for a PCA9685 that answered, its registers) and the `i2c` object in [`GET /health`](../api.md#get-health). + +The OLED check is an address ACK only. A panel that loses its setup but still answers is not detected by itself; it is set up again whenever the PCA9685 recovers. + +The PCA9685 keeps its registers across an ESP32 reset; only its own power cycle clears them. That is why setup writes `MODE2` explicitly instead of trusting the power-on default. + +Bus start: the lines are cleared the same way, then `Wire.begin(I2C_SDA, I2C_SCL)` with no explicit frequency → Arduino-ESP32 default (typically **100 kHz**). Not set in this repo. PCA9685 **VCC** = **3V3**. OLED **VCC** = **3V3**. Pull-ups are on the main control board and on a typical OLED; the Adafruit PCA9685 breakout has its own. No extra pull-ups. diff --git a/docs/hardware/testing.md b/docs/hardware/testing.md index e820ebf1..96ec127d 100644 --- a/docs/hardware/testing.md +++ b/docs/hardware/testing.md @@ -147,8 +147,14 @@ HTTP runs on STA when connected, or on setup AP at `192.168.4.1` during provisio | I2S OK but `POST /test/audio` is silent | Amp or speaker | MAX98357A **Vin**=USB 5V, GND, GP2/3/4 → BCLK/LRC/DIN, speaker on **SPK+ / SPK-** (not on the PNG, not GND) | | Servos silent / twitch / ESP32 resets during `POST /test/movement` | Power or SIG | **V+** is +5V, SIG on ch 0–4, **common GND**, supply current — see [power.md](power.md) | | RGB never goes green | GPIO10 LED path | Board is C3-Zero (LED on GPIO10). Do not expect an external NeoPixel | - -OLED absence and Wi-Fi failure are **soft** fails. PCA9685 and I2S failures **halt** in `while (true)`. +| `[i2c] pca9685 FAULT #N` and `[i2c] oled FAULT #N` together, then `[i2c] bus restarted` and `recovered` lines | Both chips stopped answering after boot; firmware restarted the bus | `sda=LOW` / `scl=LOW` on the `bus restarted` line means a line was held: supply dip or wiring. Check supply current — see [power.md](power.md) | +| `[i2c] pca9685 FAULT #N … mode1=… mode2=… prescale=…` | PCA9685 answered but lost its setup (reset to defaults, or a corrupted write) | Recovers by itself without a bus restart. Repeats point at supply dips or I2C noise | +| `[i2c] oled FAULT #N` alone | Only the OLED stopped answering | The servos are left alone. OLED **VCC**, GND, SDA, SCL, connector | +| `[i2c] still down … retry in … ms` keeps repeating | Bus or chip not coming back | Wiring, PCA9685 / OLED **VCC**, then a full power cycle | +| `[i2c] Wire.end failed` or `[i2c] Wire.begin failed` | The ESP32's I2C driver did not restart; both chips stay off until a later attempt succeeds | Report it with the serial log; a reset clears it | +| `i2c..faults` above `0` in `GET /health` | A fault happened since boot | `recoveries` equal to `faults` means all were recovered; `last_fault_ms` says when | + +OLED absence and Wi-Fi failure are **soft** fails. PCA9685 and I2S failures **halt** in `while (true)`. I2C faults after boot do not halt: the firmware sets the faulted chip up again, restarting the bus when the PCA9685 stops answering ([interfaces.md](interfaces.md#i2c)). ## Power during the servo test diff --git a/src/hardware/i2c_health.cpp b/src/hardware/i2c_health.cpp index c9d85541..e773c8ff 100644 --- a/src/hardware/i2c_health.cpp +++ b/src/hardware/i2c_health.cpp @@ -14,17 +14,49 @@ namespace { constexpr uint8_t kBusClearPulses = 9; constexpr uint32_t kBusClearHalfPeriodUs = 5; -I2cFaultTracker g_tracker; -bool g_started = false; -// The robot runs without an OLED; only watch one that was there at boot. -bool g_oledExpected = false; -bool g_servoDriverUp = true; - struct BusLines { bool sdaHigh; bool sclHigh; }; +// One watched chip. Each has its own tracker, so a chip that stays down does +// not stop the checks on the other, and a fault on one does not restart both. +struct Device { + const char* name; + bool (*healthy)(); + void (*stopWrites)(); + bool (*recover)(); + // Extra text for the fault log line; null when there is none. + void (*logFaultDetail)(); + I2cFaultTracker tracker; +}; + +bool servoDriverHealthy(); +void stopServoWrites(); +bool recoverServoDriver(); +void logServoDriverRegisters(); +bool oledHealthy(); +void stopOledWrites(); +bool recoverOled(); + +Device g_servoDriver = { + "pca9685", servoDriverHealthy, stopServoWrites, recoverServoDriver, logServoDriverRegisters, {} +}; +Device g_oled = {"oled", oledHealthy, stopOledWrites, recoverOled, nullptr, {}}; + +bool g_started = false; +// The robot runs without an OLED; only watch one that was there at boot. +bool g_oledExpected = false; +bool g_servoDriverUp = true; +// False after a failed Wire restart. Wire must not be used then: without its +// buffers a transfer leaks Wire's lock, and the next restart would wait on +// that lock forever. +bool g_wireUp = true; +// False while the last bus restart found a line held low. +bool g_linesFree = true; +// Registers from the latest PCA9685 check, for the fault log line. +ServoPwmDriverState g_servoDriverState = {}; + // A device interrupted mid-byte keeps SDA low until it has been clocked // through the rest of that byte. Clock it out, send a STOP, and report where // the lines settled: a line still low means something is holding the bus. @@ -66,105 +98,193 @@ void logLines(const BusLines& lines) { serialLogPrintln(lines.sclHigh ? "high" : "LOW"); } -void logDevices(bool servoDriver, bool oled) { - serialLogPrint(" pca9685="); - serialLogPrint(servoDriver ? "ok" : "fail"); - serialLogPrint(" oled="); - serialLogPrint(!g_oledExpected ? "absent" : oled ? "ok" : "fail"); -} +void logFault(const Device& device, uint32_t now) { + serialLogPrint("[i2c] "); + serialLogPrint(device.name); + serialLogPrint(" FAULT #"); + serialLogPrint(device.tracker.faults()); + serialLogPrint(" at "); + serialLogPrint(now); + serialLogPrint(" ms"); -// Restarts whatever is down. Returns true when every expected device is up. -bool bringUp() { - bool busUsable = true; + if (device.logFaultDetail != nullptr) { + device.logFaultDetail(); + } - // A PCA9685 that still answers proves the bus works: leave it alone and - // only retry the OLED. - if (!(g_servoDriverUp && servoPwmDriverResponds())) { - g_servoDriverUp = false; - oledAvailable = false; + serialLogPrintln(); +} - Wire.end(); - const BusLines lines = clearBus(); - Wire.begin(I2C_SDA, I2C_SCL); +void noteRecovery(Device& device, bool ok) { + const uint32_t now = millis(); + device.tracker.noteRecovery(ok, now); - serialLogPrint("[i2c] bus restarted"); - logLines(lines); + serialLogPrint("[i2c] "); + serialLogPrint(device.name); - // A held line fails every transfer; skip the probes and their timeouts. - busUsable = lines.sdaHigh && lines.sclHigh; + if (ok) { + serialLogPrint(" recovered, attempt "); + serialLogPrint(device.tracker.attempts()); + serialLogPrint(", down "); + serialLogPrint(now - device.tracker.lastFaultMs()); + serialLogPrintln(" ms"); + return; + } - if (busUsable && i2cDeviceConnected(PCA9685_ADDRESS)) { - restartServoPwmDriver(); - g_servoDriverUp = servoPwmDriverResponds(); - } + serialLogPrint(" still down, attempt "); + serialLogPrint(device.tracker.attempts()); + serialLogPrint(", retry in "); + serialLogPrint(device.tracker.retryDelayMs()); + serialLogPrintln(" ms"); +} + +// Restarts Wire with the lines freed. False when Wire did not come back or a +// line is still held: every transfer would fail, so callers skip the probes +// and their timeouts. +bool restartBus() { + g_wireUp = false; + + if (!Wire.end()) { + // Wire dropped its buffers but kept its driver. begin() would report + // success without them, so leave it down and end() again next attempt. + stopOledWrites(); + serialLogPrintln("[i2c] Wire.end failed"); + return false; } - if (busUsable && g_oledExpected && !oledAvailable && restartOled()) { - // The restart turned the panel on; a sleeping robot keeps it dark. Awake, - // the next eye frame or setup screen redraws it. - if (isSleeping()) { - blankOled(); - sleepOled(); - } + const BusLines lines = clearBus(); + g_linesFree = lines.sdaHigh && lines.sclHigh; + g_wireUp = Wire.begin(I2C_SDA, I2C_SCL); + + if (!g_wireUp) { + stopOledWrites(); } - return g_servoDriverUp && (!g_oledExpected || oledAvailable); + serialLogPrint(g_wireUp ? "[i2c] bus restarted" : "[i2c] Wire.begin failed"); + logLines(lines); + + return g_wireUp && g_linesFree; } -void check(uint32_t now) { - ServoPwmDriverState pca; - const bool servoDriver = servoPwmDriverState(pca); - const bool oled = !g_oledExpected || i2cDeviceConnected(OLED_ADDRESS); +// The restart leaves the panel lit; a sleeping robot keeps it dark. Awake, +// the next eye frame or setup screen redraws it. +bool restartOledForSleepState() { + if (!restartOled()) { + return false; + } + + if (isSleeping()) { + blankOled(); + sleepOled(); + } + + return true; +} - if (!g_tracker.noteCheck(servoDriver && oled, now)) { +// Both chips share a supply, so a PCA9685 that had to be set up again is +// reason to set the panel up again too, even when it still answers. +void restartOledAfterBusRecovery() { + if (!g_oledExpected) { return; } - // Stop writing to both: they share the bus, and each failed write blocks - // the loop for its timeout. + const bool ok = restartOledForSleepState(); + + if (g_oled.tracker.down()) { + noteRecovery(g_oled, ok); + } +} + +bool servoDriverHealthy() { + g_servoDriverState = {}; + return g_wireUp && servoPwmDriverState(g_servoDriverState); +} + +void stopServoWrites() { g_servoDriverUp = false; - oledAvailable = false; +} - serialLogPrint("[i2c] FAULT #"); - serialLogPrint(g_tracker.faults()); - serialLogPrint(" at "); - serialLogPrint(now); - serialLogPrint(" ms:"); - logDevices(servoDriver, oled); +// A PCA9685 that answered but lost its setup: the registers say what changed. +void logServoDriverRegisters() { + if (!g_servoDriverState.answered) { + return; + } + + serialLogPrint(" mode1=0x"); + serialLogPrint(g_servoDriverState.mode1, HEX); + serialLogPrint(" mode2=0x"); + serialLogPrint(g_servoDriverState.mode2, HEX); + serialLogPrint(" prescale=0x"); + serialLogPrint(g_servoDriverState.prescale, HEX); +} + +bool servoDriverAnswers() { + return g_wireUp && i2cDeviceConnected(PCA9685_ADDRESS); +} - // It answered but lost its setup: the registers say what changed. - if (!servoDriver && pca.answered) { - serialLogPrint(" mode1=0x"); - serialLogPrint(pca.mode1, HEX); - serialLogPrint(" mode2=0x"); - serialLogPrint(pca.mode2, HEX); - serialLogPrint(" prescale=0x"); - serialLogPrint(pca.prescale, HEX); +// The PCA9685 is the required chip, so its recovery is the one that restarts +// the bus. A chip that still answers proves the bus works: then only its +// setup is redone. +bool recoverServoDriver() { + bool reachable = servoDriverAnswers(); + + if (!reachable) { + reachable = restartBus() && servoDriverAnswers(); } - serialLogPrintln(); + if (!reachable) { + return false; + } + + // An answer proves the lines are free, whatever the last restart saw. + g_linesFree = true; + + g_servoDriverUp = restartServoPwmDriver(); + + if (g_servoDriverUp) { + restartOledAfterBusRecovery(); + } + + return g_servoDriverUp; } -void recover() { - const bool ok = bringUp(); - g_tracker.noteRecovery(ok, millis()); +bool oledHealthy() { + return g_wireUp && i2cDeviceConnected(OLED_ADDRESS); +} - if (ok) { - serialLogPrint("[i2c] recovered, attempt "); - serialLogPrint(g_tracker.attempts()); - serialLogPrint(", down "); - serialLogPrint(millis() - g_tracker.lastFaultMs()); - serialLogPrintln(" ms"); - return; +void stopOledWrites() { + oledAvailable = false; +} + +// Never restarts the bus: an OLED fault alone must not disturb the servos. +// A dead bus is left to the PCA9685's recovery. +bool recoverOled() { + return g_wireUp && g_linesFree && restartOledForSleepState(); +} + +void service(Device& device, uint32_t now) { + switch (device.tracker.due(now)) { + case I2cFaultAction::Check: + if (device.tracker.noteCheck(device.healthy(), now)) { + // Each failed write blocks the loop for its timeout. + device.stopWrites(); + logFault(device, now); + } + break; + case I2cFaultAction::Recover: + noteRecovery(device, device.recover()); + break; + case I2cFaultAction::None: + break; } +} - serialLogPrint("[i2c] still down, attempt "); - serialLogPrint(g_tracker.attempts()); - serialLogPrint(":"); - logDevices(g_servoDriverUp, oledAvailable); - serialLogPrint(" retry in "); - serialLogPrint(g_tracker.retryDelayMs()); - serialLogPrintln(" ms"); +I2cDeviceHealth deviceHealth(const Device& device, bool up) { + return { + up, + device.tracker.faults(), + device.tracker.recoveries(), + device.tracker.lastFaultMs() + }; } } // namespace @@ -177,7 +297,7 @@ void initI2cBus() { logLines(lines); } - Wire.begin(I2C_SDA, I2C_SCL); + g_wireUp = Wire.begin(I2C_SDA, I2C_SCL); } void beginI2cHealth() { @@ -190,15 +310,10 @@ void updateI2cHealth(uint32_t now) { return; } - switch (g_tracker.due(now)) { - case I2cFaultAction::Check: - check(now); - break; - case I2cFaultAction::Recover: - recover(); - break; - case I2cFaultAction::None: - break; + service(g_servoDriver, now); + + if (g_oledExpected) { + service(g_oled, now); } } @@ -208,10 +323,8 @@ bool i2cServoDriverUp() { I2cHealth i2cHealth() { return { - !g_tracker.down(), - g_servoDriverUp, - g_tracker.faults(), - g_tracker.recoveries(), - g_tracker.lastFaultMs() + !g_servoDriver.tracker.down() && !g_oled.tracker.down(), + deviceHealth(g_servoDriver, g_servoDriverUp), + deviceHealth(g_oled, oledAvailable) }; } diff --git a/src/hardware/i2c_health.h b/src/hardware/i2c_health.h index 39e88a98..a2ec14ca 100644 --- a/src/hardware/i2c_health.h +++ b/src/hardware/i2c_health.h @@ -12,19 +12,24 @@ void initI2cBus(); // Starts the checks. Call once the PCA9685 and the OLED are set up. void beginI2cHealth(); -// Probes the devices while the bus is up; restarts the bus and sets the -// devices up again while it is down. Cheap enough for every tick. +// Checks each chip while it is up and sets it up again while it is down. +// Cheap enough for every tick. void updateI2cHealth(uint32_t now); -// False while the PCA9685 cannot be reached: skip servo writes. +// False while the PCA9685 cannot be used: skip servo writes. bool i2cServoDriverUp(); -struct I2cHealth { +struct I2cDeviceHealth { bool ok; - bool servoDriver; uint32_t faults; uint32_t recoveries; uint32_t lastFaultMs; }; +struct I2cHealth { + bool ok; + I2cDeviceHealth servoDriver; + I2cDeviceHealth oled; +}; + I2cHealth i2cHealth(); diff --git a/src/hardware/servo_wrapper.cpp b/src/hardware/servo_wrapper.cpp index 64b2eed2..c564b84f 100644 --- a/src/hardware/servo_wrapper.cpp +++ b/src/hardware/servo_wrapper.cpp @@ -25,15 +25,19 @@ constexpr uint16_t kServoPwmHz = 50; constexpr uint8_t kPcaMode1Reg = 0x00; constexpr uint8_t kPcaMode2Reg = 0x01; -constexpr uint8_t kPcaMode1Sleep = 0x10; +// Awake with register auto-increment on: setPWM writes four registers in one +// transfer and needs it. +constexpr uint8_t kPcaMode1Mask = MODE1_SLEEP | MODE1_AI; +constexpr uint8_t kPcaMode1Servo = MODE1_AI; // Totem-pole outputs, not inverted: what a servo signal line needs. Any other // value leaves the servos without a usable pulse while every write still // succeeds. -constexpr uint8_t kPcaMode2Servo = 0x04; - -// Prescale the PCA9685 reported after setup. A chip that lost power comes -// back with its default, which is how a silent reset shows up. -static uint8_t g_pwmPrescale = 0; +constexpr uint8_t kPcaMode2Servo = MODE2_OUTDRV; +// What setPWMFreq(kServoPwmHz) programs, by the Adafruit driver's own formula. +// Computed, not read back: a readback after a lost write would bless the +// wrong value. A chip that lost power comes back with a different default. +constexpr uint8_t kPcaPrescaleServo = + static_cast(FREQUENCY_OSCILLATOR / (kServoPwmHz * 4096.0) + 0.5 - 1); static bool g_allOutputsReleased = false; static bool g_parkHoldUntilReboot = false; @@ -269,10 +273,8 @@ void writeChannel(uint8_t channel, uint16_t off) { bool readPcaRegister(uint8_t reg, uint8_t& value) { Wire.beginTransmission(PCA9685_ADDRESS); Wire.write(reg); - - if (Wire.endTransmission(false) != 0) { - return false; - } + // Without a STOP nothing is sent yet; a failure shows up in requestFrom. + Wire.endTransmission(false); if (Wire.requestFrom(PCA9685_ADDRESS, static_cast(1)) != 1) { return false; @@ -295,7 +297,6 @@ void configureServoPwmDriver() { Wire.endTransmission(); pwm.setPWMFreq(kServoPwmHz); - g_pwmPrescale = pwm.readPrescale(); } } // namespace @@ -312,21 +313,24 @@ bool servoPwmDriverState(ServoPwmDriverState& state) { state.prescale = pwm.readPrescale(); - return (state.mode1 & kPcaMode1Sleep) == 0 && + return (state.mode1 & kPcaMode1Mask) == kPcaMode1Servo && state.mode2 == kPcaMode2Servo && - state.prescale == g_pwmPrescale; -} - -bool servoPwmDriverResponds() { - ServoPwmDriverState state; - return servoPwmDriverState(state); + state.prescale == kPcaPrescaleServo; } -void restartServoPwmDriver() { +bool restartServoPwmDriver() { disableServoOutputs(); configureServoPwmDriver(); + // Outputs stay off unless the setup took: a wrong output mode or frequency + // must not reach the servos. + ServoPwmDriverState state; + + if (!servoPwmDriverState(state)) { + return false; + } + // The servos kept animating in software while the chip was unreachable; // put every channel where the firmware thinks it is. for (int i = 0; i < SERVO_COUNT; i++) { @@ -335,6 +339,7 @@ void restartServoPwmDriver() { delay(10); enableServoOutputs(); + return true; } void initServoPwmDriver() { diff --git a/src/hardware/servo_wrapper.h b/src/hardware/servo_wrapper.h index cee37966..d20014a7 100644 --- a/src/hardware/servo_wrapper.h +++ b/src/hardware/servo_wrapper.h @@ -65,12 +65,12 @@ struct ServoPwmDriverState { uint8_t prescale; }; -// Reads the PCA9685's setup registers. True when it answers, is awake, and -// still holds the output mode and prescale it was given. +// Reads the PCA9685's setup registers. True when it answers, is awake with +// auto-increment on, and holds the servo output mode and prescale. bool servoPwmDriverState(ServoPwmDriverState& state); -bool servoPwmDriverResponds(); // Sets the PCA9685 up again and rewrites every channel; keeps the current pose. -void restartServoPwmDriver(); +// False, with the outputs left disabled, when the setup did not take. +bool restartServoPwmDriver(); void servoMoveAllSmoothTo( const float targets[SERVO_COUNT], float speedDegS = SERVO_BOOT_SPEED_DEG_S diff --git a/src/http/health_handlers.cpp b/src/http/health_handlers.cpp index 71348b30..803b2ba4 100644 --- a/src/http/health_handlers.cpp +++ b/src/http/health_handlers.cpp @@ -10,6 +10,17 @@ #include "network/wifi_connect.h" #include "settings/settings.h" +namespace { + +void writeI2cDevice(JsonObject out, const I2cDeviceHealth& device) { + out["ok"] = device.ok; + out["faults"] = device.faults; + out["recoveries"] = device.recoveries; + out["last_fault_ms"] = device.lastFaultMs; +} + +} // namespace + void handleAuth(WebServer& server) { JsonDocument body; body["ok"] = true; @@ -56,9 +67,7 @@ void handleHealth(WebServer& server) { const I2cHealth bus = i2cHealth(); JsonObject i2c = body["i2c"].to(); i2c["ok"] = bus.ok; - i2c["pca9685"] = bus.servoDriver; - i2c["faults"] = bus.faults; - i2c["recoveries"] = bus.recoveries; - i2c["last_fault_ms"] = bus.lastFaultMs; + writeI2cDevice(i2c["pca9685"].to(), bus.servoDriver); + writeI2cDevice(i2c["oled"].to(), bus.oled); sendJson(server, 200, body); } From fc6a7e1e22b090fe8c6e9593e699724199f125a7 Mon Sep 17 00:00:00 2001 From: Blake Azuela Date: Mon, 16 Nov 2026 14:24:09 -0500 Subject: [PATCH 04/10] chore: ignore the PlatformIO VS Code extension's generated files The extension writes .vscode/ (IntelliSense and debug configs with absolute paths, plus its extension recommendations) and .gitignore.optional into the repo root. They are machine-specific and were showing up as untracked files. --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index b0e52111..5511fab7 100644 --- a/.gitignore +++ b/.gitignore @@ -34,6 +34,9 @@ web/print/files/ .pio .pio-home compile_commands.json +# Written by the PlatformIO VS Code extension; machine-specific paths +.vscode/ +.gitignore.optional # Prerequisites *.d From 23f9e951cb81d0739562a8082d1cc59175db065d Mon Sep 17 00:00:00 2001 From: Blake Azuela Date: Mon, 16 Nov 2026 15:28:18 -0500 Subject: [PATCH 05/10] fix(firmware): act on failed servo writes and close OLED recovery gaps A second review of the I2C recovery found these, all fixed here: - The OLED could stay dark until reboot. A failed OLED restart outside an OLED fault (inside the PCA9685's recovery, or after a failed Wire restart) cleared oledAvailable, but the check looked only at the address ACK, so the OLED never faulted and never recovered. The check now requires oledAvailable. - Each chip's tracker now reads its own time. The PCA9685's recovery stamped the OLED tracker with a later millis() than the tick's `now`, so the unsigned subtraction wrapped and the OLED retried twice in one tick. - A failed servo write now stops servo writes at once. setPWM reports failure and the firmware ignored it, so a dead bus could still stall a blocking move that never reaches the health check. The next two checks then fail and recovery sets the chip up again. - A failed write while resyncing the channels fails the recovery, instead of leaving that servo's cached pulse wrong. - Servo outputs are switched off (OE) at a PCA9685 fault, so a corrupted setup cannot keep driving them. - Recovery no longer calls pwm.begin(), which programs 1 kHz before the 50 Hz setup; on a board with OE not wired the live servos got those pulses. reset() wakes the chip instead. - The check includes MODE1's EXTCLK bit. - The stale "line held" flag that could keep the OLED waiting for the PCA9685's next retry is gone. - Boot logs a failed Wire.begin, and a sleeping robot's OLED recovery no longer sends a second blank frame. While a line is held, an update that retries both chips can take up to 101 ms (52 ms before), because the OLED retry now probes the bus. It happens once per retry interval. On the robot, with a temporary injector: a dead bus faulted both chips after 0.74 s and both recovered after release; MODE2 = 0x10 was repaired in 48 ms; an OLED-only fault recovered without touching the PCA9685; a failed servo write faulted after 0.95 s and recovered in 75 ms; oledAvailable cleared outside a fault faulted after 0.96 s and recovered in 16 ms. A 115 s soak with 25 animation changes, bells and a sleep cycle raised no fault. --- docs/api.md | 4 +- docs/hardware/interfaces.md | 5 +- docs/hardware/testing.md | 1 + src/hardware/i2c_health.cpp | 100 ++++++++++++++++++++------------- src/hardware/i2c_health.h | 12 ++-- src/hardware/servo_wrapper.cpp | 55 ++++++++++++------ src/hardware/servo_wrapper.h | 3 +- src/robot_tick.cpp | 2 +- 8 files changed, 116 insertions(+), 66 deletions(-) diff --git a/docs/api.md b/docs/api.md index edfb03c9..56ba0090 100644 --- a/docs/api.md +++ b/docs/api.md @@ -146,8 +146,8 @@ curl -H "Authorization: Bearer YOUR_TOKEN" http://tiny-engineer.local/health | `setup_ap_ssid` | Setup AP SSID when `provisioning` is true, else `""` | | `setup_ap_ip` | Setup AP IP when `provisioning` is true (usually `192.168.4.1`), else `""` | | `oled` | SSD1306 probed and initialized. Goes `false` while an I2C fault keeps the display unreachable | -| `i2c.ok` | `false` while either chip on the shared I2C bus is in a fault. The firmware checks each every 500 ms and sets a faulted chip up again, retrying with a growing delay (1 s up to 30 s) | -| `i2c.pca9685.ok` | Servo driver reachable and still holding its setup (awake, auto-increment, output mode, prescale). Servo writes are skipped while `false` | +| `i2c.ok` | `false` while either chip on the shared I2C bus cannot be used. The firmware checks each every 500 ms and sets a faulted chip up again, retrying with a growing delay (1 s up to 30 s) | +| `i2c.pca9685.ok` | Servo driver usable: reachable, holding its setup (awake, internal clock, auto-increment, output mode, prescale), and its last write succeeded. Goes `false` at the first failed servo write; servo writes and outputs stay off until the chip is set up again | | `i2c.oled.ok` | Same value as `oled`. Stays `false`, with zero counts, on a robot that booted without a display | | `i2c..faults` | Faults on that chip since boot. A bus fault that takes both chips down counts once on each | | `i2c..recoveries` | Faults on that chip the firmware recovered from since boot | diff --git a/docs/hardware/interfaces.md b/docs/hardware/interfaces.md index 3889b776..cdf7de58 100644 --- a/docs/hardware/interfaces.md +++ b/docs/hardware/interfaces.md @@ -18,8 +18,9 @@ Shared bus. That table is the boot check. After boot the firmware keeps watching both devices (`src/hardware/i2c_health.cpp`): -- **Check:** every 500 ms the OLED must answer, and the PCA9685 must answer, be awake with auto-increment on, and still hold its output mode (`MODE2 = 0x04`) and prescale. Two failed checks in a row are a fault. Each chip is tracked separately, so one that stays down does not stop the checks on the other. A healthy check costs about 1.7 ms per 500 ms. -- **While down:** writes to the faulted chip are skipped, so failed transfers do not stall audio and HTTP. +- **Check:** every 500 ms the OLED must answer, and the PCA9685 must answer, be awake on its internal clock with auto-increment on, and still hold its output mode (`MODE2 = 0x04`) and prescale. Two failed checks in a row are a fault. Each chip is tracked separately, so one that stays down does not stop the checks on the other. A healthy check costs about 1.7 ms per 500 ms. +- **Failed servo write:** the first one stops servo writes at once, even inside a blocking move that never reaches the check, and the next two checks then fail. +- **While down:** writes to the faulted chip are skipped, so failed transfers do not stall audio and HTTP. At a PCA9685 fault the servo outputs are switched off (OE), so a corrupted setup cannot keep driving them. - **PCA9685 recovery:** if the chip still answers, only its setup is redone. Otherwise restart `Wire` and clock out a device left holding SDA first. Then rewrite every servo channel, and set the OLED up again too, since both share a supply. Servos jump to where the firmware thinks they are. Outputs stay disabled if the setup does not read back correctly. - **OLED recovery:** set the panel up again. Never restarts the bus, so an OLED fault alone does not disturb the servos. - **Retry:** a failed attempt is retried after 1 s, doubling up to 30 s. diff --git a/docs/hardware/testing.md b/docs/hardware/testing.md index 96ec127d..67899223 100644 --- a/docs/hardware/testing.md +++ b/docs/hardware/testing.md @@ -149,6 +149,7 @@ HTTP runs on STA when connected, or on setup AP at `192.168.4.1` during provisio | RGB never goes green | GPIO10 LED path | Board is C3-Zero (LED on GPIO10). Do not expect an external NeoPixel | | `[i2c] pca9685 FAULT #N` and `[i2c] oled FAULT #N` together, then `[i2c] bus restarted` and `recovered` lines | Both chips stopped answering after boot; firmware restarted the bus | `sda=LOW` / `scl=LOW` on the `bus restarted` line means a line was held: supply dip or wiring. Check supply current — see [power.md](power.md) | | `[i2c] pca9685 FAULT #N … mode1=… mode2=… prescale=…` | PCA9685 answered but lost its setup (reset to defaults, or a corrupted write) | Recovers by itself without a bus restart. Repeats point at supply dips or I2C noise | +| `[i2c] pca9685 FAULT #N … write failed` | A servo write failed; servo writes stopped at once | Recovers by itself. Repeats point at supply dips or I2C noise | | `[i2c] oled FAULT #N` alone | Only the OLED stopped answering | The servos are left alone. OLED **VCC**, GND, SDA, SCL, connector | | `[i2c] still down … retry in … ms` keeps repeating | Bus or chip not coming back | Wiring, PCA9685 / OLED **VCC**, then a full power cycle | | `[i2c] Wire.end failed` or `[i2c] Wire.begin failed` | The ESP32's I2C driver did not restart; both chips stay off until a later attempt succeeds | Report it with the serial log; a reset clears it | diff --git a/src/hardware/i2c_health.cpp b/src/hardware/i2c_health.cpp index e773c8ff..2ee37b0b 100644 --- a/src/hardware/i2c_health.cpp +++ b/src/hardware/i2c_health.cpp @@ -32,28 +32,27 @@ struct Device { }; bool servoDriverHealthy(); -void stopServoWrites(); +void stopServoOutputs(); bool recoverServoDriver(); -void logServoDriverRegisters(); +void logServoDriverDetail(); bool oledHealthy(); void stopOledWrites(); bool recoverOled(); Device g_servoDriver = { - "pca9685", servoDriverHealthy, stopServoWrites, recoverServoDriver, logServoDriverRegisters, {} + "pca9685", servoDriverHealthy, stopServoOutputs, recoverServoDriver, logServoDriverDetail, {} }; Device g_oled = {"oled", oledHealthy, stopOledWrites, recoverOled, nullptr, {}}; bool g_started = false; // The robot runs without an OLED; only watch one that was there at boot. bool g_oledExpected = false; -bool g_servoDriverUp = true; +// Set by a failed servo write; cleared once the chip has been set up again. +bool g_servoWriteFailed = false; // False after a failed Wire restart. Wire must not be used then: without its // buffers a transfer leaks Wire's lock, and the next restart would wait on // that lock forever. bool g_wireUp = true; -// False while the last bus restart found a line held low. -bool g_linesFree = true; // Registers from the latest PCA9685 check, for the fault log line. ServoPwmDriverState g_servoDriverState = {}; @@ -138,8 +137,9 @@ void noteRecovery(Device& device, bool ok) { } // Restarts Wire with the lines freed. False when Wire did not come back or a -// line is still held: every transfer would fail, so callers skip the probes -// and their timeouts. +// line is still held: every transfer would fail, so the caller skips the +// probes and their timeouts. Only the PCA9685's recovery calls this, so servo +// writes are already off. bool restartBus() { g_wireUp = false; @@ -152,7 +152,6 @@ bool restartBus() { } const BusLines lines = clearBus(); - g_linesFree = lines.sdaHigh && lines.sclHigh; g_wireUp = Wire.begin(I2C_SDA, I2C_SCL); if (!g_wireUp) { @@ -162,18 +161,17 @@ bool restartBus() { serialLogPrint(g_wireUp ? "[i2c] bus restarted" : "[i2c] Wire.begin failed"); logLines(lines); - return g_wireUp && g_linesFree; + return g_wireUp && lines.sdaHigh && lines.sclHigh; } -// The restart leaves the panel lit; a sleeping robot keeps it dark. Awake, -// the next eye frame or setup screen redraws it. +// The restart leaves the panel lit and blank; a sleeping robot keeps it dark. +// Awake, the next eye frame or setup screen redraws it. bool restartOledForSleepState() { if (!restartOled()) { return false; } if (isSleeping()) { - blankOled(); sleepOled(); } @@ -181,7 +179,9 @@ bool restartOledForSleepState() { } // Both chips share a supply, so a PCA9685 that had to be set up again is -// reason to set the panel up again too, even when it still answers. +// reason to set the panel up again too, even when it still answers. A failure +// here while the OLED counts as up clears oledAvailable, and the OLED's own +// checks then take it down and recover it. void restartOledAfterBusRecovery() { if (!g_oledExpected) { return; @@ -196,15 +196,28 @@ void restartOledAfterBusRecovery() { bool servoDriverHealthy() { g_servoDriverState = {}; - return g_wireUp && servoPwmDriverState(g_servoDriverState); + + if (!g_wireUp) { + return false; + } + + // Read even after a failed write, so the fault line shows the registers. + const bool registersOk = servoPwmDriverState(g_servoDriverState); + return registersOk && !g_servoWriteFailed; } -void stopServoWrites() { - g_servoDriverUp = false; +// At a fault the outputs go off, so a corrupted setup cannot keep driving the +// servos. Recovery turns them back on once the setup reads back correctly. +void stopServoOutputs() { + disableServoOutputs(); } -// A PCA9685 that answered but lost its setup: the registers say what changed. -void logServoDriverRegisters() { +// Why the PCA9685 faulted: a failed write, and the registers it answered with. +void logServoDriverDetail() { + if (g_servoWriteFailed) { + serialLogPrint(" write failed"); + } + if (!g_servoDriverState.answered) { return; } @@ -231,24 +244,19 @@ bool recoverServoDriver() { reachable = restartBus() && servoDriverAnswers(); } - if (!reachable) { + if (!reachable || !restartServoPwmDriver()) { return false; } - // An answer proves the lines are free, whatever the last restart saw. - g_linesFree = true; - - g_servoDriverUp = restartServoPwmDriver(); - - if (g_servoDriverUp) { - restartOledAfterBusRecovery(); - } - - return g_servoDriverUp; + g_servoWriteFailed = false; + restartOledAfterBusRecovery(); + return true; } +// oledAvailable is part of the check: a restart that failed outside an OLED +// fault cleared it, and only a fault and its recovery set it again. bool oledHealthy() { - return g_wireUp && i2cDeviceConnected(OLED_ADDRESS); + return g_wireUp && oledAvailable && i2cDeviceConnected(OLED_ADDRESS); } void stopOledWrites() { @@ -258,10 +266,14 @@ void stopOledWrites() { // Never restarts the bus: an OLED fault alone must not disturb the servos. // A dead bus is left to the PCA9685's recovery. bool recoverOled() { - return g_wireUp && g_linesFree && restartOledForSleepState(); + return g_wireUp && restartOledForSleepState(); } -void service(Device& device, uint32_t now) { +void service(Device& device) { + // Read here, not once per tick: the other chip's recovery can take tens of + // milliseconds and stamp this tracker with a later time. + const uint32_t now = millis(); + switch (device.tracker.due(now)) { case I2cFaultAction::Check: if (device.tracker.noteCheck(device.healthy(), now)) { @@ -298,6 +310,10 @@ void initI2cBus() { } g_wireUp = Wire.begin(I2C_SDA, I2C_SCL); + + if (!g_wireUp) { + serialLogPrintln("[i2c] Wire.begin failed at boot"); + } } void beginI2cHealth() { @@ -305,26 +321,32 @@ void beginI2cHealth() { g_started = true; } -void updateI2cHealth(uint32_t now) { +void updateI2cHealth() { if (!g_started) { return; } - service(g_servoDriver, now); + service(g_servoDriver); if (g_oledExpected) { - service(g_oled, now); + service(g_oled); } } bool i2cServoDriverUp() { - return g_servoDriverUp; + return !g_servoDriver.tracker.down() && !g_servoWriteFailed; +} + +void i2cNoteServoWriteFailed() { + g_servoWriteFailed = true; } I2cHealth i2cHealth() { + const bool servoDriverUp = i2cServoDriverUp(); + return { - !g_servoDriver.tracker.down() && !g_oled.tracker.down(), - deviceHealth(g_servoDriver, g_servoDriverUp), + servoDriverUp && (!g_oledExpected || oledAvailable), + deviceHealth(g_servoDriver, servoDriverUp), deviceHealth(g_oled, oledAvailable) }; } diff --git a/src/hardware/i2c_health.h b/src/hardware/i2c_health.h index a2ec14ca..1a08bd96 100644 --- a/src/hardware/i2c_health.h +++ b/src/hardware/i2c_health.h @@ -2,9 +2,10 @@ #include -// Watches the I2C bus the PCA9685 and the OLED share. The Adafruit drivers -// ignore failed writes, so without this a dead bus leaves the robot frozen -// while every write waits out its timeout and starves audio and HTTP. +// Watches the I2C bus the PCA9685 and the OLED share. The SSD1306 driver +// ignores failed writes and the firmware used to ignore the PCA9685 driver's +// write status, so a dead bus left the robot frozen while every write waited +// out its timeout and starved audio and HTTP. // Frees a bus a device still holds from before a reset, then starts Wire. void initI2cBus(); @@ -14,11 +15,14 @@ void beginI2cHealth(); // Checks each chip while it is up and sets it up again while it is down. // Cheap enough for every tick. -void updateI2cHealth(uint32_t now); +void updateI2cHealth(); // False while the PCA9685 cannot be used: skip servo writes. bool i2cServoDriverUp(); +// A servo write failed. Servo writes stop until the chip has been set up again. +void i2cNoteServoWriteFailed(); + struct I2cDeviceHealth { bool ok; uint32_t faults; diff --git a/src/hardware/servo_wrapper.cpp b/src/hardware/servo_wrapper.cpp index c564b84f..fe9ebc03 100644 --- a/src/hardware/servo_wrapper.cpp +++ b/src/hardware/servo_wrapper.cpp @@ -25,9 +25,10 @@ constexpr uint16_t kServoPwmHz = 50; constexpr uint8_t kPcaMode1Reg = 0x00; constexpr uint8_t kPcaMode2Reg = 0x01; -// Awake with register auto-increment on: setPWM writes four registers in one -// transfer and needs it. -constexpr uint8_t kPcaMode1Mask = MODE1_SLEEP | MODE1_AI; +// Awake, on the internal clock, with register auto-increment on: setPWM +// writes four registers in one transfer and needs it. EXTCLK is sticky (only a +// power cycle clears it), so a corrupted one keeps the chip down, visibly. +constexpr uint8_t kPcaMode1Mask = MODE1_SLEEP | MODE1_AI | MODE1_EXTCLK; constexpr uint8_t kPcaMode1Servo = MODE1_AI; // Totem-pole outputs, not inverted: what a servo signal line needs. Any other // value leaves the servos without a usable pulse while every write still @@ -260,14 +261,18 @@ void setServoParkHoldUntilReboot(bool hold) { namespace { -// Skipped while the PCA9685 is unreachable, so a dead bus does not stall the -// loop. restartServoPwmDriver() rewrites every channel once it is back. +// Skipped while the PCA9685 is unusable, so a dead bus does not stall the +// loop. The first failed write stops the rest at once, even inside a blocking +// move that never reaches the health check; restartServoPwmDriver() rewrites +// every channel once the chip is back. void writeChannel(uint8_t channel, uint16_t off) { if (!i2cServoDriverUp()) { return; } - pwm.setPWM(channel, 0, off); + if (pwm.setPWM(channel, 0, off) != 0) { + i2cNoteServoWriteFailed(); + } } bool readPcaRegister(uint8_t reg, uint8_t& value) { @@ -284,10 +289,9 @@ bool readPcaRegister(uint8_t reg, uint8_t& value) { return true; } -// Full PCA9685 setup, shared by boot and recovery. -void configureServoPwmDriver() { - pwm.begin(); - +// Output mode and servo frequency, shared by boot and recovery. The chip must +// be awake first: setPWMFreq() restores the sleep bit it finds. +void writeServoSetup() { // The Adafruit driver never writes MODE2, so a value corrupted by a bus // glitch would outlive every restart. The chip keeps its registers across // an ESP32 reset: only its own power cycle clears them. @@ -321,7 +325,11 @@ bool servoPwmDriverState(ServoPwmDriverState& state) { bool restartServoPwmDriver() { disableServoOutputs(); - configureServoPwmDriver(); + // Not pwm.begin(): it programs 1 kHz before writeServoSetup() sets 50 Hz, + // and on a board with OE not wired the live servos would get those pulses. + // reset() wakes the chip, which is all begin() adds here. + pwm.reset(); + writeServoSetup(); // Outputs stay off unless the setup took: a wrong output mode or frequency // must not reach the servos. @@ -333,8 +341,14 @@ bool restartServoPwmDriver() { // The servos kept animating in software while the chip was unreachable; // put every channel where the firmware thinks it is. + bool resynced = true; + for (int i = 0; i < SERVO_COUNT; i++) { - g_servos[i].resyncOutput(); + resynced = g_servos[i].resyncOutput() && resynced; + } + + if (!resynced) { + return false; } delay(10); @@ -345,7 +359,8 @@ bool restartServoPwmDriver() { void initServoPwmDriver() { disableServoOutputs(); - configureServoPwmDriver(); + pwm.begin(); + writeServoSetup(); delay(100); uint8_t pose[SERVO_COUNT]; @@ -525,17 +540,23 @@ void ServoWrapper::releaseServoOutput() { released_ = true; } -void ServoWrapper::resyncOutput() { +bool ServoWrapper::resyncOutput() { // Written straight to the chip: this runs while it still counts as down. if (released_) { - pwm.setPWM(SERVO_SPECS[index_].channel, 0, kPca9685FullOff); lastPulse_ = UINT16_MAX; - return; + return pwm.setPWM(SERVO_SPECS[index_].channel, 0, kPca9685FullOff) == 0; } const uint16_t pulse = angleToPulse(angle_); - pwm.setPWM(SERVO_SPECS[index_].channel, 0, pulse); + + if (pwm.setPWM(SERVO_SPECS[index_].channel, 0, pulse) != 0) { + // Not cached: the next write for this servo must go out. + lastPulse_ = UINT16_MAX; + return false; + } + lastPulse_ = pulse; + return true; } void ServoWrapper::writeAngle(float angle, bool log, bool electrical) { diff --git a/src/hardware/servo_wrapper.h b/src/hardware/servo_wrapper.h index d20014a7..c4dcc25d 100644 --- a/src/hardware/servo_wrapper.h +++ b/src/hardware/servo_wrapper.h @@ -30,7 +30,8 @@ class ServoWrapper { void ensureServoOutput(); void releaseServoOutput(); // Rewrites this channel from the firmware's state after a driver restart. - void resyncOutput(); + // False when the write failed. + bool resyncOutput(); friend void servoMoveAllSmoothTo( const float targets[SERVO_COUNT], diff --git a/src/robot_tick.cpp b/src/robot_tick.cpp index 9a61a207..0e743292 100644 --- a/src/robot_tick.cpp +++ b/src/robot_tick.cpp @@ -13,7 +13,7 @@ void tickRobot() { const uint32_t now = millis(); - updateI2cHealth(now); + updateI2cHealth(); pollWifi(); updateProvisioningOled(now); updateAnimation(); From ebdb608fea892c31d6aae11af7bee0685192bba1 Mon Sep 17 00:00:00 2001 From: Blake Azuela Date: Mon, 16 Nov 2026 16:16:20 -0500 Subject: [PATCH 06/10] fix(firmware): fault the PCA9685 at once on a failed write and lighten recovery A third review of the I2C recovery found these, all fixed here: - A failed servo write only set a flag. The outputs stayed on until two later checks faulted the chip, and those never run inside a blocking move or during boot. The tracker now has noteFault(): a failed write takes the chip down at once and switches its outputs off. The separate g_servoWriteFailed flag is gone. - One failed write escalated to a full reset (sleep/wake cycle, OLED re-init). A chip that still holds its setup now only gets its channels rewritten and its outputs switched back on. The full restart runs only when the setup was lost. - Every PCA9685 recovery restarted the OLED, blanking a healthy panel and counting OLED faults it never had. The OLED is now set up again only when the PCA9685 lost its setup, the sign of a supply dip. - An OLED restart left the panel blank, and static screens (boot IP, servo test, orientation test) are never redrawn. The restart now keeps the frame buffer and puts it back; a sleeping robot keeps the panel dark but holds the frame for when it wakes. - A recovery inside the /play loop blocked for 50 to 60 ms between speaker writes, an audible gap. Recovery now waits while the speaker is playing (samples within the last 250 ms), for at most 5 s after the fault. - Boot turned the outputs on even when the setup did not read back correctly. Boot and recovery now share one finishing sequence: check the setup, write every channel, then enable the outputs. A boot setup that fails is a fault ("setup failed"). - The health check reads MODE1 and MODE2 in one transfer instead of two. - stopServoOutputs, a one-line wrapper, is gone. Add tracker tests for noteFault, and update hardware/interfaces.md and the troubleshooting table in hardware/testing.md. Not yet run on the robot. --- docs/hardware/interfaces.md | 11 +-- docs/hardware/testing.md | 3 +- src/audio/audio.cpp | 13 +++ src/audio/audio.h | 2 + src/display/oled.h | 6 +- src/display/oled_init.cpp | 47 +++++++++-- src/hardware/i2c_fault.cpp | 15 +++- src/hardware/i2c_fault.h | 6 ++ src/hardware/i2c_health.cpp | 110 ++++++++++++++----------- src/hardware/i2c_health.h | 6 +- src/hardware/servo_wrapper.cpp | 101 +++++++++++++++-------- src/hardware/servo_wrapper.h | 7 ++ test/test_i2c_fault/test_i2c_fault.cpp | 36 ++++++++ 13 files changed, 262 insertions(+), 101 deletions(-) diff --git a/docs/hardware/interfaces.md b/docs/hardware/interfaces.md index cdf7de58..ede31a5b 100644 --- a/docs/hardware/interfaces.md +++ b/docs/hardware/interfaces.md @@ -18,15 +18,16 @@ Shared bus. That table is the boot check. After boot the firmware keeps watching both devices (`src/hardware/i2c_health.cpp`): -- **Check:** every 500 ms the OLED must answer, and the PCA9685 must answer, be awake on its internal clock with auto-increment on, and still hold its output mode (`MODE2 = 0x04`) and prescale. Two failed checks in a row are a fault. Each chip is tracked separately, so one that stays down does not stop the checks on the other. A healthy check costs about 1.7 ms per 500 ms. -- **Failed servo write:** the first one stops servo writes at once, even inside a blocking move that never reaches the check, and the next two checks then fail. +- **Check:** every 500 ms the OLED must answer, and the PCA9685 must answer, be awake on its internal clock with auto-increment on, and still hold its output mode (`MODE2 = 0x04`) and prescale. Two failed checks in a row are a fault. Each chip is tracked separately, so one that stays down does not stop the checks on the other. A healthy PCA9685 check is two register reads: `MODE1` and `MODE2` together, then the prescale. +- **Failed servo write:** a fault on its own. The first one takes the PCA9685 down at once and switches its outputs off, even inside a blocking move that never reaches the check. So does a boot setup that does not read back correctly. - **While down:** writes to the faulted chip are skipped, so failed transfers do not stall audio and HTTP. At a PCA9685 fault the servo outputs are switched off (OE), so a corrupted setup cannot keep driving them. -- **PCA9685 recovery:** if the chip still answers, only its setup is redone. Otherwise restart `Wire` and clock out a device left holding SDA first. Then rewrite every servo channel, and set the OLED up again too, since both share a supply. Servos jump to where the firmware thinks they are. Outputs stay disabled if the setup does not read back correctly. -- **OLED recovery:** set the panel up again. Never restarts the bus, so an OLED fault alone does not disturb the servos. +- **PCA9685 recovery:** if the chip does not answer, restart `Wire` and clock out a device left holding SDA first. A chip that kept its setup only lost a write or a read: rewrite every servo channel and switch the outputs back on. A chip that lost its setup lost power or was corrupted: set it up again, rewrite every channel, and set the OLED up again too, since both share a supply. Servos jump to where the firmware thinks they are. Outputs stay disabled if the setup does not read back correctly. +- **OLED recovery:** set the panel up again and put the last frame back, so a static screen survives too. Never restarts the bus, so an OLED fault alone does not disturb the servos. +- **Audio:** a recovery blocks for tens of milliseconds, so while the speaker is playing it waits, up to 5 s after the fault. The chip stays off meanwhile. - **Retry:** a failed attempt is retried after 1 s, doubling up to 30 s. - **Report:** `[i2c]` lines on the serial log (with SDA / SCL levels and, for a PCA9685 that answered, its registers) and the `i2c` object in [`GET /health`](../api.md#get-health). -The OLED check is an address ACK only. A panel that loses its setup but still answers is not detected by itself; it is set up again whenever the PCA9685 recovers. +The OLED check is an address ACK only. A panel that loses its setup but still answers is not detected by itself; it is set up again whenever the PCA9685 recovers from a lost setup. The PCA9685 keeps its registers across an ESP32 reset; only its own power cycle clears them. That is why setup writes `MODE2` explicitly instead of trusting the power-on default. diff --git a/docs/hardware/testing.md b/docs/hardware/testing.md index 67899223..d1ed4cbd 100644 --- a/docs/hardware/testing.md +++ b/docs/hardware/testing.md @@ -149,7 +149,8 @@ HTTP runs on STA when connected, or on setup AP at `192.168.4.1` during provisio | RGB never goes green | GPIO10 LED path | Board is C3-Zero (LED on GPIO10). Do not expect an external NeoPixel | | `[i2c] pca9685 FAULT #N` and `[i2c] oled FAULT #N` together, then `[i2c] bus restarted` and `recovered` lines | Both chips stopped answering after boot; firmware restarted the bus | `sda=LOW` / `scl=LOW` on the `bus restarted` line means a line was held: supply dip or wiring. Check supply current — see [power.md](power.md) | | `[i2c] pca9685 FAULT #N … mode1=… mode2=… prescale=…` | PCA9685 answered but lost its setup (reset to defaults, or a corrupted write) | Recovers by itself without a bus restart. Repeats point at supply dips or I2C noise | -| `[i2c] pca9685 FAULT #N … write failed` | A servo write failed; servo writes stopped at once | Recovers by itself. Repeats point at supply dips or I2C noise | +| `[i2c] pca9685 FAULT #N … write failed` | A servo write failed; servo writes and outputs stopped at once | Recovers by itself, usually by rewriting the channels only. Repeats point at supply dips or I2C noise | +| `[i2c] pca9685 FAULT #N … setup failed` | The PCA9685 setup did not read back correctly at boot; outputs stayed off | Recovers by itself once the checks start. Repeats point at supply dips or I2C noise | | `[i2c] oled FAULT #N` alone | Only the OLED stopped answering | The servos are left alone. OLED **VCC**, GND, SDA, SCL, connector | | `[i2c] still down … retry in … ms` keeps repeating | Bus or chip not coming back | Wiring, PCA9685 / OLED **VCC**, then a full power cycle | | `[i2c] Wire.end failed` or `[i2c] Wire.begin failed` | The ESP32's I2C driver did not restart; both chips stay off until a later attempt succeeds | Report it with the serial log; a reset clears it | diff --git a/src/audio/audio.cpp b/src/audio/audio.cpp index 539ce50f..9f78c988 100644 --- a/src/audio/audio.cpp +++ b/src/audio/audio.cpp @@ -21,6 +21,12 @@ namespace { bool audioStorageReady = false; +// A gap this long between speaker writes means no clip is playing. Longer +// than a /play body's usual network stall. +constexpr uint32_t kSpeakerQuietMs = 250; +bool g_speakerFed = false; +uint32_t g_lastSpeakerWriteMs = 0; + constexpr const char* kAudioPartition = "spiffs"; void logAudioStorageContents() { @@ -99,9 +105,16 @@ bool initAudioStorage() { return true; } +bool speakerPlaying(uint32_t now) { + return g_speakerFed && now - g_lastSpeakerWriteMs < kSpeakerQuietMs; +} + void writeMonoToSpeaker(const int16_t* samples, size_t count) { constexpr size_t FRAMES = 256; + g_speakerFed = true; + g_lastSpeakerWriteMs = millis(); + int16_t buffer[FRAMES * 2]; const uint8_t volume = settingsVolume(); diff --git a/src/audio/audio.h b/src/audio/audio.h index ff052d91..252eb261 100644 --- a/src/audio/audio.h +++ b/src/audio/audio.h @@ -7,6 +7,8 @@ extern I2SClass I2S; bool initAudioStorage(); // Volume-scaled, duplicated to both I2S slots. Blocks until the DMA buffer accepts it. void writeMonoToSpeaker(const int16_t* samples, size_t count); +// True while a clip is playing: samples reached the speaker moments ago. +bool speakerPlaying(uint32_t now); void playTone( float frequency, int durationMs diff --git a/src/display/oled.h b/src/display/oled.h index f1556f1f..a795d63f 100644 --- a/src/display/oled.h +++ b/src/display/oled.h @@ -34,8 +34,10 @@ void showTextThenIdle(const char* line1, const char* line2 = nullptr); void showIdleScreen(); void initOled(); -// Sets the OLED up again after an I2C fault. Leaves a blank, lit panel. -bool restartOled(); +// Sets the OLED up again after an I2C fault and puts the last frame back, so +// a static screen survives too. With panelOn false the panel stays dark but +// holds the frame for when it wakes. +bool restartOled(bool panelOn); void runOledTest(); void showServoProgress( diff --git a/src/display/oled_init.cpp b/src/display/oled_init.cpp index 2e1311a3..545e5671 100644 --- a/src/display/oled_init.cpp +++ b/src/display/oled_init.cpp @@ -1,6 +1,7 @@ #include #include #include +#include #include "pins.h" #include "display/oled.h" @@ -15,8 +16,19 @@ bool i2cDeviceConnected(uint8_t address) { namespace { -// Sends the SSD1306 its full setup. Safe to repeat: a panel that lost power -// comes back dark until it gets this again. +constexpr size_t kOledFrameBytes = OLED_WIDTH * ((OLED_HEIGHT + 7) / 8); + +// The frame on the panel before a restart. begin() clears the frame buffer, +// and static screens are never drawn again, so a restart puts this back. +uint8_t g_savedFrame[kOledFrameBytes]; +// Set from the first attempt of a restart until one succeeds: a failed +// attempt's begin() has already cleared the buffer, so later attempts must not +// save it again. +bool g_frameSaved = false; + +// Sends the SSD1306 its full setup and turns the panel on. Safe to repeat: a +// panel that lost power comes back dark until it gets this again. Clears the +// frame buffer. bool configureOled() { if (!display.begin( SSD1306_SWITCHCAPVCC, @@ -26,9 +38,6 @@ bool configureOled() { } display.setRotation(settingsOledRotate180() ? 2 : 0); - - display.clearDisplay(); - display.display(); return true; } @@ -52,12 +61,36 @@ void initOled() { return; } + display.display(); oledAvailable = true; serialLogPrintln("OLED initialized"); } -bool restartOled() { +bool restartOled(bool panelOn) { + uint8_t* buffer = display.getBuffer(); + + if (buffer != nullptr && !g_frameSaved) { + memcpy(g_savedFrame, buffer, kOledFrameBytes); + g_frameSaved = true; + } + oledAvailable = i2cDeviceConnected(OLED_ADDRESS) && configureOled(); - return oledAvailable; + + if (!oledAvailable) { + return false; + } + + // Dark first: the panel takes the frame while off, and wakes showing it. + if (!panelOn) { + display.ssd1306_command(SSD1306_DISPLAYOFF); + } + + if (g_frameSaved) { + memcpy(display.getBuffer(), g_savedFrame, kOledFrameBytes); + g_frameSaved = false; + } + + display.display(); + return true; } diff --git a/src/hardware/i2c_fault.cpp b/src/hardware/i2c_fault.cpp index 7febc106..82da542b 100644 --- a/src/hardware/i2c_fault.cpp +++ b/src/hardware/i2c_fault.cpp @@ -26,6 +26,20 @@ bool I2cFaultTracker::noteCheck(bool ok, uint32_t now) { return false; } + takeDown(now); + return true; +} + +bool I2cFaultTracker::noteFault(uint32_t now) { + if (down_) { + return false; + } + + takeDown(now); + return true; +} + +void I2cFaultTracker::takeDown(uint32_t now) { down_ = true; faults_++; lastFaultMs_ = now; @@ -33,7 +47,6 @@ bool I2cFaultTracker::noteCheck(bool ok, uint32_t now) { // The first recovery attempt runs right away. retryDelayMs_ = 0; lastAttemptMs_ = now; - return true; } void I2cFaultTracker::noteRecovery(bool ok, uint32_t now) { diff --git a/src/hardware/i2c_fault.h b/src/hardware/i2c_fault.h index 8bd46c99..31d9e9a4 100644 --- a/src/hardware/i2c_fault.h +++ b/src/hardware/i2c_fault.h @@ -23,6 +23,10 @@ class I2cFaultTracker { // Result of a health check while up. True when this check took the bus down. bool noteCheck(bool ok, uint32_t now); + // A failure that counts as a fault on its own, such as a failed write: the + // bus goes down at once. True when this took it down; false when it was down. + bool noteFault(uint32_t now); + // Result of a recovery attempt while down. A failed attempt doubles the wait // before the next one, up to kI2cMaxRetryMs. void noteRecovery(bool ok, uint32_t now); @@ -37,6 +41,8 @@ class I2cFaultTracker { uint32_t retryDelayMs() const { return retryDelayMs_; } private: + void takeDown(uint32_t now); + bool down_ = false; bool checked_ = false; uint8_t failures_ = 0; diff --git a/src/hardware/i2c_health.cpp b/src/hardware/i2c_health.cpp index 2ee37b0b..41f44947 100644 --- a/src/hardware/i2c_health.cpp +++ b/src/hardware/i2c_health.cpp @@ -1,6 +1,7 @@ #include #include +#include "audio/audio.h" #include "display/oled.h" #include "hardware/i2c_fault.h" #include "hardware/i2c_health.h" @@ -13,6 +14,10 @@ namespace { constexpr uint8_t kBusClearPulses = 9; constexpr uint32_t kBusClearHalfPeriodUs = 5; +// A recovery blocks for tens of milliseconds, which is an audible gap while a +// clip plays. It waits for the speaker to go quiet, but not for longer than +// this after the fault: a long clip must not keep the robot limp. +constexpr uint32_t kRecoveryAudioWaitMaxMs = 5000; struct BusLines { bool sdaHigh; @@ -32,7 +37,6 @@ struct Device { }; bool servoDriverHealthy(); -void stopServoOutputs(); bool recoverServoDriver(); void logServoDriverDetail(); bool oledHealthy(); @@ -40,15 +44,15 @@ void stopOledWrites(); bool recoverOled(); Device g_servoDriver = { - "pca9685", servoDriverHealthy, stopServoOutputs, recoverServoDriver, logServoDriverDetail, {} + // At a fault the outputs go off, so a corrupted setup cannot keep driving + // the servos. Recovery turns them back on once the setup reads back right. + "pca9685", servoDriverHealthy, disableServoOutputs, recoverServoDriver, logServoDriverDetail, {} }; Device g_oled = {"oled", oledHealthy, stopOledWrites, recoverOled, nullptr, {}}; bool g_started = false; // The robot runs without an OLED; only watch one that was there at boot. bool g_oledExpected = false; -// Set by a failed servo write; cleared once the chip has been set up again. -bool g_servoWriteFailed = false; // False after a failed Wire restart. Wire must not be used then: without its // buffers a transfer leaks Wire's lock, and the next restart would wait on // that lock forever. @@ -97,7 +101,9 @@ void logLines(const BusLines& lines) { serialLogPrintln(lines.sclHigh ? "high" : "LOW"); } -void logFault(const Device& device, uint32_t now) { +// `cause` names a fault found outside a check; a check's fault logs the +// device's own detail instead. +void logFault(const Device& device, uint32_t now, const char* cause = nullptr) { serialLogPrint("[i2c] "); serialLogPrint(device.name); serialLogPrint(" FAULT #"); @@ -106,7 +112,10 @@ void logFault(const Device& device, uint32_t now) { serialLogPrint(now); serialLogPrint(" ms"); - if (device.logFaultDetail != nullptr) { + if (cause != nullptr) { + serialLogPrint(" "); + serialLogPrint(cause); + } else if (device.logFaultDetail != nullptr) { device.logFaultDetail(); } @@ -164,25 +173,17 @@ bool restartBus() { return g_wireUp && lines.sdaHigh && lines.sclHigh; } -// The restart leaves the panel lit and blank; a sleeping robot keeps it dark. -// Awake, the next eye frame or setup screen redraws it. +// The restart puts the last frame back on the panel; a sleeping robot keeps +// it dark. bool restartOledForSleepState() { - if (!restartOled()) { - return false; - } - - if (isSleeping()) { - sleepOled(); - } - - return true; + return restartOled(!isSleeping()); } -// Both chips share a supply, so a PCA9685 that had to be set up again is -// reason to set the panel up again too, even when it still answers. A failure -// here while the OLED counts as up clears oledAvailable, and the OLED's own -// checks then take it down and recover it. -void restartOledAfterBusRecovery() { +// Both chips share a supply, so a PCA9685 that lost its setup is reason to set +// the panel up again too, even when it still answers. A failure here while the +// OLED counts as up clears oledAvailable, and the OLED's own checks then take +// it down and recover it. +void restartOledAfterPowerLoss() { if (!g_oledExpected) { return; } @@ -201,23 +202,11 @@ bool servoDriverHealthy() { return false; } - // Read even after a failed write, so the fault line shows the registers. - const bool registersOk = servoPwmDriverState(g_servoDriverState); - return registersOk && !g_servoWriteFailed; + return servoPwmDriverState(g_servoDriverState); } -// At a fault the outputs go off, so a corrupted setup cannot keep driving the -// servos. Recovery turns them back on once the setup reads back correctly. -void stopServoOutputs() { - disableServoOutputs(); -} - -// Why the PCA9685 faulted: a failed write, and the registers it answered with. +// Why a check faulted the PCA9685: the registers it answered with. void logServoDriverDetail() { - if (g_servoWriteFailed) { - serialLogPrint(" write failed"); - } - if (!g_servoDriverState.answered) { return; } @@ -235,8 +224,10 @@ bool servoDriverAnswers() { } // The PCA9685 is the required chip, so its recovery is the one that restarts -// the bus. A chip that still answers proves the bus works: then only its -// setup is redone. +// the bus. A chip that still answers proves the bus works. One that also kept +// its setup only lost a write or a read: rewriting the channels is enough, and +// the healthy OLED is left alone. A lost setup means the chip lost power or +// was corrupted: it is set up from scratch, and so is the panel. bool recoverServoDriver() { bool reachable = servoDriverAnswers(); @@ -244,12 +235,19 @@ bool recoverServoDriver() { reachable = restartBus() && servoDriverAnswers(); } - if (!reachable || !restartServoPwmDriver()) { + if (!reachable) { + return false; + } + + if (servoPwmDriverState(g_servoDriverState)) { + return resyncServoPwmDriver(); + } + + if (!restartServoPwmDriver()) { return false; } - g_servoWriteFailed = false; - restartOledAfterBusRecovery(); + restartOledAfterPowerLoss(); return true; } @@ -269,6 +267,18 @@ bool recoverOled() { return g_wireUp && restartOledForSleepState(); } +// The device is down: stop its writes, since each failed one blocks the loop +// for its timeout. +void enterFault(Device& device, uint32_t now, const char* cause = nullptr) { + device.stopWrites(); + logFault(device, now, cause); +} + +bool recoveryWaitsForAudio(const Device& device, uint32_t now) { + return speakerPlaying(now) && + now - device.tracker.lastFaultMs() < kRecoveryAudioWaitMaxMs; +} + void service(Device& device) { // Read here, not once per tick: the other chip's recovery can take tens of // milliseconds and stamp this tracker with a later time. @@ -277,13 +287,13 @@ void service(Device& device) { switch (device.tracker.due(now)) { case I2cFaultAction::Check: if (device.tracker.noteCheck(device.healthy(), now)) { - // Each failed write blocks the loop for its timeout. - device.stopWrites(); - logFault(device, now); + enterFault(device, now); } break; case I2cFaultAction::Recover: - noteRecovery(device, device.recover()); + if (!recoveryWaitsForAudio(device, now)) { + noteRecovery(device, device.recover()); + } break; case I2cFaultAction::None: break; @@ -334,11 +344,15 @@ void updateI2cHealth() { } bool i2cServoDriverUp() { - return !g_servoDriver.tracker.down() && !g_servoWriteFailed; + return !g_servoDriver.tracker.down(); } -void i2cNoteServoWriteFailed() { - g_servoWriteFailed = true; +void i2cNoteServoDriverFault(const char* cause) { + const uint32_t now = millis(); + + if (g_servoDriver.tracker.noteFault(now)) { + enterFault(g_servoDriver, now, cause); + } } I2cHealth i2cHealth() { diff --git a/src/hardware/i2c_health.h b/src/hardware/i2c_health.h index 1a08bd96..32096d25 100644 --- a/src/hardware/i2c_health.h +++ b/src/hardware/i2c_health.h @@ -20,8 +20,10 @@ void updateI2cHealth(); // False while the PCA9685 cannot be used: skip servo writes. bool i2cServoDriverUp(); -// A servo write failed. Servo writes stop until the chip has been set up again. -void i2cNoteServoWriteFailed(); +// The PCA9685 failed outside a check, such as a failed write or a boot setup +// that did not take. It goes down at once: its outputs go off and servo writes +// stop until the recovery has set it up again. `cause` goes in the fault log. +void i2cNoteServoDriverFault(const char* cause); struct I2cDeviceHealth { bool ok; diff --git a/src/hardware/servo_wrapper.cpp b/src/hardware/servo_wrapper.cpp index fe9ebc03..edef62b0 100644 --- a/src/hardware/servo_wrapper.cpp +++ b/src/hardware/servo_wrapper.cpp @@ -262,30 +262,34 @@ void setServoParkHoldUntilReboot(bool hold) { namespace { // Skipped while the PCA9685 is unusable, so a dead bus does not stall the -// loop. The first failed write stops the rest at once, even inside a blocking -// move that never reaches the health check; restartServoPwmDriver() rewrites -// every channel once the chip is back. +// loop. The first failed write takes the chip down at once, with its outputs +// off, even inside a blocking move that never reaches the health check; the +// recovery rewrites every channel once the chip is back. void writeChannel(uint8_t channel, uint16_t off) { if (!i2cServoDriverUp()) { return; } if (pwm.setPWM(channel, 0, off) != 0) { - i2cNoteServoWriteFailed(); + i2cNoteServoDriverFault("write failed"); } } -bool readPcaRegister(uint8_t reg, uint8_t& value) { +// Reads `count` registers from `reg` in one transfer; relies on auto-increment. +bool readPcaRegisters(uint8_t reg, uint8_t* values, uint8_t count) { Wire.beginTransmission(PCA9685_ADDRESS); Wire.write(reg); // Without a STOP nothing is sent yet; a failure shows up in requestFrom. Wire.endTransmission(false); - if (Wire.requestFrom(PCA9685_ADDRESS, static_cast(1)) != 1) { + if (Wire.requestFrom(PCA9685_ADDRESS, count) != count) { return false; } - value = static_cast(Wire.read()); + for (uint8_t i = 0; i < count; i++) { + values[i] = static_cast(Wire.read()); + } + return true; } @@ -303,18 +307,48 @@ void writeServoSetup() { pwm.setPWMFreq(kServoPwmHz); } +// Writes every channel from the firmware's state, then turns the outputs on. +// False, with the outputs left off, when a write failed. +bool writeChannelsAndEnableOutputs() { + bool written = true; + + for (int i = 0; i < SERVO_COUNT; i++) { + written = g_servos[i].resyncOutput() && written; + } + + if (!written) { + return false; + } + + // Let each channel's first pulse go out before the servos see it. + delay(10); + enableServoOutputs(); + return true; +} + +// The end of boot and of a restart. Outputs stay off unless the setup took: +// a wrong output mode or frequency must not reach the servos. +bool finishServoPwmDriverSetup() { + ServoPwmDriverState state; + return servoPwmDriverState(state) && writeChannelsAndEnableOutputs(); +} + } // namespace bool servoPwmDriverState(ServoPwmDriverState& state) { state = {}; - state.answered = - readPcaRegister(kPcaMode1Reg, state.mode1) && - readPcaRegister(kPcaMode2Reg, state.mode2); + + // MODE1 and MODE2 are adjacent, so one read returns both. With + // auto-increment off the second byte repeats MODE1 and fails the MODE2 check. + uint8_t modes[2] = {}; + state.answered = readPcaRegisters(kPcaMode1Reg, modes, 2); if (!state.answered) { return false; } + state.mode1 = modes[0]; + state.mode2 = modes[1]; state.prescale = pwm.readPrescale(); return (state.mode1 & kPcaMode1Mask) == kPcaMode1Servo && @@ -331,29 +365,14 @@ bool restartServoPwmDriver() { pwm.reset(); writeServoSetup(); - // Outputs stay off unless the setup took: a wrong output mode or frequency - // must not reach the servos. - ServoPwmDriverState state; - - if (!servoPwmDriverState(state)) { - return false; - } - // The servos kept animating in software while the chip was unreachable; - // put every channel where the firmware thinks it is. - bool resynced = true; - - for (int i = 0; i < SERVO_COUNT; i++) { - resynced = g_servos[i].resyncOutput() && resynced; - } - - if (!resynced) { - return false; - } + // finishing puts every channel where the firmware thinks it is. + return finishServoPwmDriverSetup(); +} - delay(10); - enableServoOutputs(); - return true; +bool resyncServoPwmDriver() { + disableServoOutputs(); + return writeChannelsAndEnableOutputs(); } void initServoPwmDriver() { @@ -373,15 +392,18 @@ void initServoPwmDriver() { } for (int i = 0; i < SERVO_COUNT; i++) { - g_servos[i].setPosition(static_cast(pose[i])); + g_servos[i].restorePosition(static_cast(pose[i])); } memcpy(g_lastSavedPose, pose, SERVO_COUNT); g_lastSavedPoseValid = true; g_poseDirty = false; - delay(10); - enableServoOutputs(); + // Same path as a restart, so boot and recovery cannot drift apart. A setup + // that did not take leaves the outputs off and the recovery to the checks. + if (!finishServoPwmDriverSetup()) { + i2cNoteServoDriverFault("setup failed"); + } } ServoWrapper& servoAt(int index) { @@ -418,6 +440,14 @@ void ServoWrapper::setPosition(float angle) { lastUpdateMs_ = 0; } +void ServoWrapper::restorePosition(float angle) { + angle_ = clampServoAngle(index_, angle); + target_ = angle_; + lastUpdateMs_ = 0; + // Not written yet: the next write for this servo must go out. + lastPulse_ = UINT16_MAX; +} + void ServoWrapper::setNormTarget(float n, float speedDegS) { setTarget(servoNormToDeg(index_, n), speedDegS); } @@ -541,7 +571,8 @@ void ServoWrapper::releaseServoOutput() { } bool ServoWrapper::resyncOutput() { - // Written straight to the chip: this runs while it still counts as down. + // Written straight to the chip, past writeChannel()'s check: this runs at + // boot and while the chip still counts as down. if (released_) { lastPulse_ = UINT16_MAX; return pwm.setPWM(SERVO_SPECS[index_].channel, 0, kPca9685FullOff) == 0; diff --git a/src/hardware/servo_wrapper.h b/src/hardware/servo_wrapper.h index c4dcc25d..cf5ba502 100644 --- a/src/hardware/servo_wrapper.h +++ b/src/hardware/servo_wrapper.h @@ -19,6 +19,9 @@ class ServoWrapper { float speedDegS = SERVO_MAX_SPEED_DEG_S ); void setPosition(float angle); + // Sets the firmware's angle without writing the chip; boot writes every + // channel afterwards. + void restorePosition(float angle); void setNormTarget( float n, float speedDegS = SERVO_MAX_SPEED_DEG_S @@ -72,6 +75,10 @@ bool servoPwmDriverState(ServoPwmDriverState& state); // Sets the PCA9685 up again and rewrites every channel; keeps the current pose. // False, with the outputs left disabled, when the setup did not take. bool restartServoPwmDriver(); +// Rewrites every channel and turns the outputs on, for a chip that kept its +// setup (servoPwmDriverState() just passed). False, outputs off, when a write +// failed. +bool resyncServoPwmDriver(); void servoMoveAllSmoothTo( const float targets[SERVO_COUNT], float speedDegS = SERVO_BOOT_SPEED_DEG_S diff --git a/test/test_i2c_fault/test_i2c_fault.cpp b/test/test_i2c_fault/test_i2c_fault.cpp index 3cbd6f18..e8c2be61 100644 --- a/test/test_i2c_fault/test_i2c_fault.cpp +++ b/test/test_i2c_fault/test_i2c_fault.cpp @@ -112,6 +112,39 @@ void test_second_fault_starts_over() { expectAction(I2cFaultAction::Recover, tracker.due(now)); } +void test_a_fault_takes_the_bus_down_at_once() { + I2cFaultTracker tracker; + tracker.noteCheck(true, 1000); + + TEST_ASSERT_TRUE(tracker.noteFault(1200)); + TEST_ASSERT_TRUE(tracker.down()); + TEST_ASSERT_EQUAL_UINT32(1, tracker.faults()); + TEST_ASSERT_EQUAL_UINT32(1200, tracker.lastFaultMs()); + expectAction(I2cFaultAction::Recover, tracker.due(1200)); +} + +void test_a_fault_while_down_is_not_counted_again() { + I2cFaultTracker tracker; + const uint32_t at = takeDown(tracker, 1000); + tracker.noteRecovery(false, at); + + TEST_ASSERT_FALSE(tracker.noteFault(at + 10)); + TEST_ASSERT_EQUAL_UINT32(1, tracker.faults()); + TEST_ASSERT_EQUAL_UINT32(at, tracker.lastFaultMs()); + // The backoff the failed attempt started still holds. + expectAction(I2cFaultAction::None, tracker.due(at + kI2cFirstRetryMs - 1)); +} + +void test_a_fault_before_the_first_check_still_recovers() { + I2cFaultTracker tracker; + TEST_ASSERT_TRUE(tracker.noteFault(50)); + expectAction(I2cFaultAction::Recover, tracker.due(50)); + + tracker.noteRecovery(true, 60); + TEST_ASSERT_FALSE(tracker.down()); + TEST_ASSERT_EQUAL_UINT32(1, tracker.recoveries()); +} + void test_intervals_survive_millis_wrap() { I2cFaultTracker tracker; const uint32_t beforeWrap = UINT32_MAX - 100; @@ -135,6 +168,9 @@ int main(int, char**) { RUN_TEST(test_failed_recovery_backs_off_to_the_cap); RUN_TEST(test_recovery_brings_the_bus_up_and_resumes_checks); RUN_TEST(test_second_fault_starts_over); + RUN_TEST(test_a_fault_takes_the_bus_down_at_once); + RUN_TEST(test_a_fault_while_down_is_not_counted_again); + RUN_TEST(test_a_fault_before_the_first_check_still_recovers); RUN_TEST(test_intervals_survive_millis_wrap); return UNITY_END(); } From 2c63672edf8d060288f0dc63cf1ce21fbc398135 Mon Sep 17 00:00:00 2001 From: Blake Azuela Date: Mon, 16 Nov 2026 17:46:12 -0500 Subject: [PATCH 07/10] fix(firmware): verify recovery writes and close the gaps Copilot found Copilot's review of the I2C recovery found these, all fixed here: - Outputs came back on after acknowledged channel writes without reading them back, though the recovery promises a verified setup. Each resync write is now read back (four registers per channel) before OE goes low. - The channel resync kept writing after a failure. On a dead bus each further write waited out its timeout; it now stops at the first one, and the next attempt rewrites every channel anyway. - A PCA9685 recovery that restored the setup but failed a channel write lost the pending OLED restart: the retry found the setup intact, took the channel-only path, and a panel that lost power stayed dark. The restart now stays pending until it has run. - The SSD1306 driver ignores failed writes, so an OLED setup or frame that never landed still counted as recovered, and the ACK-only check never noticed. The OLED restart now sends the setup and the frame itself, checking every transfer, at boot and in recovery. Without begin() the frame buffer is not cleared, which drops the saved frame copy (512 B of RAM). - An OLED recovery applied the saved rotation over the one the setup wizard was previewing. A restart now keeps the current rotation. - The bus clear stopped clocking once SDA read high. In an interrupted read that can be a 1 data bit, and the device then pulls SDA low again under the STOP. It now always sends nine clocks, which pass an acknowledge slot. - A bus restart that left a line held kept OLED writes on, so frames went to a dead bus until two checks failed. A held line now stops them too. - POST /servos/park saved the centre pose and replied 200 while the PCA9685 was down and the move's writes were skipped. It now replies 503 "servo driver down" without saving the pose or arming the hold. The upload hook already treats a failed park as a warning. Document the readback, the checked OLED restart and the new park error. Not yet run on the robot. --- docs/api.md | 2 +- docs/hardware/interfaces.md | 4 +- src/api/commands.cpp | 8 +++ src/display/oled.h | 7 +- src/display/oled_init.cpp | 117 ++++++++++++++++++++------------- src/hardware/i2c_health.cpp | 36 +++++++--- src/hardware/servo_wrapper.cpp | 33 +++++++--- src/hardware/servo_wrapper.h | 2 +- 8 files changed, 138 insertions(+), 71 deletions(-) diff --git a/docs/api.md b/docs/api.md index 56ba0090..f0fa8058 100644 --- a/docs/api.md +++ b/docs/api.md @@ -551,7 +551,7 @@ tiny-engineer park --transport serial { "ok": true, "park": "servos", "hold": true } ``` -Pose NVS write failure → **400** `{"ok":false,"error":"pose save failed"}` (PWM hold is still armed). Details: [hardware/servos.md](hardware/servos.md#flash-safe-park). +Pose NVS write failure → **400** `{"ok":false,"error":"pose save failed"}` (PWM hold is still armed). Servo driver faulted (`i2c.pca9685.ok` false in [`GET /health`](#get-health)) → **503** `{"ok":false,"error":"servo driver down"}`: the joints did not move, so the pose is not saved and no PWM hold is armed. Details: [hardware/servos.md](hardware/servos.md#flash-safe-park). ### `POST /test/servo` diff --git a/docs/hardware/interfaces.md b/docs/hardware/interfaces.md index ede31a5b..65d041ed 100644 --- a/docs/hardware/interfaces.md +++ b/docs/hardware/interfaces.md @@ -21,8 +21,8 @@ That table is the boot check. After boot the firmware keeps watching both device - **Check:** every 500 ms the OLED must answer, and the PCA9685 must answer, be awake on its internal clock with auto-increment on, and still hold its output mode (`MODE2 = 0x04`) and prescale. Two failed checks in a row are a fault. Each chip is tracked separately, so one that stays down does not stop the checks on the other. A healthy PCA9685 check is two register reads: `MODE1` and `MODE2` together, then the prescale. - **Failed servo write:** a fault on its own. The first one takes the PCA9685 down at once and switches its outputs off, even inside a blocking move that never reaches the check. So does a boot setup that does not read back correctly. - **While down:** writes to the faulted chip are skipped, so failed transfers do not stall audio and HTTP. At a PCA9685 fault the servo outputs are switched off (OE), so a corrupted setup cannot keep driving them. -- **PCA9685 recovery:** if the chip does not answer, restart `Wire` and clock out a device left holding SDA first. A chip that kept its setup only lost a write or a read: rewrite every servo channel and switch the outputs back on. A chip that lost its setup lost power or was corrupted: set it up again, rewrite every channel, and set the OLED up again too, since both share a supply. Servos jump to where the firmware thinks they are. Outputs stay disabled if the setup does not read back correctly. -- **OLED recovery:** set the panel up again and put the last frame back, so a static screen survives too. Never restarts the bus, so an OLED fault alone does not disturb the servos. +- **PCA9685 recovery:** if the chip does not answer, restart `Wire` and clock out a device left holding SDA first. A chip that kept its setup only lost a write or a read: rewrite every servo channel, read each one back, and switch the outputs back on. A chip that lost its setup lost power or was corrupted: set it up again, rewrite every channel, and set the OLED up again too, since both share a supply. Servos jump to where the firmware thinks they are. Outputs stay disabled unless the setup and every channel read back correctly. +- **OLED recovery:** send the panel its setup and the last frame, so a static screen survives too. Every transfer is checked, since the SSD1306 driver ignores failed writes and a half-configured panel still answers its address. The current rotation is kept. Never restarts the bus, so an OLED fault alone does not disturb the servos. - **Audio:** a recovery blocks for tens of milliseconds, so while the speaker is playing it waits, up to 5 s after the fault. The chip stays off meanwhile. - **Retry:** a failed attempt is retried after 1 s, doubling up to 30 s. - **Report:** `[i2c]` lines on the serial log (with SDA / SCL levels and, for a PCA9685 that answered, its registers) and the `i2c` object in [`GET /health`](../api.md#get-health). diff --git a/src/api/commands.cpp b/src/api/commands.cpp index 27a7d7df..e187e1fd 100644 --- a/src/api/commands.cpp +++ b/src/api/commands.cpp @@ -7,6 +7,7 @@ #include "audio/audio.h" #include "display/oled.h" #include "extensions/push_button.h" +#include "hardware/i2c_health.h" #include "hardware/pca9685_servos.h" #include "hardware/rgb.h" #include "hardware/servo_wrapper.h" @@ -122,6 +123,13 @@ ApiResult apiParkServos() { setAnimationImmediately(AnimationId::None); ensureAllServoOutputs(); centerAllServos(); + + // A faulted servo driver skipped the move's writes: the joints never got to + // mid, so neither save that pose nor report a park. + if (!i2cServoDriverUp()) { + return {false, 503, "servo driver down"}; + } + const bool poseSaved = flushServoPoseToNvs(); setServoParkHoldUntilReboot(true); diff --git a/src/display/oled.h b/src/display/oled.h index a795d63f..109a3ce5 100644 --- a/src/display/oled.h +++ b/src/display/oled.h @@ -34,9 +34,10 @@ void showTextThenIdle(const char* line1, const char* line2 = nullptr); void showIdleScreen(); void initOled(); -// Sets the OLED up again after an I2C fault and puts the last frame back, so -// a static screen survives too. With panelOn false the panel stays dark but -// holds the frame for when it wakes. +// Sets the OLED up again after an I2C fault and sends the last frame, so a +// static screen survives too. Checks every transfer; false leaves the OLED +// unavailable. With panelOn false the panel stays dark but holds the frame for +// when it wakes. Keeps the current rotation. bool restartOled(bool panelOn); void runOledTest(); diff --git a/src/display/oled_init.cpp b/src/display/oled_init.cpp index 545e5671..0284fb45 100644 --- a/src/display/oled_init.cpp +++ b/src/display/oled_init.cpp @@ -1,7 +1,6 @@ #include #include #include -#include #include "pins.h" #include "display/oled.h" @@ -17,28 +16,71 @@ bool i2cDeviceConnected(uint8_t address) { namespace { constexpr size_t kOledFrameBytes = OLED_WIDTH * ((OLED_HEIGHT + 7) / 8); +// Frame bytes per transfer, well inside Wire's buffer. +constexpr size_t kOledFrameChunk = 32; + +// What begin() sends a 128x32 panel on its internal charge pump, minus the +// final DISPLAYON. The driver ignores failed writes, and a panel that missed +// part of this still answers its address, so this copy checks each transfer. +constexpr uint8_t kOledSetup[] = { + SSD1306_DISPLAYOFF, + SSD1306_SETDISPLAYCLOCKDIV, 0x80, + SSD1306_SETMULTIPLEX, OLED_HEIGHT - 1, + SSD1306_SETDISPLAYOFFSET, 0x00, + SSD1306_SETSTARTLINE | 0x00, + SSD1306_CHARGEPUMP, 0x14, + SSD1306_MEMORYMODE, 0x00, + SSD1306_SEGREMAP | 0x01, + SSD1306_COMSCANDEC, + SSD1306_SETCOMPINS, 0x02, + SSD1306_SETCONTRAST, 0x8F, + SSD1306_SETPRECHARGE, 0xF1, + SSD1306_SETVCOMDETECT, 0x40, + SSD1306_DISPLAYALLON_RESUME, + SSD1306_NORMALDISPLAY, + SSD1306_DEACTIVATE_SCROLL +}; +static_assert(OLED_WIDTH == 128 && OLED_HEIGHT == 32, "kOledSetup is for a 128x32 panel"); + +constexpr uint8_t kOledFrameWindow[] = { + SSD1306_PAGEADDR, 0x00, 0xFF, + SSD1306_COLUMNADDR, 0x00, OLED_WIDTH - 1 +}; + +// One I2C transfer: a control byte (0x00 commands, 0x40 frame data), then +// the bytes. True when the panel acknowledged all of it. +bool sendOled(uint8_t control, const uint8_t* bytes, size_t count) { + Wire.beginTransmission(OLED_ADDRESS); + Wire.write(control); + Wire.write(bytes, count); + return Wire.endTransmission() == 0; +} + +// Sends the panel its full setup and the frame buffer, then turns it on when +// panelOn. Safe to repeat: a panel that lost power comes back dark until it +// gets this again. The frame goes over while the panel is still off, so it +// wakes showing it. Leaves the frame buffer and the rotation as they are. +bool configureOled(bool panelOn) { + const uint8_t* frame = display.getBuffer(); -// The frame on the panel before a restart. begin() clears the frame buffer, -// and static screens are never drawn again, so a restart puts this back. -uint8_t g_savedFrame[kOledFrameBytes]; -// Set from the first attempt of a restart until one succeeds: a failed -// attempt's begin() has already cleared the buffer, so later attempts must not -// save it again. -bool g_frameSaved = false; - -// Sends the SSD1306 its full setup and turns the panel on. Safe to repeat: a -// panel that lost power comes back dark until it gets this again. Clears the -// frame buffer. -bool configureOled() { - if (!display.begin( - SSD1306_SWITCHCAPVCC, - OLED_ADDRESS - )) { + if (frame == nullptr || + !sendOled(0x00, kOledSetup, sizeof(kOledSetup)) || + !sendOled(0x00, kOledFrameWindow, sizeof(kOledFrameWindow))) { return false; } - display.setRotation(settingsOledRotate180() ? 2 : 0); - return true; + for (size_t offset = 0; offset < kOledFrameBytes; offset += kOledFrameChunk) { + if (!sendOled(0x40, frame + offset, kOledFrameChunk)) { + return false; + } + } + + if (!panelOn) { + return true; + } + + const uint8_t on = SSD1306_DISPLAYON; + return sendOled(0x00, &on, 1); } } // namespace @@ -55,42 +97,25 @@ void initOled() { serialLogPrintln("OLED found"); - if (!configureOled()) { + // begin() allocates and clears the frame buffer and sets the driver up; + // configureOled() then sends the setup again with each transfer checked. + if (!display.begin(SSD1306_SWITCHCAPVCC, OLED_ADDRESS) || !configureOled(true)) { serialLogPrintln("ERROR: OLED initialization failed"); oledAvailable = false; return; } - display.display(); + display.setRotation(settingsOledRotate180() ? 2 : 0); oledAvailable = true; serialLogPrintln("OLED initialized"); } +// Not begin(): it clears the frame buffer and ignores failed writes. The +// rotation is left alone too, since the setup wizard may be previewing one it +// has not saved. The frame buffer still holds the last frame, so a static +// screen comes back too. bool restartOled(bool panelOn) { - uint8_t* buffer = display.getBuffer(); - - if (buffer != nullptr && !g_frameSaved) { - memcpy(g_savedFrame, buffer, kOledFrameBytes); - g_frameSaved = true; - } - - oledAvailable = i2cDeviceConnected(OLED_ADDRESS) && configureOled(); - - if (!oledAvailable) { - return false; - } - - // Dark first: the panel takes the frame while off, and wakes showing it. - if (!panelOn) { - display.ssd1306_command(SSD1306_DISPLAYOFF); - } - - if (g_frameSaved) { - memcpy(display.getBuffer(), g_savedFrame, kOledFrameBytes); - g_frameSaved = false; - } - - display.display(); - return true; + oledAvailable = i2cDeviceConnected(OLED_ADDRESS) && configureOled(panelOn); + return oledAvailable; } diff --git a/src/hardware/i2c_health.cpp b/src/hardware/i2c_health.cpp index 41f44947..628f565f 100644 --- a/src/hardware/i2c_health.cpp +++ b/src/hardware/i2c_health.cpp @@ -57,19 +57,27 @@ bool g_oledExpected = false; // buffers a transfer leaks Wire's lock, and the next restart would wait on // that lock forever. bool g_wireUp = true; +// Set when the PCA9685 is found without its setup; cleared once the OLED has +// been set up again. Kept across failed attempts: a retry finds the PCA9685's +// setup restored and would otherwise skip the panel. +bool g_oledRestartPending = false; // Registers from the latest PCA9685 check, for the fault log line. ServoPwmDriverState g_servoDriverState = {}; -// A device interrupted mid-byte keeps SDA low until it has been clocked +// A device interrupted mid-byte keeps driving SDA until it has been clocked // through the rest of that byte. Clock it out, send a STOP, and report where // the lines settled: a line still low means something is holding the bus. +// All nine clocks, even once SDA reads high: in an interrupted read a high +// SDA can be a 1 data bit, and the device would pull the line low again under +// the STOP. Nine clocks always pass an acknowledge slot, where the released +// SDA reads as a NACK and ends the read. BusLines clearBus() { pinMode(I2C_SDA, INPUT_PULLUP); pinMode(I2C_SCL, OUTPUT_OPEN_DRAIN); digitalWrite(I2C_SCL, HIGH); delayMicroseconds(kBusClearHalfPeriodUs); - for (uint8_t i = 0; i < kBusClearPulses && digitalRead(I2C_SDA) == LOW; i++) { + for (uint8_t i = 0; i < kBusClearPulses; i++) { digitalWrite(I2C_SCL, LOW); delayMicroseconds(kBusClearHalfPeriodUs); digitalWrite(I2C_SCL, HIGH); @@ -162,15 +170,18 @@ bool restartBus() { const BusLines lines = clearBus(); g_wireUp = Wire.begin(I2C_SDA, I2C_SCL); + const bool usable = g_wireUp && lines.sdaHigh && lines.sclHigh; - if (!g_wireUp) { + // A held line fails every transfer as surely as a dead Wire: stop the + // OLED's frames, and let its own checks recover it once the bus is back. + if (!usable) { stopOledWrites(); } serialLogPrint(g_wireUp ? "[i2c] bus restarted" : "[i2c] Wire.begin failed"); logLines(lines); - return g_wireUp && lines.sdaHigh && lines.sclHigh; + return usable; } // The restart puts the last frame back on the panel; a sleeping robot keeps @@ -184,6 +195,8 @@ bool restartOledForSleepState() { // OLED counts as up clears oledAvailable, and the OLED's own checks then take // it down and recover it. void restartOledAfterPowerLoss() { + g_oledRestartPending = false; + if (!g_oledExpected) { return; } @@ -240,14 +253,21 @@ bool recoverServoDriver() { } if (servoPwmDriverState(g_servoDriverState)) { - return resyncServoPwmDriver(); + if (!resyncServoPwmDriver()) { + return false; + } + } else { + g_oledRestartPending = true; + + if (!restartServoPwmDriver()) { + return false; + } } - if (!restartServoPwmDriver()) { - return false; + if (g_oledRestartPending) { + restartOledAfterPowerLoss(); } - restartOledAfterPowerLoss(); return true; } diff --git a/src/hardware/servo_wrapper.cpp b/src/hardware/servo_wrapper.cpp index edef62b0..77db9a68 100644 --- a/src/hardware/servo_wrapper.cpp +++ b/src/hardware/servo_wrapper.cpp @@ -293,6 +293,21 @@ bool readPcaRegisters(uint8_t reg, uint8_t* values, uint8_t count) { return true; } +// Writes one channel and reads it back. Recovery and boot turn the outputs on +// only after this: an acknowledged write can still land corrupted on a bus +// that just glitched. +bool writeChannelVerified(uint8_t channel, uint16_t off) { + if (pwm.setPWM(channel, 0, off) != 0) { + return false; + } + + uint8_t got[4] = {}; + const uint8_t want[4] = {0, 0, static_cast(off & 0xFF), static_cast(off >> 8)}; + + return readPcaRegisters(PCA9685_LED0_ON_L + 4 * channel, got, 4) && + memcmp(got, want, sizeof(want)) == 0; +} + // Output mode and servo frequency, shared by boot and recovery. The chip must // be awake first: setPWMFreq() restores the sleep bit it finds. void writeServoSetup() { @@ -308,16 +323,14 @@ void writeServoSetup() { } // Writes every channel from the firmware's state, then turns the outputs on. -// False, with the outputs left off, when a write failed. +// False, with the outputs left off, when a write failed or did not read back. +// Stops at the first failure: on a dead bus each further write waits out its +// timeout, and the next attempt rewrites every channel anyway. bool writeChannelsAndEnableOutputs() { - bool written = true; - for (int i = 0; i < SERVO_COUNT; i++) { - written = g_servos[i].resyncOutput() && written; - } - - if (!written) { - return false; + if (!g_servos[i].resyncOutput()) { + return false; + } } // Let each channel's first pulse go out before the servos see it. @@ -575,12 +588,12 @@ bool ServoWrapper::resyncOutput() { // boot and while the chip still counts as down. if (released_) { lastPulse_ = UINT16_MAX; - return pwm.setPWM(SERVO_SPECS[index_].channel, 0, kPca9685FullOff) == 0; + return writeChannelVerified(SERVO_SPECS[index_].channel, kPca9685FullOff); } const uint16_t pulse = angleToPulse(angle_); - if (pwm.setPWM(SERVO_SPECS[index_].channel, 0, pulse) != 0) { + if (!writeChannelVerified(SERVO_SPECS[index_].channel, pulse)) { // Not cached: the next write for this servo must go out. lastPulse_ = UINT16_MAX; return false; diff --git a/src/hardware/servo_wrapper.h b/src/hardware/servo_wrapper.h index cf5ba502..7cffcd89 100644 --- a/src/hardware/servo_wrapper.h +++ b/src/hardware/servo_wrapper.h @@ -33,7 +33,7 @@ class ServoWrapper { void ensureServoOutput(); void releaseServoOutput(); // Rewrites this channel from the firmware's state after a driver restart. - // False when the write failed. + // False when the write failed or did not read back. bool resyncOutput(); friend void servoMoveAllSmoothTo( From fbc642b2e80a70ceaa8c3aa0f68e7bcb0efff052 Mon Sep 17 00:00:00 2001 From: Blake Azuela Date: Mon, 16 Nov 2026 20:39:14 -0500 Subject: [PATCH 08/10] fix(firmware): step the PCA9685 recovery and enable outputs between pulses Close the two Copilot findings left open in 7c59720, and a slowdown it introduced: - A recovery could still cut a gap into a clip. It waited for the speaker for up to 5 s, then ran in one go: tens of milliseconds, and each failed transfer waited out Wire's 50 ms timeout. The PCA9685's recovery now runs as steps, one per loop pass (probe, bus restart, setup check, setup, channels off, settle, channels on, OLED), a pass services at most one chip, and transfers time out after 20 ms. A pass then costs about one timeout at worst, inside the speaker's DMA buffer of about 65 ms (6 x 240 frames at 22,050 Hz). The audio wait and speakerPlaying() are gone. - Turning OE on is not synchronised to the PWM period, so enabling the outputs while a channel was mid-pulse cut that pulse short. Channel writes are: the PCA9685 applies new values at the end of a LOW cycle. Boot and recovery now turn every servo channel off, wait more than one period, turn OE on, and only then write and read back the channels, so each servo starts on a whole pulse. - The checked OLED transfers from 7c59720 ran at 100 kHz, where the SSD1306 driver switches to 400 kHz, so an OLED restart took about four times as long. They now switch the clock the same way. Not yet run on the robot. The LOW-cycle behaviour comes from the datasheet and wants a scope check. --- docs/hardware/interfaces.md | 4 +- src/audio/audio.cpp | 13 --- src/audio/audio.h | 2 - src/display/oled_init.cpp | 22 +++-- src/hardware/i2c_health.cpp | 160 +++++++++++++++++++++++---------- src/hardware/servo_wrapper.cpp | 74 +++++++++------ src/hardware/servo_wrapper.h | 21 +++-- 7 files changed, 190 insertions(+), 106 deletions(-) diff --git a/docs/hardware/interfaces.md b/docs/hardware/interfaces.md index 65d041ed..0ee2f4e0 100644 --- a/docs/hardware/interfaces.md +++ b/docs/hardware/interfaces.md @@ -21,9 +21,9 @@ That table is the boot check. After boot the firmware keeps watching both device - **Check:** every 500 ms the OLED must answer, and the PCA9685 must answer, be awake on its internal clock with auto-increment on, and still hold its output mode (`MODE2 = 0x04`) and prescale. Two failed checks in a row are a fault. Each chip is tracked separately, so one that stays down does not stop the checks on the other. A healthy PCA9685 check is two register reads: `MODE1` and `MODE2` together, then the prescale. - **Failed servo write:** a fault on its own. The first one takes the PCA9685 down at once and switches its outputs off, even inside a blocking move that never reaches the check. So does a boot setup that does not read back correctly. - **While down:** writes to the faulted chip are skipped, so failed transfers do not stall audio and HTTP. At a PCA9685 fault the servo outputs are switched off (OE), so a corrupted setup cannot keep driving them. -- **PCA9685 recovery:** if the chip does not answer, restart `Wire` and clock out a device left holding SDA first. A chip that kept its setup only lost a write or a read: rewrite every servo channel, read each one back, and switch the outputs back on. A chip that lost its setup lost power or was corrupted: set it up again, rewrite every channel, and set the OLED up again too, since both share a supply. Servos jump to where the firmware thinks they are. Outputs stay disabled unless the setup and every channel read back correctly. +- **PCA9685 recovery:** if the chip does not answer, restart `Wire` and clock out a device left holding SDA first. A chip that kept its setup only lost a write or a read. A chip that lost its setup lost power or was corrupted: set it up again, and set the OLED up again too afterwards, since both share a supply. Either way, the outputs come back on with every servo channel off, and then every channel is written and read back. The PCA9685 applies channel values at the end of a LOW cycle, so each servo starts on a whole pulse; turning OE on mid-pulse would cut one short. Servos jump to where the firmware thinks they are. Outputs stay disabled unless the setup and every channel read back correctly. - **OLED recovery:** send the panel its setup and the last frame, so a static screen survives too. Every transfer is checked, since the SSD1306 driver ignores failed writes and a half-configured panel still answers its address. The current rotation is kept. Never restarts the bus, so an OLED fault alone does not disturb the servos. -- **Audio:** a recovery blocks for tens of milliseconds, so while the speaker is playing it waits, up to 5 s after the fault. The chip stays off meanwhile. +- **Audio:** a recovery runs in steps, one per loop pass, and a pass services at most one chip. Transfers time out after 20 ms (Wire's default is 50 ms), so a pass costs about one timeout at worst, well inside the speaker's DMA buffer of about 65 ms. - **Retry:** a failed attempt is retried after 1 s, doubling up to 30 s. - **Report:** `[i2c]` lines on the serial log (with SDA / SCL levels and, for a PCA9685 that answered, its registers) and the `i2c` object in [`GET /health`](../api.md#get-health). diff --git a/src/audio/audio.cpp b/src/audio/audio.cpp index 9f78c988..539ce50f 100644 --- a/src/audio/audio.cpp +++ b/src/audio/audio.cpp @@ -21,12 +21,6 @@ namespace { bool audioStorageReady = false; -// A gap this long between speaker writes means no clip is playing. Longer -// than a /play body's usual network stall. -constexpr uint32_t kSpeakerQuietMs = 250; -bool g_speakerFed = false; -uint32_t g_lastSpeakerWriteMs = 0; - constexpr const char* kAudioPartition = "spiffs"; void logAudioStorageContents() { @@ -105,16 +99,9 @@ bool initAudioStorage() { return true; } -bool speakerPlaying(uint32_t now) { - return g_speakerFed && now - g_lastSpeakerWriteMs < kSpeakerQuietMs; -} - void writeMonoToSpeaker(const int16_t* samples, size_t count) { constexpr size_t FRAMES = 256; - g_speakerFed = true; - g_lastSpeakerWriteMs = millis(); - int16_t buffer[FRAMES * 2]; const uint8_t volume = settingsVolume(); diff --git a/src/audio/audio.h b/src/audio/audio.h index 252eb261..ff052d91 100644 --- a/src/audio/audio.h +++ b/src/audio/audio.h @@ -7,8 +7,6 @@ extern I2SClass I2S; bool initAudioStorage(); // Volume-scaled, duplicated to both I2S slots. Blocks until the DMA buffer accepts it. void writeMonoToSpeaker(const int16_t* samples, size_t count); -// True while a clip is playing: samples reached the speaker moments ago. -bool speakerPlaying(uint32_t now); void playTone( float frequency, int durationMs diff --git a/src/display/oled_init.cpp b/src/display/oled_init.cpp index 0284fb45..2680eed9 100644 --- a/src/display/oled_init.cpp +++ b/src/display/oled_init.cpp @@ -18,6 +18,11 @@ namespace { constexpr size_t kOledFrameBytes = OLED_WIDTH * ((OLED_HEIGHT + 7) / 8); // Frame bytes per transfer, well inside Wire's buffer. constexpr size_t kOledFrameChunk = 32; +// The driver runs its own transfers at 400 kHz and drops back to 100 kHz for +// the PCA9685 (its clkDuring / clkAfter defaults). The checked transfers here +// do the same, or a frame would take four times as long. +constexpr uint32_t kOledClockHz = 400000; +constexpr uint32_t kBusClockHz = 100000; // What begin() sends a 128x32 panel on its internal charge pump, minus the // final DISPLAYON. The driver ignores failed writes, and a panel that missed @@ -56,11 +61,7 @@ bool sendOled(uint8_t control, const uint8_t* bytes, size_t count) { return Wire.endTransmission() == 0; } -// Sends the panel its full setup and the frame buffer, then turns it on when -// panelOn. Safe to repeat: a panel that lost power comes back dark until it -// gets this again. The frame goes over while the panel is still off, so it -// wakes showing it. Leaves the frame buffer and the rotation as they are. -bool configureOled(bool panelOn) { +bool sendOledSetupAndFrame(bool panelOn) { const uint8_t* frame = display.getBuffer(); if (frame == nullptr || @@ -83,6 +84,17 @@ bool configureOled(bool panelOn) { return sendOled(0x00, &on, 1); } +// Sends the panel its full setup and the frame buffer, then turns it on when +// panelOn. Safe to repeat: a panel that lost power comes back dark until it +// gets this again. The frame goes over while the panel is still off, so it +// wakes showing it. Leaves the frame buffer and the rotation as they are. +bool configureOled(bool panelOn) { + Wire.setClock(kOledClockHz); + const bool ok = sendOledSetupAndFrame(panelOn); + Wire.setClock(kBusClockHz); + return ok; +} + } // namespace void initOled() { diff --git a/src/hardware/i2c_health.cpp b/src/hardware/i2c_health.cpp index 628f565f..e99adb6b 100644 --- a/src/hardware/i2c_health.cpp +++ b/src/hardware/i2c_health.cpp @@ -1,7 +1,6 @@ #include #include -#include "audio/audio.h" #include "display/oled.h" #include "hardware/i2c_fault.h" #include "hardware/i2c_health.h" @@ -14,10 +13,18 @@ namespace { constexpr uint8_t kBusClearPulses = 9; constexpr uint32_t kBusClearHalfPeriodUs = 5; -// A recovery blocks for tens of milliseconds, which is an audible gap while a -// clip plays. It waits for the speaker to go quiet, but not for longer than -// this after the fault: a long clip must not keep the robot limp. -constexpr uint32_t kRecoveryAudioWaitMaxMs = 5000; +// Wire's default is 50 ms. Every transfer here finishes in a few: the +// largest, a 128-byte OLED frame chunk, takes about 3 ms at 400 kHz and 12 ms +// at 100 kHz. On a dead bus each transfer waits this long, and a tick does at +// most one device's work, about one timeout. The speaker's DMA buffer holds +// about 65 ms (6 x 240 frames at 22,050 Hz), so a tick never starves it. +constexpr uint16_t kI2cTransferTimeoutMs = 20; + +enum class RecoveryResult : uint8_t { + InProgress, // a step ran; call again next tick + Succeeded, + Failed +}; struct BusLines { bool sdaHigh; @@ -30,18 +37,19 @@ struct Device { const char* name; bool (*healthy)(); void (*stopWrites)(); - bool (*recover)(); + // One step of a recovery, bounded by about one transfer timeout. + RecoveryResult (*recover)(uint32_t now); // Extra text for the fault log line; null when there is none. void (*logFaultDetail)(); I2cFaultTracker tracker; }; bool servoDriverHealthy(); -bool recoverServoDriver(); +RecoveryResult recoverServoDriver(uint32_t now); void logServoDriverDetail(); bool oledHealthy(); void stopOledWrites(); -bool recoverOled(); +RecoveryResult recoverOled(uint32_t now); Device g_servoDriver = { // At a fault the outputs go off, so a corrupted setup cannot keep driving @@ -61,6 +69,20 @@ bool g_wireUp = true; // been set up again. Kept across failed attempts: a retry finds the PCA9685's // setup restored and would otherwise skip the panel. bool g_oledRestartPending = false; +// Where the PCA9685's recovery is. It runs one step per tick, each a short +// burst of transfers, so a recovery never holds the loop for long. +enum class ServoRecoveryStep : uint8_t { + Probe, // does the chip answer? + RestartBus, // it did not: restart Wire, then ask again + CheckSetup, // does it still hold its setup? + SetUp, // it did not: set it up again + ChannelsOff, // every channel off, outputs off + Settle, // wait for the off values to take effect + ChannelsOn, // outputs on, then every channel written + Oled // set the panel up again after a power loss +}; +ServoRecoveryStep g_servoStep = ServoRecoveryStep::Probe; +uint32_t g_channelsOffMs = 0; // Registers from the latest PCA9685 check, for the fault log line. ServoPwmDriverState g_servoDriverState = {}; @@ -236,39 +258,75 @@ bool servoDriverAnswers() { return g_wireUp && i2cDeviceConnected(PCA9685_ADDRESS); } +RecoveryResult endServoRecovery(RecoveryResult result) { + g_servoStep = ServoRecoveryStep::Probe; + return result; +} + +RecoveryResult nextServoStep(ServoRecoveryStep step) { + g_servoStep = step; + return RecoveryResult::InProgress; +} + // The PCA9685 is the required chip, so its recovery is the one that restarts // the bus. A chip that still answers proves the bus works. One that also kept // its setup only lost a write or a read: rewriting the channels is enough, and // the healthy OLED is left alone. A lost setup means the chip lost power or // was corrupted: it is set up from scratch, and so is the panel. -bool recoverServoDriver() { - bool reachable = servoDriverAnswers(); +RecoveryResult recoverServoDriver(uint32_t now) { + switch (g_servoStep) { + case ServoRecoveryStep::Probe: + return nextServoStep( + servoDriverAnswers() ? ServoRecoveryStep::CheckSetup : ServoRecoveryStep::RestartBus + ); + + case ServoRecoveryStep::RestartBus: + if (!restartBus() || !servoDriverAnswers()) { + return endServoRecovery(RecoveryResult::Failed); + } + return nextServoStep(ServoRecoveryStep::CheckSetup); - if (!reachable) { - reachable = restartBus() && servoDriverAnswers(); - } + case ServoRecoveryStep::CheckSetup: + if (servoPwmDriverState(g_servoDriverState)) { + return nextServoStep(ServoRecoveryStep::ChannelsOff); + } + g_oledRestartPending = true; + return nextServoStep(ServoRecoveryStep::SetUp); - if (!reachable) { - return false; - } + case ServoRecoveryStep::SetUp: + if (!setUpServoPwmDriver()) { + return endServoRecovery(RecoveryResult::Failed); + } + return nextServoStep(ServoRecoveryStep::ChannelsOff); - if (servoPwmDriverState(g_servoDriverState)) { - if (!resyncServoPwmDriver()) { - return false; - } - } else { - g_oledRestartPending = true; + case ServoRecoveryStep::ChannelsOff: + if (!writeServoChannelsOff()) { + return endServoRecovery(RecoveryResult::Failed); + } + g_channelsOffMs = now; + return nextServoStep(ServoRecoveryStep::Settle); - if (!restartServoPwmDriver()) { - return false; - } - } + case ServoRecoveryStep::Settle: + if (now - g_channelsOffMs < kServoOutputSettleMs) { + return RecoveryResult::InProgress; + } + return nextServoStep(ServoRecoveryStep::ChannelsOn); + + case ServoRecoveryStep::ChannelsOn: + if (!enableServoOutputsWithChannels()) { + return endServoRecovery(RecoveryResult::Failed); + } + if (!g_oledRestartPending) { + return endServoRecovery(RecoveryResult::Succeeded); + } + return nextServoStep(ServoRecoveryStep::Oled); - if (g_oledRestartPending) { - restartOledAfterPowerLoss(); + case ServoRecoveryStep::Oled: + restartOledAfterPowerLoss(); + return endServoRecovery(RecoveryResult::Succeeded); } - return true; + return endServoRecovery(RecoveryResult::Failed); } // oledAvailable is part of the check: a restart that failed outside an OLED @@ -283,8 +341,10 @@ void stopOledWrites() { // Never restarts the bus: an OLED fault alone must not disturb the servos. // A dead bus is left to the PCA9685's recovery. -bool recoverOled() { - return g_wireUp && restartOledForSleepState(); +RecoveryResult recoverOled(uint32_t) { + return g_wireUp && restartOledForSleepState() + ? RecoveryResult::Succeeded + : RecoveryResult::Failed; } // The device is down: stop its writes, since each failed one blocks the loop @@ -294,14 +354,10 @@ void enterFault(Device& device, uint32_t now, const char* cause = nullptr) { logFault(device, now, cause); } -bool recoveryWaitsForAudio(const Device& device, uint32_t now) { - return speakerPlaying(now) && - now - device.tracker.lastFaultMs() < kRecoveryAudioWaitMaxMs; -} - -void service(Device& device) { - // Read here, not once per tick: the other chip's recovery can take tens of - // milliseconds and stamp this tracker with a later time. +// True when it did bus work: a check, or a step of a recovery. +bool service(Device& device) { + // Read here, not once per tick: the other chip's work can stamp this + // tracker with a later time. const uint32_t now = millis(); switch (device.tracker.due(now)) { @@ -309,15 +365,21 @@ void service(Device& device) { if (device.tracker.noteCheck(device.healthy(), now)) { enterFault(device, now); } - break; - case I2cFaultAction::Recover: - if (!recoveryWaitsForAudio(device, now)) { - noteRecovery(device, device.recover()); + return true; + case I2cFaultAction::Recover: { + // A recovery in progress keeps its tracker due until it ends. + const RecoveryResult result = device.recover(now); + + if (result != RecoveryResult::InProgress) { + noteRecovery(device, result == RecoveryResult::Succeeded); } - break; + return true; + } case I2cFaultAction::None: - break; + return false; } + + return false; } I2cDeviceHealth deviceHealth(const Device& device, bool up) { @@ -339,6 +401,8 @@ void initI2cBus() { logLines(lines); } + // Kept across Wire restarts: it is a member, set once. + Wire.setTimeOut(kI2cTransferTimeoutMs); g_wireUp = Wire.begin(I2C_SDA, I2C_SCL); if (!g_wireUp) { @@ -356,9 +420,9 @@ void updateI2cHealth() { return; } - service(g_servoDriver); - - if (g_oledExpected) { + // At most one chip's work per tick, so a tick costs about one transfer + // timeout at worst. The OLED's turn comes a tick later. + if (!service(g_servoDriver) && g_oledExpected) { service(g_oled); } } diff --git a/src/hardware/servo_wrapper.cpp b/src/hardware/servo_wrapper.cpp index 77db9a68..f7e86e3e 100644 --- a/src/hardware/servo_wrapper.cpp +++ b/src/hardware/servo_wrapper.cpp @@ -293,9 +293,8 @@ bool readPcaRegisters(uint8_t reg, uint8_t* values, uint8_t count) { return true; } -// Writes one channel and reads it back. Recovery and boot turn the outputs on -// only after this: an acknowledged write can still land corrupted on a bus -// that just glitched. +// Writes one channel and reads it back: an acknowledged write can still land +// corrupted on a bus that just glitched. bool writeChannelVerified(uint8_t channel, uint16_t off) { if (pwm.setPWM(channel, 0, off) != 0) { return false; @@ -322,28 +321,17 @@ void writeServoSetup() { pwm.setPWMFreq(kServoPwmHz); } -// Writes every channel from the firmware's state, then turns the outputs on. -// False, with the outputs left off, when a write failed or did not read back. -// Stops at the first failure: on a dead bus each further write waits out its -// timeout, and the next attempt rewrites every channel anyway. -bool writeChannelsAndEnableOutputs() { - for (int i = 0; i < SERVO_COUNT; i++) { - if (!g_servos[i].resyncOutput()) { - return false; - } - } - - // Let each channel's first pulse go out before the servos see it. - delay(10); - enableServoOutputs(); - return true; -} - -// The end of boot and of a restart. Outputs stay off unless the setup took: -// a wrong output mode or frequency must not reach the servos. +// The end of boot: the recovery's sequence in one go. Outputs stay off unless +// the setup took: a wrong output mode or frequency must not reach the servos. bool finishServoPwmDriverSetup() { ServoPwmDriverState state; - return servoPwmDriverState(state) && writeChannelsAndEnableOutputs(); + + if (!servoPwmDriverState(state) || !writeServoChannelsOff()) { + return false; + } + + delay(kServoOutputSettleMs); + return enableServoOutputsWithChannels(); } } // namespace @@ -369,7 +357,7 @@ bool servoPwmDriverState(ServoPwmDriverState& state) { state.prescale == kPcaPrescaleServo; } -bool restartServoPwmDriver() { +bool setUpServoPwmDriver() { disableServoOutputs(); // Not pwm.begin(): it programs 1 kHz before writeServoSetup() sets 50 Hz, @@ -378,14 +366,42 @@ bool restartServoPwmDriver() { pwm.reset(); writeServoSetup(); - // The servos kept animating in software while the chip was unreachable; - // finishing puts every channel where the firmware thinks it is. - return finishServoPwmDriverSetup(); + ServoPwmDriverState state; + return servoPwmDriverState(state); } -bool resyncServoPwmDriver() { +// Turning OE on is not synchronised to the PWM period: done while a channel is +// mid-pulse it would cut that pulse short, and the servo would read a wrong +// angle. Channel writes are: the PCA9685 applies new values at the end of a +// LOW cycle (datasheet 7.3.3). So the outputs come on with every channel off, +// and the channels are written after, each starting on a whole pulse. +bool writeServoChannelsOff() { disableServoOutputs(); - return writeChannelsAndEnableOutputs(); + + // Stops at the first failure: on a dead bus each further write waits out + // its timeout, and the next attempt writes every channel anyway. + for (int i = 0; i < SERVO_COUNT; i++) { + if (!writeChannelVerified(SERVO_SPECS[i].channel, kPca9685FullOff)) { + return false; + } + } + + return true; +} + +bool enableServoOutputsWithChannels() { + enableServoOutputs(); + + // The servos kept animating in software while the chip was unreachable; + // this puts every channel where the firmware thinks it is. + for (int i = 0; i < SERVO_COUNT; i++) { + if (!g_servos[i].resyncOutput()) { + disableServoOutputs(); + return false; + } + } + + return true; } void initServoPwmDriver() { diff --git a/src/hardware/servo_wrapper.h b/src/hardware/servo_wrapper.h index 7cffcd89..860d50f2 100644 --- a/src/hardware/servo_wrapper.h +++ b/src/hardware/servo_wrapper.h @@ -72,13 +72,20 @@ struct ServoPwmDriverState { // Reads the PCA9685's setup registers. True when it answers, is awake with // auto-increment on, and holds the servo output mode and prescale. bool servoPwmDriverState(ServoPwmDriverState& state); -// Sets the PCA9685 up again and rewrites every channel; keeps the current pose. -// False, with the outputs left disabled, when the setup did not take. -bool restartServoPwmDriver(); -// Rewrites every channel and turns the outputs on, for a chip that kept its -// setup (servoPwmDriverState() just passed). False, outputs off, when a write -// failed. -bool resyncServoPwmDriver(); +// Bringing the outputs back, in steps a caller can spread over several loop +// passes: setUpServoPwmDriver() if the setup was lost, writeServoChannelsOff(), +// at least kServoOutputSettleMs, then enableServoOutputsWithChannels(). Each +// returns false, with the outputs off, when a write failed or did not read +// back. +constexpr uint32_t kServoOutputSettleMs = 25; // more than one 20 ms period +// Wakes the PCA9685 and writes its output mode and frequency, outputs off. +// True when the setup reads back. +bool setUpServoPwmDriver(); +// Turns every servo channel fully off, outputs off. +bool writeServoChannelsOff(); +// Turns the outputs on, then writes every channel from the firmware's state; +// keeps the current pose. +bool enableServoOutputsWithChannels(); void servoMoveAllSmoothTo( const float targets[SERVO_COUNT], float speedDegS = SERVO_BOOT_SPEED_DEG_S From 03580111714eb3007d698a816a01585dad40739b Mon Sep 17 00:00:00 2001 From: Blake Azuela Date: Mon, 16 Nov 2026 21:02:55 -0500 Subject: [PATCH 09/10] fix(firmware): keep watching an OLED whose boot setup failed The I2C checks watched the OLED only if oledAvailable was true when they started. Since 7c59720 a failed checked setup transfer at boot clears it, so a panel that answered but missed part of its setup was never checked or recovered and stayed dark until reboot. Track "answered at boot and the driver started" separately from "usable now". The checks watch any panel detected at boot; one whose setup failed faults on its first checks and gets that setup retried. Boot logs "OLED setup failed; retrying after boot", added to the troubleshooting table. Not yet run on the robot. --- docs/hardware/testing.md | 1 + src/display/oled.h | 3 +++ src/display/oled_init.cpp | 21 ++++++++++++++++++++- src/hardware/i2c_health.cpp | 5 +++-- 4 files changed, 27 insertions(+), 3 deletions(-) diff --git a/docs/hardware/testing.md b/docs/hardware/testing.md index d1ed4cbd..7d3fe04e 100644 --- a/docs/hardware/testing.md +++ b/docs/hardware/testing.md @@ -140,6 +140,7 @@ HTTP runs on STA when connected, or on setup AP at `192.168.4.1` during provisio | --- | --- | --- | | OLED `ERROR: OLED not found` then rest of boot runs | Nothing ACK’d at `0x3C` | OLED **VCC=3V3**, GND, SDA=GP0, **SCL**=GP1, common ground, address jumper still 0x3C | | OLED found but `ERROR: OLED initialization failed` | ACK then `display.begin` failed | Wiring/power glitch, wrong size module, I2C noise | +| OLED found but `ERROR: OLED setup failed; retrying after boot` | ACK, but a setup transfer failed | Retries by itself: an `[i2c] oled FAULT` and `recovered` line follow. Repeats point at wiring, power or I2C noise | | OLED shows `Join this WiFi` / AP name, then `Then open` / `192.168.4.1` | Setup AP mode active | Connect to the shown AP, open `http://192.168.4.1/config`, finish the servo + screen + LED + speaker + WiFi wizard | | OLED `WiFi failed` then setup AP | No saved network connected | Join setup AP, open `http://192.168.4.1/config`, enter home WiFi again | | `ERROR: PCA9685 not found` + red RGB + **hang** | Nothing ACK’d at `0x40` | PCA9685 **VCC=3V3** (not V+), GND, SDA/SCL, I2C address pads, +5V not required for the ACK but needed later for motion | diff --git a/src/display/oled.h b/src/display/oled.h index 109a3ce5..72513b4c 100644 --- a/src/display/oled.h +++ b/src/display/oled.h @@ -34,6 +34,9 @@ void showTextThenIdle(const char* line1, const char* line2 = nullptr); void showIdleScreen(); void initOled(); +// True when the OLED answered at boot and its driver started, even if its +// setup then failed: the I2C checks watch it and retry that setup. +bool oledDetectedAtBoot(); // Sets the OLED up again after an I2C fault and sends the last frame, so a // static screen survives too. Checks every transfer; false leaves the OLED // unavailable. With panelOn false the panel stays dark but holds the frame for diff --git a/src/display/oled_init.cpp b/src/display/oled_init.cpp index 2680eed9..ac1abf68 100644 --- a/src/display/oled_init.cpp +++ b/src/display/oled_init.cpp @@ -15,6 +15,11 @@ bool i2cDeviceConnected(uint8_t address) { namespace { +// The panel answered at boot and the driver has its frame buffer, whether or +// not the setup then went through. Separate from oledAvailable, which is +// whether it can be drawn on right now. +bool g_oledDetected = false; + constexpr size_t kOledFrameBytes = OLED_WIDTH * ((OLED_HEIGHT + 7) / 8); // Frame bytes per transfer, well inside Wire's buffer. constexpr size_t kOledFrameChunk = 32; @@ -111,13 +116,23 @@ void initOled() { // begin() allocates and clears the frame buffer and sets the driver up; // configureOled() then sends the setup again with each transfer checked. - if (!display.begin(SSD1306_SWITCHCAPVCC, OLED_ADDRESS) || !configureOled(true)) { + if (!display.begin(SSD1306_SWITCHCAPVCC, OLED_ADDRESS)) { serialLogPrintln("ERROR: OLED initialization failed"); oledAvailable = false; return; } + g_oledDetected = true; display.setRotation(settingsOledRotate180() ? 2 : 0); + + // A setup that did not go through leaves the panel detected but not + // available: the I2C checks then fault it and retry the setup. + if (!configureOled(true)) { + serialLogPrintln("ERROR: OLED setup failed; retrying after boot"); + oledAvailable = false; + return; + } + oledAvailable = true; serialLogPrintln("OLED initialized"); @@ -127,6 +142,10 @@ void initOled() { // rotation is left alone too, since the setup wizard may be previewing one it // has not saved. The frame buffer still holds the last frame, so a static // screen comes back too. +bool oledDetectedAtBoot() { + return g_oledDetected; +} + bool restartOled(bool panelOn) { oledAvailable = i2cDeviceConnected(OLED_ADDRESS) && configureOled(panelOn); return oledAvailable; diff --git a/src/hardware/i2c_health.cpp b/src/hardware/i2c_health.cpp index e99adb6b..0ee60c07 100644 --- a/src/hardware/i2c_health.cpp +++ b/src/hardware/i2c_health.cpp @@ -59,7 +59,8 @@ Device g_servoDriver = { Device g_oled = {"oled", oledHealthy, stopOledWrites, recoverOled, nullptr, {}}; bool g_started = false; -// The robot runs without an OLED; only watch one that was there at boot. +// The robot runs without an OLED; only watch one that was there at boot, +// including one whose setup failed there, so that setup gets retried. bool g_oledExpected = false; // False after a failed Wire restart. Wire must not be used then: without its // buffers a transfer leaks Wire's lock, and the next restart would wait on @@ -411,7 +412,7 @@ void initI2cBus() { } void beginI2cHealth() { - g_oledExpected = oledAvailable; + g_oledExpected = oledDetectedAtBoot(); g_started = true; } From e02c2b5544e22d6307bf31b8aa3bfcc0b1fd4be8 Mon Sep 17 00:00:00 2001 From: Blake Azuela Date: Tue, 17 Nov 2026 07:12:29 -0500 Subject: [PATCH 10/10] fix(firmware): finish servo recovery with the channel resync CodeRabbit's review of the I2C recovery found these, all fixed here: - The recovery's last step could be the OLED restart, a pass after the channels were rewritten. A servo command in between was cached but not written, since the chip still counted as down, and nothing wrote it afterwards. The OLED restart now runs before the outputs come back, and the channel resync is the last step: the chip counts as up in the same pass. - The settle time started from the tick's timestamp, taken before the channel-off writes, so the writes ate into the 25 ms. It now starts after them. - The setup step ran the frequency setup and its readbacks after a failed MODE2 write, each waiting out a timeout on a dead bus. A failed MODE2 write now ends the step. - POST /servos/park centred the servos before checking the driver, so a refused park still marked the centred pose for the debounced save. It now refuses before anything changes, and still checks again after the move. - interfaces.md says a fault switches the outputs off; that needs OE driven by the firmware. testing.md says a PCA9685 failure halts; only a missing chip does, a setup that does not read back is retried. --- docs/hardware/interfaces.md | 2 +- docs/hardware/testing.md | 2 +- src/api/commands.cpp | 10 ++++++++-- src/hardware/i2c_health.cpp | 23 +++++++++++++---------- src/hardware/servo_wrapper.cpp | 21 +++++++++++++++------ 5 files changed, 38 insertions(+), 20 deletions(-) diff --git a/docs/hardware/interfaces.md b/docs/hardware/interfaces.md index 0ee2f4e0..67682f60 100644 --- a/docs/hardware/interfaces.md +++ b/docs/hardware/interfaces.md @@ -20,7 +20,7 @@ That table is the boot check. After boot the firmware keeps watching both device - **Check:** every 500 ms the OLED must answer, and the PCA9685 must answer, be awake on its internal clock with auto-increment on, and still hold its output mode (`MODE2 = 0x04`) and prescale. Two failed checks in a row are a fault. Each chip is tracked separately, so one that stays down does not stop the checks on the other. A healthy PCA9685 check is two register reads: `MODE1` and `MODE2` together, then the prescale. - **Failed servo write:** a fault on its own. The first one takes the PCA9685 down at once and switches its outputs off, even inside a blocking move that never reaches the check. So does a boot setup that does not read back correctly. -- **While down:** writes to the faulted chip are skipped, so failed transfers do not stall audio and HTTP. At a PCA9685 fault the servo outputs are switched off (OE), so a corrupted setup cannot keep driving them. +- **While down:** writes to the faulted chip are skipped, so failed transfers do not stall audio and HTTP. At a PCA9685 fault the servo outputs are switched off (OE), so a corrupted setup cannot keep driving them. That needs OE driven by the firmware, the default; with `PCA9685_OE_WIRED = false` a fault cannot switch the outputs off ([PCA9685 output enable](#pca9685-output-enable)). - **PCA9685 recovery:** if the chip does not answer, restart `Wire` and clock out a device left holding SDA first. A chip that kept its setup only lost a write or a read. A chip that lost its setup lost power or was corrupted: set it up again, and set the OLED up again too afterwards, since both share a supply. Either way, the outputs come back on with every servo channel off, and then every channel is written and read back. The PCA9685 applies channel values at the end of a LOW cycle, so each servo starts on a whole pulse; turning OE on mid-pulse would cut one short. Servos jump to where the firmware thinks they are. Outputs stay disabled unless the setup and every channel read back correctly. - **OLED recovery:** send the panel its setup and the last frame, so a static screen survives too. Every transfer is checked, since the SSD1306 driver ignores failed writes and a half-configured panel still answers its address. The current rotation is kept. Never restarts the bus, so an OLED fault alone does not disturb the servos. - **Audio:** a recovery runs in steps, one per loop pass, and a pass services at most one chip. Transfers time out after 20 ms (Wire's default is 50 ms), so a pass costs about one timeout at worst, well inside the speaker's DMA buffer of about 65 ms. diff --git a/docs/hardware/testing.md b/docs/hardware/testing.md index 7d3fe04e..98d8aa16 100644 --- a/docs/hardware/testing.md +++ b/docs/hardware/testing.md @@ -157,7 +157,7 @@ HTTP runs on STA when connected, or on setup AP at `192.168.4.1` during provisio | `[i2c] Wire.end failed` or `[i2c] Wire.begin failed` | The ESP32's I2C driver did not restart; both chips stay off until a later attempt succeeds | Report it with the serial log; a reset clears it | | `i2c..faults` above `0` in `GET /health` | A fault happened since boot | `recoveries` equal to `faults` means all were recovered; `last_fault_ms` says when | -OLED absence and Wi-Fi failure are **soft** fails. PCA9685 and I2S failures **halt** in `while (true)`. I2C faults after boot do not halt: the firmware sets the faulted chip up again, restarting the bus when the PCA9685 stops answering ([interfaces.md](interfaces.md#i2c)). +OLED absence and Wi-Fi failure are **soft** fails. A PCA9685 that does not answer at boot, and an I2S failure, **halt** in `while (true)`. A PCA9685 that answers but whose setup does not read back is a fault instead (`setup failed`): boot continues with the servo outputs off, and the checks set it up again. I2C faults after boot do not halt: the firmware sets the faulted chip up again, restarting the bus when the PCA9685 stops answering ([interfaces.md](interfaces.md#i2c)). ## Power during the servo test diff --git a/src/api/commands.cpp b/src/api/commands.cpp index e187e1fd..19935ef5 100644 --- a/src/api/commands.cpp +++ b/src/api/commands.cpp @@ -120,12 +120,18 @@ ApiResult apiMoveServo(int index, float angle) { } ApiResult apiParkServos() { + // A faulted servo driver would skip the move's writes, while the move still + // marks the centred pose for saving. Refuse before anything changes. + if (!i2cServoDriverUp()) { + return {false, 503, "servo driver down"}; + } + setAnimationImmediately(AnimationId::None); ensureAllServoOutputs(); centerAllServos(); - // A faulted servo driver skipped the move's writes: the joints never got to - // mid, so neither save that pose nor report a park. + // The driver faulted during the move: the joints never got to mid, so + // neither save that pose nor report a park. if (!i2cServoDriverUp()) { return {false, 503, "servo driver down"}; } diff --git a/src/hardware/i2c_health.cpp b/src/hardware/i2c_health.cpp index 0ee60c07..874c2d10 100644 --- a/src/hardware/i2c_health.cpp +++ b/src/hardware/i2c_health.cpp @@ -79,8 +79,8 @@ enum class ServoRecoveryStep : uint8_t { SetUp, // it did not: set it up again ChannelsOff, // every channel off, outputs off Settle, // wait for the off values to take effect - ChannelsOn, // outputs on, then every channel written - Oled // set the panel up again after a power loss + Oled, // set the panel up again after a power loss + ChannelsOn // outputs on, then every channel written; ends the recovery }; ServoRecoveryStep g_servoStep = ServoRecoveryStep::Probe; uint32_t g_channelsOffMs = 0; @@ -304,26 +304,29 @@ RecoveryResult recoverServoDriver(uint32_t now) { if (!writeServoChannelsOff()) { return endServoRecovery(RecoveryResult::Failed); } - g_channelsOffMs = now; + // After the writes, not `now`: they must not eat into the settle time. + g_channelsOffMs = millis(); return nextServoStep(ServoRecoveryStep::Settle); case ServoRecoveryStep::Settle: if (now - g_channelsOffMs < kServoOutputSettleMs) { return RecoveryResult::InProgress; } + return nextServoStep( + g_oledRestartPending ? ServoRecoveryStep::Oled : ServoRecoveryStep::ChannelsOn + ); + + case ServoRecoveryStep::Oled: + restartOledAfterPowerLoss(); return nextServoStep(ServoRecoveryStep::ChannelsOn); + // Last, so the channels written here are the latest the firmware asked + // for: the chip counts as up again in the same pass. A step after it + // would leave a pass where servo writes are still skipped but cached. case ServoRecoveryStep::ChannelsOn: if (!enableServoOutputsWithChannels()) { return endServoRecovery(RecoveryResult::Failed); } - if (!g_oledRestartPending) { - return endServoRecovery(RecoveryResult::Succeeded); - } - return nextServoStep(ServoRecoveryStep::Oled); - - case ServoRecoveryStep::Oled: - restartOledAfterPowerLoss(); return endServoRecovery(RecoveryResult::Succeeded); } diff --git a/src/hardware/servo_wrapper.cpp b/src/hardware/servo_wrapper.cpp index f7e86e3e..643124ae 100644 --- a/src/hardware/servo_wrapper.cpp +++ b/src/hardware/servo_wrapper.cpp @@ -308,17 +308,23 @@ bool writeChannelVerified(uint8_t channel, uint16_t off) { } // Output mode and servo frequency, shared by boot and recovery. The chip must -// be awake first: setPWMFreq() restores the sleep bit it finds. -void writeServoSetup() { +// be awake first: setPWMFreq() restores the sleep bit it finds. False when the +// MODE2 write failed: the frequency is then skipped, since on a dead bus each +// of its transfers would wait out a timeout too. +bool writeServoSetup() { // The Adafruit driver never writes MODE2, so a value corrupted by a bus // glitch would outlive every restart. The chip keeps its registers across // an ESP32 reset: only its own power cycle clears them. Wire.beginTransmission(PCA9685_ADDRESS); Wire.write(kPcaMode2Reg); Wire.write(kPcaMode2Servo); - Wire.endTransmission(); + + if (Wire.endTransmission() != 0) { + return false; + } pwm.setPWMFreq(kServoPwmHz); + return true; } // The end of boot: the recovery's sequence in one go. Outputs stay off unless @@ -364,7 +370,10 @@ bool setUpServoPwmDriver() { // and on a board with OE not wired the live servos would get those pulses. // reset() wakes the chip, which is all begin() adds here. pwm.reset(); - writeServoSetup(); + + if (!writeServoSetup()) { + return false; + } ServoPwmDriverState state; return servoPwmDriverState(state); @@ -408,7 +417,7 @@ void initServoPwmDriver() { disableServoOutputs(); pwm.begin(); - writeServoSetup(); + const bool setupWritten = writeServoSetup(); delay(100); uint8_t pose[SERVO_COUNT]; @@ -430,7 +439,7 @@ void initServoPwmDriver() { // Same path as a restart, so boot and recovery cannot drift apart. A setup // that did not take leaves the outputs off and the recovery to the checks. - if (!finishServoPwmDriverSetup()) { + if (!setupWritten || !finishServoPwmDriverSetup()) { i2cNoteServoDriverFault("setup failed"); } }