diff --git a/armsrc/appmain.c b/armsrc/appmain.c index e546330c3..43cf97cf1 100644 --- a/armsrc/appmain.c +++ b/armsrc/appmain.c @@ -753,6 +753,17 @@ static void SendStatus(uint32_t wait) { Dbprintf(" BWM fw version...... " _YELLOW_("%s"), "unknown"); } } + if (g_dbglevel >= DBG_DEBUG) { + // Flow-control diagnostics (since boot). Read before this reply is queued. + bwm_fc_stats_t fc; + bwm_fwd_fc_stats(&fc); + Dbprintf(" BWM fwd frames...... " _YELLOW_("%u") " sent, " _YELLOW_("%u") " acked, " _YELLOW_("%u") " errored", + fc.frames, fc.acks, fc.errors); + Dbprintf(" BWM fwd gate........ " _YELLOW_("%u") " timeouts, " _YELLOW_("%u") " forgotten, peak in flight " _YELLOW_("%u") " B", + fc.timeouts, fc.forgotten, fc.bytes_max); + Dbprintf(" BWM fwd now......... " _YELLOW_("%u") " frames / " _YELLOW_("%u") " B in flight", + fc.in_flight_frames, fc.in_flight_bytes); + } #endif #ifdef PM5 pm5_power_print_status(); diff --git a/armsrc/bwm_forward.c b/armsrc/bwm_forward.c index 36291109e..b92132159 100644 --- a/armsrc/bwm_forward.c +++ b/armsrc/bwm_forward.c @@ -14,6 +14,7 @@ #include "bwm_forward.h" #include "bwm_uart_at32.h" +#include "bwm_wifi.h" // BWM_CMD_CMD_ERROR #include "pm3_cmd.h" // PM3_CMD_DATA_SIZE, PM3_* return codes #include "ticks_apis.h" // SpinDelay #include "string.h" @@ -49,11 +50,51 @@ static uint16_t bwm_crc16(const uint8_t *data, size_t len, uint16_t crc) { static void bwm_pump(void); // fwd decl: TX gate pumps RX to collect forward-frame acks -// --- Flow control (ack window) --------------------------------------------- -// s_fwd_inflight: forward frames sent but not yet acked by the ESP. Bumped on -// send, decremented when a SLAVE_RESP echoing cmd=SEND_FORWARD_DATA arrives. -// We may send while it is below BWM_FC_WINDOW; at the cap we wait for an ack. -static volatile int16_t s_fwd_inflight = 0; +// --- Flow control (byte window) -------------------------------------------- +// Forward frames sent but not yet acked by the ESP: their lengths, oldest +// first (acks arrive in order), and their sum. Bumped on every forward frame, +// popped by the parser on the ESP's SLAVE_RESP. See BWM_FC_BYTES. +static volatile uint16_t s_fc_len[BWM_FC_MAX_FRAMES]; +static volatile uint8_t s_fc_head = 0; +static volatile uint8_t s_fc_count = 0; +static volatile uint32_t s_fc_bytes = 0; +static bwm_fc_stats_t s_fc_stats; // diagnostics since boot, see bwm_fwd_fc_stats() + +static void fc_reset(void) { + s_fc_head = 0; + s_fc_count = 0; + s_fc_bytes = 0; +} + +static void fc_push(uint16_t len) { + s_fc_stats.frames++; + s_fc_len[(uint8_t)((s_fc_head + s_fc_count) % BWM_FC_MAX_FRAMES)] = len; + s_fc_count++; + s_fc_bytes += len; + if (s_fc_bytes > s_fc_stats.bytes_max) { + s_fc_stats.bytes_max = s_fc_bytes; + } +} + +// Release the oldest frame: its ack (or CMD_ERROR) came, or the gate gave up on +// it. Acks are not matched to frames, so an ack that was eaten elsewhere (bwm_cmd() +// drains the ring, see #3648) or one that comes after the gate forgot its frame +// just shifts the window by one frame: the gate is one frame too pessimistic or +// too optimistic until the window drains, and drains it does after every command. +static void fc_pop(void) { + if (s_fc_count == 0) { + return; + } + s_fc_bytes -= s_fc_len[s_fc_head]; + s_fc_head = (uint8_t)((s_fc_head + 1) % BWM_FC_MAX_FRAMES); + s_fc_count--; +} + +void bwm_fwd_fc_stats(bwm_fc_stats_t *out) { + *out = s_fc_stats; + out->in_flight_frames = s_fc_count; + out->in_flight_bytes = s_fc_bytes; +} // Set true by the parser when a SLAVE_RESP echoing BWM_CMD_SET_UART_BAUD arrives // (the ESP's ack for a baud-set request). Consumed by bwm_fwd_negotiate_baud(). @@ -83,18 +124,23 @@ int bwm_fwd_writebuffer_sync(const uint8_t *data, size_t len) { } size_t idx = 0; - // Flow control: block while the in-flight window is full, waiting for the - // ESP to ack an earlier forward frame. bwm_pump() drains the IRQ-filled RX - // ring, so acks are collected even while we sit inside a tight download loop - // (the reply_old firehose). The spin cap is a safety valve so a dead or - // disconnected ESP can't hard-hang us. A window >= 1 means single command - // replies never block - only sustained bursts hit the cap. + // Flow control: block while this frame would not fit the in-flight byte budget + // (or the length FIFO is full). bwm_pump() drains the IRQ-filled RX ring, so + // acks land even inside a tight download loop (the reply_old firehose). The + // time cap is a safety valve so a dead or disconnected ESP can't hard-hang us. { uint32_t t0 = GetTickCount(); - while (s_fwd_inflight >= BWM_FC_WINDOW) { + uint32_t need = (uint32_t)len + BWM_TX_OVERHEAD; + while ((s_fc_count >= BWM_FC_MAX_FRAMES) || (s_fc_bytes + need > BWM_FC_BYTES)) { bwm_pump(); if (GetTickCountDelta(t0) > BWM_FC_ACK_TIMEOUT_MS) { - s_fwd_inflight = 0; // best-effort: assume the pipe cleared, never hard-hang + s_fc_stats.timeouts++; + // Never hard-hang: make room for this frame only, so a stalled ESP + // gets one frame per timeout on top of its ring, not a second window. + while (s_fc_count && ((s_fc_count >= BWM_FC_MAX_FRAMES) || (s_fc_bytes + need > BWM_FC_BYTES))) { + fc_pop(); + s_fc_stats.forgotten++; + } break; } } @@ -115,7 +161,7 @@ int bwm_fwd_writebuffer_sync(const uint8_t *data, size_t len) { frame[idx++] = (uint8_t)((crc >> 8) & 0xFF); int wr = bwm_uart_write(frame, idx); - s_fwd_inflight++; // one more forward frame awaiting its ack + fc_push((uint16_t)idx); // one more forward frame awaiting its ack return wr; } @@ -253,10 +299,14 @@ static void bwm_feed_byte(bwm_parser_t *p, uint8_t byte) { fifo_push(p->payload[i]); } } else if ((p->is_bcast == false) && p->cmd == BWM_CMD_SEND_FORWARD_DATA) { - // SLAVE_RESP ack for a forward frame -> one slot freed - if (s_fwd_inflight > 0) { - s_fwd_inflight--; - } + // SLAVE_RESP ack for a forward frame -> its bytes leave the budget + s_fc_stats.acks++; + fc_pop(); + } else if (p->is_bcast && p->cmd == BWM_CMD_CMD_ERROR && p->len >= 2 && + (((uint16_t)p->payload[0] | ((uint16_t)p->payload[1] << 8)) == BWM_CMD_SEND_FORWARD_DATA)) { + // the ESP could not deliver a forward frame: this is its answer instead of the ack + s_fc_stats.errors++; + fc_pop(); } else if ((p->is_bcast == false) && p->cmd == BWM_CMD_SET_UART_BAUD) { // SLAVE_RESP ack for a baud-set request (see negotiate below) s_baud_ack = true; @@ -406,7 +456,7 @@ static void bwm_link_reset(void) { s_p.state = S_IDLE; s_fifo_head = 0; s_fifo_tail = 0; - s_fwd_inflight = 0; + fc_reset(); } bool bwm_fwd_negotiate_baud(uint32_t target) { diff --git a/armsrc/bwm_forward.h b/armsrc/bwm_forward.h index 4623368d5..6d9e01c03 100644 --- a/armsrc/bwm_forward.h +++ b/armsrc/bwm_forward.h @@ -53,27 +53,29 @@ // echoing this cmd (len 0) at the OLD baud, then commits to the new baud. #define BWM_CMD_SET_UART_BAUD 1011 #define BWM_CMD_GET_UART_BAUD 1009 // read back the ESP's live baud (negotiation verify) -// Flow control (ack window) - ARM-side only, no BWM firmware change required. -// The ESP already replies to every forward frame with a SLAVE_RESP echoing -// cmd=SEND_FORWARD_DATA, and it sends that ack only *after* app_ble_send() has -// drained the frame to BLE. So the un-acked count is a live measure of how far -// ahead of the wireless link we are. We allow up to BWM_FC_WINDOW frames in -// flight, then block for an ack before sending more - which paces us to the real -// BLE/WiFi rate and prevents the ESP UART-RX overrun that dropped bulk downloads. -// WINDOW frames must fit the ESP UART RX FIFO + wireless send buffer. -// Ceiling on un-acked forward frames. On a download the ESP acks steadily so -// this never bites; it only matters on a bidirectional UPLOAD, where the ESP -// defers the small acks while forwarding large incoming chunks. A tight value -// (4) let inflight hit the cap and stall the AT32 past the client timeout, so -// keep enough headroom to ride out delayed acks. Only ~1 response is ever -// really in flight during an upload, so this does not risk an ESP overrun. -#define BWM_FC_WINDOW 16 // max un-acked forward frames in flight +// Flow control (byte window). The ESP acks each forward frame with a SLAVE_RESP +// echoing SEND_FORWARD_DATA once it has taken the frame out of its UART ring +// (module firmware from the companion PR; older firmware acks after the radio, +// which also works, just slower), and acks come back in order, so un-acked bytes +// bound what that ring holds. The window is sized to the ring (UART_RX_BUF_SIZE +// in the BWM firmware's app_cmd_uart.h): keep BWM_ESP_UART_RX_BUF equal to it. +// Frames run ~30 B to ~2.1 KB, hence bytes. A frame the ESP cannot deliver is +// answered with CMD_ERROR instead and leaves the window the same way. When acks +// stall on a full window the gate waits BWM_FC_ACK_TIMEOUT_MS, then forgets just +// enough of the oldest frames to send this one. Acks are counted, not matched: +// a lost or late ack shifts the window by a frame until it drains at idle. +#define BWM_ESP_UART_RX_BUF 12288 +#define BWM_FC_BYTES (BWM_ESP_UART_RX_BUF - 1024) // in-flight bytes allowed; slack for the ESP's FIFO and parser lag +#define BWM_FC_MAX_FRAMES 64 // depth of the in-flight length FIFO; also caps tiny frames in flight #ifndef BWM_FC_ACK_TIMEOUT_MS // Hard cap (ms) on how long a forward write may block the main loop waiting for // acks. A spin COUNT was unbounded in wall-clock time and could hang the main // loop long enough that the client gives up and the device looks dead (USB still // enumerates on interrupts). Time-bounded => the main loop is always serviced. -#define BWM_FC_ACK_TIMEOUT_MS 50 // safety valve: proceed if acks stall, never hard-hang +// A full window that stays full this long means the ESP is stuck on the radio +// (a phone scanning stalls BLE for a few hundred ms); every timeout then leaks +// one frame into a full ring, so wait long enough for the usual stalls to pass. +#define BWM_FC_ACK_TIMEOUT_MS 200 // safety valve: proceed if acks stall, never hard-hang #endif // safety valve: give up waiting for credit (avoid hard hang) #define BWM_CRC16_POLY 0x1021 @@ -93,6 +95,19 @@ uint32_t bwm_read_ng(uint8_t *data, size_t len); // >0 when raw bytes are waiting on the FPC USART (gate for receive_ng()). uint16_t bwm_fwd_rxdata_available(void); +// Flow-control diagnostics since boot (hw status at debug level). +typedef struct { + uint32_t frames; // forward frames sent + uint32_t acks; // SLAVE_RESP acks seen + uint32_t errors; // CMD_ERROR answers seen (undeliverable frames) + uint32_t timeouts; // gate waited BWM_FC_ACK_TIMEOUT_MS on a full window + uint32_t forgotten; // frames the gate stopped counting on those timeouts + uint32_t bytes_max; // peak bytes in flight + uint8_t in_flight_frames; // now + uint32_t in_flight_bytes; // now +} bwm_fc_stats_t; +void bwm_fwd_fc_stats(bwm_fc_stats_t *out); + // True while the ESP reports a client on BLE or on its WiFi TCP server (the // LINK_STATE broadcast, sent on change only). ESP firmware without it: never true. bool bwm_fwd_link_connected(void);