From 05137e825a80883615adbd67918d85a4f30a6033 Mon Sep 17 00:00:00 2001 From: iceman1001 Date: Sat, 12 Sep 2026 15:54:40 +0200 Subject: [PATCH] hold the spiffs file open across an upload SPIFFS has no directory. SPIFFS_open() by name walks the object lookup page of every block and reads the header of each index page it meets to strcmp the name, so reopening per packet makes an upload cost packets x filesystem size. On a 2MB part one open reads about 120KB of flash - storing a 512KB file spent 16MB of reads finding the same file 131 times over, and 249MB back when a packet was one 256 byte flash page. append_to_spiffs() now keeps the descriptor when the next packet names the same file. Everything else calls spiffs_close_cached() on entry, so a held descriptor is closed, and its writes flushed, before any other operation can look at the file - all 15 direct SPIFFS_* call sites in this file are covered. No protocol change: the CMD_SPIFFS_UNMOUNT the client already sends at the end of an upload closes it, as does any other SPIFFS command. rdv40_spiffs_unmount() closes before SPIFFS_unmount() since descriptors do not survive a mount, and the close checks the mount state so a dead handle is dropped rather than passed on. Flash reads to store a file, measured on a host build of this filesystem: 2MB fs, 512KB file, 4029 byte packets 16302 KB -> 371 KB 2MB fs, 512KB file, 256 byte packets 249415 KB -> 914 KB 2MB fs, 256KB file, 4029 byte packets 7834 KB -> 313 KB 256KB fs, 80KB file, 589 byte packets 1483 KB -> 82 KB Written bytes are unchanged; only the searching went away. Verified on the host against an order sensitive pattern - content byte identical, size right, SPIFFS_check clean - both for a straight upload and for one with a stat and an fsinfo landing mid transfer, which is the case that catches a missed invalidation. Then on hardware: 80KB uploaded, 'mem spiffs tree' in between to force a flush, downloaded again, byte identical. Co-Authored-By: Claude Opus 5 (1M context) --- CHANGELOG.md | 1 + armsrc/spiffs.c | 78 ++++++++++++++++++++++++++++++++++++++++++++----- 2 files changed, 72 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 15d9e7d33..643ce9e79 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ All notable changes to this project will be documented in this file. This project uses the changelog in accordance with [keepchangelog](http://keepachangelog.com/). Please use this to write notable changes, which is not the same as git commit log... ## [unreleased][unreleased] +- - Changed `mem spiffs upload` - the file is held open across packets instead of being reopened by name for each one (@iceman1001) - Added `hf thinfilm sniff` - passively records the frames a Thinfilm / NFC Barcode tag beams at a reader (@iceman1001) - Fixed `hf thinfilm sim` - it now holds the genuine tag's measured 65536 carrier period frame rate instead of a fixed delay (@iceman1001) - Changed `mem spiffs upload`/`mem spiffs dump` - both now print inline progress, and a refused write is reported with the byte it stopped at instead of claiming success (@iceman1001) diff --git a/armsrc/spiffs.c b/armsrc/spiffs.c index 95e34ac36..20413d528 100644 --- a/armsrc/spiffs.c +++ b/armsrc/spiffs.c @@ -175,6 +175,38 @@ static u8_t spiffs_cache_buf[RDV40_SPIFFS_CACHE_SZ] __attribute__((aligned)); static spiffs fs; +// One file descriptor held open across the packets of an upload. +// +// SPIFFS has no directory: SPIFFS_open() by name walks the object lookup page of +// every block and reads the header of each index page it meets to compare names. +// Doing that per packet makes an upload cost packets x filesystem size - measured +// on a 2MB filesystem, one open reads ~120KB of flash, so storing a 512KB file +// spent 16MB of reads finding the same file 131 times over. +// +// Only append_to_spiffs() fills this in. Every other path into the filesystem +// calls spiffs_close_cached() first, so a held descriptor can never be seen by +// another operation - the descriptor is closed, and its writes flushed, before +// anything else looks at the file. +static spiffs_file g_cached_fd = -1; +static char g_cached_fn[SPIFFS_OBJ_NAME_LEN] = {0}; + +static int rdv40_spiffs_mounted(void); + +static void spiffs_close_cached(void) { + + if (g_cached_fd < 0) { + return; + } + + // an unmount already invalidated it, do not touch the dead handle + if (rdv40_spiffs_mounted()) { + SPIFFS_close(&fs, g_cached_fd); + } + + g_cached_fd = -1; + g_cached_fn[0] = '\0'; +} + static enum spiffs_mount_status { RDV40_SPIFFS_UNMOUNTED, RDV40_SPIFFS_MOUNTED, @@ -234,6 +266,9 @@ int rdv40_spiffs_unmount(void) { return SPIFFS_ERR_NOT_MOUNTED; } + // the descriptor does not survive the unmount, flush it out first + spiffs_close_cached(); + SPIFFS_clearerr(&fs); SPIFFS_unmount(&fs); @@ -245,6 +280,7 @@ int rdv40_spiffs_unmount(void) { } int rdv40_spiffs_check(void) { + spiffs_close_cached(); rdv40_spiffs_lazy_mount(); SPIFFS_check(&fs); SPIFFS_gc_quick(&fs, 0); @@ -268,6 +304,7 @@ int rdv40_spiffs_write_status(void) { void write_to_spiffs(const char *filename, const uint8_t *src, uint32_t size) { g_spiffs_write_res = SPIFFS_OK; + spiffs_close_cached(); spiffs_file fd = SPIFFS_open(&fs, filename, SPIFFS_CREAT | SPIFFS_TRUNC | SPIFFS_RDWR, 0); if (fd < 0) { g_spiffs_write_res = SPIFFS_errno(&fs); @@ -284,21 +321,35 @@ void write_to_spiffs(const char *filename, const uint8_t *src, uint32_t size) { void append_to_spiffs(const char *filename, const uint8_t *src, uint32_t size) { g_spiffs_write_res = SPIFFS_OK; - spiffs_file fd = SPIFFS_open(&fs, filename, SPIFFS_APPEND | SPIFFS_RDWR, 0); - if (fd < 0) { - g_spiffs_write_res = SPIFFS_errno(&fs); - Dbprintf("open errno %i\n", g_spiffs_write_res); - return; + + // reuse the descriptor when this is another packet of the same file + if ((g_cached_fd < 0) || (strncmp(filename, g_cached_fn, sizeof(g_cached_fn)) != 0)) { + + spiffs_close_cached(); + + spiffs_file fd = SPIFFS_open(&fs, filename, SPIFFS_APPEND | SPIFFS_RDWR, 0); + if (fd < 0) { + g_spiffs_write_res = SPIFFS_errno(&fs); + Dbprintf("open errno %i\n", g_spiffs_write_res); + return; + } + + g_cached_fd = fd; + strncpy(g_cached_fn, filename, sizeof(g_cached_fn) - 1); + g_cached_fn[sizeof(g_cached_fn) - 1] = '\0'; } + // Note: SPIFFS_write() doesn't declare third parameter as const (but should) - if (SPIFFS_write(&fs, fd, (void *)src, size) < 0) { + if (SPIFFS_write(&fs, g_cached_fd, (void *)src, size) < 0) { g_spiffs_write_res = SPIFFS_errno(&fs); Dbprintf("errno %i\n", g_spiffs_write_res); + // do not hold on to a descriptor that just failed + spiffs_close_cached(); } - SPIFFS_close(&fs, fd); } void read_from_spiffs(const char *filename, uint8_t *dst, uint32_t size) { + spiffs_close_cached(); spiffs_file fd = SPIFFS_open(&fs, filename, SPIFFS_RDWR, 0); if (SPIFFS_read(&fs, fd, dst, size) < 0) { Dbprintf("errno %i\n", SPIFFS_errno(&fs)); @@ -307,18 +358,21 @@ void read_from_spiffs(const char *filename, uint8_t *dst, uint32_t size) { } static void rename_in_spiffs(const char *old_filename, const char *new_filename) { + spiffs_close_cached(); if (SPIFFS_rename(&fs, old_filename, new_filename) < 0) { Dbprintf("errno %i\n", SPIFFS_errno(&fs)); } } static void remove_from_spiffs(const char *filename) { + spiffs_close_cached(); if (SPIFFS_remove(&fs, filename) < 0) { Dbprintf("errno %i\n", SPIFFS_errno(&fs)); } } uint32_t size_in_spiffs(const char *filename) { + spiffs_close_cached(); spiffs_stat s; if (SPIFFS_stat(&fs, filename, &s) < 0) { Dbprintf("errno %i\n", SPIFFS_errno(&fs)); @@ -328,6 +382,7 @@ uint32_t size_in_spiffs(const char *filename) { } static rdv40_spiffs_fsinfo info_of_spiffs(void) { + spiffs_close_cached(); rdv40_spiffs_fsinfo fsinfo; fsinfo.blockSize = SPIFFS_CFG_LOG_BLOCK_SZ; fsinfo.pageSize = LOG_PAGE_SIZE; @@ -346,6 +401,7 @@ static rdv40_spiffs_fsinfo info_of_spiffs(void) { } int exists_in_spiffs(const char *filename) { + spiffs_close_cached(); spiffs_stat stat; int rc = SPIFFS_stat(&fs, filename, &stat); return (rc == SPIFFS_OK); @@ -390,6 +446,8 @@ static RDV40SpiFFSFileType filetype_in_spiffs(const char *filename) { static void copy_in_spiffs(const char *src, const char *dst) { + spiffs_close_cached(); + // no BigBuf_calloc(filesize) here: a file can be bigger than BigBuf, and bigger // than the uint16_t BigBuf_calloc() takes. Copy it in SPIFFS_WRITE_CHUNK_SIZE // chunks instead, which is also what rdv40_spiffs_write() writes in @@ -743,6 +801,8 @@ static int resolve_in_spiffs(const char *filename, char *dst, uint16_t dstlen) { // Expects the filesystem to be mounted. Returns bytes streamed, or a negative PM3_E* static int stream_from_spiffs(const char *filename, uint32_t offset, uint32_t size, uint8_t *chunkbuf, uint16_t chunklen, spiffs_chunk_cb_t cb) { + spiffs_close_cached(); + char target[SPIFFS_OBJ_NAME_LEN] = {0}; int res = resolve_in_spiffs(filename, target, sizeof(target)); if (res != PM3_SUCCESS) { @@ -866,6 +926,8 @@ void rdv40_spiffs_safe_print_tree(void) { struct spiffs_dirent e; struct spiffs_dirent *pe = &e; + spiffs_close_cached(); + char *resolvedlink = (char *)BigBuf_calloc(11 + SPIFFS_OBJ_NAME_LEN); char *linkdest = (char *)BigBuf_calloc(SPIFFS_OBJ_NAME_LEN); bool printed = false; @@ -899,6 +961,8 @@ void rdv40_spiffs_safe_print_tree(void) { void rdv40_spiffs_safe_wipe(void) { + spiffs_close_cached(); + int changed = rdv40_spiffs_lazy_mount(); spiffs_DIR d;