From 73285278b98e0b15a608325b3f7e4c5b8e0c6b75 Mon Sep 17 00:00:00 2001 From: Brian Pugh Date: Wed, 16 Aug 2023 21:46:04 -0700 Subject: [PATCH 01/23] refactor lfs_scan_for_state_updates and lfs_scan_for_superblock out of lfs_format --- lfs.c | 312 ++++++++++++++++++++++++++++++++++------------------------ 1 file changed, 181 insertions(+), 131 deletions(-) diff --git a/lfs.c b/lfs.c index 38b825dd..90e1365f 100644 --- a/lfs.c +++ b/lfs.c @@ -4237,6 +4237,175 @@ static int lfs_deinit(lfs_t *lfs) { return 0; } +static int lfs_scan_for_superblock(lfs_t *lfs, lfs_superblock_t *superblock){ + // scan directory blocks for superblock and any global updates + lfs_mdir_t dir = {.tail = {0, 1}}; + lfs_block_t tortoise[2] = {LFS_BLOCK_NULL, LFS_BLOCK_NULL}; + lfs_size_t tortoise_i = 1; + lfs_size_t tortoise_period = 1; + while (!lfs_pair_isnull(dir.tail)) { + // detect cycles with Brent's algorithm + if (lfs_pair_issync(dir.tail, tortoise)) { + LFS_WARN("Cycle detected in tail list"); + return LFS_ERR_CORRUPT; + } + if (tortoise_i == tortoise_period) { + tortoise[0] = dir.tail[0]; + tortoise[1] = dir.tail[1]; + tortoise_i = 0; + tortoise_period *= 2; + } + tortoise_i += 1; + + // fetch next block in tail list + lfs_stag_t tag = lfs_dir_fetchmatch(lfs, &dir, dir.tail, + LFS_MKTAG(0x7ff, 0x3ff, 0), + LFS_MKTAG(LFS_TYPE_SUPERBLOCK, 0, 8), + NULL, + lfs_dir_find_match, &(struct lfs_dir_find_match){ + lfs, "littlefs", 8}); + if (tag < 0) { + return tag; + } + + // has superblock? + if (tag && !lfs_tag_isdelete(tag)) { + // update root + lfs->root[0] = dir.pair[0]; + lfs->root[1] = dir.pair[1]; + + // grab superblock + tag = lfs_dir_get(lfs, &dir, LFS_MKTAG(0x7ff, 0x3ff, 0), + LFS_MKTAG(LFS_TYPE_INLINESTRUCT, 0, sizeof(*superblock)), + superblock); + if (tag < 0) { + return tag; + } + lfs_superblock_fromle32(superblock); + } + } + + return LFS_ERR_OK; +} + +static int lfs_scan_for_state_updates(lfs_t *lfs){ + int err; + // scan directory blocks for superblock and any global updates + lfs_mdir_t dir = {.tail = {0, 1}}; + lfs_block_t tortoise[2] = {LFS_BLOCK_NULL, LFS_BLOCK_NULL}; + lfs_size_t tortoise_i = 1; + lfs_size_t tortoise_period = 1; + while (!lfs_pair_isnull(dir.tail)) { + // detect cycles with Brent's algorithm + if (lfs_pair_issync(dir.tail, tortoise)) { + LFS_WARN("Cycle detected in tail list"); + return LFS_ERR_CORRUPT; + } + if (tortoise_i == tortoise_period) { + tortoise[0] = dir.tail[0]; + tortoise[1] = dir.tail[1]; + tortoise_i = 0; + tortoise_period *= 2; + } + tortoise_i += 1; + + // fetch next block in tail list + lfs_stag_t tag = lfs_dir_fetchmatch(lfs, &dir, dir.tail, + LFS_MKTAG(0x7ff, 0x3ff, 0), + LFS_MKTAG(LFS_TYPE_SUPERBLOCK, 0, 8), + NULL, + lfs_dir_find_match, &(struct lfs_dir_find_match){ + lfs, "littlefs", 8}); + if (tag < 0) { + return tag; + } + + // has gstate? + err = lfs_dir_getgstate(lfs, &dir, &lfs->gstate); + if (err) { + return err; + } + } + + return LFS_ERR_OK; +} + +static int lfs_validate_superblock(lfs_t *lfs, lfs_superblock_t *superblock){ + // check version + uint16_t major_version = (0xffff & (superblock->version >> 16)); + uint16_t minor_version = (0xffff & (superblock->version >> 0)); + if (major_version != lfs_fs_disk_version_major(lfs) + || minor_version > lfs_fs_disk_version_minor(lfs)) { + LFS_ERROR("Invalid version " + "v%"PRIu16".%"PRIu16" != v%"PRIu16".%"PRIu16, + major_version, + minor_version, + lfs_fs_disk_version_major(lfs), + lfs_fs_disk_version_minor(lfs)); + return LFS_ERR_INVAL; + } + + // found older minor version? set an in-device only bit in the + // gstate so we know we need to rewrite the superblock before + // the first write + if (minor_version < lfs_fs_disk_version_minor(lfs)) { + LFS_DEBUG("Found older minor version " + "v%"PRIu16".%"PRIu16" < v%"PRIu16".%"PRIu16, + major_version, + minor_version, + lfs_fs_disk_version_major(lfs), + lfs_fs_disk_version_minor(lfs)); + // note this bit is reserved on disk, so fetching more gstate + // will not interfere here + lfs_fs_prepsuperblock(lfs, true); + } + + // check superblock configuration + if (superblock->name_max) { + if (superblock->name_max > lfs->name_max) { + LFS_ERROR("Unsupported name_max (%"PRIu32" > %"PRIu32")", + superblock->name_max, lfs->name_max); + return LFS_ERR_INVAL; + } + lfs->name_max = superblock->name_max; + } + + if (superblock->file_max) { + if (superblock->file_max > lfs->file_max) { + LFS_ERROR("Unsupported file_max (%"PRIu32" > %"PRIu32")", + superblock->file_max, lfs->file_max); + return LFS_ERR_INVAL; + } + lfs->file_max = superblock->file_max; + } + + if (superblock->attr_max) { + if (superblock->attr_max > lfs->attr_max) { + LFS_ERROR("Unsupported attr_max (%"PRIu32" > %"PRIu32")", + superblock->attr_max, lfs->attr_max); + return LFS_ERR_INVAL; + } + lfs->attr_max = superblock->attr_max; + } + + if (superblock->block_count != lfs->cfg->block_count) { + LFS_ERROR("Invalid block count (%"PRIu32" != %"PRIu32")", + superblock->block_count, lfs->cfg->block_count); + return LFS_ERR_INVAL; + } + + if (superblock->block_size != lfs->cfg->block_size) { + LFS_ERROR("Invalid block size (%"PRIu32" != %"PRIu32")", + superblock->block_size, lfs->cfg->block_size); + return LFS_ERR_INVAL; + } + + return LFS_ERR_OK; +} + + + + #ifndef LFS_READONLY static int lfs_rawformat(lfs_t *lfs, const struct lfs_config *cfg) { int err = 0; @@ -4309,139 +4478,20 @@ static int lfs_rawmount(lfs_t *lfs, const struct lfs_config *cfg) { return err; } - // scan directory blocks for superblock and any global updates - lfs_mdir_t dir = {.tail = {0, 1}}; - lfs_block_t tortoise[2] = {LFS_BLOCK_NULL, LFS_BLOCK_NULL}; - lfs_size_t tortoise_i = 1; - lfs_size_t tortoise_period = 1; - while (!lfs_pair_isnull(dir.tail)) { - // detect cycles with Brent's algorithm - if (lfs_pair_issync(dir.tail, tortoise)) { - LFS_WARN("Cycle detected in tail list"); - err = LFS_ERR_CORRUPT; - goto cleanup; - } - if (tortoise_i == tortoise_period) { - tortoise[0] = dir.tail[0]; - tortoise[1] = dir.tail[1]; - tortoise_i = 0; - tortoise_period *= 2; - } - tortoise_i += 1; - - // fetch next block in tail list - lfs_stag_t tag = lfs_dir_fetchmatch(lfs, &dir, dir.tail, - LFS_MKTAG(0x7ff, 0x3ff, 0), - LFS_MKTAG(LFS_TYPE_SUPERBLOCK, 0, 8), - NULL, - lfs_dir_find_match, &(struct lfs_dir_find_match){ - lfs, "littlefs", 8}); - if (tag < 0) { - err = tag; - goto cleanup; - } - - // has superblock? - if (tag && !lfs_tag_isdelete(tag)) { - // update root - lfs->root[0] = dir.pair[0]; - lfs->root[1] = dir.pair[1]; - - // grab superblock - lfs_superblock_t superblock; - tag = lfs_dir_get(lfs, &dir, LFS_MKTAG(0x7ff, 0x3ff, 0), - LFS_MKTAG(LFS_TYPE_INLINESTRUCT, 0, sizeof(superblock)), - &superblock); - if (tag < 0) { - err = tag; - goto cleanup; - } - lfs_superblock_fromle32(&superblock); - - // check version - uint16_t major_version = (0xffff & (superblock.version >> 16)); - uint16_t minor_version = (0xffff & (superblock.version >> 0)); - if (major_version != lfs_fs_disk_version_major(lfs) - || minor_version > lfs_fs_disk_version_minor(lfs)) { - LFS_ERROR("Invalid version " - "v%"PRIu16".%"PRIu16" != v%"PRIu16".%"PRIu16, - major_version, - minor_version, - lfs_fs_disk_version_major(lfs), - lfs_fs_disk_version_minor(lfs)); - err = LFS_ERR_INVAL; - goto cleanup; - } - - // found older minor version? set an in-device only bit in the - // gstate so we know we need to rewrite the superblock before - // the first write - if (minor_version < lfs_fs_disk_version_minor(lfs)) { - LFS_DEBUG("Found older minor version " - "v%"PRIu16".%"PRIu16" < v%"PRIu16".%"PRIu16, - major_version, - minor_version, - lfs_fs_disk_version_major(lfs), - lfs_fs_disk_version_minor(lfs)); - // note this bit is reserved on disk, so fetching more gstate - // will not interfere here - lfs_fs_prepsuperblock(lfs, true); - } - - // check superblock configuration - if (superblock.name_max) { - if (superblock.name_max > lfs->name_max) { - LFS_ERROR("Unsupported name_max (%"PRIu32" > %"PRIu32")", - superblock.name_max, lfs->name_max); - err = LFS_ERR_INVAL; - goto cleanup; - } - - lfs->name_max = superblock.name_max; - } - - if (superblock.file_max) { - if (superblock.file_max > lfs->file_max) { - LFS_ERROR("Unsupported file_max (%"PRIu32" > %"PRIu32")", - superblock.file_max, lfs->file_max); - err = LFS_ERR_INVAL; - goto cleanup; - } - - lfs->file_max = superblock.file_max; - } - - if (superblock.attr_max) { - if (superblock.attr_max > lfs->attr_max) { - LFS_ERROR("Unsupported attr_max (%"PRIu32" > %"PRIu32")", - superblock.attr_max, lfs->attr_max); - err = LFS_ERR_INVAL; - goto cleanup; - } - - lfs->attr_max = superblock.attr_max; - } - - if (superblock.block_count != lfs->cfg->block_count) { - LFS_ERROR("Invalid block count (%"PRIu32" != %"PRIu32")", - superblock.block_count, lfs->cfg->block_count); - err = LFS_ERR_INVAL; - goto cleanup; - } + lfs_superblock_t superblock; + err = lfs_scan_for_superblock(lfs, &superblock); + if(err) { + goto cleanup; + } - if (superblock.block_size != lfs->cfg->block_size) { - LFS_ERROR("Invalid block size (%"PRIu32" != %"PRIu32")", - superblock.block_size, lfs->cfg->block_size); - err = LFS_ERR_INVAL; - goto cleanup; - } - } + err = lfs_validate_superblock(lfs, &superblock); + if(err){ + return err; + } - // has gstate? - err = lfs_dir_getgstate(lfs, &dir, &lfs->gstate); - if (err) { - goto cleanup; - } + err = lfs_scan_for_state_updates(lfs); + if(err) { + goto cleanup; } // found superblock? From 6dae7038f9be86bce1b1f5e261f386aafc69c9f6 Mon Sep 17 00:00:00 2001 From: Brian Pugh Date: Wed, 16 Aug 2023 21:46:52 -0700 Subject: [PATCH 02/23] remove redundant superblock check --- lfs.c | 6 ------ 1 file changed, 6 deletions(-) diff --git a/lfs.c b/lfs.c index 90e1365f..b9807baf 100644 --- a/lfs.c +++ b/lfs.c @@ -4494,12 +4494,6 @@ static int lfs_rawmount(lfs_t *lfs, const struct lfs_config *cfg) { goto cleanup; } - // found superblock? - if (lfs_pair_isnull(lfs->root)) { - err = LFS_ERR_INVAL; - goto cleanup; - } - // update littlefs with gstate if (!lfs_gstate_iszero(&lfs->gstate)) { LFS_DEBUG("Found pending gstate 0x%08"PRIx32"%08"PRIx32"%08"PRIx32, From be6812213dc48a14ad48db9f7d4352d6120f70d8 Mon Sep 17 00:00:00 2001 From: Brian Pugh Date: Wed, 16 Aug 2023 22:18:40 -0700 Subject: [PATCH 03/23] introduce lfs->block_count. If cfg->block_count is 0, autopopulate from superblock --- lfs.c | 54 +++++++++++++++++++++++++++++++++++++++--------------- lfs.h | 2 ++ 2 files changed, 41 insertions(+), 15 deletions(-) diff --git a/lfs.c b/lfs.c index b9807baf..d6588de7 100644 --- a/lfs.c +++ b/lfs.c @@ -46,7 +46,7 @@ static int lfs_bd_read(lfs_t *lfs, lfs_block_t block, lfs_off_t off, void *buffer, lfs_size_t size) { uint8_t *data = buffer; - if (block >= lfs->cfg->block_count || + if (block >= lfs->block_count || off+size > lfs->cfg->block_size) { return LFS_ERR_CORRUPT; } @@ -104,7 +104,7 @@ static int lfs_bd_read(lfs_t *lfs, } // load to cache, first condition can no longer fail - LFS_ASSERT(block < lfs->cfg->block_count); + LFS_ASSERT(block < lfs->block_count); rcache->block = block; rcache->off = lfs_aligndown(off, lfs->cfg->read_size); rcache->size = lfs_min( @@ -176,7 +176,7 @@ static int lfs_bd_crc(lfs_t *lfs, static int lfs_bd_flush(lfs_t *lfs, lfs_cache_t *pcache, lfs_cache_t *rcache, bool validate) { if (pcache->block != LFS_BLOCK_NULL && pcache->block != LFS_BLOCK_INLINE) { - LFS_ASSERT(pcache->block < lfs->cfg->block_count); + LFS_ASSERT(pcache->block < lfs->block_count); lfs_size_t diff = lfs_alignup(pcache->size, lfs->cfg->prog_size); int err = lfs->cfg->prog(lfs->cfg, pcache->block, pcache->off, pcache->buffer, diff); @@ -229,7 +229,7 @@ static int lfs_bd_prog(lfs_t *lfs, lfs_block_t block, lfs_off_t off, const void *buffer, lfs_size_t size) { const uint8_t *data = buffer; - LFS_ASSERT(block == LFS_BLOCK_INLINE || block < lfs->cfg->block_count); + LFS_ASSERT(block == LFS_BLOCK_INLINE || block < lfs->block_count); LFS_ASSERT(off + size <= lfs->cfg->block_size); while (size > 0) { @@ -273,7 +273,7 @@ static int lfs_bd_prog(lfs_t *lfs, #ifndef LFS_READONLY static int lfs_bd_erase(lfs_t *lfs, lfs_block_t block) { - LFS_ASSERT(block < lfs->cfg->block_count); + LFS_ASSERT(block < lfs->block_count); int err = lfs->cfg->erase(lfs->cfg, block); LFS_ASSERT(err <= 0); return err; @@ -597,7 +597,7 @@ static int lfs_rawunmount(lfs_t *lfs); static int lfs_alloc_lookahead(void *p, lfs_block_t block) { lfs_t *lfs = (lfs_t*)p; lfs_block_t off = ((block - lfs->free.off) - + lfs->cfg->block_count) % lfs->cfg->block_count; + + lfs->block_count) % lfs->block_count; if (off < lfs->free.size) { lfs->free.buffer[off / 32] |= 1U << (off % 32); @@ -611,7 +611,7 @@ static int lfs_alloc_lookahead(void *p, lfs_block_t block) { // is to prevent blocks from being garbage collected in the middle of a // commit operation static void lfs_alloc_ack(lfs_t *lfs) { - lfs->free.ack = lfs->cfg->block_count; + lfs->free.ack = lfs->block_count; } // drop the lookahead buffer, this is done during mounting and failed @@ -632,7 +632,7 @@ static int lfs_alloc(lfs_t *lfs, lfs_block_t *block) { if (!(lfs->free.buffer[off / 32] & (1U << (off % 32)))) { // found a free block - *block = (lfs->free.off + off) % lfs->cfg->block_count; + *block = (lfs->free.off + off) % lfs->block_count; // eagerly find next off so an alloc ack can // discredit old lookahead blocks @@ -655,7 +655,7 @@ static int lfs_alloc(lfs_t *lfs, lfs_block_t *block) { } lfs->free.off = (lfs->free.off + lfs->free.size) - % lfs->cfg->block_count; + % lfs->block_count; lfs->free.size = lfs_min(8*lfs->cfg->lookahead_size, lfs->free.ack); lfs->free.i = 0; @@ -1067,7 +1067,7 @@ static lfs_stag_t lfs_dir_fetchmatch(lfs_t *lfs, // if either block address is invalid we return LFS_ERR_CORRUPT here, // otherwise later writes to the pair could fail - if (pair[0] >= lfs->cfg->block_count || pair[1] >= lfs->cfg->block_count) { + if (lfs->block_count && (pair[0] >= lfs->block_count || pair[1] >= lfs->block_count)) { return LFS_ERR_CORRUPT; } @@ -2140,7 +2140,7 @@ static int lfs_dir_splittingcompact(lfs_t *lfs, lfs_mdir_t *dir, // do we have extra space? littlefs can't reclaim this space // by itself, so expand cautiously - if ((lfs_size_t)size < lfs->cfg->block_count/2) { + if ((lfs_size_t)size < lfs->block_count/2) { LFS_DEBUG("Expanding superblock at rev %"PRIu32, dir->rev); int err = lfs_dir_split(lfs, dir, attrs, attrcount, source, begin, end); @@ -4095,6 +4095,7 @@ static int lfs_rawremoveattr(lfs_t *lfs, const char *path, uint8_t type) { /// Filesystem operations /// static int lfs_init(lfs_t *lfs, const struct lfs_config *cfg) { lfs->cfg = cfg; + lfs->block_count = cfg->block_count; // May be 0 int err = 0; #ifdef LFS_MULTIVERSION @@ -4415,11 +4416,26 @@ static int lfs_rawformat(lfs_t *lfs, const struct lfs_config *cfg) { return err; } + if(cfg->block_count == 0){ + // Attempt to read a (possibly) prior superblock + lfs_superblock_t superblock; + err = lfs_scan_for_superblock(lfs, &superblock); + if(err){ + return err; + } + lfs->block_count = superblock.block_count; + + err = lfs_validate_superblock(lfs, &superblock); + if(err){ + return err; + } + } + // create free lookahead memset(lfs->free.buffer, 0, lfs->cfg->lookahead_size); lfs->free.off = 0; lfs->free.size = lfs_min(8*lfs->cfg->lookahead_size, - lfs->cfg->block_count); + lfs->block_count); lfs->free.i = 0; lfs_alloc_ack(lfs); @@ -4434,7 +4450,7 @@ static int lfs_rawformat(lfs_t *lfs, const struct lfs_config *cfg) { lfs_superblock_t superblock = { .version = lfs_fs_disk_version(lfs), .block_size = lfs->cfg->block_size, - .block_count = lfs->cfg->block_count, + .block_count = lfs->block_count, .name_max = lfs->name_max, .file_max = lfs->file_max, .attr_max = lfs->attr_max, @@ -4484,6 +4500,10 @@ static int lfs_rawmount(lfs_t *lfs, const struct lfs_config *cfg) { goto cleanup; } + if(lfs->block_count == 0){ + lfs->block_count = superblock.block_count; + } + err = lfs_validate_superblock(lfs, &superblock); if(err){ return err; @@ -4506,7 +4526,7 @@ static int lfs_rawmount(lfs_t *lfs, const struct lfs_config *cfg) { // setup free lookahead, to distribute allocations uniformly across // boots, we start the allocator at a random location - lfs->free.off = lfs->seed % lfs->cfg->block_count; + lfs->free.off = lfs->seed % lfs->block_count; lfs_alloc_drop(lfs); return 0; @@ -4815,7 +4835,7 @@ static int lfs_fs_desuperblock(lfs_t *lfs) { lfs_superblock_t superblock = { .version = lfs_fs_disk_version(lfs), .block_size = lfs->cfg->block_size, - .block_count = lfs->cfg->block_count, + .block_count = lfs->block_count, .name_max = lfs->name_max, .file_max = lfs->file_max, .attr_max = lfs->attr_max, @@ -5493,6 +5513,10 @@ static int lfs1_unmount(lfs_t *lfs) { /// v1 migration /// static int lfs_rawmigrate(lfs_t *lfs, const struct lfs_config *cfg) { struct lfs1 lfs1; + if(cfg->block_count == 0){ + // Indeterminate filesystem size not allowed for migration. + return LFS_ERR_INVAL; + } int err = lfs1_mount(lfs, &lfs1, cfg); if (err) { return err; diff --git a/lfs.h b/lfs.h index 1081735f..be5c95cf 100644 --- a/lfs.h +++ b/lfs.h @@ -437,6 +437,8 @@ typedef struct lfs { lfs_size_t file_max; lfs_size_t attr_max; + lfs_size_t block_count; + #ifdef LFS_MIGRATE struct lfs1 *lfs1; #endif From df238ebac6adb7f0111af757e79df104f64d9963 Mon Sep 17 00:00:00 2001 From: Brian Pugh Date: Wed, 16 Aug 2023 23:07:55 -0700 Subject: [PATCH 04/23] Add a unit test; currently hanging on final permutation. Some block-device bound-checks are disabled during superblock search. --- bd/lfs_emubd.c | 6 +++--- lfs.c | 8 ++++---- tests/test_superblocks.toml | 15 +++++++++++++++ 3 files changed, 22 insertions(+), 7 deletions(-) diff --git a/bd/lfs_emubd.c b/bd/lfs_emubd.c index 29925538..546a90c1 100644 --- a/bd/lfs_emubd.c +++ b/bd/lfs_emubd.c @@ -237,7 +237,7 @@ int lfs_emubd_read(const struct lfs_config *cfg, lfs_block_t block, lfs_emubd_t *bd = cfg->context; // check if read is valid - LFS_ASSERT(block < cfg->block_count); + LFS_ASSERT(!cfg->block_count || block < cfg->block_count); LFS_ASSERT(off % cfg->read_size == 0); LFS_ASSERT(size % cfg->read_size == 0); LFS_ASSERT(off+size <= cfg->block_size); @@ -287,7 +287,7 @@ int lfs_emubd_prog(const struct lfs_config *cfg, lfs_block_t block, lfs_emubd_t *bd = cfg->context; // check if write is valid - LFS_ASSERT(block < cfg->block_count); + LFS_ASSERT(!cfg->block_count || block < cfg->block_count); LFS_ASSERT(off % cfg->prog_size == 0); LFS_ASSERT(size % cfg->prog_size == 0); LFS_ASSERT(off+size <= cfg->block_size); @@ -376,7 +376,7 @@ int lfs_emubd_erase(const struct lfs_config *cfg, lfs_block_t block) { lfs_emubd_t *bd = cfg->context; // check if erase is valid - LFS_ASSERT(block < cfg->block_count); + LFS_ASSERT(!cfg->block_count || block < cfg->block_count); // get the block lfs_emubd_block_t *b = lfs_emubd_mutblock(cfg, &bd->blocks[block]); diff --git a/lfs.c b/lfs.c index d6588de7..2bb7fe74 100644 --- a/lfs.c +++ b/lfs.c @@ -46,8 +46,8 @@ static int lfs_bd_read(lfs_t *lfs, lfs_block_t block, lfs_off_t off, void *buffer, lfs_size_t size) { uint8_t *data = buffer; - if (block >= lfs->block_count || - off+size > lfs->cfg->block_size) { + if (lfs->block_count && + (block >= lfs->block_count || off+size > lfs->cfg->block_size)) { return LFS_ERR_CORRUPT; } @@ -104,7 +104,7 @@ static int lfs_bd_read(lfs_t *lfs, } // load to cache, first condition can no longer fail - LFS_ASSERT(block < lfs->block_count); + LFS_ASSERT(!lfs->block_count || block < lfs->block_count); rcache->block = block; rcache->off = lfs_aligndown(off, lfs->cfg->read_size); rcache->size = lfs_min( @@ -4389,7 +4389,7 @@ static int lfs_validate_superblock(lfs_t *lfs, lfs_superblock_t *superblock){ lfs->attr_max = superblock->attr_max; } - if (superblock->block_count != lfs->cfg->block_count) { + if (lfs->cfg->block_count && superblock->block_count != lfs->cfg->block_count) { LFS_ERROR("Invalid block count (%"PRIu32" != %"PRIu32")", superblock->block_count, lfs->cfg->block_count); return LFS_ERR_INVAL; diff --git a/tests/test_superblocks.toml b/tests/test_superblocks.toml index 0aff84ba..0622a4a3 100644 --- a/tests/test_superblocks.toml +++ b/tests/test_superblocks.toml @@ -5,6 +5,21 @@ code = ''' lfs_format(&lfs, cfg) => 0; ''' +# tests formatting from interpretting a previous superblock +[cases.test_superblocks_format_unknown_block_count] +code = ''' + lfs_t lfs; + lfs_format(&lfs, cfg) => 0; + assert(lfs.block_count == cfg->block_count); + + memset(&lfs, 0, sizeof(lfs)); + struct lfs_config tweaked_cfg = *cfg; + tweaked_cfg.block_count = 0; + lfs_format(&lfs, &tweaked_cfg) => 0; + assert(lfs.block_count == cfg->block_count); +''' + + # mount/unmount [cases.test_superblocks_mount] code = ''' From 6de3fc6ae84057062f5859b3446a33319d5c70e3 Mon Sep 17 00:00:00 2001 From: Brian Pugh Date: Thu, 17 Aug 2023 15:07:19 -0700 Subject: [PATCH 05/23] fix corruption check --- lfs.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lfs.c b/lfs.c index 2bb7fe74..16295f29 100644 --- a/lfs.c +++ b/lfs.c @@ -46,8 +46,7 @@ static int lfs_bd_read(lfs_t *lfs, lfs_block_t block, lfs_off_t off, void *buffer, lfs_size_t size) { uint8_t *data = buffer; - if (lfs->block_count && - (block >= lfs->block_count || off+size > lfs->cfg->block_size)) { + if (off+size > lfs->cfg->block_size || (lfs->block_count && block >= lfs->block_count)) { return LFS_ERR_CORRUPT; } From 3d0bcf40667bc1778ca96c7036d97a4568b2933a Mon Sep 17 00:00:00 2001 From: Brian Pugh Date: Thu, 17 Aug 2023 15:13:16 -0700 Subject: [PATCH 06/23] Add test_superblocks_mount_unknown_block_count --- tests/test_superblocks.toml | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/tests/test_superblocks.toml b/tests/test_superblocks.toml index 0622a4a3..a0d6388f 100644 --- a/tests/test_superblocks.toml +++ b/tests/test_superblocks.toml @@ -5,7 +5,7 @@ code = ''' lfs_format(&lfs, cfg) => 0; ''' -# tests formatting from interpretting a previous superblock +# formatting from interpretting a previous superblock block_count [cases.test_superblocks_format_unknown_block_count] code = ''' lfs_t lfs; @@ -29,6 +29,20 @@ code = ''' lfs_unmount(&lfs) => 0; ''' +# mount/unmount from interpretting a previous superblock block_count +[cases.test_superblocks_mount_unknown_block_count] +code = ''' + lfs_t lfs; + lfs_format(&lfs, cfg) => 0; + + memset(&lfs, 0, sizeof(lfs)); + struct lfs_config tweaked_cfg = *cfg; + tweaked_cfg.block_count = 0; + lfs_mount(&lfs, &tweaked_cfg) => 0; + assert(lfs.block_count == cfg->block_count); +''' + + # reentrant format [cases.test_superblocks_reentrant_format] reentrant = true From 2ebfec78c3c3383925f2014a95e46c99b2e4fac2 Mon Sep 17 00:00:00 2001 From: Brian Pugh Date: Thu, 17 Aug 2023 15:20:46 -0700 Subject: [PATCH 07/23] test for failure when interpretting block count when formatting without superblock --- tests/test_superblocks.toml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/test_superblocks.toml b/tests/test_superblocks.toml index a0d6388f..2dcf8806 100644 --- a/tests/test_superblocks.toml +++ b/tests/test_superblocks.toml @@ -19,6 +19,14 @@ code = ''' assert(lfs.block_count == cfg->block_count); ''' +# formatting from interpretting a non-existent previous superblock block_count +[cases.test_superblocks_format_unknown_block_count_failure] +code = ''' + lfs_t lfs; + struct lfs_config tweaked_cfg = *cfg; + tweaked_cfg.block_count = 0; + lfs_format(&lfs, &tweaked_cfg) => LFS_ERR_CORRUPT; +''' # mount/unmount [cases.test_superblocks_mount] From 7521e0a6b2017a18b498d9894174d2d2da57992e Mon Sep 17 00:00:00 2001 From: Brian Pugh Date: Thu, 17 Aug 2023 19:55:54 -0700 Subject: [PATCH 08/23] fix newly introduced missing cleanup when an invalid superblock is found. --- lfs.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lfs.c b/lfs.c index 16295f29..4afaa944 100644 --- a/lfs.c +++ b/lfs.c @@ -4420,13 +4420,13 @@ static int lfs_rawformat(lfs_t *lfs, const struct lfs_config *cfg) { lfs_superblock_t superblock; err = lfs_scan_for_superblock(lfs, &superblock); if(err){ - return err; + goto cleanup; } lfs->block_count = superblock.block_count; err = lfs_validate_superblock(lfs, &superblock); if(err){ - return err; + goto cleanup; } } @@ -4505,7 +4505,7 @@ static int lfs_rawmount(lfs_t *lfs, const struct lfs_config *cfg) { err = lfs_validate_superblock(lfs, &superblock); if(err){ - return err; + goto cleanup; } err = lfs_scan_for_state_updates(lfs); From 5caa83fb776a4c725a45ca8c5f252045efb2feab Mon Sep 17 00:00:00 2001 From: Brian Pugh Date: Thu, 17 Aug 2023 22:10:53 -0700 Subject: [PATCH 09/23] forgot to unmount lfs in test; leaking memory --- tests/test_superblocks.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_superblocks.toml b/tests/test_superblocks.toml index 2dcf8806..f9826a83 100644 --- a/tests/test_superblocks.toml +++ b/tests/test_superblocks.toml @@ -48,6 +48,7 @@ code = ''' tweaked_cfg.block_count = 0; lfs_mount(&lfs, &tweaked_cfg) => 0; assert(lfs.block_count == cfg->block_count); + lfs_unmount(&lfs) => 0; ''' From d6c0c6a786fad487581f82f2f2adacdb19113979 Mon Sep 17 00:00:00 2001 From: Brian Pugh Date: Sun, 20 Aug 2023 11:33:29 -0700 Subject: [PATCH 10/23] linting --- lfs.c | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/lfs.c b/lfs.c index 4afaa944..52ce6142 100644 --- a/lfs.c +++ b/lfs.c @@ -1066,7 +1066,8 @@ static lfs_stag_t lfs_dir_fetchmatch(lfs_t *lfs, // if either block address is invalid we return LFS_ERR_CORRUPT here, // otherwise later writes to the pair could fail - if (lfs->block_count && (pair[0] >= lfs->block_count || pair[1] >= lfs->block_count)) { + if (lfs->block_count + && (pair[0] >= lfs->block_count || pair[1] >= lfs->block_count)) { return LFS_ERR_CORRUPT; } @@ -4415,17 +4416,17 @@ static int lfs_rawformat(lfs_t *lfs, const struct lfs_config *cfg) { return err; } - if(cfg->block_count == 0){ + if (cfg->block_count == 0) { // Attempt to read a (possibly) prior superblock lfs_superblock_t superblock; err = lfs_scan_for_superblock(lfs, &superblock); - if(err){ + if (err) { goto cleanup; } lfs->block_count = superblock.block_count; err = lfs_validate_superblock(lfs, &superblock); - if(err){ + if (err) { goto cleanup; } } @@ -4495,21 +4496,21 @@ static int lfs_rawmount(lfs_t *lfs, const struct lfs_config *cfg) { lfs_superblock_t superblock; err = lfs_scan_for_superblock(lfs, &superblock); - if(err) { + if (err) { goto cleanup; } - if(lfs->block_count == 0){ + if (lfs->block_count == 0) { lfs->block_count = superblock.block_count; } err = lfs_validate_superblock(lfs, &superblock); - if(err){ + if (err) { goto cleanup; } err = lfs_scan_for_state_updates(lfs); - if(err) { + if (err) { goto cleanup; } @@ -5512,10 +5513,10 @@ static int lfs1_unmount(lfs_t *lfs) { /// v1 migration /// static int lfs_rawmigrate(lfs_t *lfs, const struct lfs_config *cfg) { struct lfs1 lfs1; - if(cfg->block_count == 0){ - // Indeterminate filesystem size not allowed for migration. - return LFS_ERR_INVAL; - } + + // Indeterminate filesystem size not allowed for migration. + LFS_ASSERT(cfg->block_count != 0); + int err = lfs1_mount(lfs, &lfs1, cfg); if (err) { return err; From d6098bd3cef6ba0f3b24aac6643fc97959e3d418 Mon Sep 17 00:00:00 2001 From: Brian Pugh Date: Sun, 20 Aug 2023 11:53:18 -0700 Subject: [PATCH 11/23] Add block_count and block_size to fsinfo --- lfs.c | 3 +++ lfs.h | 7 +++++++ 2 files changed, 10 insertions(+) diff --git a/lfs.c b/lfs.c index 52ce6142..54554859 100644 --- a/lfs.c +++ b/lfs.c @@ -4575,6 +4575,9 @@ static int lfs_fs_rawstat(lfs_t *lfs, struct lfs_fsinfo *fsinfo) { fsinfo->file_max = lfs->file_max; fsinfo->attr_max = lfs->attr_max; + fsinfo->block_count = lfs->block_count; + fsinfo->block_size = lfs->cfg->block_size; + return 0; } diff --git a/lfs.h b/lfs.h index be5c95cf..c777f301 100644 --- a/lfs.h +++ b/lfs.h @@ -301,6 +301,13 @@ struct lfs_fsinfo { // Upper limit on the size of custom attributes in bytes. lfs_size_t attr_max; + + // Number of blocks in filesystem. + // May differ from cfg->block_count if autodetected from filesystem. + lfs_size_t block_count; + + // Size of block in bytes. + lfs_size_t block_size; }; // Custom attribute structure, used to describe custom attributes From 23089d5758b2a38ffa45028e446de292791272ff Mon Sep 17 00:00:00 2001 From: Brian Pugh Date: Sun, 20 Aug 2023 14:10:12 -0700 Subject: [PATCH 12/23] remove previous block_count detection from lfs_format --- lfs.c | 15 +-------------- tests/test_superblocks.toml | 23 ----------------------- 2 files changed, 1 insertion(+), 37 deletions(-) diff --git a/lfs.c b/lfs.c index 54554859..b1851618 100644 --- a/lfs.c +++ b/lfs.c @@ -4416,20 +4416,7 @@ static int lfs_rawformat(lfs_t *lfs, const struct lfs_config *cfg) { return err; } - if (cfg->block_count == 0) { - // Attempt to read a (possibly) prior superblock - lfs_superblock_t superblock; - err = lfs_scan_for_superblock(lfs, &superblock); - if (err) { - goto cleanup; - } - lfs->block_count = superblock.block_count; - - err = lfs_validate_superblock(lfs, &superblock); - if (err) { - goto cleanup; - } - } + LFS_ASSERT(cfg->block_count != 0); // create free lookahead memset(lfs->free.buffer, 0, lfs->cfg->lookahead_size); diff --git a/tests/test_superblocks.toml b/tests/test_superblocks.toml index f9826a83..99e1dad0 100644 --- a/tests/test_superblocks.toml +++ b/tests/test_superblocks.toml @@ -5,29 +5,6 @@ code = ''' lfs_format(&lfs, cfg) => 0; ''' -# formatting from interpretting a previous superblock block_count -[cases.test_superblocks_format_unknown_block_count] -code = ''' - lfs_t lfs; - lfs_format(&lfs, cfg) => 0; - assert(lfs.block_count == cfg->block_count); - - memset(&lfs, 0, sizeof(lfs)); - struct lfs_config tweaked_cfg = *cfg; - tweaked_cfg.block_count = 0; - lfs_format(&lfs, &tweaked_cfg) => 0; - assert(lfs.block_count == cfg->block_count); -''' - -# formatting from interpretting a non-existent previous superblock block_count -[cases.test_superblocks_format_unknown_block_count_failure] -code = ''' - lfs_t lfs; - struct lfs_config tweaked_cfg = *cfg; - tweaked_cfg.block_count = 0; - lfs_format(&lfs, &tweaked_cfg) => LFS_ERR_CORRUPT; -''' - # mount/unmount [cases.test_superblocks_mount] code = ''' From 9c23329dd7daae78c3515eca8940668cfe6074dc Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Sun, 3 Sep 2023 13:19:03 -0500 Subject: [PATCH 13/23] Revert of refactor lfs_scan_* out of lfs_format This would result in two passes through the superblock chain during mount, when we can access everything we need to in one. --- lfs.c | 316 +++++++++++++++++++++++++--------------------------------- 1 file changed, 134 insertions(+), 182 deletions(-) diff --git a/lfs.c b/lfs.c index b1851618..60c5a8bd 100644 --- a/lfs.c +++ b/lfs.c @@ -4238,173 +4238,6 @@ static int lfs_deinit(lfs_t *lfs) { return 0; } -static int lfs_scan_for_superblock(lfs_t *lfs, lfs_superblock_t *superblock){ - // scan directory blocks for superblock and any global updates - lfs_mdir_t dir = {.tail = {0, 1}}; - lfs_block_t tortoise[2] = {LFS_BLOCK_NULL, LFS_BLOCK_NULL}; - lfs_size_t tortoise_i = 1; - lfs_size_t tortoise_period = 1; - while (!lfs_pair_isnull(dir.tail)) { - // detect cycles with Brent's algorithm - if (lfs_pair_issync(dir.tail, tortoise)) { - LFS_WARN("Cycle detected in tail list"); - return LFS_ERR_CORRUPT; - } - if (tortoise_i == tortoise_period) { - tortoise[0] = dir.tail[0]; - tortoise[1] = dir.tail[1]; - tortoise_i = 0; - tortoise_period *= 2; - } - tortoise_i += 1; - - // fetch next block in tail list - lfs_stag_t tag = lfs_dir_fetchmatch(lfs, &dir, dir.tail, - LFS_MKTAG(0x7ff, 0x3ff, 0), - LFS_MKTAG(LFS_TYPE_SUPERBLOCK, 0, 8), - NULL, - lfs_dir_find_match, &(struct lfs_dir_find_match){ - lfs, "littlefs", 8}); - if (tag < 0) { - return tag; - } - - // has superblock? - if (tag && !lfs_tag_isdelete(tag)) { - // update root - lfs->root[0] = dir.pair[0]; - lfs->root[1] = dir.pair[1]; - - // grab superblock - tag = lfs_dir_get(lfs, &dir, LFS_MKTAG(0x7ff, 0x3ff, 0), - LFS_MKTAG(LFS_TYPE_INLINESTRUCT, 0, sizeof(*superblock)), - superblock); - if (tag < 0) { - return tag; - } - lfs_superblock_fromle32(superblock); - } - } - - return LFS_ERR_OK; -} - -static int lfs_scan_for_state_updates(lfs_t *lfs){ - int err; - // scan directory blocks for superblock and any global updates - lfs_mdir_t dir = {.tail = {0, 1}}; - lfs_block_t tortoise[2] = {LFS_BLOCK_NULL, LFS_BLOCK_NULL}; - lfs_size_t tortoise_i = 1; - lfs_size_t tortoise_period = 1; - while (!lfs_pair_isnull(dir.tail)) { - // detect cycles with Brent's algorithm - if (lfs_pair_issync(dir.tail, tortoise)) { - LFS_WARN("Cycle detected in tail list"); - return LFS_ERR_CORRUPT; - } - if (tortoise_i == tortoise_period) { - tortoise[0] = dir.tail[0]; - tortoise[1] = dir.tail[1]; - tortoise_i = 0; - tortoise_period *= 2; - } - tortoise_i += 1; - - // fetch next block in tail list - lfs_stag_t tag = lfs_dir_fetchmatch(lfs, &dir, dir.tail, - LFS_MKTAG(0x7ff, 0x3ff, 0), - LFS_MKTAG(LFS_TYPE_SUPERBLOCK, 0, 8), - NULL, - lfs_dir_find_match, &(struct lfs_dir_find_match){ - lfs, "littlefs", 8}); - if (tag < 0) { - return tag; - } - - // has gstate? - err = lfs_dir_getgstate(lfs, &dir, &lfs->gstate); - if (err) { - return err; - } - } - - return LFS_ERR_OK; -} - -static int lfs_validate_superblock(lfs_t *lfs, lfs_superblock_t *superblock){ - // check version - uint16_t major_version = (0xffff & (superblock->version >> 16)); - uint16_t minor_version = (0xffff & (superblock->version >> 0)); - if (major_version != lfs_fs_disk_version_major(lfs) - || minor_version > lfs_fs_disk_version_minor(lfs)) { - LFS_ERROR("Invalid version " - "v%"PRIu16".%"PRIu16" != v%"PRIu16".%"PRIu16, - major_version, - minor_version, - lfs_fs_disk_version_major(lfs), - lfs_fs_disk_version_minor(lfs)); - return LFS_ERR_INVAL; - } - - // found older minor version? set an in-device only bit in the - // gstate so we know we need to rewrite the superblock before - // the first write - if (minor_version < lfs_fs_disk_version_minor(lfs)) { - LFS_DEBUG("Found older minor version " - "v%"PRIu16".%"PRIu16" < v%"PRIu16".%"PRIu16, - major_version, - minor_version, - lfs_fs_disk_version_major(lfs), - lfs_fs_disk_version_minor(lfs)); - // note this bit is reserved on disk, so fetching more gstate - // will not interfere here - lfs_fs_prepsuperblock(lfs, true); - } - - // check superblock configuration - if (superblock->name_max) { - if (superblock->name_max > lfs->name_max) { - LFS_ERROR("Unsupported name_max (%"PRIu32" > %"PRIu32")", - superblock->name_max, lfs->name_max); - return LFS_ERR_INVAL; - } - lfs->name_max = superblock->name_max; - } - - if (superblock->file_max) { - if (superblock->file_max > lfs->file_max) { - LFS_ERROR("Unsupported file_max (%"PRIu32" > %"PRIu32")", - superblock->file_max, lfs->file_max); - return LFS_ERR_INVAL; - } - lfs->file_max = superblock->file_max; - } - - if (superblock->attr_max) { - if (superblock->attr_max > lfs->attr_max) { - LFS_ERROR("Unsupported attr_max (%"PRIu32" > %"PRIu32")", - superblock->attr_max, lfs->attr_max); - return LFS_ERR_INVAL; - } - lfs->attr_max = superblock->attr_max; - } - - if (lfs->cfg->block_count && superblock->block_count != lfs->cfg->block_count) { - LFS_ERROR("Invalid block count (%"PRIu32" != %"PRIu32")", - superblock->block_count, lfs->cfg->block_count); - return LFS_ERR_INVAL; - } - - if (superblock->block_size != lfs->cfg->block_size) { - LFS_ERROR("Invalid block size (%"PRIu32" != %"PRIu32")", - superblock->block_size, lfs->cfg->block_size); - return LFS_ERR_INVAL; - } - - return LFS_ERR_OK; -} - - #ifndef LFS_READONLY @@ -4481,24 +4314,143 @@ static int lfs_rawmount(lfs_t *lfs, const struct lfs_config *cfg) { return err; } - lfs_superblock_t superblock; - err = lfs_scan_for_superblock(lfs, &superblock); - if (err) { - goto cleanup; - } + // scan directory blocks for superblock and any global updates + lfs_mdir_t dir = {.tail = {0, 1}}; + lfs_block_t tortoise[2] = {LFS_BLOCK_NULL, LFS_BLOCK_NULL}; + lfs_size_t tortoise_i = 1; + lfs_size_t tortoise_period = 1; + while (!lfs_pair_isnull(dir.tail)) { + // detect cycles with Brent's algorithm + if (lfs_pair_issync(dir.tail, tortoise)) { + LFS_WARN("Cycle detected in tail list"); + err = LFS_ERR_CORRUPT; + goto cleanup; + } + if (tortoise_i == tortoise_period) { + tortoise[0] = dir.tail[0]; + tortoise[1] = dir.tail[1]; + tortoise_i = 0; + tortoise_period *= 2; + } + tortoise_i += 1; - if (lfs->block_count == 0) { - lfs->block_count = superblock.block_count; - } + // fetch next block in tail list + lfs_stag_t tag = lfs_dir_fetchmatch(lfs, &dir, dir.tail, + LFS_MKTAG(0x7ff, 0x3ff, 0), + LFS_MKTAG(LFS_TYPE_SUPERBLOCK, 0, 8), + NULL, + lfs_dir_find_match, &(struct lfs_dir_find_match){ + lfs, "littlefs", 8}); + if (tag < 0) { + err = tag; + goto cleanup; + } - err = lfs_validate_superblock(lfs, &superblock); - if (err) { - goto cleanup; - } + // has superblock? + if (tag && !lfs_tag_isdelete(tag)) { + // update root + lfs->root[0] = dir.pair[0]; + lfs->root[1] = dir.pair[1]; - err = lfs_scan_for_state_updates(lfs); - if (err) { - goto cleanup; + // grab superblock + lfs_superblock_t superblock; + tag = lfs_dir_get(lfs, &dir, LFS_MKTAG(0x7ff, 0x3ff, 0), + LFS_MKTAG(LFS_TYPE_INLINESTRUCT, 0, sizeof(superblock)), + &superblock); + if (tag < 0) { + err = tag; + goto cleanup; + } + lfs_superblock_fromle32(&superblock); + + // check version + uint16_t major_version = (0xffff & (superblock.version >> 16)); + uint16_t minor_version = (0xffff & (superblock.version >> 0)); + if (major_version != lfs_fs_disk_version_major(lfs) + || minor_version > lfs_fs_disk_version_minor(lfs)) { + LFS_ERROR("Invalid version " + "v%"PRIu16".%"PRIu16" != v%"PRIu16".%"PRIu16, + major_version, + minor_version, + lfs_fs_disk_version_major(lfs), + lfs_fs_disk_version_minor(lfs)); + err = LFS_ERR_INVAL; + goto cleanup; + } + + // found older minor version? set an in-device only bit in the + // gstate so we know we need to rewrite the superblock before + // the first write + if (minor_version < lfs_fs_disk_version_minor(lfs)) { + LFS_DEBUG("Found older minor version " + "v%"PRIu16".%"PRIu16" < v%"PRIu16".%"PRIu16, + major_version, + minor_version, + lfs_fs_disk_version_major(lfs), + lfs_fs_disk_version_minor(lfs)); + // note this bit is reserved on disk, so fetching more gstate + // will not interfere here + lfs_fs_prepsuperblock(lfs, true); + } + + // check superblock configuration + if (superblock.name_max) { + if (superblock.name_max > lfs->name_max) { + LFS_ERROR("Unsupported name_max (%"PRIu32" > %"PRIu32")", + superblock.name_max, lfs->name_max); + err = LFS_ERR_INVAL; + goto cleanup; + } + + lfs->name_max = superblock.name_max; + } + + if (superblock.file_max) { + if (superblock.file_max > lfs->file_max) { + LFS_ERROR("Unsupported file_max (%"PRIu32" > %"PRIu32")", + superblock.file_max, lfs->file_max); + err = LFS_ERR_INVAL; + goto cleanup; + } + + lfs->file_max = superblock.file_max; + } + + if (superblock.attr_max) { + if (superblock.attr_max > lfs->attr_max) { + LFS_ERROR("Unsupported attr_max (%"PRIu32" > %"PRIu32")", + superblock.attr_max, lfs->attr_max); + err = LFS_ERR_INVAL; + goto cleanup; + } + + lfs->attr_max = superblock.attr_max; + } + + // this is where we get the block_count from disk if block_count=0 + if (lfs->cfg->block_count + && superblock.block_count != lfs->cfg->block_count) { + LFS_ERROR("Invalid block count (%"PRIu32" != %"PRIu32")", + superblock.block_count, lfs->cfg->block_count); + err = LFS_ERR_INVAL; + goto cleanup; + } + + lfs->block_count = superblock.block_count; + + if (superblock.block_size != lfs->cfg->block_size) { + LFS_ERROR("Invalid block size (%"PRIu32" != %"PRIu32")", + superblock.block_size, lfs->cfg->block_size); + err = LFS_ERR_INVAL; + goto cleanup; + } + } + + // has gstate? + err = lfs_dir_getgstate(lfs, &dir, &lfs->gstate); + if (err) { + goto cleanup; + } } // update littlefs with gstate From 027331b2f094d1327b23c25054ddaae652d30ea2 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Fri, 11 Nov 2022 15:05:34 -0600 Subject: [PATCH 14/23] Adopted erase_size/erase_count config in test block-devices/runners In separating the configuration of littlefs from the physical geometry of the underlying device, we can no longer rely solely on lfs_config to contain all of the information necessary for the simulated block devices we use for testing. This adds a new lfs_*bd_config struct for each of the block devices, and new erase_size/erase_count fields. The erase_* name was chosen since these reflect the (simulated) physical erase size and count of erase-sized blocks, unlike the block_* variants which represent logical block sizes used for littlefs's bookkeeping. It may be worth adopting erase_size/erase_count in littlefs's config at some point in the future, but at the moment doesn't seem necessary. Changing the lfs_bd_config structs to be required is probably a good idea anyways, as it moves us more towards separating the bds from littlefs. Though we can't quite get rid of the lfs_config parameter because of the block-device API in lfs_config. Eventually it would be nice to get rid of it, but that would require API breakage. --- bd/lfs_emubd.c | 105 ++++++++++++++++--------------------- bd/lfs_emubd.h | 18 +++++-- bd/lfs_filebd.c | 43 ++++++++------- bd/lfs_filebd.h | 21 +++++++- bd/lfs_rambd.c | 65 +++++++++-------------- bd/lfs_rambd.h | 19 +++++-- runners/bench_runner.c | 28 +++++----- runners/bench_runner.h | 32 ++++++----- runners/test_runner.c | 52 ++++++++++++------ runners/test_runner.h | 34 +++++++----- tests/test_alloc.toml | 16 +++--- tests/test_badblocks.toml | 6 +-- tests/test_exhaustion.toml | 10 ++-- 13 files changed, 248 insertions(+), 201 deletions(-) diff --git a/bd/lfs_emubd.c b/bd/lfs_emubd.c index 546a90c1..c27ae300 100644 --- a/bd/lfs_emubd.c +++ b/bd/lfs_emubd.c @@ -48,6 +48,7 @@ static void lfs_emubd_decblock(lfs_emubd_block_t *block) { static lfs_emubd_block_t *lfs_emubd_mutblock( const struct lfs_config *cfg, lfs_emubd_block_t **block) { + lfs_emubd_t *bd = cfg->context; lfs_emubd_block_t *block_ = *block; if (block_ && block_->rc == 1) { // rc == 1? can modify @@ -56,13 +57,13 @@ static lfs_emubd_block_t *lfs_emubd_mutblock( } else if (block_) { // rc > 1? need to create a copy lfs_emubd_block_t *nblock = malloc( - sizeof(lfs_emubd_block_t) + cfg->block_size); + sizeof(lfs_emubd_block_t) + bd->cfg->erase_size); if (!nblock) { return NULL; } memcpy(nblock, block_, - sizeof(lfs_emubd_block_t) + cfg->block_size); + sizeof(lfs_emubd_block_t) + bd->cfg->erase_size); nblock->rc = 1; lfs_emubd_decblock(block_); @@ -72,7 +73,7 @@ static lfs_emubd_block_t *lfs_emubd_mutblock( } else { // no block? need to allocate lfs_emubd_block_t *nblock = malloc( - sizeof(lfs_emubd_block_t) + cfg->block_size); + sizeof(lfs_emubd_block_t) + bd->cfg->erase_size); if (!nblock) { return NULL; } @@ -81,10 +82,9 @@ static lfs_emubd_block_t *lfs_emubd_mutblock( nblock->wear = 0; // zero for consistency - lfs_emubd_t *bd = cfg->context; memset(nblock->data, (bd->cfg->erase_value != -1) ? bd->cfg->erase_value : 0, - cfg->block_size); + bd->cfg->erase_size); *block = nblock; return nblock; @@ -94,22 +94,22 @@ static lfs_emubd_block_t *lfs_emubd_mutblock( // emubd create/destroy -int lfs_emubd_createcfg(const struct lfs_config *cfg, const char *path, +int lfs_emubd_create(const struct lfs_config *cfg, const struct lfs_emubd_config *bdcfg) { - LFS_EMUBD_TRACE("lfs_emubd_createcfg(%p {.context=%p, " - ".read=%p, .prog=%p, .erase=%p, .sync=%p, " - ".read_size=%"PRIu32", .prog_size=%"PRIu32", " - ".block_size=%"PRIu32", .block_count=%"PRIu32"}, " - "\"%s\", " - "%p {.erase_value=%"PRId32", .erase_cycles=%"PRIu32", " + LFS_EMUBD_TRACE("lfs_emubd_create(%p {.context=%p, " + ".read=%p, .prog=%p, .erase=%p, .sync=%p}, " + "%p {.read_size=%"PRIu32", .prog_size=%"PRIu32", " + ".erase_size=%"PRIu32", .erase_count=%"PRIu32", " + ".erase_value=%"PRId32", .erase_cycles=%"PRIu32", " ".badblock_behavior=%"PRIu8", .power_cycles=%"PRIu32", " ".powerloss_behavior=%"PRIu8", .powerloss_cb=%p, " ".powerloss_data=%p, .track_branches=%d})", (void*)cfg, cfg->context, (void*)(uintptr_t)cfg->read, (void*)(uintptr_t)cfg->prog, (void*)(uintptr_t)cfg->erase, (void*)(uintptr_t)cfg->sync, - cfg->read_size, cfg->prog_size, cfg->block_size, cfg->block_count, - path, (void*)bdcfg, bdcfg->erase_value, bdcfg->erase_cycles, + (void*)bdcfg, + bdcfg->read_size, bdcfg->prog_size, bdcfg->erase_size, + bdcfg->erase_count, bdcfg->erase_value, bdcfg->erase_cycles, bdcfg->badblock_behavior, bdcfg->power_cycles, bdcfg->powerloss_behavior, (void*)(uintptr_t)bdcfg->powerloss_cb, bdcfg->powerloss_data, bdcfg->track_branches); @@ -117,12 +117,12 @@ int lfs_emubd_createcfg(const struct lfs_config *cfg, const char *path, bd->cfg = bdcfg; // allocate our block array, all blocks start as uninitialized - bd->blocks = malloc(cfg->block_count * sizeof(lfs_emubd_block_t*)); + bd->blocks = malloc(bd->cfg->erase_count * sizeof(lfs_emubd_block_t*)); if (!bd->blocks) { - LFS_EMUBD_TRACE("lfs_emubd_createcfg -> %d", LFS_ERR_NOMEM); + LFS_EMUBD_TRACE("lfs_emubd_create -> %d", LFS_ERR_NOMEM); return LFS_ERR_NOMEM; } - memset(bd->blocks, 0, cfg->block_count * sizeof(lfs_emubd_block_t*)); + memset(bd->blocks, 0, bd->cfg->erase_count * sizeof(lfs_emubd_block_t*)); // setup testing things bd->readed = 0; @@ -134,7 +134,7 @@ int lfs_emubd_createcfg(const struct lfs_config *cfg, const char *path, if (bd->cfg->disk_path) { bd->disk = malloc(sizeof(lfs_emubd_disk_t)); if (!bd->disk) { - LFS_EMUBD_TRACE("lfs_emubd_createcfg -> %d", LFS_ERR_NOMEM); + LFS_EMUBD_TRACE("lfs_emubd_create -> %d", LFS_ERR_NOMEM); return LFS_ERR_NOMEM; } bd->disk->rc = 1; @@ -156,21 +156,21 @@ int lfs_emubd_createcfg(const struct lfs_config *cfg, const char *path, // if we're emulating erase values, we can keep a block around in // memory of just the erase state to speed up emulated erases if (bd->cfg->erase_value != -1) { - bd->disk->scratch = malloc(cfg->block_size); + bd->disk->scratch = malloc(bd->cfg->erase_size); if (!bd->disk->scratch) { - LFS_EMUBD_TRACE("lfs_emubd_createcfg -> %d", LFS_ERR_NOMEM); + LFS_EMUBD_TRACE("lfs_emubd_create -> %d", LFS_ERR_NOMEM); return LFS_ERR_NOMEM; } memset(bd->disk->scratch, bd->cfg->erase_value, - cfg->block_size); + bd->cfg->erase_size); // go ahead and erase all of the disk, otherwise the file will not // match our internal representation - for (size_t i = 0; i < cfg->block_count; i++) { + for (size_t i = 0; i < bd->cfg->erase_count; i++) { ssize_t res = write(bd->disk->fd, bd->disk->scratch, - cfg->block_size); + bd->cfg->erase_size); if (res < 0) { int err = -errno; LFS_EMUBD_TRACE("lfs_emubd_create -> %d", err); @@ -180,33 +180,16 @@ int lfs_emubd_createcfg(const struct lfs_config *cfg, const char *path, } } - LFS_EMUBD_TRACE("lfs_emubd_createcfg -> %d", 0); + LFS_EMUBD_TRACE("lfs_emubd_create -> %d", 0); return 0; } -int lfs_emubd_create(const struct lfs_config *cfg, const char *path) { - LFS_EMUBD_TRACE("lfs_emubd_create(%p {.context=%p, " - ".read=%p, .prog=%p, .erase=%p, .sync=%p, " - ".read_size=%"PRIu32", .prog_size=%"PRIu32", " - ".block_size=%"PRIu32", .block_count=%"PRIu32"}, " - "\"%s\")", - (void*)cfg, cfg->context, - (void*)(uintptr_t)cfg->read, (void*)(uintptr_t)cfg->prog, - (void*)(uintptr_t)cfg->erase, (void*)(uintptr_t)cfg->sync, - cfg->read_size, cfg->prog_size, cfg->block_size, cfg->block_count, - path); - static const struct lfs_emubd_config defaults = {.erase_value=-1}; - int err = lfs_emubd_createcfg(cfg, path, &defaults); - LFS_EMUBD_TRACE("lfs_emubd_create -> %d", err); - return err; -} - int lfs_emubd_destroy(const struct lfs_config *cfg) { LFS_EMUBD_TRACE("lfs_emubd_destroy(%p)", (void*)cfg); lfs_emubd_t *bd = cfg->context; // decrement reference counts - for (lfs_block_t i = 0; i < cfg->block_count; i++) { + for (lfs_block_t i = 0; i < bd->cfg->erase_count; i++) { lfs_emubd_decblock(bd->blocks[i]); } free(bd->blocks); @@ -237,10 +220,10 @@ int lfs_emubd_read(const struct lfs_config *cfg, lfs_block_t block, lfs_emubd_t *bd = cfg->context; // check if read is valid - LFS_ASSERT(!cfg->block_count || block < cfg->block_count); - LFS_ASSERT(off % cfg->read_size == 0); - LFS_ASSERT(size % cfg->read_size == 0); - LFS_ASSERT(off+size <= cfg->block_size); + LFS_ASSERT(block < bd->cfg->erase_count); + LFS_ASSERT(off % bd->cfg->read_size == 0); + LFS_ASSERT(size % bd->cfg->read_size == 0); + LFS_ASSERT(off+size <= bd->cfg->erase_size); // get the block const lfs_emubd_block_t *b = bd->blocks[block]; @@ -287,10 +270,10 @@ int lfs_emubd_prog(const struct lfs_config *cfg, lfs_block_t block, lfs_emubd_t *bd = cfg->context; // check if write is valid - LFS_ASSERT(!cfg->block_count || block < cfg->block_count); - LFS_ASSERT(off % cfg->prog_size == 0); - LFS_ASSERT(size % cfg->prog_size == 0); - LFS_ASSERT(off+size <= cfg->block_size); + LFS_ASSERT(block < bd->cfg->erase_count); + LFS_ASSERT(off % bd->cfg->prog_size == 0); + LFS_ASSERT(size % bd->cfg->prog_size == 0); + LFS_ASSERT(off+size <= bd->cfg->erase_size); // get the block lfs_emubd_block_t *b = lfs_emubd_mutblock(cfg, &bd->blocks[block]); @@ -327,7 +310,7 @@ int lfs_emubd_prog(const struct lfs_config *cfg, lfs_block_t block, // mirror to disk file? if (bd->disk) { off_t res1 = lseek(bd->disk->fd, - (off_t)block*cfg->block_size + (off_t)off, + (off_t)block*bd->cfg->erase_size + (off_t)off, SEEK_SET); if (res1 < 0) { int err = -errno; @@ -372,11 +355,11 @@ int lfs_emubd_prog(const struct lfs_config *cfg, lfs_block_t block, int lfs_emubd_erase(const struct lfs_config *cfg, lfs_block_t block) { LFS_EMUBD_TRACE("lfs_emubd_erase(%p, 0x%"PRIx32" (%"PRIu32"))", - (void*)cfg, block, cfg->block_size); + (void*)cfg, block, ((lfs_emubd_t*)cfg->context)->cfg->erase_size); lfs_emubd_t *bd = cfg->context; // check if erase is valid - LFS_ASSERT(!cfg->block_count || block < cfg->block_count); + LFS_ASSERT(block < bd->cfg->erase_count); // get the block lfs_emubd_block_t *b = lfs_emubd_mutblock(cfg, &bd->blocks[block]); @@ -405,12 +388,12 @@ int lfs_emubd_erase(const struct lfs_config *cfg, lfs_block_t block) { // emulate an erase value? if (bd->cfg->erase_value != -1) { - memset(b->data, bd->cfg->erase_value, cfg->block_size); + memset(b->data, bd->cfg->erase_value, bd->cfg->erase_size); // mirror to disk file? if (bd->disk) { off_t res1 = lseek(bd->disk->fd, - (off_t)block*cfg->block_size, + (off_t)block*bd->cfg->erase_size, SEEK_SET); if (res1 < 0) { int err = -errno; @@ -420,7 +403,7 @@ int lfs_emubd_erase(const struct lfs_config *cfg, lfs_block_t block) { ssize_t res2 = write(bd->disk->fd, bd->disk->scratch, - cfg->block_size); + bd->cfg->erase_size); if (res2 < 0) { int err = -errno; LFS_EMUBD_TRACE("lfs_emubd_erase -> %d", err); @@ -430,7 +413,7 @@ int lfs_emubd_erase(const struct lfs_config *cfg, lfs_block_t block) { } // track erases - bd->erased += cfg->block_size; + bd->erased += bd->cfg->erase_size; if (bd->cfg->erase_sleep) { int err = nanosleep(&(struct timespec){ .tv_sec=bd->cfg->erase_sleep/1000000000, @@ -573,7 +556,7 @@ lfs_emubd_swear_t lfs_emubd_wear(const struct lfs_config *cfg, lfs_emubd_t *bd = cfg->context; // check if block is valid - LFS_ASSERT(block < cfg->block_count); + LFS_ASSERT(block < bd->cfg->erase_count); // get the wear lfs_emubd_wear_t wear; @@ -595,7 +578,7 @@ int lfs_emubd_setwear(const struct lfs_config *cfg, lfs_emubd_t *bd = cfg->context; // check if block is valid - LFS_ASSERT(block < cfg->block_count); + LFS_ASSERT(block < bd->cfg->erase_count); // set the wear lfs_emubd_block_t *b = lfs_emubd_mutblock(cfg, &bd->blocks[block]); @@ -635,13 +618,13 @@ int lfs_emubd_copy(const struct lfs_config *cfg, lfs_emubd_t *copy) { lfs_emubd_t *bd = cfg->context; // lazily copy over our block array - copy->blocks = malloc(cfg->block_count * sizeof(lfs_emubd_block_t*)); + copy->blocks = malloc(bd->cfg->erase_count * sizeof(lfs_emubd_block_t*)); if (!copy->blocks) { LFS_EMUBD_TRACE("lfs_emubd_copy -> %d", LFS_ERR_NOMEM); return LFS_ERR_NOMEM; } - for (size_t i = 0; i < cfg->block_count; i++) { + for (size_t i = 0; i < bd->cfg->erase_count; i++) { copy->blocks[i] = lfs_emubd_incblock(bd->blocks[i]); } diff --git a/bd/lfs_emubd.h b/bd/lfs_emubd.h index 35a411fe..9049649f 100644 --- a/bd/lfs_emubd.h +++ b/bd/lfs_emubd.h @@ -67,6 +67,18 @@ typedef int64_t lfs_emubd_ssleep_t; // emubd config, this is required for testing struct lfs_emubd_config { + // Minimum size of a read operation in bytes. + lfs_size_t read_size; + + // Minimum size of a program operation in bytes. + lfs_size_t prog_size; + + // Size of an erase operation in bytes. + lfs_size_t erase_size; + + // Number of erase blocks on the device. + lfs_size_t erase_count; + // 8-bit erase value to use for simulating erases. -1 does not simulate // erases, which can speed up testing by avoiding the extra block-device // operations to store the erase value. @@ -149,11 +161,7 @@ typedef struct lfs_emubd { /// Block device API /// // Create an emulating block device using the geometry in lfs_config -// -// Note that filebd is used if a path is provided, if path is NULL -// emubd will use rambd which can be much faster. -int lfs_emubd_create(const struct lfs_config *cfg, const char *path); -int lfs_emubd_createcfg(const struct lfs_config *cfg, const char *path, +int lfs_emubd_create(const struct lfs_config *cfg, const struct lfs_emubd_config *bdcfg); // Clean up memory associated with block device diff --git a/bd/lfs_filebd.c b/bd/lfs_filebd.c index 780c8f90..4ff25d44 100644 --- a/bd/lfs_filebd.c +++ b/bd/lfs_filebd.c @@ -15,18 +15,22 @@ #include #endif -int lfs_filebd_create(const struct lfs_config *cfg, const char *path) { +int lfs_filebd_create(const struct lfs_config *cfg, const char *path, + const struct lfs_filebd_config *bdcfg) { LFS_FILEBD_TRACE("lfs_filebd_create(%p {.context=%p, " - ".read=%p, .prog=%p, .erase=%p, .sync=%p, " - ".read_size=%"PRIu32", .prog_size=%"PRIu32", " - ".block_size=%"PRIu32", .block_count=%"PRIu32"}, " - "\"%s\")", + ".read=%p, .prog=%p, .erase=%p, .sync=%p}, " + "\"%s\", " + "%p {.read_size=%"PRIu32", .prog_size=%"PRIu32", " + ".erase_size=%"PRIu32", .erase_count=%"PRIu32"})", (void*)cfg, cfg->context, (void*)(uintptr_t)cfg->read, (void*)(uintptr_t)cfg->prog, (void*)(uintptr_t)cfg->erase, (void*)(uintptr_t)cfg->sync, - cfg->read_size, cfg->prog_size, cfg->block_size, cfg->block_count, - path, (void*)bdcfg, bdcfg->erase_value); + path, + (void*)bdcfg, + bdcfg->read_size, bdcfg->prog_size, bdcfg->erase_size, + bdcfg->erase_count); lfs_filebd_t *bd = cfg->context; + bd->cfg = bdcfg; // open file #ifdef _WIN32 @@ -66,17 +70,17 @@ int lfs_filebd_read(const struct lfs_config *cfg, lfs_block_t block, lfs_filebd_t *bd = cfg->context; // check if read is valid - LFS_ASSERT(block < cfg->block_count); - LFS_ASSERT(off % cfg->read_size == 0); - LFS_ASSERT(size % cfg->read_size == 0); - LFS_ASSERT(off+size <= cfg->block_size); + LFS_ASSERT(block < bd->cfg->erase_count); + LFS_ASSERT(off % bd->cfg->read_size == 0); + LFS_ASSERT(size % bd->cfg->read_size == 0); + LFS_ASSERT(off+size <= bd->cfg->erase_size); // zero for reproducibility (in case file is truncated) memset(buffer, 0, size); // read off_t res1 = lseek(bd->fd, - (off_t)block*cfg->block_size + (off_t)off, SEEK_SET); + (off_t)block*bd->cfg->erase_size + (off_t)off, SEEK_SET); if (res1 < 0) { int err = -errno; LFS_FILEBD_TRACE("lfs_filebd_read -> %d", err); @@ -102,14 +106,14 @@ int lfs_filebd_prog(const struct lfs_config *cfg, lfs_block_t block, lfs_filebd_t *bd = cfg->context; // check if write is valid - LFS_ASSERT(block < cfg->block_count); - LFS_ASSERT(off % cfg->prog_size == 0); - LFS_ASSERT(size % cfg->prog_size == 0); - LFS_ASSERT(off+size <= cfg->block_size); + LFS_ASSERT(block < bd->cfg->erase_count); + LFS_ASSERT(off % bd->cfg->prog_size == 0); + LFS_ASSERT(size % bd->cfg->prog_size == 0); + LFS_ASSERT(off+size <= bd->cfg->erase_size); // program data off_t res1 = lseek(bd->fd, - (off_t)block*cfg->block_size + (off_t)off, SEEK_SET); + (off_t)block*bd->cfg->erase_size + (off_t)off, SEEK_SET); if (res1 < 0) { int err = -errno; LFS_FILEBD_TRACE("lfs_filebd_prog -> %d", err); @@ -129,10 +133,11 @@ int lfs_filebd_prog(const struct lfs_config *cfg, lfs_block_t block, int lfs_filebd_erase(const struct lfs_config *cfg, lfs_block_t block) { LFS_FILEBD_TRACE("lfs_filebd_erase(%p, 0x%"PRIx32" (%"PRIu32"))", - (void*)cfg, block, cfg->block_size); + (void*)cfg, block, ((lfs_file_t*)cfg->context)->cfg->erase_size); + lfs_filebd_t *bd = cfg->context; // check if erase is valid - LFS_ASSERT(block < cfg->block_count); + LFS_ASSERT(block < bd->cfg->erase_count); // erase is a noop (void)block; diff --git a/bd/lfs_filebd.h b/bd/lfs_filebd.h index 0f24996a..d7d2fd95 100644 --- a/bd/lfs_filebd.h +++ b/bd/lfs_filebd.h @@ -26,14 +26,31 @@ extern "C" #endif #endif +// filebd config +struct lfs_filebd_config { + // Minimum size of a read operation in bytes. + lfs_size_t read_size; + + // Minimum size of a program operation in bytes. + lfs_size_t prog_size; + + // Size of an erase operation in bytes. + lfs_size_t erase_size; + + // Number of erase blocks on the device. + lfs_size_t erase_count; +}; + // filebd state typedef struct lfs_filebd { int fd; + const struct lfs_filebd_config *cfg; } lfs_filebd_t; -// Create a file block device using the geometry in lfs_config -int lfs_filebd_create(const struct lfs_config *cfg, const char *path); +// Create a file block device +int lfs_filebd_create(const struct lfs_config *cfg, const char *path, + const struct lfs_filebd_config *bdcfg); // Clean up memory associated with block device int lfs_filebd_destroy(const struct lfs_config *cfg); diff --git a/bd/lfs_rambd.c b/bd/lfs_rambd.c index ab180b93..a6a05727 100644 --- a/bd/lfs_rambd.c +++ b/bd/lfs_rambd.c @@ -7,18 +7,19 @@ */ #include "bd/lfs_rambd.h" -int lfs_rambd_createcfg(const struct lfs_config *cfg, +int lfs_rambd_create(const struct lfs_config *cfg, const struct lfs_rambd_config *bdcfg) { - LFS_RAMBD_TRACE("lfs_rambd_createcfg(%p {.context=%p, " - ".read=%p, .prog=%p, .erase=%p, .sync=%p, " - ".read_size=%"PRIu32", .prog_size=%"PRIu32", " - ".block_size=%"PRIu32", .block_count=%"PRIu32"}, " - "%p {.buffer=%p})", + LFS_RAMBD_TRACE("lfs_rambd_create(%p {.context=%p, " + ".read=%p, .prog=%p, .erase=%p, .sync=%p}, " + "%p {.read_size=%"PRIu32", .prog_size=%"PRIu32", " + ".erase_size=%"PRIu32", .erase_count=%"PRIu32", " + ".buffer=%p})", (void*)cfg, cfg->context, (void*)(uintptr_t)cfg->read, (void*)(uintptr_t)cfg->prog, (void*)(uintptr_t)cfg->erase, (void*)(uintptr_t)cfg->sync, - cfg->read_size, cfg->prog_size, cfg->block_size, cfg->block_count, - (void*)bdcfg, bdcfg->buffer); + (void*)bdcfg, + bdcfg->read_size, bdcfg->prog_size, bdcfg->erase_size, + bdcfg->erase_count, bdcfg->buffer); lfs_rambd_t *bd = cfg->context; bd->cfg = bdcfg; @@ -26,35 +27,20 @@ int lfs_rambd_createcfg(const struct lfs_config *cfg, if (bd->cfg->buffer) { bd->buffer = bd->cfg->buffer; } else { - bd->buffer = lfs_malloc(cfg->block_size * cfg->block_count); + bd->buffer = lfs_malloc(bd->cfg->erase_size * bd->cfg->erase_count); if (!bd->buffer) { - LFS_RAMBD_TRACE("lfs_rambd_createcfg -> %d", LFS_ERR_NOMEM); + LFS_RAMBD_TRACE("lfs_rambd_create -> %d", LFS_ERR_NOMEM); return LFS_ERR_NOMEM; } } // zero for reproducibility - memset(bd->buffer, 0, cfg->block_size * cfg->block_count); + memset(bd->buffer, 0, bd->cfg->erase_size * bd->cfg->erase_count); - LFS_RAMBD_TRACE("lfs_rambd_createcfg -> %d", 0); + LFS_RAMBD_TRACE("lfs_rambd_create -> %d", 0); return 0; } -int lfs_rambd_create(const struct lfs_config *cfg) { - LFS_RAMBD_TRACE("lfs_rambd_create(%p {.context=%p, " - ".read=%p, .prog=%p, .erase=%p, .sync=%p, " - ".read_size=%"PRIu32", .prog_size=%"PRIu32", " - ".block_size=%"PRIu32", .block_count=%"PRIu32"})", - (void*)cfg, cfg->context, - (void*)(uintptr_t)cfg->read, (void*)(uintptr_t)cfg->prog, - (void*)(uintptr_t)cfg->erase, (void*)(uintptr_t)cfg->sync, - cfg->read_size, cfg->prog_size, cfg->block_size, cfg->block_count); - static const struct lfs_rambd_config defaults = {0}; - int err = lfs_rambd_createcfg(cfg, &defaults); - LFS_RAMBD_TRACE("lfs_rambd_create -> %d", err); - return err; -} - int lfs_rambd_destroy(const struct lfs_config *cfg) { LFS_RAMBD_TRACE("lfs_rambd_destroy(%p)", (void*)cfg); // clean up memory @@ -74,13 +60,13 @@ int lfs_rambd_read(const struct lfs_config *cfg, lfs_block_t block, lfs_rambd_t *bd = cfg->context; // check if read is valid - LFS_ASSERT(block < cfg->block_count); - LFS_ASSERT(off % cfg->read_size == 0); - LFS_ASSERT(size % cfg->read_size == 0); - LFS_ASSERT(off+size <= cfg->block_size); + LFS_ASSERT(block < bd->cfg->erase_count); + LFS_ASSERT(off % bd->cfg->read_size == 0); + LFS_ASSERT(size % bd->cfg->read_size == 0); + LFS_ASSERT(off+size <= bd->cfg->erase_size); // read data - memcpy(buffer, &bd->buffer[block*cfg->block_size + off], size); + memcpy(buffer, &bd->buffer[block*bd->cfg->erase_size + off], size); LFS_RAMBD_TRACE("lfs_rambd_read -> %d", 0); return 0; @@ -94,13 +80,13 @@ int lfs_rambd_prog(const struct lfs_config *cfg, lfs_block_t block, lfs_rambd_t *bd = cfg->context; // check if write is valid - LFS_ASSERT(block < cfg->block_count); - LFS_ASSERT(off % cfg->prog_size == 0); - LFS_ASSERT(size % cfg->prog_size == 0); - LFS_ASSERT(off+size <= cfg->block_size); + LFS_ASSERT(block < bd->cfg->erase_count); + LFS_ASSERT(off % bd->cfg->prog_size == 0); + LFS_ASSERT(size % bd->cfg->prog_size == 0); + LFS_ASSERT(off+size <= bd->cfg->erase_size); // program data - memcpy(&bd->buffer[block*cfg->block_size + off], buffer, size); + memcpy(&bd->buffer[block*bd->cfg->erase_size + off], buffer, size); LFS_RAMBD_TRACE("lfs_rambd_prog -> %d", 0); return 0; @@ -108,10 +94,11 @@ int lfs_rambd_prog(const struct lfs_config *cfg, lfs_block_t block, int lfs_rambd_erase(const struct lfs_config *cfg, lfs_block_t block) { LFS_RAMBD_TRACE("lfs_rambd_erase(%p, 0x%"PRIx32" (%"PRIu32"))", - (void*)cfg, block, cfg->block_size); + (void*)cfg, block, ((lfs_rambd_t*)cfg->context)->cfg->erase_size); + lfs_rambd_t *bd = cfg->context; // check if erase is valid - LFS_ASSERT(block < cfg->block_count); + LFS_ASSERT(block < bd->cfg->erase_count); // erase is a noop (void)block; diff --git a/bd/lfs_rambd.h b/bd/lfs_rambd.h index 34246802..86637026 100644 --- a/bd/lfs_rambd.h +++ b/bd/lfs_rambd.h @@ -26,8 +26,20 @@ extern "C" #endif #endif -// rambd config (optional) +// rambd config struct lfs_rambd_config { + // Minimum size of a read operation in bytes. + lfs_size_t read_size; + + // Minimum size of a program operation in bytes. + lfs_size_t prog_size; + + // Size of an erase operation in bytes. + lfs_size_t erase_size; + + // Number of erase blocks on the device. + lfs_size_t erase_count; + // Optional statically allocated buffer for the block device. void *buffer; }; @@ -39,9 +51,8 @@ typedef struct lfs_rambd { } lfs_rambd_t; -// Create a RAM block device using the geometry in lfs_config -int lfs_rambd_create(const struct lfs_config *cfg); -int lfs_rambd_createcfg(const struct lfs_config *cfg, +// Create a RAM block device +int lfs_rambd_create(const struct lfs_config *cfg, const struct lfs_rambd_config *bdcfg); // Clean up memory associated with block device diff --git a/runners/bench_runner.c b/runners/bench_runner.c index ba791b25..f4dce22b 100644 --- a/runners/bench_runner.c +++ b/runners/bench_runner.c @@ -1271,9 +1271,9 @@ static void list_geometries(void) { builtin_geometries[g].name, READ_SIZE, PROG_SIZE, - BLOCK_SIZE, - BLOCK_COUNT, - BLOCK_SIZE*BLOCK_COUNT); + ERASE_SIZE, + ERASE_COUNT, + ERASE_SIZE*ERASE_COUNT); } } @@ -1324,6 +1324,10 @@ void perm_run( }; struct lfs_emubd_config bdcfg = { + .read_size = READ_SIZE, + .prog_size = PROG_SIZE, + .erase_size = ERASE_SIZE, + .erase_count = ERASE_COUNT, .erase_value = ERASE_VALUE, .erase_cycles = ERASE_CYCLES, .badblock_behavior = BADBLOCK_BEHAVIOR, @@ -1333,7 +1337,7 @@ void perm_run( .erase_sleep = bench_erase_sleep, }; - int err = lfs_emubd_createcfg(&cfg, bench_disk_path, &bdcfg); + int err = lfs_emubd_create(&cfg, &bdcfg); if (err) { fprintf(stderr, "error: could not create block device: %d\n", err); exit(-1); @@ -1761,19 +1765,19 @@ int main(int argc, char **argv) { = BENCH_LIT(sizes[0]); geometry->defines[PROG_SIZE_i] = BENCH_LIT(sizes[1]); - geometry->defines[BLOCK_SIZE_i] + geometry->defines[ERASE_SIZE_i] = BENCH_LIT(sizes[2]); } else if (count >= 2) { geometry->defines[PROG_SIZE_i] = BENCH_LIT(sizes[0]); - geometry->defines[BLOCK_SIZE_i] + geometry->defines[ERASE_SIZE_i] = BENCH_LIT(sizes[1]); } else { - geometry->defines[BLOCK_SIZE_i] + geometry->defines[ERASE_SIZE_i] = BENCH_LIT(sizes[0]); } if (count >= 4) { - geometry->defines[BLOCK_COUNT_i] + geometry->defines[ERASE_COUNT_i] = BENCH_LIT(sizes[3]); } optarg = s; @@ -1805,19 +1809,19 @@ int main(int argc, char **argv) { = BENCH_LIT(sizes[0]); geometry->defines[PROG_SIZE_i] = BENCH_LIT(sizes[1]); - geometry->defines[BLOCK_SIZE_i] + geometry->defines[ERASE_SIZE_i] = BENCH_LIT(sizes[2]); } else if (count >= 2) { geometry->defines[PROG_SIZE_i] = BENCH_LIT(sizes[0]); - geometry->defines[BLOCK_SIZE_i] + geometry->defines[ERASE_SIZE_i] = BENCH_LIT(sizes[1]); } else { - geometry->defines[BLOCK_SIZE_i] + geometry->defines[ERASE_SIZE_i] = BENCH_LIT(sizes[0]); } if (count >= 4) { - geometry->defines[BLOCK_COUNT_i] + geometry->defines[ERASE_COUNT_i] = BENCH_LIT(sizes[3]); } optarg = s; diff --git a/runners/bench_runner.h b/runners/bench_runner.h index 6296c091..b072970e 100644 --- a/runners/bench_runner.h +++ b/runners/bench_runner.h @@ -89,18 +89,22 @@ intmax_t bench_define(size_t define); #define READ_SIZE_i 0 #define PROG_SIZE_i 1 -#define BLOCK_SIZE_i 2 -#define BLOCK_COUNT_i 3 -#define CACHE_SIZE_i 4 -#define LOOKAHEAD_SIZE_i 5 -#define BLOCK_CYCLES_i 6 -#define ERASE_VALUE_i 7 -#define ERASE_CYCLES_i 8 -#define BADBLOCK_BEHAVIOR_i 9 -#define POWERLOSS_BEHAVIOR_i 10 +#define ERASE_SIZE_i 2 +#define ERASE_COUNT_i 3 +#define BLOCK_SIZE_i 4 +#define BLOCK_COUNT_i 5 +#define CACHE_SIZE_i 6 +#define LOOKAHEAD_SIZE_i 7 +#define BLOCK_CYCLES_i 8 +#define ERASE_VALUE_i 9 +#define ERASE_CYCLES_i 10 +#define BADBLOCK_BEHAVIOR_i 11 +#define POWERLOSS_BEHAVIOR_i 12 #define READ_SIZE bench_define(READ_SIZE_i) #define PROG_SIZE bench_define(PROG_SIZE_i) +#define ERASE_SIZE bench_define(ERASE_SIZE_i) +#define ERASE_COUNT bench_define(ERASE_COUNT_i) #define BLOCK_SIZE bench_define(BLOCK_SIZE_i) #define BLOCK_COUNT bench_define(BLOCK_COUNT_i) #define CACHE_SIZE bench_define(CACHE_SIZE_i) @@ -113,9 +117,11 @@ intmax_t bench_define(size_t define); #define BENCH_IMPLICIT_DEFINES \ BENCH_DEF(READ_SIZE, PROG_SIZE) \ - BENCH_DEF(PROG_SIZE, BLOCK_SIZE) \ - BENCH_DEF(BLOCK_SIZE, 0) \ - BENCH_DEF(BLOCK_COUNT, (1024*1024)/BLOCK_SIZE) \ + BENCH_DEF(PROG_SIZE, ERASE_SIZE) \ + BENCH_DEF(ERASE_SIZE, 0) \ + BENCH_DEF(ERASE_COUNT, (1024*1024)/BLOCK_SIZE) \ + BENCH_DEF(BLOCK_SIZE, ERASE_SIZE) \ + BENCH_DEF(BLOCK_COUNT, ERASE_COUNT/lfs_max(BLOCK_SIZE/ERASE_SIZE,1))\ BENCH_DEF(CACHE_SIZE, lfs_max(64,lfs_max(READ_SIZE,PROG_SIZE))) \ BENCH_DEF(LOOKAHEAD_SIZE, 16) \ BENCH_DEF(BLOCK_CYCLES, -1) \ @@ -125,7 +131,7 @@ intmax_t bench_define(size_t define); BENCH_DEF(POWERLOSS_BEHAVIOR, LFS_EMUBD_POWERLOSS_NOOP) #define BENCH_GEOMETRY_DEFINE_COUNT 4 -#define BENCH_IMPLICIT_DEFINE_COUNT 11 +#define BENCH_IMPLICIT_DEFINE_COUNT 13 #endif diff --git a/runners/test_runner.c b/runners/test_runner.c index 27f85249..13befdc3 100644 --- a/runners/test_runner.c +++ b/runners/test_runner.c @@ -1312,9 +1312,9 @@ static void list_geometries(void) { builtin_geometries[g].name, READ_SIZE, PROG_SIZE, - BLOCK_SIZE, - BLOCK_COUNT, - BLOCK_SIZE*BLOCK_COUNT); + ERASE_SIZE, + ERASE_COUNT, + ERASE_SIZE*ERASE_COUNT); } } @@ -1352,6 +1352,10 @@ static void run_powerloss_none( }; struct lfs_emubd_config bdcfg = { + .read_size = READ_SIZE, + .prog_size = PROG_SIZE, + .erase_size = ERASE_SIZE, + .erase_count = ERASE_COUNT, .erase_value = ERASE_VALUE, .erase_cycles = ERASE_CYCLES, .badblock_behavior = BADBLOCK_BEHAVIOR, @@ -1361,7 +1365,7 @@ static void run_powerloss_none( .erase_sleep = test_erase_sleep, }; - int err = lfs_emubd_createcfg(&cfg, test_disk_path, &bdcfg); + int err = lfs_emubd_create(&cfg, &bdcfg); if (err) { fprintf(stderr, "error: could not create block device: %d\n", err); exit(-1); @@ -1424,6 +1428,10 @@ static void run_powerloss_linear( }; struct lfs_emubd_config bdcfg = { + .read_size = READ_SIZE, + .prog_size = PROG_SIZE, + .erase_size = ERASE_SIZE, + .erase_count = ERASE_COUNT, .erase_value = ERASE_VALUE, .erase_cycles = ERASE_CYCLES, .badblock_behavior = BADBLOCK_BEHAVIOR, @@ -1437,7 +1445,7 @@ static void run_powerloss_linear( .powerloss_data = &powerloss_jmp, }; - int err = lfs_emubd_createcfg(&cfg, test_disk_path, &bdcfg); + int err = lfs_emubd_create(&cfg, &bdcfg); if (err) { fprintf(stderr, "error: could not create block device: %d\n", err); exit(-1); @@ -1513,6 +1521,10 @@ static void run_powerloss_log( }; struct lfs_emubd_config bdcfg = { + .read_size = READ_SIZE, + .prog_size = PROG_SIZE, + .erase_size = ERASE_SIZE, + .erase_count = ERASE_COUNT, .erase_value = ERASE_VALUE, .erase_cycles = ERASE_CYCLES, .badblock_behavior = BADBLOCK_BEHAVIOR, @@ -1526,7 +1538,7 @@ static void run_powerloss_log( .powerloss_data = &powerloss_jmp, }; - int err = lfs_emubd_createcfg(&cfg, test_disk_path, &bdcfg); + int err = lfs_emubd_create(&cfg, &bdcfg); if (err) { fprintf(stderr, "error: could not create block device: %d\n", err); exit(-1); @@ -1600,6 +1612,10 @@ static void run_powerloss_cycles( }; struct lfs_emubd_config bdcfg = { + .read_size = READ_SIZE, + .prog_size = PROG_SIZE, + .erase_size = ERASE_SIZE, + .erase_count = ERASE_COUNT, .erase_value = ERASE_VALUE, .erase_cycles = ERASE_CYCLES, .badblock_behavior = BADBLOCK_BEHAVIOR, @@ -1613,7 +1629,7 @@ static void run_powerloss_cycles( .powerloss_data = &powerloss_jmp, }; - int err = lfs_emubd_createcfg(&cfg, test_disk_path, &bdcfg); + int err = lfs_emubd_create(&cfg, &bdcfg); if (err) { fprintf(stderr, "error: could not create block device: %d\n", err); exit(-1); @@ -1785,6 +1801,10 @@ static void run_powerloss_exhaustive( }; struct lfs_emubd_config bdcfg = { + .read_size = READ_SIZE, + .prog_size = PROG_SIZE, + .erase_size = ERASE_SIZE, + .erase_count = ERASE_COUNT, .erase_value = ERASE_VALUE, .erase_cycles = ERASE_CYCLES, .badblock_behavior = BADBLOCK_BEHAVIOR, @@ -1797,7 +1817,7 @@ static void run_powerloss_exhaustive( .powerloss_data = NULL, }; - int err = lfs_emubd_createcfg(&cfg, test_disk_path, &bdcfg); + int err = lfs_emubd_create(&cfg, &bdcfg); if (err) { fprintf(stderr, "error: could not create block device: %d\n", err); exit(-1); @@ -2314,19 +2334,19 @@ int main(int argc, char **argv) { = TEST_LIT(sizes[0]); geometry->defines[PROG_SIZE_i] = TEST_LIT(sizes[1]); - geometry->defines[BLOCK_SIZE_i] + geometry->defines[ERASE_SIZE_i] = TEST_LIT(sizes[2]); } else if (count >= 2) { geometry->defines[PROG_SIZE_i] = TEST_LIT(sizes[0]); - geometry->defines[BLOCK_SIZE_i] + geometry->defines[ERASE_SIZE_i] = TEST_LIT(sizes[1]); } else { - geometry->defines[BLOCK_SIZE_i] + geometry->defines[ERASE_SIZE_i] = TEST_LIT(sizes[0]); } if (count >= 4) { - geometry->defines[BLOCK_COUNT_i] + geometry->defines[ERASE_COUNT_i] = TEST_LIT(sizes[3]); } optarg = s; @@ -2358,19 +2378,19 @@ int main(int argc, char **argv) { = TEST_LIT(sizes[0]); geometry->defines[PROG_SIZE_i] = TEST_LIT(sizes[1]); - geometry->defines[BLOCK_SIZE_i] + geometry->defines[ERASE_SIZE_i] = TEST_LIT(sizes[2]); } else if (count >= 2) { geometry->defines[PROG_SIZE_i] = TEST_LIT(sizes[0]); - geometry->defines[BLOCK_SIZE_i] + geometry->defines[ERASE_SIZE_i] = TEST_LIT(sizes[1]); } else { - geometry->defines[BLOCK_SIZE_i] + geometry->defines[ERASE_SIZE_i] = TEST_LIT(sizes[0]); } if (count >= 4) { - geometry->defines[BLOCK_COUNT_i] + geometry->defines[ERASE_COUNT_i] = TEST_LIT(sizes[3]); } optarg = s; diff --git a/runners/test_runner.h b/runners/test_runner.h index e30d4928..4be72e42 100644 --- a/runners/test_runner.h +++ b/runners/test_runner.h @@ -82,19 +82,23 @@ intmax_t test_define(size_t define); #define READ_SIZE_i 0 #define PROG_SIZE_i 1 -#define BLOCK_SIZE_i 2 -#define BLOCK_COUNT_i 3 -#define CACHE_SIZE_i 4 -#define LOOKAHEAD_SIZE_i 5 -#define BLOCK_CYCLES_i 6 -#define ERASE_VALUE_i 7 -#define ERASE_CYCLES_i 8 -#define BADBLOCK_BEHAVIOR_i 9 -#define POWERLOSS_BEHAVIOR_i 10 -#define DISK_VERSION_i 11 +#define ERASE_SIZE_i 2 +#define ERASE_COUNT_i 3 +#define BLOCK_SIZE_i 4 +#define BLOCK_COUNT_i 5 +#define CACHE_SIZE_i 6 +#define LOOKAHEAD_SIZE_i 7 +#define BLOCK_CYCLES_i 8 +#define ERASE_VALUE_i 9 +#define ERASE_CYCLES_i 10 +#define BADBLOCK_BEHAVIOR_i 11 +#define POWERLOSS_BEHAVIOR_i 12 +#define DISK_VERSION_i 13 #define READ_SIZE TEST_DEFINE(READ_SIZE_i) #define PROG_SIZE TEST_DEFINE(PROG_SIZE_i) +#define ERASE_SIZE TEST_DEFINE(ERASE_SIZE_i) +#define ERASE_COUNT TEST_DEFINE(ERASE_COUNT_i) #define BLOCK_SIZE TEST_DEFINE(BLOCK_SIZE_i) #define BLOCK_COUNT TEST_DEFINE(BLOCK_COUNT_i) #define CACHE_SIZE TEST_DEFINE(CACHE_SIZE_i) @@ -108,9 +112,11 @@ intmax_t test_define(size_t define); #define TEST_IMPLICIT_DEFINES \ TEST_DEF(READ_SIZE, PROG_SIZE) \ - TEST_DEF(PROG_SIZE, BLOCK_SIZE) \ - TEST_DEF(BLOCK_SIZE, 0) \ - TEST_DEF(BLOCK_COUNT, (1024*1024)/BLOCK_SIZE) \ + TEST_DEF(PROG_SIZE, ERASE_SIZE) \ + TEST_DEF(ERASE_SIZE, 0) \ + TEST_DEF(ERASE_COUNT, (1024*1024)/ERASE_SIZE) \ + TEST_DEF(BLOCK_SIZE, ERASE_SIZE) \ + TEST_DEF(BLOCK_COUNT, ERASE_COUNT/lfs_max(BLOCK_SIZE/ERASE_SIZE,1)) \ TEST_DEF(CACHE_SIZE, lfs_max(64,lfs_max(READ_SIZE,PROG_SIZE))) \ TEST_DEF(LOOKAHEAD_SIZE, 16) \ TEST_DEF(BLOCK_CYCLES, -1) \ @@ -120,8 +126,8 @@ intmax_t test_define(size_t define); TEST_DEF(POWERLOSS_BEHAVIOR, LFS_EMUBD_POWERLOSS_NOOP) \ TEST_DEF(DISK_VERSION, 0) -#define TEST_IMPLICIT_DEFINE_COUNT 12 #define TEST_GEOMETRY_DEFINE_COUNT 4 +#define TEST_IMPLICIT_DEFINE_COUNT 14 #endif diff --git a/tests/test_alloc.toml b/tests/test_alloc.toml index 205efbb1..916fc2a5 100644 --- a/tests/test_alloc.toml +++ b/tests/test_alloc.toml @@ -460,8 +460,8 @@ code = ''' # chained dir exhaustion test [cases.test_alloc_chained_dir_exhaustion] -if = 'BLOCK_SIZE == 512' -defines.BLOCK_COUNT = 1024 +if = 'ERASE_SIZE == 512' +defines.ERASE_COUNT = 1024 code = ''' lfs_t lfs; lfs_format(&lfs, cfg) => 0; @@ -538,8 +538,8 @@ code = ''' # split dir test [cases.test_alloc_split_dir] -if = 'BLOCK_SIZE == 512' -defines.BLOCK_COUNT = 1024 +if = 'ERASE_SIZE == 512' +defines.ERASE_COUNT = 1024 code = ''' lfs_t lfs; lfs_format(&lfs, cfg) => 0; @@ -587,8 +587,8 @@ code = ''' # outdated lookahead test [cases.test_alloc_outdated_lookahead] -if = 'BLOCK_SIZE == 512' -defines.BLOCK_COUNT = 1024 +if = 'ERASE_SIZE == 512' +defines.ERASE_COUNT = 1024 code = ''' lfs_t lfs; lfs_format(&lfs, cfg) => 0; @@ -655,8 +655,8 @@ code = ''' # outdated lookahead and split dir test [cases.test_alloc_outdated_lookahead_split_dir] -if = 'BLOCK_SIZE == 512' -defines.BLOCK_COUNT = 1024 +if = 'ERASE_SIZE == 512' +defines.ERASE_COUNT = 1024 code = ''' lfs_t lfs; lfs_format(&lfs, cfg) => 0; diff --git a/tests/test_badblocks.toml b/tests/test_badblocks.toml index b50b3933..04ed73f8 100644 --- a/tests/test_badblocks.toml +++ b/tests/test_badblocks.toml @@ -2,7 +2,7 @@ if = '(int32_t)BLOCK_CYCLES == -1' [cases.test_badblocks_single] -defines.BLOCK_COUNT = 256 # small bd so test runs faster +defines.ERASE_COUNT = 256 # small bd so test runs faster defines.ERASE_CYCLES = 0xffffffff defines.ERASE_VALUE = [0x00, 0xff, -1] defines.BADBLOCK_BEHAVIOR = [ @@ -82,7 +82,7 @@ code = ''' ''' [cases.test_badblocks_region_corruption] # (causes cascading failures) -defines.BLOCK_COUNT = 256 # small bd so test runs faster +defines.ERASE_COUNT = 256 # small bd so test runs faster defines.ERASE_CYCLES = 0xffffffff defines.ERASE_VALUE = [0x00, 0xff, -1] defines.BADBLOCK_BEHAVIOR = [ @@ -161,7 +161,7 @@ code = ''' ''' [cases.test_badblocks_alternating_corruption] # (causes cascading failures) -defines.BLOCK_COUNT = 256 # small bd so test runs faster +defines.ERASE_COUNT = 256 # small bd so test runs faster defines.ERASE_CYCLES = 0xffffffff defines.ERASE_VALUE = [0x00, 0xff, -1] defines.BADBLOCK_BEHAVIOR = [ diff --git a/tests/test_exhaustion.toml b/tests/test_exhaustion.toml index 2cf6aed1..273dd98c 100644 --- a/tests/test_exhaustion.toml +++ b/tests/test_exhaustion.toml @@ -1,7 +1,7 @@ # test running a filesystem to exhaustion [cases.test_exhaustion_normal] defines.ERASE_CYCLES = 10 -defines.BLOCK_COUNT = 256 # small bd so test runs faster +defines.ERASE_COUNT = 256 # small bd so test runs faster defines.BLOCK_CYCLES = 'ERASE_CYCLES / 2' defines.BADBLOCK_BEHAVIOR = [ 'LFS_EMUBD_BADBLOCK_PROGERROR', @@ -94,7 +94,7 @@ exhausted: # which also requires expanding superblocks [cases.test_exhaustion_superblocks] defines.ERASE_CYCLES = 10 -defines.BLOCK_COUNT = 256 # small bd so test runs faster +defines.ERASE_COUNT = 256 # small bd so test runs faster defines.BLOCK_CYCLES = 'ERASE_CYCLES / 2' defines.BADBLOCK_BEHAVIOR = [ 'LFS_EMUBD_BADBLOCK_PROGERROR', @@ -188,7 +188,7 @@ exhausted: # wear-level test running a filesystem to exhaustion [cases.test_exhuastion_wear_leveling] defines.ERASE_CYCLES = 20 -defines.BLOCK_COUNT = 256 # small bd so test runs faster +defines.ERASE_COUNT = 256 # small bd so test runs faster defines.BLOCK_CYCLES = 'ERASE_CYCLES / 2' defines.FILES = 10 code = ''' @@ -288,7 +288,7 @@ exhausted: # wear-level test + expanding superblock [cases.test_exhaustion_wear_leveling_superblocks] defines.ERASE_CYCLES = 20 -defines.BLOCK_COUNT = 256 # small bd so test runs faster +defines.ERASE_COUNT = 256 # small bd so test runs faster defines.BLOCK_CYCLES = 'ERASE_CYCLES / 2' defines.FILES = 10 code = ''' @@ -385,7 +385,7 @@ exhausted: # test that we wear blocks roughly evenly [cases.test_exhaustion_wear_distribution] defines.ERASE_CYCLES = 0xffffffff -defines.BLOCK_COUNT = 256 # small bd so test runs faster +defines.ERASE_COUNT = 256 # small bd so test runs faster defines.BLOCK_CYCLES = [5, 4, 3, 2, 1] defines.CYCLES = 100 defines.FILES = 10 From 127d84b68146490730c2648bff533a3969add267 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Sat, 12 Nov 2022 14:31:52 -0600 Subject: [PATCH 15/23] Added a couple mixed/unknown block_count tests These were cherry-picked from some previous work on a related feature. --- tests/test_superblocks.toml | 147 ++++++++++++++++++++++++++++++++++++ 1 file changed, 147 insertions(+) diff --git a/tests/test_superblocks.toml b/tests/test_superblocks.toml index 99e1dad0..a99a5a47 100644 --- a/tests/test_superblocks.toml +++ b/tests/test_superblocks.toml @@ -212,3 +212,150 @@ code = ''' assert(info.type == LFS_TYPE_REG); lfs_unmount(&lfs) => 0; ''' + +# mount with unknown block_count +[cases.test_superblocks_unknown_blocks] +code = ''' + lfs_t lfs; + lfs_format(&lfs, cfg) => 0; + + // known block_size/block_count + cfg->block_size = BLOCK_SIZE; + cfg->block_count = BLOCK_COUNT; + lfs_mount(&lfs, cfg) => 0; + struct lfs_fsinfo fsinfo; + lfs_fs_stat(&lfs, &fsinfo) => 0; + assert(fsinfo.block_size == BLOCK_SIZE); + assert(fsinfo.block_count == BLOCK_COUNT); + lfs_unmount(&lfs) => 0; + + // unknown block_count + cfg->block_size = BLOCK_SIZE; + cfg->block_count = 0; + lfs_mount(&lfs, cfg) => 0; + lfs_fs_stat(&lfs, &fsinfo) => 0; + assert(fsinfo.block_size == BLOCK_SIZE); + assert(fsinfo.block_count == BLOCK_COUNT); + lfs_unmount(&lfs) => 0; + + // do some work + lfs_mount(&lfs, cfg) => 0; + lfs_fs_stat(&lfs, &fsinfo) => 0; + assert(fsinfo.block_size == BLOCK_SIZE); + assert(fsinfo.block_count == BLOCK_COUNT); + lfs_file_t file; + lfs_file_open(&lfs, &file, "test", + LFS_O_CREAT | LFS_O_EXCL | LFS_O_WRONLY) => 0; + lfs_file_write(&lfs, &file, "hello!", 6) => 6; + lfs_file_close(&lfs, &file) => 0; + lfs_unmount(&lfs) => 0; + + lfs_mount(&lfs, cfg) => 0; + lfs_fs_stat(&lfs, &fsinfo) => 0; + assert(fsinfo.block_size == BLOCK_SIZE); + assert(fsinfo.block_count == BLOCK_COUNT); + lfs_file_open(&lfs, &file, "test", LFS_O_RDONLY) => 0; + uint8_t buffer[256]; + lfs_file_read(&lfs, &file, buffer, sizeof(buffer)) => 6; + lfs_file_close(&lfs, &file) => 0; + assert(memcmp(buffer, "hello!", 6) == 0); + lfs_unmount(&lfs) => 0; +''' + +# mount with blocks fewer than the erase_count +[cases.test_superblocks_fewer_blocks] +defines.BLOCK_COUNT = ['ERASE_COUNT/2', 'ERASE_COUNT/4', '2'] +code = ''' + lfs_t lfs; + lfs_format(&lfs, cfg) => 0; + + // known block_size/block_count + cfg->block_size = BLOCK_SIZE; + cfg->block_count = BLOCK_COUNT; + lfs_mount(&lfs, cfg) => 0; + struct lfs_fsinfo fsinfo; + lfs_fs_stat(&lfs, &fsinfo) => 0; + assert(fsinfo.block_size == BLOCK_SIZE); + assert(fsinfo.block_count == BLOCK_COUNT); + lfs_unmount(&lfs) => 0; + + // incorrect block_count + cfg->block_size = BLOCK_SIZE; + cfg->block_count = ERASE_COUNT; + lfs_mount(&lfs, cfg) => LFS_ERR_INVAL; + + // unknown block_count + cfg->block_size = BLOCK_SIZE; + cfg->block_count = 0; + lfs_mount(&lfs, cfg) => 0; + lfs_fs_stat(&lfs, &fsinfo) => 0; + assert(fsinfo.block_size == BLOCK_SIZE); + assert(fsinfo.block_count == BLOCK_COUNT); + lfs_unmount(&lfs) => 0; + + // do some work + lfs_mount(&lfs, cfg) => 0; + lfs_fs_stat(&lfs, &fsinfo) => 0; + assert(fsinfo.block_size == BLOCK_SIZE); + assert(fsinfo.block_count == BLOCK_COUNT); + lfs_file_t file; + lfs_file_open(&lfs, &file, "test", + LFS_O_CREAT | LFS_O_EXCL | LFS_O_WRONLY) => 0; + lfs_file_write(&lfs, &file, "hello!", 6) => 6; + lfs_file_close(&lfs, &file) => 0; + lfs_unmount(&lfs) => 0; + + lfs_mount(&lfs, cfg) => 0; + lfs_fs_stat(&lfs, &fsinfo) => 0; + assert(fsinfo.block_size == BLOCK_SIZE); + assert(fsinfo.block_count == BLOCK_COUNT); + lfs_file_open(&lfs, &file, "test", LFS_O_RDONLY) => 0; + uint8_t buffer[256]; + lfs_file_read(&lfs, &file, buffer, sizeof(buffer)) => 6; + lfs_file_close(&lfs, &file) => 0; + assert(memcmp(buffer, "hello!", 6) == 0); + lfs_unmount(&lfs) => 0; +''' + +# mount with more blocks than the erase_count +[cases.test_superblocks_more_blocks] +defines.FORMAT_BLOCK_COUNT = '2*ERASE_COUNT' +in = 'lfs.c' +code = ''' + lfs_t lfs; + lfs_init(&lfs, cfg) => 0; + lfs.block_count = BLOCK_COUNT; + + lfs_mdir_t root = { + .pair = {0, 0}, // make sure this goes into block 0 + .rev = 0, + .off = sizeof(uint32_t), + .etag = 0xffffffff, + .count = 0, + .tail = {LFS_BLOCK_NULL, LFS_BLOCK_NULL}, + .erased = false, + .split = false, + }; + + lfs_superblock_t superblock = { + .version = LFS_DISK_VERSION, + .block_size = BLOCK_SIZE, + .block_count = FORMAT_BLOCK_COUNT, + .name_max = LFS_NAME_MAX, + .file_max = LFS_FILE_MAX, + .attr_max = LFS_ATTR_MAX, + }; + + lfs_superblock_tole32(&superblock); + lfs_dir_commit(&lfs, &root, LFS_MKATTRS( + {LFS_MKTAG(LFS_TYPE_CREATE, 0, 0), NULL}, + {LFS_MKTAG(LFS_TYPE_SUPERBLOCK, 0, 8), "littlefs"}, + {LFS_MKTAG(LFS_TYPE_INLINESTRUCT, 0, sizeof(superblock)), + &superblock})) => 0; + lfs_deinit(&lfs) => 0; + + // known block_size/block_count + cfg->block_size = BLOCK_SIZE; + cfg->block_count = BLOCK_COUNT; + lfs_mount(&lfs, cfg) => LFS_ERR_INVAL; +''' From 2c222af17d8c3066e663b3adca422bf8ea50c243 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Mon, 11 Sep 2023 23:23:07 -0500 Subject: [PATCH 16/23] Tweaked lfs_fsinfo block_size/block_count fields Mainly to match superblock ordering and emphasize these are logical blocks. --- lfs.c | 10 ++++++---- lfs.h | 16 +++++++--------- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/lfs.c b/lfs.c index 60c5a8bd..99550fe8 100644 --- a/lfs.c +++ b/lfs.c @@ -46,7 +46,8 @@ static int lfs_bd_read(lfs_t *lfs, lfs_block_t block, lfs_off_t off, void *buffer, lfs_size_t size) { uint8_t *data = buffer; - if (off+size > lfs->cfg->block_size || (lfs->block_count && block >= lfs->block_count)) { + if (off+size > lfs->cfg->block_size + || (lfs->block_count && block >= lfs->block_count)) { return LFS_ERR_CORRUPT; } @@ -4509,14 +4510,15 @@ static int lfs_fs_rawstat(lfs_t *lfs, struct lfs_fsinfo *fsinfo) { fsinfo->disk_version = superblock.version; } + // filesystem geometry + fsinfo->block_size = lfs->cfg->block_size; + fsinfo->block_count = lfs->block_count; + // other on-disk configuration, we cache all of these for internal use fsinfo->name_max = lfs->name_max; fsinfo->file_max = lfs->file_max; fsinfo->attr_max = lfs->attr_max; - fsinfo->block_count = lfs->block_count; - fsinfo->block_size = lfs->cfg->block_size; - return 0; } diff --git a/lfs.h b/lfs.h index c777f301..6f40c550 100644 --- a/lfs.h +++ b/lfs.h @@ -293,6 +293,12 @@ struct lfs_fsinfo { // On-disk version. uint32_t disk_version; + // Size of a logical block in bytes. + lfs_size_t block_size; + + // Number of logical blocks in filesystem. + lfs_size_t block_count; + // Upper limit on the length of file names in bytes. lfs_size_t name_max; @@ -301,13 +307,6 @@ struct lfs_fsinfo { // Upper limit on the size of custom attributes in bytes. lfs_size_t attr_max; - - // Number of blocks in filesystem. - // May differ from cfg->block_count if autodetected from filesystem. - lfs_size_t block_count; - - // Size of block in bytes. - lfs_size_t block_size; }; // Custom attribute structure, used to describe custom attributes @@ -440,12 +439,11 @@ typedef struct lfs { } free; const struct lfs_config *cfg; + lfs_size_t block_count; lfs_size_t name_max; lfs_size_t file_max; lfs_size_t attr_max; - lfs_size_t block_count; - #ifdef LFS_MIGRATE struct lfs1 *lfs1; #endif From 23505fa9fa4f2d863f046c0bfad8db2ca63cfb90 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Sun, 13 Nov 2022 13:09:21 -0600 Subject: [PATCH 17/23] Added lfs_fs_grow for growing the filesystem to a different block_count The initial implementation for this was provided by kaetemi, originally as a mount flag. However, it has been modified here to be self-contained in an explicit runtime function that can be called after mount. The reasons for an explicit function: 1. lfs_mount stays a strictly readonly operation, and avoids pulling in all of the write machinery. 2. filesystem-wide operations such as lfs_fs_grow can be a bit risky, and irreversable. The action of growing the filesystem should be very intentional. --- One concern with this change is that this will be the first function that changes metadata in the superblock. This might break tools that expect the first valid superblock entry to contain the most recent metadata, since only the last superblock in the superblock chain will contain the updated metadata. --- lfs.c | 54 +++++++++++++++++++ lfs.h | 10 ++++ tests/test_superblocks.toml | 105 ++++++++++++++++++++++++++++++++++++ 3 files changed, 169 insertions(+) diff --git a/lfs.c b/lfs.c index 99550fe8..85660acd 100644 --- a/lfs.c +++ b/lfs.c @@ -5033,6 +5033,44 @@ static lfs_ssize_t lfs_fs_rawsize(lfs_t *lfs) { return size; } +#ifndef LFS_READONLY +int lfs_fs_rawgrow(lfs_t *lfs, lfs_size_t block_count) { + // shrinking is not supported + LFS_ASSERT(block_count >= lfs->block_count); + + if (block_count > lfs->block_count) { + lfs->block_count = block_count; + + // fetch the root + lfs_mdir_t root; + int err = lfs_dir_fetch(lfs, &root, lfs->root); + if (err) { + return err; + } + + // update the superblock + lfs_superblock_t superblock; + lfs_stag_t tag = lfs_dir_get(lfs, &root, LFS_MKTAG(0x7ff, 0x3ff, 0), + LFS_MKTAG(LFS_TYPE_INLINESTRUCT, 0, sizeof(superblock)), + &superblock); + if (tag < 0) { + return tag; + } + lfs_superblock_fromle32(&superblock); + + superblock.block_count = lfs->block_count; + + lfs_superblock_tole32(&superblock); + err = lfs_dir_commit(lfs, &root, LFS_MKATTRS( + {tag, &superblock})); + if (err) { + return err; + } + } + + return 0; +} +#endif #ifdef LFS_MIGRATE ////// Migration from littelfs v1 below this ////// @@ -6216,6 +6254,22 @@ int lfs_fs_mkconsistent(lfs_t *lfs) { } #endif +#ifndef LFS_READONLY +int lfs_fs_grow(lfs_t *lfs, lfs_size_t block_count) { + int err = LFS_LOCK(lfs->cfg); + if (err) { + return err; + } + LFS_TRACE("lfs_fs_grow(%p, %"PRIu32")", (void*)lfs, block_count); + + err = lfs_fs_rawgrow(lfs, block_count); + + LFS_TRACE("lfs_fs_grow -> %d", err); + LFS_UNLOCK(lfs->cfg); + return err; +} +#endif + #ifdef LFS_MIGRATE int lfs_migrate(lfs_t *lfs, const struct lfs_config *cfg) { int err = LFS_LOCK(cfg); diff --git a/lfs.h b/lfs.h index 6f40c550..291dbb51 100644 --- a/lfs.h +++ b/lfs.h @@ -724,6 +724,16 @@ int lfs_fs_traverse(lfs_t *lfs, int (*cb)(void*, lfs_block_t), void *data); int lfs_fs_mkconsistent(lfs_t *lfs); #endif +#ifndef LFS_READONLY +// Grows the filesystem to a new size, updating the superblock with the new +// block count. +// +// Note: This is irreversible. +// +// Returns a negative error code on failure. +int lfs_fs_grow(lfs_t *lfs, lfs_size_t block_count); +#endif + #ifndef LFS_READONLY #ifdef LFS_MIGRATE // Attempts to migrate a previous version of littlefs diff --git a/tests/test_superblocks.toml b/tests/test_superblocks.toml index a99a5a47..6d4a3e4f 100644 --- a/tests/test_superblocks.toml +++ b/tests/test_superblocks.toml @@ -359,3 +359,108 @@ code = ''' cfg->block_count = BLOCK_COUNT; lfs_mount(&lfs, cfg) => LFS_ERR_INVAL; ''' + +# mount and grow the filesystem +[cases.test_superblocks_grow] +defines.BLOCK_COUNT = ['ERASE_COUNT/2', 'ERASE_COUNT/4', '2'] +defines.BLOCK_COUNT_2 = 'ERASE_COUNT' +defines.KNOWN_BLOCK_COUNT = [true, false] +code = ''' + lfs_t lfs; + lfs_format(&lfs, cfg) => 0; + + if (KNOWN_BLOCK_COUNT) { + cfg->block_count = BLOCK_COUNT; + } else { + cfg->block_count = 0; + } + + // mount with block_size < erase_size + lfs_mount(&lfs, cfg) => 0; + struct lfs_fsinfo fsinfo; + lfs_fs_stat(&lfs, &fsinfo) => 0; + assert(fsinfo.block_size == BLOCK_SIZE); + assert(fsinfo.block_count == BLOCK_COUNT); + lfs_unmount(&lfs) => 0; + + // same size is a noop + lfs_mount(&lfs, cfg) => 0; + lfs_fs_grow(&lfs, BLOCK_COUNT) => 0; + lfs_fs_stat(&lfs, &fsinfo) => 0; + assert(fsinfo.block_size == BLOCK_SIZE); + assert(fsinfo.block_count == BLOCK_COUNT); + lfs_unmount(&lfs) => 0; + + lfs_mount(&lfs, cfg) => 0; + lfs_fs_stat(&lfs, &fsinfo) => 0; + assert(fsinfo.block_size == BLOCK_SIZE); + assert(fsinfo.block_count == BLOCK_COUNT); + lfs_unmount(&lfs) => 0; + + // grow to new size + lfs_mount(&lfs, cfg) => 0; + lfs_fs_grow(&lfs, BLOCK_COUNT_2) => 0; + lfs_fs_stat(&lfs, &fsinfo) => 0; + assert(fsinfo.block_size == BLOCK_SIZE); + assert(fsinfo.block_count == BLOCK_COUNT_2); + lfs_unmount(&lfs) => 0; + + if (KNOWN_BLOCK_COUNT) { + cfg->block_count = BLOCK_COUNT_2; + } else { + cfg->block_count = 0; + } + + lfs_mount(&lfs, cfg) => 0; + lfs_fs_stat(&lfs, &fsinfo) => 0; + assert(fsinfo.block_size == BLOCK_SIZE); + assert(fsinfo.block_count == BLOCK_COUNT_2); + lfs_unmount(&lfs) => 0; + + // mounting with the previous size should fail + cfg->block_count = BLOCK_COUNT; + lfs_mount(&lfs, cfg) => LFS_ERR_INVAL; + + if (KNOWN_BLOCK_COUNT) { + cfg->block_count = BLOCK_COUNT_2; + } else { + cfg->block_count = 0; + } + + // same size is a noop + lfs_mount(&lfs, cfg) => 0; + lfs_fs_grow(&lfs, BLOCK_COUNT_2) => 0; + lfs_fs_stat(&lfs, &fsinfo) => 0; + assert(fsinfo.block_size == BLOCK_SIZE); + assert(fsinfo.block_count == BLOCK_COUNT_2); + lfs_unmount(&lfs) => 0; + + lfs_mount(&lfs, cfg) => 0; + lfs_fs_stat(&lfs, &fsinfo) => 0; + assert(fsinfo.block_size == BLOCK_SIZE); + assert(fsinfo.block_count == BLOCK_COUNT_2); + lfs_unmount(&lfs) => 0; + + // do some work + lfs_mount(&lfs, cfg) => 0; + lfs_fs_stat(&lfs, &fsinfo) => 0; + assert(fsinfo.block_size == BLOCK_SIZE); + assert(fsinfo.block_count == BLOCK_COUNT_2); + lfs_file_t file; + lfs_file_open(&lfs, &file, "test", + LFS_O_CREAT | LFS_O_EXCL | LFS_O_WRONLY) => 0; + lfs_file_write(&lfs, &file, "hello!", 6) => 6; + lfs_file_close(&lfs, &file) => 0; + lfs_unmount(&lfs) => 0; + + lfs_mount(&lfs, cfg) => 0; + lfs_fs_stat(&lfs, &fsinfo) => 0; + assert(fsinfo.block_size == BLOCK_SIZE); + assert(fsinfo.block_count == BLOCK_COUNT_2); + lfs_file_open(&lfs, &file, "test", LFS_O_RDONLY) => 0; + uint8_t buffer[256]; + lfs_file_read(&lfs, &file, buffer, sizeof(buffer)) => 6; + lfs_file_close(&lfs, &file) => 0; + assert(memcmp(buffer, "hello!", 6) == 0); + lfs_unmount(&lfs) => 0; +''' From b6373792106f04d8de6b959be61062b89d5b793a Mon Sep 17 00:00:00 2001 From: ondrap Date: Wed, 2 Aug 2023 11:51:52 +0200 Subject: [PATCH 18/23] Update lfs_find_free_blocks to match the latest changes. --- lfs.c | 27 +++++++++++++++++---------- lfs.h | 4 ++++ 2 files changed, 21 insertions(+), 10 deletions(-) diff --git a/lfs.c b/lfs.c index fbd0ca60..250e7346 100644 --- a/lfs.c +++ b/lfs.c @@ -654,20 +654,27 @@ static int lfs_alloc(lfs_t *lfs, lfs_block_t *block) { return LFS_ERR_NOSPC; } - lfs->free.off = (lfs->free.off + lfs->free.size) - % lfs->block_count; - lfs->free.size = lfs_min(8*lfs->cfg->lookahead_size, lfs->free.ack); - lfs->free.i = 0; - - // find mask of free blocks from tree - memset(lfs->free.buffer, 0, lfs->cfg->lookahead_size); - int err = lfs_fs_rawtraverse(lfs, lfs_alloc_lookahead, lfs, true); - if (err) { - lfs_alloc_drop(lfs); + int err = lfs_find_free_blocks(lfs); + if(err) { return err; } } } + +int lfs_find_free_blocks(lfs_t *lfs){ + lfs->free.off = (lfs->free.off + lfs->free.size) + % lfs->block_count; + lfs->free.size = lfs_min(8*lfs->cfg->lookahead_size, lfs->free.ack); + lfs->free.i = 0; + + // find mask of free blocks from tree + memset(lfs->free.buffer, 0, lfs->cfg->lookahead_size); + int const err = lfs_fs_rawtraverse(lfs, lfs_alloc_lookahead, lfs, true); + if (err) { + lfs_alloc_drop(lfs); + } + return err; +} #endif /// Metadata pair and directory operations /// diff --git a/lfs.h b/lfs.h index 291dbb51..f574b317 100644 --- a/lfs.h +++ b/lfs.h @@ -712,6 +712,10 @@ lfs_ssize_t lfs_fs_size(lfs_t *lfs); // Returns a negative error code on failure. int lfs_fs_traverse(lfs_t *lfs, int (*cb)(void*, lfs_block_t), void *data); +// Use Traverse function and try to find free blocks. LittleFS free blocks search is unpredictable. +// Search is costly operation which may delay write. In realtime write scenarios can be better to find them before a write. +int lfs_find_free_blocks(lfs_t *lfs); + #ifndef LFS_READONLY // Attempt to make the filesystem consistent and ready for writing // From d85a0fe2e2f1a2c48d80b05a29c3c60a28c7e3b0 Mon Sep 17 00:00:00 2001 From: ondrap Date: Tue, 29 Aug 2023 12:50:16 +0200 Subject: [PATCH 19/23] Move lookahead buffer offset at the first free block if such block doesn't exist move it for whole lookahead size. --- lfs.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lfs.c b/lfs.c index 250e7346..3664e801 100644 --- a/lfs.c +++ b/lfs.c @@ -662,7 +662,9 @@ static int lfs_alloc(lfs_t *lfs, lfs_block_t *block) { } int lfs_find_free_blocks(lfs_t *lfs){ - lfs->free.off = (lfs->free.off + lfs->free.size) + // Move free offset at the first unused block (lfs->free.i) + // lfs->free.i is equal lfs->free.size when all blocks are used + lfs->free.off = (lfs->free.off + lfs->free.i) % lfs->block_count; lfs->free.size = lfs_min(8*lfs->cfg->lookahead_size, lfs->free.ack); lfs->free.i = 0; From dbe4598c12a9b1e6cab2f8dc78158755b64a1e6d Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Mon, 11 Sep 2023 23:42:37 -0500 Subject: [PATCH 20/23] Added API boilerplate for lfs_fs_findfreeblocks and consistent style This adds the tracing and optional locking for the littlefs API. Also updated to match the code style, and added LFS_READONLY guards where necessary. --- lfs.c | 55 +++++++++++++++++++++++++++++++++++++------------------ lfs.h | 9 ++++++--- 2 files changed, 43 insertions(+), 21 deletions(-) diff --git a/lfs.c b/lfs.c index 3664e801..dafe0805 100644 --- a/lfs.c +++ b/lfs.c @@ -622,6 +622,26 @@ static void lfs_alloc_drop(lfs_t *lfs) { lfs_alloc_ack(lfs); } +#ifndef LFS_READONLY +static int lfs_fs_rawfindfreeblocks(lfs_t *lfs) { + // Move free offset at the first unused block (lfs->free.i) + // lfs->free.i is equal lfs->free.size when all blocks are used + lfs->free.off = (lfs->free.off + lfs->free.i) % lfs->block_count; + lfs->free.size = lfs_min(8*lfs->cfg->lookahead_size, lfs->free.ack); + lfs->free.i = 0; + + // find mask of free blocks from tree + memset(lfs->free.buffer, 0, lfs->cfg->lookahead_size); + int err = lfs_fs_rawtraverse(lfs, lfs_alloc_lookahead, lfs, true); + if (err) { + lfs_alloc_drop(lfs); + return err; + } + + return 0; +} +#endif + #ifndef LFS_READONLY static int lfs_alloc(lfs_t *lfs, lfs_block_t *block) { while (true) { @@ -654,29 +674,12 @@ static int lfs_alloc(lfs_t *lfs, lfs_block_t *block) { return LFS_ERR_NOSPC; } - int err = lfs_find_free_blocks(lfs); + int err = lfs_fs_rawfindfreeblocks(lfs); if(err) { return err; } } } - -int lfs_find_free_blocks(lfs_t *lfs){ - // Move free offset at the first unused block (lfs->free.i) - // lfs->free.i is equal lfs->free.size when all blocks are used - lfs->free.off = (lfs->free.off + lfs->free.i) - % lfs->block_count; - lfs->free.size = lfs_min(8*lfs->cfg->lookahead_size, lfs->free.ack); - lfs->free.i = 0; - - // find mask of free blocks from tree - memset(lfs->free.buffer, 0, lfs->cfg->lookahead_size); - int const err = lfs_fs_rawtraverse(lfs, lfs_alloc_lookahead, lfs, true); - if (err) { - lfs_alloc_drop(lfs); - } - return err; -} #endif /// Metadata pair and directory operations /// @@ -6247,6 +6250,22 @@ int lfs_fs_traverse(lfs_t *lfs, int (*cb)(void *, lfs_block_t), void *data) { return err; } +#ifndef LFS_READONLY +int lfs_fs_findfreeblocks(lfs_t *lfs) { + int err = LFS_LOCK(lfs->cfg); + if (err) { + return err; + } + LFS_TRACE("lfs_fs_findfreeblocks(%p)", (void*)lfs); + + err = lfs_fs_rawfindfreeblocks(lfs); + + LFS_TRACE("lfs_fs_findfreeblocks -> %d", err); + LFS_UNLOCK(lfs->cfg); + return err; +} +#endif + #ifndef LFS_READONLY int lfs_fs_mkconsistent(lfs_t *lfs) { int err = LFS_LOCK(lfs->cfg); diff --git a/lfs.h b/lfs.h index f574b317..a0dce9d8 100644 --- a/lfs.h +++ b/lfs.h @@ -712,9 +712,12 @@ lfs_ssize_t lfs_fs_size(lfs_t *lfs); // Returns a negative error code on failure. int lfs_fs_traverse(lfs_t *lfs, int (*cb)(void*, lfs_block_t), void *data); -// Use Traverse function and try to find free blocks. LittleFS free blocks search is unpredictable. -// Search is costly operation which may delay write. In realtime write scenarios can be better to find them before a write. -int lfs_find_free_blocks(lfs_t *lfs); +// Use Traverse function and try to find free blocks. LittleFS free blocks +// search is unpredictable. +// +// Search is costly operation which may delay write. In realtime write +// scenarios can be better to find them before a write. +int lfs_fs_findfreeblocks(lfs_t *lfs); #ifndef LFS_READONLY // Attempt to make the filesystem consistent and ready for writing From 63e4408f2af1464df5b9c574661e923dc8be2386 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Mon, 11 Sep 2023 23:56:49 -0500 Subject: [PATCH 21/23] Extended alloc tests to test some properties of lfs_fs_findfreeblocks - Test that the code actually runs. - Test that lfs_fs_findfreeblocks does not break block allocations. - Test that lfs_fs_findfreeblocks does not error when no space is available, it should only errors when the block is actually needed. --- tests/test_alloc.toml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/tests/test_alloc.toml b/tests/test_alloc.toml index 916fc2a5..50baa7d0 100644 --- a/tests/test_alloc.toml +++ b/tests/test_alloc.toml @@ -6,6 +6,7 @@ if = 'BLOCK_CYCLES == -1' [cases.test_alloc_parallel] defines.FILES = 3 defines.SIZE = '(((BLOCK_SIZE-8)*(BLOCK_COUNT-6)) / FILES)' +defines.GC = [false, true] code = ''' const char *names[] = {"bacon", "eggs", "pancakes"}; lfs_file_t files[FILES]; @@ -24,6 +25,9 @@ code = ''' LFS_O_WRONLY | LFS_O_CREAT | LFS_O_APPEND) => 0; } for (int n = 0; n < FILES; n++) { + if (GC) { + lfs_fs_findfreeblocks(&lfs) => 0; + } size_t size = strlen(names[n]); for (lfs_size_t i = 0; i < SIZE; i += size) { lfs_file_write(&lfs, &files[n], names[n], size) => size; @@ -55,6 +59,7 @@ code = ''' [cases.test_alloc_serial] defines.FILES = 3 defines.SIZE = '(((BLOCK_SIZE-8)*(BLOCK_COUNT-6)) / FILES)' +defines.GC = [false, true] code = ''' const char *names[] = {"bacon", "eggs", "pancakes"}; @@ -75,6 +80,9 @@ code = ''' uint8_t buffer[1024]; memcpy(buffer, names[n], size); for (int i = 0; i < SIZE; i += size) { + if (GC) { + lfs_fs_findfreeblocks(&lfs) => 0; + } lfs_file_write(&lfs, &file, buffer, size) => size; } lfs_file_close(&lfs, &file) => 0; @@ -247,6 +255,9 @@ code = ''' } res => LFS_ERR_NOSPC; + // note that lfs_fs_findfreeblocks should not error here + lfs_fs_findfreeblocks(&lfs) => 0; + lfs_file_close(&lfs, &file) => 0; lfs_unmount(&lfs) => 0; @@ -298,6 +309,9 @@ code = ''' } res => LFS_ERR_NOSPC; + // note that lfs_fs_findfreeblocks should not error here + lfs_fs_findfreeblocks(&lfs) => 0; + lfs_file_close(&lfs, &file) => 0; lfs_unmount(&lfs) => 0; @@ -337,6 +351,8 @@ code = ''' count += 1; } err => LFS_ERR_NOSPC; + // note that lfs_fs_findfreeblocks should not error here + lfs_fs_findfreeblocks(&lfs) => 0; lfs_file_close(&lfs, &file) => 0; lfs_remove(&lfs, "exhaustion") => 0; @@ -435,6 +451,8 @@ code = ''' break; } } + // note that lfs_fs_findfreeblocks should not error here + lfs_fs_findfreeblocks(&lfs) => 0; lfs_file_close(&lfs, &file) => 0; lfs_unmount(&lfs) => 0; From 6b33ee5e34a29508e928dce2e6f32d3c6936131a Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Tue, 12 Sep 2023 00:06:04 -0500 Subject: [PATCH 22/23] Renamed lfs_fs_findfreeblocks -> lfs_fs_gc, tweaked documentation The idea is in the future this function may be extended to support other block janitorial work. In such a case calling this lfs_fs_gc provides a more general name that can include other operations. This is currently just wishful thinking, however. --- lfs.c | 12 ++++++------ lfs.h | 15 ++++++++++----- tests/test_alloc.toml | 20 ++++++++++---------- 3 files changed, 26 insertions(+), 21 deletions(-) diff --git a/lfs.c b/lfs.c index dafe0805..0827331c 100644 --- a/lfs.c +++ b/lfs.c @@ -623,7 +623,7 @@ static void lfs_alloc_drop(lfs_t *lfs) { } #ifndef LFS_READONLY -static int lfs_fs_rawfindfreeblocks(lfs_t *lfs) { +static int lfs_fs_rawgc(lfs_t *lfs) { // Move free offset at the first unused block (lfs->free.i) // lfs->free.i is equal lfs->free.size when all blocks are used lfs->free.off = (lfs->free.off + lfs->free.i) % lfs->block_count; @@ -674,7 +674,7 @@ static int lfs_alloc(lfs_t *lfs, lfs_block_t *block) { return LFS_ERR_NOSPC; } - int err = lfs_fs_rawfindfreeblocks(lfs); + int err = lfs_fs_rawgc(lfs); if(err) { return err; } @@ -6251,16 +6251,16 @@ int lfs_fs_traverse(lfs_t *lfs, int (*cb)(void *, lfs_block_t), void *data) { } #ifndef LFS_READONLY -int lfs_fs_findfreeblocks(lfs_t *lfs) { +int lfs_fs_gc(lfs_t *lfs) { int err = LFS_LOCK(lfs->cfg); if (err) { return err; } - LFS_TRACE("lfs_fs_findfreeblocks(%p)", (void*)lfs); + LFS_TRACE("lfs_fs_gc(%p)", (void*)lfs); - err = lfs_fs_rawfindfreeblocks(lfs); + err = lfs_fs_rawgc(lfs); - LFS_TRACE("lfs_fs_findfreeblocks -> %d", err); + LFS_TRACE("lfs_fs_gc -> %d", err); LFS_UNLOCK(lfs->cfg); return err; } diff --git a/lfs.h b/lfs.h index a0dce9d8..6535eedb 100644 --- a/lfs.h +++ b/lfs.h @@ -712,12 +712,17 @@ lfs_ssize_t lfs_fs_size(lfs_t *lfs); // Returns a negative error code on failure. int lfs_fs_traverse(lfs_t *lfs, int (*cb)(void*, lfs_block_t), void *data); -// Use Traverse function and try to find free blocks. LittleFS free blocks -// search is unpredictable. +// Attempt to proactively find free blocks // -// Search is costly operation which may delay write. In realtime write -// scenarios can be better to find them before a write. -int lfs_fs_findfreeblocks(lfs_t *lfs); +// Calling this function is not required, but may allowing the offloading of +// the expensive block allocation scan to a less time-critical code path. +// +// Note: littlefs currently does not persist any found free blocks to disk. +// This may change in the future. +// +// Returns a negative error code on failure. Finding no free blocks is +// not an error. +int lfs_fs_gc(lfs_t *lfs); #ifndef LFS_READONLY // Attempt to make the filesystem consistent and ready for writing diff --git a/tests/test_alloc.toml b/tests/test_alloc.toml index 50baa7d0..e6fba975 100644 --- a/tests/test_alloc.toml +++ b/tests/test_alloc.toml @@ -26,7 +26,7 @@ code = ''' } for (int n = 0; n < FILES; n++) { if (GC) { - lfs_fs_findfreeblocks(&lfs) => 0; + lfs_fs_gc(&lfs) => 0; } size_t size = strlen(names[n]); for (lfs_size_t i = 0; i < SIZE; i += size) { @@ -81,7 +81,7 @@ code = ''' memcpy(buffer, names[n], size); for (int i = 0; i < SIZE; i += size) { if (GC) { - lfs_fs_findfreeblocks(&lfs) => 0; + lfs_fs_gc(&lfs) => 0; } lfs_file_write(&lfs, &file, buffer, size) => size; } @@ -255,8 +255,8 @@ code = ''' } res => LFS_ERR_NOSPC; - // note that lfs_fs_findfreeblocks should not error here - lfs_fs_findfreeblocks(&lfs) => 0; + // note that lfs_fs_gc should not error here + lfs_fs_gc(&lfs) => 0; lfs_file_close(&lfs, &file) => 0; lfs_unmount(&lfs) => 0; @@ -309,8 +309,8 @@ code = ''' } res => LFS_ERR_NOSPC; - // note that lfs_fs_findfreeblocks should not error here - lfs_fs_findfreeblocks(&lfs) => 0; + // note that lfs_fs_gc should not error here + lfs_fs_gc(&lfs) => 0; lfs_file_close(&lfs, &file) => 0; lfs_unmount(&lfs) => 0; @@ -351,8 +351,8 @@ code = ''' count += 1; } err => LFS_ERR_NOSPC; - // note that lfs_fs_findfreeblocks should not error here - lfs_fs_findfreeblocks(&lfs) => 0; + // note that lfs_fs_gc should not error here + lfs_fs_gc(&lfs) => 0; lfs_file_close(&lfs, &file) => 0; lfs_remove(&lfs, "exhaustion") => 0; @@ -451,8 +451,8 @@ code = ''' break; } } - // note that lfs_fs_findfreeblocks should not error here - lfs_fs_findfreeblocks(&lfs) => 0; + // note that lfs_fs_gc should not error here + lfs_fs_gc(&lfs) => 0; lfs_file_close(&lfs, &file) => 0; lfs_unmount(&lfs) => 0; From f91c5bd6870395b351af0f0c6d8abd0b3de4845b Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Thu, 21 Sep 2023 13:02:09 -0500 Subject: [PATCH 23/23] Bumped minor version to v2.8 --- lfs.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lfs.h b/lfs.h index 6535eedb..9eeab230 100644 --- a/lfs.h +++ b/lfs.h @@ -21,7 +21,7 @@ extern "C" // Software library version // Major (top-nibble), incremented on backwards incompatible changes // Minor (bottom-nibble), incremented on feature additions -#define LFS_VERSION 0x00020007 +#define LFS_VERSION 0x00020008 #define LFS_VERSION_MAJOR (0xffff & (LFS_VERSION >> 16)) #define LFS_VERSION_MINOR (0xffff & (LFS_VERSION >> 0))