-
Notifications
You must be signed in to change notification settings - Fork 6.5k
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
A new non volatile storage system #77930
Open
rghaddab
wants to merge
3
commits into
zephyrproject-rtos:main
Choose a base branch
from
rghaddab:rghaddab/zms
base: main
Could not load branches
Branch not found: {{ refName }}
Loading
Could not load tags
Nothing to show
Loading
Are you sure you want to change the base?
Some commits from the old base branch may be removed from the timeline,
and old review comments may become outdated.
+3,497
−0
Open
Changes from all commits
Commits
Show all changes
3 commits
Select commit
Hold shift + click to select a range
File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,211 @@ | ||
/* ZMS: Zephyr Memory Storage | ||
* | ||
* Copyright (c) 2024 BayLibre SAS | ||
* | ||
* SPDX-License-Identifier: Apache-2.0 | ||
*/ | ||
#ifndef ZEPHYR_INCLUDE_FS_ZMS_H_ | ||
#define ZEPHYR_INCLUDE_FS_ZMS_H_ | ||
|
||
#include <sys/types.h> | ||
#include <zephyr/kernel.h> | ||
#include <zephyr/device.h> | ||
#include <zephyr/toolchain.h> | ||
|
||
#ifdef __cplusplus | ||
extern "C" { | ||
#endif | ||
|
||
/** | ||
* @brief Zephyr Memory Storage (ZMS) | ||
* @defgroup zms Zephyr Memory Storage (ZMS) | ||
* @ingroup file_system_storage | ||
* @{ | ||
* @} | ||
*/ | ||
|
||
/** | ||
* @brief Zephyr Memory Storage Data Structures | ||
* @defgroup zms_data_structures Zephyr Memory Storage Data Structures | ||
* @ingroup zms | ||
* @{ | ||
*/ | ||
|
||
/** | ||
* @brief Zephyr Memory Storage File system structure | ||
*/ | ||
struct zms_fs { | ||
/** File system offset in flash **/ | ||
off_t offset; | ||
/** Allocation table entry write address. | ||
* Addresses are stored as uint64_t: | ||
* - high 4 bytes correspond to the sector | ||
* - low 4 bytes are the offset in the sector | ||
*/ | ||
uint64_t ate_wra; | ||
/** Data write address */ | ||
uint64_t data_wra; | ||
/** Storage system is split into sectors, each sector size must be multiple of erase-blocks | ||
* if the device has erase capabilities | ||
*/ | ||
uint32_t sector_size; | ||
/** Number of sectors in the file system */ | ||
uint32_t sector_count; | ||
/** Current cycle counter of the active sector (pointed by ate_wra)*/ | ||
uint8_t sector_cycle; | ||
/** Flag indicating if the file system is initialized */ | ||
bool ready; | ||
/** Mutex */ | ||
struct k_mutex zms_lock; | ||
/** Flash device runtime structure */ | ||
const struct device *flash_device; | ||
/** Flash memory parameters structure */ | ||
const struct flash_parameters *flash_parameters; | ||
#if CONFIG_ZMS_LOOKUP_CACHE | ||
uint64_t lookup_cache[CONFIG_ZMS_LOOKUP_CACHE_SIZE]; | ||
#endif | ||
}; | ||
|
||
/** | ||
* @} | ||
*/ | ||
|
||
/** | ||
* @brief Zephyr Memory Storage APIs | ||
* @defgroup zms_high_level_api Zephyr Memory Storage APIs | ||
* @ingroup zms | ||
* @{ | ||
*/ | ||
|
||
/** | ||
* @brief Mount a ZMS file system onto the device specified in @p fs. | ||
* | ||
* @param fs Pointer to file system | ||
* @retval 0 Success | ||
* @retval -ERRNO errno code if error | ||
*/ | ||
int zms_mount(struct zms_fs *fs); | ||
|
||
/** | ||
* @brief Clear the ZMS file system from device. | ||
* | ||
* @param fs Pointer to file system | ||
* @retval 0 Success | ||
* @retval -ERRNO errno code if error | ||
*/ | ||
int zms_clear(struct zms_fs *fs); | ||
|
||
/** | ||
* @brief Write an entry to the file system. | ||
* | ||
* @note When @p len parameter is equal to @p 0 then entry is effectively removed (it is | ||
* equivalent to calling of zms_delete). It is not possible to distinguish between a deleted | ||
* entry and an entry with data of length 0. | ||
* | ||
* @param fs Pointer to file system | ||
* @param id Id of the entry to be written | ||
* @param data Pointer to the data to be written | ||
* @param len Number of bytes to be written (maximum 64 KB) | ||
* | ||
* @return Number of bytes written. On success, it will be equal to the number of bytes requested | ||
* to be written. When a rewrite of the same data already stored is attempted, nothing is written | ||
* to flash, thus 0 is returned. On error, returns negative value of errno.h defined error codes. | ||
*/ | ||
ssize_t zms_write(struct zms_fs *fs, uint32_t id, const void *data, size_t len); | ||
|
||
/** | ||
* @brief Delete an entry from the file system | ||
* | ||
* @param fs Pointer to file system | ||
* @param id Id of the entry to be deleted | ||
* @retval 0 Success | ||
* @retval -ERRNO errno code if error | ||
*/ | ||
int zms_delete(struct zms_fs *fs, uint32_t id); | ||
|
||
/** | ||
* @brief Read an entry from the file system. | ||
* | ||
* @param fs Pointer to file system | ||
* @param id Id of the entry to be read | ||
* @param data Pointer to data buffer | ||
* @param len Number of bytes to be read (or size of the allocated read buffer) | ||
* | ||
* @return Number of bytes read. On success, it will be equal to the number of bytes requested | ||
* to be read. When the return value is less than the number of bytes requested to read this | ||
* indicates that ATE contain less data than requested. On error, returns negative value of | ||
* errno.h defined error codes. | ||
*/ | ||
ssize_t zms_read(struct zms_fs *fs, uint32_t id, void *data, size_t len); | ||
|
||
/** | ||
* @brief Read a history entry from the file system. | ||
* | ||
* @param fs Pointer to file system | ||
* @param id Id of the entry to be read | ||
* @param data Pointer to data buffer | ||
* @param len Number of bytes to be read | ||
* @param cnt History counter: 0: latest entry, 1: one before latest ... | ||
* | ||
* @return Number of bytes read. On success, it will be equal to the number of bytes requested | ||
* to be read. When the return value is larger than the number of bytes requested to read this | ||
* indicates not all bytes were read, and more data is available. On error, returns negative | ||
* value of errno.h defined error codes. | ||
*/ | ||
ssize_t zms_read_hist(struct zms_fs *fs, uint32_t id, void *data, size_t len, uint32_t cnt); | ||
|
||
/** | ||
* @brief Gets the data size that is stored in an entry with a given id | ||
* | ||
* @param fs Pointer to file system | ||
* @param id Id of the entry that we want to get its data length | ||
* | ||
* @return Data length contained in the ATE. On success, it will be equal to the number of bytes | ||
* in the ATE. On error, returns negative value of errno.h defined error codes. | ||
*/ | ||
ssize_t zms_get_data_length(struct zms_fs *fs, uint32_t id); | ||
/** | ||
* @brief Calculate the available free space in the file system. | ||
* | ||
* @param fs Pointer to file system | ||
* | ||
* @return Number of bytes free. On success, it will be equal to the number of bytes that can | ||
* still be written to the file system. | ||
* Calculating the free space is a time consuming operation, especially on spi flash. | ||
* On error, returns negative value of errno.h defined error codes. | ||
*/ | ||
ssize_t zms_calc_free_space(struct zms_fs *fs); | ||
|
||
/** | ||
* @brief Tell how many contiguous free space remains in the currently active ZMS sector. | ||
* | ||
* @param fs Pointer to the file system. | ||
* | ||
* @return Number of free bytes. | ||
*/ | ||
size_t zms_sector_max_data_size(struct zms_fs *fs); | ||
|
||
/** | ||
* @brief Close the currently active sector and switch to the next one. | ||
* | ||
* @note The garbage collector is called on the new sector. | ||
* | ||
* @warning This routine is made available for specific use cases. | ||
* It breaks the aim of the ZMS to avoid any unnecessary flash erases. | ||
* Using this routine extensively can result in premature failure of the flash device. | ||
* | ||
* @param fs Pointer to the file system. | ||
* | ||
* @return 0 on success. On error, returns negative value of errno.h defined error codes. | ||
*/ | ||
int zms_sector_use_next(struct zms_fs *fs); | ||
rghaddab marked this conversation as resolved.
Show resolved
Hide resolved
|
||
|
||
/** | ||
* @} | ||
*/ | ||
|
||
#ifdef __cplusplus | ||
} | ||
#endif | ||
|
||
#endif /* ZEPHYR_INCLUDE_FS_ZMS_H_ */ |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,10 @@ | ||
# SPDX-License-Identifier: Apache-2.0 | ||
|
||
cmake_minimum_required(VERSION 3.20.0) | ||
|
||
find_package(Zephyr REQUIRED HINTS $ENV{ZEPHYR_BASE}) | ||
project(zms) | ||
|
||
|
||
target_sources(app PRIVATE src/main.c) | ||
target_include_directories(app PRIVATE ${ZEPHYR_BASE}/subsys/fs/zms) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,93 @@ | ||
.. zephyr:code-sample:: zms | ||
:name: Zephyr Memory Storage (ZMS) | ||
:relevant-api: zms_high_level_api | ||
|
||
Store and retrieve data from storage using the ZMS API. | ||
|
||
Overview | ||
******** | ||
The sample shows how to use ZMS to store ID/VALUE pairs and reads them back. | ||
Deleting an ID/VALE pair is also shown in this sample. | ||
|
||
The sample stores the following items: | ||
1- A string representing an IP-address: stored at id=1, data="192.168.1.1" | ||
2- A binary blob representing a key/value pair: stored at id=0xbeefdead, | ||
data={0xDE, 0xAD, 0xBE, 0xEF, 0xDE, 0xAD, 0xBE, 0xEF} | ||
3- A variable (32bit): stored at id=2, data=cnt | ||
4- A long set of data (128 bytes) | ||
A loop is executed where we mount the storage system, and then write all set | ||
of data. | ||
Each DELETE_ITERATION period, we delete all set of data and verify that it has been deleted. | ||
We generate as well incremented ID/value pairs, we store them until storage is full, then we | ||
delete them and verify that storage is empty. | ||
|
||
Requirements | ||
************ | ||
|
||
* A board with flash support or native_sim target | ||
|
||
Building and Running | ||
******************** | ||
|
||
This sample can be found under :zephyr_file:`samples/subsys/fs/zms` in the Zephyr tree. | ||
|
||
The sample can be built for several platforms, but for the moment it has been tested only | ||
on native_sim target | ||
|
||
.. zephyr-app-commands:: | ||
:zephyr-app: samples/subsys/fs/zms | ||
:goals: build | ||
:compact: | ||
|
||
After running the generated image on a native_sim target, the output on the console shows the | ||
multiple Iterations of read/write/delete exectuted. | ||
|
||
Sample Output | ||
============= | ||
|
||
.. code-block:: console | ||
|
||
*** Booting Zephyr OS build v3.7.0-2383-g624f75400242 *** | ||
[00:00:00.000,000] <inf> fs_zms: 3 Sectors of 4096 bytes | ||
[00:00:00.000,000] <inf> fs_zms: alloc wra: 0, fc0 | ||
[00:00:00.000,000] <inf> fs_zms: data wra: 0, 0 | ||
ITERATION: 0 | ||
Adding IP_ADDRESS 172.16.254.1 at id 1 | ||
Adding key/value at id beefdead | ||
Adding counter at id 2 | ||
Adding Longarray at id 3 | ||
[00:00:00.000,000] <inf> fs_zms: 3 Sectors of 4096 bytes | ||
[00:00:00.000,000] <inf> fs_zms: alloc wra: 0, f80 | ||
[00:00:00.000,000] <inf> fs_zms: data wra: 0, 8c | ||
ITERATION: 1 | ||
ID: 1, IP Address: 172.16.254.1 | ||
Adding IP_ADDRESS 172.16.254.1 at id 1 | ||
Id: beefdead, Key: de ad be ef de ad be ef | ||
Adding key/value at id beefdead | ||
Id: 2, loop_cnt: 0 | ||
Adding counter at id 2 | ||
Id: 3, Longarray: 0 1 2 3 4 5 6 7 8 9 a b c d e f 10 11 12 13 14 15 16 17 18 19 1a 1b 1c 1d 1e 1f 20 21 22 23 24 25 26 27 28 29 2a 2b 2c 2d 2e 2f 30 31 32 33 34 35 36 37 38 39 3a 3b 3c 3d 3e 3f 40 41 42 43 44 45 46 47 48 49 4a 4b 4c 4d 4e 4f 50 51 52 53 5 | ||
4 55 56 57 58 59 5a 5b 5c 5d 5e 5f 60 61 62 63 64 65 66 67 68 69 6a 6b 6c 6d 6e 6f 70 71 72 73 74 75 76 77 78 79 7a 7b 7c 7d 7e 7f | ||
Adding Longarray at id 3 | ||
. | ||
. | ||
. | ||
. | ||
. | ||
. | ||
[00:00:00.000,000] <inf> fs_zms: 3 Sectors of 4096 bytes | ||
[00:00:00.000,000] <inf> fs_zms: alloc wra: 0, f40 | ||
[00:00:00.000,000] <inf> fs_zms: data wra: 0, 80 | ||
ITERATION: 299 | ||
ID: 1, IP Address: 172.16.254.1 | ||
Adding IP_ADDRESS 172.16.254.1 at id 1 | ||
Id: beefdead, Key: de ad be ef de ad be ef | ||
Adding key/value at id beefdead | ||
Id: 2, loop_cnt: 298 | ||
Adding counter at id 2 | ||
Id: 3, Longarray: 0 1 2 3 4 5 6 7 8 9 a b c d e f 10 11 12 13 14 15 16 17 18 19 1a 1b 1c 1d 1e 1f 20 21 22 23 24 25 26 27 28 29 2a 2b 2c 2d 2e 2f 30 31 32 33 34 35 36 37 38 39 3a 3b 3c 3d 3e 3f 40 41 42 43 44 45 46 47 48 49 4a 4b 4c 4d 4e 4f 50 51 52 53 5 | ||
4 55 56 57 58 59 5a 5b 5c 5d 5e 5f 60 61 62 63 64 65 66 67 68 69 6a 6b 6c 6d 6e 6f 70 71 72 73 74 75 76 77 78 79 7a 7b 7c 7d 7e 7f | ||
Adding Longarray at id 3 | ||
Memory is full let's delete all items | ||
Free space in storage is 8064 bytes | ||
Sample code finished Successfully |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,5 @@ | ||
CONFIG_FLASH=y | ||
CONFIG_FLASH_MAP=y | ||
|
||
CONFIG_ZMS=y | ||
CONFIG_LOG=y |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,10 @@ | ||
sample: | ||
name: ZMS Sample | ||
|
||
tests: | ||
sample.zms.basic: | ||
tags: zms | ||
depends_on: zms | ||
platform_allow: | ||
- qemu_x86 | ||
- native_posix |
Oops, something went wrong.
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
I think this was a weird design choice of nvs (not being able to store 0-length values) and we had to implement various workarounds to this limitation. It's worth thinking twice if we want to adopt this to the new filesystem.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Can you explain more the usecases where we need to store 0-length values ?
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
For example, there can be a device label that has a default value but a user may configure it to an empty string. I remember we once had to add setting a magic value that represents an empty string to be able to support this use case :).
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
This could be done through a combination between len and metadata (union with data) field that I introduce in the ATE
We can have a field in the metadata to indicate that this is an empty value ATE if len == 0
if (len == 0) and (metadata_bit_n == 0) => deleted ATE
if (len == 0) and (metadata_bit_n == 1) => ATE with empty value
Does that resolves the usecase ?
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Yes, I'm curious what others think about this but for me it makes sense to distinguish between non-existent and empty values.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
@Damian-Nordic whether we are keeping the interpretation of len == 0 as a delete ATE or that we are going to add an empty value ATE, this is feasible with the current design (could be decided/done in a future commit)
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
The empty value has sense.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
It has sense also for me.
Why keeping the
len == 0
check for a delete ATE if we have the metadata_bit_n telling us already that the ATE is deleted ?There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
@RICCIARDI-Adrien actually both len == 0 and metadata_bit_n are needed.
metadata is part of data for normal ATEs with data <= 8 bytes (metadata_bit_n could take any value)
We can interpret meta_data_bit_n for example when len == 0
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Oh yes, you are correct !